diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 53ca9e0fa2e..ee5afa17669 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -36,6 +36,8 @@ Improvements Other Changes ---------------------- +* SOLR-14656: Autoscaling framework removed (Ishan Chattopadhyaya, noble, Ilan Ginzburg) + * LUCENE-9391: Upgrade HPPC to 0.8.2. (Haoyu Zhai) * SOLR-10288: Remove non-minified JavaScript from the webapp. (Erik Hatcher, marcussorealheis) diff --git a/solr/bin/solr b/solr/bin/solr index 83d247ccefa..7e3cf0c62d6 100755 --- a/solr/bin/solr +++ b/solr/bin/solr @@ -330,7 +330,7 @@ function print_usage() { if [ -z "$CMD" ]; then echo "" echo "Usage: solr COMMAND OPTIONS" - echo " where COMMAND is one of: start, stop, restart, status, healthcheck, create, create_core, create_collection, delete, version, zk, auth, assert, config, autoscaling, export" + echo " where COMMAND is one of: start, stop, restart, status, healthcheck, create, create_core, create_collection, delete, version, zk, auth, assert, config, export" echo "" echo " Standalone server example (start Solr running in the background on port 8984):" echo "" @@ -1417,11 +1417,6 @@ if [[ "$SCRIPT_CMD" == "zk" ]]; then fi -if [[ "$SCRIPT_CMD" == "autoscaling" ]]; then - run_tool autoscaling $@ - exit $? -fi - if [[ "$SCRIPT_CMD" == "export" ]]; then run_tool export $@ exit $? diff --git a/solr/bin/solr.cmd b/solr/bin/solr.cmd index 4b4d8132c5b..8fb5e7ad4e5 100755 --- a/solr/bin/solr.cmd +++ b/solr/bin/solr.cmd @@ -224,7 +224,6 @@ IF "%1"=="version" goto get_version IF "%1"=="-v" goto get_version IF "%1"=="-version" goto get_version IF "%1"=="assert" goto run_assert -IF "%1"=="autoscaling" goto run_autoscaling IF "%1"=="export" goto run_export IF "%1"=="package" goto run_package @@ -303,7 +302,7 @@ goto done :script_usage @echo. @echo Usage: solr COMMAND OPTIONS -@echo where COMMAND is one of: start, stop, restart, healthcheck, create, create_core, create_collection, delete, version, zk, auth, assert, config, autoscaling, export +@echo where COMMAND is one of: start, stop, restart, healthcheck, create, create_core, create_collection, delete, version, zk, auth, assert, config, export @echo. @echo Standalone server example (start Solr running in the background on port 8984): @echo. @@ -1454,13 +1453,6 @@ if errorlevel 1 ( ) goto done -:run_autoscaling -"%JAVA%" %SOLR_SSL_OPTS% %AUTHC_OPTS% %SOLR_ZK_CREDS_AND_ACLS% -Dsolr.install.dir="%SOLR_TIP%" ^ - -Dlog4j.configurationFile="file:///%DEFAULT_SERVER_DIR%\resources\log4j2-console.xml" ^ - -classpath "%DEFAULT_SERVER_DIR%\solr-webapp\webapp\WEB-INF\lib\*;%DEFAULT_SERVER_DIR%\lib\ext\*" ^ - org.apache.solr.util.SolrCLI %* -goto done: - :run_export "%JAVA%" %SOLR_SSL_OPTS% %AUTHC_OPTS% %SOLR_ZK_CREDS_AND_ACLS% -Dsolr.install.dir="%SOLR_TIP%" ^ -Dlog4j.configurationFile="file:///%DEFAULT_SERVER_DIR%\resources\log4j2-console.xml" ^ diff --git a/solr/core/src/java/org/apache/solr/cloud/CloudUtil.java b/solr/core/src/java/org/apache/solr/cloud/CloudUtil.java index ecc653b52f0..65de4ca6804 100644 --- a/solr/core/src/java/org/apache/solr/cloud/CloudUtil.java +++ b/solr/core/src/java/org/apache/solr/cloud/CloudUtil.java @@ -235,7 +235,6 @@ public class CloudUtil { * Return a {@link CollectionStatePredicate} that returns true if a collection has the expected * number of shards and replicas. *

Note: for shards marked as inactive the current Solr behavior is that replicas remain active. - * {@link org.apache.solr.cloud.autoscaling.sim.SimCloudManager} follows this behavior.

* @param expectedShards expected number of shards * @param expectedReplicas expected number of active replicas per shard * @param withInactive if true then count also inactive shards diff --git a/solr/core/src/java/org/apache/solr/cloud/Overseer.java b/solr/core/src/java/org/apache/solr/cloud/Overseer.java index 3dfe6c593e9..bb405ad4863 100644 --- a/solr/core/src/java/org/apache/solr/cloud/Overseer.java +++ b/solr/core/src/java/org/apache/solr/cloud/Overseer.java @@ -39,7 +39,6 @@ import org.apache.solr.client.solrj.request.CollectionAdminRequest; import org.apache.solr.client.solrj.response.CollectionAdminResponse; import org.apache.solr.cloud.api.collections.CreateCollectionCmd; import org.apache.solr.cloud.api.collections.OverseerCollectionMessageHandler; -import org.apache.solr.cloud.autoscaling.OverseerTriggerThread; import org.apache.solr.cloud.overseer.ClusterStateMutator; import org.apache.solr.cloud.overseer.CollectionMutator; import org.apache.solr.cloud.overseer.NodeMutator; @@ -83,16 +82,14 @@ import com.codahale.metrics.Timer; * collections, shards, replicas and setting various properties.

* *

The Overseer is a single elected node in the SolrCloud cluster that is in charge of interactions with - * ZooKeeper that require global synchronization. It also hosts the Collection API implementation and the - * Autoscaling framework.

+ * ZooKeeper that require global synchronization.

* *

The Overseer deals with:

* * *

The nodes in the cluster communicate with the Overseer over queues implemented in ZooKeeper. There are essentially @@ -644,14 +641,8 @@ public class Overseer implements SolrCloseable { ccThread = new OverseerThread(ccTg, overseerCollectionConfigSetProcessor, "OverseerCollectionConfigSetProcessor-" + id); ccThread.setDaemon(true); - ThreadGroup triggerThreadGroup = new ThreadGroup("Overseer autoscaling triggers"); - OverseerTriggerThread trigger = new OverseerTriggerThread(zkController.getCoreContainer().getResourceLoader(), - zkController.getSolrCloudManager()); - triggerThread = new OverseerThread(triggerThreadGroup, trigger, "OverseerAutoScalingTriggerThread-" + id); - updaterThread.start(); ccThread.start(); - triggerThread.start(); systemCollectionCompatCheck(new BiConsumer() { boolean firstPair = true; diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkController.java b/solr/core/src/java/org/apache/solr/cloud/ZkController.java index c50581cc539..ca44ece55cf 100644 --- a/solr/core/src/java/org/apache/solr/cloud/ZkController.java +++ b/solr/core/src/java/org/apache/solr/cloud/ZkController.java @@ -60,7 +60,6 @@ import org.apache.solr.client.solrj.impl.HttpSolrClient.Builder; import org.apache.solr.client.solrj.impl.SolrClientCloudManager; import org.apache.solr.client.solrj.impl.ZkClientClusterStateProvider; import org.apache.solr.client.solrj.request.CoreAdminRequest.WaitForState; -import org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventType; import org.apache.solr.cloud.overseer.OverseerAction; import org.apache.solr.cloud.overseer.SliceMutator; import org.apache.solr.common.AlreadyClosedException; @@ -99,7 +98,6 @@ import org.apache.solr.common.util.IOUtils; import org.apache.solr.common.util.ObjectReleaseTracker; import org.apache.solr.common.util.SolrNamedThreadFactory; import org.apache.solr.common.util.StrUtils; -import org.apache.solr.common.util.TimeSource; import org.apache.solr.common.util.URLUtil; import org.apache.solr.common.util.Utils; import org.apache.solr.core.CloseHook; @@ -897,13 +895,8 @@ public class ZkController implements Closeable { cmdExecutor.ensureExists(ZkStateReader.LIVE_NODES_ZKNODE, zkClient); cmdExecutor.ensureExists(ZkStateReader.COLLECTIONS_ZKNODE, zkClient); cmdExecutor.ensureExists(ZkStateReader.ALIASES, zkClient); - cmdExecutor.ensureExists(ZkStateReader.SOLR_AUTOSCALING_EVENTS_PATH, zkClient); - cmdExecutor.ensureExists(ZkStateReader.SOLR_AUTOSCALING_TRIGGER_STATE_PATH, zkClient); - cmdExecutor.ensureExists(ZkStateReader.SOLR_AUTOSCALING_NODE_ADDED_PATH, zkClient); - cmdExecutor.ensureExists(ZkStateReader.SOLR_AUTOSCALING_NODE_LOST_PATH, zkClient); byte[] emptyJson = "{}".getBytes(StandardCharsets.UTF_8); cmdExecutor.ensureExists(ZkStateReader.SOLR_SECURITY_CONF_PATH, emptyJson, CreateMode.PERSISTENT, zkClient); - cmdExecutor.ensureExists(ZkStateReader.SOLR_AUTOSCALING_CONF_PATH, emptyJson, CreateMode.PERSISTENT, zkClient); bootstrapDefaultConfigSet(zkClient); } @@ -1044,29 +1037,6 @@ public class ZkController implements Closeable { } i++; } - - // retrieve current trigger config - if there are no nodeLost triggers - // then don't create markers - boolean createNodes = false; - try { - createNodes = zkStateReader.getAutoScalingConfig().hasTriggerForEvents(TriggerEventType.NODELOST); - } catch (KeeperException | InterruptedException e1) { - log.warn("Unable to read autoscaling.json", e1); - } - if (createNodes) { - byte[] json = Utils.toJSON(Collections.singletonMap("timestamp", getSolrCloudManager().getTimeSource().getEpochTimeNs())); - for (String n : oldNodes) { - String path = ZkStateReader.SOLR_AUTOSCALING_NODE_LOST_PATH + "/" + n; - - try { - zkClient.create(path, json, CreateMode.PERSISTENT, true); - } catch (KeeperException.NodeExistsException e) { - // someone else already created this node - ignore - } catch (KeeperException | InterruptedException e1) { - log.warn("Unable to register nodeLost path for {}", n, e1); - } - } - } return false; }; zkStateReader.registerLiveNodesListener(listener); @@ -1152,18 +1122,9 @@ public class ZkController implements Closeable { } String nodeName = getNodeName(); String nodePath = ZkStateReader.LIVE_NODES_ZKNODE + "/" + nodeName; - String nodeAddedPath = ZkStateReader.SOLR_AUTOSCALING_NODE_ADDED_PATH + "/" + nodeName; log.info("Register node as live in ZooKeeper:{}", nodePath); List ops = new ArrayList<>(2); ops.add(Op.create(nodePath, null, zkClient.getZkACLProvider().getACLsToAdd(nodePath), CreateMode.EPHEMERAL)); - // if there are nodeAdded triggers don't create nodeAdded markers - boolean createMarkerNode = zkStateReader.getAutoScalingConfig().hasTriggerForEvents(TriggerEventType.NODEADDED); - if (createMarkerNode && !zkClient.exists(nodeAddedPath, true)) { - // use EPHEMERAL so that it disappears if this node goes down - // and no other action is taken - byte[] json = Utils.toJSON(Collections.singletonMap("timestamp", TimeSource.NANO_TIME.getEpochTimeNs())); - ops.add(Op.create(nodeAddedPath, json, zkClient.getZkACLProvider().getACLsToAdd(nodeAddedPath), CreateMode.EPHEMERAL)); - } zkClient.multi(ops, true); } @@ -1173,11 +1134,9 @@ public class ZkController implements Closeable { } String nodeName = getNodeName(); String nodePath = ZkStateReader.LIVE_NODES_ZKNODE + "/" + nodeName; - String nodeAddedPath = ZkStateReader.SOLR_AUTOSCALING_NODE_ADDED_PATH + "/" + nodeName; log.info("Remove node as live in ZooKeeper:{}", nodePath); List ops = new ArrayList<>(2); ops.add(Op.delete(nodePath, -1)); - ops.add(Op.delete(nodeAddedPath, -1)); try { zkClient.multi(ops, true); diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/AddReplicaCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/AddReplicaCmd.java index 1c2146bbfb3..ca5639f4989 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/AddReplicaCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/AddReplicaCmd.java @@ -44,12 +44,10 @@ import java.util.List; import java.util.Locale; import java.util.Map; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; import org.apache.solr.client.solrj.cloud.SolrCloudManager; -import org.apache.solr.client.solrj.cloud.autoscaling.PolicyHelper; import org.apache.solr.cloud.ActiveReplicaWatcher; import org.apache.solr.cloud.Overseer; import org.apache.solr.cloud.api.collections.OverseerCollectionMessageHandler.ShardRequestTracker; @@ -148,18 +146,11 @@ public class AddReplicaCmd implements OverseerCollectionMessageHandler.Cmd { } } - AtomicReference sessionWrapper = new AtomicReference<>(); - List createReplicas; - try { - createReplicas = buildReplicaPositions(ocmh.cloudManager, clusterState, collectionName, message, replicaTypesVsCount, sessionWrapper) + List createReplicas = buildReplicaPositions(ocmh.cloudManager, clusterState, collectionName, message, replicaTypesVsCount) .stream() .map(replicaPosition -> assignReplicaDetails(ocmh.cloudManager, clusterState, message, replicaPosition)) .collect(Collectors.toList()); - } finally { - if (sessionWrapper.get() != null) { - sessionWrapper.get().release(); - } - } + ShardHandler shardHandler = ocmh.shardHandlerFactory.getShardHandler(); ZkStateReader zkStateReader = ocmh.zkStateReader; @@ -339,8 +330,7 @@ public class AddReplicaCmd implements OverseerCollectionMessageHandler.Cmd { public static List buildReplicaPositions(SolrCloudManager cloudManager, ClusterState clusterState, String collectionName, ZkNodeProps message, - EnumMap replicaTypeVsCount, - AtomicReference< PolicyHelper.SessionWrapper> sessionWrapper) throws IOException, InterruptedException { + EnumMap replicaTypeVsCount) throws IOException, InterruptedException { boolean skipCreateReplicaInClusterState = message.getBool(SKIP_CREATE_REPLICA_IN_CLUSTER_STATE, false); boolean skipNodeAssignment = message.getBool(CollectionAdminParams.SKIP_NODE_ASSIGNMENT, false); String sliceName = message.getStr(SHARD_ID_PROP); @@ -365,7 +355,6 @@ public class AddReplicaCmd implements OverseerCollectionMessageHandler.Cmd { positions = Assign.getNodesForNewReplicas(clusterState, collection.getName(), sliceName, numNrtReplicas, numTlogReplicas, numPullReplicas, createNodeSetStr, cloudManager); - sessionWrapper.set(PolicyHelper.getLastSessionWrapper(true)); } if (positions == null) { diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/Assign.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/Assign.java index 98a399a79d5..909b3eddc99 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/Assign.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/Assign.java @@ -16,6 +16,10 @@ */ package org.apache.solr.cloud.api.collections; +import static org.apache.solr.cloud.api.collections.OverseerCollectionMessageHandler.CREATE_NODE_SET; +import static org.apache.solr.common.cloud.DocCollection.SNITCH; +import static org.apache.solr.common.cloud.ZkStateReader.CORE_NAME_PROP; + import java.io.IOException; import java.lang.invoke.MethodHandles; import java.util.ArrayList; @@ -29,19 +33,15 @@ import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Objects; -import java.util.Optional; import java.util.Random; import java.util.Set; import java.util.stream.Collectors; -import com.google.common.collect.ImmutableMap; import org.apache.solr.client.solrj.cloud.DistribStateManager; import org.apache.solr.client.solrj.cloud.SolrCloudManager; -import org.apache.solr.client.solrj.cloud.autoscaling.AlreadyExistsException; -import org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig; -import org.apache.solr.client.solrj.cloud.autoscaling.BadVersionException; -import org.apache.solr.client.solrj.cloud.autoscaling.PolicyHelper; -import org.apache.solr.client.solrj.cloud.autoscaling.VersionedData; +import org.apache.solr.client.solrj.cloud.AlreadyExistsException; +import org.apache.solr.client.solrj.cloud.BadVersionException; +import org.apache.solr.client.solrj.cloud.VersionedData; import org.apache.solr.cloud.rule.ReplicaAssigner; import org.apache.solr.cloud.rule.Rule; import org.apache.solr.common.SolrException; @@ -52,19 +52,14 @@ import org.apache.solr.common.cloud.ReplicaPosition; import org.apache.solr.common.cloud.Slice; import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.cloud.ZkStateReader; -import org.apache.solr.common.params.CollectionAdminParams; import org.apache.solr.common.util.StrUtils; -import org.apache.solr.common.util.Utils; import org.apache.solr.util.NumberUtils; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static org.apache.solr.client.solrj.cloud.autoscaling.Policy.POLICY; -import static org.apache.solr.cloud.api.collections.OverseerCollectionMessageHandler.CREATE_NODE_SET; -import static org.apache.solr.common.cloud.DocCollection.SNITCH; -import static org.apache.solr.common.cloud.ZkStateReader.CORE_NAME_PROP; +import com.google.common.collect.ImmutableMap; public class Assign { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); @@ -255,57 +250,6 @@ public class Assign { return nodeList; } - /** - * Note: where possible, the {@link #usePolicyFramework(DocCollection, SolrCloudManager)} method should - * be used instead of this method - * - * @return true if autoscaling policy framework should be used for replica placement - */ - public static boolean usePolicyFramework(SolrCloudManager cloudManager) throws IOException, InterruptedException { - Objects.requireNonNull(cloudManager, "The SolrCloudManager instance cannot be null"); - return usePolicyFramework(Optional.empty(), cloudManager); - } - - /** - * @return true if auto scaling policy framework should be used for replica placement - * for this collection, otherwise false - */ - public static boolean usePolicyFramework(DocCollection collection, SolrCloudManager cloudManager) - throws IOException, InterruptedException { - Objects.requireNonNull(collection, "The DocCollection instance cannot be null"); - Objects.requireNonNull(cloudManager, "The SolrCloudManager instance cannot be null"); - return usePolicyFramework(Optional.of(collection), cloudManager); - } - - @SuppressWarnings({"unchecked"}) - private static boolean usePolicyFramework(Optional collection, SolrCloudManager cloudManager) throws IOException, InterruptedException { - boolean useLegacyAssignment = true; - Map clusterProperties = cloudManager.getClusterStateProvider().getClusterProperties(); - if (clusterProperties.containsKey(CollectionAdminParams.DEFAULTS)) { - Map defaults = (Map) clusterProperties.get(CollectionAdminParams.DEFAULTS); - Map collectionDefaults = (Map) defaults.getOrDefault(CollectionAdminParams.CLUSTER, Collections.emptyMap()); - useLegacyAssignment = Boolean.parseBoolean(collectionDefaults.getOrDefault(CollectionAdminParams.USE_LEGACY_REPLICA_ASSIGNMENT, "true").toString()); - } - - if (!useLegacyAssignment) { - // if legacy assignment is not selected then autoscaling is always available through the implicit policy/preferences - return true; - } - - // legacy assignment is turned on, which means we must look at the actual autoscaling config - // to determine whether policy framework can be used or not for this collection - - AutoScalingConfig autoScalingConfig = cloudManager.getDistribStateManager().getAutoScalingConfig(); - // if no autoscaling configuration exists then obviously we cannot use the policy framework - if (autoScalingConfig.getPolicy().isEmpty()) return false; - // do custom preferences exist - if (!autoScalingConfig.getPolicy().hasEmptyPreferences()) return true; - // does a cluster policy exist - if (!autoScalingConfig.getPolicy().getClusterPolicy().isEmpty()) return true; - // finally we check if the current collection has a policy - return !collection.isPresent() || collection.get().getPolicyName() != null; - } - static class ReplicaCount { public final String nodeName; public int thisCollectionNodes = 0; @@ -358,44 +302,6 @@ public class Assign { return assignStrategy.assign(cloudManager, assignRequest); } - public static List getPositionsUsingPolicy(String collName, List shardNames, - int nrtReplicas, - int tlogReplicas, - int pullReplicas, - String policyName, SolrCloudManager cloudManager, - List nodesList) throws IOException, InterruptedException, AssignmentException { - log.debug("shardnames {} NRT {} TLOG {} PULL {} , policy {}, nodeList {}", shardNames, nrtReplicas, tlogReplicas, pullReplicas, policyName, nodesList); - List replicaPositions = null; - AutoScalingConfig autoScalingConfig = cloudManager.getDistribStateManager().getAutoScalingConfig(); - try { - Map kvMap = Collections.singletonMap(collName, policyName); - replicaPositions = PolicyHelper.getReplicaLocations( - collName, - autoScalingConfig, - cloudManager, - kvMap, - shardNames, - nrtReplicas, - tlogReplicas, - pullReplicas, - nodesList); - return replicaPositions; - } catch (Exception e) { - throw new AssignmentException("Error getting replica locations : " + e.getMessage(), e); - } finally { - if (log.isTraceEnabled()) { - if (replicaPositions != null) { - if (log.isTraceEnabled()) { - log.trace("REPLICA_POSITIONS: {}", Utils.toJSONString(Utils.getDeepCopy(replicaPositions, 7, true))); - } - } - if (log.isTraceEnabled()) { - log.trace("AUTOSCALING_CONF: {}", Utils.toJSONString(autoScalingConfig)); - } - } - } - } - static HashMap getNodeNameVsShardCount(String collectionName, ClusterState clusterState, List createNodeList) { HashMap nodeNameVsShardCount = new HashMap<>(); @@ -637,22 +543,6 @@ public class Assign { } } - public static class PolicyBasedAssignStrategy implements AssignStrategy { - public String policyName; - - public PolicyBasedAssignStrategy(String policyName) { - this.policyName = policyName; - } - - @Override - public List assign(SolrCloudManager solrCloudManager, AssignRequest assignRequest) throws Assign.AssignmentException, IOException, InterruptedException { - return Assign.getPositionsUsingPolicy(assignRequest.collectionName, - assignRequest.shardNames, assignRequest.numNrtReplicas, - assignRequest.numTlogReplicas, assignRequest.numPullReplicas, - policyName, solrCloudManager, assignRequest.nodes); - } - } - public static class AssignStrategyFactory { public SolrCloudManager solrCloudManager; @@ -663,19 +553,16 @@ public class Assign { public AssignStrategy create(ClusterState clusterState, DocCollection collection) throws IOException, InterruptedException { @SuppressWarnings({"unchecked", "rawtypes"}) List ruleMaps = (List) collection.get("rule"); - String policyName = collection.getStr(POLICY); @SuppressWarnings({"rawtypes"}) List snitches = (List) collection.get(SNITCH); Strategy strategy = null; - if ((ruleMaps == null || ruleMaps.isEmpty()) && !usePolicyFramework(collection, solrCloudManager)) { - strategy = Strategy.LEGACY; - } else if (ruleMaps != null && !ruleMaps.isEmpty()) { + if (ruleMaps != null && !ruleMaps.isEmpty()) { strategy = Strategy.RULES; } else { - strategy = Strategy.POLICY; + strategy = Strategy.LEGACY; } - + switch (strategy) { case LEGACY: return new LegacyAssignStrategy(); @@ -683,15 +570,13 @@ public class Assign { List rules = new ArrayList<>(); for (Object map : ruleMaps) rules.add(new Rule((Map) map)); return new RulesBasedAssignStrategy(rules, snitches, clusterState); - case POLICY: - return new PolicyBasedAssignStrategy(policyName); default: throw new Assign.AssignmentException("Unknown strategy type: " + strategy); } } private enum Strategy { - LEGACY, RULES, POLICY; + LEGACY, RULES; } } } diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java index c3e9a3e64fb..cfad397fbd5 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java @@ -31,15 +31,13 @@ import java.util.NoSuchElementException; import java.util.Properties; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; -import java.util.concurrent.atomic.AtomicReference; import org.apache.solr.client.solrj.cloud.DistribStateManager; import org.apache.solr.client.solrj.cloud.SolrCloudManager; -import org.apache.solr.client.solrj.cloud.autoscaling.AlreadyExistsException; -import org.apache.solr.client.solrj.cloud.autoscaling.BadVersionException; -import org.apache.solr.client.solrj.cloud.autoscaling.NotEmptyException; -import org.apache.solr.client.solrj.cloud.autoscaling.PolicyHelper; -import org.apache.solr.client.solrj.cloud.autoscaling.VersionedData; +import org.apache.solr.client.solrj.cloud.AlreadyExistsException; +import org.apache.solr.client.solrj.cloud.BadVersionException; +import org.apache.solr.client.solrj.cloud.NotEmptyException; +import org.apache.solr.client.solrj.cloud.VersionedData; import org.apache.solr.cloud.Overseer; import org.apache.solr.cloud.ZkController; import org.apache.solr.cloud.api.collections.OverseerCollectionMessageHandler.ShardRequestTracker; @@ -147,7 +145,7 @@ public class CreateCollectionCmd implements OverseerCollectionMessageHandler.Cmd List shardNames = populateShardNames(message, router); checkReplicaTypes(message); - AtomicReference sessionWrapper = new AtomicReference<>(); + try { @@ -187,7 +185,7 @@ public class CreateCollectionCmd implements OverseerCollectionMessageHandler.Cmd List replicaPositions = null; try { - replicaPositions = buildReplicaPositions(ocmh.cloudManager, clusterState, clusterState.getCollection(collectionName), message, shardNames, sessionWrapper); + replicaPositions = buildReplicaPositions(ocmh.cloudManager, clusterState, clusterState.getCollection(collectionName), message, shardNames); } catch (Assign.AssignmentException e) { ZkNodeProps deleteMessage = new ZkNodeProps("name", collectionName); new DeleteCollectionCmd(ocmh).call(clusterState, deleteMessage, results); @@ -335,16 +333,13 @@ public class CreateCollectionCmd implements OverseerCollectionMessageHandler.Cmd throw ex; } catch (Exception ex) { throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, null, ex); - } finally { - if (sessionWrapper.get() != null) sessionWrapper.get().release(); } } public static List buildReplicaPositions(SolrCloudManager cloudManager, ClusterState clusterState, DocCollection docCollection, ZkNodeProps message, - List shardNames, - AtomicReference sessionWrapper) throws IOException, InterruptedException, Assign.AssignmentException { + List shardNames) throws IOException, InterruptedException, Assign.AssignmentException { final String collectionName = message.getStr(NAME); // look at the replication factor and see if it matches reality // if it does not, find best nodes to create more cores @@ -386,7 +381,6 @@ public class CreateCollectionCmd implements OverseerCollectionMessageHandler.Cmd Assign.AssignStrategyFactory assignStrategyFactory = new Assign.AssignStrategyFactory(cloudManager); Assign.AssignStrategy assignStrategy = assignStrategyFactory.create(clusterState, docCollection); replicaPositions = assignStrategy.assign(cloudManager, assignRequest); - sessionWrapper.set(PolicyHelper.getLastSessionWrapper(true)); } return replicaPositions; } diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerCollectionMessageHandler.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerCollectionMessageHandler.java index e1fca930be4..74112b48acf 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerCollectionMessageHandler.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerCollectionMessageHandler.java @@ -38,8 +38,8 @@ import org.apache.solr.client.solrj.SolrResponse; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.cloud.DistribStateManager; import org.apache.solr.client.solrj.cloud.SolrCloudManager; -import org.apache.solr.client.solrj.cloud.autoscaling.AlreadyExistsException; -import org.apache.solr.client.solrj.cloud.autoscaling.BadVersionException; +import org.apache.solr.client.solrj.cloud.AlreadyExistsException; +import org.apache.solr.client.solrj.cloud.BadVersionException; import org.apache.solr.client.solrj.impl.BaseHttpSolrClient; import org.apache.solr.client.solrj.impl.HttpSolrClient; import org.apache.solr.client.solrj.request.AbstractUpdateRequest; @@ -91,7 +91,6 @@ import org.apache.zookeeper.KeeperException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static org.apache.solr.client.solrj.cloud.autoscaling.Policy.POLICY; import static org.apache.solr.common.cloud.DocCollection.SNITCH; import static org.apache.solr.common.cloud.ZkStateReader.BASE_URL_PROP; import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP; @@ -146,9 +145,7 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler, ZkStateReader.NRT_REPLICAS, "1", ZkStateReader.TLOG_REPLICAS, "0", ZkStateReader.PULL_REPLICAS, "0", - ZkStateReader.AUTO_ADD_REPLICAS, "false", DocCollection.RULE, null, - POLICY, null, SNITCH, null, WITH_COLLECTION, null, COLOCATED_WITH, null)); @@ -238,7 +235,6 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler, .put(ADDREPLICA, new AddReplicaCmd(this)) .put(MOVEREPLICA, new MoveReplicaCmd(this)) .put(REINDEXCOLLECTION, new ReindexCollectionCmd(this)) - .put(UTILIZENODE, new UtilizeNodeCmd(this)) .put(RENAME, new RenameCmd(this)) .build() ; diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/ReindexCollectionCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/ReindexCollectionCmd.java index d98d50a3f4d..c45c772e518 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/ReindexCollectionCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/ReindexCollectionCmd.java @@ -35,7 +35,6 @@ import com.google.common.annotations.VisibleForTesting; import org.apache.http.client.HttpClient; import org.apache.solr.client.solrj.SolrResponse; import org.apache.solr.client.solrj.cloud.DistribStateManager; -import org.apache.solr.client.solrj.cloud.autoscaling.Policy; import org.apache.solr.client.solrj.impl.CloudSolrClient; import org.apache.solr.client.solrj.impl.HttpSolrClient; import org.apache.solr.client.solrj.request.CollectionAdminRequest; @@ -110,10 +109,8 @@ public class ReindexCollectionCmd implements OverseerCollectionMessageHandler.Cm ZkStateReader.TLOG_REPLICAS, ZkStateReader.REPLICATION_FACTOR, "shards", - Policy.POLICY, CollectionAdminParams.CREATE_NODE_SET_PARAM, - CollectionAdminParams.CREATE_NODE_SET_SHUFFLE_PARAM, - ZkStateReader.AUTO_ADD_REPLICAS + CollectionAdminParams.CREATE_NODE_SET_SHUFFLE_PARAM ); private final OverseerCollectionMessageHandler ocmh; diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/ReplaceNodeCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/ReplaceNodeCmd.java index f1c1f8cc8a3..aa10bb1e4b7 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/ReplaceNodeCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/ReplaceNodeCmd.java @@ -27,9 +27,6 @@ import java.util.Locale; import java.util.Map; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicReference; - -import org.apache.solr.client.solrj.cloud.autoscaling.PolicyHelper; import org.apache.solr.cloud.ActiveReplicaWatcher; import org.apache.solr.common.SolrCloseableLatch; import org.apache.solr.common.SolrException; @@ -101,7 +98,6 @@ public class ReplaceNodeCmd implements OverseerCollectionMessageHandler.Cmd { SolrCloseableLatch countDownLatch = new SolrCloseableLatch(sourceReplicas.size(), ocmh); SolrCloseableLatch replicasToRecover = new SolrCloseableLatch(numLeaders, ocmh); - AtomicReference sessionWrapperRef = new AtomicReference<>(); try { for (ZkNodeProps sourceReplica : sourceReplicas) { @SuppressWarnings({"rawtypes"}) @@ -127,7 +123,6 @@ public class ReplaceNodeCmd implements OverseerCollectionMessageHandler.Cmd { Assign.AssignStrategyFactory assignStrategyFactory = new Assign.AssignStrategyFactory(ocmh.cloudManager); Assign.AssignStrategy assignStrategy = assignStrategyFactory.create(clusterState, clusterState.getCollection(sourceCollection)); targetNode = assignStrategy.assign(ocmh.cloudManager, assignRequest).get(0).node; - sessionWrapperRef.set(PolicyHelper.getLastSessionWrapper(true)); } ZkNodeProps msg = sourceReplica.plus("parallel", String.valueOf(parallel)).plus(CoreAdminParams.NODE, targetNode); if (async != null) msg.getProperties().put(ASYNC, async); @@ -185,8 +180,6 @@ public class ReplaceNodeCmd implements OverseerCollectionMessageHandler.Cmd { log.debug("Finished waiting for replicas to be added"); } } finally { - PolicyHelper.SessionWrapper sw = sessionWrapperRef.get(); - if (sw != null) sw.release(); } // now wait for leader replicas to recover log.debug("Waiting for {} leader replicas to recover", numLeaders); diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/RestoreCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/RestoreCmd.java index f80097913dc..552c40a0dea 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/RestoreCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/RestoreCmd.java @@ -35,7 +35,6 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; -import org.apache.solr.client.solrj.cloud.autoscaling.PolicyHelper; import org.apache.solr.cloud.Overseer; import org.apache.solr.cloud.api.collections.OverseerCollectionMessageHandler.ShardRequestTracker; import org.apache.solr.cloud.overseer.OverseerAction; @@ -110,15 +109,15 @@ public class RestoreCmd implements OverseerCollectionMessageHandler.Cmd { Object format = properties.get("stateFormat"); if (format != null && !"2".equals(format)) { throw new SolrException(ErrorCode.BAD_REQUEST, "Collection " + backupCollection + " is in stateFormat=" + format + - " no longer supported in Solr 9 and above. It can't be restored. If it originates in Solr 8 you can restore" + - " it there, migrate it to stateFormat=2 and backup again, it will then be restorable on Solr 9"); + " no longer supported in Solr 9 and above. It can't be restored. If it originates in Solr 8 you can restore" + + " it there, migrate it to stateFormat=2 and backup again, it will then be restorable on Solr 9"); } String backupCollectionAlias = properties.getProperty(BackupManager.COLLECTION_ALIAS_PROP); DocCollection backupCollectionState = backupMgr.readCollectionState(location, backupName, backupCollection); // Get the Solr nodes to restore a collection. final List nodeList = Assign.getLiveOrLiveAndCreateNodeSetList( - zkStateReader.getClusterState().getLiveNodes(), message, OverseerCollectionMessageHandler.RANDOM); + zkStateReader.getClusterState().getLiveNodes(), message, OverseerCollectionMessageHandler.RANDOM); int numShards = backupCollectionState.getActiveSlices().size(); @@ -136,7 +135,7 @@ public class RestoreCmd implements OverseerCollectionMessageHandler.Cmd { int numPullReplicas = getInt(message, PULL_REPLICAS, backupCollectionState.getNumPullReplicas(), 0); int totalReplicasPerShard = numNrtReplicas + numTlogReplicas + numPullReplicas; assert totalReplicasPerShard > 0; - + //Upload the configs String configName = (String) properties.get(CollectionAdminParams.COLL_CONF); String restoreConfigName = message.getStr(CollectionAdminParams.COLL_CONF, configName); @@ -149,7 +148,7 @@ public class RestoreCmd implements OverseerCollectionMessageHandler.Cmd { } log.info("Starting restore into collection={} with backup_name={} at location={}", restoreCollectionName, backupName, - location); + location); //Create core-less collection { @@ -190,7 +189,7 @@ public class RestoreCmd implements OverseerCollectionMessageHandler.Cmd { Map newSlices = new LinkedHashMap<>(backupSlices.size()); for (Slice backupSlice : backupSlices) { newSlices.put(backupSlice.getName(), - new Slice(backupSlice.getName(), Collections.emptyMap(), backupSlice.getProperties(),restoreCollectionName)); + new Slice(backupSlice.getName(), Collections.emptyMap(), backupSlice.getProperties(), restoreCollectionName)); } propMap.put(OverseerCollectionMessageHandler.SHARDS_PROP, newSlices); } @@ -222,216 +221,211 @@ public class RestoreCmd implements OverseerCollectionMessageHandler.Cmd { List sliceNames = new ArrayList<>(); restoreCollection.getSlices().forEach(x -> sliceNames.add(x.getName())); - PolicyHelper.SessionWrapper sessionWrapper = null; - try { - Assign.AssignRequest assignRequest = new Assign.AssignRequestBuilder() - .forCollection(restoreCollectionName) - .forShard(sliceNames) - .assignNrtReplicas(numNrtReplicas) - .assignTlogReplicas(numTlogReplicas) - .assignPullReplicas(numPullReplicas) - .onNodes(nodeList) - .build(); - Assign.AssignStrategyFactory assignStrategyFactory = new Assign.AssignStrategyFactory(ocmh.cloudManager); - Assign.AssignStrategy assignStrategy = assignStrategyFactory.create(clusterState, restoreCollection); - List replicaPositions = assignStrategy.assign(ocmh.cloudManager, assignRequest); - sessionWrapper = PolicyHelper.getLastSessionWrapper(true); + Assign.AssignRequest assignRequest = new Assign.AssignRequestBuilder() + .forCollection(restoreCollectionName) + .forShard(sliceNames) + .assignNrtReplicas(numNrtReplicas) + .assignTlogReplicas(numTlogReplicas) + .assignPullReplicas(numPullReplicas) + .onNodes(nodeList) + .build(); + Assign.AssignStrategyFactory assignStrategyFactory = new Assign.AssignStrategyFactory(ocmh.cloudManager); + Assign.AssignStrategy assignStrategy = assignStrategyFactory.create(clusterState, restoreCollection); + List replicaPositions = assignStrategy.assign(ocmh.cloudManager, assignRequest); - CountDownLatch countDownLatch = new CountDownLatch(restoreCollection.getSlices().size()); + CountDownLatch countDownLatch = new CountDownLatch(restoreCollection.getSlices().size()); - //Create one replica per shard and copy backed up data to it - for (Slice slice : restoreCollection.getSlices()) { - if (log.isInfoEnabled()) { - log.info("Adding replica for shard={} collection={} ", slice.getName(), restoreCollection); + //Create one replica per shard and copy backed up data to it + for (Slice slice : restoreCollection.getSlices()) { + if (log.isInfoEnabled()) { + log.info("Adding replica for shard={} collection={} ", slice.getName(), restoreCollection); + } + HashMap propMap = new HashMap<>(); + propMap.put(Overseer.QUEUE_OPERATION, CREATESHARD); + propMap.put(COLLECTION_PROP, restoreCollectionName); + propMap.put(SHARD_ID_PROP, slice.getName()); + + if (numNrtReplicas >= 1) { + propMap.put(REPLICA_TYPE, Replica.Type.NRT.name()); + } else if (numTlogReplicas >= 1) { + propMap.put(REPLICA_TYPE, Replica.Type.TLOG.name()); + } else { + throw new SolrException(ErrorCode.BAD_REQUEST, "Unexpected number of replicas, replicationFactor, " + + Replica.Type.NRT + " or " + Replica.Type.TLOG + " must be greater than 0"); + } + + // Get the first node matching the shard to restore in + String node; + for (ReplicaPosition replicaPosition : replicaPositions) { + if (Objects.equals(replicaPosition.shard, slice.getName())) { + node = replicaPosition.node; + propMap.put(CoreAdminParams.NODE, node); + replicaPositions.remove(replicaPosition); + break; } - HashMap propMap = new HashMap<>(); - propMap.put(Overseer.QUEUE_OPERATION, CREATESHARD); - propMap.put(COLLECTION_PROP, restoreCollectionName); - propMap.put(SHARD_ID_PROP, slice.getName()); + } - if (numNrtReplicas >= 1) { - propMap.put(REPLICA_TYPE, Replica.Type.NRT.name()); - } else if (numTlogReplicas >= 1) { - propMap.put(REPLICA_TYPE, Replica.Type.TLOG.name()); + // add async param + if (asyncId != null) { + propMap.put(ASYNC, asyncId); + } + ocmh.addPropertyParams(message, propMap); + final NamedList addReplicaResult = new NamedList(); + ocmh.addReplica(clusterState, new ZkNodeProps(propMap), addReplicaResult, () -> { + Object addResultFailure = addReplicaResult.get("failure"); + if (addResultFailure != null) { + SimpleOrderedMap failure = (SimpleOrderedMap) results.get("failure"); + if (failure == null) { + failure = new SimpleOrderedMap(); + results.add("failure", failure); + } + failure.addAll((NamedList) addResultFailure); } else { - throw new SolrException(ErrorCode.BAD_REQUEST, "Unexpected number of replicas, replicationFactor, " + - Replica.Type.NRT + " or " + Replica.Type.TLOG + " must be greater than 0"); - } - - // Get the first node matching the shard to restore in - String node; - for (ReplicaPosition replicaPosition : replicaPositions) { - if (Objects.equals(replicaPosition.shard, slice.getName())) { - node = replicaPosition.node; - propMap.put(CoreAdminParams.NODE, node); - replicaPositions.remove(replicaPosition); - break; + SimpleOrderedMap success = (SimpleOrderedMap) results.get("success"); + if (success == null) { + success = new SimpleOrderedMap(); + results.add("success", success); } + success.addAll((NamedList) addReplicaResult.get("success")); } - - // add async param - if (asyncId != null) { - propMap.put(ASYNC, asyncId); - } - ocmh.addPropertyParams(message, propMap); - final NamedList addReplicaResult = new NamedList(); - ocmh.addReplica(clusterState, new ZkNodeProps(propMap), addReplicaResult, () -> { - Object addResultFailure = addReplicaResult.get("failure"); - if (addResultFailure != null) { - SimpleOrderedMap failure = (SimpleOrderedMap) results.get("failure"); - if (failure == null) { - failure = new SimpleOrderedMap(); - results.add("failure", failure); - } - failure.addAll((NamedList) addResultFailure); - } else { - SimpleOrderedMap success = (SimpleOrderedMap) results.get("success"); - if (success == null) { - success = new SimpleOrderedMap(); - results.add("success", success); - } - success.addAll((NamedList) addReplicaResult.get("success")); - } - countDownLatch.countDown(); - }); - } - - boolean allIsDone = countDownLatch.await(1, TimeUnit.HOURS); - if (!allIsDone) { - throw new TimeoutException("Initial replicas were not created within 1 hour. Timing out."); - } - Object failures = results.get("failure"); - if (failures != null && ((SimpleOrderedMap) failures).size() > 0) { - log.error("Restore failed to create initial replicas."); - ocmh.cleanupCollection(restoreCollectionName, new NamedList()); - return; - } - - //refresh the location copy of collection state - restoreCollection = zkStateReader.getClusterState().getCollection(restoreCollectionName); - - { - ShardRequestTracker shardRequestTracker = ocmh.asyncRequestTracker(asyncId); - // Copy data from backed up index to each replica - for (Slice slice : restoreCollection.getSlices()) { - ModifiableSolrParams params = new ModifiableSolrParams(); - params.set(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.RESTORECORE.toString()); - params.set(NAME, "snapshot." + slice.getName()); - params.set(CoreAdminParams.BACKUP_LOCATION, backupPath.toASCIIString()); - params.set(CoreAdminParams.BACKUP_REPOSITORY, repo); - shardRequestTracker.sliceCmd(clusterState, params, null, slice, shardHandler); - } - shardRequestTracker.processResponses(new NamedList(), shardHandler, true, "Could not restore core"); - } - - { - ShardRequestTracker shardRequestTracker = ocmh.asyncRequestTracker(asyncId); - - for (Slice s : restoreCollection.getSlices()) { - for (Replica r : s.getReplicas()) { - String nodeName = r.getNodeName(); - String coreNodeName = r.getCoreName(); - Replica.State stateRep = r.getState(); - - if (log.isDebugEnabled()) { - log.debug("Calling REQUESTAPPLYUPDATES on: nodeName={}, coreNodeName={}, state={}", nodeName, coreNodeName, - stateRep.name()); - } - - ModifiableSolrParams params = new ModifiableSolrParams(); - params.set(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.REQUESTAPPLYUPDATES.toString()); - params.set(CoreAdminParams.NAME, coreNodeName); - - shardRequestTracker.sendShardRequest(nodeName, params, shardHandler); - } - - shardRequestTracker.processResponses(new NamedList(), shardHandler, true, - "REQUESTAPPLYUPDATES calls did not succeed"); - } - } - - //Mark all shards in ACTIVE STATE - { - HashMap propMap = new HashMap<>(); - propMap.put(Overseer.QUEUE_OPERATION, OverseerAction.UPDATESHARDSTATE.toLower()); - propMap.put(ZkStateReader.COLLECTION_PROP, restoreCollectionName); - for (Slice shard : restoreCollection.getSlices()) { - propMap.put(shard.getName(), Slice.State.ACTIVE.toString()); - } - ocmh.overseer.offerStateUpdate((Utils.toJSON(new ZkNodeProps(propMap)))); - } - - if (totalReplicasPerShard > 1) { - if (log.isInfoEnabled()) { - log.info("Adding replicas to restored collection={}", restoreCollection.getName()); - } - for (Slice slice : restoreCollection.getSlices()) { - - //Add the remaining replicas for each shard, considering it's type - int createdNrtReplicas = 0, createdTlogReplicas = 0, createdPullReplicas = 0; - - // We already created either a NRT or an TLOG replica as leader - if (numNrtReplicas > 0) { - createdNrtReplicas++; - } else if (createdTlogReplicas > 0) { - createdTlogReplicas++; - } - - for (int i = 1; i < totalReplicasPerShard; i++) { - Replica.Type typeToCreate; - if (createdNrtReplicas < numNrtReplicas) { - createdNrtReplicas++; - typeToCreate = Replica.Type.NRT; - } else if (createdTlogReplicas < numTlogReplicas) { - createdTlogReplicas++; - typeToCreate = Replica.Type.TLOG; - } else { - createdPullReplicas++; - typeToCreate = Replica.Type.PULL; - assert createdPullReplicas <= numPullReplicas: "Unexpected number of replicas"; - } - - if (log.isDebugEnabled()) { - log.debug("Adding replica for shard={} collection={} of type {} ", slice.getName(), restoreCollection, typeToCreate); - } - HashMap propMap = new HashMap<>(); - propMap.put(COLLECTION_PROP, restoreCollectionName); - propMap.put(SHARD_ID_PROP, slice.getName()); - propMap.put(REPLICA_TYPE, typeToCreate.name()); - - // Get the first node matching the shard to restore in - String node; - for (ReplicaPosition replicaPosition : replicaPositions) { - if (Objects.equals(replicaPosition.shard, slice.getName())) { - node = replicaPosition.node; - propMap.put(CoreAdminParams.NODE, node); - replicaPositions.remove(replicaPosition); - break; - } - } - - // add async param - if (asyncId != null) { - propMap.put(ASYNC, asyncId); - } - ocmh.addPropertyParams(message, propMap); - - ocmh.addReplica(zkStateReader.getClusterState(), new ZkNodeProps(propMap), results, null); - } - } - } - - if (backupCollectionAlias != null && !backupCollectionAlias.equals(backupCollection)) { - log.debug("Restoring alias {} -> {}", backupCollectionAlias, backupCollection); - ocmh.zkStateReader.aliasesManager - .applyModificationAndExportToZk(a -> a.cloneWithCollectionAlias(backupCollectionAlias, backupCollection)); - } - - log.info("Completed restoring collection={} backupName={}", restoreCollection, backupName); - } finally { - if (sessionWrapper != null) sessionWrapper.release(); + countDownLatch.countDown(); + }); } + + boolean allIsDone = countDownLatch.await(1, TimeUnit.HOURS); + if (!allIsDone) { + throw new TimeoutException("Initial replicas were not created within 1 hour. Timing out."); + } + Object failures = results.get("failure"); + if (failures != null && ((SimpleOrderedMap) failures).size() > 0) { + log.error("Restore failed to create initial replicas."); + ocmh.cleanupCollection(restoreCollectionName, new NamedList()); + return; + } + + //refresh the location copy of collection state + restoreCollection = zkStateReader.getClusterState().getCollection(restoreCollectionName); + + { + ShardRequestTracker shardRequestTracker = ocmh.asyncRequestTracker(asyncId); + // Copy data from backed up index to each replica + for (Slice slice : restoreCollection.getSlices()) { + ModifiableSolrParams params = new ModifiableSolrParams(); + params.set(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.RESTORECORE.toString()); + params.set(NAME, "snapshot." + slice.getName()); + params.set(CoreAdminParams.BACKUP_LOCATION, backupPath.toASCIIString()); + params.set(CoreAdminParams.BACKUP_REPOSITORY, repo); + shardRequestTracker.sliceCmd(clusterState, params, null, slice, shardHandler); + } + shardRequestTracker.processResponses(new NamedList(), shardHandler, true, "Could not restore core"); + } + + { + ShardRequestTracker shardRequestTracker = ocmh.asyncRequestTracker(asyncId); + + for (Slice s : restoreCollection.getSlices()) { + for (Replica r : s.getReplicas()) { + String nodeName = r.getNodeName(); + String coreNodeName = r.getCoreName(); + Replica.State stateRep = r.getState(); + + if (log.isDebugEnabled()) { + log.debug("Calling REQUESTAPPLYUPDATES on: nodeName={}, coreNodeName={}, state={}", nodeName, coreNodeName, + stateRep.name()); + } + + ModifiableSolrParams params = new ModifiableSolrParams(); + params.set(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.REQUESTAPPLYUPDATES.toString()); + params.set(CoreAdminParams.NAME, coreNodeName); + + shardRequestTracker.sendShardRequest(nodeName, params, shardHandler); + } + + shardRequestTracker.processResponses(new NamedList(), shardHandler, true, + "REQUESTAPPLYUPDATES calls did not succeed"); + } + } + + //Mark all shards in ACTIVE STATE + { + HashMap propMap = new HashMap<>(); + propMap.put(Overseer.QUEUE_OPERATION, OverseerAction.UPDATESHARDSTATE.toLower()); + propMap.put(ZkStateReader.COLLECTION_PROP, restoreCollectionName); + for (Slice shard : restoreCollection.getSlices()) { + propMap.put(shard.getName(), Slice.State.ACTIVE.toString()); + } + ocmh.overseer.offerStateUpdate((Utils.toJSON(new ZkNodeProps(propMap)))); + } + + if (totalReplicasPerShard > 1) { + if (log.isInfoEnabled()) { + log.info("Adding replicas to restored collection={}", restoreCollection.getName()); + } + for (Slice slice : restoreCollection.getSlices()) { + + //Add the remaining replicas for each shard, considering it's type + int createdNrtReplicas = 0, createdTlogReplicas = 0, createdPullReplicas = 0; + + // We already created either a NRT or an TLOG replica as leader + if (numNrtReplicas > 0) { + createdNrtReplicas++; + } else if (createdTlogReplicas > 0) { + createdTlogReplicas++; + } + + for (int i = 1; i < totalReplicasPerShard; i++) { + Replica.Type typeToCreate; + if (createdNrtReplicas < numNrtReplicas) { + createdNrtReplicas++; + typeToCreate = Replica.Type.NRT; + } else if (createdTlogReplicas < numTlogReplicas) { + createdTlogReplicas++; + typeToCreate = Replica.Type.TLOG; + } else { + createdPullReplicas++; + typeToCreate = Replica.Type.PULL; + assert createdPullReplicas <= numPullReplicas : "Unexpected number of replicas"; + } + + if (log.isDebugEnabled()) { + log.debug("Adding replica for shard={} collection={} of type {} ", slice.getName(), restoreCollection, typeToCreate); + } + HashMap propMap = new HashMap<>(); + propMap.put(COLLECTION_PROP, restoreCollectionName); + propMap.put(SHARD_ID_PROP, slice.getName()); + propMap.put(REPLICA_TYPE, typeToCreate.name()); + + // Get the first node matching the shard to restore in + String node; + for (ReplicaPosition replicaPosition : replicaPositions) { + if (Objects.equals(replicaPosition.shard, slice.getName())) { + node = replicaPosition.node; + propMap.put(CoreAdminParams.NODE, node); + replicaPositions.remove(replicaPosition); + break; + } + } + + // add async param + if (asyncId != null) { + propMap.put(ASYNC, asyncId); + } + ocmh.addPropertyParams(message, propMap); + + ocmh.addReplica(zkStateReader.getClusterState(), new ZkNodeProps(propMap), results, null); + } + } + } + + if (backupCollectionAlias != null && !backupCollectionAlias.equals(backupCollection)) { + log.debug("Restoring alias {} -> {}", backupCollectionAlias, backupCollection); + ocmh.zkStateReader.aliasesManager + .applyModificationAndExportToZk(a -> a.cloneWithCollectionAlias(backupCollectionAlias, backupCollection)); + } + + log.info("Completed restoring collection={} backupName={}", restoreCollection, backupName); + } private int getInt(ZkNodeProps message, String propertyName, Integer count, int defaultValue) { diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java index 0df9d09f03b..495bf65985e 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java @@ -18,41 +18,17 @@ package org.apache.solr.cloud.api.collections; -import java.lang.invoke.MethodHandles; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.NoSuchElementException; -import java.util.Set; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicReference; - import org.apache.solr.client.solrj.cloud.DistribStateManager; import org.apache.solr.client.solrj.cloud.NodeStateProvider; import org.apache.solr.client.solrj.cloud.SolrCloudManager; -import org.apache.solr.client.solrj.cloud.autoscaling.PolicyHelper; -import org.apache.solr.client.solrj.cloud.autoscaling.Variable.Type; -import org.apache.solr.client.solrj.cloud.autoscaling.VersionedData; +import org.apache.solr.client.solrj.cloud.VersionedData; import org.apache.solr.client.solrj.request.CoreAdminRequest; import org.apache.solr.cloud.Overseer; import org.apache.solr.cloud.api.collections.OverseerCollectionMessageHandler.ShardRequestTracker; import org.apache.solr.cloud.overseer.OverseerAction; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException.ErrorCode; -import org.apache.solr.common.cloud.ClusterState; -import org.apache.solr.common.cloud.CompositeIdRouter; -import org.apache.solr.common.cloud.DocCollection; -import org.apache.solr.common.cloud.DocRouter; -import org.apache.solr.common.cloud.PlainIdRouter; -import org.apache.solr.common.cloud.Replica; -import org.apache.solr.common.cloud.ReplicaPosition; -import org.apache.solr.common.cloud.Slice; -import org.apache.solr.common.cloud.ZkNodeProps; -import org.apache.solr.common.cloud.ZkStateReader; +import org.apache.solr.common.cloud.*; import org.apache.solr.common.cloud.rule.ImplicitSnitch; import org.apache.solr.common.params.CommonAdminParams; import org.apache.solr.common.params.CommonParams; @@ -71,13 +47,15 @@ import org.apache.zookeeper.data.Stat; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP; -import static org.apache.solr.common.cloud.ZkStateReader.REPLICA_TYPE; -import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP; +import java.lang.invoke.MethodHandles; +import java.util.*; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +import static org.apache.solr.client.solrj.impl.SolrClientNodeStateProvider.Variable.CORE_IDX; +import static org.apache.solr.common.cloud.ZkStateReader.*; import static org.apache.solr.common.params.CollectionAdminParams.FOLLOW_ALIASES; -import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDREPLICA; -import static org.apache.solr.common.params.CollectionParams.CollectionAction.CREATESHARD; -import static org.apache.solr.common.params.CollectionParams.CollectionAction.DELETESHARD; +import static org.apache.solr.common.params.CollectionParams.CollectionAction.*; import static org.apache.solr.common.params.CommonAdminParams.ASYNC; import static org.apache.solr.common.params.CommonAdminParams.NUM_SUB_SHARDS; @@ -135,7 +113,6 @@ public class SplitShardCmd implements OverseerCollectionMessageHandler.Cmd { String splitKey = message.getStr("split.key"); DocCollection collection = clusterState.getCollection(collectionName); - PolicyHelper.SessionWrapper sessionWrapper = null; Slice parentSlice = getParentSlice(clusterState, collectionName, slice, splitKey); if (parentSlice.getState() != Slice.State.ACTIVE) { @@ -427,7 +404,6 @@ public class SplitShardCmd implements OverseerCollectionMessageHandler.Cmd { log.debug("Successfully applied buffered updates on : {}", subShardNames); // Replica creation for the new Slices - // replica placement is controlled by the autoscaling policy framework Set nodes = clusterState.getLiveNodes(); List nodeList = new ArrayList<>(nodes.size()); @@ -458,7 +434,6 @@ public class SplitShardCmd implements OverseerCollectionMessageHandler.Cmd { Assign.AssignStrategyFactory assignStrategyFactory = new Assign.AssignStrategyFactory(ocmh.cloudManager); Assign.AssignStrategy assignStrategy = assignStrategyFactory.create(clusterState, collection); List replicaPositions = assignStrategy.assign(ocmh.cloudManager, assignRequest); - sessionWrapper = PolicyHelper.getLastSessionWrapper(true); t.stop(); t = timings.sub("createReplicaPlaceholders"); @@ -611,7 +586,6 @@ public class SplitShardCmd implements OverseerCollectionMessageHandler.Cmd { log.error("Error executing split operation for collection: {} parent shard: {}", collectionName, slice, e); throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, null, e); } finally { - if (sessionWrapper != null) sessionWrapper.release(); if (!success) { cleanupAfterFailure(zkStateReader, collectionName, parentSlice.getName(), subSlices, offlineSlices); unlockForSplit(ocmh.cloudManager, collectionName, parentSlice.getName()); @@ -638,7 +612,7 @@ public class SplitShardCmd implements OverseerCollectionMessageHandler.Cmd { Map nodeValues = nodeStateProvider.getNodeValues(parentShardLeader.getNodeName(), Collections.singletonList(ImplicitSnitch.DISK)); Map>> infos = nodeStateProvider.getReplicaInfo(parentShardLeader.getNodeName(), - Collections.singletonList(Type.CORE_IDX.metricsAttribute)); + Collections.singletonList(CORE_IDX.metricsAttribute)); if (infos.get(collection) == null || infos.get(collection).get(shard) == null) { throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "missing replica information for parent shard leader"); } @@ -647,11 +621,11 @@ public class SplitShardCmd implements OverseerCollectionMessageHandler.Cmd { Double indexSize = null; for (Replica info : lst) { if (info.getCoreName().equals(parentShardLeader.getCoreName())) { - Number size = (Number)info.get(Type.CORE_IDX.metricsAttribute); + Number size = (Number)info.get( CORE_IDX.metricsAttribute); if (size == null) { throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "missing index size information for parent shard leader"); } - indexSize = (Double) Type.CORE_IDX.convertVal(size); + indexSize = (Double) CORE_IDX.convertVal(size); break; } } diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/UtilizeNodeCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/UtilizeNodeCmd.java deleted file mode 100644 index 0fd19ac21db..00000000000 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/UtilizeNodeCmd.java +++ /dev/null @@ -1,139 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.cloud.api.collections; - -import java.lang.invoke.MethodHandles; -import java.util.ArrayList; -import java.util.List; -import java.util.Objects; - -import org.apache.solr.client.solrj.SolrRequest; -import org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig; -import org.apache.solr.client.solrj.cloud.autoscaling.Policy; -import org.apache.solr.client.solrj.cloud.autoscaling.PolicyHelper; -import org.apache.solr.client.solrj.cloud.autoscaling.Suggester; -import org.apache.solr.client.solrj.request.V2Request; -import org.apache.solr.common.cloud.ClusterState; -import org.apache.solr.common.cloud.ZkNodeProps; -import org.apache.solr.common.params.CollectionParams; -import org.apache.solr.common.util.NamedList; -import org.apache.solr.common.util.StrUtils; -import org.apache.solr.common.util.Utils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP; -import static org.apache.solr.common.cloud.ZkStateReader.REPLICA_PROP; -import static org.apache.solr.common.params.AutoScalingParams.NODE; -import static org.apache.solr.common.params.CollectionParams.CollectionAction.MOVEREPLICA; -import static org.apache.solr.common.params.CommonAdminParams.ASYNC; - -public class UtilizeNodeCmd implements OverseerCollectionMessageHandler.Cmd { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - private final OverseerCollectionMessageHandler ocmh; - - public UtilizeNodeCmd(OverseerCollectionMessageHandler ocmh) { - this.ocmh = ocmh; - } - - @Override - public void call(ClusterState state, ZkNodeProps message, @SuppressWarnings({"rawtypes"})NamedList results) throws Exception { - ocmh.checkRequired(message, NODE); - String nodeName = message.getStr(NODE); - String async = message.getStr(ASYNC); - AutoScalingConfig autoScalingConfig = ocmh.overseer.getSolrCloudManager().getDistribStateManager().getAutoScalingConfig(); - - //first look for any violation that may use this replica - List requests = new ArrayList<>(); - //first look for suggestions if any - List suggestions = PolicyHelper.getSuggestions(autoScalingConfig, ocmh.overseer.getSolrCloudManager()); - for (Suggester.SuggestionInfo suggestionInfo : suggestions) { - if (log.isInfoEnabled()) { - log.info("op: {}", suggestionInfo.getOperation()); - } - String coll = null; - List pieces = StrUtils.splitSmart(suggestionInfo.getOperation().getPath(), '/'); - if (pieces.size() > 1) { - coll = pieces.get(2); - } else { - continue; - } - log.info("coll: {}", coll); - if (suggestionInfo.getOperation() instanceof V2Request) { - String targetNode = (String) Utils.getObjectByPath(suggestionInfo.getOperation(), true, "command/move-replica/targetNode"); - if (Objects.equals(targetNode, nodeName)) { - String replica = (String) Utils.getObjectByPath(suggestionInfo.getOperation(), true, "command/move-replica/replica"); - requests.add(new ZkNodeProps(COLLECTION_PROP, coll, - CollectionParams.TARGET_NODE, targetNode, - ASYNC, async, - REPLICA_PROP, replica)); - } - } - } - executeAll(requests); - PolicyHelper.SessionWrapper sessionWrapper = PolicyHelper.getSession(ocmh.overseer.getSolrCloudManager()); - Policy.Session session = sessionWrapper.get(); - Suggester initialsuggester = session.getSuggester(MOVEREPLICA) - .hint(Suggester.Hint.TARGET_NODE, nodeName); - Suggester suggester = null; - for (; ; ) { - suggester = session.getSuggester(MOVEREPLICA) - .hint(Suggester.Hint.TARGET_NODE, nodeName); - @SuppressWarnings({"rawtypes"}) - SolrRequest request = suggester.getSuggestion(); - if (requests.size() > 10) { - log.info("too_many_suggestions"); - PolicyHelper.logState(ocmh.overseer.getSolrCloudManager(), initialsuggester); - break; - } - log.info("SUGGESTION: {}", request); - if (request == null) break; - session = suggester.getSession(); - requests.add(new ZkNodeProps(COLLECTION_PROP, request.getParams().get(COLLECTION_PROP), - CollectionParams.TARGET_NODE, request.getParams().get(CollectionParams.TARGET_NODE), - REPLICA_PROP, request.getParams().get(REPLICA_PROP), - ASYNC, request.getParams().get(ASYNC))); - } - if (log.isInfoEnabled()) { - log.info("total_suggestions: {}", requests.size()); - } - if (requests.size() == 0) { - PolicyHelper.logState(ocmh.overseer.getSolrCloudManager(), initialsuggester); - } - sessionWrapper.returnSession(session); - try { - executeAll(requests); - } finally { - sessionWrapper.release(); - } - } - - private void executeAll(List requests) throws Exception { - if (requests.isEmpty()) return; - for (ZkNodeProps props : requests) { - @SuppressWarnings({"rawtypes"}) - NamedList result = new NamedList(); - ocmh.commandMap.get(MOVEREPLICA) - .call(ocmh.overseer.getSolrCloudManager().getClusterStateProvider().getClusterState(), - props, - result); - } - requests.clear(); - } - -} diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/ActionContext.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/ActionContext.java deleted file mode 100644 index 8487d3d7cf2..00000000000 --- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/ActionContext.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.cloud.autoscaling; - -import java.io.IOException; -import java.util.Map; - -import org.apache.solr.client.solrj.cloud.SolrCloudManager; -import org.apache.solr.common.MapWriter; - -/** - * Provides additional context for the TriggerAction such as the trigger instance on - * which the action is being executed as well as helper methods to pass computed information along - * to the next action - */ -public class ActionContext implements MapWriter { - - private final SolrCloudManager cloudManager; - private final AutoScaling.Trigger source; - private final Map properties; - - public ActionContext(SolrCloudManager cloudManager, AutoScaling.Trigger source, Map properties) { - this.cloudManager = cloudManager; - this.source = source; - this.properties = properties; - } - - public SolrCloudManager getCloudManager() { - return cloudManager; - } - - public AutoScaling.Trigger getSource() { - return source; - } - - public Map getProperties() { - return properties; - } - - public Object getProperty(String name) { - return properties != null ? properties.get(name) : null; - } - - @Override - public void writeMap(EntryWriter ew) throws IOException { - ew.put("source", source.getName()); - if (properties != null) { - for (Map.Entry entry : properties.entrySet()) { - ew.put("properties." + entry.getKey(), entry.getValue()); - } - } - } -} diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoAddReplicasPlanAction.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoAddReplicasPlanAction.java deleted file mode 100644 index d129fdbefe9..00000000000 --- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoAddReplicasPlanAction.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.cloud.autoscaling; - - -import java.util.Collections; -import java.util.Map; - -import org.apache.solr.client.solrj.cloud.SolrCloudManager; -import org.apache.solr.core.SolrResourceLoader; - -import static org.apache.solr.common.cloud.ZkStateReader.AUTO_ADD_REPLICAS; - -/** - * This class configures the parent ComputePlanAction to compute plan - * only for collections which have autoAddReplicas=true. - */ -public class AutoAddReplicasPlanAction extends ComputePlanAction { - - @Override - public void configure(SolrResourceLoader loader, SolrCloudManager cloudManager, Map properties) throws TriggerValidationException { - properties.put("collections", Collections.singletonMap(AUTO_ADD_REPLICAS, "true")); - super.configure(loader, cloudManager, properties); - } -} diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScaling.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScaling.java deleted file mode 100644 index 1a191ee858b..00000000000 --- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScaling.java +++ /dev/null @@ -1,246 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.cloud.autoscaling; - -import java.io.Closeable; -import java.io.IOException; -import java.util.List; -import java.util.Map; -import java.util.Objects; - -import org.apache.lucene.store.AlreadyClosedException; -import org.apache.solr.client.solrj.cloud.SolrCloudManager; -import org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventType; -import org.apache.solr.common.util.Utils; -import org.apache.solr.core.SolrResourceLoader; - -public class AutoScaling { - - /** - * Implementation of this interface is used for processing events generated by a trigger. - */ - public interface TriggerEventProcessor { - - /** - * This method is executed for events produced by {@link Trigger#run()}. - * - * @param event a subclass of {@link TriggerEvent} - * @return true if the processor was ready to perform actions on the event, false - * otherwise. If false was returned then callers should assume the event was discarded. - */ - boolean process(TriggerEvent event); - } - - /** - * Interface for a Solr trigger. Each trigger implements Runnable and Closeable interface. A trigger - * is scheduled using a {@link java.util.concurrent.ScheduledExecutorService} so it is executed as - * per a configured schedule to check whether the trigger is ready to fire. The {@link AutoScaling.Trigger#setProcessor(AutoScaling.TriggerEventProcessor)} - * method should be used to set a processor which is used by implementation of this class whenever - * ready. - *

- * As per the guarantees made by the {@link java.util.concurrent.ScheduledExecutorService} a trigger - * implementation is only ever called sequentially and therefore need not be thread safe. However, it - * is encouraged that implementations be immutable with the exception of the associated listener - * which can be get/set by a different thread than the one executing the trigger. Therefore, implementations - * should use appropriate synchronization around the listener. - *

- * When a trigger is ready to fire, it calls the {@link TriggerEventProcessor#process(TriggerEvent)} event - * with the proper trigger event object. If that method returns false then it should be interpreted to mean - * that Solr is not ready to process this trigger event and therefore we should retain the state and fire - * at the next invocation of the run() method. - */ - public interface Trigger extends Closeable, Runnable { - /** - * Trigger name. - */ - String getName(); - - /** - * Event type generated by this trigger. - */ - TriggerEventType getEventType(); - - /** Returns true if this trigger is enabled. */ - boolean isEnabled(); - - /** Trigger properties. */ - Map getProperties(); - - /** Number of seconds to wait between fired events ("waitFor" property). */ - int getWaitForSecond(); - - /** Actions to execute when event is fired. */ - List getActions(); - - /** Set event processor to call when event is fired. */ - void setProcessor(TriggerEventProcessor processor); - - /** Get event processor. */ - TriggerEventProcessor getProcessor(); - - /** Return true when this trigger is closed and cannot be used. */ - boolean isClosed(); - - /** Set internal state of this trigger from another instance. */ - void restoreState(Trigger old); - - /** Save internal state of this trigger in ZooKeeper. */ - void saveState(); - - /** Restore internal state of this trigger from ZooKeeper. */ - void restoreState(); - - /** - * Called when trigger is created but before it's initialized or scheduled for use. - * This method should also verify that the trigger configuration parameters are correct. It may - * be called multiple times. - * @param properties configuration properties - * @throws TriggerValidationException contains details of invalid configuration parameters. - */ - void configure(SolrResourceLoader loader, SolrCloudManager cloudManager, Map properties) throws TriggerValidationException; - - /** - * Called before a trigger is scheduled. Any heavy object creation or initialisation should - * be done in this method instead of the Trigger's constructor. - */ - void init() throws Exception; - } - - /** - * Factory to produce instances of {@link Trigger}. - */ - public static abstract class TriggerFactory implements Closeable { - protected boolean isClosed = false; - - public abstract Trigger create(TriggerEventType type, String name, Map props) throws TriggerValidationException; - - @Override - public void close() throws IOException { - synchronized (this) { - isClosed = true; - } - } - } - - /** - * Default implementation of {@link TriggerFactory}. - */ - public static class TriggerFactoryImpl extends TriggerFactory { - - private final SolrCloudManager cloudManager; - private final SolrResourceLoader loader; - - public TriggerFactoryImpl(SolrResourceLoader loader, SolrCloudManager cloudManager) { - Objects.requireNonNull(cloudManager); - Objects.requireNonNull(loader); - this.cloudManager = cloudManager; - this.loader = loader; - } - - @Override - public synchronized Trigger create(TriggerEventType type, String name, Map props) throws TriggerValidationException { - if (isClosed) { - throw new AlreadyClosedException("TriggerFactory has already been closed, cannot create new triggers"); - } - if (type == null) { - throw new IllegalArgumentException("Trigger type must not be null"); - } - if (name == null || name.isEmpty()) { - throw new IllegalArgumentException("Trigger name must not be empty"); - } - Trigger t; - switch (type) { - case NODEADDED: - t = new NodeAddedTrigger(name); - break; - case NODELOST: - t = new NodeLostTrigger(name); - break; - case SEARCHRATE: - t = new SearchRateTrigger(name); - break; - case METRIC: - t = new MetricTrigger(name); - break; - case SCHEDULED: - t = new ScheduledTrigger(name); - break; - case INDEXSIZE: - t = new IndexSizeTrigger(name); - break; - default: - throw new IllegalArgumentException("Unknown event type: " + type + " in trigger: " + name); - } - t.configure(loader, cloudManager, props); - return t; - } - - } - - public static final String AUTO_ADD_REPLICAS_TRIGGER_NAME = ".auto_add_replicas"; - - public static final String AUTO_ADD_REPLICAS_TRIGGER_DSL = - " {" + - " 'name' : '" + AUTO_ADD_REPLICAS_TRIGGER_NAME + "'," + - " 'event' : 'nodeLost'," + - " 'waitFor' : -1," + - " 'enabled' : true," + - " 'actions' : [" + - " {" + - " 'name':'auto_add_replicas_plan'," + - " 'class':'solr.AutoAddReplicasPlanAction'" + - " }," + - " {" + - " 'name':'execute_plan'," + - " 'class':'solr.ExecutePlanAction'" + - " }" + - " ]" + - " }"; - - @SuppressWarnings({"unchecked"}) - public static final Map AUTO_ADD_REPLICAS_TRIGGER_PROPS = (Map) Utils.fromJSONString(AUTO_ADD_REPLICAS_TRIGGER_DSL); - - public static final String SCHEDULED_MAINTENANCE_TRIGGER_NAME = ".scheduled_maintenance"; - - public static final String SCHEDULED_MAINTENANCE_TRIGGER_DSL = - " {" + - " 'name' : '" + SCHEDULED_MAINTENANCE_TRIGGER_NAME + "'," + - " 'event' : 'scheduled'," + - " 'startTime' : 'NOW'," + - " 'every' : '+1DAY'," + - " 'enabled' : true," + - " 'actions' : [" + - " {" + - " 'name':'inactive_shard_plan'," + - " 'class':'solr.InactiveShardPlanAction'" + - " }," + - " {" + - " 'name':'inactive_markers_plan'," + - " 'class':'solr.InactiveMarkersPlanAction'" + - " }," + - " {" + - " 'name':'execute_plan'," + - " 'class':'solr.ExecutePlanAction'" + - " }" + - " ]" + - " }"; - - @SuppressWarnings({"unchecked"}) - public static final Map SCHEDULED_MAINTENANCE_TRIGGER_PROPS = (Map) Utils.fromJSONString(SCHEDULED_MAINTENANCE_TRIGGER_DSL); - -} diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScalingHandler.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScalingHandler.java deleted file mode 100644 index 23ec0754cbc..00000000000 --- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScalingHandler.java +++ /dev/null @@ -1,742 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.cloud.autoscaling; - -import java.io.IOException; -import java.lang.invoke.MethodHandles; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.Date; -import java.util.EnumSet; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.TimeUnit; -import java.util.function.BiConsumer; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import org.apache.solr.api.Api; -import org.apache.solr.api.ApiBag; -import org.apache.solr.client.solrj.cloud.SolrCloudManager; -import org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig; -import org.apache.solr.client.solrj.cloud.autoscaling.BadVersionException; -import org.apache.solr.client.solrj.cloud.autoscaling.Clause; -import org.apache.solr.client.solrj.cloud.autoscaling.Policy; -import org.apache.solr.client.solrj.cloud.autoscaling.PolicyHelper; -import org.apache.solr.client.solrj.cloud.autoscaling.Preference; -import org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventProcessorStage; -import org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventType; -import org.apache.solr.common.MapWriter; -import org.apache.solr.common.SolrException; -import org.apache.solr.common.params.AutoScalingParams; -import org.apache.solr.common.params.CollectionAdminParams; -import org.apache.solr.common.params.SolrParams; -import org.apache.solr.common.util.CommandOperation; -import org.apache.solr.common.util.IOUtils; -import org.apache.solr.common.util.StrUtils; -import org.apache.solr.common.util.TimeSource; -import org.apache.solr.common.util.Utils; -import org.apache.solr.core.SolrResourceLoader; -import org.apache.solr.handler.RequestHandlerBase; -import org.apache.solr.handler.RequestHandlerUtils; -import org.apache.solr.request.SolrQueryRequest; -import org.apache.solr.request.SolrRequestHandler; -import org.apache.solr.response.SolrQueryResponse; -import org.apache.solr.security.AuthorizationContext; -import org.apache.solr.security.PermissionNameProvider; -import org.apache.zookeeper.KeeperException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import static java.util.stream.Collectors.collectingAndThen; -import static java.util.stream.Collectors.toSet; -import static org.apache.solr.common.cloud.ZkStateReader.SOLR_AUTOSCALING_CONF_PATH; -import static org.apache.solr.common.params.AutoScalingParams.*; -import static org.apache.solr.common.params.CommonParams.JSON; - -/** - * Handler for /cluster/autoscaling - */ -public class AutoScalingHandler extends RequestHandlerBase implements PermissionNameProvider { - public static final String HANDLER_PATH = "/admin/autoscaling"; - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - protected final SolrCloudManager cloudManager; - protected final SolrResourceLoader loader; - protected final AutoScaling.TriggerFactory triggerFactory; - private final List> DEFAULT_ACTIONS = new ArrayList<>(3); - private static Set singletonCommands = Stream.of("set-cluster-preferences", "set-cluster-policy") - .collect(collectingAndThen(toSet(), Collections::unmodifiableSet)); - - private final TimeSource timeSource; - - public AutoScalingHandler(SolrCloudManager cloudManager, SolrResourceLoader loader) { - this.cloudManager = cloudManager; - this.loader = loader; - this.triggerFactory = new AutoScaling.TriggerFactoryImpl(loader, cloudManager); - this.timeSource = cloudManager.getTimeSource(); - Map map = new HashMap<>(2); - map.put(NAME, "compute_plan"); - map.put(CLASS, "solr.ComputePlanAction"); - DEFAULT_ACTIONS.add(map); - map = new HashMap<>(2); - map.put(NAME, "execute_plan"); - map.put(CLASS, "solr.ExecutePlanAction"); - DEFAULT_ACTIONS.add(map); - } - - Optional> getSubpathExecutor(List path, SolrQueryRequest request) { - if (path.size() == 3) { - if (DIAGNOSTICS.equals(path.get(2))) { - return Optional.of((rsp, autoScalingConf) -> handleDiagnostics(rsp, autoScalingConf)); - } else if (SUGGESTIONS.equals(path.get(2))) { - return Optional.of((rsp, autoScalingConf) -> handleSuggestions(rsp, autoScalingConf, request.getParams())); - } else { - return Optional.empty(); - } - - } - return Optional.empty(); - } - - @Override - @SuppressWarnings({"unchecked", "rawtypes"}) - public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception { - try { - String httpMethod = (String) req.getContext().get("httpMethod"); - RequestHandlerUtils.setWt(req, JSON); - - if ("GET".equals(httpMethod)) { - String path = (String) req.getContext().get("path"); - if (path == null) path = "/cluster/autoscaling"; - List parts = StrUtils.splitSmart(path, '/', true); - - if (parts.size() < 2 || parts.size() > 3) { - // invalid - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Unknown path: " + path); - } - - AutoScalingConfig autoScalingConf = cloudManager.getDistribStateManager().getAutoScalingConfig(); - if (parts.size() == 2) { - autoScalingConf.writeMap(new MapWriter.EntryWriter() { - - @Override - public MapWriter.EntryWriter put(CharSequence k, Object v) { - rsp.getValues().add(k.toString(), v); - return this; - } - }); - } else { - getSubpathExecutor(parts, req).ifPresent(it -> it.accept(rsp, autoScalingConf)); - } - } else { - if (req.getContentStreams() == null) { - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "No commands specified for autoscaling"); - } - String path = (String) req.getContext().get("path"); - if (path != null) { - List parts = StrUtils.splitSmart(path, '/', true); - if(parts.size() == 3){ - getSubpathExecutor(parts, req).ifPresent(it -> { - Map map = null; - try { - map = (Map) Utils.fromJSON(req.getContentStreams().iterator().next().getStream()); - } catch (IOException e1) { - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "error parsing payload", e1); - } - it.accept(rsp, new AutoScalingConfig(map)); - }); - - return; - } - - } - List ops = CommandOperation.readCommands(req.getContentStreams(), rsp.getValues(), singletonCommands); - if (ops == null) { - // errors have already been added to the response so there's nothing left to do - return; - } - processOps(req, rsp, ops); - } - - } catch (Exception e) { - rsp.getValues().add("result", "failure"); - throw e; - } finally { - RequestHandlerUtils.addExperimentalFormatWarning(rsp); - } - } - - - @SuppressWarnings({"unchecked"}) - private void handleSuggestions(SolrQueryResponse rsp, AutoScalingConfig autoScalingConf, SolrParams params) { - rsp.getValues().add("suggestions", - PolicyHelper.getSuggestions(autoScalingConf, cloudManager, params)); - } - - @SuppressWarnings({"unchecked", "rawtypes"}) - public void processOps(SolrQueryRequest req, SolrQueryResponse rsp, List ops) - throws KeeperException, InterruptedException, IOException { - while (true) { - AutoScalingConfig initialConfig = cloudManager.getDistribStateManager().getAutoScalingConfig(); - AutoScalingConfig currentConfig = initialConfig; - for (CommandOperation op : ops) { - switch (op.name) { - case CMD_SET_TRIGGER: - currentConfig = handleSetTrigger(req, rsp, op, currentConfig); - break; - case CMD_REMOVE_TRIGGER: - currentConfig = handleRemoveTrigger(req, rsp, op, currentConfig); - break; - case CMD_SET_LISTENER: - currentConfig = handleSetListener(req, rsp, op, currentConfig); - break; - case CMD_REMOVE_LISTENER: - currentConfig = handleRemoveListener(req, rsp, op, currentConfig); - break; - case CMD_SUSPEND_TRIGGER: - currentConfig = handleSuspendTrigger(req, rsp, op, currentConfig); - break; - case CMD_RESUME_TRIGGER: - currentConfig = handleResumeTrigger(req, rsp, op, currentConfig); - break; - case CMD_SET_POLICY: - currentConfig = handleSetPolicies(req, rsp, op, currentConfig); - break; - case CMD_REMOVE_POLICY: - currentConfig = handleRemovePolicy(req, rsp, op, currentConfig); - break; - case CMD_SET_CLUSTER_PREFERENCES: - currentConfig = handleSetClusterPreferences(req, rsp, op, currentConfig); - break; - case CMD_SET_CLUSTER_POLICY: - currentConfig = handleSetClusterPolicy(req, rsp, op, currentConfig); - break; - case CMD_SET_PROPERTIES: - currentConfig = handleSetProperties(req, rsp, op, currentConfig); - break; - default: - op.addError("Unknown command: " + op.name); - } - } - List errs = CommandOperation.captureErrors(ops); - if (!errs.isEmpty()) { - throw new ApiBag.ExceptionWithErrObject(SolrException.ErrorCode.BAD_REQUEST, "Error in command payload", errs); - } - - if (!currentConfig.equals(initialConfig)) { - // update in ZK - if (setAutoScalingConfig(currentConfig)) { - break; - } else { - // someone else updated the config, get the latest one and re-apply our ops - rsp.getValues().add("retry", "initialVersion=" + initialConfig.getZkVersion()); - continue; - } - } else { - // no changes - break; - } - } - rsp.getValues().add("result", "success"); - } - - private AutoScalingConfig handleSetProperties(SolrQueryRequest req, SolrQueryResponse rsp, CommandOperation op, AutoScalingConfig currentConfig) { - Map map = op.getDataMap() == null ? Collections.emptyMap() : op.getDataMap(); - Map configProps = new HashMap<>(currentConfig.getProperties()); - configProps.putAll(map); - // remove a key which is set to null - map.forEach((k, v) -> { - if (v == null) configProps.remove(k); - }); - return currentConfig.withProperties(configProps); - } - - @SuppressWarnings({"unchecked"}) - private void handleDiagnostics(SolrQueryResponse rsp, AutoScalingConfig autoScalingConf) { - Policy policy = autoScalingConf.getPolicy(); - rsp.getValues().add("diagnostics", PolicyHelper.getDiagnostics(policy, cloudManager)); - } - - @SuppressWarnings({"unchecked"}) - private AutoScalingConfig handleSetClusterPolicy(SolrQueryRequest req, SolrQueryResponse rsp, CommandOperation op, - AutoScalingConfig currentConfig) throws KeeperException, InterruptedException, IOException { - List> clusterPolicy = (List>) op.getCommandData(); - if (clusterPolicy == null || !(clusterPolicy instanceof List)) { - op.addError("set-cluster-policy expects an array of objects"); - return currentConfig; - } - List cp = null; - try { - cp = clusterPolicy.stream().map(Clause::create).collect(Collectors.toList()); - } catch (Exception e) { - op.addError(e.getMessage()); - return currentConfig; - } - Policy p = currentConfig.getPolicy().withClusterPolicy(cp); - currentConfig = currentConfig.withPolicy(p); - return currentConfig; - } - - @SuppressWarnings({"unchecked"}) - private AutoScalingConfig handleSetClusterPreferences(SolrQueryRequest req, SolrQueryResponse rsp, CommandOperation op, - AutoScalingConfig currentConfig) throws KeeperException, InterruptedException, IOException { - List> preferences = (List>) op.getCommandData(); - if (preferences == null || !(preferences instanceof List)) { - op.addError("A list of cluster preferences not found"); - return currentConfig; - } - List prefs = null; - try { - prefs = preferences.stream().map(Preference::new).collect(Collectors.toList()); - } catch (Exception e) { - op.addError(e.getMessage()); - return currentConfig; - } - Policy p = currentConfig.getPolicy().withClusterPreferences(prefs); - currentConfig = currentConfig.withPolicy(p); - return currentConfig; - } - - private AutoScalingConfig handleRemovePolicy(SolrQueryRequest req, SolrQueryResponse rsp, CommandOperation op, - AutoScalingConfig currentConfig) throws KeeperException, InterruptedException, IOException { - String policyName = (String) op.getVal(""); - - if (op.hasError()) return currentConfig; - - Map> policies = currentConfig.getPolicy().getPolicies(); - if (policies == null || !policies.containsKey(policyName)) { - op.addError("No policy exists with name: " + policyName); - return currentConfig; - } - - cloudManager.getClusterStateProvider().getClusterState().forEachCollection(coll -> { - if (policyName.equals(coll.getPolicyName())) - op.addError(StrUtils.formatString("policy : {0} is being used by collection {1}", policyName, coll.getName())); - }); - if (op.hasError()) return currentConfig; - policies = new HashMap<>(policies); - policies.remove(policyName); - Policy p = currentConfig.getPolicy().withPolicies(policies); - currentConfig = currentConfig.withPolicy(p); - return currentConfig; - } - - @SuppressWarnings({"unchecked"}) - private AutoScalingConfig handleSetPolicies(SolrQueryRequest req, SolrQueryResponse rsp, CommandOperation op, - AutoScalingConfig currentConfig) throws KeeperException, InterruptedException, IOException { - Map policiesMap = op.getDataMap(); - for (Map.Entry policy : policiesMap.entrySet()) { - String policyName = policy.getKey(); - if (policyName == null || policyName.trim().length() == 0) { - op.addError("The policy name cannot be null or empty"); - return currentConfig; - } - } - Map> currentClauses = new HashMap<>(currentConfig.getPolicy().getPolicies()); - Map> newClauses = null; - try { - newClauses = Policy.clausesFromMap((Map>>) op.getCommandData(), - new ArrayList<>() ); - } catch (Exception e) { - op.addError(e.getMessage()); - return currentConfig; - } - currentClauses.putAll(newClauses); - Policy p = currentConfig.getPolicy().withPolicies(currentClauses); - currentConfig = currentConfig.withPolicy(p); - return currentConfig; - } - - @SuppressWarnings({"unchecked"}) - private AutoScalingConfig handleResumeTrigger(SolrQueryRequest req, SolrQueryResponse rsp, CommandOperation op, - AutoScalingConfig currentConfig) throws KeeperException, InterruptedException { - String triggerName = op.getStr(NAME); - if (op.hasError()) return currentConfig; - Map triggers = currentConfig.getTriggerConfigs(); - Set changed = new HashSet<>(); - if (!Policy.EACH.equals(triggerName) && !triggers.containsKey(triggerName)) { - op.addError("No trigger exists with name: " + triggerName); - return currentConfig; - } - Map newTriggers = new HashMap<>(); - for (Map.Entry entry : triggers.entrySet()) { - if (Policy.EACH.equals(triggerName) || triggerName.equals(entry.getKey())) { - AutoScalingConfig.TriggerConfig trigger = entry.getValue(); - if (!trigger.enabled) { - trigger = trigger.withEnabled(true); - newTriggers.put(entry.getKey(), trigger); - changed.add(entry.getKey()); - } else { - newTriggers.put(entry.getKey(), entry.getValue()); - } - } else { - newTriggers.put(entry.getKey(), entry.getValue()); - } - } - rsp.getValues().add("changed", changed); - if (!changed.isEmpty()) { - currentConfig = currentConfig.withTriggerConfigs(newTriggers); - } - return currentConfig; - } - - @SuppressWarnings({"unchecked"}) - private AutoScalingConfig handleSuspendTrigger(SolrQueryRequest req, SolrQueryResponse rsp, CommandOperation op, - AutoScalingConfig currentConfig) throws KeeperException, InterruptedException { - String triggerName = op.getStr(NAME); - if (op.hasError()) return currentConfig; - String timeout = op.getStr(TIMEOUT, null); - Date resumeTime = null; - if (timeout != null) { - try { - int timeoutSeconds = parseHumanTime(timeout); - resumeTime = new Date(TimeUnit.MILLISECONDS.convert(timeSource.getTimeNs(), TimeUnit.NANOSECONDS) - + TimeUnit.MILLISECONDS.convert(timeoutSeconds, TimeUnit.SECONDS)); - } catch (IllegalArgumentException e) { - op.addError("Invalid 'timeout' value for suspend trigger: " + triggerName); - return currentConfig; - } - } - - Map triggers = currentConfig.getTriggerConfigs(); - Set changed = new HashSet<>(); - - if (!Policy.EACH.equals(triggerName) && !triggers.containsKey(triggerName)) { - op.addError("No trigger exists with name: " + triggerName); - return currentConfig; - } - Map newTriggers = new HashMap<>(); - for (Map.Entry entry : triggers.entrySet()) { - if (Policy.EACH.equals(triggerName) || triggerName.equals(entry.getKey())) { - AutoScalingConfig.TriggerConfig trigger = entry.getValue(); - if (trigger.enabled) { - trigger = trigger.withEnabled(false); - if (resumeTime != null) { - trigger = trigger.withProperty(RESUME_AT, resumeTime.getTime()); - } - newTriggers.put(entry.getKey(), trigger); - changed.add(trigger.name); - } else { - newTriggers.put(entry.getKey(), entry.getValue()); - } - } else { - newTriggers.put(entry.getKey(), entry.getValue()); - } - } - rsp.getValues().add("changed", changed); - if (!changed.isEmpty()) { - currentConfig = currentConfig.withTriggerConfigs(newTriggers); - } - return currentConfig; - } - - private AutoScalingConfig handleRemoveListener(SolrQueryRequest req, SolrQueryResponse rsp, CommandOperation op, - AutoScalingConfig currentConfig) throws KeeperException, InterruptedException { - String listenerName = op.getStr(NAME); - - if (op.hasError()) return currentConfig; - Map listeners = currentConfig.getTriggerListenerConfigs(); - if (listeners == null || !listeners.containsKey(listenerName)) { - op.addError("No listener exists with name: " + listenerName); - return currentConfig; - } - currentConfig = currentConfig.withoutTriggerListenerConfig(listenerName); - return currentConfig; - } - - private AutoScalingConfig handleSetListener(SolrQueryRequest req, SolrQueryResponse rsp, CommandOperation op, - AutoScalingConfig currentConfig) throws KeeperException, InterruptedException { - String listenerName = op.getStr(NAME); - String triggerName = op.getStr(TRIGGER); - List stageNames = op.getStrs(STAGE, Collections.emptyList()); - String listenerClass = op.getStr(CLASS); - List beforeActions = op.getStrs(BEFORE_ACTION, Collections.emptyList()); - List afterActions = op.getStrs(AFTER_ACTION, Collections.emptyList()); - - if (op.hasError()) return currentConfig; - - Map triggers = currentConfig.getTriggerConfigs(); - if (triggers == null || !triggers.containsKey(triggerName)) { - op.addError("A trigger with the name " + triggerName + " does not exist"); - return currentConfig; - } - AutoScalingConfig.TriggerConfig triggerConfig = triggers.get(triggerName); - - if (stageNames.isEmpty() && beforeActions.isEmpty() && afterActions.isEmpty()) { - op.addError("Either 'stage' or 'beforeAction' or 'afterAction' must be specified"); - return currentConfig; - } - - for (String stage : stageNames) { - try { - TriggerEventProcessorStage.valueOf(stage); - } catch (IllegalArgumentException e) { - op.addError("Invalid stage name: " + stage); - } - } - if (op.hasError()) return currentConfig; - - AutoScalingConfig.TriggerListenerConfig listenerConfig = new AutoScalingConfig.TriggerListenerConfig(listenerName, op.getValuesExcluding("name")); - - // validate that we can load the listener class - // todo allow creation from blobstore - TriggerListener listener = null; - try { - listener = loader.newInstance(listenerClass, TriggerListener.class); - listener.configure(loader, cloudManager, listenerConfig); - } catch (TriggerValidationException e) { - log.warn("invalid listener configuration", e); - op.addError("invalid listener configuration: " + e.toString()); - return currentConfig; - } catch (Exception e) { - log.warn("error loading listener class ", e); - op.addError("Listener not found: " + listenerClass + ". error message:" + e.getMessage()); - return currentConfig; - } finally { - if (listener != null) { - IOUtils.closeQuietly(listener); - } - } - - Set actionNames = new HashSet<>(); - actionNames.addAll(beforeActions); - actionNames.addAll(afterActions); - for (AutoScalingConfig.ActionConfig action : triggerConfig.actions) { - actionNames.remove(action.name); - } - if (!actionNames.isEmpty()) { - op.addError("The trigger '" + triggerName + "' does not have actions named: " + actionNames); - return currentConfig; - } - // todo - handle races between competing set-trigger and set-listener invocations - currentConfig = currentConfig.withTriggerListenerConfig(listenerConfig); - return currentConfig; - } - - @SuppressWarnings({"unchecked"}) - private AutoScalingConfig handleSetTrigger(SolrQueryRequest req, SolrQueryResponse rsp, CommandOperation op, - AutoScalingConfig currentConfig) throws KeeperException, InterruptedException { - // we're going to modify the op - use a copy - String triggerName = op.getStr(NAME); - String eventTypeStr = op.getStr(EVENT); - - if (op.hasError()) return currentConfig; - TriggerEventType eventType = TriggerEventType.valueOf(eventTypeStr.trim().toUpperCase(Locale.ROOT)); - - String waitForStr = op.getStr(WAIT_FOR, null); - - CommandOperation opCopy = new CommandOperation(op.name, Utils.getDeepCopy((Map) op.getCommandData(), 10)); - - if (waitForStr != null) { - int seconds = 0; - try { - seconds = parseHumanTime(waitForStr); - } catch (IllegalArgumentException e) { - op.addError("Invalid 'waitFor' value '" + waitForStr + "' in trigger: " + triggerName); - return currentConfig; - } - opCopy.getDataMap().put(WAIT_FOR, seconds); - } - - Integer lowerBound = op.getInt(LOWER_BOUND, null); - Integer upperBound = op.getInt(UPPER_BOUND, null); - - List> actions = (List>) op.getVal(ACTIONS); - if (actions == null) { - actions = DEFAULT_ACTIONS; - opCopy.getDataMap().put(ACTIONS, actions); - } - - // validate that we can load all the actions - // todo allow creation from blobstore - for (Map action : actions) { - if (!action.containsKey(NAME) || !action.containsKey(CLASS)) { - op.addError("No 'name' or 'class' specified for action: " + action); - return currentConfig; - } - String klass = action.get(CLASS); - try { - loader.findClass(klass, TriggerAction.class); - } catch (Exception e) { - log.warn("Could not load class : ", e); - op.addError("Action not found: " + klass + " " + e.getMessage()); - return currentConfig; - } - } - AutoScalingConfig.TriggerConfig trigger = new AutoScalingConfig.TriggerConfig(triggerName, opCopy.getValuesExcluding("name")); - // validate trigger config - AutoScaling.Trigger t = null; - try { - t = triggerFactory.create(trigger.event, trigger.name, trigger.properties); - } catch (Exception e) { - op.addError("Error validating trigger config " + trigger.name + ": " + e.toString()); - return currentConfig; - } finally { - if (t != null) { - IOUtils.closeQuietly(t); - } - } - currentConfig = currentConfig.withTriggerConfig(trigger); - // check that there's a default SystemLogListener, unless user specified another one - return withSystemLogListener(currentConfig, triggerName); - } - - private static String fullName = SystemLogListener.class.getName(); - private static String solrName = "solr." + SystemLogListener.class.getSimpleName(); - - public static AutoScalingConfig withSystemLogListener(AutoScalingConfig autoScalingConfig, String triggerName) { - Map configs = autoScalingConfig.getTriggerListenerConfigs(); - for (AutoScalingConfig.TriggerListenerConfig cfg : configs.values()) { - if (triggerName.equals(cfg.trigger)) { - // already has some listener config - return autoScalingConfig; - } - } - // need to add - Map properties = new HashMap<>(); - properties.put(AutoScalingParams.CLASS, SystemLogListener.class.getName()); - properties.put(AutoScalingParams.TRIGGER, triggerName); - properties.put(AutoScalingParams.STAGE, EnumSet.allOf(TriggerEventProcessorStage.class)); - AutoScalingConfig.TriggerListenerConfig listener = - new AutoScalingConfig.TriggerListenerConfig(triggerName + CollectionAdminParams.SYSTEM_COLL, properties); - autoScalingConfig = autoScalingConfig.withTriggerListenerConfig(listener); - return autoScalingConfig; - } - - private int parseHumanTime(String timeStr) { - char c = timeStr.charAt(timeStr.length() - 1); - long timeValue = Long.parseLong(timeStr.substring(0, timeStr.length() - 1)); - int seconds; - switch (c) { - case 'h': - seconds = (int) TimeUnit.HOURS.toSeconds(timeValue); - break; - case 'm': - seconds = (int) TimeUnit.MINUTES.toSeconds(timeValue); - break; - case 's': - seconds = (int) timeValue; - break; - default: - throw new IllegalArgumentException("Invalid time value"); - } - return seconds; - } - - private AutoScalingConfig handleRemoveTrigger(SolrQueryRequest req, SolrQueryResponse rsp, CommandOperation op, - AutoScalingConfig currentConfig) throws KeeperException, InterruptedException { - String triggerName = op.getStr(NAME); - boolean removeListeners = op.getBoolean(REMOVE_LISTENERS, false); - - if (op.hasError()) return currentConfig; - Map triggerConfigs = currentConfig.getTriggerConfigs(); - if (!triggerConfigs.containsKey(triggerName)) { - op.addError("No trigger exists with name: " + triggerName); - return currentConfig; - } - triggerConfigs = new HashMap<>(triggerConfigs); - Set activeListeners = new HashSet<>(); - Map listeners = currentConfig.getTriggerListenerConfigs(); - for (AutoScalingConfig.TriggerListenerConfig listener : listeners.values()) { - if (triggerName.equals(listener.trigger)) { - activeListeners.add(listener.name); - } - } - if (!activeListeners.isEmpty()) { - boolean onlySystemLog = false; - if (activeListeners.size() == 1) { - AutoScalingConfig.TriggerListenerConfig cfg = listeners.get(activeListeners.iterator().next()); - if (SystemLogListener.class.getName().equals(cfg.listenerClass) || - ("solr." + SystemLogListener.class.getSimpleName()).equals(cfg.listenerClass)) { - onlySystemLog = true; - } - } - if (removeListeners || onlySystemLog) { - listeners = new HashMap<>(listeners); - listeners.keySet().removeAll(activeListeners); - } else { - op.addError("Cannot remove trigger: " + triggerName + " because it has active listeners: " + activeListeners); - return currentConfig; - } - } - triggerConfigs.remove(triggerName); - currentConfig = currentConfig.withTriggerConfigs(triggerConfigs).withTriggerListenerConfigs(listeners); - return currentConfig; - } - - - private boolean setAutoScalingConfig(AutoScalingConfig currentConfig) throws KeeperException, InterruptedException, IOException { - verifyAutoScalingConf(currentConfig); - try { - cloudManager.getDistribStateManager().setData(SOLR_AUTOSCALING_CONF_PATH, Utils.toJSON(currentConfig), currentConfig.getZkVersion()); - } catch (BadVersionException bve) { - // somebody else has changed the configuration so we must retry - return false; - } - //log.debug("-- saved version " + currentConfig.getZkVersion() + ": " + currentConfig); - return true; - } - - private void verifyAutoScalingConf(AutoScalingConfig autoScalingConf) throws IOException { - Policy.Session session = autoScalingConf.getPolicy() - .createSession(cloudManager); - log.debug("Verified autoscaling configuration"); - } - - @Override - public String getDescription() { - return "A handler for autoscaling configuration"; - } - - @Override - public Name getPermissionName(AuthorizationContext request) { - switch (request.getHttpMethod()) { - case "GET": - return Name.AUTOSCALING_READ_PERM; - case "POST": { - return StrUtils.splitSmart(request.getResource(), '/', true).size() == 3 ? - Name.AUTOSCALING_READ_PERM : - Name.AUTOSCALING_WRITE_PERM; - } - default: - return null; - } - } - - @Override - public Collection getApis() { - return ApiBag.wrapRequestHandlers(this, "autoscaling.Commands"); - } - - @Override - public Boolean registerV2() { - return Boolean.TRUE; - } - - @Override - public SolrRequestHandler getSubHandler(String path) { - if (path.equals("/diagnostics") || path.equals("/suggestions")) return this; - return null; - } -} diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/ComputePlanAction.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/ComputePlanAction.java deleted file mode 100644 index 33bf6b04dab..00000000000 --- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/ComputePlanAction.java +++ /dev/null @@ -1,355 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.cloud.autoscaling; - -import org.apache.solr.client.solrj.SolrRequest; -import org.apache.solr.client.solrj.cloud.SolrCloudManager; -import org.apache.solr.client.solrj.cloud.autoscaling.*; -import org.apache.solr.common.SolrException; -import org.apache.solr.common.cloud.ClusterState; -import org.apache.solr.common.cloud.DocCollection; -import org.apache.solr.common.cloud.Replica; -import org.apache.solr.common.params.AutoScalingParams; -import org.apache.solr.common.params.CollectionParams; -import org.apache.solr.common.util.Pair; -import org.apache.solr.common.util.StrUtils; -import org.apache.solr.core.SolrResourceLoader; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.lang.invoke.MethodHandles; -import java.util.*; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.Predicate; -import java.util.stream.Collectors; - -import static org.apache.solr.cloud.autoscaling.TriggerEvent.NODE_NAMES; - -/** - * This class is responsible for using the configured policy and preferences - * with the hints provided by the trigger event to compute the required cluster operations. - *

- * The cluster operations computed here are put into the {@link ActionContext}'s properties - * with the key name "operations". The value is a List of SolrRequest objects. - */ -public class ComputePlanAction extends TriggerActionBase { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - // accept all collections by default - Predicate collectionsPredicate = s -> true; - - public ComputePlanAction() { - super(); - TriggerUtils.validProperties(validProperties, "collections"); - } - - - @Override - public void configure(SolrResourceLoader loader, SolrCloudManager cloudManager, Map properties) throws TriggerValidationException { - super.configure(loader, cloudManager, properties); - - Object value = properties.get("collections"); - if (value instanceof String) { - String colString = (String) value; - if (!colString.isEmpty()) { - List whiteListedCollections = StrUtils.splitSmart(colString, ','); - collectionsPredicate = whiteListedCollections::contains; - } - } else if (value instanceof Map) { - @SuppressWarnings({"unchecked"}) - Map matchConditions = (Map) value; - collectionsPredicate = collectionName -> { - try { - DocCollection collection = cloudManager.getClusterStateProvider().getCollection(collectionName); - if (collection == null) { - log.debug("Collection: {} was not found while evaluating conditions", collectionName); - return false; - } - for (Map.Entry entry : matchConditions.entrySet()) { - if (!entry.getValue().equals(collection.get(entry.getKey()))) { - if (log.isDebugEnabled()) { - log.debug("Collection: {} does not match condition: {}:{}", collectionName, entry.getKey(), entry.getValue()); - } - return false; - } - } - return true; - } catch (IOException e) { - log.error("Exception fetching collection information for: {}", collectionName, e); - return false; - } - }; - } - } - - @Override - public void process(TriggerEvent event, ActionContext context) throws Exception { - if (log.isDebugEnabled()) { - log.debug("-- processing event: {} with context properties: {}", event, context.getProperties()); - } - SolrCloudManager cloudManager = context.getCloudManager(); - try { - AutoScalingConfig autoScalingConf = cloudManager.getDistribStateManager().getAutoScalingConfig(); - if (autoScalingConf.isEmpty()) { - throw new Exception("Action: " + getName() + " executed but no policy is configured"); - } - PolicyHelper.SessionWrapper sessionWrapper = PolicyHelper.getSession(cloudManager); - Policy.Session session = sessionWrapper.get(); - ClusterState clusterState = cloudManager.getClusterStateProvider().getClusterState(); - if (log.isTraceEnabled()) { - log.trace("-- session: {}", session); - log.trace("-- state: {}", clusterState); - } - try { - Suggester suggester = getSuggester(session, event, context, cloudManager); - int maxOperations = getMaxNumOps(event, autoScalingConf, clusterState); - int requestedOperations = getRequestedNumOps(event); - if (requestedOperations > maxOperations) { - log.warn("Requested number of operations {} higher than maximum {}, adjusting...", - requestedOperations, maxOperations); - } - int opCount = 0; - int opLimit = maxOperations; - if (requestedOperations > 0) { - opLimit = requestedOperations; - } - do { - // computing changes in large clusters may take a long time - if (Thread.currentThread().isInterrupted()) { - throw new InterruptedException("stopping - thread was interrupted"); - } - @SuppressWarnings({"rawtypes"}) - SolrRequest operation = suggester.getSuggestion(); - opCount++; - // prepare suggester for the next iteration - if (suggester.getSession() != null) { - session = suggester.getSession(); - } - suggester = getSuggester(session, event, context, cloudManager); - - // break on first null op - // unless a specific number of ops was requested - // uncomment the following to log too many operations - /*if (opCount > 10) { - PolicyHelper.logState(cloudManager, initialSuggester); - }*/ - - if (operation == null) { - if (requestedOperations < 0) { - //uncomment the following to log zero operations -// PolicyHelper.logState(cloudManager, initialSuggester); - break; - } else { - log.info("Computed plan empty, remained {} requested ops to try.", opCount - opLimit); - continue; - } - } - if (log.isDebugEnabled()) { - log.debug("Computed Plan: {}", operation.getParams()); - } - Map props = context.getProperties(); - props.compute("operations", (k, v) -> { - @SuppressWarnings({"unchecked", "rawtypes"}) - List operations = (List) v; - if (operations == null) operations = new ArrayList<>(); - operations.add(operation); - return operations; - }); - } while (opCount < opLimit); - } finally { - releasePolicySession(sessionWrapper, session); - } - } catch (Exception e) { - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, - "Unexpected exception while processing event: " + event, e); - } - } - - private void releasePolicySession(PolicyHelper.SessionWrapper sessionWrapper, Policy.Session session) { - sessionWrapper.returnSession(session); - sessionWrapper.release(); - - } - - protected int getMaxNumOps(TriggerEvent event, AutoScalingConfig autoScalingConfig, ClusterState clusterState) { - // estimate a maximum default limit that should be sufficient for most purposes: - // number of nodes * total number of replicas * 3 - AtomicInteger totalRF = new AtomicInteger(); - clusterState.forEachCollection(coll -> { - Integer rf = coll.getReplicationFactor(); - if (rf == null) { - if (coll.getSlices().isEmpty()) { - rf = 1; // ??? - } else { - rf = coll.getReplicas().size() / coll.getSlices().size(); - } - } - totalRF.addAndGet(rf * coll.getSlices().size()); - }); - int totalMax = clusterState.getLiveNodes().size() * totalRF.get() * 3; - int maxOp = (Integer) autoScalingConfig.getProperties().getOrDefault(AutoScalingParams.MAX_COMPUTE_OPERATIONS, totalMax); - Object o = event.getProperty(AutoScalingParams.MAX_COMPUTE_OPERATIONS, maxOp); - try { - return Integer.parseInt(String.valueOf(o)); - } catch (Exception e) { - log.warn("Invalid '{}' event property: {}, using default {}", AutoScalingParams.MAX_COMPUTE_OPERATIONS, o, maxOp); - return maxOp; - } - } - - protected int getRequestedNumOps(TriggerEvent event) { - @SuppressWarnings({"unchecked"}) - Collection ops = (Collection) event.getProperty(TriggerEvent.REQUESTED_OPS, Collections.emptyList()); - if (ops.isEmpty()) { - return -1; - } else { - return ops.size(); - } - } - - private static final String START = "__start__"; - - protected Suggester getSuggester(Policy.Session session, TriggerEvent event, ActionContext context, SolrCloudManager cloudManager) throws IOException { - Suggester suggester; - switch (event.getEventType()) { - case NODEADDED: - suggester = getNodeAddedSuggester(cloudManager, session, event); - break; - case NODELOST: - suggester = getNodeLostSuggester(cloudManager, session, event); - break; - case SEARCHRATE: - case METRIC: - case INDEXSIZE: - @SuppressWarnings({"unchecked"}) - List ops = (List)event.getProperty(TriggerEvent.REQUESTED_OPS, Collections.emptyList()); - int start = (Integer)event.getProperty(START, 0); - if (ops.isEmpty() || start >= ops.size()) { - return NoneSuggester.get(session); - } - TriggerEvent.Op op = ops.get(start); - suggester = session.getSuggester(op.getAction()); - if (suggester instanceof UnsupportedSuggester) { - @SuppressWarnings({"unchecked"}) - List unsupportedOps = (List)context.getProperties().computeIfAbsent(TriggerEvent.UNSUPPORTED_OPS, k -> new ArrayList()); - unsupportedOps.add(op); - } - for (Map.Entry e : op.getHints().entrySet()) { - suggester = suggester.hint(e.getKey(), e.getValue()); - } - if (applyCollectionHints(cloudManager, suggester) == 0) return NoneSuggester.get(session); - suggester = suggester.forceOperation(true); - event.getProperties().put(START, ++start); - break; - case SCHEDULED: - String preferredOp = (String) event.getProperty(AutoScalingParams.PREFERRED_OP, CollectionParams.CollectionAction.MOVEREPLICA.toLower()); - CollectionParams.CollectionAction action = CollectionParams.CollectionAction.get(preferredOp); - suggester = session.getSuggester(action); - if (applyCollectionHints(cloudManager, suggester) == 0) return NoneSuggester.get(session); - break; - default: - throw new UnsupportedOperationException("No support for events other than nodeAdded, nodeLost, searchRate, metric, scheduled and indexSize. Received: " + event.getEventType()); - } - return suggester; - } - - private Suggester getNodeLostSuggester(SolrCloudManager cloudManager, Policy.Session session, TriggerEvent event) throws IOException { - String preferredOp = (String) event.getProperty(AutoScalingParams.PREFERRED_OP, CollectionParams.CollectionAction.MOVEREPLICA.toLower()); - CollectionParams.CollectionAction action = CollectionParams.CollectionAction.get(preferredOp); - switch (action) { - case MOVEREPLICA: - Suggester s = session.getSuggester(action) - .hint(Suggester.Hint.SRC_NODE, event.getProperty(NODE_NAMES)); - if (applyCollectionHints(cloudManager, s) == 0) return NoneSuggester.get(session); - return s; - case DELETENODE: - int start = (Integer)event.getProperty(START, 0); - @SuppressWarnings({"unchecked"}) - List srcNodes = (List) event.getProperty(NODE_NAMES); - if (srcNodes.isEmpty() || start >= srcNodes.size()) { - return NoneSuggester.get(session); - } - String sourceNode = srcNodes.get(start); - s = session.getSuggester(action) - .hint(Suggester.Hint.SRC_NODE, event.getProperty(NODE_NAMES)); - if (applyCollectionHints(cloudManager, s) == 0) return NoneSuggester.get(session); - s.hint(Suggester.Hint.SRC_NODE, Collections.singletonList(sourceNode)); - event.getProperties().put(START, ++start); - return s; - case NONE: - return NoneSuggester.get(session); - default: - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unsupported preferredOperation: " + action.toLower() + " specified for node lost trigger"); - } - } - - /** - * Applies collection hints for all collections that match the {@link #collectionsPredicate} - * and returns the number of collections that matched. - * @return number of collections that match the {@link #collectionsPredicate} - * @throws IOException if {@link org.apache.solr.client.solrj.impl.ClusterStateProvider} throws IOException - */ - private int applyCollectionHints(SolrCloudManager cloudManager, Suggester s) throws IOException { - ClusterState clusterState = cloudManager.getClusterStateProvider().getClusterState(); - Set set = clusterState.getCollectionStates().keySet().stream() - .filter(collectionRef -> collectionsPredicate.test(collectionRef)) - .collect(Collectors.toSet()); - if (set.size() < clusterState.getCollectionStates().size()) { - // apply hints only if a subset of collections are selected - set.forEach(c -> s.hint(Suggester.Hint.COLL, c)); - } - return set.size(); - } - - private Suggester getNodeAddedSuggester(SolrCloudManager cloudManager, Policy.Session session, TriggerEvent event) throws IOException { - String preferredOp = (String) event.getProperty(AutoScalingParams.PREFERRED_OP, CollectionParams.CollectionAction.MOVEREPLICA.toLower()); - Replica.Type replicaType = (Replica.Type) event.getProperty(AutoScalingParams.REPLICA_TYPE, Replica.Type.NRT); - CollectionParams.CollectionAction action = CollectionParams.CollectionAction.get(preferredOp); - - Suggester suggester = session.getSuggester(action) - .hint(Suggester.Hint.TARGET_NODE, event.getProperty(NODE_NAMES)); - switch (action) { - case ADDREPLICA: - // add all collection/shard pairs and let policy engine figure out which one - // to place on the target node - ClusterState clusterState = cloudManager.getClusterStateProvider().getClusterState(); - Set> collShards = new HashSet<>(); - clusterState.getCollectionStates().entrySet().stream() - .filter(e -> collectionsPredicate.test(e.getKey())) - .forEach(entry -> { - DocCollection docCollection = entry.getValue().get(); - if (docCollection != null) { - docCollection.getActiveSlices().stream() - .map(slice -> new Pair<>(entry.getKey(), slice.getName())) - .forEach(collShards::add); - } - }); - suggester.hint(Suggester.Hint.COLL_SHARD, collShards); - suggester.hint(Suggester.Hint.REPLICATYPE, replicaType); - break; - case MOVEREPLICA: - case NONE: - break; - default: - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, - "Unsupported preferredOperation=" + preferredOp + " for node added event"); - } - return suggester; - } -} diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/ExecutePlanAction.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/ExecutePlanAction.java deleted file mode 100644 index 1dfc3b1581c..00000000000 --- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/ExecutePlanAction.java +++ /dev/null @@ -1,250 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.cloud.autoscaling; - -import java.io.IOException; -import java.lang.invoke.MethodHandles; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; - -import org.apache.commons.lang3.exception.ExceptionUtils; -import org.apache.solr.client.solrj.SolrRequest; -import org.apache.solr.client.solrj.SolrResponse; -import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.client.solrj.cloud.autoscaling.AlreadyExistsException; -import org.apache.solr.client.solrj.cloud.DistribStateManager; -import org.apache.solr.client.solrj.cloud.SolrCloudManager; -import org.apache.solr.client.solrj.request.CollectionAdminRequest; -import org.apache.solr.client.solrj.response.RequestStatusState; -import org.apache.solr.common.SolrException; -import org.apache.solr.common.cloud.ZkStateReader; -import org.apache.solr.common.util.NamedList; -import org.apache.solr.common.util.Utils; -import org.apache.solr.core.SolrResourceLoader; -import org.apache.solr.util.TestInjection; -import org.apache.zookeeper.CreateMode; -import org.apache.zookeeper.KeeperException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * This class is responsible for executing cluster operations read from the {@link ActionContext}'s properties - * with the key name "operations". - */ -public class ExecutePlanAction extends TriggerActionBase { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - private static final String PREFIX = "op-"; - - static final int DEFAULT_TASK_TIMEOUT_SECONDS = 120; - public static final String TASK_TIMEOUT_SECONDS = "taskTimeoutSeconds"; - public static final String TASK_TIMEOUT_FAIL = "taskTimeoutFail"; - - int taskTimeoutSeconds; - boolean taskTimeoutFail; - - public ExecutePlanAction() { - TriggerUtils.validProperties(validProperties, TASK_TIMEOUT_SECONDS, TASK_TIMEOUT_FAIL); - } - - @Override - public void configure(SolrResourceLoader loader, SolrCloudManager cloudManager, Map properties) throws TriggerValidationException { - super.configure(loader, cloudManager, properties); - String str = String.valueOf(properties.getOrDefault(TASK_TIMEOUT_SECONDS, DEFAULT_TASK_TIMEOUT_SECONDS)); - taskTimeoutSeconds = Integer.parseInt(str); - str = String.valueOf(properties.getOrDefault(TASK_TIMEOUT_FAIL, false)); - taskTimeoutFail = Boolean.parseBoolean(str); - } - - @Override - @SuppressWarnings({"unchecked", "rawtypes"}) - public void process(TriggerEvent event, ActionContext context) throws Exception { - if (log.isDebugEnabled()) { - log.debug("-- processing event: {} with context properties: {}", event, context.getProperties()); - } - SolrCloudManager cloudManager = context.getCloudManager(); - List operations = (List) context.getProperty("operations"); - if (operations == null || operations.isEmpty()) { - log.info("No operations to execute for event: {}", event); - return; - } - try { - int counter = 0; - for (SolrRequest operation : operations) { - if (log.isDebugEnabled()) { - log.debug("Executing operation: {}", operation.getParams()); - } - try { - SolrResponse response = null; - if (operation instanceof CollectionAdminRequest.AsyncCollectionAdminRequest) { - CollectionAdminRequest.AsyncCollectionAdminRequest req = (CollectionAdminRequest.AsyncCollectionAdminRequest) operation; - // waitForFinalState so that the end effects of operations are visible - req.setWaitForFinalState(true); - String asyncId = event.getSource() + '/' + event.getId() + '/' + counter; - String znode = saveAsyncId(cloudManager.getDistribStateManager(), event, asyncId); - log.trace("Saved requestId: {} in znode: {}", asyncId, znode); - // TODO: find a better way of using async calls using dataProvider API !!! - req.setAsyncId(asyncId); - if (TestInjection.delayInExecutePlanAction != null) { - cloudManager.getTimeSource().sleep(TestInjection.delayInExecutePlanAction); - } - CollectionAdminRequest.RequestStatusResponse statusResponse = null; - RequestStatusState state = RequestStatusState.FAILED; - if (!TestInjection.failInExecutePlanAction) { - SolrResponse asyncResponse = cloudManager.request(req); - if (asyncResponse.getResponse().get("error") != null) { - throw new IOException("" + asyncResponse.getResponse().get("error")); - } - asyncId = (String)asyncResponse.getResponse().get("requestid"); - statusResponse = waitForTaskToFinish(cloudManager, asyncId, - taskTimeoutSeconds, TimeUnit.SECONDS); - } - if (statusResponse != null) { - state = statusResponse.getRequestStatus(); - // overwrite to test a long-running task - if (TestInjection.delayInExecutePlanAction != null && - TestInjection.delayInExecutePlanAction > TimeUnit.MILLISECONDS.convert(taskTimeoutSeconds, TimeUnit.SECONDS)) { - state = RequestStatusState.RUNNING; - } - if (TestInjection.failInExecutePlanAction) { - state = RequestStatusState.FAILED; - } - // should we accept partial success here? i.e. some operations won't be completed - // successfully but the event processing will still be declared a success - if (state == RequestStatusState.COMPLETED || state == RequestStatusState.FAILED || state == RequestStatusState.NOT_FOUND) { - // remove pending task marker for this request - try { - cloudManager.getDistribStateManager().removeData(znode, -1); - } catch (Exception e) { - log.warn("Unexpected exception while trying to delete znode: {}", znode, e); - } - } - response = statusResponse; - } - if (state == RequestStatusState.RUNNING || state == RequestStatusState.SUBMITTED) { - String msg = String.format(Locale.ROOT, "Task %s is still running after " + taskTimeoutSeconds + " seconds. Consider increasing " + - TASK_TIMEOUT_SECONDS + " action property or `waitFor` of the trigger %s. Operation: %s", - asyncId, event.source, req); - if (taskTimeoutFail) { - throw new IOException(msg); - } else { - log.warn(msg); - } - } else if (state == RequestStatusState.FAILED) { - // remove it as a pending task - try { - cloudManager.getDistribStateManager().removeData(znode, -1); - } catch (Exception e) { - log.warn("Unexpected exception while trying to delete znode: {}", znode, e); - } - throw new IOException("Task " + asyncId + " failed: " + (statusResponse != null ? statusResponse : " timed out. Operation: " + req)); - } - } else { - // generic response - can't easily determine success or failure - response = cloudManager.request(operation); - } - NamedList result = response.getResponse(); - context.getProperties().compute("responses", (s, o) -> { - @SuppressWarnings({"unchecked"}) - List> responses = (List>) o; - if (responses == null) responses = new ArrayList<>(operations.size()); - responses.add(result); - return responses; - }); - counter++; - } catch (IOException e) { - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, - "Unexpected exception executing operation: " + operation.getParams(), e); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "ExecutePlanAction was interrupted", e); - } catch (Exception e) { - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, - "Unexpected exception executing operation: " + operation.getParams(), e); - } - } - } catch (Exception e) { - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, - "Unexpected exception while processing event: " + event, e); - } - } - - - static CollectionAdminRequest.RequestStatusResponse waitForTaskToFinish(SolrCloudManager cloudManager, String requestId, long duration, TimeUnit timeUnit) throws IOException, InterruptedException { - long timeoutSeconds = timeUnit.toSeconds(duration); - RequestStatusState state = RequestStatusState.NOT_FOUND; - CollectionAdminRequest.RequestStatusResponse statusResponse = null; - for (int i = 0; i < timeoutSeconds; i++) { - try { - statusResponse = (CollectionAdminRequest.RequestStatusResponse)cloudManager.request(CollectionAdminRequest.requestStatus(requestId)); - state = statusResponse.getRequestStatus(); - if (state == RequestStatusState.COMPLETED || state == RequestStatusState.FAILED) { - log.trace("Task with requestId={} finished with state={} in {}s", requestId, state, i * 5); - cloudManager.request(CollectionAdminRequest.deleteAsyncId(requestId)); - return statusResponse; - } else if (state == RequestStatusState.NOT_FOUND) { - // the request for this id was never actually submitted! no harm done, just bail out - log.warn("Task with requestId={} was not found on overseer", requestId); - cloudManager.request(CollectionAdminRequest.deleteAsyncId(requestId)); - return statusResponse; - } - } catch (Exception e) { - Throwable rootCause = ExceptionUtils.getRootCause(e); - if (rootCause instanceof IllegalStateException && rootCause.getMessage().contains("Connection pool shut down")) { - throw e; - } - if (rootCause instanceof TimeoutException && rootCause.getMessage().contains("Could not connect to ZooKeeper")) { - throw e; - } - if (rootCause instanceof SolrServerException) { - throw e; - } - log.error("Unexpected Exception while querying status of requestId={}", requestId, e); - throw e; - } - if (i > 0 && i % 5 == 0) { - log.trace("Task with requestId={} still not complete after {}s. Last state={}", requestId, i * 5, state); - } - cloudManager.getTimeSource().sleep(5000); - } - log.debug("Task with requestId={} did not complete within {} seconds. Last state={}", timeoutSeconds, requestId, state); - return statusResponse; - } - - /** - * Saves the given asyncId in ZK as a persistent sequential node. This allows us to wait for the completion - * of pending tasks from this event in {@link ScheduledTriggers} - * before starting the actions of the next event. - * - * @return the path of the newly created node in ZooKeeper - */ - private String saveAsyncId(DistribStateManager stateManager, TriggerEvent event, String asyncId) throws InterruptedException, AlreadyExistsException, IOException, KeeperException { - String parentPath = ZkStateReader.SOLR_AUTOSCALING_TRIGGER_STATE_PATH + "/" + event.getSource() + "/" + getName(); - try { - stateManager.makePath(parentPath); - } catch (AlreadyExistsException e) { - // ignore - } - return stateManager.createData(parentPath + "/" + PREFIX, Utils.toJSON(Collections.singletonMap("requestid", asyncId)), CreateMode.PERSISTENT_SEQUENTIAL); - } - -} diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/HttpTriggerListener.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/HttpTriggerListener.java deleted file mode 100644 index 139efe0946f..00000000000 --- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/HttpTriggerListener.java +++ /dev/null @@ -1,164 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.cloud.autoscaling; - -import java.io.IOException; -import java.lang.invoke.MethodHandles; -import java.util.HashMap; -import java.util.Map; -import java.util.Properties; -import java.util.StringJoiner; - -import org.apache.solr.client.solrj.SolrRequest; -import org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig; -import org.apache.solr.client.solrj.cloud.SolrCloudManager; -import org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventProcessorStage; -import org.apache.solr.client.solrj.impl.HttpClientUtil; -import org.apache.solr.common.util.Utils; -import org.apache.solr.core.SolrResourceLoader; -import org.apache.solr.util.PropertiesUtil; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Simple HTTP callback that POSTs event data to a URL. - * URL, payload and headers may contain property substitution patterns, with the following properties available: - *
    - *
  • config.* - listener configuration
    • - *
  • event.* - event properties
    • - *
  • stage - current stage of event processing
    • - *
  • actionName - optional current action name
    • - *
  • context.* - optional {@link ActionContext} properties
    • - *
  • error - optional error string (from {@link Throwable#toString()})
    • - *
  • message - optional message
    • - *
- * The following listener configuration is supported: - *
    - *
  • url - a URL template
    • - *
  • payload - string, optional payload template. If absent a JSON map of all properties listed above will be used.
    • - *
  • contentType - string, optional payload content type. If absent then application/json will be used.
    • - *
  • header.* - string, optional header template(s). The name of the property without "header." prefix defines the literal header name.
    • - *
  • timeout - int, optional connection and socket timeout in milliseconds. Default is 60 seconds.
    • - *
  • followRedirects - boolean, optional setting to follow redirects. Default is false.
    • - *
- */ -public class HttpTriggerListener extends TriggerListenerBase { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - private String urlTemplate; - private String payloadTemplate; - private String contentType; - private Map headerTemplates = new HashMap<>(); - private int timeout = HttpClientUtil.DEFAULT_CONNECT_TIMEOUT; - private boolean followRedirects; - - public HttpTriggerListener() { - super(); - TriggerUtils.requiredProperties(requiredProperties, validProperties, "url"); - TriggerUtils.validProperties(validProperties, "payload", "contentType", "timeout", "followRedirects"); - validPropertyPrefixes.add("header."); - } - - @Override - public void configure(SolrResourceLoader loader, SolrCloudManager cloudManager, AutoScalingConfig.TriggerListenerConfig config) throws TriggerValidationException { - super.configure(loader, cloudManager, config); - urlTemplate = (String)config.properties.get("url"); - payloadTemplate = (String)config.properties.get("payload"); - contentType = (String)config.properties.get("contentType"); - config.properties.forEach((k, v) -> { - if (k.startsWith("header.")) { - headerTemplates.put(k.substring(7), String.valueOf(v)); - } - }); - timeout = PropertiesUtil.toInteger(String.valueOf(config.properties.get("timeout")), HttpClientUtil.DEFAULT_CONNECT_TIMEOUT); - followRedirects = PropertiesUtil.toBoolean(String.valueOf(config.properties.get("followRedirects"))); - } - - @Override - public void onEvent(TriggerEvent event, TriggerEventProcessorStage stage, String actionName, ActionContext context, Throwable error, String message) { - Properties properties = new Properties(); - properties.setProperty("stage", stage.toString()); - // if configuration used "actionName" but we're in a non-action related stage then PropertiesUtil will - // throws an exception on missing value - so replace it with an empty string - if (actionName == null) { - actionName = ""; - } - properties.setProperty("actionName", actionName); - if (context != null) { - context.getProperties().forEach((k, v) -> { - properties.setProperty("context." + k, String.valueOf(v)); - }); - } - if (error != null) { - properties.setProperty("error", error.toString()); - } else { - properties.setProperty("error", ""); - } - if (message != null) { - properties.setProperty("message", message); - } else { - properties.setProperty("message", ""); - } - // add event properties - properties.setProperty("event.id", event.getId()); - properties.setProperty("event.source", event.getSource()); - properties.setProperty("event.eventTime", String.valueOf(event.eventTime)); - properties.setProperty("event.eventType", event.getEventType().toString()); - event.getProperties().forEach((k, v) -> { - properties.setProperty("event.properties." + k, String.valueOf(v)); - }); - // add config properties - properties.setProperty("config.name", config.name); - properties.setProperty("config.trigger", config.trigger); - properties.setProperty("config.listenerClass", config.listenerClass); - properties.setProperty("config.beforeActions", String.join(",", config.beforeActions)); - properties.setProperty("config.afterActions", String.join(",", config.afterActions)); - StringJoiner joiner = new StringJoiner(","); - config.stages.forEach(s -> joiner.add(s.toString())); - properties.setProperty("config.stages", joiner.toString()); - config.properties.forEach((k, v) -> { - properties.setProperty("config.properties." + k, String.valueOf(v)); - }); - String url = PropertiesUtil.substituteProperty(urlTemplate, properties); - String payload; - String type; - if (payloadTemplate != null) { - payload = PropertiesUtil.substituteProperty(payloadTemplate, properties); - if (contentType != null) { - type = contentType; - } else { - type = "application/json"; - } - } else { - payload = Utils.toJSONString(properties); - type = "application/json"; - } - Map headers = new HashMap<>(); - headerTemplates.forEach((k, v) -> { - String headerVal = PropertiesUtil.substituteProperty(v, properties); - if (!headerVal.isEmpty()) { - headers.put(k, headerVal); - } - }); - headers.put("Content-Type", type); - try { - cloudManager.httpRequest(url, SolrRequest.METHOD.POST, headers, payload, timeout, followRedirects); - } catch (IOException e) { - log.warn("Exception sending request for event {}", event, e); - } - } -} diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/InactiveMarkersPlanAction.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/InactiveMarkersPlanAction.java deleted file mode 100644 index 648254203fc..00000000000 --- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/InactiveMarkersPlanAction.java +++ /dev/null @@ -1,141 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.cloud.autoscaling; - -import java.io.IOException; -import java.lang.invoke.MethodHandles; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.NoSuchElementException; -import java.util.Set; -import java.util.TreeSet; -import java.util.concurrent.TimeUnit; - -import org.apache.solr.client.solrj.cloud.DistribStateManager; -import org.apache.solr.client.solrj.cloud.SolrCloudManager; -import org.apache.solr.client.solrj.cloud.autoscaling.BadVersionException; -import org.apache.solr.client.solrj.cloud.autoscaling.NotEmptyException; -import org.apache.solr.common.cloud.ZkStateReader; -import org.apache.solr.common.util.Utils; -import org.apache.solr.core.SolrResourceLoader; -import org.apache.zookeeper.KeeperException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import static org.apache.solr.cloud.autoscaling.OverseerTriggerThread.MARKER_ACTIVE; -import static org.apache.solr.cloud.autoscaling.OverseerTriggerThread.MARKER_STATE; - -/** - * This plan simply removes nodeAdded and nodeLost markers from Zookeeper if their TTL has - * expired. These markers are used by {@link NodeAddedTrigger} and {@link NodeLostTrigger} to - * ensure fault tolerance in case of Overseer leader crash. - */ -public class InactiveMarkersPlanAction extends TriggerActionBase { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - public static final String TTL_PROP = "ttl"; - - public static final int DEFAULT_TTL_SECONDS = 3600 * 24 * 2; - - private int cleanupTTL; - - public InactiveMarkersPlanAction() { - super(); - TriggerUtils.validProperties(validProperties, TTL_PROP); - } - - @Override - public void configure(SolrResourceLoader loader, SolrCloudManager cloudManager, Map properties) throws TriggerValidationException { - super.configure(loader, cloudManager, properties); - String cleanupStr = String.valueOf(properties.getOrDefault(TTL_PROP, String.valueOf(DEFAULT_TTL_SECONDS))); - try { - cleanupTTL = Integer.parseInt(cleanupStr); - } catch (Exception e) { - throw new TriggerValidationException(getName(), TTL_PROP, "invalid value '" + cleanupStr + "': " + e.toString()); - } - if (cleanupTTL < 0) { - throw new TriggerValidationException(getName(), TTL_PROP, "invalid value '" + cleanupStr + "', should be > 0. "); - } - } - - @Override - public void process(TriggerEvent event, ActionContext context) throws Exception { - if (log.isTraceEnabled()) { - log.trace("-- {} cleaning markers", getName()); - } - // use epoch time to track this across JVMs and nodes - long currentTimeNs = cloudManager.getTimeSource().getEpochTimeNs(); - Map results = new LinkedHashMap<>(); - Set cleanedUp = new TreeSet<>(); - cleanupMarkers(ZkStateReader.SOLR_AUTOSCALING_NODE_ADDED_PATH, currentTimeNs, cleanedUp); - if (!cleanedUp.isEmpty()) { - results.put("nodeAdded", cleanedUp); - cleanedUp = new TreeSet<>(); - } - cleanupMarkers(ZkStateReader.SOLR_AUTOSCALING_NODE_LOST_PATH, currentTimeNs, cleanedUp); - if (!cleanedUp.isEmpty()) { - results.put("nodeLost", cleanedUp); - } - if (!results.isEmpty()) { - context.getProperties().put(getName(), results); - } - } - - private void cleanupMarkers(String path, long currentTimeNs, Set cleanedUp) throws Exception { - DistribStateManager stateManager = cloudManager.getDistribStateManager(); - if (!stateManager.hasData(path)) { - return; - } - List markers = stateManager.listData(path); - markers.forEach(m -> { - String markerPath = path + "/" + m; - try { - Map payload = Utils.getJson(stateManager, markerPath); - if (payload.isEmpty()) { - log.trace(" -- ignore {}: either missing or unsupported format", markerPath); - return; - } - boolean activeMarker = payload.getOrDefault(MARKER_STATE, MARKER_ACTIVE) - .equals(MARKER_ACTIVE); - long timestamp = ((Number)payload.get("timestamp")).longValue(); - long delta = TimeUnit.NANOSECONDS.toSeconds(currentTimeNs - timestamp); - if (delta > cleanupTTL || !activeMarker) { - try { - stateManager.removeData(markerPath, -1); - log.trace(" -- remove {}, delta={}, ttl={}, active={}", markerPath, delta, cleanupTTL, activeMarker); - cleanedUp.add(m); - } catch (NoSuchElementException nse) { - // someone already removed it - ignore - return; - } catch (BadVersionException be) { - throw new RuntimeException("should never happen", be); - } catch (NotEmptyException ne) { - log.error("Marker znode should be empty but it's not! Ignoring {} ({})", markerPath, ne); - } - } else { - log.trace(" -- keep {}, delta={}, ttl={}, active={}", markerPath, delta, cleanupTTL, activeMarker); - } - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - return; - } catch (IOException | KeeperException e) { - log.warn("Could not cleanup marker at {}, skipping... ", markerPath, e); - } - }); - } -} diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/InactiveShardPlanAction.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/InactiveShardPlanAction.java deleted file mode 100644 index d3de649cf7a..00000000000 --- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/InactiveShardPlanAction.java +++ /dev/null @@ -1,163 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.cloud.autoscaling; - -import java.lang.invoke.MethodHandles; -import java.util.ArrayList; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.NoSuchElementException; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; - -import org.apache.solr.client.solrj.SolrRequest; -import org.apache.solr.client.solrj.cloud.SolrCloudManager; -import org.apache.solr.client.solrj.request.CollectionAdminRequest; -import org.apache.solr.common.cloud.ClusterState; -import org.apache.solr.common.cloud.Slice; -import org.apache.solr.common.cloud.ZkStateReader; -import org.apache.solr.common.util.Utils; -import org.apache.solr.core.SolrResourceLoader; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * This class checks whether there are shards that have been inactive for a long - * time (which usually means they are left-overs from shard splitting) and requests their removal - * after their cleanup TTL period elapsed. - *

Shard delete requests are put into the {@link ActionContext}'s properties - * with the key name "operations". The value is a List of SolrRequest objects.

- */ -public class InactiveShardPlanAction extends TriggerActionBase { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - public static final String TTL_PROP = "ttl"; - - public static final int DEFAULT_TTL_SECONDS = 3600 * 24 * 2; - - private int cleanupTTL; - - public InactiveShardPlanAction() { - super(); - TriggerUtils.validProperties(validProperties, TTL_PROP); - } - - @Override - public void configure(SolrResourceLoader loader, SolrCloudManager cloudManager, Map properties) throws TriggerValidationException { - super.configure(loader, cloudManager, properties); - String cleanupStr = String.valueOf(properties.getOrDefault(TTL_PROP, String.valueOf(DEFAULT_TTL_SECONDS))); - try { - cleanupTTL = Integer.parseInt(cleanupStr); - } catch (Exception e) { - throw new TriggerValidationException(getName(), TTL_PROP, "invalid value '" + cleanupStr + "': " + e.toString()); - } - if (cleanupTTL < 0) { - throw new TriggerValidationException(getName(), TTL_PROP, "invalid value '" + cleanupStr + "', should be > 0. "); - } - } - - @Override - public void process(TriggerEvent event, ActionContext context) throws Exception { - SolrCloudManager cloudManager = context.getCloudManager(); - ClusterState state = cloudManager.getClusterStateProvider().getClusterState(); - Map> cleanup = new LinkedHashMap<>(); - Map> inactive = new LinkedHashMap<>(); - Map> staleLocks = new LinkedHashMap<>(); - state.forEachCollection(coll -> - coll.getSlices().forEach(s -> { - if (Slice.State.INACTIVE.equals(s.getState())) { - inactive.computeIfAbsent(coll.getName(), c -> new ArrayList<>()).add(s.getName()); - String tstampStr = s.getStr(ZkStateReader.STATE_TIMESTAMP_PROP); - if (tstampStr == null || tstampStr.isEmpty()) { - return; - } - long timestamp = Long.parseLong(tstampStr); - // this timestamp uses epoch time - long currentTime = cloudManager.getTimeSource().getEpochTimeNs(); - long delta = TimeUnit.NANOSECONDS.toSeconds(currentTime - timestamp); - if (log.isDebugEnabled()) { - log.debug("{}/{}: tstamp={}, time={}, delta={}", coll.getName(), s.getName(), timestamp, currentTime, delta); - } - if (delta > cleanupTTL) { - if (log.isDebugEnabled()) { - log.debug("-- delete inactive {} / {}", coll.getName(), s.getName()); - } - @SuppressWarnings({"unchecked", "rawtypes"}) - List operations = (List)context.getProperties().computeIfAbsent("operations", k -> new ArrayList<>()); - operations.add(CollectionAdminRequest.deleteShard(coll.getName(), s.getName())); - cleanup.computeIfAbsent(coll.getName(), c -> new ArrayList<>()).add(s.getName()); - } - } - // check for stale shard split locks - String parentPath = ZkStateReader.COLLECTIONS_ZKNODE + "/" + coll.getName(); - List locks; - try { - locks = cloudManager.getDistribStateManager().listData(parentPath).stream() - .filter(name -> name.endsWith("-splitting")) - .collect(Collectors.toList()); - for (String lock : locks) { - try { - String lockPath = parentPath + "/" + lock; - Map lockData = Utils.getJson(cloudManager.getDistribStateManager(), lockPath); - String tstampStr = (String)lockData.get(ZkStateReader.STATE_TIMESTAMP_PROP); - if (tstampStr == null || tstampStr.isEmpty()) { - return; - } - long timestamp = Long.parseLong(tstampStr); - // this timestamp uses epoch time - long currentTime = cloudManager.getTimeSource().getEpochTimeNs(); - long delta = TimeUnit.NANOSECONDS.toSeconds(currentTime - timestamp); - if (log.isDebugEnabled()) { - log.debug("{}/{}: locktstamp={}, time={}, delta={}", coll.getName(), lock, timestamp, currentTime, delta); - } - if (delta > cleanupTTL) { - if (log.isDebugEnabled()) { - log.debug("-- delete inactive split lock for {}/{}, delta={}", coll.getName(), lock, delta); - } - cloudManager.getDistribStateManager().removeData(lockPath, -1); - lockData.put("currentTimeNs", currentTime); - lockData.put("deltaSec", delta); - lockData.put("ttlSec", cleanupTTL); - staleLocks.put(coll.getName() + "/" + lock, lockData); - } else { - if (log.isDebugEnabled()) { - log.debug("-- lock {}/{} still active (delta={})", coll.getName(), lock, delta); - } - } - } catch (NoSuchElementException nse) { - // already removed by someone else - ignore - } - } - } catch (Exception e) { - log.warn("Exception checking for inactive shard split locks in {}", parentPath, e); - } - }) - ); - Map results = new LinkedHashMap<>(); - if (!cleanup.isEmpty()) { - results.put("inactive", inactive); - results.put("cleanup", cleanup); - } - if (!staleLocks.isEmpty()) { - results.put("staleLocks", staleLocks); - } - if (!results.isEmpty()) { - context.getProperties().put(getName(), results); - } - } -} \ No newline at end of file diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/IndexSizeTrigger.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/IndexSizeTrigger.java deleted file mode 100644 index e9878281dec..00000000000 --- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/IndexSizeTrigger.java +++ /dev/null @@ -1,562 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.cloud.autoscaling; - -import java.io.IOException; -import java.lang.invoke.MethodHandles; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.TreeMap; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicLong; -import java.util.Locale; - -import org.apache.solr.client.solrj.cloud.SolrCloudManager; -import org.apache.solr.client.solrj.cloud.autoscaling.Suggester; -import org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventType; -import org.apache.solr.common.SolrException; -import org.apache.solr.common.cloud.ClusterState; -import org.apache.solr.common.cloud.DocCollection; -import org.apache.solr.common.cloud.Replica; -import org.apache.solr.common.cloud.Slice; -import org.apache.solr.common.params.CollectionParams; -import org.apache.solr.common.params.CommonAdminParams; -import org.apache.solr.common.util.Pair; -import org.apache.solr.common.util.StrUtils; -import org.apache.solr.common.util.Utils; -import org.apache.solr.core.SolrResourceLoader; -import org.apache.solr.metrics.SolrCoreMetricManager; -import org.apache.solr.update.SolrIndexSplitter; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import static org.apache.solr.client.solrj.cloud.autoscaling.Variable.Type.CORE_IDX; - -/** - * - */ -public class IndexSizeTrigger extends TriggerBase { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - // configuration properties - public static final String ABOVE_BYTES_PROP = "aboveBytes"; - public static final String ABOVE_DOCS_PROP = "aboveDocs"; - public static final String ABOVE_OP_PROP = "aboveOp"; - public static final String BELOW_BYTES_PROP = "belowBytes"; - public static final String BELOW_DOCS_PROP = "belowDocs"; - public static final String BELOW_OP_PROP = "belowOp"; - public static final String COLLECTIONS_PROP = "collections"; - public static final String MAX_OPS_PROP = "maxOps"; - public static final String SPLIT_FUZZ_PROP = CommonAdminParams.SPLIT_FUZZ; - public static final String SPLIT_METHOD_PROP = CommonAdminParams.SPLIT_METHOD; - public static final String SPLIT_BY_PREFIX = CommonAdminParams.SPLIT_BY_PREFIX; - - // event properties - public static final String BYTES_SIZE_KEY = "__bytes__"; - public static final String TOTAL_BYTES_SIZE_KEY = "__total_bytes__"; - public static final String DOCS_SIZE_KEY = "__docs__"; - public static final String MAX_DOC_KEY = "__maxDoc__"; - public static final String COMMIT_SIZE_KEY = "__commitBytes__"; - public static final String ABOVE_SIZE_KEY = "aboveSize"; - public static final String BELOW_SIZE_KEY = "belowSize"; - public static final String VIOLATION_KEY = "violationType"; - - public static final int DEFAULT_MAX_OPS = 10; - - public enum Unit { bytes, docs } - - private long aboveBytes, aboveDocs, belowBytes, belowDocs; - private int maxOps; - private SolrIndexSplitter.SplitMethod splitMethod; - private boolean splitByPrefix; - private float splitFuzz; - private CollectionParams.CollectionAction aboveOp, belowOp; - private final Set collections = new HashSet<>(); - private final Map lastAboveEventMap = new ConcurrentHashMap<>(); - private final Map lastBelowEventMap = new ConcurrentHashMap<>(); - - public IndexSizeTrigger(String name) { - super(TriggerEventType.INDEXSIZE, name); - TriggerUtils.validProperties(validProperties, - ABOVE_BYTES_PROP, ABOVE_DOCS_PROP, ABOVE_OP_PROP, - BELOW_BYTES_PROP, BELOW_DOCS_PROP, BELOW_OP_PROP, - COLLECTIONS_PROP, MAX_OPS_PROP, - SPLIT_METHOD_PROP, SPLIT_FUZZ_PROP, SPLIT_BY_PREFIX); - } - - @Override - public void configure(SolrResourceLoader loader, SolrCloudManager cloudManager, Map properties) throws TriggerValidationException { - super.configure(loader, cloudManager, properties); - String aboveStr = String.valueOf(properties.getOrDefault(ABOVE_BYTES_PROP, Long.MAX_VALUE)); - String belowStr = String.valueOf(properties.getOrDefault(BELOW_BYTES_PROP, -1)); - try { - aboveBytes = Long.parseLong(aboveStr); - if (aboveBytes <= 0) { - throw new Exception("value must be > 0"); - } - } catch (Exception e) { - throw new TriggerValidationException(getName(), ABOVE_BYTES_PROP, "invalid value '" + aboveStr + "': " + e.toString()); - } - try { - belowBytes = Long.parseLong(belowStr); - if (belowBytes < 0) { - belowBytes = -1; - } - } catch (Exception e) { - throw new TriggerValidationException(getName(), BELOW_BYTES_PROP, "invalid value '" + belowStr + "': " + e.toString()); - } - // below must be at least 2x smaller than above, otherwise splitting a shard - // would immediately put the shard below the threshold and cause the mergeshards action - if (belowBytes > 0 && (belowBytes * 2 > aboveBytes)) { - throw new TriggerValidationException(getName(), BELOW_BYTES_PROP, - "invalid value " + belowBytes + ", should be less than half of '" + ABOVE_BYTES_PROP + "' value, which is " + aboveBytes); - } - // do the same for docs bounds - aboveStr = String.valueOf(properties.getOrDefault(ABOVE_DOCS_PROP, Long.MAX_VALUE)); - belowStr = String.valueOf(properties.getOrDefault(BELOW_DOCS_PROP, -1)); - try { - aboveDocs = Long.parseLong(aboveStr); - if (aboveDocs <= 0) { - throw new Exception("value must be > 0"); - } - } catch (Exception e) { - throw new TriggerValidationException(getName(), ABOVE_DOCS_PROP, "invalid value '" + aboveStr + "': " + e.toString()); - } - try { - belowDocs = Long.parseLong(belowStr); - if (belowDocs < 0) { - belowDocs = -1; - } - } catch (Exception e) { - throw new TriggerValidationException(getName(), BELOW_DOCS_PROP, "invalid value '" + belowStr + "': " + e.toString()); - } - // below must be at least 2x smaller than above, otherwise splitting a shard - // would immediately put the shard below the threshold and cause the mergeshards action - if (belowDocs > 0 && (belowDocs * 2 > aboveDocs)) { - throw new TriggerValidationException(getName(), BELOW_DOCS_PROP, - "invalid value " + belowDocs + ", should be less than half of '" + ABOVE_DOCS_PROP + "' value, which is " + aboveDocs); - } - - String collectionsString = (String) properties.get(COLLECTIONS_PROP); - if (collectionsString != null && !collectionsString.isEmpty()) { - collections.addAll(StrUtils.splitSmart(collectionsString, ',')); - } - String aboveOpStr = String.valueOf(properties.getOrDefault(ABOVE_OP_PROP, CollectionParams.CollectionAction.SPLITSHARD.toLower())); - // TODO: this is a placeholder until SOLR-9407 is implemented - String belowOpStr = String.valueOf(properties.getOrDefault(BELOW_OP_PROP, CollectionParams.CollectionAction.MERGESHARDS.toLower())); - aboveOp = CollectionParams.CollectionAction.get(aboveOpStr); - if (aboveOp == null) { - throw new TriggerValidationException(getName(), ABOVE_OP_PROP, "unrecognized value of: '" + aboveOpStr + "'"); - } - belowOp = CollectionParams.CollectionAction.get(belowOpStr); - if (belowOp == null) { - throw new TriggerValidationException(getName(), BELOW_OP_PROP, "unrecognized value of: '" + belowOpStr + "'"); - } - String maxOpsStr = String.valueOf(properties.getOrDefault(MAX_OPS_PROP, DEFAULT_MAX_OPS)); - try { - maxOps = Integer.parseInt(maxOpsStr); - if (maxOps < 1) { - throw new Exception("must be > 1"); - } - } catch (Exception e) { - throw new TriggerValidationException(getName(), MAX_OPS_PROP, "invalid value: '" + maxOpsStr + "': " + e.getMessage()); - } - String methodStr = (String)properties.getOrDefault(SPLIT_METHOD_PROP, SolrIndexSplitter.SplitMethod.LINK.toLower()); - splitMethod = SolrIndexSplitter.SplitMethod.get(methodStr); - if (splitMethod == null) { - throw new TriggerValidationException(getName(), SPLIT_METHOD_PROP, "unrecognized value of: '" + methodStr + "'"); - } - String fuzzStr = String.valueOf(properties.getOrDefault(SPLIT_FUZZ_PROP, 0.0f)); - try { - splitFuzz = Float.parseFloat(fuzzStr); - } catch (Exception e) { - throw new TriggerValidationException(getName(), SPLIT_FUZZ_PROP, "invalid value: '" + fuzzStr + "': " + e.getMessage()); - } - String splitByPrefixStr = String.valueOf(properties.getOrDefault(SPLIT_BY_PREFIX, false)); - try { - splitByPrefix = getValidBool(splitByPrefixStr); - } catch (Exception e) { - throw new TriggerValidationException(getName(), SPLIT_BY_PREFIX, "invalid value: '" + splitByPrefixStr + "': " + e.getMessage()); - } - } - - private boolean getValidBool(String str) throws Exception { - if (str != null && (str.toLowerCase(Locale.ROOT).equals("true") || str.toLowerCase(Locale.ROOT).equals("false"))) { - return Boolean.parseBoolean(str); - } - throw new IllegalArgumentException("Expected a valid boolean value but got " + str); - } - - @Override - protected Map getState() { - Map state = new HashMap<>(); - state.put("lastAboveEventMap", lastAboveEventMap); - state.put("lastBelowEventMap", lastBelowEventMap); - return state; - } - - @Override - @SuppressWarnings({"unchecked"}) - protected void setState(Map state) { - this.lastAboveEventMap.clear(); - this.lastBelowEventMap.clear(); - Map replicaVsTime = (Map)state.get("lastAboveEventMap"); - if (replicaVsTime != null) { - this.lastAboveEventMap.putAll(replicaVsTime); - } - replicaVsTime = (Map)state.get("lastBelowEventMap"); - if (replicaVsTime != null) { - this.lastBelowEventMap.putAll(replicaVsTime); - } - } - - @Override - public void restoreState(AutoScaling.Trigger old) { - assert old.isClosed(); - if (old instanceof IndexSizeTrigger) { - IndexSizeTrigger that = (IndexSizeTrigger)old; - assert this.name.equals(that.name); - this.lastAboveEventMap.clear(); - this.lastBelowEventMap.clear(); - this.lastAboveEventMap.putAll(that.lastAboveEventMap); - this.lastBelowEventMap.putAll(that.lastBelowEventMap); - } else { - throw new SolrException(SolrException.ErrorCode.INVALID_STATE, - "Unable to restore state from an unknown type of trigger"); - } - } - - @Override - @SuppressWarnings({"unchecked", "rawtypes"}) - public void run() { - synchronized(this) { - if (isClosed) { - log.warn("{} ran but was already closed", getName()); - return; - } - } - AutoScaling.TriggerEventProcessor processor = processorRef.get(); - if (processor == null) { - return; - } - - // replica name / info + size, retrieved from leaders only - Map currentSizes = new HashMap<>(); - - try { - ClusterState clusterState = cloudManager.getClusterStateProvider().getClusterState(); - for (String node : clusterState.getLiveNodes()) { - Map metricTags = new HashMap<>(); - // coll, shard, replica - Map>> infos = cloudManager.getNodeStateProvider().getReplicaInfo(node, Collections.emptyList()); - infos.forEach((coll, shards) -> { - if (!collections.isEmpty() && !collections.contains(coll)) { - return; - } - DocCollection docCollection = clusterState.getCollection(coll); - - shards.forEach((sh, replicas) -> { - // check only the leader replica in an active shard - Slice s = docCollection.getSlice(sh); - if (s.getState() != Slice.State.ACTIVE) { - return; - } - Replica r = s.getLeader(); - // no leader - don't do anything - if (r == null) { - return; - } - // not on this node - if (!r.getNodeName().equals(node)) { - return; - } - // find ReplicaInfo - Replica info = null; - for (Replica ri : replicas) { - if (r.getCoreName().equals(ri.getCoreName())) { - info = ri; - break; - } - } - if (info == null) { - // probably replica is not on this node? - return; - } - // we have to translate to the metrics registry name, which uses "_replica_nN" as suffix - String replicaName = Utils.parseMetricsReplicaName(coll, info.getCoreName()); - if (replicaName == null) { // should never happen??? - replicaName = info.getName(); // which is actually coreNode name... - } - String registry = SolrCoreMetricManager.createRegistryName(true, coll, sh, replicaName, null); - String tag = "metrics:" + registry + ":" + CORE_IDX.metricsAttribute; - metricTags.put(tag, info); - tag = "metrics:" + registry + ":SEARCHER.searcher.numDocs"; - metricTags.put(tag, info); - tag = "metrics:" + registry + ":SEARCHER.searcher.maxDoc"; - metricTags.put(tag, info); - tag = "metrics:" + registry + ":SEARCHER.searcher.indexCommitSize"; - metricTags.put(tag, info); - }); - }); - if (metricTags.isEmpty()) { - continue; - } - Map sizes = cloudManager.getNodeStateProvider().getNodeValues(node, metricTags.keySet()); - sizes.forEach((tag, size) -> { - final Replica info = metricTags.get(tag); - if (info == null) { - log.warn("Missing replica info for response tag {}", tag); - } else { - // verify that it's a Number - if (!(size instanceof Number)) { - log.warn("invalid size value for tag {} - not a number: '{}' is {}", tag, size, size.getClass().getName()); - return; - } - - Replica currentInfo = currentSizes.computeIfAbsent(info.getCoreName(), k -> (Replica) info.clone()); - if (tag.contains("INDEX")) { - currentInfo.getProperties().put(TOTAL_BYTES_SIZE_KEY, ((Number) size).longValue()); - } else if (tag.endsWith("SEARCHER.searcher.numDocs")) { - currentInfo.getProperties().put(DOCS_SIZE_KEY, ((Number) size).longValue()); - } else if (tag.endsWith("SEARCHER.searcher.maxDoc")) { - currentInfo.getProperties().put(MAX_DOC_KEY, ((Number) size).longValue()); - } else if (tag.endsWith("SEARCHER.searcher.indexCommitSize")) { - currentInfo.getProperties().put(COMMIT_SIZE_KEY, ((Number) size).longValue()); - } - } - }); - } - } catch (IOException e) { - log.warn("Error running trigger {}", getName(), e); - return; - } - - long now = cloudManager.getTimeSource().getTimeNs(); - - // now check thresholds - - // collection / list(info) - Map> aboveSize = new HashMap<>(); - - Set splittable = new HashSet<>(); - - currentSizes.forEach((coreName, info) -> { - // calculate estimated bytes - long maxDoc = (Long)info.get(MAX_DOC_KEY); - long numDocs = (Long)info.get(DOCS_SIZE_KEY); - long commitSize = (Long)info.get(COMMIT_SIZE_KEY, 0L); - if (commitSize <= 0) { - commitSize = (Long)info.get(TOTAL_BYTES_SIZE_KEY); - } - // calculate estimated size as a side-effect - commitSize = estimatedSize(maxDoc, numDocs, commitSize); - info.getProperties().put(BYTES_SIZE_KEY, commitSize); - - if ((Long)info.get(BYTES_SIZE_KEY) > aboveBytes || - (Long)info.get(DOCS_SIZE_KEY) > aboveDocs) { - if (waitForElapsed(coreName, now, lastAboveEventMap)) { - List infos = aboveSize.computeIfAbsent(info.getCollection(), c -> new ArrayList<>()); - if (!infos.contains(info)) { - if ((Long)info.get(BYTES_SIZE_KEY) > aboveBytes) { - info.getProperties().put(VIOLATION_KEY, ABOVE_BYTES_PROP); - } else { - info.getProperties().put(VIOLATION_KEY, ABOVE_DOCS_PROP); - } - infos.add(info); - splittable.add(info.getName()); - } - } - } else { - // no violation - clear waitForElapsed - lastAboveEventMap.remove(coreName); - } - }); - - // collection / list(info) - Map> belowSize = new HashMap<>(); - - currentSizes.forEach((coreName, info) -> { - if (((Long)info.get(BYTES_SIZE_KEY) < belowBytes || - (Long)info.get(DOCS_SIZE_KEY) < belowDocs) && - // make sure we don't produce conflicting ops - !splittable.contains(info.getName())) { - if (waitForElapsed(coreName, now, lastBelowEventMap)) { - List infos = belowSize.computeIfAbsent(info.getCollection(), c -> new ArrayList<>()); - if (!infos.contains(info)) { - if ((Long)info.get(BYTES_SIZE_KEY) < belowBytes) { - info.getProperties().put(VIOLATION_KEY, BELOW_BYTES_PROP); - } else { - info.getProperties().put(VIOLATION_KEY, BELOW_DOCS_PROP); - } - infos.add(info); - } - } - } else { - // no violation - clear waitForElapsed - lastBelowEventMap.remove(coreName); - } - }); - - if (aboveSize.isEmpty() && belowSize.isEmpty()) { - log.trace("NO VIOLATIONS: Now={}", now); - log.trace("lastAbove={}", lastAboveEventMap); - log.trace("lastBelow={}", lastBelowEventMap); - return; - } - - // find the earliest time when a condition was exceeded - final AtomicLong eventTime = new AtomicLong(now); - - // calculate ops - final List ops = new ArrayList<>(); - aboveSize.forEach((coll, replicas) -> { - // sort by decreasing size to first split the largest ones - // XXX see the comment below about using DOCS_SIZE_PROP in lieu of BYTES_SIZE_PROP - replicas.sort((r1, r2) -> { - long delta = (Long) r1.get(DOCS_SIZE_KEY) - (Long) r2.get(DOCS_SIZE_KEY); - if (delta > 0) { - return -1; - } else if (delta < 0) { - return 1; - } else { - return 0; - } - }); - replicas.forEach(r -> { - if (ops.size() >= maxOps) { - return; - } - TriggerEvent.Op op = new TriggerEvent.Op(aboveOp); - op.addHint(Suggester.Hint.COLL_SHARD, new Pair<>(coll, r.getShard())); - Map params = new HashMap<>(); - params.put(SPLIT_METHOD_PROP, splitMethod.toLower()); - if (splitFuzz > 0) { - params.put(SPLIT_FUZZ_PROP, splitFuzz); - } - params.put(SPLIT_BY_PREFIX, splitByPrefix); - op.addHint(Suggester.Hint.PARAMS, params); - ops.add(op); - Long time = lastAboveEventMap.get(r.getCoreName()); - if (time != null && eventTime.get() > time) { - eventTime.set(time); - } - }); - }); - belowSize.forEach((coll, replicas) -> { - if (replicas.size() < 2) { - return; - } - if (ops.size() >= maxOps) { - return; - } - // sort by increasing size - replicas.sort((r1, r2) -> { - // XXX this is not quite correct - if BYTES_SIZE_PROP decided that replica got here - // then we should be sorting by BYTES_SIZE_PROP. However, since DOCS and BYTES are - // loosely correlated it's simpler to sort just by docs (which better reflects the "too small" - // condition than index size, due to possibly existing deleted docs that still occupy space) - long delta = (Long) r1.get(DOCS_SIZE_KEY) - (Long) r2.get(DOCS_SIZE_KEY); - if (delta > 0) { - return 1; - } else if (delta < 0) { - return -1; - } else { - return 0; - } - }); - - // TODO: MERGESHARDS is not implemented yet. For now take the top two smallest shards - // TODO: but in the future we probably need to get ones with adjacent ranges. - - // TODO: generate as many MERGESHARDS as needed to consume all belowSize shards - TriggerEvent.Op op = new TriggerEvent.Op(belowOp); - op.addHint(Suggester.Hint.COLL_SHARD, new Pair(coll, replicas.get(0).getShard())); - op.addHint(Suggester.Hint.COLL_SHARD, new Pair(coll, replicas.get(1).getShard())); - ops.add(op); - Long time = lastBelowEventMap.get(replicas.get(0).getCoreName()); - if (time != null && eventTime.get() > time) { - eventTime.set(time); - } - time = lastBelowEventMap.get(replicas.get(1).getCoreName()); - if (time != null && eventTime.get() > time) { - eventTime.set(time); - } - }); - - if (ops.isEmpty()) { - return; - } - if (processor.process(new IndexSizeEvent(getName(), eventTime.get(), ops, aboveSize, belowSize))) { - // update last event times - aboveSize.forEach((coll, replicas) -> { - replicas.forEach(r -> lastAboveEventMap.put(r.getCoreName(), now)); - }); - belowSize.forEach((coll, replicas) -> { - if (replicas.size() < 2) { - return; - } - lastBelowEventMap.put(replicas.get(0).getCoreName(), now); - lastBelowEventMap.put(replicas.get(1).getCoreName(), now); - }); - } - } - - public static long estimatedSize(long maxDoc, long numDocs, long commitSize) { - if (maxDoc == 0) { - return 0; - } - if (maxDoc == numDocs) { - return commitSize; - } - return commitSize * numDocs / maxDoc; - } - - private boolean waitForElapsed(String name, long now, Map lastEventMap) { - Long lastTime = lastEventMap.computeIfAbsent(name, s -> now); - long elapsed = TimeUnit.SECONDS.convert(now - lastTime, TimeUnit.NANOSECONDS); - log.trace("name={}, lastTime={}, elapsed={}", name, lastTime, elapsed); - if (TimeUnit.SECONDS.convert(now - lastTime, TimeUnit.NANOSECONDS) < getWaitForSecond()) { - return false; - } - return true; - } - - public static class IndexSizeEvent extends TriggerEvent { - public IndexSizeEvent(String source, long eventTime, List ops, Map> aboveSize, - Map> belowSize) { - super(TriggerEventType.INDEXSIZE, source, eventTime, null); - properties.put(TriggerEvent.REQUESTED_OPS, ops); - // avoid passing very large amounts of data here - just use replica names - TreeMap above = new TreeMap<>(); - aboveSize.forEach((coll, replicas) -> - replicas.forEach(r -> above.put(r.getCoreName(), "docs=" + r.get(DOCS_SIZE_KEY) + ", bytes=" + r.get(BYTES_SIZE_KEY)))); - properties.put(ABOVE_SIZE_KEY, above); - TreeMap below = new TreeMap<>(); - belowSize.forEach((coll, replicas) -> - replicas.forEach(r -> below.put(r.getCoreName(), "docs=" + r.get(DOCS_SIZE_KEY) + ", bytes=" + r.get(BYTES_SIZE_KEY)))); - properties.put(BELOW_SIZE_KEY, below); - } - } - -} diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/LoggingListener.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/LoggingListener.java deleted file mode 100644 index a7dcf63c81f..00000000000 --- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/LoggingListener.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.cloud.autoscaling; - -import java.lang.invoke.MethodHandles; - -import org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventProcessorStage; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Implementation of {@link TriggerListener} that reports - * events to a log. - */ -public class LoggingListener extends TriggerListenerBase { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - @Override - public void onEvent(TriggerEvent event, TriggerEventProcessorStage stage, String actionName, ActionContext context, - Throwable error, String message) { - log.info("{}: stage={}, actionName={}, event={}, error={}, messsage={}", config.name, stage, actionName, event, error, message); - } -} diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/MetricTrigger.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/MetricTrigger.java deleted file mode 100644 index 573ac77bd6b..00000000000 --- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/MetricTrigger.java +++ /dev/null @@ -1,220 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.cloud.autoscaling; - -import java.lang.invoke.MethodHandles; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicLong; -import java.util.stream.Collectors; - -import org.apache.solr.client.solrj.cloud.autoscaling.Policy; -import org.apache.solr.client.solrj.cloud.SolrCloudManager; -import org.apache.solr.client.solrj.cloud.autoscaling.Suggester; -import org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventType; -import org.apache.solr.common.SolrException; -import org.apache.solr.common.cloud.ClusterState; -import org.apache.solr.common.cloud.DocCollection; -import org.apache.solr.common.cloud.Slice; -import org.apache.solr.common.params.AutoScalingParams; -import org.apache.solr.common.params.CollectionParams; -import org.apache.solr.common.util.Pair; -import org.apache.solr.core.SolrResourceLoader; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import static org.apache.solr.common.params.AutoScalingParams.ABOVE; -import static org.apache.solr.common.params.AutoScalingParams.BELOW; -import static org.apache.solr.common.params.AutoScalingParams.METRIC; -import static org.apache.solr.common.params.AutoScalingParams.PREFERRED_OP; - -public class MetricTrigger extends TriggerBase { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - private String metric; - private Number above, below; - private String collection, shard, node, preferredOp; - - private final Map lastNodeEvent = new ConcurrentHashMap<>(); - - public MetricTrigger(String name) { - super(TriggerEventType.METRIC, name); - TriggerUtils.requiredProperties(requiredProperties, validProperties, METRIC); - TriggerUtils.validProperties(validProperties, ABOVE, BELOW, PREFERRED_OP, - AutoScalingParams.COLLECTION, - AutoScalingParams.SHARD, - AutoScalingParams.NODE); - } - - @Override - public void configure(SolrResourceLoader loader, SolrCloudManager cloudManager, Map properties) throws TriggerValidationException { - super.configure(loader, cloudManager, properties); - this.metric = (String) properties.get(METRIC); - this.above = (Number) properties.get(ABOVE); - this.below = (Number) properties.get(BELOW); - this.collection = (String) properties.getOrDefault(AutoScalingParams.COLLECTION, Policy.ANY); - shard = (String) properties.getOrDefault(AutoScalingParams.SHARD, Policy.ANY); - if (collection.equals(Policy.ANY) && !shard.equals(Policy.ANY)) { - throw new TriggerValidationException("shard", "When 'shard' is other than #ANY then collection name must be also other than #ANY"); - } - node = (String) properties.getOrDefault(AutoScalingParams.NODE, Policy.ANY); - preferredOp = (String) properties.getOrDefault(PREFERRED_OP, CollectionParams.CollectionAction.MOVEREPLICA.toLower()); - } - - @Override - protected Map getState() { - return null; - } - - @Override - protected void setState(Map state) { - lastNodeEvent.clear(); - @SuppressWarnings({"unchecked"}) - Map nodeTimes = (Map) state.get("lastNodeEvent"); - if (nodeTimes != null) { - lastNodeEvent.putAll(nodeTimes); - } - } - - @Override - public void restoreState(AutoScaling.Trigger old) { - assert old.isClosed(); - if (old instanceof MetricTrigger) { - MetricTrigger that = (MetricTrigger) old; - assert this.name.equals(that.name); - this.lastNodeEvent.clear(); - this.lastNodeEvent.putAll(that.lastNodeEvent); - } else { - throw new SolrException(SolrException.ErrorCode.INVALID_STATE, - "Unable to restore state from an unknown type of trigger"); - } - } - - @Override - public void run() { - AutoScaling.TriggerEventProcessor processor = processorRef.get(); - if (processor == null) { - return; - } - - Set liveNodes = null; - if (node.equals(Policy.ANY)) { - if (collection.equals(Policy.ANY)) { - liveNodes = cloudManager.getClusterStateProvider().getLiveNodes(); - } else { - final Set nodes = new HashSet<>(); - ClusterState.CollectionRef ref = cloudManager.getClusterStateProvider().getState(collection); - DocCollection docCollection; - if (ref == null || (docCollection = ref.get()) == null) { - log.warn("MetricTrigger could not find collection: {}", collection); - return; - } - if (shard.equals(Policy.ANY)) { - docCollection.getReplicas().forEach(replica -> { - nodes.add(replica.getNodeName()); - }); - } else { - Slice slice = docCollection.getSlice(shard); - if (slice == null) { - log.warn("MetricTrigger could not find collection: {} shard: {}", collection, shard); - return; - } - slice.getReplicas().forEach(replica -> nodes.add(replica.getNodeName())); - } - liveNodes = nodes; - } - } else { - liveNodes = Collections.singleton(node); - } - - Map rates = new HashMap<>(liveNodes.size()); - for (String node : liveNodes) { - Map values = cloudManager.getNodeStateProvider().getNodeValues(node, Collections.singletonList(metric)); - values.forEach((tag, rate) -> rates.computeIfAbsent(node, s -> (Number) rate)); - } - - long now = cloudManager.getTimeSource().getTimeNs(); - // check for exceeded rates and filter out those with less than waitFor from previous events - Map hotNodes = rates.entrySet().stream() - .filter(entry -> waitForElapsed(entry.getKey(), now, lastNodeEvent)) - .filter(entry -> (below != null && Double.compare(entry.getValue().doubleValue(), below.doubleValue()) < 0) || (above != null && Double.compare(entry.getValue().doubleValue(), above.doubleValue()) > 0)) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - - if (hotNodes.isEmpty()) return; - - final AtomicLong eventTime = new AtomicLong(now); - hotNodes.forEach((n, r) -> { - long time = lastNodeEvent.get(n); - if (eventTime.get() > time) { - eventTime.set(time); - } - }); - - if (processor.process(new MetricBreachedEvent(getName(), collection, shard, preferredOp, eventTime.get(), metric, hotNodes))) { - hotNodes.keySet().forEach(node -> lastNodeEvent.put(node, now)); - } - } - - private boolean waitForElapsed(String name, long now, Map lastEventMap) { - Long lastTime = lastEventMap.computeIfAbsent(name, s -> now); - long elapsed = TimeUnit.SECONDS.convert(now - lastTime, TimeUnit.NANOSECONDS); - log.trace("name={}, lastTime={}, elapsed={}", name, lastTime, elapsed); - if (TimeUnit.SECONDS.convert(now - lastTime, TimeUnit.NANOSECONDS) < getWaitForSecond()) { - return false; - } - return true; - } - - public static class MetricBreachedEvent extends TriggerEvent { - public MetricBreachedEvent(String source, String collection, String shard, String preferredOp, long eventTime, String metric, Map hotNodes) { - super(TriggerEventType.METRIC, source, eventTime, null); - properties.put(METRIC, metric); - properties.put(AutoScalingParams.NODE, hotNodes); - if (!collection.equals(Policy.ANY)) { - properties.put(AutoScalingParams.COLLECTION, collection); - } - if (!shard.equals(Policy.ANY)) { - properties.put(AutoScalingParams.SHARD, shard); - } - properties.put(PREFERRED_OP, preferredOp); - - // specify requested ops - List ops = new ArrayList<>(hotNodes.size()); - for (String n : hotNodes.keySet()) { - Op op = new Op(CollectionParams.CollectionAction.get(preferredOp)); - op.addHint(Suggester.Hint.SRC_NODE, n); - if (!collection.equals(Policy.ANY)) { - if (!shard.equals(Policy.ANY)) { - op.addHint(Suggester.Hint.COLL_SHARD, new Pair<>(collection, shard)); - } else { - op.addHint(Suggester.Hint.COLL, collection); - } - } - ops.add(op); - } - properties.put(TriggerEvent.REQUESTED_OPS, ops); - } - } -} diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeAddedTrigger.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeAddedTrigger.java deleted file mode 100644 index 42188e4587a..00000000000 --- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeAddedTrigger.java +++ /dev/null @@ -1,250 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.cloud.autoscaling; - -import java.io.IOException; -import java.lang.invoke.MethodHandles; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.NoSuchElementException; -import java.util.Set; -import java.util.concurrent.TimeUnit; - -import org.apache.solr.client.solrj.cloud.SolrCloudManager; -import org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventType; -import org.apache.solr.common.SolrException; -import org.apache.solr.common.cloud.Replica; -import org.apache.solr.common.cloud.ZkStateReader; -import org.apache.solr.common.params.CollectionParams; -import org.apache.solr.common.util.Utils; -import org.apache.solr.core.SolrResourceLoader; -import org.apache.zookeeper.KeeperException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import static org.apache.solr.cloud.autoscaling.OverseerTriggerThread.MARKER_ACTIVE; -import static org.apache.solr.cloud.autoscaling.OverseerTriggerThread.MARKER_INACTIVE; -import static org.apache.solr.cloud.autoscaling.OverseerTriggerThread.MARKER_STATE; -import static org.apache.solr.common.params.AutoScalingParams.PREFERRED_OP; -import static org.apache.solr.common.params.AutoScalingParams.REPLICA_TYPE; - -/** - * Trigger for the {@link TriggerEventType#NODEADDED} event - */ -public class NodeAddedTrigger extends TriggerBase { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - private Set lastLiveNodes = new HashSet<>(); - - private Map nodeNameVsTimeAdded = new HashMap<>(); - - private String preferredOp; - private Replica.Type replicaType = Replica.Type.NRT; - - public NodeAddedTrigger(String name) { - super(TriggerEventType.NODEADDED, name); - TriggerUtils.validProperties(validProperties, PREFERRED_OP, REPLICA_TYPE); - } - - @Override - public void init() throws Exception { - super.init(); - lastLiveNodes = new HashSet<>(cloudManager.getClusterStateProvider().getLiveNodes()); - log.debug("NodeAddedTrigger {} - Initial livenodes: {}", name, lastLiveNodes); - log.debug("NodeAddedTrigger {} instantiated with properties: {}", name, properties); - // pick up added nodes for which marker paths were created - try { - List added = stateManager.listData(ZkStateReader.SOLR_AUTOSCALING_NODE_ADDED_PATH); - added.forEach(n -> { - String markerPath = ZkStateReader.SOLR_AUTOSCALING_NODE_ADDED_PATH + "/" + n; - try { - Map markerData = Utils.getJson(stateManager, markerPath); - // skip inactive markers - if (markerData.getOrDefault(MARKER_STATE, MARKER_ACTIVE).equals(MARKER_INACTIVE)) { - return; - } - } catch (InterruptedException | IOException | KeeperException e) { - log.debug("-- ignoring marker {} state due to error{}", markerPath, e); - } - // don't add nodes that have since gone away - if (lastLiveNodes.contains(n) && !nodeNameVsTimeAdded.containsKey(n)) { - // since {@code #restoreState(AutoScaling.Trigger)} is called first, the timeAdded for a node may also be restored - log.debug("Adding node from marker path: {}", n); - nodeNameVsTimeAdded.put(n, cloudManager.getTimeSource().getTimeNs()); - } - }); - } catch (NoSuchElementException e) { - // ignore - } catch (Exception e) { - log.warn("Exception retrieving nodeLost markers", e); - } - } - - @Override - public void configure(SolrResourceLoader loader, SolrCloudManager cloudManager, Map properties) throws TriggerValidationException { - super.configure(loader, cloudManager, properties); - preferredOp = (String) properties.getOrDefault(PREFERRED_OP, CollectionParams.CollectionAction.MOVEREPLICA.toLower()); - preferredOp = preferredOp.toLowerCase(Locale.ROOT); - String replicaTypeStr = (String) properties.getOrDefault(REPLICA_TYPE, Replica.Type.NRT.name()); - // verify - try { - replicaType = Replica.Type.valueOf(replicaTypeStr); - } catch (IllegalArgumentException | NullPointerException e) { - throw new TriggerValidationException("Unsupported replicaType=" + replicaTypeStr + " specified for node added trigger"); - } - - CollectionParams.CollectionAction action = CollectionParams.CollectionAction.get(preferredOp); - switch (action) { - case ADDREPLICA: - case MOVEREPLICA: - case NONE: - break; - default: - throw new TriggerValidationException("Unsupported preferredOperation=" + preferredOp + " specified for node added trigger"); - } - } - - @Override - public void restoreState(AutoScaling.Trigger old) { - assert old.isClosed(); - if (old instanceof NodeAddedTrigger) { - NodeAddedTrigger that = (NodeAddedTrigger) old; - assert this.name.equals(that.name); - this.lastLiveNodes.clear(); - this.lastLiveNodes.addAll(that.lastLiveNodes); - this.nodeNameVsTimeAdded.clear(); - this.nodeNameVsTimeAdded.putAll(that.nodeNameVsTimeAdded); - } else { - throw new SolrException(SolrException.ErrorCode.INVALID_STATE, - "Unable to restore state from an unknown type of trigger"); - } - } - - @Override - protected Map getState() { - Map state = new HashMap<>(); - state.put("lastLiveNodes", lastLiveNodes); - state.put("nodeNameVsTimeAdded", nodeNameVsTimeAdded); - return state; - } - - @Override - protected void setState(Map state) { - this.lastLiveNodes.clear(); - this.nodeNameVsTimeAdded.clear(); - @SuppressWarnings({"unchecked"}) - Collection lastLiveNodes = (Collection)state.get("lastLiveNodes"); - if (lastLiveNodes != null) { - this.lastLiveNodes.addAll(lastLiveNodes); - } - @SuppressWarnings({"unchecked"}) - Map nodeNameVsTimeAdded = (Map)state.get("nodeNameVsTimeAdded"); - if (nodeNameVsTimeAdded != null) { - this.nodeNameVsTimeAdded.putAll(nodeNameVsTimeAdded); - } - } - - @Override - public void run() { - try { - synchronized (this) { - if (isClosed) { - log.warn("NodeAddedTrigger ran but was already closed"); - throw new RuntimeException("Trigger has been closed"); - } - } - log.debug("Running NodeAddedTrigger {}", name); - - Set newLiveNodes = new HashSet<>(cloudManager.getClusterStateProvider().getLiveNodes()); - if (log.isDebugEnabled()) { - log.debug("Found livenodes: {}", newLiveNodes.size()); - } - - // have any nodes that we were tracking been removed from the cluster? - // if so, remove them from the tracking map - Set trackingKeySet = nodeNameVsTimeAdded.keySet(); - trackingKeySet.retainAll(newLiveNodes); - - // have any new nodes been added? - Set copyOfNew = new HashSet<>(newLiveNodes); - copyOfNew.removeAll(lastLiveNodes); - copyOfNew.forEach(n -> { - long eventTime = cloudManager.getTimeSource().getTimeNs(); - log.debug("Tracking new node: {} at time {}", n, eventTime); - nodeNameVsTimeAdded.put(n, eventTime); - }); - - // has enough time expired to trigger events for a node? - List nodeNames = new ArrayList<>(); - List times = new ArrayList<>(); - for (Iterator> it = nodeNameVsTimeAdded.entrySet().iterator(); it.hasNext(); ) { - Map.Entry entry = it.next(); - String nodeName = entry.getKey(); - Long timeAdded = entry.getValue(); - long now = cloudManager.getTimeSource().getTimeNs(); - if (TimeUnit.SECONDS.convert(now - timeAdded, TimeUnit.NANOSECONDS) >= getWaitForSecond()) { - nodeNames.add(nodeName); - times.add(timeAdded); - } - } - AutoScaling.TriggerEventProcessor processor = processorRef.get(); - if (!nodeNames.isEmpty()) { - if (processor != null) { - if (log.isDebugEnabled()) { - log.debug("NodeAddedTrigger {} firing registered processor for nodes: {} added at times {}, now={}", name, - nodeNames, times, cloudManager.getTimeSource().getTimeNs()); - } - if (processor.process(new NodeAddedEvent(getEventType(), getName(), times, nodeNames, preferredOp, replicaType))) { - // remove from tracking set only if the fire was accepted - nodeNames.forEach(n -> { - log.debug("Removing new node from tracking: {}", n); - nodeNameVsTimeAdded.remove(n); - }); - } else { - log.debug("Processor returned false for {}!", nodeNames); - } - } else { - nodeNames.forEach(n -> { - nodeNameVsTimeAdded.remove(n); - }); - } - } - lastLiveNodes = new HashSet<>(newLiveNodes); - } catch (RuntimeException e) { - log.error("Unexpected exception in NodeAddedTrigger", e); - } - } - - public static class NodeAddedEvent extends TriggerEvent { - - public NodeAddedEvent(TriggerEventType eventType, String source, List times, List nodeNames, String preferredOp, Replica.Type replicaType) { - // use the oldest time as the time of the event - super(eventType, source, times.get(0), null); - properties.put(NODE_NAMES, nodeNames); - properties.put(EVENT_TIMES, times); - properties.put(PREFERRED_OP, preferredOp); - properties.put(REPLICA_TYPE, replicaType); - } - } -} diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeLostTrigger.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeLostTrigger.java deleted file mode 100644 index b1c58183a39..00000000000 --- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeLostTrigger.java +++ /dev/null @@ -1,240 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.cloud.autoscaling; - -import java.io.IOException; -import java.lang.invoke.MethodHandles; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.NoSuchElementException; -import java.util.Set; -import java.util.concurrent.TimeUnit; - -import org.apache.solr.client.solrj.cloud.SolrCloudManager; -import org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventType; -import org.apache.solr.common.AlreadyClosedException; -import org.apache.solr.common.SolrException; -import org.apache.solr.common.cloud.ZkStateReader; -import org.apache.solr.common.params.CollectionParams; -import org.apache.solr.common.util.Utils; -import org.apache.solr.core.SolrResourceLoader; -import org.apache.zookeeper.KeeperException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import static org.apache.solr.cloud.autoscaling.OverseerTriggerThread.MARKER_ACTIVE; -import static org.apache.solr.cloud.autoscaling.OverseerTriggerThread.MARKER_INACTIVE; -import static org.apache.solr.cloud.autoscaling.OverseerTriggerThread.MARKER_STATE; -import static org.apache.solr.common.params.AutoScalingParams.PREFERRED_OP; - -/** - * Trigger for the {@link TriggerEventType#NODELOST} event - */ -public class NodeLostTrigger extends TriggerBase { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - private Set lastLiveNodes = new HashSet<>(); - - private Map nodeNameVsTimeRemoved = new HashMap<>(); - - private String preferredOp; - - public NodeLostTrigger(String name) { - super(TriggerEventType.NODELOST, name); - TriggerUtils.validProperties(validProperties, PREFERRED_OP); - } - - @Override - public void init() throws Exception { - super.init(); - lastLiveNodes = new HashSet<>(cloudManager.getClusterStateProvider().getLiveNodes()); - log.debug("NodeLostTrigger {} - Initial livenodes: {}", name, lastLiveNodes); - // pick up lost nodes for which marker paths were created - try { - List lost = stateManager.listData(ZkStateReader.SOLR_AUTOSCALING_NODE_LOST_PATH); - lost.forEach(n -> { - String markerPath = ZkStateReader.SOLR_AUTOSCALING_NODE_LOST_PATH + "/" + n; - try { - Map markerData = Utils.getJson(stateManager, markerPath); - // skip inactive markers - if (markerData.getOrDefault(MARKER_STATE, MARKER_ACTIVE).equals(MARKER_INACTIVE)) { - return; - } - } catch (InterruptedException | IOException | KeeperException e) { - log.debug("-- ignoring marker {} state due to error", markerPath, e); - } - // don't add nodes that have since came back - if (!lastLiveNodes.contains(n) && !nodeNameVsTimeRemoved.containsKey(n)) { - // since {@code #restoreState(AutoScaling.Trigger)} is called first, the timeRemoved for a node may also be restored - log.debug("Adding lost node from marker path: {}", n); - nodeNameVsTimeRemoved.put(n, cloudManager.getTimeSource().getTimeNs()); - } - }); - } catch (NoSuchElementException e) { - // ignore - } catch (Exception e) { - log.warn("Exception retrieving nodeLost markers", e); - } - } - - @Override - public void configure(SolrResourceLoader loader, SolrCloudManager cloudManager, Map properties) throws TriggerValidationException { - super.configure(loader, cloudManager, properties); - preferredOp = (String) properties.getOrDefault(PREFERRED_OP, CollectionParams.CollectionAction.MOVEREPLICA.toLower()); - preferredOp = preferredOp.toLowerCase(Locale.ROOT); - // verify - CollectionParams.CollectionAction action = CollectionParams.CollectionAction.get(preferredOp); - switch (action) { - case MOVEREPLICA: - case DELETENODE: - case NONE: - break; - default: - throw new TriggerValidationException("Unsupported preferredOperation=" + preferredOp + " specified for node lost trigger"); - } - } - - @Override - public void restoreState(AutoScaling.Trigger old) { - assert old.isClosed(); - if (old instanceof NodeLostTrigger) { - NodeLostTrigger that = (NodeLostTrigger) old; - assert this.name.equals(that.name); - this.lastLiveNodes.clear(); - this.lastLiveNodes.addAll(that.lastLiveNodes); - this.nodeNameVsTimeRemoved.clear(); - this.nodeNameVsTimeRemoved.putAll(that.nodeNameVsTimeRemoved); - } else { - throw new SolrException(SolrException.ErrorCode.INVALID_STATE, - "Unable to restore state from an unknown type of trigger"); - } - } - - @Override - protected Map getState() { - Map state = new HashMap<>(); - state.put("lastLiveNodes", lastLiveNodes); - state.put("nodeNameVsTimeRemoved", nodeNameVsTimeRemoved); - return state; - } - - @Override - protected void setState(Map state) { - this.lastLiveNodes.clear(); - this.nodeNameVsTimeRemoved.clear(); - @SuppressWarnings({"unchecked"}) - Collection lastLiveNodes = (Collection)state.get("lastLiveNodes"); - if (lastLiveNodes != null) { - this.lastLiveNodes.addAll(lastLiveNodes); - } - @SuppressWarnings({"unchecked"}) - Map nodeNameVsTimeRemoved = (Map)state.get("nodeNameVsTimeRemoved"); - if (nodeNameVsTimeRemoved != null) { - this.nodeNameVsTimeRemoved.putAll(nodeNameVsTimeRemoved); - } - } - - @Override - public void run() { - try { - synchronized (this) { - if (isClosed) { - log.warn("NodeLostTrigger ran but was already closed"); - return; - } - } - - Set newLiveNodes = new HashSet<>(cloudManager.getClusterStateProvider().getLiveNodes()); - if (log.isDebugEnabled()) { - log.debug("Running NodeLostTrigger: {} with currently live nodes: {} and last live nodes: {}", name, newLiveNodes.size(), lastLiveNodes.size()); - } - log.trace("Current Live Nodes for {}: {}", name, newLiveNodes); - log.trace("Last Live Nodes for {}: {}", name, lastLiveNodes); - - // have any nodes that we were tracking been added to the cluster? - // if so, remove them from the tracking map - Set trackingKeySet = nodeNameVsTimeRemoved.keySet(); - trackingKeySet.removeAll(newLiveNodes); - - // have any nodes been removed? - Set copyOfLastLiveNodes = new HashSet<>(lastLiveNodes); - copyOfLastLiveNodes.removeAll(newLiveNodes); - copyOfLastLiveNodes.forEach(n -> { - log.debug("Tracking lost node: {}", n); - nodeNameVsTimeRemoved.put(n, cloudManager.getTimeSource().getTimeNs()); - }); - - // has enough time expired to trigger events for a node? - List nodeNames = new ArrayList<>(); - List times = new ArrayList<>(); - for (Iterator> it = nodeNameVsTimeRemoved.entrySet().iterator(); it.hasNext(); ) { - Map.Entry entry = it.next(); - String nodeName = entry.getKey(); - Long timeRemoved = entry.getValue(); - long now = cloudManager.getTimeSource().getTimeNs(); - long te = TimeUnit.SECONDS.convert(now - timeRemoved, TimeUnit.NANOSECONDS); - if (te >= getWaitForSecond()) { - nodeNames.add(nodeName); - times.add(timeRemoved); - } - } - // fire! - AutoScaling.TriggerEventProcessor processor = processorRef.get(); - if (!nodeNames.isEmpty()) { - if (processor != null) { - log.debug("NodeLostTrigger firing registered processor for lost nodes: {}", nodeNames); - if (processor.process(new NodeLostEvent(getEventType(), getName(), times, nodeNames, preferredOp))) { - // remove from tracking set only if the fire was accepted - nodeNames.forEach(n -> { - nodeNameVsTimeRemoved.remove(n); - }); - } else { - log.debug("NodeLostTrigger processor for lost nodes: {} is not ready, will try later", nodeNames); - } - } else { - log.debug("NodeLostTrigger firing, but no processor - so removing lost nodes: {}", nodeNames); - nodeNames.forEach(n -> { - nodeNameVsTimeRemoved.remove(n); - }); - } - } - lastLiveNodes = new HashSet<>(newLiveNodes); - } catch (AlreadyClosedException e) { - - } catch (RuntimeException e) { - log.error("Unexpected exception in NodeLostTrigger", e); - } - } - - public static class NodeLostEvent extends TriggerEvent { - - public NodeLostEvent(TriggerEventType eventType, String source, List times, List nodeNames, String preferredOp) { - // use the oldest time as the time of the event - super(eventType, source, times.get(0), null); - properties.put(NODE_NAMES, nodeNames); - properties.put(EVENT_TIMES, times); - properties.put(PREFERRED_OP, preferredOp); - } - } -} diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/OverseerTriggerThread.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/OverseerTriggerThread.java deleted file mode 100644 index fa27942bd9c..00000000000 --- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/OverseerTriggerThread.java +++ /dev/null @@ -1,407 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.cloud.autoscaling; - -import java.io.IOException; -import java.lang.invoke.MethodHandles; -import java.net.ConnectException; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.NoSuchElementException; -import java.util.Set; -import java.util.concurrent.locks.Condition; -import java.util.concurrent.locks.ReentrantLock; - -import org.apache.solr.client.solrj.cloud.DistribStateManager; -import org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig; -import org.apache.solr.client.solrj.cloud.autoscaling.BadVersionException; -import org.apache.solr.client.solrj.cloud.SolrCloudManager; -import org.apache.solr.client.solrj.cloud.autoscaling.Policy; -import org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventType; -import org.apache.solr.common.AlreadyClosedException; -import org.apache.solr.common.SolrCloseable; -import org.apache.solr.common.cloud.ZkStateReader; -import org.apache.solr.common.util.IOUtils; -import org.apache.solr.common.util.Utils; -import org.apache.solr.core.SolrResourceLoader; -import org.apache.zookeeper.KeeperException; -import org.apache.zookeeper.WatchedEvent; -import org.apache.zookeeper.Watcher; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import static org.apache.solr.common.cloud.ZkStateReader.SOLR_AUTOSCALING_CONF_PATH; - -/** - * Overseer thread responsible for reading triggers from zookeeper and - * adding/removing them from {@link ScheduledTriggers} - */ -public class OverseerTriggerThread implements Runnable, SolrCloseable { - - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - public static final String MARKER_STATE = "state"; - public static final String MARKER_ACTIVE = "active"; - public static final String MARKER_INACTIVE = "inactive"; - public static final int DEFAULT_AUTO_ADD_REPLICA_WAIT_FOR_SECONDS = 120; - - - private final SolrCloudManager cloudManager; - - private final ScheduledTriggers scheduledTriggers; - - private final AutoScaling.TriggerFactory triggerFactory; - - private final ReentrantLock updateLock = new ReentrantLock(); - - private final Condition updated = updateLock.newCondition(); - - /* - Following variables are only accessed or modified when updateLock is held - */ - private int znodeVersion = 0; - - private Map activeTriggers = new HashMap<>(); - - private volatile int processedZnodeVersion = -1; - - private volatile boolean isClosed = false; - - private AutoScalingConfig autoScalingConfig; - - public OverseerTriggerThread(SolrResourceLoader loader, SolrCloudManager cloudManager) { - this.cloudManager = cloudManager; - scheduledTriggers = new ScheduledTriggers(loader, cloudManager); - triggerFactory = new AutoScaling.TriggerFactoryImpl(loader, cloudManager); - } - - @Override - public void close() throws IOException { - updateLock.lock(); - try { - isClosed = true; - activeTriggers.clear(); - updated.signalAll(); - } finally { - updateLock.unlock(); - } - IOUtils.closeQuietly(triggerFactory); - IOUtils.closeQuietly(scheduledTriggers); - log.debug("OverseerTriggerThread has been closed explicitly"); - } - - /** - * For tests. - * @lucene.internal - * @return current {@link ScheduledTriggers} instance - */ - public ScheduledTriggers getScheduledTriggers() { - return scheduledTriggers; - } - - /** - * For tests, to ensure that all processing has been completed in response to an update of /autoscaling.json. - * @lucene.internal - * @return version of /autoscaling.json for which all configuration updates & processing have been completed. - * Until then this value will always be smaller than the current znodeVersion of /autoscaling.json. - */ - public int getProcessedZnodeVersion() { - return processedZnodeVersion; - } - - @Override - public boolean isClosed() { - return isClosed; - } - - @Override - public void run() { - int lastZnodeVersion = znodeVersion; - - // we automatically add a trigger for auto add replicas if it does not exists already - // we also automatically add a scheduled maintenance trigger - while (!isClosed) { - try { - if (Thread.currentThread().isInterrupted()) { - log.warn("Interrupted"); - break; - } - AutoScalingConfig autoScalingConfig = cloudManager.getDistribStateManager().getAutoScalingConfig(); - AutoScalingConfig updatedConfig = withDefaultPolicy(autoScalingConfig); - updatedConfig = withAutoAddReplicasTrigger(updatedConfig); - updatedConfig = withScheduledMaintenanceTrigger(updatedConfig); - if (updatedConfig.equals(autoScalingConfig)) break; - log.debug("Adding .auto_add_replicas and .scheduled_maintenance triggers"); - cloudManager.getDistribStateManager().setData(SOLR_AUTOSCALING_CONF_PATH, Utils.toJSON(updatedConfig), updatedConfig.getZkVersion()); - break; - } catch (AlreadyClosedException e) { - break; - } catch (BadVersionException bve) { - // somebody else has changed the configuration so we must retry - } catch (InterruptedException e) { - // Restore the interrupted status - Thread.currentThread().interrupt(); - log.warn("Interrupted", e); - break; - } - catch (IOException | KeeperException e) { - if (e instanceof KeeperException.SessionExpiredException || - (e.getCause()!=null && e.getCause() instanceof KeeperException.SessionExpiredException)) { - log.warn("Solr cannot talk to ZK, exiting {} main queue loop" - , getClass().getSimpleName(), e); - return; - } else { - log.error("A ZK error has occurred", e); - } - } - } - - if (isClosed || Thread.currentThread().isInterrupted()) return; - - try { - refreshAutoScalingConf(new AutoScalingWatcher()); - } catch (ConnectException e) { - log.warn("ZooKeeper watch triggered for autoscaling conf, but Solr cannot talk to ZK: ", e); - } catch (InterruptedException e) { - // Restore the interrupted status - Thread.currentThread().interrupt(); - log.warn("Interrupted", e); - } catch (Exception e) { - log.error("Unexpected exception", e); - } - - while (true) { - Map copy = null; - try { - - updateLock.lockInterruptibly(); - try { - // must check for close here before we await on the condition otherwise we can - // only be woken up on interruption - if (isClosed) { - log.info("OverseerTriggerThread has been closed, exiting."); - break; - } - - log.debug("Current znodeVersion {}, lastZnodeVersion {}", znodeVersion, lastZnodeVersion); - - if (znodeVersion == lastZnodeVersion) { - updated.await(); - - // are we closed? - if (isClosed) { - log.info("OverseerTriggerThread woken up but we are closed, exiting."); - break; - } - - // spurious wakeup? - if (znodeVersion == lastZnodeVersion) continue; - } - copy = new HashMap<>(activeTriggers); - lastZnodeVersion = znodeVersion; - log.debug("Processed trigger updates upto znodeVersion {}", znodeVersion); - } finally { - updateLock.unlock(); - } - } catch (InterruptedException e) { - // Restore the interrupted status - Thread.currentThread().interrupt(); - log.warn("Interrupted", e); - break; - } - - // update the current config - scheduledTriggers.setAutoScalingConfig(autoScalingConfig); - - Set managedTriggerNames = scheduledTriggers.getScheduledTriggerNames(); - // remove the triggers which are no longer active - for (String managedTriggerName : managedTriggerNames) { - if (!copy.containsKey(managedTriggerName)) { - scheduledTriggers.remove(managedTriggerName); - } - } - // nodeLost / nodeAdded markers are checked by triggers during their init() call - // which is invoked in scheduledTriggers.add(), so once this is done we can remove them - try { - // add new triggers and/or replace and close the replaced triggers - for (Map.Entry entry : copy.entrySet()) { - try { - scheduledTriggers.add(entry.getValue()); - } catch (AlreadyClosedException e) { - - } catch (Exception e) { - log.warn("Exception initializing trigger {}, configuration ignored", entry.getKey(), e); - } - } - } catch (AlreadyClosedException e) { - // this _should_ mean that we're closing, complain loudly if that's not the case - if (isClosed) { - return; - } else { - throw new IllegalStateException("Caught AlreadyClosedException from ScheduledTriggers, but we're not closed yet!", e); - } - } - log.debug("-- deactivating old nodeLost / nodeAdded markers"); - deactivateMarkers(ZkStateReader.SOLR_AUTOSCALING_NODE_LOST_PATH); - deactivateMarkers(ZkStateReader.SOLR_AUTOSCALING_NODE_ADDED_PATH); - processedZnodeVersion = znodeVersion; - } - } - - private void deactivateMarkers(String path) { - DistribStateManager stateManager = cloudManager.getDistribStateManager(); - try { - List markers = stateManager.listData(path); - for (String marker : markers) { - String markerPath = path + "/" + marker; - try { - Map markerMap = new HashMap<>(Utils.getJson(stateManager, markerPath)); - markerMap.put(MARKER_STATE, MARKER_INACTIVE); - stateManager.setData(markerPath, Utils.toJSON(markerMap), -1); - } catch (NoSuchElementException e) { - // ignore - already deleted - } - } - } catch (NoSuchElementException e) { - // ignore - } catch (Exception e) { - log.warn("Error deactivating old markers", e); - } - } - - class AutoScalingWatcher implements Watcher { - @Override - public void process(WatchedEvent watchedEvent) { - // session events are not change events, and do not remove the watcher - if (Event.EventType.None.equals(watchedEvent.getType())) { - return; - } - - try { - refreshAutoScalingConf(this); - } catch (ConnectException e) { - log.warn("ZooKeeper watch triggered for autoscaling conf, but we cannot talk to ZK: ", e); - } catch (InterruptedException e) { - // Restore the interrupted status - Thread.currentThread().interrupt(); - log.warn("Interrupted", e); - } catch (Exception e) { - log.error("Unexpected exception", e); - } - } - - } - - private void refreshAutoScalingConf(Watcher watcher) throws InterruptedException, IOException { - updateLock.lock(); - try { - if (isClosed) { - return; - } - AutoScalingConfig currentConfig = cloudManager.getDistribStateManager().getAutoScalingConfig(watcher); - if (log.isDebugEnabled()) { - log.debug("Refreshing {} with znode version {}", ZkStateReader.SOLR_AUTOSCALING_CONF_PATH, currentConfig.getZkVersion()); - } - if (znodeVersion >= currentConfig.getZkVersion()) { - // protect against reordered watcher fires by ensuring that we only move forward - return; - } - autoScalingConfig = currentConfig; - znodeVersion = autoScalingConfig.getZkVersion(); - Map triggerMap = loadTriggers(triggerFactory, autoScalingConfig); - - // remove all active triggers that have been removed from ZK - Set trackingKeySet = activeTriggers.keySet(); - trackingKeySet.retainAll(triggerMap.keySet()); - - // now lets add or remove triggers which have been enabled or disabled respectively - for (Map.Entry entry : triggerMap.entrySet()) { - String triggerName = entry.getKey(); - AutoScaling.Trigger trigger = entry.getValue(); - if (trigger.isEnabled()) { - activeTriggers.put(triggerName, trigger); - } else { - activeTriggers.remove(triggerName); - } - } - updated.signalAll(); - } finally { - updateLock.unlock(); - } - } - - private AutoScalingConfig withDefaultPolicy(AutoScalingConfig autoScalingConfig) { - Policy policy = autoScalingConfig.getPolicy(); - if (policy.hasEmptyClusterPolicy()) { - policy = policy.withClusterPolicy(Policy.DEFAULT_CLUSTER_POLICY); - autoScalingConfig = autoScalingConfig.withPolicy(policy); - } - return autoScalingConfig; - } - - private AutoScalingConfig withAutoAddReplicasTrigger(AutoScalingConfig autoScalingConfig) { - Map triggerProps = AutoScaling.AUTO_ADD_REPLICAS_TRIGGER_PROPS; - return withDefaultTrigger(triggerProps, autoScalingConfig); - } - - private AutoScalingConfig withScheduledMaintenanceTrigger(AutoScalingConfig autoScalingConfig) { - Map triggerProps = AutoScaling.SCHEDULED_MAINTENANCE_TRIGGER_PROPS; - return withDefaultTrigger(triggerProps, autoScalingConfig); - } - - private AutoScalingConfig withDefaultTrigger(Map triggerProps, AutoScalingConfig autoScalingConfig) { - String triggerName = (String) triggerProps.get("name"); - Map configs = autoScalingConfig.getTriggerConfigs(); - for (AutoScalingConfig.TriggerConfig cfg : configs.values()) { - if (triggerName.equals(cfg.name)) { - // already has this trigger - return autoScalingConfig; - } - } - // need to add - triggerProps.computeIfPresent("waitFor", (k, v) -> (long) (DEFAULT_AUTO_ADD_REPLICA_WAIT_FOR_SECONDS)); - AutoScalingConfig.TriggerConfig config = new AutoScalingConfig.TriggerConfig(triggerName, triggerProps); - autoScalingConfig = autoScalingConfig.withTriggerConfig(config); - // need to add SystemLogListener explicitly here - autoScalingConfig = AutoScalingHandler.withSystemLogListener(autoScalingConfig, triggerName); - return autoScalingConfig; - } - - private static Map loadTriggers(AutoScaling.TriggerFactory triggerFactory, AutoScalingConfig autoScalingConfig) { - Map triggers = autoScalingConfig.getTriggerConfigs(); - if (triggers == null) { - return Collections.emptyMap(); - } - - Map triggerMap = new HashMap<>(triggers.size()); - - for (Map.Entry entry : triggers.entrySet()) { - AutoScalingConfig.TriggerConfig cfg = entry.getValue(); - TriggerEventType eventType = cfg.event; - String triggerName = entry.getKey(); - try { - triggerMap.put(triggerName, triggerFactory.create(eventType, triggerName, cfg.properties)); - } catch (TriggerValidationException e) { - log.warn("Error in trigger '{}' configuration, trigger config ignored: {}", triggerName, cfg, e); - } - } - return triggerMap; - } -} diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/ScheduledTrigger.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/ScheduledTrigger.java deleted file mode 100644 index 98a367c08e0..00000000000 --- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/ScheduledTrigger.java +++ /dev/null @@ -1,227 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.cloud.autoscaling; - -import java.lang.invoke.MethodHandles; -import java.text.ParseException; -import java.time.Instant; -import java.time.format.DateTimeFormatter; -import java.time.format.DateTimeFormatterBuilder; -import java.time.temporal.ChronoField; -import java.util.Collections; -import java.util.Date; -import java.util.Locale; -import java.util.Map; -import java.util.TimeZone; -import java.util.concurrent.TimeUnit; - -import org.apache.solr.client.solrj.cloud.SolrCloudManager; -import org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventType; -import org.apache.solr.common.SolrException; -import org.apache.solr.common.params.AutoScalingParams; -import org.apache.solr.common.params.CollectionParams; -import org.apache.solr.common.util.TimeSource; -import org.apache.solr.core.SolrResourceLoader; -import org.apache.solr.util.DateMathParser; -import org.apache.solr.util.TimeZoneUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import static org.apache.solr.common.params.AutoScalingParams.PREFERRED_OP; - -/** - * A trigger which creates {@link TriggerEventType#SCHEDULED} events as per the configured schedule - */ -public class ScheduledTrigger extends TriggerBase { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - private static final String DEFAULT_GRACE_DURATION = "+15MINUTES"; - private static final String LAST_RUN_AT = "lastRunAt"; - static final String ACTUAL_EVENT_TIME = "actualEventTime"; - - private String everyStr; - - private String graceDurationStr; - - private String preferredOp; - - private TimeZone timeZone; - - private Instant lastRunAt; - - public ScheduledTrigger(String name) { - super(TriggerEventType.SCHEDULED, name); - TriggerUtils.requiredProperties(requiredProperties, validProperties, "startTime", "every"); - TriggerUtils.validProperties(validProperties, "timeZone", "graceDuration", AutoScalingParams.PREFERRED_OP); - } - - @Override - public void configure(SolrResourceLoader loader, SolrCloudManager cloudManager, Map properties) throws TriggerValidationException { - super.configure(loader, cloudManager, properties); - String timeZoneStr = (String) properties.get("timeZone"); - this.timeZone = TimeZoneUtils.parseTimezone(timeZoneStr); // defaults to UTC - - String startTimeStr = (String) properties.get("startTime"); - this.everyStr = (String) properties.get("every"); - this.graceDurationStr = (String) properties.getOrDefault("graceDuration", DEFAULT_GRACE_DURATION); - - preferredOp = (String) properties.get(PREFERRED_OP); - if (preferredOp != null && - CollectionParams.CollectionAction.get(preferredOp) == null) { - throw new TriggerValidationException(getName(), PREFERRED_OP, "unrecognized value of: '" + preferredOp + "'"); - } - - // attempt parsing to validate date math strings - // explicitly set NOW because it may be different for simulated time - Date now = new Date(TimeUnit.NANOSECONDS.toMillis(cloudManager.getTimeSource().getEpochTimeNs())); - Instant startTime = parseStartTime(now, startTimeStr, timeZoneStr); - DateMathParser.parseMath(now, startTime + everyStr, timeZone); - DateMathParser.parseMath(now, startTime + graceDurationStr, timeZone); - - // We set lastRunAt to be the startTime (which could be a date math expression such as 'NOW') - // Ordinarily, NOW will always be evaluated in this constructor so it may seem that - // the trigger will always fire the first time. - // However, the lastRunAt is overwritten with the value from ZK - // during restoreState() operation (which is performed before run()) so the trigger works correctly - this.lastRunAt = startTime; - } - - private Instant parseStartTime(Date now, String startTimeStr, String timeZoneStr) throws TriggerValidationException { - try { - // try parsing startTime as an ISO-8601 date time string - return DateMathParser.parseMath(now, startTimeStr).toInstant(); - } catch (SolrException e) { - if (e.code() != SolrException.ErrorCode.BAD_REQUEST.code) { - throw new TriggerValidationException("startTime", "error parsing value '" + startTimeStr + "': " + e.toString()); - } - } - if (timeZoneStr == null) { - throw new TriggerValidationException("timeZone", - "Either 'startTime' should be an ISO-8601 date time string or 'timeZone' must be not be null"); - } - TimeZone timeZone = TimeZone.getTimeZone(timeZoneStr); - DateTimeFormatter dateTimeFormatter = new DateTimeFormatterBuilder() - .append(DateTimeFormatter.ISO_LOCAL_DATE).appendPattern("['T'[HH[:mm[:ss]]]]") - .parseDefaulting(ChronoField.HOUR_OF_DAY, 0) - .parseDefaulting(ChronoField.MINUTE_OF_HOUR, 0) - .parseDefaulting(ChronoField.SECOND_OF_MINUTE, 0) - .toFormatter(Locale.ROOT).withZone(timeZone.toZoneId()); - try { - return Instant.from(dateTimeFormatter.parse(startTimeStr)); - } catch (Exception e) { - throw new TriggerValidationException("startTime", "error parsing startTime '" + startTimeStr + "': " + e.toString()); - } - } - - @Override - protected Map getState() { - return Collections.singletonMap(LAST_RUN_AT, lastRunAt.toEpochMilli()); - } - - @Override - protected void setState(Map state) { - if (state.containsKey(LAST_RUN_AT)) { - this.lastRunAt = Instant.ofEpochMilli((Long) state.get(LAST_RUN_AT)); - } - } - - @Override - public void restoreState(AutoScaling.Trigger old) { - assert old.isClosed(); - if (old instanceof ScheduledTrigger) { - ScheduledTrigger scheduledTrigger = (ScheduledTrigger) old; - this.lastRunAt = scheduledTrigger.lastRunAt; - } else { - throw new SolrException(SolrException.ErrorCode.INVALID_STATE, - "Unable to restore state from an unknown type of trigger"); - } - } - - @Override - public void run() { - synchronized (this) { - if (isClosed) { - log.debug("ScheduledTrigger ran but was already closed"); - return; - } - } - - TimeSource timeSource = cloudManager.getTimeSource(); - DateMathParser dateMathParser = new DateMathParser(timeZone); - dateMathParser.setNow(new Date(lastRunAt.toEpochMilli())); - Instant nextRunTime, nextPlusGrace; - try { - Date next = dateMathParser.parseMath(everyStr); - dateMathParser.setNow(next); - nextPlusGrace = dateMathParser.parseMath(graceDurationStr).toInstant(); - nextRunTime = next.toInstant(); - } catch (ParseException e) { - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, - "Unable to calculate next run time. lastRan: " + lastRunAt.toString() + " and date math string: " + everyStr, e); - } - - Instant now = Instant.ofEpochMilli( - TimeUnit.NANOSECONDS.toMillis(timeSource.getEpochTimeNs())); - AutoScaling.TriggerEventProcessor processor = processorRef.get(); - - if (now.isBefore(nextRunTime)) { - return; // it's not time yet - } - if (now.isAfter(nextPlusGrace)) { - // we are past time and we could not run per schedule so skip this event - if (log.isWarnEnabled()) { - log.warn("ScheduledTrigger was not able to run event at scheduled time: {}. Now: {}", - nextRunTime, now); - } - // Even though we are skipping the event, we need to notify any listeners of the IGNORED stage - // so we create a dummy event with the ignored=true flag and ScheduledTriggers will do the rest - if (processor != null && processor.process(new ScheduledEvent(getEventType(), getName(), timeSource.getTimeNs(), - preferredOp, now.toEpochMilli(), true))) { - lastRunAt = nextRunTime; - return; - } - } - - if (processor != null) { - if (log.isDebugEnabled()) { - log.debug("ScheduledTrigger {} firing registered processor for scheduled time {}, now={}", name, - nextRunTime, now); - } - if (processor.process(new ScheduledEvent(getEventType(), getName(), timeSource.getTimeNs(), - preferredOp, now.toEpochMilli()))) { - lastRunAt = nextRunTime; // set to nextRunTime instead of now to avoid drift - } - } else { - lastRunAt = nextRunTime; // set to nextRunTime instead of now to avoid drift - } - } - - public static class ScheduledEvent extends TriggerEvent { - public ScheduledEvent(TriggerEventType eventType, String source, long eventTime, String preferredOp, long actualEventTime) { - this(eventType, source, eventTime, preferredOp, actualEventTime, false); - } - - public ScheduledEvent(TriggerEventType eventType, String source, long eventTime, String preferredOp, long actualEventTime, boolean ignored) { - super(eventType, source, eventTime, null, ignored); - if (preferredOp != null) { - properties.put(PREFERRED_OP, preferredOp); - } - properties.put(ACTUAL_EVENT_TIME, actualEventTime); - } - } -} diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/ScheduledTriggers.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/ScheduledTriggers.java deleted file mode 100644 index e080eecc8ab..00000000000 --- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/ScheduledTriggers.java +++ /dev/null @@ -1,913 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.cloud.autoscaling; - -import java.io.Closeable; -import java.io.IOException; -import java.lang.invoke.MethodHandles; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Random; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledFuture; -import java.util.concurrent.ScheduledThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.locks.ReentrantLock; -import java.util.stream.Collectors; - -import org.apache.commons.lang3.exception.ExceptionUtils; -import org.apache.solr.client.solrj.cloud.DistribStateManager; -import org.apache.solr.client.solrj.cloud.SolrCloudManager; -import org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig; -import org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventProcessorStage; -import org.apache.solr.client.solrj.cloud.autoscaling.VersionedData; -import org.apache.solr.client.solrj.request.CollectionAdminRequest.RequestStatusResponse; -import org.apache.solr.client.solrj.response.RequestStatusState; -import org.apache.solr.cloud.Stats; -import org.apache.solr.common.AlreadyClosedException; -import org.apache.solr.common.SolrException; -import org.apache.solr.common.cloud.ZkStateReader; -import org.apache.solr.common.util.ExecutorUtil; -import org.apache.solr.common.util.IOUtils; -import org.apache.solr.common.util.Utils; -import org.apache.solr.core.SolrResourceLoader; -import org.apache.solr.common.util.SolrNamedThreadFactory; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import static org.apache.solr.cloud.autoscaling.ExecutePlanAction.waitForTaskToFinish; -import static org.apache.solr.common.params.AutoScalingParams.ACTION_THROTTLE_PERIOD_SECONDS; -import static org.apache.solr.common.params.AutoScalingParams.TRIGGER_COOLDOWN_PERIOD_SECONDS; -import static org.apache.solr.common.params.AutoScalingParams.TRIGGER_CORE_POOL_SIZE; -import static org.apache.solr.common.params.AutoScalingParams.TRIGGER_SCHEDULE_DELAY_SECONDS; -import static org.apache.solr.common.util.ExecutorUtil.awaitTermination; - -/** - * Responsible for scheduling active triggers, starting and stopping them and - * performing actions when they fire - */ -public class ScheduledTriggers implements Closeable { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - public static final int DEFAULT_SCHEDULED_TRIGGER_DELAY_SECONDS = 1; - public static final int DEFAULT_ACTION_THROTTLE_PERIOD_SECONDS = 5; - public static final int DEFAULT_COOLDOWN_PERIOD_SECONDS = 5; - public static final int DEFAULT_TRIGGER_CORE_POOL_SIZE = 4; - - static final Map DEFAULT_PROPERTIES = new HashMap<>(); - - static { - DEFAULT_PROPERTIES.put(TRIGGER_SCHEDULE_DELAY_SECONDS, DEFAULT_SCHEDULED_TRIGGER_DELAY_SECONDS); - DEFAULT_PROPERTIES.put(TRIGGER_COOLDOWN_PERIOD_SECONDS, DEFAULT_COOLDOWN_PERIOD_SECONDS); - DEFAULT_PROPERTIES.put(TRIGGER_CORE_POOL_SIZE, DEFAULT_TRIGGER_CORE_POOL_SIZE); - DEFAULT_PROPERTIES.put(ACTION_THROTTLE_PERIOD_SECONDS, DEFAULT_ACTION_THROTTLE_PERIOD_SECONDS); - } - - protected static final Random RANDOM; - static { - // We try to make things reproducible in the context of our tests by initializing the random instance - // based on the current seed - String seed = System.getProperty("tests.seed"); - if (seed == null) { - RANDOM = new Random(); - } else { - RANDOM = new Random(seed.hashCode()); - } - } - - private final Map scheduledTriggerWrappers = new ConcurrentHashMap<>(); - - /** - * Thread pool for scheduling the triggers - */ - private final ScheduledThreadPoolExecutor scheduledThreadPoolExecutor; - - /** - * Single threaded executor to run the actions upon a trigger event. We rely on this being a single - * threaded executor to ensure that trigger fires do not step on each other as well as to ensure - * that we do not run scheduled trigger threads while an action has been submitted to this executor - */ - private final ExecutorService actionExecutor; - - private boolean isClosed = false; - - private final AtomicBoolean hasPendingActions = new AtomicBoolean(false); - - private final AtomicLong cooldownStart = new AtomicLong(); - - private final AtomicLong cooldownPeriod = new AtomicLong(TimeUnit.SECONDS.toNanos(DEFAULT_COOLDOWN_PERIOD_SECONDS)); - - private final AtomicLong triggerDelay = new AtomicLong(DEFAULT_SCHEDULED_TRIGGER_DELAY_SECONDS); - - private final SolrCloudManager cloudManager; - - private final DistribStateManager stateManager; - - private final SolrResourceLoader loader; - - private final Stats queueStats; - - private final TriggerListeners listeners; - - private final List additionalListeners = new ArrayList<>(); - - private AutoScalingConfig autoScalingConfig; - - public ScheduledTriggers(SolrResourceLoader loader, SolrCloudManager cloudManager) { - scheduledThreadPoolExecutor = (ScheduledThreadPoolExecutor) Executors.newScheduledThreadPool(DEFAULT_TRIGGER_CORE_POOL_SIZE, - new SolrNamedThreadFactory("ScheduledTrigger")); - scheduledThreadPoolExecutor.setRemoveOnCancelPolicy(true); - scheduledThreadPoolExecutor.setExecuteExistingDelayedTasksAfterShutdownPolicy(false); - actionExecutor = ExecutorUtil.newMDCAwareSingleThreadExecutor(new SolrNamedThreadFactory("AutoscalingActionExecutor")); - this.cloudManager = cloudManager; - this.stateManager = cloudManager.getDistribStateManager(); - this.loader = loader; - queueStats = new Stats(); - listeners = new TriggerListeners(); - // initialize cooldown timer - cooldownStart.set(cloudManager.getTimeSource().getTimeNs() - cooldownPeriod.get()); - } - - /** - * Set the current autoscaling config. This is invoked by {@link OverseerTriggerThread} when autoscaling.json is updated, - * and it re-initializes trigger listeners and other properties used by the framework - * @param autoScalingConfig current autoscaling.json - */ - public void setAutoScalingConfig(AutoScalingConfig autoScalingConfig) { - Map currentProps = new HashMap<>(DEFAULT_PROPERTIES); - if (this.autoScalingConfig != null) { - currentProps.putAll(this.autoScalingConfig.getProperties()); - } - - // reset listeners early in order to capture first execution of newly scheduled triggers - listeners.setAutoScalingConfig(autoScalingConfig); - - for (Map.Entry entry : currentProps.entrySet()) { - Map newProps = autoScalingConfig.getProperties(); - String key = entry.getKey(); - if (newProps.containsKey(key) && !entry.getValue().equals(newProps.get(key))) { - if (log.isDebugEnabled()) { - log.debug("Changing value of autoscaling property: {} from: {} to: {}", key, entry.getValue(), newProps.get(key)); - } - switch (key) { - case TRIGGER_SCHEDULE_DELAY_SECONDS: - triggerDelay.set(((Number) newProps.get(key)).intValue()); - synchronized (this) { - scheduledTriggerWrappers.forEach((s, triggerWrapper) -> { - if (triggerWrapper.scheduledFuture.cancel(false)) { - triggerWrapper.scheduledFuture = scheduledThreadPoolExecutor.scheduleWithFixedDelay( - triggerWrapper, 0, - cloudManager.getTimeSource().convertDelay(TimeUnit.SECONDS, triggerDelay.get(), TimeUnit.MILLISECONDS), - TimeUnit.MILLISECONDS); - } else { - log.debug("Failed to cancel scheduled task: {}", s); - } - }); - } - break; - case TRIGGER_COOLDOWN_PERIOD_SECONDS: - cooldownPeriod.set(TimeUnit.SECONDS.toNanos(((Number) newProps.get(key)).longValue())); - break; - case TRIGGER_CORE_POOL_SIZE: - this.scheduledThreadPoolExecutor.setCorePoolSize(((Number) newProps.get(key)).intValue()); - break; - } - } - } - - this.autoScalingConfig = autoScalingConfig; - // reset cooldown - cooldownStart.set(cloudManager.getTimeSource().getTimeNs() - cooldownPeriod.get()); - } - - /** - * Adds a new trigger or replaces an existing one. The replaced trigger, if any, is closed - * before the new trigger is run. If a trigger is replaced with itself then this - * operation becomes a no-op. - * - * @param newTrigger the trigger to be managed - * @throws AlreadyClosedException if this class has already been closed - */ - public synchronized void add(AutoScaling.Trigger newTrigger) throws Exception { - if (isClosed) { - throw new AlreadyClosedException("ScheduledTriggers has been closed and cannot be used anymore"); - } - TriggerWrapper st; - try { - st = new TriggerWrapper(newTrigger, cloudManager, queueStats); - } catch (Exception e) { - if (isClosed || e instanceof AlreadyClosedException) { - throw new AlreadyClosedException("ScheduledTriggers has been closed and cannot be used anymore"); - } - if (cloudManager.isClosed()) { - log.error("Failed to add trigger {} - closing or disconnected from data provider", newTrigger.getName(), e); - } else { - log.error("Failed to add trigger {}", newTrigger.getName(), e); - } - return; - } - TriggerWrapper triggerWrapper = st; - - TriggerWrapper old = scheduledTriggerWrappers.putIfAbsent(newTrigger.getName(), triggerWrapper); - if (old != null) { - if (old.trigger.equals(newTrigger)) { - // the trigger wasn't actually modified so we do nothing - return; - } - IOUtils.closeQuietly(old); - newTrigger.restoreState(old.trigger); - triggerWrapper.setReplay(false); - scheduledTriggerWrappers.replace(newTrigger.getName(), triggerWrapper); - } - newTrigger.setProcessor(event -> { - TriggerListeners triggerListeners = listeners.copy(); - if (cloudManager.isClosed()) { - String msg = String.format(Locale.ROOT, "Ignoring autoscaling event %s because Solr has been shutdown.", event.toString()); - log.warn(msg); - triggerListeners.fireListeners(event.getSource(), event, TriggerEventProcessorStage.ABORTED, msg); - return false; - } - TriggerWrapper scheduledSource = scheduledTriggerWrappers.get(event.getSource()); - if (scheduledSource == null) { - String msg = String.format(Locale.ROOT, "Ignoring autoscaling event %s because the source trigger: %s doesn't exist.", event.toString(), event.getSource()); - triggerListeners.fireListeners(event.getSource(), event, TriggerEventProcessorStage.FAILED, msg); - log.warn(msg); - return false; - } - boolean replaying = event.getProperty(TriggerEvent.REPLAYING) != null ? (Boolean)event.getProperty(TriggerEvent.REPLAYING) : false; - AutoScaling.Trigger source = scheduledSource.trigger; - if (scheduledSource.isClosed || source.isClosed()) { - String msg = String.format(Locale.ROOT, "Ignoring autoscaling event %s because the source trigger: %s has already been closed", event.toString(), source); - triggerListeners.fireListeners(event.getSource(), event, TriggerEventProcessorStage.ABORTED, msg); - log.warn(msg); - // we do not want to lose this event just because the trigger was closed, perhaps a replacement will need it - return false; - } - if (event.isIgnored()) { - log.debug("-------- Ignoring event: {}", event); - event.getProperties().put(TriggerEvent.IGNORED, true); - triggerListeners.fireListeners(event.getSource(), event, TriggerEventProcessorStage.IGNORED, "Event was ignored."); - return true; // always return true for ignored events - } - // even though we pause all triggers during action execution there is a possibility that a trigger was already - // running at the time and would have already created an event so we reject such events during cooldown period - if (cooldownStart.get() + cooldownPeriod.get() > cloudManager.getTimeSource().getTimeNs()) { - log.debug("-------- Cooldown period - rejecting event: {}", event); - event.getProperties().put(TriggerEvent.COOLDOWN, true); - triggerListeners.fireListeners(event.getSource(), event, TriggerEventProcessorStage.IGNORED, "In cooldown period."); - return false; - } else { - log.debug("++++++++ Cooldown inactive - processing event: {}", event); - // start cooldown here to immediately reject other events - cooldownStart.set(cloudManager.getTimeSource().getTimeNs()); - } - if (hasPendingActions.compareAndSet(false, true)) { - // pause all triggers while we execute actions so triggers do not operate on a cluster in transition - pauseTriggers(); - - final boolean enqueued; - if (replaying) { - enqueued = false; - } else { - enqueued = triggerWrapper.enqueue(event); - } - // fire STARTED event listeners after enqueuing the event is successful - triggerListeners.fireListeners(event.getSource(), event, TriggerEventProcessorStage.STARTED); - List actions = source.getActions(); - if (actions != null) { - if (actionExecutor.isShutdown()) { - String msg = String.format(Locale.ROOT, "Ignoring autoscaling event %s from trigger %s because the executor has already been closed", event.toString(), source); - triggerListeners.fireListeners(event.getSource(), event, TriggerEventProcessorStage.ABORTED, msg); - log.warn(msg); - hasPendingActions.set(false); - // we do not want to lose this event just because the trigger was closed, perhaps a replacement will need it - return false; - } - actionExecutor.submit(() -> { - assert hasPendingActions.get(); - long eventProcessingStart = cloudManager.getTimeSource().getTimeNs(); - TriggerListeners triggerListeners1 = triggerListeners.copy(); - log.debug("-- processing actions for {}", event); - try { - // in future, we could wait for pending tasks in a different thread and re-enqueue - // this event so that we continue processing other events and not block this action executor - waitForPendingTasks(newTrigger, actions); - - ActionContext actionContext = new ActionContext(cloudManager, newTrigger, new HashMap<>()); - for (TriggerAction action : actions) { - @SuppressWarnings({"unchecked"}) - List beforeActions = (List) actionContext.getProperties().computeIfAbsent(TriggerEventProcessorStage.BEFORE_ACTION.toString(), k -> new ArrayList()); - beforeActions.add(action.getName()); - triggerListeners1.fireListeners(event.getSource(), event, TriggerEventProcessorStage.BEFORE_ACTION, action.getName(), actionContext); - try { - action.process(event, actionContext); - } catch (Exception e) { - triggerListeners1.fireListeners(event.getSource(), event, TriggerEventProcessorStage.FAILED, action.getName(), actionContext, e, null); - throw new TriggerActionException(event.getSource(), action.getName(), "Error processing action for trigger event: " + event, e); - } - @SuppressWarnings({"unchecked"}) - List afterActions = (List) actionContext.getProperties().computeIfAbsent(TriggerEventProcessorStage.AFTER_ACTION.toString(), k -> new ArrayList()); - afterActions.add(action.getName()); - triggerListeners1.fireListeners(event.getSource(), event, TriggerEventProcessorStage.AFTER_ACTION, action.getName(), actionContext); - } - if (enqueued) { - TriggerEvent ev = triggerWrapper.dequeue(); - assert ev.getId().equals(event.getId()); - } - triggerListeners1.fireListeners(event.getSource(), event, TriggerEventProcessorStage.SUCCEEDED); - } catch (TriggerActionException e) { - log.warn("Exception executing actions", e); - } catch (Exception e) { - triggerListeners1.fireListeners(event.getSource(), event, TriggerEventProcessorStage.FAILED); - log.warn("Unhandled exception executing actions", e); - } finally { - // update cooldown to the time when we actually finished processing the actions - cooldownStart.set(cloudManager.getTimeSource().getTimeNs()); - hasPendingActions.set(false); - // resume triggers after cool down period - resumeTriggers(cloudManager.getTimeSource().convertDelay(TimeUnit.NANOSECONDS, cooldownPeriod.get(), TimeUnit.MILLISECONDS)); - } - if (log.isDebugEnabled()) { - log.debug("-- processing took {} ms for event id={}", - TimeUnit.NANOSECONDS.toMillis(cloudManager.getTimeSource().getTimeNs() - eventProcessingStart), event.id); - } - }); - } else { - if (enqueued) { - TriggerEvent ev = triggerWrapper.dequeue(); - if (!ev.getId().equals(event.getId())) { - throw new RuntimeException("Wrong event dequeued, queue of " + triggerWrapper.trigger.getName() - + " is broken! Expected event=" + event + " but got " + ev); - } - } - triggerListeners.fireListeners(event.getSource(), event, TriggerEventProcessorStage.SUCCEEDED); - hasPendingActions.set(false); - // resume triggers now - resumeTriggers(0); - } - return true; - } else { - log.debug("Ignoring event {}, already processing other actions.", event.id); - // there is an action in the queue and we don't want to enqueue another until it is complete - triggerListeners.fireListeners(event.getSource(), event, TriggerEventProcessorStage.IGNORED, "Already processing another event."); - return false; - } - }); - newTrigger.init(); // mark as ready for scheduling - triggerWrapper.scheduledFuture = scheduledThreadPoolExecutor.scheduleWithFixedDelay(triggerWrapper, 0, - cloudManager.getTimeSource().convertDelay(TimeUnit.SECONDS, triggerDelay.get(), TimeUnit.MILLISECONDS), - TimeUnit.MILLISECONDS); - } - - /** - * Pauses all scheduled trigger invocations without interrupting any that are in progress - * @lucene.internal - */ - public synchronized void pauseTriggers() { - if (log.isDebugEnabled()) { - log.debug("Pausing all triggers: {}", scheduledTriggerWrappers.keySet()); - } - scheduledTriggerWrappers.forEach((s, triggerWrapper) -> triggerWrapper.scheduledFuture.cancel(false)); - } - - /** - * Resumes all previously cancelled triggers to be scheduled after the given initial delay - * @param afterDelayMillis the initial delay in milliseconds after which triggers should be resumed - * @lucene.internal - */ - public synchronized void resumeTriggers(long afterDelayMillis) { - List> entries = new ArrayList<>(scheduledTriggerWrappers.entrySet()); - Collections.shuffle(entries, RANDOM); - entries.forEach(e -> { - String key = e.getKey(); - TriggerWrapper triggerWrapper = e.getValue(); - if (triggerWrapper.scheduledFuture.isCancelled()) { - log.debug("Resuming trigger: {} after {}ms", key, afterDelayMillis); - triggerWrapper.scheduledFuture = scheduledThreadPoolExecutor.scheduleWithFixedDelay(triggerWrapper, afterDelayMillis, - cloudManager.getTimeSource().convertDelay(TimeUnit.SECONDS, triggerDelay.get(), TimeUnit.MILLISECONDS), TimeUnit.MILLISECONDS); - } - }); - } - - private void waitForPendingTasks(AutoScaling.Trigger newTrigger, List actions) throws AlreadyClosedException { - DistribStateManager stateManager = cloudManager.getDistribStateManager(); - try { - - for (TriggerAction action : actions) { - if (action instanceof ExecutePlanAction) { - String parentPath = ZkStateReader.SOLR_AUTOSCALING_TRIGGER_STATE_PATH + "/" + newTrigger.getName() + "/" + action.getName(); - if (!stateManager.hasData(parentPath)) { - break; - } - List children = stateManager.listData(parentPath); - if (children != null) { - for (String child : children) { - String path = parentPath + '/' + child; - VersionedData data = stateManager.getData(path, null); - if (data != null) { - @SuppressWarnings({"rawtypes"}) - Map map = (Map) Utils.fromJSON(data.getData()); - String requestid = (String) map.get("requestid"); - try { - log.debug("Found pending task with requestid={}", requestid); - RequestStatusResponse statusResponse = waitForTaskToFinish(cloudManager, requestid, - ExecutePlanAction.DEFAULT_TASK_TIMEOUT_SECONDS, TimeUnit.SECONDS); - if (statusResponse != null) { - RequestStatusState state = statusResponse.getRequestStatus(); - if (state == RequestStatusState.COMPLETED || state == RequestStatusState.FAILED || state == RequestStatusState.NOT_FOUND) { - stateManager.removeData(path, -1); - } - } - } catch (Exception e) { - if (cloudManager.isClosed()) { - throw e; // propagate the abort to the caller - } - Throwable rootCause = ExceptionUtils.getRootCause(e); - if (rootCause instanceof IllegalStateException && rootCause.getMessage().contains("Connection pool shut down")) { - throw e; - } - if (rootCause instanceof TimeoutException && rootCause.getMessage().contains("Could not connect to ZooKeeper")) { - throw e; - } - log.error("Unexpected exception while waiting for pending task with requestid: {} to finish", requestid, e); - } - } - } - } - } - } - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Thread interrupted", e); - } catch (Exception e) { - if (cloudManager.isClosed()) { - throw new AlreadyClosedException("The Solr instance has been shutdown"); - } - // we catch but don't rethrow because a failure to wait for pending tasks - // should not keep the actions from executing - log.error("Unexpected exception while waiting for pending tasks to finish", e); - } - } - - /** - * Remove and stop all triggers. Also cleans up any leftover - * state / events in ZK. - */ - public synchronized void removeAll() { - getScheduledTriggerNames().forEach(t -> { - log.info("-- removing trigger: {}", t); - remove(t); - }); - } - - /** - * Removes and stops the trigger with the given name. Also cleans up any leftover - * state / events in ZK. - * - * @param triggerName the name of the trigger to be removed - */ - public synchronized void remove(String triggerName) { - TriggerWrapper removed = scheduledTriggerWrappers.remove(triggerName); - IOUtils.closeQuietly(removed); - removeTriggerZKData(triggerName); - } - - private void removeTriggerZKData(String triggerName) { - String statePath = ZkStateReader.SOLR_AUTOSCALING_TRIGGER_STATE_PATH + "/" + triggerName; - String eventsPath = ZkStateReader.SOLR_AUTOSCALING_EVENTS_PATH + "/" + triggerName; - try { - stateManager.removeRecursively(statePath, true, true); - } catch (Exception e) { - log.warn("Failed to remove state for removed trigger {}", statePath, e); - } - try { - stateManager.removeRecursively(eventsPath, true, true); - } catch (Exception e) { - log.warn("Failed to remove events for removed trigger {}", eventsPath, e); - } - } - - /** - * @return an unmodifiable set of names of all triggers being managed by this class - */ - public synchronized Set getScheduledTriggerNames() { - return Set.copyOf(scheduledTriggerWrappers.keySet()); // shallow copy - } - - /** - * For use in white/grey box testing: The Trigger returned may be inspected, - * but should not be modified in any way. - * - * @param name the name of an existing trigger - * @return the current scheduled trigger with that name, or null if none exists - * @lucene.internal - */ - public synchronized AutoScaling.Trigger getTrigger(String name) { - TriggerWrapper w = scheduledTriggerWrappers.get(name); - return (null == w) ? null : w.trigger; - } - - @Override - public void close() throws IOException { - synchronized (this) { - // mark that we are closed - isClosed = true; - for (TriggerWrapper triggerWrapper : scheduledTriggerWrappers.values()) { - IOUtils.closeQuietly(triggerWrapper); - } - scheduledTriggerWrappers.clear(); - } - // shutdown and interrupt all running tasks because there's no longer any - // guarantee about cluster state - log.debug("Shutting down scheduled thread pool executor now"); - scheduledThreadPoolExecutor.shutdownNow(); - - log.debug("Shutting down action executor now"); - actionExecutor.shutdownNow(); - - listeners.close(); - - log.debug("Awaiting termination for action executor"); - awaitTermination(actionExecutor); - - log.debug("Awaiting termination for scheduled thread pool executor"); - awaitTermination(scheduledThreadPoolExecutor); - - log.debug("ScheduledTriggers closed completely"); - } - - /** - * Add a temporary listener for internal use (tests, simulation). - * @param listener listener instance - */ - public void addAdditionalListener(TriggerListener listener) { - listeners.addAdditionalListener(listener); - } - - /** - * Remove a temporary listener for internal use (tests, simulation). - * @param listener listener instance - */ - public void removeAdditionalListener(TriggerListener listener) { - listeners.removeAdditionalListener(listener); - } - - private class TriggerWrapper implements Runnable, Closeable { - AutoScaling.Trigger trigger; - ScheduledFuture scheduledFuture; - TriggerEventQueue queue; - boolean replay; - volatile boolean isClosed; - - TriggerWrapper(AutoScaling.Trigger trigger, SolrCloudManager cloudManager, Stats stats) throws IOException { - this.trigger = trigger; - this.queue = new TriggerEventQueue(cloudManager, trigger.getName(), stats); - this.replay = true; - this.isClosed = false; - } - - public void setReplay(boolean replay) { - this.replay = replay; - } - - public boolean enqueue(TriggerEvent event) { - if (isClosed) { - throw new AlreadyClosedException("ScheduledTrigger " + trigger.getName() + " has been closed."); - } - return queue.offerEvent(event); - } - - public TriggerEvent dequeue() { - if (isClosed) { - throw new AlreadyClosedException("ScheduledTrigger " + trigger.getName() + " has been closed."); - } - TriggerEvent event = queue.pollEvent(); - return event; - } - - @Override - public void run() { - if (isClosed) { - throw new AlreadyClosedException("ScheduledTrigger " + trigger.getName() + " has been closed."); - } - // fire a trigger only if an action is not pending - // note this is not fool proof e.g. it does not prevent an action being executed while a trigger - // is still executing. There is additional protection against that scenario in the event listener. - if (!hasPendingActions.get()) { - // this synchronization is usually never under contention - // but the only reason to have it here is to ensure that when the set-properties API is used - // to change the schedule delay, we can safely cancel the old scheduled task - // and create another one with the new delay without worrying about concurrent - // execution of the same trigger instance - synchronized (TriggerWrapper.this) { - // replay accumulated events on first run, if any - - try { - if (replay) { - TriggerEvent event; - // peek first without removing - we may crash before calling the listener - while ((event = queue.peekEvent()) != null) { - // override REPLAYING=true - event.getProperties().put(TriggerEvent.REPLAYING, true); - if (!trigger.getProcessor().process(event)) { - log.error("Failed to re-play event, discarding: {}", event); - } - queue.pollEvent(); // always remove it from queue - } - // now restore saved state to possibly generate new events from old state on the first run - try { - trigger.restoreState(); - } catch (Exception e) { - // log but don't throw - see below - log.error("Error restoring trigger state {}", trigger.getName(), e); - } - replay = false; - } - } catch (AlreadyClosedException e) { - - } catch (Exception e) { - log.error("Unexpected exception from trigger: {}", trigger.getName(), e); - } - try { - trigger.run(); - } catch (AlreadyClosedException e) { - - } catch (Exception e) { - // log but do not propagate exception because an exception thrown from a scheduled operation - // will suppress future executions - log.error("Unexpected exception from trigger: {}", trigger.getName(), e); - } finally { - // checkpoint after each run - trigger.saveState(); - } - } - } - } - - @Override - public void close() throws IOException { - isClosed = true; - if (scheduledFuture != null) { - scheduledFuture.cancel(true); - } - IOUtils.closeQuietly(trigger); - } - } - - private class TriggerListeners { - Map>> listenersPerStage = new HashMap<>(); - Map listenersPerName = new HashMap<>(); - List additionalListeners = new ArrayList<>(); - ReentrantLock updateLock = new ReentrantLock(); - - public TriggerListeners() { - - } - - private TriggerListeners(Map>> listenersPerStage, - Map listenersPerName) { - this.listenersPerStage = new HashMap<>(); - listenersPerStage.forEach((n, listeners) -> { - Map> perStage = this.listenersPerStage.computeIfAbsent(n, name -> new HashMap<>()); - listeners.forEach((s, lst) -> { - List newLst = perStage.computeIfAbsent(s, stage -> new ArrayList<>()); - newLst.addAll(lst); - }); - }); - this.listenersPerName = new HashMap<>(listenersPerName); - } - - public TriggerListeners copy() { - return new TriggerListeners(listenersPerStage, listenersPerName); - } - - public void addAdditionalListener(TriggerListener listener) { - updateLock.lock(); - try { - AutoScalingConfig.TriggerListenerConfig config = listener.getConfig(); - for (TriggerEventProcessorStage stage : config.stages) { - addPerStage(config.trigger, stage, listener); - } - // add also for beforeAction / afterAction TriggerStage - if (!config.beforeActions.isEmpty()) { - addPerStage(config.trigger, TriggerEventProcessorStage.BEFORE_ACTION, listener); - } - if (!config.afterActions.isEmpty()) { - addPerStage(config.trigger, TriggerEventProcessorStage.AFTER_ACTION, listener); - } - additionalListeners.add(listener); - } finally { - updateLock.unlock(); - } - } - - public void removeAdditionalListener(TriggerListener listener) { - updateLock.lock(); - try { - listenersPerName.remove(listener.getConfig().name); - listenersPerStage.forEach((trigger, perStage) -> { - perStage.forEach((stage, listeners) -> { - listeners.remove(listener); - }); - }); - additionalListeners.remove(listener); - } finally { - updateLock.unlock(); - } - } - - void setAutoScalingConfig(AutoScalingConfig autoScalingConfig) { - updateLock.lock(); - // we will recreate this from scratch - listenersPerStage.clear(); - try { - Set triggerNames = autoScalingConfig.getTriggerConfigs().keySet(); - Map configs = autoScalingConfig.getTriggerListenerConfigs(); - Set listenerNames = configs.entrySet().stream().map(entry -> entry.getValue().name).collect(Collectors.toSet()); - // close those for non-existent triggers and nonexistent listener configs - for (Iterator> it = listenersPerName.entrySet().iterator(); it.hasNext(); ) { - Map.Entry entry = it.next(); - String name = entry.getKey(); - TriggerListener listener = entry.getValue(); - if (!triggerNames.contains(listener.getConfig().trigger) || !listenerNames.contains(name)) { - try { - listener.close(); - } catch (Exception e) { - log.warn("Exception closing old listener {}", listener.getConfig(), e); - } - it.remove(); - } - } - for (Map.Entry entry : configs.entrySet()) { - AutoScalingConfig.TriggerListenerConfig config = entry.getValue(); - if (!triggerNames.contains(config.trigger)) { - log.debug("-- skipping listener for non-existent trigger: {}", config); - continue; - } - // find previous instance and reuse if possible - TriggerListener oldListener = listenersPerName.get(config.name); - TriggerListener listener = null; - if (oldListener != null) { - if (!oldListener.getConfig().equals(config)) { // changed config - try { - oldListener.close(); - } catch (Exception e) { - log.warn("Exception closing old listener {}", oldListener.getConfig(), e); - } - } else { - listener = oldListener; // reuse - } - } - if (listener == null) { // create new instance - String clazz = config.listenerClass; - try { - listener = loader.newInstance(clazz, TriggerListener.class); - } catch (Exception e) { - log.warn("Invalid TriggerListener class name '{}', skipping...", clazz, e); - } - if (listener != null) { - try { - listener.configure(loader, cloudManager, config); - listener.init(); - listenersPerName.put(config.name, listener); - } catch (Exception e) { - log.warn("Error initializing TriggerListener {}", config, e); - IOUtils.closeQuietly(listener); - listener = null; - } - } - } - if (listener == null) { - continue; - } - // add per stage - for (TriggerEventProcessorStage stage : config.stages) { - addPerStage(config.trigger, stage, listener); - } - // add also for beforeAction / afterAction TriggerStage - if (!config.beforeActions.isEmpty()) { - addPerStage(config.trigger, TriggerEventProcessorStage.BEFORE_ACTION, listener); - } - if (!config.afterActions.isEmpty()) { - addPerStage(config.trigger, TriggerEventProcessorStage.AFTER_ACTION, listener); - } - } - // re-add additional listeners - List additional = new ArrayList<>(additionalListeners); - additionalListeners.clear(); - for (TriggerListener listener : additional) { - addAdditionalListener(listener); - } - - } finally { - updateLock.unlock(); - } - } - - private void addPerStage(String triggerName, TriggerEventProcessorStage stage, TriggerListener listener) { - Map> perStage = - listenersPerStage.computeIfAbsent(triggerName, k -> new HashMap<>()); - List lst = perStage.computeIfAbsent(stage, k -> new ArrayList<>(3)); - lst.add(listener); - } - - void reset() { - updateLock.lock(); - try { - listenersPerStage.clear(); - for (TriggerListener listener : listenersPerName.values()) { - IOUtils.closeQuietly(listener); - } - listenersPerName.clear(); - } finally { - updateLock.unlock(); - } - } - - void close() { - reset(); - } - - List getTriggerListeners(String trigger, TriggerEventProcessorStage stage) { - Map> perStage = listenersPerStage.get(trigger); - if (perStage == null) { - return Collections.emptyList(); - } - List lst = perStage.get(stage); - if (lst == null) { - return Collections.emptyList(); - } else { - return Collections.unmodifiableList(lst); - } - } - - void fireListeners(String trigger, TriggerEvent event, TriggerEventProcessorStage stage) { - fireListeners(trigger, event, stage, null, null, null, null); - } - - void fireListeners(String trigger, TriggerEvent event, TriggerEventProcessorStage stage, String message) { - fireListeners(trigger, event, stage, null, null, null, message); - } - - void fireListeners(String trigger, TriggerEvent event, TriggerEventProcessorStage stage, String actionName, - ActionContext context) { - fireListeners(trigger, event, stage, actionName, context, null, null); - } - - void fireListeners(String trigger, TriggerEvent event, TriggerEventProcessorStage stage, String actionName, - ActionContext context, Throwable error, String message) { - updateLock.lock(); - try { - for (TriggerListener listener : getTriggerListeners(trigger, stage)) { - if (!listener.isEnabled()) { - continue; - } - if (actionName != null) { - AutoScalingConfig.TriggerListenerConfig config = listener.getConfig(); - if (stage == TriggerEventProcessorStage.BEFORE_ACTION) { - if (!config.beforeActions.contains(actionName)) { - continue; - } - } else if (stage == TriggerEventProcessorStage.AFTER_ACTION) { - if (!config.afterActions.contains(actionName)) { - continue; - } - } - } - try { - listener.onEvent(event, stage, actionName, context, error, message); - } catch (Exception e) { - log.warn("Exception running listener {}", listener.getConfig(), e); - } - } - } finally { - updateLock.unlock(); - } - } - } -} diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/SearchRateTrigger.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/SearchRateTrigger.java deleted file mode 100644 index 505c5113df0..00000000000 --- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/SearchRateTrigger.java +++ /dev/null @@ -1,805 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.cloud.autoscaling; - -import java.io.IOException; -import java.lang.invoke.MethodHandles; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicLong; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.util.concurrent.AtomicDouble; -import org.apache.solr.client.solrj.cloud.autoscaling.Policy; -import org.apache.solr.client.solrj.cloud.SolrCloudManager; -import org.apache.solr.client.solrj.cloud.autoscaling.Suggester; -import org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventType; -import org.apache.solr.common.SolrException; -import org.apache.solr.common.cloud.ClusterState; -import org.apache.solr.common.cloud.Replica; -import org.apache.solr.common.cloud.ZkStateReader; -import org.apache.solr.common.params.AutoScalingParams; -import org.apache.solr.common.params.CollectionParams; -import org.apache.solr.common.util.Pair; -import org.apache.solr.common.util.StrUtils; -import org.apache.solr.common.util.Utils; -import org.apache.solr.core.SolrResourceLoader; -import org.apache.solr.metrics.SolrCoreMetricManager; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Trigger for the {@link org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventType#SEARCHRATE} event. - */ -public class SearchRateTrigger extends TriggerBase { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - public static final String COLLECTIONS_PROP = "collections"; - public static final String METRIC_PROP = "metric"; - public static final String MAX_OPS_PROP = "maxOps"; - public static final String MIN_REPLICAS_PROP = "minReplicas"; - public static final String ABOVE_RATE_PROP = "aboveRate"; - public static final String BELOW_RATE_PROP = "belowRate"; - public static final String ABOVE_NODE_RATE_PROP = "aboveNodeRate"; - public static final String BELOW_NODE_RATE_PROP = "belowNodeRate"; - public static final String ABOVE_OP_PROP = "aboveOp"; - public static final String BELOW_OP_PROP = "belowOp"; - public static final String ABOVE_NODE_OP_PROP = "aboveNodeOp"; - public static final String BELOW_NODE_OP_PROP = "belowNodeOp"; - - // back-compat - public static final String BC_COLLECTION_PROP = "collection"; - public static final String BC_RATE_PROP = "rate"; - - - public static final String HOT_NODES = "hotNodes"; - public static final String HOT_COLLECTIONS = "hotCollections"; - public static final String HOT_SHARDS = "hotShards"; - public static final String HOT_REPLICAS = "hotReplicas"; - public static final String COLD_NODES = "coldNodes"; - public static final String COLD_COLLECTIONS = "coldCollections"; - public static final String COLD_SHARDS = "coldShards"; - public static final String COLD_REPLICAS = "coldReplicas"; - public static final String VIOLATION_PROP = "violationType"; - - public static final int DEFAULT_MAX_OPS = 3; - public static final String DEFAULT_METRIC = "QUERY./select.requestTimes:1minRate"; - - private String metric; - private int maxOps; - private Integer minReplicas = null; - private final Set collections = new HashSet<>(); - private String shard; - private String node; - private double aboveRate; - private double belowRate; - private double aboveNodeRate; - private double belowNodeRate; - private CollectionParams.CollectionAction aboveOp, belowOp, aboveNodeOp, belowNodeOp; - private final Map lastCollectionEvent = new ConcurrentHashMap<>(); - private final Map lastNodeEvent = new ConcurrentHashMap<>(); - private final Map lastShardEvent = new ConcurrentHashMap<>(); - private final Map lastReplicaEvent = new ConcurrentHashMap<>(); - private final Map state = new HashMap<>(); - - public SearchRateTrigger(String name) { - super(TriggerEventType.SEARCHRATE, name); - this.state.put("lastCollectionEvent", lastCollectionEvent); - this.state.put("lastNodeEvent", lastNodeEvent); - this.state.put("lastShardEvent", lastShardEvent); - this.state.put("lastReplicaEvent", lastReplicaEvent); - TriggerUtils.validProperties(validProperties, - COLLECTIONS_PROP, AutoScalingParams.SHARD, AutoScalingParams.NODE, - METRIC_PROP, - MAX_OPS_PROP, - MIN_REPLICAS_PROP, - ABOVE_OP_PROP, - BELOW_OP_PROP, - ABOVE_NODE_OP_PROP, - BELOW_NODE_OP_PROP, - ABOVE_RATE_PROP, - BELOW_RATE_PROP, - ABOVE_NODE_RATE_PROP, - BELOW_NODE_RATE_PROP, - // back-compat props - BC_COLLECTION_PROP, - BC_RATE_PROP); - } - - @Override - public void configure(SolrResourceLoader loader, SolrCloudManager cloudManager, Map properties) throws TriggerValidationException { - super.configure(loader, cloudManager, properties); - // parse config options - String collectionsStr = (String)properties.get(COLLECTIONS_PROP); - if (collectionsStr != null) { - collections.addAll(StrUtils.splitSmart(collectionsStr, ',')); - } - // check back-compat collection prop - collectionsStr = (String)properties.get(BC_COLLECTION_PROP); - if (collectionsStr != null) { - if (!collectionsStr.equals(Policy.ANY)) { - collections.add(collectionsStr); - } - } - shard = (String)properties.getOrDefault(AutoScalingParams.SHARD, Policy.ANY); - if (!shard.equals(Policy.ANY) && (collections.isEmpty() || collections.size() > 1)) { - throw new TriggerValidationException(name, AutoScalingParams.SHARD, "When 'shard' is other than #ANY then exactly one collection name must be set"); - } - node = (String)properties.getOrDefault(AutoScalingParams.NODE, Policy.ANY); - metric = (String)properties.getOrDefault(METRIC_PROP, DEFAULT_METRIC); - - String maxOpsStr = String.valueOf(properties.getOrDefault(MAX_OPS_PROP, DEFAULT_MAX_OPS)); - try { - maxOps = Integer.parseInt(maxOpsStr); - } catch (Exception e) { - throw new TriggerValidationException(name, MAX_OPS_PROP, "invalid value '" + maxOpsStr + "': " + e.toString()); - } - - Object o = properties.get(MIN_REPLICAS_PROP); - if (o != null) { - try { - minReplicas = Integer.parseInt(o.toString()); - if (minReplicas < 1) { - throw new Exception("must be at least 1, or not set to use 'replicationFactor'"); - } - } catch (Exception e) { - throw new TriggerValidationException(name, MIN_REPLICAS_PROP, "invalid value '" + o + "': " + e.toString()); - } - } - - Object above = properties.get(ABOVE_RATE_PROP); - Object below = properties.get(BELOW_RATE_PROP); - // back-compat rate prop - if (properties.containsKey(BC_RATE_PROP)) { - above = properties.get(BC_RATE_PROP); - } - if (above == null && below == null) { - throw new TriggerValidationException(name, ABOVE_RATE_PROP, "at least one of '" + - ABOVE_RATE_PROP + "' or '" + BELOW_RATE_PROP + "' must be set"); - } - if (above != null) { - try { - aboveRate = Double.parseDouble(String.valueOf(above)); - } catch (Exception e) { - throw new TriggerValidationException(name, ABOVE_RATE_PROP, "Invalid configuration value: '" + above + "': " + e.toString()); - } - } else { - aboveRate = Double.MAX_VALUE; - } - if (below != null) { - try { - belowRate = Double.parseDouble(String.valueOf(below)); - } catch (Exception e) { - throw new TriggerValidationException(name, BELOW_RATE_PROP, "Invalid configuration value: '" + below + "': " + e.toString()); - } - } else { - belowRate = -1; - } - - // node rates - above = properties.get(ABOVE_NODE_RATE_PROP); - below = properties.get(BELOW_NODE_RATE_PROP); - if (above != null) { - try { - aboveNodeRate = Double.parseDouble(String.valueOf(above)); - } catch (Exception e) { - throw new TriggerValidationException(name, ABOVE_NODE_RATE_PROP, "Invalid configuration value: '" + above + "': " + e.toString()); - } - } else { - aboveNodeRate = Double.MAX_VALUE; - } - if (below != null) { - try { - belowNodeRate = Double.parseDouble(String.valueOf(below)); - } catch (Exception e) { - throw new TriggerValidationException(name, BELOW_NODE_RATE_PROP, "Invalid configuration value: '" + below + "': " + e.toString()); - } - } else { - belowNodeRate = -1; - } - - String aboveOpStr = String.valueOf(properties.getOrDefault(ABOVE_OP_PROP, CollectionParams.CollectionAction.ADDREPLICA.toLower())); - String belowOpStr = String.valueOf(properties.getOrDefault(BELOW_OP_PROP, CollectionParams.CollectionAction.DELETEREPLICA.toLower())); - aboveOp = CollectionParams.CollectionAction.get(aboveOpStr); - if (aboveOp == null) { - throw new TriggerValidationException(getName(), ABOVE_OP_PROP, "unrecognized value: '" + aboveOpStr + "'"); - } - belowOp = CollectionParams.CollectionAction.get(belowOpStr); - if (belowOp == null) { - throw new TriggerValidationException(getName(), BELOW_OP_PROP, "unrecognized value: '" + belowOpStr + "'"); - } - Object aboveNodeObj = properties.getOrDefault(ABOVE_NODE_OP_PROP, CollectionParams.CollectionAction.MOVEREPLICA.toLower()); - // do NOT set the default to DELETENODE - Object belowNodeObj = properties.get(BELOW_NODE_OP_PROP); - try { - aboveNodeOp = CollectionParams.CollectionAction.get(String.valueOf(aboveNodeObj)); - } catch (Exception e) { - throw new TriggerValidationException(getName(), ABOVE_NODE_OP_PROP, "unrecognized value: '" + aboveNodeObj + "'"); - } - if (belowNodeObj != null) { - try { - belowNodeOp = CollectionParams.CollectionAction.get(String.valueOf(belowNodeObj)); - } catch (Exception e) { - throw new TriggerValidationException(getName(), BELOW_NODE_OP_PROP, "unrecognized value: '" + belowNodeObj + "'"); - } - } - } - - @VisibleForTesting - Map getConfig() { - Map config = new HashMap<>(); - config.put("name", name); - config.put(COLLECTIONS_PROP, collections); - config.put(AutoScalingParams.SHARD, shard); - config.put(AutoScalingParams.NODE, node); - config.put(METRIC_PROP, metric); - config.put(MAX_OPS_PROP, maxOps); - config.put(MIN_REPLICAS_PROP, minReplicas); - config.put(ABOVE_RATE_PROP, aboveRate); - config.put(BELOW_RATE_PROP, belowRate); - config.put(ABOVE_NODE_RATE_PROP, aboveNodeRate); - config.put(BELOW_NODE_RATE_PROP, belowNodeRate); - config.put(ABOVE_OP_PROP, aboveOp); - config.put(ABOVE_NODE_OP_PROP, aboveNodeOp); - config.put(BELOW_OP_PROP, belowOp); - config.put(BELOW_NODE_OP_PROP, belowNodeOp); - return config; - } - - @Override - protected Map getState() { - return state; - } - - @Override - protected void setState(Map state) { - lastCollectionEvent.clear(); - lastNodeEvent.clear(); - lastShardEvent.clear(); - lastReplicaEvent.clear(); - @SuppressWarnings({"unchecked"}) - Map collTimes = (Map)state.get("lastCollectionEvent"); - if (collTimes != null) { - lastCollectionEvent.putAll(collTimes); - } - @SuppressWarnings({"unchecked"}) - Map nodeTimes = (Map)state.get("lastNodeEvent"); - if (nodeTimes != null) { - lastNodeEvent.putAll(nodeTimes); - } - @SuppressWarnings({"unchecked"}) - Map shardTimes = (Map)state.get("lastShardEvent"); - if (shardTimes != null) { - lastShardEvent.putAll(shardTimes); - } - @SuppressWarnings({"unchecked"}) - Map replicaTimes = (Map)state.get("lastReplicaEvent"); - if (replicaTimes != null) { - lastReplicaEvent.putAll(replicaTimes); - } - } - - @Override - public void restoreState(AutoScaling.Trigger old) { - assert old.isClosed(); - if (old instanceof SearchRateTrigger) { - SearchRateTrigger that = (SearchRateTrigger)old; - assert this.name.equals(that.name); - this.lastCollectionEvent.clear(); - this.lastNodeEvent.clear(); - this.lastShardEvent.clear(); - this.lastReplicaEvent.clear(); - this.lastCollectionEvent.putAll(that.lastCollectionEvent); - this.lastNodeEvent.putAll(that.lastNodeEvent); - this.lastShardEvent.putAll(that.lastShardEvent); - this.lastReplicaEvent.putAll(that.lastReplicaEvent); - } else { - throw new SolrException(SolrException.ErrorCode.INVALID_STATE, - "Unable to restore state from an unknown type of trigger"); - } - - } - - @Override - public void run() { - AutoScaling.TriggerEventProcessor processor = processorRef.get(); - if (processor == null) { - return; - } - - // collection, shard, list(replica + rate) - Map>> collectionRates = new HashMap<>(); - // node, rate - Map nodeRates = new HashMap<>(); - // this replication factor only considers replica types that are searchable - // collection, shard, RF - Map> searchableReplicationFactors = new HashMap<>(); - - ClusterState clusterState = null; - try { - clusterState = cloudManager.getClusterStateProvider().getClusterState(); - } catch (IOException e) { - log.warn("Error getting ClusterState", e); - return; - } - for (String node : cloudManager.getClusterStateProvider().getLiveNodes()) { - Map metricTags = new HashMap<>(); - // coll, shard, replica - Map>> infos = cloudManager.getNodeStateProvider().getReplicaInfo(node, Collections.emptyList()); - infos.forEach((coll, shards) -> { - Map replPerShard = searchableReplicationFactors.computeIfAbsent(coll, c -> new HashMap<>()); - shards.forEach((sh, replicas) -> { - AtomicInteger repl = replPerShard.computeIfAbsent(sh, s -> new AtomicInteger()); - replicas.forEach(replica -> { - // skip non-active replicas - if (replica.getState() != Replica.State.ACTIVE) { - return; - } - repl.incrementAndGet(); - // we have to translate to the metrics registry name, which uses "_replica_nN" as suffix - String replicaName = Utils.parseMetricsReplicaName(coll, replica.getCoreName()); - if (replicaName == null) { // should never happen??? - replicaName = replica.getName(); // which is actually coreNode name... - } - String registry = SolrCoreMetricManager.createRegistryName(true, coll, sh, replicaName, null); - String tag = "metrics:" + registry + ":" + metric; - metricTags.put(tag, replica); - }); - }); - }); - if (metricTags.isEmpty()) { - continue; - } - Map rates = cloudManager.getNodeStateProvider().getNodeValues(node, metricTags.keySet()); - if (log.isDebugEnabled()) { - log.debug("### rates for node {}", node); - rates.forEach((tag, rate) -> log.debug("### " + tag + "\t" + rate)); // logOk - } - rates.forEach((tag, rate) -> { - Replica info = metricTags.get(tag); - if (info == null) { - log.warn("Missing replica info for response tag {}", tag); - } else { - Map> perCollection = collectionRates.computeIfAbsent(info.getCollection(), s -> new HashMap<>()); - List perShard = perCollection.computeIfAbsent(info.getShard(), s -> new ArrayList<>()); - info = (Replica)info.clone(); - info.getProperties().put(AutoScalingParams.RATE, ((Number)rate).doubleValue()); - perShard.add(info); - AtomicDouble perNode = nodeRates.computeIfAbsent(node, s -> new AtomicDouble()); - perNode.addAndGet(((Number)rate).doubleValue()); - } - }); - } - - if (log.isDebugEnabled()) { - collectionRates.forEach((coll, collRates) -> { - log.debug("## Collection: {}", coll); - collRates.forEach((s, replicas) -> { - log.debug("## - {}", s); - replicas.forEach(ri -> log.debug("## {} {}", ri.getCoreName(), ri.get(AutoScalingParams.RATE))); //logOk - }); - }); - } - long now = cloudManager.getTimeSource().getTimeNs(); - Map hotNodes = new HashMap<>(); - Map coldNodes = new HashMap<>(); - - // check for exceeded rates and filter out those with less than waitFor from previous events - nodeRates.entrySet().stream() - .filter(entry -> node.equals(Policy.ANY) || node.equals(entry.getKey())) - .forEach(entry -> { - if (entry.getValue().get() > aboveNodeRate) { - if (waitForElapsed(entry.getKey(), now, lastNodeEvent)) { - hotNodes.put(entry.getKey(), entry.getValue().get()); - } - } else if (entry.getValue().get() < belowNodeRate) { - if (waitForElapsed(entry.getKey(), now, lastNodeEvent)) { - coldNodes.put(entry.getKey(), entry.getValue().get()); - } - } else { - // no violation - clear waitForElapsed - // (violation is only valid if it persists throughout waitFor) - lastNodeEvent.remove(entry.getKey()); - } - }); - - Map> hotShards = new HashMap<>(); - Map> coldShards = new HashMap<>(); - List hotReplicas = new ArrayList<>(); - List coldReplicas = new ArrayList<>(); - collectionRates.forEach((coll, shardRates) -> { - shardRates.forEach((sh, replicaRates) -> { - double totalShardRate = replicaRates.stream() - .map(r -> { - String elapsedKey = r.getCollection() + "." + r.getCoreName(); - if ((Double)r.get(AutoScalingParams.RATE) > aboveRate) { - if (waitForElapsed(elapsedKey, now, lastReplicaEvent)) { - hotReplicas.add(r); - } - } else if ((Double)r.get(AutoScalingParams.RATE) < belowRate) { - if (waitForElapsed(elapsedKey, now, lastReplicaEvent)) { - coldReplicas.add(r); - } - } else { - // no violation - clear waitForElapsed - lastReplicaEvent.remove(elapsedKey); - } - return r; - }) - .mapToDouble(r -> (Double)r.get(AutoScalingParams.RATE)).sum(); - // calculate average shard rate over all searchable replicas (see SOLR-12470) - double shardRate = totalShardRate / searchableReplicationFactors.get(coll).get(sh).doubleValue(); - String elapsedKey = coll + "." + sh; - log.debug("-- {}: totalShardRate={}, shardRate={}", elapsedKey, totalShardRate, shardRate); - if ((collections.isEmpty() || collections.contains(coll)) && - (shard.equals(Policy.ANY) || shard.equals(sh))) { - if (shardRate > aboveRate) { - if (waitForElapsed(elapsedKey, now, lastShardEvent)) { - hotShards.computeIfAbsent(coll, s -> new HashMap<>()).put(sh, shardRate); - } - } else if (shardRate < belowRate) { - if (waitForElapsed(elapsedKey, now, lastShardEvent)) { - coldShards.computeIfAbsent(coll, s -> new HashMap<>()).put(sh, shardRate); - log.debug("-- coldShard waitFor elapsed {}", elapsedKey); - } else { - if (log.isDebugEnabled()) { - Long lastTime = lastShardEvent.computeIfAbsent(elapsedKey, s -> now); - long elapsed = TimeUnit.SECONDS.convert(now - lastTime, TimeUnit.NANOSECONDS); - if (log.isDebugEnabled()) { - log.debug("-- waitFor didn't elapse for {}, waitFor={}, elapsed={}", elapsedKey, getWaitForSecond(), elapsed); - } - } - } - } else { - // no violation - clear waitForElapsed - lastShardEvent.remove(elapsedKey); - } - } - }); - }); - - Map hotCollections = new HashMap<>(); - Map coldCollections = new HashMap<>(); - collectionRates.forEach((coll, shardRates) -> { - double total = shardRates.entrySet().stream() - .mapToDouble(e -> e.getValue().stream() - .mapToDouble(r -> (Double)r.get(AutoScalingParams.RATE)).sum()).sum(); - if (collections.isEmpty() || collections.contains(coll)) { - if (total > aboveRate) { - if (waitForElapsed(coll, now, lastCollectionEvent)) { - hotCollections.put(coll, total); - } - } else if (total < belowRate) { - if (waitForElapsed(coll, now, lastCollectionEvent)) { - coldCollections.put(coll, total); - } - } else { - // no violation - clear waitForElapsed - lastCollectionEvent.remove(coll); - } - } - }); - - if (hotCollections.isEmpty() && - hotShards.isEmpty() && - hotReplicas.isEmpty() && - hotNodes.isEmpty() && - coldCollections.isEmpty() && - coldShards.isEmpty() && - coldReplicas.isEmpty() && - coldNodes.isEmpty()) { - return; - } - - // generate event - - // find the earliest time when a condition was exceeded - final AtomicLong eventTime = new AtomicLong(now); - hotCollections.forEach((c, r) -> { - long time = lastCollectionEvent.get(c); - if (eventTime.get() > time) { - eventTime.set(time); - } - }); - coldCollections.forEach((c, r) -> { - long time = lastCollectionEvent.get(c); - if (eventTime.get() > time) { - eventTime.set(time); - } - }); - hotShards.forEach((c, shards) -> { - shards.forEach((s, r) -> { - long time = lastShardEvent.get(c + "." + s); - if (eventTime.get() > time) { - eventTime.set(time); - } - }); - }); - coldShards.forEach((c, shards) -> { - shards.forEach((s, r) -> { - long time = lastShardEvent.get(c + "." + s); - if (eventTime.get() > time) { - eventTime.set(time); - } - }); - }); - hotReplicas.forEach(r -> { - long time = lastReplicaEvent.get(r.getCollection() + "." + r.getCoreName()); - if (eventTime.get() > time) { - eventTime.set(time); - } - }); - coldReplicas.forEach(r -> { - long time = lastReplicaEvent.get(r.getCollection() + "." + r.getCoreName()); - if (eventTime.get() > time) { - eventTime.set(time); - } - }); - hotNodes.forEach((n, r) -> { - long time = lastNodeEvent.get(n); - if (eventTime.get() > time) { - eventTime.set(time); - } - }); - coldNodes.forEach((n, r) -> { - long time = lastNodeEvent.get(n); - if (eventTime.get() > time) { - eventTime.set(time); - } - }); - - final List ops = new ArrayList<>(); - final Set violations = new HashSet<>(); - - calculateHotOps(ops, violations, searchableReplicationFactors, hotNodes, hotCollections, hotShards, hotReplicas); - calculateColdOps(ops, violations, clusterState, searchableReplicationFactors, coldNodes, coldCollections, coldShards, coldReplicas); - - if (ops.isEmpty()) { - return; - } - - if (processor.process(new SearchRateEvent(getName(), eventTime.get(), ops, - hotNodes, hotCollections, hotShards, hotReplicas, - coldNodes, coldCollections, coldShards, coldReplicas, violations))) { - // update lastEvent times - hotNodes.keySet().forEach(node -> lastNodeEvent.put(node, now)); - coldNodes.keySet().forEach(node -> lastNodeEvent.put(node, now)); - hotCollections.keySet().forEach(coll -> lastCollectionEvent.put(coll, now)); - coldCollections.keySet().forEach(coll -> lastCollectionEvent.put(coll, now)); - hotShards.entrySet().forEach(e -> e.getValue() - .forEach((sh, rate) -> lastShardEvent.put(e.getKey() + "." + sh, now))); - coldShards.entrySet().forEach(e -> e.getValue() - .forEach((sh, rate) -> lastShardEvent.put(e.getKey() + "." + sh, now))); - hotReplicas.forEach(r -> lastReplicaEvent.put(r.getCollection() + "." + r.getCoreName(), now)); - coldReplicas.forEach(r -> lastReplicaEvent.put(r.getCollection() + "." + r.getCoreName(), now)); - } - } - - private void calculateHotOps(List ops, - Set violations, - Map> searchableReplicationFactors, - Map hotNodes, - Map hotCollections, - Map> hotShards, - List hotReplicas) { - // calculate the number of replicas to add to each hot shard, based on how much the rate was - // exceeded - but within limits. - - // first resolve a situation when only a node is hot but no collection / shard is hot - // TODO: eventually we may want to commission a new node - if (!hotNodes.isEmpty()) { - if (hotShards.isEmpty() && hotCollections.isEmpty()) { - // move replicas around - if (aboveNodeOp != null) { - hotNodes.forEach((n, r) -> { - ops.add(new TriggerEvent.Op(aboveNodeOp, Suggester.Hint.SRC_NODE, n)); - violations.add(HOT_NODES); - }); - } - } else { - // ignore - hot shards will result in changes that will change hot node status anyway - } - } - // add replicas - Map>>> hints = new HashMap<>(); - - // HOT COLLECTIONS - // currently we don't do anything for hot collections. Theoretically we could add - // 1 replica more to each shard, based on how close to the threshold each shard is - // but it's probably better to wait for a shard to become hot and be more precise. - - // HOT SHARDS - - hotShards.forEach((coll, shards) -> shards.forEach((s, r) -> { - List> perShard = hints - .computeIfAbsent(coll, c -> new HashMap<>()) - .computeIfAbsent(s, sh -> new ArrayList<>()); - addReplicaHints(coll, s, r, searchableReplicationFactors.get(coll).get(s).get(), perShard); - violations.add(HOT_SHARDS); - })); - - // HOT REPLICAS - // Hot replicas (while their shards are not hot) may be caused by - // dumb clients that use direct replica URLs - this is beyond our control - // so ignore them. - - hints.values().forEach(m -> m.values().forEach(lst -> lst.forEach(p -> { - ops.add(new TriggerEvent.Op(aboveOp, Suggester.Hint.COLL_SHARD, p)); - }))); - - } - - /** - * This method implements a primitive form of proportional controller with a limiter. - */ - @SuppressWarnings({"unchecked", "rawtypes"}) - private void addReplicaHints(String collection, String shard, double r, int replicationFactor, List> hints) { - int numReplicas = (int)Math.round((r - aboveRate) / (double) replicationFactor); - // in one event add at least 1 replica - if (numReplicas < 1) { - numReplicas = 1; - } - // ... and at most maxOps replicas - if (numReplicas > maxOps) { - numReplicas = maxOps; - } - for (int i = 0; i < numReplicas; i++) { - hints.add(new Pair(collection, shard)); - } - } - - private void calculateColdOps(List ops, - Set violations, - ClusterState clusterState, - Map> searchableReplicationFactors, - Map coldNodes, - Map coldCollections, - Map> coldShards, - List coldReplicas) { - // COLD COLLECTIONS - // Probably can't do anything reasonable about whole cold collections - // because they may be needed even if not used. - - // COLD SHARDS & COLD REPLICAS: - // We remove cold replicas only from cold shards, otherwise we are susceptible to uneven - // replica routing (which is beyond our control). - // If we removed replicas from non-cold shards we could accidentally bring that shard into - // the hot range, which would result in adding replica, and that replica could again stay cold due to - // the same routing issue, which then would lead to removing that replica, etc, etc... - - // Remove cold replicas but only when there's at least a minimum number of searchable - // replicas still available (additional non-searchable replicas may exist, too) - // NOTE: do this before adding ops for DELETENODE because we don't want to attempt - // deleting replicas that have been already moved elsewhere - Map>> byCollectionByShard = new HashMap<>(); - coldReplicas.forEach(ri -> { - byCollectionByShard.computeIfAbsent(ri.getCollection(), c -> new HashMap<>()) - .computeIfAbsent(ri.getShard(), s -> new ArrayList<>()) - .add(ri); - }); - coldShards.forEach((coll, perShard) -> { - perShard.forEach((shard, rate) -> { - List replicas = byCollectionByShard - .getOrDefault(coll, Collections.emptyMap()) - .getOrDefault(shard, Collections.emptyList()); - if (replicas.isEmpty()) { - return; - } - // only delete if there's at least minRF searchable replicas left - int rf = searchableReplicationFactors.get(coll).get(shard).get(); - // assume first that we only really need a leader and we may be - // allowed to remove other replicas - int minRF = 1; - // but check the official RF and don't go below that - Integer RF = clusterState.getCollection(coll).getReplicationFactor(); - if (RF != null) { - minRF = RF; - } - // unless minReplicas is set explicitly - if (minReplicas != null) { - minRF = minReplicas; - } - if (minRF < 1) { - minRF = 1; - } - if (rf > minRF) { - // delete at most maxOps replicas at a time - AtomicInteger limit = new AtomicInteger(Math.min(maxOps, rf - minRF)); - replicas.forEach(ri -> { - if (limit.get() == 0) { - return; - } - // don't delete a leader - if (ri.getBool(ZkStateReader.LEADER_PROP, false)) { - return; - } - TriggerEvent.Op op = new TriggerEvent.Op(belowOp, - Suggester.Hint.COLL_SHARD, new Pair<>(ri.getCollection(), ri.getShard())); - op.addHint(Suggester.Hint.REPLICA, ri.getName()); - ops.add(op); - violations.add(COLD_SHARDS); - limit.decrementAndGet(); - }); - } - }); - }); - - // COLD NODES: - // Unlike the case of hot nodes, if a node is cold then any monitored - // collections / shards / replicas located on that node are cold, too. - // HOWEVER, we check only replicas from selected collections / shards, - // so deleting a cold node is dangerous because it may interfere with these - // non-monitored resources - this is the reason the default belowNodeOp is null / ignored. - // - // Also, note that due to the way activity is measured only nodes that contain any - // monitored resources are considered - there may be cold nodes in the cluster that don't - // belong to the monitored collections and they will be ignored. - if (belowNodeOp != null) { - coldNodes.forEach((node, rate) -> { - ops.add(new TriggerEvent.Op(belowNodeOp, Suggester.Hint.SRC_NODE, node)); - violations.add(COLD_NODES); - }); - } - - - } - - private boolean waitForElapsed(String name, long now, Map lastEventMap) { - Long lastTime = lastEventMap.computeIfAbsent(name, s -> now); - long elapsed = TimeUnit.SECONDS.convert(now - lastTime, TimeUnit.NANOSECONDS); - if (log.isTraceEnabled()) { - log.trace("name={}, lastTime={}, elapsed={}, waitFor={}", name, lastTime, elapsed, getWaitForSecond()); - } - if (TimeUnit.SECONDS.convert(now - lastTime, TimeUnit.NANOSECONDS) < getWaitForSecond()) { - return false; - } - return true; - } - - public static class SearchRateEvent extends TriggerEvent { - public SearchRateEvent(String source, long eventTime, List ops, - Map hotNodes, - Map hotCollections, - Map> hotShards, - List hotReplicas, - Map coldNodes, - Map coldCollections, - Map> coldShards, - List coldReplicas, - Set violations) { - super(TriggerEventType.SEARCHRATE, source, eventTime, null); - properties.put(TriggerEvent.REQUESTED_OPS, ops); - properties.put(HOT_NODES, hotNodes); - properties.put(HOT_COLLECTIONS, hotCollections); - properties.put(HOT_SHARDS, hotShards); - properties.put(HOT_REPLICAS, hotReplicas); - properties.put(COLD_NODES, coldNodes); - properties.put(COLD_COLLECTIONS, coldCollections); - properties.put(COLD_SHARDS, coldShards); - properties.put(COLD_REPLICAS, coldReplicas); - properties.put(VIOLATION_PROP, violations); - } - } -} \ No newline at end of file diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/SystemLogListener.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/SystemLogListener.java deleted file mode 100644 index b8414784447..00000000000 --- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/SystemLogListener.java +++ /dev/null @@ -1,221 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.cloud.autoscaling; - -import java.io.PrintWriter; -import java.io.StringWriter; -import java.lang.invoke.MethodHandles; -import java.util.Collection; -import java.util.Date; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.StringJoiner; - -import org.apache.solr.client.solrj.SolrRequest; -import org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig; -import org.apache.solr.client.solrj.cloud.SolrCloudManager; -import org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventProcessorStage; -import org.apache.solr.client.solrj.request.UpdateRequest; -import org.apache.solr.common.SolrException; -import org.apache.solr.common.SolrInputDocument; -import org.apache.solr.common.cloud.ClusterState; -import org.apache.solr.common.cloud.DocCollection; -import org.apache.solr.common.params.CollectionAdminParams; -import org.apache.solr.common.params.CommonParams; -import org.apache.solr.common.params.SolrParams; -import org.apache.solr.common.util.NamedList; -import org.apache.solr.common.util.Utils; -import org.apache.solr.core.SolrResourceLoader; -import org.apache.solr.util.IdUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * This listener saves events to the {@link CollectionAdminParams#SYSTEM_COLL} collection. - *

Configuration properties:

- *
    - *
  • collection - optional string, specifies what collection should be used for storing events. Default value - * is {@link CollectionAdminParams#SYSTEM_COLL}.
    • - *
- */ -public class SystemLogListener extends TriggerListenerBase { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - public static final String SOURCE_FIELD = "source_s"; - public static final String EVENT_SOURCE_FIELD = "event.source_s"; - public static final String EVENT_TYPE_FIELD = "event.type_s"; - public static final String STAGE_FIELD = "stage_s"; - public static final String ACTION_FIELD = "action_s"; - public static final String MESSAGE_FIELD = "message_t"; - public static final String BEFORE_ACTIONS_FIELD = "before.actions_ss"; - public static final String AFTER_ACTIONS_FIELD = "after.actions_ss"; - public static final String COLLECTIONS_FIELD = "collections_ss"; - public static final String SOURCE = SystemLogListener.class.getSimpleName(); - public static final String DOC_TYPE = "autoscaling_event"; - - private String collection = CollectionAdminParams.SYSTEM_COLL; - - @Override - public void configure(SolrResourceLoader loader, SolrCloudManager cloudManager, AutoScalingConfig.TriggerListenerConfig config) throws TriggerValidationException { - super.configure(loader, cloudManager, config); - collection = (String)config.properties.getOrDefault(CollectionAdminParams.COLLECTION, CollectionAdminParams.SYSTEM_COLL); - } - - @Override - @SuppressWarnings({"unchecked", "rawtypes"}) - public void onEvent(TriggerEvent event, TriggerEventProcessorStage stage, String actionName, ActionContext context, - Throwable error, String message) throws Exception { - try { - ClusterState clusterState = cloudManager.getClusterStateProvider().getClusterState(); - DocCollection coll = clusterState.getCollectionOrNull(collection); - if (coll == null) { - log.debug("Collection {} missing, skip sending event {}", collection, event); - return; - } - SolrInputDocument doc = new SolrInputDocument(); - doc.addField(CommonParams.TYPE, DOC_TYPE); - doc.addField(SOURCE_FIELD, SOURCE); - doc.addField("id", IdUtils.timeRandomId()); - doc.addField("event.id_s", event.getId()); - doc.addField(EVENT_TYPE_FIELD, event.getEventType().toString()); - doc.addField(EVENT_SOURCE_FIELD, event.getSource()); - doc.addField("event.time_l", event.getEventTime()); - doc.addField("timestamp", new Date()); - addMap("event.property.", doc, event.getProperties()); - doc.addField(STAGE_FIELD, stage.toString()); - if (actionName != null) { - doc.addField(ACTION_FIELD, actionName); - } - if (message != null) { - doc.addField(MESSAGE_FIELD, message); - } - addError(doc, error); - // add JSON versions of event and context - String eventJson = Utils.toJSONString(event); - doc.addField("event_str", eventJson); - if (context != null) { - // capture specifics of operations after compute_plan action - addOperations(doc, (List)context.getProperties().get("operations")); - // capture specifics of responses after execute_plan action - addResponses(doc, (List>)context.getProperties().get("responses")); - addActions(BEFORE_ACTIONS_FIELD, doc, (List)context.getProperties().get(TriggerEventProcessorStage.BEFORE_ACTION.toString())); - addActions(AFTER_ACTIONS_FIELD, doc, (List)context.getProperties().get(TriggerEventProcessorStage.AFTER_ACTION.toString())); - String contextJson = Utils.toJSONString(context); - doc.addField("context_str", contextJson); - } - UpdateRequest req = new UpdateRequest(); - req.add(doc); - req.setParam(CollectionAdminParams.COLLECTION, collection); - cloudManager.request(req); - } catch (Exception e) { - if ((e instanceof SolrException) && e.getMessage().contains("Collection not found")) { - // relatively benign but log this - collection still existed when we started - log.info("Collection {} missing, skip sending event {}", collection, event); - } else { - log.warn("Exception sending event. Collection: {}, event: {}, exception: {}", collection, event, e); - } - } - } - - private void addActions(String field, SolrInputDocument doc, List actions) { - if (actions == null) { - return; - } - actions.forEach(a -> doc.addField(field, a)); - } - - private void addMap(String prefix, SolrInputDocument doc, Map map) { - map.forEach((k, v) -> { - if (v instanceof Collection) { - for (Object o : (Collection)v) { - doc.addField(prefix + k + "_ss", String.valueOf(o)); - } - } else { - doc.addField(prefix + k + "_ss", String.valueOf(v)); - } - }); - } - - @SuppressWarnings({"rawtypes"}) - private void addOperations(SolrInputDocument doc, List operations) { - if (operations == null || operations.isEmpty()) { - return; - } - Set collections = new HashSet<>(); - for (SolrRequest req : operations) { - SolrParams params = req.getParams(); - if (params == null) { - continue; - } - if (params.get(CollectionAdminParams.COLLECTION) != null) { - collections.add(params.get(CollectionAdminParams.COLLECTION)); - } - // build a whitespace-separated param string - StringJoiner paramJoiner = new StringJoiner(" "); - paramJoiner.setEmptyValue(""); - for (Iterator it = params.getParameterNamesIterator(); it.hasNext(); ) { - final String name = it.next(); - final String [] values = params.getParams(name); - for (String value : values) { - paramJoiner.add(name + "=" + value); - } - } - String paramString = paramJoiner.toString(); - if (!paramString.isEmpty()) { - doc.addField("operations.params_ts", paramString); - } - } - if (!collections.isEmpty()) { - doc.addField(COLLECTIONS_FIELD, collections); - } - } - - private void addResponses(SolrInputDocument doc, List> responses) { - if (responses == null || responses.isEmpty()) { - return; - } - for (NamedList rsp : responses) { - Object o = rsp.get("success"); - if (o != null) { - doc.addField("responses_ts", "success " + o); - } else { - o = rsp.get("failure"); - if (o != null) { - doc.addField("responses_ts", "failure " + o); - } else { // something else - doc.addField("responses_ts", Utils.toJSONString(rsp)); - } - } - } - } - - private void addError(SolrInputDocument doc, Throwable error) { - if (error == null) { - return; - } - StringWriter sw = new StringWriter(); - PrintWriter pw = new PrintWriter(sw); - error.printStackTrace(pw); - pw.flush(); pw.close(); - doc.addField("error.message_t", error.getMessage()); - doc.addField("error.details_t", sw.toString()); - } -} diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/TriggerAction.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/TriggerAction.java deleted file mode 100644 index b873ee61b93..00000000000 --- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/TriggerAction.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.cloud.autoscaling; - -import java.io.Closeable; -import java.util.Map; - -import org.apache.solr.client.solrj.cloud.SolrCloudManager; -import org.apache.solr.core.SolrResourceLoader; - -/** - * Interface for actions performed in response to a trigger being activated - */ -public interface TriggerAction extends Closeable { - - /** - * Called when action is created but before it's initialized and used. - * This method should also verify that the configuration parameters are correct. - * It may be called multiple times. - * @param loader loader to use for instantiating sub-components - * @param cloudManager current instance of SolrCloudManager - * @param properties configuration properties - * @throws TriggerValidationException contains details of invalid configuration parameters. - */ - void configure(SolrResourceLoader loader, SolrCloudManager cloudManager, Map properties) throws TriggerValidationException; - - /** - * Called before an action is first used. Any heavy object creation or initialization should - * be done in this method instead of the constructor or {@link #configure(SolrResourceLoader, SolrCloudManager, Map)} method. - */ - void init() throws Exception; - - String getName(); - - void process(TriggerEvent event, ActionContext context) throws Exception; -} diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/TriggerActionBase.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/TriggerActionBase.java deleted file mode 100644 index 7a9f34b7710..00000000000 --- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/TriggerActionBase.java +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.cloud.autoscaling; - -import java.io.IOException; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; - -import org.apache.solr.client.solrj.cloud.SolrCloudManager; -import org.apache.solr.core.SolrResourceLoader; - -/** - * Base class for {@link TriggerAction} implementations. - */ -public abstract class TriggerActionBase implements TriggerAction { - - protected Map properties = new HashMap<>(); - protected SolrResourceLoader loader; - protected SolrCloudManager cloudManager; - /** - * Set of valid property names. Subclasses may add to this set - * using {@link TriggerUtils#validProperties(Set, String...)} - */ - protected final Set validProperties = new HashSet<>(); - /** - * Set of required property names. Subclasses may add to this set - * using {@link TriggerUtils#requiredProperties(Set, Set, String...)} - * (required properties are also valid properties). - */ - protected final Set requiredProperties = new HashSet<>(); - - protected TriggerActionBase() { - // not strictly needed here because they are already checked during instantiation - TriggerUtils.validProperties(validProperties, "name", "class"); - } - - @Override - public String getName() { - String name = (String) properties.get("name"); - if (name != null) { - return name; - } else { - return getClass().getSimpleName(); - } - } - - @Override - public void close() throws IOException { - - } - - @Override - public void configure(SolrResourceLoader loader, SolrCloudManager cloudManager, Map properties) throws TriggerValidationException { - this.loader = loader; - this.cloudManager = cloudManager; - if (properties != null) { - this.properties.putAll(properties); - } - // validate the config - Map results = new HashMap<>(); - TriggerUtils.checkProperties(this.properties, results, requiredProperties, validProperties); - if (!results.isEmpty()) { - throw new TriggerValidationException(getName(), results); - } - } - - @Override - public void init() throws Exception { - - } -} diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/TriggerActionException.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/TriggerActionException.java deleted file mode 100644 index 624ce68a34b..00000000000 --- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/TriggerActionException.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.cloud.autoscaling; - -/** - * Trigger action-specific exception. - */ -public class TriggerActionException extends Exception { - - public final String triggerName; - public final String actionName; - - public TriggerActionException(String triggerName, String actionName, String message, Throwable cause) { - super(message, cause); - this.triggerName = triggerName; - this.actionName = actionName; - } -} diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/TriggerBase.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/TriggerBase.java deleted file mode 100644 index d045f6ab2fc..00000000000 --- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/TriggerBase.java +++ /dev/null @@ -1,300 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.cloud.autoscaling; - -import java.io.IOException; -import java.lang.invoke.MethodHandles; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.concurrent.atomic.AtomicReference; - -import org.apache.lucene.util.IOUtils; -import org.apache.solr.client.solrj.cloud.autoscaling.AlreadyExistsException; -import org.apache.solr.client.solrj.cloud.autoscaling.BadVersionException; -import org.apache.solr.client.solrj.cloud.DistribStateManager; -import org.apache.solr.client.solrj.cloud.SolrCloudManager; -import org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventType; - -import org.apache.solr.client.solrj.cloud.autoscaling.VersionedData; -import org.apache.solr.common.AlreadyClosedException; -import org.apache.solr.common.cloud.ZkStateReader; -import org.apache.solr.common.util.Utils; -import org.apache.solr.core.SolrResourceLoader; -import org.apache.zookeeper.CreateMode; -import org.apache.zookeeper.KeeperException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Base class for {@link org.apache.solr.cloud.autoscaling.AutoScaling.Trigger} implementations. - * It handles state snapshot / restore in ZK. - */ -public abstract class TriggerBase implements AutoScaling.Trigger { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - protected final String name; - protected SolrCloudManager cloudManager; - protected SolrResourceLoader loader; - protected DistribStateManager stateManager; - protected final Map properties = new HashMap<>(); - /** - * Set of valid property names. Subclasses may add to this set - * using {@link TriggerUtils#validProperties(Set, String...)} - */ - protected final Set validProperties = new HashSet<>(); - /** - * Set of required property names. Subclasses may add to this set - * using {@link TriggerUtils#requiredProperties(Set, Set, String...)} - * (required properties are also valid properties). - */ - protected final Set requiredProperties = new HashSet<>(); - protected final TriggerEventType eventType; - protected int waitForSecond; - protected Map lastState; - protected final AtomicReference processorRef = new AtomicReference<>(); - protected List actions; - protected boolean enabled; - protected boolean isClosed; - - - protected TriggerBase(TriggerEventType eventType, String name) { - this.eventType = eventType; - this.name = name; - - // subclasses may further modify this set to include other supported properties - TriggerUtils.validProperties(validProperties, "name", "class", "event", "enabled", "waitFor", "actions"); - } - - /** - * Return a set of valid property names supported by this trigger. - */ - public final Set getValidProperties() { - return Collections.unmodifiableSet(this.validProperties); - } - - /** - * Return a set of required property names supported by this trigger. - */ - public final Set getRequiredProperties() { - return Collections.unmodifiableSet(this.requiredProperties); - } - - @Override - public void configure(SolrResourceLoader loader, SolrCloudManager cloudManager, Map properties) throws TriggerValidationException { - this.cloudManager = cloudManager; - this.loader = loader; - this.stateManager = cloudManager.getDistribStateManager(); - if (properties != null) { - this.properties.putAll(properties); - } - this.enabled = Boolean.parseBoolean(String.valueOf(this.properties.getOrDefault("enabled", "true"))); - this.waitForSecond = ((Number) this.properties.getOrDefault("waitFor", -1L)).intValue(); - @SuppressWarnings({"unchecked"}) - List> o = (List>) properties.get("actions"); - if (o != null && !o.isEmpty()) { - actions = new ArrayList<>(3); - for (Map map : o) { - TriggerAction action = null; - try { - action = loader.newInstance((String)map.get("class"), TriggerAction.class); - } catch (Exception e) { - throw new TriggerValidationException("action", "exception creating action " + map + ": " + e.toString()); - } - action.configure(loader, cloudManager, map); - actions.add(action); - } - } else { - actions = Collections.emptyList(); - } - - - Map results = new HashMap<>(); - TriggerUtils.checkProperties(this.properties, results, requiredProperties, validProperties); - if (!results.isEmpty()) { - throw new TriggerValidationException(name, results); - } - } - - @Override - public void init() throws Exception { - try { - if (!stateManager.hasData(ZkStateReader.SOLR_AUTOSCALING_TRIGGER_STATE_PATH)) { - stateManager.makePath(ZkStateReader.SOLR_AUTOSCALING_TRIGGER_STATE_PATH); - } - } catch (AlreadyExistsException e) { - // ignore - } catch (InterruptedException | KeeperException | IOException e) { - log.warn("Exception checking ZK path {}", ZkStateReader.SOLR_AUTOSCALING_TRIGGER_STATE_PATH, e); - throw e; - } - for (TriggerAction action : actions) { - action.init(); - } - } - - @Override - public void setProcessor(AutoScaling.TriggerEventProcessor processor) { - processorRef.set(processor); - } - - @Override - public AutoScaling.TriggerEventProcessor getProcessor() { - return processorRef.get(); - } - - @Override - public String getName() { - return name; - } - - @Override - public TriggerEventType getEventType() { - return eventType; - } - - @Override - public boolean isEnabled() { - return enabled; - } - - @Override - public int getWaitForSecond() { - return waitForSecond; - } - - @Override - public Map getProperties() { - return properties; - } - - @Override - public List getActions() { - return actions; - } - - @Override - public boolean isClosed() { - synchronized (this) { - return isClosed; - } - } - - @Override - public void close() throws IOException { - synchronized (this) { - isClosed = true; - IOUtils.closeWhileHandlingException(actions); - } - } - - @Override - public int hashCode() { - return Objects.hash(name, properties); - } - - @Override - public boolean equals(Object obj) { - if (obj == null) { - return false; - } - if (obj.getClass().equals(this.getClass())) { - TriggerBase that = (TriggerBase) obj; - return this.name.equals(that.name) - && this.properties.equals(that.properties); - } - return false; - } - - /** - * Prepare and return internal state of this trigger in a format suitable for persisting in ZK. - * @return map of internal state properties. Note: values must be supported by {@link Utils#toJSON(Object)}. - */ - protected abstract Map getState(); - - /** - * Restore internal state of this trigger from properties retrieved from ZK. - * @param state never null but may be empty. - */ - protected abstract void setState(Map state); - - /** - * Returns an immutable deep copy of this trigger's state, suitible for saving. - * This method is public only for tests that wish to do grey-box introspection - * - * @see #getState - * @lucene.internal - */ - @SuppressWarnings({"unchecked"}) - public Map deepCopyState() { - return Utils.getDeepCopy(getState(), 10, false, true); - } - - @Override - public void saveState() { - Map state = deepCopyState(); - if (lastState != null && lastState.equals(state)) { - // skip saving if identical - return; - } - byte[] data = Utils.toJSON(state); - String path = ZkStateReader.SOLR_AUTOSCALING_TRIGGER_STATE_PATH + "/" + getName(); - try { - if (stateManager.hasData(path)) { - // update - stateManager.setData(path, data, -1); - } else { - // create - stateManager.createData(path, data, CreateMode.PERSISTENT); - } - lastState = state; - } catch (AlreadyExistsException e) { - - } catch (InterruptedException | BadVersionException | IOException | KeeperException e) { - log.warn("Exception updating trigger state '{}'", path, e); - } - } - - @Override - @SuppressWarnings({"unchecked"}) - public void restoreState() { - byte[] data = null; - String path = ZkStateReader.SOLR_AUTOSCALING_TRIGGER_STATE_PATH + "/" + getName(); - try { - if (stateManager.hasData(path)) { - VersionedData versionedData = stateManager.getData(path); - data = versionedData.getData(); - } - } catch (AlreadyClosedException e) { - - } catch (Exception e) { - log.warn("Exception getting trigger state '{}'", path, e); - } - if (data != null) { - Map restoredState = (Map)Utils.fromJSON(data); - // make sure lastState is sorted - restoredState = Utils.getDeepCopy(restoredState, 10, false, true); - setState(restoredState); - lastState = restoredState; - } - } -} diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/TriggerEvent.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/TriggerEvent.java deleted file mode 100644 index 91482e5083c..00000000000 --- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/TriggerEvent.java +++ /dev/null @@ -1,315 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.cloud.autoscaling; - -import java.io.IOException; -import java.util.Collection; -import java.util.Collections; -import java.util.EnumMap; -import java.util.HashMap; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import org.apache.solr.client.solrj.cloud.autoscaling.Suggester; -import org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventType; -import org.apache.solr.common.MapWriter; -import org.apache.solr.common.params.CollectionParams; -import org.apache.solr.common.util.Pair; -import org.apache.solr.common.util.Utils; -import org.apache.solr.util.IdUtils; - -/** - * Trigger event. - */ -public class TriggerEvent implements MapWriter { - public static final String IGNORED = "ignored"; - public static final String COOLDOWN = "cooldown"; - public static final String REPLAYING = "replaying"; - public static final String NODE_NAMES = "nodeNames"; - public static final String EVENT_TIMES = "eventTimes"; - public static final String REQUESTED_OPS = "requestedOps"; - public static final String UNSUPPORTED_OPS = "unsupportedOps"; - - public static final class Op implements MapWriter { - private final CollectionParams.CollectionAction action; - private final EnumMap hints = new EnumMap<>(Suggester.Hint.class); - - public Op(CollectionParams.CollectionAction action) { - this.action = action; - } - - public Op(CollectionParams.CollectionAction action, Suggester.Hint hint, Object hintValue) { - this.action = action; - addHint(hint, hintValue); - } - - @SuppressWarnings({"unchecked"}) - public void addHint(Suggester.Hint hint, Object value) { - hint.validator.accept(value); - if (hint.multiValued) { - Collection values = value instanceof Collection ? (Collection) value : Collections.singletonList(value); - ((Set) hints.computeIfAbsent(hint, h -> new LinkedHashSet<>())).addAll(values); - } else if (value instanceof Map) { - hints.put(hint, value); - } else { - hints.put(hint, value == null ? null : String.valueOf(value)); - } - } - - public CollectionParams.CollectionAction getAction() { - return action; - } - - public EnumMap getHints() { - return hints; - } - - @Override - public void writeMap(EntryWriter ew) throws IOException { - ew.put("action", action); - ew.put("hints", hints); - } - - @SuppressWarnings({"unchecked", "rawtypes"}) - public static Op fromMap(Map map) { - if (!map.containsKey("action")) { - return null; - } - CollectionParams.CollectionAction action = CollectionParams.CollectionAction.get(String.valueOf(map.get("action"))); - if (action == null) { - return null; - } - Op op = new Op(action); - Map hints = (Map)map.get("hints"); - if (hints != null && !hints.isEmpty()) { - hints.forEach((k, v) -> { - Suggester.Hint h = Suggester.Hint.get(k.toString()); - if (h == null) { - return; - } - if (!(v instanceof Collection)) { - v = Collections.singletonList(v); - } - ((Collection)v).forEach(vv -> { - if (vv instanceof Map) { - // maybe it's a Pair? - Map m = (Map)vv; - if (m.containsKey("first") && m.containsKey("second")) { - Pair p = Pair.parse(m); - if (p != null) { - op.addHint(h, p); - return; - } - } - } - op.addHint(h, vv); - }); - }); - } - return op; - } - - @Override - public String toString() { - return "Op{" + - "action=" + action + - ", hints=" + hints + - '}'; - } - } - - protected final String id; - protected final String source; - protected final long eventTime; - protected final TriggerEventType eventType; - protected final Map properties = new HashMap<>(); - protected final boolean ignored; - - public TriggerEvent(TriggerEventType eventType, String source, long eventTime, - Map properties) { - this(IdUtils.timeRandomId(eventTime), eventType, source, eventTime, properties, false); - } - - public TriggerEvent(TriggerEventType eventType, String source, long eventTime, - Map properties, boolean ignored) { - this(IdUtils.timeRandomId(eventTime), eventType, source, eventTime, properties, ignored); - } - - public TriggerEvent(String id, TriggerEventType eventType, String source, long eventTime, - Map properties) { - this(id, eventType, source, eventTime, properties, false); - } - - public TriggerEvent(String id, TriggerEventType eventType, String source, long eventTime, - Map properties, boolean ignored) { - this.id = id; - this.eventType = eventType; - this.source = source; - this.eventTime = eventTime; - if (properties != null) { - this.properties.putAll(properties); - } - this.ignored = ignored; - } - - /** - * Unique event id. - */ - public String getId() { - return id; - } - - /** - * Name of the trigger that fired the event. - */ - public String getSource() { - return source; - } - - /** - * Timestamp of the actual event, in nanoseconds. - * NOTE: this is NOT the timestamp when the event was fired - events may be fired - * much later than the actual condition that generated the event, due to the "waitFor" limit. - */ - public long getEventTime() { - return eventTime; - } - - /** - * Get event properties (modifiable). - */ - public Map getProperties() { - return properties; - } - - /** - * Get a named event property or null if missing. - */ - public Object getProperty(String name) { - return properties.get(name); - } - - /** - * Get a named event property or default value if missing. - */ - public Object getProperty(String name, Object defaultValue) { - Object v = properties.get(name); - if (v == null) { - return defaultValue; - } else { - return v; - } - } - - /** - * Event type. - */ - public TriggerEventType getEventType() { - return eventType; - } - - public boolean isIgnored() { - return ignored; - } - - /** - * Set event properties. - * - * @param properties may be null. A shallow copy of this parameter is used. - */ - public void setProperties(Map properties) { - this.properties.clear(); - if (properties != null) { - this.properties.putAll(properties); - } - } - - @Override - public void writeMap(EntryWriter ew) throws IOException { - ew.put("id", id); - ew.put("source", source); - ew.put("eventTime", eventTime); - ew.put("eventType", eventType.toString()); - ew.put("properties", properties); - if (ignored) { - ew.put("ignored", true); - } - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - - TriggerEvent that = (TriggerEvent) o; - - if (eventTime != that.eventTime) return false; - if (!id.equals(that.id)) return false; - if (!source.equals(that.source)) return false; - if (eventType != that.eventType) return false; - if (ignored != that.ignored) return false; - return properties.equals(that.properties); - } - - @Override - public int hashCode() { - int result = id.hashCode(); - result = 31 * result + source.hashCode(); - result = 31 * result + (int) (eventTime ^ (eventTime >>> 32)); - result = 31 * result + eventType.hashCode(); - result = 31 * result + properties.hashCode(); - result = 31 * result + Boolean.hashCode(ignored); - return result; - } - - @Override - public String toString() { - return Utils.toJSONString(this); - } - - @SuppressWarnings({"unchecked"}) - public static TriggerEvent fromMap(Map map) { - String id = (String)map.get("id"); - String source = (String)map.get("source"); - long eventTime = ((Number)map.get("eventTime")).longValue(); - TriggerEventType eventType = TriggerEventType.valueOf((String)map.get("eventType")); - Map properties = (Map)map.get("properties"); - // properly deserialize some well-known complex properties - fixOps(TriggerEvent.REQUESTED_OPS, properties); - fixOps(TriggerEvent.UNSUPPORTED_OPS, properties); - TriggerEvent res = new TriggerEvent(id, eventType, source, eventTime, properties); - return res; - } - - @SuppressWarnings({"unchecked"}) - public static void fixOps(String type, Map properties) { - List ops = (List)properties.get(type); - if (ops != null && !ops.isEmpty()) { - for (int i = 0; i < ops.size(); i++) { - Object o = ops.get(i); - if (o instanceof Map) { - TriggerEvent.Op op = TriggerEvent.Op.fromMap((Map)o); - if (op != null) { - ops.set(i, op); - } - } - } - } - } -} diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/TriggerEventQueue.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/TriggerEventQueue.java deleted file mode 100644 index ec41495ba80..00000000000 --- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/TriggerEventQueue.java +++ /dev/null @@ -1,121 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.cloud.autoscaling; - -import java.io.IOException; -import java.lang.invoke.MethodHandles; -import java.nio.charset.StandardCharsets; -import java.util.Map; - -import org.apache.solr.client.solrj.cloud.DistributedQueue; -import org.apache.solr.client.solrj.cloud.SolrCloudManager; -import org.apache.solr.cloud.Stats; -import org.apache.solr.common.AlreadyClosedException; -import org.apache.solr.common.cloud.ZkStateReader; -import org.apache.solr.common.util.Utils; -import org.apache.solr.common.util.TimeSource; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * - */ -public class TriggerEventQueue { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - public static final String ENQUEUE_TIME = "_enqueue_time_"; - public static final String DEQUEUE_TIME = "_dequeue_time_"; - - private final String triggerName; - private final TimeSource timeSource; - private final DistributedQueue delegate; - - public TriggerEventQueue(SolrCloudManager cloudManager, String triggerName, Stats stats) throws IOException { - // TODO: collect stats - this.delegate = cloudManager.getDistributedQueueFactory().makeQueue(ZkStateReader.SOLR_AUTOSCALING_EVENTS_PATH + "/" + triggerName); - this.triggerName = triggerName; - this.timeSource = cloudManager.getTimeSource(); - } - - public boolean offerEvent(TriggerEvent event) { - event.getProperties().put(ENQUEUE_TIME, timeSource.getTimeNs()); - try { - byte[] data = Utils.toJSON(event); - delegate.offer(data); - return true; - } catch (Exception e) { - log.warn("Exception adding event {} to queue {}", event, triggerName, e); - return false; - } - } - - public TriggerEvent peekEvent() { - byte[] data; - try { - while ((data = delegate.peek()) != null) { - if (data.length == 0) { - log.warn("ignoring empty data..."); - continue; - } - try { - @SuppressWarnings({"unchecked"}) - Map map = (Map) Utils.fromJSON(data); - return fromMap(map); - } catch (Exception e) { - log.warn("Invalid event data, ignoring: {}", new String(data, StandardCharsets.UTF_8)); - continue; - } - } - } - catch (AlreadyClosedException e) { - - } - catch (Exception e) { - log.warn("Exception peeking queue of trigger {}", triggerName, e); - } - return null; - } - - public TriggerEvent pollEvent() { - byte[] data; - try { - while ((data = delegate.poll()) != null) { - if (data.length == 0) { - log.warn("ignoring empty data..."); - continue; - } - try { - @SuppressWarnings({"unchecked"}) - Map map = (Map) Utils.fromJSON(data); - return fromMap(map); - } catch (Exception e) { - log.warn("Invalid event data, ignoring: {}", new String(data, StandardCharsets.UTF_8)); - continue; - } - } - } catch (Exception e) { - log.warn("Exception polling queue of trigger {}", triggerName, e); - } - return null; - } - - private TriggerEvent fromMap(Map map) { - TriggerEvent res = TriggerEvent.fromMap(map); - res.getProperties().put(DEQUEUE_TIME, timeSource.getTimeNs()); - return res; - } -} diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/TriggerListener.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/TriggerListener.java deleted file mode 100644 index 234387f0875..00000000000 --- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/TriggerListener.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.cloud.autoscaling; - -import java.io.Closeable; - -import org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig; -import org.apache.solr.client.solrj.cloud.SolrCloudManager; -import org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventProcessorStage; -import org.apache.solr.core.SolrResourceLoader; - -/** - * Implementations of this interface are notified of stages in event processing that they were - * registered for. Note: instances may be closed and re-created on each auto-scaling config update. - */ -public interface TriggerListener extends Closeable { - - /** - * Called when listener is created but before it's initialized and used. - * This method should also verify that the configuration parameters are correct. - * It may be called multiple times. - * @param loader loader to use for instantiating sub-components - * @param cloudManager current instance of SolrCloudManager - * @param config coniguration - * @throws TriggerValidationException contains details of invalid configuration parameters. - */ - void configure(SolrResourceLoader loader, SolrCloudManager cloudManager, AutoScalingConfig.TriggerListenerConfig config) throws TriggerValidationException; - - /** - * If this method returns false then the listener's {@link #onEvent(TriggerEvent, TriggerEventProcessorStage, String, ActionContext, Throwable, String)} - * method should not be called. - */ - boolean isEnabled(); - - void init() throws Exception; - - AutoScalingConfig.TriggerListenerConfig getConfig(); - - /** - * This method is called when either a particular stage or - * actionName is reached during event processing. - * @param event current event being processed - * @param stage {@link TriggerEventProcessorStage} that this listener was registered for, or null - * @param actionName {@link TriggerAction} name that this listener was registered for, or null - * @param context optional {@link ActionContext} when the processing stage is related to an action, or null - * @param error optional {@link Throwable} error, or null - * @param message optional message - */ - void onEvent(TriggerEvent event, TriggerEventProcessorStage stage, String actionName, ActionContext context, - Throwable error, String message) throws Exception; -} diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/TriggerListenerBase.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/TriggerListenerBase.java deleted file mode 100644 index 7a323c77391..00000000000 --- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/TriggerListenerBase.java +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.cloud.autoscaling; - -import java.io.IOException; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; - -import org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig; -import org.apache.solr.client.solrj.cloud.SolrCloudManager; -import org.apache.solr.core.SolrResourceLoader; - -/** - * Base class for implementations of {@link TriggerListener}. - */ -public abstract class TriggerListenerBase implements TriggerListener { - - protected AutoScalingConfig.TriggerListenerConfig config; - protected SolrCloudManager cloudManager; - protected SolrResourceLoader loader; - protected boolean enabled; - /** - * Set of valid property names. Subclasses may add to this set - * using {@link TriggerUtils#validProperties(Set, String...)} - */ - protected final Set validProperties = new HashSet<>(); - /** - * Set of required property names. Subclasses may add to this set - * using {@link TriggerUtils#requiredProperties(Set, Set, String...)} - * (required properties are also valid properties). - */ - protected final Set requiredProperties = new HashSet<>(); - /** - * Subclasses can add to this set if they want to allow arbitrary properties that - * start with one of valid prefixes. - */ - protected final Set validPropertyPrefixes = new HashSet<>(); - - protected TriggerListenerBase() { - TriggerUtils.requiredProperties(requiredProperties, validProperties, "trigger"); - TriggerUtils.validProperties(validProperties, "name", "class", "stage", "beforeAction", "afterAction", "enabled"); - } - - @Override - public void configure(SolrResourceLoader loader, SolrCloudManager cloudManager, AutoScalingConfig.TriggerListenerConfig config) throws TriggerValidationException { - this.loader = loader; - this.cloudManager = cloudManager; - this.config = config; - this.enabled = Boolean.parseBoolean(String.valueOf(config.properties.getOrDefault("enabled", true))); - // validate the config - Map results = new HashMap<>(); - // prepare a copy to treat the prefix-based properties - Map propsToCheck = new HashMap<>(config.properties); - propsToCheck.keySet().removeIf(k -> - validPropertyPrefixes.stream().anyMatch(p -> k.startsWith(p))); - TriggerUtils.checkProperties(propsToCheck, results, requiredProperties, validProperties); - if (!results.isEmpty()) { - throw new TriggerValidationException(config.name, results); - } - } - - @Override - public AutoScalingConfig.TriggerListenerConfig getConfig() { - return config; - } - - @Override - public boolean isEnabled() { - return enabled; - } - - @Override - public void init() throws Exception { - - } - - @Override - public void close() throws IOException { - - } -} diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/TriggerUtils.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/TriggerUtils.java deleted file mode 100644 index cecd933fd2f..00000000000 --- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/TriggerUtils.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.cloud.autoscaling; - -import java.util.Arrays; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; - -/** - * - */ -public class TriggerUtils { - // validation helper methods - - public static void requiredProperties(Set required, Set valid, String... propertyNames) { - required.addAll(Arrays.asList(propertyNames)); - valid.addAll(Arrays.asList(propertyNames)); - } - - public static void validProperties(Set valid, String... propertyNames) { - valid.addAll(Arrays.asList(propertyNames)); - } - - public static void checkProperties(Map properties, Map results, Set required, Set valid) { - checkValidPropertyNames(properties, results, valid); - checkRequiredPropertyNames(properties, results, required); - } - - public static void checkValidPropertyNames(Map properties, Map results, Set valid) { - Set currentNames = new HashSet<>(properties.keySet()); - currentNames.removeAll(valid); - if (!currentNames.isEmpty()) { - for (String name : currentNames) { - results.put(name, "unknown property"); - } - } - } - - public static void checkRequiredPropertyNames(Map properties, Map results, Set required) { - Set requiredNames = new HashSet<>(required); - requiredNames.removeAll(properties.keySet()); - if (!requiredNames.isEmpty()) { - for (String name : requiredNames) { - results.put(name, "missing required property"); - } - } - } - - @SuppressWarnings({"unchecked", "rawtypes"}) - public static void checkProperty(Map properties, Map results, String name, boolean required, Class... acceptClasses) { - Object value = properties.get(name); - if (value == null) { - if (required) { - results.put(name, "missing required value"); - } else { - return; - } - } - if (acceptClasses == null || acceptClasses.length == 0) { - return; - } - boolean accepted = false; - for (Class clz : acceptClasses) { - if (clz.isAssignableFrom(value.getClass())) { - accepted = true; - break; - } - } - if (!accepted) { - results.put(name, "value is not an expected type"); - } - } -} diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/TriggerValidationException.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/TriggerValidationException.java deleted file mode 100644 index 648e1e4a779..00000000000 --- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/TriggerValidationException.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.cloud.autoscaling; - -import java.util.HashMap; -import java.util.Map; - -/** - * This class represents errors found when validating trigger configuration. - */ -public class TriggerValidationException extends Exception { - private final Map details = new HashMap<>(); - private final String name; - - /** - * Create an exception. - * @param name name of the trigger / action / listener that caused the exception - * @param details details of invalid configuration - key is a property name, - * value is an error message. - */ - public TriggerValidationException(String name, Map details) { - super(); - this.name = name; - if (details != null) { - this.details.putAll(details); - } - } - - /** - * Create an exception. - * @param name name of the trigger / action / listener that caused the exception - * @param keyValues zero or even number of arguments representing symbolic key - * (eg. property name) and the corresponding validation error message. - */ - public TriggerValidationException(String name, String... keyValues) { - super(); - this.name = name; - if (keyValues == null || keyValues.length == 0) { - return; - } - if (keyValues.length % 2 != 0) { - throw new IllegalArgumentException("number of arguments representing key & value pairs must be even"); - } - for (int i = 0; i < keyValues.length; i += 2) { - details.put(keyValues[i], keyValues[i + 1]); - } - } - - public Map getDetails() { - return details; - } - - @Override - public String toString() { - return "TriggerValidationException{" + - "name=" + name + - ", details='" + details + '\'' + - '}'; - } -} diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/package-info.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/package-info.java deleted file mode 100644 index d3447aa2ca1..00000000000 --- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/package-info.java +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * Package for classes related to autoscaling - */ -package org.apache.solr.cloud.autoscaling; \ No newline at end of file diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/sim/ActionError.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/sim/ActionError.java deleted file mode 100644 index c1c070d923c..00000000000 --- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/sim/ActionError.java +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.cloud.autoscaling.sim; - -/** - * Interface that helps simulating action errors. - */ -public interface ActionError { - boolean shouldFail(String... args); -} diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/sim/FakeDocIterator.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/sim/FakeDocIterator.java deleted file mode 100644 index fbe66aca118..00000000000 --- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/sim/FakeDocIterator.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.cloud.autoscaling.sim; - -import java.util.Iterator; - -import org.apache.solr.common.SolrInputDocument; -import org.apache.solr.common.SolrInputField; - -/** - * Lightweight generator of fake documents - * NOTE: this iterator only ever returns the same document N times, which works ok - * for our "bulk index update" simulation. Obviously don't use this for real indexing. - */ -public class FakeDocIterator implements Iterator { - final SolrInputDocument doc = new SolrInputDocument(); - final SolrInputField idField = new SolrInputField("id"); - - final long start, count; - - long current, max; - - FakeDocIterator(long start, long count) { - this.start = start; - this.count = count; - current = start; - max = start + count; - doc.put("id", idField); - idField.setValue("foo"); - } - - @Override - public boolean hasNext() { - return current < max; - } - - @Override - public SolrInputDocument next() { - current++; - return doc; - } -} diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/sim/GenericDistributedQueue.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/sim/GenericDistributedQueue.java deleted file mode 100644 index 109c516ab26..00000000000 --- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/sim/GenericDistributedQueue.java +++ /dev/null @@ -1,601 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.cloud.autoscaling.sim; - -import java.io.IOException; -import java.lang.invoke.MethodHandles; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.NoSuchElementException; -import java.util.TreeSet; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.locks.Condition; -import java.util.concurrent.locks.ReentrantLock; -import java.util.function.Predicate; - -import com.codahale.metrics.Timer; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; -import org.apache.solr.client.solrj.cloud.DistributedQueue; -import org.apache.solr.client.solrj.cloud.autoscaling.AlreadyExistsException; -import org.apache.solr.client.solrj.cloud.DistribStateManager; -import org.apache.solr.client.solrj.cloud.autoscaling.VersionedData; -import org.apache.solr.cloud.OverseerTaskQueue; -import org.apache.solr.cloud.Stats; -import org.apache.solr.common.SolrException; -import org.apache.solr.common.SolrException.ErrorCode; -import org.apache.solr.common.util.Pair; -import org.apache.zookeeper.CreateMode; -import org.apache.zookeeper.KeeperException; -import org.apache.zookeeper.Op; -import org.apache.zookeeper.WatchedEvent; -import org.apache.zookeeper.Watcher; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * A distributed queue that uses {@link DistribStateManager} as the underlying distributed store. - * Implementation based on {@link org.apache.solr.cloud.ZkDistributedQueue} - */ -public class GenericDistributedQueue implements DistributedQueue { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - static final String PREFIX = "qn-"; - - /** - * Theory of operation: - *

- * Under ordinary circumstances we neither watch nor poll for children in ZK. - * Instead we keep an in-memory list of known child names. When the in-memory - * list is exhausted, we then fetch from ZK. - *

- * We only bother setting a child watcher when the queue has no children in ZK. - */ - private static final Object _IMPLEMENTATION_NOTES = null; - - final String dir; - - final DistribStateManager stateManager; - - final Stats stats; - - /** - * A lock that guards all of the mutable state that follows. - */ - private final ReentrantLock updateLock = new ReentrantLock(); - - /** - * Contains the last set of children fetched from ZK. Elements are removed from the head of - * this in-memory set as they are consumed from the queue. Due to the distributed nature - * of the queue, elements may appear in this set whose underlying nodes have been consumed in ZK. - * Therefore, methods like {@link #peek()} have to double-check actual node existence, and methods - * like {@link #poll()} must resolve any races by attempting to delete the underlying node. - */ - private TreeSet knownChildren = new TreeSet<>(); - - /** - * Used to wait on ZK changes to the child list; you must hold {@link #updateLock} before waiting on this condition. - */ - private final Condition changed = updateLock.newCondition(); - - private boolean isDirty = true; - - private int watcherCount = 0; - - private final int maxQueueSize; - - /** - * If {@link #maxQueueSize} is set, the number of items we can queue without rechecking the server. - */ - private final AtomicInteger offerPermits = new AtomicInteger(0); - - public GenericDistributedQueue(DistribStateManager stateManager, String dir) { - this(stateManager, dir, new Stats()); - } - - public GenericDistributedQueue(DistribStateManager stateManager, String dir, Stats stats) { - this(stateManager, dir, stats, 0); - } - - public GenericDistributedQueue(DistribStateManager stateManager, String dir, Stats stats, int maxQueueSize) { - this.dir = dir; - - try { - if (!stateManager.hasData(dir)) { - try { - stateManager.makePath(dir); - } catch (AlreadyExistsException e) { - // ignore - } - } - } catch (IOException | KeeperException e) { - throw new SolrException(ErrorCode.SERVER_ERROR, e); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new SolrException(ErrorCode.SERVER_ERROR, e); - } - - this.stateManager = stateManager; - this.stats = stats; - this.maxQueueSize = maxQueueSize; - } - - /** - * Returns the data at the first element of the queue, or null if the queue is - * empty. - * - * @return data at the first element of the queue, or null. - */ - @Override - public byte[] peek() throws Exception { - Timer.Context time = stats.time(dir + "_peek"); - try { - return firstElement(); - } finally { - time.stop(); - } - } - - /** - * Returns the data at the first element of the queue, or null if the queue is - * empty and block is false. - * - * @param block if true, blocks until an element enters the queue - * @return data at the first element of the queue, or null. - */ - @Override - public byte[] peek(boolean block) throws Exception { - return block ? peek(Long.MAX_VALUE) : peek(); - } - - /** - * Returns the data at the first element of the queue, or null if the queue is - * empty after wait ms. - * - * @param wait max wait time in ms. - * @return data at the first element of the queue, or null. - */ - @Override - public byte[] peek(long wait) throws Exception { - Preconditions.checkArgument(wait > 0); - Timer.Context time; - if (wait == Long.MAX_VALUE) { - time = stats.time(dir + "_peek_wait_forever"); - } else { - time = stats.time(dir + "_peek_wait" + wait); - } - updateLock.lockInterruptibly(); - try { - long waitNanos = TimeUnit.MILLISECONDS.toNanos(wait); - while (waitNanos > 0) { - byte[] result = firstElement(); - if (result != null) { - return result; - } - waitNanos = changed.awaitNanos(waitNanos); - } - return null; - } finally { - updateLock.unlock(); - time.stop(); - } - } - - /** - * Attempts to remove the head of the queue and return it. Returns null if the - * queue is empty. - * - * @return Head of the queue or null. - */ - @Override - public byte[] poll() throws Exception { - Timer.Context time = stats.time(dir + "_poll"); - try { - return removeFirst(); - } finally { - time.stop(); - } - } - - /** - * Attempts to remove the head of the queue and return it. - * - * @return The former head of the queue - */ - @Override - public byte[] remove() throws Exception { - Timer.Context time = stats.time(dir + "_remove"); - try { - byte[] result = removeFirst(); - if (result == null) { - throw new NoSuchElementException(); - } - return result; - } finally { - time.stop(); - } - } - - public void remove(Collection paths) throws Exception { - if (paths.isEmpty()) return; - List ops = new ArrayList<>(); - for (String path : paths) { - ops.add(Op.delete(dir + "/" + path, -1)); - } - for (int from = 0; from < ops.size(); from += 1000) { - int to = Math.min(from + 1000, ops.size()); - if (from < to) { - try { - stateManager.multi(ops.subList(from, to)); - } catch (NoSuchElementException e) { - // don't know which nodes are not exist, so try to delete one by one node - for (int j = from; j < to; j++) { - try { - stateManager.removeData(ops.get(j).getPath(), -1); - } catch (NoSuchElementException e2) { - if (log.isDebugEnabled()) { - log.debug("Can not remove node which is not exist : {}", ops.get(j).getPath()); - } - } - } - } - } - } - - int cacheSizeBefore = knownChildren.size(); - knownChildren.removeAll(paths); - if (cacheSizeBefore - paths.size() == knownChildren.size() && knownChildren.size() != 0) { - stats.setQueueLength(knownChildren.size()); - } else { - // There are elements get deleted but not present in the cache, - // the cache seems not valid anymore - knownChildren.clear(); - isDirty = true; - } - } - - /** - * Removes the head of the queue and returns it, blocks until it succeeds. - * - * @return The former head of the queue - */ - @Override - public byte[] take() throws Exception { - // Same as for element. Should refactor this. - Timer.Context timer = stats.time(dir + "_take"); - updateLock.lockInterruptibly(); - try { - while (true) { - byte[] result = removeFirst(); - if (result != null) { - return result; - } - changed.await(); - } - } finally { - updateLock.unlock(); - timer.stop(); - } - } - - /** - * Inserts data into queue. If there are no other queue consumers, the offered element - * will be immediately visible when this method returns. - */ - @Override - public void offer(byte[] data) throws Exception { - Timer.Context time = stats.time(dir + "_offer"); - try { - while (true) { - try { - if (maxQueueSize > 0) { - if (offerPermits.get() <= 0 || offerPermits.getAndDecrement() <= 0) { - // If a max queue size is set, check it before creating a new queue item. - if (!stateManager.hasData(dir)) { - // jump to the code below, which tries to create dir if it doesn't exist - throw new NoSuchElementException(); - } - List children = stateManager.listData(dir); - int remainingCapacity = maxQueueSize - children.size(); - if (remainingCapacity <= 0) { - throw new IllegalStateException("queue is full"); - } - - // Allow this client to push up to 1% of the remaining queue capacity without rechecking. - offerPermits.set(remainingCapacity / 100); - } - } - - // Explicitly set isDirty here so that synchronous same-thread calls behave as expected. - // This will get set again when the watcher actually fires, but that's ok. - stateManager.createData(dir + "/" + PREFIX, data, CreateMode.PERSISTENT_SEQUENTIAL); - isDirty = true; - return; - } catch (NoSuchElementException e) { - try { - stateManager.createData(dir, new byte[0], CreateMode.PERSISTENT); - } catch (NoSuchElementException ne) { - // someone created it - } - } - } - } finally { - time.stop(); - } - } - - public Stats getZkStats() { - return stats; - } - - @Override - public Map getStats() { - if (stats == null) { - return Collections.emptyMap(); - } - Map res = new HashMap<>(); - res.put("queueLength", stats.getQueueLength()); - final Map statsMap = new HashMap<>(); - res.put("stats", statsMap); - stats.getStats().forEach((op, stat) -> { - final Map statMap = new HashMap<>(); - statMap.put("success", stat.success.get()); - statMap.put("errors", stat.errors.get()); - final List> failed = new ArrayList<>(stat.failureDetails.size()); - statMap.put("failureDetails", failed); - stat.failureDetails.forEach(failedOp -> { - Map fo = new HashMap<>(); - fo.put("req", failedOp.req); - fo.put("resp", failedOp.resp); - }); - statsMap.put(op, statMap); - }); - return res; - } - - /** - * Returns the name if the first known child node, or {@code null} if the queue is empty. - * This is the only place {@link #knownChildren} is ever updated! - * The caller must double check that the actual node still exists, since the in-memory - * list is inherently stale. - */ - private String firstChild(boolean remove, boolean refetchIfDirty) throws Exception { - updateLock.lockInterruptibly(); - try { - // We always return from cache first, the cache will be cleared if the node is not exist - if (!knownChildren.isEmpty() && !(isDirty && refetchIfDirty)) { - return remove ? knownChildren.pollFirst() : knownChildren.first(); - } - - if (!isDirty && knownChildren.isEmpty()) { - return null; - } - - // Dirty, try to fetch an updated list of children from ZK. - // Only set a new watcher if there isn't already a watcher. - ChildWatcher newWatcher = (watcherCount == 0) ? new ChildWatcher() : null; - knownChildren = fetchZkChildren(newWatcher); - if (newWatcher != null) { - watcherCount++; // watcher was successfully set - } - isDirty = false; - if (knownChildren.isEmpty()) { - return null; - } - changed.signalAll(); - return remove ? knownChildren.pollFirst() : knownChildren.first(); - } finally { - updateLock.unlock(); - } - } - - /** - * Return the current set of children from ZK; does not change internal state. - */ - TreeSet fetchZkChildren(Watcher watcher) throws Exception { - while (true) { - try { - TreeSet orderedChildren = new TreeSet<>(); - - List childNames = stateManager.listData(dir, watcher); - stats.setQueueLength(childNames.size()); - for (String childName : childNames) { - // Check format - if (!childName.regionMatches(0, PREFIX, 0, PREFIX.length())) { - log.debug("Found child node with improper name: {}", childName); - continue; - } - orderedChildren.add(childName); - } - return orderedChildren; - } catch (NoSuchElementException e) { - try { - stateManager.makePath(dir); - } catch (AlreadyExistsException e2) { - // ignore - } - // go back to the loop and try again - } - } - } - - /** - * Return the currently-known set of elements, using child names from memory. If no children are found, or no - * children pass {@code acceptFilter}, waits up to {@code waitMillis} for at least one child to become available. - *

- * Package-private to support {@link OverseerTaskQueue} specifically.

- */ - @Override - public Collection> peekElements(int max, long waitMillis, Predicate acceptFilter) throws Exception { - List foundChildren = new ArrayList<>(); - long waitNanos = TimeUnit.MILLISECONDS.toNanos(waitMillis); - boolean first = true; - while (true) { - // Trigger a refresh, but only force it if this is not the first iteration. - firstChild(false, !first); - - updateLock.lockInterruptibly(); - try { - for (String child : knownChildren) { - if (acceptFilter.test(child)) { - foundChildren.add(child); - } - } - if (!foundChildren.isEmpty()) { - break; - } - if (waitNanos <= 0) { - break; - } - - // If this is our first time through, force a refresh before waiting. - if (first) { - first = false; - continue; - } - - waitNanos = changed.awaitNanos(waitNanos); - } finally { - updateLock.unlock(); - } - - if (!foundChildren.isEmpty()) { - break; - } - } - - // Technically we could restart the method if we fail to actually obtain any valid children - // from ZK, but this is a super rare case, and the latency of the ZK fetches would require - // much more sophisticated waitNanos tracking. - List> result = new ArrayList<>(); - for (String child : foundChildren) { - if (result.size() >= max) { - break; - } - try { - VersionedData data = stateManager.getData(dir + "/" + child); - result.add(new Pair<>(child, data.getData())); - } catch (NoSuchElementException e) { - // Another client deleted the node first, remove the in-memory and continue. - updateLock.lockInterruptibly(); - try { - knownChildren.remove(child); - } finally { - updateLock.unlock(); - } - } - } - return result; - } - - /** - * Return the head of the queue without modifying the queue. - * - * @return the data at the head of the queue. - */ - private byte[] firstElement() throws Exception { - while (true) { - String firstChild = firstChild(false, false); - if (firstChild == null) { - return null; - } - try { - VersionedData data = stateManager.getData(dir + "/" + firstChild); - return data != null ? data.getData() : null; - } catch (NoSuchElementException e) { - // Another client deleted the node first, remove the in-memory and retry. - updateLock.lockInterruptibly(); - try { - // Efficient only for single-consumer - knownChildren.clear(); - isDirty = true; - } finally { - updateLock.unlock(); - } - } - } - } - - private byte[] removeFirst() throws Exception { - while (true) { - String firstChild = firstChild(true, false); - if (firstChild == null) { - return null; - } - try { - String path = dir + "/" + firstChild; - VersionedData result = stateManager.getData(path); - stateManager.removeData(path, -1); - stats.setQueueLength(knownChildren.size()); - return result.getData(); - } catch (NoSuchElementException e) { - // Another client deleted the node first, remove the in-memory and retry. - updateLock.lockInterruptibly(); - try { - // Efficient only for single-consumer - knownChildren.clear(); - isDirty = true; - } finally { - updateLock.unlock(); - } - } - } - } - - @VisibleForTesting int watcherCount() throws InterruptedException { - updateLock.lockInterruptibly(); - try { - return watcherCount; - } finally { - updateLock.unlock(); - } - } - - @VisibleForTesting boolean isDirty() throws InterruptedException { - updateLock.lockInterruptibly(); - try { - return isDirty; - } finally { - updateLock.unlock(); - } - } - - @VisibleForTesting class ChildWatcher implements Watcher { - - @Override - public void process(WatchedEvent event) { - // session events are not change events, and do not remove the watcher; except for Expired - if (Event.EventType.None.equals(event.getType()) && !Event.KeeperState.Expired.equals(event.getState())) { - return; - } - updateLock.lock(); - try { - isDirty = true; - watcherCount--; - // optimistically signal any waiters that the queue may not be empty now, so they can wake up and retry - changed.signalAll(); - } finally { - updateLock.unlock(); - } - } - } -} diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/sim/GenericDistributedQueueFactory.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/sim/GenericDistributedQueueFactory.java deleted file mode 100644 index d4d7e2f3798..00000000000 --- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/sim/GenericDistributedQueueFactory.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.cloud.autoscaling.sim; - -import java.io.IOException; - -import org.apache.solr.client.solrj.cloud.DistributedQueue; -import org.apache.solr.client.solrj.cloud.DistributedQueueFactory; -import org.apache.solr.client.solrj.cloud.DistribStateManager; - -/** - * Factory for {@link GenericDistributedQueue}. - */ -public class GenericDistributedQueueFactory implements DistributedQueueFactory { - - private final DistribStateManager stateManager; - - public GenericDistributedQueueFactory(DistribStateManager stateManager) { - this.stateManager = stateManager; - } - - @Override - public DistributedQueue makeQueue(String path) throws IOException { - return new GenericDistributedQueue(stateManager, path); - } - - @Override - public void removeQueue(String path) throws IOException { - - } -} diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/sim/LiveNodesSet.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/sim/LiveNodesSet.java deleted file mode 100644 index 5f120044dbf..00000000000 --- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/sim/LiveNodesSet.java +++ /dev/null @@ -1,113 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.cloud.autoscaling.sim; - -import java.util.Collection; -import java.util.Collections; -import java.util.Iterator; -import java.util.Set; -import java.util.SortedSet; -import java.util.TreeSet; -import java.util.concurrent.ConcurrentHashMap; - -import org.apache.solr.common.cloud.LiveNodesListener; - -/** - * This class represents a set of live nodes and allows adding listeners to track their state. - */ -public class LiveNodesSet implements Iterable { - - private final Set set = ConcurrentHashMap.newKeySet(); - private final Set listeners = ConcurrentHashMap.newKeySet(); - - public Set get() { - return Collections.unmodifiableSet(set); - } - - public int size() { - return set.size(); - } - - public void registerLiveNodesListener(LiveNodesListener listener) { - listeners.add(listener); - } - - public void removeLiveNodesListener(LiveNodesListener listener) { - listeners.remove(listener); - } - - public void removeAllLiveNodesListeners() { - listeners.clear(); - } - - private void fireListeners(SortedSet oldNodes, SortedSet newNodes) { - for (LiveNodesListener listener : listeners) { - listener.onChange(oldNodes, newNodes); - } - } - - public boolean isEmpty() { - return set.isEmpty(); - } - - public boolean contains(String id) { - return set.contains(id); - } - - public synchronized boolean add(String id) { - if (set.contains(id)) { - return false; - } - TreeSet oldNodes = new TreeSet<>(set); - set.add(id); - TreeSet newNodes = new TreeSet<>(set); - fireListeners(oldNodes, newNodes); - return true; - } - - public synchronized boolean addAll(Collection nodes) { - TreeSet oldNodes = new TreeSet<>(set); - boolean changed = set.addAll(nodes); - TreeSet newNodes = new TreeSet<>(set); - if (changed) { - fireListeners(oldNodes, newNodes); - } - return changed; - } - - public synchronized boolean remove(String id) { - if (!set.contains(id)) { - return false; - } - TreeSet oldNodes = new TreeSet<>(set); - set.remove(id); - TreeSet newNodes = new TreeSet<>(set); - fireListeners(oldNodes, newNodes); - return true; - } - - public synchronized void clear() { - TreeSet oldNodes = new TreeSet<>(set); - set.clear(); - fireListeners(oldNodes, Collections.emptySortedSet()); - } - - @Override - public Iterator iterator() { - return set.iterator(); - } -} \ No newline at end of file diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/sim/NoopDistributedQueueFactory.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/sim/NoopDistributedQueueFactory.java deleted file mode 100644 index b04d38ee2c9..00000000000 --- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/sim/NoopDistributedQueueFactory.java +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.cloud.autoscaling.sim; - -import java.io.IOException; -import java.util.Collection; -import java.util.Collections; -import java.util.Map; -import java.util.function.Predicate; - -import org.apache.solr.client.solrj.cloud.DistributedQueue; -import org.apache.solr.client.solrj.cloud.DistributedQueueFactory; -import org.apache.solr.common.util.Pair; - -/** - * A queue factory implementation that does nothing. - */ -public class NoopDistributedQueueFactory implements DistributedQueueFactory { - - public static final DistributedQueueFactory INSTANCE = new NoopDistributedQueueFactory(); - - @Override - public DistributedQueue makeQueue(String path) throws IOException { - return NoopDistributedQueue.INSTANCE; - } - - @Override - public void removeQueue(String path) throws IOException { - - } - - private static final class NoopDistributedQueue implements DistributedQueue { - static final DistributedQueue INSTANCE = new NoopDistributedQueue(); - - @Override - public byte[] peek() throws Exception { - return new byte[0]; - } - - @Override - public byte[] peek(boolean block) throws Exception { - return new byte[0]; - } - - @Override - public byte[] peek(long wait) throws Exception { - return new byte[0]; - } - - @Override - public byte[] poll() throws Exception { - return new byte[0]; - } - - @Override - public byte[] remove() throws Exception { - return new byte[0]; - } - - @Override - public byte[] take() throws Exception { - return new byte[0]; - } - - @Override - public void offer(byte[] data) throws Exception { - - } - - @Override - public Map getStats() { - return Collections.emptyMap(); - } - - @Override - public Collection> peekElements(int max, long waitMillis, Predicate acceptFilter) throws Exception { - return Collections.emptyList(); - } - } -} diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/sim/SimCloudManager.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/sim/SimCloudManager.java deleted file mode 100644 index b8ac3131dea..00000000000 --- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/sim/SimCloudManager.java +++ /dev/null @@ -1,1060 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.cloud.autoscaling.sim; - -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.IOException; -import java.lang.invoke.MethodHandles; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Random; -import java.util.Set; -import java.util.TreeMap; -import java.util.concurrent.Callable; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentSkipListMap; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicLong; - -import com.codahale.metrics.jvm.ClassLoadingGaugeSet; -import com.codahale.metrics.jvm.GarbageCollectorMetricSet; -import com.codahale.metrics.jvm.MemoryUsageGaugeSet; -import com.codahale.metrics.jvm.ThreadStatesGaugeSet; -import org.apache.solr.client.solrj.SolrClient; -import org.apache.solr.client.solrj.SolrRequest; -import org.apache.solr.client.solrj.SolrResponse; -import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.client.solrj.cloud.DistribStateManager; -import org.apache.solr.client.solrj.cloud.DistributedQueueFactory; -import org.apache.solr.client.solrj.cloud.NodeStateProvider; -import org.apache.solr.client.solrj.cloud.SolrCloudManager; -import org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig; -import org.apache.solr.client.solrj.cloud.autoscaling.Variable; -import org.apache.solr.client.solrj.impl.ClusterStateProvider; -import org.apache.solr.client.solrj.request.AbstractUpdateRequest; -import org.apache.solr.client.solrj.request.CollectionAdminRequest; -import org.apache.solr.client.solrj.request.QueryRequest; -import org.apache.solr.client.solrj.request.RequestWriter; -import org.apache.solr.client.solrj.request.UpdateRequest; -import org.apache.solr.client.solrj.request.V2Request; -import org.apache.solr.client.solrj.response.RequestStatusState; -import org.apache.solr.client.solrj.response.SolrResponseBase; -import org.apache.solr.client.solrj.response.UpdateResponse; -import org.apache.solr.cloud.Overseer; -import org.apache.solr.cloud.autoscaling.AutoScalingHandler; -import org.apache.solr.cloud.autoscaling.OverseerTriggerThread; -import org.apache.solr.common.SolrException; -import org.apache.solr.common.SolrInputDocument; -import org.apache.solr.common.cloud.ClusterState; -import org.apache.solr.common.cloud.Replica; -import org.apache.solr.common.cloud.ZkNodeProps; -import org.apache.solr.common.cloud.ZkStateReader; -import org.apache.solr.common.cloud.rule.ImplicitSnitch; -import org.apache.solr.common.params.CollectionAdminParams; -import org.apache.solr.common.params.CollectionParams; -import org.apache.solr.common.params.CommonAdminParams; -import org.apache.solr.common.params.CommonParams; -import org.apache.solr.common.params.CoreAdminParams; -import org.apache.solr.common.params.ModifiableSolrParams; -import org.apache.solr.common.params.SolrParams; -import org.apache.solr.common.util.ContentStreamBase; -import org.apache.solr.common.util.ExecutorUtil; -import org.apache.solr.common.util.IOUtils; -import org.apache.solr.common.util.NamedList; -import org.apache.solr.common.util.ObjectCache; -import org.apache.solr.common.util.SimpleOrderedMap; -import org.apache.solr.common.util.TimeSource; -import org.apache.solr.core.SolrInfoBean; -import org.apache.solr.core.SolrResourceLoader; -import org.apache.solr.handler.admin.MetricsHandler; -import org.apache.solr.handler.admin.MetricsHistoryHandler; -import org.apache.solr.metrics.AltBufferPoolMetricSet; -import org.apache.solr.metrics.MetricsMap; -import org.apache.solr.metrics.OperatingSystemMetricSet; -import org.apache.solr.metrics.SolrMetricManager; -import org.apache.solr.metrics.SolrMetricsContext; -import org.apache.solr.request.LocalSolrQueryRequest; -import org.apache.solr.response.SolrQueryResponse; -import org.apache.solr.common.util.SolrNamedThreadFactory; -import org.apache.solr.util.MockSearchableSolrClient; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import static org.apache.solr.cloud.api.collections.OverseerCollectionMessageHandler.REQUESTID; - -/** - * Simulated {@link SolrCloudManager}. - */ -public class SimCloudManager implements SolrCloudManager { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - private static final Random random; - - static { - String seed = System.getProperty("tests.seed"); - if (seed == null) { - random = new Random(); - } else { - random = new Random(seed.hashCode()); - } - } - - private final SimDistribStateManager stateManager; - private final SimClusterStateProvider clusterStateProvider; - private final SimNodeStateProvider nodeStateProvider; - private final AutoScalingHandler autoScalingHandler; - private final LiveNodesSet liveNodesSet = new LiveNodesSet(); - private final DistributedQueueFactory queueFactory; - private final ObjectCache objectCache = new ObjectCache(); - private final SolrMetricManager metricManager = new SolrMetricManager(); - private final String metricTag; - - private final List systemColl = Collections.synchronizedList(new ArrayList<>()); - private final Map> eventCounts = new ConcurrentHashMap<>(); - private final MockSearchableSolrClient solrClient; - private final Map opCounts = new ConcurrentSkipListMap<>(); - /** - * @see #submit - * @see #getBackgroundTaskFailureCount - * @see LoggingCallable - */ - private final AtomicLong backgroundTaskFailureCounter = new AtomicLong(0); - - private ExecutorService simCloudManagerPool; - private Overseer.OverseerThread triggerThread; - private ThreadGroup triggerThreadGroup; - private SolrResourceLoader loader; - private MetricsHandler metricsHandler; - private MetricsHistoryHandler metricsHistoryHandler; - private TimeSource timeSource; - private boolean useSystemCollection = true; - - private static int nodeIdPort = 10000; - public static int DEFAULT_FREE_DISK = 10240; // 10 TiB - public static int DEFAULT_TOTAL_DISK = 10240; // 10 TiB - public static long DEFAULT_IDX_SIZE_BYTES = 10240; // 10 kiB - - /** - * Create a simulated cluster. This cluster uses the following components: - *
    - *
  • {@link SimDistribStateManager} with non-shared root node.
    • - *
  • {@link SimClusterStateProvider}
    • - *
  • {@link SimNodeStateProvider}, where node values are automatically initialized when using - * {@link #simAddNode()} method.
    • - *
  • {@link GenericDistributedQueueFactory} that uses {@link SimDistribStateManager} as its storage.
    • - *
  • an instance of {@link AutoScalingHandler} for managing AutoScalingConfig.
    • - *
  • an instance of {@link OverseerTriggerThread} for managing triggers and processing events.
    • - *
- * @param timeSource time source to use. - */ - public SimCloudManager(TimeSource timeSource) throws Exception { - this(timeSource, null); - } - - SimCloudManager(TimeSource timeSource, SimDistribStateManager distribStateManager) throws Exception { - this.loader = new SolrResourceLoader(); - if (distribStateManager == null) { - this.stateManager = new SimDistribStateManager(SimDistribStateManager.createNewRootNode()); - // init common paths - stateManager.makePath(ZkStateReader.CLUSTER_PROPS); - stateManager.makePath(ZkStateReader.SOLR_AUTOSCALING_CONF_PATH); - stateManager.makePath(ZkStateReader.LIVE_NODES_ZKNODE); - stateManager.makePath(ZkStateReader.ROLES); - stateManager.makePath(ZkStateReader.SOLR_AUTOSCALING_EVENTS_PATH); - stateManager.makePath(ZkStateReader.SOLR_AUTOSCALING_TRIGGER_STATE_PATH); - stateManager.makePath(ZkStateReader.SOLR_AUTOSCALING_NODE_LOST_PATH); - stateManager.makePath(ZkStateReader.SOLR_AUTOSCALING_NODE_ADDED_PATH); - stateManager.makePath(Overseer.OVERSEER_ELECT); - } else { - this.stateManager = distribStateManager; - } - - // register common metrics - metricTag = Integer.toHexString(hashCode()); - String registryName = SolrMetricManager.getRegistryName(SolrInfoBean.Group.jvm); - metricManager.registerAll(registryName, new AltBufferPoolMetricSet(), SolrMetricManager.ResolutionStrategy.REPLACE, "buffers"); - metricManager.registerAll(registryName, new ClassLoadingGaugeSet(), SolrMetricManager.ResolutionStrategy.REPLACE, "classes"); - metricManager.registerAll(registryName, new OperatingSystemMetricSet(), SolrMetricManager.ResolutionStrategy.REPLACE, "os"); - metricManager.registerAll(registryName, new GarbageCollectorMetricSet(), SolrMetricManager.ResolutionStrategy.REPLACE, "gc"); - metricManager.registerAll(registryName, new MemoryUsageGaugeSet(), SolrMetricManager.ResolutionStrategy.REPLACE, "memory"); - metricManager.registerAll(registryName, new ThreadStatesGaugeSet(), SolrMetricManager.ResolutionStrategy.REPLACE, "threads"); // todo should we use CachedThreadStatesGaugeSet instead? - MetricsMap sysprops = new MetricsMap((detailed, map) -> { - System.getProperties().forEach((k, v) -> { - map.put(String.valueOf(k), v); - }); - }); - metricManager.registerGauge(null, registryName, sysprops, metricTag, true, "properties", "system"); - - registryName = SolrMetricManager.getRegistryName(SolrInfoBean.Group.node); - metricManager.registerGauge(null, registryName, () -> new File("/").getUsableSpace(), - metricTag, true, "usableSpace", SolrInfoBean.Category.CONTAINER.toString(), "fs", "coreRoot"); - - solrClient = new MockSearchableSolrClient() { - @Override - @SuppressWarnings({"rawtypes"}) - public NamedList request(SolrRequest request, String collection) throws SolrServerException, IOException { - if (collection != null) { - if (request instanceof AbstractUpdateRequest) { - ((AbstractUpdateRequest)request).setParam("collection", collection); - } else if (request instanceof QueryRequest) { - if (request.getPath() != null && ( - request.getPath().startsWith("/admin/autoscaling") || - request.getPath().startsWith("/cluster/autoscaling") || - request.getPath().startsWith("/admin/metrics/history") || - request.getPath().startsWith("/cluster/metrics/history") - )) { - // forward it - ModifiableSolrParams params = new ModifiableSolrParams(request.getParams()); - params.set("collection", collection); - request = new QueryRequest(params); - } else { - // search request - if (collection.equals(CollectionAdminParams.SYSTEM_COLL)) { - return super.request(request, collection); - } else { - // forward it - ModifiableSolrParams params = new ModifiableSolrParams(request.getParams()); - params.set("collection", collection); - request = new QueryRequest(params); - } - } - } else { - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "when collection != null only UpdateRequest and QueryRequest are supported: request=" + request + ", collection=" + collection); - } - } - try { - SolrResponse rsp = SimCloudManager.this.request(request); - return rsp.getResponse(); - } catch (UnsupportedOperationException e) { - throw new SolrServerException(e); - } - } - }; - - - this.timeSource = timeSource != null ? timeSource : TimeSource.NANO_TIME; - this.clusterStateProvider = new SimClusterStateProvider(liveNodesSet, this); - this.nodeStateProvider = new SimNodeStateProvider(liveNodesSet, this.stateManager, this.clusterStateProvider, null); - this.queueFactory = new GenericDistributedQueueFactory(stateManager); - this.simCloudManagerPool = ExecutorUtil.newMDCAwareFixedThreadPool(200, new SolrNamedThreadFactory("simCloudManagerPool")); - - this.autoScalingHandler = new AutoScalingHandler(this, loader); - - - triggerThreadGroup = new ThreadGroup("Simulated Overseer autoscaling triggers"); - OverseerTriggerThread trigger = new OverseerTriggerThread(loader, this); - triggerThread = new Overseer.OverseerThread(triggerThreadGroup, trigger, "Simulated OverseerAutoScalingTriggerThread"); - triggerThread.start(); - } - - // ---------- simulator setup methods ----------- - - /** - * Create a cluster with the specified number of nodes. Node metrics are pre-populated. - * @param numNodes number of nodes to create - * @param timeSource time source - * @return instance of simulated cluster - */ - public static SimCloudManager createCluster(int numNodes, TimeSource timeSource) throws Exception { - SimCloudManager cloudManager = new SimCloudManager(timeSource); - for (int i = 1; i <= numNodes; i++) { - cloudManager.simAddNode(); - } - return cloudManager; - } - - /** - * Create a cluster initialized from the provided cluster state. - * @param initialState existing cluster state - * @param timeSource time source - * @return instance of simulated cluster with the same layout as the provided cluster state. - */ - public static SimCloudManager createCluster(ClusterState initialState, TimeSource timeSource) throws Exception { - SimCloudManager cloudManager = new SimCloudManager(timeSource); - cloudManager.getSimClusterStateProvider().simSetClusterState(initialState); - for (String node : cloudManager.getClusterStateProvider().getLiveNodes()) { - cloudManager.getSimNodeStateProvider().simSetNodeValues(node, createNodeValues(node)); - } - return cloudManager; - } - - public static SimCloudManager createCluster(SolrCloudManager other, AutoScalingConfig config, TimeSource timeSource) throws Exception { - SimDistribStateManager distribStateManager = new SimDistribStateManager(SimDistribStateManager.createNewRootNode()); - distribStateManager.copyFrom(other.getDistribStateManager(), false); - SimCloudManager cloudManager = new SimCloudManager(timeSource, distribStateManager); - if (config != null) { - cloudManager.getSimDistribStateManager().simSetAutoScalingConfig(config); - } else { - config = cloudManager.getDistribStateManager().getAutoScalingConfig(); - } - Set nodeTags = new HashSet<>(SimUtils.COMMON_NODE_TAGS); - nodeTags.addAll(config.getPolicy().getParamNames()); - Set replicaTags = new HashSet<>(SimUtils.COMMON_REPLICA_TAGS); - replicaTags.addAll(config.getPolicy().getPerReplicaAttributes()); - cloudManager.getSimClusterStateProvider().copyFrom(other.getClusterStateProvider()); - for (String node : other.getClusterStateProvider().getLiveNodes()) { - SimClusterStateProvider simClusterStateProvider = cloudManager.getSimClusterStateProvider(); - cloudManager.getSimNodeStateProvider().simSetNodeValues(node, other.getNodeStateProvider().getNodeValues(node, nodeTags)); - Map>> infos = other.getNodeStateProvider().getReplicaInfo(node, replicaTags); - simClusterStateProvider.simSetReplicaValues(node, infos, true); - } - SimUtils.checkConsistency(cloudManager, config); - return cloudManager; - } - - /** - * Create simulated node values (metrics) for a node. - * @param nodeName node name (eg. '127.0.0.1:10000_solr'). If null then a new node name will be - * created using sequentially increasing port number. - * @return node values - */ - public static Map createNodeValues(String nodeName) { - Map values = new HashMap<>(); - String host, nodeId; - int port; - if (nodeName == null) { - host = "127.0.0.1"; - port = nodeIdPort++; - nodeId = host + ":" + port + "_solr"; - values.put("ip_1", "127"); - values.put("ip_2", "0"); - values.put("ip_3", "0"); - values.put("ip_4", "1"); - } else { - String[] hostPortCtx = nodeName.split(":"); - if (hostPortCtx.length != 2) { - throw new RuntimeException("Invalid nodeName " + nodeName); - } - host = hostPortCtx[0]; - String[] portCtx = hostPortCtx[1].split("_"); - if (portCtx.length != 2) { - throw new RuntimeException("Invalid port_context in nodeName " + nodeName); - } - port = Integer.parseInt(portCtx[0]); - nodeId = host + ":" + port + "_" + portCtx[1]; - String[] ip = host.split("\\."); - if (ip.length == 4) { - values.put("ip_1", ip[0]); - values.put("ip_2", ip[1]); - values.put("ip_3", ip[2]); - values.put("ip_4", ip[3]); - } - } - values.put(ImplicitSnitch.HOST, host); - values.put(ImplicitSnitch.PORT, port); - values.put(ImplicitSnitch.NODE, nodeId); - values.put(ImplicitSnitch.CORES, 0); - values.put(ImplicitSnitch.DISK, DEFAULT_FREE_DISK); - values.put(Variable.Type.TOTALDISK.tagName, DEFAULT_TOTAL_DISK); - values.put(ImplicitSnitch.SYSLOADAVG, 1.0); - values.put(ImplicitSnitch.HEAPUSAGE, 123450000); - values.put("sysprop.java.version", System.getProperty("java.version")); - values.put("sysprop.java.vendor", System.getProperty("java.vendor")); - // fake some metrics expected in tests - values.put("metrics:solr.node:ADMIN./admin/authorization.clientErrors:count", 0); - values.put("metrics:solr.jvm:buffers.direct.Count", 0); - return values; - } - - public void disableMetricsHistory() { - metricsHistoryHandler.close(); - } - - public String dumpClusterState(boolean withCollections) throws Exception { - StringBuilder sb = new StringBuilder(); - sb.append("#######################################\n"); - sb.append("############ CLUSTER STATE ############\n"); - sb.append("#######################################\n"); - sb.append("## Live nodes:\t\t").append(getLiveNodesSet().size()).append("\n"); - int emptyNodes = 0; - int maxReplicas = 0; - int minReplicas = Integer.MAX_VALUE; - Map> replicaStates = new TreeMap<>(); - int numReplicas = 0; - for (String node : getLiveNodesSet().get()) { - List replicas = getSimClusterStateProvider().simGetReplicaInfos(node); - numReplicas += replicas.size(); - if (replicas.size() > maxReplicas) { - maxReplicas = replicas.size(); - } - if (minReplicas > replicas.size()) { - minReplicas = replicas.size(); - } - for (Replica ri : replicas) { - replicaStates.computeIfAbsent(ri.getCollection(), c -> new TreeMap<>()) - .computeIfAbsent(ri.getState(), s -> new AtomicInteger()) - .incrementAndGet(); - } - if (replicas.isEmpty()) { - emptyNodes++; - } - } - if (minReplicas == Integer.MAX_VALUE) { - minReplicas = 0; - } - sb.append("## Empty nodes:\t").append(emptyNodes).append("\n"); - Set deadNodes = getSimNodeStateProvider().simGetDeadNodes(); - sb.append("## Dead nodes:\t\t").append(deadNodes.size()).append("\n"); - deadNodes.forEach(n -> sb.append("##\t\t").append(n).append("\n")); - sb.append("## Collections:\n"); - clusterStateProvider.simGetCollectionStats().forEach((coll, stats) -> { - sb.append("## * ").append(coll).append('\n'); - stats.forEach((k, v) -> { - sb.append("## ").append(k).append("\t").append(v).append("\n"); - }); - }); - if (withCollections) { - ClusterState state = clusterStateProvider.getClusterState(); - state.forEachCollection(coll -> sb.append(coll.toString()).append("\n")); - } - sb.append("## Max replicas per node:\t").append(maxReplicas).append("\n"); - sb.append("## Min replicas per node:\t").append(minReplicas).append("\n"); - sb.append("## Total replicas:\t\t").append(numReplicas).append("\n"); - replicaStates.forEach((c, map) -> { - AtomicInteger repCnt = new AtomicInteger(); - map.forEach((s, cnt) -> repCnt.addAndGet(cnt.get())); - sb.append("## * ").append(c).append("\t\t").append(repCnt.get()).append("\n"); - map.forEach((s, cnt) -> sb.append("##\t\t- ").append(String.format(Locale.ROOT, "%-12s %4d", s, cnt.get())).append("\n")); - }); - sb.append("######### Solr op counts ##########\n"); - simGetOpCounts().forEach((k, cnt) -> sb.append("##\t\t- ").append(String.format(Locale.ROOT, "%-14s %4d", k, cnt.get())).append("\n")); - sb.append("######### Autoscaling event counts ###########\n"); - Map> counts = simGetEventCounts(); - counts.forEach((trigger, map) -> { - sb.append("## * Trigger: ").append(trigger).append("\n"); - map.forEach((s, cnt) -> sb.append("##\t\t- ").append(String.format(Locale.ROOT, "%-11s %4d", s, cnt.get())).append("\n")); - }); - return sb.toString(); - } - - /** - * Get the instance of {@link SolrResourceLoader} that is used by the cluster components. - */ - public SolrResourceLoader getLoader() { - return loader; - } - - /** - * Get the source of randomness (usually initialized by the test suite). - */ - public Random getRandom() { - return random; - } - - /** - * Add a new node and initialize its node values (metrics). The - * /live_nodes list is updated with the new node id. - * @return new node id - */ - public String simAddNode() throws Exception { - Map values = createNodeValues(null); - String nodeId = (String)values.get(ImplicitSnitch.NODE); - nodeStateProvider.simSetNodeValues(nodeId, values); - clusterStateProvider.simAddNode(nodeId); - log.trace("-- added node {}", nodeId); - // initialize history handler if this is the first node - if (metricsHistoryHandler == null && liveNodesSet.size() == 1) { - metricsHandler = new MetricsHandler(metricManager); - metricsHistoryHandler = new MetricsHistoryHandler(nodeId, metricsHandler, solrClient, this, new HashMap<>()); - SolrMetricsContext solrMetricsContext = new SolrMetricsContext(metricManager, SolrMetricManager.getRegistryName(SolrInfoBean.Group.node), metricTag); - metricsHistoryHandler.initializeMetrics(solrMetricsContext, CommonParams.METRICS_HISTORY_PATH); - } - return nodeId; - } - - /** - * Remove a node from the cluster. This simulates a node lost scenario. - * Node id is removed from the /live_nodes list. - * @param nodeId node id - * @param withValues when true, remove also simulated node values. If false - * then node values are retained to later simulate - * a node that comes back up - */ - public void simRemoveNode(String nodeId, boolean withValues) throws Exception { - clusterStateProvider.simRemoveNode(nodeId); - if (withValues) { - nodeStateProvider.simRemoveNodeValues(nodeId); - } - if (liveNodesSet.isEmpty()) { - // remove handlers - if (metricsHistoryHandler != null) { - IOUtils.closeQuietly(metricsHistoryHandler); - metricsHistoryHandler = null; - } - if (metricsHandler != null) { - metricsHandler = null; - } - } - log.trace("-- removed node {}", nodeId); - } - - /** - * Remove a number of randomly selected nodes - * @param number number of nodes to remove - * @param withValues when true, remove also simulated node values. If false - * then node values are retained to later simulate - * a node that comes back up - * @param random random - */ - public void simRemoveRandomNodes(int number, boolean withValues, Random random) throws Exception { - List nodes = new ArrayList<>(liveNodesSet.get()); - Collections.shuffle(nodes, random); - int count = Math.min(number, nodes.size()); - for (int i = 0; i < count; i++) { - simRemoveNode(nodes.get(i), withValues); - } - } - - public void simSetUseSystemCollection(boolean useSystemCollection) { - this.useSystemCollection = useSystemCollection; - } - - /** - * Clear the (simulated) .system collection. - */ - public void simClearSystemCollection() { - systemColl.clear(); - } - - /** - * Get the content of (simulated) .system collection. - * @return documents in the collection, in chronological order starting from the oldest. - */ - public List simGetSystemCollection() { - return systemColl; - } - - public Map> simGetEventCounts() { - TreeMap> counts = new TreeMap<>(eventCounts); - return counts; - } - - /** - * Get a {@link SolrClient} implementation where calls are forwarded to this - * instance of the cluster. - * @return simulated SolrClient. - */ - public SolrClient simGetSolrClient() { - return solrClient; -// return new SolrClient() { -// @Override -// public NamedList request(SolrRequest request, String collection) throws SolrServerException, IOException { -// if (collection != null) { -// if (request instanceof AbstractUpdateRequest) { -// ((AbstractUpdateRequest)request).setParam("collection", collection); -// } else if (request instanceof QueryRequest) { -// ModifiableSolrParams params = new ModifiableSolrParams(request.getParams()); -// params.set("collection", collection); -// request = new QueryRequest(params); -// } else { -// throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "when collection != null only UpdateRequest and QueryRequest are supported: request=" + request + ", collection=" + collection); -// } -// } -// SolrResponse rsp = SimCloudManager.this.request(request); -// return rsp.getResponse(); -// } -// -// @Override -// public void close() throws IOException { -// -// } -// }; - } - - /** - * Simulate the effect of restarting Overseer leader - in this case this means closing the current - * {@link OverseerTriggerThread} (and optionally killing a node) then starting a new - * {@link OverseerTriggerThread}. - * All background tasks currently in progress will be interrupted. - * @param killNodeId optional nodeId to kill. If null then don't kill any node, just restart the thread - * @see #getOverseerTriggerThread - */ - public void simRestartOverseer(String killNodeId) throws Exception { - log.info("=== Restarting OverseerTriggerThread and clearing object cache..."); - triggerThread.interrupt(); - IOUtils.closeQuietly(triggerThread); - if (killNodeId != null) { - log.info(" = killing node {}", killNodeId); - simRemoveNode(killNodeId, false); - } - objectCache.clear(); - - try { - simCloudManagerPool.shutdownNow(); - } catch (Exception e) { - // ignore - } - simCloudManagerPool = ExecutorUtil.newMDCAwareFixedThreadPool(200, new SolrNamedThreadFactory("simCloudManagerPool")); - - OverseerTriggerThread trigger = new OverseerTriggerThread(loader, this); - triggerThread = new Overseer.OverseerThread(triggerThreadGroup, trigger, "Simulated OverseerAutoScalingTriggerThread"); - triggerThread.start(); - - } - - /** - * Submit a task to execute in a thread pool. - * Every callable submitted will be wrapped such that errors not handled w/in the callable - * will be logged and counted for later assertions. - * - * @param callable task to execute - * @return future to obtain results - * @see #getBackgroundTaskFailureCount - */ - @SuppressWarnings({"unchecked", "rawtypes"}) - public Future submit(Callable callable) { - return simCloudManagerPool.submit(new LoggingCallable(backgroundTaskFailureCounter, callable)); - } - /** - * Returns a total count of the number of tasks submitted to {@link #submit} that have failed - * with any throwable other then InteruptedException - * - * @see #submit - */ - public long getBackgroundTaskFailureCount() { - return backgroundTaskFailureCounter.get(); - } - - // ---------- type-safe methods to obtain simulator components ---------- - public SimClusterStateProvider getSimClusterStateProvider() { - return clusterStateProvider; - } - - public SimNodeStateProvider getSimNodeStateProvider() { - return nodeStateProvider; - } - - public SimDistribStateManager getSimDistribStateManager() { - return stateManager; - } - - public LiveNodesSet getLiveNodesSet() { - return liveNodesSet; - } - - /** - * Get the number and type of operations processed by this cluster. - */ - public Map simGetOpCounts() { - return opCounts; - } - - public void simResetOpCounts() { - opCounts.clear(); - } - - /** - * Get the number of processed operations of a specified type. - * @param op operation name, eg. MOVEREPLICA - * @return number of operations - */ - public long simGetOpCount(String op) { - AtomicLong count = opCounts.get(op); - return count != null ? count.get() : 0L; - } - - public SolrMetricManager getMetricManager() { - return metricManager; - } - - // --------- interface methods ----------- - - - @Override - public ObjectCache getObjectCache() { - return objectCache; - } - - @Override - public TimeSource getTimeSource() { - return timeSource; - } - - @Override - public ClusterStateProvider getClusterStateProvider() { - return clusterStateProvider; - } - - @Override - public NodeStateProvider getNodeStateProvider() { - return nodeStateProvider; - } - - @Override - public DistribStateManager getDistribStateManager() { - return stateManager; - } - - @Override - public DistributedQueueFactory getDistributedQueueFactory() { - return queueFactory; - } - - @Override - @SuppressWarnings({"rawtypes"}) - public SolrResponse request(SolrRequest req) throws IOException { - try { - // NOTE: we're doing 2 odd things here: - // 1) rather then calling simHandleSolrRequest directly, we're submitting it to the - // executor service and immediately waiting on the Future. - // - This can introduce a delays if there are a lot of existing background tasks submitted - // 2) we use simCloudManagerPool directly, instead of using the public submit() method - // - this is because there may be "user level" errors (ie: bad input) deliberately generated - // by the testcase. we're going to immediately catch & re-throw any exceptions, so we don't - // need/want to be wrapped in a LoggingCallable w/getBackgroundTaskFailureCount() tracking - Future rsp = simCloudManagerPool.submit(() -> simHandleSolrRequest(req)); - return rsp.get(120, TimeUnit.SECONDS); // longer then this and something is seriously wrong - } catch (Exception e) { - throw new IOException(e); - } - } - - private void incrementCount(String op) { - AtomicLong count = opCounts.computeIfAbsent(op, o -> new AtomicLong()); - count.incrementAndGet(); - } - - /** - * Handler method for autoscaling requests. NOTE: only a specific subset of autoscaling requests is - * supported! - * @param req autoscaling request - * @return results - */ - - @SuppressWarnings({"unchecked", "rawtypes"}) - public SolrResponse simHandleSolrRequest(SolrRequest req) throws IOException, InterruptedException { - // pay the penalty for remote request, at least 5 ms - timeSource.sleep(5); - - if (log.isTraceEnabled()) { - log.trace("--- got SolrRequest: {} {} {}", req.getMethod(), req.getPath(), - (req.getParams() != null ? " " + req.getParams() : "")); // logOk - } - if (req.getPath() != null) { - if (req.getPath().startsWith("/admin/autoscaling") || - req.getPath().startsWith("/cluster/autoscaling") || - req.getPath().startsWith("/admin/metrics") || - req.getPath().startsWith("/cluster/metrics") - ) { - metricManager.registry("solr.node").counter("ADMIN." + req.getPath() + ".requests").inc(); - boolean autoscaling = req.getPath().contains("autoscaling"); - boolean history = req.getPath().contains("history"); - if (autoscaling) { - incrementCount("autoscaling"); - } else if (history) { - incrementCount("metricsHistory"); - } else { - incrementCount("metrics"); - } - ModifiableSolrParams params = new ModifiableSolrParams(req.getParams()); - params.set(CommonParams.PATH, req.getPath()); - LocalSolrQueryRequest queryRequest = new LocalSolrQueryRequest(null, params); - if (autoscaling) { - RequestWriter.ContentWriter cw = req.getContentWriter("application/json"); - if (null != cw) { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - cw.write(baos); - String payload = baos.toString("UTF-8"); - log.trace("-- payload: {}", payload); - queryRequest.setContentStreams(Collections.singletonList(new ContentStreamBase.StringStream(payload))); - } - } - queryRequest.getContext().put("httpMethod", req.getMethod().toString()); - SolrQueryResponse queryResponse = new SolrQueryResponse(); - queryResponse.addResponseHeader(new SimpleOrderedMap<>()); - if (autoscaling) { - autoScalingHandler.handleRequest(queryRequest, queryResponse); - } else { - if (history) { - if (metricsHistoryHandler != null) { - metricsHistoryHandler.handleRequest(queryRequest, queryResponse); - } else { - queryRequest.close(); - throw new UnsupportedOperationException("must add at least 1 node first"); - } - } else { - if (metricsHandler != null) { - metricsHandler.handleRequest(queryRequest, queryResponse); - } else { - queryRequest.close(); - throw new UnsupportedOperationException("must add at least 1 node first"); - } - } - } - if (queryResponse.getException() != null) { - if (log.isDebugEnabled()) { - log.debug("-- exception handling request", queryResponse.getException()); - } - throw new IOException(queryResponse.getException()); - } - SolrResponse rsp = new SolrResponseBase(); - rsp.setResponse(queryResponse.getValues()); - log.trace("-- response: {}", rsp); - return rsp; - } else if (req instanceof QueryRequest) { - incrementCount("query"); - return clusterStateProvider.simQuery((QueryRequest)req); - } - } - if (req instanceof UpdateRequest) { - incrementCount("update"); - UpdateRequest ureq = (UpdateRequest)req; - String collection = ureq.getCollection(); - UpdateResponse rsp = clusterStateProvider.simUpdate(ureq); - if (collection == null || collection.equals(CollectionAdminParams.SYSTEM_COLL)) { - List docs = ureq.getDocuments(); - if (docs != null) { - if (useSystemCollection) { - systemColl.addAll(docs); - } - for (SolrInputDocument d : docs) { - if (!"autoscaling_event".equals(d.getFieldValue("type"))) { - continue; - } - eventCounts.computeIfAbsent((String)d.getFieldValue("event.source_s"), s -> new ConcurrentHashMap<>()) - .computeIfAbsent((String)d.getFieldValue("stage_s"), s -> new AtomicInteger()) - .incrementAndGet(); - } - } - return new UpdateResponse(); - } else { - return rsp; - } - } - // support only a specific subset of collection admin ops - SolrParams params = req.getParams(); - String a = params != null ? params.get(CoreAdminParams.ACTION) : null; - SolrResponse rsp = new SolrResponseBase(); - rsp.setResponse(new NamedList<>()); - String path = params != null ? params.get("path") : null; - if (!(req instanceof CollectionAdminRequest)) { - // maybe a V2Request? - if (req instanceof V2Request) { - params = SimUtils.v2AdminRequestToV1Params((V2Request)req); - a = params.get(CoreAdminParams.ACTION); - } else if (path != null && (path.startsWith("/admin/") || path.startsWith("/cluster/"))) { - // pass it through, it's likely a generic request containing admin params - } else { - throw new UnsupportedOperationException("Only some CollectionAdminRequest-s are supported: " + req.getClass().getName() + ": " + req.getPath() + " " + req.getParams()); - } - } - metricManager.registry("solr.node").counter("ADMIN." + req.getPath() + ".requests").inc(); - if (a != null) { - CollectionParams.CollectionAction action = CollectionParams.CollectionAction.get(a); - if (action == null) { - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Unknown action: " + a); - } - if (log.isTraceEnabled()) { - log.trace("Invoking Collection Action :{} with params {}", action.toLower(), params.toQueryString()); - } - @SuppressWarnings({"rawtypes"}) - NamedList results = new NamedList(); - rsp.setResponse(results); - incrementCount(action.name()); - switch (action) { - case REQUESTSTATUS: - // we complete all async ops immediately - String requestId = params.get(REQUESTID); - SimpleOrderedMap status = new SimpleOrderedMap<>(); - status.add("state", RequestStatusState.COMPLETED.getKey()); - status.add("msg", "found [" + requestId + "] in completed tasks"); - results.add("status", status); - results.add("success", ""); - // ExecutePlanAction expects a specific response class - rsp = new CollectionAdminRequest.RequestStatusResponse(); - rsp.setResponse(results); - break; - case DELETESTATUS: - requestId = params.get(REQUESTID); - results.add("status", "successfully removed stored response for [" + requestId + "]"); - results.add("success", ""); - break; - case CREATE: - try { - clusterStateProvider.simCreateCollection(new ZkNodeProps(params.toNamedList().asMap(10)), results); - } catch (Exception e) { - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e); - } - break; - case DELETE: - try { - clusterStateProvider.simDeleteCollection(params.get(CommonParams.NAME), - params.get(CommonAdminParams.ASYNC), results); - } catch (Exception e) { - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e); - } - break; - case LIST: - results.add("collections", clusterStateProvider.simListCollections()); - break; - case ADDREPLICA: - try { - clusterStateProvider.simAddReplica(new ZkNodeProps(params.toNamedList().asMap(10)), results); - } catch (Exception e) { - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e); - } - break; - case MOVEREPLICA: - try { - clusterStateProvider.simMoveReplica(new ZkNodeProps(params.toNamedList().asMap(10)), results); - } catch (Exception e) { - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e); - } - break; - case OVERSEERSTATUS: - if (params.get(CommonAdminParams.ASYNC) != null) { - results.add(REQUESTID, params.get(CommonAdminParams.ASYNC)); - } - if (!liveNodesSet.get().isEmpty()) { - results.add("leader", liveNodesSet.get().iterator().next()); - } - results.add("overseer_queue_size", 0); - results.add("overseer_work_queue_size", 0); - results.add("overseer_collection_queue_size", 0); - results.add("success", ""); - break; - case ADDROLE: - nodeStateProvider.simSetNodeValue(params.get("node"), "nodeRole", params.get("role")); - break; - case CREATESHARD: - try { - clusterStateProvider.simCreateShard(new ZkNodeProps(params.toNamedList().asMap(10)), results); - } catch (Exception e) { - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e); - } - break; - case SPLITSHARD: - try { - clusterStateProvider.simSplitShard(new ZkNodeProps(params.toNamedList().asMap(10)), results); - } catch (Exception e) { - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e); - } - break; - case DELETESHARD: - try { - clusterStateProvider.simDeleteShard(new ZkNodeProps(params.toNamedList().asMap(10)), results); - } catch (Exception e) { - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e); - } - break; - default: - throw new UnsupportedOperationException("Unsupported collection admin action=" + action + " in request: " + params); - } - } else { - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "action is a required param in request: " + params); - } - return rsp; - - } - - /** - * HTTP requests are not supported by this implementation. - */ - @Override - public byte[] httpRequest(String url, SolrRequest.METHOD method, Map headers, String payload, int timeout, boolean followRedirects) throws IOException { - throw new UnsupportedOperationException("general HTTP requests are not supported yet"); - } - - @Override - public void close() throws IOException { - // make sure we shutdown the pool first, so any in active background tasks get interupted - // before we start closing resources they may be using. - simCloudManagerPool.shutdownNow(); - - if (metricsHistoryHandler != null) { - IOUtils.closeQuietly(metricsHistoryHandler); - } - IOUtils.closeQuietly(clusterStateProvider); - IOUtils.closeQuietly(nodeStateProvider); - IOUtils.closeQuietly(stateManager); - triggerThread.interrupt(); - IOUtils.closeQuietly(triggerThread); - triggerThread.interrupt(); - try { - triggerThread.join(); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - IOUtils.closeQuietly(objectCache); - } - - /** - * Direct access to the current {@link OverseerTriggerThread} - * @see #simRestartOverseer - */ - public OverseerTriggerThread getOverseerTriggerThread() { - return ((OverseerTriggerThread) triggerThread.getThread()); - } - - /** - * Wrapper for any Callable that will log a warn/error in the event of InterruptException/Throwable. - * Also increments the passed in counter so the CloudManger can later report total errors programatically. - * - * @see #submit - * @see #getBackgroundTaskFailureCount - */ - private static final class LoggingCallable implements Callable { - - final AtomicLong failCounter; - final Callable inner; - - public LoggingCallable(final AtomicLong failCounter, final Callable inner) { - assert null != failCounter; - assert null != inner; - this.failCounter = failCounter; - this.inner = inner; - } - - public T call() throws Exception { - try { - return inner.call(); - } catch (InterruptedException ignored) { - log.warn("Callable interupted", ignored); - throw ignored; - } catch (Throwable t) { - // be forgiving of errors that occured as a result of interuption, even if - // the inner Callable didn't realize it... - if (Thread.currentThread().isInterrupted()) { - log.warn("Callable interrupted w/o noticing", t); - throw t; - } - Throwable cause = t; - while ((cause = cause.getCause()) != null) { - if (cause instanceof InterruptedException) { - log.warn("Callable threw wrapped InterruptedException", t); - throw t; - } - } - - // in all other situations, this is a problem that should be tracked in the failCounter - failCounter.incrementAndGet(); - log.error("Callable failed", t); - throw t; - } - } - } -} diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/sim/SimClusterStateProvider.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/sim/SimClusterStateProvider.java deleted file mode 100644 index 19866d6d7bc..00000000000 --- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/sim/SimClusterStateProvider.java +++ /dev/null @@ -1,2574 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.cloud.autoscaling.sim; - -import java.io.IOException; -import java.lang.invoke.MethodHandles; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.EnumMap; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.NoSuchElementException; -import java.util.Random; -import java.util.Set; -import java.util.TreeMap; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.atomic.AtomicReference; -import java.util.concurrent.locks.ReentrantLock; -import java.util.stream.Collectors; - -import com.google.common.util.concurrent.AtomicDouble; -import org.apache.commons.math3.stat.descriptive.SummaryStatistics; -import org.apache.solr.client.solrj.cloud.DistribStateManager; -import org.apache.solr.client.solrj.cloud.autoscaling.AlreadyExistsException; -import org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig; -import org.apache.solr.client.solrj.cloud.autoscaling.BadVersionException; -import org.apache.solr.client.solrj.cloud.autoscaling.Policy; -import org.apache.solr.client.solrj.cloud.autoscaling.PolicyHelper; -import org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventType; -import org.apache.solr.client.solrj.cloud.autoscaling.Variable; -import org.apache.solr.client.solrj.cloud.autoscaling.Variable.Type; -import org.apache.solr.client.solrj.cloud.autoscaling.VersionedData; -import org.apache.solr.client.solrj.impl.ClusterStateProvider; -import org.apache.solr.client.solrj.request.QueryRequest; -import org.apache.solr.client.solrj.request.UpdateRequest; -import org.apache.solr.client.solrj.response.QueryResponse; -import org.apache.solr.client.solrj.response.UpdateResponse; -import org.apache.solr.cloud.ActionThrottle; -import org.apache.solr.cloud.CloudUtil; -import org.apache.solr.cloud.Overseer; -import org.apache.solr.cloud.api.collections.AddReplicaCmd; -import org.apache.solr.cloud.api.collections.Assign; -import org.apache.solr.cloud.api.collections.CreateCollectionCmd; -import org.apache.solr.cloud.api.collections.CreateShardCmd; -import org.apache.solr.cloud.api.collections.OverseerCollectionMessageHandler; -import org.apache.solr.cloud.api.collections.SplitShardCmd; -import org.apache.solr.cloud.overseer.ClusterStateMutator; -import org.apache.solr.cloud.overseer.CollectionMutator; -import org.apache.solr.cloud.overseer.ZkWriteCommand; -import org.apache.solr.common.SolrDocumentList; -import org.apache.solr.common.SolrException; -import org.apache.solr.common.SolrInputDocument; -import org.apache.solr.common.cloud.ClusterState; -import org.apache.solr.common.cloud.DocCollection; -import org.apache.solr.common.cloud.DocRouter; -import org.apache.solr.common.cloud.Replica; -import org.apache.solr.common.cloud.ReplicaPosition; -import org.apache.solr.common.cloud.Slice; -import org.apache.solr.common.cloud.ZkNodeProps; -import org.apache.solr.common.cloud.ZkStateReader; -import org.apache.solr.common.cloud.rule.ImplicitSnitch; -import org.apache.solr.common.params.CollectionAdminParams; -import org.apache.solr.common.params.CollectionParams; -import org.apache.solr.common.params.CommonAdminParams; -import org.apache.solr.common.params.CommonParams; -import org.apache.solr.common.params.CoreAdminParams; -import org.apache.solr.common.params.SolrParams; -import org.apache.solr.common.params.UpdateParams; -import org.apache.solr.common.util.NamedList; -import org.apache.solr.common.util.Utils; -import org.apache.solr.core.SolrInfoBean; -import org.apache.solr.metrics.SolrMetricManager; -import org.apache.solr.update.SolrIndexSplitter; -import org.apache.zookeeper.CreateMode; -import org.apache.zookeeper.KeeperException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP; -import static org.apache.solr.common.cloud.ZkStateReader.NRT_REPLICAS; -import static org.apache.solr.common.cloud.ZkStateReader.PULL_REPLICAS; -import static org.apache.solr.common.cloud.ZkStateReader.REPLICATION_FACTOR; -import static org.apache.solr.common.cloud.ZkStateReader.REPLICA_PROP; -import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP; -import static org.apache.solr.common.cloud.ZkStateReader.TLOG_REPLICAS; -import static org.apache.solr.common.params.CollectionParams.CollectionAction.MODIFYCOLLECTION; -import static org.apache.solr.common.params.CommonParams.NAME; - -/** - * Simulated {@link ClusterStateProvider}. - *

The following behaviors are supported:

- *
    - *
  • using autoscaling policy for replica placements
    • - *
  • maintaining and up-to-date list of /live_nodes and nodeAdded / nodeLost markers
    • - *
  • running a simulated leader election on collection changes (with throttling), when needed
    • - *
  • maintaining an up-to-date /state.json per-collection files, which also track replica states, - * leader election changes, replica property changes, etc. Note: these files are only written, - * but never read by the framework!
    • - *
  • maintaining an up-to-date /clusterprops.json. Note: this file is only written, but never read by the - * framework!
    • - *
- */ -public class SimClusterStateProvider implements ClusterStateProvider { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - public static final long DEFAULT_DOC_SIZE_BYTES = 2048; - - private static final String BUFFERED_UPDATES = "__buffered_updates__"; - - private final LiveNodesSet liveNodes; - private final SimDistribStateManager stateManager; - private final SimCloudManager cloudManager; - - private final Map> nodeReplicaMap = new ConcurrentHashMap<>(); - private final Map>> colShardReplicaMap = new ConcurrentHashMap<>(); - private final Map clusterProperties = new ConcurrentHashMap<>(); - private final Map> collProperties = new ConcurrentHashMap<>(); - private final Map>> sliceProperties = new ConcurrentHashMap<>(); - - private final ReentrantLock lock = new ReentrantLock(); - - private final Map> leaderThrottles = new ConcurrentHashMap<>(); - - // default map of: operation -> delay - private final Map defaultOpDelays = new ConcurrentHashMap<>(); - // per-collection map of: collection -> op -> delay - private final Map> opDelays = new ConcurrentHashMap<>(); - - - private volatile String overseerLeader = null; - - private volatile Map lastSavedProperties = null; - - private class CachedCollectionRef { - private final String name; - private int zkVersion; - private DocCollection coll; - ReentrantLock lock = new ReentrantLock(); - - CachedCollectionRef(String name, int zkVersion) { - this.name = name; - this.zkVersion = zkVersion; - } - - @SuppressWarnings({"unchecked", "rawtypes"}) - public DocCollection getColl() throws InterruptedException, IOException { - DocCollection dc = coll; - if (dc != null) { - return dc; - } - lock.lock(); - try { - if (coll != null) { - return coll; - } else { - Map>> collMap = new HashMap<>(); - nodeReplicaMap.forEach((n, replicas) -> { - synchronized (replicas) { - replicas.forEach(ri -> { - if (!ri.getCollection().equals(name)) { - return; - } - Map props; - synchronized (ri) { - props = new HashMap<>(ri.getProperties()); - } - props.put(ZkStateReader.NODE_NAME_PROP, n); - props.put(ZkStateReader.CORE_NAME_PROP, ri.getCoreName()); - props.put(ZkStateReader.REPLICA_TYPE, ri.getType().toString()); - props.put(ZkStateReader.STATE_PROP, ri.getState().toString()); - Replica r = new Replica(ri.getName(), props, ri.getCollection(), ri.getShard()); - collMap.computeIfAbsent(ri.getCollection(), c -> new HashMap<>()) - .computeIfAbsent(ri.getShard(), s -> new HashMap<>()) - .put(ri.getName(), r); - }); - } - }); - - // add empty slices - sliceProperties.forEach((c, perSliceProps) -> { - if (!c.equals(name)) { - return; - } - perSliceProps.forEach((slice, props) -> { - collMap.computeIfAbsent(c, co -> new ConcurrentHashMap<>()).computeIfAbsent(slice, s -> new ConcurrentHashMap<>()); - }); - }); - // add empty collections - collProperties.keySet().forEach(c -> { - if (!c.equals(name)) { - return; - } - collMap.computeIfAbsent(c, co -> new ConcurrentHashMap<>()); - }); - - Map> shards = collMap.get(name); - Map slices = new HashMap<>(); - shards.forEach((s, replicas) -> { - Map sliceProps = sliceProperties.computeIfAbsent(name, c -> new ConcurrentHashMap<>()).computeIfAbsent(s, sl -> new ConcurrentHashMap<>()); - Slice slice = new Slice(s, replicas, sliceProps, name); - slices.put(s, slice); - }); - Map collProps = collProperties.computeIfAbsent(name, c -> new ConcurrentHashMap<>()); - Map routerProp = (Map) collProps.getOrDefault(DocCollection.DOC_ROUTER, Collections.singletonMap("name", DocRouter.DEFAULT_NAME)); - DocRouter router = DocRouter.getDocRouter((String)routerProp.getOrDefault("name", DocRouter.DEFAULT_NAME)); - String path = ZkStateReader.getCollectionPath(name); - coll = new DocCollection(name, slices, collProps, router, zkVersion + 1); - try { - SimDistribStateManager stateManager = cloudManager.getSimDistribStateManager(); - byte[] data = Utils.toJSON(Collections.singletonMap(name, coll)); - if (!stateManager.hasData(path)) { - try { - stateManager.makePath(path, data, CreateMode.PERSISTENT, true); - } catch (AlreadyExistsException e) { - // try updating - stateManager.setData(path, data, zkVersion); - } - } else { - stateManager.setData(path, data, zkVersion); - } - // verify version - VersionedData vd = stateManager.getData(path); - assert vd.getVersion() == zkVersion + 1; - zkVersion++; - } catch (KeeperException | BadVersionException e) { - // should never happen? - throw new RuntimeException("error saving " + coll, e); - } - } - } finally { - lock.unlock(); - } - return coll; - } - - public int getZkVersion() { - lock.lock(); - try { - return zkVersion; - } finally { - lock.unlock(); - } - } - - public void invalidate() { - lock.lock(); - try { - coll = null; - } finally { - lock.unlock(); - } - } - } - - private final Map collectionsStatesRef = new ConcurrentHashMap<>(); - - private final Random bulkUpdateRandom = new Random(0); - - private transient boolean closed; - - /** - * The instance needs to be initialized using the sim* methods in order - * to ensure proper behavior, otherwise it will behave as a cluster with zero replicas. - */ - public SimClusterStateProvider(LiveNodesSet liveNodes, SimCloudManager cloudManager) throws Exception { - this.liveNodes = liveNodes; - for (String nodeId : liveNodes.get()) { - createEphemeralLiveNode(nodeId); - } - this.cloudManager = cloudManager; - this.stateManager = cloudManager.getSimDistribStateManager(); - // names are CollectionAction operation names, delays are in ms (simulated time) - defaultOpDelays.put(CollectionParams.CollectionAction.MOVEREPLICA.name(), 5000L); - defaultOpDelays.put(CollectionParams.CollectionAction.DELETEREPLICA.name(), 5000L); - defaultOpDelays.put(CollectionParams.CollectionAction.ADDREPLICA.name(), 500L); - defaultOpDelays.put(CollectionParams.CollectionAction.SPLITSHARD.name(), 5000L); - defaultOpDelays.put(CollectionParams.CollectionAction.CREATESHARD.name(), 5000L); - defaultOpDelays.put(CollectionParams.CollectionAction.DELETESHARD.name(), 5000L); - defaultOpDelays.put(CollectionParams.CollectionAction.CREATE.name(), 500L); - defaultOpDelays.put(CollectionParams.CollectionAction.DELETE.name(), 5000L); - } - - // ============== SIMULATOR SETUP METHODS ==================== - - public void copyFrom(ClusterStateProvider other) throws Exception { - ClusterState state = other.getClusterState(); - simSetClusterState(state); - clusterProperties.clear(); - clusterProperties.putAll(other.getClusterProperties()); - } - - /** - * Initialize from an existing cluster state - * @param initialState initial cluster state - */ - @SuppressWarnings({"unchecked"}) - public void simSetClusterState(ClusterState initialState) throws Exception { - lock.lockInterruptibly(); - try { - collProperties.clear(); - colShardReplicaMap.clear(); - sliceProperties.clear(); - nodeReplicaMap.clear(); - liveNodes.clear(); - collectionsStatesRef.clear(); - for (String nodeId : stateManager.listData(ZkStateReader.LIVE_NODES_ZKNODE)) { - if (stateManager.hasData(ZkStateReader.LIVE_NODES_ZKNODE + "/" + nodeId)) { - stateManager.removeData(ZkStateReader.LIVE_NODES_ZKNODE + "/" + nodeId, -1); - } - if (stateManager.hasData(ZkStateReader.SOLR_AUTOSCALING_NODE_ADDED_PATH + "/" + nodeId)) { - stateManager.removeData(ZkStateReader.SOLR_AUTOSCALING_NODE_ADDED_PATH + "/" + nodeId, -1); - } - if (stateManager.hasData(ZkStateReader.SOLR_AUTOSCALING_NODE_LOST_PATH + "/" + nodeId)) { - stateManager.removeData(ZkStateReader.SOLR_AUTOSCALING_NODE_LOST_PATH + "/" + nodeId, -1); - } - } - liveNodes.addAll(initialState.getLiveNodes()); - for (String nodeId : liveNodes.get()) { - createEphemeralLiveNode(nodeId); - } - initialState.forEachCollection(dc -> { - // DocCollection will be created later - collectionsStatesRef.put(dc.getName(), new CachedCollectionRef(dc.getName(), dc.getZNodeVersion())); - collProperties.computeIfAbsent(dc.getName(), name -> new ConcurrentHashMap<>()).putAll(dc.getProperties()); - opDelays.computeIfAbsent(dc.getName(), Utils.NEW_HASHMAP_FUN).putAll(defaultOpDelays); - dc.getSlices().forEach(s -> { - sliceProperties.computeIfAbsent(dc.getName(), name -> new ConcurrentHashMap<>()) - .computeIfAbsent(s.getName(), Utils.NEW_HASHMAP_FUN).putAll(s.getProperties()); - Replica leader = s.getLeader(); - s.getReplicas().forEach(r -> { - Map props = new HashMap<>(r.getProperties()); - if (leader != null && r.getName().equals(leader.getName())) { - props.put("leader", "true"); - } - Replica ri = new Replica(r.getName(), r.getNodeName(), dc.getName(), s.getName(), r.getCoreName(), - r.getState(), r.getType(), props); - if (leader != null && r.getName().equals(leader.getName())) { - ri.getProperties().put("leader", "true"); - } - if (liveNodes.get().contains(r.getNodeName())) { - nodeReplicaMap.computeIfAbsent(r.getNodeName(), Utils.NEW_SYNCHRONIZED_ARRAYLIST_FUN).add(ri); - colShardReplicaMap.computeIfAbsent(ri.getCollection(), name -> new ConcurrentHashMap<>()) - .computeIfAbsent(ri.getShard(), shard -> new ArrayList<>()).add(ri); - } else { - log.warn("- dropping replica because its node {} is not live: {}", r.getNodeName(), r); - } - }); - }); - }); - } finally { - lock.unlock(); - } - } - - /** - * Reset the leader election throttles. - */ - public void simResetLeaderThrottles() { - leaderThrottles.clear(); - } - - private ActionThrottle getThrottle(String collection, String shard) { - return leaderThrottles.computeIfAbsent(collection, coll -> new ConcurrentHashMap<>()) - .computeIfAbsent(shard, s -> new ActionThrottle("leader", 5000, cloudManager.getTimeSource())); - } - - /** - * Get random node id. - * @return one of the live nodes - */ - public String simGetRandomNode() { - return simGetRandomNode(cloudManager.getRandom()); - } - - /** - * Get random node id. - * @param random instance of random. - * @return one of the live nodes - */ - public String simGetRandomNode(Random random) { - if (liveNodes.isEmpty()) { - return null; - } - List nodes = new ArrayList<>(liveNodes.get()); - return nodes.get(random.nextInt(nodes.size())); - } - - private Replica getReplicaInfo(Replica r) { - @SuppressWarnings({"unchecked"}) - final List list = nodeReplicaMap.computeIfAbsent - (r.getNodeName(), Utils.NEW_SYNCHRONIZED_ARRAYLIST_FUN); - synchronized (list) { - for (Replica ri : list) { - if (r.getCoreName().equals(ri.getCoreName()) && r.getName().equals(ri.getName())) { - return ri; - } - } - } - return null; - } - - /** - * Add a new node to the cluster. - * @param nodeId unique node id - */ - @SuppressWarnings({"unchecked"}) - public void simAddNode(String nodeId) throws Exception { - ensureNotClosed(); - if (liveNodes.contains(nodeId)) { - throw new Exception("Node " + nodeId + " already exists"); - } - createEphemeralLiveNode(nodeId); - nodeReplicaMap.computeIfAbsent(nodeId, Utils.NEW_SYNCHRONIZED_ARRAYLIST_FUN); - liveNodes.add(nodeId); - updateOverseerLeader(); - } - - /** - * Remove node from a cluster. This is equivalent to a situation when a node is lost. - * All replicas that were assigned to this node are marked as DOWN. - * @param nodeId node id - * @return true if a node existed and was removed - */ - public boolean simRemoveNode(String nodeId) throws Exception { - ensureNotClosed(); - lock.lockInterruptibly(); - try { - Set collections = new HashSet<>(); - // mark every replica on that node as down - boolean res = liveNodes.remove(nodeId); - setReplicaStates(nodeId, Replica.State.DOWN, collections); - for (String collection : collections) { - collectionsStatesRef.get(collection).invalidate();; - } - // remove ephemeral nodes - stateManager.getRoot().removeEphemeralChildren(nodeId); - // create a nodeLost marker if needed - AutoScalingConfig cfg = stateManager.getAutoScalingConfig(null); - if (cfg.hasTriggerForEvents(TriggerEventType.NODELOST)) { - String path = ZkStateReader.SOLR_AUTOSCALING_NODE_LOST_PATH + "/" + nodeId; - byte[] json = Utils.toJSON(Collections.singletonMap("timestamp", cloudManager.getTimeSource().getEpochTimeNs())); - stateManager.makePath(path, - json, CreateMode.PERSISTENT, false); - log.debug(" -- created marker: {}", path); - } - updateOverseerLeader(); - if (!collections.isEmpty()) { - simRunLeaderElection(collections, true); - } - return res; - } finally { - lock.unlock(); - } - } - - /** - * Remove all replica information related to dead nodes. - */ - public void simRemoveDeadNodes() throws Exception { - lock.lockInterruptibly(); - try { - Set myNodes = new HashSet<>(nodeReplicaMap.keySet()); - myNodes.removeAll(liveNodes.get()); - } finally { - lock.unlock(); - } - } - - private synchronized void updateOverseerLeader() throws Exception { - if (overseerLeader != null && liveNodes.contains(overseerLeader)) { - return; - } - String path = Overseer.OVERSEER_ELECT + "/leader"; - if (liveNodes.isEmpty()) { - overseerLeader = null; - // remove it from ZK - try { - cloudManager.getDistribStateManager().removeData(path, -1); - } catch (NoSuchElementException e) { - // ignore - } - return; - } - // pick first - overseerLeader = liveNodes.iterator().next(); - log.debug("--- new Overseer leader: {}", overseerLeader); - // record it in ZK - Map id = new HashMap<>(); - id.put("id", cloudManager.getTimeSource().getTimeNs() + - "-" + overseerLeader + "-n_0000000000"); - try { - cloudManager.getDistribStateManager().makePath(path, Utils.toJSON(id), CreateMode.EPHEMERAL, false); - } catch (Exception e) { - log.warn("Exception saving overseer leader id", e); - } - } - - public synchronized String simGetOverseerLeader() { - return overseerLeader; - } - - // this method needs to be called under a lock - private void setReplicaStates(String nodeId, Replica.State state, Set changedCollections) { - @SuppressWarnings({"unchecked"}) - List replicas = nodeReplicaMap.computeIfAbsent(nodeId, Utils.NEW_SYNCHRONIZED_ARRAYLIST_FUN); - synchronized (replicas) { - replicas.forEach(r -> { - r.setState(state); - //r.getProperties().put(ZkStateReader.STATE_PROP, state.toString()); - if (state != Replica.State.ACTIVE) { - r.getProperties().remove(ZkStateReader.LEADER_PROP); - } - changedCollections.add(r.getCollection()); - }); - } - } - - // this method needs to be called under a lock - private void createEphemeralLiveNode(String nodeId) throws Exception { - DistribStateManager mgr = stateManager.withEphemeralId(nodeId); - mgr.makePath(ZkStateReader.LIVE_NODES_ZKNODE + "/" + nodeId, null, CreateMode.EPHEMERAL, true); - AutoScalingConfig cfg = stateManager.getAutoScalingConfig(null); - if (cfg.hasTriggerForEvents(TriggerEventType.NODEADDED)) { - byte[] json = Utils.toJSON(Collections.singletonMap("timestamp", cloudManager.getTimeSource().getEpochTimeNs())); - String path = ZkStateReader.SOLR_AUTOSCALING_NODE_ADDED_PATH + "/" + nodeId; - log.debug("-- creating marker: {}", path); - mgr.makePath(path, json, CreateMode.EPHEMERAL, true); - } - } - - /** - * Restore a previously removed node. This also simulates a short replica recovery state. - * @param nodeId node id to restore - * @return true when this operation restored any replicas, false otherwise (empty node). - */ - public boolean simRestoreNode(String nodeId) throws Exception { - liveNodes.add(nodeId); - createEphemeralLiveNode(nodeId); - Set collections = new HashSet<>(); - - lock.lockInterruptibly(); - try { - setReplicaStates(nodeId, Replica.State.RECOVERING, collections); - } finally { - lock.unlock(); - } - - cloudManager.getTimeSource().sleep(1000); - - lock.lockInterruptibly(); - try { - setReplicaStates(nodeId, Replica.State.ACTIVE, collections); - if (!collections.isEmpty()) { - collections.forEach(c -> collectionsStatesRef.get(c).invalidate()); - simRunLeaderElection(collections, true); - return true; - } else { - return false; - } - } finally { - lock.unlock(); - } - } - - /** - * Add a new replica. Note that if any details of a replica (node, coreNodeName, SolrCore name, etc) - * are missing they will be filled in using the policy framework. - * @param message replica details - * @param results result of the operation - */ - @SuppressWarnings({"unchecked", "rawtypes"}) - public void simAddReplica(ZkNodeProps message, NamedList results) throws Exception { - if (message.getStr(CommonAdminParams.ASYNC) != null) { - results.add(CoreAdminParams.REQUESTID, message.getStr(CommonAdminParams.ASYNC)); - } - ClusterState clusterState = getClusterState(); - DocCollection coll = clusterState.getCollection(message.getStr(ZkStateReader.COLLECTION_PROP)); - AtomicReference sessionWrapper = new AtomicReference<>(); - - Replica.Type replicaType = Replica.Type.valueOf(message.getStr(ZkStateReader.REPLICA_TYPE, Replica.Type.NRT.name()).toUpperCase(Locale.ROOT)); - EnumMap replicaTypesVsCount = new EnumMap<>(Replica.Type.class); - replicaTypesVsCount.put(Replica.Type.NRT, message.getInt(NRT_REPLICAS, replicaType == Replica.Type.NRT ? 1 : 0)); - replicaTypesVsCount.put(Replica.Type.TLOG, message.getInt(TLOG_REPLICAS, replicaType == Replica.Type.TLOG ? 1 : 0)); - replicaTypesVsCount.put(Replica.Type.PULL, message.getInt(PULL_REPLICAS, replicaType == Replica.Type.PULL ? 1 : 0)); - - int totalReplicas = 0; - for (Map.Entry entry : replicaTypesVsCount.entrySet()) { - totalReplicas += entry.getValue(); - } - if (totalReplicas > 1) { - if (message.getStr(CoreAdminParams.NAME) != null) { - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Cannot create " + totalReplicas + " replicas if 'name' parameter is specified"); - } - if (message.getStr(CoreAdminParams.CORE_NODE_NAME) != null) { - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Cannot create " + totalReplicas + " replicas if 'coreNodeName' parameter is specified"); - } - } - - List replicaPositions = AddReplicaCmd.buildReplicaPositions(cloudManager, clusterState, coll.getName(), message, replicaTypesVsCount, sessionWrapper); - for (ReplicaPosition replicaPosition : replicaPositions) { - AddReplicaCmd.CreateReplica createReplica = AddReplicaCmd.assignReplicaDetails(cloudManager, clusterState, message, replicaPosition); - if (message.getStr(CoreAdminParams.CORE_NODE_NAME) == null) { - createReplica.coreNodeName = Assign.assignCoreNodeName(stateManager, coll); - } - Replica ri = new Replica( - createReplica.coreNodeName, - createReplica.node, - createReplica.collectionName, - createReplica.sliceName, - createReplica.coreName, - Replica.State.DOWN, - createReplica.replicaType, - message.getProperties() - ); - simAddReplica(ri.getNodeName(), ri, true); - } - if (sessionWrapper.get() != null) { - sessionWrapper.get().release(); - } - results.add("success", ""); - } - - /** - * Add a replica. Note that all details of the replica must be present here, including - * node, coreNodeName and SolrCore name. - * @param nodeId node id where the replica will be added - * @param replicaInfo replica info - * @param runLeaderElection if true then run a leader election after adding the replica. - */ - @SuppressWarnings({"unchecked"}) - public void simAddReplica(String nodeId, Replica replicaInfo, boolean runLeaderElection) throws Exception { - ensureNotClosed(); - lock.lockInterruptibly(); - try { - - // make sure SolrCore name is unique across cluster and coreNodeName within collection - for (Map.Entry> e : nodeReplicaMap.entrySet()) { - final List replicas = e.getValue(); - synchronized (replicas) { - for (Replica ri : replicas) { - if (ri.getCoreName().equals(replicaInfo.getCoreName())) { - throw new Exception("Duplicate SolrCore name for existing=" + ri + " on node " + e.getKey() + " and new=" + replicaInfo); - } - if (ri.getName().equals(replicaInfo.getName()) && ri.getCollection().equals(replicaInfo.getCollection())) { - throw new Exception("Duplicate coreNode name for existing=" + ri + " on node " + e.getKey() + " and new=" + replicaInfo); - } - } - } - } - if (!liveNodes.contains(nodeId)) { - throw new Exception("Target node " + nodeId + " is not live: " + liveNodes); - } - // verify info - if (replicaInfo.getCoreName() == null) { - throw new Exception("Missing core: " + replicaInfo); - } - // XXX replica info is not supposed to have this as a variable - replicaInfo.getProperties().remove(ZkStateReader.SHARD_ID_PROP); - if (replicaInfo.getName() == null) { - throw new Exception("Missing name: " + replicaInfo); - } - if (replicaInfo.getNodeName() == null) { - throw new Exception("Missing node: " + replicaInfo); - } - if (!replicaInfo.getNodeName().equals(nodeId)) { - throw new Exception("Wrong node (not " + nodeId + "): " + replicaInfo); - } - - opDelay(replicaInfo.getCollection(), CollectionParams.CollectionAction.ADDREPLICA.name()); - - // mark replica as active - replicaInfo.setState(Replica.State.ACTIVE); - // add a property expected in Policy calculations, if missing - if (replicaInfo.get(Type.CORE_IDX.metricsAttribute) == null) { - replicaInfo.getProperties().put(Type.CORE_IDX.metricsAttribute, new AtomicLong(SimCloudManager.DEFAULT_IDX_SIZE_BYTES)); - replicaInfo.getProperties().put(Variable.coreidxsize, - new AtomicDouble((Double)Type.CORE_IDX.convertVal(SimCloudManager.DEFAULT_IDX_SIZE_BYTES))); - } - nodeReplicaMap.computeIfAbsent(nodeId, Utils.NEW_SYNCHRONIZED_ARRAYLIST_FUN).add(replicaInfo); - colShardReplicaMap.computeIfAbsent(replicaInfo.getCollection(), c -> new ConcurrentHashMap<>()) - .computeIfAbsent(replicaInfo.getShard(), s -> new ArrayList<>()) - .add(replicaInfo); - - Map values = cloudManager.getSimNodeStateProvider().simGetAllNodeValues() - .computeIfAbsent(nodeId, id -> new ConcurrentHashMap<>(SimCloudManager.createNodeValues(id))); - // update the number of cores and freedisk in node values - Number cores = (Number)values.get(ImplicitSnitch.CORES); - if (cores == null) { - cores = 0; - } - cloudManager.getSimNodeStateProvider().simSetNodeValue(nodeId, ImplicitSnitch.CORES, cores.intValue() + 1); - Number disk = (Number)values.get(ImplicitSnitch.DISK); - if (disk == null) { - throw new Exception("Missing '" + ImplicitSnitch.DISK + "' in node metrics for node " + nodeId); - //disk = SimCloudManager.DEFAULT_FREE_DISK; - } - long replicaSize = ((Number)replicaInfo.get(Type.CORE_IDX.metricsAttribute)).longValue(); - Number replicaSizeGB = (Number)Type.CORE_IDX.convertVal(replicaSize); - cloudManager.getSimNodeStateProvider().simSetNodeValue(nodeId, ImplicitSnitch.DISK, disk.doubleValue() - replicaSizeGB.doubleValue()); - // fake metrics - String registry = SolrMetricManager.getRegistryName(SolrInfoBean.Group.core, replicaInfo.getCollection(), - replicaInfo.getShard(), - Utils.parseMetricsReplicaName(replicaInfo.getCollection(), replicaInfo.getCoreName())); - cloudManager.getMetricManager().registry(registry).counter("UPDATE./update.requests"); - cloudManager.getMetricManager().registry(registry).counter("QUERY./select.requests"); - cloudManager.getMetricManager().registerGauge(null, registry, - () -> replicaSize, "", true, Type.CORE_IDX.metricsAttribute); - // at this point nuke our cached DocCollection state - collectionsStatesRef.get(replicaInfo.getCollection()).invalidate(); - log.trace("-- simAddReplica {}", replicaInfo); - if (runLeaderElection) { - simRunLeaderElection(replicaInfo.getCollection(), replicaInfo.getShard(), true); - } - } finally { - lock.unlock(); - } - } - - /** - * Remove replica. - * @param nodeId node id - * @param coreNodeName coreNodeName - */ - public void simRemoveReplica(String nodeId, String collection, String coreNodeName) throws Exception { - ensureNotClosed(); - - lock.lockInterruptibly(); - try { - @SuppressWarnings({"unchecked"}) - final List replicas = nodeReplicaMap.computeIfAbsent - (nodeId, Utils.NEW_SYNCHRONIZED_ARRAYLIST_FUN); - synchronized (replicas) { - for (int i = 0; i < replicas.size(); i++) { - if (collection.equals(replicas.get(i).getCollection()) && coreNodeName.equals(replicas.get(i).getName())) { - Replica ri = replicas.remove(i); - colShardReplicaMap.computeIfAbsent(ri.getCollection(), c -> new ConcurrentHashMap<>()) - .computeIfAbsent(ri.getShard(), s -> new ArrayList<>()) - .remove(ri); - collectionsStatesRef.get(ri.getCollection()).invalidate(); - - opDelay(ri.getCollection(), CollectionParams.CollectionAction.DELETEREPLICA.name()); - - // update the number of cores in node values, if node is live - if (liveNodes.contains(nodeId)) { - Number cores = (Number)cloudManager.getSimNodeStateProvider().simGetNodeValue(nodeId, ImplicitSnitch.CORES); - if (cores == null || cores.intValue() == 0) { - throw new Exception("Unexpected value of 'cores' (" + cores + ") on node: " + nodeId); - } - cloudManager.getSimNodeStateProvider().simSetNodeValue(nodeId, ImplicitSnitch.CORES, cores.intValue() - 1); - Number disk = (Number)cloudManager.getSimNodeStateProvider().simGetNodeValue(nodeId, ImplicitSnitch.DISK); - if (disk == null || disk.doubleValue() == 0.0) { - throw new Exception("Unexpected value of 'freedisk' (" + disk + ") on node: " + nodeId); - } - if (ri.get(Type.CORE_IDX.metricsAttribute) == null) { - throw new RuntimeException("Missing replica size: " + ri); - } - long replicaSize = ((Number)ri.get(Type.CORE_IDX.metricsAttribute)).longValue(); - Number replicaSizeGB = (Number)Type.CORE_IDX.convertVal(replicaSize); - cloudManager.getSimNodeStateProvider().simSetNodeValue(nodeId, ImplicitSnitch.DISK, disk.doubleValue() + replicaSizeGB.doubleValue()); - } - log.trace("-- simRemoveReplica {}", ri); - simRunLeaderElection(ri.getCollection(), ri.getShard(), true); - - return; - } - } - } - throw new Exception("Replica " + coreNodeName + " not found on node " + nodeId); - } finally { - lock.unlock(); - } - } - - /** - * Delay an operation by a configured amount. - * @param collection collection name - * @param op operation name. - */ - private void opDelay(String collection, String op) throws InterruptedException { - Map delays = opDelays.get(collection); - if (delays == null || delays.isEmpty() || !delays.containsKey(op)) { - return; - } - cloudManager.getTimeSource().sleep(delays.get(op)); - } - - public void simSetOpDelays(String collection, Map delays) { - Map currentDelays = opDelays.getOrDefault(collection, Collections.emptyMap()); - Map newDelays = new HashMap<>(currentDelays); - delays.forEach((k, v) -> { - if (v == null) { - newDelays.remove(k); - } else { - newDelays.put(k, v); - } - }); - opDelays.put(collection, newDelays); - } - - /** - * Simulate running a shard leader election. This operation is a no-op if a leader already exists. - * If a new leader is elected the cluster state is saved. - * @param collections list of affected collections - * @param saveClusterState if true then save cluster state regardless of changes. - */ - private void simRunLeaderElection(Collection collections, boolean saveClusterState) throws Exception { - ensureNotClosed(); - if (saveClusterState) { - lock.lockInterruptibly(); - try { - collections.forEach(c -> collectionsStatesRef.get(c).invalidate()); - } finally { - lock.unlock(); - } - } - ClusterState state = getClusterState(); - state.forEachCollection(dc -> { - if (!collections.contains(dc.getName())) { - return; - } - dc.getSlices().forEach(s -> { - if (log.isTraceEnabled()) { - log.trace("-- submit leader election for {} / {}", dc.getName(), s.getName()); - } - cloudManager.submit(() -> { - simRunLeaderElection(dc.getName(), s.getName(), saveClusterState); - return true; - }); - }); - }); - } - - private void simRunLeaderElection(final String collection, final String slice, - final boolean saveState) throws Exception { - - log.trace("Attempting leader election ({} / {})", collection, slice); - final AtomicBoolean stateChanged = new AtomicBoolean(Boolean.FALSE); - - lock.lockInterruptibly(); - try { - final ClusterState state = getClusterState(); - final DocCollection col = state.getCollectionOrNull(collection); - - if (null == col) { - log.trace("-- collection does not exist (anymore), skipping leader election ({} / {})", - collection, slice); - return; - } - final Slice s = col.getSlice(slice); - if (null == s) { - log.trace("-- slice does not exist, skipping leader election ({} / {})", - collection, slice); - return; - } - if (s.getState() == Slice.State.INACTIVE) { - if (log.isTraceEnabled()) { - log.trace("-- slice state is {}, skipping leader election ({} / {})", - s.getState(), collection, slice); - } - return; - } - if (s.getReplicas().isEmpty()) { - log.trace("-- no replicas, skipping leader election ({} / {})", collection, slice); - return; - } - - final Replica leader = s.getLeader(); - if (null != leader && liveNodes.contains(leader.getNodeName())) { - log.trace("-- already has livenode leader, skipping leader election {} / {}", - collection, slice); - return; - } - - if (s.getState() != Slice.State.ACTIVE) { - if (log.isTraceEnabled()) { - log.trace("-- slice state is {}, but I will run leader election anyway ({} / {})", - s.getState(), collection, slice); - } - } - - log.debug("Running leader election ({} / {})", collection, slice); - ActionThrottle lt = getThrottle(collection, s.getName()); - synchronized (lt) { - // collect all active and live - List active = new ArrayList<>(); - AtomicBoolean alreadyHasLeader = new AtomicBoolean(false); - s.getReplicas().forEach(r -> { - // find our ReplicaInfo for this replica - Replica ri = getReplicaInfo(r); - if (ri == null) { - throw new IllegalStateException("-- could not find ReplicaInfo for replica " + r); - } - synchronized (ri) { - if (r.isActive(liveNodes.get())) { - if (ri.getProperties().get(ZkStateReader.LEADER_PROP) != null) { - if (log.isTraceEnabled()) { - log.trace("-- found existing leader {} / {}: {}, {}", collection, s.getName(), ri, r); - } - alreadyHasLeader.set(true); - return; - } else { - active.add(ri); - } - } else { // if it's on a node that is not live mark it down - if (log.isTraceEnabled()) { - log.trace("-- replica not active on live nodes: {}, {}", liveNodes.get(), r); - } - if (!liveNodes.contains(r.getNodeName())) { - ri.getProperties().put(ZkStateReader.STATE_PROP, Replica.State.DOWN.toString()); - ri.getProperties().remove(ZkStateReader.LEADER_PROP); - stateChanged.set(true); - } - } - } - }); - if (alreadyHasLeader.get()) { - if (log.isTraceEnabled()) { - log.trace("-- already has leader {} / {}: {}", collection, s.getName(), s); - } - return; - } - if (active.isEmpty()) { - if (log.isWarnEnabled()) { - log.warn("Can't find any active replicas for {} / {}: {}", collection, s.getName(), s); - } - if (log.isDebugEnabled()) { - log.debug("-- liveNodes: {}", liveNodes.get()); - } - return; - } - // pick first active one - Replica ri = null; - for (Replica a : active) { - if (!a.getType().equals(Replica.Type.PULL)) { - ri = a; - break; - } - } - if (ri == null) { - log.warn("-- can't find any suitable replica type for {} / {}: {}", collection, s.getName(), s); - return; - } - // now mark the leader election throttle - lt.minimumWaitBetweenActions(); - lt.markAttemptingAction(); - synchronized (ri) { - ri.getProperties().put(ZkStateReader.LEADER_PROP, "true"); - } - if (log.isDebugEnabled()) { - log.debug("-- elected new leader for {} / {} (currentVersion={}): {}", collection, - s.getName(), col.getZNodeVersion(), ri); - } - stateChanged.set(true); - } - } finally { - if (stateChanged.get() || saveState) { - collectionsStatesRef.get(collection).invalidate(); - } - lock.unlock(); - } - } - - /** - * Create a new collection. This operation uses policy framework for node and replica assignments. - * @param props collection details - * @param results results of the operation. - */ - - @SuppressWarnings({"unchecked", "rawtypes"}) - public void simCreateCollection(ZkNodeProps props, NamedList results) throws Exception { - ensureNotClosed(); - if (props.getStr(CommonAdminParams.ASYNC) != null) { - results.add(CoreAdminParams.REQUESTID, props.getStr(CommonAdminParams.ASYNC)); - } - boolean waitForFinalState = props.getBool(CommonAdminParams.WAIT_FOR_FINAL_STATE, false); - final String collectionName = props.getStr(NAME); - log.debug("-- simCreateCollection {}", collectionName); - - String router = props.getStr("router.name", DocRouter.DEFAULT_NAME); - String policy = props.getStr(Policy.POLICY); - AutoScalingConfig autoScalingConfig = cloudManager.getDistribStateManager().getAutoScalingConfig(); - boolean usePolicyFramework = !autoScalingConfig.getPolicy().getClusterPolicy().isEmpty() || policy != null; - - // fail fast if parameters are wrong or incomplete - List shardNames = CreateCollectionCmd.populateShardNames(props, router); - CreateCollectionCmd.checkReplicaTypes(props); - - // always force getting fresh state - final ClusterState clusterState = getClusterState(); - - String withCollection = props.getStr(CollectionAdminParams.WITH_COLLECTION); - String wcShard = null; - if (withCollection != null) { - if (!clusterState.hasCollection(withCollection)) { - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "The 'withCollection' does not exist: " + withCollection); - } else { - DocCollection collection = clusterState.getCollection(withCollection); - if (collection.getActiveSlices().size() > 1) { - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "The `withCollection` must have only one shard, found: " + collection.getActiveSlices().size()); - } - wcShard = collection.getActiveSlices().iterator().next().getName(); - } - } - final String withCollectionShard = wcShard; - - ZkWriteCommand cmd = ZkWriteCommand.noop(); - - lock.lockInterruptibly(); - try { - cmd = new ClusterStateMutator(cloudManager).createCollection(clusterState, props); - if (cmd.noop) { - log.warn("Collection {} already exists. exit", collectionName); - log.debug("-- collection: {}, clusterState: {}", collectionName, clusterState); - results.add("success", "no-op"); - return; - } - // add collection props - DocCollection coll = cmd.collection; - collProperties.computeIfAbsent(collectionName, c -> new ConcurrentHashMap<>()).putAll(coll.getProperties()); - colShardReplicaMap.computeIfAbsent(collectionName, c -> new ConcurrentHashMap<>()); - // add slice props - coll.getSlices().forEach(s -> { - Map sliceProps = sliceProperties.computeIfAbsent(coll.getName(), c -> new ConcurrentHashMap<>()) - .computeIfAbsent(s.getName(), slice -> new ConcurrentHashMap<>()); - s.getProperties().forEach((k, v) -> { - if (k != null && v != null) { - sliceProps.put(k, v); - } - }); - colShardReplicaMap.computeIfAbsent(collectionName, c -> new ConcurrentHashMap<>()) - .computeIfAbsent(s.getName(), sh -> new ArrayList<>()); - }); - - // modify the `withCollection` and store this new collection's name with it - if (withCollection != null) { - ZkNodeProps message = new ZkNodeProps( - Overseer.QUEUE_OPERATION, MODIFYCOLLECTION.toString(), - ZkStateReader.COLLECTION_PROP, withCollection, - CollectionAdminParams.COLOCATED_WITH, collectionName); - cmd = new CollectionMutator(cloudManager).modifyCollection(clusterState,message); - } - collectionsStatesRef.put(collectionName, new CachedCollectionRef(collectionName, 0)); - - } finally { - lock.unlock(); - } - opDelays.computeIfAbsent(collectionName, c -> new HashMap<>()).putAll(defaultOpDelays); - - opDelay(collectionName, CollectionParams.CollectionAction.CREATE.name()); - - AtomicReference sessionWrapper = new AtomicReference<>(); - List replicaPositions = CreateCollectionCmd.buildReplicaPositions(cloudManager, getClusterState(), cmd.collection, props, - shardNames, sessionWrapper); - if (sessionWrapper.get() != null) { - sessionWrapper.get().release(); - } - // calculate expected number of positions - int numTlogReplicas = props.getInt(TLOG_REPLICAS, 0); - int numNrtReplicas = props.getInt(NRT_REPLICAS, props.getInt(REPLICATION_FACTOR, numTlogReplicas>0?0:1)); - int numPullReplicas = props.getInt(PULL_REPLICAS, 0); - int totalReplicas = shardNames.size() * (numNrtReplicas + numPullReplicas + numTlogReplicas); - if (totalReplicas != replicaPositions.size()) { - throw new RuntimeException("unexpected number of replica positions: expected " + totalReplicas + " but got " + replicaPositions.size()); - } - final CountDownLatch finalStateLatch = new CountDownLatch(replicaPositions.size()); - AtomicInteger replicaNum = new AtomicInteger(1); - replicaPositions.forEach(pos -> { - - if (withCollection != null) { - // check that we have a replica of `withCollection` on this node and if not, create one - DocCollection collection = clusterState.getCollection(withCollection); - List replicas = collection.getReplicas(pos.node); - if (replicas == null || replicas.isEmpty()) { - Map replicaProps = new HashMap<>(); - replicaProps.put(ZkStateReader.NODE_NAME_PROP, pos.node); - replicaProps.put(ZkStateReader.REPLICA_TYPE, pos.type.toString()); - String coreName = String.format(Locale.ROOT, "%s_%s_replica_%s%s", withCollection, withCollectionShard, pos.type.name().substring(0,1).toLowerCase(Locale.ROOT), - collection.getReplicas().size() + 1); - try { - replicaProps.put(ZkStateReader.CORE_NAME_PROP, coreName); - replicaProps.put("SEARCHER.searcher.deletedDocs", new AtomicLong(0)); - replicaProps.put("SEARCHER.searcher.numDocs", new AtomicLong(0)); - replicaProps.put("SEARCHER.searcher.maxDoc", new AtomicLong(0)); - Replica ri = new Replica("core_node" + Assign.incAndGetId(stateManager, withCollection, 0), - pos.node, withCollection, withCollectionShard, coreName, Replica.State.DOWN, - pos.type, replicaProps); - cloudManager.submit(() -> { - simAddReplica(pos.node, ri, false); - // do not count down the latch here - return true; - }); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - } - - Map replicaProps = new HashMap<>(); - replicaProps.put(ZkStateReader.NODE_NAME_PROP, pos.node); - replicaProps.put(ZkStateReader.REPLICA_TYPE, pos.type.toString()); - String coreName = String.format(Locale.ROOT, "%s_%s_replica_%s%s", collectionName, pos.shard, pos.type.name().substring(0,1).toLowerCase(Locale.ROOT), - replicaNum.getAndIncrement()); - try { - replicaProps.put(ZkStateReader.CORE_NAME_PROP, coreName); - replicaProps.put("SEARCHER.searcher.deletedDocs", new AtomicLong(0)); - replicaProps.put("SEARCHER.searcher.numDocs", new AtomicLong(0)); - replicaProps.put("SEARCHER.searcher.maxDoc", new AtomicLong(0)); - Replica ri = new Replica("core_node" + Assign.incAndGetId(stateManager, collectionName, 0), - pos.node, collectionName, pos.shard, coreName, Replica.State.DOWN, - pos.type, replicaProps); - cloudManager.submit(() -> { - simAddReplica(pos.node, ri, true); - finalStateLatch.countDown(); - return true; - }); - } catch (Exception e) { - throw new RuntimeException(e); - } - }); - - // force recreation of collection states - lock.lockInterruptibly(); - try { - collectionsStatesRef.get(collectionName).invalidate(); - } finally { - lock.unlock(); - } - //simRunLeaderElection(Collections.singleton(collectionName), true); - if (waitForFinalState) { - boolean finished = finalStateLatch.await(cloudManager.getTimeSource().convertDelay(TimeUnit.SECONDS, 60, TimeUnit.MILLISECONDS), - TimeUnit.MILLISECONDS); - if (!finished) { - results.add("failure", "Timeout waiting for all replicas to become active."); - return; - } - } - results.add("success", ""); - log.debug("-- finished createCollection {}", collectionName); - } - - /** - * Delete a collection - * @param collection collection name - * @param async async id - * @param results results of the operation - */ - @SuppressWarnings({"unchecked", "rawtypes"}) - public void simDeleteCollection(String collection, String async, NamedList results) throws Exception { - ensureNotClosed(); - if (async != null) { - results.add(CoreAdminParams.REQUESTID, async); - } - - lock.lockInterruptibly(); - try { - collProperties.remove(collection); - sliceProperties.remove(collection); - leaderThrottles.remove(collection); - colShardReplicaMap.remove(collection); - SplitShardCmd.unlockForSplit(cloudManager, collection, null); - - opDelay(collection, CollectionParams.CollectionAction.DELETE.name()); - - opDelays.remove(collection); - nodeReplicaMap.forEach((n, replicas) -> { - synchronized (replicas) { - for (Iterator it = replicas.iterator(); it.hasNext(); ) { - Replica ri = it.next(); - if (ri.getCollection().equals(collection)) { - it.remove(); - // update the number of cores in node values - Number cores = (Number) cloudManager.getSimNodeStateProvider().simGetNodeValue(n, "cores"); - if (cores != null) { // node is still up - if (cores.intValue() == 0) { - throw new RuntimeException("Unexpected value of 'cores' (" + cores + ") on node: " + n); - } - try { - cloudManager.getSimNodeStateProvider().simSetNodeValue(n, "cores", cores.intValue() - 1); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new RuntimeException("interrupted"); - } - } - } - } - } - }); - cloudManager.getDistribStateManager().removeRecursively(ZkStateReader.getCollectionPath(collection), true, true); - collectionsStatesRef.remove(collection); - results.add("success", ""); - } catch (Exception e) { - log.warn("Exception", e); - } finally { - lock.unlock(); - } - } - - /** - * Remove all collections. - */ - public void simDeleteAllCollections() throws Exception { - lock.lockInterruptibly(); - try { - collectionsStatesRef.keySet().forEach(name -> { - try { - cloudManager.getDistribStateManager().removeRecursively(ZkStateReader.getCollectionPath(name), true, true); - } catch (Exception e) { - log.error("Unable to delete collection state.json"); - } - }); - - collProperties.clear(); - sliceProperties.clear(); - leaderThrottles.clear(); - nodeReplicaMap.clear(); - colShardReplicaMap.clear(); - cloudManager.getSimNodeStateProvider().simGetAllNodeValues().forEach((n, values) -> { - values.put(ImplicitSnitch.CORES, 0); - values.put(ImplicitSnitch.DISK, SimCloudManager.DEFAULT_FREE_DISK); - values.put(Variable.Type.TOTALDISK.tagName, SimCloudManager.DEFAULT_TOTAL_DISK); - values.put(ImplicitSnitch.SYSLOADAVG, 1.0); - values.put(ImplicitSnitch.HEAPUSAGE, 123450000); - }); - cloudManager.getDistribStateManager().removeRecursively(ZkStateReader.COLLECTIONS_ZKNODE, true, false); - } finally { - lock.unlock(); - } - } - - private static final Set NO_COPY_PROPS = new HashSet<>(Arrays.asList( - ZkStateReader.CORE_NODE_NAME_PROP, - ZkStateReader.NODE_NAME_PROP, - ZkStateReader.BASE_URL_PROP, - ZkStateReader.CORE_NAME_PROP - )); - - /** - * Move replica. This uses a similar algorithm as {@link org.apache.solr.cloud.api.collections.MoveReplicaCmd} moveNormalReplica(...) method. - * @param message operation details - * @param results operation results. - */ - @SuppressWarnings({"unchecked", "rawtypes"}) - public void simMoveReplica(ZkNodeProps message, NamedList results) throws Exception { - ensureNotClosed(); - if (message.getStr(CommonAdminParams.ASYNC) != null) { - results.add(CoreAdminParams.REQUESTID, message.getStr(CommonAdminParams.ASYNC)); - } - String collection = message.getStr(COLLECTION_PROP); - String targetNode = message.getStr("targetNode"); - ClusterState clusterState = getClusterState(); - DocCollection coll = clusterState.getCollection(collection); - if (coll == null) { - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Collection: " + collection + " does not exist"); - } - String replicaName = message.getStr(REPLICA_PROP); - Replica replica = coll.getReplica(replicaName); - if (replica == null) { - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, - "Collection: " + collection + " replica: " + replicaName + " does not exist"); - } - Slice slice = null; - for (Slice s : coll.getSlices()) { - if (s.getReplicas().contains(replica)) { - slice = s; - } - } - if (slice == null) { - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Replica has no 'slice' property! : " + replica); - } - - opDelay(collection, CollectionParams.CollectionAction.MOVEREPLICA.name()); - Replica ri = getReplicaInfo(replica); - if (ri != null) { - if (ri.get(Type.CORE_IDX.tagName) != null) { - // simulate very large replicas - add additional delay of 5s / GB - long sizeInGB = ((Number)ri.get(Type.CORE_IDX.tagName)).longValue(); - long opDelay = opDelays.getOrDefault(ri.getCollection(), Collections.emptyMap()) - .getOrDefault(CollectionParams.CollectionAction.MOVEREPLICA.name(), defaultOpDelays.get(CollectionParams.CollectionAction.MOVEREPLICA.name())); - opDelay = TimeUnit.MILLISECONDS.toSeconds(opDelay); - if (sizeInGB > opDelay) { - // add 5s per each GB above the threshold - cloudManager.getTimeSource().sleep(TimeUnit.SECONDS.toMillis(sizeInGB - opDelay) * 5); - } - } - } - - // TODO: for now simulate moveNormalReplica sequence, where we first add new replica and then delete the old one - - String newSolrCoreName = Assign.buildSolrCoreName(stateManager, coll, slice.getName(), replica.getType()); - String coreNodeName = Assign.assignCoreNodeName(stateManager, coll); - // copy other properties - Map props = replica.getProperties().entrySet().stream() - .filter(e -> !NO_COPY_PROPS.contains(e.getKey())) - .collect(Collectors.toMap(e -> e.getKey(), e -> e.getValue())); - Replica newReplica = new Replica(coreNodeName, targetNode, collection, slice.getName(), newSolrCoreName, - Replica.State.DOWN, replica.getType(), props); - log.debug("-- new replica: {}", newReplica); - // xxx should run leader election here already? - simAddReplica(targetNode, newReplica, false); - // this will trigger leader election - simRemoveReplica(replica.getNodeName(), collection, replica.getName()); - results.add("success", ""); - } - - /** - * Create a new shard. This uses a similar algorithm as {@link CreateShardCmd}. - * @param message operation details - * @param results operation results - */ - @SuppressWarnings({"unchecked", "rawtypes"}) - public void simCreateShard(ZkNodeProps message, NamedList results) throws Exception { - ensureNotClosed(); - if (message.getStr(CommonAdminParams.ASYNC) != null) { - results.add(CoreAdminParams.REQUESTID, message.getStr(CommonAdminParams.ASYNC)); - } - String collectionName = message.getStr(COLLECTION_PROP); - String sliceName = message.getStr(SHARD_ID_PROP); - ClusterState clusterState = getClusterState(); - lock.lockInterruptibly(); - try { - ZkWriteCommand cmd = new CollectionMutator(cloudManager).createShard(clusterState, message); - if (cmd.noop) { - results.add("success", "no-op"); - return; - } - - opDelay(collectionName, CollectionParams.CollectionAction.CREATESHARD.name()); - - // copy shard properties -- our equivalent of creating an empty shard in cluster state - DocCollection collection = cmd.collection; - Slice slice = collection.getSlice(sliceName); - Map props = sliceProperties.computeIfAbsent(collection.getName(), c -> new ConcurrentHashMap<>()) - .computeIfAbsent(sliceName, s -> new ConcurrentHashMap<>()); - props.clear(); - slice.getProperties().entrySet().stream() - .filter(e -> !e.getKey().equals("range")) - .filter(e -> !e.getKey().equals("replicas")) - .forEach(e -> props.put(e.getKey(), e.getValue())); - // 2. create new replicas - EnumMap replicaTypesVsCount = new EnumMap<>(Replica.Type.class); - int numNrtReplicas = message.getInt(NRT_REPLICAS, message.getInt(REPLICATION_FACTOR, collection.getInt(NRT_REPLICAS, collection.getInt(REPLICATION_FACTOR, 1)))); - int numTlogReplicas = message.getInt(TLOG_REPLICAS, message.getInt(TLOG_REPLICAS, collection.getInt(TLOG_REPLICAS, 0))); - int numPullReplicas = message.getInt(PULL_REPLICAS, message.getInt(PULL_REPLICAS, collection.getInt(PULL_REPLICAS, 0))); - replicaTypesVsCount.put(Replica.Type.NRT, numNrtReplicas); - replicaTypesVsCount.put(Replica.Type.TLOG, numTlogReplicas); - replicaTypesVsCount.put(Replica.Type.PULL, numPullReplicas); - - ZkNodeProps addReplicasProps = new ZkNodeProps( - COLLECTION_PROP, collectionName, - SHARD_ID_PROP, sliceName, - ZkStateReader.NRT_REPLICAS, String.valueOf(replicaTypesVsCount.get(Replica.Type.NRT)), - ZkStateReader.TLOG_REPLICAS, String.valueOf(replicaTypesVsCount.get(Replica.Type.TLOG)), - ZkStateReader.PULL_REPLICAS, String.valueOf(replicaTypesVsCount.get(Replica.Type.PULL)), - OverseerCollectionMessageHandler.CREATE_NODE_SET, message.getStr(OverseerCollectionMessageHandler.CREATE_NODE_SET) - ); - - try { - // this also takes care of leader election - simAddReplica(addReplicasProps, results); - } catch (Exception e) { - throw new RuntimeException(e); - } - - collProperties.computeIfAbsent(collectionName, c -> new ConcurrentHashMap<>()); - results.add("success", ""); - } finally { - lock.unlock(); - } - } - - /** - * Split a shard. This uses a similar algorithm as {@link SplitShardCmd}, including simulating its - * quirks, and leaving the original parent slice in place. - * @param message operation details - * @param results operation results. - */ - @SuppressWarnings({"unchecked", "rawtypes"}) - public void simSplitShard(ZkNodeProps message, NamedList results) throws Exception { - ensureNotClosed(); - if (message.getStr(CommonAdminParams.ASYNC) != null) { - results.add(CoreAdminParams.REQUESTID, message.getStr(CommonAdminParams.ASYNC)); - } - String collectionName = message.getStr(COLLECTION_PROP); - AtomicReference sliceName = new AtomicReference<>(); - sliceName.set(message.getStr(SHARD_ID_PROP)); - String splitKey = message.getStr("split.key"); - String methodStr = message.getStr(CommonAdminParams.SPLIT_METHOD, SolrIndexSplitter.SplitMethod.REWRITE.toLower()); - SolrIndexSplitter.SplitMethod splitMethod = SolrIndexSplitter.SplitMethod.get(methodStr); - if (splitMethod == null) { - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Unknown value '" + CommonAdminParams.SPLIT_METHOD + - ": " + methodStr); - } - - ClusterState clusterState = getClusterState(); - DocCollection collection = clusterState.getCollection(collectionName); - Slice parentSlice = SplitShardCmd.getParentSlice(clusterState, collectionName, sliceName, splitKey); - Replica leader = parentSlice.getLeader(); - // XXX leader election may not have happened yet - should we require it? - if (leader == null) { - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Shard " + collectionName + - " / " + sliceName.get() + " has no leader and can't be split"); - } - SplitShardCmd.checkDiskSpace(collectionName, sliceName.get(), leader, splitMethod, cloudManager); - SplitShardCmd.lockForSplit(cloudManager, collectionName, sliceName.get()); - // start counting buffered updates - Map props = sliceProperties.computeIfAbsent(collectionName, c -> new ConcurrentHashMap<>()) - .computeIfAbsent(sliceName.get(), ss -> new ConcurrentHashMap<>()); - if (props.containsKey(BUFFERED_UPDATES)) { - SplitShardCmd.unlockForSplit(cloudManager, collectionName, sliceName.get()); - throw new Exception("--- SOLR-12729: Overlapping splitShard commands for " + collectionName + "/" + sliceName.get()); - } - props.put(BUFFERED_UPDATES, new AtomicLong()); - - List subRanges = new ArrayList<>(); - List subSlices = new ArrayList<>(); - List subShardNames = new ArrayList<>(); - - opDelay(collectionName, CollectionParams.CollectionAction.SPLITSHARD.name()); - - SplitShardCmd.fillRanges(cloudManager, message, collection, parentSlice, subRanges, subSlices, subShardNames, true); - // add replicas for new subShards - int repFactor = parentSlice.getReplicas().size(); - Assign.AssignRequest assignRequest = new Assign.AssignRequestBuilder() - .forCollection(collectionName) - .forShard(subSlices) - .assignNrtReplicas(repFactor) - .assignTlogReplicas(0) - .assignPullReplicas(0) - .onNodes(new ArrayList<>(clusterState.getLiveNodes())) - .build(); - Assign.AssignStrategyFactory assignStrategyFactory = new Assign.AssignStrategyFactory(cloudManager); - Assign.AssignStrategy assignStrategy = assignStrategyFactory.create(clusterState, collection); - // reproduce the bug - List replicaPositions = assignStrategy.assign(cloudManager, assignRequest); - PolicyHelper.SessionWrapper sessionWrapper = PolicyHelper.getLastSessionWrapper(true); - if (sessionWrapper != null) sessionWrapper.release(); - - // adjust numDocs / deletedDocs / maxDoc - String numDocsStr = String.valueOf(getReplicaInfo(leader).get("SEARCHER.searcher.numDocs", "0")); - long numDocs = Long.parseLong(numDocsStr); - long newNumDocs = numDocs / subSlices.size(); - long remainderDocs = numDocs % subSlices.size(); - long newIndexSize = SimCloudManager.DEFAULT_IDX_SIZE_BYTES + newNumDocs * DEFAULT_DOC_SIZE_BYTES; - long remainderIndexSize = SimCloudManager.DEFAULT_IDX_SIZE_BYTES + remainderDocs * DEFAULT_DOC_SIZE_BYTES; - String remainderSlice = null; - - // add slice props - for (int i = 0; i < subRanges.size(); i++) { - String subSlice = subSlices.get(i); - DocRouter.Range range = subRanges.get(i); - Map sliceProps = sliceProperties.computeIfAbsent(collectionName, c -> new ConcurrentHashMap<>()) - .computeIfAbsent(subSlice, ss -> new ConcurrentHashMap<>()); - sliceProps.put(Slice.RANGE, range); - sliceProps.put(Slice.PARENT, sliceName.get()); - sliceProps.put(ZkStateReader.STATE_PROP, Slice.State.CONSTRUCTION.toString()); - sliceProps.put(ZkStateReader.STATE_TIMESTAMP_PROP, String.valueOf(cloudManager.getTimeSource().getEpochTimeNs())); - } - // add replicas - for (ReplicaPosition replicaPosition : replicaPositions) { - String subSliceName = replicaPosition.shard; - String subShardNodeName = replicaPosition.node; -// String solrCoreName = collectionName + "_" + subSliceName + "_replica_n" + (replicaPosition.index); - String solrCoreName = Assign.buildSolrCoreName(collectionName, subSliceName, replicaPosition.type, Assign.incAndGetId(stateManager, collectionName, 0)); - Map replicaProps = new HashMap<>(); - replicaProps.put(ZkStateReader.SHARD_ID_PROP, replicaPosition.shard); - replicaProps.put(ZkStateReader.NODE_NAME_PROP, replicaPosition.node); - replicaProps.put(ZkStateReader.REPLICA_TYPE, replicaPosition.type.toString()); - replicaProps.put(ZkStateReader.BASE_URL_PROP, Utils.getBaseUrlForNodeName(subShardNodeName, "http")); - - long replicasNumDocs = newNumDocs; - long replicasIndexSize = newIndexSize; - if (remainderSlice == null) { - remainderSlice = subSliceName; - } - if (remainderSlice.equals(subSliceName)) { // only add to one sub slice - replicasNumDocs += remainderDocs; - replicasIndexSize += remainderIndexSize; - } - replicaProps.put("SEARCHER.searcher.numDocs", new AtomicLong(replicasNumDocs)); - replicaProps.put("SEARCHER.searcher.maxDoc", new AtomicLong(replicasNumDocs)); - replicaProps.put("SEARCHER.searcher.deletedDocs", new AtomicLong(0)); - replicaProps.put(Type.CORE_IDX.metricsAttribute, new AtomicLong(replicasIndexSize)); - replicaProps.put(Variable.coreidxsize, new AtomicDouble((Double)Type.CORE_IDX.convertVal(replicasIndexSize))); - - Replica ri = new Replica("core_node" + Assign.incAndGetId(stateManager, collectionName, 0), - subShardNodeName, collectionName, replicaPosition.shard, solrCoreName, - Replica.State.DOWN, replicaPosition.type, replicaProps); - simAddReplica(replicaPosition.node, ri, false); - } - simRunLeaderElection(Collections.singleton(collectionName), true); - - // delay it once again to better simulate replica recoveries - //opDelay(collectionName, CollectionParams.CollectionAction.SPLITSHARD.name()); - - boolean success = false; - try { - CloudUtil.waitForState(cloudManager, collectionName, 30, TimeUnit.SECONDS, (liveNodes, state) -> { - for (String subSlice : subSlices) { - Slice s = state.getSlice(subSlice); - if (s.getLeader() == null) { - log.debug("** no leader in {} / {}", collectionName, s); - return false; - } - if (s.getReplicas().size() < repFactor) { - if (log.isDebugEnabled()) { - log.debug("** expected {} repFactor but there are {} replicas", repFactor, s.getReplicas().size()); - } - return false; - } - } - return true; - }); - success = true; - } finally { - if (!success) { - Map sProps = sliceProperties.computeIfAbsent(collectionName, c -> new ConcurrentHashMap<>()) - .computeIfAbsent(sliceName.get(), s -> new ConcurrentHashMap<>()); - sProps.remove(BUFFERED_UPDATES); - SplitShardCmd.unlockForSplit(cloudManager, collectionName, sliceName.get()); - } - } - // mark the new slices as active and the old slice as inactive - if (log.isTraceEnabled()) { - log.trace("-- switching slice states after split shard: collection={}, parent={}, subSlices={}", collectionName, - sliceName.get(), subSlices); - } - lock.lockInterruptibly(); - try { - Map sProps = sliceProperties.computeIfAbsent(collectionName, c -> new ConcurrentHashMap<>()) - .computeIfAbsent(sliceName.get(), s -> new ConcurrentHashMap<>()); - sProps.put(ZkStateReader.STATE_PROP, Slice.State.INACTIVE.toString()); - sProps.put(ZkStateReader.STATE_TIMESTAMP_PROP, String.valueOf(cloudManager.getTimeSource().getEpochTimeNs())); - AtomicLong bufferedUpdates = (AtomicLong)sProps.remove(BUFFERED_UPDATES); - if (bufferedUpdates.get() > 0) { - // apply buffered updates - long perShard = bufferedUpdates.get() / subSlices.size(); - long remainder = bufferedUpdates.get() % subSlices.size(); - if (log.isDebugEnabled()) { - log.debug("-- applying {} buffered docs from {} / {}, perShard={}, remainder={}", bufferedUpdates.get(), - collectionName, parentSlice.getName(), perShard, remainder); - } - for (int i = 0; i < subSlices.size(); i++) { - String sub = subSlices.get(i); - long numUpdates = perShard; - if (i == 0) { - numUpdates += remainder; - } - simSetShardValue(collectionName, sub, "SEARCHER.searcher.numDocs", numUpdates, true, false); - simSetShardValue(collectionName, sub, "SEARCHER.searcher.maxDoc", numUpdates, true, false); - } - } - // XXX also mark replicas as down? currently SplitShardCmd doesn't do this - - for (String s : subSlices) { - Map sliceProps = sliceProperties.computeIfAbsent(collectionName, c -> new ConcurrentHashMap<>()) - .computeIfAbsent(s, ss -> new ConcurrentHashMap<>()); - sliceProps.put(ZkStateReader.STATE_PROP, Slice.State.ACTIVE.toString()); - sliceProps.put(ZkStateReader.STATE_TIMESTAMP_PROP, String.valueOf(cloudManager.getTimeSource().getEpochTimeNs())); - } - - // invalidate cached state - collectionsStatesRef.get(collectionName).invalidate(); - } finally { - SplitShardCmd.unlockForSplit(cloudManager, collectionName, sliceName.get()); - lock.unlock(); - } - results.add("success", ""); - - } - - /** - * Delete a shard. This uses a similar algorithm as {@link org.apache.solr.cloud.api.collections.DeleteShardCmd} - * @param message operation details - * @param results operation results - */ - @SuppressWarnings({"unchecked", "rawtypes"}) - public void simDeleteShard(ZkNodeProps message, NamedList results) throws Exception { - ensureNotClosed(); - if (message.getStr(CommonAdminParams.ASYNC) != null) { - results.add(CoreAdminParams.REQUESTID, message.getStr(CommonAdminParams.ASYNC)); - } - String collectionName = message.getStr(COLLECTION_PROP); - String sliceName = message.getStr(SHARD_ID_PROP); - ClusterState clusterState = getClusterState(); - DocCollection collection = clusterState.getCollection(collectionName); - if (collection == null) { - throw new Exception("Collection " + collectionName + " doesn't exist"); - } - Slice slice = collection.getSlice(sliceName); - if (slice == null) { - throw new Exception(" Collection " + collectionName + " slice " + sliceName + " doesn't exist."); - } - - opDelay(collectionName, CollectionParams.CollectionAction.DELETESHARD.name()); - - lock.lockInterruptibly(); - try { - sliceProperties.computeIfAbsent(collectionName, c -> new ConcurrentHashMap<>()).remove(sliceName); - colShardReplicaMap.computeIfAbsent(collectionName, c -> new ConcurrentHashMap<>()).remove(sliceName); - nodeReplicaMap.forEach((n, replicas) -> { - synchronized (replicas) { - Iterator it = replicas.iterator(); - while (it.hasNext()) { - Replica ri = it.next(); - if (ri.getCollection().equals(collectionName) && ri.getShard().equals(sliceName)) { - it.remove(); - } - } - } - }); - collectionsStatesRef.get(collectionName).invalidate(); - results.add("success", ""); - } catch (Exception e) { - results.add("failure", e.toString()); - } finally { - lock.unlock(); - } - } - - @SuppressWarnings({"rawtypes"}) - public void createSystemCollection() throws IOException { - try { - - synchronized (this) { - if (colShardReplicaMap.containsKey(CollectionAdminParams.SYSTEM_COLL)) { - return; - } - } - String repFactor = String.valueOf(Math.min(3, liveNodes.size())); - ZkNodeProps props = new ZkNodeProps( - NAME, CollectionAdminParams.SYSTEM_COLL, - REPLICATION_FACTOR, repFactor, - OverseerCollectionMessageHandler.NUM_SLICES, "1", - CommonAdminParams.WAIT_FOR_FINAL_STATE, "true"); - simCreateCollection(props, new NamedList()); - CloudUtil.waitForState(cloudManager, CollectionAdminParams.SYSTEM_COLL, 120, TimeUnit.SECONDS, - CloudUtil.clusterShape(1, Integer.parseInt(repFactor), false, true)); - } catch (Exception e) { - throw new IOException(e); - } - } - - /** - * Simulate an update by modifying replica metrics. - * The following core metrics are updated: - *
    - *
  • SEARCHER.searcher.numDocs - increased by added docs, decreased by deleteById and deleteByQuery
    • - *
  • SEARCHER.searcher.deletedDocs - decreased by deleteById and deleteByQuery by up to numDocs
    • - *
  • SEARCHER.searcher.maxDoc - always increased by the number of added docs.
    • - *
- *

IMPORTANT limitations:

- *
    - *
  • document replacements are always counted as new docs
    • - *
  • delete by ID always succeeds (unless numDocs == 0)
    • - *
  • deleteByQuery is not supported unless the query is *:*
    • - *
- * @param req update request. This request MUST have the collection param set. - * @return {@link UpdateResponse} - * @throws SolrException on errors, such as nonexistent collection or unsupported deleteByQuery - */ - public UpdateResponse simUpdate(UpdateRequest req) throws SolrException, InterruptedException, IOException { - ensureNotClosed(); - String collection = req.getCollection(); - if (collection == null) { - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Collection not set"); - } - ensureSystemCollection(collection); - DocCollection coll = getClusterState().getCollection(collection); - DocRouter router = coll.getRouter(); - List deletes = req.getDeleteById(); - Map freediskDeltaPerNode = new HashMap<>(); - if (deletes != null && !deletes.isEmpty()) { - Map deletesPerShard = new HashMap<>(); - Map indexSizePerShard = new HashMap<>(); - for (String id : deletes) { - Slice s = router.getTargetSlice(id, null, null, req.getParams(), coll); - Replica leader = s.getLeader(); - if (leader == null) { - throw new IOException("-- no leader in " + s); - } - cloudManager.getMetricManager().registry(createRegistryName(collection, s.getName(), leader)).counter("UPDATE./update.requests").inc(); - Replica ri = getReplicaInfo(leader); - Number numDocs = (Number)ri.get("SEARCHER.searcher.numDocs"); - if (numDocs == null || numDocs.intValue() <= 0) { - if (log.isDebugEnabled()) { - log.debug("-- attempting to delete nonexistent doc {} from {}", id, s.getLeader()); - } - continue; - } - - // this is somewhat wrong - we should wait until buffered updates are applied - // but this way the freedisk changes are much easier to track - s.getReplicas().forEach(r -> - freediskDeltaPerNode.computeIfAbsent(r.getNodeName(), node -> new AtomicLong(0)) - .addAndGet(DEFAULT_DOC_SIZE_BYTES)); - - AtomicLong bufferedUpdates = (AtomicLong)sliceProperties.get(collection).get(s.getName()).get(BUFFERED_UPDATES); - if (bufferedUpdates != null) { - if (bufferedUpdates.get() > 0) { - bufferedUpdates.decrementAndGet(); - } else { - if (log.isDebugEnabled()) { - log.debug("-- attempting to delete nonexistent buffered doc {} from {}", id, s.getLeader()); - } - } - continue; - } - deletesPerShard.computeIfAbsent(s.getName(), slice -> new AtomicLong(0)).incrementAndGet(); - Number indexSize = (Number)ri.get(Type.CORE_IDX.metricsAttribute); - if (indexSize != null) { - indexSizePerShard.put(s.getName(), indexSize); - } - } - if (!deletesPerShard.isEmpty()) { - lock.lockInterruptibly(); - try { - for (Map.Entry entry : deletesPerShard.entrySet()) { - String shard = entry.getKey(); - simSetShardValue(collection, shard, "SEARCHER.searcher.deletedDocs", entry.getValue().get(), true, false); - simSetShardValue(collection, shard, "SEARCHER.searcher.numDocs", -entry.getValue().get(), true, false); - Number indexSize = indexSizePerShard.get(shard); - long delSize = DEFAULT_DOC_SIZE_BYTES * entry.getValue().get(); - if (indexSize != null) { - indexSize = indexSize.longValue() - delSize; - if (indexSize.longValue() < SimCloudManager.DEFAULT_IDX_SIZE_BYTES) { - indexSize = SimCloudManager.DEFAULT_IDX_SIZE_BYTES; - } - simSetShardValue(collection, shard, Type.CORE_IDX.metricsAttribute, - new AtomicLong(indexSize.longValue()), false, false); - simSetShardValue(collection, shard, Variable.coreidxsize, - new AtomicDouble((Double)Type.CORE_IDX.convertVal(indexSize)), false, false); - } else { - throw new Exception("unexpected indexSize for collection=" + collection + ", shard=" + shard + ": " + indexSize); - } - } - } catch (Exception e) { - throw new IOException(e); - } finally { - lock.unlock(); - } - } - } - deletes = req.getDeleteQuery(); - if (deletes != null && !deletes.isEmpty()) { - for (String q : deletes) { - if (!"*:*".equals(q)) { - throw new UnsupportedOperationException("Only '*:*' query is supported in deleteByQuery"); - } - //log.debug("-- req delByQ {}", collection); - for (Slice s : coll.getSlices()) { - Replica leader = s.getLeader(); - if (leader == null) { - throw new IOException("-- no leader in " + s); - } - - cloudManager.getMetricManager().registry(createRegistryName(collection, s.getName(), leader)).counter("UPDATE./update.requests").inc(); - Replica ri = getReplicaInfo(leader); - Number numDocs = (Number)ri.get("SEARCHER.searcher.numDocs"); - if (numDocs == null || numDocs.intValue() == 0) { - continue; - } - lock.lockInterruptibly(); - try { - Number indexSize = (Number)ri.get(Type.CORE_IDX.metricsAttribute); - if (indexSize != null) { - long delta = indexSize.longValue() < SimCloudManager.DEFAULT_IDX_SIZE_BYTES ? 0 : - indexSize.longValue() - SimCloudManager.DEFAULT_IDX_SIZE_BYTES; - s.getReplicas().forEach(r -> - freediskDeltaPerNode.computeIfAbsent(r.getNodeName(), node -> new AtomicLong(0)) - .addAndGet(delta)); - } else { - throw new RuntimeException("Missing index size in " + ri); - } - simSetShardValue(collection, s.getName(), "SEARCHER.searcher.deletedDocs", new AtomicLong(numDocs.longValue()), false, false); - simSetShardValue(collection, s.getName(), "SEARCHER.searcher.numDocs", new AtomicLong(0), false, false); - simSetShardValue(collection, s.getName(), Type.CORE_IDX.metricsAttribute, - new AtomicLong(SimCloudManager.DEFAULT_IDX_SIZE_BYTES), false, false); - simSetShardValue(collection, s.getName(), Variable.coreidxsize, - new AtomicDouble((Double)Type.CORE_IDX.convertVal(SimCloudManager.DEFAULT_IDX_SIZE_BYTES)), false, false); - } catch (Exception e) { - throw new IOException(e); - } finally { - lock.unlock(); - } - } - } - } - List docs = req.getDocuments(); - int docCount = 0; - Iterator it = null; - if (docs != null) { - docCount = docs.size(); - } else { - it = req.getDocIterator(); - if (it != null) { - while (it.hasNext()) { - it.next(); - docCount++; - } - } - } - if (docCount > 0) { - //log.debug("-- req update {}/{}", collection, docCount); - // this approach to updating counters and metrics drastically increases performance - // of bulk updates, because simSetShardValue is relatively costly - - Map docUpdates = new HashMap<>(); - Map> metricUpdates = new HashMap<>(); - - // XXX don't add more than 2bln docs in one request - boolean modified = false; - lock.lockInterruptibly(); - try { - coll = getClusterState().getCollection(collection); - Slice[] slices = coll.getActiveSlicesArr(); - if (slices.length == 0) { - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Collection without slices"); - } - int[] perSlice = new int[slices.length]; - - if (it != null) { - // BULK UPDATE: simulate random doc assignment without actually calling DocRouter, - // which adds significant overhead - - int totalAdded = 0; - for (int i = 0; i < slices.length; i++) { - Slice s = slices[i]; - long count = (long) docCount * ((long) s.getRange().max - (long) s.getRange().min) / 0x100000000L; - perSlice[i] = (int) count; - totalAdded += perSlice[i]; - } - // loss of precision due to integer math - int diff = docCount - totalAdded; - if (diff > 0) { - // spread the remainder more or less equally - int perRemain = diff / slices.length; - int remainder = diff % slices.length; - int remainderSlice = slices.length > 1 ? bulkUpdateRandom.nextInt(slices.length) : 0; - for (int i = 0; i < slices.length; i++) { - perSlice[i] += perRemain; - if (i == remainderSlice) { - perSlice[i] += remainder; - } - } - } - for (int i = 0; i < slices.length; i++) { - Slice s = slices[i]; - Replica leader = s.getLeader(); - if (leader == null) { - throw new IOException("-- no leader in " + s); - } - metricUpdates.computeIfAbsent(s.getName(), sh -> new HashMap<>()) - .computeIfAbsent(leader.getCoreName(), cn -> new AtomicLong()) - .addAndGet(perSlice[i]); - modified = true; - long perSliceCount = perSlice[i]; - s.getReplicas().forEach(r -> - freediskDeltaPerNode.computeIfAbsent(r.getNodeName(), node -> new AtomicLong(0)) - .addAndGet(-perSliceCount * DEFAULT_DOC_SIZE_BYTES)); - AtomicLong bufferedUpdates = (AtomicLong)sliceProperties.get(collection).get(s.getName()).get(BUFFERED_UPDATES); - if (bufferedUpdates != null) { - bufferedUpdates.addAndGet(perSlice[i]); - continue; - } - docUpdates.computeIfAbsent(s.getName(), sh -> new AtomicLong()) - .addAndGet(perSlice[i]); - } - } else { - // SMALL UPDATE: use exact assignment via DocRouter - for (SolrInputDocument doc : docs) { - String id = (String) doc.getFieldValue("id"); - if (id == null) { - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Document without id: " + doc); - } - Slice s = coll.getRouter().getTargetSlice(id, doc, null, null, coll); - Replica leader = s.getLeader(); - if (leader == null) { - throw new IOException("-- no leader in " + s); - } - metricUpdates.computeIfAbsent(s.getName(), sh -> new HashMap<>()) - .computeIfAbsent(leader.getCoreName(), cn -> new AtomicLong()) - .incrementAndGet(); - modified = true; - s.getReplicas().forEach(r -> - freediskDeltaPerNode.computeIfAbsent(r.getNodeName(), node -> new AtomicLong()) - .addAndGet(-DEFAULT_DOC_SIZE_BYTES)); - AtomicLong bufferedUpdates = (AtomicLong)sliceProperties.get(collection).get(s.getName()).get(BUFFERED_UPDATES); - if (bufferedUpdates != null) { - bufferedUpdates.incrementAndGet(); - continue; - } - docUpdates.computeIfAbsent(s.getName(), sh -> new AtomicLong()) - .incrementAndGet(); - } - } - - if (modified) { - docUpdates.forEach((sh, count) -> { - try { - simSetShardValue(collection, sh, "SEARCHER.searcher.numDocs", count.get(), true, false); - simSetShardValue(collection, sh, "SEARCHER.searcher.maxDoc", count.get(), true, false); - simSetShardValue(collection, sh, "UPDATE./update.requests", count.get(), true, false); - // for each new document increase the size by DEFAULT_DOC_SIZE_BYTES - simSetShardValue(collection, sh, Type.CORE_IDX.metricsAttribute, - DEFAULT_DOC_SIZE_BYTES * count.get(), true, false); - simSetShardValue(collection, sh, Variable.coreidxsize, - Type.CORE_IDX.convertVal(DEFAULT_DOC_SIZE_BYTES * count.get()), true, false); - } catch (Exception e) { - throw new RuntimeException(e); - } - }); - metricUpdates.forEach((sh, cores) -> { - cores.forEach((core, count) -> { - String registry = SolrMetricManager.getRegistryName(SolrInfoBean.Group.core, collection, sh, - Utils.parseMetricsReplicaName(collection, core)); - cloudManager.getMetricManager().registry(registry).counter("UPDATE./update.requests").inc(count.get()); - }); - }); - } - } finally { - lock.unlock(); - } - } - if (!freediskDeltaPerNode.isEmpty()) { - SimNodeStateProvider nodeStateProvider = cloudManager.getSimNodeStateProvider(); - freediskDeltaPerNode.forEach((node, delta) -> { - if (delta.get() == 0) { - return; - } - try { - // this method does its own locking to prevent races - nodeStateProvider.simUpdateNodeValue(node, Type.FREEDISK.tagName, val -> { - if (val == null) { - throw new RuntimeException("no freedisk for node " + node); - } - double freedisk = ((Number) val).doubleValue(); - double deltaGB = (Double) Type.FREEDISK.convertVal(delta.get()); - freedisk += deltaGB; - if (freedisk < 0) { - log.warn("-- freedisk={} - ran out of disk space on node {}", freedisk, node); - freedisk = 0; - } - return freedisk; - }); - } catch (Exception e) { - throw new RuntimeException(e); - } - }); - } - SolrParams params = req.getParams(); - if (params != null && (params.getBool(UpdateParams.OPTIMIZE, false) || params.getBool(UpdateParams.EXPUNGE_DELETES, false))) { - lock.lockInterruptibly(); - try { - coll.getSlices().forEach(s -> { - Replica leader = s.getLeader(); - Replica ri = getReplicaInfo(leader); - Number numDocs = (Number)ri.get("SEARCHER.searcher.numDocs"); - if (numDocs == null || numDocs.intValue() == 0) { - numDocs = 0; - } - try { - simSetShardValue(ri.getCollection(), ri.getShard(), "SEARCHER.searcher.maxDoc", numDocs, false, false); - simSetShardValue(ri.getCollection(), ri.getShard(), "SEARCHER.searcher.deletedDocs", 0, false, false); - } catch (Exception e) { - throw new RuntimeException(e); - } - }); - } finally { - lock.unlock(); - } - } - return new UpdateResponse(); - } - - public QueryResponse simQuery(QueryRequest req) throws SolrException, InterruptedException, IOException { - ensureNotClosed(); - String collection = req.getCollection(); - if (collection == null) { - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Collection not set"); - } - ensureSystemCollection(collection); - if (!colShardReplicaMap.containsKey(collection)) { - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Collection does not exist"); - } - String query = req.getParams().get(CommonParams.Q); - if (query == null || !query.equals("*:*")) { - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Only '*:*' query is supported"); - } - ClusterState clusterState = getClusterState(); - DocCollection coll = clusterState.getCollection(collection); - AtomicLong count = new AtomicLong(); - for (Slice s : coll.getActiveSlicesArr()) { - Replica r = s.getLeader(); - if (r == null) { - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, collection + "/" + s.getName() + " has no leader"); - } - Replica ri = getReplicaInfo(r); - Number numDocs = (Number)ri.get("SEARCHER.searcher.numDocs", 0L); - count.addAndGet(numDocs.longValue()); - AtomicLong bufferedUpdates = (AtomicLong)sliceProperties.get(collection).get(s.getName()).get(BUFFERED_UPDATES); - if (bufferedUpdates != null) { - count.addAndGet(bufferedUpdates.get()); - } - } - QueryResponse rsp = new QueryResponse(); - NamedList values = new NamedList<>(); - values.add("responseHeader", new NamedList<>()); - SolrDocumentList docs = new SolrDocumentList(); - docs.setNumFound(count.get()); - values.add("response", docs); - rsp.setResponse(values); - return rsp; - } - - private void ensureSystemCollection(String collection) throws InterruptedException, IOException { - if (!simListCollections().contains(collection)) { - if (CollectionAdminParams.SYSTEM_COLL.equals(collection)) { - // auto-create - createSystemCollection(); - } else { - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Collection '" + collection + "' doesn't exist"); - } - } - } - - private static String createRegistryName(String collection, String shard, Replica r) { - return SolrMetricManager.getRegistryName(SolrInfoBean.Group.core, collection, shard, - Utils.parseMetricsReplicaName(collection, r.getCoreName())); - } - - /** - * Saves cluster properties to clusterprops.json. - * @return current properties - */ - @SuppressWarnings({"unchecked"}) - private synchronized Map saveClusterProperties() throws Exception { - if (lastSavedProperties != null && lastSavedProperties.equals(clusterProperties)) { - return lastSavedProperties; - } - byte[] data = Utils.toJSON(clusterProperties); - VersionedData oldData = stateManager.getData(ZkStateReader.CLUSTER_PROPS); - int version = oldData != null ? oldData.getVersion() : -1; - stateManager.setData(ZkStateReader.CLUSTER_PROPS, data, version); - lastSavedProperties = new ConcurrentHashMap<>((Map)Utils.fromJSON(data)); - return lastSavedProperties; - } - - /** - * Set all cluster properties. This also updates the clusterprops.json data in - * {@link DistribStateManager} - * @param properties properties to set - */ - public void simSetClusterProperties(Map properties) throws Exception { - lock.lockInterruptibly(); - try { - clusterProperties.clear(); - if (properties != null) { - this.clusterProperties.putAll(properties); - } - saveClusterProperties(); - } finally { - lock.unlock(); - } - } - - /** - * Set a cluster property. This also updates the clusterprops.json data in - * {@link DistribStateManager} - * @param key property name - * @param value property value - */ - public void simSetClusterProperty(String key, Object value) throws Exception { - lock.lockInterruptibly(); - try { - if (value != null) { - clusterProperties.put(key, value); - } else { - clusterProperties.remove(key); - } - saveClusterProperties(); - } finally { - lock.unlock(); - } - } - - /** - * Set collection properties. - * @param coll collection name - * @param properties properties - */ - public void simSetCollectionProperties(String coll, Map properties) throws Exception { - lock.lockInterruptibly(); - try { - if (properties == null) { - collProperties.remove(coll); - } else { - Map props = collProperties.computeIfAbsent(coll, c -> new HashMap<>()); - props.clear(); - props.putAll(properties); - } - collectionsStatesRef.get(coll).invalidate(); - } finally { - lock.unlock(); - } - } - - /** - * Set collection property. - * @param coll collection name - * @param key property name - * @param value property value - */ - public void simSetCollectionProperty(String coll, String key, String value) throws Exception { - lock.lockInterruptibly(); - try { - final Map props = collProperties.computeIfAbsent(coll, c -> new HashMap<>()); - if (value == null) { - props.remove(key); - } else { - props.put(key, value); - } - collectionsStatesRef.get(coll).invalidate(); - } finally { - lock.unlock(); - } - } - - /** - * Set slice properties. - * @param coll collection name - * @param slice slice name - * @param properties slice properties - */ - public void simSetSliceProperties(String coll, String slice, Map properties) throws Exception { - lock.lockInterruptibly(); - try { - final Map sliceProps = sliceProperties.computeIfAbsent - (coll, c -> new HashMap<>()).computeIfAbsent(slice, s -> new HashMap<>()); - sliceProps.clear(); - if (properties != null) { - sliceProps.putAll(properties); - } - collectionsStatesRef.get(coll).invalidate(); - } finally { - lock.unlock(); - } - } - - /** - * Set per-collection value (eg. a metric). This value will be applied to each replica. - * @param collection collection name - * @param key property name - * @param value property value - */ - public void simSetCollectionValue(String collection, String key, Object value) throws Exception { - simSetCollectionValue(collection, key, value, false, false); - } - - /** - * Set per-collection value (eg. a metric). This value will be applied to each replica. - * @param collection collection name - * @param key property name - * @param value property value - * @param divide if the value is a {@link Number} and this param is true, then the value will be evenly - * divided by the number of replicas. - */ - public void simSetCollectionValue(String collection, String key, Object value, boolean delta, boolean divide) throws Exception { - simSetShardValue(collection, null, key, value, delta, divide); - } - - /** - * Set per-collection value (eg. a metric). This value will be applied to each replica in a selected shard. - * @param collection collection name - * @param shard shard name. If null then all shards will be affected. - * @param key property name - * @param value property value - */ - public void simSetShardValue(String collection, String shard, String key, Object value) throws Exception { - simSetShardValue(collection, shard, key, value, false, false); - } - - /** - * Set per-collection value (eg. a metric). This value will be applied to each replica in a selected shard. - * @param collection collection name - * @param shard shard name. If null then all shards will be affected. - * @param key property name - * @param value property value - * @param delta if true then treat the numeric value as delta to add to the existing value - * (or set the value to delta if missing) - * @param divide if the value is a {@link Number} and this is true, then the value will be evenly - * divided by the number of replicas. - */ - public void simSetShardValue(String collection, String shard, String key, Object value, boolean delta, boolean divide) throws Exception { - final List infos; - if (shard == null) { - infos = new ArrayList<>(); - colShardReplicaMap.computeIfAbsent(collection, c -> new ConcurrentHashMap<>()) - .forEach((sh, replicas) -> infos.addAll(replicas)); - } else { - infos = colShardReplicaMap.computeIfAbsent(collection, c -> new ConcurrentHashMap<>()) - .computeIfAbsent(shard, s -> new ArrayList<>()); - } - if (infos.isEmpty()) { - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Collection " + collection + " doesn't exist (shard=" + shard + ")."); - } - if (divide && value != null && (value instanceof Number)) { - if ((value instanceof Long) || (value instanceof Integer)) { - value = ((Number) value).longValue() / infos.size(); - } else { - value = ((Number) value).doubleValue() / infos.size(); - } - } - for (Replica r : infos) { - synchronized (r) { - if (value == null) { - r.getProperties().remove(key); - } else { - if (delta) { - Object prevValue = r.getProperties().get(key); - if (prevValue != null) { - if ((prevValue instanceof Number) && (value instanceof Number)) { - if (((prevValue instanceof Long) || (prevValue instanceof Integer) || - (prevValue instanceof AtomicLong) || (prevValue instanceof AtomicInteger)) && - ((value instanceof Long) || (value instanceof Integer))) { - long newValue = ((Number)prevValue).longValue() + ((Number)value).longValue(); - // minimize object allocations - if (prevValue instanceof AtomicLong) { - ((AtomicLong)prevValue).set(newValue); - } else if (prevValue instanceof AtomicInteger) { - ((AtomicInteger)prevValue).set(((Number)prevValue).intValue() + ((Number)value).intValue()); - } else { - r.getProperties().put(key, newValue); - } - } else { - double newValue = ((Number)prevValue).doubleValue() + ((Number)value).doubleValue(); - if (prevValue instanceof AtomicDouble) { - ((AtomicDouble)prevValue).set(newValue); - } else { - r.getProperties().put(key, newValue); - } - } - } else { - throw new UnsupportedOperationException("delta cannot be applied to non-numeric values: " + prevValue + " and " + value); - } - } else { - if (value instanceof Integer) { - r.getProperties().put(key, new AtomicInteger((Integer)value)); - } else if (value instanceof Long) { - r.getProperties().put(key, new AtomicLong((Long)value)); - } else if (value instanceof Double) { - r.getProperties().put(key, new AtomicDouble((Double)value)); - } else { - r.getProperties().put(key, value); - } - } - } else { - if (value instanceof Integer) { - r.getProperties().put(key, new AtomicInteger((Integer)value)); - } else if (value instanceof Long) { - r.getProperties().put(key, new AtomicLong((Long)value)); - } else if (value instanceof Double) { - r.getProperties().put(key, new AtomicDouble((Double)value)); - } else { - r.getProperties().put(key, value); - } - } - } - } - } - } - - @SuppressWarnings({"unchecked"}) - public void simSetReplicaValues(String node, Map>> source, boolean overwrite) { - List infos = nodeReplicaMap.get(node); - if (infos == null) { - throw new RuntimeException("Node not present: " + node); - } - // core_node_name is not unique across collections - Map> infoMap = new HashMap<>(); - infos.forEach(ri -> infoMap.computeIfAbsent(ri.getCollection(), Utils.NEW_HASHMAP_FUN).put(ri.getName(), ri)); - source.forEach((coll, shards) -> shards.forEach((shard, replicas) -> replicas.forEach(r -> { - Replica target = infoMap.getOrDefault(coll, Collections.emptyMap()).get(r.getName()); - if (target == null) { - throw new RuntimeException("Unable to find simulated replica of " + r); - } - r.getProperties().forEach((k, v) -> { - if (target.getProperties().containsKey(k)) { - if (overwrite) { - target.getProperties().put(k, v); - } - } else { - target.getProperties().put(k, v); - } - }); - }))); - } - - /** - * Return all replica infos for a node. - * @param node node id - * @return copy of the list of replicas on that node, or empty list if none - */ - public List simGetReplicaInfos(String node) { - @SuppressWarnings({"unchecked"}) - final List replicas = nodeReplicaMap.computeIfAbsent - (node, Utils.NEW_SYNCHRONIZED_ARRAYLIST_FUN); - // make a defensive copy to avoid ConcurrentModificationException - return Arrays.asList(replicas.toArray(new Replica[replicas.size()])); - } - - public List simGetReplicaInfos(String collection, String shard) { - List replicas = colShardReplicaMap.computeIfAbsent(collection, c -> new ConcurrentHashMap<>()) - .computeIfAbsent(shard, s -> new ArrayList<>()); - if (replicas == null) { - return Collections.emptyList(); - } else { - // make a defensive copy to avoid ConcurrentModificationException - return Arrays.asList(replicas.toArray(new Replica[replicas.size()])); - } - } - - public Replica simGetReplicaInfo(String collection, String coreNode) { - Map> shardsReplicas = colShardReplicaMap.computeIfAbsent(collection, c -> new ConcurrentHashMap<>()); - for (List replicas : shardsReplicas.values()) { - for (Replica ri : replicas) { - if (ri.getName().equals(coreNode)) { - return ri; - } - } - } - return null; - } - - /** - * List collections. - * @return list of existing collections. - */ - public List simListCollections() throws InterruptedException { - return new ArrayList<>(colShardReplicaMap.keySet()); - } - - public Map> simGetCollectionStats() throws IOException, InterruptedException { - lock.lockInterruptibly(); - try { - final Map> stats = new TreeMap<>(); - ClusterState state = getClusterState(); - state.forEachCollection(coll -> { - Map perColl = new LinkedHashMap<>(); - stats.put(coll.getName(), perColl); - perColl.put("shardsTotal", coll.getSlices().size()); - Map shardState = new TreeMap<>(); - int noLeader = 0; - - SummaryStatistics docs = new SummaryStatistics(); - SummaryStatistics bytes = new SummaryStatistics(); - SummaryStatistics inactiveDocs = new SummaryStatistics(); - SummaryStatistics inactiveBytes = new SummaryStatistics(); - - long deletedDocs = 0; - long bufferedDocs = 0; - int totalReplicas = 0; - int activeReplicas = 0; - - for (Slice s : coll.getSlices()) { - shardState.computeIfAbsent(s.getState().toString(), st -> new AtomicInteger()) - .incrementAndGet(); - totalReplicas += s.getReplicas().size(); - if (s.getState() != Slice.State.ACTIVE) { - if (!s.getReplicas().isEmpty()) { - Replica ri = getReplicaInfo(s.getReplicas().iterator().next()); - if (ri != null) { - Number numDocs = (Number)ri.get("SEARCHER.searcher.numDocs"); - Number numBytes = (Number)ri.get(Type.CORE_IDX.metricsAttribute); - if (numDocs != null) { - inactiveDocs.addValue(numDocs.doubleValue()); - } - if (numBytes != null) { - inactiveBytes.addValue(numBytes.doubleValue()); - } - } - } - continue; - } - AtomicLong buffered = (AtomicLong)sliceProperties - .getOrDefault(coll.getName(), Collections.emptyMap()) - .getOrDefault(s.getName(), Collections.emptyMap()).get(BUFFERED_UPDATES); - if (buffered != null) { - bufferedDocs += buffered.get(); - } - - for (Replica r : s.getReplicas()) { - if (r.getState() == Replica.State.ACTIVE) { - activeReplicas++; - } - } - Replica leader = s.getLeader(); - if (leader == null) { - noLeader++; - if (!s.getReplicas().isEmpty()) { - leader = s.getReplicas().iterator().next(); - } - } - Replica ri = null; - if (leader != null) { - ri = getReplicaInfo(leader); - if (ri == null) { - log.warn("Unknown ReplicaInfo for {}", leader); - } - } - if (ri != null) { - Number numDocs = (Number)ri.get("SEARCHER.searcher.numDocs"); - Number delDocs = (Number)ri.get("SEARCHER.searcher.deleteDocs"); - Number numBytes = (Number)ri.get(Type.CORE_IDX.metricsAttribute); - if (numDocs != null) { - docs.addValue(numDocs.doubleValue()); - } - if (delDocs != null) { - deletedDocs += delDocs.longValue(); - } - if (numBytes != null) { - bytes.addValue(numBytes.doubleValue()); - } - } - } - perColl.put("shardsState", shardState); - perColl.put(" shardsWithoutLeader", noLeader); - perColl.put("totalReplicas", totalReplicas); - perColl.put(" activeReplicas", activeReplicas); - perColl.put(" inactiveReplicas", totalReplicas - activeReplicas); - long totalDocs = (long)docs.getSum() + bufferedDocs; - perColl.put("totalActiveDocs", String.format(Locale.ROOT, "%,d", totalDocs)); - perColl.put(" bufferedDocs", String.format(Locale.ROOT, "%,d", bufferedDocs)); - perColl.put(" maxActiveSliceDocs", String.format(Locale.ROOT, "%,d", (long)docs.getMax())); - perColl.put(" minActiveSliceDocs", String.format(Locale.ROOT, "%,d", (long)docs.getMin())); - perColl.put(" avgActiveSliceDocs", String.format(Locale.ROOT, "%,.0f", docs.getMean())); - perColl.put("totalInactiveDocs", String.format(Locale.ROOT, "%,d", (long)inactiveDocs.getSum())); - perColl.put(" maxInactiveSliceDocs", String.format(Locale.ROOT, "%,d", (long)inactiveDocs.getMax())); - perColl.put(" minInactiveSliceDocs", String.format(Locale.ROOT, "%,d", (long)inactiveDocs.getMin())); - perColl.put(" avgInactiveSliceDocs", String.format(Locale.ROOT, "%,.0f", inactiveDocs.getMean())); - perColl.put("totalActiveBytes", String.format(Locale.ROOT, "%,d", (long)bytes.getSum())); - perColl.put(" maxActiveSliceBytes", String.format(Locale.ROOT, "%,d", (long)bytes.getMax())); - perColl.put(" minActiveSliceBytes", String.format(Locale.ROOT, "%,d", (long)bytes.getMin())); - perColl.put(" avgActiveSliceBytes", String.format(Locale.ROOT, "%,.0f", bytes.getMean())); - perColl.put("totalInactiveBytes", String.format(Locale.ROOT, "%,d", (long)inactiveBytes.getSum())); - perColl.put(" maxInactiveSliceBytes", String.format(Locale.ROOT, "%,d", (long)inactiveBytes.getMax())); - perColl.put(" minInactiveSliceBytes", String.format(Locale.ROOT, "%,d", (long)inactiveBytes.getMin())); - perColl.put(" avgInactiveSliceBytes", String.format(Locale.ROOT, "%,.0f", inactiveBytes.getMean())); - perColl.put("totalActiveDeletedDocs", String.format(Locale.ROOT, "%,d", deletedDocs)); - }); - return stats; - } finally { - lock.unlock(); - } - } - - // interface methods - - @Override - public ClusterState.CollectionRef getState(String collection) { - try { - return getClusterState().getCollectionRef(collection); - } catch (IOException e) { - return null; - } - } - - @Override - public Set getLiveNodes() { - return liveNodes.get(); - } - - @Override - public List resolveAlias(String alias) { - throw new UnsupportedOperationException("resolveAlias not implemented"); - } - - @Override - public Map getAliasProperties(String alias) { - throw new UnsupportedOperationException("getAliasProperties not implemented"); - } - - @Override - public ClusterState getClusterState() throws IOException { - ensureNotClosed(); - try { - lock.lockInterruptibly(); - try { - Map states = getCollectionStates(); - ClusterState state = new ClusterState(liveNodes.get(), states); - return state; - } finally { - lock.unlock(); - } - } catch (InterruptedException e) { - throw new IOException(e); - } - } - - private Map getCollectionStates() throws IOException, InterruptedException { - lock.lockInterruptibly(); - try { - Map collectionStates = new HashMap<>(); - collectionsStatesRef.forEach((name, cached) -> { - try { - collectionStates.put(name, cached.getColl()); - } catch (Exception e) { - throw new RuntimeException("error building collection " + name + " state", e); - } - }); - return collectionStates; - } finally { - lock.unlock(); - } - } - - @Override - public Map getClusterProperties() { - return clusterProperties; - } - - @Override - public String getPolicyNameByCollection(String coll) { - Map props = collProperties.computeIfAbsent(coll, c -> new HashMap<>()); - return (String)props.get("policy"); - } - - @Override - public void connect() { - - } - - @Override - public void close() throws IOException { - closed = true; - } - - @Override - public boolean isClosed() { - return closed; - } - - private void ensureNotClosed() throws IOException { - if (closed) { - throw new IOException("already closed"); - } - } -} diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/sim/SimDistribStateManager.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/sim/SimDistribStateManager.java deleted file mode 100644 index ea9fa550f59..00000000000 --- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/sim/SimDistribStateManager.java +++ /dev/null @@ -1,649 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.cloud.autoscaling.sim; - -import java.io.IOException; -import java.lang.invoke.MethodHandles; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.NoSuchElementException; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.atomic.AtomicReference; -import java.util.concurrent.locks.ReentrantLock; - -import org.apache.jute.Record; -import org.apache.solr.client.solrj.cloud.autoscaling.AlreadyExistsException; -import org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig; -import org.apache.solr.client.solrj.cloud.autoscaling.BadVersionException; -import org.apache.solr.client.solrj.cloud.DistribStateManager; -import org.apache.solr.client.solrj.cloud.autoscaling.NotEmptyException; -import org.apache.solr.client.solrj.cloud.autoscaling.VersionedData; -import org.apache.solr.cloud.ActionThrottle; -import org.apache.solr.common.cloud.ZkStateReader; -import org.apache.solr.common.params.AutoScalingParams; -import org.apache.solr.common.util.ExecutorUtil; -import org.apache.solr.common.util.Utils; -import org.apache.solr.common.util.SolrNamedThreadFactory; -import org.apache.solr.util.IdUtils; -import org.apache.zookeeper.CreateMode; -import org.apache.zookeeper.KeeperException; -import org.apache.zookeeper.Op; -import org.apache.zookeeper.OpResult; -import org.apache.zookeeper.WatchedEvent; -import org.apache.zookeeper.Watcher; -import org.apache.zookeeper.data.Stat; -import org.apache.zookeeper.proto.CheckVersionRequest; -import org.apache.zookeeper.proto.CreateRequest; -import org.apache.zookeeper.proto.DeleteRequest; -import org.apache.zookeeper.proto.SetDataRequest; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Simulated {@link DistribStateManager} that keeps all data locally in a static structure. Instances of this - * class are identified by their id in order to simulate the deletion of ephemeral nodes when {@link #close()} is - * invoked. - */ -public class SimDistribStateManager implements DistribStateManager { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - public static final class Node { - ReentrantLock dataLock = new ReentrantLock(); - private int version = 0; - private int seq = 0; - private final CreateMode mode; - // copyFrom needs to modify this - private String owner; - private final String path; - private final String name; - private final Node parent; - private byte[] data = null; - private Map children = new ConcurrentHashMap<>(); - Set dataWatches = ConcurrentHashMap.newKeySet(); - Set childrenWatches = ConcurrentHashMap.newKeySet(); - - Node(Node parent, String name, String path, CreateMode mode, String owner) { - this.parent = parent; - this.name = name; - this.path = path; - this.mode = mode; - this.owner = owner; - } - - Node(Node parent, String name, String path, byte[] data, CreateMode mode, String owner) { - this(parent, name, path, mode, owner); - this.data = data; - } - - public void clear() { - dataLock.lock(); - try { - children.clear(); - version = 0; - seq = 0; - dataWatches.clear(); - childrenWatches.clear(); - data = null; - } finally { - dataLock.unlock(); - } - } - - public void setData(byte[] data, int version) throws BadVersionException, IOException { - Set currentWatchers = new HashSet<>(dataWatches); - dataLock.lock(); - try { - if (version != -1 && version != this.version) { - throw new BadVersionException(version, path); - } - if (data != null) { - this.data = Arrays.copyOf(data, data.length); - } else { - this.data = null; - } - this.version++; - dataWatches.clear(); - } finally { - dataLock.unlock(); - } - for (Watcher w : currentWatchers) { - w.process(new WatchedEvent(Watcher.Event.EventType.NodeDataChanged, Watcher.Event.KeeperState.SyncConnected, path)); - } - } - - public VersionedData getData(Watcher w) { - dataLock.lock(); - try { - VersionedData res = new VersionedData(version, data, mode, owner); - if (w != null && !dataWatches.contains(w)) { - dataWatches.add(w); - } - return res; - } finally { - dataLock.unlock(); - } - } - - public void setChild(String name, Node child) { - assert child.name.equals(name); - Set currentWatchers = new HashSet<>(childrenWatches); - dataLock.lock(); - try { - children.put(name, child); - childrenWatches.clear(); - } finally { - dataLock.unlock(); - } - for (Watcher w : currentWatchers) { - w.process(new WatchedEvent(Watcher.Event.EventType.NodeChildrenChanged, Watcher.Event.KeeperState.SyncConnected, path)); - } - } - - public void removeChild(String name, int version) throws NoSuchElementException, BadVersionException, IOException { - Node n = children.get(name); - if (n == null) { - throw new NoSuchElementException(path + "/" + name); - } - if (version != -1 && version != n.version) { - throw new BadVersionException(version, path); - } - children.remove(name); - Set currentWatchers = new HashSet<>(childrenWatches); - childrenWatches.clear(); - for (Watcher w : currentWatchers) { - w.process(new WatchedEvent(Watcher.Event.EventType.NodeChildrenChanged, Watcher.Event.KeeperState.SyncConnected, path)); - } - currentWatchers = new HashSet<>(n.dataWatches); - n.dataWatches.clear(); - for (Watcher w : currentWatchers) { - w.process(new WatchedEvent(Watcher.Event.EventType.NodeDeleted, Watcher.Event.KeeperState.SyncConnected, n.path)); - } - // TODO: not sure if it's correct to recurse and fire watches??? - Set kids = new HashSet<>(n.children.keySet()); - for (String kid : kids) { - n.removeChild(kid, -1); - } - } - - public void removeEphemeralChildren(String id) throws NoSuchElementException, BadVersionException, IOException { - Set kids = new HashSet<>(children.keySet()); - for (String kid : kids) { - Node n = children.get(kid); - if (n == null) { - continue; - } - if ((CreateMode.EPHEMERAL == n.mode || CreateMode.EPHEMERAL_SEQUENTIAL == n.mode) && - id.equals(n.owner)) { - removeChild(n.name, -1); - } else { - n.removeEphemeralChildren(id); - } - } - } - - } - - private final ReentrantLock multiLock = new ReentrantLock(); - - public static Node createNewRootNode() { - return new Node(null, "", "/", CreateMode.PERSISTENT, "0"); - } - - private final ExecutorService watchersPool; - - private final AtomicReference throttleRef = new AtomicReference<>(); - private final AtomicReference errorRef = new AtomicReference<>(); - private final String id; - private final Node root; - - private int juteMaxbuffer = 0xfffff; - - public SimDistribStateManager() { - this(null); - } - - /** - * Construct new state manager that uses provided root node for storing data. - * @param root if null then a new root node will be created. - */ - public SimDistribStateManager(Node root) { - this.id = IdUtils.timeRandomId(); - this.root = root != null ? root : createNewRootNode(); - watchersPool = ExecutorUtil.newMDCAwareFixedThreadPool(10, new SolrNamedThreadFactory("sim-watchers")); - String bufferSize = System.getProperty("jute.maxbuffer", Integer.toString(0xffffff)); - juteMaxbuffer = Integer.parseInt(bufferSize); - } - - /** - * Copy all content from another DistribStateManager. - * @param other another state manager. - * @param failOnExists abort copy when one or more paths already exist (the state of this manager remains unchanged). - */ - public void copyFrom(DistribStateManager other, boolean failOnExists) throws InterruptedException, IOException, KeeperException, AlreadyExistsException, BadVersionException { - List tree = other.listTree("/"); - if (log.isInfoEnabled()) { - log.info("- copying {} resources...", tree.size()); - } - // check if any node exists - for (String path : tree) { - if (hasData(path) && failOnExists) { - throw new AlreadyExistsException(path); - } - } - for (String path : tree) { - VersionedData data = other.getData(path); - if (hasData(path)) { - setData(path, data.getData(), -1); - } else { - makePath(path, data.getData(), data.getMode(), failOnExists); - } - // hack: set the version and owner to be the same as the source - Node n = traverse(path, false, CreateMode.PERSISTENT); - n.version = data.getVersion(); - n.owner = data.getOwner(); - } - } - - public SimDistribStateManager(ActionThrottle actionThrottle, ActionError actionError) { - this(null, actionThrottle, actionError); - } - - public SimDistribStateManager(Node root, ActionThrottle actionThrottle, ActionError actionError) { - this(root); - this.throttleRef.set(actionThrottle); - this.errorRef.set(actionError); - } - - private SimDistribStateManager(String id, ExecutorService watchersPool, Node root, ActionThrottle actionThrottle, - ActionError actionError) { - this.id = id; - this.watchersPool = watchersPool; - this.root = root; - this.throttleRef.set(actionThrottle); - this.errorRef.set(actionError); - } - - /** - * Create a copy of this instance using a specified ephemeral owner id. This is useful when performing - * node operations that require using a specific id. Note: this instance should never be closed, it can - * be just discarded after use. - * @param id ephemeral owner id - */ - public SimDistribStateManager withEphemeralId(String id) { - return new SimDistribStateManager(id, watchersPool, root, throttleRef.get(), errorRef.get()) { - @Override - public void close() { - throw new UnsupportedOperationException("this instance should never be closed - instead close the parent instance."); - } - }; - } - - /** - * Get the root node of the tree used by this instance. It could be a static shared root node. - */ - public Node getRoot() { - return root; - } - - /** - * Clear this instance. All nodes, watchers and data is deleted. - */ - public void clear() { - root.clear(); - } - - private void throttleOrError(String path) throws IOException { - ActionError err = errorRef.get(); - if (err != null && err.shouldFail(path)) { - throw new IOException("Simulated error, path=" + path); - } - ActionThrottle throttle = throttleRef.get(); - if (throttle != null) { - throttle.minimumWaitBetweenActions(); - throttle.markAttemptingAction(); - } - } - - // this method should always be invoked under lock - private Node traverse(String path, boolean create, CreateMode mode) throws IOException { - if (path == null || path.isEmpty()) { - return null; - } - throttleOrError(path); - if (path.equals("/")) { - return root; - } - if (path.charAt(0) == '/') { - path = path.substring(1); - } - StringBuilder currentPath = new StringBuilder(); - String[] elements = path.split("/"); - Node parentNode = root; - Node n = null; - for (int i = 0; i < elements.length; i++) { - String currentName = elements[i]; - currentPath.append('/'); - n = parentNode.children != null ? parentNode.children.get(currentName) : null; - if (n == null) { - if (create) { - n = createNode(parentNode, mode, currentPath, currentName,null, true); - } else { - break; - } - } else { - currentPath.append(currentName); - } - parentNode = n; - } - return n; - } - - private Node createNode(Node parentNode, CreateMode mode, StringBuilder fullChildPath, String baseChildName, byte[] data, boolean attachToParent) throws IOException { - String nodeName = baseChildName; - if ((parentNode.mode == CreateMode.EPHEMERAL || parentNode.mode == CreateMode.EPHEMERAL_SEQUENTIAL) && - (mode == CreateMode.EPHEMERAL || mode == CreateMode.EPHEMERAL_SEQUENTIAL)) { - throw new IOException("NoChildrenEphemerals for " + parentNode.path); - } - if (CreateMode.PERSISTENT_SEQUENTIAL == mode || CreateMode.EPHEMERAL_SEQUENTIAL == mode) { - nodeName = nodeName + String.format(Locale.ROOT, "%010d", parentNode.seq); - parentNode.seq++; - } - - fullChildPath.append(nodeName); - String owner = mode == CreateMode.EPHEMERAL || mode == CreateMode.EPHEMERAL_SEQUENTIAL ? id : "0"; - Node child = new Node(parentNode, nodeName, fullChildPath.toString(), data, mode, owner); - - if (attachToParent) { - parentNode.setChild(nodeName, child); - } - return child; - } - - @Override - public void close() throws IOException { - multiLock.lock(); - try { - // remove all my ephemeral nodes - root.removeEphemeralChildren(id); - } catch (BadVersionException e) { - // not happening - } finally { - multiLock.unlock(); - } - - } - - @Override - public boolean hasData(String path) throws IOException { - multiLock.lock(); - try { - return traverse(path, false, CreateMode.PERSISTENT) != null; - } finally { - multiLock.unlock(); - } - } - - @Override - public List listData(String path) throws NoSuchElementException, IOException { - multiLock.lock(); - try { - Node n = traverse(path, false, CreateMode.PERSISTENT); - if (n == null) { - throw new NoSuchElementException(path); - } - List res = new ArrayList<>(n.children.keySet()); - Collections.sort(res); - return res; - } finally { - multiLock.unlock(); - } - } - - @Override - public List listData(String path, Watcher watcher) throws NoSuchElementException, IOException { - Node n; - List res; - multiLock.lock(); - try { - n = traverse(path, false, CreateMode.PERSISTENT); - if (n == null) { - throw new NoSuchElementException(path); - } - res = new ArrayList<>(n.children.keySet()); - Collections.sort(res); - } finally { - multiLock.unlock(); - } - if (watcher != null) { - n.dataWatches.add(watcher); - n.childrenWatches.add(watcher); - } - return res; - } - - @Override - public VersionedData getData(String path, Watcher watcher) throws NoSuchElementException, IOException { - Node n = null; - multiLock.lock(); - try { - n = traverse(path, false, CreateMode.PERSISTENT); - if (n == null) { - throw new NoSuchElementException(path); - } - } finally { - multiLock.unlock(); - } - return n.getData(watcher); - } - - @Override - public void makePath(String path) throws IOException { - multiLock.lock(); - try { - traverse(path, true, CreateMode.PERSISTENT); - } finally { - multiLock.unlock(); - } - } - - @Override - public void makePath(String path, byte[] data, CreateMode createMode, boolean failOnExists) throws AlreadyExistsException, IOException, KeeperException, InterruptedException { - Node n = null; - multiLock.lock(); - try { - if (failOnExists && hasData(path)) { - throw new AlreadyExistsException(path); - } - n = traverse(path, true, createMode); - } finally { - multiLock.unlock(); - } - try { - n.setData(data, -1); - } catch (BadVersionException e) { - throw new IOException("should not happen!", e); - } - } - - @Override - public String createData(String path, byte[] data, CreateMode mode) throws AlreadyExistsException, NoSuchElementException, IOException { - if ((CreateMode.EPHEMERAL == mode || CreateMode.PERSISTENT == mode) && hasData(path)) { - throw new AlreadyExistsException(path); - } - - String relPath = path.charAt(0) == '/' ? path.substring(1) : path; - if (relPath.length() == 0) { //Trying to create root-node, return null. - // TODO should trying to create a root node throw an exception since its always init'd in the ctor? - return null; - } - - String[] elements = relPath.split("/"); - StringBuilder parentStringBuilder = new StringBuilder(); - Node parentNode = null; - if (elements.length == 1) { // Direct descendant of '/'. - parentNode = getRoot(); - } else { // Indirect descendant of '/', lookup parent node - for (int i = 0; i < elements.length - 1; i++) { - parentStringBuilder.append('/'); - parentStringBuilder.append(elements[i]); - } - if (!hasData(parentStringBuilder.toString())) { - throw new NoSuchElementException(parentStringBuilder.toString()); - } - parentNode = traverse(parentStringBuilder.toString(), false, mode); - } - - multiLock.lock(); - try { - String nodeName = elements[elements.length-1]; - Node childNode = createNode(parentNode, mode, parentStringBuilder.append("/"), nodeName, data,false); - parentNode.setChild(childNode.name, childNode); - return childNode.path; - } finally { - multiLock.unlock(); - } - - } - - @Override - public void removeData(String path, int version) throws NoSuchElementException, NotEmptyException, BadVersionException, IOException { - multiLock.lock(); - Node parent; - Node n; - try { - n = traverse(path, false, CreateMode.PERSISTENT); - if (n == null) { - throw new NoSuchElementException(path); - } - parent = n.parent; - if (parent == null) { - throw new IOException("Cannot remove root node"); - } - if (!n.children.isEmpty()) { - throw new NotEmptyException(path); - } - } finally { - multiLock.unlock(); - } - - // outside the lock to avoid deadlock with update lock - parent.removeChild(n.name, version); - } - - @Override - public void setData(String path, byte[] data, int version) throws NoSuchElementException, BadVersionException, IOException { - if (data != null && data.length > juteMaxbuffer) { - throw new IOException("Len error " + data.length); - } - multiLock.lock(); - Node n = null; - try { - n = traverse(path, false, CreateMode.PERSISTENT); - if (n == null) { - throw new NoSuchElementException(path); - } - } finally { - multiLock.unlock(); - } - n.setData(data, version); - } - - @Override - public List multi(Iterable ops) throws BadVersionException, NoSuchElementException, AlreadyExistsException, IOException, KeeperException, InterruptedException { - multiLock.lock(); - List res = new ArrayList<>(); - try { - for (Op op : ops) { - Record r = op.toRequestRecord(); - try { - if (op instanceof Op.Check) { - CheckVersionRequest rr = (CheckVersionRequest)r; - Node n = traverse(rr.getPath(), false, CreateMode.PERSISTENT); - if (n == null) { - throw new NoSuchElementException(rr.getPath()); - } - if (rr.getVersion() != -1 && n.version != rr.getVersion()) { - throw new Exception("version mismatch"); - } - // everything ok - res.add(new OpResult.CheckResult()); - } else if (op instanceof Op.Create) { - CreateRequest rr = (CreateRequest)r; - createData(rr.getPath(), rr.getData(), CreateMode.fromFlag(rr.getFlags())); - res.add(new OpResult.CreateResult(rr.getPath())); - } else if (op instanceof Op.Delete) { - DeleteRequest rr = (DeleteRequest)r; - removeData(rr.getPath(), rr.getVersion()); - res.add(new OpResult.DeleteResult()); - } else if (op instanceof Op.SetData) { - SetDataRequest rr = (SetDataRequest)r; - setData(rr.getPath(), rr.getData(), rr.getVersion()); - VersionedData vd = getData(rr.getPath()); - Stat s = new Stat(); - s.setVersion(vd.getVersion()); - res.add(new OpResult.SetDataResult(s)); - } else { - throw new Exception("Unknown Op: " + op); - } - } catch (Exception e) { - res.add(new OpResult.ErrorResult(KeeperException.Code.APIERROR.intValue())); - } - } - } finally { - multiLock.unlock(); - } - return res; - } - - @Override - @SuppressWarnings({"unchecked"}) - public AutoScalingConfig getAutoScalingConfig(Watcher watcher) throws InterruptedException, IOException { - Map map = new HashMap<>(); - int version = 0; - try { - VersionedData data = getData(ZkStateReader.SOLR_AUTOSCALING_CONF_PATH, watcher); - if (data != null && data.getData() != null && data.getData().length > 0) { - map = (Map) Utils.fromJSON(data.getData()); - version = data.getVersion(); - } - } catch (NoSuchElementException e) { - // ignore - } - map.put(AutoScalingParams.ZK_VERSION, version); - return new AutoScalingConfig(map); - } - - // ------------ simulator methods -------------- - - public void simSetAutoScalingConfig(AutoScalingConfig cfg) throws Exception { - try { - makePath(ZkStateReader.SOLR_AUTOSCALING_CONF_PATH); - } catch (Exception e) { - // ignore - } - setData(ZkStateReader.SOLR_AUTOSCALING_CONF_PATH, Utils.toJSON(cfg), -1); - } -} diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/sim/SimDistributedQueueFactory.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/sim/SimDistributedQueueFactory.java deleted file mode 100644 index fb17881c00f..00000000000 --- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/sim/SimDistributedQueueFactory.java +++ /dev/null @@ -1,287 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.cloud.autoscaling.sim; - -import java.io.IOException; -import java.lang.invoke.MethodHandles; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Queue; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentLinkedQueue; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicReference; -import java.util.concurrent.locks.Condition; -import java.util.concurrent.locks.ReentrantLock; -import java.util.function.Predicate; - -import com.codahale.metrics.Timer; -import com.google.common.base.Preconditions; -import org.apache.solr.client.solrj.cloud.DistributedQueue; -import org.apache.solr.client.solrj.cloud.DistributedQueueFactory; -import org.apache.solr.cloud.Stats; -import org.apache.solr.common.util.Pair; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Simulated {@link DistributedQueueFactory} that keeps all data in memory. Unlike - * the {@link GenericDistributedQueueFactory} this queue implementation data is not - * exposed anywhere. - */ -public class SimDistributedQueueFactory implements DistributedQueueFactory { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - Map queues = new ConcurrentHashMap<>(); - - public SimDistributedQueueFactory() { - } - - @Override - public DistributedQueue makeQueue(final String path) throws IOException { - return queues.computeIfAbsent(path, p -> new SimDistributedQueue(path)); - } - - @Override - public void removeQueue(String path) throws IOException { - queues.remove(path); - } - - public static class SimDistributedQueue implements DistributedQueue { - private final Queue> queue = new ConcurrentLinkedQueue<>(); - private final ReentrantLock updateLock = new ReentrantLock(); - private final Condition changed = updateLock.newCondition(); - private final Stats stats = new Stats(); - private final String dir; - private int seq = 0; - - public SimDistributedQueue(String dir) { - this.dir = dir; - } - - @Override - public byte[] peek() throws Exception { - Timer.Context time = stats.time(dir + "_peek"); - try { - Pair pair = queue.peek(); - return pair != null ? pair.second() : null; - } finally { - time.stop(); - } - } - - @Override - public byte[] peek(boolean block) throws Exception { - return block ? peek(Long.MAX_VALUE) : peek(); - } - - @Override - public byte[] peek(long wait) throws Exception { - Timer.Context time; - if (wait == Long.MAX_VALUE) { - time = stats.time(dir + "_peek_wait_forever"); - } else { - time = stats.time(dir + "_peek_wait" + wait); - } - try { - Pair pair = peekInternal(wait); - return pair != null ? pair.second() : null; - } finally { - time.stop(); - } - } - - private Pair peekInternal(long wait) throws Exception { - Preconditions.checkArgument(wait > 0); - long waitNanos = TimeUnit.MILLISECONDS.toNanos(wait); - updateLock.lockInterruptibly(); - try { - while (waitNanos > 0) { - Pair pair = queue.peek(); - if (pair != null) { - return pair; - } - waitNanos = changed.awaitNanos(waitNanos); - if (waitNanos < 0) { // timed out - return null; - } - } - } finally { - updateLock.unlock(); - } - return null; - } - - @Override - public byte[] poll() throws Exception { - Timer.Context time = stats.time(dir + "_poll"); - updateLock.lockInterruptibly(); - try { - Pair pair = queue.poll(); - if (pair != null) { - changed.signalAll(); - return pair.second(); - } else { - return null; - } - } finally { - updateLock.unlock(); - time.stop(); - } - } - - @Override - public byte[] remove() throws Exception { - Timer.Context time = stats.time(dir + "_remove"); - updateLock.lockInterruptibly(); - try { - byte[] res = queue.remove().second(); - changed.signalAll(); - return res; - } finally { - updateLock.unlock(); - time.stop(); - } - } - - @Override - public byte[] take() throws Exception { - Timer.Context timer = stats.time(dir + "_take"); - updateLock.lockInterruptibly(); - try { - while (true) { - byte[] result = poll(); - if (result != null) { - return result; - } - changed.await(); - } - } finally { - updateLock.unlock(); - timer.stop(); - } - } - - @Override - @SuppressWarnings({"unchecked", "rawtypes"}) - public void offer(byte[] data) throws Exception { - Timer.Context time = stats.time(dir + "_offer"); - updateLock.lockInterruptibly(); - try { - queue.offer(new Pair(String.format(Locale.ROOT, "qn-%010d", seq), data)); - seq++; - if (log.isTraceEnabled()) { - log.trace("=== offer {}", System.nanoTime()); - } - changed.signalAll(); - } finally { - updateLock.unlock(); - time.stop(); - } - } - - @Override - public Collection> peekElements(int max, long waitMillis, Predicate acceptFilter) throws Exception { - updateLock.lockInterruptibly(); - try { - List> res = new LinkedList<>(); - final int maximum = max < 0 ? Integer.MAX_VALUE : max; - final AtomicReference> pairRef = new AtomicReference<>(); - queue.forEach(pair -> { - if (acceptFilter != null && !acceptFilter.test(pair.first())) { - return; - } - if (res.size() < maximum) { - pairRef.set(pair); - res.add(pair); - } - }); - if (res.size() < maximum && waitMillis > 0) { - long waitNanos = TimeUnit.MILLISECONDS.toNanos(waitMillis); - waitNanos = changed.awaitNanos(waitNanos); - if (waitNanos < 0) { - return res; - } - AtomicBoolean seen = new AtomicBoolean(false); - queue.forEach(pair -> { - if (!seen.get()) { - if (pairRef.get() == null) { - seen.set(true); - } else { - if (pairRef.get().first().equals(pair.first())) { - seen.set(true); - return; - } - } - } - if (!seen.get()) { - return; - } - if (!acceptFilter.test(pair.first())) { - return; - } - if (res.size() < maximum) { - res.add(pair); - pairRef.set(pair); - } else { - return; - } - }); - } - return res; - } finally { - updateLock.unlock(); - } - } - - public Stats getZkStats() { - return stats; - } - - @Override - public Map getStats() { - if (stats == null) { - return Collections.emptyMap(); - } - Map res = new HashMap<>(); - res.put("queueLength", stats.getQueueLength()); - final Map statsMap = new HashMap<>(); - res.put("stats", statsMap); - stats.getStats().forEach((op, stat) -> { - final Map statMap = new HashMap<>(); - statMap.put("success", stat.success.get()); - statMap.put("errors", stat.errors.get()); - final List> failed = new ArrayList<>(stat.failureDetails.size()); - statMap.put("failureDetails", failed); - stat.failureDetails.forEach(failedOp -> { - Map fo = new HashMap<>(); - fo.put("req", failedOp.req); - fo.put("resp", failedOp.resp); - }); - statsMap.put(op, statMap); - }); - return res; - } - } -} diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/sim/SimNodeStateProvider.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/sim/SimNodeStateProvider.java deleted file mode 100644 index e7a1fd471de..00000000000 --- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/sim/SimNodeStateProvider.java +++ /dev/null @@ -1,359 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.cloud.autoscaling.sim; - -import java.io.IOException; -import java.lang.invoke.MethodHandles; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.TreeSet; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.locks.ReentrantLock; -import java.util.function.Function; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import java.util.stream.Collectors; - -import org.apache.solr.client.solrj.cloud.NodeStateProvider; -import org.apache.solr.common.cloud.Replica; -import org.apache.solr.common.cloud.ZkStateReader; -import org.apache.solr.common.util.Utils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Simulated {@link NodeStateProvider}. - * Note: in order to setup node-level metrics use {@link #simSetNodeValues(String, Map)}. However, in order - * to setup core-level metrics use {@link SimClusterStateProvider#simSetCollectionValue(String, String, Object, boolean, boolean)}. - */ -public class SimNodeStateProvider implements NodeStateProvider { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - private final Map> nodeValues = new ConcurrentHashMap<>(); - private final SimClusterStateProvider clusterStateProvider; - private final SimDistribStateManager stateManager; - private final LiveNodesSet liveNodesSet; - private final ReentrantLock lock = new ReentrantLock(); - - public SimNodeStateProvider(LiveNodesSet liveNodesSet, SimDistribStateManager stateManager, - SimClusterStateProvider clusterStateProvider, - Map> nodeValues) { - this.liveNodesSet = liveNodesSet; - this.stateManager = stateManager; - this.clusterStateProvider = clusterStateProvider; - if (nodeValues != null) { - this.nodeValues.putAll(nodeValues); - } - } - - // -------- simulator setup methods ------------ - - /** - * Get a node value - * @param node node id - * @param key property name - * @return property value or null if property or node doesn't exist. - */ - public Object simGetNodeValue(String node, String key) { - Map values = nodeValues.get(node); - if (values == null) { - return null; - } - return values.get(key); - } - - /** - * Atomically update a node value. - * @param node node id - * @param key property name - * @param updater updater function - * @return previous property value or null if property or node didn't exist. - */ - public Object simUpdateNodeValue(String node, String key, Function updater) throws InterruptedException { - lock.lockInterruptibly(); - try { - Map values = nodeValues.computeIfAbsent(node, n -> new ConcurrentHashMap<>()); - return values.put(key, updater.apply(values.get(key))); - } finally { - lock.unlock(); - } - } - - /** - * Set node values. - * NOTE: if values contain 'nodeRole' key then /roles.json is updated. - * @param node node id - * @param values values. - */ - public void simSetNodeValues(String node, Map values) throws InterruptedException { - lock.lockInterruptibly(); - try { - Map existing = nodeValues.computeIfAbsent(node, n -> new ConcurrentHashMap<>()); - existing.clear(); - if (values != null) { - existing.putAll(values); - } - if (values == null || values.isEmpty() || values.containsKey("nodeRole")) { - saveRoles(); - } - } finally { - lock.unlock(); - } - } - - /** - * Set a node value, replacing any previous value. - * NOTE: if key is 'nodeRole' then /roles.json is updated. - * @param node node id - * @param key property name - * @param value property value - */ - public void simSetNodeValue(String node, String key, Object value) throws InterruptedException { - lock.lockInterruptibly(); - try { - Map existing = nodeValues.computeIfAbsent(node, n -> new ConcurrentHashMap<>()); - if (value == null) { - existing.remove(key); - } else { - existing.put(key, value); - } - if (key.equals("nodeRole")) { - saveRoles(); - } - } finally { - lock.unlock(); - } - } - - /** - * Add a node value, creating a list of values if necessary. - * NOTE: if key is 'nodeRole' then /roles.json is updated. - * @param node node id - * @param key property name - * @param value property value. - */ - @SuppressWarnings({"unchecked"}) - public void simAddNodeValue(String node, String key, Object value) throws InterruptedException { - lock.lockInterruptibly(); - try { - Map values = nodeValues.computeIfAbsent(node, n -> new ConcurrentHashMap<>()); - Object existing = values.get(key); - if (existing == null) { - values.put(key, value); - } else if (existing instanceof Set) { - ((Set)existing).add(value); - } else { - Set vals = new HashSet<>(); - vals.add(existing); - vals.add(value); - values.put(key, vals); - } - if (key.equals("nodeRole")) { - saveRoles(); - } - } finally { - lock.unlock(); - } - } - - /** - * Remove node values. If values contained a 'nodeRole' key then - * /roles.json is updated. - * @param node node id - */ - public void simRemoveNodeValues(String node) throws InterruptedException { - log.debug("--removing value for {}", node); - lock.lockInterruptibly(); - try { - Map values = nodeValues.remove(node); - if (values != null && values.containsKey("nodeRole")) { - saveRoles(); - } - } finally { - lock.unlock(); - } - } - - /** - * Remove values that correspond to dead nodes. If values contained a 'nodeRole' - * key then /roles.json is updated. - */ - public void simRemoveDeadNodes() throws InterruptedException { - Set myNodes = new HashSet<>(nodeValues.keySet()); - myNodes.removeAll(liveNodesSet.get()); - lock.lockInterruptibly(); - try { - AtomicBoolean updateRoles = new AtomicBoolean(false); - myNodes.forEach(n -> { - log.debug("- removing dead node values: {}", n); - Map vals = nodeValues.remove(n); - if (vals.containsKey("nodeRole")) { - updateRoles.set(true); - } - }); - if (updateRoles.get()) { - saveRoles(); - } - } finally { - lock.unlock(); - } - } - - /** - * Return a set of nodes that are not live but their values are still present. - */ - public Set simGetDeadNodes() { - Set myNodes = new TreeSet<>(nodeValues.keySet()); - myNodes.removeAll(liveNodesSet.get()); - return myNodes; - } - - /** - * Get all node values. - */ - public Map> simGetAllNodeValues() { - return nodeValues; - } - - /** Get all values for a selected node. */ - public Map simGetNodeValues(String node) { - return nodeValues.getOrDefault(node, Collections.emptyMap()); - } - - private void saveRoles() { - final Map> roles = new HashMap<>(); - nodeValues.forEach((n, values) -> { - String nodeRole = (String)values.get("nodeRole"); - if (nodeRole != null) { - roles.computeIfAbsent(nodeRole, role -> new HashSet<>()).add(n); - } - }); - try { - stateManager.setData(ZkStateReader.ROLES, Utils.toJSON(roles), -1); - } catch (Exception e) { - throw new RuntimeException("Unexpected exception saving roles " + roles, e); - } - } - - private static final Pattern REGISTRY_PATTERN = Pattern.compile("^solr\\.core\\.([\\w.-_]+?)\\.(shard[\\d_]+?)\\.(replica.*)"); - private static final Pattern METRIC_KEY_PATTERN = Pattern.compile("^metrics:([^:]+?):([^:]+?)(:([^:]+))?$"); - /** - * Simulate getting replica metrics values. This uses per-replica properties set in - * {@link SimClusterStateProvider#simSetCollectionValue(String, String, Object, boolean, boolean)} and - * similar methods. - * @param node node id - * @param tags metrics names - * @return map of metrics names / values - */ - public Map getReplicaMetricsValues(String node, Collection tags) { - if (!liveNodesSet.contains(node)) { - throw new RuntimeException("non-live node " + node); - } - Map values = new HashMap<>(); - for (String tag : tags) { - Matcher m = METRIC_KEY_PATTERN.matcher(tag); - if (!m.matches() || m.groupCount() < 2) { - log.warn("Invalid metrics: tag: {}", tag); - continue; - } - String registryName = m.group(1); - String key = m.group(3) != null ? m.group(2) + m.group(3) : m.group(2); - if (!registryName.startsWith("solr.core.")) { - // skip - this is probably solr.node or solr.jvm metric - continue; - } - m = REGISTRY_PATTERN.matcher(registryName); - - if (!m.matches()) { - log.warn("Invalid registry name: {}", registryName); - continue; - } - String collection = m.group(1); - String shard = m.group(2); - String replica = m.group(3); - List replicas = clusterStateProvider.simGetReplicaInfos(collection, shard); - replicas.forEach(r -> { - if (r.getNodeName().equals(node) && r.getCoreName().endsWith(replica)) { - Object value = r.getProperties().get(key); - if (value != null) { - values.put(tag, value); - } else { - value = r.getProperties().get(tag); - if (value != null) { - values.put(tag, value); - } - } - } - }); - } - return values; - } - - // ---------- interface methods ------------- - - @Override - public Map getNodeValues(String node, Collection tags) { - log.trace("-- requested values for {}: {}", node, tags); - if (!liveNodesSet.contains(node)) { - throw new RuntimeException("non-live node " + node); - } - if (tags.isEmpty()) { - return new HashMap<>(); - } - Map metrics = getReplicaMetricsValues(node, tags.stream().filter(s -> s.startsWith("metrics:solr.core.")).collect(Collectors.toList())); - Map result = new HashMap<>(metrics); - Map values = nodeValues.get(node); - if (values == null) { - return result; - } - result.putAll(values.entrySet().stream() - .filter(e -> tags.contains(e.getKey())) - .filter(e -> e.getValue() != null) - .collect(Collectors.toMap(e -> e.getKey(), e -> e.getValue()))); - return result; - } - - @Override - public Map>> getReplicaInfo(String node, Collection keys) { - List replicas = clusterStateProvider.simGetReplicaInfos(node); - if (replicas == null || replicas.isEmpty()) { - return new HashMap<>(); - } - Map>> res = new HashMap<>(); - // TODO: probably needs special treatment for "metrics:solr.core..." tags - for (Replica r : replicas) { - @SuppressWarnings({"unchecked"}) - Map> perCollection = res.computeIfAbsent(r.getCollection(), Utils.NEW_HASHMAP_FUN); - @SuppressWarnings({"unchecked"}) - List perShard = perCollection.computeIfAbsent(r.getShard(), Utils.NEW_ARRAYLIST_FUN); - // XXX filter out some properties? - perShard.add(r); - } - return res; - } - - @Override - public void close() throws IOException { - - } -} diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/sim/SimScenario.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/sim/SimScenario.java deleted file mode 100644 index 5cba5f5c46c..00000000000 --- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/sim/SimScenario.java +++ /dev/null @@ -1,1129 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.cloud.autoscaling.sim; - -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.PrintStream; -import java.io.Reader; -import java.lang.invoke.MethodHandles; -import java.net.URLDecoder; -import java.nio.charset.Charset; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Optional; -import java.util.Properties; -import java.util.Set; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; - -import org.apache.solr.client.solrj.SolrRequest; -import org.apache.solr.client.solrj.SolrResponse; -import org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig; -import org.apache.solr.client.solrj.cloud.autoscaling.Clause; -import org.apache.solr.client.solrj.cloud.autoscaling.Policy; -import org.apache.solr.client.solrj.cloud.autoscaling.PolicyHelper; -import org.apache.solr.client.solrj.cloud.autoscaling.Suggester; -import org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventProcessorStage; -import org.apache.solr.client.solrj.cloud.autoscaling.Variable; -import org.apache.solr.client.solrj.impl.CloudSolrClient; -import org.apache.solr.client.solrj.impl.SolrClientCloudManager; -import org.apache.solr.client.solrj.request.GenericSolrRequest; -import org.apache.solr.client.solrj.request.RequestWriter; -import org.apache.solr.client.solrj.request.UpdateRequest; -import org.apache.solr.client.solrj.request.V2Request; -import org.apache.solr.cloud.CloudUtil; -import org.apache.solr.cloud.autoscaling.ActionContext; -import org.apache.solr.cloud.autoscaling.AutoScaling; -import org.apache.solr.cloud.autoscaling.AutoScalingHandler; -import org.apache.solr.cloud.autoscaling.TriggerEvent; -import org.apache.solr.cloud.autoscaling.TriggerListener; -import org.apache.solr.cloud.autoscaling.TriggerListenerBase; -import org.apache.solr.common.cloud.Replica; -import org.apache.solr.common.params.AutoScalingParams; -import org.apache.solr.common.params.CollectionAdminParams; -import org.apache.solr.common.params.CollectionParams; -import org.apache.solr.common.params.ModifiableSolrParams; -import org.apache.solr.common.params.SolrParams; -import org.apache.solr.common.util.IOUtils; -import org.apache.solr.common.util.TimeSource; -import org.apache.solr.common.util.Utils; -import org.apache.solr.util.CLIO; -import org.apache.solr.util.PropertiesUtil; -import org.apache.solr.util.RedactionUtils; -import org.apache.solr.util.TimeOut; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * This class represents an autoscaling scenario consisting of a series of autoscaling - * operations on a simulated cluster. - */ -public class SimScenario implements AutoCloseable { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - /** Context variable: Random live node name. */ - public static final String RANDOM_NODE_CTX_PROP = "_random_node_"; - /** Context variable: Node name of the current Overseer leader. */ - public static final String OVERSEER_LEADER_CTX_PROP = "_overseer_leader_"; - /** Context variable: List of live nodes. */ - public static final String LIVE_NODES_CTX_PROP = "_live_nodes_"; - /** Context variable: List of collections. */ - public static final String COLLECTIONS_CTX_PROP = "_collections_"; - /** Context variable: List of calculated suggestions. */ - public static final String SUGGESTIONS_CTX_PROP = "_suggestions_"; - /** Context variable: List of SolrResponses of SOLR_REQUEST operations. */ - public static final String RESPONSES_CTX_PROP = "_responses_"; - /** Context variable: Current loop iteration or none if outside of loop. */ - public static final String LOOP_ITER_PROP = "_loop_iter_"; - /** Last trigger event captured by WAIT_EVENT. */ - public static final String TRIGGER_EVENT_PREFIX = "_trigger_event_"; - - public SimCloudManager cluster; - public AutoScalingConfig config; - public List ops = new ArrayList<>(); - public Map context = new HashMap<>(); - public PrintStream console = CLIO.getErrStream(); - public boolean verbose; - public boolean abortLoop; - public boolean abortScenario; - - /** Base class for implementation of scenario DSL actions. */ - public static abstract class SimOp { - ModifiableSolrParams initParams; - ModifiableSolrParams params; - - public void init(SolrParams params) { - this.initParams = new ModifiableSolrParams(params); - } - - /** - * This method prepares a copy of initial params (and sets the value of {@link #params} - * with all property references resolved against the current {@link SimScenario#context} - * and system properties. This method should always be called before invoking - * {@link #execute(SimScenario)}. - * @param scenario current scenario - */ - @SuppressWarnings({"unchecked"}) - public void prepareCurrentParams(SimScenario scenario) { - Properties props = new Properties(); - scenario.context.forEach((k, v) -> { - if (v instanceof String[]) { - v = String.join(",", (String[]) v); - } else if (v instanceof Collection) { - StringBuilder sb = new StringBuilder(); - for (Object o : (Collection)v) { - if (sb.length() > 0) { - sb.append(','); - } - if ((o instanceof String) || (o instanceof Number)) { - sb.append(o); - } else { - // skip all values - return; - } - } - v = sb.toString(); - } else if ((v instanceof String) || (v instanceof Number)) { - // don't convert, put as is - } else { - // skip - return; - } - props.put(k, v); - }); - ModifiableSolrParams currentParams = new ModifiableSolrParams(); - initParams.forEach(e -> { - String newKey = PropertiesUtil.substituteProperty(e.getKey(), props); - if (newKey == null) { - newKey = e.getKey(); - } - String[] newValues; - if (e.getValue() != null && e.getValue().length > 0) { - String[] values = e.getValue(); - newValues = new String[values.length]; - for (int k = 0; k < values.length; k++) { - String newVal = PropertiesUtil.substituteProperty(values[k], props); - if (newVal == null) { - newVal = values[k]; - } - newValues[k] = newVal; - } - } else { - newValues = e.getValue(); - } - currentParams.add(newKey, newValues); - }); - params = currentParams; - } - - /** - * Execute the operation. - * @param scenario current scenario. - */ - public abstract void execute (SimScenario scenario) throws Exception; - } - - - /** - * Actions supported by the scenario. - */ - public enum SimAction { - /** Create a new simulated cluster. */ - CREATE_CLUSTER, - /** Create a simulated cluster from autoscaling snapshot. */ - LOAD_SNAPSHOT, - /** Save autoscaling snapshot of the current simulated cluster. */ - SAVE_SNAPSHOT, - /** Calculate autoscaling suggestions and put them in the scenario's context. */ - CALCULATE_SUGGESTIONS, - /** Apply previously calculated autoscaling suggestions. */ - APPLY_SUGGESTIONS, - /** Kill specific nodes, or a number of randomly selected nodes. */ - KILL_NODES, - /** Add new nodes. */ - ADD_NODES, - /** Load autoscaling.json configuration from a file. */ - LOAD_AUTOSCALING, - /** Start a loop. */ - LOOP_START, - /** End a loop. */ - LOOP_END, - /** Set operation delays to simulate long-running actions. */ - SET_OP_DELAYS, - /** Execute a SolrRequest (must be supported by {@link SimCloudManager}). */ - SOLR_REQUEST, - /** Wait for a collection to reach the indicated number of shards and replicas. */ - WAIT_COLLECTION, - /** Prepare a listener to listen for an autoscaling event. */ - EVENT_LISTENER, - /** Wait for an autoscaling event using previously prepared listener. */ - WAIT_EVENT, - /** Run the simulation for a while, allowing background tasks to execute. */ - RUN, - /** Dump the internal state of the simulator to console. */ - DUMP, - /** Set a variable in context. */ - CTX_SET, - /** Remove a variable from context. */ - CTX_REMOVE, - /** Set metrics for a node. */ - SET_NODE_METRICS, - /** Set metrics for each replica of a collection's shard(s). */ - SET_SHARD_METRICS, - /** Bulk index a number of simulated documents. */ - INDEX_DOCS, - /** Assert a condition. */ - ASSERT; - - public static SimAction get(String str) { - if (str != null) { - try { - return SimAction.valueOf(str.toUpperCase(Locale.ROOT)); - } catch (Exception e) { - return null; - } - } else { - return null; - } - } - - public String toLower() { - return toString().toLowerCase(Locale.ROOT); - } - } - - public static Map> simOps = new HashMap<>(); - static { - simOps.put(SimAction.CREATE_CLUSTER, CreateCluster.class); - simOps.put(SimAction.LOAD_SNAPSHOT, LoadSnapshot.class); - simOps.put(SimAction.SAVE_SNAPSHOT, SaveSnapshot.class); - simOps.put(SimAction.LOAD_AUTOSCALING, LoadAutoscaling.class); - simOps.put(SimAction.CALCULATE_SUGGESTIONS, CalculateSuggestions.class); - simOps.put(SimAction.APPLY_SUGGESTIONS, ApplySuggestions.class); - simOps.put(SimAction.KILL_NODES, KillNodes.class); - simOps.put(SimAction.ADD_NODES, AddNodes.class); - simOps.put(SimAction.LOOP_START, LoopOp.class); - simOps.put(SimAction.LOOP_END, null); - simOps.put(SimAction.SET_OP_DELAYS, SetOpDelays.class); - simOps.put(SimAction.SOLR_REQUEST, RunSolrRequest.class); - simOps.put(SimAction.RUN, RunSimulator.class); - simOps.put(SimAction.WAIT_COLLECTION, WaitCollection.class); - simOps.put(SimAction.EVENT_LISTENER, SetEventListener.class); - simOps.put(SimAction.WAIT_EVENT, WaitEvent.class); - simOps.put(SimAction.CTX_SET, CtxSet.class); - simOps.put(SimAction.CTX_REMOVE, CtxRemove.class); - simOps.put(SimAction.DUMP, Dump.class); - simOps.put(SimAction.SET_NODE_METRICS, SetNodeMetrics.class); - simOps.put(SimAction.SET_SHARD_METRICS, SetShardMetrics.class); - simOps.put(SimAction.INDEX_DOCS, IndexDocs.class); - simOps.put(SimAction.ASSERT, Assert.class); - } - - /** - * Loop action. - */ - public static class LoopOp extends SimOp { - // populated by the DSL parser - List ops = new ArrayList<>(); - int iterations; - - @Override - public void execute(SimScenario scenario) throws Exception { - iterations = Integer.parseInt(params.get("iterations", "10")); - for (int i = 0; i < iterations; i++) { - if (scenario.abortLoop) { - log.info(" -- abortLoop requested, aborting after {} iterations.", i); - return; - } - scenario.context.put(LOOP_ITER_PROP, String.valueOf(i)); - log.info(" * iter {} :", i + 1); // logOK - for (SimOp op : ops) { - op.prepareCurrentParams(scenario); - if (log.isInfoEnabled()) { - log.info(" - {}\t{})", op.getClass().getSimpleName(), op.params); - } - op.execute(scenario); - if (scenario.abortLoop) { - log.info(" -- abortLoop requested, aborting after {} iterations.", i); - return; - } - } - } - } - } - - /** - * Set a context property. - */ - public static class CtxSet extends SimOp { - @Override - public void execute(SimScenario scenario) throws Exception { - String key = params.required().get("key"); - String[] values = params.required().getParams("value"); - if (values != null) { - scenario.context.put(key, Arrays.asList(values)); - } else { - scenario.context.remove(key); - } - } - } - - /** - * Remove a context property. - */ - public static class CtxRemove extends SimOp { - @Override - public void execute(SimScenario scenario) throws Exception { - String key = params.required().get("key"); - scenario.context.remove(key); - } - } - - /** - * Create a simulated cluster. - */ - public static class CreateCluster extends SimOp { - @Override - public void execute(SimScenario scenario) throws Exception { - int numNodes = Integer.parseInt(params.get("numNodes", "5")); - boolean disableMetricsHistory = Boolean.parseBoolean(params.get("disableMetricsHistory", "false")); - String timeSourceStr = params.get("timeSource", "simTime:50"); - if (scenario.cluster != null) { // close & reset - IOUtils.closeQuietly(scenario.cluster); - scenario.context.clear(); - } - scenario.cluster = SimCloudManager.createCluster(numNodes, TimeSource.get(timeSourceStr)); - if (disableMetricsHistory) { - scenario.cluster.disableMetricsHistory(); - } - scenario.config = scenario.cluster.getDistribStateManager().getAutoScalingConfig(); - } - } - - /** - * Create a simulated cluster from an autoscaling snapshot. - */ - public static class LoadSnapshot extends SimOp { - @Override - public void execute(SimScenario scenario) throws Exception { - String path = params.get("path"); - SnapshotCloudManager snapshotCloudManager; - if (path == null) { - String zkHost = params.get("zkHost"); - if (zkHost == null) { - throw new IOException(SimAction.LOAD_SNAPSHOT + " must specify 'path' or 'zkHost'"); - } else { - try (CloudSolrClient cloudSolrClient = new CloudSolrClient.Builder(Collections.singletonList(zkHost), Optional.empty()).build()) { - cloudSolrClient.connect(); - try (SolrClientCloudManager realCloudManager = new SolrClientCloudManager(NoopDistributedQueueFactory.INSTANCE, cloudSolrClient)) { - snapshotCloudManager = new SnapshotCloudManager(realCloudManager, null); - } - } - } - } else { - snapshotCloudManager = SnapshotCloudManager.readSnapshot(new File(path)); - } - scenario.cluster = SimCloudManager.createCluster(snapshotCloudManager, null, snapshotCloudManager.getTimeSource()); - scenario.config = scenario.cluster.getDistribStateManager().getAutoScalingConfig(); - } - } - - /** - * Save an autoscaling snapshot. - */ - public static class SaveSnapshot extends SimOp { - @Override - public void execute(SimScenario scenario) throws Exception { - String path = params.get("path"); - if (path == null) { - throw new IOException(SimAction.SAVE_SNAPSHOT + " must specify 'path'"); - } - boolean redact = Boolean.parseBoolean(params.get("redact", "false")); - try (SnapshotCloudManager snapshotCloudManager = new SnapshotCloudManager(scenario.cluster, null)) { - snapshotCloudManager.saveSnapshot(new File(path), true, redact); - } - } - } - - /** - * Load autoscaling.json configuration. - */ - public static class LoadAutoscaling extends SimOp { - @Override - @SuppressWarnings({"unchecked"}) - public void execute(SimScenario scenario) throws Exception { - Map map; - boolean addDefaults = Boolean.parseBoolean(params.get("withDefaultTriggers", "true")); - int defaultWaitFor = Integer.parseInt(params.get("defaultWaitFor", "120")); - String path = params.get("path"); - if (path == null) { - String json = params.get("json"); - if (json == null) { - throw new IOException(SimAction.LOAD_AUTOSCALING + " must specify either 'path' or 'json'"); - } else { - map = (Map) Utils.fromJSONString(json); - } - } else { - File f = new File(path); - Reader r; - if (f.exists()) { - r = new InputStreamReader(new FileInputStream(f), Charset.forName("UTF-8")); - } else { - InputStream is = getClass().getResourceAsStream(path); - if (is == null) { - throw new IOException("path " + path + " does not exist and it's not a resource"); - } - r = new InputStreamReader(is, Charset.forName("UTF-8")); - } - map = (Map) Utils.fromJSON(r); - } - AutoScalingConfig config = new AutoScalingConfig(map); - if (addDefaults) { - // add default triggers - if (!config.getTriggerConfigs().containsKey(AutoScaling.AUTO_ADD_REPLICAS_TRIGGER_NAME)) { - Map props = new HashMap<>(AutoScaling.AUTO_ADD_REPLICAS_TRIGGER_PROPS); - props.put("waitFor", defaultWaitFor); - AutoScalingConfig.TriggerConfig trigger = new AutoScalingConfig.TriggerConfig(AutoScaling.AUTO_ADD_REPLICAS_TRIGGER_NAME, props); - config = config.withTriggerConfig(trigger); - config = AutoScalingHandler.withSystemLogListener(config, AutoScaling.AUTO_ADD_REPLICAS_TRIGGER_NAME); - } - if (!config.getTriggerConfigs().containsKey(AutoScaling.SCHEDULED_MAINTENANCE_TRIGGER_NAME)) { - AutoScalingConfig.TriggerConfig trigger = new AutoScalingConfig.TriggerConfig(AutoScaling.SCHEDULED_MAINTENANCE_TRIGGER_NAME, AutoScaling.SCHEDULED_MAINTENANCE_TRIGGER_PROPS); - config = config.withTriggerConfig(trigger); - config = AutoScalingHandler.withSystemLogListener(config, AutoScaling.SCHEDULED_MAINTENANCE_TRIGGER_NAME); - } - } - scenario.config = config; - // set this config on the simulator - scenario.cluster.getSimDistribStateManager().simSetAutoScalingConfig(config); - // wait until it finished processing the config - (new TimeOut(30, TimeUnit.SECONDS, scenario.cluster.getTimeSource())) - .waitFor("OverseerTriggerThread never caught up to the latest znodeVersion", () -> { - try { - AutoScalingConfig autoscalingConfig = scenario.cluster.getDistribStateManager().getAutoScalingConfig(); - return autoscalingConfig.getZkVersion() == scenario.cluster.getOverseerTriggerThread().getProcessedZnodeVersion(); - } catch (Exception e) { - throw new RuntimeException("FAILED", e); - } - }); - - } - } - - /** - * Kill one or more nodes. - */ - public static class KillNodes extends SimOp { - @Override - public void execute(SimScenario scenario) throws Exception { - if (params.get("numNodes") != null) { - int numNodes = Integer.parseInt(params.get("numNodes")); - scenario.cluster.simRemoveRandomNodes(numNodes, false, scenario.cluster.getRandom()); - } else if (params.get("nodes") != null || params.get("node") != null) { - Set nodes = new HashSet<>(); - String[] nodesValues = params.getParams("nodes"); - if (nodesValues != null) { - for (String nodesValue : nodesValues) { - String[] vals = nodesValue.split(","); - nodes.addAll(Arrays.asList(vals)); - } - } - nodesValues = params.getParams("node"); - if (nodesValues != null) { - nodes.addAll(Arrays.asList(nodesValues)); - } - for (String node : nodes) { - scenario.cluster.simRemoveNode(node, false); - } - } - } - } - - /** - * Add one or more nodes. - */ - public static class AddNodes extends SimOp { - @Override - public void execute(SimScenario scenario) throws Exception { - int numNodes = Integer.parseInt(params.get("numNodes")); - for (int i = 0; i < numNodes; i++) { - scenario.cluster.simAddNode(); - } - } - } - - /** - * Calculate autoscaling suggestions. - */ - public static class CalculateSuggestions extends SimOp { - @Override - public void execute(SimScenario scenario) throws Exception { - List suggestions = PolicyHelper.getSuggestions(scenario.config, scenario.cluster); - scenario.context.put(SUGGESTIONS_CTX_PROP, suggestions); - if (log.isInfoEnabled()) { - log.info(" - {} suggestions", suggestions.size()); - } - if (suggestions.isEmpty()) { - scenario.abortLoop = true; - } - } - } - - /** - * Apply autoscaling suggestions. - */ - public static class ApplySuggestions extends SimOp { - @Override - public void execute(SimScenario scenario) throws Exception { - @SuppressWarnings({"unchecked"}) - List suggestions = (List) scenario.context.getOrDefault(SUGGESTIONS_CTX_PROP, Collections.emptyList()); - int unresolvedCount = 0; - for (Suggester.SuggestionInfo suggestion : suggestions) { - @SuppressWarnings({"rawtypes"}) - SolrRequest operation = suggestion.getOperation(); - if (operation == null) { - unresolvedCount++; - if (suggestion.getViolation() == null) { - log.error(" -- ignoring suggestion without violation and without operation: {}", suggestion); - } - continue; - } - SolrParams params = operation.getParams(); - if (operation instanceof V2Request) { - params = SimUtils.v2AdminRequestToV1Params((V2Request)operation); - } - Map paramsMap = new LinkedHashMap<>(); - params.toMap(paramsMap); - Replica info = scenario.cluster.getSimClusterStateProvider().simGetReplicaInfo( - params.get(CollectionAdminParams.COLLECTION), params.get("replica")); - if (info == null) { - log.error("Could not find ReplicaInfo for params: {}", params); - } else if (scenario.verbose) { - paramsMap.put("replicaInfo", info); - } else if (info.get(Variable.Type.CORE_IDX.tagName) != null) { - paramsMap.put(Variable.Type.CORE_IDX.tagName, info.get(Variable.Type.CORE_IDX.tagName)); - } - try { - scenario.cluster.request(operation); - } catch (Exception e) { - log.error("Aborting - error executing suggestion {}", suggestion, e); - break; - } - } - if (suggestions.size() > 0 && unresolvedCount == suggestions.size()) { - log.info(" -- aborting simulation, only {} unresolved violations remain.", unresolvedCount); - scenario.abortLoop = true; - } - } - } - - /** - * Execute a SolrRequest supported by {@link SimCloudManager}. - */ - public static class RunSolrRequest extends SimOp { - @Override - public void execute(SimScenario scenario) throws Exception { - String path = params.get("path", "/"); - SolrRequest.METHOD m = SolrRequest.METHOD.valueOf(params.get("httpMethod", "GET")); - params.remove("httpMethod"); - String streamBody = params.get("stream.body"); - params.remove("stream.body"); - GenericSolrRequest req = new GenericSolrRequest(m, path, params); - if (streamBody != null) { - req.setContentWriter(new RequestWriter.StringPayloadContentWriter(streamBody, "application/json")); - } - SolrResponse rsp = scenario.cluster.request(req); - @SuppressWarnings({"unchecked"}) - List responses = (List) scenario.context.computeIfAbsent(RESPONSES_CTX_PROP, Utils.NEW_ARRAYLIST_FUN); - responses.add(rsp); - } - } - - /** - * Set delays for specified collection operations in order to simulate slow execution. - */ - public static class SetOpDelays extends SimOp { - @Override - public void execute(SimScenario scenario) throws Exception { - String[] collections = params.remove("collection"); - if (collections == null || collections.length == 0) { - throw new IOException("'collection' param is required but missing: " + params); - } - Map delays = new HashMap<>(); - params.forEach(e -> { - String key = e.getKey(); - CollectionParams.CollectionAction a = CollectionParams.CollectionAction.get(key); - if (a == null) { - log.warn("Invalid collection action {}, skipping...", key); - return; - } - String[] values = e.getValue(); - if (values == null || values[0].isBlank()) { - delays.put(a.name(), null); - } else { - Long value = Long.parseLong(values[0]); - delays.put(a.name(), value); - } - }); - for (String collection : collections) { - scenario.cluster.getSimClusterStateProvider().simSetOpDelays(collection, delays); - } - } - } - - /** - * Run the simulator for a while. - */ - public static class RunSimulator extends SimOp { - @Override - public void execute(SimScenario scenario) throws Exception { - int timeMs = Integer.parseInt(params.get("time", "60000")); - scenario.cluster.getTimeSource().sleep(timeMs); - } - } - - /** - * Wait for a specific collection shape. - */ - public static class WaitCollection extends SimOp { - @Override - public void execute(SimScenario scenario) throws Exception { - String collection = params.required().get("collection"); - int shards = Integer.parseInt(params.required().get("shards")); - int replicas = Integer.parseInt(params.required().get("replicas")); - boolean withInactive = Boolean.parseBoolean(params.get("withInactive", "false")); - boolean requireLeaders = Boolean.parseBoolean(params.get("requireLeaders", "true")); - int waitSec = Integer.parseInt(params.required().get("wait", "" + CloudUtil.DEFAULT_TIMEOUT)); - CloudUtil.waitForState(scenario.cluster, collection, waitSec, TimeUnit.SECONDS, - CloudUtil.clusterShape(shards, replicas, withInactive, requireLeaders)); - } - } - - private static class SimWaitListener extends TriggerListenerBase { - private final TimeSource timeSource; - private final AutoScalingConfig.TriggerListenerConfig config; - private CountDownLatch triggerFired = new CountDownLatch(1); - private TriggerEvent event; - - SimWaitListener(TimeSource timeSource, AutoScalingConfig.TriggerListenerConfig config) { - this.timeSource = timeSource; - this.config = config; - } - - @Override - public AutoScalingConfig.TriggerListenerConfig getConfig() { - return config; - } - - @Override - public boolean isEnabled() { - return true; - } - - @Override - public void onEvent(TriggerEvent event, TriggerEventProcessorStage stage, String actionName, ActionContext context, Throwable error, String message) throws Exception { - triggerFired.countDown(); - this.event = event; - } - - public TriggerEvent getEvent() { - return event; - } - - public void wait(int waitSec) throws Exception { - long waitTime = timeSource.convertDelay(TimeUnit.SECONDS, waitSec, TimeUnit.MILLISECONDS); - boolean await = triggerFired.await(waitTime, TimeUnit.MILLISECONDS); - if (!await) { - throw new IOException("Timed out waiting for trigger " + config.trigger + " to fire after simulated " + - waitSec + "s (real " + waitTime + "ms)."); - } - } - } - - /** - * Set a temporary listener to wait for a specific trigger event processing. - */ - @SuppressWarnings({"unchecked"}) - public static class SetEventListener extends SimOp { - @Override - public void execute(SimScenario scenario) throws Exception { - String trigger = params.required().get(AutoScalingParams.TRIGGER); - Map cfgMap = new HashMap<>(); - String name = ".sim_wait_event_" + trigger; - cfgMap.put(AutoScalingParams.NAME, name); - cfgMap.put(AutoScalingParams.TRIGGER, trigger); - - String[] beforeActions = params.getParams(AutoScalingParams.BEFORE_ACTION); - String[] afterActions = params.getParams(AutoScalingParams.AFTER_ACTION); - if (beforeActions != null) { - for (String beforeAction : beforeActions) { - ((List)cfgMap.computeIfAbsent(AutoScalingParams.BEFORE_ACTION, Utils.NEW_ARRAYLIST_FUN)).add(beforeAction); - } - } - if (afterActions != null) { - for (String afterAction : afterActions) { - ((List)cfgMap.computeIfAbsent(AutoScalingParams.AFTER_ACTION, Utils.NEW_ARRAYLIST_FUN)).add(afterAction); - } - } - String[] stages = params.required().getParams(AutoScalingParams.STAGE); - for (String stage : stages) { - String[] lst = stage.split("[,\\s]+"); - for (String val : lst) { - try { - TriggerEventProcessorStage.valueOf(val); - ((List)cfgMap.computeIfAbsent(AutoScalingParams.STAGE, Utils.NEW_ARRAYLIST_FUN)).add(val); - } catch (IllegalArgumentException e) { - throw new IOException("Invalid stage name '" + val + "'"); - } - } - } - final AutoScalingConfig.TriggerListenerConfig listenerConfig = new AutoScalingConfig.TriggerListenerConfig(name, cfgMap); - if (scenario.context.containsKey("_sim_waitListener_" + trigger)) { - throw new IOException("currently only one listener can be set per trigger. Trigger name: " + trigger); - } - TriggerListener listener = new SimWaitListener(scenario.cluster.getTimeSource(), listenerConfig); - scenario.context.put("_sim_waitListener_" + trigger, listener); - scenario.cluster.getOverseerTriggerThread().getScheduledTriggers().addAdditionalListener(listener); - } - } - - /** - * Wait for the previously set listener to capture an event. - */ - public static class WaitEvent extends SimOp { - @Override - public void execute(SimScenario scenario) throws Exception { - String trigger = params.required().get(AutoScalingParams.TRIGGER); - int waitSec = Integer.parseInt(params.get("wait", "" + CloudUtil.DEFAULT_TIMEOUT)); - SimWaitListener listener = (SimWaitListener)scenario.context.remove("_sim_waitListener_" + trigger); - if (listener == null) { - throw new IOException(SimAction.WAIT_EVENT + " must be preceded by " + SimAction.EVENT_LISTENER + " for trigger " + trigger); - } - try { - listener.wait(waitSec); - scenario.context.remove(TRIGGER_EVENT_PREFIX + trigger); - if (listener.getEvent() != null) { - @SuppressWarnings({"unchecked"}) - Map ev = listener.getEvent().toMap(new LinkedHashMap<>()); - scenario.context.put(TRIGGER_EVENT_PREFIX + trigger, ev); - } - } finally { - scenario.cluster.getOverseerTriggerThread().getScheduledTriggers().removeAdditionalListener(listener); - } - } - } - - public static class SetNodeMetrics extends SimOp { - - @Override - public void execute(SimScenario scenario) throws Exception { - String nodeset = params.required().get(Clause.NODESET); - Set nodes = new HashSet<>(); - if (nodeset.equals(Policy.ANY)) { - nodes.addAll(scenario.cluster.getLiveNodesSet().get()); - } else { - String[] list = nodeset.split("[,\\s]+"); - for (String node : list) { - if (node.isBlank()) { - continue; - } - nodes.add(node); - } - } - Map values = new HashMap<>(); - params.remove(Clause.NODESET); - for (String key : params.getParameterNames()) { - String strVal = params.get(key); - Object val; - // try auto-converting to a number - try { - val = Long.parseLong(strVal); - } catch (NumberFormatException nfe) { - try { - val = Double.parseDouble(strVal); - } catch (NumberFormatException nfe1) { - val = strVal; - } - } - values.put(key, val); - } - for (String node : nodes) { - Map newValues = new HashMap<>(scenario.cluster.getSimNodeStateProvider().simGetNodeValues(node)); - newValues.putAll(values); - scenario.cluster.getSimNodeStateProvider().simSetNodeValues(node, newValues); - } - } - } - - public static class SetShardMetrics extends SimOp { - - @Override - public void execute(SimScenario scenario) throws Exception { - String collection = params.required().get("collection"); - String shard = params.get("shard"); - boolean delta = params.getBool("delta", false); - boolean divide = params.getBool("divide", false); - params.remove("collection"); - params.remove("shard"); - params.remove("delta"); - params.remove("divide"); - Map values = new HashMap<>(); - for (String key : params.getParameterNames()) { - // try guessing if it's a number - try { - Integer i = Integer.valueOf(params.get(key)); - values.put(key, i); - } catch (NumberFormatException nfe) { - try { - Double d = Double.valueOf(params.get(key)); - values.put(key, d); - } catch (NumberFormatException nfe1) { - // not a number - values.put(key, params.get(key)); - } - } - } - values.forEach((k, v) -> { - try { - scenario.cluster.getSimClusterStateProvider().simSetShardValue(collection, shard, k, v, delta, divide); - } catch (Exception e) { - throw new RuntimeException("Error setting shard value", e); - } - }); - } - } - - public static class IndexDocs extends SimOp { - - @Override - public void execute(SimScenario scenario) throws Exception { - String collection = params.required().get("collection"); - long numDocs = params.required().getLong("numDocs"); - long start = params.getLong("start", 0L); - - UpdateRequest ureq = new UpdateRequest(); - ureq.setParam("collection", collection); - ureq.setDocIterator(new FakeDocIterator(start, numDocs)); - scenario.cluster.simGetSolrClient().request(ureq); - } - } - - public enum Condition { - EQUALS, - NOT_EQUALS, - NULL, - NOT_NULL; - - public static Condition get(String p) { - if (p == null) { - return null; - } else { - try { - return Condition.valueOf(p.toUpperCase(Locale.ROOT)); - } catch (Exception e) { - return null; - } - } - } - } - - public static class Assert extends SimOp { - - @Override - public void execute(SimScenario scenario) throws Exception { - String key = params.get("key"); - Condition condition = Condition.get(params.required().get("condition")); - if (condition == null) { - throw new IOException("Invalid 'condition' in params: " + params); - } - String expected = params.get("expected"); - if (condition != Condition.NOT_NULL && condition != Condition.NULL && expected == null) { - throw new IOException("'expected' param is required when condition is " + condition); - } - Object value; - if (key != null) { - if (key.contains("/")) { - value = Utils.getObjectByPath(scenario.context, true, key); - } else { - value = scenario.context.get(key); - } - } else { - value = params.required().get("value"); - } - switch (condition) { - case NULL: - if (value != null) { - throw new IOException("expected value should be null but was '" + value + "'"); - } - break; - case NOT_NULL: - if (value == null) { - throw new IOException("expected value should not be null"); - } - break; - case EQUALS: - if (!expected.equals(String.valueOf(value))) { - throw new IOException("expected value is '" + expected + "' but actual value is '" + value + "'"); - } - break; - case NOT_EQUALS: - if (expected.equals(String.valueOf(value))) { - throw new IOException("expected value is '" + expected + "' and actual value is the same while it should be different"); - } - break; - } - } - } - - - /** - * Dump the simulator state to the console. - */ - public static class Dump extends SimOp { - @Override - @SuppressWarnings({"unchecked"}) - public void execute(SimScenario scenario) throws Exception { - boolean redact = Boolean.parseBoolean(params.get("redact", "false")); - boolean withData = Boolean.parseBoolean(params.get("withData", "false")); - boolean withStats = Boolean.parseBoolean(params.get("withStats", "false")); - boolean withSuggestions = Boolean.parseBoolean(params.get("withSuggestions", "true")); - boolean withDiagnostics = Boolean.parseBoolean(params.get("withDiagnostics", "false")); - boolean withNodeState = Boolean.parseBoolean(params.get("withNodeState", "false")); - boolean withClusterState = Boolean.parseBoolean(params.get("withClusterState", "false")); - boolean withManagerState = Boolean.parseBoolean(params.get("withManagerState", "false")); - SnapshotCloudManager snapshotCloudManager = new SnapshotCloudManager(scenario.cluster, null); - Map snapshot = snapshotCloudManager.getSnapshot(true, redact); - if (!withData) { - snapshot.remove(SnapshotCloudManager.DISTRIB_STATE_KEY); - } - if (!withNodeState) { - snapshot.remove(SnapshotCloudManager.NODE_STATE_KEY); - } - if (!withClusterState) { - snapshot.remove(SnapshotCloudManager.CLUSTER_STATE_KEY); - } - if (!withStats) { - snapshot.remove(SnapshotCloudManager.STATISTICS_STATE_KEY); - } - if (!withManagerState) { - snapshot.remove(SnapshotCloudManager.MANAGER_STATE_KEY); - } - if (!withDiagnostics) { - ((Map)snapshot.get(SnapshotCloudManager.AUTOSCALING_STATE_KEY)).remove("diagnostics"); - } - if (!withSuggestions) { - ((Map)snapshot.get(SnapshotCloudManager.AUTOSCALING_STATE_KEY)).remove("suggestions"); - } - String data = Utils.toJSONString(snapshot); - if (redact) { - RedactionUtils.RedactionContext ctx = SimUtils.getRedactionContext(snapshotCloudManager.getClusterStateProvider().getClusterState()); - data = RedactionUtils.redactNames(ctx.getRedactions(), data); - } - snapshotCloudManager.close(); - scenario.console.println(data); - } - } - - /** - * Parse a DSL string and create a scenario ready to run. - * @param data DSL string with commands and parameters - * @return configured scenario - * @throws Exception on syntax errors - */ - public static SimScenario load(String data) throws Exception { - @SuppressWarnings("resource") - SimScenario scenario = new SimScenario(); - String[] lines = data.split("\\r?\\n"); - for (int i = 0; i < lines.length; i++) { - String line = lines[i]; - line = line.trim(); - if (line.isBlank() || line.startsWith("#") || line.startsWith("//")) { - continue; - } - // remove trailing / / comments - String[] comments = line.split("//"); - String expr = comments[0]; - // split on blank - String[] parts = expr.split("\\s+"); - if (parts.length > 2) { - log.warn("Invalid line - wrong number of parts {}, skipping: {}", parts.length, line); - continue; - } - SimAction action = SimAction.get(parts[0]); - if (action == null) { - log.warn("Invalid scenario action {}, skipping...", parts[0]); - continue; - } - if (action == SimAction.LOOP_END) { - if (!scenario.context.containsKey("loop")) { - throw new IOException("LOOP_END without start!"); - } - scenario.context.remove("loop"); - continue; - } - Class opClass = simOps.get(action); - SimOp op = opClass.getConstructor().newInstance(); - ModifiableSolrParams params = new ModifiableSolrParams(); - if (parts.length > 1) { - String paramsString = parts[1]; - if (parts[1].contains("?")) { // url-like with path?params... - String[] urlParts = parts[1].split("\\?"); - params.set("path", urlParts[0]); - paramsString = urlParts.length > 1 ? urlParts[1] : ""; - } - String[] paramsParts = paramsString.split("&"); - for (String paramPair : paramsParts) { - String[] paramKV = paramPair.split("="); - String k = URLDecoder.decode(paramKV[0], "UTF-8"); - String v = paramKV.length > 1 ? URLDecoder.decode(paramKV[1], "UTF-8") : null; - params.add(k, v); - } - } - op.init(params); - // loop handling - if (action == SimAction.LOOP_START) { - if (scenario.context.containsKey("loop")) { - throw new IOException("only one loop level is allowed"); - } - scenario.context.put("loop", op); - scenario.ops.add(op); - continue; - } - LoopOp currentLoop = (LoopOp) scenario.context.get("loop"); - if (currentLoop != null) { - currentLoop.ops.add(op); - } else { - scenario.ops.add(op); - } - } - if (scenario.context.containsKey("loop")) { - throw new IOException("Unterminated loop statement"); - } - // sanity check set_listener / wait_listener - int numSets = 0, numWaits = 0; - for (SimOp op : scenario.ops) { - if (op instanceof SetEventListener) { - numSets++; - } else if (op instanceof WaitEvent) { - numWaits++; - } - if (numWaits > numSets) { - throw new Exception("Unexpected " + SimAction.WAIT_EVENT + " without previous " + SimAction.EVENT_LISTENER); - } - } - if (numSets > numWaits) { - throw new Exception(SimAction.EVENT_LISTENER + " count should be equal to " + SimAction.WAIT_EVENT + " count but was " + - numSets + " > " + numWaits); - } - return scenario; - } - - /** - * Run the scenario. - */ - public void run() throws Exception { - for (int i = 0; i < ops.size(); i++) { - if (abortScenario) { - log.info("-- abortScenario requested, aborting after {} ops.", i); - return; - } - SimOp op = ops.get(i); - if (log.isInfoEnabled()) { - log.info("{}.\t{}\t{}", i + 1, op.getClass().getSimpleName(), op.initParams); // logOk - } - // substitute parameters based on the current context - if (cluster != null && cluster.getLiveNodesSet().size() > 0) { - context.put(LIVE_NODES_CTX_PROP, new ArrayList<>(cluster.getLiveNodesSet().get())); - context.put(RANDOM_NODE_CTX_PROP, cluster.getSimClusterStateProvider().simGetRandomNode()); - context.put(COLLECTIONS_CTX_PROP, cluster.getSimClusterStateProvider().simListCollections()); - context.put(OVERSEER_LEADER_CTX_PROP, cluster.getSimClusterStateProvider().simGetOverseerLeader()); - } else { - context.remove(LIVE_NODES_CTX_PROP); - context.remove(COLLECTIONS_CTX_PROP); - context.remove(RANDOM_NODE_CTX_PROP); - context.remove(SUGGESTIONS_CTX_PROP); - context.remove(OVERSEER_LEADER_CTX_PROP); - } - op.prepareCurrentParams(this); - if (log.isInfoEnabled()) { - log.info("\t\t{}\t{}", op.getClass().getSimpleName(), op.params); - } - op.execute(this); - } - } - - @Override - public void close() throws IOException { - if (cluster != null) { - cluster.close(); - cluster = null; - } - } -} diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/sim/SimUtils.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/sim/SimUtils.java deleted file mode 100644 index 944049e16bd..00000000000 --- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/sim/SimUtils.java +++ /dev/null @@ -1,434 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.cloud.autoscaling.sim; - -import java.lang.invoke.MethodHandles; -import java.net.MalformedURLException; -import java.net.URL; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.TreeMap; -import java.util.TreeSet; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.stream.Collectors; - -import org.apache.solr.client.solrj.cloud.SolrCloudManager; -import org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig; -import org.apache.solr.client.solrj.cloud.autoscaling.Cell; -import org.apache.solr.client.solrj.cloud.autoscaling.Policy; -import org.apache.solr.client.solrj.cloud.autoscaling.Row; -import org.apache.solr.client.solrj.cloud.autoscaling.Variable; -import org.apache.solr.client.solrj.request.CollectionApiMapping; -import org.apache.solr.client.solrj.request.V2Request; -import org.apache.solr.common.cloud.ClusterState; -import org.apache.solr.common.cloud.Replica; -import org.apache.solr.common.params.CollectionAdminParams; -import org.apache.solr.common.params.CoreAdminParams; -import org.apache.solr.common.params.ModifiableSolrParams; -import org.apache.solr.common.util.Utils; -import org.apache.solr.util.RedactionUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Various utility methods useful for autoscaling simulations and snapshots. - */ -public class SimUtils { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - - public static final Set COMMON_REPLICA_TAGS = new HashSet<>(Arrays.asList( - Variable.Type.CORE_IDX.metricsAttribute, - Variable.Type.CORE_IDX.tagName, - "SEARCHER.searcher.numDocs", - "SEARCHER.searcher.maxDoc", - "SEARCHER.searcher.indexCommitSize", - "QUERY./select.requests", - "UPDATE./update.requests" - )); - - public static final Set COMMON_NODE_TAGS = new HashSet<>(Arrays.asList( - Variable.Type.CORES.tagName, - Variable.Type.FREEDISK.tagName, - Variable.Type.NODE.tagName, - Variable.Type.NODE_ROLE.tagName, - Variable.Type.TOTALDISK.tagName, - Variable.Type.DISKTYPE.tagName, - Variable.Type.HEAPUSAGE.tagName, - Variable.Type.HOST.tagName, - Variable.Type.IP_1.tagName, - Variable.Type.IP_2.tagName, - Variable.Type.IP_3.tagName, - Variable.Type.IP_4.tagName, - Variable.Type.PORT.tagName, - Variable.Type.SYSLOADAVG.tagName, - "withCollection" - )); - - /** - * Check consistency of data in a {@link SolrCloudManager}. This may be needed when constructing a simulated - * instance from potentially inconsistent data (eg. partial snapshots taken at different points in time). - * @param solrCloudManager source manager - * @param config optional {@link AutoScalingConfig} instance used to determine what node and replica metrics to check. - */ - public static void checkConsistency(SolrCloudManager solrCloudManager, AutoScalingConfig config) throws Exception { - if (config == null) { - config = solrCloudManager.getDistribStateManager().getAutoScalingConfig(); - } - Set replicaTags = new HashSet<>(COMMON_REPLICA_TAGS); - replicaTags.addAll(config.getPolicy().getPerReplicaAttributes()); - - // verify replicas are consistent and data is available - Map> allReplicas = new HashMap<>(); - solrCloudManager.getClusterStateProvider().getClusterState().forEachCollection(coll -> { - coll.getReplicas().forEach(r -> { - if (allReplicas.containsKey(r.getName())) { - throw new RuntimeException("duplicate core_node name in clusterState: " + allReplicas.get(r.getName()) + " versus " + r); - } else { - allReplicas.computeIfAbsent(coll.getName(), c -> new HashMap<>()).put(r.getName(), r); - } - }); - }); - Map> allReplicaInfos = new HashMap<>(); - solrCloudManager.getClusterStateProvider().getLiveNodes().forEach(n -> { - Map>> infos = solrCloudManager.getNodeStateProvider().getReplicaInfo(n, replicaTags); - infos.forEach((coll, shards) -> shards.forEach((shard, replicas) -> replicas.forEach(r -> { - if (allReplicaInfos.containsKey(r.getName())) { - throw new RuntimeException("duplicate core_node name in NodeStateProvider: " + allReplicaInfos.get(r.getName()) + " versus " + r); - } else { - allReplicaInfos.computeIfAbsent(coll, c -> new HashMap<>()).put(r.getName(), r); - } - }))); - }); - if (!allReplicaInfos.keySet().equals(allReplicas.keySet())) { - Set notInClusterState = allReplicaInfos.keySet().stream() - .filter(k -> !allReplicas.containsKey(k)) - .collect(Collectors.toSet()); - Set notInNodeProvider = allReplicas.keySet().stream() - .filter(k -> !allReplicaInfos.containsKey(k)) - .collect(Collectors.toSet()); - throw new RuntimeException("Mismatched replica data between ClusterState and NodeStateProvider:\n\t" + - "collection not in ClusterState: " + notInClusterState + "\n\t" + - "collection not in NodeStateProvider: " + notInNodeProvider); - } - allReplicaInfos.keySet().forEach(collection -> { - Set infosCores = allReplicaInfos.getOrDefault(collection, Collections.emptyMap()).keySet(); - Map replicas = allReplicas.getOrDefault(collection, Collections.emptyMap()); - Set csCores = replicas.keySet(); - if (!infosCores.equals(csCores)) { - Set notInClusterState = infosCores.stream() - .filter(k -> !csCores.contains(k)) - .collect(Collectors.toSet()); - Set notInNodeProvider = csCores.stream() - .filter(k -> !infosCores.contains(k) && replicas.get(k).isActive(solrCloudManager.getClusterStateProvider().getLiveNodes())) - .collect(Collectors.toSet()); - if (!notInClusterState.isEmpty() || !notInNodeProvider.isEmpty()) { - throw new RuntimeException("Mismatched replica data for collection " + collection + " between ClusterState and NodeStateProvider:\n\t" + - "replica in NodeStateProvider but not in ClusterState: " + notInClusterState + "\n\t" + - "replica in ClusterState but not in NodeStateProvider: " + notInNodeProvider); - } - } - }); - // verify all replicas have size info - allReplicaInfos.forEach((coll, replicas) -> replicas.forEach((core, ri) -> { - Number size = (Number) ri.get(Variable.Type.CORE_IDX.metricsAttribute); - if (size == null) { - size = (Number) ri.get(Variable.Type.CORE_IDX.tagName); - if (size == null) { -// for (String node : solrCloudManager.getClusterStateProvider().getLiveNodes()) { -// log.error("Check for missing values: {}: {}", node, solrCloudManager.getNodeStateProvider().getReplicaInfo(node, SnapshotNodeStateProvider.REPLICA_TAGS)); -// } - throw new RuntimeException("missing replica size information: " + ri); - } - } - } - )); - } - - /** - * Calculate statistics of node / collection and replica layouts for the provided {@link SolrCloudManager}. - * @param cloudManager manager - * @param config autoscaling config, or null if the one from the provided manager should be used - * @param verbose if true then add more details about replicas. - * @return a map containing detailed statistics - */ - public static Map calculateStats(SolrCloudManager cloudManager, AutoScalingConfig config, boolean verbose) throws Exception { - ClusterState clusterState = cloudManager.getClusterStateProvider().getClusterState(); - Map> collStats = new TreeMap<>(); - Policy.Session session = config.getPolicy().createSession(cloudManager); - clusterState.forEachCollection(coll -> { - Map perColl = collStats.computeIfAbsent(coll.getName(), n -> new LinkedHashMap<>()); - AtomicInteger numCores = new AtomicInteger(); - HashMap> nodes = new HashMap<>(); - coll.getSlices().forEach(s -> { - numCores.addAndGet(s.getReplicas().size()); - s.getReplicas().forEach(r -> { - nodes.computeIfAbsent(r.getNodeName(), n -> new HashMap<>()) - .computeIfAbsent(s.getName(), slice -> new AtomicInteger()).incrementAndGet(); - }); - }); - int maxCoresPerNode = 0; - int minCoresPerNode = 0; - int maxActualShardsPerNode = 0; - int minActualShardsPerNode = 0; - int maxShardReplicasPerNode = 0; - int minShardReplicasPerNode = 0; - if (!nodes.isEmpty()) { - minCoresPerNode = Integer.MAX_VALUE; - minActualShardsPerNode = Integer.MAX_VALUE; - minShardReplicasPerNode = Integer.MAX_VALUE; - for (Map counts : nodes.values()) { - int total = counts.values().stream().mapToInt(c -> c.get()).sum(); - for (AtomicInteger count : counts.values()) { - if (count.get() > maxShardReplicasPerNode) { - maxShardReplicasPerNode = count.get(); - } - if (count.get() < minShardReplicasPerNode) { - minShardReplicasPerNode = count.get(); - } - } - if (total > maxCoresPerNode) { - maxCoresPerNode = total; - } - if (total < minCoresPerNode) { - minCoresPerNode = total; - } - if (counts.size() > maxActualShardsPerNode) { - maxActualShardsPerNode = counts.size(); - } - if (counts.size() < minActualShardsPerNode) { - minActualShardsPerNode = counts.size(); - } - } - } - perColl.put("activeShards", coll.getActiveSlices().size()); - perColl.put("inactiveShards", coll.getSlices().size() - coll.getActiveSlices().size()); - perColl.put("rf", coll.getReplicationFactor()); - perColl.put("maxActualShardsPerNode", maxActualShardsPerNode); - perColl.put("minActualShardsPerNode", minActualShardsPerNode); - perColl.put("maxShardReplicasPerNode", maxShardReplicasPerNode); - perColl.put("minShardReplicasPerNode", minShardReplicasPerNode); - perColl.put("numCores", numCores.get()); - perColl.put("numNodes", nodes.size()); - perColl.put("maxCoresPerNode", maxCoresPerNode); - perColl.put("minCoresPerNode", minCoresPerNode); - }); - Map> nodeStats = new TreeMap<>(); - Map coreStats = new TreeMap<>(); - List rows = session.getSortedNodes(); - // check consistency - if (rows.size() != clusterState.getLiveNodes().size()) { - throw new Exception("Mismatch between autoscaling matrix size (" + rows.size() + ") and liveNodes size (" + clusterState.getLiveNodes().size() + ")"); - } - for (Row row : rows) { - Map nodeStat = nodeStats.computeIfAbsent(row.node, n -> new LinkedHashMap<>()); - nodeStat.put("isLive", row.isLive()); - for (Cell cell : row.getCells()) { - nodeStat.put(cell.getName(), cell.getValue()); - } -// nodeStat.put("freedisk", row.getVal("freedisk", 0)); -// nodeStat.put("totaldisk", row.getVal("totaldisk", 0)); - int cores = ((Number)row.getVal("cores", 0)).intValue(); -// nodeStat.put("cores", cores); - coreStats.computeIfAbsent(cores, num -> new AtomicInteger()).incrementAndGet(); - Map>> collReplicas = new TreeMap<>(); - // check consistency - AtomicInteger rowCores = new AtomicInteger(); - row.forEachReplica(ri -> rowCores.incrementAndGet()); - if (cores != rowCores.get()) { - throw new Exception("Mismatch between autoscaling matrix row replicas (" + rowCores.get() + ") and number of cores (" + cores + ")"); - } - row.forEachReplica(ri -> { - Map perReplica = collReplicas.computeIfAbsent(ri.getCollection(), c -> new TreeMap<>()) - .computeIfAbsent(ri.getCoreName().substring(ri.getCollection().length() + 1), core -> new LinkedHashMap<>()); -// if (ri.getVariable(Variable.Type.CORE_IDX.tagName) != null) { -// perReplica.put(Variable.Type.CORE_IDX.tagName, ri.getVariable(Variable.Type.CORE_IDX.tagName)); -// } - if (ri.get(Variable.Type.CORE_IDX.metricsAttribute) != null) { - perReplica.put(Variable.Type.CORE_IDX.metricsAttribute, ri.get(Variable.Type.CORE_IDX.metricsAttribute)); - if (ri.get(Variable.Type.CORE_IDX.tagName) != null) { - perReplica.put(Variable.Type.CORE_IDX.tagName, ri.get(Variable.Type.CORE_IDX.tagName)); - } else { - perReplica.put(Variable.Type.CORE_IDX.tagName, - Variable.Type.CORE_IDX.convertVal(ri.get(Variable.Type.CORE_IDX.metricsAttribute))); - } - } - perReplica.put("coreNode", ri.getName()); - if (ri.isLeader() || ri.getBool("leader", false)) { - perReplica.put("leader", true); - Double totalSize = (Double)collStats.computeIfAbsent(ri.getCollection(), c -> new HashMap<>()) - .computeIfAbsent("avgShardSize", size -> 0.0); - Number riSize = (Number)ri.get(Variable.Type.CORE_IDX.metricsAttribute); - if (riSize != null) { - totalSize += riSize.doubleValue(); - collStats.get(ri.getCollection()).put("avgShardSize", totalSize); - Double max = (Double)collStats.get(ri.getCollection()).get("maxShardSize"); - if (max == null) max = 0.0; - if (riSize.doubleValue() > max) { - collStats.get(ri.getCollection()).put("maxShardSize", riSize.doubleValue()); - } - Double min = (Double)collStats.get(ri.getCollection()).get("minShardSize"); - if (min == null) min = Double.MAX_VALUE; - if (riSize.doubleValue() < min) { - collStats.get(ri.getCollection()).put("minShardSize", riSize.doubleValue()); - } - } else { - throw new RuntimeException("ReplicaInfo without size information: " + ri); - } - } - if (verbose) { - nodeStat.put("replicas", collReplicas); - } - }); - } - - // calculate average per shard and convert the units - for (Map perColl : collStats.values()) { - Number avg = perColl.get("avgShardSize"); - if (avg != null) { - avg = avg.doubleValue() / perColl.get("activeShards").doubleValue(); - perColl.put("avgShardSize", (Number)Variable.Type.CORE_IDX.convertVal(avg)); - } - Number num = perColl.get("maxShardSize"); - if (num != null) { - perColl.put("maxShardSize", (Number)Variable.Type.CORE_IDX.convertVal(num)); - } - num = perColl.get("minShardSize"); - if (num != null) { - perColl.put("minShardSize", (Number)Variable.Type.CORE_IDX.convertVal(num)); - } - } - Map stats = new LinkedHashMap<>(); - stats.put("coresPerNodes", coreStats); - stats.put("sortedNodeStats", nodeStats); - stats.put("collectionStats", collStats); - return stats; - } - - private static final Map v2v1Mapping = new HashMap<>(); - static { - for (CollectionApiMapping.Meta meta : CollectionApiMapping.Meta.values()) { - if (meta.action != null) { - String key; - if (meta.commandName != null) { - key = meta.commandName; - } else { - key = meta.action.toLower(); - } - v2v1Mapping.put(key, meta.action.toLower()); - } else { - log.warn("V2 action {} has no equivalent V1 action", meta); - } - } - } - - /** - * Convert a V2 {@link org.apache.solr.client.solrj.request.CollectionAdminRequest} to regular {@link org.apache.solr.common.params.SolrParams} - * @param req request - * @return request payload and parameters converted to V1 params - */ - @SuppressWarnings({"unchecked"}) - public static ModifiableSolrParams v2AdminRequestToV1Params(V2Request req) { - Map reqMap = new HashMap<>(); - req.toMap(reqMap); - String path = (String)reqMap.get("path"); - if (!(path.startsWith("/c/") || path.startsWith("/collections/")) || path.length() < 4) { - throw new UnsupportedOperationException("Unsupported V2 request path: " + reqMap); - } - Map cmd; - Object cmdObj = reqMap.get("command"); - if (cmdObj instanceof String) { - cmd = (Map)Utils.fromJSONString((String)cmdObj); - } else if (cmdObj instanceof Map) { - cmd = (Map)cmdObj; - } else { - throw new UnsupportedOperationException("Unsupported 'command': " + cmdObj + " (of type " + cmdObj.getClass() + ")"); - } - if (cmd.size() != 1) { - throw new UnsupportedOperationException("Unsupported multi-command V2 request: " + reqMap); - } - String a = cmd.keySet().iterator().next(); - ModifiableSolrParams params = new ModifiableSolrParams(); - params.set("path", "/admin/collections"); - if (req.getParams() != null) { - params.add(req.getParams()); - } - Map reqParams = (Map)cmd.get(a); - for (Map.Entry e : reqParams.entrySet()) { - params.add(e.getKey(), e.getValue().toString()); - } - // trim the leading / - path = path.substring(1); - String[] pathEls = path.split("/"); - if (pathEls.length < 2) { - throw new UnsupportedOperationException("Unsupported V2 request path: " + reqMap); - } - params.set(CollectionAdminParams.COLLECTION, pathEls[1]); - if (pathEls.length > 3) { - if (!pathEls[2].equals("shards")) { - throw new UnsupportedOperationException("Invalid V2 request path: expected 'shards' but was '" + pathEls[2] + "'"); - } - if (!pathEls[3].isBlank()) { - params.set("shard", pathEls[3]); - } - } - if (pathEls.length > 4 && !pathEls[4].isBlank()) { - params.set("replica", pathEls[4]); - } - // re-map from v2 to v1 action - a = v2v1Mapping.get(a); - if (a == null) { - throw new UnsupportedOperationException("Unsupported V2 request: " + reqMap); - } - params.add(CoreAdminParams.ACTION, a); - return params; - } - - /** - * Prepare collection and node / host names for redaction. - * @param clusterState cluster state - */ - public static RedactionUtils.RedactionContext getRedactionContext(ClusterState clusterState) { - RedactionUtils.RedactionContext ctx = new RedactionUtils.RedactionContext(); - TreeSet names = new TreeSet<>(clusterState.getLiveNodes()); - for (String nodeName : names) { - String urlString = Utils.getBaseUrlForNodeName(nodeName, "http"); - try { - URL u = new URL(urlString); - // protocol format - String hostPort = u.getHost() + ":" + u.getPort(); - ctx.addName(u.getHost() + ":" + u.getPort(), RedactionUtils.NODE_REDACTION_PREFIX); - // node name format - ctx.addEquivalentName(hostPort, u.getHost() + "_" + u.getPort() + "_", RedactionUtils.NODE_REDACTION_PREFIX); - } catch (MalformedURLException e) { - log.warn("Invalid URL for node name {}, replacing including protocol and path", nodeName, e); - ctx.addName(urlString, RedactionUtils.NODE_REDACTION_PREFIX); - ctx.addEquivalentName(urlString, Utils.getBaseUrlForNodeName(nodeName, "https"), RedactionUtils.NODE_REDACTION_PREFIX); - } - } - names.clear(); - names.addAll(clusterState.getCollectionStates().keySet()); - names.forEach(n -> ctx.addName(n, RedactionUtils.COLL_REDACTION_PREFIX)); - return ctx; - } -} diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/sim/SnapshotCloudManager.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/sim/SnapshotCloudManager.java deleted file mode 100644 index 2a0a25d7d71..00000000000 --- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/sim/SnapshotCloudManager.java +++ /dev/null @@ -1,260 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.cloud.autoscaling.sim; - -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.lang.invoke.MethodHandles; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; - -import org.apache.commons.io.IOUtils; -import org.apache.solr.client.solrj.SolrRequest; -import org.apache.solr.client.solrj.SolrResponse; -import org.apache.solr.client.solrj.cloud.DistribStateManager; -import org.apache.solr.client.solrj.cloud.DistributedQueueFactory; -import org.apache.solr.client.solrj.cloud.NodeStateProvider; -import org.apache.solr.client.solrj.cloud.SolrCloudManager; -import org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig; -import org.apache.solr.client.solrj.cloud.autoscaling.Policy; -import org.apache.solr.client.solrj.cloud.autoscaling.PolicyHelper; -import org.apache.solr.client.solrj.cloud.autoscaling.Suggester; -import org.apache.solr.client.solrj.impl.ClusterStateProvider; -import org.apache.solr.client.solrj.request.V2Request; -import org.apache.solr.common.cloud.ClusterState; -import org.apache.solr.common.cloud.Replica; -import org.apache.solr.common.params.CollectionAdminParams; -import org.apache.solr.common.params.SolrParams; -import org.apache.solr.common.util.ObjectCache; -import org.apache.solr.common.util.TimeSource; -import org.apache.solr.common.util.Utils; -import org.apache.solr.util.RedactionUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Read-only snapshot of another {@link SolrCloudManager}. - */ -public class SnapshotCloudManager implements SolrCloudManager { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - private ObjectCache objectCache = new ObjectCache(); - private SnapshotClusterStateProvider clusterStateProvider; - private SnapshotNodeStateProvider nodeStateProvider; - private SnapshotDistribStateManager distribStateManager; - private TimeSource timeSource; - - public static final String MANAGER_STATE_KEY = "managerState"; - public static final String CLUSTER_STATE_KEY = "clusterState"; - public static final String NODE_STATE_KEY = "nodeState"; - public static final String DISTRIB_STATE_KEY = "distribState"; - public static final String AUTOSCALING_STATE_KEY = "autoscalingState"; - public static final String STATISTICS_STATE_KEY = "statistics"; - public static final String AUTOSCALING_JSON_KEY = "autoscaling"; - - public static final List REQUIRED_KEYS = Arrays.asList( - MANAGER_STATE_KEY, - CLUSTER_STATE_KEY, - NODE_STATE_KEY, - DISTRIB_STATE_KEY - ); - - public SnapshotCloudManager(SolrCloudManager other, AutoScalingConfig config) throws Exception { - this.timeSource = other.getTimeSource(); - this.clusterStateProvider = new SnapshotClusterStateProvider(other.getClusterStateProvider()); - this.nodeStateProvider = new SnapshotNodeStateProvider(other, config); - this.distribStateManager = new SnapshotDistribStateManager(other.getDistribStateManager(), config); - SimUtils.checkConsistency(this, config); - } - - @SuppressWarnings({"unchecked"}) - public SnapshotCloudManager(Map snapshot) throws Exception { - Objects.requireNonNull(snapshot); - init( - (Map)snapshot.getOrDefault(MANAGER_STATE_KEY, Collections.emptyMap()), - (Map)snapshot.getOrDefault(CLUSTER_STATE_KEY, Collections.emptyMap()), - (Map)snapshot.getOrDefault(NODE_STATE_KEY, Collections.emptyMap()), - (Map)snapshot.getOrDefault(DISTRIB_STATE_KEY, Collections.emptyMap()), - (Map)snapshot.getOrDefault(AUTOSCALING_JSON_KEY, Collections.emptyMap()) - ); - } - - public void saveSnapshot(File targetDir, boolean withAutoscaling, boolean redact) throws Exception { - Map snapshot = getSnapshot(withAutoscaling, redact); - ClusterState clusterState = getClusterStateProvider().getClusterState(); - RedactionUtils.RedactionContext ctx = SimUtils.getRedactionContext(clusterState); - targetDir.mkdirs(); - for (Map.Entry e : snapshot.entrySet()) { - FileOutputStream out = new FileOutputStream(new File(targetDir, e.getKey() + ".json")); - if (redact) { - String data = Utils.toJSONString(e.getValue()); - data = RedactionUtils.redactNames(ctx.getRedactions(), data); - IOUtils.write(data.getBytes("UTF-8"), out); - } else { - IOUtils.write(Utils.toJSON(e.getValue()), out); - } - out.flush(); - out.close(); - } - } - - @SuppressWarnings({"unchecked"}) - public static SnapshotCloudManager readSnapshot(File sourceDir) throws Exception { - if (!sourceDir.exists()) { - throw new Exception("Source path doesn't exist: " + sourceDir); - } - if (!sourceDir.isDirectory()) { - throw new Exception("Source path is not a directory: " + sourceDir); - } - Map snapshot = new HashMap<>(); - List allKeys = new ArrayList<>(REQUIRED_KEYS); - allKeys.add(AUTOSCALING_JSON_KEY); - int validData = 0; - for (String key : allKeys) { - File src = new File(sourceDir, key + ".json"); - if (src.exists()) { - InputStream is = new FileInputStream(src); - Map data = (Map)Utils.fromJSON(is); - is.close(); - snapshot.put(key, data); - if (REQUIRED_KEYS.contains(key)) { - validData++; - } - } - } - if (validData < REQUIRED_KEYS.size()) { - throw new Exception("Some data is missing - expected: " + REQUIRED_KEYS + ", found: " + snapshot.keySet()); - } - return new SnapshotCloudManager(snapshot); - } - - private void init(Map managerState, Map clusterState, Map nodeState, - Map distribState, Map autoscalingJson) throws Exception { - Objects.requireNonNull(managerState); - Objects.requireNonNull(clusterState); - Objects.requireNonNull(nodeState); - Objects.requireNonNull(distribState); - this.timeSource = TimeSource.get((String)managerState.getOrDefault("timeSource", "simTime:50")); - this.clusterStateProvider = new SnapshotClusterStateProvider(clusterState); - this.nodeStateProvider = new SnapshotNodeStateProvider(nodeState); - if (autoscalingJson == null || autoscalingJson.isEmpty()) { - this.distribStateManager = new SnapshotDistribStateManager(distribState); - } else { - this.distribStateManager = new SnapshotDistribStateManager(distribState, new AutoScalingConfig(autoscalingJson)); - } - - SimUtils.checkConsistency(this, null); - } - - public Map getSnapshot(boolean withAutoscaling, boolean redact) throws Exception { - Map snapshot = new LinkedHashMap<>(4); - Map managerState = new HashMap<>(); - managerState.put("timeSource", timeSource.toString()); - snapshot.put(MANAGER_STATE_KEY, managerState); - RedactionUtils.RedactionContext ctx = redact ? SimUtils.getRedactionContext(clusterStateProvider.getClusterState()) : null; - snapshot.put(CLUSTER_STATE_KEY, clusterStateProvider.getSnapshot()); - snapshot.put(NODE_STATE_KEY, nodeStateProvider.getSnapshot()); - snapshot.put(DISTRIB_STATE_KEY, distribStateManager.getSnapshot(ctx)); - if (withAutoscaling) { - AutoScalingConfig config = distribStateManager.getAutoScalingConfig(); - Policy.Session session = config.getPolicy().createSession(this); - List suggestions = PolicyHelper.getSuggestions(config, this); - Map diagnostics = new LinkedHashMap<>(); - PolicyHelper.getDiagnostics(session).toMap(diagnostics); - List> suggestionDetails = new ArrayList<>(suggestions.size()); - suggestions.forEach(s -> { - Map map = new LinkedHashMap<>(); - map.put("suggestion", s); - if (s.getOperation() != null) { - SolrParams params = s.getOperation().getParams(); - if (s.getOperation() instanceof V2Request) { - params = SimUtils.v2AdminRequestToV1Params((V2Request)s.getOperation()); - } - Replica info = nodeStateProvider.getReplicaInfo( - params.get(CollectionAdminParams.COLLECTION), params.get("replica")); - if (info == null) { - log.warn("Can't find ReplicaInfo for suggested operation: {}", s); - } else { - map.put("replica", info); - } - } - suggestionDetails.add(map); - }); - Map autoscaling = new LinkedHashMap<>(); - autoscaling.put("suggestions", suggestionDetails); - autoscaling.put("diagnostics", diagnostics); - snapshot.put(AUTOSCALING_STATE_KEY, autoscaling); - } - snapshot.put(STATISTICS_STATE_KEY, SimUtils.calculateStats(this, distribStateManager.getAutoScalingConfig(), true)); - return snapshot; - } - - @Override - public ClusterStateProvider getClusterStateProvider() { - return clusterStateProvider; - } - - @Override - public NodeStateProvider getNodeStateProvider() { - return nodeStateProvider; - } - - @Override - public DistribStateManager getDistribStateManager() { - return distribStateManager; - } - - @Override - public DistributedQueueFactory getDistributedQueueFactory() { - return NoopDistributedQueueFactory.INSTANCE; - } - - @Override - public ObjectCache getObjectCache() { - return objectCache; - } - - @Override - public TimeSource getTimeSource() { - return timeSource; - } - - @Override - @SuppressWarnings({"rawtypes"}) - public SolrResponse request(SolrRequest req) throws IOException { - throw new UnsupportedOperationException("request"); - } - - @Override - public byte[] httpRequest(String url, SolrRequest.METHOD method, Map headers, String payload, int timeout, boolean followRedirects) throws IOException { - throw new UnsupportedOperationException("httpRequest"); - } - - @Override - public void close() throws IOException { - - } -} diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/sim/SnapshotClusterStateProvider.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/sim/SnapshotClusterStateProvider.java deleted file mode 100644 index af385c16607..00000000000 --- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/sim/SnapshotClusterStateProvider.java +++ /dev/null @@ -1,152 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.cloud.autoscaling.sim; - -import java.io.IOException; -import java.io.UnsupportedEncodingException; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; - -import org.apache.solr.client.solrj.impl.ClusterStateProvider; -import org.apache.solr.common.cloud.ClusterState; -import org.apache.solr.common.cloud.DocCollection; -import org.apache.solr.common.util.Utils; -import org.noggit.CharArr; -import org.noggit.JSONWriter; - -/** - * Read-only snapshot of another {@link ClusterStateProvider}. - */ -public class SnapshotClusterStateProvider implements ClusterStateProvider { - - final Set liveNodes; - final ClusterState clusterState; - final Map clusterProperties; - - public SnapshotClusterStateProvider(ClusterStateProvider other) throws Exception { - liveNodes = Set.copyOf(other.getLiveNodes()); - ClusterState otherState = other.getClusterState(); - clusterState = new ClusterState(liveNodes, otherState.getCollectionsMap()); - clusterProperties = new HashMap<>(other.getClusterProperties()); - } - - @SuppressWarnings({"unchecked"}) - public SnapshotClusterStateProvider(Map snapshot) { - Objects.requireNonNull(snapshot); - liveNodes = Set.copyOf((Collection)snapshot.getOrDefault("liveNodes", Collections.emptySet())); - clusterProperties = (Map)snapshot.getOrDefault("clusterProperties", Collections.emptyMap()); - Map stateMap = new HashMap<>((Map)snapshot.getOrDefault("clusterState", Collections.emptyMap())); - Map collectionStates = new HashMap<>(); - // back-compat with format = 1 - Integer stateVersion = Integer.valueOf(String.valueOf(stateMap.getOrDefault("version", 0))); - stateMap.remove("version"); - stateMap.forEach((name, state) -> { - Map mutableState = (Map)state; - Map collMap = (Map) mutableState.get(name); - if (collMap == null) { - // snapshot in format 1 - collMap = mutableState; - mutableState = Collections.singletonMap(name, state); - } - int version = Integer.parseInt(String.valueOf(collMap.getOrDefault("zNodeVersion", stateVersion))); - collMap.remove("zNodeVersion"); - byte[] data = Utils.toJSON(mutableState); - ClusterState collState = ClusterState.createFromJson(version, data, Collections.emptySet()); - collectionStates.put(name, collState.getCollection(name)); - }); - clusterState = new ClusterState(liveNodes, collectionStates); - } - - public Map getSnapshot() { - Map snapshot = new HashMap<>(); - snapshot.put("liveNodes", liveNodes); - if (clusterProperties != null) { - snapshot.put("clusterProperties", clusterProperties); - } - Map stateMap = new HashMap<>(); - snapshot.put("clusterState", stateMap); - clusterState.forEachCollection(coll -> { - CharArr out = new CharArr(); - JSONWriter writer = new JSONWriter(out, 2); - coll.write(writer); - String json = out.toString(); - try { - @SuppressWarnings({"unchecked"}) - Map collMap = new LinkedHashMap<>((Map)Utils.fromJSON(json.getBytes("UTF-8"))); - collMap.put("zNodeVersion", coll.getZNodeVersion()); - // format compatible with the real /state.json, which uses a mini-ClusterState - // consisting of a single collection - stateMap.put(coll.getName(), Collections.singletonMap(coll.getName(), collMap)); - } catch (UnsupportedEncodingException e) { - throw new RuntimeException("should not happen!", e); - } - }); - return snapshot; - } - - @Override - public ClusterState.CollectionRef getState(String collection) { - return clusterState.getCollectionRef(collection); - } - - @Override - public Set getLiveNodes() { - return liveNodes; - } - - @Override - public List resolveAlias(String alias) { - throw new UnsupportedOperationException("resolveAlias"); - } - - @Override - public Map getAliasProperties(String alias) { - throw new UnsupportedOperationException("getAliasProperties"); - } - - @Override - public ClusterState getClusterState() throws IOException { - return clusterState; - } - - @Override - public Map getClusterProperties() { - return clusterProperties; - } - - @Override - public String getPolicyNameByCollection(String coll) { - DocCollection collection = clusterState.getCollectionOrNull(coll); - return collection == null ? null : (String)collection.getProperties().get("policy"); - } - - @Override - public void connect() { - - } - - @Override - public void close() throws IOException { - - } -} diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/sim/SnapshotDistribStateManager.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/sim/SnapshotDistribStateManager.java deleted file mode 100644 index eb3a29fb399..00000000000 --- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/sim/SnapshotDistribStateManager.java +++ /dev/null @@ -1,228 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.cloud.autoscaling.sim; - -import java.io.IOException; -import java.lang.invoke.MethodHandles; -import java.nio.charset.Charset; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.NoSuchElementException; -import java.util.Set; -import java.util.regex.Pattern; -import java.util.stream.Collectors; - -import org.apache.solr.client.solrj.cloud.DistribStateManager; -import org.apache.solr.client.solrj.cloud.autoscaling.AlreadyExistsException; -import org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig; -import org.apache.solr.client.solrj.cloud.autoscaling.BadVersionException; -import org.apache.solr.client.solrj.cloud.autoscaling.NotEmptyException; -import org.apache.solr.client.solrj.cloud.autoscaling.VersionedData; -import org.apache.solr.common.cloud.ZkStateReader; -import org.apache.solr.common.params.AutoScalingParams; -import org.apache.solr.common.util.Base64; -import org.apache.solr.common.util.Utils; -import org.apache.solr.util.RedactionUtils; -import org.apache.zookeeper.CreateMode; -import org.apache.zookeeper.KeeperException; -import org.apache.zookeeper.Op; -import org.apache.zookeeper.OpResult; -import org.apache.zookeeper.Watcher; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Read-only snapshot of another {@link DistribStateManager} - */ -public class SnapshotDistribStateManager implements DistribStateManager { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - LinkedHashMap dataMap = new LinkedHashMap<>(); - - /** - * Populate this instance from another {@link DistribStateManager} instance. - * @param other another instance - * @param config optional {@link AutoScalingConfig}, which will overwrite any existing config. - */ - public SnapshotDistribStateManager(DistribStateManager other, AutoScalingConfig config) throws Exception { - List tree = other.listTree("/"); - if (log.isDebugEnabled()) { - log.debug("- copying {} resources from {}", tree.size(), other.getClass().getSimpleName()); - } - for (String path : tree) { - dataMap.put(path, other.getData(path)); - } - if (config != null) { // overwrite existing - VersionedData vd = new VersionedData(config.getZkVersion(), Utils.toJSON(config), CreateMode.PERSISTENT, "0"); - dataMap.put(ZkStateReader.SOLR_AUTOSCALING_CONF_PATH, vd); - } - } - - /** - * Populate this instance from a previously generated snapshot. - * @param snapshot previous snapshot created using this class. - */ - public SnapshotDistribStateManager(Map snapshot) { - this(snapshot, null); - } - /** - * Populate this instance from a previously generated snapshot. - * @param snapshot previous snapshot created using this class. - * @param config optional config to override the one from snapshot, may be null - */ - public SnapshotDistribStateManager(Map snapshot, AutoScalingConfig config) { - snapshot.forEach((path, value) -> { - @SuppressWarnings({"unchecked"}) - Map map = (Map)value; - Number version = (Number)map.getOrDefault("version", 0); - String owner = (String)map.get("owner"); - String modeStr = (String)map.getOrDefault("mode", CreateMode.PERSISTENT.toString()); - CreateMode mode = CreateMode.valueOf(modeStr); - byte[] bytes = null; - if (map.containsKey("data")) { - bytes = Base64.base64ToByteArray((String)map.get("data")); - } - dataMap.put(path, new VersionedData(version.intValue(), bytes, mode, owner)); - }); - if (config != null) { // overwrite existing - VersionedData vd = new VersionedData(config.getZkVersion(), Utils.toJSON(config), CreateMode.PERSISTENT, "0"); - dataMap.put(ZkStateReader.SOLR_AUTOSCALING_CONF_PATH, vd); - } - if (log.isDebugEnabled()) { - log.debug("- loaded snapshot of {} resources", dataMap.size()); - } - } - - // content of these nodes is a UTF-8 String and it needs to be redacted - private static final Set REDACTED = new HashSet<>() {{ - add(Pattern.compile("/aliases\\.json")); - add(Pattern.compile("/autoscaling\\.json")); - add(Pattern.compile("/clusterstate\\.json")); - add(Pattern.compile("/collections/.*?/state\\.json")); - add(Pattern.compile("/collections/.*?/leaders/shard.*?/leader")); - add(Pattern.compile("/overseer_elect/leader")); - }}; - /** - * Create a snapshot of all content in this instance. - */ - public Map getSnapshot(RedactionUtils.RedactionContext ctx) { - Map snapshot = new LinkedHashMap<>(); - dataMap.forEach((path, vd) -> { - Map data = new HashMap<>(); - if (vd.getData() != null && ctx != null && REDACTED.stream().anyMatch(p -> p.matcher(path).matches())) { - String str = new String(vd.getData(), Charset.forName("UTF-8")); - str = RedactionUtils.redactNames(ctx.getRedactions(), str); - vd = new VersionedData(vd.getVersion(), str.getBytes(Charset.forName("UTF-8")), vd.getMode(), vd.getOwner()); - } - vd.toMap(data); - snapshot.put(path, data); - }); - return snapshot; - } - - @Override - public boolean hasData(String path) throws IOException, KeeperException, InterruptedException { - return dataMap.containsKey(path); - } - - @Override - public List listData(String path) throws NoSuchElementException, IOException, KeeperException, InterruptedException { - return listData(path, null); - } - - @Override - public List listTree(String path) { - return dataMap.keySet().stream() - .filter(p -> p.startsWith(path)) - .collect(Collectors.toList()); - } - - @Override - public List listData(String path, Watcher watcher) throws NoSuchElementException, IOException, KeeperException, InterruptedException { - final String prefix = path + "/"; - return dataMap.entrySet().stream() - .filter(e -> e.getKey().startsWith(prefix)) - .map(e -> { - String suffix = e.getKey().substring(prefix.length()); - int idx = suffix.indexOf('/'); - if (idx == -1) { - return suffix; - } else { - return suffix.substring(0, idx); - } - }) - .collect(Collectors.toList()); - } - - @Override - public VersionedData getData(String path, Watcher watcher) throws NoSuchElementException, IOException, KeeperException, InterruptedException { - if (!dataMap.containsKey(path)) { - throw new NoSuchElementException(path); - } - return dataMap.get(path); - } - - @Override - public void makePath(String path) throws AlreadyExistsException, IOException, KeeperException, InterruptedException { - throw new UnsupportedOperationException("makePath"); - } - - @Override - public void makePath(String path, byte[] data, CreateMode createMode, boolean failOnExists) throws AlreadyExistsException, IOException, KeeperException, InterruptedException { - throw new UnsupportedOperationException("makePath"); - } - - @Override - public String createData(String path, byte[] data, CreateMode mode) throws AlreadyExistsException, IOException, KeeperException, InterruptedException { - throw new UnsupportedOperationException("createData"); - } - - @Override - public void removeData(String path, int version) throws NoSuchElementException, IOException, NotEmptyException, KeeperException, InterruptedException, BadVersionException { - throw new UnsupportedOperationException("removeData"); - } - - @Override - public void setData(String path, byte[] data, int version) throws BadVersionException, NoSuchElementException, IOException, KeeperException, InterruptedException { - throw new UnsupportedOperationException("setData"); - } - - @Override - public List multi(Iterable ops) throws BadVersionException, NoSuchElementException, AlreadyExistsException, IOException, KeeperException, InterruptedException { - throw new UnsupportedOperationException("multi"); - } - - @Override - @SuppressWarnings({"unchecked"}) - public AutoScalingConfig getAutoScalingConfig(Watcher watcher) throws InterruptedException, IOException { - VersionedData vd = dataMap.get(ZkStateReader.SOLR_AUTOSCALING_CONF_PATH); - Map map = new HashMap<>(); - if (vd != null && vd.getData() != null && vd.getData().length > 0) { - map = (Map) Utils.fromJSON(vd.getData()); - map.put(AutoScalingParams.ZK_VERSION, vd.getVersion()); - } - return new AutoScalingConfig(map); - } - - @Override - public void close() throws IOException { - - } -} diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/sim/SnapshotNodeStateProvider.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/sim/SnapshotNodeStateProvider.java deleted file mode 100644 index fca154a256f..00000000000 --- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/sim/SnapshotNodeStateProvider.java +++ /dev/null @@ -1,203 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.cloud.autoscaling.sim; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; - -import org.apache.solr.client.solrj.cloud.NodeStateProvider; -import org.apache.solr.client.solrj.cloud.SolrCloudManager; -import org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig; -import org.apache.solr.client.solrj.cloud.autoscaling.Variable; -import org.apache.solr.common.cloud.Replica; - -/** - * Read-only snapshot of another {@link NodeStateProvider}. - */ -public class SnapshotNodeStateProvider implements NodeStateProvider { - private Map> nodeValues = new LinkedHashMap<>(); - private Map>>> replicaInfos = new LinkedHashMap<>(); - - private static double GB = 1024.0d * 1024.0d * 1024.0d; - - /** - * Populate this instance from another instance of {@link SolrCloudManager}. - * @param other another instance - * @param config optional {@link AutoScalingConfig}, which will be used to determine what node and - * replica tags to retrieve. If this is null then the other instance's config will be used. - */ - @SuppressWarnings({"unchecked"}) - public SnapshotNodeStateProvider(SolrCloudManager other, AutoScalingConfig config) throws Exception { - if (config == null) { - config = other.getDistribStateManager().getAutoScalingConfig(); - } - Set nodeTags = new HashSet<>(SimUtils.COMMON_NODE_TAGS); - nodeTags.addAll(config.getPolicy().getParamNames()); - Set replicaTags = new HashSet<>(SimUtils.COMMON_REPLICA_TAGS); - replicaTags.addAll(config.getPolicy().getPerReplicaAttributes()); - for (String node : other.getClusterStateProvider().getLiveNodes()) { - nodeValues.put(node, new LinkedHashMap<>(other.getNodeStateProvider().getNodeValues(node, nodeTags))); - Map>> infos = other.getNodeStateProvider().getReplicaInfo(node, replicaTags); - infos.forEach((collection, shards) -> { - shards.forEach((shard, replicas) -> { - replicas.forEach(r -> { - List myReplicas = replicaInfos - .computeIfAbsent(node, n -> new LinkedHashMap<>()) - .computeIfAbsent(collection, c -> new LinkedHashMap<>()) - .computeIfAbsent(shard, s -> new ArrayList<>()); - Map rMap = new LinkedHashMap<>(); - r.toMap(rMap); - if (r.isLeader()) { // ReplicaInfo.toMap doesn't write this!!! - ((Map)rMap.values().iterator().next()).put("leader", "true"); - } - Replica ri = new Replica(rMap); - // put in "leader" again if present - if (r.isLeader()) { - ri.getProperties().put("leader", "true"); - } - // externally produced snapshots may not include the right units - if (ri.get(Variable.Type.CORE_IDX.metricsAttribute) == null) { - if (ri.get(Variable.Type.CORE_IDX.tagName) != null) { - Number indexSizeGB = (Number) ri.get(Variable.Type.CORE_IDX.tagName); - ri.getProperties().put(Variable.Type.CORE_IDX.metricsAttribute, indexSizeGB.doubleValue() * GB); - } else { - throw new RuntimeException("Missing size information for replica: " + ri); - } - } - myReplicas.add(ri); - }); - }); - }); - } - } - - /** - * Populate this instance from a previously generated snapshot. - * @param snapshot previous snapshot created using this class. - */ - @SuppressWarnings({"unchecked"}) - public SnapshotNodeStateProvider(Map snapshot) { - Objects.requireNonNull(snapshot); - nodeValues = (Map>)snapshot.getOrDefault("nodeValues", Collections.emptyMap()); - ((Map)snapshot.getOrDefault("replicaInfos", Collections.emptyMap())).forEach((node, v) -> { - Map>> perNode = replicaInfos.computeIfAbsent(node, n -> new LinkedHashMap<>()); - ((Map)v).forEach((collection, shards) -> { - Map> perColl = perNode.computeIfAbsent(collection, c -> new LinkedHashMap<>()); - ((Map)shards).forEach((shard, replicas) -> { - List infos = perColl.computeIfAbsent(shard, s -> new ArrayList<>()); - ((List>)replicas).forEach(replicaMap -> { - Replica ri = new Replica(new LinkedHashMap<>(replicaMap)); // constructor modifies this map - if (ri.isLeader()) { - ri.getProperties().put("leader", "true"); - } - // externally produced snapshots may not include the right units - if (ri.get(Variable.Type.CORE_IDX.metricsAttribute) == null) { - if (ri.get(Variable.Type.CORE_IDX.tagName) != null) { - Number indexSizeGB = (Number) ri.get(Variable.Type.CORE_IDX.tagName); - ri.getProperties().put(Variable.Type.CORE_IDX.metricsAttribute, indexSizeGB.doubleValue() * GB); - } else { - throw new RuntimeException("Missing size information for replica: " + ri); - } - } - infos.add(ri); - }); - }); - }); - }); - } - - /** - * Create a snapshot of all node and replica tag values available from the original source, per the original - * autoscaling configuration. Note: - */ - @SuppressWarnings({"unchecked"}) - public Map getSnapshot() { - Map snapshot = new LinkedHashMap<>(); - snapshot.put("nodeValues", nodeValues); - Map>>>> replicaInfosMap = new LinkedHashMap<>(); - snapshot.put("replicaInfos", replicaInfosMap); - replicaInfos.forEach((node, perNode) -> { - perNode.forEach((collection, shards) -> { - shards.forEach((shard, replicas) -> { - replicas.forEach(r -> { - List> myReplicas = replicaInfosMap - .computeIfAbsent(node, n -> new LinkedHashMap<>()) - .computeIfAbsent(collection, c -> new LinkedHashMap<>()) - .computeIfAbsent(shard, s -> new ArrayList<>()); - Map rMap = new LinkedHashMap<>(); - r.toMap(rMap); - if (r.isLeader()) { // ReplicaInfo.toMap doesn't write this!!! - ((Map)rMap.values().iterator().next()).put("leader", "true"); - } - myReplicas.add(rMap); - }); - }); - }); - }); - return snapshot; - } - - @Override - public Map getNodeValues(String node, Collection tags) { - return new LinkedHashMap<>(nodeValues.getOrDefault(node, Collections.emptyMap())); - } - - @Override - public Map>> getReplicaInfo(String node, Collection keys) { - Map>> result = new LinkedHashMap<>(); - Map>> infos = replicaInfos.getOrDefault(node, Collections.emptyMap()); - // deep copy - infos.forEach((coll, shards) -> { - shards.forEach((shard, replicas) -> { - replicas.forEach(ri -> { - List myReplicas = result - .computeIfAbsent(coll, c -> new LinkedHashMap<>()) - .computeIfAbsent(shard, s -> new ArrayList<>()); - Replica myReplica = (Replica)ri.clone(); - myReplicas.add(myReplica); - }); - }); - }); - return result; - } - - public Replica getReplicaInfo(String collection, String coreNode) { - for (Map>> perNode : replicaInfos.values()) { - for (List perShard : perNode.getOrDefault(collection, Collections.emptyMap()).values()) { - for (Replica ri : perShard) { - if (ri.getName().equals(coreNode)) { - return (Replica)ri.clone(); - } - } - } - } - return null; - } - - @Override - public void close() throws IOException { - - } -} diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/sim/package-info.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/sim/package-info.java deleted file mode 100644 index 71d4d29d29c..00000000000 --- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/sim/package-info.java +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -/** - * Simulated environment for autoscaling. - * - *

Goals

- *
    - *
  • Use the actual unchanged autoscaling code for cluster state monitoring and autoscaling plan execution.
    • - *
  • Support testing large clusters (> 100 nodes).
    • - *
  • Support fast testing using accelerated time (eg. 100x faster).
    • - *
  • Support enough of other Solr functionality for the test results to be meaningful.
    • - *
- * - *

Simulated SolrCloudManager - {@link org.apache.solr.cloud.autoscaling.sim.SimCloudManager}

- * This implementation of {@link org.apache.solr.client.solrj.cloud.SolrCloudManager} - * uses the following simulated components: - *
    - *
  • {@link org.apache.solr.cloud.autoscaling.sim.SimDistribStateManager} - in-memory ZK look-alike, with support for Watcher-s, ephemeral and sequential nodes.
    • - *
  • {@link org.apache.solr.cloud.autoscaling.sim.SimClusterStateProvider} - manages collection, replica infos, states and replica metrics.
    • - *
  • {@link org.apache.solr.cloud.autoscaling.sim.SimNodeStateProvider} - manages node metrics.
    • - *
  • {@link org.apache.solr.cloud.autoscaling.sim.GenericDistributedQueue} - DistributedQueue that uses SimDistribStateManager.
    • - *
- * SimCloudManager also maintains an up-to-date /live_nodes in SimDistribStateManager, provides a SolrClient instance for use in tests, - * and provides several convenience methods for setting up simulated clusters, populating node and replica metrics, collecting - * autoscaling-related event history, collecting autoscaling event statistics, etc. - * - * SimCloudManager runs actual {@link org.apache.solr.cloud.autoscaling.OverseerTriggerThread} so that it - * uses real trigger and trigger action implementations, as well as real event scheduling and processing code. - * It also provides methods for simulating Overseer leader change. - * - * An important part of the SimCloudManager is also a request handler that processes common autoscaling - * and collection admin requests. Autoscaling requests are processes by an instance of - * {@link org.apache.solr.cloud.autoscaling.AutoScalingHandler} (and result in changes in respective - * data stored in {@link org.apache.solr.cloud.autoscaling.sim.SimDistribStateManager}). Collection - * admin commands are simulated, ie. they don't use actual {@link org.apache.solr.handler.admin.CollectionsHandler} - * due to the complex dependencies on real components. - * - *

{@link org.apache.solr.cloud.autoscaling.sim.SimClusterStateProvider}

- * This components maintains collection and replica states: - *
    - *
  • Simulates delays between request and the actual cluster state changes
    • - *
  • Marks replicas as down when a node goes down (optionally preserving the replica metrics in order to simulate a node coming back), and keeps track of per-node cores and disk space.
    • - *
  • Runs a shard leader election look-alike on collection state updates.
    • - *
  • Maintains up-to-date /clusterstate.json and /clusterprops.json in SimDistribStateManager (which in turn notifies Watcher-s about collection updates). - * Currently for simplicity it uses the old single /clusterstate.json format for representing ClusterState.
    • - *
- * - *

{@link org.apache.solr.cloud.autoscaling.sim.SimNodeStateProvider}

- * This component maintains node metrics. When a simulated cluster is set up using eg. - * {@link org.apache.solr.cloud.autoscaling.sim.SimCloudManager#createCluster(int, org.apache.solr.common.util.TimeSource)} - * method, each simulated node is initialized with some basic metrics that are expected by the autoscaling - * framework, such as node name, fake system load average, heap usage and disk usage. - * - * The number of cores and disk space metrics may be used in autoscaling calculations, so they are - * tracked and adjusted by {@link org.apache.solr.cloud.autoscaling.sim.SimClusterStateProvider} according - * to the currently active replicas located on each node. - * - *

Limitations of the simulation framework

- * Currently the simulation framework is limited to testing the core autoscaling API in a single JVM. - * Using it for other purposes would require extensive modifications in Solr and in the framework code. - * - * Specifically, the framework supports testing the following autoscaling components: - *
    - *
  • OverseerTriggerThread and components that it uses.
    • - *
  • Autoscaling config, triggers, trigger listeners, ScheduledTriggers, trigger event queues, ComputePlanAction / ExecutePlanAction, etc.
    • - *
- * Overseer and CollectionsHandler Cmd implementations are NOT used, so cannot be properly tested - some of their functionality is simulated. - * Other SolrCloud components make too many direct references to ZkStateReader, or direct HTTP requests, or rely on too many other components and require much more complex functionality - they may be refactored later but the effort may be too high. - * - * Simulation framework definitely does not support the following functionality: - *
    - *
  • Solr searching and indexing
    • - *
  • Any component that uses ZkController (eg. CoreContainer)
    • - *
  • Any component that uses ShardHandler (eg. CollectionsHandler Cmd-s)
    • - *
- * - */ -package org.apache.solr.cloud.autoscaling.sim; - - diff --git a/solr/core/src/java/org/apache/solr/cloud/overseer/ReplicaMutator.java b/solr/core/src/java/org/apache/solr/cloud/overseer/ReplicaMutator.java index 3654ea6714b..f849143142f 100644 --- a/solr/core/src/java/org/apache/solr/cloud/overseer/ReplicaMutator.java +++ b/solr/core/src/java/org/apache/solr/cloud/overseer/ReplicaMutator.java @@ -30,7 +30,7 @@ import java.util.concurrent.TimeUnit; import org.apache.commons.lang3.StringUtils; import org.apache.solr.client.solrj.cloud.DistribStateManager; import org.apache.solr.client.solrj.cloud.SolrCloudManager; -import org.apache.solr.client.solrj.cloud.autoscaling.VersionedData; +import org.apache.solr.client.solrj.cloud.VersionedData; import org.apache.solr.cloud.CloudUtil; import org.apache.solr.cloud.Overseer; import org.apache.solr.cloud.api.collections.Assign; diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java index 77983fd9b36..35787317ea2 100644 --- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java +++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java @@ -70,7 +70,6 @@ import org.apache.solr.client.solrj.util.SolrIdentifierValidator; import org.apache.solr.cloud.CloudDescriptor; import org.apache.solr.cloud.OverseerTaskQueue; import org.apache.solr.cloud.ZkController; -import org.apache.solr.cloud.autoscaling.AutoScalingHandler; import org.apache.solr.common.AlreadyClosedException; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException.ErrorCode; @@ -90,7 +89,6 @@ import org.apache.solr.core.backup.repository.BackupRepositoryFactory; import org.apache.solr.filestore.PackageStoreAPI; import org.apache.solr.handler.RequestHandlerBase; import org.apache.solr.handler.SnapShooter; -import org.apache.solr.handler.admin.AutoscalingHistoryHandler; import org.apache.solr.handler.admin.CollectionsHandler; import org.apache.solr.handler.admin.ConfigSetsHandler; import org.apache.solr.handler.admin.ContainerPluginsApi; @@ -137,7 +135,6 @@ import org.slf4j.LoggerFactory; import static java.util.Objects.requireNonNull; import static org.apache.solr.common.params.CommonParams.AUTHC_PATH; import static org.apache.solr.common.params.CommonParams.AUTHZ_PATH; -import static org.apache.solr.common.params.CommonParams.AUTOSCALING_HISTORY_PATH; import static org.apache.solr.common.params.CommonParams.COLLECTIONS_HANDLER_PATH; import static org.apache.solr.common.params.CommonParams.CONFIGSETS_HANDLER_PATH; import static org.apache.solr.common.params.CommonParams.CORES_HANDLER_PATH; @@ -242,8 +239,6 @@ public class CoreContainer { protected volatile MetricsCollectorHandler metricsCollectorHandler; - protected volatile AutoscalingHistoryHandler autoscalingHistoryHandler; - private volatile SolrClientCache solrClientCache; private final ObjectCache objectCache = new ObjectCache(); @@ -259,8 +254,6 @@ public class CoreContainer { public final static long INITIAL_CORE_LOAD_COMPLETE = 0x4L; private volatile long status = 0L; - protected volatile AutoScalingHandler autoScalingHandler; - private ExecutorService coreContainerAsyncTaskExecutor = ExecutorUtil.newMDCAwareCachedThreadPool("Core Container Async Task"); private enum CoreInitFailedAction {fromleader, none} @@ -737,7 +730,6 @@ public class CoreContainer { createMetricsHistoryHandler(); - autoscalingHistoryHandler = createHandler(AUTOSCALING_HISTORY_PATH, AutoscalingHistoryHandler.class.getName(), AutoscalingHistoryHandler.class); metricsCollectorHandler = createHandler(MetricsCollectorHandler.HANDLER_PATH, MetricsCollectorHandler.class.getName(), MetricsCollectorHandler.class); // may want to add some configuration here in the future metricsCollectorHandler.init(null); @@ -889,10 +881,6 @@ public class CoreContainer { containerHandlers.getApiBag().registerObject(containerPluginsApi.readAPI); containerHandlers.getApiBag().registerObject(containerPluginsApi.editAPI); zkSys.getZkController().checkOverseerDesignate(); - // initialize this handler here when SolrCloudManager is ready - autoScalingHandler = new AutoScalingHandler(getZkController().getSolrCloudManager(), loader); - containerHandlers.put(AutoScalingHandler.HANDLER_PATH, autoScalingHandler); - autoScalingHandler.initializeMetrics(solrMetricsContext, AutoScalingHandler.HANDLER_PATH); } // This is a bit redundant but these are two distinct concepts for all they're accomplished at the same time. status |= LOAD_COMPLETE | INITIAL_CORE_LOAD_COMPLETE; diff --git a/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java b/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java index 4e7e15ccdb6..cfd36906e62 100644 --- a/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java +++ b/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java @@ -67,8 +67,7 @@ public class SolrResourceLoader implements ResourceLoader, Closeable, SolrClassL private static final String[] packages = { "", "analysis.", "schema.", "handler.", "handler.tagger.", "search.", "update.", "core.", "response.", "request.", "update.processor.", "util.", "spelling.", "handler.component.", "handler.dataimport.", - "spelling.suggest.", "spelling.suggest.fst.", "rest.schema.analysis.", "security.", "handler.admin.", - "cloud.autoscaling." + "spelling.suggest.", "spelling.suggest.fst.", "rest.schema.analysis.", "security.", "handler.admin." }; private static final Charset UTF_8 = StandardCharsets.UTF_8; diff --git a/solr/core/src/java/org/apache/solr/handler/admin/AutoscalingHistoryHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/AutoscalingHistoryHandler.java deleted file mode 100644 index 5fb618e88d8..00000000000 --- a/solr/core/src/java/org/apache/solr/handler/admin/AutoscalingHistoryHandler.java +++ /dev/null @@ -1,166 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.handler.admin; - -import java.lang.invoke.MethodHandles; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.Locale; -import java.util.Map; -import java.util.Optional; - -import org.apache.solr.api.Api; -import org.apache.solr.api.ApiBag; -import org.apache.solr.client.solrj.impl.CloudSolrClient; -import org.apache.solr.client.solrj.response.QueryResponse; -import org.apache.solr.cloud.autoscaling.SystemLogListener; -import org.apache.solr.cloud.autoscaling.TriggerEvent; -import org.apache.solr.common.SolrException; -import org.apache.solr.common.params.AutoScalingParams; -import org.apache.solr.common.params.CollectionAdminParams; -import org.apache.solr.common.params.CommonParams; -import org.apache.solr.common.params.ModifiableSolrParams; -import org.apache.solr.core.CoreContainer; -import org.apache.solr.handler.RequestHandlerBase; -import org.apache.solr.request.SolrQueryRequest; -import org.apache.solr.response.SolrQueryResponse; -import org.apache.solr.security.AuthorizationContext; -import org.apache.solr.security.PermissionNameProvider; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * This handler makes it easier to retrieve a history of autoscaling events from the .system - * collection. - */ -public class AutoscalingHistoryHandler extends RequestHandlerBase implements PermissionNameProvider { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - public static final String SYSTEM_COLLECTION_PARAM = "systemCollection"; - - public static final String ACTION_PARAM = "action"; - public static final String MESSAGE_PARAM = "message"; - public static final String TRIGGER_PARAM = AutoScalingParams.TRIGGER; - public static final String TYPE_PARAM = "eventType"; - public static final String NODE_PARAM = "node"; - public static final String COLLECTION_PARAM = CollectionAdminParams.COLLECTION; - public static final String STAGE_PARAM = AutoScalingParams.STAGE; - public static final String BEFORE_ACTION_PARAM = AutoScalingParams.BEFORE_ACTION; - public static final String AFTER_ACTION_PARAM = AutoScalingParams.AFTER_ACTION; - - private static final String EVENTS_FQ = "{!term f=" + CommonParams.TYPE + "}" + SystemLogListener.DOC_TYPE; - - private static final String ACTION_FQ_FORMAT = "{!term f=" + SystemLogListener.ACTION_FIELD + "}%s"; - private static final String MESSAGE_FQ_FORMAT = "{!lucene}" + SystemLogListener.MESSAGE_FIELD + ":%s"; - private static final String TRIGGER_FQ_FORMAT = "{!term f=" + SystemLogListener.EVENT_SOURCE_FIELD + "}%s"; - private static final String STAGE_FQ_FORMAT = "{!term f=" + SystemLogListener.STAGE_FIELD + "}%s"; - private static final String COLLECTION_FQ_FORMAT = "{!term f=" + SystemLogListener.COLLECTIONS_FIELD + "}%s"; - private static final String TYPE_FQ_FORMAT = "{!term f=" + SystemLogListener.EVENT_TYPE_FIELD + "}%s"; - private static final String NODE_FQ_FORMAT = "{!term f=event.property." + TriggerEvent.NODE_NAMES + "_ss}%s"; - private static final String BEFORE_ACTION_FQ_FORMAT = "{!term f=" + SystemLogListener.BEFORE_ACTIONS_FIELD + "}%s"; - private static final String AFTER_ACTION_FQ_FORMAT = "{!term f=" + SystemLogListener.AFTER_ACTIONS_FIELD + "}%s"; - - private static final Map formats = new HashMap() {{ - put(ACTION_PARAM, ACTION_FQ_FORMAT); - put(MESSAGE_PARAM, MESSAGE_FQ_FORMAT); - put(TRIGGER_PARAM, TRIGGER_FQ_FORMAT); - put(TYPE_PARAM, TYPE_FQ_FORMAT); - put(STAGE_PARAM, STAGE_FQ_FORMAT); - put(NODE_PARAM, NODE_FQ_FORMAT); - put(COLLECTION_PARAM, COLLECTION_FQ_FORMAT); - put(BEFORE_ACTION_PARAM, BEFORE_ACTION_FQ_FORMAT); - put(AFTER_ACTION_PARAM, AFTER_ACTION_FQ_FORMAT); - }}; - - private final CoreContainer coreContainer; - - - public AutoscalingHistoryHandler(CoreContainer coreContainer) { - this.coreContainer = coreContainer; - } - - @Override - public Name getPermissionName(AuthorizationContext request) { - return Name.AUTOSCALING_HISTORY_READ_PERM; - } - - @Override - @SuppressWarnings({"unchecked"}) - public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception { - ModifiableSolrParams params = new ModifiableSolrParams(req.getParams()); - String collection = params.get(SYSTEM_COLLECTION_PARAM, CollectionAdminParams.SYSTEM_COLL); - params.remove(SYSTEM_COLLECTION_PARAM); - params.remove(CommonParams.QT); - // check that we have the main query, if not then use *:* - if (params.get(CommonParams.Q) == null) { - params.add(CommonParams.Q, "*:*"); - } - // sort by doc id, which are time-based, unless specified otherwise - if (params.get(CommonParams.SORT) == null) { - params.add(CommonParams.SORT, "id asc"); - } - // filter query to pick only autoscaling events - params.remove(CommonParams.FQ, EVENTS_FQ); - params.add(CommonParams.FQ, EVENTS_FQ); - // add filters translated from simplified parameters - for (Map.Entry e : formats.entrySet()) { - String[] values = params.remove(e.getKey()); - if (values != null) { - for (String value : values) { - params.add(CommonParams.FQ, String.format(Locale.ROOT, e.getValue(), value)); - } - } - } - try (CloudSolrClient cloudSolrClient = new CloudSolrClient.Builder(Collections.singletonList(coreContainer.getZkController().getZkServerAddress()), Optional.empty()).withSocketTimeout(30000).withConnectionTimeout(15000) - .withHttpClient(coreContainer.getUpdateShardHandler().getDefaultHttpClient()) - .build()) { - QueryResponse qr = cloudSolrClient.query(collection, params); - rsp.setAllValues(qr.getResponse()); - } catch (Exception e) { - if ((e instanceof SolrException) && e.getMessage().contains("Collection not found")) { - // relatively benign - String msg = "Collection " + collection + " does not exist."; - log.info(msg); - rsp.getValues().add("error", msg); - } else { - throw e; - } - } - } - - @Override - public String getDescription() { - return "A handler to return autoscaling event history"; - } - - @Override - public Category getCategory() { - return Category.ADMIN; - } - - @Override - public Boolean registerV2() { - return Boolean.TRUE; - } - - @Override - public Collection getApis() { - return ApiBag.wrapRequestHandlers(this, "autoscaling.history"); - } - -} diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java index 9b1cf78cf05..06226410df1 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java @@ -56,7 +56,6 @@ import org.apache.solr.common.cloud.ZkCmdExecutor; import org.apache.solr.common.cloud.ZkCoreNodeProps; import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.cloud.ZkStateReader; -import org.apache.solr.common.params.AutoScalingParams; import org.apache.solr.common.params.CollectionAdminParams; import org.apache.solr.common.params.CollectionParams; import org.apache.solr.common.params.CollectionParams.CollectionAction; @@ -103,7 +102,6 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.stream.Collectors; -import static org.apache.solr.client.solrj.cloud.autoscaling.Policy.POLICY; import static org.apache.solr.client.solrj.response.RequestStatusState.COMPLETED; import static org.apache.solr.client.solrj.response.RequestStatusState.FAILED; import static org.apache.solr.client.solrj.response.RequestStatusState.NOT_FOUND; @@ -125,7 +123,6 @@ import static org.apache.solr.common.SolrException.ErrorCode.BAD_REQUEST; import static org.apache.solr.common.cloud.DocCollection.DOC_ROUTER; import static org.apache.solr.common.cloud.DocCollection.RULE; import static org.apache.solr.common.cloud.DocCollection.SNITCH; -import static org.apache.solr.common.cloud.ZkStateReader.AUTO_ADD_REPLICAS; import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP; import static org.apache.solr.common.cloud.ZkStateReader.NRT_REPLICAS; import static org.apache.solr.common.cloud.ZkStateReader.PROPERTY_PROP; @@ -462,13 +459,11 @@ public class CollectionsHandler extends RequestHandlerBase implements Permission CREATE_NODE_SET, CREATE_NODE_SET_SHUFFLE, SHARDS_PROP, - AUTO_ADD_REPLICAS, RULE, SNITCH, PULL_REPLICAS, TLOG_REPLICAS, NRT_REPLICAS, - POLICY, WAIT_FOR_FINAL_STATE, WITH_COLLECTION, ALIAS); @@ -558,10 +553,8 @@ public class CollectionsHandler extends RequestHandlerBase implements Permission PULL_REPLICAS, TLOG_REPLICAS, REPLICATION_FACTOR, - POLICY, CREATE_NODE_SET, CREATE_NODE_SET_SHUFFLE, - AUTO_ADD_REPLICAS, "shards", CommonParams.ROWS, CommonParams.Q, @@ -974,9 +967,6 @@ public class CollectionsHandler extends RequestHandlerBase implements Permission new ZkNodeProps(all)).getClusterStatus(rsp.getValues()); return null; }), - UTILIZENODE_OP(UTILIZENODE, (req, rsp, h) -> { - return copy(req.getParams().required(), null, AutoScalingParams.NODE); - }), ADDREPLICAPROP_OP(ADDREPLICAPROP, (req, rsp, h) -> { Map map = copy(req.getParams().required(), null, COLLECTION_PROP, @@ -1169,7 +1159,7 @@ public class CollectionsHandler extends RequestHandlerBase implements Permission } // from CREATE_OP: copy(req.getParams(), params, COLL_CONF, REPLICATION_FACTOR, NRT_REPLICAS, TLOG_REPLICAS, - PULL_REPLICAS, AUTO_ADD_REPLICAS, CREATE_NODE_SET, CREATE_NODE_SET_SHUFFLE); + PULL_REPLICAS, CREATE_NODE_SET, CREATE_NODE_SET_SHUFFLE); copyPropertiesWithPrefix(req.getParams(), params, COLL_PROP_PREFIX); return params; }), diff --git a/solr/core/src/java/org/apache/solr/handler/admin/MetricsHistoryHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/MetricsHistoryHandler.java index 607fb8b4d2c..8173f4534ea 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/MetricsHistoryHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/MetricsHistoryHandler.java @@ -60,9 +60,9 @@ import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.client.solrj.cloud.NodeStateProvider; import org.apache.solr.client.solrj.cloud.SolrCloudManager; -import org.apache.solr.client.solrj.cloud.autoscaling.Variable; -import org.apache.solr.client.solrj.cloud.autoscaling.VersionedData; +import org.apache.solr.client.solrj.cloud.VersionedData; import org.apache.solr.client.solrj.impl.HttpClientUtil; +import org.apache.solr.client.solrj.impl.SolrClientNodeStateProvider.Variable; import org.apache.solr.cloud.LeaderElector; import org.apache.solr.cloud.Overseer; import org.apache.solr.common.SolrException; @@ -136,7 +136,7 @@ public class MetricsHistoryHandler extends RequestHandlerBase implements Permiss DEFAULT_NODE_GAUGES.add("CONTAINER.fs.coreRoot.usableSpace"); - DEFAULT_CORE_GAUGES.add(Variable.Type.CORE_IDX.metricsAttribute); + DEFAULT_CORE_GAUGES.add(Variable.CORE_IDX.metricsAttribute); DEFAULT_CORE_COUNTERS.add("QUERY./select.requests"); DEFAULT_CORE_COUNTERS.add("UPDATE./update.requests"); diff --git a/solr/core/src/java/org/apache/solr/security/PermissionNameProvider.java b/solr/core/src/java/org/apache/solr/security/PermissionNameProvider.java index 618a57241ac..5d24a1dc862 100644 --- a/solr/core/src/java/org/apache/solr/security/PermissionNameProvider.java +++ b/solr/core/src/java/org/apache/solr/security/PermissionNameProvider.java @@ -48,9 +48,6 @@ public interface PermissionNameProvider { SECURITY_EDIT_PERM("security-edit", null), SECURITY_READ_PERM("security-read", null), METRICS_READ_PERM("metrics-read", null), - AUTOSCALING_READ_PERM("autoscaling-read", null), - AUTOSCALING_WRITE_PERM("autoscaling-write", null), - AUTOSCALING_HISTORY_READ_PERM("autoscaling-history-read", null), METRICS_HISTORY_READ_PERM("metrics-history-read", null), FILESTORE_READ_PERM("filestore-read", null), FILESTORE_WRITE_PERM("filestore-write", null), diff --git a/solr/core/src/java/org/apache/solr/util/SolrCLI.java b/solr/core/src/java/org/apache/solr/util/SolrCLI.java index 922fe631eb6..19de535c35c 100755 --- a/solr/core/src/java/org/apache/solr/util/SolrCLI.java +++ b/solr/core/src/java/org/apache/solr/util/SolrCLI.java @@ -19,7 +19,6 @@ package org.apache.solr.util; import javax.net.ssl.SSLPeerUnverifiedException; import java.io.Console; import java.io.File; -import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; @@ -44,7 +43,6 @@ import java.util.Collection; import java.util.Collections; import java.util.Enumeration; import java.util.HashMap; -import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; @@ -74,7 +72,6 @@ import org.apache.commons.exec.Executor; import org.apache.commons.exec.OS; import org.apache.commons.exec.environment.EnvironmentUtils; import org.apache.commons.io.FileUtils; -import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.BooleanUtils; import org.apache.commons.lang3.SystemUtils; import org.apache.http.HttpEntity; @@ -96,28 +93,14 @@ import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.client.solrj.cloud.SolrCloudManager; -import org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig; -import org.apache.solr.client.solrj.cloud.autoscaling.Policy; -import org.apache.solr.client.solrj.cloud.autoscaling.PolicyHelper; -import org.apache.solr.client.solrj.cloud.autoscaling.Suggester; -import org.apache.solr.client.solrj.cloud.autoscaling.Variable; import org.apache.solr.client.solrj.impl.CloudSolrClient; import org.apache.solr.client.solrj.impl.HttpClientUtil; import org.apache.solr.client.solrj.impl.HttpSolrClient; -import org.apache.solr.client.solrj.impl.SolrClientCloudManager; import org.apache.solr.client.solrj.impl.ZkClientClusterStateProvider; import org.apache.solr.client.solrj.request.CollectionAdminRequest; import org.apache.solr.client.solrj.request.ContentStreamUpdateRequest; -import org.apache.solr.client.solrj.request.V2Request; import org.apache.solr.client.solrj.response.CollectionAdminResponse; import org.apache.solr.client.solrj.response.QueryResponse; -import org.apache.solr.cloud.autoscaling.sim.NoopDistributedQueueFactory; -import org.apache.solr.cloud.autoscaling.sim.SimCloudManager; -import org.apache.solr.cloud.autoscaling.sim.SimScenario; -import org.apache.solr.cloud.autoscaling.sim.SimUtils; -import org.apache.solr.cloud.autoscaling.sim.SnapshotCloudManager; -import org.apache.solr.common.MapWriter; import org.apache.solr.common.SolrException; import org.apache.solr.common.cloud.ClusterState; import org.apache.solr.common.cloud.DocCollection; @@ -130,12 +113,9 @@ import org.apache.solr.common.cloud.ZkStateReader; import org.apache.solr.common.params.CollectionAdminParams; import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.params.ModifiableSolrParams; -import org.apache.solr.common.params.SolrParams; import org.apache.solr.common.util.ContentStreamBase; import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.StrUtils; -import org.apache.solr.common.util.TimeSource; -import org.apache.solr.common.util.Utils; import org.apache.solr.security.Sha256AuthenticationProvider; import org.apache.solr.util.configuration.SSLConfigurationsFactory; import org.noggit.CharArr; @@ -412,8 +392,6 @@ public class SolrCLI implements CLIO { return new UtilsTool(); else if ("auth".equals(toolType)) return new AuthTool(); - else if ("autoscaling".equals(toolType)) - return new AutoscalingTool(); else if ("export".equals(toolType)) return new ExportTool(); else if ("package".equals(toolType)) @@ -436,7 +414,6 @@ public class SolrCLI implements CLIO { formatter.printHelp("healthcheck", getToolOptions(new HealthcheckTool())); formatter.printHelp("status", getToolOptions(new StatusTool())); formatter.printHelp("api", getToolOptions(new ApiTool())); - formatter.printHelp("autoscaling", getToolOptions(new AutoscalingTool())); formatter.printHelp("create_collection", getToolOptions(new CreateCollectionTool())); formatter.printHelp("create_core", getToolOptions(new CreateCoreTool())); formatter.printHelp("create", getToolOptions(new CreateTool())); @@ -859,373 +836,6 @@ public class SolrCLI implements CLIO { return result; } - - public static class AutoscalingTool extends ToolBase { - - public AutoscalingTool() { - this(CLIO.getOutStream()); - } - - public AutoscalingTool(PrintStream stdout) { - super(stdout); - } - - @Override - public Option[] getOptions() { - return new Option[] { - Option.builder("zkHost") - .argName("HOST") - .hasArg() - .required(false) - .desc("Address of the Zookeeper ensemble; defaults to: "+ZK_HOST) - .build(), - Option.builder("a") - .argName("CONFIG") - .hasArg() - .required(false) - .desc("Autoscaling config file, defaults to the one deployed in the cluster.") - .longOpt("config") - .build(), - Option.builder("s") - .desc("Show calculated suggestions") - .longOpt("suggestions") - .build(), - Option.builder("c") - .desc("Show ClusterState (collections layout)") - .longOpt("clusterState") - .build(), - Option.builder("d") - .desc("Show calculated diagnostics") - .longOpt("diagnostics") - .build(), - Option.builder("n") - .desc("Show sorted nodes with diagnostics") - .longOpt("sortedNodes") - .build(), - Option.builder("r") - .desc("Redact node and collection names (original names will be consistently randomized)") - .longOpt("redact") - .build(), - Option.builder("stats") - .desc("Show summarized collection & node statistics.") - .build(), - Option.builder("save") - .desc("Store autoscaling snapshot of the current cluster.") - .argName("DIR") - .hasArg() - .build(), - Option.builder("load") - .desc("Load autoscaling snapshot of the cluster instead of using the real one.") - .argName("DIR") - .hasArg() - .build(), - Option.builder("simulate") - .desc("Simulate execution of all suggestions.") - .build(), - Option.builder("i") - .desc("Max number of simulation iterations.") - .argName("NUMBER") - .hasArg() - .longOpt("iterations") - .build(), - Option.builder("ss") - .desc("Save autoscaling snapshots at each step of simulated execution.") - .argName("DIR") - .longOpt("saveSimulated") - .hasArg() - .build(), - Option.builder("scenario") - .desc("Execute a scenario from a file (and ignore all other options).") - .argName("FILE") - .hasArg() - .build(), - Option.builder("all") - .desc("Turn on all options to get all available information.") - .build() - - }; - } - - @Override - public String getName() { - return "autoscaling"; - } - - protected void runImpl(CommandLine cli) throws Exception { - raiseLogLevelUnlessVerbose(cli); - if (cli.hasOption("scenario")) { - String data = IOUtils.toString(new FileInputStream(cli.getOptionValue("scenario")), "UTF-8"); - try (SimScenario scenario = SimScenario.load(data)) { - scenario.verbose = verbose; - scenario.console = CLIO.getOutStream(); - scenario.run(); - } - return; - } - SnapshotCloudManager cloudManager; - AutoScalingConfig config = null; - String configFile = cli.getOptionValue("a"); - if (configFile != null) { - CLIO.err("- reading autoscaling config from " + configFile); - config = new AutoScalingConfig(IOUtils.toByteArray(new FileInputStream(configFile))); - } - if (cli.hasOption("load")) { - File sourceDir = new File(cli.getOptionValue("load")); - CLIO.err("- loading autoscaling snapshot from " + sourceDir.getAbsolutePath()); - cloudManager = SnapshotCloudManager.readSnapshot(sourceDir); - if (config == null) { - CLIO.err("- reading autoscaling config from the snapshot."); - config = cloudManager.getDistribStateManager().getAutoScalingConfig(); - } - } else { - String zkHost = cli.getOptionValue("zkHost", ZK_HOST); - - log.debug("Connecting to Solr cluster: {}", zkHost); - try (CloudSolrClient cloudSolrClient = new CloudSolrClient.Builder(Collections.singletonList(zkHost), Optional.empty()).build()) { - - String collection = cli.getOptionValue("collection"); - if (collection != null) - cloudSolrClient.setDefaultCollection(collection); - - cloudSolrClient.connect(); - try (SolrClientCloudManager realCloudManager = new SolrClientCloudManager(NoopDistributedQueueFactory.INSTANCE, cloudSolrClient)) { - if (config == null) { - CLIO.err("- reading autoscaling config from the cluster."); - config = realCloudManager.getDistribStateManager().getAutoScalingConfig(); - } - cloudManager = new SnapshotCloudManager(realCloudManager, config); - } - } - } - boolean redact = cli.hasOption("r"); - if (cli.hasOption("save")) { - File targetDir = new File(cli.getOptionValue("save")); - cloudManager.saveSnapshot(targetDir, true, redact); - CLIO.err("- saved autoscaling snapshot to " + targetDir.getAbsolutePath()); - } - HashSet liveNodes = new HashSet<>(cloudManager.getClusterStateProvider().getLiveNodes()); - boolean withSuggestions = cli.hasOption("s"); - boolean withDiagnostics = cli.hasOption("d") || cli.hasOption("n"); - boolean withSortedNodes = cli.hasOption("n"); - boolean withClusterState = cli.hasOption("c"); - boolean withStats = cli.hasOption("stats"); - if (cli.hasOption("all")) { - withSuggestions = true; - withDiagnostics = true; - withSortedNodes = true; - withClusterState = true; - withStats = true; - } - // prepare to redact also host names / IPs in base_url and other properties - ClusterState clusterState = cloudManager.getClusterStateProvider().getClusterState(); - RedactionUtils.RedactionContext ctx = null; - if (redact) { - ctx = SimUtils.getRedactionContext(clusterState); - } - if (!withSuggestions && !withDiagnostics) { - withSuggestions = true; - } - Map results = prepareResults(cloudManager, config, withClusterState, - withStats, withSuggestions, withSortedNodes, withDiagnostics); - if (cli.hasOption("simulate")) { - String iterStr = cli.getOptionValue("i", "10"); - String saveSimulated = cli.getOptionValue("saveSimulated"); - int iterations; - try { - iterations = Integer.parseInt(iterStr); - } catch (Exception e) { - log.warn("Invalid option 'i' value, using default 10:", e); - iterations = 10; - } - Map simulationResults = new HashMap<>(); - simulate(cloudManager, config, simulationResults, saveSimulated, withClusterState, - withStats, withSuggestions, withSortedNodes, withDiagnostics, iterations, redact); - results.put("simulation", simulationResults); - } - String data = Utils.toJSONString(results); - if (redact) { - data = RedactionUtils.redactNames(ctx.getRedactions(), data); - } - stdout.println(data); - } - - private Map prepareResults(SolrCloudManager clientCloudManager, - AutoScalingConfig config, - boolean withClusterState, - boolean withStats, - boolean withSuggestions, - boolean withSortedNodes, - boolean withDiagnostics) throws Exception { - Policy.Session session = config.getPolicy().createSession(clientCloudManager); - ClusterState clusterState = clientCloudManager.getClusterStateProvider().getClusterState(); - List suggestions = Collections.emptyList(); - long start, end; - if (withSuggestions) { - CLIO.err("- calculating suggestions..."); - start = TimeSource.NANO_TIME.getTimeNs(); - suggestions = PolicyHelper.getSuggestions(config, clientCloudManager); - end = TimeSource.NANO_TIME.getTimeNs(); - CLIO.err(" (took " + TimeUnit.NANOSECONDS.toMillis(end - start) + " ms)"); - } - Map diagnostics = Collections.emptyMap(); - if (withDiagnostics) { - CLIO.err("- calculating diagnostics..."); - start = TimeSource.NANO_TIME.getTimeNs(); - MapWriter mw = PolicyHelper.getDiagnostics(session); - diagnostics = new LinkedHashMap<>(); - mw.toMap(diagnostics); - end = TimeSource.NANO_TIME.getTimeNs(); - CLIO.err(" (took " + TimeUnit.NANOSECONDS.toMillis(end - start) + " ms)"); - } - Map results = new LinkedHashMap<>(); - if (withClusterState) { - Map map = new LinkedHashMap<>(); - map.put("liveNodes", new TreeSet<>(clusterState.getLiveNodes())); - map.put("collections", clusterState.getCollectionsMap()); - results.put("CLUSTERSTATE", map); - } - if (withStats) { - results.put("STATISTICS", SimUtils.calculateStats(clientCloudManager, config, verbose)); - } - if (withSuggestions) { - results.put("SUGGESTIONS", suggestions); - } - if (!withSortedNodes) { - diagnostics.remove("sortedNodes"); - } - if (withDiagnostics) { - results.put("DIAGNOSTICS", diagnostics); - } - return results; - } - - - private void simulate(SolrCloudManager cloudManager, - AutoScalingConfig config, - Map results, - String saveSimulated, - boolean withClusterState, - boolean withStats, - boolean withSuggestions, - boolean withSortedNodes, - boolean withDiagnostics, int iterations, boolean redact) throws Exception { - File saveDir = null; - if (saveSimulated != null) { - saveDir = new File(saveSimulated); - if (!saveDir.exists()) { - if (!saveDir.mkdirs()) { - throw new Exception("Unable to create 'saveSimulated' directory: " + saveDir.getAbsolutePath()); - } - } else if (!saveDir.isDirectory()) { - throw new Exception("'saveSimulated' path exists and is not a directory! " + saveDir.getAbsolutePath()); - } - } - int SPEED = 50; - SimCloudManager simCloudManager = SimCloudManager.createCluster(cloudManager, config, TimeSource.get("simTime:" + SPEED)); - int loop = 0; - List suggestions = Collections.emptyList(); - Map intermediate = new LinkedHashMap<>(); - results.put("intermediate", intermediate); - while (loop < iterations) { - LinkedHashMap perStep = new LinkedHashMap<>(); - long start = TimeSource.NANO_TIME.getTimeNs(); - suggestions = PolicyHelper.getSuggestions(config, simCloudManager); - CLIO.err("-- step " + loop + ", " + suggestions.size() + " suggestions."); - long end = TimeSource.NANO_TIME.getTimeNs(); - CLIO.err(" - calculated in " + TimeUnit.NANOSECONDS.toMillis(end - start) + " ms (real time ≈ simulated time)"); - if (suggestions.isEmpty()) { - break; - } - SnapshotCloudManager snapshotCloudManager = new SnapshotCloudManager(simCloudManager, config); - if (saveDir != null) { - File target = new File(saveDir, "step" + loop + "_start"); - snapshotCloudManager.saveSnapshot(target, true, redact); - } - if (verbose) { - Map snapshot = snapshotCloudManager.getSnapshot(false, redact); - snapshot.remove(SnapshotCloudManager.DISTRIB_STATE_KEY); - snapshot.remove(SnapshotCloudManager.MANAGER_STATE_KEY); - perStep.put("snapshotStart", snapshot); - } - intermediate.put("step" + loop, perStep); - int unresolvedCount = 0; - start = TimeSource.NANO_TIME.getTimeNs(); - List> perStepOps = new ArrayList<>(suggestions.size()); - if (withSuggestions) { - perStep.put("suggestions", suggestions); - perStep.put("opDetails", perStepOps); - } - for (Suggester.SuggestionInfo suggestion : suggestions) { - SolrRequest operation = suggestion.getOperation(); - if (operation == null) { - unresolvedCount++; - if (suggestion.getViolation() == null) { - CLIO.err(" - ignoring suggestion without violation and without operation: " + suggestion); - } - continue; - } - SolrParams params = operation.getParams(); - if (operation instanceof V2Request) { - params = SimUtils.v2AdminRequestToV1Params((V2Request)operation); - } - Map paramsMap = new LinkedHashMap<>(); - params.toMap(paramsMap); - Replica info = simCloudManager.getSimClusterStateProvider().simGetReplicaInfo( - params.get(CollectionAdminParams.COLLECTION), params.get("replica")); - if (info == null) { - CLIO.err("Could not find ReplicaInfo for params: " + params); - } else if (verbose) { - paramsMap.put("replicaInfo", info); - } else if (info.get(Variable.Type.CORE_IDX.tagName) != null) { - paramsMap.put(Variable.Type.CORE_IDX.tagName, info.get(Variable.Type.CORE_IDX.tagName)); - } - if (withSuggestions) { - perStepOps.add(paramsMap); - } - try { - simCloudManager.request(operation); - } catch (Exception e) { - CLIO.err("Aborting - error executing suggestion " + suggestion + ": " + e); - Map error = new HashMap<>(); - error.put("suggestion", suggestion); - error.put("replicaInfo", info); - error.put("exception", e); - perStep.put("error", error); - break; - } - } - end = TimeSource.NANO_TIME.getTimeNs(); - long realTime = TimeUnit.NANOSECONDS.toMillis(end - start); - long simTime = realTime * SPEED; - CLIO.err(" - executed in " + realTime + " ms (real time), " + simTime + " ms (simulated time)"); - if (unresolvedCount == suggestions.size()) { - CLIO.err("--- aborting simulation, only unresolved violations remain"); - break; - } - if (withStats) { - perStep.put("statsExecutionStop", SimUtils.calculateStats(simCloudManager, config, verbose)); - } - snapshotCloudManager = new SnapshotCloudManager(simCloudManager, config); - if (saveDir != null) { - File target = new File(saveDir, "step" + loop + "_stop"); - snapshotCloudManager.saveSnapshot(target, true, redact); - } - if (verbose) { - Map snapshot = snapshotCloudManager.getSnapshot(false, redact); - snapshot.remove(SnapshotCloudManager.DISTRIB_STATE_KEY); - snapshot.remove(SnapshotCloudManager.MANAGER_STATE_KEY); - perStep.put("snapshotStop", snapshot); - } - loop++; - } - if (loop == iterations && !suggestions.isEmpty()) { - CLIO.err("### Failed to apply all suggestions in " + iterations + " steps. Remaining suggestions: " + suggestions + "\n"); - } - results.put("finalState", prepareResults(simCloudManager, config, withClusterState, withStats, - withSuggestions, withSortedNodes, withDiagnostics)); - } - } - /** * Get the status of a Solr server. */ diff --git a/solr/core/src/test-files/solr/simSnapshot/autoscalingState.json b/solr/core/src/test-files/solr/simSnapshot/autoscalingState.json deleted file mode 100644 index 9ce3f6f4a65..00000000000 --- a/solr/core/src/test-files/solr/simSnapshot/autoscalingState.json +++ /dev/null @@ -1,3923 +0,0 @@ -{ - "suggestions":[{ - "suggestion":{ - "type":"improvement", - "operation":{ - "method":"POST", - "path":"/c/COLL_q", - "command":{"move-replica":{ - "targetNode":"N_b9_solr", - "inPlaceMove":"true", - "replica":"core_node3"}}}}, - "replica":{"core_node3":{ - "core":"COLL_q_shard1_replica_n1", - "shard":"shard1", - "collection":"COLL_q", - "node_name":"N_7e_solr", - "type":"NRT", - "base_url":"http://N_7e/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":4.9242789E8, - "INDEX.sizeInGB":0.45860921032726765}}}], - "diagnostics":{ - "sortedNodes":[ - { - "node":"N_7e_solr", - "isLive":true, - "cores":13.0, - "freedisk":873.6022491455078, - "sysprop.pool":"pool-03", - "sysprop.az":"us-east-1b", - "totaldisk":999.51171875, - "replicas":{ - "COLL_22":{"shard1":[{"core_node6":{ - "core":"COLL_22_shard1_replica_n4", - "shard":"shard1", - "collection":"COLL_22", - "node_name":"N_7e_solr", - "type":"NRT", - "base_url":"http://N_7e/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":2.4348956483E10, - "INDEX.sizeInGB":22.676732840947807}}]}, - "COLL_q":{"shard1":[{"core_node3":{ - "core":"COLL_q_shard1_replica_n1", - "shard":"shard1", - "collection":"COLL_q", - "node_name":"N_7e_solr", - "type":"NRT", - "base_url":"http://N_7e/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":4.9242789E8, - "INDEX.sizeInGB":0.45860921032726765}}]}, - "COLL_1b":{"shard1":[{"core_node3":{ - "core":"COLL_1b_shard1_replica_n1", - "shard":"shard1", - "collection":"COLL_1b", - "node_name":"N_7e_solr", - "type":"NRT", - "base_url":"http://N_7e/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":135.0, - "INDEX.sizeInGB":1.257285475730896E-7}}]}, - "COLL_1t":{"shard1":[{"core_node3":{ - "core":"COLL_1t_shard1_replica_n1", - "shard":"shard1", - "collection":"COLL_1t", - "node_name":"N_7e_solr", - "type":"NRT", - "base_url":"http://N_7e/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":8.5774407615E10, - "INDEX.sizeInGB":79.8836421361193}}]}, - "COLL_x":{"shard1":[{"core_node3":{ - "core":"COLL_x_shard1_replica_n1", - "shard":"shard1", - "collection":"COLL_x", - "node_name":"N_7e_solr", - "type":"NRT", - "base_url":"http://N_7e/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":3.18270873E8, - "INDEX.sizeInGB":0.296412848867476}}]}, - "COLL_2k":{"shard1":[{"core_node3":{ - "core":"COLL_2k_shard1_replica_n1", - "shard":"shard1", - "collection":"COLL_2k", - "node_name":"N_7e_solr", - "type":"NRT", - "base_url":"http://N_7e/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":135.0, - "INDEX.sizeInGB":1.257285475730896E-7}}]}, - "COLL_1r":{"shard1":[{"core_node3":{ - "core":"COLL_1r_shard1_replica_n1", - "shard":"shard1", - "collection":"COLL_1r", - "node_name":"N_7e_solr", - "type":"NRT", - "base_url":"http://N_7e/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":4.12015174E8, - "INDEX.sizeInGB":0.38371903263032436}}]}, - "COLL_8":{"shard1":[{"core_node3":{ - "core":"COLL_8_shard1_replica_n1", - "shard":"shard1", - "collection":"COLL_8", - "node_name":"N_7e_solr", - "type":"NRT", - "base_url":"http://N_7e/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":356048.0, - "INDEX.sizeInGB":3.315955400466919E-4}}]}, - "COLL_5":{"shard1":[{"core_node2":{ - "core":"COLL_5_shard1_replica_n1", - "shard":"shard1", - "collection":"COLL_5", - "node_name":"N_7e_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":5.854396964E9, - "base_url":"http://N_7e/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":5.452332053333521}}]}, - "COLL_l":{"shard1":[{"core_node3":{ - "core":"COLL_l_shard1_replica_n1", - "shard":"shard1", - "collection":"COLL_l", - "node_name":"N_7e_solr", - "type":"NRT", - "base_url":"http://N_7e/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":135.0, - "INDEX.sizeInGB":1.257285475730896E-7}}]}, - "COLL_1x":{"shard1":[{"core_node3":{ - "core":"COLL_1x_shard1_replica_n1", - "shard":"shard1", - "collection":"COLL_1x", - "node_name":"N_7e_solr", - "type":"NRT", - "base_url":"http://N_7e/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":4248411.0, - "INDEX.sizeInGB":0.00395664107054472}}]}, - "COLL_4":{"shard1":[{"core_node3":{ - "core":"COLL_4_shard1_replica_n1", - "shard":"shard1", - "collection":"COLL_4", - "node_name":"N_7e_solr", - "type":"NRT", - "base_url":"http://N_7e/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":2.58881858E8, - "INDEX.sizeInGB":0.2411025185137987}}]}, - "COLL_6":{"shard1":[{"core_node3":{ - "core":"COLL_6_shard1_replica_n1", - "shard":"shard1", - "collection":"COLL_6", - "node_name":"N_7e_solr", - "type":"NRT", - "base_url":"http://N_7e/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.6446420654E10, - "INDEX.sizeInGB":15.316922826692462}}]}}}, - { - "node":"N_0_solr", - "isLive":true, - "cores":12.0, - "freedisk":719.6562576293945, - "sysprop.pool":"pool-03", - "sysprop.az":"us-east-1a", - "totaldisk":999.51171875, - "replicas":{ - "COLL_22":{"shard1":[{"core_node10":{ - "core":"COLL_22_shard1_replica_n9", - "shard":"shard1", - "collection":"COLL_22", - "node_name":"N_0_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":2.4351639993E10, - "base_url":"http://N_0/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":22.679232054390013}}]}, - "COLL_q":{"shard1":[{"core_node10":{ - "core":"COLL_q_shard1_replica_n9", - "shard":"shard1", - "collection":"COLL_q", - "node_name":"N_0_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":4.9242789E8, - "base_url":"http://N_0/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.45860921032726765}}]}, - "COLL_1b":{"shard1":[{"core_node10":{ - "core":"COLL_1b_shard1_replica_n9", - "shard":"shard1", - "collection":"COLL_1b", - "node_name":"N_0_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":135.0, - "base_url":"http://N_0/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}}]}, - "COLL_1t":{"shard1":[{"core_node5":{ - "core":"COLL_1t_shard1_replica_n2", - "shard":"shard1", - "collection":"COLL_1t", - "node_name":"N_0_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":8.7485800719E10, - "base_url":"http://N_0/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":81.47750116791576}}]}, - "COLL_x":{"shard1":[{"core_node10":{ - "core":"COLL_x_shard1_replica_n9", - "shard":"shard1", - "collection":"COLL_x", - "node_name":"N_0_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":3.0928583E8, - "base_url":"http://N_0/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.2880448754876852}}]}, - "COLL_2k":{"shard1":[{"core_node10":{ - "core":"COLL_2k_shard1_replica_n9", - "shard":"shard1", - "collection":"COLL_2k", - "node_name":"N_0_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":135.0, - "base_url":"http://N_0/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}}]}, - "COLL_1r":{"shard1":[{"core_node5":{ - "core":"COLL_1r_shard1_replica_n2", - "shard":"shard1", - "collection":"COLL_1r", - "node_name":"N_0_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":4.25884524E8, - "base_url":"http://N_0/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.39663587138056755}}]}, - "COLL_8":{"shard1":[{"core_node5":{ - "core":"COLL_8_shard1_replica_n2", - "shard":"shard1", - "collection":"COLL_8", - "node_name":"N_0_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":399225.0, - "base_url":"http://N_0/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":3.718072548508644E-4}}]}, - "COLL_l":{"shard1":[{"core_node10":{ - "core":"COLL_l_shard1_replica_n9", - "shard":"shard1", - "collection":"COLL_l", - "node_name":"N_0_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":135.0, - "base_url":"http://N_0/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}}]}, - "COLL_1x":{"shard1":[{"core_node10":{ - "core":"COLL_1x_shard1_replica_n9", - "shard":"shard1", - "collection":"COLL_1x", - "node_name":"N_0_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":4264901.0, - "base_url":"http://N_0/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.003971998579800129}}]}, - "COLL_4":{"shard1":[{"core_node5":{ - "core":"COLL_4_shard1_replica_n2", - "shard":"shard1", - "collection":"COLL_4", - "node_name":"N_0_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":2.58797271E8, - "base_url":"http://N_0/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.24102374073117971}}]}, - "COLL_6":{"shard1":[{"core_node6":{ - "core":"COLL_6_shard1_replica_n4", - "shard":"shard1", - "collection":"COLL_6", - "node_name":"N_0_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":4.4921656871E10, - "base_url":"http://N_0/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":41.83655313309282}}]}}}, - { - "node":"N_4_solr", - "isLive":true, - "cores":12.0, - "freedisk":875.4758682250977, - "sysprop.pool":"pool-03", - "sysprop.az":"us-east-1c", - "totaldisk":999.51171875, - "replicas":{ - "COLL_22":{"shard1":[{"core_node5":{ - "core":"COLL_22_shard1_replica_n2", - "shard":"shard1", - "collection":"COLL_22", - "node_name":"N_4_solr", - "type":"NRT", - "base_url":"http://N_4/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":2.436290627E10, - "INDEX.sizeInGB":22.689724592491984}}]}, - "COLL_q":{"shard1":[{"core_node6":{ - "core":"COLL_q_shard1_replica_n4", - "shard":"shard1", - "collection":"COLL_q", - "node_name":"N_4_solr", - "type":"NRT", - "base_url":"http://N_4/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":4.9242789E8, - "INDEX.sizeInGB":0.45860921032726765}}]}, - "COLL_1b":{"shard1":[{"core_node6":{ - "core":"COLL_1b_shard1_replica_n4", - "shard":"shard1", - "collection":"COLL_1b", - "node_name":"N_4_solr", - "type":"NRT", - "base_url":"http://N_4/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":135.0, - "INDEX.sizeInGB":1.257285475730896E-7}}]}, - "COLL_1t":{"shard1":[{"core_node6":{ - "core":"COLL_1t_shard1_replica_n4", - "shard":"shard1", - "collection":"COLL_1t", - "node_name":"N_4_solr", - "type":"NRT", - "base_url":"http://N_4/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":8.5380419785E10, - "INDEX.sizeInGB":79.51671237591654}}]}, - "COLL_x":{"shard1":[{"core_node6":{ - "core":"COLL_x_shard1_replica_n4", - "shard":"shard1", - "collection":"COLL_x", - "node_name":"N_4_solr", - "type":"NRT", - "base_url":"http://N_4/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":3.03301808E8, - "INDEX.sizeInGB":0.28247182071208954}}]}, - "COLL_2k":{"shard1":[{"core_node6":{ - "core":"COLL_2k_shard1_replica_n4", - "shard":"shard1", - "collection":"COLL_2k", - "node_name":"N_4_solr", - "type":"NRT", - "base_url":"http://N_4/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":135.0, - "INDEX.sizeInGB":1.257285475730896E-7}}]}, - "COLL_1r":{"shard1":[{"core_node6":{ - "core":"COLL_1r_shard1_replica_n4", - "shard":"shard1", - "collection":"COLL_1r", - "node_name":"N_4_solr", - "type":"NRT", - "base_url":"http://N_4/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":4.46826689E8, - "INDEX.sizeInGB":0.4161397824063897}}]}, - "COLL_8":{"shard1":[{"core_node6":{ - "core":"COLL_8_shard1_replica_n4", - "shard":"shard1", - "collection":"COLL_8", - "node_name":"N_4_solr", - "type":"NRT", - "base_url":"http://N_4/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":356048.0, - "INDEX.sizeInGB":3.315955400466919E-4}}]}, - "COLL_l":{"shard1":[{"core_node6":{ - "core":"COLL_l_shard1_replica_n4", - "shard":"shard1", - "collection":"COLL_l", - "node_name":"N_4_solr", - "type":"NRT", - "base_url":"http://N_4/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":135.0, - "INDEX.sizeInGB":1.257285475730896E-7}}]}, - "COLL_1x":{"shard1":[{"core_node6":{ - "core":"COLL_1x_shard1_replica_n4", - "shard":"shard1", - "collection":"COLL_1x", - "node_name":"N_4_solr", - "type":"NRT", - "base_url":"http://N_4/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":4255591.0, - "INDEX.sizeInGB":0.003963327966630459}}]}, - "COLL_4":{"shard1":[{"core_node6":{ - "core":"COLL_4_shard1_replica_n4", - "shard":"shard1", - "collection":"COLL_4", - "node_name":"N_4_solr", - "type":"NRT", - "base_url":"http://N_4/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":2.59832461E8, - "INDEX.sizeInGB":0.2419878365471959}}]}, - "COLL_6":{"shard1":[{"core_node5":{ - "core":"COLL_6_shard1_replica_n2", - "shard":"shard1", - "collection":"COLL_6", - "node_name":"N_4_solr", - "type":"NRT", - "base_url":"http://N_4/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":2.0738852096E10, - "INDEX.sizeInGB":19.314561128616333}}]}}}, - { - "node":"N_g_solr", - "isLive":true, - "cores":6.0, - "freedisk":4007.3253440856934, - "sysprop.pool":"pool-01", - "sysprop.az":"us-east-1a", - "totaldisk":4998.009765625, - "replicas":{"COLL_2":{ - "shard2_1_0":[{"core_node1681":{ - "core":"COLL_2_shard2_1_0_replica_n1680", - "shard":"shard2_1_0", - "collection":"COLL_2", - "node_name":"N_g_solr", - "type":"NRT", - "base_url":"http://N_g/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.3012044407E11, - "INDEX.sizeInGB":121.18410698138177}}], - "shard5_0_1":[{"core_node1771":{ - "core":"COLL_2_shard5_0_1_replica_n1770", - "shard":"shard5_0_1", - "collection":"COLL_2", - "node_name":"N_g_solr", - "type":"NRT", - "base_url":"http://N_g/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.31464210597E11, - "INDEX.sizeInGB":122.43558708298951}}], - "shard5_1_0":[{"core_node1783":{ - "core":"COLL_2_shard5_1_0_replica_n1782", - "shard":"shard5_1_0", - "collection":"COLL_2", - "node_name":"N_g_solr", - "type":"NRT", - "base_url":"http://N_g/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.30012462556E11, - "INDEX.sizeInGB":121.08354135975242}}], - "shard5_1_1":[{"core_node861":{ - "core":"COLL_2_shard5_1_1_replica_n859", - "shard":"shard5_1_1", - "collection":"COLL_2", - "node_name":"N_g_solr", - "type":"NRT", - "base_url":"http://N_g/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.29967769078E11, - "INDEX.sizeInGB":121.04191731475294}}], - "shard5_0_0":[{"core_node1769":{ - "core":"COLL_2_shard5_0_0_replica_n1768", - "shard":"shard5_0_0", - "collection":"COLL_2", - "node_name":"N_g_solr", - "type":"NRT", - "base_url":"http://N_g/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.31922267714E11, - "INDEX.sizeInGB":122.8621860165149}}], - "shard9_0_0":[{"core_node1683":{ - "core":"COLL_2_shard9_0_0_replica_n1682", - "shard":"shard9_0_0", - "collection":"COLL_2", - "node_name":"N_g_solr", - "type":"NRT", - "property.preferredleader":"true", - "INDEX.sizeInBytes":1.29248772716E11, - "base_url":"http://N_g/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":120.37229977175593}}]}}}, - { - "node":"N_17_solr", - "isLive":true, - "cores":6.0, - "freedisk":4093.756145477295, - "sysprop.pool":"pool-01", - "sysprop.az":"us-east-1a", - "totaldisk":4998.009765625, - "replicas":{"COLL_2":{ - "shard11_1_1":[{"core_node768":{ - "core":"COLL_2_shard11_1_1_replica_n762", - "shard":"shard11_1_1", - "collection":"COLL_2", - "node_name":"N_17_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":1.30871431234E11, - "base_url":"http://N_17/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":121.88351828046143}}], - "shard14_0_0":[{"core_node1121":{ - "core":"COLL_2_shard14_0_0_replica_n1120", - "shard":"shard14_0_0", - "collection":"COLL_2", - "node_name":"N_17_solr", - "type":"NRT", - "base_url":"http://N_17/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.3029908264E11, - "INDEX.sizeInGB":121.3504771143198}}], - "shard18_0_1":[{"core_node877":{ - "core":"COLL_2_shard18_0_1_replica_n2", - "shard":"shard18_0_1", - "collection":"COLL_2", - "node_name":"N_17_solr", - "type":"NRT", - "base_url":"http://N_17/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.28174988934E11, - "INDEX.sizeInGB":119.37226069532335}}], - "shard12_0_1":[{"core_node1699":{ - "core":"COLL_2_shard12_0_1_replica_n1698", - "shard":"shard12_0_1", - "collection":"COLL_2", - "node_name":"N_17_solr", - "type":"NRT", - "base_url":"http://N_17/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.30350286057E11, - "INDEX.sizeInGB":121.39816401246935}}], - "shard12_0_0":[{"core_node1751":{ - "core":"COLL_2_shard12_0_0_replica_n1750", - "shard":"shard12_0_0", - "collection":"COLL_2", - "node_name":"N_17_solr", - "type":"NRT", - "base_url":"http://N_17/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.2936875619E11, - "INDEX.sizeInGB":120.48404308967292}}], - "shard14_0_1":[{"core_node1123":{ - "core":"COLL_2_shard14_0_1_replica_n1122", - "shard":"shard14_0_1", - "collection":"COLL_2", - "node_name":"N_17_solr", - "type":"NRT", - "base_url":"http://N_17/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.31146492351E11, - "INDEX.sizeInGB":122.13968890812248}}]}}}, - { - "node":"N_303_solr", - "isLive":true, - "cores":6.0, - "freedisk":4111.4668045043945, - "sysprop.pool":"pool-01", - "sysprop.az":"us-east-1c", - "totaldisk":4998.009765625, - "replicas":{"COLL_2":{ - "shard16_0_1":[{"core_node987":{ - "core":"COLL_2_shard16_0_1_replica_n986", - "shard":"shard16_0_1", - "collection":"COLL_2", - "node_name":"N_303_solr", - "type":"NRT", - "base_url":"http://N_303/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.30738903625E11, - "INDEX.sizeInGB":121.76009232643992}}], - "shard16_0_0":[{"core_node1785":{ - "core":"COLL_2_shard16_0_0_replica_n1784", - "shard":"shard16_0_0", - "collection":"COLL_2", - "node_name":"N_303_solr", - "type":"NRT", - "base_url":"http://N_303/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.26747476604E11, - "INDEX.sizeInGB":118.04278623685241}}], - "shard3_0_0":[{"core_node544":{ - "core":"COLL_2_shard3_0_0_replica_n2", - "shard":"shard3_0_0", - "collection":"COLL_2", - "node_name":"N_303_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":1.29792212268E11, - "base_url":"http://N_303/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":120.87841729447246}}], - "shard9_1_1":[{"core_node1163":{ - "core":"COLL_2_shard9_1_1_replica_n1162", - "shard":"shard9_1_1", - "collection":"COLL_2", - "node_name":"N_303_solr", - "type":"NRT", - "base_url":"http://N_303/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.36568824379E11, - "INDEX.sizeInGB":127.18962913285941}}], - "shard9_1_0":[{"core_node1151":{ - "core":"COLL_2_shard9_1_0_replica_n1150", - "shard":"shard9_1_0", - "collection":"COLL_2", - "node_name":"N_303_solr", - "type":"NRT", - "base_url":"http://N_303/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.31117387108E11, - "INDEX.sizeInGB":122.11258253827691}}], - "shard4_0_1":[{"core_node1773":{ - "core":"COLL_2_shard4_0_1_replica_n1772", - "shard":"shard4_0_1", - "collection":"COLL_2", - "node_name":"N_303_solr", - "type":"NRT", - "base_url":"http://N_303/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.28126128215E11, - "INDEX.sizeInGB":119.3267556047067}}]}}}, - { - "node":"N_dj_solr", - "isLive":true, - "cores":6.0, - "freedisk":4162.087951660156, - "sysprop.pool":"pool-01", - "sysprop.az":"us-east-1c", - "totaldisk":4998.009765625, - "replicas":{"COLL_2":{ - "shard1_1_0":[{"core_node471":{ - "core":"COLL_2_shard1_1_0_replica_n1", - "shard":"shard1_1_0", - "collection":"COLL_2", - "node_name":"N_dj_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":1.29057719236E11, - "base_url":"http://N_dj/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":120.19436735287309}}], - "shard7_1_0":[{"core_node928":{ - "core":"COLL_2_shard7_1_0_replica_n926", - "shard":"shard7_1_0", - "collection":"COLL_2", - "node_name":"N_dj_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":1.29963886019E11, - "base_url":"http://N_dj/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":121.03830093424767}}], - "shard7_1_1":[{"core_node941":{ - "core":"COLL_2_shard7_1_1_replica_n927", - "shard":"shard7_1_1", - "collection":"COLL_2", - "node_name":"N_dj_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":1.28538540188E11, - "base_url":"http://N_dj/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":119.71084418520331}}], - "shard18_0_1":[{"core_node773":{ - "core":"COLL_2_shard18_0_1_replica_n771", - "shard":"shard18_0_1", - "collection":"COLL_2", - "node_name":"N_dj_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":1.30821199599E11, - "base_url":"http://N_dj/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":121.83673642482609}}], - "shard13_0_1":[{"core_node1715":{ - "core":"COLL_2_shard13_0_1_replica_n1714", - "shard":"shard13_0_1", - "collection":"COLL_2", - "node_name":"N_dj_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":1.30355121703E11, - "base_url":"http://N_dj/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":121.402667558752}}], - "shard13_0_0":[{"core_node1749":{ - "core":"COLL_2_shard13_0_0_replica_n1748", - "shard":"shard13_0_0", - "collection":"COLL_2", - "node_name":"N_dj_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":1.30427736106E11, - "base_url":"http://N_dj/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":121.47029499150813}}]}}}, - { - "node":"N_1c_solr", - "isLive":true, - "cores":6.0, - "freedisk":4181.229598999023, - "sysprop.pool":"pool-01", - "sysprop.az":"us-east-1b", - "totaldisk":4998.009765625, - "replicas":{"COLL_2":{ - "shard5_0_1":[{"core_node1703":{ - "core":"COLL_2_shard5_0_1_replica_n1702", - "shard":"shard5_0_1", - "collection":"COLL_2", - "node_name":"N_1c_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":1.31521149156E11, - "base_url":"http://N_1c/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":122.48861524835229}}], - "shard5_1_0":[{"core_node1135":{ - "core":"COLL_2_shard5_1_0_replica_n1134", - "shard":"shard5_1_0", - "collection":"COLL_2", - "node_name":"N_1c_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":1.30030877168E11, - "base_url":"http://N_1c/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":121.1006913036108}}], - "shard18_0_0":[{"core_node874":{ - "core":"COLL_2_shard18_0_0_replica_n1", - "shard":"shard18_0_0", - "collection":"COLL_2", - "node_name":"N_1c_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":1.28011422432E11, - "base_url":"http://N_1c/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":119.21992751955986}}], - "shard5_1_1":[{"core_node1141":{ - "core":"COLL_2_shard5_1_1_replica_n1140", - "shard":"shard5_1_1", - "collection":"COLL_2", - "node_name":"N_1c_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":1.29917464329E11, - "base_url":"http://N_1c/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":120.99506736639887}}], - "shard5_0_0":[{"core_node999":{ - "core":"COLL_2_shard5_0_0_replica_n998", - "shard":"shard5_0_0", - "collection":"COLL_2", - "node_name":"N_1c_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":1.31937405764E11, - "base_url":"http://N_1c/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":122.87628442421556}}], - "shard18_0_1":[{"core_node876":{ - "core":"COLL_2_shard18_0_1_replica_n1", - "shard":"shard18_0_1", - "collection":"COLL_2", - "node_name":"N_1c_solr", - "type":"NRT", - "base_url":"http://N_1c/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.30729375574E11, - "INDEX.sizeInGB":121.75121863745153}}]}}}, - { - "node":"N_z_solr", - "isLive":true, - "cores":6.0, - "freedisk":4215.115695953369, - "sysprop.pool":"pool-01", - "sysprop.az":"us-east-1c", - "totaldisk":4998.009765625, - "replicas":{"COLL_2":{ - "shard1_0_0":[{"core_node1717":{ - "core":"COLL_2_shard1_0_0_replica_n1716", - "shard":"shard1_0_0", - "collection":"COLL_2", - "node_name":"N_z_solr", - "type":"NRT", - "base_url":"http://N_z/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":5.7185112146E10, - "INDEX.sizeInGB":53.25778587348759}}], - "shard8_1_0":[{"core_node1707":{ - "core":"COLL_2_shard8_1_0_replica_n1706", - "shard":"shard8_1_0", - "collection":"COLL_2", - "node_name":"N_z_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":1.35679630668E11, - "base_url":"http://N_z/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":126.361502956599}}], - "shard8_0_0":[{"core_node1731":{ - "core":"COLL_2_shard8_0_0_replica_n1730", - "shard":"shard8_0_0", - "collection":"COLL_2", - "node_name":"N_z_solr", - "type":"NRT", - "base_url":"http://N_z/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.30170301246E11, - "INDEX.sizeInGB":121.23054009489715}}], - "shard8_0_1":[{"core_node1695":{ - "core":"COLL_2_shard8_0_1_replica_n1694", - "shard":"shard8_0_1", - "collection":"COLL_2", - "node_name":"N_z_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":1.39918850407E11, - "base_url":"http://N_z/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":130.30958399828523}}], - "shard8_1_1":[{"core_node1755":{ - "core":"COLL_2_shard8_1_1_replica_n1754", - "shard":"shard8_1_1", - "collection":"COLL_2", - "node_name":"N_z_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":1.33314153125E11, - "base_url":"http://N_z/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":124.15848032105714}}], - "shard14_1_0":[{"core_node1127":{ - "core":"COLL_2_shard14_1_0_replica_n1126", - "shard":"shard14_1_0", - "collection":"COLL_2", - "node_name":"N_z_solr", - "type":"NRT", - "base_url":"http://N_z/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.27443177079E11, - "INDEX.sizeInGB":118.69070779439062}}]}}}, - { - "node":"N_6_solr", - "isLive":true, - "cores":6.0, - "freedisk":4252.47643661499, - "sysprop.pool":"pool-01", - "sysprop.az":"us-east-1b", - "totaldisk":4998.009765625, - "replicas":{"COLL_2":{ - "shard8_1_0":[{"core_node1811":{ - "core":"COLL_2_shard8_1_0_replica_n1810", - "shard":"shard8_1_0", - "collection":"COLL_2", - "node_name":"N_6_solr", - "type":"NRT", - "base_url":"http://N_6/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.35679249773E11, - "INDEX.sizeInGB":126.36114822048694}}], - "shard4_0_0":[{"core_node520":{ - "core":"COLL_2_shard4_0_0_replica_n2", - "shard":"shard4_0_0", - "collection":"COLL_2", - "node_name":"N_6_solr", - "type":"NRT", - "base_url":"http://N_6/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.28680029361E11, - "INDEX.sizeInGB":119.84261624608189}}], - "shard4_0_1":[{"core_node1803":{ - "core":"COLL_2_shard4_0_1_replica_n1802", - "shard":"shard4_0_1", - "collection":"COLL_2", - "node_name":"N_6_solr", - "type":"NRT", - "base_url":"http://N_6/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.28153346526E11, - "INDEX.sizeInGB":119.35210463218391}}], - "shard9_0_0":[{"core_node1799":{ - "core":"COLL_2_shard9_0_0_replica_n1798", - "shard":"shard9_0_0", - "collection":"COLL_2", - "node_name":"N_6_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":1.35157081196E11, - "base_url":"http://N_6/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":125.874840836972}}], - "shard3_1_0":[{"core_node459":{ - "core":"COLL_2_shard3_1_0_replica_n1", - "shard":"shard3_1_0", - "collection":"COLL_2", - "node_name":"N_6_solr", - "type":"NRT", - "base_url":"http://N_6/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.32652501535E11, - "INDEX.sizeInGB":123.54226925875992}}], - "shard15_1_1":[{"core_node1709":{ - "core":"COLL_2_shard15_1_1_replica_n1708", - "shard":"shard15_1_1", - "collection":"COLL_2", - "node_name":"N_6_solr", - "type":"NRT", - "base_url":"http://N_6/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.30846984322E11, - "INDEX.sizeInGB":121.86075031943619}}]}}}, - { - "node":"N_1m_solr", - "isLive":true, - "cores":6.0, - "freedisk":4257.921604156494, - "sysprop.pool":"pool-01", - "sysprop.az":"us-east-1b", - "totaldisk":4998.009765625, - "replicas":{"COLL_2":{ - "shard6_1_1":[{"core_node1745":{ - "core":"COLL_2_shard6_1_1_replica_n1744", - "shard":"shard6_1_1", - "collection":"COLL_2", - "node_name":"N_1m_solr", - "type":"NRT", - "base_url":"http://N_1m/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.31273933482E11, - "INDEX.sizeInGB":122.25837771035731}}], - "shard1_1_0":[{"core_node1679":{ - "core":"COLL_2_shard1_1_0_replica_n1678", - "shard":"shard1_1_0", - "collection":"COLL_2", - "node_name":"N_1m_solr", - "type":"NRT", - "base_url":"http://N_1m/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.28970690262E11, - "INDEX.sizeInGB":120.11331530474126}}], - "shard8_0_0":[{"core_node887":{ - "core":"COLL_2_shard8_0_0_replica_n886", - "shard":"shard8_0_0", - "collection":"COLL_2", - "node_name":"N_1m_solr", - "type":"NRT", - "base_url":"http://N_1m/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.30145902623E11, - "INDEX.sizeInGB":121.20781710650772}}], - "shard8_0_1":[{"core_node893":{ - "core":"COLL_2_shard8_0_1_replica_n892", - "shard":"shard8_0_1", - "collection":"COLL_2", - "node_name":"N_1m_solr", - "type":"NRT", - "base_url":"http://N_1m/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.32681734677E11, - "INDEX.sizeInGB":123.56949474383146}}], - "shard8_1_1":[{"core_node1711":{ - "core":"COLL_2_shard8_1_1_replica_n1710", - "shard":"shard8_1_1", - "collection":"COLL_2", - "node_name":"N_1m_solr", - "type":"NRT", - "base_url":"http://N_1m/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.33374089494E11, - "INDEX.sizeInGB":124.21430041454732}}], - "shard6_1_0":[{"core_node1167":{ - "core":"COLL_2_shard6_1_0_replica_n1166", - "shard":"shard6_1_0", - "collection":"COLL_2", - "node_name":"N_1m_solr", - "type":"NRT", - "base_url":"http://N_1m/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.29376799009E11, - "INDEX.sizeInGB":120.49153354857117}}]}}}, - { - "node":"N_4g_solr", - "isLive":true, - "cores":6.0, - "freedisk":4259.9677734375, - "sysprop.pool":"pool-01", - "sysprop.az":"us-east-1a", - "totaldisk":4998.009765625, - "replicas":{"COLL_2":{ - "shard8_1_1":[{"core_node1795":{ - "core":"COLL_2_shard8_1_1_replica_n1794", - "shard":"shard8_1_1", - "collection":"COLL_2", - "node_name":"N_4g_solr", - "type":"NRT", - "base_url":"http://N_4g/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.33276674177E11, - "INDEX.sizeInGB":124.1235753307119}}], - "shard9_1_1":[{"core_node944":{ - "core":"COLL_2_shard9_1_1_replica_n930", - "shard":"shard9_1_1", - "collection":"COLL_2", - "node_name":"N_4g_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":1.33928213329E11, - "base_url":"http://N_4g/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":124.73036845121533}}], - "shard9_1_0":[{"core_node931":{ - "core":"COLL_2_shard9_1_0_replica_n929", - "shard":"shard9_1_0", - "collection":"COLL_2", - "node_name":"N_4g_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":1.31111103315E11, - "base_url":"http://N_4g/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":122.1067303000018}}], - "shard18_1_1":[{"core_node626":{ - "core":"COLL_2_shard18_1_1_replica_n624", - "shard":"shard18_1_1", - "collection":"COLL_2", - "node_name":"N_4g_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":1.28190099634E11, - "base_url":"http://N_4g/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":119.38633363135159}}], - "shard18_1_0":[{"core_node625":{ - "core":"COLL_2_shard18_1_0_replica_n623", - "shard":"shard18_1_0", - "collection":"COLL_2", - "node_name":"N_4g_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":1.28955475131E11, - "base_url":"http://N_4g/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":120.09914510976523}}], - "shard2_1_1":[{"core_node1813":{ - "core":"COLL_2_shard2_1_1_replica_n1812", - "shard":"shard2_1_1", - "collection":"COLL_2", - "node_name":"N_4g_solr", - "type":"NRT", - "base_url":"http://N_4g/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.28164947427E11, - "INDEX.sizeInGB":119.36290881317109}}]}}}, - { - "node":"N_cs_solr", - "isLive":true, - "cores":6.0, - "freedisk":4260.629165649414, - "sysprop.pool":"pool-01", - "sysprop.az":"us-east-1c", - "totaldisk":4998.009765625, - "replicas":{"COLL_2":{ - "shard6_1_1":[{"core_node1705":{ - "core":"COLL_2_shard6_1_1_replica_n1704", - "shard":"shard6_1_1", - "collection":"COLL_2", - "node_name":"N_cs_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":1.31274462707E11, - "base_url":"http://N_cs/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":122.25887058954686}}], - "shard10_0_1":[{"core_node828":{ - "core":"COLL_2_shard10_0_1_replica_n826", - "shard":"shard10_0_1", - "collection":"COLL_2", - "node_name":"N_cs_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":1.28038688927E11, - "base_url":"http://N_cs/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":119.245321421884}}], - "shard6_1_0":[{"core_node937":{ - "core":"COLL_2_shard6_1_0_replica_n935", - "shard":"shard6_1_0", - "collection":"COLL_2", - "node_name":"N_cs_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":1.29597529819E11, - "base_url":"http://N_cs/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":120.69710513483733}}], - "shard15_1_0":[{"core_node955":{ - "core":"COLL_2_shard15_1_0_replica_n953", - "shard":"shard15_1_0", - "collection":"COLL_2", - "node_name":"N_cs_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":1.33515745782E11, - "base_url":"http://N_cs/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":124.34622811339796}}], - "shard10_0_0":[{"core_node827":{ - "core":"COLL_2_shard10_0_0_replica_n825", - "shard":"shard10_0_0", - "collection":"COLL_2", - "node_name":"N_cs_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":1.29486149433E11, - "base_url":"http://N_cs/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":120.59337406698614}}], - "shard15_1_1":[{"core_node956":{ - "core":"COLL_2_shard15_1_1_replica_n954", - "shard":"shard15_1_1", - "collection":"COLL_2", - "node_name":"N_cs_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":1.30865977458E11, - "base_url":"http://N_cs/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":121.87843905575573}}]}}}, - { - "node":"N_1f_solr", - "isLive":true, - "cores":6.0, - "freedisk":4260.807849884033, - "sysprop.pool":"pool-01", - "sysprop.az":"us-east-1c", - "totaldisk":4998.009765625, - "replicas":{"COLL_2":{ - "shard11_0_1":[{"core_node1223":{ - "core":"COLL_2_shard11_0_1_replica_n1222", - "shard":"shard11_0_1", - "collection":"COLL_2", - "node_name":"N_1f_solr", - "type":"NRT", - "base_url":"http://N_1f/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.27989218509E11, - "INDEX.sizeInGB":119.19924850482494}}], - "shard11_1_0":[{"core_node779":{ - "core":"COLL_2_shard11_1_0_replica_n778", - "shard":"shard11_1_0", - "collection":"COLL_2", - "node_name":"N_1f_solr", - "type":"NRT", - "base_url":"http://N_1f/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.32552454912E11, - "INDEX.sizeInGB":123.44909358024597}}], - "shard11_0_0":[{"core_node1217":{ - "core":"COLL_2_shard11_0_0_replica_n1216", - "shard":"shard11_0_0", - "collection":"COLL_2", - "node_name":"N_1f_solr", - "type":"NRT", - "base_url":"http://N_1f/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.27720861488E11, - "INDEX.sizeInGB":118.94932155311108}}], - "shard11_1_1":[{"core_node783":{ - "core":"COLL_2_shard11_1_1_replica_n782", - "shard":"shard11_1_1", - "collection":"COLL_2", - "node_name":"N_1f_solr", - "type":"NRT", - "base_url":"http://N_1f/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.30995783614E11, - "INDEX.sizeInGB":121.99933045916259}}], - "shard5_0_1":[{"core_node1003":{ - "core":"COLL_2_shard5_0_1_replica_n1002", - "shard":"shard5_0_1", - "collection":"COLL_2", - "node_name":"N_1f_solr", - "type":"NRT", - "base_url":"http://N_1f/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.31534942129E11, - "INDEX.sizeInGB":122.50146095547825}}], - "shard5_0_0":[{"core_node1001":{ - "core":"COLL_2_shard5_0_0_replica_n1000", - "shard":"shard5_0_0", - "collection":"COLL_2", - "node_name":"N_1f_solr", - "type":"NRT", - "base_url":"http://N_1f/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.31960210955E11, - "INDEX.sizeInGB":122.89752341341227}}]}}}, - { - "node":"N_65p_solr", - "isLive":true, - "cores":6.0, - "freedisk":4260.997627258301, - "sysprop.pool":"pool-01", - "sysprop.az":"us-east-1a", - "totaldisk":4998.009765625, - "replicas":{"COLL_2":{ - "shard7_0_0":[{"core_node774":{ - "core":"COLL_2_shard7_0_0_replica_n1", - "shard":"shard7_0_0", - "collection":"COLL_2", - "node_name":"N_65p_solr", - "type":"NRT", - "base_url":"http://N_65p/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.29027793373E11, - "INDEX.sizeInGB":120.16649672109634}}], - "shard10_1_0":[{"core_node1797":{ - "core":"COLL_2_shard10_1_0_replica_n1796", - "shard":"shard10_1_0", - "collection":"COLL_2", - "node_name":"N_65p_solr", - "type":"NRT", - "base_url":"http://N_65p/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.27583656591E11, - "INDEX.sizeInGB":118.82153953518718}}], - "shard3_0_0":[{"core_node543":{ - "core":"COLL_2_shard3_0_0_replica_n1", - "shard":"shard3_0_0", - "collection":"COLL_2", - "node_name":"N_65p_solr", - "type":"NRT", - "base_url":"http://N_65p/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.29871412511E11, - "INDEX.sizeInGB":120.95217826869339}}], - "shard3_0_1":[{"core_node545":{ - "core":"COLL_2_shard3_0_1_replica_n1", - "shard":"shard3_0_1", - "collection":"COLL_2", - "node_name":"N_65p_solr", - "type":"NRT", - "base_url":"http://N_65p/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.31838835644E11, - "INDEX.sizeInGB":122.784483846277}}], - "shard15_1_0":[{"core_node1173":{ - "core":"COLL_2_shard15_1_0_replica_n1172", - "shard":"shard15_1_0", - "collection":"COLL_2", - "node_name":"N_65p_solr", - "type":"NRT", - "base_url":"http://N_65p/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.33316507698E11, - "INDEX.sizeInGB":124.16067318804562}}], - "shard15_1_1":[{"core_node1747":{ - "core":"COLL_2_shard15_1_1_replica_n1746", - "shard":"shard15_1_1", - "collection":"COLL_2", - "node_name":"N_65p_solr", - "type":"NRT", - "base_url":"http://N_65p/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.30883359905E11, - "INDEX.sizeInGB":121.89462772104889}}]}}}, - { - "node":"N_u_solr", - "isLive":true, - "cores":6.0, - "freedisk":4260.821304321289, - "sysprop.pool":"pool-01", - "sysprop.az":"us-east-1a", - "totaldisk":4998.009765625, - "replicas":{"COLL_2":{ - "shard8_1_0":[{"core_node1765":{ - "core":"COLL_2_shard8_1_0_replica_n1764", - "shard":"shard8_1_0", - "collection":"COLL_2", - "node_name":"N_u_solr", - "type":"NRT", - "base_url":"http://N_u/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.35571920799E11, - "INDEX.sizeInGB":126.26119032409042}}], - "shard13_1_1":[{"core_node921":{ - "core":"COLL_2_shard13_1_1_replica_n920", - "shard":"shard13_1_1", - "collection":"COLL_2", - "node_name":"N_u_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":1.29634542289E11, - "base_url":"http://N_u/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":120.73157568369061}}], - "shard15_0_1":[{"core_node734":{ - "core":"COLL_2_shard15_0_1_replica_n2", - "shard":"shard15_0_1", - "collection":"COLL_2", - "node_name":"N_u_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":1.27250282639E11, - "base_url":"http://N_u/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":118.51106084790081}}], - "shard13_0_1":[{"core_node1263":{ - "core":"COLL_2_shard13_0_1_replica_n1262", - "shard":"shard13_0_1", - "collection":"COLL_2", - "node_name":"N_u_solr", - "type":"NRT", - "base_url":"http://N_u/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.30321828131E11, - "INDEX.sizeInGB":121.37166050355881}}], - "shard13_1_0":[{"core_node1763":{ - "core":"COLL_2_shard13_1_0_replica_n1762", - "shard":"shard13_1_0", - "collection":"COLL_2", - "node_name":"N_u_solr", - "type":"NRT", - "base_url":"http://N_u/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.29567251239E11, - "INDEX.sizeInGB":120.66890600975603}}], - "shard13_0_0":[{"core_node1257":{ - "core":"COLL_2_shard13_0_0_replica_n1256", - "shard":"shard13_0_0", - "collection":"COLL_2", - "node_name":"N_u_solr", - "type":"NRT", - "base_url":"http://N_u/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.30381429251E11, - "INDEX.sizeInGB":121.42716837208718}}]}}}, - { - "node":"N_a_solr", - "isLive":true, - "cores":6.0, - "freedisk":4262.172649383545, - "sysprop.pool":"pool-01", - "sysprop.az":"us-east-1b", - "totaldisk":4998.009765625, - "replicas":{"COLL_2":{ - "shard3_0_0":[{"core_node1809":{ - "core":"COLL_2_shard3_0_0_replica_n1808", - "shard":"shard3_0_0", - "collection":"COLL_2", - "node_name":"N_a_solr", - "type":"NRT", - "base_url":"http://N_a/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.29798330608E11, - "INDEX.sizeInGB":120.88411544263363}}], - "shard14_0_0":[{"core_node1119":{ - "core":"COLL_2_shard14_0_0_replica_n1118", - "shard":"shard14_0_0", - "collection":"COLL_2", - "node_name":"N_a_solr", - "type":"NRT", - "base_url":"http://N_a/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.30313698451E11, - "INDEX.sizeInGB":121.36408914905041}}], - "shard15_1_0":[{"core_node1175":{ - "core":"COLL_2_shard15_1_0_replica_n1174", - "shard":"shard15_1_0", - "collection":"COLL_2", - "node_name":"N_a_solr", - "type":"NRT", - "base_url":"http://N_a/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.33321224738E11, - "INDEX.sizeInGB":124.16506627388299}}], - "shard14_1_1":[{"core_node836":{ - "core":"COLL_2_shard14_1_1_replica_n834", - "shard":"shard14_1_1", - "collection":"COLL_2", - "node_name":"N_a_solr", - "type":"NRT", - "base_url":"http://N_a/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.29318568492E11, - "INDEX.sizeInGB":120.43730215355754}}], - "shard14_0_1":[{"core_node1125":{ - "core":"COLL_2_shard14_0_1_replica_n1124", - "shard":"shard14_0_1", - "collection":"COLL_2", - "node_name":"N_a_solr", - "type":"NRT", - "base_url":"http://N_a/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.31102045065E11, - "INDEX.sizeInGB":122.09829414729029}}], - "shard14_1_0":[{"core_node835":{ - "core":"COLL_2_shard14_1_0_replica_n833", - "shard":"shard14_1_0", - "collection":"COLL_2", - "node_name":"N_a_solr", - "type":"NRT", - "base_url":"http://N_a/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.27418065808E11, - "INDEX.sizeInGB":118.66732110083103}}]}}}, - { - "node":"N_8_solr", - "isLive":true, - "cores":6.0, - "freedisk":4262.037788391113, - "sysprop.pool":"pool-01", - "sysprop.az":"us-east-1b", - "totaldisk":4998.009765625, - "replicas":{"COLL_2":{ - "shard16_1_1":[{"core_node853":{ - "core":"COLL_2_shard16_1_1_replica_n851", - "shard":"shard16_1_1", - "collection":"COLL_2", - "node_name":"N_8_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":1.33685050832E11, - "base_url":"http://N_8/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":124.50390572845936}}], - "shard16_0_1":[{"core_node857":{ - "core":"COLL_2_shard16_0_1_replica_n855", - "shard":"shard16_0_1", - "collection":"COLL_2", - "node_name":"N_8_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":1.30788718518E11, - "base_url":"http://N_8/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":121.80648606084287}}], - "shard16_1_0":[{"core_node852":{ - "core":"COLL_2_shard16_1_0_replica_n850", - "shard":"shard16_1_0", - "collection":"COLL_2", - "node_name":"N_8_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":1.28801317856E11, - "base_url":"http://N_8/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":119.95557495951653}}], - "shard16_0_0":[{"core_node856":{ - "core":"COLL_2_shard16_0_0_replica_n854", - "shard":"shard16_0_0", - "collection":"COLL_2", - "node_name":"N_8_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":1.2677230126E11, - "base_url":"http://N_8/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":118.06590599939227}}], - "shard2_0_0":[{"core_node796":{ - "core":"COLL_2_shard2_0_0_replica_n794", - "shard":"shard2_0_0", - "collection":"COLL_2", - "node_name":"N_8_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":1.29517293483E11, - "base_url":"http://N_8/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":120.6223792238161}}], - "shard2_0_1":[{"core_node800":{ - "core":"COLL_2_shard2_0_1_replica_n795", - "shard":"shard2_0_1", - "collection":"COLL_2", - "node_name":"N_8_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":1.31328007233E11, - "base_url":"http://N_8/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":122.30873781535774}}]}}}, - { - "node":"N_3a7_solr", - "isLive":true, - "cores":6.0, - "freedisk":4263.317134857178, - "sysprop.pool":"pool-01", - "sysprop.az":"us-east-1c", - "totaldisk":4998.009765625, - "replicas":{"COLL_2":{ - "shard7_0_0":[{"core_node775":{ - "core":"COLL_2_shard7_0_0_replica_n2", - "shard":"shard7_0_0", - "collection":"COLL_2", - "node_name":"N_3a7_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":1.29074533898E11, - "base_url":"http://N_3a7/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":120.21002722717822}}], - "shard2_0_0":[{"core_node1823":{ - "core":"COLL_2_shard2_0_0_replica_n1822", - "shard":"shard2_0_0", - "collection":"COLL_2", - "node_name":"N_3a7_solr", - "type":"NRT", - "base_url":"http://N_3a7/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.29476268104E11, - "INDEX.sizeInGB":120.58417136222124}}], - "shard14_0_0":[{"core_node839":{ - "core":"COLL_2_shard14_0_0_replica_n837", - "shard":"shard14_0_0", - "collection":"COLL_2", - "node_name":"N_3a7_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":1.30330451538E11, - "base_url":"http://N_3a7/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":121.37969167716801}}], - "shard3_1_1":[{"core_node462":{ - "core":"COLL_2_shard3_1_1_replica_n2", - "shard":"shard3_1_1", - "collection":"COLL_2", - "node_name":"N_3a7_solr", - "type":"NRT", - "base_url":"http://N_3a7/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.2992912768E11, - "INDEX.sizeInGB":121.00592970848083}}], - "shard14_1_1":[{"core_node1825":{ - "core":"COLL_2_shard14_1_1_replica_n1824", - "shard":"shard14_1_1", - "collection":"COLL_2", - "node_name":"N_3a7_solr", - "type":"NRT", - "base_url":"http://N_3a7/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.300425186E11, - "INDEX.sizeInGB":121.11153323203325}}], - "shard14_0_1":[{"core_node841":{ - "core":"COLL_2_shard14_0_1_replica_n838", - "shard":"shard14_0_1", - "collection":"COLL_2", - "node_name":"N_3a7_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":1.31168916273E11, - "base_url":"http://N_3a7/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":122.1605728128925}}]}}}, - { - "node":"N_11_solr", - "isLive":true, - "cores":6.0, - "freedisk":4264.325901031494, - "sysprop.pool":"pool-01", - "sysprop.az":"us-east-1b", - "totaldisk":4998.009765625, - "replicas":{"COLL_2":{ - "shard6_0_0":[{"core_node1210":{ - "core":"COLL_2_shard6_0_0_replica_n1209", - "shard":"shard6_0_0", - "collection":"COLL_2", - "node_name":"N_11_solr", - "type":"NRT", - "base_url":"http://N_11/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.28939953876E11, - "INDEX.sizeInGB":120.08468981459737}}], - "shard6_0_1":[{"core_node1212":{ - "core":"COLL_2_shard6_0_1_replica_n1211", - "shard":"shard6_0_1", - "collection":"COLL_2", - "node_name":"N_11_solr", - "type":"NRT", - "base_url":"http://N_11/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.28744354495E11, - "INDEX.sizeInGB":119.90252369549125}}], - "shard9_1_1":[{"core_node1155":{ - "core":"COLL_2_shard9_1_1_replica_n1154", - "shard":"shard9_1_1", - "collection":"COLL_2", - "node_name":"N_11_solr", - "type":"NRT", - "base_url":"http://N_11/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.33894519282E11, - "INDEX.sizeInGB":124.69898842461407}}], - "shard9_1_0":[{"core_node1153":{ - "core":"COLL_2_shard9_1_0_replica_n1152", - "shard":"shard9_1_0", - "collection":"COLL_2", - "node_name":"N_11_solr", - "type":"NRT", - "base_url":"http://N_11/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.31406038908E11, - "INDEX.sizeInGB":122.3814104758203}}], - "shard9_0_1":[{"core_node438":{ - "core":"COLL_2_shard9_0_1_replica_n436", - "shard":"shard9_0_1", - "collection":"COLL_2", - "node_name":"N_11_solr", - "type":"NRT", - "base_url":"http://N_11/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.29282915395E11, - "INDEX.sizeInGB":120.40409761946648}}], - "shard12_1_1":[{"core_node662":{ - "core":"COLL_2_shard12_1_1_replica_n2", - "shard":"shard12_1_1", - "collection":"COLL_2", - "node_name":"N_11_solr", - "type":"NRT", - "base_url":"http://N_11/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.26693447901E11, - "INDEX.sizeInGB":117.99246808607131}}]}}}, - { - "node":"N_4f_solr", - "isLive":true, - "cores":6.0, - "freedisk":4264.210151672363, - "sysprop.pool":"pool-01", - "sysprop.az":"us-east-1c", - "totaldisk":4998.009765625, - "replicas":{"COLL_2":{ - "shard2_0_1":[{"core_node915":{ - "core":"COLL_2_shard2_0_1_replica_n914", - "shard":"shard2_0_1", - "collection":"COLL_2", - "node_name":"N_4f_solr", - "type":"NRT", - "base_url":"http://N_4f/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.31386626219E11, - "INDEX.sizeInGB":122.36333100032061}}], - "shard2_1_0":[{"core_node975":{ - "core":"COLL_2_shard2_1_0_replica_n974", - "shard":"shard2_1_0", - "collection":"COLL_2", - "node_name":"N_4f_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":1.3001251468E11, - "base_url":"http://N_4f/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":121.0835899040103}}], - "shard6_0_0":[{"core_node1182":{ - "core":"COLL_2_shard6_0_0_replica_n1180", - "shard":"shard6_0_0", - "collection":"COLL_2", - "node_name":"N_4f_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":1.28922958966E11, - "base_url":"http://N_4f/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":120.06886207126081}}], - "shard6_0_1":[{"core_node1189":{ - "core":"COLL_2_shard6_0_1_replica_n1181", - "shard":"shard6_0_1", - "collection":"COLL_2", - "node_name":"N_4f_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":1.28773562289E11, - "base_url":"http://N_4f/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":119.92972557339817}}], - "shard3_0_1":[{"core_node546":{ - "core":"COLL_2_shard3_0_1_replica_n2", - "shard":"shard3_0_1", - "collection":"COLL_2", - "node_name":"N_4f_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":1.31838927317E11, - "base_url":"http://N_4f/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":122.78456922341138}}], - "shard2_1_1":[{"core_node1685":{ - "core":"COLL_2_shard2_1_1_replica_n1684", - "shard":"shard2_1_1", - "collection":"COLL_2", - "node_name":"N_4f_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":1.2812596905E11, - "base_url":"http://N_4f/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":119.32660737074912}}]}}}, - { - "node":"N_1i_solr", - "isLive":true, - "cores":6.0, - "freedisk":4266.027156829834, - "sysprop.pool":"pool-01", - "sysprop.az":"us-east-1c", - "totaldisk":4998.009765625, - "replicas":{"COLL_2":{ - "shard17_1_0":[{"core_node1200":{ - "core":"COLL_2_shard17_1_0_replica_n1198", - "shard":"shard17_1_0", - "collection":"COLL_2", - "node_name":"N_1i_solr", - "type":"NRT", - "base_url":"http://N_1i/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.29069936299E11, - "INDEX.sizeInGB":120.20574537944049}}], - "shard17_0_1":[{"core_node1117":{ - "core":"COLL_2_shard17_0_1_replica_n1116", - "shard":"shard17_0_1", - "collection":"COLL_2", - "node_name":"N_1i_solr", - "type":"NRT", - "base_url":"http://N_1i/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.30694171889E11, - "INDEX.sizeInGB":121.71843265090138}}], - "shard10_1_1":[{"core_node1779":{ - "core":"COLL_2_shard10_1_1_replica_n1778", - "shard":"shard10_1_1", - "collection":"COLL_2", - "node_name":"N_1i_solr", - "type":"NRT", - "base_url":"http://N_1i/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.30255789623E11, - "INDEX.sizeInGB":121.31015735026449}}], - "shard17_0_0":[{"core_node1781":{ - "core":"COLL_2_shard17_0_0_replica_n1780", - "shard":"shard17_0_0", - "collection":"COLL_2", - "node_name":"N_1i_solr", - "type":"NRT", - "base_url":"http://N_1i/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.30702509646E11, - "INDEX.sizeInGB":121.72619779221714}}], - "shard10_1_0":[{"core_node1693":{ - "core":"COLL_2_shard10_1_0_replica_n1692", - "shard":"shard10_1_0", - "collection":"COLL_2", - "node_name":"N_1i_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":1.27561685082E11, - "base_url":"http://N_1i/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":118.80107697285712}}], - "shard17_1_1":[{"core_node1203":{ - "core":"COLL_2_shard17_1_1_replica_n1199", - "shard":"shard17_1_1", - "collection":"COLL_2", - "node_name":"N_1i_solr", - "type":"NRT", - "base_url":"http://N_1i/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.28764084367E11, - "INDEX.sizeInGB":119.92089857067913}}]}}}, - { - "node":"N_9o_solr", - "isLive":true, - "cores":6.0, - "freedisk":4265.881809234619, - "sysprop.pool":"pool-01", - "sysprop.az":"us-east-1b", - "totaldisk":4998.009765625, - "replicas":{"COLL_2":{ - "shard11_0_1":[{"core_node1221":{ - "core":"COLL_2_shard11_0_1_replica_n1220", - "shard":"shard11_0_1", - "collection":"COLL_2", - "node_name":"N_9o_solr", - "type":"NRT", - "base_url":"http://N_9o/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.28020049235E11, - "INDEX.sizeInGB":119.22796185594052}}], - "shard11_1_0":[{"core_node781":{ - "core":"COLL_2_shard11_1_0_replica_n780", - "shard":"shard11_1_0", - "collection":"COLL_2", - "node_name":"N_9o_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":1.32420261013E11, - "base_url":"http://N_9o/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":123.32597841788083}}], - "shard11_0_0":[{"core_node1219":{ - "core":"COLL_2_shard11_0_0_replica_n1218", - "shard":"shard11_0_0", - "collection":"COLL_2", - "node_name":"N_9o_solr", - "type":"NRT", - "base_url":"http://N_9o/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.28002391411E11, - "INDEX.sizeInGB":119.21151672583073}}], - "shard7_0_0":[{"core_node766":{ - "core":"COLL_2_shard7_0_0_replica_n764", - "shard":"shard7_0_0", - "collection":"COLL_2", - "node_name":"N_9o_solr", - "type":"NRT", - "base_url":"http://N_9o/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.28994593549E11, - "INDEX.sizeInGB":120.13557697553188}}], - "shard11_1_1":[{"core_node785":{ - "core":"COLL_2_shard11_1_1_replica_n784", - "shard":"shard11_1_1", - "collection":"COLL_2", - "node_name":"N_9o_solr", - "type":"NRT", - "base_url":"http://N_9o/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.30909357727E11, - "INDEX.sizeInGB":121.91884007956833}}], - "shard7_0_1":[{"core_node769":{ - "core":"COLL_2_shard7_0_1_replica_n765", - "shard":"shard7_0_1", - "collection":"COLL_2", - "node_name":"N_9o_solr", - "type":"NRT", - "base_url":"http://N_9o/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.28908501869E11, - "INDEX.sizeInGB":120.0553978504613}}]}}}, - { - "node":"N_2_solr", - "isLive":true, - "cores":6.0, - "freedisk":4266.604637145996, - "sysprop.pool":"pool-01", - "sysprop.az":"us-east-1c", - "totaldisk":4998.009765625, - "replicas":{"COLL_2":{ - "shard5_1_0":[{"core_node1137":{ - "core":"COLL_2_shard5_1_0_replica_n1136", - "shard":"shard5_1_0", - "collection":"COLL_2", - "node_name":"N_2_solr", - "type":"NRT", - "base_url":"http://N_2/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":7.6877250282E10, - "INDEX.sizeInGB":71.59751866199076}}], - "shard5_1_1":[{"core_node1139":{ - "core":"COLL_2_shard5_1_1_replica_n1138", - "shard":"shard5_1_1", - "collection":"COLL_2", - "node_name":"N_2_solr", - "type":"NRT", - "base_url":"http://N_2/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.29952609098E11, - "INDEX.sizeInGB":121.02779848314822}}], - "shard7_0_1":[{"core_node776":{ - "core":"COLL_2_shard7_0_1_replica_n1", - "shard":"shard7_0_1", - "collection":"COLL_2", - "node_name":"N_2_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":1.2890128588E11, - "base_url":"http://N_2/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":120.04867743700743}}], - "shard9_0_1":[{"core_node478":{ - "core":"COLL_2_shard9_0_1_replica_n2", - "shard":"shard9_0_1", - "collection":"COLL_2", - "node_name":"N_2_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":1.29212951693E11, - "base_url":"http://N_2/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":120.33893884439021}}], - "shard12_0_1":[{"core_node1255":{ - "core":"COLL_2_shard12_0_1_replica_n1254", - "shard":"shard12_0_1", - "collection":"COLL_2", - "node_name":"N_2_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":1.30384315739E11, - "base_url":"http://N_2/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":121.42985662352294}}], - "shard12_0_0":[{"core_node1249":{ - "core":"COLL_2_shard12_0_0_replica_n1248", - "shard":"shard12_0_0", - "collection":"COLL_2", - "node_name":"N_2_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":1.29421522442E11, - "base_url":"http://N_2/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":120.53318549133837}}]}}}, - { - "node":"N_2u_solr", - "isLive":true, - "cores":6.0, - "freedisk":4266.648368835449, - "sysprop.pool":"pool-01", - "sysprop.az":"us-east-1b", - "totaldisk":4998.009765625, - "replicas":{"COLL_2":{ - "shard17_1_0":[{"core_node1225":{ - "core":"COLL_2_shard17_1_0_replica_n1224", - "shard":"shard17_1_0", - "collection":"COLL_2", - "node_name":"N_2u_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":1.29066474889E11, - "base_url":"http://N_2u/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":120.20252169016749}}], - "shard17_0_1":[{"core_node1115":{ - "core":"COLL_2_shard17_0_1_replica_n1114", - "shard":"shard17_0_1", - "collection":"COLL_2", - "node_name":"N_2u_solr", - "type":"NRT", - "base_url":"http://N_2u/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.30049647193E11, - "INDEX.sizeInGB":121.1181722516194}}], - "shard17_0_0":[{"core_node1735":{ - "core":"COLL_2_shard17_0_0_replica_n1734", - "shard":"shard17_0_0", - "collection":"COLL_2", - "node_name":"N_2u_solr", - "type":"NRT", - "base_url":"http://N_2u/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.31102615765E11, - "INDEX.sizeInGB":122.09882565308362}}], - "shard3_1_1":[{"core_node461":{ - "core":"COLL_2_shard3_1_1_replica_n1", - "shard":"shard3_1_1", - "collection":"COLL_2", - "node_name":"N_2u_solr", - "type":"NRT", - "base_url":"http://N_2u/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.29953637358E11, - "INDEX.sizeInGB":121.02875612489879}}], - "shard17_1_1":[{"core_node1231":{ - "core":"COLL_2_shard17_1_1_replica_n1230", - "shard":"shard17_1_1", - "collection":"COLL_2", - "node_name":"N_2u_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":1.287734207E11, - "base_url":"http://N_2u/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":119.92959370836616}}], - "shard12_1_0":[{"core_node660":{ - "core":"COLL_2_shard12_1_0_replica_n2", - "shard":"shard12_1_0", - "collection":"COLL_2", - "node_name":"N_2u_solr", - "type":"NRT", - "base_url":"http://N_2u/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.27387972534E11, - "INDEX.sizeInGB":118.63929455541074}}]}}}, - { - "node":"N_m_solr", - "isLive":true, - "cores":6.0, - "freedisk":4267.171646118164, - "sysprop.pool":"pool-01", - "sysprop.az":"us-east-1a", - "totaldisk":4998.009765625, - "replicas":{"COLL_2":{ - "shard6_1_1":[{"core_node1171":{ - "core":"COLL_2_shard6_1_1_replica_n1170", - "shard":"shard6_1_1", - "collection":"COLL_2", - "node_name":"N_m_solr", - "type":"NRT", - "base_url":"http://N_m/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.31256081422E11, - "INDEX.sizeInGB":122.24175168387592}}], - "shard17_1_0":[{"core_node1227":{ - "core":"COLL_2_shard17_1_0_replica_n1226", - "shard":"shard17_1_0", - "collection":"COLL_2", - "node_name":"N_m_solr", - "type":"NRT", - "base_url":"http://N_m/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.29049722959E11, - "INDEX.sizeInGB":120.18692023959011}}], - "shard6_0_0":[{"core_node1208":{ - "core":"COLL_2_shard6_0_0_replica_n1207", - "shard":"shard6_0_0", - "collection":"COLL_2", - "node_name":"N_m_solr", - "type":"NRT", - "base_url":"http://N_m/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.28936808614E11, - "INDEX.sizeInGB":120.08176056109369}}], - "shard6_0_1":[{"core_node1214":{ - "core":"COLL_2_shard6_0_1_replica_n1213", - "shard":"shard6_0_1", - "collection":"COLL_2", - "node_name":"N_m_solr", - "type":"NRT", - "base_url":"http://N_m/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.28745543493E11, - "INDEX.sizeInGB":119.90363103616983}}], - "shard9_0_1":[{"core_node477":{ - "core":"COLL_2_shard9_0_1_replica_n1", - "shard":"shard9_0_1", - "collection":"COLL_2", - "node_name":"N_m_solr", - "type":"NRT", - "base_url":"http://N_m/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.29063920601E11, - "INDEX.sizeInGB":120.20014282409102}}], - "shard17_1_1":[{"core_node1229":{ - "core":"COLL_2_shard17_1_1_replica_n1228", - "shard":"shard17_1_1", - "collection":"COLL_2", - "node_name":"N_m_solr", - "type":"NRT", - "base_url":"http://N_m/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.28816978409E11, - "INDEX.sizeInGB":119.97015998605639}}]}}}, - { - "node":"N_t_solr", - "isLive":true, - "cores":6.0, - "freedisk":4266.856658935547, - "sysprop.pool":"pool-01", - "sysprop.az":"us-east-1a", - "totaldisk":4998.009765625, - "replicas":{"COLL_2":{ - "shard11_0_1":[{"core_node1195":{ - "core":"COLL_2_shard11_0_1_replica_n1184", - "shard":"shard11_0_1", - "collection":"COLL_2", - "node_name":"N_t_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":1.27980394382E11, - "base_url":"http://N_t/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":119.19103039614856}}], - "shard11_1_0":[{"core_node1791":{ - "core":"COLL_2_shard11_1_0_replica_n1790", - "shard":"shard11_1_0", - "collection":"COLL_2", - "node_name":"N_t_solr", - "type":"NRT", - "base_url":"http://N_t/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.32416023485E11, - "INDEX.sizeInGB":123.32203191239387}}], - "shard11_0_0":[{"core_node1185":{ - "core":"COLL_2_shard11_0_0_replica_n1183", - "shard":"shard11_0_0", - "collection":"COLL_2", - "node_name":"N_t_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":1.2777477116E11, - "base_url":"http://N_t/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":118.99952884763479}}], - "shard10_1_1":[{"core_node1743":{ - "core":"COLL_2_shard10_1_1_replica_n1742", - "shard":"shard10_1_1", - "collection":"COLL_2", - "node_name":"N_t_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":1.30757016285E11, - "base_url":"http://N_t/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":121.77696105558425}}], - "shard10_0_1":[{"core_node905":{ - "core":"COLL_2_shard10_0_1_replica_n904", - "shard":"shard10_0_1", - "collection":"COLL_2", - "node_name":"N_t_solr", - "type":"NRT", - "base_url":"http://N_t/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.28142990156E11, - "INDEX.sizeInGB":119.34245951101184}}], - "shard10_0_0":[{"core_node1733":{ - "core":"COLL_2_shard10_0_0_replica_n1732", - "shard":"shard10_0_0", - "collection":"COLL_2", - "node_name":"N_t_solr", - "type":"NRT", - "base_url":"http://N_t/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.2914349283E11, - "INDEX.sizeInGB":120.27425023727119}}]}}}, - { - "node":"N_7_solr", - "isLive":true, - "cores":6.0, - "freedisk":4268.472709655762, - "sysprop.pool":"pool-01", - "sysprop.az":"us-east-1b", - "totaldisk":4998.009765625, - "replicas":{"COLL_2":{ - "shard13_1_1":[{"core_node808":{ - "core":"COLL_2_shard13_1_1_replica_n806", - "shard":"shard13_1_1", - "collection":"COLL_2", - "node_name":"N_7_solr", - "type":"NRT", - "base_url":"http://N_7/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.2961448776E11, - "INDEX.sizeInGB":120.71289844810963}}], - "shard15_0_1":[{"core_node610":{ - "core":"COLL_2_shard15_0_1_replica_n608", - "shard":"shard15_0_1", - "collection":"COLL_2", - "node_name":"N_7_solr", - "type":"NRT", - "base_url":"http://N_7/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.2722802278E11, - "INDEX.sizeInGB":118.49032973870635}}], - "shard15_0_0":[{"core_node609":{ - "core":"COLL_2_shard15_0_0_replica_n607", - "shard":"shard15_0_0", - "collection":"COLL_2", - "node_name":"N_7_solr", - "type":"NRT", - "base_url":"http://N_7/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.27258670055E11, - "INDEX.sizeInGB":118.5188722377643}}], - "shard13_0_1":[{"core_node1767":{ - "core":"COLL_2_shard13_0_1_replica_n1766", - "shard":"shard13_0_1", - "collection":"COLL_2", - "node_name":"N_7_solr", - "type":"NRT", - "base_url":"http://N_7/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.30339106107E11, - "INDEX.sizeInGB":121.38775187265128}}], - "shard13_1_0":[{"core_node1689":{ - "core":"COLL_2_shard13_1_0_replica_n1688", - "shard":"shard13_1_0", - "collection":"COLL_2", - "node_name":"N_7_solr", - "type":"NRT", - "base_url":"http://N_7/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.29592823396E11, - "INDEX.sizeInGB":120.69272193685174}}], - "shard13_0_0":[{"core_node1713":{ - "core":"COLL_2_shard13_0_0_replica_n1712", - "shard":"shard13_0_0", - "collection":"COLL_2", - "node_name":"N_7_solr", - "type":"NRT", - "base_url":"http://N_7/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.30437704659E11, - "INDEX.sizeInGB":121.47957892995328}}]}}}, - { - "node":"N_6c_solr", - "isLive":true, - "cores":6.0, - "freedisk":4269.135753631592, - "sysprop.pool":"pool-01", - "sysprop.az":"us-east-1a", - "totaldisk":4998.009765625, - "replicas":{"COLL_2":{ - "shard17_0_1":[{"core_node848":{ - "core":"COLL_2_shard17_0_1_replica_n843", - "shard":"shard17_0_1", - "collection":"COLL_2", - "node_name":"N_6c_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":1.30730929322E11, - "base_url":"http://N_6c/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":121.7526656780392}}], - "shard17_0_0":[{"core_node844":{ - "core":"COLL_2_shard17_0_0_replica_n842", - "shard":"shard17_0_0", - "collection":"COLL_2", - "node_name":"N_6c_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":1.30743109221E11, - "base_url":"http://N_6c/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":121.76400909293443}}], - "shard4_0_0":[{"core_node445":{ - "core":"COLL_2_shard4_0_0_replica_n443", - "shard":"shard4_0_0", - "collection":"COLL_2", - "node_name":"N_6c_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":1.28741762257E11, - "base_url":"http://N_6c/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":119.90010948572308}}], - "shard4_1_0":[{"core_node457":{ - "core":"COLL_2_shard4_1_0_replica_n455", - "shard":"shard4_1_0", - "collection":"COLL_2", - "node_name":"N_6c_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":1.27664473589E11, - "base_url":"http://N_6c/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":118.89680622983724}}], - "shard4_0_1":[{"core_node446":{ - "core":"COLL_2_shard4_0_1_replica_n444", - "shard":"shard4_0_1", - "collection":"COLL_2", - "node_name":"N_6c_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":1.28032413116E11, - "base_url":"http://N_6c/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":119.23947661742568}}], - "shard4_1_1":[{"core_node458":{ - "core":"COLL_2_shard4_1_1_replica_n456", - "shard":"shard4_1_1", - "collection":"COLL_2", - "node_name":"N_6c_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":1.27865802727E11, - "base_url":"http://N_6c/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":119.08430860098451}}]}}}, - { - "node":"N_6i_solr", - "isLive":true, - "cores":6.0, - "freedisk":4269.712917327881, - "sysprop.pool":"pool-01", - "sysprop.az":"us-east-1b", - "totaldisk":4998.009765625, - "replicas":{"COLL_2":{ - "shard10_1_1":[{"core_node840":{ - "core":"COLL_2_shard10_1_1_replica_n830", - "shard":"shard10_1_1", - "collection":"COLL_2", - "node_name":"N_6i_solr", - "type":"NRT", - "base_url":"http://N_6i/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.30273229534E11, - "INDEX.sizeInGB":121.32639953307807}}], - "shard10_1_0":[{"core_node831":{ - "core":"COLL_2_shard10_1_0_replica_n829", - "shard":"shard10_1_0", - "collection":"COLL_2", - "node_name":"N_6i_solr", - "type":"NRT", - "base_url":"http://N_6i/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.27564995026E11, - "INDEX.sizeInGB":118.80415959842503}}], - "shard10_0_1":[{"core_node1739":{ - "core":"COLL_2_shard10_0_1_replica_n1738", - "shard":"shard10_0_1", - "collection":"COLL_2", - "node_name":"N_6i_solr", - "type":"NRT", - "base_url":"http://N_6i/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.28024871739E11, - "INDEX.sizeInGB":119.2324531627819}}], - "shard2_1_0":[{"core_node1727":{ - "core":"COLL_2_shard2_1_0_replica_n1726", - "shard":"shard2_1_0", - "collection":"COLL_2", - "node_name":"N_6i_solr", - "type":"NRT", - "base_url":"http://N_6i/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.30025926492E11, - "INDEX.sizeInGB":121.0960806272924}}], - "shard10_0_0":[{"core_node897":{ - "core":"COLL_2_shard10_0_0_replica_n896", - "shard":"shard10_0_0", - "collection":"COLL_2", - "node_name":"N_6i_solr", - "type":"NRT", - "base_url":"http://N_6i/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.29103730913E11, - "INDEX.sizeInGB":120.2372190663591}}], - "shard2_1_1":[{"core_node979":{ - "core":"COLL_2_shard2_1_1_replica_n978", - "shard":"shard2_1_1", - "collection":"COLL_2", - "node_name":"N_6i_solr", - "type":"NRT", - "base_url":"http://N_6i/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.2815510735E11, - "INDEX.sizeInGB":119.35374452732503}}]}}}, - { - "node":"N_3_solr", - "isLive":true, - "cores":6.0, - "freedisk":4272.45711517334, - "sysprop.pool":"pool-01", - "sysprop.az":"us-east-1c", - "totaldisk":4998.009765625, - "replicas":{"COLL_2":{ - "shard16_1_1":[{"core_node997":{ - "core":"COLL_2_shard16_1_1_replica_n996", - "shard":"shard16_1_1", - "collection":"COLL_2", - "node_name":"N_3_solr", - "type":"NRT", - "base_url":"http://N_3/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.26611980672E11, - "INDEX.sizeInGB":117.91659581661224}}], - "shard16_1_0":[{"core_node991":{ - "core":"COLL_2_shard16_1_0_replica_n990", - "shard":"shard16_1_0", - "collection":"COLL_2", - "node_name":"N_3_solr", - "type":"NRT", - "base_url":"http://N_3/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.28724323652E11, - "INDEX.sizeInGB":119.88386851921678}}], - "shard1_1_1":[{"core_node474":{ - "core":"COLL_2_shard1_1_1_replica_n2", - "shard":"shard1_1_1", - "collection":"COLL_2", - "node_name":"N_3_solr", - "type":"NRT", - "base_url":"http://N_3/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.29556889925E11, - "INDEX.sizeInGB":120.65925628412515}}], - "shard4_0_0":[{"core_node1737":{ - "core":"COLL_2_shard4_0_0_replica_n1736", - "shard":"shard4_0_0", - "collection":"COLL_2", - "node_name":"N_3_solr", - "type":"NRT", - "base_url":"http://N_3/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.28645187639E11, - "INDEX.sizeInGB":119.81016736384481}}], - "shard4_1_0":[{"core_node523":{ - "core":"COLL_2_shard4_1_0_replica_n1", - "shard":"shard4_1_0", - "collection":"COLL_2", - "node_name":"N_3_solr", - "type":"NRT", - "base_url":"http://N_3/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.27649471364E11, - "INDEX.sizeInGB":118.88283431902528}}], - "shard9_0_0":[{"core_node1815":{ - "core":"COLL_2_shard9_0_0_replica_n1814", - "shard":"shard9_0_0", - "collection":"COLL_2", - "node_name":"N_3_solr", - "type":"NRT", - "base_url":"http://N_3/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.29037175651E11, - "INDEX.sizeInGB":120.17523464839906}}]}}}, - { - "node":"N_1d_solr", - "isLive":true, - "cores":6.0, - "freedisk":4273.009799957275, - "sysprop.pool":"pool-01", - "sysprop.az":"us-east-1a", - "totaldisk":4998.009765625, - "replicas":{"COLL_2":{ - "shard3_1_0":[{"core_node425":{ - "core":"COLL_2_shard3_1_0_replica_n423", - "shard":"shard3_1_0", - "collection":"COLL_2", - "node_name":"N_1d_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":1.2828759808E11, - "base_url":"http://N_1d/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":119.47713613510132}}], - "shard3_1_1":[{"core_node426":{ - "core":"COLL_2_shard3_1_1_replica_n424", - "shard":"shard3_1_1", - "collection":"COLL_2", - "node_name":"N_1d_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":1.29948029547E11, - "base_url":"http://N_1d/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":121.02353344392031}}], - "shard15_0_0":[{"core_node732":{ - "core":"COLL_2_shard15_0_0_replica_n2", - "shard":"shard15_0_0", - "collection":"COLL_2", - "node_name":"N_1d_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":1.27262832088E11, - "base_url":"http://N_1d/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":118.5227484330535}}], - "shard12_1_0":[{"core_node1789":{ - "core":"COLL_2_shard12_1_0_replica_n1788", - "shard":"shard12_1_0", - "collection":"COLL_2", - "node_name":"N_1d_solr", - "type":"NRT", - "base_url":"http://N_1d/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.27487519935E11, - "INDEX.sizeInGB":118.73200529720634}}], - "shard14_1_1":[{"core_node1741":{ - "core":"COLL_2_shard14_1_1_replica_n1740", - "shard":"shard14_1_1", - "collection":"COLL_2", - "node_name":"N_1d_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":1.29231781669E11, - "base_url":"http://N_1d/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":120.35647562611848}}], - "shard14_1_0":[{"core_node1129":{ - "core":"COLL_2_shard14_1_0_replica_n1128", - "shard":"shard14_1_0", - "collection":"COLL_2", - "node_name":"N_1d_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":1.27407685053E11, - "base_url":"http://N_1d/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":118.65765326935798}}]}}}, - { - "node":"N_1_solr", - "isLive":true, - "cores":6.0, - "freedisk":4274.765396118164, - "sysprop.pool":"pool-01", - "sysprop.az":"us-east-1a", - "totaldisk":4998.009765625, - "replicas":{"COLL_2":{ - "shard16_1_1":[{"core_node995":{ - "core":"COLL_2_shard16_1_1_replica_n994", - "shard":"shard16_1_1", - "collection":"COLL_2", - "node_name":"N_1_solr", - "type":"NRT", - "base_url":"http://N_1/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.26672765511E11, - "INDEX.sizeInGB":117.97320610936731}}], - "shard16_0_1":[{"core_node989":{ - "core":"COLL_2_shard16_0_1_replica_n988", - "shard":"shard16_0_1", - "collection":"COLL_2", - "node_name":"N_1_solr", - "type":"NRT", - "base_url":"http://N_1/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.3069803609E11, - "INDEX.sizeInGB":121.72203146852553}}], - "shard16_1_0":[{"core_node993":{ - "core":"COLL_2_shard16_1_0_replica_n992", - "shard":"shard16_1_0", - "collection":"COLL_2", - "node_name":"N_1_solr", - "type":"NRT", - "base_url":"http://N_1/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.28812502313E11, - "INDEX.sizeInGB":119.96599129680544}}], - "shard16_0_0":[{"core_node983":{ - "core":"COLL_2_shard16_0_0_replica_n982", - "shard":"shard16_0_0", - "collection":"COLL_2", - "node_name":"N_1_solr", - "type":"NRT", - "base_url":"http://N_1/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.26766519189E11, - "INDEX.sizeInGB":118.06052102614194}}], - "shard18_0_0":[{"core_node875":{ - "core":"COLL_2_shard18_0_0_replica_n2", - "shard":"shard18_0_0", - "collection":"COLL_2", - "node_name":"N_1_solr", - "type":"NRT", - "base_url":"http://N_1/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.28033512867E11, - "INDEX.sizeInGB":119.24050084035844}}], - "shard12_1_1":[{"core_node586":{ - "core":"COLL_2_shard12_1_1_replica_n584", - "shard":"shard12_1_1", - "collection":"COLL_2", - "node_name":"N_1_solr", - "type":"NRT", - "base_url":"http://N_1/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.2671712403E11, - "INDEX.sizeInGB":118.01451819948852}}]}}}, - { - "node":"N_aw_solr", - "isLive":true, - "cores":6.0, - "freedisk":4276.759601593018, - "sysprop.pool":"pool-01", - "sysprop.az":"us-east-1c", - "totaldisk":4998.009765625, - "replicas":{"COLL_2":{ - "shard18_1_1":[{"core_node1821":{ - "core":"COLL_2_shard18_1_1_replica_n1820", - "shard":"shard18_1_1", - "collection":"COLL_2", - "node_name":"N_aw_solr", - "type":"NRT", - "base_url":"http://N_aw/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.28188518759E11, - "INDEX.sizeInGB":119.38486132677644}}], - "shard4_1_1":[{"core_node525":{ - "core":"COLL_2_shard4_1_1_replica_n1", - "shard":"shard4_1_1", - "collection":"COLL_2", - "node_name":"N_aw_solr", - "type":"NRT", - "base_url":"http://N_aw/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.27899653279E11, - "INDEX.sizeInGB":119.11583438422531}}], - "shard3_1_0":[{"core_node460":{ - "core":"COLL_2_shard3_1_0_replica_n2", - "shard":"shard3_1_0", - "collection":"COLL_2", - "node_name":"N_aw_solr", - "type":"NRT", - "base_url":"http://N_aw/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.28273400877E11, - "INDEX.sizeInGB":119.46391395945102}}], - "shard15_0_1":[{"core_node1817":{ - "core":"COLL_2_shard15_0_1_replica_n1816", - "shard":"shard15_0_1", - "collection":"COLL_2", - "node_name":"N_aw_solr", - "type":"NRT", - "base_url":"http://N_aw/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.27129784031E11, - "INDEX.sizeInGB":118.39883777406067}}], - "shard12_1_1":[{"core_node661":{ - "core":"COLL_2_shard12_1_1_replica_n1", - "shard":"shard12_1_1", - "collection":"COLL_2", - "node_name":"N_aw_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":1.26701654869E11, - "base_url":"http://N_aw/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":118.00011142063886}}], - "shard12_1_0":[{"core_node659":{ - "core":"COLL_2_shard12_1_0_replica_n1", - "shard":"shard12_1_0", - "collection":"COLL_2", - "node_name":"N_aw_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":1.27434400341E11, - "base_url":"http://N_aw/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":118.68253382015973}}]}}}, - { - "node":"N_1h_solr", - "isLive":true, - "cores":6.0, - "freedisk":4297.329685211182, - "sysprop.pool":"pool-01", - "sysprop.az":"us-east-1b", - "totaldisk":4998.009765625, - "replicas":{"COLL_2":{ - "shard1_0_0":[{"core_node1729":{ - "core":"COLL_2_shard1_0_0_replica_n1728", - "shard":"shard1_0_0", - "collection":"COLL_2", - "node_name":"N_1h_solr", - "type":"NRT", - "base_url":"http://N_1h/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":5.7176945428E10, - "INDEX.sizeInGB":53.25018002465367}}], - "shard7_1_0":[{"core_node1145":{ - "core":"COLL_2_shard7_1_0_replica_n1144", - "shard":"shard7_1_0", - "collection":"COLL_2", - "node_name":"N_1h_solr", - "type":"NRT", - "base_url":"http://N_1h/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.2949609012E11, - "INDEX.sizeInGB":120.60263205319643}}], - "shard7_1_1":[{"core_node1701":{ - "core":"COLL_2_shard7_1_1_replica_n1700", - "shard":"shard7_1_1", - "collection":"COLL_2", - "node_name":"N_1h_solr", - "type":"NRT", - "base_url":"http://N_1h/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.28489170345E11, - "INDEX.sizeInGB":119.66486493591219}}], - "shard3_0_1":[{"core_node510":{ - "core":"COLL_2_shard3_0_1_replica_n508", - "shard":"shard3_0_1", - "collection":"COLL_2", - "node_name":"N_1h_solr", - "type":"NRT", - "base_url":"http://N_1h/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.31866901019E11, - "INDEX.sizeInGB":122.81062176357955}}], - "shard12_0_1":[{"core_node1761":{ - "core":"COLL_2_shard12_0_1_replica_n1760", - "shard":"shard12_0_1", - "collection":"COLL_2", - "node_name":"N_1h_solr", - "type":"NRT", - "base_url":"http://N_1h/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.30342308934E11, - "INDEX.sizeInGB":121.39073473773897}}], - "shard12_0_0":[{"core_node1697":{ - "core":"COLL_2_shard12_0_0_replica_n1696", - "shard":"shard12_0_0", - "collection":"COLL_2", - "node_name":"N_1h_solr", - "type":"NRT", - "base_url":"http://N_1h/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.29369271388E11, - "INDEX.sizeInGB":120.48452290520072}}]}}}, - { - "node":"N_29_solr", - "isLive":true, - "cores":6.0, - "freedisk":4303.548599243164, - "sysprop.pool":"pool-01", - "sysprop.az":"us-east-1a", - "totaldisk":4998.009765625, - "replicas":{"COLL_2":{ - "shard8_0_0":[{"core_node1691":{ - "core":"COLL_2_shard8_0_0_replica_n1690", - "shard":"shard8_0_0", - "collection":"COLL_2", - "node_name":"N_29_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":1.30176337999E11, - "base_url":"http://N_29/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":121.23616225924343}}], - "shard8_0_1":[{"core_node1787":{ - "core":"COLL_2_shard8_0_1_replica_n1786", - "shard":"shard8_0_1", - "collection":"COLL_2", - "node_name":"N_29_solr", - "type":"NRT", - "base_url":"http://N_29/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.32692723859E11, - "INDEX.sizeInGB":123.57972921710461}}], - "shard7_1_0":[{"core_node1143":{ - "core":"COLL_2_shard7_1_0_replica_n1142", - "shard":"shard7_1_0", - "collection":"COLL_2", - "node_name":"N_29_solr", - "type":"NRT", - "base_url":"http://N_29/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.2946739865E11, - "INDEX.sizeInGB":120.57591103948653}}], - "shard7_0_1":[{"core_node777":{ - "core":"COLL_2_shard7_0_1_replica_n2", - "shard":"shard7_0_1", - "collection":"COLL_2", - "node_name":"N_29_solr", - "type":"NRT", - "base_url":"http://N_29/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":8.6794048237E10, - "INDEX.sizeInGB":80.83325646538287}}], - "shard7_1_1":[{"core_node1759":{ - "core":"COLL_2_shard7_1_1_replica_n1758", - "shard":"shard7_1_1", - "collection":"COLL_2", - "node_name":"N_29_solr", - "type":"NRT", - "base_url":"http://N_29/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.28546712309E11, - "INDEX.sizeInGB":119.7184550659731}}], - "shard6_1_0":[{"core_node1793":{ - "core":"COLL_2_shard6_1_0_replica_n1792", - "shard":"shard6_1_0", - "collection":"COLL_2", - "node_name":"N_29_solr", - "type":"NRT", - "base_url":"http://N_29/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.29365181039E11, - "INDEX.sizeInGB":120.48071347083896}}]}}}, - { - "node":"N_e_solr", - "isLive":true, - "cores":6.0, - "freedisk":4334.874732971191, - "sysprop.pool":"pool-01", - "sysprop.az":"us-east-1c", - "totaldisk":4998.009765625, - "replicas":{"COLL_2":{ - "shard1_0_1":[{"core_node1719":{ - "core":"COLL_2_shard1_0_1_replica_n1718", - "shard":"shard1_0_1", - "collection":"COLL_2", - "node_name":"N_e_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":5.9506746089E10, - "base_url":"http://N_e/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":55.41997597459704}}], - "shard18_0_0":[{"core_node1819":{ - "core":"COLL_2_shard18_0_0_replica_n1818", - "shard":"shard18_0_0", - "collection":"COLL_2", - "node_name":"N_e_solr", - "type":"NRT", - "base_url":"http://N_e/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.28218931509E11, - "INDEX.sizeInGB":119.41318540740758}}], - "shard13_1_1":[{"core_node925":{ - "core":"COLL_2_shard13_1_1_replica_n924", - "shard":"shard13_1_1", - "collection":"COLL_2", - "node_name":"N_e_solr", - "type":"NRT", - "base_url":"http://N_e/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.29598508564E11, - "INDEX.sizeInGB":120.69801666215062}}], - "shard18_1_0":[{"core_node672":{ - "core":"COLL_2_shard18_1_0_replica_n2", - "shard":"shard18_1_0", - "collection":"COLL_2", - "node_name":"N_e_solr", - "type":"NRT", - "base_url":"http://N_e/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.29108586002E11, - "INDEX.sizeInGB":120.24174072034657}}], - "shard15_0_0":[{"core_node731":{ - "core":"COLL_2_shard15_0_0_replica_n1", - "shard":"shard15_0_0", - "collection":"COLL_2", - "node_name":"N_e_solr", - "type":"NRT", - "base_url":"http://N_e/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.27235871561E11, - "INDEX.sizeInGB":118.49763948563486}}], - "shard13_1_0":[{"core_node923":{ - "core":"COLL_2_shard13_1_0_replica_n922", - "shard":"shard13_1_0", - "collection":"COLL_2", - "node_name":"N_e_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":1.29514183189E11, - "base_url":"http://N_e/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":120.6194825368002}}]}}}, - { - "node":"N_2w_solr", - "isLive":true, - "cores":6.0, - "freedisk":4336.208312988281, - "sysprop.pool":"pool-01", - "sysprop.az":"us-east-1b", - "totaldisk":4998.009765625, - "replicas":{"COLL_2":{ - "shard1_0_1":[{"core_node1677":{ - "core":"COLL_2_shard1_0_1_replica_n1676", - "shard":"shard1_0_1", - "collection":"COLL_2", - "node_name":"N_2w_solr", - "type":"NRT", - "base_url":"http://N_2w/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":5.9557275352E10, - "INDEX.sizeInGB":55.46703501790762}}], - "shard1_1_1":[{"core_node1807":{ - "core":"COLL_2_shard1_1_1_replica_n1806", - "shard":"shard1_1_1", - "collection":"COLL_2", - "node_name":"N_2w_solr", - "type":"NRT", - "base_url":"http://N_2w/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.2954748046E11, - "INDEX.sizeInGB":120.6504930369556}}], - "shard4_1_0":[{"core_node1775":{ - "core":"COLL_2_shard4_1_0_replica_n1774", - "shard":"shard4_1_0", - "collection":"COLL_2", - "node_name":"N_2w_solr", - "type":"NRT", - "base_url":"http://N_2w/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.27659935903E11, - "INDEX.sizeInGB":118.89258018042892}}], - "shard18_1_1":[{"core_node673":{ - "core":"COLL_2_shard18_1_1_replica_n1", - "shard":"shard18_1_1", - "collection":"COLL_2", - "node_name":"N_2w_solr", - "type":"NRT", - "base_url":"http://N_2w/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.28226679933E11, - "INDEX.sizeInGB":119.42040168959647}}], - "shard4_1_1":[{"core_node1805":{ - "core":"COLL_2_shard4_1_1_replica_n1804", - "shard":"shard4_1_1", - "collection":"COLL_2", - "node_name":"N_2w_solr", - "type":"NRT", - "base_url":"http://N_2w/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.27878088796E11, - "INDEX.sizeInGB":119.0957508943975}}], - "shard18_1_0":[{"core_node671":{ - "core":"COLL_2_shard18_1_0_replica_n1", - "shard":"shard18_1_0", - "collection":"COLL_2", - "node_name":"N_2w_solr", - "type":"NRT", - "base_url":"http://N_2w/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.28884297502E11, - "INDEX.sizeInGB":120.03285577706993}}]}}}, - { - "node":"N_5_solr", - "isLive":true, - "cores":6.0, - "freedisk":4397.149795532227, - "sysprop.pool":"pool-01", - "sysprop.az":"us-east-1a", - "totaldisk":4998.009765625, - "replicas":{"COLL_2":{ - "shard1_1_0":[{"core_node1721":{ - "core":"COLL_2_shard1_1_0_replica_n1720", - "shard":"shard1_1_0", - "collection":"COLL_2", - "node_name":"N_5_solr", - "type":"NRT", - "base_url":"http://N_5/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.29009851855E11, - "INDEX.sizeInGB":120.14978738036007}}], - "shard1_0_1":[{"core_node1669":{ - "core":"COLL_2_shard1_0_1_replica_n1668", - "shard":"shard1_0_1", - "collection":"COLL_2", - "node_name":"N_5_solr", - "type":"NRT", - "base_url":"http://N_5/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":5.9574276743E10, - "INDEX.sizeInGB":55.482868797145784}}], - "shard1_1_1":[{"core_node418":{ - "core":"COLL_2_shard1_1_1_replica_n416", - "shard":"shard1_1_1", - "collection":"COLL_2", - "node_name":"N_5_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":1.29698716918E11, - "base_url":"http://N_5/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":120.79134296439588}}], - "shard2_0_0":[{"core_node911":{ - "core":"COLL_2_shard2_0_0_replica_n910", - "shard":"shard2_0_0", - "collection":"COLL_2", - "node_name":"N_5_solr", - "type":"NRT", - "base_url":"http://N_5/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.29504451209E11, - "INDEX.sizeInGB":120.6104189241305}}], - "shard2_0_1":[{"core_node917":{ - "core":"COLL_2_shard2_0_1_replica_n916", - "shard":"shard2_0_1", - "collection":"COLL_2", - "node_name":"N_5_solr", - "type":"NRT", - "base_url":"http://N_5/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.31334463143E11, - "INDEX.sizeInGB":122.31475035008043}}], - "shard1_0_0":[{"core_node1725":{ - "core":"COLL_2_shard1_0_0_replica_n1724", - "shard":"shard1_0_0", - "collection":"COLL_2", - "node_name":"N_5_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":5.7183711221E10, - "base_url":"http://N_5/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":53.25648116040975}}]}}}, - { - "node":"N_do_solr", - "isLive":true, - "cores":5.0, - "freedisk":407.25314712524414, - "sysprop.pool":"pool-02", - "sysprop.az":"us-east-1c", - "totaldisk":999.51171875, - "replicas":{ - "COLL_1":{ - "shard3_0_0":[{"core_node112":{ - "core":"COLL_1_shard3_0_0_replica_n111", - "shard":"shard3_0_0", - "collection":"COLL_1", - "node_name":"N_do_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":4.4957115524E10, - "base_url":"http://N_do/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":41.86957657709718}}], - "shard3_1_0":[{"core_node116":{ - "core":"COLL_1_shard3_1_0_replica_n115", - "shard":"shard3_1_0", - "collection":"COLL_1", - "node_name":"N_do_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":4.3732753925E10, - "base_url":"http://N_do/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":40.72930098045617}}], - "shard3_0_1":[{"core_node114":{ - "core":"COLL_1_shard3_0_1_replica_n113", - "shard":"shard3_0_1", - "collection":"COLL_1", - "node_name":"N_do_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":4.577095697E10, - "base_url":"http://N_do/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":42.62752548791468}}], - "shard3_1_1":[{"core_node118":{ - "core":"COLL_1_shard3_1_1_replica_n117", - "shard":"shard3_1_1", - "collection":"COLL_1", - "node_name":"N_do_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":4.8532509927E10, - "base_url":"http://N_do/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":45.19942209776491}}]}, - "COLL_0":{"shard3":[{"core_node15":{ - "core":"COLL_0_shard3_replica_n12", - "shard":"shard3", - "collection":"COLL_0", - "node_name":"N_do_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":3.1297025422E10, - "base_url":"http://N_do/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":29.147626293823123}}]}}}, - { - "node":"N_3a_solr", - "isLive":true, - "cores":5.0, - "freedisk":407.706729888916, - "sysprop.pool":"pool-02", - "sysprop.az":"us-east-1b", - "totaldisk":999.51171875, - "replicas":{ - "COLL_1":{ - "shard3_0_0":[{"core_node73":{ - "core":"COLL_1_shard3_0_0_replica_n71", - "shard":"shard3_0_0", - "collection":"COLL_1", - "node_name":"N_3a_solr", - "type":"NRT", - "base_url":"http://N_3a/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":4.5160600486E10, - "INDEX.sizeInGB":42.05908671580255}}], - "shard3_1_0":[{"core_node77":{ - "core":"COLL_1_shard3_1_0_replica_n75", - "shard":"shard3_1_0", - "collection":"COLL_1", - "node_name":"N_3a_solr", - "type":"NRT", - "base_url":"http://N_3a/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":4.5090380622E10, - "INDEX.sizeInGB":41.99368937127292}}], - "shard3_0_1":[{"core_node74":{ - "core":"COLL_1_shard3_0_1_replica_n72", - "shard":"shard3_0_1", - "collection":"COLL_1", - "node_name":"N_3a_solr", - "type":"NRT", - "base_url":"http://N_3a/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":4.5879426317E10, - "INDEX.sizeInGB":42.72854543942958}}], - "shard3_1_1":[{"core_node78":{ - "core":"COLL_1_shard3_1_1_replica_n76", - "shard":"shard3_1_1", - "collection":"COLL_1", - "node_name":"N_3a_solr", - "type":"NRT", - "base_url":"http://N_3a/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":4.6849085882E10, - "INDEX.sizeInGB":43.631611282005906}}]}, - "COLL_0":{"shard3":[{"core_node17":{ - "core":"COLL_0_shard3_replica_n14", - "shard":"shard3", - "collection":"COLL_0", - "node_name":"N_3a_solr", - "type":"NRT", - "base_url":"http://N_3a/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":3.0819950704E10, - "INDEX.sizeInGB":28.70331583917141}}]}}}, - { - "node":"N_v_solr", - "isLive":true, - "cores":5.0, - "freedisk":412.18456649780273, - "sysprop.pool":"pool-02", - "sysprop.az":"us-east-1a", - "totaldisk":999.51171875, - "replicas":{ - "COLL_1":{ - "shard3_0_0":[{"core_node120":{ - "core":"COLL_1_shard3_0_0_replica_n119", - "shard":"shard3_0_0", - "collection":"COLL_1", - "node_name":"N_v_solr", - "type":"NRT", - "base_url":"http://N_v/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":4.3809517838E10, - "INDEX.sizeInGB":40.80079294554889}}], - "shard3_1_0":[{"core_node124":{ - "core":"COLL_1_shard3_1_0_replica_n123", - "shard":"shard3_1_0", - "collection":"COLL_1", - "node_name":"N_v_solr", - "type":"NRT", - "base_url":"http://N_v/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":4.5638162031E10, - "INDEX.sizeInGB":42.503850563429296}}], - "shard3_0_1":[{"core_node122":{ - "core":"COLL_1_shard3_0_1_replica_n121", - "shard":"shard3_0_1", - "collection":"COLL_1", - "node_name":"N_v_solr", - "type":"NRT", - "base_url":"http://N_v/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":4.6310602091E10, - "INDEX.sizeInGB":43.13010917138308}}], - "shard3_1_1":[{"core_node126":{ - "core":"COLL_1_shard3_1_1_replica_n125", - "shard":"shard3_1_1", - "collection":"COLL_1", - "node_name":"N_v_solr", - "type":"NRT", - "base_url":"http://N_v/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":4.4257494507E10, - "INDEX.sizeInGB":41.21800373028964}}]}, - "COLL_0":{"shard3":[{"core_node18":{ - "core":"COLL_0_shard3_replica_n16", - "shard":"shard3", - "collection":"COLL_0", - "node_name":"N_v_solr", - "type":"NRT", - "base_url":"http://N_v/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":2.8932093807E10, - "INDEX.sizeInGB":26.94511209335178}}]}}}, - { - "node":"N_13_solr", - "isLive":true, - "cores":5.0, - "freedisk":718.1634063720703, - "sysprop.pool":"pool-02", - "sysprop.az":"us-east-1c", - "totaldisk":999.51171875, - "replicas":{ - "COLL_1":{ - "shard1_1_0":[{"core_node61":{ - "core":"COLL_1_shard1_1_0_replica_n59", - "shard":"shard1_1_0", - "collection":"COLL_1", - "node_name":"N_13_solr", - "type":"NRT", - "base_url":"http://N_13/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":4.3783419579E10, - "INDEX.sizeInGB":40.77648704778403}}], - "shard1_0_1":[{"core_node58":{ - "core":"COLL_1_shard1_0_1_replica_n56", - "shard":"shard1_0_1", - "collection":"COLL_1", - "node_name":"N_13_solr", - "type":"NRT", - "base_url":"http://N_13/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":4.4932001726E10, - "INDEX.sizeInGB":41.846187530085444}}], - "shard1_1_1":[{"core_node62":{ - "core":"COLL_1_shard1_1_1_replica_n60", - "shard":"shard1_1_1", - "collection":"COLL_1", - "node_name":"N_13_solr", - "type":"NRT", - "base_url":"http://N_13/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":4.811959042E10, - "INDEX.sizeInGB":44.814860839396715}}], - "shard1_0_0":[{"core_node57":{ - "core":"COLL_1_shard1_0_0_replica_n55", - "shard":"shard1_0_0", - "collection":"COLL_1", - "node_name":"N_13_solr", - "type":"NRT", - "base_url":"http://N_13/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":4.5921892273E10, - "INDEX.sizeInGB":42.76809494290501}}]}, - "COLL_0":{"shard2":[{"core_node13":{ - "core":"COLL_0_shard2_replica_n10", - "shard":"shard2", - "collection":"COLL_0", - "node_name":"N_13_solr", - "type":"NRT", - "base_url":"http://N_13/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":3.4248182159E10, - "INDEX.sizeInGB":31.896105184219778}}]}}}, - { - "node":"N_3to_solr", - "isLive":true, - "cores":5.0, - "freedisk":794.5433731079102, - "sysprop.pool":"pool-02", - "sysprop.az":"us-east-1a", - "totaldisk":999.51171875, - "replicas":{ - "COLL_1":{ - "shard1_1_0":[{"core_node84":{ - "core":"COLL_1_shard1_1_0_replica_n83", - "shard":"shard1_1_0", - "collection":"COLL_1", - "node_name":"N_3to_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":4.3892348528E10, - "base_url":"http://N_3to/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":40.87793503701687}}], - "shard1_0_1":[{"core_node82":{ - "core":"COLL_1_shard1_0_1_replica_n81", - "shard":"shard1_0_1", - "collection":"COLL_1", - "node_name":"N_3to_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":4.4936912617E10, - "base_url":"http://N_3to/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":41.85076115373522}}], - "shard1_1_1":[{"core_node86":{ - "core":"COLL_1_shard1_1_1_replica_n85", - "shard":"shard1_1_1", - "collection":"COLL_1", - "node_name":"N_3to_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":5.1015133973E10, - "base_url":"http://N_3to/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":47.511545916087925}}], - "shard1_0_0":[{"core_node80":{ - "core":"COLL_1_shard1_0_0_replica_n79", - "shard":"shard1_0_0", - "collection":"COLL_1", - "node_name":"N_3to_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":4.644843302E10, - "base_url":"http://N_3to/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":43.258474227041006}}]}, - "COLL_0":{"shard2":[{"core_node11":{ - "core":"COLL_0_shard2_replica_n8", - "shard":"shard2", - "collection":"COLL_0", - "node_name":"N_3to_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":3.0722710385E10, - "base_url":"http://N_3to/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":28.6127537349239}}]}}}, - { - "node":"N_16_solr", - "isLive":true, - "cores":5.0, - "freedisk":795.7872657775879, - "sysprop.pool":"pool-02", - "sysprop.az":"us-east-1b", - "totaldisk":999.51171875, - "replicas":{ - "COLL_1":{ - "shard2_0_0":[{"core_node100":{ - "core":"COLL_1_shard2_0_0_replica_n99", - "shard":"shard2_0_0", - "collection":"COLL_1", - "node_name":"N_16_solr", - "type":"NRT", - "base_url":"http://N_16/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":4.8764329025E10, - "INDEX.sizeInGB":45.41532045695931}}], - "shard2_0_1":[{"core_node102":{ - "core":"COLL_1_shard2_0_1_replica_n101", - "shard":"shard2_0_1", - "collection":"COLL_1", - "node_name":"N_16_solr", - "type":"NRT", - "base_url":"http://N_16/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":4.3740343099E10, - "INDEX.sizeInGB":40.73636894952506}}], - "shard2_1_0":[{"core_node96":{ - "core":"COLL_1_shard2_1_0_replica_n95", - "shard":"shard2_1_0", - "collection":"COLL_1", - "node_name":"N_16_solr", - "type":"NRT", - "base_url":"http://N_16/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":4.5585236311E10, - "INDEX.sizeInGB":42.45455964561552}}], - "shard2_1_1":[{"core_node98":{ - "core":"COLL_1_shard2_1_1_replica_n97", - "shard":"shard2_1_1", - "collection":"COLL_1", - "node_name":"N_16_solr", - "type":"NRT", - "base_url":"http://N_16/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":4.527594328E10, - "INDEX.sizeInGB":42.16650806367397}}]}, - "COLL_0":{"shard1":[{"core_node5":{ - "core":"COLL_0_shard1_replica_n2", - "shard":"shard1", - "collection":"COLL_0", - "node_name":"N_16_solr", - "type":"NRT", - "base_url":"http://N_16/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":3.3775978753E10, - "INDEX.sizeInGB":31.45633149240166}}]}}}, - { - "node":"N_d4_solr", - "isLive":true, - "cores":5.0, - "freedisk":797.2159843444824, - "sysprop.pool":"pool-02", - "sysprop.az":"us-east-1c", - "totaldisk":999.51171875, - "replicas":{ - "COLL_1":{ - "shard2_0_0":[{"core_node69":{ - "core":"COLL_1_shard2_0_0_replica_n67", - "shard":"shard2_0_0", - "collection":"COLL_1", - "node_name":"N_d4_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":4.497304707E10, - "base_url":"http://N_d4/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":41.8844139855355}}], - "shard2_0_1":[{"core_node70":{ - "core":"COLL_1_shard2_0_1_replica_n68", - "shard":"shard2_0_1", - "collection":"COLL_1", - "node_name":"N_d4_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":4.5692831033E10, - "base_url":"http://N_d4/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":42.554765039123595}}], - "shard2_1_0":[{"core_node65":{ - "core":"COLL_1_shard2_1_0_replica_n63", - "shard":"shard2_1_0", - "collection":"COLL_1", - "node_name":"N_d4_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":4.5935880044E10, - "base_url":"http://N_d4/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":42.78112206980586}}], - "shard2_1_1":[{"core_node66":{ - "core":"COLL_1_shard2_1_1_replica_n64", - "shard":"shard2_1_1", - "collection":"COLL_1", - "node_name":"N_d4_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":4.5166045429E10, - "base_url":"http://N_d4/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":42.064157714135945}}]}, - "COLL_0":{"shard1":[{"core_node3":{ - "core":"COLL_0_shard1_replica_n1", - "shard":"shard1", - "collection":"COLL_0", - "node_name":"N_d4_solr", - "type":"NRT", - "leader":"true", - "INDEX.sizeInBytes":3.401835331E10, - "base_url":"http://N_d4/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":31.682060388848186}}]}}}, - { - "node":"N_b9_solr", - "isLive":true, - "cores":5.0, - "freedisk":801.2417984008789, - "sysprop.pool":"pool-02", - "sysprop.az":"us-east-1b", - "totaldisk":999.51171875, - "replicas":{ - "COLL_1":{ - "shard1_1_0":[{"core_node92":{ - "core":"COLL_1_shard1_1_0_replica_n91", - "shard":"shard1_1_0", - "collection":"COLL_1", - "node_name":"N_b9_solr", - "type":"NRT", - "base_url":"http://N_b9/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":4.5724314347E10, - "INDEX.sizeInGB":42.5840861601755}}], - "shard1_0_1":[{"core_node90":{ - "core":"COLL_1_shard1_0_1_replica_n89", - "shard":"shard1_0_1", - "collection":"COLL_1", - "node_name":"N_b9_solr", - "type":"NRT", - "base_url":"http://N_b9/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":4.6030616744E10, - "INDEX.sizeInGB":42.869352497160435}}], - "shard1_1_1":[{"core_node94":{ - "core":"COLL_1_shard1_1_1_replica_n93", - "shard":"shard1_1_1", - "collection":"COLL_1", - "node_name":"N_b9_solr", - "type":"NRT", - "base_url":"http://N_b9/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":4.574559386E10, - "INDEX.sizeInGB":42.603904251009226}}], - "shard1_0_0":[{"core_node88":{ - "core":"COLL_1_shard1_0_0_replica_n87", - "shard":"shard1_0_0", - "collection":"COLL_1", - "node_name":"N_b9_solr", - "type":"NRT", - "base_url":"http://N_b9/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":4.5100613575E10, - "INDEX.sizeInGB":42.0032195514068}}]}, - "COLL_0":{"shard2":[{"core_node9":{ - "core":"COLL_0_shard2_replica_n6", - "shard":"shard2", - "collection":"COLL_0", - "node_name":"N_b9_solr", - "type":"NRT", - "base_url":"http://N_b9/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":2.8865621899E10, - "INDEX.sizeInGB":26.883205304853618}}]}}}, - { - "node":"N_74_solr", - "isLive":true, - "cores":5.0, - "freedisk":802.5921897888184, - "sysprop.pool":"pool-02", - "sysprop.az":"us-east-1a", - "totaldisk":999.51171875, - "replicas":{ - "COLL_1":{ - "shard2_0_0":[{"core_node108":{ - "core":"COLL_1_shard2_0_0_replica_n107", - "shard":"shard2_0_0", - "collection":"COLL_1", - "node_name":"N_74_solr", - "type":"NRT", - "base_url":"http://N_74/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":4.3767024396E10, - "INDEX.sizeInGB":40.76121784374118}}], - "shard2_0_1":[{"core_node110":{ - "core":"COLL_1_shard2_0_1_replica_n109", - "shard":"shard2_0_1", - "collection":"COLL_1", - "node_name":"N_74_solr", - "type":"NRT", - "base_url":"http://N_74/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":4.8622428842E10, - "INDEX.sizeInGB":45.28316561318934}}], - "shard2_1_0":[{"core_node104":{ - "core":"COLL_1_shard2_1_0_replica_n103", - "shard":"shard2_1_0", - "collection":"COLL_1", - "node_name":"N_74_solr", - "type":"NRT", - "base_url":"http://N_74/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":4.4599223614E10, - "INDEX.sizeInGB":41.536263762041926}}], - "shard2_1_1":[{"core_node106":{ - "core":"COLL_1_shard2_1_1_replica_n105", - "shard":"shard2_1_1", - "collection":"COLL_1", - "node_name":"N_74_solr", - "type":"NRT", - "base_url":"http://N_74/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":4.3768191618E10, - "INDEX.sizeInGB":40.762304903939366}}]}, - "COLL_0":{"shard1":[{"core_node7":{ - "core":"COLL_0_shard1_replica_n4", - "shard":"shard1", - "collection":"COLL_0", - "node_name":"N_74_solr", - "type":"NRT", - "base_url":"http://N_74/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":2.9252853492E10, - "INDEX.sizeInGB":27.24384282901883}}]}}}], - "liveNodes":[ - "N_7e_solr", - "N_dj_solr", - "N_b9_solr", - "N_m_solr", - "N_1i_solr", - "N_17_solr", - "N_1_solr", - "N_g_solr", - "N_e_solr", - "N_4_solr", - "N_a_solr", - "N_16_solr", - "N_2w_solr", - "N_13_solr", - "N_2_solr", - "N_1m_solr", - "N_5_solr", - "N_do_solr", - "N_3a_solr", - "N_6i_solr", - "N_cs_solr", - "N_1f_solr", - "N_65p_solr", - "N_1c_solr", - "N_1d_solr", - "N_d4_solr", - "N_2u_solr", - "N_3to_solr", - "N_v_solr", - "N_3a7_solr", - "N_74_solr", - "N_t_solr", - "N_9o_solr", - "N_11_solr", - "N_0_solr", - "N_8_solr", - "N_7_solr", - "N_303_solr", - "N_6_solr", - "N_29_solr", - "N_3_solr", - "N_1h_solr", - "N_aw_solr", - "N_6c_solr", - "N_z_solr", - "N_4f_solr", - "N_4g_solr", - "N_u_solr"], - "violations":[], - "config":{ - "cluster-preferences":[ - { - "minimize":"cores", - "precision":1}, - { - "maximize":"freedisk", - "precision":10}], - "cluster-policy":[ - { - "replica":"#ALL", - "collection":"COLL_2", - "sysprop.pool":"pool-01"}, - { - "replica":"#ALL", - "collection":"COLL_1", - "sysprop.pool":"pool-02"}, - { - "replica":"#ALL", - "collection":"COLL_0", - "sysprop.pool":"pool-02"}, - { - "replica":"<2", - "shard":"#EACH", - "node":"#ANY"}, - { - "replica":"#EQUAL", - "shard":"#EACH", - "sysprop.az":"#EACH"}]}}} \ No newline at end of file diff --git a/solr/core/src/test-files/solr/simSnapshot/clusterState.json b/solr/core/src/test-files/solr/simSnapshot/clusterState.json deleted file mode 100644 index acaa3409a79..00000000000 --- a/solr/core/src/test-files/solr/simSnapshot/clusterState.json +++ /dev/null @@ -1,2838 +0,0 @@ -{ - "clusterProperties":{}, - "liveNodes":[ - "N_7e_solr", - "N_dj_solr", - "N_b9_solr", - "N_m_solr", - "N_1i_solr", - "N_17_solr", - "N_1_solr", - "N_g_solr", - "N_e_solr", - "N_4_solr", - "N_a_solr", - "N_16_solr", - "N_2w_solr", - "N_13_solr", - "N_2_solr", - "N_1m_solr", - "N_5_solr", - "N_do_solr", - "N_3a_solr", - "N_6i_solr", - "N_cs_solr", - "N_1f_solr", - "N_65p_solr", - "N_1c_solr", - "N_1d_solr", - "N_d4_solr", - "N_2u_solr", - "N_3to_solr", - "N_v_solr", - "N_3a7_solr", - "N_74_solr", - "N_t_solr", - "N_9o_solr", - "N_11_solr", - "N_0_solr", - "N_8_solr", - "N_7_solr", - "N_303_solr", - "N_6_solr", - "N_29_solr", - "N_3_solr", - "N_1h_solr", - "N_aw_solr", - "N_6c_solr", - "N_z_solr", - "N_4f_solr", - "N_4g_solr", - "N_u_solr"], - "clusterState":{ - "COLL_1t":{ - "pullReplicas":"0", - "replicationFactor":"3", - "shards":{"shard1":{ - "range":"80000000-7fffffff", - "state":"active", - "replicas":{ - "core_node3":{ - "core":"COLL_1t_shard1_replica_n1", - "base_url":"http://N_7e/solr", - "node_name":"N_7e_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node5":{ - "core":"COLL_1t_shard1_replica_n2", - "base_url":"http://N_0/solr", - "node_name":"N_0_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node6":{ - "core":"COLL_1t_shard1_replica_n4", - "base_url":"http://N_4/solr", - "node_name":"N_4_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}}}, - "router":{"name":"compositeId"}, - "autoAddReplicas":"true", - "nrtReplicas":"3", - "tlogReplicas":"0"}, - "COLL_x":{ - "pullReplicas":"0", - "replicationFactor":"3", - "shards":{"shard1":{ - "range":"80000000-7fffffff", - "state":"active", - "replicas":{ - "core_node3":{ - "core":"COLL_x_shard1_replica_n1", - "base_url":"http://N_7e/solr", - "node_name":"N_7e_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node6":{ - "core":"COLL_x_shard1_replica_n4", - "base_url":"http://N_4/solr", - "node_name":"N_4_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node10":{ - "core":"COLL_x_shard1_replica_n9", - "base_url":"http://N_0/solr", - "node_name":"N_0_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}}, - "router":{"name":"compositeId"}, - "autoAddReplicas":"true", - "nrtReplicas":"3", - "tlogReplicas":"0"}, - "COLL_2k":{ - "pullReplicas":"0", - "replicationFactor":"3", - "shards":{"shard1":{ - "range":"80000000-7fffffff", - "state":"active", - "replicas":{ - "core_node3":{ - "core":"COLL_2k_shard1_replica_n1", - "base_url":"http://N_7e/solr", - "node_name":"N_7e_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node6":{ - "core":"COLL_2k_shard1_replica_n4", - "base_url":"http://N_4/solr", - "node_name":"N_4_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node10":{ - "core":"COLL_2k_shard1_replica_n9", - "base_url":"http://N_0/solr", - "node_name":"N_0_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}}, - "router":{"name":"compositeId"}, - "autoAddReplicas":"true", - "nrtReplicas":"3", - "tlogReplicas":"0"}, - "COLL_1r":{ - "pullReplicas":"0", - "replicationFactor":"3", - "shards":{"shard1":{ - "range":"80000000-7fffffff", - "state":"active", - "replicas":{ - "core_node3":{ - "core":"COLL_1r_shard1_replica_n1", - "base_url":"http://N_7e/solr", - "node_name":"N_7e_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node5":{ - "core":"COLL_1r_shard1_replica_n2", - "base_url":"http://N_0/solr", - "node_name":"N_0_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node6":{ - "core":"COLL_1r_shard1_replica_n4", - "base_url":"http://N_4/solr", - "node_name":"N_4_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}}}, - "router":{"name":"compositeId"}, - "autoAddReplicas":"true", - "nrtReplicas":"3", - "tlogReplicas":"0"}, - "COLL_8":{ - "pullReplicas":"0", - "replicationFactor":"3", - "shards":{"shard1":{ - "range":"80000000-7fffffff", - "state":"active", - "replicas":{ - "core_node3":{ - "core":"COLL_8_shard1_replica_n1", - "base_url":"http://N_7e/solr", - "node_name":"N_7e_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node5":{ - "core":"COLL_8_shard1_replica_n2", - "base_url":"http://N_0/solr", - "node_name":"N_0_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node6":{ - "core":"COLL_8_shard1_replica_n4", - "base_url":"http://N_4/solr", - "node_name":"N_4_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}}}, - "router":{"name":"compositeId"}, - "autoAddReplicas":"true", - "nrtReplicas":"3", - "tlogReplicas":"0"}, - "COLL_1":{ - "pullReplicas":"0", - "replicationFactor":"3", - "shards":{ - "shard1_0_0":{ - "range":"80000000-9554ffff", - "state":"active", - "replicas":{ - "core_node57":{ - "core":"COLL_1_shard1_0_0_replica_n55", - "base_url":"http://N_13/solr", - "node_name":"N_13_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node80":{ - "core":"COLL_1_shard1_0_0_replica_n79", - "base_url":"http://N_3to/solr", - "node_name":"N_3to_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node88":{ - "core":"COLL_1_shard1_0_0_replica_n87", - "base_url":"http://N_b9/solr", - "node_name":"N_b9_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}, - "stateTimestamp":"1562040263462532068"}, - "shard1_0_1":{ - "range":"95550000-aaa9ffff", - "state":"active", - "replicas":{ - "core_node58":{ - "core":"COLL_1_shard1_0_1_replica_n56", - "base_url":"http://N_13/solr", - "node_name":"N_13_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node82":{ - "core":"COLL_1_shard1_0_1_replica_n81", - "base_url":"http://N_3to/solr", - "node_name":"N_3to_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node90":{ - "core":"COLL_1_shard1_0_1_replica_n89", - "base_url":"http://N_b9/solr", - "node_name":"N_b9_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}, - "stateTimestamp":"1562040263462514239"}, - "shard1_1_0":{ - "range":"aaaa0000-bffeffff", - "state":"active", - "replicas":{ - "core_node61":{ - "core":"COLL_1_shard1_1_0_replica_n59", - "base_url":"http://N_13/solr", - "node_name":"N_13_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node84":{ - "core":"COLL_1_shard1_1_0_replica_n83", - "base_url":"http://N_3to/solr", - "node_name":"N_3to_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node92":{ - "core":"COLL_1_shard1_1_0_replica_n91", - "base_url":"http://N_b9/solr", - "node_name":"N_b9_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}, - "stateTimestamp":"1562040278815865699"}, - "shard1_1_1":{ - "range":"bfff0000-d554ffff", - "state":"active", - "replicas":{ - "core_node62":{ - "core":"COLL_1_shard1_1_1_replica_n60", - "base_url":"http://N_13/solr", - "node_name":"N_13_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node86":{ - "core":"COLL_1_shard1_1_1_replica_n85", - "base_url":"http://N_3to/solr", - "node_name":"N_3to_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node94":{ - "core":"COLL_1_shard1_1_1_replica_n93", - "base_url":"http://N_b9/solr", - "node_name":"N_b9_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}, - "stateTimestamp":"1562040278815883523"}, - "shard2_1_0":{ - "range":"ffff0000-1553ffff", - "state":"active", - "replicas":{ - "core_node65":{ - "core":"COLL_1_shard2_1_0_replica_n63", - "base_url":"http://N_d4/solr", - "node_name":"N_d4_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node96":{ - "core":"COLL_1_shard2_1_0_replica_n95", - "base_url":"http://N_16/solr", - "node_name":"N_16_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node104":{ - "core":"COLL_1_shard2_1_0_replica_n103", - "base_url":"http://N_74/solr", - "node_name":"N_74_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}, - "stateTimestamp":"1562040347757791179"}, - "shard2_1_1":{ - "range":"15540000-2aa9ffff", - "state":"active", - "replicas":{ - "core_node66":{ - "core":"COLL_1_shard2_1_1_replica_n64", - "base_url":"http://N_d4/solr", - "node_name":"N_d4_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node98":{ - "core":"COLL_1_shard2_1_1_replica_n97", - "base_url":"http://N_16/solr", - "node_name":"N_16_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node106":{ - "core":"COLL_1_shard2_1_1_replica_n105", - "base_url":"http://N_74/solr", - "node_name":"N_74_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}, - "stateTimestamp":"1562040347757805057"}, - "shard2_0_0":{ - "range":"d5550000-eaa9ffff", - "state":"active", - "replicas":{ - "core_node69":{ - "core":"COLL_1_shard2_0_0_replica_n67", - "base_url":"http://N_d4/solr", - "node_name":"N_d4_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node100":{ - "core":"COLL_1_shard2_0_0_replica_n99", - "base_url":"http://N_16/solr", - "node_name":"N_16_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node108":{ - "core":"COLL_1_shard2_0_0_replica_n107", - "base_url":"http://N_74/solr", - "node_name":"N_74_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}, - "stateTimestamp":"1562040365925094823"}, - "shard2_0_1":{ - "range":"eaaa0000-fffeffff", - "state":"active", - "replicas":{ - "core_node70":{ - "core":"COLL_1_shard2_0_1_replica_n68", - "base_url":"http://N_d4/solr", - "node_name":"N_d4_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node102":{ - "core":"COLL_1_shard2_0_1_replica_n101", - "base_url":"http://N_16/solr", - "node_name":"N_16_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node110":{ - "core":"COLL_1_shard2_0_1_replica_n109", - "base_url":"http://N_74/solr", - "node_name":"N_74_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}, - "stateTimestamp":"1562040365925105897"}, - "shard3_0_0":{ - "range":"2aaa0000-3ffeffff", - "state":"active", - "replicas":{ - "core_node73":{ - "core":"COLL_1_shard3_0_0_replica_n71", - "base_url":"http://N_3a/solr", - "node_name":"N_3a_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node112":{ - "core":"COLL_1_shard3_0_0_replica_n111", - "base_url":"http://N_do/solr", - "node_name":"N_do_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node120":{ - "core":"COLL_1_shard3_0_0_replica_n119", - "base_url":"http://N_v/solr", - "node_name":"N_v_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}, - "stateTimestamp":"1562040222670106007"}, - "shard3_0_1":{ - "range":"3fff0000-5554ffff", - "state":"active", - "replicas":{ - "core_node74":{ - "core":"COLL_1_shard3_0_1_replica_n72", - "base_url":"http://N_3a/solr", - "node_name":"N_3a_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node114":{ - "core":"COLL_1_shard3_0_1_replica_n113", - "base_url":"http://N_do/solr", - "node_name":"N_do_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node122":{ - "core":"COLL_1_shard3_0_1_replica_n121", - "base_url":"http://N_v/solr", - "node_name":"N_v_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}, - "stateTimestamp":"1562040222670118507"}, - "shard3_1_0":{ - "range":"55550000-6aa9ffff", - "state":"active", - "replicas":{ - "core_node77":{ - "core":"COLL_1_shard3_1_0_replica_n75", - "base_url":"http://N_3a/solr", - "node_name":"N_3a_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node116":{ - "core":"COLL_1_shard3_1_0_replica_n115", - "base_url":"http://N_do/solr", - "node_name":"N_do_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node124":{ - "core":"COLL_1_shard3_1_0_replica_n123", - "base_url":"http://N_v/solr", - "node_name":"N_v_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}, - "stateTimestamp":"1562040233681530342"}, - "shard3_1_1":{ - "range":"6aaa0000-7fffffff", - "state":"active", - "replicas":{ - "core_node78":{ - "core":"COLL_1_shard3_1_1_replica_n76", - "base_url":"http://N_3a/solr", - "node_name":"N_3a_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node118":{ - "core":"COLL_1_shard3_1_1_replica_n117", - "base_url":"http://N_do/solr", - "node_name":"N_do_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node126":{ - "core":"COLL_1_shard3_1_1_replica_n125", - "base_url":"http://N_v/solr", - "node_name":"N_v_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}, - "stateTimestamp":"1562040233681548279"}}, - "router":{"name":"compositeId"}, - "autoAddReplicas":"true", - "nrtReplicas":"3", - "tlogReplicas":"0"}, - "version":0, - "COLL_4":{ - "pullReplicas":"0", - "replicationFactor":"3", - "shards":{"shard1":{ - "range":"80000000-7fffffff", - "state":"active", - "replicas":{ - "core_node3":{ - "core":"COLL_4_shard1_replica_n1", - "base_url":"http://N_7e/solr", - "node_name":"N_7e_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node5":{ - "core":"COLL_4_shard1_replica_n2", - "base_url":"http://N_0/solr", - "node_name":"N_0_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node6":{ - "core":"COLL_4_shard1_replica_n4", - "base_url":"http://N_4/solr", - "node_name":"N_4_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}}}, - "router":{"name":"compositeId"}, - "autoAddReplicas":"true", - "nrtReplicas":"3", - "tlogReplicas":"0"}, - "COLL_2":{ - "pullReplicas":"0", - "replicationFactor":"2", - "shards":{ - "shard1_0_0":{ - "range":"80000000-838dffff", - "state":"active", - "replicas":{ - "core_node1717":{ - "core":"COLL_2_shard1_0_0_replica_n1716", - "base_url":"http://N_z/solr", - "node_name":"N_z_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node1725":{ - "core":"COLL_2_shard1_0_0_replica_n1724", - "base_url":"http://N_5/solr", - "node_name":"N_5_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node1729":{ - "core":"COLL_2_shard1_0_0_replica_n1728", - "base_url":"http://N_1h/solr", - "node_name":"N_1h_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}, - "stateTimestamp":"1565148935777897324"}, - "shard1_0_1":{ - "range":"838e0000-871bffff", - "state":"active", - "replicas":{ - "core_node1669":{ - "core":"COLL_2_shard1_0_1_replica_n1668", - "base_url":"http://N_5/solr", - "node_name":"N_5_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node1677":{ - "core":"COLL_2_shard1_0_1_replica_n1676", - "base_url":"http://N_2w/solr", - "node_name":"N_2w_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node1719":{ - "core":"COLL_2_shard1_0_1_replica_n1718", - "base_url":"http://N_e/solr", - "node_name":"N_e_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}, - "stateTimestamp":"1565148935777884238"}, - "shard1_1_0":{ - "range":"871c0000-8aa9ffff", - "state":"active", - "replicas":{ - "core_node471":{ - "core":"COLL_2_shard1_1_0_replica_n1", - "base_url":"http://N_dj/solr", - "node_name":"N_dj_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node1679":{ - "core":"COLL_2_shard1_1_0_replica_n1678", - "base_url":"http://N_1m/solr", - "node_name":"N_1m_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node1721":{ - "core":"COLL_2_shard1_1_0_replica_n1720", - "base_url":"http://N_5/solr", - "node_name":"N_5_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}, - "stateTimestamp":"1565151683385910884"}, - "shard1_1_1":{ - "range":"8aaa0000-8e37ffff", - "state":"active", - "replicas":{ - "core_node418":{ - "core":"COLL_2_shard1_1_1_replica_n416", - "base_url":"http://N_5/solr", - "node_name":"N_5_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node474":{ - "core":"COLL_2_shard1_1_1_replica_n2", - "base_url":"http://N_3/solr", - "node_name":"N_3_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node1807":{ - "core":"COLL_2_shard1_1_1_replica_n1806", - "base_url":"http://N_2w/solr", - "node_name":"N_2w_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}, - "stateTimestamp":"1565151683385930901"}, - "shard3_1_0":{ - "range":"a38d0000-a71affff", - "state":"active", - "replicas":{ - "core_node425":{ - "core":"COLL_2_shard3_1_0_replica_n423", - "base_url":"http://N_1d/solr", - "node_name":"N_1d_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node459":{ - "core":"COLL_2_shard3_1_0_replica_n1", - "base_url":"http://N_6/solr", - "node_name":"N_6_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node460":{ - "core":"COLL_2_shard3_1_0_replica_n2", - "base_url":"http://N_aw/solr", - "node_name":"N_aw_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}, - "stateTimestamp":"1565145109110689908"}, - "shard3_1_1":{ - "range":"a71b0000-aaa9ffff", - "state":"active", - "replicas":{ - "core_node426":{ - "core":"COLL_2_shard3_1_1_replica_n424", - "base_url":"http://N_1d/solr", - "node_name":"N_1d_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node461":{ - "core":"COLL_2_shard3_1_1_replica_n1", - "base_url":"http://N_2u/solr", - "node_name":"N_2u_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node462":{ - "core":"COLL_2_shard3_1_1_replica_n2", - "base_url":"http://N_3a7/solr", - "node_name":"N_3a7_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}, - "stateTimestamp":"1565145109110736393"}, - "shard9_0_0":{ - "range":"f1c70000-f554ffff", - "state":"active", - "replicas":{ - "core_node1683":{ - "core":"COLL_2_shard9_0_0_replica_n1682", - "base_url":"http://N_g/solr", - "node_name":"N_g_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "property.preferredleader":"true"}, - "core_node1799":{ - "core":"COLL_2_shard9_0_0_replica_n1798", - "base_url":"http://N_6/solr", - "node_name":"N_6_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node1815":{ - "core":"COLL_2_shard9_0_0_replica_n1814", - "base_url":"http://N_3/solr", - "node_name":"N_3_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}, - "stateTimestamp":"1565151698484117350"}, - "shard9_0_1":{ - "range":"f5550000-f8e2ffff", - "state":"active", - "replicas":{ - "core_node438":{ - "core":"COLL_2_shard9_0_1_replica_n436", - "base_url":"http://N_11/solr", - "node_name":"N_11_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node477":{ - "core":"COLL_2_shard9_0_1_replica_n1", - "base_url":"http://N_m/solr", - "node_name":"N_m_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node478":{ - "core":"COLL_2_shard9_0_1_replica_n2", - "base_url":"http://N_2/solr", - "node_name":"N_2_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}, - "stateTimestamp":"1565151698484109022"}, - "shard4_0_0":{ - "range":"aaaa0000-ae37ffff", - "state":"active", - "replicas":{ - "core_node445":{ - "core":"COLL_2_shard4_0_0_replica_n443", - "base_url":"http://N_6c/solr", - "node_name":"N_6c_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node520":{ - "core":"COLL_2_shard4_0_0_replica_n2", - "base_url":"http://N_6/solr", - "node_name":"N_6_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node1737":{ - "core":"COLL_2_shard4_0_0_replica_n1736", - "base_url":"http://N_3/solr", - "node_name":"N_3_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}, - "stateTimestamp":"1565158279227348339"}, - "shard4_0_1":{ - "range":"ae380000-b1c5ffff", - "state":"active", - "replicas":{ - "core_node446":{ - "core":"COLL_2_shard4_0_1_replica_n444", - "base_url":"http://N_6c/solr", - "node_name":"N_6c_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node1773":{ - "core":"COLL_2_shard4_0_1_replica_n1772", - "base_url":"http://N_303/solr", - "node_name":"N_303_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node1803":{ - "core":"COLL_2_shard4_0_1_replica_n1802", - "base_url":"http://N_6/solr", - "node_name":"N_6_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}, - "stateTimestamp":"1565158279227361478"}, - "shard4_1_0":{ - "range":"b1c60000-b553ffff", - "state":"active", - "replicas":{ - "core_node457":{ - "core":"COLL_2_shard4_1_0_replica_n455", - "base_url":"http://N_6c/solr", - "node_name":"N_6c_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node523":{ - "core":"COLL_2_shard4_1_0_replica_n1", - "base_url":"http://N_3/solr", - "node_name":"N_3_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node1775":{ - "core":"COLL_2_shard4_1_0_replica_n1774", - "base_url":"http://N_2w/solr", - "node_name":"N_2w_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}, - "stateTimestamp":"1565161247490072659"}, - "shard4_1_1":{ - "range":"b5540000-b8e2ffff", - "state":"active", - "replicas":{ - "core_node458":{ - "core":"COLL_2_shard4_1_1_replica_n456", - "base_url":"http://N_6c/solr", - "node_name":"N_6c_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node525":{ - "core":"COLL_2_shard4_1_1_replica_n1", - "base_url":"http://N_aw/solr", - "node_name":"N_aw_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node1805":{ - "core":"COLL_2_shard4_1_1_replica_n1804", - "base_url":"http://N_2w/solr", - "node_name":"N_2w_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}, - "stateTimestamp":"1565161247490078977"}, - "shard3_0_0":{ - "range":"9c710000-9ffeffff", - "state":"active", - "replicas":{ - "core_node543":{ - "core":"COLL_2_shard3_0_0_replica_n1", - "base_url":"http://N_65p/solr", - "node_name":"N_65p_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node544":{ - "core":"COLL_2_shard3_0_0_replica_n2", - "base_url":"http://N_303/solr", - "node_name":"N_303_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node1809":{ - "core":"COLL_2_shard3_0_0_replica_n1808", - "base_url":"http://N_a/solr", - "node_name":"N_a_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}, - "stateTimestamp":"1565183961448368559"}, - "shard3_0_1":{ - "range":"9fff0000-a38cffff", - "state":"active", - "replicas":{ - "core_node510":{ - "core":"COLL_2_shard3_0_1_replica_n508", - "base_url":"http://N_1h/solr", - "node_name":"N_1h_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node545":{ - "core":"COLL_2_shard3_0_1_replica_n1", - "base_url":"http://N_65p/solr", - "node_name":"N_65p_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node546":{ - "core":"COLL_2_shard3_0_1_replica_n2", - "base_url":"http://N_4f/solr", - "node_name":"N_4f_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}, - "stateTimestamp":"1565183961448383802"}, - "shard12_1_0":{ - "range":"238d0000-271affff", - "state":"active", - "replicas":{ - "core_node659":{ - "core":"COLL_2_shard12_1_0_replica_n1", - "base_url":"http://N_aw/solr", - "node_name":"N_aw_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node660":{ - "core":"COLL_2_shard12_1_0_replica_n2", - "base_url":"http://N_2u/solr", - "node_name":"N_2u_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node1789":{ - "core":"COLL_2_shard12_1_0_replica_n1788", - "base_url":"http://N_1d/solr", - "node_name":"N_1d_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}, - "stateTimestamp":"1565207157854150844"}, - "shard12_1_1":{ - "range":"271b0000-2aa9ffff", - "state":"active", - "replicas":{ - "core_node586":{ - "core":"COLL_2_shard12_1_1_replica_n584", - "base_url":"http://N_1/solr", - "node_name":"N_1_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node661":{ - "core":"COLL_2_shard12_1_1_replica_n1", - "base_url":"http://N_aw/solr", - "node_name":"N_aw_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node662":{ - "core":"COLL_2_shard12_1_1_replica_n2", - "base_url":"http://N_11/solr", - "node_name":"N_11_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}, - "stateTimestamp":"1565207157854141908"}, - "shard15_0_0":{ - "range":"471c0000-4aa9ffff", - "state":"active", - "replicas":{ - "core_node609":{ - "core":"COLL_2_shard15_0_0_replica_n607", - "base_url":"http://N_7/solr", - "node_name":"N_7_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node731":{ - "core":"COLL_2_shard15_0_0_replica_n1", - "base_url":"http://N_e/solr", - "node_name":"N_e_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node732":{ - "core":"COLL_2_shard15_0_0_replica_n2", - "base_url":"http://N_1d/solr", - "node_name":"N_1d_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}, - "stateTimestamp":"1565218606036002434"}, - "shard15_0_1":{ - "range":"4aaa0000-4e37ffff", - "state":"active", - "replicas":{ - "core_node610":{ - "core":"COLL_2_shard15_0_1_replica_n608", - "base_url":"http://N_7/solr", - "node_name":"N_7_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node734":{ - "core":"COLL_2_shard15_0_1_replica_n2", - "base_url":"http://N_u/solr", - "node_name":"N_u_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node1817":{ - "core":"COLL_2_shard15_0_1_replica_n1816", - "base_url":"http://N_aw/solr", - "node_name":"N_aw_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}, - "stateTimestamp":"1565218606035996309"}, - "shard18_1_0":{ - "range":"78e30000-7c70ffff", - "state":"active", - "replicas":{ - "core_node625":{ - "core":"COLL_2_shard18_1_0_replica_n623", - "base_url":"http://N_4g/solr", - "node_name":"N_4g_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node671":{ - "core":"COLL_2_shard18_1_0_replica_n1", - "base_url":"http://N_2w/solr", - "node_name":"N_2w_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node672":{ - "core":"COLL_2_shard18_1_0_replica_n2", - "base_url":"http://N_e/solr", - "node_name":"N_e_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}, - "stateTimestamp":"1565207836146608524"}, - "shard18_1_1":{ - "range":"7c710000-7fffffff", - "state":"active", - "replicas":{ - "core_node626":{ - "core":"COLL_2_shard18_1_1_replica_n624", - "base_url":"http://N_4g/solr", - "node_name":"N_4g_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node673":{ - "core":"COLL_2_shard18_1_1_replica_n1", - "base_url":"http://N_2w/solr", - "node_name":"N_2w_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node1821":{ - "core":"COLL_2_shard18_1_1_replica_n1820", - "base_url":"http://N_aw/solr", - "node_name":"N_aw_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}, - "stateTimestamp":"1565207836146601465"}, - "shard11_1_0":{ - "range":"15540000-18e1ffff", - "state":"active", - "replicas":{ - "core_node779":{ - "core":"COLL_2_shard11_1_0_replica_n778", - "base_url":"http://N_1f/solr", - "node_name":"N_1f_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node781":{ - "core":"COLL_2_shard11_1_0_replica_n780", - "base_url":"http://N_9o/solr", - "node_name":"N_9o_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node1791":{ - "core":"COLL_2_shard11_1_0_replica_n1790", - "base_url":"http://N_t/solr", - "node_name":"N_t_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}, - "stateTimestamp":"1565245269658561257"}, - "shard7_0_0":{ - "range":"d5550000-d8e2ffff", - "state":"active", - "replicas":{ - "core_node766":{ - "core":"COLL_2_shard7_0_0_replica_n764", - "base_url":"http://N_9o/solr", - "node_name":"N_9o_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node774":{ - "core":"COLL_2_shard7_0_0_replica_n1", - "base_url":"http://N_65p/solr", - "node_name":"N_65p_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node775":{ - "core":"COLL_2_shard7_0_0_replica_n2", - "base_url":"http://N_3a7/solr", - "node_name":"N_3a7_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}, - "stateTimestamp":"1565248636417965858"}, - "shard11_1_1":{ - "range":"18e20000-1c70ffff", - "state":"active", - "replicas":{ - "core_node768":{ - "core":"COLL_2_shard11_1_1_replica_n762", - "base_url":"http://N_17/solr", - "node_name":"N_17_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node783":{ - "core":"COLL_2_shard11_1_1_replica_n782", - "base_url":"http://N_1f/solr", - "node_name":"N_1f_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node785":{ - "core":"COLL_2_shard11_1_1_replica_n784", - "base_url":"http://N_9o/solr", - "node_name":"N_9o_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}, - "stateTimestamp":"1565245269658580580"}, - "shard7_0_1":{ - "range":"d8e30000-dc70ffff", - "state":"active", - "replicas":{ - "core_node769":{ - "core":"COLL_2_shard7_0_1_replica_n765", - "base_url":"http://N_9o/solr", - "node_name":"N_9o_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node776":{ - "core":"COLL_2_shard7_0_1_replica_n1", - "base_url":"http://N_2/solr", - "node_name":"N_2_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node777":{ - "core":"COLL_2_shard7_0_1_replica_n2", - "base_url":"http://N_29/solr", - "node_name":"N_29_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}, - "stateTimestamp":"1565248636417971231"}, - "shard18_0_0":{ - "range":"71c70000-7554ffff", - "state":"active", - "replicas":{ - "core_node874":{ - "core":"COLL_2_shard18_0_0_replica_n1", - "base_url":"http://N_1c/solr", - "node_name":"N_1c_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node875":{ - "core":"COLL_2_shard18_0_0_replica_n2", - "base_url":"http://N_1/solr", - "node_name":"N_1_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node1819":{ - "core":"COLL_2_shard18_0_0_replica_n1818", - "base_url":"http://N_e/solr", - "node_name":"N_e_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}, - "stateTimestamp":"1565255619352465985"}, - "shard18_0_1":{ - "range":"75550000-78e2ffff", - "state":"active", - "replicas":{ - "core_node773":{ - "core":"COLL_2_shard18_0_1_replica_n771", - "base_url":"http://N_dj/solr", - "node_name":"N_dj_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node876":{ - "core":"COLL_2_shard18_0_1_replica_n1", - "base_url":"http://N_1c/solr", - "node_name":"N_1c_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node877":{ - "core":"COLL_2_shard18_0_1_replica_n2", - "base_url":"http://N_17/solr", - "node_name":"N_17_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}, - "stateTimestamp":"1565255619352471980"}, - "shard2_0_0":{ - "range":"8e380000-91c5ffff", - "state":"active", - "replicas":{ - "core_node796":{ - "core":"COLL_2_shard2_0_0_replica_n794", - "base_url":"http://N_8/solr", - "node_name":"N_8_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node911":{ - "core":"COLL_2_shard2_0_0_replica_n910", - "base_url":"http://N_5/solr", - "node_name":"N_5_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node1823":{ - "core":"COLL_2_shard2_0_0_replica_n1822", - "base_url":"http://N_3a7/solr", - "node_name":"N_3a7_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}, - "stateTimestamp":"1565270983762975731"}, - "shard2_1_0":{ - "range":"95540000-98e1ffff", - "state":"active", - "replicas":{ - "core_node975":{ - "core":"COLL_2_shard2_1_0_replica_n974", - "base_url":"http://N_4f/solr", - "node_name":"N_4f_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node1681":{ - "core":"COLL_2_shard2_1_0_replica_n1680", - "base_url":"http://N_g/solr", - "node_name":"N_g_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node1727":{ - "core":"COLL_2_shard2_1_0_replica_n1726", - "base_url":"http://N_6i/solr", - "node_name":"N_6i_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}, - "stateTimestamp":"1565271264820336969"}, - "shard2_0_1":{ - "range":"91c60000-9553ffff", - "state":"active", - "replicas":{ - "core_node800":{ - "core":"COLL_2_shard2_0_1_replica_n795", - "base_url":"http://N_8/solr", - "node_name":"N_8_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node915":{ - "core":"COLL_2_shard2_0_1_replica_n914", - "base_url":"http://N_4f/solr", - "node_name":"N_4f_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node917":{ - "core":"COLL_2_shard2_0_1_replica_n916", - "base_url":"http://N_5/solr", - "node_name":"N_5_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}, - "stateTimestamp":"1565270983762987501"}, - "shard2_1_1":{ - "range":"98e20000-9c70ffff", - "state":"active", - "replicas":{ - "core_node979":{ - "core":"COLL_2_shard2_1_1_replica_n978", - "base_url":"http://N_6i/solr", - "node_name":"N_6i_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node1685":{ - "core":"COLL_2_shard2_1_1_replica_n1684", - "base_url":"http://N_4f/solr", - "node_name":"N_4f_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node1813":{ - "core":"COLL_2_shard2_1_1_replica_n1812", - "base_url":"http://N_4g/solr", - "node_name":"N_4g_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}, - "stateTimestamp":"1565271264820364576"}, - "shard13_1_0":{ - "range":"31c60000-3553ffff", - "state":"active", - "replicas":{ - "core_node923":{ - "core":"COLL_2_shard13_1_0_replica_n922", - "base_url":"http://N_e/solr", - "node_name":"N_e_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node1689":{ - "core":"COLL_2_shard13_1_0_replica_n1688", - "base_url":"http://N_7/solr", - "node_name":"N_7_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node1763":{ - "core":"COLL_2_shard13_1_0_replica_n1762", - "base_url":"http://N_u/solr", - "node_name":"N_u_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}, - "stateTimestamp":"1565268739309072068"}, - "shard13_1_1":{ - "range":"35540000-38e2ffff", - "state":"active", - "replicas":{ - "core_node808":{ - "core":"COLL_2_shard13_1_1_replica_n806", - "base_url":"http://N_7/solr", - "node_name":"N_7_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node921":{ - "core":"COLL_2_shard13_1_1_replica_n920", - "base_url":"http://N_u/solr", - "node_name":"N_u_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node925":{ - "core":"COLL_2_shard13_1_1_replica_n924", - "base_url":"http://N_e/solr", - "node_name":"N_e_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}, - "stateTimestamp":"1565268739309062648"}, - "shard8_0_0":{ - "range":"e38e0000-e71bffff", - "state":"active", - "replicas":{ - "core_node887":{ - "core":"COLL_2_shard8_0_0_replica_n886", - "base_url":"http://N_1m/solr", - "node_name":"N_1m_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node1691":{ - "core":"COLL_2_shard8_0_0_replica_n1690", - "base_url":"http://N_29/solr", - "node_name":"N_29_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node1731":{ - "core":"COLL_2_shard8_0_0_replica_n1730", - "base_url":"http://N_z/solr", - "node_name":"N_z_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}, - "stateTimestamp":"1565264342830784906"}, - "shard10_0_0":{ - "range":"0-38dffff", - "state":"active", - "replicas":{ - "core_node827":{ - "core":"COLL_2_shard10_0_0_replica_n825", - "base_url":"http://N_cs/solr", - "node_name":"N_cs_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node897":{ - "core":"COLL_2_shard10_0_0_replica_n896", - "base_url":"http://N_6i/solr", - "node_name":"N_6i_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node1733":{ - "core":"COLL_2_shard10_0_0_replica_n1732", - "base_url":"http://N_t/solr", - "node_name":"N_t_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}, - "stateTimestamp":"1565267042378799051"}, - "shard10_0_1":{ - "range":"38e0000-71bffff", - "state":"active", - "replicas":{ - "core_node828":{ - "core":"COLL_2_shard10_0_1_replica_n826", - "base_url":"http://N_cs/solr", - "node_name":"N_cs_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node905":{ - "core":"COLL_2_shard10_0_1_replica_n904", - "base_url":"http://N_t/solr", - "node_name":"N_t_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node1739":{ - "core":"COLL_2_shard10_0_1_replica_n1738", - "base_url":"http://N_6i/solr", - "node_name":"N_6i_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}, - "stateTimestamp":"1565267042378772588"}, - "shard10_1_0":{ - "range":"71c0000-aa9ffff", - "state":"active", - "replicas":{ - "core_node831":{ - "core":"COLL_2_shard10_1_0_replica_n829", - "base_url":"http://N_6i/solr", - "node_name":"N_6i_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node1693":{ - "core":"COLL_2_shard10_1_0_replica_n1692", - "base_url":"http://N_1i/solr", - "node_name":"N_1i_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node1797":{ - "core":"COLL_2_shard10_1_0_replica_n1796", - "base_url":"http://N_65p/solr", - "node_name":"N_65p_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}, - "stateTimestamp":"1565264436499940709"}, - "shard8_0_1":{ - "range":"e71c0000-eaa9ffff", - "state":"active", - "replicas":{ - "core_node893":{ - "core":"COLL_2_shard8_0_1_replica_n892", - "base_url":"http://N_1m/solr", - "node_name":"N_1m_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node1695":{ - "core":"COLL_2_shard8_0_1_replica_n1694", - "base_url":"http://N_z/solr", - "node_name":"N_z_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node1787":{ - "core":"COLL_2_shard8_0_1_replica_n1786", - "base_url":"http://N_29/solr", - "node_name":"N_29_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}, - "stateTimestamp":"1565264342830792972"}, - "shard14_1_0":{ - "range":"3fff0000-438cffff", - "state":"active", - "replicas":{ - "core_node835":{ - "core":"COLL_2_shard14_1_0_replica_n833", - "base_url":"http://N_a/solr", - "node_name":"N_a_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node1127":{ - "core":"COLL_2_shard14_1_0_replica_n1126", - "base_url":"http://N_z/solr", - "node_name":"N_z_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node1129":{ - "core":"COLL_2_shard14_1_0_replica_n1128", - "base_url":"http://N_1d/solr", - "node_name":"N_1d_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}, - "stateTimestamp":"1565264301050142265"}, - "shard14_1_1":{ - "range":"438d0000-471bffff", - "state":"active", - "replicas":{ - "core_node836":{ - "core":"COLL_2_shard14_1_1_replica_n834", - "base_url":"http://N_a/solr", - "node_name":"N_a_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node1741":{ - "core":"COLL_2_shard14_1_1_replica_n1740", - "base_url":"http://N_1d/solr", - "node_name":"N_1d_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node1825":{ - "core":"COLL_2_shard14_1_1_replica_n1824", - "base_url":"http://N_3a7/solr", - "node_name":"N_3a7_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}, - "stateTimestamp":"1565264301050133844"}, - "shard14_0_0":{ - "range":"38e30000-3c70ffff", - "state":"active", - "replicas":{ - "core_node839":{ - "core":"COLL_2_shard14_0_0_replica_n837", - "base_url":"http://N_3a7/solr", - "node_name":"N_3a7_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node1119":{ - "core":"COLL_2_shard14_0_0_replica_n1118", - "base_url":"http://N_a/solr", - "node_name":"N_a_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node1121":{ - "core":"COLL_2_shard14_0_0_replica_n1120", - "base_url":"http://N_17/solr", - "node_name":"N_17_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}, - "stateTimestamp":"1565265507181833703"}, - "shard10_1_1":{ - "range":"aaa0000-e37ffff", - "state":"active", - "replicas":{ - "core_node840":{ - "core":"COLL_2_shard10_1_1_replica_n830", - "base_url":"http://N_6i/solr", - "node_name":"N_6i_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node1743":{ - "core":"COLL_2_shard10_1_1_replica_n1742", - "base_url":"http://N_t/solr", - "node_name":"N_t_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node1779":{ - "core":"COLL_2_shard10_1_1_replica_n1778", - "base_url":"http://N_1i/solr", - "node_name":"N_1i_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}, - "stateTimestamp":"1565264436499929417"}, - "shard14_0_1":{ - "range":"3c710000-3ffeffff", - "state":"active", - "replicas":{ - "core_node841":{ - "core":"COLL_2_shard14_0_1_replica_n838", - "base_url":"http://N_3a7/solr", - "node_name":"N_3a7_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node1123":{ - "core":"COLL_2_shard14_0_1_replica_n1122", - "base_url":"http://N_17/solr", - "node_name":"N_17_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node1125":{ - "core":"COLL_2_shard14_0_1_replica_n1124", - "base_url":"http://N_a/solr", - "node_name":"N_a_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}, - "stateTimestamp":"1565265507181840877"}, - "shard17_0_0":{ - "range":"638e0000-671bffff", - "state":"active", - "replicas":{ - "core_node844":{ - "core":"COLL_2_shard17_0_0_replica_n842", - "base_url":"http://N_6c/solr", - "node_name":"N_6c_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node1735":{ - "core":"COLL_2_shard17_0_0_replica_n1734", - "base_url":"http://N_2u/solr", - "node_name":"N_2u_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node1781":{ - "core":"COLL_2_shard17_0_0_replica_n1780", - "base_url":"http://N_1i/solr", - "node_name":"N_1i_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}, - "stateTimestamp":"1565266637880800687"}, - "shard17_0_1":{ - "range":"671c0000-6aa9ffff", - "state":"active", - "replicas":{ - "core_node848":{ - "core":"COLL_2_shard17_0_1_replica_n843", - "base_url":"http://N_6c/solr", - "node_name":"N_6c_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node1115":{ - "core":"COLL_2_shard17_0_1_replica_n1114", - "base_url":"http://N_2u/solr", - "node_name":"N_2u_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node1117":{ - "core":"COLL_2_shard17_0_1_replica_n1116", - "base_url":"http://N_1i/solr", - "node_name":"N_1i_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}, - "stateTimestamp":"1565266637880794287"}, - "shard16_1_0":{ - "range":"5c710000-5ffeffff", - "state":"active", - "replicas":{ - "core_node852":{ - "core":"COLL_2_shard16_1_0_replica_n850", - "base_url":"http://N_8/solr", - "node_name":"N_8_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node991":{ - "core":"COLL_2_shard16_1_0_replica_n990", - "base_url":"http://N_3/solr", - "node_name":"N_3_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node993":{ - "core":"COLL_2_shard16_1_0_replica_n992", - "base_url":"http://N_1/solr", - "node_name":"N_1_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}, - "stateTimestamp":"1565275919174780002"}, - "shard16_1_1":{ - "range":"5fff0000-638dffff", - "state":"active", - "replicas":{ - "core_node853":{ - "core":"COLL_2_shard16_1_1_replica_n851", - "base_url":"http://N_8/solr", - "node_name":"N_8_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node995":{ - "core":"COLL_2_shard16_1_1_replica_n994", - "base_url":"http://N_1/solr", - "node_name":"N_1_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node997":{ - "core":"COLL_2_shard16_1_1_replica_n996", - "base_url":"http://N_3/solr", - "node_name":"N_3_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}, - "stateTimestamp":"1565275919174771365"}, - "shard16_0_0":{ - "range":"55550000-58e2ffff", - "state":"active", - "replicas":{ - "core_node856":{ - "core":"COLL_2_shard16_0_0_replica_n854", - "base_url":"http://N_8/solr", - "node_name":"N_8_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node983":{ - "core":"COLL_2_shard16_0_0_replica_n982", - "base_url":"http://N_1/solr", - "node_name":"N_1_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node1785":{ - "core":"COLL_2_shard16_0_0_replica_n1784", - "base_url":"http://N_303/solr", - "node_name":"N_303_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}, - "stateTimestamp":"1565275747479472229"}, - "shard16_0_1":{ - "range":"58e30000-5c70ffff", - "state":"active", - "replicas":{ - "core_node857":{ - "core":"COLL_2_shard16_0_1_replica_n855", - "base_url":"http://N_8/solr", - "node_name":"N_8_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node987":{ - "core":"COLL_2_shard16_0_1_replica_n986", - "base_url":"http://N_303/solr", - "node_name":"N_303_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node989":{ - "core":"COLL_2_shard16_0_1_replica_n988", - "base_url":"http://N_1/solr", - "node_name":"N_1_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}, - "stateTimestamp":"1565275747479466413"}, - "shard5_1_0":{ - "range":"bfff0000-c38cffff", - "state":"active", - "replicas":{ - "core_node1135":{ - "core":"COLL_2_shard5_1_0_replica_n1134", - "base_url":"http://N_1c/solr", - "node_name":"N_1c_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node1137":{ - "core":"COLL_2_shard5_1_0_replica_n1136", - "base_url":"http://N_2/solr", - "node_name":"N_2_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node1783":{ - "core":"COLL_2_shard5_1_0_replica_n1782", - "base_url":"http://N_g/solr", - "node_name":"N_g_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}, - "stateTimestamp":"1565275178824240086"}, - "shard5_1_1":{ - "range":"c38d0000-c71bffff", - "state":"active", - "replicas":{ - "core_node861":{ - "core":"COLL_2_shard5_1_1_replica_n859", - "base_url":"http://N_g/solr", - "node_name":"N_g_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node1139":{ - "core":"COLL_2_shard5_1_1_replica_n1138", - "base_url":"http://N_2/solr", - "node_name":"N_2_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node1141":{ - "core":"COLL_2_shard5_1_1_replica_n1140", - "base_url":"http://N_1c/solr", - "node_name":"N_1c_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}, - "stateTimestamp":"1565275178824249126"}, - "shard5_0_0":{ - "range":"b8e30000-bc70ffff", - "state":"active", - "replicas":{ - "core_node999":{ - "core":"COLL_2_shard5_0_0_replica_n998", - "base_url":"http://N_1c/solr", - "node_name":"N_1c_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node1001":{ - "core":"COLL_2_shard5_0_0_replica_n1000", - "base_url":"http://N_1f/solr", - "node_name":"N_1f_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node1769":{ - "core":"COLL_2_shard5_0_0_replica_n1768", - "base_url":"http://N_g/solr", - "node_name":"N_g_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}, - "stateTimestamp":"1565275212524831473"}, - "shard5_0_1":{ - "range":"bc710000-bffeffff", - "state":"active", - "replicas":{ - "core_node1003":{ - "core":"COLL_2_shard5_0_1_replica_n1002", - "base_url":"http://N_1f/solr", - "node_name":"N_1f_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node1703":{ - "core":"COLL_2_shard5_0_1_replica_n1702", - "base_url":"http://N_1c/solr", - "node_name":"N_1c_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node1771":{ - "core":"COLL_2_shard5_0_1_replica_n1770", - "base_url":"http://N_g/solr", - "node_name":"N_g_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}, - "stateTimestamp":"1565275212524825994"}, - "shard7_1_0":{ - "range":"dc710000-dffeffff", - "state":"active", - "replicas":{ - "core_node928":{ - "core":"COLL_2_shard7_1_0_replica_n926", - "base_url":"http://N_dj/solr", - "node_name":"N_dj_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node1143":{ - "core":"COLL_2_shard7_1_0_replica_n1142", - "base_url":"http://N_29/solr", - "node_name":"N_29_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node1145":{ - "core":"COLL_2_shard7_1_0_replica_n1144", - "base_url":"http://N_1h/solr", - "node_name":"N_1h_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}, - "stateTimestamp":"1565302558938511755"}, - "shard9_1_0":{ - "range":"f8e30000-fc70ffff", - "state":"active", - "replicas":{ - "core_node931":{ - "core":"COLL_2_shard9_1_0_replica_n929", - "base_url":"http://N_4g/solr", - "node_name":"N_4g_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node1151":{ - "core":"COLL_2_shard9_1_0_replica_n1150", - "base_url":"http://N_303/solr", - "node_name":"N_303_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node1153":{ - "core":"COLL_2_shard9_1_0_replica_n1152", - "base_url":"http://N_11/solr", - "node_name":"N_11_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}, - "stateTimestamp":"1565302364380675715"}, - "shard6_1_0":{ - "range":"ce380000-d1c5ffff", - "state":"active", - "replicas":{ - "core_node937":{ - "core":"COLL_2_shard6_1_0_replica_n935", - "base_url":"http://N_cs/solr", - "node_name":"N_cs_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node1167":{ - "core":"COLL_2_shard6_1_0_replica_n1166", - "base_url":"http://N_1m/solr", - "node_name":"N_1m_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node1793":{ - "core":"COLL_2_shard6_1_0_replica_n1792", - "base_url":"http://N_29/solr", - "node_name":"N_29_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}, - "stateTimestamp":"1565308352274112068"}, - "shard7_1_1":{ - "range":"dfff0000-e38dffff", - "state":"active", - "replicas":{ - "core_node941":{ - "core":"COLL_2_shard7_1_1_replica_n927", - "base_url":"http://N_dj/solr", - "node_name":"N_dj_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node1701":{ - "core":"COLL_2_shard7_1_1_replica_n1700", - "base_url":"http://N_1h/solr", - "node_name":"N_1h_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node1759":{ - "core":"COLL_2_shard7_1_1_replica_n1758", - "base_url":"http://N_29/solr", - "node_name":"N_29_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}, - "stateTimestamp":"1565302558938518446"}, - "shard9_1_1":{ - "range":"fc710000-ffffffff", - "state":"active", - "replicas":{ - "core_node944":{ - "core":"COLL_2_shard9_1_1_replica_n930", - "base_url":"http://N_4g/solr", - "node_name":"N_4g_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node1155":{ - "core":"COLL_2_shard9_1_1_replica_n1154", - "base_url":"http://N_11/solr", - "node_name":"N_11_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node1163":{ - "core":"COLL_2_shard9_1_1_replica_n1162", - "base_url":"http://N_303/solr", - "node_name":"N_303_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}, - "stateTimestamp":"1565302364380668948"}, - "shard6_1_1":{ - "range":"d1c60000-d554ffff", - "state":"active", - "replicas":{ - "core_node1171":{ - "core":"COLL_2_shard6_1_1_replica_n1170", - "base_url":"http://N_m/solr", - "node_name":"N_m_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node1705":{ - "core":"COLL_2_shard6_1_1_replica_n1704", - "base_url":"http://N_cs/solr", - "node_name":"N_cs_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node1745":{ - "core":"COLL_2_shard6_1_1_replica_n1744", - "base_url":"http://N_1m/solr", - "node_name":"N_1m_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}, - "stateTimestamp":"1565308352274105667"}, - "shard15_1_0":{ - "range":"4e380000-51c5ffff", - "state":"active", - "replicas":{ - "core_node955":{ - "core":"COLL_2_shard15_1_0_replica_n953", - "base_url":"http://N_cs/solr", - "node_name":"N_cs_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node1173":{ - "core":"COLL_2_shard15_1_0_replica_n1172", - "base_url":"http://N_65p/solr", - "node_name":"N_65p_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node1175":{ - "core":"COLL_2_shard15_1_0_replica_n1174", - "base_url":"http://N_a/solr", - "node_name":"N_a_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}, - "stateTimestamp":"1565308280442325340"}, - "shard15_1_1":{ - "range":"51c60000-5554ffff", - "state":"active", - "replicas":{ - "core_node956":{ - "core":"COLL_2_shard15_1_1_replica_n954", - "base_url":"http://N_cs/solr", - "node_name":"N_cs_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node1709":{ - "core":"COLL_2_shard15_1_1_replica_n1708", - "base_url":"http://N_6/solr", - "node_name":"N_6_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node1747":{ - "core":"COLL_2_shard15_1_1_replica_n1746", - "base_url":"http://N_65p/solr", - "node_name":"N_65p_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}, - "stateTimestamp":"1565308280442332742"}, - "shard6_0_0":{ - "range":"c71c0000-caa9ffff", - "state":"active", - "replicas":{ - "core_node1182":{ - "core":"COLL_2_shard6_0_0_replica_n1180", - "base_url":"http://N_4f/solr", - "node_name":"N_4f_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node1208":{ - "core":"COLL_2_shard6_0_0_replica_n1207", - "base_url":"http://N_m/solr", - "node_name":"N_m_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node1210":{ - "core":"COLL_2_shard6_0_0_replica_n1209", - "base_url":"http://N_11/solr", - "node_name":"N_11_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}, - "stateTimestamp":"1565327402584689595"}, - "shard11_0_0":{ - "range":"e380000-11c5ffff", - "state":"active", - "replicas":{ - "core_node1185":{ - "core":"COLL_2_shard11_0_0_replica_n1183", - "base_url":"http://N_t/solr", - "node_name":"N_t_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node1217":{ - "core":"COLL_2_shard11_0_0_replica_n1216", - "base_url":"http://N_1f/solr", - "node_name":"N_1f_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node1219":{ - "core":"COLL_2_shard11_0_0_replica_n1218", - "base_url":"http://N_9o/solr", - "node_name":"N_9o_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}, - "stateTimestamp":"1565326848103929784"}, - "shard6_0_1":{ - "range":"caaa0000-ce37ffff", - "state":"active", - "replicas":{ - "core_node1189":{ - "core":"COLL_2_shard6_0_1_replica_n1181", - "base_url":"http://N_4f/solr", - "node_name":"N_4f_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node1212":{ - "core":"COLL_2_shard6_0_1_replica_n1211", - "base_url":"http://N_11/solr", - "node_name":"N_11_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node1214":{ - "core":"COLL_2_shard6_0_1_replica_n1213", - "base_url":"http://N_m/solr", - "node_name":"N_m_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}, - "stateTimestamp":"1565327402584696403"}, - "shard11_0_1":{ - "range":"11c60000-1553ffff", - "state":"active", - "replicas":{ - "core_node1195":{ - "core":"COLL_2_shard11_0_1_replica_n1184", - "base_url":"http://N_t/solr", - "node_name":"N_t_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node1221":{ - "core":"COLL_2_shard11_0_1_replica_n1220", - "base_url":"http://N_9o/solr", - "node_name":"N_9o_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node1223":{ - "core":"COLL_2_shard11_0_1_replica_n1222", - "base_url":"http://N_1f/solr", - "node_name":"N_1f_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}, - "stateTimestamp":"1565326848103918157"}, - "shard17_1_0":{ - "range":"6aaa0000-6e37ffff", - "state":"active", - "replicas":{ - "core_node1200":{ - "core":"COLL_2_shard17_1_0_replica_n1198", - "base_url":"http://N_1i/solr", - "node_name":"N_1i_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node1225":{ - "core":"COLL_2_shard17_1_0_replica_n1224", - "base_url":"http://N_2u/solr", - "node_name":"N_2u_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node1227":{ - "core":"COLL_2_shard17_1_0_replica_n1226", - "base_url":"http://N_m/solr", - "node_name":"N_m_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}, - "stateTimestamp":"1565327203156720131"}, - "shard17_1_1":{ - "range":"6e380000-71c6ffff", - "state":"active", - "replicas":{ - "core_node1203":{ - "core":"COLL_2_shard17_1_1_replica_n1199", - "base_url":"http://N_1i/solr", - "node_name":"N_1i_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node1229":{ - "core":"COLL_2_shard17_1_1_replica_n1228", - "base_url":"http://N_m/solr", - "node_name":"N_m_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node1231":{ - "core":"COLL_2_shard17_1_1_replica_n1230", - "base_url":"http://N_2u/solr", - "node_name":"N_2u_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}, - "stateTimestamp":"1565327203156741532"}, - "shard12_0_0":{ - "range":"1c710000-1ffeffff", - "state":"active", - "replicas":{ - "core_node1249":{ - "core":"COLL_2_shard12_0_0_replica_n1248", - "base_url":"http://N_2/solr", - "node_name":"N_2_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node1697":{ - "core":"COLL_2_shard12_0_0_replica_n1696", - "base_url":"http://N_1h/solr", - "node_name":"N_1h_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node1751":{ - "core":"COLL_2_shard12_0_0_replica_n1750", - "base_url":"http://N_17/solr", - "node_name":"N_17_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}, - "stateTimestamp":"1565348748808724074"}, - "shard12_0_1":{ - "range":"1fff0000-238cffff", - "state":"active", - "replicas":{ - "core_node1255":{ - "core":"COLL_2_shard12_0_1_replica_n1254", - "base_url":"http://N_2/solr", - "node_name":"N_2_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node1699":{ - "core":"COLL_2_shard12_0_1_replica_n1698", - "base_url":"http://N_17/solr", - "node_name":"N_17_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node1761":{ - "core":"COLL_2_shard12_0_1_replica_n1760", - "base_url":"http://N_1h/solr", - "node_name":"N_1h_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}, - "stateTimestamp":"1565348748808712354"}, - "shard8_1_0":{ - "range":"eaaa0000-ee37ffff", - "state":"active", - "replicas":{ - "core_node1707":{ - "core":"COLL_2_shard8_1_0_replica_n1706", - "base_url":"http://N_z/solr", - "node_name":"N_z_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node1765":{ - "core":"COLL_2_shard8_1_0_replica_n1764", - "base_url":"http://N_u/solr", - "node_name":"N_u_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node1811":{ - "core":"COLL_2_shard8_1_0_replica_n1810", - "base_url":"http://N_6/solr", - "node_name":"N_6_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}, - "stateTimestamp":"1565367832373747746"}, - "shard8_1_1":{ - "range":"ee380000-f1c6ffff", - "state":"active", - "replicas":{ - "core_node1711":{ - "core":"COLL_2_shard8_1_1_replica_n1710", - "base_url":"http://N_1m/solr", - "node_name":"N_1m_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node1755":{ - "core":"COLL_2_shard8_1_1_replica_n1754", - "base_url":"http://N_z/solr", - "node_name":"N_z_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node1795":{ - "core":"COLL_2_shard8_1_1_replica_n1794", - "base_url":"http://N_4g/solr", - "node_name":"N_4g_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}, - "stateTimestamp":"1565367832373770614"}, - "shard13_0_0":{ - "range":"2aaa0000-2e37ffff", - "state":"active", - "replicas":{ - "core_node1257":{ - "core":"COLL_2_shard13_0_0_replica_n1256", - "base_url":"http://N_u/solr", - "node_name":"N_u_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node1713":{ - "core":"COLL_2_shard13_0_0_replica_n1712", - "base_url":"http://N_7/solr", - "node_name":"N_7_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node1749":{ - "core":"COLL_2_shard13_0_0_replica_n1748", - "base_url":"http://N_dj/solr", - "node_name":"N_dj_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}, - "stateTimestamp":"1565369006475868394"}, - "shard13_0_1":{ - "range":"2e380000-31c5ffff", - "state":"active", - "replicas":{ - "core_node1263":{ - "core":"COLL_2_shard13_0_1_replica_n1262", - "base_url":"http://N_u/solr", - "node_name":"N_u_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node1715":{ - "core":"COLL_2_shard13_0_1_replica_n1714", - "base_url":"http://N_dj/solr", - "node_name":"N_dj_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node1767":{ - "core":"COLL_2_shard13_0_1_replica_n1766", - "base_url":"http://N_7/solr", - "node_name":"N_7_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}, - "stateTimestamp":"1565369006475856752"}}, - "router":{"name":"compositeId"}, - "autoAddReplicas":"true", - "nrtReplicas":"2", - "tlogReplicas":"0"}, - "COLL_q":{ - "pullReplicas":"0", - "replicationFactor":"3", - "shards":{"shard1":{ - "range":"80000000-7fffffff", - "state":"active", - "replicas":{ - "core_node3":{ - "core":"COLL_q_shard1_replica_n1", - "base_url":"http://N_7e/solr", - "node_name":"N_7e_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node6":{ - "core":"COLL_q_shard1_replica_n4", - "base_url":"http://N_4/solr", - "node_name":"N_4_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node10":{ - "core":"COLL_q_shard1_replica_n9", - "base_url":"http://N_0/solr", - "node_name":"N_0_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}}, - "router":{"name":"compositeId"}, - "autoAddReplicas":"true", - "nrtReplicas":"3", - "tlogReplicas":"0"}, - "COLL_22":{ - "pullReplicas":"0", - "replicationFactor":"3", - "shards":{"shard1":{ - "range":"80000000-7fffffff", - "state":"active", - "replicas":{ - "core_node5":{ - "core":"COLL_22_shard1_replica_n2", - "base_url":"http://N_4/solr", - "node_name":"N_4_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node6":{ - "core":"COLL_22_shard1_replica_n4", - "base_url":"http://N_7e/solr", - "node_name":"N_7e_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node10":{ - "core":"COLL_22_shard1_replica_n9", - "base_url":"http://N_0/solr", - "node_name":"N_0_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}}, - "router":{"name":"compositeId"}, - "autoAddReplicas":"true", - "nrtReplicas":"3", - "tlogReplicas":"0"}, - "COLL_1b":{ - "pullReplicas":"0", - "replicationFactor":"3", - "shards":{"shard1":{ - "range":"80000000-7fffffff", - "state":"active", - "replicas":{ - "core_node3":{ - "core":"COLL_1b_shard1_replica_n1", - "base_url":"http://N_7e/solr", - "node_name":"N_7e_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node6":{ - "core":"COLL_1b_shard1_replica_n4", - "base_url":"http://N_4/solr", - "node_name":"N_4_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node10":{ - "core":"COLL_1b_shard1_replica_n9", - "base_url":"http://N_0/solr", - "node_name":"N_0_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}}, - "router":{"name":"compositeId"}, - "autoAddReplicas":"true", - "nrtReplicas":"3", - "tlogReplicas":"0"}, - "COLL_5":{ - "pullReplicas":"0", - "replicationFactor":"1", - "shards":{"shard1":{ - "range":"80000000-7fffffff", - "state":"active", - "replicas":{"core_node2":{ - "core":"COLL_5_shard1_replica_n1", - "base_url":"http://N_7e/solr", - "node_name":"N_7e_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}}, - "router":{"name":"compositeId"}, - "autoAddReplicas":"true", - "nrtReplicas":"1", - "tlogReplicas":"0"}, - "COLL_1x":{ - "pullReplicas":"0", - "replicationFactor":"3", - "shards":{"shard1":{ - "range":"80000000-7fffffff", - "state":"active", - "replicas":{ - "core_node3":{ - "core":"COLL_1x_shard1_replica_n1", - "base_url":"http://N_7e/solr", - "node_name":"N_7e_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node6":{ - "core":"COLL_1x_shard1_replica_n4", - "base_url":"http://N_4/solr", - "node_name":"N_4_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node10":{ - "core":"COLL_1x_shard1_replica_n9", - "base_url":"http://N_0/solr", - "node_name":"N_0_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}}, - "router":{"name":"compositeId"}, - "autoAddReplicas":"true", - "nrtReplicas":"3", - "tlogReplicas":"0"}, - "COLL_l":{ - "pullReplicas":"0", - "replicationFactor":"3", - "shards":{"shard1":{ - "range":"80000000-7fffffff", - "state":"active", - "replicas":{ - "core_node3":{ - "core":"COLL_l_shard1_replica_n1", - "base_url":"http://N_7e/solr", - "node_name":"N_7e_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node6":{ - "core":"COLL_l_shard1_replica_n4", - "base_url":"http://N_4/solr", - "node_name":"N_4_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node10":{ - "core":"COLL_l_shard1_replica_n9", - "base_url":"http://N_0/solr", - "node_name":"N_0_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}}, - "router":{"name":"compositeId"}, - "autoAddReplicas":"true", - "nrtReplicas":"3", - "tlogReplicas":"0"}, - "COLL_0":{ - "pullReplicas":"0", - "replicationFactor":"3", - "shards":{ - "shard1":{ - "range":"80000000-d554ffff", - "state":"active", - "replicas":{ - "core_node3":{ - "core":"COLL_0_shard1_replica_n1", - "base_url":"http://N_d4/solr", - "node_name":"N_d4_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node5":{ - "core":"COLL_0_shard1_replica_n2", - "base_url":"http://N_16/solr", - "node_name":"N_16_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node7":{ - "core":"COLL_0_shard1_replica_n4", - "base_url":"http://N_74/solr", - "node_name":"N_74_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}}, - "shard2":{ - "range":"d5550000-2aa9ffff", - "state":"active", - "replicas":{ - "core_node9":{ - "core":"COLL_0_shard2_replica_n6", - "base_url":"http://N_b9/solr", - "node_name":"N_b9_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node11":{ - "core":"COLL_0_shard2_replica_n8", - "base_url":"http://N_3to/solr", - "node_name":"N_3to_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node13":{ - "core":"COLL_0_shard2_replica_n10", - "base_url":"http://N_13/solr", - "node_name":"N_13_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}}, - "shard3":{ - "range":"2aaa0000-7fffffff", - "state":"active", - "replicas":{ - "core_node15":{ - "core":"COLL_0_shard3_replica_n12", - "base_url":"http://N_do/solr", - "node_name":"N_do_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node17":{ - "core":"COLL_0_shard3_replica_n14", - "base_url":"http://N_3a/solr", - "node_name":"N_3a_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node18":{ - "core":"COLL_0_shard3_replica_n16", - "base_url":"http://N_v/solr", - "node_name":"N_v_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}}}, - "router":{"name":"compositeId"}, - "autoAddReplicas":"true", - "nrtReplicas":"3", - "tlogReplicas":"0"}, - "COLL_6":{ - "pullReplicas":"0", - "replicationFactor":"3", - "shards":{"shard1":{ - "range":"80000000-7fffffff", - "state":"active", - "replicas":{ - "core_node3":{ - "core":"COLL_6_shard1_replica_n1", - "base_url":"http://N_7e/solr", - "node_name":"N_7e_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node5":{ - "core":"COLL_6_shard1_replica_n2", - "base_url":"http://N_4/solr", - "node_name":"N_4_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node6":{ - "core":"COLL_6_shard1_replica_n4", - "base_url":"http://N_0/solr", - "node_name":"N_0_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}}, - "router":{"name":"compositeId"}, - "autoAddReplicas":"true", - "nrtReplicas":"3", - "tlogReplicas":"0"}}} diff --git a/solr/core/src/test-files/solr/simSnapshot/distribState.json b/solr/core/src/test-files/solr/simSnapshot/distribState.json deleted file mode 100644 index f59ab577e40..00000000000 --- a/solr/core/src/test-files/solr/simSnapshot/distribState.json +++ /dev/null @@ -1,206 +0,0 @@ -{ - "/clusterstate.json":{ - "owner":"0", - "mode":"PERSISTENT", - "version":0}, - "/live_nodes":{ - "owner":"0", - "mode":"PERSISTENT", - "version":0}, - "/live_nodes/N_11_solr":{ - "owner":"0", - "mode":"EPHEMERAL", - "version":0}, - "/live_nodes/N_65p_solr":{ - "owner":"0", - "mode":"EPHEMERAL", - "version":0}, - "/live_nodes/N_8_solr":{ - "owner":"0", - "mode":"EPHEMERAL", - "version":0}, - "/live_nodes/N_74_solr":{ - "owner":"0", - "mode":"EPHEMERAL", - "version":0}, - "/live_nodes/N_1i_solr":{ - "owner":"0", - "mode":"EPHEMERAL", - "version":0}, - "/live_nodes/N_4g_solr":{ - "owner":"0", - "mode":"EPHEMERAL", - "version":0}, - "/live_nodes/N_2_solr":{ - "owner":"0", - "mode":"EPHEMERAL", - "version":0}, - "/live_nodes/N_a_solr":{ - "owner":"0", - "mode":"EPHEMERAL", - "version":0}, - "/live_nodes/N_1c_solr":{ - "owner":"0", - "mode":"EPHEMERAL", - "version":0}, - "/live_nodes/N_16_solr":{ - "owner":"0", - "mode":"EPHEMERAL", - "version":0}, - "/live_nodes/N_6c_solr":{ - "owner":"0", - "mode":"EPHEMERAL", - "version":0}, - "/live_nodes/N_v_solr":{ - "owner":"0", - "mode":"EPHEMERAL", - "version":0}, - "/live_nodes/N_3a_solr":{ - "owner":"0", - "mode":"EPHEMERAL", - "version":0}, - "/live_nodes/N_1d_solr":{ - "owner":"0", - "mode":"EPHEMERAL", - "version":0}, - "/live_nodes/N_1_solr":{ - "owner":"0", - "mode":"EPHEMERAL", - "version":0}, - "/live_nodes/N_u_solr":{ - "owner":"0", - "mode":"EPHEMERAL", - "version":0}, - "/live_nodes/N_7_solr":{ - "owner":"0", - "mode":"EPHEMERAL", - "version":0}, - "/live_nodes/N_t_solr":{ - "owner":"0", - "mode":"EPHEMERAL", - "version":0}, - "/live_nodes/N_6i_solr":{ - "owner":"0", - "mode":"EPHEMERAL", - "version":0}, - "/live_nodes/N_d4_solr":{ - "owner":"0", - "mode":"EPHEMERAL", - "version":0}, - "/live_nodes/N_17_solr":{ - "owner":"0", - "mode":"EPHEMERAL", - "version":0}, - "/live_nodes/N_1f_solr":{ - "owner":"0", - "mode":"EPHEMERAL", - "version":0}, - "/live_nodes/N_do_solr":{ - "owner":"0", - "mode":"EPHEMERAL", - "version":0}, - "/live_nodes/N_3_solr":{ - "owner":"0", - "mode":"EPHEMERAL", - "version":0}, - "/live_nodes/N_303_solr":{ - "owner":"0", - "mode":"EPHEMERAL", - "version":0}, - "/live_nodes/N_29_solr":{ - "owner":"0", - "mode":"EPHEMERAL", - "version":0}, - "/live_nodes/N_9o_solr":{ - "owner":"0", - "mode":"EPHEMERAL", - "version":0}, - "/live_nodes/N_7e_solr":{ - "owner":"0", - "mode":"EPHEMERAL", - "version":0}, - "/live_nodes/N_1m_solr":{ - "owner":"0", - "mode":"EPHEMERAL", - "version":0}, - "/live_nodes/N_4_solr":{ - "owner":"0", - "mode":"EPHEMERAL", - "version":0}, - "/live_nodes/N_m_solr":{ - "owner":"0", - "mode":"EPHEMERAL", - "version":0}, - "/live_nodes/N_dj_solr":{ - "owner":"0", - "mode":"EPHEMERAL", - "version":0}, - "/live_nodes/N_e_solr":{ - "owner":"0", - "mode":"EPHEMERAL", - "version":0}, - "/live_nodes/N_13_solr":{ - "owner":"0", - "mode":"EPHEMERAL", - "version":0}, - "/live_nodes/N_g_solr":{ - "owner":"0", - "mode":"EPHEMERAL", - "version":0}, - "/live_nodes/N_2w_solr":{ - "owner":"0", - "mode":"EPHEMERAL", - "version":0}, - "/live_nodes/N_z_solr":{ - "owner":"0", - "mode":"EPHEMERAL", - "version":0}, - "/live_nodes/N_cs_solr":{ - "owner":"0", - "mode":"EPHEMERAL", - "version":0}, - "/live_nodes/N_3a7_solr":{ - "owner":"0", - "mode":"EPHEMERAL", - "version":0}, - "/live_nodes/N_aw_solr":{ - "owner":"0", - "mode":"EPHEMERAL", - "version":0}, - "/live_nodes/N_0_solr":{ - "owner":"0", - "mode":"EPHEMERAL", - "version":0}, - "/live_nodes/N_3to_solr":{ - "owner":"0", - "mode":"EPHEMERAL", - "version":0}, - "/live_nodes/N_4f_solr":{ - "owner":"0", - "mode":"EPHEMERAL", - "version":0}, - "/live_nodes/N_1h_solr":{ - "owner":"0", - "mode":"EPHEMERAL", - "version":0}, - "/live_nodes/N_2u_solr":{ - "owner":"0", - "mode":"EPHEMERAL", - "version":0}, - "/live_nodes/N_b9_solr":{ - "owner":"0", - "mode":"EPHEMERAL", - "version":0}, - "/live_nodes/N_6_solr":{ - "owner":"0", - "mode":"EPHEMERAL", - "version":0}, - "/live_nodes/N_5_solr":{ - "owner":"0", - "mode":"EPHEMERAL", - "version":0}, - "/autoscaling.json":{ - "owner":"0", - "mode":"PERSISTENT", - "data":"ewogICJjbHVzdGVyLXByZWZlcmVuY2VzIjpbCiAgICB7CiAgICAgICJtaW5pbWl6ZSI6ImNvcmVzIiwKICAgICAgInByZWNpc2lvbiI6MX0sCiAgICB7CiAgICAgICJtYXhpbWl6ZSI6ImZyZWVkaXNrIiwKICAgICAgInByZWNpc2lvbiI6MTB9XSwKICAiY2x1c3Rlci1wb2xpY3kiOlsKICAgIHsKICAgICAgInJlcGxpY2EiOiIjQUxMIiwKICAgICAgImNvbGxlY3Rpb24iOiJDT0xMXzIiLAogICAgICAic3lzcHJvcC5wb29sIjoicG9vbC0wMSJ9LAogICAgewogICAgICAicmVwbGljYSI6IiNBTEwiLAogICAgICAiY29sbGVjdGlvbiI6IkNPTExfMSIsCiAgICAgICJzeXNwcm9wLnBvb2wiOiJwb29sLTAyIn0sCiAgICB7CiAgICAgICJyZXBsaWNhIjoiI0FMTCIsCiAgICAgICJjb2xsZWN0aW9uIjoiQ09MTF8wIiwKICAgICAgInN5c3Byb3AucG9vbCI6InBvb2wtMDIifSwKICAgIHsKICAgICAgInJlcGxpY2EiOiI8MiIsCiAgICAgICJzaGFyZCI6IiNFQUNIIiwKICAgICAgIm5vZGUiOiIjQU5ZIn0sCiAgICB7CiAgICAgICJyZXBsaWNhIjoiI0VRVUFMIiwKICAgICAgInNoYXJkIjoiI0VBQ0giLAogICAgICAic3lzcHJvcC5heiI6IiNFQUNIIn1dLAogICJ0cmlnZ2VycyI6e30sCiAgImxpc3RlbmVycyI6e30sCiAgInByb3BlcnRpZXMiOnt9fQ==", - "version":0}} \ No newline at end of file diff --git a/solr/core/src/test-files/solr/simSnapshot/managerState.json b/solr/core/src/test-files/solr/simSnapshot/managerState.json deleted file mode 100644 index b96ebf4ca94..00000000000 --- a/solr/core/src/test-files/solr/simSnapshot/managerState.json +++ /dev/null @@ -1 +0,0 @@ -{"timeSource":"SimTimeSource:50.0"} \ No newline at end of file diff --git a/solr/core/src/test-files/solr/simSnapshot/nodeState.json b/solr/core/src/test-files/solr/simSnapshot/nodeState.json deleted file mode 100644 index e923736bada..00000000000 --- a/solr/core/src/test-files/solr/simSnapshot/nodeState.json +++ /dev/null @@ -1,3823 +0,0 @@ -{ - "nodeValues":{ - "N_7e_solr":{ - "node":"N_7e_solr", - "isLive":true, - "cores":13, - "freedisk":873.6022491455078, - "sysprop.pool":"pool-03", - "sysprop.az":"us-east-1b", - "totaldisk":999.51171875}, - "N_0_solr":{ - "node":"N_0_solr", - "isLive":true, - "cores":12, - "freedisk":719.6562576293945, - "sysprop.pool":"pool-03", - "sysprop.az":"us-east-1a", - "totaldisk":999.51171875}, - "N_4_solr":{ - "node":"N_4_solr", - "isLive":true, - "cores":12, - "freedisk":875.4758682250977, - "sysprop.pool":"pool-03", - "sysprop.az":"us-east-1c", - "totaldisk":999.51171875}, - "N_g_solr":{ - "node":"N_g_solr", - "isLive":true, - "cores":6, - "freedisk":4007.3253440856934, - "sysprop.pool":"pool-01", - "sysprop.az":"us-east-1a", - "totaldisk":4998.009765625}, - "N_17_solr":{ - "node":"N_17_solr", - "isLive":true, - "cores":6, - "freedisk":4093.756145477295, - "sysprop.pool":"pool-01", - "sysprop.az":"us-east-1a", - "totaldisk":4998.009765625}, - "N_303_solr":{ - "node":"N_303_solr", - "isLive":true, - "cores":6, - "freedisk":4111.4668045043945, - "sysprop.pool":"pool-01", - "sysprop.az":"us-east-1c", - "totaldisk":4998.009765625}, - "N_dj_solr":{ - "node":"N_dj_solr", - "isLive":true, - "cores":6, - "freedisk":4162.087951660156, - "sysprop.pool":"pool-01", - "sysprop.az":"us-east-1c", - "totaldisk":4998.009765625}, - "N_1c_solr":{ - "node":"N_1c_solr", - "isLive":true, - "cores":6, - "freedisk":4181.229598999023, - "sysprop.pool":"pool-01", - "sysprop.az":"us-east-1b", - "totaldisk":4998.009765625}, - "N_z_solr":{ - "node":"N_z_solr", - "isLive":true, - "cores":6, - "freedisk":4215.115695953369, - "sysprop.pool":"pool-01", - "sysprop.az":"us-east-1c", - "totaldisk":4998.009765625}, - "N_6_solr":{ - "node":"N_6_solr", - "isLive":true, - "cores":6, - "freedisk":4252.47643661499, - "sysprop.pool":"pool-01", - "sysprop.az":"us-east-1b", - "totaldisk":4998.009765625}, - "N_1m_solr":{ - "node":"N_1m_solr", - "isLive":true, - "cores":6, - "freedisk":4257.921604156494, - "sysprop.pool":"pool-01", - "sysprop.az":"us-east-1b", - "totaldisk":4998.009765625}, - "N_4g_solr":{ - "node":"N_4g_solr", - "isLive":true, - "cores":6, - "freedisk":4259.9677734375, - "sysprop.pool":"pool-01", - "sysprop.az":"us-east-1a", - "totaldisk":4998.009765625}, - "N_65p_solr":{ - "node":"N_65p_solr", - "isLive":true, - "cores":6, - "freedisk":4260.997627258301, - "sysprop.pool":"pool-01", - "sysprop.az":"us-east-1a", - "totaldisk":4998.009765625}, - "N_u_solr":{ - "node":"N_u_solr", - "isLive":true, - "cores":6, - "freedisk":4260.821304321289, - "sysprop.pool":"pool-01", - "sysprop.az":"us-east-1a", - "totaldisk":4998.009765625}, - "N_1f_solr":{ - "node":"N_1f_solr", - "isLive":true, - "cores":6, - "freedisk":4260.807849884033, - "sysprop.pool":"pool-01", - "sysprop.az":"us-east-1c", - "totaldisk":4998.009765625}, - "N_cs_solr":{ - "node":"N_cs_solr", - "isLive":true, - "cores":6, - "freedisk":4260.629165649414, - "sysprop.pool":"pool-01", - "sysprop.az":"us-east-1c", - "totaldisk":4998.009765625}, - "N_8_solr":{ - "node":"N_8_solr", - "isLive":true, - "cores":6, - "freedisk":4262.037788391113, - "sysprop.pool":"pool-01", - "sysprop.az":"us-east-1b", - "totaldisk":4998.009765625}, - "N_a_solr":{ - "node":"N_a_solr", - "isLive":true, - "cores":6, - "freedisk":4262.172649383545, - "sysprop.pool":"pool-01", - "sysprop.az":"us-east-1b", - "totaldisk":4998.009765625}, - "N_3a7_solr":{ - "node":"N_3a7_solr", - "isLive":true, - "cores":6, - "freedisk":4263.317134857178, - "sysprop.pool":"pool-01", - "sysprop.az":"us-east-1c", - "totaldisk":4998.009765625}, - "N_11_solr":{ - "node":"N_11_solr", - "isLive":true, - "cores":6, - "freedisk":4264.325901031494, - "sysprop.pool":"pool-01", - "sysprop.az":"us-east-1b", - "totaldisk":4998.009765625}, - "N_4f_solr":{ - "node":"N_4f_solr", - "isLive":true, - "cores":6, - "freedisk":4264.210151672363, - "sysprop.pool":"pool-01", - "sysprop.az":"us-east-1c", - "totaldisk":4998.009765625}, - "N_1i_solr":{ - "node":"N_1i_solr", - "isLive":true, - "cores":6, - "freedisk":4266.027156829834, - "sysprop.pool":"pool-01", - "sysprop.az":"us-east-1c", - "totaldisk":4998.009765625}, - "N_9o_solr":{ - "node":"N_9o_solr", - "isLive":true, - "cores":6, - "freedisk":4265.881809234619, - "sysprop.pool":"pool-01", - "sysprop.az":"us-east-1b", - "totaldisk":4998.009765625}, - "N_2_solr":{ - "node":"N_2_solr", - "isLive":true, - "cores":6, - "freedisk":4266.604637145996, - "sysprop.pool":"pool-01", - "sysprop.az":"us-east-1c", - "totaldisk":4998.009765625}, - "N_t_solr":{ - "node":"N_t_solr", - "isLive":true, - "cores":6, - "freedisk":4266.856658935547, - "sysprop.pool":"pool-01", - "sysprop.az":"us-east-1a", - "totaldisk":4998.009765625}, - "N_2u_solr":{ - "node":"N_2u_solr", - "isLive":true, - "cores":6, - "freedisk":4266.648368835449, - "sysprop.pool":"pool-01", - "sysprop.az":"us-east-1b", - "totaldisk":4998.009765625}, - "N_m_solr":{ - "node":"N_m_solr", - "isLive":true, - "cores":6, - "freedisk":4267.171646118164, - "sysprop.pool":"pool-01", - "sysprop.az":"us-east-1a", - "totaldisk":4998.009765625}, - "N_7_solr":{ - "node":"N_7_solr", - "isLive":true, - "cores":6, - "freedisk":4268.472709655762, - "sysprop.pool":"pool-01", - "sysprop.az":"us-east-1b", - "totaldisk":4998.009765625}, - "N_6c_solr":{ - "node":"N_6c_solr", - "isLive":true, - "cores":6, - "freedisk":4269.135753631592, - "sysprop.pool":"pool-01", - "sysprop.az":"us-east-1a", - "totaldisk":4998.009765625}, - "N_6i_solr":{ - "node":"N_6i_solr", - "isLive":true, - "cores":6, - "freedisk":4269.712917327881, - "sysprop.pool":"pool-01", - "sysprop.az":"us-east-1b", - "totaldisk":4998.009765625}, - "N_3_solr":{ - "node":"N_3_solr", - "isLive":true, - "cores":6, - "freedisk":4272.45711517334, - "sysprop.pool":"pool-01", - "sysprop.az":"us-east-1c", - "totaldisk":4998.009765625}, - "N_1d_solr":{ - "node":"N_1d_solr", - "isLive":true, - "cores":6, - "freedisk":4273.009799957275, - "sysprop.pool":"pool-01", - "sysprop.az":"us-east-1a", - "totaldisk":4998.009765625}, - "N_1_solr":{ - "node":"N_1_solr", - "isLive":true, - "cores":6, - "freedisk":4274.765396118164, - "sysprop.pool":"pool-01", - "sysprop.az":"us-east-1a", - "totaldisk":4998.009765625}, - "N_aw_solr":{ - "node":"N_aw_solr", - "isLive":true, - "cores":6, - "freedisk":4276.759601593018, - "sysprop.pool":"pool-01", - "sysprop.az":"us-east-1c", - "totaldisk":4998.009765625}, - "N_1h_solr":{ - "node":"N_1h_solr", - "isLive":true, - "cores":6, - "freedisk":4297.329685211182, - "sysprop.pool":"pool-01", - "sysprop.az":"us-east-1b", - "totaldisk":4998.009765625}, - "N_29_solr":{ - "node":"N_29_solr", - "isLive":true, - "cores":6, - "freedisk":4303.548599243164, - "sysprop.pool":"pool-01", - "sysprop.az":"us-east-1a", - "totaldisk":4998.009765625}, - "N_e_solr":{ - "node":"N_e_solr", - "isLive":true, - "cores":6, - "freedisk":4334.874732971191, - "sysprop.pool":"pool-01", - "sysprop.az":"us-east-1c", - "totaldisk":4998.009765625}, - "N_2w_solr":{ - "node":"N_2w_solr", - "isLive":true, - "cores":6, - "freedisk":4336.208312988281, - "sysprop.pool":"pool-01", - "sysprop.az":"us-east-1b", - "totaldisk":4998.009765625}, - "N_5_solr":{ - "node":"N_5_solr", - "isLive":true, - "cores":6, - "freedisk":4397.149795532227, - "sysprop.pool":"pool-01", - "sysprop.az":"us-east-1a", - "totaldisk":4998.009765625}, - "N_do_solr":{ - "node":"N_do_solr", - "isLive":true, - "cores":5, - "freedisk":407.25314712524414, - "sysprop.pool":"pool-02", - "sysprop.az":"us-east-1c", - "totaldisk":999.51171875}, - "N_3a_solr":{ - "node":"N_3a_solr", - "isLive":true, - "cores":5, - "freedisk":407.706729888916, - "sysprop.pool":"pool-02", - "sysprop.az":"us-east-1b", - "totaldisk":999.51171875}, - "N_v_solr":{ - "node":"N_v_solr", - "isLive":true, - "cores":5, - "freedisk":412.18456649780273, - "sysprop.pool":"pool-02", - "sysprop.az":"us-east-1a", - "totaldisk":999.51171875}, - "N_13_solr":{ - "node":"N_13_solr", - "isLive":true, - "cores":5, - "freedisk":718.1634063720703, - "sysprop.pool":"pool-02", - "sysprop.az":"us-east-1c", - "totaldisk":999.51171875}, - "N_3to_solr":{ - "node":"N_3to_solr", - "isLive":true, - "cores":5, - "freedisk":794.5433731079102, - "sysprop.pool":"pool-02", - "sysprop.az":"us-east-1a", - "totaldisk":999.51171875}, - "N_16_solr":{ - "node":"N_16_solr", - "isLive":true, - "cores":5, - "freedisk":795.7872657775879, - "sysprop.pool":"pool-02", - "sysprop.az":"us-east-1b", - "totaldisk":999.51171875}, - "N_d4_solr":{ - "node":"N_d4_solr", - "isLive":true, - "cores":5, - "freedisk":797.2159843444824, - "sysprop.pool":"pool-02", - "sysprop.az":"us-east-1c", - "totaldisk":999.51171875}, - "N_b9_solr":{ - "node":"N_b9_solr", - "isLive":true, - "cores":5, - "freedisk":801.2417984008789, - "sysprop.pool":"pool-02", - "sysprop.az":"us-east-1b", - "totaldisk":999.51171875}, - "N_74_solr":{ - "node":"N_74_solr", - "isLive":true, - "cores":5, - "freedisk":802.5921897888184, - "sysprop.pool":"pool-02", - "sysprop.az":"us-east-1a", - "totaldisk":999.51171875}}, - "replicaInfos":{ - "N_7e_solr":{ - "COLL_22":{"shard1":[{"core_node6":{ - "core":"COLL_22_shard1_replica_n4", - "shard":"shard1", - "collection":"COLL_22", - "node_name":"N_7e_solr", - "type":"NRT", - "base_url":"http://N_7e/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":2.4348956483E10, - "INDEX.sizeInGB":22.676732840947807}}]}, - "COLL_q":{"shard1":[{"core_node3":{ - "core":"COLL_q_shard1_replica_n1", - "shard":"shard1", - "collection":"COLL_q", - "node_name":"N_7e_solr", - "type":"NRT", - "base_url":"http://N_7e/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":4.9242789E8, - "INDEX.sizeInGB":0.45860921032726765}}]}, - "COLL_1b":{"shard1":[{"core_node3":{ - "core":"COLL_1b_shard1_replica_n1", - "shard":"shard1", - "collection":"COLL_1b", - "node_name":"N_7e_solr", - "type":"NRT", - "base_url":"http://N_7e/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":135.0, - "INDEX.sizeInGB":1.257285475730896E-7}}]}, - "COLL_1t":{"shard1":[{"core_node3":{ - "core":"COLL_1t_shard1_replica_n1", - "shard":"shard1", - "collection":"COLL_1t", - "node_name":"N_7e_solr", - "type":"NRT", - "base_url":"http://N_7e/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":8.5774407615E10, - "INDEX.sizeInGB":79.8836421361193}}]}, - "COLL_x":{"shard1":[{"core_node3":{ - "core":"COLL_x_shard1_replica_n1", - "shard":"shard1", - "collection":"COLL_x", - "node_name":"N_7e_solr", - "type":"NRT", - "base_url":"http://N_7e/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":3.18270873E8, - "INDEX.sizeInGB":0.296412848867476}}]}, - "COLL_2k":{"shard1":[{"core_node3":{ - "core":"COLL_2k_shard1_replica_n1", - "shard":"shard1", - "collection":"COLL_2k", - "node_name":"N_7e_solr", - "type":"NRT", - "base_url":"http://N_7e/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":135.0, - "INDEX.sizeInGB":1.257285475730896E-7}}]}, - "COLL_1r":{"shard1":[{"core_node3":{ - "core":"COLL_1r_shard1_replica_n1", - "shard":"shard1", - "collection":"COLL_1r", - "node_name":"N_7e_solr", - "type":"NRT", - "base_url":"http://N_7e/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":4.12015174E8, - "INDEX.sizeInGB":0.38371903263032436}}]}, - "COLL_8":{"shard1":[{"core_node3":{ - "core":"COLL_8_shard1_replica_n1", - "shard":"shard1", - "collection":"COLL_8", - "node_name":"N_7e_solr", - "type":"NRT", - "base_url":"http://N_7e/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":356048.0, - "INDEX.sizeInGB":3.315955400466919E-4}}]}, - "COLL_5":{"shard1":[{"core_node2":{ - "core":"COLL_5_shard1_replica_n1", - "shard":"shard1", - "collection":"COLL_5", - "node_name":"N_7e_solr", - "type":"NRT", - "base_url":"http://N_7e/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":5.854396964E9, - "INDEX.sizeInGB":5.452332053333521}}]}, - "COLL_l":{"shard1":[{"core_node3":{ - "core":"COLL_l_shard1_replica_n1", - "shard":"shard1", - "collection":"COLL_l", - "node_name":"N_7e_solr", - "type":"NRT", - "base_url":"http://N_7e/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":135.0, - "INDEX.sizeInGB":1.257285475730896E-7}}]}, - "COLL_1x":{"shard1":[{"core_node3":{ - "core":"COLL_1x_shard1_replica_n1", - "shard":"shard1", - "collection":"COLL_1x", - "node_name":"N_7e_solr", - "type":"NRT", - "base_url":"http://N_7e/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":4248411.0, - "INDEX.sizeInGB":0.00395664107054472}}]}, - "COLL_4":{"shard1":[{"core_node3":{ - "core":"COLL_4_shard1_replica_n1", - "shard":"shard1", - "collection":"COLL_4", - "node_name":"N_7e_solr", - "type":"NRT", - "base_url":"http://N_7e/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":2.58881858E8, - "INDEX.sizeInGB":0.2411025185137987}}]}, - "COLL_6":{"shard1":[{"core_node3":{ - "core":"COLL_6_shard1_replica_n1", - "shard":"shard1", - "collection":"COLL_6", - "node_name":"N_7e_solr", - "type":"NRT", - "base_url":"http://N_7e/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.6446420654E10, - "INDEX.sizeInGB":15.316922826692462}}]}}, - "N_0_solr":{ - "COLL_22":{"shard1":[{"core_node10":{ - "core":"COLL_22_shard1_replica_n9", - "shard":"shard1", - "collection":"COLL_22", - "node_name":"N_0_solr", - "type":"NRT", - "base_url":"http://N_0/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":2.4351639993E10, - "INDEX.sizeInGB":22.679232054390013}}]}, - "COLL_q":{"shard1":[{"core_node10":{ - "core":"COLL_q_shard1_replica_n9", - "shard":"shard1", - "collection":"COLL_q", - "node_name":"N_0_solr", - "type":"NRT", - "base_url":"http://N_0/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":4.9242789E8, - "INDEX.sizeInGB":0.45860921032726765}}]}, - "COLL_1b":{"shard1":[{"core_node10":{ - "core":"COLL_1b_shard1_replica_n9", - "shard":"shard1", - "collection":"COLL_1b", - "node_name":"N_0_solr", - "type":"NRT", - "base_url":"http://N_0/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":135.0, - "INDEX.sizeInGB":1.257285475730896E-7}}]}, - "COLL_1t":{"shard1":[{"core_node5":{ - "core":"COLL_1t_shard1_replica_n2", - "shard":"shard1", - "collection":"COLL_1t", - "node_name":"N_0_solr", - "type":"NRT", - "base_url":"http://N_0/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":8.7485800719E10, - "INDEX.sizeInGB":81.47750116791576}}]}, - "COLL_x":{"shard1":[{"core_node10":{ - "core":"COLL_x_shard1_replica_n9", - "shard":"shard1", - "collection":"COLL_x", - "node_name":"N_0_solr", - "type":"NRT", - "base_url":"http://N_0/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":3.0928583E8, - "INDEX.sizeInGB":0.2880448754876852}}]}, - "COLL_2k":{"shard1":[{"core_node10":{ - "core":"COLL_2k_shard1_replica_n9", - "shard":"shard1", - "collection":"COLL_2k", - "node_name":"N_0_solr", - "type":"NRT", - "base_url":"http://N_0/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":135.0, - "INDEX.sizeInGB":1.257285475730896E-7}}]}, - "COLL_1r":{"shard1":[{"core_node5":{ - "core":"COLL_1r_shard1_replica_n2", - "shard":"shard1", - "collection":"COLL_1r", - "node_name":"N_0_solr", - "type":"NRT", - "base_url":"http://N_0/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":4.25884524E8, - "INDEX.sizeInGB":0.39663587138056755}}]}, - "COLL_8":{"shard1":[{"core_node5":{ - "core":"COLL_8_shard1_replica_n2", - "shard":"shard1", - "collection":"COLL_8", - "node_name":"N_0_solr", - "type":"NRT", - "base_url":"http://N_0/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":399225.0, - "INDEX.sizeInGB":3.718072548508644E-4}}]}, - "COLL_l":{"shard1":[{"core_node10":{ - "core":"COLL_l_shard1_replica_n9", - "shard":"shard1", - "collection":"COLL_l", - "node_name":"N_0_solr", - "type":"NRT", - "base_url":"http://N_0/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":135.0, - "INDEX.sizeInGB":1.257285475730896E-7}}]}, - "COLL_1x":{"shard1":[{"core_node10":{ - "core":"COLL_1x_shard1_replica_n9", - "shard":"shard1", - "collection":"COLL_1x", - "node_name":"N_0_solr", - "type":"NRT", - "base_url":"http://N_0/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":4264901.0, - "INDEX.sizeInGB":0.003971998579800129}}]}, - "COLL_4":{"shard1":[{"core_node5":{ - "core":"COLL_4_shard1_replica_n2", - "shard":"shard1", - "collection":"COLL_4", - "node_name":"N_0_solr", - "type":"NRT", - "base_url":"http://N_0/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":2.58797271E8, - "INDEX.sizeInGB":0.24102374073117971}}]}, - "COLL_6":{"shard1":[{"core_node6":{ - "core":"COLL_6_shard1_replica_n4", - "shard":"shard1", - "collection":"COLL_6", - "node_name":"N_0_solr", - "type":"NRT", - "base_url":"http://N_0/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":4.4921656871E10, - "INDEX.sizeInGB":41.83655313309282}}]}}, - "N_4_solr":{ - "COLL_22":{"shard1":[{"core_node5":{ - "core":"COLL_22_shard1_replica_n2", - "shard":"shard1", - "collection":"COLL_22", - "node_name":"N_4_solr", - "type":"NRT", - "base_url":"http://N_4/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":2.436290627E10, - "INDEX.sizeInGB":22.689724592491984}}]}, - "COLL_q":{"shard1":[{"core_node6":{ - "core":"COLL_q_shard1_replica_n4", - "shard":"shard1", - "collection":"COLL_q", - "node_name":"N_4_solr", - "type":"NRT", - "base_url":"http://N_4/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":4.9242789E8, - "INDEX.sizeInGB":0.45860921032726765}}]}, - "COLL_1b":{"shard1":[{"core_node6":{ - "core":"COLL_1b_shard1_replica_n4", - "shard":"shard1", - "collection":"COLL_1b", - "node_name":"N_4_solr", - "type":"NRT", - "base_url":"http://N_4/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":135.0, - "INDEX.sizeInGB":1.257285475730896E-7}}]}, - "COLL_1t":{"shard1":[{"core_node6":{ - "core":"COLL_1t_shard1_replica_n4", - "shard":"shard1", - "collection":"COLL_1t", - "node_name":"N_4_solr", - "type":"NRT", - "base_url":"http://N_4/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":8.5380419785E10, - "INDEX.sizeInGB":79.51671237591654}}]}, - "COLL_x":{"shard1":[{"core_node6":{ - "core":"COLL_x_shard1_replica_n4", - "shard":"shard1", - "collection":"COLL_x", - "node_name":"N_4_solr", - "type":"NRT", - "base_url":"http://N_4/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":3.03301808E8, - "INDEX.sizeInGB":0.28247182071208954}}]}, - "COLL_2k":{"shard1":[{"core_node6":{ - "core":"COLL_2k_shard1_replica_n4", - "shard":"shard1", - "collection":"COLL_2k", - "node_name":"N_4_solr", - "type":"NRT", - "base_url":"http://N_4/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":135.0, - "INDEX.sizeInGB":1.257285475730896E-7}}]}, - "COLL_1r":{"shard1":[{"core_node6":{ - "core":"COLL_1r_shard1_replica_n4", - "shard":"shard1", - "collection":"COLL_1r", - "node_name":"N_4_solr", - "type":"NRT", - "base_url":"http://N_4/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":4.46826689E8, - "INDEX.sizeInGB":0.4161397824063897}}]}, - "COLL_8":{"shard1":[{"core_node6":{ - "core":"COLL_8_shard1_replica_n4", - "shard":"shard1", - "collection":"COLL_8", - "node_name":"N_4_solr", - "type":"NRT", - "base_url":"http://N_4/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":356048.0, - "INDEX.sizeInGB":3.315955400466919E-4}}]}, - "COLL_l":{"shard1":[{"core_node6":{ - "core":"COLL_l_shard1_replica_n4", - "shard":"shard1", - "collection":"COLL_l", - "node_name":"N_4_solr", - "type":"NRT", - "base_url":"http://N_4/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":135.0, - "INDEX.sizeInGB":1.257285475730896E-7}}]}, - "COLL_1x":{"shard1":[{"core_node6":{ - "core":"COLL_1x_shard1_replica_n4", - "shard":"shard1", - "collection":"COLL_1x", - "node_name":"N_4_solr", - "type":"NRT", - "base_url":"http://N_4/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":4255591.0, - "INDEX.sizeInGB":0.003963327966630459}}]}, - "COLL_4":{"shard1":[{"core_node6":{ - "core":"COLL_4_shard1_replica_n4", - "shard":"shard1", - "collection":"COLL_4", - "node_name":"N_4_solr", - "type":"NRT", - "base_url":"http://N_4/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":2.59832461E8, - "INDEX.sizeInGB":0.2419878365471959}}]}, - "COLL_6":{"shard1":[{"core_node5":{ - "core":"COLL_6_shard1_replica_n2", - "shard":"shard1", - "collection":"COLL_6", - "node_name":"N_4_solr", - "type":"NRT", - "base_url":"http://N_4/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":2.0738852096E10, - "INDEX.sizeInGB":19.314561128616333}}]}}, - "N_g_solr":{"COLL_2":{ - "shard2_1_0":[{"core_node1681":{ - "core":"COLL_2_shard2_1_0_replica_n1680", - "shard":"shard2_1_0", - "collection":"COLL_2", - "node_name":"N_g_solr", - "type":"NRT", - "base_url":"http://N_g/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.3012044407E11, - "INDEX.sizeInGB":121.18410698138177}}], - "shard5_0_1":[{"core_node1771":{ - "core":"COLL_2_shard5_0_1_replica_n1770", - "shard":"shard5_0_1", - "collection":"COLL_2", - "node_name":"N_g_solr", - "type":"NRT", - "base_url":"http://N_g/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.31464210597E11, - "INDEX.sizeInGB":122.43558708298951}}], - "shard5_1_0":[{"core_node1783":{ - "core":"COLL_2_shard5_1_0_replica_n1782", - "shard":"shard5_1_0", - "collection":"COLL_2", - "node_name":"N_g_solr", - "type":"NRT", - "base_url":"http://N_g/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.30012462556E11, - "INDEX.sizeInGB":121.08354135975242}}], - "shard5_1_1":[{"core_node861":{ - "core":"COLL_2_shard5_1_1_replica_n859", - "shard":"shard5_1_1", - "collection":"COLL_2", - "node_name":"N_g_solr", - "type":"NRT", - "base_url":"http://N_g/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.29967769078E11, - "INDEX.sizeInGB":121.04191731475294}}], - "shard5_0_0":[{"core_node1769":{ - "core":"COLL_2_shard5_0_0_replica_n1768", - "shard":"shard5_0_0", - "collection":"COLL_2", - "node_name":"N_g_solr", - "type":"NRT", - "base_url":"http://N_g/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.31922267714E11, - "INDEX.sizeInGB":122.8621860165149}}], - "shard9_0_0":[{"core_node1683":{ - "core":"COLL_2_shard9_0_0_replica_n1682", - "shard":"shard9_0_0", - "collection":"COLL_2", - "node_name":"N_g_solr", - "type":"NRT", - "property.preferredleader":"true", - "base_url":"http://N_g/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.29248772716E11, - "INDEX.sizeInGB":120.37229977175593}}]}}, - "N_17_solr":{"COLL_2":{ - "shard11_1_1":[{"core_node768":{ - "core":"COLL_2_shard11_1_1_replica_n762", - "shard":"shard11_1_1", - "collection":"COLL_2", - "node_name":"N_17_solr", - "type":"NRT", - "base_url":"http://N_17/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.30871431234E11, - "INDEX.sizeInGB":121.88351828046143}}], - "shard14_0_0":[{"core_node1121":{ - "core":"COLL_2_shard14_0_0_replica_n1120", - "shard":"shard14_0_0", - "collection":"COLL_2", - "node_name":"N_17_solr", - "type":"NRT", - "base_url":"http://N_17/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.3029908264E11, - "INDEX.sizeInGB":121.3504771143198}}], - "shard18_0_1":[{"core_node877":{ - "core":"COLL_2_shard18_0_1_replica_n2", - "shard":"shard18_0_1", - "collection":"COLL_2", - "node_name":"N_17_solr", - "type":"NRT", - "base_url":"http://N_17/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.28174988934E11, - "INDEX.sizeInGB":119.37226069532335}}], - "shard12_0_1":[{"core_node1699":{ - "core":"COLL_2_shard12_0_1_replica_n1698", - "shard":"shard12_0_1", - "collection":"COLL_2", - "node_name":"N_17_solr", - "type":"NRT", - "base_url":"http://N_17/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.30350286057E11, - "INDEX.sizeInGB":121.39816401246935}}], - "shard12_0_0":[{"core_node1751":{ - "core":"COLL_2_shard12_0_0_replica_n1750", - "shard":"shard12_0_0", - "collection":"COLL_2", - "node_name":"N_17_solr", - "type":"NRT", - "base_url":"http://N_17/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.2936875619E11, - "INDEX.sizeInGB":120.48404308967292}}], - "shard14_0_1":[{"core_node1123":{ - "core":"COLL_2_shard14_0_1_replica_n1122", - "shard":"shard14_0_1", - "collection":"COLL_2", - "node_name":"N_17_solr", - "type":"NRT", - "base_url":"http://N_17/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.31146492351E11, - "INDEX.sizeInGB":122.13968890812248}}]}}, - "N_303_solr":{"COLL_2":{ - "shard16_0_1":[{"core_node987":{ - "core":"COLL_2_shard16_0_1_replica_n986", - "shard":"shard16_0_1", - "collection":"COLL_2", - "node_name":"N_303_solr", - "type":"NRT", - "base_url":"http://N_303/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.30738903625E11, - "INDEX.sizeInGB":121.76009232643992}}], - "shard16_0_0":[{"core_node1785":{ - "core":"COLL_2_shard16_0_0_replica_n1784", - "shard":"shard16_0_0", - "collection":"COLL_2", - "node_name":"N_303_solr", - "type":"NRT", - "base_url":"http://N_303/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.26747476604E11, - "INDEX.sizeInGB":118.04278623685241}}], - "shard3_0_0":[{"core_node544":{ - "core":"COLL_2_shard3_0_0_replica_n2", - "shard":"shard3_0_0", - "collection":"COLL_2", - "node_name":"N_303_solr", - "type":"NRT", - "base_url":"http://N_303/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.29792212268E11, - "INDEX.sizeInGB":120.87841729447246}}], - "shard9_1_1":[{"core_node1163":{ - "core":"COLL_2_shard9_1_1_replica_n1162", - "shard":"shard9_1_1", - "collection":"COLL_2", - "node_name":"N_303_solr", - "type":"NRT", - "base_url":"http://N_303/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.36568824379E11, - "INDEX.sizeInGB":127.18962913285941}}], - "shard9_1_0":[{"core_node1151":{ - "core":"COLL_2_shard9_1_0_replica_n1150", - "shard":"shard9_1_0", - "collection":"COLL_2", - "node_name":"N_303_solr", - "type":"NRT", - "base_url":"http://N_303/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.31117387108E11, - "INDEX.sizeInGB":122.11258253827691}}], - "shard4_0_1":[{"core_node1773":{ - "core":"COLL_2_shard4_0_1_replica_n1772", - "shard":"shard4_0_1", - "collection":"COLL_2", - "node_name":"N_303_solr", - "type":"NRT", - "base_url":"http://N_303/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.28126128215E11, - "INDEX.sizeInGB":119.3267556047067}}]}}, - "N_dj_solr":{"COLL_2":{ - "shard1_1_0":[{"core_node471":{ - "core":"COLL_2_shard1_1_0_replica_n1", - "shard":"shard1_1_0", - "collection":"COLL_2", - "node_name":"N_dj_solr", - "type":"NRT", - "base_url":"http://N_dj/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.29057719236E11, - "INDEX.sizeInGB":120.19436735287309}}], - "shard7_1_0":[{"core_node928":{ - "core":"COLL_2_shard7_1_0_replica_n926", - "shard":"shard7_1_0", - "collection":"COLL_2", - "node_name":"N_dj_solr", - "type":"NRT", - "base_url":"http://N_dj/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.29963886019E11, - "INDEX.sizeInGB":121.03830093424767}}], - "shard7_1_1":[{"core_node941":{ - "core":"COLL_2_shard7_1_1_replica_n927", - "shard":"shard7_1_1", - "collection":"COLL_2", - "node_name":"N_dj_solr", - "type":"NRT", - "base_url":"http://N_dj/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.28538540188E11, - "INDEX.sizeInGB":119.71084418520331}}], - "shard18_0_1":[{"core_node773":{ - "core":"COLL_2_shard18_0_1_replica_n771", - "shard":"shard18_0_1", - "collection":"COLL_2", - "node_name":"N_dj_solr", - "type":"NRT", - "base_url":"http://N_dj/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.30821199599E11, - "INDEX.sizeInGB":121.83673642482609}}], - "shard13_0_1":[{"core_node1715":{ - "core":"COLL_2_shard13_0_1_replica_n1714", - "shard":"shard13_0_1", - "collection":"COLL_2", - "node_name":"N_dj_solr", - "type":"NRT", - "base_url":"http://N_dj/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.30355121703E11, - "INDEX.sizeInGB":121.402667558752}}], - "shard13_0_0":[{"core_node1749":{ - "core":"COLL_2_shard13_0_0_replica_n1748", - "shard":"shard13_0_0", - "collection":"COLL_2", - "node_name":"N_dj_solr", - "type":"NRT", - "base_url":"http://N_dj/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.30427736106E11, - "INDEX.sizeInGB":121.47029499150813}}]}}, - "N_1c_solr":{"COLL_2":{ - "shard5_0_1":[{"core_node1703":{ - "core":"COLL_2_shard5_0_1_replica_n1702", - "shard":"shard5_0_1", - "collection":"COLL_2", - "node_name":"N_1c_solr", - "type":"NRT", - "base_url":"http://N_1c/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.31521149156E11, - "INDEX.sizeInGB":122.48861524835229}}], - "shard5_1_0":[{"core_node1135":{ - "core":"COLL_2_shard5_1_0_replica_n1134", - "shard":"shard5_1_0", - "collection":"COLL_2", - "node_name":"N_1c_solr", - "type":"NRT", - "base_url":"http://N_1c/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.30030877168E11, - "INDEX.sizeInGB":121.1006913036108}}], - "shard18_0_0":[{"core_node874":{ - "core":"COLL_2_shard18_0_0_replica_n1", - "shard":"shard18_0_0", - "collection":"COLL_2", - "node_name":"N_1c_solr", - "type":"NRT", - "base_url":"http://N_1c/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.28011422432E11, - "INDEX.sizeInGB":119.21992751955986}}], - "shard5_1_1":[{"core_node1141":{ - "core":"COLL_2_shard5_1_1_replica_n1140", - "shard":"shard5_1_1", - "collection":"COLL_2", - "node_name":"N_1c_solr", - "type":"NRT", - "base_url":"http://N_1c/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.29917464329E11, - "INDEX.sizeInGB":120.99506736639887}}], - "shard5_0_0":[{"core_node999":{ - "core":"COLL_2_shard5_0_0_replica_n998", - "shard":"shard5_0_0", - "collection":"COLL_2", - "node_name":"N_1c_solr", - "type":"NRT", - "base_url":"http://N_1c/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.31937405764E11, - "INDEX.sizeInGB":122.87628442421556}}], - "shard18_0_1":[{"core_node876":{ - "core":"COLL_2_shard18_0_1_replica_n1", - "shard":"shard18_0_1", - "collection":"COLL_2", - "node_name":"N_1c_solr", - "type":"NRT", - "base_url":"http://N_1c/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.30729375574E11, - "INDEX.sizeInGB":121.75121863745153}}]}}, - "N_z_solr":{"COLL_2":{ - "shard1_0_0":[{"core_node1717":{ - "core":"COLL_2_shard1_0_0_replica_n1716", - "shard":"shard1_0_0", - "collection":"COLL_2", - "node_name":"N_z_solr", - "type":"NRT", - "base_url":"http://N_z/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":5.7185112146E10, - "INDEX.sizeInGB":53.25778587348759}}], - "shard8_1_0":[{"core_node1707":{ - "core":"COLL_2_shard8_1_0_replica_n1706", - "shard":"shard8_1_0", - "collection":"COLL_2", - "node_name":"N_z_solr", - "type":"NRT", - "base_url":"http://N_z/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.35679630668E11, - "INDEX.sizeInGB":126.361502956599}}], - "shard8_0_0":[{"core_node1731":{ - "core":"COLL_2_shard8_0_0_replica_n1730", - "shard":"shard8_0_0", - "collection":"COLL_2", - "node_name":"N_z_solr", - "type":"NRT", - "base_url":"http://N_z/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.30170301246E11, - "INDEX.sizeInGB":121.23054009489715}}], - "shard8_0_1":[{"core_node1695":{ - "core":"COLL_2_shard8_0_1_replica_n1694", - "shard":"shard8_0_1", - "collection":"COLL_2", - "node_name":"N_z_solr", - "type":"NRT", - "base_url":"http://N_z/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.39918850407E11, - "INDEX.sizeInGB":130.30958399828523}}], - "shard8_1_1":[{"core_node1755":{ - "core":"COLL_2_shard8_1_1_replica_n1754", - "shard":"shard8_1_1", - "collection":"COLL_2", - "node_name":"N_z_solr", - "type":"NRT", - "base_url":"http://N_z/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.33314153125E11, - "INDEX.sizeInGB":124.15848032105714}}], - "shard14_1_0":[{"core_node1127":{ - "core":"COLL_2_shard14_1_0_replica_n1126", - "shard":"shard14_1_0", - "collection":"COLL_2", - "node_name":"N_z_solr", - "type":"NRT", - "base_url":"http://N_z/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.27443177079E11, - "INDEX.sizeInGB":118.69070779439062}}]}}, - "N_6_solr":{"COLL_2":{ - "shard8_1_0":[{"core_node1811":{ - "core":"COLL_2_shard8_1_0_replica_n1810", - "shard":"shard8_1_0", - "collection":"COLL_2", - "node_name":"N_6_solr", - "type":"NRT", - "base_url":"http://N_6/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.35679249773E11, - "INDEX.sizeInGB":126.36114822048694}}], - "shard4_0_0":[{"core_node520":{ - "core":"COLL_2_shard4_0_0_replica_n2", - "shard":"shard4_0_0", - "collection":"COLL_2", - "node_name":"N_6_solr", - "type":"NRT", - "base_url":"http://N_6/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.28680029361E11, - "INDEX.sizeInGB":119.84261624608189}}], - "shard4_0_1":[{"core_node1803":{ - "core":"COLL_2_shard4_0_1_replica_n1802", - "shard":"shard4_0_1", - "collection":"COLL_2", - "node_name":"N_6_solr", - "type":"NRT", - "base_url":"http://N_6/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.28153346526E11, - "INDEX.sizeInGB":119.35210463218391}}], - "shard9_0_0":[{"core_node1799":{ - "core":"COLL_2_shard9_0_0_replica_n1798", - "shard":"shard9_0_0", - "collection":"COLL_2", - "node_name":"N_6_solr", - "type":"NRT", - "base_url":"http://N_6/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.35157081196E11, - "INDEX.sizeInGB":125.874840836972}}], - "shard3_1_0":[{"core_node459":{ - "core":"COLL_2_shard3_1_0_replica_n1", - "shard":"shard3_1_0", - "collection":"COLL_2", - "node_name":"N_6_solr", - "type":"NRT", - "base_url":"http://N_6/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.32652501535E11, - "INDEX.sizeInGB":123.54226925875992}}], - "shard15_1_1":[{"core_node1709":{ - "core":"COLL_2_shard15_1_1_replica_n1708", - "shard":"shard15_1_1", - "collection":"COLL_2", - "node_name":"N_6_solr", - "type":"NRT", - "base_url":"http://N_6/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.30846984322E11, - "INDEX.sizeInGB":121.86075031943619}}]}}, - "N_1m_solr":{"COLL_2":{ - "shard6_1_1":[{"core_node1745":{ - "core":"COLL_2_shard6_1_1_replica_n1744", - "shard":"shard6_1_1", - "collection":"COLL_2", - "node_name":"N_1m_solr", - "type":"NRT", - "base_url":"http://N_1m/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.31273933482E11, - "INDEX.sizeInGB":122.25837771035731}}], - "shard1_1_0":[{"core_node1679":{ - "core":"COLL_2_shard1_1_0_replica_n1678", - "shard":"shard1_1_0", - "collection":"COLL_2", - "node_name":"N_1m_solr", - "type":"NRT", - "base_url":"http://N_1m/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.28970690262E11, - "INDEX.sizeInGB":120.11331530474126}}], - "shard8_0_0":[{"core_node887":{ - "core":"COLL_2_shard8_0_0_replica_n886", - "shard":"shard8_0_0", - "collection":"COLL_2", - "node_name":"N_1m_solr", - "type":"NRT", - "base_url":"http://N_1m/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.30145902623E11, - "INDEX.sizeInGB":121.20781710650772}}], - "shard8_0_1":[{"core_node893":{ - "core":"COLL_2_shard8_0_1_replica_n892", - "shard":"shard8_0_1", - "collection":"COLL_2", - "node_name":"N_1m_solr", - "type":"NRT", - "base_url":"http://N_1m/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.32681734677E11, - "INDEX.sizeInGB":123.56949474383146}}], - "shard8_1_1":[{"core_node1711":{ - "core":"COLL_2_shard8_1_1_replica_n1710", - "shard":"shard8_1_1", - "collection":"COLL_2", - "node_name":"N_1m_solr", - "type":"NRT", - "base_url":"http://N_1m/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.33374089494E11, - "INDEX.sizeInGB":124.21430041454732}}], - "shard6_1_0":[{"core_node1167":{ - "core":"COLL_2_shard6_1_0_replica_n1166", - "shard":"shard6_1_0", - "collection":"COLL_2", - "node_name":"N_1m_solr", - "type":"NRT", - "base_url":"http://N_1m/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.29376799009E11, - "INDEX.sizeInGB":120.49153354857117}}]}}, - "N_4g_solr":{"COLL_2":{ - "shard8_1_1":[{"core_node1795":{ - "core":"COLL_2_shard8_1_1_replica_n1794", - "shard":"shard8_1_1", - "collection":"COLL_2", - "node_name":"N_4g_solr", - "type":"NRT", - "base_url":"http://N_4g/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.33276674177E11, - "INDEX.sizeInGB":124.1235753307119}}], - "shard9_1_1":[{"core_node944":{ - "core":"COLL_2_shard9_1_1_replica_n930", - "shard":"shard9_1_1", - "collection":"COLL_2", - "node_name":"N_4g_solr", - "type":"NRT", - "base_url":"http://N_4g/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.33928213329E11, - "INDEX.sizeInGB":124.73036845121533}}], - "shard9_1_0":[{"core_node931":{ - "core":"COLL_2_shard9_1_0_replica_n929", - "shard":"shard9_1_0", - "collection":"COLL_2", - "node_name":"N_4g_solr", - "type":"NRT", - "base_url":"http://N_4g/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.31111103315E11, - "INDEX.sizeInGB":122.1067303000018}}], - "shard18_1_1":[{"core_node626":{ - "core":"COLL_2_shard18_1_1_replica_n624", - "shard":"shard18_1_1", - "collection":"COLL_2", - "node_name":"N_4g_solr", - "type":"NRT", - "base_url":"http://N_4g/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.28190099634E11, - "INDEX.sizeInGB":119.38633363135159}}], - "shard18_1_0":[{"core_node625":{ - "core":"COLL_2_shard18_1_0_replica_n623", - "shard":"shard18_1_0", - "collection":"COLL_2", - "node_name":"N_4g_solr", - "type":"NRT", - "base_url":"http://N_4g/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.28955475131E11, - "INDEX.sizeInGB":120.09914510976523}}], - "shard2_1_1":[{"core_node1813":{ - "core":"COLL_2_shard2_1_1_replica_n1812", - "shard":"shard2_1_1", - "collection":"COLL_2", - "node_name":"N_4g_solr", - "type":"NRT", - "base_url":"http://N_4g/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.28164947427E11, - "INDEX.sizeInGB":119.36290881317109}}]}}, - "N_65p_solr":{"COLL_2":{ - "shard7_0_0":[{"core_node774":{ - "core":"COLL_2_shard7_0_0_replica_n1", - "shard":"shard7_0_0", - "collection":"COLL_2", - "node_name":"N_65p_solr", - "type":"NRT", - "base_url":"http://N_65p/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.29027793373E11, - "INDEX.sizeInGB":120.16649672109634}}], - "shard10_1_0":[{"core_node1797":{ - "core":"COLL_2_shard10_1_0_replica_n1796", - "shard":"shard10_1_0", - "collection":"COLL_2", - "node_name":"N_65p_solr", - "type":"NRT", - "base_url":"http://N_65p/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.27583656591E11, - "INDEX.sizeInGB":118.82153953518718}}], - "shard3_0_0":[{"core_node543":{ - "core":"COLL_2_shard3_0_0_replica_n1", - "shard":"shard3_0_0", - "collection":"COLL_2", - "node_name":"N_65p_solr", - "type":"NRT", - "base_url":"http://N_65p/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.29871412511E11, - "INDEX.sizeInGB":120.95217826869339}}], - "shard3_0_1":[{"core_node545":{ - "core":"COLL_2_shard3_0_1_replica_n1", - "shard":"shard3_0_1", - "collection":"COLL_2", - "node_name":"N_65p_solr", - "type":"NRT", - "base_url":"http://N_65p/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.31838835644E11, - "INDEX.sizeInGB":122.784483846277}}], - "shard15_1_0":[{"core_node1173":{ - "core":"COLL_2_shard15_1_0_replica_n1172", - "shard":"shard15_1_0", - "collection":"COLL_2", - "node_name":"N_65p_solr", - "type":"NRT", - "base_url":"http://N_65p/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.33316507698E11, - "INDEX.sizeInGB":124.16067318804562}}], - "shard15_1_1":[{"core_node1747":{ - "core":"COLL_2_shard15_1_1_replica_n1746", - "shard":"shard15_1_1", - "collection":"COLL_2", - "node_name":"N_65p_solr", - "type":"NRT", - "base_url":"http://N_65p/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.30883359905E11, - "INDEX.sizeInGB":121.89462772104889}}]}}, - "N_u_solr":{"COLL_2":{ - "shard8_1_0":[{"core_node1765":{ - "core":"COLL_2_shard8_1_0_replica_n1764", - "shard":"shard8_1_0", - "collection":"COLL_2", - "node_name":"N_u_solr", - "type":"NRT", - "base_url":"http://N_u/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.35571920799E11, - "INDEX.sizeInGB":126.26119032409042}}], - "shard13_1_1":[{"core_node921":{ - "core":"COLL_2_shard13_1_1_replica_n920", - "shard":"shard13_1_1", - "collection":"COLL_2", - "node_name":"N_u_solr", - "type":"NRT", - "base_url":"http://N_u/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.29634542289E11, - "INDEX.sizeInGB":120.73157568369061}}], - "shard15_0_1":[{"core_node734":{ - "core":"COLL_2_shard15_0_1_replica_n2", - "shard":"shard15_0_1", - "collection":"COLL_2", - "node_name":"N_u_solr", - "type":"NRT", - "base_url":"http://N_u/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.27250282639E11, - "INDEX.sizeInGB":118.51106084790081}}], - "shard13_0_1":[{"core_node1263":{ - "core":"COLL_2_shard13_0_1_replica_n1262", - "shard":"shard13_0_1", - "collection":"COLL_2", - "node_name":"N_u_solr", - "type":"NRT", - "base_url":"http://N_u/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.30321828131E11, - "INDEX.sizeInGB":121.37166050355881}}], - "shard13_1_0":[{"core_node1763":{ - "core":"COLL_2_shard13_1_0_replica_n1762", - "shard":"shard13_1_0", - "collection":"COLL_2", - "node_name":"N_u_solr", - "type":"NRT", - "base_url":"http://N_u/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.29567251239E11, - "INDEX.sizeInGB":120.66890600975603}}], - "shard13_0_0":[{"core_node1257":{ - "core":"COLL_2_shard13_0_0_replica_n1256", - "shard":"shard13_0_0", - "collection":"COLL_2", - "node_name":"N_u_solr", - "type":"NRT", - "base_url":"http://N_u/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.30381429251E11, - "INDEX.sizeInGB":121.42716837208718}}]}}, - "N_1f_solr":{"COLL_2":{ - "shard11_0_1":[{"core_node1223":{ - "core":"COLL_2_shard11_0_1_replica_n1222", - "shard":"shard11_0_1", - "collection":"COLL_2", - "node_name":"N_1f_solr", - "type":"NRT", - "base_url":"http://N_1f/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.27989218509E11, - "INDEX.sizeInGB":119.19924850482494}}], - "shard11_1_0":[{"core_node779":{ - "core":"COLL_2_shard11_1_0_replica_n778", - "shard":"shard11_1_0", - "collection":"COLL_2", - "node_name":"N_1f_solr", - "type":"NRT", - "base_url":"http://N_1f/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.32552454912E11, - "INDEX.sizeInGB":123.44909358024597}}], - "shard11_0_0":[{"core_node1217":{ - "core":"COLL_2_shard11_0_0_replica_n1216", - "shard":"shard11_0_0", - "collection":"COLL_2", - "node_name":"N_1f_solr", - "type":"NRT", - "base_url":"http://N_1f/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.27720861488E11, - "INDEX.sizeInGB":118.94932155311108}}], - "shard11_1_1":[{"core_node783":{ - "core":"COLL_2_shard11_1_1_replica_n782", - "shard":"shard11_1_1", - "collection":"COLL_2", - "node_name":"N_1f_solr", - "type":"NRT", - "base_url":"http://N_1f/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.30995783614E11, - "INDEX.sizeInGB":121.99933045916259}}], - "shard5_0_1":[{"core_node1003":{ - "core":"COLL_2_shard5_0_1_replica_n1002", - "shard":"shard5_0_1", - "collection":"COLL_2", - "node_name":"N_1f_solr", - "type":"NRT", - "base_url":"http://N_1f/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.31534942129E11, - "INDEX.sizeInGB":122.50146095547825}}], - "shard5_0_0":[{"core_node1001":{ - "core":"COLL_2_shard5_0_0_replica_n1000", - "shard":"shard5_0_0", - "collection":"COLL_2", - "node_name":"N_1f_solr", - "type":"NRT", - "base_url":"http://N_1f/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.31960210955E11, - "INDEX.sizeInGB":122.89752341341227}}]}}, - "N_cs_solr":{"COLL_2":{ - "shard6_1_1":[{"core_node1705":{ - "core":"COLL_2_shard6_1_1_replica_n1704", - "shard":"shard6_1_1", - "collection":"COLL_2", - "node_name":"N_cs_solr", - "type":"NRT", - "base_url":"http://N_cs/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.31274462707E11, - "INDEX.sizeInGB":122.25887058954686}}], - "shard10_0_1":[{"core_node828":{ - "core":"COLL_2_shard10_0_1_replica_n826", - "shard":"shard10_0_1", - "collection":"COLL_2", - "node_name":"N_cs_solr", - "type":"NRT", - "base_url":"http://N_cs/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.28038688927E11, - "INDEX.sizeInGB":119.245321421884}}], - "shard6_1_0":[{"core_node937":{ - "core":"COLL_2_shard6_1_0_replica_n935", - "shard":"shard6_1_0", - "collection":"COLL_2", - "node_name":"N_cs_solr", - "type":"NRT", - "base_url":"http://N_cs/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.29597529819E11, - "INDEX.sizeInGB":120.69710513483733}}], - "shard15_1_0":[{"core_node955":{ - "core":"COLL_2_shard15_1_0_replica_n953", - "shard":"shard15_1_0", - "collection":"COLL_2", - "node_name":"N_cs_solr", - "type":"NRT", - "base_url":"http://N_cs/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.33515745782E11, - "INDEX.sizeInGB":124.34622811339796}}], - "shard10_0_0":[{"core_node827":{ - "core":"COLL_2_shard10_0_0_replica_n825", - "shard":"shard10_0_0", - "collection":"COLL_2", - "node_name":"N_cs_solr", - "type":"NRT", - "base_url":"http://N_cs/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.29486149433E11, - "INDEX.sizeInGB":120.59337406698614}}], - "shard15_1_1":[{"core_node956":{ - "core":"COLL_2_shard15_1_1_replica_n954", - "shard":"shard15_1_1", - "collection":"COLL_2", - "node_name":"N_cs_solr", - "type":"NRT", - "base_url":"http://N_cs/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.30865977458E11, - "INDEX.sizeInGB":121.87843905575573}}]}}, - "N_8_solr":{"COLL_2":{ - "shard16_1_1":[{"core_node853":{ - "core":"COLL_2_shard16_1_1_replica_n851", - "shard":"shard16_1_1", - "collection":"COLL_2", - "node_name":"N_8_solr", - "type":"NRT", - "base_url":"http://N_8/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.33685050832E11, - "INDEX.sizeInGB":124.50390572845936}}], - "shard16_0_1":[{"core_node857":{ - "core":"COLL_2_shard16_0_1_replica_n855", - "shard":"shard16_0_1", - "collection":"COLL_2", - "node_name":"N_8_solr", - "type":"NRT", - "base_url":"http://N_8/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.30788718518E11, - "INDEX.sizeInGB":121.80648606084287}}], - "shard16_1_0":[{"core_node852":{ - "core":"COLL_2_shard16_1_0_replica_n850", - "shard":"shard16_1_0", - "collection":"COLL_2", - "node_name":"N_8_solr", - "type":"NRT", - "base_url":"http://N_8/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.28801317856E11, - "INDEX.sizeInGB":119.95557495951653}}], - "shard16_0_0":[{"core_node856":{ - "core":"COLL_2_shard16_0_0_replica_n854", - "shard":"shard16_0_0", - "collection":"COLL_2", - "node_name":"N_8_solr", - "type":"NRT", - "base_url":"http://N_8/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.2677230126E11, - "INDEX.sizeInGB":118.06590599939227}}], - "shard2_0_0":[{"core_node796":{ - "core":"COLL_2_shard2_0_0_replica_n794", - "shard":"shard2_0_0", - "collection":"COLL_2", - "node_name":"N_8_solr", - "type":"NRT", - "base_url":"http://N_8/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.29517293483E11, - "INDEX.sizeInGB":120.6223792238161}}], - "shard2_0_1":[{"core_node800":{ - "core":"COLL_2_shard2_0_1_replica_n795", - "shard":"shard2_0_1", - "collection":"COLL_2", - "node_name":"N_8_solr", - "type":"NRT", - "base_url":"http://N_8/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.31328007233E11, - "INDEX.sizeInGB":122.30873781535774}}]}}, - "N_a_solr":{"COLL_2":{ - "shard3_0_0":[{"core_node1809":{ - "core":"COLL_2_shard3_0_0_replica_n1808", - "shard":"shard3_0_0", - "collection":"COLL_2", - "node_name":"N_a_solr", - "type":"NRT", - "base_url":"http://N_a/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.29798330608E11, - "INDEX.sizeInGB":120.88411544263363}}], - "shard14_0_0":[{"core_node1119":{ - "core":"COLL_2_shard14_0_0_replica_n1118", - "shard":"shard14_0_0", - "collection":"COLL_2", - "node_name":"N_a_solr", - "type":"NRT", - "base_url":"http://N_a/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.30313698451E11, - "INDEX.sizeInGB":121.36408914905041}}], - "shard15_1_0":[{"core_node1175":{ - "core":"COLL_2_shard15_1_0_replica_n1174", - "shard":"shard15_1_0", - "collection":"COLL_2", - "node_name":"N_a_solr", - "type":"NRT", - "base_url":"http://N_a/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.33321224738E11, - "INDEX.sizeInGB":124.16506627388299}}], - "shard14_1_1":[{"core_node836":{ - "core":"COLL_2_shard14_1_1_replica_n834", - "shard":"shard14_1_1", - "collection":"COLL_2", - "node_name":"N_a_solr", - "type":"NRT", - "base_url":"http://N_a/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.29318568492E11, - "INDEX.sizeInGB":120.43730215355754}}], - "shard14_0_1":[{"core_node1125":{ - "core":"COLL_2_shard14_0_1_replica_n1124", - "shard":"shard14_0_1", - "collection":"COLL_2", - "node_name":"N_a_solr", - "type":"NRT", - "base_url":"http://N_a/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.31102045065E11, - "INDEX.sizeInGB":122.09829414729029}}], - "shard14_1_0":[{"core_node835":{ - "core":"COLL_2_shard14_1_0_replica_n833", - "shard":"shard14_1_0", - "collection":"COLL_2", - "node_name":"N_a_solr", - "type":"NRT", - "base_url":"http://N_a/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.27418065808E11, - "INDEX.sizeInGB":118.66732110083103}}]}}, - "N_3a7_solr":{"COLL_2":{ - "shard7_0_0":[{"core_node775":{ - "core":"COLL_2_shard7_0_0_replica_n2", - "shard":"shard7_0_0", - "collection":"COLL_2", - "node_name":"N_3a7_solr", - "type":"NRT", - "base_url":"http://N_3a7/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.29074533898E11, - "INDEX.sizeInGB":120.21002722717822}}], - "shard2_0_0":[{"core_node1823":{ - "core":"COLL_2_shard2_0_0_replica_n1822", - "shard":"shard2_0_0", - "collection":"COLL_2", - "node_name":"N_3a7_solr", - "type":"NRT", - "base_url":"http://N_3a7/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.29476268104E11, - "INDEX.sizeInGB":120.58417136222124}}], - "shard14_0_0":[{"core_node839":{ - "core":"COLL_2_shard14_0_0_replica_n837", - "shard":"shard14_0_0", - "collection":"COLL_2", - "node_name":"N_3a7_solr", - "type":"NRT", - "base_url":"http://N_3a7/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.30330451538E11, - "INDEX.sizeInGB":121.37969167716801}}], - "shard3_1_1":[{"core_node462":{ - "core":"COLL_2_shard3_1_1_replica_n2", - "shard":"shard3_1_1", - "collection":"COLL_2", - "node_name":"N_3a7_solr", - "type":"NRT", - "base_url":"http://N_3a7/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.2992912768E11, - "INDEX.sizeInGB":121.00592970848083}}], - "shard14_1_1":[{"core_node1825":{ - "core":"COLL_2_shard14_1_1_replica_n1824", - "shard":"shard14_1_1", - "collection":"COLL_2", - "node_name":"N_3a7_solr", - "type":"NRT", - "base_url":"http://N_3a7/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.300425186E11, - "INDEX.sizeInGB":121.11153323203325}}], - "shard14_0_1":[{"core_node841":{ - "core":"COLL_2_shard14_0_1_replica_n838", - "shard":"shard14_0_1", - "collection":"COLL_2", - "node_name":"N_3a7_solr", - "type":"NRT", - "base_url":"http://N_3a7/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.31168916273E11, - "INDEX.sizeInGB":122.1605728128925}}]}}, - "N_11_solr":{"COLL_2":{ - "shard6_0_0":[{"core_node1210":{ - "core":"COLL_2_shard6_0_0_replica_n1209", - "shard":"shard6_0_0", - "collection":"COLL_2", - "node_name":"N_11_solr", - "type":"NRT", - "base_url":"http://N_11/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.28939953876E11, - "INDEX.sizeInGB":120.08468981459737}}], - "shard6_0_1":[{"core_node1212":{ - "core":"COLL_2_shard6_0_1_replica_n1211", - "shard":"shard6_0_1", - "collection":"COLL_2", - "node_name":"N_11_solr", - "type":"NRT", - "base_url":"http://N_11/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.28744354495E11, - "INDEX.sizeInGB":119.90252369549125}}], - "shard9_1_1":[{"core_node1155":{ - "core":"COLL_2_shard9_1_1_replica_n1154", - "shard":"shard9_1_1", - "collection":"COLL_2", - "node_name":"N_11_solr", - "type":"NRT", - "base_url":"http://N_11/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.33894519282E11, - "INDEX.sizeInGB":124.69898842461407}}], - "shard9_1_0":[{"core_node1153":{ - "core":"COLL_2_shard9_1_0_replica_n1152", - "shard":"shard9_1_0", - "collection":"COLL_2", - "node_name":"N_11_solr", - "type":"NRT", - "base_url":"http://N_11/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.31406038908E11, - "INDEX.sizeInGB":122.3814104758203}}], - "shard9_0_1":[{"core_node438":{ - "core":"COLL_2_shard9_0_1_replica_n436", - "shard":"shard9_0_1", - "collection":"COLL_2", - "node_name":"N_11_solr", - "type":"NRT", - "base_url":"http://N_11/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.29282915395E11, - "INDEX.sizeInGB":120.40409761946648}}], - "shard12_1_1":[{"core_node662":{ - "core":"COLL_2_shard12_1_1_replica_n2", - "shard":"shard12_1_1", - "collection":"COLL_2", - "node_name":"N_11_solr", - "type":"NRT", - "base_url":"http://N_11/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.26693447901E11, - "INDEX.sizeInGB":117.99246808607131}}]}}, - "N_4f_solr":{"COLL_2":{ - "shard2_0_1":[{"core_node915":{ - "core":"COLL_2_shard2_0_1_replica_n914", - "shard":"shard2_0_1", - "collection":"COLL_2", - "node_name":"N_4f_solr", - "type":"NRT", - "base_url":"http://N_4f/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.31386626219E11, - "INDEX.sizeInGB":122.36333100032061}}], - "shard2_1_0":[{"core_node975":{ - "core":"COLL_2_shard2_1_0_replica_n974", - "shard":"shard2_1_0", - "collection":"COLL_2", - "node_name":"N_4f_solr", - "type":"NRT", - "base_url":"http://N_4f/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.3001251468E11, - "INDEX.sizeInGB":121.0835899040103}}], - "shard6_0_0":[{"core_node1182":{ - "core":"COLL_2_shard6_0_0_replica_n1180", - "shard":"shard6_0_0", - "collection":"COLL_2", - "node_name":"N_4f_solr", - "type":"NRT", - "base_url":"http://N_4f/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.28922958966E11, - "INDEX.sizeInGB":120.06886207126081}}], - "shard6_0_1":[{"core_node1189":{ - "core":"COLL_2_shard6_0_1_replica_n1181", - "shard":"shard6_0_1", - "collection":"COLL_2", - "node_name":"N_4f_solr", - "type":"NRT", - "base_url":"http://N_4f/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.28773562289E11, - "INDEX.sizeInGB":119.92972557339817}}], - "shard3_0_1":[{"core_node546":{ - "core":"COLL_2_shard3_0_1_replica_n2", - "shard":"shard3_0_1", - "collection":"COLL_2", - "node_name":"N_4f_solr", - "type":"NRT", - "base_url":"http://N_4f/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.31838927317E11, - "INDEX.sizeInGB":122.78456922341138}}], - "shard2_1_1":[{"core_node1685":{ - "core":"COLL_2_shard2_1_1_replica_n1684", - "shard":"shard2_1_1", - "collection":"COLL_2", - "node_name":"N_4f_solr", - "type":"NRT", - "base_url":"http://N_4f/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.2812596905E11, - "INDEX.sizeInGB":119.32660737074912}}]}}, - "N_1i_solr":{"COLL_2":{ - "shard17_1_0":[{"core_node1200":{ - "core":"COLL_2_shard17_1_0_replica_n1198", - "shard":"shard17_1_0", - "collection":"COLL_2", - "node_name":"N_1i_solr", - "type":"NRT", - "base_url":"http://N_1i/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.29069936299E11, - "INDEX.sizeInGB":120.20574537944049}}], - "shard17_0_1":[{"core_node1117":{ - "core":"COLL_2_shard17_0_1_replica_n1116", - "shard":"shard17_0_1", - "collection":"COLL_2", - "node_name":"N_1i_solr", - "type":"NRT", - "base_url":"http://N_1i/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.30694171889E11, - "INDEX.sizeInGB":121.71843265090138}}], - "shard10_1_1":[{"core_node1779":{ - "core":"COLL_2_shard10_1_1_replica_n1778", - "shard":"shard10_1_1", - "collection":"COLL_2", - "node_name":"N_1i_solr", - "type":"NRT", - "base_url":"http://N_1i/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.30255789623E11, - "INDEX.sizeInGB":121.31015735026449}}], - "shard17_0_0":[{"core_node1781":{ - "core":"COLL_2_shard17_0_0_replica_n1780", - "shard":"shard17_0_0", - "collection":"COLL_2", - "node_name":"N_1i_solr", - "type":"NRT", - "base_url":"http://N_1i/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.30702509646E11, - "INDEX.sizeInGB":121.72619779221714}}], - "shard10_1_0":[{"core_node1693":{ - "core":"COLL_2_shard10_1_0_replica_n1692", - "shard":"shard10_1_0", - "collection":"COLL_2", - "node_name":"N_1i_solr", - "type":"NRT", - "base_url":"http://N_1i/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.27561685082E11, - "INDEX.sizeInGB":118.80107697285712}}], - "shard17_1_1":[{"core_node1203":{ - "core":"COLL_2_shard17_1_1_replica_n1199", - "shard":"shard17_1_1", - "collection":"COLL_2", - "node_name":"N_1i_solr", - "type":"NRT", - "base_url":"http://N_1i/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.28764084367E11, - "INDEX.sizeInGB":119.92089857067913}}]}}, - "N_9o_solr":{"COLL_2":{ - "shard11_0_1":[{"core_node1221":{ - "core":"COLL_2_shard11_0_1_replica_n1220", - "shard":"shard11_0_1", - "collection":"COLL_2", - "node_name":"N_9o_solr", - "type":"NRT", - "base_url":"http://N_9o/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.28020049235E11, - "INDEX.sizeInGB":119.22796185594052}}], - "shard11_1_0":[{"core_node781":{ - "core":"COLL_2_shard11_1_0_replica_n780", - "shard":"shard11_1_0", - "collection":"COLL_2", - "node_name":"N_9o_solr", - "type":"NRT", - "base_url":"http://N_9o/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.32420261013E11, - "INDEX.sizeInGB":123.32597841788083}}], - "shard11_0_0":[{"core_node1219":{ - "core":"COLL_2_shard11_0_0_replica_n1218", - "shard":"shard11_0_0", - "collection":"COLL_2", - "node_name":"N_9o_solr", - "type":"NRT", - "base_url":"http://N_9o/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.28002391411E11, - "INDEX.sizeInGB":119.21151672583073}}], - "shard7_0_0":[{"core_node766":{ - "core":"COLL_2_shard7_0_0_replica_n764", - "shard":"shard7_0_0", - "collection":"COLL_2", - "node_name":"N_9o_solr", - "type":"NRT", - "base_url":"http://N_9o/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.28994593549E11, - "INDEX.sizeInGB":120.13557697553188}}], - "shard11_1_1":[{"core_node785":{ - "core":"COLL_2_shard11_1_1_replica_n784", - "shard":"shard11_1_1", - "collection":"COLL_2", - "node_name":"N_9o_solr", - "type":"NRT", - "base_url":"http://N_9o/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.30909357727E11, - "INDEX.sizeInGB":121.91884007956833}}], - "shard7_0_1":[{"core_node769":{ - "core":"COLL_2_shard7_0_1_replica_n765", - "shard":"shard7_0_1", - "collection":"COLL_2", - "node_name":"N_9o_solr", - "type":"NRT", - "base_url":"http://N_9o/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.28908501869E11, - "INDEX.sizeInGB":120.0553978504613}}]}}, - "N_2_solr":{"COLL_2":{ - "shard5_1_0":[{"core_node1137":{ - "core":"COLL_2_shard5_1_0_replica_n1136", - "shard":"shard5_1_0", - "collection":"COLL_2", - "node_name":"N_2_solr", - "type":"NRT", - "base_url":"http://N_2/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":7.6877250282E10, - "INDEX.sizeInGB":71.59751866199076}}], - "shard5_1_1":[{"core_node1139":{ - "core":"COLL_2_shard5_1_1_replica_n1138", - "shard":"shard5_1_1", - "collection":"COLL_2", - "node_name":"N_2_solr", - "type":"NRT", - "base_url":"http://N_2/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.29952609098E11, - "INDEX.sizeInGB":121.02779848314822}}], - "shard7_0_1":[{"core_node776":{ - "core":"COLL_2_shard7_0_1_replica_n1", - "shard":"shard7_0_1", - "collection":"COLL_2", - "node_name":"N_2_solr", - "type":"NRT", - "base_url":"http://N_2/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.2890128588E11, - "INDEX.sizeInGB":120.04867743700743}}], - "shard9_0_1":[{"core_node478":{ - "core":"COLL_2_shard9_0_1_replica_n2", - "shard":"shard9_0_1", - "collection":"COLL_2", - "node_name":"N_2_solr", - "type":"NRT", - "base_url":"http://N_2/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.29212951693E11, - "INDEX.sizeInGB":120.33893884439021}}], - "shard12_0_1":[{"core_node1255":{ - "core":"COLL_2_shard12_0_1_replica_n1254", - "shard":"shard12_0_1", - "collection":"COLL_2", - "node_name":"N_2_solr", - "type":"NRT", - "base_url":"http://N_2/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.30384315739E11, - "INDEX.sizeInGB":121.42985662352294}}], - "shard12_0_0":[{"core_node1249":{ - "core":"COLL_2_shard12_0_0_replica_n1248", - "shard":"shard12_0_0", - "collection":"COLL_2", - "node_name":"N_2_solr", - "type":"NRT", - "base_url":"http://N_2/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.29421522442E11, - "INDEX.sizeInGB":120.53318549133837}}]}}, - "N_t_solr":{"COLL_2":{ - "shard11_0_1":[{"core_node1195":{ - "core":"COLL_2_shard11_0_1_replica_n1184", - "shard":"shard11_0_1", - "collection":"COLL_2", - "node_name":"N_t_solr", - "type":"NRT", - "base_url":"http://N_t/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.27980394382E11, - "INDEX.sizeInGB":119.19103039614856}}], - "shard11_1_0":[{"core_node1791":{ - "core":"COLL_2_shard11_1_0_replica_n1790", - "shard":"shard11_1_0", - "collection":"COLL_2", - "node_name":"N_t_solr", - "type":"NRT", - "base_url":"http://N_t/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.32416023485E11, - "INDEX.sizeInGB":123.32203191239387}}], - "shard11_0_0":[{"core_node1185":{ - "core":"COLL_2_shard11_0_0_replica_n1183", - "shard":"shard11_0_0", - "collection":"COLL_2", - "node_name":"N_t_solr", - "type":"NRT", - "base_url":"http://N_t/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.2777477116E11, - "INDEX.sizeInGB":118.99952884763479}}], - "shard10_1_1":[{"core_node1743":{ - "core":"COLL_2_shard10_1_1_replica_n1742", - "shard":"shard10_1_1", - "collection":"COLL_2", - "node_name":"N_t_solr", - "type":"NRT", - "base_url":"http://N_t/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.30757016285E11, - "INDEX.sizeInGB":121.77696105558425}}], - "shard10_0_1":[{"core_node905":{ - "core":"COLL_2_shard10_0_1_replica_n904", - "shard":"shard10_0_1", - "collection":"COLL_2", - "node_name":"N_t_solr", - "type":"NRT", - "base_url":"http://N_t/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.28142990156E11, - "INDEX.sizeInGB":119.34245951101184}}], - "shard10_0_0":[{"core_node1733":{ - "core":"COLL_2_shard10_0_0_replica_n1732", - "shard":"shard10_0_0", - "collection":"COLL_2", - "node_name":"N_t_solr", - "type":"NRT", - "base_url":"http://N_t/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.2914349283E11, - "INDEX.sizeInGB":120.27425023727119}}]}}, - "N_2u_solr":{"COLL_2":{ - "shard17_1_0":[{"core_node1225":{ - "core":"COLL_2_shard17_1_0_replica_n1224", - "shard":"shard17_1_0", - "collection":"COLL_2", - "node_name":"N_2u_solr", - "type":"NRT", - "base_url":"http://N_2u/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.29066474889E11, - "INDEX.sizeInGB":120.20252169016749}}], - "shard17_0_1":[{"core_node1115":{ - "core":"COLL_2_shard17_0_1_replica_n1114", - "shard":"shard17_0_1", - "collection":"COLL_2", - "node_name":"N_2u_solr", - "type":"NRT", - "base_url":"http://N_2u/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.30049647193E11, - "INDEX.sizeInGB":121.1181722516194}}], - "shard17_0_0":[{"core_node1735":{ - "core":"COLL_2_shard17_0_0_replica_n1734", - "shard":"shard17_0_0", - "collection":"COLL_2", - "node_name":"N_2u_solr", - "type":"NRT", - "base_url":"http://N_2u/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.31102615765E11, - "INDEX.sizeInGB":122.09882565308362}}], - "shard3_1_1":[{"core_node461":{ - "core":"COLL_2_shard3_1_1_replica_n1", - "shard":"shard3_1_1", - "collection":"COLL_2", - "node_name":"N_2u_solr", - "type":"NRT", - "base_url":"http://N_2u/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.29953637358E11, - "INDEX.sizeInGB":121.02875612489879}}], - "shard17_1_1":[{"core_node1231":{ - "core":"COLL_2_shard17_1_1_replica_n1230", - "shard":"shard17_1_1", - "collection":"COLL_2", - "node_name":"N_2u_solr", - "type":"NRT", - "base_url":"http://N_2u/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.287734207E11, - "INDEX.sizeInGB":119.92959370836616}}], - "shard12_1_0":[{"core_node660":{ - "core":"COLL_2_shard12_1_0_replica_n2", - "shard":"shard12_1_0", - "collection":"COLL_2", - "node_name":"N_2u_solr", - "type":"NRT", - "base_url":"http://N_2u/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.27387972534E11, - "INDEX.sizeInGB":118.63929455541074}}]}}, - "N_m_solr":{"COLL_2":{ - "shard6_1_1":[{"core_node1171":{ - "core":"COLL_2_shard6_1_1_replica_n1170", - "shard":"shard6_1_1", - "collection":"COLL_2", - "node_name":"N_m_solr", - "type":"NRT", - "base_url":"http://N_m/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.31256081422E11, - "INDEX.sizeInGB":122.24175168387592}}], - "shard17_1_0":[{"core_node1227":{ - "core":"COLL_2_shard17_1_0_replica_n1226", - "shard":"shard17_1_0", - "collection":"COLL_2", - "node_name":"N_m_solr", - "type":"NRT", - "base_url":"http://N_m/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.29049722959E11, - "INDEX.sizeInGB":120.18692023959011}}], - "shard6_0_0":[{"core_node1208":{ - "core":"COLL_2_shard6_0_0_replica_n1207", - "shard":"shard6_0_0", - "collection":"COLL_2", - "node_name":"N_m_solr", - "type":"NRT", - "base_url":"http://N_m/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.28936808614E11, - "INDEX.sizeInGB":120.08176056109369}}], - "shard6_0_1":[{"core_node1214":{ - "core":"COLL_2_shard6_0_1_replica_n1213", - "shard":"shard6_0_1", - "collection":"COLL_2", - "node_name":"N_m_solr", - "type":"NRT", - "base_url":"http://N_m/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.28745543493E11, - "INDEX.sizeInGB":119.90363103616983}}], - "shard9_0_1":[{"core_node477":{ - "core":"COLL_2_shard9_0_1_replica_n1", - "shard":"shard9_0_1", - "collection":"COLL_2", - "node_name":"N_m_solr", - "type":"NRT", - "base_url":"http://N_m/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.29063920601E11, - "INDEX.sizeInGB":120.20014282409102}}], - "shard17_1_1":[{"core_node1229":{ - "core":"COLL_2_shard17_1_1_replica_n1228", - "shard":"shard17_1_1", - "collection":"COLL_2", - "node_name":"N_m_solr", - "type":"NRT", - "base_url":"http://N_m/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.28816978409E11, - "INDEX.sizeInGB":119.97015998605639}}]}}, - "N_7_solr":{"COLL_2":{ - "shard13_1_1":[{"core_node808":{ - "core":"COLL_2_shard13_1_1_replica_n806", - "shard":"shard13_1_1", - "collection":"COLL_2", - "node_name":"N_7_solr", - "type":"NRT", - "base_url":"http://N_7/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.2961448776E11, - "INDEX.sizeInGB":120.71289844810963}}], - "shard15_0_1":[{"core_node610":{ - "core":"COLL_2_shard15_0_1_replica_n608", - "shard":"shard15_0_1", - "collection":"COLL_2", - "node_name":"N_7_solr", - "type":"NRT", - "base_url":"http://N_7/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.2722802278E11, - "INDEX.sizeInGB":118.49032973870635}}], - "shard15_0_0":[{"core_node609":{ - "core":"COLL_2_shard15_0_0_replica_n607", - "shard":"shard15_0_0", - "collection":"COLL_2", - "node_name":"N_7_solr", - "type":"NRT", - "base_url":"http://N_7/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.27258670055E11, - "INDEX.sizeInGB":118.5188722377643}}], - "shard13_0_1":[{"core_node1767":{ - "core":"COLL_2_shard13_0_1_replica_n1766", - "shard":"shard13_0_1", - "collection":"COLL_2", - "node_name":"N_7_solr", - "type":"NRT", - "base_url":"http://N_7/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.30339106107E11, - "INDEX.sizeInGB":121.38775187265128}}], - "shard13_1_0":[{"core_node1689":{ - "core":"COLL_2_shard13_1_0_replica_n1688", - "shard":"shard13_1_0", - "collection":"COLL_2", - "node_name":"N_7_solr", - "type":"NRT", - "base_url":"http://N_7/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.29592823396E11, - "INDEX.sizeInGB":120.69272193685174}}], - "shard13_0_0":[{"core_node1713":{ - "core":"COLL_2_shard13_0_0_replica_n1712", - "shard":"shard13_0_0", - "collection":"COLL_2", - "node_name":"N_7_solr", - "type":"NRT", - "base_url":"http://N_7/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.30437704659E11, - "INDEX.sizeInGB":121.47957892995328}}]}}, - "N_6c_solr":{"COLL_2":{ - "shard17_0_1":[{"core_node848":{ - "core":"COLL_2_shard17_0_1_replica_n843", - "shard":"shard17_0_1", - "collection":"COLL_2", - "node_name":"N_6c_solr", - "type":"NRT", - "base_url":"http://N_6c/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.30730929322E11, - "INDEX.sizeInGB":121.7526656780392}}], - "shard17_0_0":[{"core_node844":{ - "core":"COLL_2_shard17_0_0_replica_n842", - "shard":"shard17_0_0", - "collection":"COLL_2", - "node_name":"N_6c_solr", - "type":"NRT", - "base_url":"http://N_6c/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.30743109221E11, - "INDEX.sizeInGB":121.76400909293443}}], - "shard4_0_0":[{"core_node445":{ - "core":"COLL_2_shard4_0_0_replica_n443", - "shard":"shard4_0_0", - "collection":"COLL_2", - "node_name":"N_6c_solr", - "type":"NRT", - "base_url":"http://N_6c/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.28741762257E11, - "INDEX.sizeInGB":119.90010948572308}}], - "shard4_1_0":[{"core_node457":{ - "core":"COLL_2_shard4_1_0_replica_n455", - "shard":"shard4_1_0", - "collection":"COLL_2", - "node_name":"N_6c_solr", - "type":"NRT", - "base_url":"http://N_6c/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.27664473589E11, - "INDEX.sizeInGB":118.89680622983724}}], - "shard4_0_1":[{"core_node446":{ - "core":"COLL_2_shard4_0_1_replica_n444", - "shard":"shard4_0_1", - "collection":"COLL_2", - "node_name":"N_6c_solr", - "type":"NRT", - "base_url":"http://N_6c/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.28032413116E11, - "INDEX.sizeInGB":119.23947661742568}}], - "shard4_1_1":[{"core_node458":{ - "core":"COLL_2_shard4_1_1_replica_n456", - "shard":"shard4_1_1", - "collection":"COLL_2", - "node_name":"N_6c_solr", - "type":"NRT", - "base_url":"http://N_6c/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.27865802727E11, - "INDEX.sizeInGB":119.08430860098451}}]}}, - "N_6i_solr":{"COLL_2":{ - "shard10_1_1":[{"core_node840":{ - "core":"COLL_2_shard10_1_1_replica_n830", - "shard":"shard10_1_1", - "collection":"COLL_2", - "node_name":"N_6i_solr", - "type":"NRT", - "base_url":"http://N_6i/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.30273229534E11, - "INDEX.sizeInGB":121.32639953307807}}], - "shard10_1_0":[{"core_node831":{ - "core":"COLL_2_shard10_1_0_replica_n829", - "shard":"shard10_1_0", - "collection":"COLL_2", - "node_name":"N_6i_solr", - "type":"NRT", - "base_url":"http://N_6i/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.27564995026E11, - "INDEX.sizeInGB":118.80415959842503}}], - "shard10_0_1":[{"core_node1739":{ - "core":"COLL_2_shard10_0_1_replica_n1738", - "shard":"shard10_0_1", - "collection":"COLL_2", - "node_name":"N_6i_solr", - "type":"NRT", - "base_url":"http://N_6i/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.28024871739E11, - "INDEX.sizeInGB":119.2324531627819}}], - "shard2_1_0":[{"core_node1727":{ - "core":"COLL_2_shard2_1_0_replica_n1726", - "shard":"shard2_1_0", - "collection":"COLL_2", - "node_name":"N_6i_solr", - "type":"NRT", - "base_url":"http://N_6i/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.30025926492E11, - "INDEX.sizeInGB":121.0960806272924}}], - "shard10_0_0":[{"core_node897":{ - "core":"COLL_2_shard10_0_0_replica_n896", - "shard":"shard10_0_0", - "collection":"COLL_2", - "node_name":"N_6i_solr", - "type":"NRT", - "base_url":"http://N_6i/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.29103730913E11, - "INDEX.sizeInGB":120.2372190663591}}], - "shard2_1_1":[{"core_node979":{ - "core":"COLL_2_shard2_1_1_replica_n978", - "shard":"shard2_1_1", - "collection":"COLL_2", - "node_name":"N_6i_solr", - "type":"NRT", - "base_url":"http://N_6i/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.2815510735E11, - "INDEX.sizeInGB":119.35374452732503}}]}}, - "N_3_solr":{"COLL_2":{ - "shard16_1_1":[{"core_node997":{ - "core":"COLL_2_shard16_1_1_replica_n996", - "shard":"shard16_1_1", - "collection":"COLL_2", - "node_name":"N_3_solr", - "type":"NRT", - "base_url":"http://N_3/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.26611980672E11, - "INDEX.sizeInGB":117.91659581661224}}], - "shard16_1_0":[{"core_node991":{ - "core":"COLL_2_shard16_1_0_replica_n990", - "shard":"shard16_1_0", - "collection":"COLL_2", - "node_name":"N_3_solr", - "type":"NRT", - "base_url":"http://N_3/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.28724323652E11, - "INDEX.sizeInGB":119.88386851921678}}], - "shard1_1_1":[{"core_node474":{ - "core":"COLL_2_shard1_1_1_replica_n2", - "shard":"shard1_1_1", - "collection":"COLL_2", - "node_name":"N_3_solr", - "type":"NRT", - "base_url":"http://N_3/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.29556889925E11, - "INDEX.sizeInGB":120.65925628412515}}], - "shard4_0_0":[{"core_node1737":{ - "core":"COLL_2_shard4_0_0_replica_n1736", - "shard":"shard4_0_0", - "collection":"COLL_2", - "node_name":"N_3_solr", - "type":"NRT", - "base_url":"http://N_3/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.28645187639E11, - "INDEX.sizeInGB":119.81016736384481}}], - "shard4_1_0":[{"core_node523":{ - "core":"COLL_2_shard4_1_0_replica_n1", - "shard":"shard4_1_0", - "collection":"COLL_2", - "node_name":"N_3_solr", - "type":"NRT", - "base_url":"http://N_3/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.27649471364E11, - "INDEX.sizeInGB":118.88283431902528}}], - "shard9_0_0":[{"core_node1815":{ - "core":"COLL_2_shard9_0_0_replica_n1814", - "shard":"shard9_0_0", - "collection":"COLL_2", - "node_name":"N_3_solr", - "type":"NRT", - "base_url":"http://N_3/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.29037175651E11, - "INDEX.sizeInGB":120.17523464839906}}]}}, - "N_1d_solr":{"COLL_2":{ - "shard3_1_0":[{"core_node425":{ - "core":"COLL_2_shard3_1_0_replica_n423", - "shard":"shard3_1_0", - "collection":"COLL_2", - "node_name":"N_1d_solr", - "type":"NRT", - "base_url":"http://N_1d/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.2828759808E11, - "INDEX.sizeInGB":119.47713613510132}}], - "shard3_1_1":[{"core_node426":{ - "core":"COLL_2_shard3_1_1_replica_n424", - "shard":"shard3_1_1", - "collection":"COLL_2", - "node_name":"N_1d_solr", - "type":"NRT", - "base_url":"http://N_1d/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.29948029547E11, - "INDEX.sizeInGB":121.02353344392031}}], - "shard15_0_0":[{"core_node732":{ - "core":"COLL_2_shard15_0_0_replica_n2", - "shard":"shard15_0_0", - "collection":"COLL_2", - "node_name":"N_1d_solr", - "type":"NRT", - "base_url":"http://N_1d/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.27262832088E11, - "INDEX.sizeInGB":118.5227484330535}}], - "shard12_1_0":[{"core_node1789":{ - "core":"COLL_2_shard12_1_0_replica_n1788", - "shard":"shard12_1_0", - "collection":"COLL_2", - "node_name":"N_1d_solr", - "type":"NRT", - "base_url":"http://N_1d/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.27487519935E11, - "INDEX.sizeInGB":118.73200529720634}}], - "shard14_1_1":[{"core_node1741":{ - "core":"COLL_2_shard14_1_1_replica_n1740", - "shard":"shard14_1_1", - "collection":"COLL_2", - "node_name":"N_1d_solr", - "type":"NRT", - "base_url":"http://N_1d/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.29231781669E11, - "INDEX.sizeInGB":120.35647562611848}}], - "shard14_1_0":[{"core_node1129":{ - "core":"COLL_2_shard14_1_0_replica_n1128", - "shard":"shard14_1_0", - "collection":"COLL_2", - "node_name":"N_1d_solr", - "type":"NRT", - "base_url":"http://N_1d/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.27407685053E11, - "INDEX.sizeInGB":118.65765326935798}}]}}, - "N_1_solr":{"COLL_2":{ - "shard16_1_1":[{"core_node995":{ - "core":"COLL_2_shard16_1_1_replica_n994", - "shard":"shard16_1_1", - "collection":"COLL_2", - "node_name":"N_1_solr", - "type":"NRT", - "base_url":"http://N_1/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.26672765511E11, - "INDEX.sizeInGB":117.97320610936731}}], - "shard16_0_1":[{"core_node989":{ - "core":"COLL_2_shard16_0_1_replica_n988", - "shard":"shard16_0_1", - "collection":"COLL_2", - "node_name":"N_1_solr", - "type":"NRT", - "base_url":"http://N_1/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.3069803609E11, - "INDEX.sizeInGB":121.72203146852553}}], - "shard16_1_0":[{"core_node993":{ - "core":"COLL_2_shard16_1_0_replica_n992", - "shard":"shard16_1_0", - "collection":"COLL_2", - "node_name":"N_1_solr", - "type":"NRT", - "base_url":"http://N_1/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.28812502313E11, - "INDEX.sizeInGB":119.96599129680544}}], - "shard16_0_0":[{"core_node983":{ - "core":"COLL_2_shard16_0_0_replica_n982", - "shard":"shard16_0_0", - "collection":"COLL_2", - "node_name":"N_1_solr", - "type":"NRT", - "base_url":"http://N_1/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.26766519189E11, - "INDEX.sizeInGB":118.06052102614194}}], - "shard18_0_0":[{"core_node875":{ - "core":"COLL_2_shard18_0_0_replica_n2", - "shard":"shard18_0_0", - "collection":"COLL_2", - "node_name":"N_1_solr", - "type":"NRT", - "base_url":"http://N_1/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.28033512867E11, - "INDEX.sizeInGB":119.24050084035844}}], - "shard12_1_1":[{"core_node586":{ - "core":"COLL_2_shard12_1_1_replica_n584", - "shard":"shard12_1_1", - "collection":"COLL_2", - "node_name":"N_1_solr", - "type":"NRT", - "base_url":"http://N_1/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.2671712403E11, - "INDEX.sizeInGB":118.01451819948852}}]}}, - "N_aw_solr":{"COLL_2":{ - "shard18_1_1":[{"core_node1821":{ - "core":"COLL_2_shard18_1_1_replica_n1820", - "shard":"shard18_1_1", - "collection":"COLL_2", - "node_name":"N_aw_solr", - "type":"NRT", - "base_url":"http://N_aw/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.28188518759E11, - "INDEX.sizeInGB":119.38486132677644}}], - "shard4_1_1":[{"core_node525":{ - "core":"COLL_2_shard4_1_1_replica_n1", - "shard":"shard4_1_1", - "collection":"COLL_2", - "node_name":"N_aw_solr", - "type":"NRT", - "base_url":"http://N_aw/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.27899653279E11, - "INDEX.sizeInGB":119.11583438422531}}], - "shard3_1_0":[{"core_node460":{ - "core":"COLL_2_shard3_1_0_replica_n2", - "shard":"shard3_1_0", - "collection":"COLL_2", - "node_name":"N_aw_solr", - "type":"NRT", - "base_url":"http://N_aw/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.28273400877E11, - "INDEX.sizeInGB":119.46391395945102}}], - "shard15_0_1":[{"core_node1817":{ - "core":"COLL_2_shard15_0_1_replica_n1816", - "shard":"shard15_0_1", - "collection":"COLL_2", - "node_name":"N_aw_solr", - "type":"NRT", - "base_url":"http://N_aw/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.27129784031E11, - "INDEX.sizeInGB":118.39883777406067}}], - "shard12_1_1":[{"core_node661":{ - "core":"COLL_2_shard12_1_1_replica_n1", - "shard":"shard12_1_1", - "collection":"COLL_2", - "node_name":"N_aw_solr", - "type":"NRT", - "base_url":"http://N_aw/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.26701654869E11, - "INDEX.sizeInGB":118.00011142063886}}], - "shard12_1_0":[{"core_node659":{ - "core":"COLL_2_shard12_1_0_replica_n1", - "shard":"shard12_1_0", - "collection":"COLL_2", - "node_name":"N_aw_solr", - "type":"NRT", - "base_url":"http://N_aw/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.27434400341E11, - "INDEX.sizeInGB":118.68253382015973}}]}}, - "N_1h_solr":{"COLL_2":{ - "shard1_0_0":[{"core_node1729":{ - "core":"COLL_2_shard1_0_0_replica_n1728", - "shard":"shard1_0_0", - "collection":"COLL_2", - "node_name":"N_1h_solr", - "type":"NRT", - "base_url":"http://N_1h/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":5.7176945428E10, - "INDEX.sizeInGB":53.25018002465367}}], - "shard7_1_0":[{"core_node1145":{ - "core":"COLL_2_shard7_1_0_replica_n1144", - "shard":"shard7_1_0", - "collection":"COLL_2", - "node_name":"N_1h_solr", - "type":"NRT", - "base_url":"http://N_1h/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.2949609012E11, - "INDEX.sizeInGB":120.60263205319643}}], - "shard7_1_1":[{"core_node1701":{ - "core":"COLL_2_shard7_1_1_replica_n1700", - "shard":"shard7_1_1", - "collection":"COLL_2", - "node_name":"N_1h_solr", - "type":"NRT", - "base_url":"http://N_1h/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.28489170345E11, - "INDEX.sizeInGB":119.66486493591219}}], - "shard3_0_1":[{"core_node510":{ - "core":"COLL_2_shard3_0_1_replica_n508", - "shard":"shard3_0_1", - "collection":"COLL_2", - "node_name":"N_1h_solr", - "type":"NRT", - "base_url":"http://N_1h/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.31866901019E11, - "INDEX.sizeInGB":122.81062176357955}}], - "shard12_0_1":[{"core_node1761":{ - "core":"COLL_2_shard12_0_1_replica_n1760", - "shard":"shard12_0_1", - "collection":"COLL_2", - "node_name":"N_1h_solr", - "type":"NRT", - "base_url":"http://N_1h/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.30342308934E11, - "INDEX.sizeInGB":121.39073473773897}}], - "shard12_0_0":[{"core_node1697":{ - "core":"COLL_2_shard12_0_0_replica_n1696", - "shard":"shard12_0_0", - "collection":"COLL_2", - "node_name":"N_1h_solr", - "type":"NRT", - "base_url":"http://N_1h/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.29369271388E11, - "INDEX.sizeInGB":120.48452290520072}}]}}, - "N_29_solr":{"COLL_2":{ - "shard8_0_0":[{"core_node1691":{ - "core":"COLL_2_shard8_0_0_replica_n1690", - "shard":"shard8_0_0", - "collection":"COLL_2", - "node_name":"N_29_solr", - "type":"NRT", - "base_url":"http://N_29/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.30176337999E11, - "INDEX.sizeInGB":121.23616225924343}}], - "shard8_0_1":[{"core_node1787":{ - "core":"COLL_2_shard8_0_1_replica_n1786", - "shard":"shard8_0_1", - "collection":"COLL_2", - "node_name":"N_29_solr", - "type":"NRT", - "base_url":"http://N_29/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.32692723859E11, - "INDEX.sizeInGB":123.57972921710461}}], - "shard7_1_0":[{"core_node1143":{ - "core":"COLL_2_shard7_1_0_replica_n1142", - "shard":"shard7_1_0", - "collection":"COLL_2", - "node_name":"N_29_solr", - "type":"NRT", - "base_url":"http://N_29/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.2946739865E11, - "INDEX.sizeInGB":120.57591103948653}}], - "shard7_0_1":[{"core_node777":{ - "core":"COLL_2_shard7_0_1_replica_n2", - "shard":"shard7_0_1", - "collection":"COLL_2", - "node_name":"N_29_solr", - "type":"NRT", - "base_url":"http://N_29/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":8.6794048237E10, - "INDEX.sizeInGB":80.83325646538287}}], - "shard7_1_1":[{"core_node1759":{ - "core":"COLL_2_shard7_1_1_replica_n1758", - "shard":"shard7_1_1", - "collection":"COLL_2", - "node_name":"N_29_solr", - "type":"NRT", - "base_url":"http://N_29/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.28546712309E11, - "INDEX.sizeInGB":119.7184550659731}}], - "shard6_1_0":[{"core_node1793":{ - "core":"COLL_2_shard6_1_0_replica_n1792", - "shard":"shard6_1_0", - "collection":"COLL_2", - "node_name":"N_29_solr", - "type":"NRT", - "base_url":"http://N_29/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.29365181039E11, - "INDEX.sizeInGB":120.48071347083896}}]}}, - "N_e_solr":{"COLL_2":{ - "shard1_0_1":[{"core_node1719":{ - "core":"COLL_2_shard1_0_1_replica_n1718", - "shard":"shard1_0_1", - "collection":"COLL_2", - "node_name":"N_e_solr", - "type":"NRT", - "base_url":"http://N_e/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":5.9506746089E10, - "INDEX.sizeInGB":55.41997597459704}}], - "shard18_0_0":[{"core_node1819":{ - "core":"COLL_2_shard18_0_0_replica_n1818", - "shard":"shard18_0_0", - "collection":"COLL_2", - "node_name":"N_e_solr", - "type":"NRT", - "base_url":"http://N_e/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.28218931509E11, - "INDEX.sizeInGB":119.41318540740758}}], - "shard13_1_1":[{"core_node925":{ - "core":"COLL_2_shard13_1_1_replica_n924", - "shard":"shard13_1_1", - "collection":"COLL_2", - "node_name":"N_e_solr", - "type":"NRT", - "base_url":"http://N_e/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.29598508564E11, - "INDEX.sizeInGB":120.69801666215062}}], - "shard18_1_0":[{"core_node672":{ - "core":"COLL_2_shard18_1_0_replica_n2", - "shard":"shard18_1_0", - "collection":"COLL_2", - "node_name":"N_e_solr", - "type":"NRT", - "base_url":"http://N_e/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.29108586002E11, - "INDEX.sizeInGB":120.24174072034657}}], - "shard15_0_0":[{"core_node731":{ - "core":"COLL_2_shard15_0_0_replica_n1", - "shard":"shard15_0_0", - "collection":"COLL_2", - "node_name":"N_e_solr", - "type":"NRT", - "base_url":"http://N_e/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.27235871561E11, - "INDEX.sizeInGB":118.49763948563486}}], - "shard13_1_0":[{"core_node923":{ - "core":"COLL_2_shard13_1_0_replica_n922", - "shard":"shard13_1_0", - "collection":"COLL_2", - "node_name":"N_e_solr", - "type":"NRT", - "base_url":"http://N_e/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.29514183189E11, - "INDEX.sizeInGB":120.6194825368002}}]}}, - "N_2w_solr":{"COLL_2":{ - "shard1_0_1":[{"core_node1677":{ - "core":"COLL_2_shard1_0_1_replica_n1676", - "shard":"shard1_0_1", - "collection":"COLL_2", - "node_name":"N_2w_solr", - "type":"NRT", - "base_url":"http://N_2w/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":5.9557275352E10, - "INDEX.sizeInGB":55.46703501790762}}], - "shard1_1_1":[{"core_node1807":{ - "core":"COLL_2_shard1_1_1_replica_n1806", - "shard":"shard1_1_1", - "collection":"COLL_2", - "node_name":"N_2w_solr", - "type":"NRT", - "base_url":"http://N_2w/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.2954748046E11, - "INDEX.sizeInGB":120.6504930369556}}], - "shard4_1_0":[{"core_node1775":{ - "core":"COLL_2_shard4_1_0_replica_n1774", - "shard":"shard4_1_0", - "collection":"COLL_2", - "node_name":"N_2w_solr", - "type":"NRT", - "base_url":"http://N_2w/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.27659935903E11, - "INDEX.sizeInGB":118.89258018042892}}], - "shard18_1_1":[{"core_node673":{ - "core":"COLL_2_shard18_1_1_replica_n1", - "shard":"shard18_1_1", - "collection":"COLL_2", - "node_name":"N_2w_solr", - "type":"NRT", - "base_url":"http://N_2w/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.28226679933E11, - "INDEX.sizeInGB":119.42040168959647}}], - "shard4_1_1":[{"core_node1805":{ - "core":"COLL_2_shard4_1_1_replica_n1804", - "shard":"shard4_1_1", - "collection":"COLL_2", - "node_name":"N_2w_solr", - "type":"NRT", - "base_url":"http://N_2w/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.27878088796E11, - "INDEX.sizeInGB":119.0957508943975}}], - "shard18_1_0":[{"core_node671":{ - "core":"COLL_2_shard18_1_0_replica_n1", - "shard":"shard18_1_0", - "collection":"COLL_2", - "node_name":"N_2w_solr", - "type":"NRT", - "base_url":"http://N_2w/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.28884297502E11, - "INDEX.sizeInGB":120.03285577706993}}]}}, - "N_5_solr":{"COLL_2":{ - "shard1_1_0":[{"core_node1721":{ - "core":"COLL_2_shard1_1_0_replica_n1720", - "shard":"shard1_1_0", - "collection":"COLL_2", - "node_name":"N_5_solr", - "type":"NRT", - "base_url":"http://N_5/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.29009851855E11, - "INDEX.sizeInGB":120.14978738036007}}], - "shard1_0_1":[{"core_node1669":{ - "core":"COLL_2_shard1_0_1_replica_n1668", - "shard":"shard1_0_1", - "collection":"COLL_2", - "node_name":"N_5_solr", - "type":"NRT", - "base_url":"http://N_5/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":5.9574276743E10, - "INDEX.sizeInGB":55.482868797145784}}], - "shard1_1_1":[{"core_node418":{ - "core":"COLL_2_shard1_1_1_replica_n416", - "shard":"shard1_1_1", - "collection":"COLL_2", - "node_name":"N_5_solr", - "type":"NRT", - "base_url":"http://N_5/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.29698716918E11, - "INDEX.sizeInGB":120.79134296439588}}], - "shard2_0_0":[{"core_node911":{ - "core":"COLL_2_shard2_0_0_replica_n910", - "shard":"shard2_0_0", - "collection":"COLL_2", - "node_name":"N_5_solr", - "type":"NRT", - "base_url":"http://N_5/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.29504451209E11, - "INDEX.sizeInGB":120.6104189241305}}], - "shard2_0_1":[{"core_node917":{ - "core":"COLL_2_shard2_0_1_replica_n916", - "shard":"shard2_0_1", - "collection":"COLL_2", - "node_name":"N_5_solr", - "type":"NRT", - "base_url":"http://N_5/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":1.31334463143E11, - "INDEX.sizeInGB":122.31475035008043}}], - "shard1_0_0":[{"core_node1725":{ - "core":"COLL_2_shard1_0_0_replica_n1724", - "shard":"shard1_0_0", - "collection":"COLL_2", - "node_name":"N_5_solr", - "type":"NRT", - "base_url":"http://N_5/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":5.7183711221E10, - "INDEX.sizeInGB":53.25648116040975}}]}}, - "N_do_solr":{ - "COLL_1":{ - "shard3_0_0":[{"core_node112":{ - "core":"COLL_1_shard3_0_0_replica_n111", - "shard":"shard3_0_0", - "collection":"COLL_1", - "node_name":"N_do_solr", - "type":"NRT", - "base_url":"http://N_do/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":4.4957115524E10, - "INDEX.sizeInGB":41.86957657709718}}], - "shard3_1_0":[{"core_node116":{ - "core":"COLL_1_shard3_1_0_replica_n115", - "shard":"shard3_1_0", - "collection":"COLL_1", - "node_name":"N_do_solr", - "type":"NRT", - "base_url":"http://N_do/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":4.3732753925E10, - "INDEX.sizeInGB":40.72930098045617}}], - "shard3_0_1":[{"core_node114":{ - "core":"COLL_1_shard3_0_1_replica_n113", - "shard":"shard3_0_1", - "collection":"COLL_1", - "node_name":"N_do_solr", - "type":"NRT", - "base_url":"http://N_do/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":4.577095697E10, - "INDEX.sizeInGB":42.62752548791468}}], - "shard3_1_1":[{"core_node118":{ - "core":"COLL_1_shard3_1_1_replica_n117", - "shard":"shard3_1_1", - "collection":"COLL_1", - "node_name":"N_do_solr", - "type":"NRT", - "base_url":"http://N_do/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":4.8532509927E10, - "INDEX.sizeInGB":45.19942209776491}}]}, - "COLL_0":{"shard3":[{"core_node15":{ - "core":"COLL_0_shard3_replica_n12", - "shard":"shard3", - "collection":"COLL_0", - "node_name":"N_do_solr", - "type":"NRT", - "base_url":"http://N_do/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":3.1297025422E10, - "INDEX.sizeInGB":29.147626293823123}}]}}, - "N_3a_solr":{ - "COLL_1":{ - "shard3_0_0":[{"core_node73":{ - "core":"COLL_1_shard3_0_0_replica_n71", - "shard":"shard3_0_0", - "collection":"COLL_1", - "node_name":"N_3a_solr", - "type":"NRT", - "base_url":"http://N_3a/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":4.5160600486E10, - "INDEX.sizeInGB":42.05908671580255}}], - "shard3_1_0":[{"core_node77":{ - "core":"COLL_1_shard3_1_0_replica_n75", - "shard":"shard3_1_0", - "collection":"COLL_1", - "node_name":"N_3a_solr", - "type":"NRT", - "base_url":"http://N_3a/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":4.5090380622E10, - "INDEX.sizeInGB":41.99368937127292}}], - "shard3_0_1":[{"core_node74":{ - "core":"COLL_1_shard3_0_1_replica_n72", - "shard":"shard3_0_1", - "collection":"COLL_1", - "node_name":"N_3a_solr", - "type":"NRT", - "base_url":"http://N_3a/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":4.5879426317E10, - "INDEX.sizeInGB":42.72854543942958}}], - "shard3_1_1":[{"core_node78":{ - "core":"COLL_1_shard3_1_1_replica_n76", - "shard":"shard3_1_1", - "collection":"COLL_1", - "node_name":"N_3a_solr", - "type":"NRT", - "base_url":"http://N_3a/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":4.6849085882E10, - "INDEX.sizeInGB":43.631611282005906}}]}, - "COLL_0":{"shard3":[{"core_node17":{ - "core":"COLL_0_shard3_replica_n14", - "shard":"shard3", - "collection":"COLL_0", - "node_name":"N_3a_solr", - "type":"NRT", - "base_url":"http://N_3a/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":3.0819950704E10, - "INDEX.sizeInGB":28.70331583917141}}]}}, - "N_v_solr":{ - "COLL_1":{ - "shard3_0_0":[{"core_node120":{ - "core":"COLL_1_shard3_0_0_replica_n119", - "shard":"shard3_0_0", - "collection":"COLL_1", - "node_name":"N_v_solr", - "type":"NRT", - "base_url":"http://N_v/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":4.3809517838E10, - "INDEX.sizeInGB":40.80079294554889}}], - "shard3_1_0":[{"core_node124":{ - "core":"COLL_1_shard3_1_0_replica_n123", - "shard":"shard3_1_0", - "collection":"COLL_1", - "node_name":"N_v_solr", - "type":"NRT", - "base_url":"http://N_v/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":4.5638162031E10, - "INDEX.sizeInGB":42.503850563429296}}], - "shard3_0_1":[{"core_node122":{ - "core":"COLL_1_shard3_0_1_replica_n121", - "shard":"shard3_0_1", - "collection":"COLL_1", - "node_name":"N_v_solr", - "type":"NRT", - "base_url":"http://N_v/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":4.6310602091E10, - "INDEX.sizeInGB":43.13010917138308}}], - "shard3_1_1":[{"core_node126":{ - "core":"COLL_1_shard3_1_1_replica_n125", - "shard":"shard3_1_1", - "collection":"COLL_1", - "node_name":"N_v_solr", - "type":"NRT", - "base_url":"http://N_v/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":4.4257494507E10, - "INDEX.sizeInGB":41.21800373028964}}]}, - "COLL_0":{"shard3":[{"core_node18":{ - "core":"COLL_0_shard3_replica_n16", - "shard":"shard3", - "collection":"COLL_0", - "node_name":"N_v_solr", - "type":"NRT", - "base_url":"http://N_v/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":2.8932093807E10, - "INDEX.sizeInGB":26.94511209335178}}]}}, - "N_13_solr":{ - "COLL_1":{ - "shard1_1_0":[{"core_node61":{ - "core":"COLL_1_shard1_1_0_replica_n59", - "shard":"shard1_1_0", - "collection":"COLL_1", - "node_name":"N_13_solr", - "type":"NRT", - "base_url":"http://N_13/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":4.3783419579E10, - "INDEX.sizeInGB":40.77648704778403}}], - "shard1_0_1":[{"core_node58":{ - "core":"COLL_1_shard1_0_1_replica_n56", - "shard":"shard1_0_1", - "collection":"COLL_1", - "node_name":"N_13_solr", - "type":"NRT", - "base_url":"http://N_13/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":4.4932001726E10, - "INDEX.sizeInGB":41.846187530085444}}], - "shard1_1_1":[{"core_node62":{ - "core":"COLL_1_shard1_1_1_replica_n60", - "shard":"shard1_1_1", - "collection":"COLL_1", - "node_name":"N_13_solr", - "type":"NRT", - "base_url":"http://N_13/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":4.811959042E10, - "INDEX.sizeInGB":44.814860839396715}}], - "shard1_0_0":[{"core_node57":{ - "core":"COLL_1_shard1_0_0_replica_n55", - "shard":"shard1_0_0", - "collection":"COLL_1", - "node_name":"N_13_solr", - "type":"NRT", - "base_url":"http://N_13/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":4.5921892273E10, - "INDEX.sizeInGB":42.76809494290501}}]}, - "COLL_0":{"shard2":[{"core_node13":{ - "core":"COLL_0_shard2_replica_n10", - "shard":"shard2", - "collection":"COLL_0", - "node_name":"N_13_solr", - "type":"NRT", - "base_url":"http://N_13/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":3.4248182159E10, - "INDEX.sizeInGB":31.896105184219778}}]}}, - "N_3to_solr":{ - "COLL_1":{ - "shard1_1_0":[{"core_node84":{ - "core":"COLL_1_shard1_1_0_replica_n83", - "shard":"shard1_1_0", - "collection":"COLL_1", - "node_name":"N_3to_solr", - "type":"NRT", - "base_url":"http://N_3to/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":4.3892348528E10, - "INDEX.sizeInGB":40.87793503701687}}], - "shard1_0_1":[{"core_node82":{ - "core":"COLL_1_shard1_0_1_replica_n81", - "shard":"shard1_0_1", - "collection":"COLL_1", - "node_name":"N_3to_solr", - "type":"NRT", - "base_url":"http://N_3to/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":4.4936912617E10, - "INDEX.sizeInGB":41.85076115373522}}], - "shard1_1_1":[{"core_node86":{ - "core":"COLL_1_shard1_1_1_replica_n85", - "shard":"shard1_1_1", - "collection":"COLL_1", - "node_name":"N_3to_solr", - "type":"NRT", - "base_url":"http://N_3to/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":5.1015133973E10, - "INDEX.sizeInGB":47.511545916087925}}], - "shard1_0_0":[{"core_node80":{ - "core":"COLL_1_shard1_0_0_replica_n79", - "shard":"shard1_0_0", - "collection":"COLL_1", - "node_name":"N_3to_solr", - "type":"NRT", - "base_url":"http://N_3to/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":4.644843302E10, - "INDEX.sizeInGB":43.258474227041006}}]}, - "COLL_0":{"shard2":[{"core_node11":{ - "core":"COLL_0_shard2_replica_n8", - "shard":"shard2", - "collection":"COLL_0", - "node_name":"N_3to_solr", - "type":"NRT", - "base_url":"http://N_3to/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":3.0722710385E10, - "INDEX.sizeInGB":28.6127537349239}}]}}, - "N_16_solr":{ - "COLL_1":{ - "shard2_0_0":[{"core_node100":{ - "core":"COLL_1_shard2_0_0_replica_n99", - "shard":"shard2_0_0", - "collection":"COLL_1", - "node_name":"N_16_solr", - "type":"NRT", - "base_url":"http://N_16/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":4.8764329025E10, - "INDEX.sizeInGB":45.41532045695931}}], - "shard2_0_1":[{"core_node102":{ - "core":"COLL_1_shard2_0_1_replica_n101", - "shard":"shard2_0_1", - "collection":"COLL_1", - "node_name":"N_16_solr", - "type":"NRT", - "base_url":"http://N_16/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":4.3740343099E10, - "INDEX.sizeInGB":40.73636894952506}}], - "shard2_1_0":[{"core_node96":{ - "core":"COLL_1_shard2_1_0_replica_n95", - "shard":"shard2_1_0", - "collection":"COLL_1", - "node_name":"N_16_solr", - "type":"NRT", - "base_url":"http://N_16/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":4.5585236311E10, - "INDEX.sizeInGB":42.45455964561552}}], - "shard2_1_1":[{"core_node98":{ - "core":"COLL_1_shard2_1_1_replica_n97", - "shard":"shard2_1_1", - "collection":"COLL_1", - "node_name":"N_16_solr", - "type":"NRT", - "base_url":"http://N_16/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":4.527594328E10, - "INDEX.sizeInGB":42.16650806367397}}]}, - "COLL_0":{"shard1":[{"core_node5":{ - "core":"COLL_0_shard1_replica_n2", - "shard":"shard1", - "collection":"COLL_0", - "node_name":"N_16_solr", - "type":"NRT", - "base_url":"http://N_16/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":3.3775978753E10, - "INDEX.sizeInGB":31.45633149240166}}]}}, - "N_d4_solr":{ - "COLL_1":{ - "shard2_0_0":[{"core_node69":{ - "core":"COLL_1_shard2_0_0_replica_n67", - "shard":"shard2_0_0", - "collection":"COLL_1", - "node_name":"N_d4_solr", - "type":"NRT", - "base_url":"http://N_d4/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":4.497304707E10, - "INDEX.sizeInGB":41.8844139855355}}], - "shard2_0_1":[{"core_node70":{ - "core":"COLL_1_shard2_0_1_replica_n68", - "shard":"shard2_0_1", - "collection":"COLL_1", - "node_name":"N_d4_solr", - "type":"NRT", - "base_url":"http://N_d4/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":4.5692831033E10, - "INDEX.sizeInGB":42.554765039123595}}], - "shard2_1_0":[{"core_node65":{ - "core":"COLL_1_shard2_1_0_replica_n63", - "shard":"shard2_1_0", - "collection":"COLL_1", - "node_name":"N_d4_solr", - "type":"NRT", - "base_url":"http://N_d4/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":4.5935880044E10, - "INDEX.sizeInGB":42.78112206980586}}], - "shard2_1_1":[{"core_node66":{ - "core":"COLL_1_shard2_1_1_replica_n64", - "shard":"shard2_1_1", - "collection":"COLL_1", - "node_name":"N_d4_solr", - "type":"NRT", - "base_url":"http://N_d4/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":4.5166045429E10, - "INDEX.sizeInGB":42.064157714135945}}]}, - "COLL_0":{"shard1":[{"core_node3":{ - "core":"COLL_0_shard1_replica_n1", - "shard":"shard1", - "collection":"COLL_0", - "node_name":"N_d4_solr", - "type":"NRT", - "base_url":"http://N_d4/solr", - "leader":"true", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":3.401835331E10, - "INDEX.sizeInGB":31.682060388848186}}]}}, - "N_b9_solr":{ - "COLL_1":{ - "shard1_1_0":[{"core_node92":{ - "core":"COLL_1_shard1_1_0_replica_n91", - "shard":"shard1_1_0", - "collection":"COLL_1", - "node_name":"N_b9_solr", - "type":"NRT", - "base_url":"http://N_b9/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":4.5724314347E10, - "INDEX.sizeInGB":42.5840861601755}}], - "shard1_0_1":[{"core_node90":{ - "core":"COLL_1_shard1_0_1_replica_n89", - "shard":"shard1_0_1", - "collection":"COLL_1", - "node_name":"N_b9_solr", - "type":"NRT", - "base_url":"http://N_b9/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":4.6030616744E10, - "INDEX.sizeInGB":42.869352497160435}}], - "shard1_1_1":[{"core_node94":{ - "core":"COLL_1_shard1_1_1_replica_n93", - "shard":"shard1_1_1", - "collection":"COLL_1", - "node_name":"N_b9_solr", - "type":"NRT", - "base_url":"http://N_b9/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":4.574559386E10, - "INDEX.sizeInGB":42.603904251009226}}], - "shard1_0_0":[{"core_node88":{ - "core":"COLL_1_shard1_0_0_replica_n87", - "shard":"shard1_0_0", - "collection":"COLL_1", - "node_name":"N_b9_solr", - "type":"NRT", - "base_url":"http://N_b9/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":4.5100613575E10, - "INDEX.sizeInGB":42.0032195514068}}]}, - "COLL_0":{"shard2":[{"core_node9":{ - "core":"COLL_0_shard2_replica_n6", - "shard":"shard2", - "collection":"COLL_0", - "node_name":"N_b9_solr", - "type":"NRT", - "base_url":"http://N_b9/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":2.8865621899E10, - "INDEX.sizeInGB":26.883205304853618}}]}}, - "N_74_solr":{ - "COLL_1":{ - "shard2_0_0":[{"core_node108":{ - "core":"COLL_1_shard2_0_0_replica_n107", - "shard":"shard2_0_0", - "collection":"COLL_1", - "node_name":"N_74_solr", - "type":"NRT", - "base_url":"http://N_74/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":4.3767024396E10, - "INDEX.sizeInGB":40.76121784374118}}], - "shard2_0_1":[{"core_node110":{ - "core":"COLL_1_shard2_0_1_replica_n109", - "shard":"shard2_0_1", - "collection":"COLL_1", - "node_name":"N_74_solr", - "type":"NRT", - "base_url":"http://N_74/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":4.8622428842E10, - "INDEX.sizeInGB":45.28316561318934}}], - "shard2_1_0":[{"core_node104":{ - "core":"COLL_1_shard2_1_0_replica_n103", - "shard":"shard2_1_0", - "collection":"COLL_1", - "node_name":"N_74_solr", - "type":"NRT", - "base_url":"http://N_74/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":4.4599223614E10, - "INDEX.sizeInGB":41.536263762041926}}], - "shard2_1_1":[{"core_node106":{ - "core":"COLL_1_shard2_1_1_replica_n105", - "shard":"shard2_1_1", - "collection":"COLL_1", - "node_name":"N_74_solr", - "type":"NRT", - "base_url":"http://N_74/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":4.3768191618E10, - "INDEX.sizeInGB":40.762304903939366}}]}, - "COLL_0":{"shard1":[{"core_node7":{ - "core":"COLL_0_shard1_replica_n4", - "shard":"shard1", - "collection":"COLL_0", - "node_name":"N_74_solr", - "type":"NRT", - "base_url":"http://N_74/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInBytes":2.9252853492E10, - "INDEX.sizeInGB":27.24384282901883}}]}}}} \ No newline at end of file diff --git a/solr/core/src/test-files/solr/simSnapshot/statistics.json b/solr/core/src/test-files/solr/simSnapshot/statistics.json deleted file mode 100644 index 3e10d61f2a4..00000000000 --- a/solr/core/src/test-files/solr/simSnapshot/statistics.json +++ /dev/null @@ -1,2029 +0,0 @@ -{ - "coresPerNodes":{ - "5":9, - "6":36, - "12":2, - "13":1}, - "sortedNodeStats":{ - "N_0_solr":{ - "isLive":true, - "cores":12.0, - "freedisk":719.6562576293945, - "sysprop.pool":"pool-03", - "node":"N_0_solr", - "sysprop.az":"us-east-1a", - "totaldisk":999.51171875, - "withCollection":null, - "replicas":{ - "COLL_6":{"shard1_replica_n4":{ - "INDEX.sizeInBytes":4.4921656871E10, - "INDEX.sizeInGB":41.83655313309282, - "coreNode":"core_node6", - "leader":true}}, - "COLL_l":{"shard1_replica_n9":{ - "INDEX.sizeInBytes":135.0, - "INDEX.sizeInGB":1.257285475730896E-7, - "coreNode":"core_node10", - "leader":true}}, - "COLL_x":{"shard1_replica_n9":{ - "INDEX.sizeInBytes":3.0928583E8, - "INDEX.sizeInGB":0.2880448754876852, - "coreNode":"core_node10", - "leader":true}}, - "COLL_1b":{"shard1_replica_n9":{ - "INDEX.sizeInBytes":135.0, - "INDEX.sizeInGB":1.257285475730896E-7, - "coreNode":"core_node10", - "leader":true}}, - "COLL_1r":{"shard1_replica_n2":{ - "INDEX.sizeInBytes":4.25884524E8, - "INDEX.sizeInGB":0.39663587138056755, - "coreNode":"core_node5", - "leader":true}}, - "COLL_8":{"shard1_replica_n2":{ - "INDEX.sizeInBytes":399225.0, - "INDEX.sizeInGB":3.718072548508644E-4, - "coreNode":"core_node5", - "leader":true}}, - "COLL_q":{"shard1_replica_n9":{ - "INDEX.sizeInBytes":4.9242789E8, - "INDEX.sizeInGB":0.45860921032726765, - "coreNode":"core_node10", - "leader":true}}, - "COLL_4":{"shard1_replica_n2":{ - "INDEX.sizeInBytes":2.58797271E8, - "INDEX.sizeInGB":0.24102374073117971, - "coreNode":"core_node5", - "leader":true}}, - "COLL_1x":{"shard1_replica_n9":{ - "INDEX.sizeInBytes":4264901.0, - "INDEX.sizeInGB":0.003971998579800129, - "coreNode":"core_node10", - "leader":true}}, - "COLL_1t":{"shard1_replica_n2":{ - "INDEX.sizeInBytes":8.7485800719E10, - "INDEX.sizeInGB":81.47750116791576, - "coreNode":"core_node5", - "leader":true}}, - "COLL_2k":{"shard1_replica_n9":{ - "INDEX.sizeInBytes":135.0, - "INDEX.sizeInGB":1.257285475730896E-7, - "coreNode":"core_node10", - "leader":true}}, - "COLL_22":{"shard1_replica_n9":{ - "INDEX.sizeInBytes":2.4351639993E10, - "INDEX.sizeInGB":22.679232054390013, - "coreNode":"core_node10", - "leader":true}}}}, - "N_3_solr":{ - "isLive":true, - "cores":6.0, - "freedisk":4272.45711517334, - "sysprop.pool":"pool-01", - "node":"N_3_solr", - "sysprop.az":"us-east-1c", - "totaldisk":4998.009765625, - "withCollection":null, - "replicas":{"COLL_2":{ - "shard16_1_0_replica_n990":{ - "INDEX.sizeInBytes":1.28724323652E11, - "INDEX.sizeInGB":119.88386851921678, - "coreNode":"core_node991"}, - "shard16_1_1_replica_n996":{ - "INDEX.sizeInBytes":1.26611980672E11, - "INDEX.sizeInGB":117.91659581661224, - "coreNode":"core_node997"}, - "shard1_1_1_replica_n2":{ - "INDEX.sizeInBytes":1.29556889925E11, - "INDEX.sizeInGB":120.65925628412515, - "coreNode":"core_node474"}, - "shard4_0_0_replica_n1736":{ - "INDEX.sizeInBytes":1.28645187639E11, - "INDEX.sizeInGB":119.81016736384481, - "coreNode":"core_node1737"}, - "shard4_1_0_replica_n1":{ - "INDEX.sizeInBytes":1.27649471364E11, - "INDEX.sizeInGB":118.88283431902528, - "coreNode":"core_node523"}, - "shard9_0_0_replica_n1814":{ - "INDEX.sizeInBytes":1.29037175651E11, - "INDEX.sizeInGB":120.17523464839906, - "coreNode":"core_node1815"}}}}, - "N_1_solr":{ - "isLive":true, - "cores":6.0, - "freedisk":4274.765396118164, - "sysprop.pool":"pool-01", - "node":"N_1_solr", - "sysprop.az":"us-east-1a", - "totaldisk":4998.009765625, - "withCollection":null, - "replicas":{"COLL_2":{ - "shard12_1_1_replica_n584":{ - "INDEX.sizeInBytes":1.2671712403E11, - "INDEX.sizeInGB":118.01451819948852, - "coreNode":"core_node586"}, - "shard16_0_0_replica_n982":{ - "INDEX.sizeInBytes":1.26766519189E11, - "INDEX.sizeInGB":118.06052102614194, - "coreNode":"core_node983"}, - "shard16_0_1_replica_n988":{ - "INDEX.sizeInBytes":1.3069803609E11, - "INDEX.sizeInGB":121.72203146852553, - "coreNode":"core_node989"}, - "shard16_1_0_replica_n992":{ - "INDEX.sizeInBytes":1.28812502313E11, - "INDEX.sizeInGB":119.96599129680544, - "coreNode":"core_node993"}, - "shard16_1_1_replica_n994":{ - "INDEX.sizeInBytes":1.26672765511E11, - "INDEX.sizeInGB":117.97320610936731, - "coreNode":"core_node995"}, - "shard18_0_0_replica_n2":{ - "INDEX.sizeInBytes":1.28033512867E11, - "INDEX.sizeInGB":119.24050084035844, - "coreNode":"core_node875"}}}}, - "N_2_solr":{ - "isLive":true, - "cores":6.0, - "freedisk":4266.604637145996, - "sysprop.pool":"pool-01", - "node":"N_2_solr", - "sysprop.az":"us-east-1c", - "totaldisk":4998.009765625, - "withCollection":null, - "replicas":{"COLL_2":{ - "shard12_0_0_replica_n1248":{ - "INDEX.sizeInBytes":1.29421522442E11, - "INDEX.sizeInGB":120.53318549133837, - "coreNode":"core_node1249", - "leader":true}, - "shard12_0_1_replica_n1254":{ - "INDEX.sizeInBytes":1.30384315739E11, - "INDEX.sizeInGB":121.42985662352294, - "coreNode":"core_node1255", - "leader":true}, - "shard5_1_0_replica_n1136":{ - "INDEX.sizeInBytes":7.6877250282E10, - "INDEX.sizeInGB":71.59751866199076, - "coreNode":"core_node1137"}, - "shard5_1_1_replica_n1138":{ - "INDEX.sizeInBytes":1.29952609098E11, - "INDEX.sizeInGB":121.02779848314822, - "coreNode":"core_node1139"}, - "shard7_0_1_replica_n1":{ - "INDEX.sizeInBytes":1.2890128588E11, - "INDEX.sizeInGB":120.04867743700743, - "coreNode":"core_node776", - "leader":true}, - "shard9_0_1_replica_n2":{ - "INDEX.sizeInBytes":1.29212951693E11, - "INDEX.sizeInGB":120.33893884439021, - "coreNode":"core_node478", - "leader":true}}}}, - "N_6_solr":{ - "isLive":true, - "cores":6.0, - "freedisk":4252.47643661499, - "sysprop.pool":"pool-01", - "node":"N_6_solr", - "sysprop.az":"us-east-1b", - "totaldisk":4998.009765625, - "withCollection":null, - "replicas":{"COLL_2":{ - "shard15_1_1_replica_n1708":{ - "INDEX.sizeInBytes":1.30846984322E11, - "INDEX.sizeInGB":121.86075031943619, - "coreNode":"core_node1709"}, - "shard3_1_0_replica_n1":{ - "INDEX.sizeInBytes":1.32652501535E11, - "INDEX.sizeInGB":123.54226925875992, - "coreNode":"core_node459"}, - "shard4_0_0_replica_n2":{ - "INDEX.sizeInBytes":1.28680029361E11, - "INDEX.sizeInGB":119.84261624608189, - "coreNode":"core_node520"}, - "shard4_0_1_replica_n1802":{ - "INDEX.sizeInBytes":1.28153346526E11, - "INDEX.sizeInGB":119.35210463218391, - "coreNode":"core_node1803"}, - "shard8_1_0_replica_n1810":{ - "INDEX.sizeInBytes":1.35679249773E11, - "INDEX.sizeInGB":126.36114822048694, - "coreNode":"core_node1811"}, - "shard9_0_0_replica_n1798":{ - "INDEX.sizeInBytes":1.35157081196E11, - "INDEX.sizeInGB":125.874840836972, - "coreNode":"core_node1799", - "leader":true}}}}, - "N_5_solr":{ - "isLive":true, - "cores":6.0, - "freedisk":4397.149795532227, - "sysprop.pool":"pool-01", - "node":"N_5_solr", - "sysprop.az":"us-east-1a", - "totaldisk":4998.009765625, - "withCollection":null, - "replicas":{"COLL_2":{ - "shard1_0_0_replica_n1724":{ - "INDEX.sizeInBytes":5.7183711221E10, - "INDEX.sizeInGB":53.25648116040975, - "coreNode":"core_node1725", - "leader":true}, - "shard1_0_1_replica_n1668":{ - "INDEX.sizeInBytes":5.9574276743E10, - "INDEX.sizeInGB":55.482868797145784, - "coreNode":"core_node1669"}, - "shard1_1_0_replica_n1720":{ - "INDEX.sizeInBytes":1.29009851855E11, - "INDEX.sizeInGB":120.14978738036007, - "coreNode":"core_node1721"}, - "shard1_1_1_replica_n416":{ - "INDEX.sizeInBytes":1.29698716918E11, - "INDEX.sizeInGB":120.79134296439588, - "coreNode":"core_node418", - "leader":true}, - "shard2_0_0_replica_n910":{ - "INDEX.sizeInBytes":1.29504451209E11, - "INDEX.sizeInGB":120.6104189241305, - "coreNode":"core_node911"}, - "shard2_0_1_replica_n916":{ - "INDEX.sizeInBytes":1.31334463143E11, - "INDEX.sizeInGB":122.31475035008043, - "coreNode":"core_node917"}}}}, - "N_4_solr":{ - "isLive":true, - "cores":12.0, - "freedisk":875.4758682250977, - "sysprop.pool":"pool-03", - "node":"N_4_solr", - "sysprop.az":"us-east-1c", - "totaldisk":999.51171875, - "withCollection":null, - "replicas":{ - "COLL_6":{"shard1_replica_n2":{ - "INDEX.sizeInBytes":2.0738852096E10, - "INDEX.sizeInGB":19.314561128616333, - "coreNode":"core_node5"}}, - "COLL_l":{"shard1_replica_n4":{ - "INDEX.sizeInBytes":135.0, - "INDEX.sizeInGB":1.257285475730896E-7, - "coreNode":"core_node6"}}, - "COLL_x":{"shard1_replica_n4":{ - "INDEX.sizeInBytes":3.03301808E8, - "INDEX.sizeInGB":0.28247182071208954, - "coreNode":"core_node6"}}, - "COLL_1b":{"shard1_replica_n4":{ - "INDEX.sizeInBytes":135.0, - "INDEX.sizeInGB":1.257285475730896E-7, - "coreNode":"core_node6"}}, - "COLL_1r":{"shard1_replica_n4":{ - "INDEX.sizeInBytes":4.46826689E8, - "INDEX.sizeInGB":0.4161397824063897, - "coreNode":"core_node6"}}, - "COLL_8":{"shard1_replica_n4":{ - "INDEX.sizeInBytes":356048.0, - "INDEX.sizeInGB":3.315955400466919E-4, - "coreNode":"core_node6"}}, - "COLL_q":{"shard1_replica_n4":{ - "INDEX.sizeInBytes":4.9242789E8, - "INDEX.sizeInGB":0.45860921032726765, - "coreNode":"core_node6"}}, - "COLL_4":{"shard1_replica_n4":{ - "INDEX.sizeInBytes":2.59832461E8, - "INDEX.sizeInGB":0.2419878365471959, - "coreNode":"core_node6"}}, - "COLL_1x":{"shard1_replica_n4":{ - "INDEX.sizeInBytes":4255591.0, - "INDEX.sizeInGB":0.003963327966630459, - "coreNode":"core_node6"}}, - "COLL_1t":{"shard1_replica_n4":{ - "INDEX.sizeInBytes":8.5380419785E10, - "INDEX.sizeInGB":79.51671237591654, - "coreNode":"core_node6"}}, - "COLL_2k":{"shard1_replica_n4":{ - "INDEX.sizeInBytes":135.0, - "INDEX.sizeInGB":1.257285475730896E-7, - "coreNode":"core_node6"}}, - "COLL_22":{"shard1_replica_n2":{ - "INDEX.sizeInBytes":2.436290627E10, - "INDEX.sizeInGB":22.689724592491984, - "coreNode":"core_node5"}}}}, - "N_7_solr":{ - "isLive":true, - "cores":6.0, - "freedisk":4268.472709655762, - "sysprop.pool":"pool-01", - "node":"N_7_solr", - "sysprop.az":"us-east-1b", - "totaldisk":4998.009765625, - "withCollection":null, - "replicas":{"COLL_2":{ - "shard13_0_0_replica_n1712":{ - "INDEX.sizeInBytes":1.30437704659E11, - "INDEX.sizeInGB":121.47957892995328, - "coreNode":"core_node1713"}, - "shard13_0_1_replica_n1766":{ - "INDEX.sizeInBytes":1.30339106107E11, - "INDEX.sizeInGB":121.38775187265128, - "coreNode":"core_node1767"}, - "shard13_1_0_replica_n1688":{ - "INDEX.sizeInBytes":1.29592823396E11, - "INDEX.sizeInGB":120.69272193685174, - "coreNode":"core_node1689"}, - "shard13_1_1_replica_n806":{ - "INDEX.sizeInBytes":1.2961448776E11, - "INDEX.sizeInGB":120.71289844810963, - "coreNode":"core_node808"}, - "shard15_0_0_replica_n607":{ - "INDEX.sizeInBytes":1.27258670055E11, - "INDEX.sizeInGB":118.5188722377643, - "coreNode":"core_node609"}, - "shard15_0_1_replica_n608":{ - "INDEX.sizeInBytes":1.2722802278E11, - "INDEX.sizeInGB":118.49032973870635, - "coreNode":"core_node610"}}}}, - "N_a_solr":{ - "isLive":true, - "cores":6.0, - "freedisk":4262.172649383545, - "sysprop.pool":"pool-01", - "node":"N_a_solr", - "sysprop.az":"us-east-1b", - "totaldisk":4998.009765625, - "withCollection":null, - "replicas":{"COLL_2":{ - "shard14_0_0_replica_n1118":{ - "INDEX.sizeInBytes":1.30313698451E11, - "INDEX.sizeInGB":121.36408914905041, - "coreNode":"core_node1119"}, - "shard14_0_1_replica_n1124":{ - "INDEX.sizeInBytes":1.31102045065E11, - "INDEX.sizeInGB":122.09829414729029, - "coreNode":"core_node1125"}, - "shard14_1_0_replica_n833":{ - "INDEX.sizeInBytes":1.27418065808E11, - "INDEX.sizeInGB":118.66732110083103, - "coreNode":"core_node835"}, - "shard14_1_1_replica_n834":{ - "INDEX.sizeInBytes":1.29318568492E11, - "INDEX.sizeInGB":120.43730215355754, - "coreNode":"core_node836"}, - "shard15_1_0_replica_n1174":{ - "INDEX.sizeInBytes":1.33321224738E11, - "INDEX.sizeInGB":124.16506627388299, - "coreNode":"core_node1175"}, - "shard3_0_0_replica_n1808":{ - "INDEX.sizeInBytes":1.29798330608E11, - "INDEX.sizeInGB":120.88411544263363, - "coreNode":"core_node1809"}}}}, - "N_13_solr":{ - "isLive":true, - "cores":5.0, - "freedisk":718.1634063720703, - "sysprop.pool":"pool-02", - "node":"N_13_solr", - "sysprop.az":"us-east-1c", - "totaldisk":999.51171875, - "withCollection":null, - "replicas":{ - "COLL_1":{ - "shard1_0_0_replica_n55":{ - "INDEX.sizeInBytes":4.5921892273E10, - "INDEX.sizeInGB":42.76809494290501, - "coreNode":"core_node57"}, - "shard1_0_1_replica_n56":{ - "INDEX.sizeInBytes":4.4932001726E10, - "INDEX.sizeInGB":41.846187530085444, - "coreNode":"core_node58"}, - "shard1_1_0_replica_n59":{ - "INDEX.sizeInBytes":4.3783419579E10, - "INDEX.sizeInGB":40.77648704778403, - "coreNode":"core_node61"}, - "shard1_1_1_replica_n60":{ - "INDEX.sizeInBytes":4.811959042E10, - "INDEX.sizeInGB":44.814860839396715, - "coreNode":"core_node62"}}, - "COLL_0":{"shard2_replica_n10":{ - "INDEX.sizeInBytes":3.4248182159E10, - "INDEX.sizeInGB":31.896105184219778, - "coreNode":"core_node13"}}}}, - "N_1d_solr":{ - "isLive":true, - "cores":6.0, - "freedisk":4273.009799957275, - "sysprop.pool":"pool-01", - "node":"N_1d_solr", - "sysprop.az":"us-east-1a", - "totaldisk":4998.009765625, - "withCollection":null, - "replicas":{"COLL_2":{ - "shard12_1_0_replica_n1788":{ - "INDEX.sizeInBytes":1.27487519935E11, - "INDEX.sizeInGB":118.73200529720634, - "coreNode":"core_node1789"}, - "shard14_1_0_replica_n1128":{ - "INDEX.sizeInBytes":1.27407685053E11, - "INDEX.sizeInGB":118.65765326935798, - "coreNode":"core_node1129", - "leader":true}, - "shard14_1_1_replica_n1740":{ - "INDEX.sizeInBytes":1.29231781669E11, - "INDEX.sizeInGB":120.35647562611848, - "coreNode":"core_node1741", - "leader":true}, - "shard15_0_0_replica_n2":{ - "INDEX.sizeInBytes":1.27262832088E11, - "INDEX.sizeInGB":118.5227484330535, - "coreNode":"core_node732", - "leader":true}, - "shard3_1_0_replica_n423":{ - "INDEX.sizeInBytes":1.2828759808E11, - "INDEX.sizeInGB":119.47713613510132, - "coreNode":"core_node425", - "leader":true}, - "shard3_1_1_replica_n424":{ - "INDEX.sizeInBytes":1.29948029547E11, - "INDEX.sizeInGB":121.02353344392031, - "coreNode":"core_node426", - "leader":true}}}}, - "N_1m_solr":{ - "isLive":true, - "cores":6.0, - "freedisk":4257.921604156494, - "sysprop.pool":"pool-01", - "node":"N_1m_solr", - "sysprop.az":"us-east-1b", - "totaldisk":4998.009765625, - "withCollection":null, - "replicas":{"COLL_2":{ - "shard1_1_0_replica_n1678":{ - "INDEX.sizeInBytes":1.28970690262E11, - "INDEX.sizeInGB":120.11331530474126, - "coreNode":"core_node1679"}, - "shard6_1_0_replica_n1166":{ - "INDEX.sizeInBytes":1.29376799009E11, - "INDEX.sizeInGB":120.49153354857117, - "coreNode":"core_node1167"}, - "shard6_1_1_replica_n1744":{ - "INDEX.sizeInBytes":1.31273933482E11, - "INDEX.sizeInGB":122.25837771035731, - "coreNode":"core_node1745"}, - "shard8_0_0_replica_n886":{ - "INDEX.sizeInBytes":1.30145902623E11, - "INDEX.sizeInGB":121.20781710650772, - "coreNode":"core_node887"}, - "shard8_0_1_replica_n892":{ - "INDEX.sizeInBytes":1.32681734677E11, - "INDEX.sizeInGB":123.56949474383146, - "coreNode":"core_node893"}, - "shard8_1_1_replica_n1710":{ - "INDEX.sizeInBytes":1.33374089494E11, - "INDEX.sizeInGB":124.21430041454732, - "coreNode":"core_node1711"}}}}, - "N_17_solr":{ - "isLive":true, - "cores":6.0, - "freedisk":4093.756145477295, - "sysprop.pool":"pool-01", - "node":"N_17_solr", - "sysprop.az":"us-east-1a", - "totaldisk":4998.009765625, - "withCollection":null, - "replicas":{"COLL_2":{ - "shard11_1_1_replica_n762":{ - "INDEX.sizeInBytes":1.30871431234E11, - "INDEX.sizeInGB":121.88351828046143, - "coreNode":"core_node768", - "leader":true}, - "shard12_0_0_replica_n1750":{ - "INDEX.sizeInBytes":1.2936875619E11, - "INDEX.sizeInGB":120.48404308967292, - "coreNode":"core_node1751"}, - "shard12_0_1_replica_n1698":{ - "INDEX.sizeInBytes":1.30350286057E11, - "INDEX.sizeInGB":121.39816401246935, - "coreNode":"core_node1699"}, - "shard14_0_0_replica_n1120":{ - "INDEX.sizeInBytes":1.3029908264E11, - "INDEX.sizeInGB":121.3504771143198, - "coreNode":"core_node1121"}, - "shard14_0_1_replica_n1122":{ - "INDEX.sizeInBytes":1.31146492351E11, - "INDEX.sizeInGB":122.13968890812248, - "coreNode":"core_node1123"}, - "shard18_0_1_replica_n2":{ - "INDEX.sizeInBytes":1.28174988934E11, - "INDEX.sizeInGB":119.37226069532335, - "coreNode":"core_node877"}}}}, - "N_11_solr":{ - "isLive":true, - "cores":6.0, - "freedisk":4264.325901031494, - "sysprop.pool":"pool-01", - "node":"N_11_solr", - "sysprop.az":"us-east-1b", - "totaldisk":4998.009765625, - "withCollection":null, - "replicas":{"COLL_2":{ - "shard12_1_1_replica_n2":{ - "INDEX.sizeInBytes":1.26693447901E11, - "INDEX.sizeInGB":117.99246808607131, - "coreNode":"core_node662"}, - "shard6_0_0_replica_n1209":{ - "INDEX.sizeInBytes":1.28939953876E11, - "INDEX.sizeInGB":120.08468981459737, - "coreNode":"core_node1210"}, - "shard6_0_1_replica_n1211":{ - "INDEX.sizeInBytes":1.28744354495E11, - "INDEX.sizeInGB":119.90252369549125, - "coreNode":"core_node1212"}, - "shard9_0_1_replica_n436":{ - "INDEX.sizeInBytes":1.29282915395E11, - "INDEX.sizeInGB":120.40409761946648, - "coreNode":"core_node438"}, - "shard9_1_0_replica_n1152":{ - "INDEX.sizeInBytes":1.31406038908E11, - "INDEX.sizeInGB":122.3814104758203, - "coreNode":"core_node1153"}, - "shard9_1_1_replica_n1154":{ - "INDEX.sizeInBytes":1.33894519282E11, - "INDEX.sizeInGB":124.69898842461407, - "coreNode":"core_node1155"}}}}, - "N_z_solr":{ - "isLive":true, - "cores":6.0, - "freedisk":4215.115695953369, - "sysprop.pool":"pool-01", - "node":"N_z_solr", - "sysprop.az":"us-east-1c", - "totaldisk":4998.009765625, - "withCollection":null, - "replicas":{"COLL_2":{ - "shard14_1_0_replica_n1126":{ - "INDEX.sizeInBytes":1.27443177079E11, - "INDEX.sizeInGB":118.69070779439062, - "coreNode":"core_node1127"}, - "shard1_0_0_replica_n1716":{ - "INDEX.sizeInBytes":5.7185112146E10, - "INDEX.sizeInGB":53.25778587348759, - "coreNode":"core_node1717"}, - "shard8_0_0_replica_n1730":{ - "INDEX.sizeInBytes":1.30170301246E11, - "INDEX.sizeInGB":121.23054009489715, - "coreNode":"core_node1731"}, - "shard8_0_1_replica_n1694":{ - "INDEX.sizeInBytes":1.39918850407E11, - "INDEX.sizeInGB":130.30958399828523, - "coreNode":"core_node1695", - "leader":true}, - "shard8_1_0_replica_n1706":{ - "INDEX.sizeInBytes":1.35679630668E11, - "INDEX.sizeInGB":126.361502956599, - "coreNode":"core_node1707", - "leader":true}, - "shard8_1_1_replica_n1754":{ - "INDEX.sizeInBytes":1.33314153125E11, - "INDEX.sizeInGB":124.15848032105714, - "coreNode":"core_node1755", - "leader":true}}}}, - "N_t_solr":{ - "isLive":true, - "cores":6.0, - "freedisk":4266.856658935547, - "sysprop.pool":"pool-01", - "node":"N_t_solr", - "sysprop.az":"us-east-1a", - "totaldisk":4998.009765625, - "withCollection":null, - "replicas":{"COLL_2":{ - "shard10_0_0_replica_n1732":{ - "INDEX.sizeInBytes":1.2914349283E11, - "INDEX.sizeInGB":120.27425023727119, - "coreNode":"core_node1733"}, - "shard10_0_1_replica_n904":{ - "INDEX.sizeInBytes":1.28142990156E11, - "INDEX.sizeInGB":119.34245951101184, - "coreNode":"core_node905"}, - "shard10_1_1_replica_n1742":{ - "INDEX.sizeInBytes":1.30757016285E11, - "INDEX.sizeInGB":121.77696105558425, - "coreNode":"core_node1743", - "leader":true}, - "shard11_0_0_replica_n1183":{ - "INDEX.sizeInBytes":1.2777477116E11, - "INDEX.sizeInGB":118.99952884763479, - "coreNode":"core_node1185", - "leader":true}, - "shard11_0_1_replica_n1184":{ - "INDEX.sizeInBytes":1.27980394382E11, - "INDEX.sizeInGB":119.19103039614856, - "coreNode":"core_node1195", - "leader":true}, - "shard11_1_0_replica_n1790":{ - "INDEX.sizeInBytes":1.32416023485E11, - "INDEX.sizeInGB":123.32203191239387, - "coreNode":"core_node1791"}}}}, - "N_1c_solr":{ - "isLive":true, - "cores":6.0, - "freedisk":4181.229598999023, - "sysprop.pool":"pool-01", - "node":"N_1c_solr", - "sysprop.az":"us-east-1b", - "totaldisk":4998.009765625, - "withCollection":null, - "replicas":{"COLL_2":{ - "shard18_0_0_replica_n1":{ - "INDEX.sizeInBytes":1.28011422432E11, - "INDEX.sizeInGB":119.21992751955986, - "coreNode":"core_node874", - "leader":true}, - "shard18_0_1_replica_n1":{ - "INDEX.sizeInBytes":1.30729375574E11, - "INDEX.sizeInGB":121.75121863745153, - "coreNode":"core_node876"}, - "shard5_0_0_replica_n998":{ - "INDEX.sizeInBytes":1.31937405764E11, - "INDEX.sizeInGB":122.87628442421556, - "coreNode":"core_node999", - "leader":true}, - "shard5_0_1_replica_n1702":{ - "INDEX.sizeInBytes":1.31521149156E11, - "INDEX.sizeInGB":122.48861524835229, - "coreNode":"core_node1703", - "leader":true}, - "shard5_1_0_replica_n1134":{ - "INDEX.sizeInBytes":1.30030877168E11, - "INDEX.sizeInGB":121.1006913036108, - "coreNode":"core_node1135", - "leader":true}, - "shard5_1_1_replica_n1140":{ - "INDEX.sizeInBytes":1.29917464329E11, - "INDEX.sizeInGB":120.99506736639887, - "coreNode":"core_node1141", - "leader":true}}}}, - "N_1i_solr":{ - "isLive":true, - "cores":6.0, - "freedisk":4266.027156829834, - "sysprop.pool":"pool-01", - "node":"N_1i_solr", - "sysprop.az":"us-east-1c", - "totaldisk":4998.009765625, - "withCollection":null, - "replicas":{"COLL_2":{ - "shard10_1_0_replica_n1692":{ - "INDEX.sizeInBytes":1.27561685082E11, - "INDEX.sizeInGB":118.80107697285712, - "coreNode":"core_node1693", - "leader":true}, - "shard10_1_1_replica_n1778":{ - "INDEX.sizeInBytes":1.30255789623E11, - "INDEX.sizeInGB":121.31015735026449, - "coreNode":"core_node1779"}, - "shard17_0_0_replica_n1780":{ - "INDEX.sizeInBytes":1.30702509646E11, - "INDEX.sizeInGB":121.72619779221714, - "coreNode":"core_node1781"}, - "shard17_0_1_replica_n1116":{ - "INDEX.sizeInBytes":1.30694171889E11, - "INDEX.sizeInGB":121.71843265090138, - "coreNode":"core_node1117"}, - "shard17_1_0_replica_n1198":{ - "INDEX.sizeInBytes":1.29069936299E11, - "INDEX.sizeInGB":120.20574537944049, - "coreNode":"core_node1200"}, - "shard17_1_1_replica_n1199":{ - "INDEX.sizeInBytes":1.28764084367E11, - "INDEX.sizeInGB":119.92089857067913, - "coreNode":"core_node1203"}}}}, - "N_g_solr":{ - "isLive":true, - "cores":6.0, - "freedisk":4007.3253440856934, - "sysprop.pool":"pool-01", - "node":"N_g_solr", - "sysprop.az":"us-east-1a", - "totaldisk":4998.009765625, - "withCollection":null, - "replicas":{"COLL_2":{ - "shard2_1_0_replica_n1680":{ - "INDEX.sizeInBytes":1.3012044407E11, - "INDEX.sizeInGB":121.18410698138177, - "coreNode":"core_node1681"}, - "shard5_0_0_replica_n1768":{ - "INDEX.sizeInBytes":1.31922267714E11, - "INDEX.sizeInGB":122.8621860165149, - "coreNode":"core_node1769"}, - "shard5_0_1_replica_n1770":{ - "INDEX.sizeInBytes":1.31464210597E11, - "INDEX.sizeInGB":122.43558708298951, - "coreNode":"core_node1771"}, - "shard5_1_0_replica_n1782":{ - "INDEX.sizeInBytes":1.30012462556E11, - "INDEX.sizeInGB":121.08354135975242, - "coreNode":"core_node1783"}, - "shard5_1_1_replica_n859":{ - "INDEX.sizeInBytes":1.29967769078E11, - "INDEX.sizeInGB":121.04191731475294, - "coreNode":"core_node861"}, - "shard9_0_0_replica_n1682":{ - "INDEX.sizeInBytes":1.29248772716E11, - "INDEX.sizeInGB":120.37229977175593, - "coreNode":"core_node1683"}}}}, - "N_8_solr":{ - "isLive":true, - "cores":6.0, - "freedisk":4262.037788391113, - "sysprop.pool":"pool-01", - "node":"N_8_solr", - "sysprop.az":"us-east-1b", - "totaldisk":4998.009765625, - "withCollection":null, - "replicas":{"COLL_2":{ - "shard16_0_0_replica_n854":{ - "INDEX.sizeInBytes":1.2677230126E11, - "INDEX.sizeInGB":118.06590599939227, - "coreNode":"core_node856", - "leader":true}, - "shard16_0_1_replica_n855":{ - "INDEX.sizeInBytes":1.30788718518E11, - "INDEX.sizeInGB":121.80648606084287, - "coreNode":"core_node857", - "leader":true}, - "shard16_1_0_replica_n850":{ - "INDEX.sizeInBytes":1.28801317856E11, - "INDEX.sizeInGB":119.95557495951653, - "coreNode":"core_node852", - "leader":true}, - "shard16_1_1_replica_n851":{ - "INDEX.sizeInBytes":1.33685050832E11, - "INDEX.sizeInGB":124.50390572845936, - "coreNode":"core_node853", - "leader":true}, - "shard2_0_0_replica_n794":{ - "INDEX.sizeInBytes":1.29517293483E11, - "INDEX.sizeInGB":120.6223792238161, - "coreNode":"core_node796", - "leader":true}, - "shard2_0_1_replica_n795":{ - "INDEX.sizeInBytes":1.31328007233E11, - "INDEX.sizeInGB":122.30873781535774, - "coreNode":"core_node800", - "leader":true}}}}, - "N_1f_solr":{ - "isLive":true, - "cores":6.0, - "freedisk":4260.807849884033, - "sysprop.pool":"pool-01", - "node":"N_1f_solr", - "sysprop.az":"us-east-1c", - "totaldisk":4998.009765625, - "withCollection":null, - "replicas":{"COLL_2":{ - "shard11_0_0_replica_n1216":{ - "INDEX.sizeInBytes":1.27720861488E11, - "INDEX.sizeInGB":118.94932155311108, - "coreNode":"core_node1217"}, - "shard11_0_1_replica_n1222":{ - "INDEX.sizeInBytes":1.27989218509E11, - "INDEX.sizeInGB":119.19924850482494, - "coreNode":"core_node1223"}, - "shard11_1_0_replica_n778":{ - "INDEX.sizeInBytes":1.32552454912E11, - "INDEX.sizeInGB":123.44909358024597, - "coreNode":"core_node779"}, - "shard11_1_1_replica_n782":{ - "INDEX.sizeInBytes":1.30995783614E11, - "INDEX.sizeInGB":121.99933045916259, - "coreNode":"core_node783"}, - "shard5_0_0_replica_n1000":{ - "INDEX.sizeInBytes":1.31960210955E11, - "INDEX.sizeInGB":122.89752341341227, - "coreNode":"core_node1001"}, - "shard5_0_1_replica_n1002":{ - "INDEX.sizeInBytes":1.31534942129E11, - "INDEX.sizeInGB":122.50146095547825, - "coreNode":"core_node1003"}}}}, - "N_v_solr":{ - "isLive":true, - "cores":5.0, - "freedisk":412.18456649780273, - "sysprop.pool":"pool-02", - "node":"N_v_solr", - "sysprop.az":"us-east-1a", - "totaldisk":999.51171875, - "withCollection":null, - "replicas":{ - "COLL_1":{ - "shard3_0_0_replica_n119":{ - "INDEX.sizeInBytes":4.3809517838E10, - "INDEX.sizeInGB":40.80079294554889, - "coreNode":"core_node120"}, - "shard3_0_1_replica_n121":{ - "INDEX.sizeInBytes":4.6310602091E10, - "INDEX.sizeInGB":43.13010917138308, - "coreNode":"core_node122"}, - "shard3_1_0_replica_n123":{ - "INDEX.sizeInBytes":4.5638162031E10, - "INDEX.sizeInGB":42.503850563429296, - "coreNode":"core_node124"}, - "shard3_1_1_replica_n125":{ - "INDEX.sizeInBytes":4.4257494507E10, - "INDEX.sizeInGB":41.21800373028964, - "coreNode":"core_node126"}}, - "COLL_0":{"shard3_replica_n16":{ - "INDEX.sizeInBytes":2.8932093807E10, - "INDEX.sizeInGB":26.94511209335178, - "coreNode":"core_node18"}}}}, - "N_m_solr":{ - "isLive":true, - "cores":6.0, - "freedisk":4267.171646118164, - "sysprop.pool":"pool-01", - "node":"N_m_solr", - "sysprop.az":"us-east-1a", - "totaldisk":4998.009765625, - "withCollection":null, - "replicas":{"COLL_2":{ - "shard17_1_0_replica_n1226":{ - "INDEX.sizeInBytes":1.29049722959E11, - "INDEX.sizeInGB":120.18692023959011, - "coreNode":"core_node1227"}, - "shard17_1_1_replica_n1228":{ - "INDEX.sizeInBytes":1.28816978409E11, - "INDEX.sizeInGB":119.97015998605639, - "coreNode":"core_node1229"}, - "shard6_0_0_replica_n1207":{ - "INDEX.sizeInBytes":1.28936808614E11, - "INDEX.sizeInGB":120.08176056109369, - "coreNode":"core_node1208"}, - "shard6_0_1_replica_n1213":{ - "INDEX.sizeInBytes":1.28745543493E11, - "INDEX.sizeInGB":119.90363103616983, - "coreNode":"core_node1214"}, - "shard6_1_1_replica_n1170":{ - "INDEX.sizeInBytes":1.31256081422E11, - "INDEX.sizeInGB":122.24175168387592, - "coreNode":"core_node1171"}, - "shard9_0_1_replica_n1":{ - "INDEX.sizeInBytes":1.29063920601E11, - "INDEX.sizeInGB":120.20014282409102, - "coreNode":"core_node477"}}}}, - "N_16_solr":{ - "isLive":true, - "cores":5.0, - "freedisk":795.7872657775879, - "sysprop.pool":"pool-02", - "node":"N_16_solr", - "sysprop.az":"us-east-1b", - "totaldisk":999.51171875, - "withCollection":null, - "replicas":{ - "COLL_1":{ - "shard2_0_0_replica_n99":{ - "INDEX.sizeInBytes":4.8764329025E10, - "INDEX.sizeInGB":45.41532045695931, - "coreNode":"core_node100"}, - "shard2_0_1_replica_n101":{ - "INDEX.sizeInBytes":4.3740343099E10, - "INDEX.sizeInGB":40.73636894952506, - "coreNode":"core_node102"}, - "shard2_1_0_replica_n95":{ - "INDEX.sizeInBytes":4.5585236311E10, - "INDEX.sizeInGB":42.45455964561552, - "coreNode":"core_node96"}, - "shard2_1_1_replica_n97":{ - "INDEX.sizeInBytes":4.527594328E10, - "INDEX.sizeInGB":42.16650806367397, - "coreNode":"core_node98"}}, - "COLL_0":{"shard1_replica_n2":{ - "INDEX.sizeInBytes":3.3775978753E10, - "INDEX.sizeInGB":31.45633149240166, - "coreNode":"core_node5"}}}}, - "N_3a_solr":{ - "isLive":true, - "cores":5.0, - "freedisk":407.706729888916, - "sysprop.pool":"pool-02", - "node":"N_3a_solr", - "sysprop.az":"us-east-1b", - "totaldisk":999.51171875, - "withCollection":null, - "replicas":{ - "COLL_1":{ - "shard3_0_0_replica_n71":{ - "INDEX.sizeInBytes":4.5160600486E10, - "INDEX.sizeInGB":42.05908671580255, - "coreNode":"core_node73"}, - "shard3_0_1_replica_n72":{ - "INDEX.sizeInBytes":4.5879426317E10, - "INDEX.sizeInGB":42.72854543942958, - "coreNode":"core_node74"}, - "shard3_1_0_replica_n75":{ - "INDEX.sizeInBytes":4.5090380622E10, - "INDEX.sizeInGB":41.99368937127292, - "coreNode":"core_node77"}, - "shard3_1_1_replica_n76":{ - "INDEX.sizeInBytes":4.6849085882E10, - "INDEX.sizeInGB":43.631611282005906, - "coreNode":"core_node78"}}, - "COLL_0":{"shard3_replica_n14":{ - "INDEX.sizeInBytes":3.0819950704E10, - "INDEX.sizeInGB":28.70331583917141, - "coreNode":"core_node17"}}}}, - "N_u_solr":{ - "isLive":true, - "cores":6.0, - "freedisk":4260.821304321289, - "sysprop.pool":"pool-01", - "node":"N_u_solr", - "sysprop.az":"us-east-1a", - "totaldisk":4998.009765625, - "withCollection":null, - "replicas":{"COLL_2":{ - "shard13_0_0_replica_n1256":{ - "INDEX.sizeInBytes":1.30381429251E11, - "INDEX.sizeInGB":121.42716837208718, - "coreNode":"core_node1257"}, - "shard13_0_1_replica_n1262":{ - "INDEX.sizeInBytes":1.30321828131E11, - "INDEX.sizeInGB":121.37166050355881, - "coreNode":"core_node1263"}, - "shard13_1_0_replica_n1762":{ - "INDEX.sizeInBytes":1.29567251239E11, - "INDEX.sizeInGB":120.66890600975603, - "coreNode":"core_node1763"}, - "shard13_1_1_replica_n920":{ - "INDEX.sizeInBytes":1.29634542289E11, - "INDEX.sizeInGB":120.73157568369061, - "coreNode":"core_node921", - "leader":true}, - "shard15_0_1_replica_n2":{ - "INDEX.sizeInBytes":1.27250282639E11, - "INDEX.sizeInGB":118.51106084790081, - "coreNode":"core_node734", - "leader":true}, - "shard8_1_0_replica_n1764":{ - "INDEX.sizeInBytes":1.35571920799E11, - "INDEX.sizeInGB":126.26119032409042, - "coreNode":"core_node1765"}}}}, - "N_e_solr":{ - "isLive":true, - "cores":6.0, - "freedisk":4334.874732971191, - "sysprop.pool":"pool-01", - "node":"N_e_solr", - "sysprop.az":"us-east-1c", - "totaldisk":4998.009765625, - "withCollection":null, - "replicas":{"COLL_2":{ - "shard13_1_0_replica_n922":{ - "INDEX.sizeInBytes":1.29514183189E11, - "INDEX.sizeInGB":120.6194825368002, - "coreNode":"core_node923", - "leader":true}, - "shard13_1_1_replica_n924":{ - "INDEX.sizeInBytes":1.29598508564E11, - "INDEX.sizeInGB":120.69801666215062, - "coreNode":"core_node925"}, - "shard15_0_0_replica_n1":{ - "INDEX.sizeInBytes":1.27235871561E11, - "INDEX.sizeInGB":118.49763948563486, - "coreNode":"core_node731"}, - "shard18_0_0_replica_n1818":{ - "INDEX.sizeInBytes":1.28218931509E11, - "INDEX.sizeInGB":119.41318540740758, - "coreNode":"core_node1819"}, - "shard18_1_0_replica_n2":{ - "INDEX.sizeInBytes":1.29108586002E11, - "INDEX.sizeInGB":120.24174072034657, - "coreNode":"core_node672"}, - "shard1_0_1_replica_n1718":{ - "INDEX.sizeInBytes":5.9506746089E10, - "INDEX.sizeInGB":55.41997597459704, - "coreNode":"core_node1719", - "leader":true}}}}, - "N_29_solr":{ - "isLive":true, - "cores":6.0, - "freedisk":4303.548599243164, - "sysprop.pool":"pool-01", - "node":"N_29_solr", - "sysprop.az":"us-east-1a", - "totaldisk":4998.009765625, - "withCollection":null, - "replicas":{"COLL_2":{ - "shard6_1_0_replica_n1792":{ - "INDEX.sizeInBytes":1.29365181039E11, - "INDEX.sizeInGB":120.48071347083896, - "coreNode":"core_node1793"}, - "shard7_0_1_replica_n2":{ - "INDEX.sizeInBytes":8.6794048237E10, - "INDEX.sizeInGB":80.83325646538287, - "coreNode":"core_node777"}, - "shard7_1_0_replica_n1142":{ - "INDEX.sizeInBytes":1.2946739865E11, - "INDEX.sizeInGB":120.57591103948653, - "coreNode":"core_node1143"}, - "shard7_1_1_replica_n1758":{ - "INDEX.sizeInBytes":1.28546712309E11, - "INDEX.sizeInGB":119.7184550659731, - "coreNode":"core_node1759"}, - "shard8_0_0_replica_n1690":{ - "INDEX.sizeInBytes":1.30176337999E11, - "INDEX.sizeInGB":121.23616225924343, - "coreNode":"core_node1691", - "leader":true}, - "shard8_0_1_replica_n1786":{ - "INDEX.sizeInBytes":1.32692723859E11, - "INDEX.sizeInGB":123.57972921710461, - "coreNode":"core_node1787"}}}}, - "N_2u_solr":{ - "isLive":true, - "cores":6.0, - "freedisk":4266.648368835449, - "sysprop.pool":"pool-01", - "node":"N_2u_solr", - "sysprop.az":"us-east-1b", - "totaldisk":4998.009765625, - "withCollection":null, - "replicas":{"COLL_2":{ - "shard12_1_0_replica_n2":{ - "INDEX.sizeInBytes":1.27387972534E11, - "INDEX.sizeInGB":118.63929455541074, - "coreNode":"core_node660"}, - "shard17_0_0_replica_n1734":{ - "INDEX.sizeInBytes":1.31102615765E11, - "INDEX.sizeInGB":122.09882565308362, - "coreNode":"core_node1735"}, - "shard17_0_1_replica_n1114":{ - "INDEX.sizeInBytes":1.30049647193E11, - "INDEX.sizeInGB":121.1181722516194, - "coreNode":"core_node1115"}, - "shard17_1_0_replica_n1224":{ - "INDEX.sizeInBytes":1.29066474889E11, - "INDEX.sizeInGB":120.20252169016749, - "coreNode":"core_node1225", - "leader":true}, - "shard17_1_1_replica_n1230":{ - "INDEX.sizeInBytes":1.287734207E11, - "INDEX.sizeInGB":119.92959370836616, - "coreNode":"core_node1231", - "leader":true}, - "shard3_1_1_replica_n1":{ - "INDEX.sizeInBytes":1.29953637358E11, - "INDEX.sizeInGB":121.02875612489879, - "coreNode":"core_node461"}}}}, - "N_2w_solr":{ - "isLive":true, - "cores":6.0, - "freedisk":4336.208312988281, - "sysprop.pool":"pool-01", - "node":"N_2w_solr", - "sysprop.az":"us-east-1b", - "totaldisk":4998.009765625, - "withCollection":null, - "replicas":{"COLL_2":{ - "shard18_1_0_replica_n1":{ - "INDEX.sizeInBytes":1.28884297502E11, - "INDEX.sizeInGB":120.03285577706993, - "coreNode":"core_node671"}, - "shard18_1_1_replica_n1":{ - "INDEX.sizeInBytes":1.28226679933E11, - "INDEX.sizeInGB":119.42040168959647, - "coreNode":"core_node673"}, - "shard1_0_1_replica_n1676":{ - "INDEX.sizeInBytes":5.9557275352E10, - "INDEX.sizeInGB":55.46703501790762, - "coreNode":"core_node1677"}, - "shard1_1_1_replica_n1806":{ - "INDEX.sizeInBytes":1.2954748046E11, - "INDEX.sizeInGB":120.6504930369556, - "coreNode":"core_node1807"}, - "shard4_1_0_replica_n1774":{ - "INDEX.sizeInBytes":1.27659935903E11, - "INDEX.sizeInGB":118.89258018042892, - "coreNode":"core_node1775"}, - "shard4_1_1_replica_n1804":{ - "INDEX.sizeInBytes":1.27878088796E11, - "INDEX.sizeInGB":119.0957508943975, - "coreNode":"core_node1805"}}}}, - "N_74_solr":{ - "isLive":true, - "cores":5.0, - "freedisk":802.5921897888184, - "sysprop.pool":"pool-02", - "node":"N_74_solr", - "sysprop.az":"us-east-1a", - "totaldisk":999.51171875, - "withCollection":null, - "replicas":{ - "COLL_1":{ - "shard2_0_0_replica_n107":{ - "INDEX.sizeInBytes":4.3767024396E10, - "INDEX.sizeInGB":40.76121784374118, - "coreNode":"core_node108"}, - "shard2_0_1_replica_n109":{ - "INDEX.sizeInBytes":4.8622428842E10, - "INDEX.sizeInGB":45.28316561318934, - "coreNode":"core_node110"}, - "shard2_1_0_replica_n103":{ - "INDEX.sizeInBytes":4.4599223614E10, - "INDEX.sizeInGB":41.536263762041926, - "coreNode":"core_node104"}, - "shard2_1_1_replica_n105":{ - "INDEX.sizeInBytes":4.3768191618E10, - "INDEX.sizeInGB":40.762304903939366, - "coreNode":"core_node106"}}, - "COLL_0":{"shard1_replica_n4":{ - "INDEX.sizeInBytes":2.9252853492E10, - "INDEX.sizeInGB":27.24384282901883, - "coreNode":"core_node7"}}}}, - "N_6i_solr":{ - "isLive":true, - "cores":6.0, - "freedisk":4269.712917327881, - "sysprop.pool":"pool-01", - "node":"N_6i_solr", - "sysprop.az":"us-east-1b", - "totaldisk":4998.009765625, - "withCollection":null, - "replicas":{"COLL_2":{ - "shard10_0_0_replica_n896":{ - "INDEX.sizeInBytes":1.29103730913E11, - "INDEX.sizeInGB":120.2372190663591, - "coreNode":"core_node897"}, - "shard10_0_1_replica_n1738":{ - "INDEX.sizeInBytes":1.28024871739E11, - "INDEX.sizeInGB":119.2324531627819, - "coreNode":"core_node1739"}, - "shard10_1_0_replica_n829":{ - "INDEX.sizeInBytes":1.27564995026E11, - "INDEX.sizeInGB":118.80415959842503, - "coreNode":"core_node831"}, - "shard10_1_1_replica_n830":{ - "INDEX.sizeInBytes":1.30273229534E11, - "INDEX.sizeInGB":121.32639953307807, - "coreNode":"core_node840"}, - "shard2_1_0_replica_n1726":{ - "INDEX.sizeInBytes":1.30025926492E11, - "INDEX.sizeInGB":121.0960806272924, - "coreNode":"core_node1727"}, - "shard2_1_1_replica_n978":{ - "INDEX.sizeInBytes":1.2815510735E11, - "INDEX.sizeInGB":119.35374452732503, - "coreNode":"core_node979"}}}}, - "N_dj_solr":{ - "isLive":true, - "cores":6.0, - "freedisk":4162.087951660156, - "sysprop.pool":"pool-01", - "node":"N_dj_solr", - "sysprop.az":"us-east-1c", - "totaldisk":4998.009765625, - "withCollection":null, - "replicas":{"COLL_2":{ - "shard13_0_0_replica_n1748":{ - "INDEX.sizeInBytes":1.30427736106E11, - "INDEX.sizeInGB":121.47029499150813, - "coreNode":"core_node1749", - "leader":true}, - "shard13_0_1_replica_n1714":{ - "INDEX.sizeInBytes":1.30355121703E11, - "INDEX.sizeInGB":121.402667558752, - "coreNode":"core_node1715", - "leader":true}, - "shard18_0_1_replica_n771":{ - "INDEX.sizeInBytes":1.30821199599E11, - "INDEX.sizeInGB":121.83673642482609, - "coreNode":"core_node773", - "leader":true}, - "shard1_1_0_replica_n1":{ - "INDEX.sizeInBytes":1.29057719236E11, - "INDEX.sizeInGB":120.19436735287309, - "coreNode":"core_node471", - "leader":true}, - "shard7_1_0_replica_n926":{ - "INDEX.sizeInBytes":1.29963886019E11, - "INDEX.sizeInGB":121.03830093424767, - "coreNode":"core_node928", - "leader":true}, - "shard7_1_1_replica_n927":{ - "INDEX.sizeInBytes":1.28538540188E11, - "INDEX.sizeInGB":119.71084418520331, - "coreNode":"core_node941", - "leader":true}}}}, - "N_6c_solr":{ - "isLive":true, - "cores":6.0, - "freedisk":4269.135753631592, - "sysprop.pool":"pool-01", - "node":"N_6c_solr", - "sysprop.az":"us-east-1a", - "totaldisk":4998.009765625, - "withCollection":null, - "replicas":{"COLL_2":{ - "shard17_0_0_replica_n842":{ - "INDEX.sizeInBytes":1.30743109221E11, - "INDEX.sizeInGB":121.76400909293443, - "coreNode":"core_node844", - "leader":true}, - "shard17_0_1_replica_n843":{ - "INDEX.sizeInBytes":1.30730929322E11, - "INDEX.sizeInGB":121.7526656780392, - "coreNode":"core_node848", - "leader":true}, - "shard4_0_0_replica_n443":{ - "INDEX.sizeInBytes":1.28741762257E11, - "INDEX.sizeInGB":119.90010948572308, - "coreNode":"core_node445", - "leader":true}, - "shard4_0_1_replica_n444":{ - "INDEX.sizeInBytes":1.28032413116E11, - "INDEX.sizeInGB":119.23947661742568, - "coreNode":"core_node446", - "leader":true}, - "shard4_1_0_replica_n455":{ - "INDEX.sizeInBytes":1.27664473589E11, - "INDEX.sizeInGB":118.89680622983724, - "coreNode":"core_node457", - "leader":true}, - "shard4_1_1_replica_n456":{ - "INDEX.sizeInBytes":1.27865802727E11, - "INDEX.sizeInGB":119.08430860098451, - "coreNode":"core_node458", - "leader":true}}}}, - "N_9o_solr":{ - "isLive":true, - "cores":6.0, - "freedisk":4265.881809234619, - "sysprop.pool":"pool-01", - "node":"N_9o_solr", - "sysprop.az":"us-east-1b", - "totaldisk":4998.009765625, - "withCollection":null, - "replicas":{"COLL_2":{ - "shard11_0_0_replica_n1218":{ - "INDEX.sizeInBytes":1.28002391411E11, - "INDEX.sizeInGB":119.21151672583073, - "coreNode":"core_node1219"}, - "shard11_0_1_replica_n1220":{ - "INDEX.sizeInBytes":1.28020049235E11, - "INDEX.sizeInGB":119.22796185594052, - "coreNode":"core_node1221"}, - "shard11_1_0_replica_n780":{ - "INDEX.sizeInBytes":1.32420261013E11, - "INDEX.sizeInGB":123.32597841788083, - "coreNode":"core_node781", - "leader":true}, - "shard11_1_1_replica_n784":{ - "INDEX.sizeInBytes":1.30909357727E11, - "INDEX.sizeInGB":121.91884007956833, - "coreNode":"core_node785"}, - "shard7_0_0_replica_n764":{ - "INDEX.sizeInBytes":1.28994593549E11, - "INDEX.sizeInGB":120.13557697553188, - "coreNode":"core_node766"}, - "shard7_0_1_replica_n765":{ - "INDEX.sizeInBytes":1.28908501869E11, - "INDEX.sizeInGB":120.0553978504613, - "coreNode":"core_node769"}}}}, - "N_4g_solr":{ - "isLive":true, - "cores":6.0, - "freedisk":4259.9677734375, - "sysprop.pool":"pool-01", - "node":"N_4g_solr", - "sysprop.az":"us-east-1a", - "totaldisk":4998.009765625, - "withCollection":null, - "replicas":{"COLL_2":{ - "shard18_1_0_replica_n623":{ - "INDEX.sizeInBytes":1.28955475131E11, - "INDEX.sizeInGB":120.09914510976523, - "coreNode":"core_node625", - "leader":true}, - "shard18_1_1_replica_n624":{ - "INDEX.sizeInBytes":1.28190099634E11, - "INDEX.sizeInGB":119.38633363135159, - "coreNode":"core_node626", - "leader":true}, - "shard2_1_1_replica_n1812":{ - "INDEX.sizeInBytes":1.28164947427E11, - "INDEX.sizeInGB":119.36290881317109, - "coreNode":"core_node1813"}, - "shard8_1_1_replica_n1794":{ - "INDEX.sizeInBytes":1.33276674177E11, - "INDEX.sizeInGB":124.1235753307119, - "coreNode":"core_node1795"}, - "shard9_1_0_replica_n929":{ - "INDEX.sizeInBytes":1.31111103315E11, - "INDEX.sizeInGB":122.1067303000018, - "coreNode":"core_node931", - "leader":true}, - "shard9_1_1_replica_n930":{ - "INDEX.sizeInBytes":1.33928213329E11, - "INDEX.sizeInGB":124.73036845121533, - "coreNode":"core_node944", - "leader":true}}}}, - "N_cs_solr":{ - "isLive":true, - "cores":6.0, - "freedisk":4260.629165649414, - "sysprop.pool":"pool-01", - "node":"N_cs_solr", - "sysprop.az":"us-east-1c", - "totaldisk":4998.009765625, - "withCollection":null, - "replicas":{"COLL_2":{ - "shard10_0_0_replica_n825":{ - "INDEX.sizeInBytes":1.29486149433E11, - "INDEX.sizeInGB":120.59337406698614, - "coreNode":"core_node827", - "leader":true}, - "shard10_0_1_replica_n826":{ - "INDEX.sizeInBytes":1.28038688927E11, - "INDEX.sizeInGB":119.245321421884, - "coreNode":"core_node828", - "leader":true}, - "shard15_1_0_replica_n953":{ - "INDEX.sizeInBytes":1.33515745782E11, - "INDEX.sizeInGB":124.34622811339796, - "coreNode":"core_node955", - "leader":true}, - "shard15_1_1_replica_n954":{ - "INDEX.sizeInBytes":1.30865977458E11, - "INDEX.sizeInGB":121.87843905575573, - "coreNode":"core_node956", - "leader":true}, - "shard6_1_0_replica_n935":{ - "INDEX.sizeInBytes":1.29597529819E11, - "INDEX.sizeInGB":120.69710513483733, - "coreNode":"core_node937", - "leader":true}, - "shard6_1_1_replica_n1704":{ - "INDEX.sizeInBytes":1.31274462707E11, - "INDEX.sizeInGB":122.25887058954686, - "coreNode":"core_node1705", - "leader":true}}}}, - "N_d4_solr":{ - "isLive":true, - "cores":5.0, - "freedisk":797.2159843444824, - "sysprop.pool":"pool-02", - "node":"N_d4_solr", - "sysprop.az":"us-east-1c", - "totaldisk":999.51171875, - "withCollection":null, - "replicas":{ - "COLL_1":{ - "shard2_0_0_replica_n67":{ - "INDEX.sizeInBytes":4.497304707E10, - "INDEX.sizeInGB":41.8844139855355, - "coreNode":"core_node69", - "leader":true}, - "shard2_0_1_replica_n68":{ - "INDEX.sizeInBytes":4.5692831033E10, - "INDEX.sizeInGB":42.554765039123595, - "coreNode":"core_node70", - "leader":true}, - "shard2_1_0_replica_n63":{ - "INDEX.sizeInBytes":4.5935880044E10, - "INDEX.sizeInGB":42.78112206980586, - "coreNode":"core_node65", - "leader":true}, - "shard2_1_1_replica_n64":{ - "INDEX.sizeInBytes":4.5166045429E10, - "INDEX.sizeInGB":42.064157714135945, - "coreNode":"core_node66", - "leader":true}}, - "COLL_0":{"shard1_replica_n1":{ - "INDEX.sizeInBytes":3.401835331E10, - "INDEX.sizeInGB":31.682060388848186, - "coreNode":"core_node3", - "leader":true}}}}, - "N_do_solr":{ - "isLive":true, - "cores":5.0, - "freedisk":407.25314712524414, - "sysprop.pool":"pool-02", - "node":"N_do_solr", - "sysprop.az":"us-east-1c", - "totaldisk":999.51171875, - "withCollection":null, - "replicas":{ - "COLL_1":{ - "shard3_0_0_replica_n111":{ - "INDEX.sizeInBytes":4.4957115524E10, - "INDEX.sizeInGB":41.86957657709718, - "coreNode":"core_node112", - "leader":true}, - "shard3_0_1_replica_n113":{ - "INDEX.sizeInBytes":4.577095697E10, - "INDEX.sizeInGB":42.62752548791468, - "coreNode":"core_node114", - "leader":true}, - "shard3_1_0_replica_n115":{ - "INDEX.sizeInBytes":4.3732753925E10, - "INDEX.sizeInGB":40.72930098045617, - "coreNode":"core_node116", - "leader":true}, - "shard3_1_1_replica_n117":{ - "INDEX.sizeInBytes":4.8532509927E10, - "INDEX.sizeInGB":45.19942209776491, - "coreNode":"core_node118", - "leader":true}}, - "COLL_0":{"shard3_replica_n12":{ - "INDEX.sizeInBytes":3.1297025422E10, - "INDEX.sizeInGB":29.147626293823123, - "coreNode":"core_node15", - "leader":true}}}}, - "N_4f_solr":{ - "isLive":true, - "cores":6.0, - "freedisk":4264.210151672363, - "sysprop.pool":"pool-01", - "node":"N_4f_solr", - "sysprop.az":"us-east-1c", - "totaldisk":4998.009765625, - "withCollection":null, - "replicas":{"COLL_2":{ - "shard2_0_1_replica_n914":{ - "INDEX.sizeInBytes":1.31386626219E11, - "INDEX.sizeInGB":122.36333100032061, - "coreNode":"core_node915"}, - "shard2_1_0_replica_n974":{ - "INDEX.sizeInBytes":1.3001251468E11, - "INDEX.sizeInGB":121.0835899040103, - "coreNode":"core_node975", - "leader":true}, - "shard2_1_1_replica_n1684":{ - "INDEX.sizeInBytes":1.2812596905E11, - "INDEX.sizeInGB":119.32660737074912, - "coreNode":"core_node1685", - "leader":true}, - "shard3_0_1_replica_n2":{ - "INDEX.sizeInBytes":1.31838927317E11, - "INDEX.sizeInGB":122.78456922341138, - "coreNode":"core_node546", - "leader":true}, - "shard6_0_0_replica_n1180":{ - "INDEX.sizeInBytes":1.28922958966E11, - "INDEX.sizeInGB":120.06886207126081, - "coreNode":"core_node1182", - "leader":true}, - "shard6_0_1_replica_n1181":{ - "INDEX.sizeInBytes":1.28773562289E11, - "INDEX.sizeInGB":119.92972557339817, - "coreNode":"core_node1189", - "leader":true}}}}, - "N_1h_solr":{ - "isLive":true, - "cores":6.0, - "freedisk":4297.329685211182, - "sysprop.pool":"pool-01", - "node":"N_1h_solr", - "sysprop.az":"us-east-1b", - "totaldisk":4998.009765625, - "withCollection":null, - "replicas":{"COLL_2":{ - "shard12_0_0_replica_n1696":{ - "INDEX.sizeInBytes":1.29369271388E11, - "INDEX.sizeInGB":120.48452290520072, - "coreNode":"core_node1697"}, - "shard12_0_1_replica_n1760":{ - "INDEX.sizeInBytes":1.30342308934E11, - "INDEX.sizeInGB":121.39073473773897, - "coreNode":"core_node1761"}, - "shard1_0_0_replica_n1728":{ - "INDEX.sizeInBytes":5.7176945428E10, - "INDEX.sizeInGB":53.25018002465367, - "coreNode":"core_node1729"}, - "shard3_0_1_replica_n508":{ - "INDEX.sizeInBytes":1.31866901019E11, - "INDEX.sizeInGB":122.81062176357955, - "coreNode":"core_node510"}, - "shard7_1_0_replica_n1144":{ - "INDEX.sizeInBytes":1.2949609012E11, - "INDEX.sizeInGB":120.60263205319643, - "coreNode":"core_node1145"}, - "shard7_1_1_replica_n1700":{ - "INDEX.sizeInBytes":1.28489170345E11, - "INDEX.sizeInGB":119.66486493591219, - "coreNode":"core_node1701"}}}}, - "N_7e_solr":{ - "isLive":true, - "cores":13.0, - "freedisk":873.6022491455078, - "sysprop.pool":"pool-03", - "node":"N_7e_solr", - "sysprop.az":"us-east-1b", - "totaldisk":999.51171875, - "withCollection":null, - "replicas":{ - "COLL_6":{"shard1_replica_n1":{ - "INDEX.sizeInBytes":1.6446420654E10, - "INDEX.sizeInGB":15.316922826692462, - "coreNode":"core_node3"}}, - "COLL_5":{"shard1_replica_n1":{ - "INDEX.sizeInBytes":5.854396964E9, - "INDEX.sizeInGB":5.452332053333521, - "coreNode":"core_node2", - "leader":true}}, - "COLL_l":{"shard1_replica_n1":{ - "INDEX.sizeInBytes":135.0, - "INDEX.sizeInGB":1.257285475730896E-7, - "coreNode":"core_node3"}}, - "COLL_x":{"shard1_replica_n1":{ - "INDEX.sizeInBytes":3.18270873E8, - "INDEX.sizeInGB":0.296412848867476, - "coreNode":"core_node3"}}, - "COLL_1b":{"shard1_replica_n1":{ - "INDEX.sizeInBytes":135.0, - "INDEX.sizeInGB":1.257285475730896E-7, - "coreNode":"core_node3"}}, - "COLL_1r":{"shard1_replica_n1":{ - "INDEX.sizeInBytes":4.12015174E8, - "INDEX.sizeInGB":0.38371903263032436, - "coreNode":"core_node3"}}, - "COLL_8":{"shard1_replica_n1":{ - "INDEX.sizeInBytes":356048.0, - "INDEX.sizeInGB":3.315955400466919E-4, - "coreNode":"core_node3"}}, - "COLL_q":{"shard1_replica_n1":{ - "INDEX.sizeInBytes":4.9242789E8, - "INDEX.sizeInGB":0.45860921032726765, - "coreNode":"core_node3"}}, - "COLL_4":{"shard1_replica_n1":{ - "INDEX.sizeInBytes":2.58881858E8, - "INDEX.sizeInGB":0.2411025185137987, - "coreNode":"core_node3"}}, - "COLL_1x":{"shard1_replica_n1":{ - "INDEX.sizeInBytes":4248411.0, - "INDEX.sizeInGB":0.00395664107054472, - "coreNode":"core_node3"}}, - "COLL_1t":{"shard1_replica_n1":{ - "INDEX.sizeInBytes":8.5774407615E10, - "INDEX.sizeInGB":79.8836421361193, - "coreNode":"core_node3"}}, - "COLL_2k":{"shard1_replica_n1":{ - "INDEX.sizeInBytes":135.0, - "INDEX.sizeInGB":1.257285475730896E-7, - "coreNode":"core_node3"}}, - "COLL_22":{"shard1_replica_n4":{ - "INDEX.sizeInBytes":2.4348956483E10, - "INDEX.sizeInGB":22.676732840947807, - "coreNode":"core_node6"}}}}, - "N_b9_solr":{ - "isLive":true, - "cores":5.0, - "freedisk":801.2417984008789, - "sysprop.pool":"pool-02", - "node":"N_b9_solr", - "sysprop.az":"us-east-1b", - "totaldisk":999.51171875, - "withCollection":null, - "replicas":{ - "COLL_1":{ - "shard1_0_0_replica_n87":{ - "INDEX.sizeInBytes":4.5100613575E10, - "INDEX.sizeInGB":42.0032195514068, - "coreNode":"core_node88"}, - "shard1_0_1_replica_n89":{ - "INDEX.sizeInBytes":4.6030616744E10, - "INDEX.sizeInGB":42.869352497160435, - "coreNode":"core_node90"}, - "shard1_1_0_replica_n91":{ - "INDEX.sizeInBytes":4.5724314347E10, - "INDEX.sizeInGB":42.5840861601755, - "coreNode":"core_node92"}, - "shard1_1_1_replica_n93":{ - "INDEX.sizeInBytes":4.574559386E10, - "INDEX.sizeInGB":42.603904251009226, - "coreNode":"core_node94"}}, - "COLL_0":{"shard2_replica_n6":{ - "INDEX.sizeInBytes":2.8865621899E10, - "INDEX.sizeInGB":26.883205304853618, - "coreNode":"core_node9"}}}}, - "N_aw_solr":{ - "isLive":true, - "cores":6.0, - "freedisk":4276.759601593018, - "sysprop.pool":"pool-01", - "node":"N_aw_solr", - "sysprop.az":"us-east-1c", - "totaldisk":4998.009765625, - "withCollection":null, - "replicas":{"COLL_2":{ - "shard12_1_0_replica_n1":{ - "INDEX.sizeInBytes":1.27434400341E11, - "INDEX.sizeInGB":118.68253382015973, - "coreNode":"core_node659", - "leader":true}, - "shard12_1_1_replica_n1":{ - "INDEX.sizeInBytes":1.26701654869E11, - "INDEX.sizeInGB":118.00011142063886, - "coreNode":"core_node661", - "leader":true}, - "shard15_0_1_replica_n1816":{ - "INDEX.sizeInBytes":1.27129784031E11, - "INDEX.sizeInGB":118.39883777406067, - "coreNode":"core_node1817"}, - "shard18_1_1_replica_n1820":{ - "INDEX.sizeInBytes":1.28188518759E11, - "INDEX.sizeInGB":119.38486132677644, - "coreNode":"core_node1821"}, - "shard3_1_0_replica_n2":{ - "INDEX.sizeInBytes":1.28273400877E11, - "INDEX.sizeInGB":119.46391395945102, - "coreNode":"core_node460"}, - "shard4_1_1_replica_n1":{ - "INDEX.sizeInBytes":1.27899653279E11, - "INDEX.sizeInGB":119.11583438422531, - "coreNode":"core_node525"}}}}, - "N_3a7_solr":{ - "isLive":true, - "cores":6.0, - "freedisk":4263.317134857178, - "sysprop.pool":"pool-01", - "node":"N_3a7_solr", - "sysprop.az":"us-east-1c", - "totaldisk":4998.009765625, - "withCollection":null, - "replicas":{"COLL_2":{ - "shard14_0_0_replica_n837":{ - "INDEX.sizeInBytes":1.30330451538E11, - "INDEX.sizeInGB":121.37969167716801, - "coreNode":"core_node839", - "leader":true}, - "shard14_0_1_replica_n838":{ - "INDEX.sizeInBytes":1.31168916273E11, - "INDEX.sizeInGB":122.1605728128925, - "coreNode":"core_node841", - "leader":true}, - "shard14_1_1_replica_n1824":{ - "INDEX.sizeInBytes":1.300425186E11, - "INDEX.sizeInGB":121.11153323203325, - "coreNode":"core_node1825"}, - "shard2_0_0_replica_n1822":{ - "INDEX.sizeInBytes":1.29476268104E11, - "INDEX.sizeInGB":120.58417136222124, - "coreNode":"core_node1823"}, - "shard3_1_1_replica_n2":{ - "INDEX.sizeInBytes":1.2992912768E11, - "INDEX.sizeInGB":121.00592970848083, - "coreNode":"core_node462"}, - "shard7_0_0_replica_n2":{ - "INDEX.sizeInBytes":1.29074533898E11, - "INDEX.sizeInGB":120.21002722717822, - "coreNode":"core_node775", - "leader":true}}}}, - "N_303_solr":{ - "isLive":true, - "cores":6.0, - "freedisk":4111.4668045043945, - "sysprop.pool":"pool-01", - "node":"N_303_solr", - "sysprop.az":"us-east-1c", - "totaldisk":4998.009765625, - "withCollection":null, - "replicas":{"COLL_2":{ - "shard16_0_0_replica_n1784":{ - "INDEX.sizeInBytes":1.26747476604E11, - "INDEX.sizeInGB":118.04278623685241, - "coreNode":"core_node1785"}, - "shard16_0_1_replica_n986":{ - "INDEX.sizeInBytes":1.30738903625E11, - "INDEX.sizeInGB":121.76009232643992, - "coreNode":"core_node987"}, - "shard3_0_0_replica_n2":{ - "INDEX.sizeInBytes":1.29792212268E11, - "INDEX.sizeInGB":120.87841729447246, - "coreNode":"core_node544", - "leader":true}, - "shard4_0_1_replica_n1772":{ - "INDEX.sizeInBytes":1.28126128215E11, - "INDEX.sizeInGB":119.3267556047067, - "coreNode":"core_node1773"}, - "shard9_1_0_replica_n1150":{ - "INDEX.sizeInBytes":1.31117387108E11, - "INDEX.sizeInGB":122.11258253827691, - "coreNode":"core_node1151"}, - "shard9_1_1_replica_n1162":{ - "INDEX.sizeInBytes":1.36568824379E11, - "INDEX.sizeInGB":127.18962913285941, - "coreNode":"core_node1163"}}}}, - "N_3to_solr":{ - "isLive":true, - "cores":5.0, - "freedisk":794.5433731079102, - "sysprop.pool":"pool-02", - "node":"N_3to_solr", - "sysprop.az":"us-east-1a", - "totaldisk":999.51171875, - "withCollection":null, - "replicas":{ - "COLL_1":{ - "shard1_0_0_replica_n79":{ - "INDEX.sizeInBytes":4.644843302E10, - "INDEX.sizeInGB":43.258474227041006, - "coreNode":"core_node80", - "leader":true}, - "shard1_0_1_replica_n81":{ - "INDEX.sizeInBytes":4.4936912617E10, - "INDEX.sizeInGB":41.85076115373522, - "coreNode":"core_node82", - "leader":true}, - "shard1_1_0_replica_n83":{ - "INDEX.sizeInBytes":4.3892348528E10, - "INDEX.sizeInGB":40.87793503701687, - "coreNode":"core_node84", - "leader":true}, - "shard1_1_1_replica_n85":{ - "INDEX.sizeInBytes":5.1015133973E10, - "INDEX.sizeInGB":47.511545916087925, - "coreNode":"core_node86", - "leader":true}}, - "COLL_0":{"shard2_replica_n8":{ - "INDEX.sizeInBytes":3.0722710385E10, - "INDEX.sizeInGB":28.6127537349239, - "coreNode":"core_node11", - "leader":true}}}}, - "N_65p_solr":{ - "isLive":true, - "cores":6.0, - "freedisk":4260.997627258301, - "sysprop.pool":"pool-01", - "node":"N_65p_solr", - "sysprop.az":"us-east-1a", - "totaldisk":4998.009765625, - "withCollection":null, - "replicas":{"COLL_2":{ - "shard10_1_0_replica_n1796":{ - "INDEX.sizeInBytes":1.27583656591E11, - "INDEX.sizeInGB":118.82153953518718, - "coreNode":"core_node1797"}, - "shard15_1_0_replica_n1172":{ - "INDEX.sizeInBytes":1.33316507698E11, - "INDEX.sizeInGB":124.16067318804562, - "coreNode":"core_node1173"}, - "shard15_1_1_replica_n1746":{ - "INDEX.sizeInBytes":1.30883359905E11, - "INDEX.sizeInGB":121.89462772104889, - "coreNode":"core_node1747"}, - "shard3_0_0_replica_n1":{ - "INDEX.sizeInBytes":1.29871412511E11, - "INDEX.sizeInGB":120.95217826869339, - "coreNode":"core_node543"}, - "shard3_0_1_replica_n1":{ - "INDEX.sizeInBytes":1.31838835644E11, - "INDEX.sizeInGB":122.784483846277, - "coreNode":"core_node545"}, - "shard7_0_0_replica_n1":{ - "INDEX.sizeInBytes":1.29027793373E11, - "INDEX.sizeInGB":120.16649672109634, - "coreNode":"core_node774"}}}}}, - "collectionStats":{ - "COLL_2":{ - "activeShards":72, - "inactiveShards":0, - "rf":2, - "maxActualShardsPerNode":6, - "minActualShardsPerNode":6, - "maxShardReplicasPerNode":1, - "minShardReplicasPerNode":1, - "numCores":216, - "numNodes":36, - "maxCoresPerNode":6, - "minCoresPerNode":6, - "avgShardSize":119.19235682340029, - "maxShardSize":130.30958399828523, - "minShardSize":53.25648116040975}, - "COLL_1":{ - "activeShards":12, - "inactiveShards":0, - "rf":3, - "maxActualShardsPerNode":4, - "minActualShardsPerNode":4, - "maxShardReplicasPerNode":1, - "minShardReplicasPerNode":1, - "numCores":36, - "numNodes":9, - "maxCoresPerNode":4, - "minCoresPerNode":4, - "avgShardSize":42.76741669047624, - "maxShardSize":47.511545916087925, - "minShardSize":40.72930098045617}, - "COLL_0":{ - "activeShards":3, - "inactiveShards":0, - "rf":3, - "maxActualShardsPerNode":1, - "minActualShardsPerNode":1, - "maxShardReplicasPerNode":1, - "minShardReplicasPerNode":1, - "numCores":9, - "numNodes":9, - "maxCoresPerNode":1, - "minCoresPerNode":1, - "avgShardSize":29.814146805865068, - "maxShardSize":31.682060388848186, - "minShardSize":28.6127537349239}, - "COLL_6":{ - "activeShards":1, - "inactiveShards":0, - "rf":3, - "maxActualShardsPerNode":1, - "minActualShardsPerNode":1, - "maxShardReplicasPerNode":1, - "minShardReplicasPerNode":1, - "numCores":3, - "numNodes":3, - "maxCoresPerNode":1, - "minCoresPerNode":1, - "avgShardSize":41.83655313309282, - "maxShardSize":41.83655313309282, - "minShardSize":41.83655313309282}, - "COLL_5":{ - "activeShards":1, - "inactiveShards":0, - "rf":1, - "maxActualShardsPerNode":1, - "minActualShardsPerNode":1, - "maxShardReplicasPerNode":1, - "minShardReplicasPerNode":1, - "numCores":1, - "numNodes":1, - "maxCoresPerNode":1, - "minCoresPerNode":1, - "avgShardSize":5.452332053333521, - "maxShardSize":5.452332053333521, - "minShardSize":5.452332053333521}, - "COLL_l":{ - "activeShards":1, - "inactiveShards":0, - "rf":3, - "maxActualShardsPerNode":1, - "minActualShardsPerNode":1, - "maxShardReplicasPerNode":1, - "minShardReplicasPerNode":1, - "numCores":3, - "numNodes":3, - "maxCoresPerNode":1, - "minCoresPerNode":1, - "avgShardSize":1.257285475730896E-7, - "maxShardSize":1.257285475730896E-7, - "minShardSize":1.257285475730896E-7}, - "COLL_x":{ - "activeShards":1, - "inactiveShards":0, - "rf":3, - "maxActualShardsPerNode":1, - "minActualShardsPerNode":1, - "maxShardReplicasPerNode":1, - "minShardReplicasPerNode":1, - "numCores":3, - "numNodes":3, - "maxCoresPerNode":1, - "minCoresPerNode":1, - "avgShardSize":0.2880448754876852, - "maxShardSize":0.2880448754876852, - "minShardSize":0.2880448754876852}, - "COLL_1b":{ - "activeShards":1, - "inactiveShards":0, - "rf":3, - "maxActualShardsPerNode":1, - "minActualShardsPerNode":1, - "maxShardReplicasPerNode":1, - "minShardReplicasPerNode":1, - "numCores":3, - "numNodes":3, - "maxCoresPerNode":1, - "minCoresPerNode":1, - "avgShardSize":1.257285475730896E-7, - "maxShardSize":1.257285475730896E-7, - "minShardSize":1.257285475730896E-7}, - "COLL_1r":{ - "activeShards":1, - "inactiveShards":0, - "rf":3, - "maxActualShardsPerNode":1, - "minActualShardsPerNode":1, - "maxShardReplicasPerNode":1, - "minShardReplicasPerNode":1, - "numCores":3, - "numNodes":3, - "maxCoresPerNode":1, - "minCoresPerNode":1, - "avgShardSize":0.39663587138056755, - "maxShardSize":0.39663587138056755, - "minShardSize":0.39663587138056755}, - "COLL_8":{ - "activeShards":1, - "inactiveShards":0, - "rf":3, - "maxActualShardsPerNode":1, - "minActualShardsPerNode":1, - "maxShardReplicasPerNode":1, - "minShardReplicasPerNode":1, - "numCores":3, - "numNodes":3, - "maxCoresPerNode":1, - "minCoresPerNode":1, - "avgShardSize":3.718072548508644E-4, - "maxShardSize":3.718072548508644E-4, - "minShardSize":3.718072548508644E-4}, - "COLL_q":{ - "activeShards":1, - "inactiveShards":0, - "rf":3, - "maxActualShardsPerNode":1, - "minActualShardsPerNode":1, - "maxShardReplicasPerNode":1, - "minShardReplicasPerNode":1, - "numCores":3, - "numNodes":3, - "maxCoresPerNode":1, - "minCoresPerNode":1, - "avgShardSize":0.45860921032726765, - "maxShardSize":0.45860921032726765, - "minShardSize":0.45860921032726765}, - "COLL_4":{ - "activeShards":1, - "inactiveShards":0, - "rf":3, - "maxActualShardsPerNode":1, - "minActualShardsPerNode":1, - "maxShardReplicasPerNode":1, - "minShardReplicasPerNode":1, - "numCores":3, - "numNodes":3, - "maxCoresPerNode":1, - "minCoresPerNode":1, - "avgShardSize":0.24102374073117971, - "maxShardSize":0.24102374073117971, - "minShardSize":0.24102374073117971}, - "COLL_1x":{ - "activeShards":1, - "inactiveShards":0, - "rf":3, - "maxActualShardsPerNode":1, - "minActualShardsPerNode":1, - "maxShardReplicasPerNode":1, - "minShardReplicasPerNode":1, - "numCores":3, - "numNodes":3, - "maxCoresPerNode":1, - "minCoresPerNode":1, - "avgShardSize":0.003971998579800129, - "maxShardSize":0.003971998579800129, - "minShardSize":0.003971998579800129}, - "COLL_1t":{ - "activeShards":1, - "inactiveShards":0, - "rf":3, - "maxActualShardsPerNode":1, - "minActualShardsPerNode":1, - "maxShardReplicasPerNode":1, - "minShardReplicasPerNode":1, - "numCores":3, - "numNodes":3, - "maxCoresPerNode":1, - "minCoresPerNode":1, - "avgShardSize":81.47750116791576, - "maxShardSize":81.47750116791576, - "minShardSize":81.47750116791576}, - "COLL_2k":{ - "activeShards":1, - "inactiveShards":0, - "rf":3, - "maxActualShardsPerNode":1, - "minActualShardsPerNode":1, - "maxShardReplicasPerNode":1, - "minShardReplicasPerNode":1, - "numCores":3, - "numNodes":3, - "maxCoresPerNode":1, - "minCoresPerNode":1, - "avgShardSize":1.257285475730896E-7, - "maxShardSize":1.257285475730896E-7, - "minShardSize":1.257285475730896E-7}, - "COLL_22":{ - "activeShards":1, - "inactiveShards":0, - "rf":3, - "maxActualShardsPerNode":1, - "minActualShardsPerNode":1, - "maxShardReplicasPerNode":1, - "minShardReplicasPerNode":1, - "numCores":3, - "numNodes":3, - "maxCoresPerNode":1, - "minCoresPerNode":1, - "avgShardSize":22.679232054390013, - "maxShardSize":22.679232054390013, - "minShardSize":22.679232054390013}}} diff --git a/solr/core/src/test/org/apache/solr/cloud/CloudTestUtils.java b/solr/core/src/test/org/apache/solr/cloud/CloudTestUtils.java deleted file mode 100644 index 07e279b7992..00000000000 --- a/solr/core/src/test/org/apache/solr/cloud/CloudTestUtils.java +++ /dev/null @@ -1,195 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.cloud; - -import java.io.IOException; -import java.util.Map; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; - -import org.apache.lucene.util.LuceneTestCase; -import org.apache.solr.client.solrj.SolrClient; -import org.apache.solr.client.solrj.SolrRequest; -import org.apache.solr.client.solrj.SolrResponse; -import org.apache.solr.client.solrj.cloud.SolrCloudManager; -import org.apache.solr.client.solrj.request.RequestWriter; -import org.apache.solr.client.solrj.request.RequestWriter.StringPayloadContentWriter; -import org.apache.solr.client.solrj.request.V2Request; -import org.apache.solr.client.solrj.response.SolrResponseBase; -import org.apache.solr.common.params.SolrParams; -import org.apache.solr.util.TimeOut; -import org.junit.Assert; - -import static org.apache.solr.common.params.CommonParams.JSON_MIME; - - -/** - * Some useful methods for SolrCloud tests. - */ -public class CloudTestUtils { - - public static final int DEFAULT_TIMEOUT = 90; - - /** - * Wait for a particular named trigger to be scheduled. - *

- * This is a convenience method that polls the autoscaling API looking for a trigger with the - * specified name using the {@link #DEFAULT_TIMEOUT}. It is particularly useful for tests - * that want to know when the Overseer has finished scheduling the automatic triggers on startup. - *

- * - * @param cloudManager current instance of {@link SolrCloudManager} - * @param triggerName the name of the trigger we need to see sheduled in order to return successfully - * @see #suspendTrigger - */ - public static long waitForTriggerToBeScheduled(final SolrCloudManager cloudManager, - final String triggerName) - throws InterruptedException, TimeoutException, IOException { - - TimeOut timeout = new TimeOut(DEFAULT_TIMEOUT, TimeUnit.SECONDS, cloudManager.getTimeSource()); - while (!timeout.hasTimedOut()) { - final SolrResponse response = cloudManager.request(AutoScalingRequest.create(SolrRequest.METHOD.GET, null)); - @SuppressWarnings({"unchecked"}) - final Map triggers = (Map) response.getResponse().get("triggers"); - Assert.assertNotNull("null triggers in response from autoscaling request", triggers); - - if ( triggers.containsKey(triggerName) ) { - return timeout.timeElapsed(TimeUnit.MILLISECONDS); - } - timeout.sleep(100); - } - throw new TimeoutException("Never saw trigger with name: " + triggerName); - } - - /** - * Suspends the trigger with the specified name - *

- * This is a convenience method that sends a suspend-trigger command to the autoscaling - * API for the specified trigger. It is particularly useful for tests that may need to disable automatic - * triggers such as .scheduled_maintenance in order to test their own - * triggers. - *

- * - * @param cloudManager current instance of {@link SolrCloudManager} - * @param triggerName the name of the trigger to suspend. This must already be scheduled. - * @see #assertAutoScalingRequest - * @see #waitForTriggerToBeScheduled - */ - public static void suspendTrigger(final SolrCloudManager cloudManager, - final String triggerName) throws IOException { - assertAutoScalingRequest(cloudManager, "{'suspend-trigger' : {'name' : '"+triggerName+"'} }"); - } - - /** - * Creates & executes an autoscaling request against the current cluster, asserting that - * the result is a success. - * - * @param cloudManager current instance of {@link SolrCloudManager} - * @param json The request to POST to the AutoScaling Handler - * @see AutoScalingRequest#create - */ - public static void assertAutoScalingRequest(final SolrCloudManager cloudManager, - final String json) throws IOException { - // TODO: a lot of code that directly uses AutoScalingRequest.create should use this method - - @SuppressWarnings({"rawtypes"}) - final SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, json); - final SolrResponse rsp = cloudManager.request(req); - final String result = rsp.getResponse().get("result").toString(); - Assert.assertEquals("Unexpected result from auto-scaling command: " + json + " -> " + rsp, - "success", result); - } - - - /** - * Helper class for sending (JSON) autoscaling requests that can randomize between V1 and V2 requests - */ - @SuppressWarnings({"rawtypes"}) - public static class AutoScalingRequest extends SolrRequest { - private SolrParams params = null; - /** - * Creates a request using a randomized root path (V1 vs V2) - * - * @param m HTTP Method to use - * @aram message JSON payload, may be null - */ - @SuppressWarnings({"rawtypes"}) - public static SolrRequest create(SolrRequest.METHOD m, String message) { - return create(m, null, message); - } - /** - * Creates a request using a randomized root path (V1 vs V2) - * - * @param m HTTP Method to use - * @param subPath optional sub-path under "$ROOT/autoscaling". may be null, - * otherwise must start with "/" - * @param message JSON payload, may be null - */ - @SuppressWarnings({"rawtypes"}) - public static SolrRequest create(SolrRequest.METHOD m, String subPath, String message) { - return create(m,subPath,message,null); - - } - @SuppressWarnings({"rawtypes"}) - public static SolrRequest create(SolrRequest.METHOD m, String subPath, String message, SolrParams params) { - final boolean useV1 = LuceneTestCase.random().nextBoolean(); - String path = useV1 ? "/admin/autoscaling" : "/cluster/autoscaling"; - if (null != subPath) { - assert subPath.startsWith("/"); - path += subPath; - } - return useV1 - ? new AutoScalingRequest(m, path, message).withParams(params) - : new V2Request.Builder(path).withMethod(m).withParams(params).withPayload(message).build(); - - } - - protected final String message; - - /** - * Simple request - * @param m HTTP Method to use - * @param path path to send request to - * @param message JSON payload, may be null - */ - private AutoScalingRequest(METHOD m, String path, String message) { - super(m, path); - this.message = message; - } - - - AutoScalingRequest withParams(SolrParams params){ - this.params = params; - return this; - } - @Override - public SolrParams getParams() { - return params; - } - - @Override - public RequestWriter.ContentWriter getContentWriter(String expectedType) { - return message == null ? null : new StringPayloadContentWriter(message, JSON_MIME); - } - - @Override - protected SolrResponse createResponse(SolrClient client) { - return new SolrResponseBase(); - } - } -} diff --git a/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java b/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java index 35daa446941..a6d3d811f25 100644 --- a/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java @@ -20,7 +20,6 @@ import static java.util.Arrays.asList; import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_DEF; import static org.apache.solr.common.cloud.ZkStateReader.NRT_REPLICAS; import static org.apache.solr.common.cloud.ZkStateReader.NUM_SHARDS_PROP; -import static org.apache.solr.common.cloud.ZkStateReader.SOLR_AUTOSCALING_CONF_PATH; import static org.apache.solr.common.params.CollectionAdminParams.COLLECTION; import static org.apache.solr.common.params.CollectionAdminParams.DEFAULTS; @@ -63,7 +62,6 @@ import org.apache.solr.common.cloud.ClusterState; import org.apache.solr.common.cloud.DocCollection; import org.apache.solr.common.cloud.Replica; import org.apache.solr.common.cloud.Slice; -import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.cloud.ZkStateReader; import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.params.CoreAdminParams; @@ -90,8 +88,6 @@ public class CollectionsAPISolrJTest extends SolrCloudTestCase { .addConfig("conf2", configset("cloud-dynamic")) .configure(); - // clear any persisted auto scaling configuration - zkClient().setData(SOLR_AUTOSCALING_CONF_PATH, Utils.toJSON(new ZkNodeProps()), true); } @After @@ -550,22 +546,20 @@ public class CollectionsAPISolrJTest extends SolrCloudTestCase { // sanity check our expected default final ClusterProperties props = new ClusterProperties(zkClient()); - assertEquals("Expecting prop to default to unset, test needs upated", - props.getClusterProperty(ZkStateReader.AUTO_ADD_REPLICAS, null), null); - CollectionAdminResponse response = CollectionAdminRequest.setClusterProperty(ZkStateReader.AUTO_ADD_REPLICAS, "true") + CollectionAdminResponse response = CollectionAdminRequest.setClusterProperty(ZkStateReader.MAX_CORES_PER_NODE, "42") .process(cluster.getSolrClient()); assertEquals(0, response.getStatus()); - assertEquals("Cluster property was not set", props.getClusterProperty(ZkStateReader.AUTO_ADD_REPLICAS, null), "true"); + assertEquals("Cluster property was not set", props.getClusterProperty(ZkStateReader.MAX_CORES_PER_NODE, null), "42"); // Unset ClusterProp that we set. - CollectionAdminRequest.setClusterProperty(ZkStateReader.AUTO_ADD_REPLICAS, null).process(cluster.getSolrClient()); - assertEquals("Cluster property was not unset", props.getClusterProperty(ZkStateReader.AUTO_ADD_REPLICAS, null), null); + CollectionAdminRequest.setClusterProperty(ZkStateReader.MAX_CORES_PER_NODE, null).process(cluster.getSolrClient()); + assertEquals("Cluster property was not unset", props.getClusterProperty(ZkStateReader.MAX_CORES_PER_NODE, null), null); - response = CollectionAdminRequest.setClusterProperty(ZkStateReader.AUTO_ADD_REPLICAS, "false") + response = CollectionAdminRequest.setClusterProperty(ZkStateReader.MAX_CORES_PER_NODE, "1") .process(cluster.getSolrClient()); assertEquals(0, response.getStatus()); - assertEquals("Cluster property was not set", props.getClusterProperty(ZkStateReader.AUTO_ADD_REPLICAS, null), "false"); + assertEquals("Cluster property was not set", props.getClusterProperty(ZkStateReader.MAX_CORES_PER_NODE, null), "1"); } @Test diff --git a/solr/core/src/test/org/apache/solr/cloud/MetricsHistoryIntegrationTest.java b/solr/core/src/test/org/apache/solr/cloud/MetricsHistoryIntegrationTest.java index a720677a5e4..897f060575e 100644 --- a/solr/core/src/test/org/apache/solr/cloud/MetricsHistoryIntegrationTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/MetricsHistoryIntegrationTest.java @@ -30,7 +30,6 @@ import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.client.solrj.cloud.SolrCloudManager; import org.apache.solr.client.solrj.request.CollectionAdminRequest; import org.apache.solr.client.solrj.request.GenericSolrRequest; -import org.apache.solr.cloud.autoscaling.sim.SimCloudManager; import org.apache.solr.common.params.CollectionAdminParams; import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.params.SolrParams; @@ -49,6 +48,7 @@ import org.slf4j.LoggerFactory; * */ @LuceneTestCase.Slow +@LuceneTestCase.Nightly @LogLevel("org.apache.solr.handler.admin=DEBUG") public class MetricsHistoryIntegrationTest extends SolrCloudTestCase { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); @@ -59,18 +59,11 @@ public class MetricsHistoryIntegrationTest extends SolrCloudTestCase { @BeforeClass public static void setupCluster() throws Exception { - boolean simulated = TEST_NIGHTLY ? random().nextBoolean() : true; - if (simulated) { - cloudManager = SimCloudManager.createCluster(1, TimeSource.get("simTime:50")); - solrClient = ((SimCloudManager)cloudManager).simGetSolrClient(); - } configureCluster(1) .addConfig("conf", configset("cloud-minimal")) .configure(); - if (!simulated) { - cloudManager = cluster.getJettySolrRunner(0).getCoreContainer().getZkController().getSolrCloudManager(); - solrClient = cluster.getSolrClient(); - } + cloudManager = cluster.getJettySolrRunner(0).getCoreContainer().getZkController().getSolrCloudManager(); + solrClient = cluster.getSolrClient(); timeSource = cloudManager.getTimeSource(); // create .system CollectionAdminRequest.createCollection(CollectionAdminParams.SYSTEM_COLL, null, 1, 1) @@ -78,19 +71,12 @@ public class MetricsHistoryIntegrationTest extends SolrCloudTestCase { CloudUtil.waitForState(cloudManager, CollectionAdminParams.SYSTEM_COLL, 30, TimeUnit.SECONDS, CloudUtil.clusterShape(1, 1)); solrClient.query(CollectionAdminParams.SYSTEM_COLL, params(CommonParams.Q, "*:*")); - // sleep a little to allow the handler to collect some metrics - if (simulated) { - timeSource.sleep(90000); - } else { - timeSource.sleep(100000); - } + // sleep until next generation of kids grow up to allow the handler to collect some metrics + timeSource.sleep(100000); } @AfterClass public static void teardown() throws Exception { - if (cloudManager instanceof SimCloudManager) { - cloudManager.close(); - } solrClient = null; cloudManager = null; } diff --git a/solr/core/src/test/org/apache/solr/cloud/MoveReplicaTest.java b/solr/core/src/test/org/apache/solr/cloud/MoveReplicaTest.java index f4191d8aa5d..53c47dfd546 100644 --- a/solr/core/src/test/org/apache/solr/cloud/MoveReplicaTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/MoveReplicaTest.java @@ -105,7 +105,6 @@ public class MoveReplicaTest extends SolrCloudTestCase { // random create tlog or pull type replicas with nrt boolean isTlog = random().nextBoolean(); CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(coll, "conf1", 2, 1, isTlog ? 1 : 0, !isTlog ? 1 : 0); - create.setAutoAddReplicas(false); cloudClient.request(create); addDocs(coll, 100); @@ -249,7 +248,6 @@ public class MoveReplicaTest extends SolrCloudTestCase { // random create tlog or pull type replicas with nrt boolean isTlog = random().nextBoolean(); CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(coll, "conf1", 2, 1, isTlog ? 1 : 0, !isTlog ? 1 : 0); - create.setAutoAddReplicas(false); cloudClient.request(create); addDocs(coll, 100); diff --git a/solr/core/src/test/org/apache/solr/cloud/OverseerCollectionConfigSetProcessorTest.java b/solr/core/src/test/org/apache/solr/cloud/OverseerCollectionConfigSetProcessorTest.java index 3d4f69137ea..c811bf81020 100644 --- a/solr/core/src/test/org/apache/solr/cloud/OverseerCollectionConfigSetProcessorTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/OverseerCollectionConfigSetProcessorTest.java @@ -37,8 +37,7 @@ import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.client.solrj.SolrResponse; import org.apache.solr.client.solrj.cloud.DistribStateManager; import org.apache.solr.client.solrj.cloud.SolrCloudManager; -import org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig; -import org.apache.solr.client.solrj.cloud.autoscaling.VersionedData; +import org.apache.solr.client.solrj.cloud.VersionedData; import org.apache.solr.client.solrj.impl.ClusterStateProvider; import org.apache.solr.cloud.Overseer.LeaderStatus; import org.apache.solr.cloud.OverseerTaskQueue.QueueEvent; @@ -81,9 +80,6 @@ import org.mockito.stubbing.Answer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static org.apache.solr.common.params.CollectionAdminParams.CLUSTER; -import static org.apache.solr.common.params.CollectionAdminParams.DEFAULTS; -import static org.apache.solr.common.params.CollectionAdminParams.USE_LEGACY_REPLICA_ASSIGNMENT; import static org.mockito.Mockito.any; import static org.mockito.Mockito.anyBoolean; import static org.mockito.Mockito.anyInt; @@ -127,7 +123,6 @@ public class OverseerCollectionConfigSetProcessorTest extends SolrTestCaseJ4 { private static HttpClient httpClientMock; private static ObjectCache objectCache; - private static AutoScalingConfig autoScalingConfig = new AutoScalingConfig(Collections.emptyMap()); private Map zkClientData = new HashMap<>(); private final Map collectionsSet = new HashMap<>(); private final List replicas = new ArrayList<>(); @@ -294,7 +289,6 @@ public class OverseerCollectionConfigSetProcessorTest extends SolrTestCaseJ4 { when(zkStateReaderMock.getZkClient()).thenReturn(solrZkClientMock); when(zkStateReaderMock.getClusterState()).thenReturn(clusterStateMock); - when(zkStateReaderMock.getAutoScalingConfig()).thenReturn(autoScalingConfig); when(zkStateReaderMock.getAliases()).thenReturn(Aliases.EMPTY); when(clusterStateMock.getCollection(anyString())).thenAnswer(invocation -> { @@ -373,10 +367,8 @@ public class OverseerCollectionConfigSetProcessorTest extends SolrTestCaseJ4 { when(cloudDataProviderMock.getClusterStateProvider()).thenReturn(clusterStateProviderMock); when(clusterStateProviderMock.getClusterState()).thenReturn(clusterStateMock); when(clusterStateProviderMock.getLiveNodes()).thenReturn(liveNodes); - when(clusterStateProviderMock.getClusterProperties()).thenReturn(Utils.makeMap(DEFAULTS, Utils.makeMap(CLUSTER, Utils.makeMap(USE_LEGACY_REPLICA_ASSIGNMENT, true)))); when(cloudDataProviderMock.getDistribStateManager()).thenReturn(stateManagerMock); when(cloudManagerMock.getDistribStateManager()).thenReturn(distribStateManagerMock); - when(distribStateManagerMock.getAutoScalingConfig()).thenReturn(new AutoScalingConfig(Collections.emptyMap())); Mockito.doAnswer( new Answer() { @@ -456,7 +448,6 @@ public class OverseerCollectionConfigSetProcessorTest extends SolrTestCaseJ4 { when(zkControllerMock.getZkClient()).thenReturn(solrZkClientMock); when(cloudManagerMock.getDistribStateManager()).thenReturn(distribStateManagerMock); - when(distribStateManagerMock.getAutoScalingConfig()).thenReturn(new AutoScalingConfig(Collections.emptyMap())); Mockito.doAnswer( new Answer() { diff --git a/solr/core/src/test/org/apache/solr/cloud/ReplaceNodeNoTargetTest.java b/solr/core/src/test/org/apache/solr/cloud/ReplaceNodeNoTargetTest.java deleted file mode 100644 index cd5d4be5c6a..00000000000 --- a/solr/core/src/test/org/apache/solr/cloud/ReplaceNodeNoTargetTest.java +++ /dev/null @@ -1,127 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.cloud; - - -import java.lang.invoke.MethodHandles; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Set; - -import org.apache.lucene.util.LuceneTestCase; -import org.apache.solr.client.solrj.SolrRequest; -import org.apache.solr.client.solrj.impl.CloudSolrClient; -import org.apache.solr.client.solrj.impl.HttpSolrClient; -import org.apache.solr.client.solrj.request.CollectionAdminRequest; -import org.apache.solr.client.solrj.request.CoreAdminRequest; -import org.apache.solr.client.solrj.response.CoreAdminResponse; -import org.apache.solr.client.solrj.response.RequestStatusState; -import org.apache.solr.cloud.CloudTestUtils.AutoScalingRequest; - -import org.junit.BeforeClass; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import static org.apache.solr.cloud.ReplaceNodeTest.createReplaceNodeRequest; - -@LuceneTestCase.AwaitsFix(bugUrl = "https://issues.apache.org/jira/browse/SOLR-11067") -public class ReplaceNodeNoTargetTest extends SolrCloudTestCase { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - @BeforeClass - public static void setupCluster() throws Exception { - configureCluster(6) - .addConfig("conf1", TEST_PATH().resolve("configsets").resolve("cloud-dynamic").resolve("conf")) - .configure(); - } - - protected String getSolrXml() { - return "solr.xml"; - } - - - @Test - @LuceneTestCase.AwaitsFix(bugUrl = "https://issues.apache.org/jira/browse/SOLR-11067") - public void test() throws Exception { - String coll = "replacenodetest_coll_notarget"; - if (log.isInfoEnabled()) { - log.info("total_jettys: {}", cluster.getJettySolrRunners().size()); - } - - CloudSolrClient cloudClient = cluster.getSolrClient(); - Set liveNodes = cloudClient.getZkStateReader().getClusterState().getLiveNodes(); - ArrayList l = new ArrayList<>(liveNodes); - Collections.shuffle(l, random()); - String node2bdecommissioned = l.get(0); - CloudSolrClient solrClient = cluster.getSolrClient(); - String setClusterPolicyCommand = "{" + - " 'set-cluster-policy': [" + - " {'replica':'<5', 'shard': '#EACH', 'node': '#ANY'}]}"; - @SuppressWarnings({"rawtypes"}) - SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setClusterPolicyCommand); - solrClient.request(req); - - log.info("Creating collection..."); - CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(coll, "conf1", 5, 2, 0, 0); - cloudClient.request(create); - cluster.waitForActiveCollection(coll, 5, 10); - - if (log.isInfoEnabled()) { - log.info("Current core status list for node we plan to decommision: {} => {}", - node2bdecommissioned, - getCoreStatusForNamedNode(cloudClient, node2bdecommissioned).getCoreStatus()); - log.info("Decommisioning node: {}", node2bdecommissioned); - } - - createReplaceNodeRequest(node2bdecommissioned, null, null).processAsync("001", cloudClient); - CollectionAdminRequest.RequestStatus requestStatus = CollectionAdminRequest.requestStatus("001"); - boolean success = false; - CollectionAdminRequest.RequestStatusResponse rsp = null; - for (int i = 0; i < 300; i++) { - rsp = requestStatus.process(cloudClient); - if (rsp.getRequestStatus() == RequestStatusState.COMPLETED) { - success = true; - break; - } - assertFalse("async replace node request aparently failed: " + rsp.toString(), - - rsp.getRequestStatus() == RequestStatusState.FAILED); - Thread.sleep(50); - } - assertTrue("async replace node request should have finished successfully by now, last status: " + rsp, - success); - CoreAdminResponse status = getCoreStatusForNamedNode(cloudClient, node2bdecommissioned); - assertEquals("Expected no cores for decommisioned node: " - + status.getCoreStatus().toString(), - 0, status.getCoreStatus().size()); - } - - /** - * Given a cloud client and a nodename, build an HTTP client for that node, and ask it for it's core status - */ - private CoreAdminResponse getCoreStatusForNamedNode(final CloudSolrClient cloudClient, - final String nodeName) throws Exception { - - try (HttpSolrClient coreclient = getHttpSolrClient - (cloudClient.getZkStateReader().getBaseUrlForNodeName(nodeName))) { - return CoreAdminRequest.getStatus(null, coreclient); - } - } - -} diff --git a/solr/core/src/test/org/apache/solr/cloud/RoutingToNodesWithPropertiesTest.java b/solr/core/src/test/org/apache/solr/cloud/RoutingToNodesWithPropertiesTest.java deleted file mode 100644 index 4f9cbfc09ba..00000000000 --- a/solr/core/src/test/org/apache/solr/cloud/RoutingToNodesWithPropertiesTest.java +++ /dev/null @@ -1,242 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.cloud; - -import java.lang.invoke.MethodHandles; -import java.util.ArrayList; -import java.util.Collections; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; - -import com.google.common.collect.ImmutableMap; -import org.apache.solr.client.solrj.SolrQuery; -import org.apache.solr.client.solrj.SolrRequest; -import org.apache.solr.client.solrj.cloud.NodeStateProvider; -import org.apache.solr.client.solrj.cloud.SolrCloudManager; -import org.apache.solr.client.solrj.embedded.JettySolrRunner; -import org.apache.solr.client.solrj.impl.SolrClientCloudManager; -import org.apache.solr.client.solrj.request.CollectionAdminRequest; -import org.apache.solr.client.solrj.request.UpdateRequest; -import org.apache.solr.client.solrj.response.QueryResponse; -import org.apache.solr.common.cloud.Replica; -import org.apache.solr.common.cloud.Slice; -import org.apache.solr.common.cloud.ZkStateReader; -import org.apache.solr.common.params.ModifiableSolrParams; -import org.apache.solr.common.params.ShardParams; -import org.apache.solr.common.util.CommonTestInjection; -import org.apache.solr.common.util.SimpleOrderedMap; -import org.apache.solr.common.util.TimeSource; -import org.apache.solr.handler.component.TrackingShardHandlerFactory; -import org.apache.solr.util.TestInjection; -import org.apache.solr.util.TimeOut; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import static org.apache.solr.common.cloud.rule.ImplicitSnitch.SYSPROP; - -public class RoutingToNodesWithPropertiesTest extends SolrCloudTestCase { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - private static final String PROP_NAME = SYSPROP + "zone"; - final static String COLLECTION = "coll"; - - private final List zone1Nodes = new ArrayList<>(); - private final List zone2Nodes = new ArrayList<>(); - private final LinkedList zone1Queue = new LinkedList<>(); - private final LinkedList zone2Queue = new LinkedList<>(); - - @Before - public void setupCluster() throws Exception { - CommonTestInjection.setAdditionalProps(ImmutableMap.of("zone", "us-west1")); - configureCluster(2) - .withSolrXml(TEST_PATH().resolve("solr-trackingshardhandler.xml")) - .addConfig("config", TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf")) - .configure(); - - zone1Nodes.addAll(cluster.getJettySolrRunners().stream().map(JettySolrRunner::getNodeName).collect(Collectors.toSet())); - CommonTestInjection.setAdditionalProps(ImmutableMap.of("zone", "us-west2")); - zone2Nodes.add(cluster.startJettySolrRunner().getNodeName()); - zone2Nodes.add(cluster.startJettySolrRunner().getNodeName()); - - String commands = "{set-cluster-policy :[{" + - " 'replica':'#EQUAL'," + - " 'shard':'#EACH'," + - " 'sysprop.zone':'#EACH'}]}"; - - @SuppressWarnings({"rawtypes"}) - SolrRequest req = CloudTestUtils.AutoScalingRequest.create(SolrRequest.METHOD.POST, commands); - cluster.getSolrClient().request(req); - - CollectionAdminRequest.createCollection(COLLECTION, 2, 2) - .process(cluster.getSolrClient()); - cluster.waitForActiveCollection(COLLECTION, 2, 4); - - // Checking putting replicas - for (Slice slice : getCollectionState(COLLECTION).getSlices()) { - int numReplicaInZone1 = 0; - int numReplicaInZone2 = 0; - for (Replica replica : slice.getReplicas()) { - if (zone1Nodes.contains(replica.getNodeName())) - numReplicaInZone1++; - if (zone2Nodes.contains(replica.getNodeName())) - numReplicaInZone2++; - } - - assertEquals(1, numReplicaInZone1); - assertEquals(1, numReplicaInZone2); - } - - // check inject props - try (SolrCloudManager cloudManager = new SolrClientCloudManager(new ZkDistributedQueueFactory(cluster.getZkClient()), - cluster.getSolrClient())) { - for (String zone1Node: zone1Nodes) { - NodeStateProvider nodeStateProvider = cloudManager.getNodeStateProvider(); - Map map = nodeStateProvider.getNodeValues(zone1Node, Collections.singletonList(PROP_NAME)); - assertEquals("us-west1", map.get(PROP_NAME)); - } - - for (String zone2Node: zone2Nodes) { - NodeStateProvider nodeStateProvider = cloudManager.getNodeStateProvider(); - Map map = nodeStateProvider.getNodeValues(zone2Node, Collections.singletonList(PROP_NAME)); - assertEquals("us-west2", map.get(PROP_NAME)); - } - - for (JettySolrRunner jetty : cluster.getJettySolrRunners()) { - if (zone1Nodes.contains(jetty.getNodeName())) { - ((TrackingShardHandlerFactory)jetty.getCoreContainer().getShardHandlerFactory()).setTrackingQueue(zone1Queue); - } else { - ((TrackingShardHandlerFactory)jetty.getCoreContainer().getShardHandlerFactory()).setTrackingQueue(zone2Queue); - } - } - - for (int i = 0; i < 20; i++) { - new UpdateRequest() - .add("id", String.valueOf(i)) - .process(cluster.getSolrClient(), COLLECTION); - } - - new UpdateRequest() - .commit(cluster.getSolrClient(), COLLECTION); - } - } - - @After - public void after() { - TestInjection.reset(); - } - - @Test - public void test() throws Exception { - final int NUM_TRY = 10; - CollectionAdminRequest - .setClusterProperty(ZkStateReader.DEFAULT_SHARD_PREFERENCES, ShardParams.SHARDS_PREFERENCE_NODE_WITH_SAME_SYSPROP +":"+PROP_NAME) - .process(cluster.getSolrClient()); - { - TimeOut timeOut = new TimeOut(20, TimeUnit.SECONDS, TimeSource.NANO_TIME); - timeOut.waitFor("Timeout waiting for sysprops are cached in all nodes", () -> { - int total = 0; - for (JettySolrRunner runner : cluster.getJettySolrRunners()) { - total += runner.getCoreContainer().getZkController().getSysPropsCacher().getCacheSize(); - } - return total == cluster.getJettySolrRunners().size() * cluster.getJettySolrRunners().size(); - }); - } - - for (int i = 0; i < NUM_TRY; i++) { - SolrQuery qRequest = new SolrQuery("*:*"); - ModifiableSolrParams qParams = new ModifiableSolrParams(); - qParams.add(ShardParams.SHARDS_INFO, "true"); - qRequest.add(qParams); - QueryResponse qResponse = cluster.getSolrClient().query(COLLECTION, qRequest); - - Object shardsInfo = qResponse.getResponse().get(ShardParams.SHARDS_INFO); - assertNotNull("Unable to obtain "+ShardParams.SHARDS_INFO, shardsInfo); - SimpleOrderedMap shardsInfoMap = (SimpleOrderedMap)shardsInfo; - String firstReplicaAddr = ((SimpleOrderedMap) shardsInfoMap.getVal(0)).get("shardAddress").toString(); - String secondReplicaAddr = ((SimpleOrderedMap) shardsInfoMap.getVal(1)).get("shardAddress").toString(); - boolean firstReplicaInZone1 = false; - boolean secondReplicaInZone1 = false; - for (String zone1Node : zone1Nodes) { - zone1Node = zone1Node.replace("_solr", ""); - firstReplicaInZone1 = firstReplicaInZone1 || firstReplicaAddr.contains(zone1Node); - secondReplicaInZone1 = secondReplicaInZone1 || secondReplicaAddr.contains(zone1Node); - } - - assertEquals(firstReplicaInZone1, secondReplicaInZone1); - } - - // intense asserting using TrackingShardHandlerFactory - assertRoutingToSameZone(); - - // Cachers should be stop running - CollectionAdminRequest - .setClusterProperty(ZkStateReader.DEFAULT_SHARD_PREFERENCES, ShardParams.SHARDS_PREFERENCE_REPLICA_TYPE+":PULL") - .process(cluster.getSolrClient()); - { - TimeOut timeOut = new TimeOut(20, TimeUnit.SECONDS, TimeSource.NANO_TIME); - timeOut.waitFor("Timeout waiting for sysPropsCache stop", () -> { - int numNodeStillRunningCache = 0; - for (JettySolrRunner runner: cluster.getJettySolrRunners()) { - if (runner.getCoreContainer().getZkController().getSysPropsCacher().isRunning()) { - numNodeStillRunningCache++; - } - } - return numNodeStillRunningCache == 0; - }); - } - - // Testing disable default shard preferences - CollectionAdminRequest - .setClusterProperty(ZkStateReader.DEFAULT_SHARD_PREFERENCES, null) - .process(cluster.getSolrClient()); - { - TimeOut timeOut = new TimeOut(20, TimeUnit.SECONDS, TimeSource.NANO_TIME); - timeOut.waitFor("Timeout waiting cluster properties get updated", () -> { - int numNodeGetUpdatedPref = 0; - int numNodeStillRunningCache = 0; - for (JettySolrRunner runner: cluster.getJettySolrRunners()) { - if (runner.getCoreContainer().getZkController() - .getZkStateReader().getClusterProperties().containsKey(ZkStateReader.DEFAULT_SHARD_PREFERENCES)) { - numNodeGetUpdatedPref++; - } - if (runner.getCoreContainer().getZkController().getSysPropsCacher().isRunning()) { - numNodeStillRunningCache++; - } - } - return numNodeGetUpdatedPref == 0 && numNodeStillRunningCache == 0; - }); - } - - } - - private void assertRoutingToSameZone() { - for (TrackingShardHandlerFactory.ShardRequestAndParams sreq: zone1Queue) { - String firstNode = sreq.shard.split("\\|")[0]; - assertTrue(zone1Nodes.stream().anyMatch(s -> firstNode.contains(s.replace('_','/')))); - } - for (TrackingShardHandlerFactory.ShardRequestAndParams sreq: zone2Queue) { - String firstNode = sreq.shard.split("\\|")[0]; - assertTrue(zone2Nodes.stream().anyMatch(s -> firstNode.contains(s.replace('_','/')))); - } - } -} diff --git a/solr/core/src/test/org/apache/solr/cloud/SharedFSAutoReplicaFailoverTest.java b/solr/core/src/test/org/apache/solr/cloud/SharedFSAutoReplicaFailoverTest.java index d9f8a9829bb..f0a107a8337 100644 --- a/solr/core/src/test/org/apache/solr/cloud/SharedFSAutoReplicaFailoverTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/SharedFSAutoReplicaFailoverTest.java @@ -82,7 +82,7 @@ import org.slf4j.LoggerFactory; QuickPatchThreadsFilter.class, BadHdfsThreadsFilter.class // hdfs currently leaks thread(s) }) -@LogLevel("org.apache.solr.cloud.autoscaling=DEBUG;org.apache.solr.cloud.*=DEBUG") +@LogLevel("org.apache.solr.cloud.*=DEBUG") @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Jul-2018 public class SharedFSAutoReplicaFailoverTest extends AbstractFullDistribZkTestBase { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); @@ -161,8 +161,7 @@ public class SharedFSAutoReplicaFailoverTest extends AbstractFullDistribZkTestBa private void testBasics() throws Exception { String collection1 = "solrj_collection"; Create createCollectionRequest = CollectionAdminRequest.createCollection(collection1,"conf1",2,2) - .setRouterField("myOwnField") - .setAutoAddReplicas(true); + .setRouterField("myOwnField"); CollectionAdminResponse response = createCollectionRequest.process(cloudClient); assertEquals(0, response.getStatus()); @@ -171,8 +170,7 @@ public class SharedFSAutoReplicaFailoverTest extends AbstractFullDistribZkTestBa String collection2 = "solrj_collection2"; createCollectionRequest = CollectionAdminRequest.createCollection(collection2,"conf1",2,2) - .setRouterField("myOwnField") - .setAutoAddReplicas(false); + .setRouterField("myOwnField"); CollectionAdminResponse response2 = createCollectionRequest.process(getCommonCloudSolrClient()); assertEquals(0, response2.getStatus()); @@ -182,8 +180,7 @@ public class SharedFSAutoReplicaFailoverTest extends AbstractFullDistribZkTestBa String collection3 = "solrj_collection3"; createCollectionRequest = CollectionAdminRequest.createCollection(collection3,"conf1",5,1) - .setRouterField("myOwnField") - .setAutoAddReplicas(true); + .setRouterField("myOwnField"); CollectionAdminResponse response3 = createCollectionRequest.process(getCommonCloudSolrClient()); assertEquals(0, response3.getStatus()); @@ -194,8 +191,7 @@ public class SharedFSAutoReplicaFailoverTest extends AbstractFullDistribZkTestBa // a collection has only 1 replica per a shard String collection4 = "solrj_collection4"; createCollectionRequest = CollectionAdminRequest.createCollection(collection4,"conf1",5,1) - .setRouterField("text") - .setAutoAddReplicas(true); + .setRouterField("text"); CollectionAdminResponse response4 = createCollectionRequest.process(getCommonCloudSolrClient()); assertEquals(0, response4.getStatus()); diff --git a/solr/core/src/test/org/apache/solr/cloud/TestClusterProperties.java b/solr/core/src/test/org/apache/solr/cloud/TestClusterProperties.java index aa272e5fa0d..03f6afd2eeb 100644 --- a/solr/core/src/test/org/apache/solr/cloud/TestClusterProperties.java +++ b/solr/core/src/test/org/apache/solr/cloud/TestClusterProperties.java @@ -20,7 +20,6 @@ package org.apache.solr.cloud; import org.apache.solr.client.solrj.request.CollectionAdminRequest; import org.apache.solr.common.SolrException; import org.apache.solr.common.cloud.ClusterProperties; -import org.apache.solr.common.cloud.ZkStateReader; import org.junit.BeforeClass; import org.junit.Test; @@ -38,17 +37,6 @@ public class TestClusterProperties extends SolrCloudTestCase { super.setUp(); props = new ClusterProperties(zkClient()); } - - @Test - public void testClusterProperties() throws Exception { - assertEquals("false", props.getClusterProperty(ZkStateReader.AUTO_ADD_REPLICAS, "false")); - - CollectionAdminRequest.setClusterProperty(ZkStateReader.AUTO_ADD_REPLICAS, "true").process(cluster.getSolrClient()); - assertEquals("true", props.getClusterProperty(ZkStateReader.AUTO_ADD_REPLICAS, "false")); - - CollectionAdminRequest.setClusterProperty(ZkStateReader.AUTO_ADD_REPLICAS, "false").process(cluster.getSolrClient()); - assertEquals("false", props.getClusterProperty(ZkStateReader.AUTO_ADD_REPLICAS, "true")); - } @Test public void testSetPluginClusterProperty() throws Exception { diff --git a/solr/core/src/test/org/apache/solr/cloud/TestUtilizeNode.java b/solr/core/src/test/org/apache/solr/cloud/TestUtilizeNode.java deleted file mode 100644 index 68e1e882a1e..00000000000 --- a/solr/core/src/test/org/apache/solr/cloud/TestUtilizeNode.java +++ /dev/null @@ -1,184 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.cloud; - -import java.io.IOException; -import java.lang.invoke.MethodHandles; -import java.util.ArrayList; -import java.util.List; - -import org.apache.solr.client.solrj.SolrRequest; -import org.apache.solr.client.solrj.embedded.JettySolrRunner; -import org.apache.solr.client.solrj.impl.CloudSolrClient; -import org.apache.solr.client.solrj.request.CollectionAdminRequest; -import org.apache.solr.cloud.CloudTestUtils.AutoScalingRequest; -import org.apache.solr.common.cloud.DocCollection; -import org.apache.solr.common.cloud.Replica; -import org.apache.solr.common.util.NamedList; -import org.apache.solr.util.LogLevel; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -@LogLevel("org.apache.solr.cloud.autoscaling=DEBUG;org.apache.solr.cloud.Overseer=DEBUG;org.apache.solr.cloud.overseer=DEBUG;org.apache.solr.client.solrj.impl.SolrClientDataProvider=DEBUG;org.apache.solr.client.solrj.cloud.autoscaling.PolicyHelper=TRACE") -public class TestUtilizeNode extends SolrCloudTestCase { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - @BeforeClass - public static void setupCluster() throws Exception { - configureCluster(3) - .addConfig("conf1", TEST_PATH().resolve("configsets").resolve("cloud-dynamic").resolve("conf")) - .configure(); - NamedList overSeerStatus = cluster.getSolrClient().request(CollectionAdminRequest.getOverseerStatus()); - JettySolrRunner overseerJetty = null; - String overseerLeader = (String) overSeerStatus.get("leader"); - for (int i = 0; i < cluster.getJettySolrRunners().size(); i++) { - JettySolrRunner jetty = cluster.getJettySolrRunner(i); - if (jetty.getNodeName().equals(overseerLeader)) { - overseerJetty = jetty; - break; - } - } - if (overseerJetty == null) { - fail("no overseer leader!"); - } - } - - protected String getSolrXml() { - return "solr.xml"; - } - - @Before - public void beforeTest() throws Exception { - cluster.deleteAllCollections(); - } - - @Test - public void test() throws Exception { - cluster.waitForAllNodes(5); - String coll = "utilizenodecoll"; - CloudSolrClient cloudClient = cluster.getSolrClient(); - - log.info("Creating Collection..."); - CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(coll, "conf1", 2, 2); - cloudClient.request(create); - - log.info("Spinning up additional jettyX..."); - JettySolrRunner jettyX = cluster.startJettySolrRunner(); - cluster.waitForAllNodes(30); - - assertNoReplicas("jettyX should not yet be utilized: ", coll, jettyX); - - if (log.isInfoEnabled()) { - log.info("Sending UTILIZE command for jettyX ({})", jettyX.getNodeName()); - } - cloudClient.request(new CollectionAdminRequest.UtilizeNode(jettyX.getNodeName())); - - // TODO: aparently we can't assert this? ... - // - // assertSomeReplicas("jettyX should now be utilized: ", coll, jettyX); - // - // ... it appears from the docs that unless there are policy violations, - // this can be ignored unless jettyX has less "load" then other jetty instances? - // - // if the above is true, that means that this test is incredibly weak... - // unless we know jettyX has at least one replica, then all the subsequent testing of the - // port blacklist & additional UTILIZE command for jettyY are a waste of time. - // - // should we skip spinning up a *new* jettyX, and instead just pick an existing jetty? - - if (log.isInfoEnabled()) { - log.info("jettyX replicas prior to being blacklisted: {}", getReplicaList(coll, jettyX)); - } - - String setClusterPolicyCommand = "{" + - " 'set-cluster-policy': [" + - " {'port':" + jettyX.getLocalPort() + - " , 'replica':0}" + - " ]" + - "}"; - if (log.isInfoEnabled()) { - log.info("Setting new policy to blacklist jettyX ({}) port={}", - jettyX.getNodeName(), jettyX.getLocalPort()); - } - @SuppressWarnings({"rawtypes"}) - SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setClusterPolicyCommand); - NamedList response = cloudClient.request(req); - assertEquals(req + " => " + response, - "success", response.get("result").toString()); - - log.info("Spinning up additional jettyY..."); - JettySolrRunner jettyY = cluster.startJettySolrRunner(); - cluster.waitForAllNodes(30); - - assertNoReplicas("jettyY should not yet be utilized: ", coll, jettyY); - if (log.isInfoEnabled()) { - log.info("jettyX replicas prior to utilizing jettyY: {}", getReplicaList(coll, jettyX)); - log.info("Sending UTILIZE command for jettyY ({})", jettyY.getNodeName()); // logOk - } - cloudClient.request(new CollectionAdminRequest.UtilizeNode(jettyY.getNodeName())); - - assertSomeReplicas("jettyY should now be utilized: ", coll, jettyY); - } - - /** - * Gets the list of replicas for the specified collection hosted on the specified node - * and then asserts that it has no replicas - */ - private void assertNoReplicas(String prefix, String collectionName, JettySolrRunner jettyNode) throws IOException { - - final List replicas = getReplicaList(collectionName, jettyNode); - assertEquals(prefix + " " + jettyNode.getNodeName() + " => " + replicas, - 0, replicas.size()); - } - - /** - * Gets the list of replicas for the specified collection hosted on the specified node - * and then asserts that it there is at least one - */ - private void assertSomeReplicas(String prefix, String collectionName, JettySolrRunner jettyNode) throws IOException { - - final List replicas = getReplicaList(collectionName, jettyNode); - assertTrue(prefix + " " + jettyNode.getNodeName() + " => " + replicas, - 0 < replicas.size()); - } - - /** - * Returns a list of all Replicas for the specified collection hosted on the specified node using - * an uncached ClusterState call (so it should be authoritative from ZK). - */ - private List getReplicaList(String collectionName, JettySolrRunner jettyNode) throws IOException { - DocCollection collection = cluster.getSolrClient().getClusterStateProvider() - // we do *NOT* want to trust the cache, because anytime we call this method we have just - // done a lot of mucking with the cluster - .getClusterState().getCollectionOrNull(collectionName, false); - - List results = new ArrayList<>(3); - if (collection != null) { - collection.forEachReplica((s, replica) -> { - if (replica.getNodeName().equals(jettyNode.getNodeName())) { - results.add(replica); - } - }); - } - return results; - } - -} diff --git a/solr/core/src/test/org/apache/solr/cloud/TestWithCollection.java b/solr/core/src/test/org/apache/solr/cloud/TestWithCollection.java deleted file mode 100644 index 2ee47f4e79a..00000000000 --- a/solr/core/src/test/org/apache/solr/cloud/TestWithCollection.java +++ /dev/null @@ -1,611 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.cloud; - -import java.io.IOException; -import java.lang.invoke.MethodHandles; -import java.nio.charset.StandardCharsets; -import java.util.List; -import java.util.Optional; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; - -import org.apache.solr.client.solrj.SolrRequest; -import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.client.solrj.cloud.SolrCloudManager; -import org.apache.solr.client.solrj.embedded.JettySolrRunner; -import org.apache.solr.client.solrj.impl.BaseHttpSolrClient; -import org.apache.solr.client.solrj.impl.CloudSolrClient; -import org.apache.solr.client.solrj.request.CollectionAdminRequest; -import org.apache.solr.client.solrj.response.CollectionAdminResponse; -import org.apache.solr.cloud.CloudTestUtils.AutoScalingRequest; -import org.apache.solr.cloud.autoscaling.ActionContext; -import org.apache.solr.cloud.autoscaling.ComputePlanAction; -import org.apache.solr.cloud.autoscaling.ExecutePlanAction; -import org.apache.solr.cloud.autoscaling.TriggerActionBase; -import org.apache.solr.cloud.autoscaling.TriggerEvent; -import org.apache.solr.common.cloud.DocCollection; -import org.apache.solr.common.cloud.Replica; -import org.apache.solr.common.cloud.ZkStateReader; -import org.apache.solr.common.util.TimeSource; -import org.apache.solr.util.LogLevel; -import org.apache.solr.util.TimeOut; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import static org.apache.solr.common.params.CollectionAdminParams.WITH_COLLECTION; - -/** - * Tests for co-locating a collection with another collection such that any Collection API - * always ensures that the co-location is never broken. - * - * See SOLR-11990 for more details. - */ -@LogLevel("org.apache.solr.cloud.autoscaling=TRACE;org.apache.solr.client.solrj.cloud.autoscaling=DEBUG;org.apache.solr.cloud.overseer=DEBUG") -public class TestWithCollection extends SolrCloudTestCase { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - private static SolrCloudManager cloudManager; - - private static final int NUM_JETTIES = 2; - - @Before - public void setupCluster() throws Exception { - configureCluster(NUM_JETTIES) - .addConfig("conf", configset("cloud-minimal")) - .configure(); - - if (zkClient().exists(ZkStateReader.SOLR_AUTOSCALING_CONF_PATH, true)) { - zkClient().setData(ZkStateReader.SOLR_AUTOSCALING_CONF_PATH, "{}".getBytes(StandardCharsets.UTF_8), true); - } - - cluster.getSolrClient().setDefaultCollection(null); - - cloudManager = cluster.getJettySolrRunner(0).getCoreContainer().getZkController().getSolrCloudManager(); - deleteChildrenRecursively(ZkStateReader.SOLR_AUTOSCALING_EVENTS_PATH); - deleteChildrenRecursively(ZkStateReader.SOLR_AUTOSCALING_TRIGGER_STATE_PATH); - deleteChildrenRecursively(ZkStateReader.SOLR_AUTOSCALING_NODE_LOST_PATH); - deleteChildrenRecursively(ZkStateReader.SOLR_AUTOSCALING_NODE_ADDED_PATH); - LATCH = new CountDownLatch(1); - } - - @After - public void teardownCluster() throws Exception { - shutdownCluster(); - } - - @AfterClass - public static void cleanUpAfterClass() throws Exception { - cloudManager = null; - } - - private void deleteChildrenRecursively(String path) throws Exception { - cloudManager.getDistribStateManager().removeRecursively(path, true, false); - } - - @Test - public void testCreateCollectionNoWithCollection() throws IOException, SolrServerException { - String prefix = "testCreateCollectionNoWithCollection"; - String xyz = prefix + "_xyz"; - String abc = prefix + "_abc"; - - CloudSolrClient solrClient = cluster.getSolrClient(); - try { - - CollectionAdminRequest.createCollection(xyz, 1, 1) - .setWithCollection(abc).process(solrClient); - } catch (BaseHttpSolrClient.RemoteSolrException e) { - assertTrue(e.getMessage().contains("The 'withCollection' does not exist")); - } - - CollectionAdminRequest.createCollection(abc, 2, 1) - .process(solrClient); - try { - CollectionAdminRequest.createCollection(xyz, 1, 1) - .setWithCollection(abc).process(solrClient); - } catch (BaseHttpSolrClient.RemoteSolrException e) { - assertTrue(e.getMessage().contains("The `withCollection` must have only one shard, found: 2")); - } - } - - public void testCreateCollection() throws Exception { - String prefix = "testCreateCollection"; - String xyz = prefix + "_xyz"; - String abc = prefix + "_abc"; - - CloudSolrClient solrClient = cluster.getSolrClient(); - - String setClusterPolicyCommand = "{" + - " 'set-cluster-policy': [" + - " {'cores':'<10', 'node':'#ANY'}," + - " ]" + - "}"; - @SuppressWarnings({"rawtypes"}) - SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setClusterPolicyCommand); - solrClient.request(req); - - String chosenNode = cluster.getRandomJetty(random()).getNodeName(); - CollectionAdminRequest.createCollection(abc, 1, 1) - .setCreateNodeSet(chosenNode) // randomize to avoid choosing the first node always - .process(solrClient); - CollectionAdminRequest.createCollection(xyz, 1, 1) - .setWithCollection(abc) - .process(solrClient); - - DocCollection c1 = cluster.getSolrClient().getZkStateReader().getClusterState().getCollection(xyz); - assertNotNull(c1); - assertEquals(abc, c1.getStr(WITH_COLLECTION)); - Replica replica = c1.getReplicas().get(0); - String nodeName = replica.getNodeName(); - - assertEquals(chosenNode, nodeName); - } - - @Test - //Commented 14-Oct-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 23-Aug-2018 - public void testDeleteWithCollection() throws IOException, SolrServerException, InterruptedException { - String prefix = "testDeleteWithCollection"; - String xyz = prefix + "_xyz"; - String abc = prefix + "_abc"; - - CloudSolrClient solrClient = cluster.getSolrClient(); - CollectionAdminRequest.createCollection(abc, 1, 1) - .process(solrClient); - CollectionAdminRequest.createCollection(xyz, 1, 1) - .setWithCollection(abc) - .process(solrClient); - try { - CollectionAdminResponse response = CollectionAdminRequest.deleteCollection(abc).process(solrClient); - fail("Deleting collection: " + abc + " should have failed with an exception. Instead response was: " + response.getResponse()); - } catch (BaseHttpSolrClient.RemoteSolrException e) { - assertTrue(e.getMessage().contains("is co-located with collection")); - } - - // delete the co-located collection first - CollectionAdminRequest.deleteCollection(xyz).process(solrClient); - // deleting the with collection should succeed now - CollectionAdminRequest.deleteCollection(abc).process(solrClient); - - xyz = xyz + "_2"; - abc = abc + "_2"; - CollectionAdminRequest.createCollection(abc, 1, 1) - .process(solrClient); - CollectionAdminRequest.createCollection(xyz, 1, 1) - .setWithCollection(abc) - .process(solrClient); - // sanity check - try { - CollectionAdminResponse response = CollectionAdminRequest.deleteCollection(abc).process(solrClient); - fail("Deleting collection: " + abc + " should have failed with an exception. Instead response was: " + response.getResponse()); - } catch (BaseHttpSolrClient.RemoteSolrException e) { - assertTrue(e.getMessage().contains("is co-located with collection")); - } - - CollectionAdminRequest.modifyCollection(xyz, null) - .unsetAttribute("withCollection") - .process(solrClient); - TimeOut timeOut = new TimeOut(5, TimeUnit.SECONDS, TimeSource.NANO_TIME); - while (!timeOut.hasTimedOut()) { - DocCollection c1 = cluster.getSolrClient().getZkStateReader().getClusterState().getCollection(xyz); - if (c1.getStr("withCollection") == null) break; - Thread.sleep(200); - } - DocCollection c1 = cluster.getSolrClient().getZkStateReader().getClusterState().getCollection(xyz); - assertNull(c1.getStr("withCollection")); - CollectionAdminRequest.deleteCollection(abc).process(solrClient); - } - - @Test - public void testAddReplicaSimple() throws Exception { - String prefix = "testAddReplica"; - String xyz = prefix + "_xyz"; - String abc = prefix + "_abc"; - - CloudSolrClient solrClient = cluster.getSolrClient(); - String chosenNode = cluster.getRandomJetty(random()).getNodeName(); - log.info("Chosen node {} for collection {}", chosenNode, abc); - CollectionAdminRequest.createCollection(abc, 1, 1) - .setCreateNodeSet(chosenNode) // randomize to avoid choosing the first node always - .process(solrClient); - CollectionAdminRequest.createCollection(xyz, 1, 1) - .setWithCollection(abc) - .process(solrClient); - - String otherNode = null; - for (JettySolrRunner jettySolrRunner : cluster.getJettySolrRunners()) { - if (!chosenNode.equals(jettySolrRunner.getNodeName())) { - otherNode = jettySolrRunner.getNodeName(); - } - } - CollectionAdminRequest.addReplicaToShard(xyz, "shard1") - .setNode(otherNode) - .process(solrClient); - DocCollection collection = solrClient.getZkStateReader().getClusterState().getCollection(xyz); - DocCollection withCollection = solrClient.getZkStateReader().getClusterState().getCollection(abc); - - assertTrue(collection.getReplicas().stream().noneMatch(replica -> withCollection.getReplicas(replica.getNodeName()).isEmpty())); - } - - @Test - // commented out on: 17-Feb-2019 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018 - public void testAddReplicaWithPolicy() throws Exception { - String prefix = "testAddReplicaWithPolicy"; - String xyz = prefix + "_xyz"; - String abc = prefix + "_abc"; - - CloudSolrClient solrClient = cluster.getSolrClient(); - String setClusterPolicyCommand = "{" + - " 'set-cluster-policy': [" + - " {'cores':'<10', 'node':'#ANY'}," + - " {'replica':'<2', 'node':'#ANY'}," + - " ]" + - "}"; - @SuppressWarnings({"rawtypes"}) - SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setClusterPolicyCommand); - solrClient.request(req); - - String chosenNode = cluster.getRandomJetty(random()).getNodeName(); - log.info("Chosen node {} for collection {}", chosenNode, abc); - CollectionAdminRequest.createCollection(abc, 1, 1) - .setCreateNodeSet(chosenNode) // randomize to avoid choosing the first node always - .process(solrClient); - CollectionAdminRequest.createCollection(xyz, 1, 1) - .setWithCollection(abc) - .process(solrClient); - - DocCollection collection = solrClient.getZkStateReader().getClusterState().getCollection(xyz); - assertEquals(chosenNode, collection.getReplicas().iterator().next().getNodeName()); - -// zkClient().printLayoutToStdOut(); - - CollectionAdminRequest.addReplicaToShard(xyz, "shard1") - .process(solrClient); - collection = solrClient.getZkStateReader().getClusterState().getCollection(xyz); - DocCollection withCollection = solrClient.getZkStateReader().getClusterState().getCollection(abc); - - assertTrue(collection.getReplicas().stream().noneMatch( - replica -> withCollection.getReplicas(replica.getNodeName()) == null - || withCollection.getReplicas(replica.getNodeName()).isEmpty())); - } - - @Test - public void testMoveReplicaMainCollection() throws Exception { - String prefix = "testMoveReplicaMainCollection"; - String xyz = prefix + "_xyz"; - String abc = prefix + "_abc"; - - CloudSolrClient solrClient = cluster.getSolrClient(); - - String setClusterPolicyCommand = "{" + - " 'set-cluster-policy': [" + - " {'cores':'<10', 'node':'#ANY'}," + - " {'replica':'<2', 'node':'#ANY'}," + - " ]" + - "}"; - @SuppressWarnings({"rawtypes"}) - SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setClusterPolicyCommand); - solrClient.request(req); - - String chosenNode = cluster.getRandomJetty(random()).getNodeName(); - log.info("Chosen node {} for collection {}", chosenNode, abc); - CollectionAdminRequest.createCollection(abc, 1, 1) - .setCreateNodeSet(chosenNode) // randomize to avoid choosing the first node always - .process(solrClient); - CollectionAdminRequest.createCollection(xyz, 1, 1) - .setWithCollection(abc) - .process(solrClient); - - String otherNode = null; - for (JettySolrRunner jettySolrRunner : cluster.getJettySolrRunners()) { - if (!chosenNode.equals(jettySolrRunner.getNodeName())) { - otherNode = jettySolrRunner.getNodeName(); - } - } - - DocCollection collection = solrClient.getZkStateReader().getClusterState().getCollection(xyz); - DocCollection withCollection = solrClient.getZkStateReader().getClusterState().getCollection(abc); - assertNull(collection.getReplicas(otherNode)); // sanity check - assertNull(withCollection.getReplicas(otherNode)); // sanity check - - CollectionAdminRequest.MoveReplica moveReplica = new CollectionAdminRequest.MoveReplica(xyz, collection.getReplicas().iterator().next().getName(), otherNode); - moveReplica.setWaitForFinalState(true); - moveReplica.process(solrClient); -// zkClient().printLayoutToStdOut(); - collection = solrClient.getZkStateReader().getClusterState().getCollection(xyz); // refresh - DocCollection withCollectionRefreshed = solrClient.getZkStateReader().getClusterState().getCollection(abc); // refresh - assertTrue(collection.getReplicas().stream().noneMatch( - replica -> withCollectionRefreshed.getReplicas(replica.getNodeName()) == null - || withCollectionRefreshed.getReplicas(replica.getNodeName()).isEmpty())); - } - - @Test - //Commented 14-Oct-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 23-Aug-2018 - public void testMoveReplicaWithCollection() throws Exception { - String prefix = "testMoveReplicaWithCollection"; - String xyz = prefix + "_xyz"; - String abc = prefix + "_abc"; - - CloudSolrClient solrClient = cluster.getSolrClient(); - - String setClusterPolicyCommand = "{" + - " 'set-cluster-policy': [" + - " {'cores':'<10', 'node':'#ANY'}," + - " {'replica':'<2', 'node':'#ANY'}," + - " ]" + - "}"; - @SuppressWarnings({"rawtypes"}) - SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setClusterPolicyCommand); - solrClient.request(req); - - String chosenNode = cluster.getRandomJetty(random()).getNodeName(); - log.info("Chosen node {} for collection {}", chosenNode, abc); - CollectionAdminRequest.createCollection(abc, 1, 1) - .setCreateNodeSet(chosenNode) // randomize to avoid choosing the first node always - .process(solrClient); - CollectionAdminRequest.createCollection(xyz, 1, 1) - .setWithCollection(abc) - .process(solrClient); - - DocCollection collection = solrClient.getZkStateReader().getClusterState().getCollection(xyz); - assertEquals(chosenNode, collection.getReplicas().iterator().next().getNodeName()); - - String otherNode = null; - for (JettySolrRunner jettySolrRunner : cluster.getJettySolrRunners()) { - if (!chosenNode.equals(jettySolrRunner.getNodeName())) { - otherNode = jettySolrRunner.getNodeName(); - } - } - - collection = solrClient.getZkStateReader().getClusterState().getCollection(xyz); - DocCollection withCollection = solrClient.getZkStateReader().getClusterState().getCollection(abc); - assertNull(collection.getReplicas(otherNode)); // sanity check - assertNull(withCollection.getReplicas(otherNode)); // sanity check - - try { - new CollectionAdminRequest.MoveReplica(abc, collection.getReplicas().iterator().next().getName(), otherNode) - .process(solrClient); - fail("Expected moving a replica of 'withCollection': " + abc + " to fail"); - } catch (BaseHttpSolrClient.RemoteSolrException e) { - assertTrue(e.getMessage().contains("Collection: testMoveReplicaWithCollection_abc is co-located with collection: testMoveReplicaWithCollection_xyz")); - } -// zkClient().printLayoutToStdOut(); - collection = solrClient.getZkStateReader().getClusterState().getCollection(xyz); // refresh - DocCollection withCollectionRefreshed = solrClient.getZkStateReader().getClusterState().getCollection(abc); // refresh - - // sanity check that the failed move operation didn't actually change our co-location guarantees - assertTrue(collection.getReplicas().stream().noneMatch( - replica -> withCollectionRefreshed.getReplicas(replica.getNodeName()) == null - || withCollectionRefreshed.getReplicas(replica.getNodeName()).isEmpty())); - } - - /** - * Tests that when a new node is added to the cluster and autoscaling framework - * moves replicas to the new node, we maintain all co-locating guarantees - */ - // commented out on: 01-Apr-2019 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 15-Sep-2018 - public void testNodeAdded() throws Exception { - String prefix = "testNodeAdded"; - String xyz = prefix + "_xyz"; - String abc = prefix + "_abc"; - - CloudSolrClient solrClient = cluster.getSolrClient(); - - String setClusterPolicyCommand = "{" + - " 'set-cluster-policy': [" + - " {'cores':'<10', 'node':'#ANY'}," + - " {'replica':'<2', 'node':'#ANY'}," + - " ]" + - "}"; - @SuppressWarnings({"rawtypes"}) - SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setClusterPolicyCommand); - solrClient.request(req); - - String chosenNode = cluster.getRandomJetty(random()).getNodeName(); - log.info("Chosen node {} for collection {}", chosenNode, abc); - CollectionAdminRequest.createCollection(abc, 1, 1) - .setCreateNodeSet(chosenNode) // randomize to avoid choosing the first node always - .process(solrClient); - CollectionAdminRequest.createCollection(xyz, 1, 1) - .setWithCollection(abc) - .process(solrClient); - - DocCollection collection = solrClient.getZkStateReader().getClusterState().getCollection(xyz); - assertEquals(chosenNode, collection.getReplicas().iterator().next().getNodeName()); - - String setTriggerCommand = "{" + - "'set-trigger' : {" + - "'name' : 'node_added_trigger1'," + - "'event' : 'nodeAdded'," + - "'waitFor' : '0s'," + - "'actions' : [" + - "{'name' : 'compute', 'class' : '" + ComputePlanAction.class.getName() + "'}" + - "{'name' : 'execute', 'class' : '" + ExecutePlanAction.class.getName() + "'}" + - "{'name' : 'compute', 'class' : '" + CapturingAction.class.getName() + "'}" + - "]" + - "}}"; - req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand); - solrClient.request(req); - - Optional other = cluster.getJettySolrRunners() - .stream().filter(j -> !chosenNode.equals(j.getNodeName())).findAny(); - String otherNode = other.orElseThrow(AssertionError::new).getNodeName(); - - // add an extra replica of abc collection on a different node - CollectionAdminRequest.AddReplica addReplica = CollectionAdminRequest.addReplicaToShard(abc, "shard1") - .setNode(otherNode); - addReplica.setWaitForFinalState(true); - addReplica.process(solrClient); - - // refresh - collection = solrClient.getZkStateReader().getClusterState().getCollection(xyz); - DocCollection withCollection = solrClient.getZkStateReader().getClusterState().getCollection(abc); - - // sanity check - assertColocated(collection, otherNode, withCollection); - - assertEquals(1, collection.getReplicas().size()); - Replica xyzReplica = collection.getReplicas().get(0); - - // start a new node - JettySolrRunner newNode = cluster.startJettySolrRunner(); - assertTrue("Action was not fired till 30 seconds", LATCH.await(30, TimeUnit.SECONDS)); - // refresh - collection = solrClient.getZkStateReader().getClusterState().getCollection(xyz); - withCollection = solrClient.getZkStateReader().getClusterState().getCollection(abc); - - // sanity check - assertColocated(collection, otherNode, withCollection); - - // assert that the replica of xyz collection was not moved - assertNotNull(collection.getReplica(xyzReplica.getName())); - assertEquals(chosenNode, collection.getReplicas().get(0).getNodeName()); - - // add an extra replica of xyz collection -- this should be placed on the 'otherNode' - addReplica = CollectionAdminRequest.addReplicaToShard(xyz, "shard1"); - addReplica.setWaitForFinalState(true); - addReplica.process(solrClient); - - // refresh - collection = solrClient.getZkStateReader().getClusterState().getCollection(xyz); - withCollection = solrClient.getZkStateReader().getClusterState().getCollection(abc); - - List replicas = collection.getReplicas(otherNode); - assertNotNull(replicas); - assertEquals(1, replicas.size()); - replicas = withCollection.getReplicas(otherNode); - assertNotNull(replicas); - assertEquals(1, replicas.size()); - - // add an extra replica of xyz collection -- this should be placed on the 'newNode' - addReplica = CollectionAdminRequest.addReplicaToShard(xyz, "shard1"); - addReplica.setWaitForFinalState(true); - addReplica.process(solrClient); - - // refresh - collection = solrClient.getZkStateReader().getClusterState().getCollection(xyz); - withCollection = solrClient.getZkStateReader().getClusterState().getCollection(abc); - - assertNotNull(collection.getReplicas(newNode.getNodeName())); - replicas = collection.getReplicas(newNode.getNodeName()); - assertNotNull(replicas); - assertEquals(1, replicas.size()); - replicas = withCollection.getReplicas(newNode.getNodeName()); - assertNotNull(replicas); - assertEquals(1, replicas.size()); - } - - public void testMultipleWithCollections() throws Exception { - String prefix = "testMultipleWithCollections"; - String xyz = prefix + "_xyz"; - String xyz2 = prefix + "_xyz2"; - String abc = prefix + "_abc"; - String abc2 = prefix + "_abc2"; - - // start 2 more nodes so we have 4 in total - cluster.startJettySolrRunner(); - cluster.startJettySolrRunner(); - cluster.waitForAllNodes(30); - - CloudSolrClient solrClient = cluster.getSolrClient(); - - String setClusterPolicyCommand = "{" + - " 'set-cluster-policy': [" + - " {'cores':'<10', 'node':'#ANY'}," + - " {'replica':'<2', 'node':'#ANY'}," + - " ]" + - "}"; - @SuppressWarnings({"rawtypes"}) - SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setClusterPolicyCommand); - solrClient.request(req); - - String chosenNode = cluster.getJettySolrRunner(0).getNodeName(); - log.info("Chosen node {} for collection {}", chosenNode, abc); - - CollectionAdminRequest.createCollection(abc, 1, 1) - .setCreateNodeSet(chosenNode) - .process(solrClient); - CollectionAdminRequest.createCollection(xyz, 1, 1) - .setWithCollection(abc) - .process(solrClient); - - String chosenNode2 = cluster.getJettySolrRunner(1).getNodeName(); - log.info("Chosen node {} for collection {}", chosenNode2, abc2); - CollectionAdminRequest.createCollection(abc2, 1, 1) - .setCreateNodeSet(chosenNode2) - .process(solrClient); - CollectionAdminRequest.createCollection(xyz2, 1, 1) - .setWithCollection(abc2) - .process(solrClient); - - // refresh - DocCollection collection = solrClient.getZkStateReader().getClusterState().getCollection(xyz); - DocCollection collection2 = solrClient.getZkStateReader().getClusterState().getCollection(xyz2); - DocCollection withCollection = solrClient.getZkStateReader().getClusterState().getCollection(abc); - DocCollection withCollection2 = solrClient.getZkStateReader().getClusterState().getCollection(abc2); - - // sanity check - assertColocated(collection, chosenNode2, withCollection); // no replica should be on chosenNode2 - assertColocated(collection2, chosenNode, withCollection2); // no replica should be on chosenNode - - String chosenNode3 = cluster.getJettySolrRunner(2).getNodeName(); - CollectionAdminRequest.addReplicaToShard(xyz, "shard1") - .setNode(chosenNode3) - .process(solrClient); - String chosenNode4 = cluster.getJettySolrRunner(2).getNodeName(); - CollectionAdminRequest.addReplicaToShard(xyz2, "shard1") - .setNode(chosenNode4) - .process(solrClient); - - collection = solrClient.getZkStateReader().getClusterState().getCollection(xyz); - collection2 = solrClient.getZkStateReader().getClusterState().getCollection(xyz2); - withCollection = solrClient.getZkStateReader().getClusterState().getCollection(abc); - withCollection2 = solrClient.getZkStateReader().getClusterState().getCollection(abc2); - - // sanity check - assertColocated(collection, null, withCollection); - assertColocated(collection2, null, withCollection2); - } - - /** - * Asserts that all replicas of collection are colocated with at least one - * replica of the withCollection and none of them should be on the given 'noneOnNode'. - */ - private void assertColocated(DocCollection collection, String noneOnNode, DocCollection withCollection) { - // sanity check - assertTrue(collection.getReplicas().stream().noneMatch( - replica -> withCollection.getReplicas(replica.getNodeName()) == null - || withCollection.getReplicas(replica.getNodeName()).isEmpty())); - - if (noneOnNode != null) { - assertTrue(collection.getReplicas().stream().noneMatch( - replica -> noneOnNode.equals(replica.getNodeName()))); - } - } - - private static CountDownLatch LATCH = new CountDownLatch(1); - public static class CapturingAction extends TriggerActionBase { - @Override - public void process(TriggerEvent event, ActionContext context) throws Exception { - LATCH.countDown(); - } - } -} diff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/AbstractCloudBackupRestoreTestCase.java b/solr/core/src/test/org/apache/solr/cloud/api/collections/AbstractCloudBackupRestoreTestCase.java index 74bcd753aaa..ccfc78dc9ee 100644 --- a/solr/core/src/test/org/apache/solr/cloud/api/collections/AbstractCloudBackupRestoreTestCase.java +++ b/solr/core/src/test/org/apache/solr/cloud/api/collections/AbstractCloudBackupRestoreTestCase.java @@ -121,9 +121,6 @@ public abstract class AbstractCloudBackupRestoreTestCase extends SolrCloudTestCa CollectionAdminRequest.createCollectionWithImplicitRouter(getCollectionName(), "conf1", "shard1,shard2", replFactor, numTlogReplicas, numPullReplicas) : CollectionAdminRequest.createCollection(getCollectionName(), "conf1", NUM_SHARDS, replFactor, numTlogReplicas, numPullReplicas); - if (random().nextBoolean()) { - create.setAutoAddReplicas(true);//just to assert it survives the restoration - } Properties coreProps = new Properties(); coreProps.put("customKey", "customValue");//just to assert it survives the restoration create.setProperties(coreProps); @@ -366,7 +363,6 @@ public abstract class AbstractCloudBackupRestoreTestCase extends SolrCloudTestCa assertEquals(origShardToDocCount, getShardToDocCountMap(client, restoreCollection)); } - assertEquals(backupCollection.getAutoAddReplicas(), restoreCollection.getAutoAddReplicas()); assertEquals(sameConfig ? "conf1" : "customConfigName", cluster.getSolrClient().getZkStateReader().readConfigName(restoreCollectionName)); diff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/AssignTest.java b/solr/core/src/test/org/apache/solr/cloud/api/collections/AssignTest.java deleted file mode 100644 index 962d04e0fd9..00000000000 --- a/solr/core/src/test/org/apache/solr/cloud/api/collections/AssignTest.java +++ /dev/null @@ -1,204 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.cloud.api.collections; - -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Future; -import java.util.stream.Collectors; - -import org.apache.solr.SolrTestCaseJ4; -import org.apache.solr.client.solrj.cloud.DistribStateManager; -import org.apache.solr.client.solrj.cloud.SolrCloudManager; -import org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig; -import org.apache.solr.client.solrj.cloud.autoscaling.Policy; -import org.apache.solr.client.solrj.impl.ClusterStateProvider; -import org.apache.solr.client.solrj.impl.ZkDistribStateManager; -import org.apache.solr.cloud.ZkTestServer; -import org.apache.solr.common.cloud.DocCollection; -import org.apache.solr.common.cloud.DocRouter; -import org.apache.solr.common.cloud.Replica; -import org.apache.solr.common.cloud.Slice; -import org.apache.solr.common.cloud.SolrZkClient; -import org.apache.solr.common.util.ExecutorUtil; -import org.apache.solr.common.util.Utils; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyBoolean; -import static org.mockito.ArgumentMatchers.anyInt; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -public class AssignTest extends SolrTestCaseJ4 { - - @Override - @Before - public void setUp() throws Exception { - super.setUp(); - - } - - @Override - @After - public void tearDown() throws Exception { - super.tearDown(); - } - - @Test - public void testAssignNode() throws Exception { - assumeWorkingMockito(); - - SolrZkClient zkClient = mock(SolrZkClient.class); - Map zkClientData = new HashMap<>(); - when(zkClient.setData(anyString(), any(), anyInt(), anyBoolean())).then(invocation -> { - zkClientData.put(invocation.getArgument(0), invocation.getArgument(1)); - return null; - } - ); - when(zkClient.getData(anyString(), any(), any(), anyBoolean())).then(invocation -> - zkClientData.get(invocation.getArgument(0))); - // TODO: fix this to be independent of ZK - ZkDistribStateManager stateManager = new ZkDistribStateManager(zkClient); - String nodeName = Assign.assignCoreNodeName(stateManager, new DocCollection("collection1", new HashMap<>(), new HashMap<>(), DocRouter.DEFAULT)); - assertEquals("core_node1", nodeName); - nodeName = Assign.assignCoreNodeName(stateManager, new DocCollection("collection2", new HashMap<>(), new HashMap<>(), DocRouter.DEFAULT)); - assertEquals("core_node1", nodeName); - nodeName = Assign.assignCoreNodeName(stateManager, new DocCollection("collection1", new HashMap<>(), new HashMap<>(), DocRouter.DEFAULT)); - assertEquals("core_node2", nodeName); - } - - @Test - public void testIdIsUnique() throws Exception { - Path zkDir = createTempDir("zkData"); - ZkTestServer server = new ZkTestServer(zkDir); - Object fixedValue = new Object(); - String[] collections = new String[]{"c1","c2","c3","c4","c5","c6","c7","c8","c9"}; - Map> collectionUniqueIds = new HashMap<>(); - for (String c : collections) { - collectionUniqueIds.put(c, new ConcurrentHashMap<>()); - } - - ExecutorService executor = ExecutorUtil.newMDCAwareCachedThreadPool("threadpool"); - try { - server.run(); - - try (SolrZkClient zkClient = new SolrZkClient(server.getZkAddress(), 10000)) { - assertTrue(zkClient.isConnected()); - for (String c : collections) { - zkClient.makePath("/collections/" + c, true); - } - // TODO: fix this to be independent of ZK - ZkDistribStateManager stateManager = new ZkDistribStateManager(zkClient); - List> futures = new ArrayList<>(); - for (int i = 0; i < 73; i++) { - futures.add(executor.submit(() -> { - String collection = collections[random().nextInt(collections.length)]; - int id = Assign.incAndGetId(stateManager, collection, 0); - Object val = collectionUniqueIds.get(collection).put(id, fixedValue); - if (val != null) { - fail("ZkController do not generate unique id for " + collection); - } - })); - } - for (Future future : futures) { - future.get(); - } - } - assertEquals(73, (long) collectionUniqueIds.values().stream() - .map(ConcurrentHashMap::size) - .reduce((m1, m2) -> m1 + m2).get()); - } finally { - server.shutdown(); - ExecutorUtil.shutdownAndAwaitTermination(executor); - } - } - - - @Test - public void testBuildCoreName() throws Exception { - Path zkDir = createTempDir("zkData"); - ZkTestServer server = new ZkTestServer(zkDir); - server.run(); - try (SolrZkClient zkClient = new SolrZkClient(server.getZkAddress(), 10000)) { - // TODO: fix this to be independent of ZK - ZkDistribStateManager stateManager = new ZkDistribStateManager(zkClient); - Map slices = new HashMap<>(); - slices.put("shard1", new Slice("shard1", new HashMap<>(), null,"collection1")); - slices.put("shard2", new Slice("shard2", new HashMap<>(), null,"collection1")); - - DocCollection docCollection = new DocCollection("collection1", slices, null, DocRouter.DEFAULT); - assertEquals("Core name pattern changed", "collection1_shard1_replica_n1", Assign.buildSolrCoreName(stateManager, docCollection, "shard1", Replica.Type.NRT)); - assertEquals("Core name pattern changed", "collection1_shard2_replica_p2", Assign.buildSolrCoreName(stateManager, docCollection, "shard2", Replica.Type.PULL)); - } finally { - server.shutdown(); - } - } - - @Test - public void testUseLegacyByDefault() throws Exception { - assumeWorkingMockito(); - - SolrCloudManager solrCloudManager = mock(SolrCloudManager.class); - ClusterStateProvider clusterStateProvider = mock(ClusterStateProvider.class); - when(solrCloudManager.getClusterStateProvider()).thenReturn(clusterStateProvider); - DistribStateManager distribStateManager = mock(DistribStateManager.class); - when(solrCloudManager.getDistribStateManager()).thenReturn(distribStateManager); - when(distribStateManager.getAutoScalingConfig()).thenReturn(new AutoScalingConfig(Collections.emptyMap())); - - // first we don't set any cluster property and assert that legacy assignment is used - when(clusterStateProvider.getClusterProperties()).thenReturn(Collections.emptyMap()); - // verify - boolean usePolicyFramework = Assign.usePolicyFramework(solrCloudManager); - assertFalse(usePolicyFramework); - // another sanity check - when(clusterStateProvider.getClusterProperties()).thenReturn(Utils.makeMap("defaults", Collections.emptyMap())); - // verify - usePolicyFramework = Assign.usePolicyFramework(solrCloudManager); - assertFalse(usePolicyFramework); - - // first we set useLegacyReplicaAssignment=false, so autoscaling should always be used - when(clusterStateProvider.getClusterProperties()).thenReturn(Utils.makeMap("defaults", Utils.makeMap("cluster", Utils.makeMap("useLegacyReplicaAssignment", false)))); - // verify - usePolicyFramework = Assign.usePolicyFramework(solrCloudManager); - assertTrue(usePolicyFramework); - - // now we set useLegacyReplicaAssignment=true, so autoscaling can only be used if an explicit policy or preference exists - when(clusterStateProvider.getClusterProperties()).thenReturn(Utils.makeMap("defaults", Utils.makeMap("cluster", Utils.makeMap("useLegacyReplicaAssignment", true)))); - when(distribStateManager.getAutoScalingConfig()).thenReturn(new AutoScalingConfig(Collections.emptyMap())); - assertFalse(Assign.usePolicyFramework(solrCloudManager)); - - // lets provide a custom preference and assert that autoscaling is used even if useLegacyReplicaAssignment=false - // our custom preferences are exactly the same as the default ones - // but because we are providing them explicitly, they must cause autoscaling to turn on - @SuppressWarnings({"rawtypes"}) - List customPreferences = Policy.DEFAULT_PREFERENCES - .stream().map(preference -> preference.getOriginal()).collect(Collectors.toList()); - - when(distribStateManager.getAutoScalingConfig()).thenReturn(new AutoScalingConfig(Utils.makeMap("cluster-preferences", customPreferences))); - assertTrue(Assign.usePolicyFramework(solrCloudManager)); - } -} diff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionTooManyReplicasTest.java b/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionTooManyReplicasTest.java index 6489f1c3fc5..46a36554d73 100644 --- a/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionTooManyReplicasTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionTooManyReplicasTest.java @@ -21,10 +21,8 @@ import java.util.Map; import java.util.stream.Collectors; import org.apache.lucene.util.LuceneTestCase.Slow; -import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.client.solrj.embedded.JettySolrRunner; import org.apache.solr.client.solrj.request.CollectionAdminRequest; -import org.apache.solr.cloud.CloudTestUtils; import org.apache.solr.cloud.SolrCloudTestCase; import org.apache.solr.common.cloud.DocCollection; import org.apache.solr.common.cloud.Replica; @@ -32,6 +30,7 @@ import org.apache.solr.common.cloud.Slice; import org.apache.zookeeper.KeeperException; import org.junit.Before; import org.junit.BeforeClass; +import org.junit.Ignore; import org.junit.Test; @Slow @@ -50,6 +49,7 @@ public class CollectionTooManyReplicasTest extends SolrCloudTestCase { } @Test + @Ignore // Since maxShardsPerNode was removed in SOLR-12847 and autoscaling framework was removed in SOLR-14656, this test is broken public void testAddTooManyReplicas() throws Exception { final String collectionName = "TooManyReplicasInSeveralFlavors"; @@ -70,8 +70,8 @@ public class CollectionTooManyReplicasTest extends SolrCloudTestCase { .process(cluster.getSolrClient()); // equivalent to maxShardsPerNode=1 - String commands = "{ set-cluster-policy: [ {replica: '<2', shard: '#ANY', node: '#ANY', strict: true} ] }"; - cluster.getSolrClient().request(CloudTestUtils.AutoScalingRequest.create(SolrRequest.METHOD.POST, commands)); + // String commands = "{ set-cluster-policy: [ {replica: '<2', shard: '#ANY', node: '#ANY', strict: true} ] }"; + // cluster.getSolrClient().request(CloudTestUtils.AutoScalingRequest.create(SolrRequest.METHOD.POST, commands)); // this should fail because the policy prevents it Exception e = expectThrows(Exception.class, () -> { @@ -116,10 +116,11 @@ public class CollectionTooManyReplicasTest extends SolrCloudTestCase { } @Test + @Ignore // Since maxShardsPerNode was removed in SOLR-12847 and autoscaling framework was removed in SOLR-14656, this test is broken public void testAddShard() throws Exception { // equivalent to maxShardsPerNode=2 - String commands = "{ set-cluster-policy: [ {replica: '<3', shard: '#ANY', node: '#ANY', strict: true} ] }"; - cluster.getSolrClient().request(CloudTestUtils.AutoScalingRequest.create(SolrRequest.METHOD.POST, commands)); + // String commands = "{ set-cluster-policy: [ {replica: '<3', shard: '#ANY', node: '#ANY', strict: true} ] }"; + // cluster.getSolrClient().request(CloudTestUtils.AutoScalingRequest.create(SolrRequest.METHOD.POST, commands)); String collectionName = "TooManyReplicasWhenAddingShards"; CollectionAdminRequest.createCollectionWithImplicitRouter(collectionName, "conf", "shardstart", 2) diff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/ConcurrentCreateCollectionTest.java b/solr/core/src/test/org/apache/solr/cloud/api/collections/ConcurrentCreateCollectionTest.java deleted file mode 100644 index cc3bbd6d0dd..00000000000 --- a/solr/core/src/test/org/apache/solr/cloud/api/collections/ConcurrentCreateCollectionTest.java +++ /dev/null @@ -1,292 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.cloud.api.collections; - -import java.lang.invoke.MethodHandles; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.atomic.AtomicInteger; - -import org.apache.solr.client.solrj.SolrRequest; -import org.apache.solr.client.solrj.embedded.JettySolrRunner; -import org.apache.solr.client.solrj.impl.CloudSolrClient; -import org.apache.solr.client.solrj.request.CollectionAdminRequest; -import org.apache.solr.cloud.CloudTestUtils; -import org.apache.solr.cloud.SolrCloudTestCase; -import org.apache.solr.common.cloud.ClusterState; -import org.apache.solr.common.cloud.DocCollection; -import org.apache.solr.common.cloud.Replica; -import org.apache.solr.common.cloud.Slice; -import org.junit.After; -import org.junit.Before; -import org.junit.BeforeClass; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - - -public class ConcurrentCreateCollectionTest extends SolrCloudTestCase { - - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - static int NODES = 2; - - @BeforeClass - public static void setupCluster() throws Exception { - configureCluster(NODES) - .addConfig("conf", configset("cloud-minimal")) - //.addConfig("conf", configset("_default")) - .configure(); - } - - @Before - @Override - public void setUp() throws Exception { - super.setUp(); - } - - @After - @Override - public void tearDown() throws Exception { - super.tearDown(); - cluster.deleteAllCollections(); - } - - - private CollectionAdminRequest.Create createCollectionRequest(String cname, int numShards, int numReplicas) throws Exception { - CollectionAdminRequest.Create creq = CollectionAdminRequest - // .createCollection(cname, "conf", NODES - 1, NODES - 1) - .createCollection(cname, "conf", numShards, numReplicas); - creq.setWaitForFinalState(true); - creq.setAutoAddReplicas(true); - return creq; - } - - public void testConcurrentCreatePlacement() throws Exception { - final int nThreads = 2; - final int createsPerThread = 1; - final int nShards = 1; - final int repFactor = 2; - final boolean useClusterPolicy = false; - final boolean useCollectionPolicy = true; - final boolean startUnbalanced = true; // can help make a smaller test that can still reproduce an issue. - final int unbalancedSize = 1; // the number of replicas to create first - final boolean stopNode = false; // only applicable when startUnbalanced==true... stops a node during first collection creation, then restarts - - final CloudSolrClient client = cluster.getSolrClient(); - - - if (startUnbalanced) { - /*** This produces a failure (multiple replicas of single shard on same node) when run with NODES=4 and - final int nThreads = 2; - final int createsPerThread = 1; - final int nShards = 2; - final int repFactor = 2; - final boolean useClusterPolicy = false; - final boolean useCollectionPolicy = true; - final boolean startUnbalanced = true; - // NOTE: useClusterPolicy=true seems to fix it! So does putting both creates in a single thread! - // NOTE: even creating a single replica to start with causes failure later on. - - Also reproduced with smaller cluster: NODES=2 and - final int nThreads = 2; - final int createsPerThread = 1; - final int nShards = 1; - final int repFactor = 2; - final boolean useClusterPolicy = false; - final boolean useCollectionPolicy = true; - final boolean startUnbalanced = true; - - Also, with NODES=3: - final int nThreads = 2; - final int createsPerThread = 1; - final int nShards = 1; - final int repFactor = 2; - final boolean useClusterPolicy = false; - final boolean useCollectionPolicy = true; - final boolean startUnbalanced = false; - - // Also succeeded in replicating a bug where all 5 replicas were on a single node: CORES=5, nThreads=5, repFactor=5, - // unbalancedSize = 16 (4 replicas on each of the up nodes), stopNode=true - ***/ - - - JettySolrRunner downJetty = cluster.getJettySolrRunners().get(0); - if (stopNode) { - cluster.stopJettySolrRunner(downJetty); - } - - String cname = "STARTCOLLECTION"; - CollectionAdminRequest.Create creq = CollectionAdminRequest - // .createCollection(cname, "conf", NODES - 1, NODES - 1) - .createCollection(cname, "conf", unbalancedSize, 1); - creq.setWaitForFinalState(true); - // creq.setAutoAddReplicas(true); - if (useCollectionPolicy) { creq.setPolicy("policy1"); } - creq.process(client); - - if (stopNode) { - // this will start it with a new port.... does it matter? - cluster.startJettySolrRunner(downJetty); - } - } - - - - if (useClusterPolicy) { - String setClusterPolicyCommand = "{" + - " 'set-cluster-policy': [" + - // " {'cores':'<100', 'node':'#ANY'}," + - " {'replica':'<2', 'shard': '#EACH', 'node': '#ANY'}," + - // " {'replica':'<2', 'node': '#ANY'}," + - " ]" + - "}"; - - @SuppressWarnings({"rawtypes"}) - SolrRequest req = CloudTestUtils.AutoScalingRequest.create(SolrRequest.METHOD.POST, setClusterPolicyCommand); - client.request(req); - } - - if (useCollectionPolicy) { - // NOTE: the mere act of setting this named policy prevents LegacyAssignStrategy from being used, even if the policy is - // not used during collection creation. - String commands = "{set-policy : {" + - " policy1 : [{replica:'<2' , node:'#ANY'}]" + - ",policy2 : [{replica:'<2' , shard:'#EACH', node:'#ANY'}]" + - "}}"; - client.request(CloudTestUtils.AutoScalingRequest.create(SolrRequest.METHOD.POST, commands)); - - /*** take defaults for cluster preferences - String cmd = "{" + - " 'set-cluster-preferences': [" + - // " {'cores':'<100', 'node':'#ANY'}," + - " {minimize:cores}" + - " ]" + - "}"; - - SolrRequest req = CloudTestUtils.AutoScalingRequest.create(SolrRequest.METHOD.POST, cmd); - client.request(req); - ***/ - } - - /*** - SolrRequest req = CloudTestUtils.AutoScalingRequest.create(SolrRequest.METHOD.GET, null); - SolrResponse response = req.process(client); - log.info("######### AUTOSCALE {}", response); - ***/ - - - byte[] data = client.getZkStateReader().getZkClient().getData("/autoscaling.json", null, null, true); - if (log.isInfoEnabled()) { - log.info("AUTOSCALE DATA: {}", new String(data, "UTF-8")); - } - - final AtomicInteger collectionNum = new AtomicInteger(); - Thread[] indexThreads = new Thread[nThreads]; - - for (int i=0; i { - try { - for (int j=0; j> replicaMap = new HashMap<>(); - ClusterState cstate = client.getZkStateReader().getClusterState(); - for (DocCollection collection : cstate.getCollectionsMap().values()) { - for (Replica replica : collection.getReplicas()) { - String url = replica.getBaseUrl(); - List replicas = replicaMap.get(url); - if (replicas == null) { - replicas = new ArrayList<>(); - replicaMap.put(url, replicas); - } - replicas.add(replica); - } - } - - // check if nodes are balanced - boolean failed = false; - for (List replicas : replicaMap.values()) { - if (replicas.size() != expectedPerNode ) { - if (expectBalanced) { - failed = true; - } - log.error("UNBALANCED CLUSTER: expected replicas per node {} but got {}", expectedPerNode, replicas.size()); - } - } - - // check if there were multiple replicas of the same shard placed on the same node - for (DocCollection collection : cstate.getCollectionsMap().values()) { - for (Slice slice : collection.getSlices()) { - Map nodeToReplica = new HashMap<>(); - for (Replica replica : slice.getReplicas()) { - Replica prev = nodeToReplica.put(replica.getBaseUrl(), replica); - if (prev != null) { - failed = true; - // NOTE: with a replication factor > 2, this will print multiple times per bad slice. - log.error("MULTIPLE REPLICAS OF SINGLE SHARD ON SAME NODE: r1={} r2={}", prev, replica); - } - } - } - } - - if (failed) { - log.error("Cluster state {}", cstate.getCollectionsMap()); - } - - assertEquals(replicaMap.size(), NODES); // make sure something was created - - assertTrue(!failed); - } - - - -} diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoAddReplicasIntegrationTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoAddReplicasIntegrationTest.java deleted file mode 100644 index b744ac19105..00000000000 --- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoAddReplicasIntegrationTest.java +++ /dev/null @@ -1,472 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.cloud.autoscaling; - -import static org.apache.solr.common.util.Utils.makeMap; - -import java.lang.invoke.MethodHandles; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; - -import org.apache.solr.client.solrj.SolrRequest; -import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.client.solrj.embedded.JettySolrRunner; -import org.apache.solr.client.solrj.request.CollectionAdminRequest; -import org.apache.solr.client.solrj.request.QueryRequest; -import org.apache.solr.client.solrj.request.V2Request; -import org.apache.solr.cloud.MiniSolrCloudCluster; -import org.apache.solr.cloud.SolrCloudTestCase; -import org.apache.solr.common.cloud.CollectionStatePredicate; -import org.apache.solr.common.cloud.ClusterStateUtil; -import org.apache.solr.common.cloud.DocCollection; -import org.apache.solr.common.cloud.Replica; -import org.apache.solr.common.cloud.ZkStateReader; -import org.apache.solr.common.params.CollectionParams; -import org.apache.solr.common.params.MapSolrParams; -import org.apache.solr.common.params.ModifiableSolrParams; -import org.apache.solr.common.params.SolrParams; -import org.apache.solr.common.util.TimeSource; -import org.apache.solr.util.TimeOut; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -@org.apache.solr.util.LogLevel("org.apache.solr.cloud.autoscaling=DEBUG;org.apache.solr.cloud.autoscaling.NodeLostTrigger=TRACE;org.apache.solr.cloud.Overseer=DEBUG;org.apache.solr.cloud.overseer=DEBUG") -public class AutoAddReplicasIntegrationTest extends SolrCloudTestCase { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - - protected String getConfigSet() { - return "cloud-minimal"; - } - - @Before - public void setupCluster() throws Exception { - configureCluster(3) - .addConfig("conf", configset(getConfigSet())) - .withSolrXml(TEST_PATH().resolve("solr.xml")) - .configure(); - - new V2Request.Builder("/cluster") - .withMethod(SolrRequest.METHOD.POST) - .withPayload("{set-obj-property:{defaults : {cluster: {useLegacyReplicaAssignment:true}}}}") - .build() - .process(cluster.getSolrClient()); - - new V2Request.Builder("/cluster/autoscaling") - .withMethod(SolrRequest.METHOD.POST) - .withPayload("{'set-trigger':{'name':'.auto_add_replicas','event':'nodeLost','waitFor':'5s','enabled':'true','actions':[{'name':'auto_add_replicas_plan','class':'solr.AutoAddReplicasPlanAction'},{'name':'execute_plan','class':'solr.ExecutePlanAction'}]}}") - .build() - .process(cluster.getSolrClient()); - } - - @After - public void tearDown() throws Exception { - try { - shutdownCluster(); - } finally { - super.tearDown(); - } - } - - /** - * Test that basic autoAddReplicaLogic kicks in when a node is lost - */ - @Test - public void testSimple() throws Exception { - final String COLLECTION = "test_simple"; - final ZkStateReader zkStateReader = cluster.getSolrClient().getZkStateReader(); - final JettySolrRunner jetty1 = cluster.getJettySolrRunner(1); - final JettySolrRunner jetty2 = cluster.getJettySolrRunner(2); - if (log.isInfoEnabled()) { - log.info("Creating {} using jetty1:{}/{} and jetty2:{}/{}", COLLECTION, - jetty1.getNodeName(), jetty1.getLocalPort(), - jetty2.getNodeName(), jetty2.getLocalPort()); - } - - CollectionAdminRequest.createCollection(COLLECTION, "conf", 2, 2) - .setCreateNodeSet(jetty1.getNodeName()+","+jetty2.getNodeName()) - .setAutoAddReplicas(true) - .process(cluster.getSolrClient()); - - cluster.waitForActiveCollection(COLLECTION, 2, 4); - - // start the tests - JettySolrRunner lostJetty = random().nextBoolean() ? jetty1 : jetty2; - String lostNodeName = lostJetty.getNodeName(); - List replacedHdfsReplicas = getReplacedSharedFsReplicas(COLLECTION, zkStateReader, lostNodeName); - if (log.isInfoEnabled()) { - log.info("Stopping random node: {} / {}", lostNodeName, lostJetty.getLocalPort()); - } - lostJetty.stop(); - - cluster.waitForJettyToStop(lostJetty); - waitForNodeLeave(lostNodeName); - - waitForState(COLLECTION + "=(2,4) w/o down replicas", - COLLECTION, clusterShapeNoDownReplicas(2,4), 90, TimeUnit.SECONDS); - - checkSharedFsReplicasMovedCorrectly(replacedHdfsReplicas, zkStateReader, COLLECTION); - - if (log.isInfoEnabled()) { - log.info("Re-starting (same) random node: {} / {}", lostNodeName, lostJetty.getLocalPort()); - } - lostJetty.start(); - - waitForNodeLive(lostJetty); - - assertTrue("Timeout waiting for all live and active", - ClusterStateUtil.waitForAllActiveAndLiveReplicas(zkStateReader, 90000)); - - } - - /** - * Test that basic autoAddReplicaLogic logic is not used if the cluster prop for it is disabled - * (even if sys prop is set after collection is created) - */ - @Test - public void testClusterPropOverridesCollecitonProp() throws Exception { - final String COLLECTION = "test_clusterprop"; - final ZkStateReader zkStateReader = cluster.getSolrClient().getZkStateReader(); - final JettySolrRunner jetty1 = cluster.getJettySolrRunner(1); - final JettySolrRunner jetty2 = cluster.getJettySolrRunner(2); - - if (log.isInfoEnabled()) { - log.info("Creating {} using jetty1:{}/{} and jetty2:{}/{}", COLLECTION, - jetty1.getNodeName(), jetty1.getLocalPort(), - jetty2.getNodeName(), jetty2.getLocalPort()); - } - - CollectionAdminRequest.createCollection(COLLECTION, "conf", 2, 2) - .setCreateNodeSet(jetty1.getNodeName()+","+jetty2.getNodeName()) - .setAutoAddReplicas(true) - .process(cluster.getSolrClient()); - - cluster.waitForActiveCollection(COLLECTION, 2, 4); - - // check cluster property is considered - disableAutoAddReplicasInCluster(); - - JettySolrRunner lostJetty = random().nextBoolean() ? jetty1 : jetty2; - String lostNodeName = lostJetty.getNodeName(); - List replacedHdfsReplicas = getReplacedSharedFsReplicas(COLLECTION, zkStateReader, lostNodeName); - - if (log.isInfoEnabled()) { - log.info("Stopping random node: {} / {}", lostNodeName, lostJetty.getLocalPort()); - } - lostJetty.stop(); - - cluster.waitForJettyToStop(lostJetty); - - waitForNodeLeave(lostNodeName); - - waitForState(COLLECTION + "=(2,2)", COLLECTION, - clusterShape(2, 2), 90, TimeUnit.SECONDS); - - - if (log.isInfoEnabled()) { - log.info("Re-starting (same) random node: {} / {}", lostNodeName, lostJetty.getLocalPort()); - } - lostJetty.start(); - - waitForNodeLive(lostJetty); - - assertTrue("Timeout waiting for all live and active", - ClusterStateUtil.waitForAllActiveAndLiveReplicas(zkStateReader, 90000)); - - waitForState(COLLECTION + "=(2,4) w/o down replicas", - COLLECTION, clusterShapeNoDownReplicas(2,4), 90, TimeUnit.SECONDS); - - } - - /** - * Test that we can modify a collection after creation to add autoAddReplicas. - */ - @Test - public void testAddCollectionPropAfterCreation() throws Exception { - final String COLLECTION = "test_addprop"; - final ZkStateReader zkStateReader = cluster.getSolrClient().getZkStateReader(); - final JettySolrRunner jetty1 = cluster.getJettySolrRunner(1); - final JettySolrRunner jetty2 = cluster.getJettySolrRunner(2); - - if (log.isInfoEnabled()) { - log.info("Creating {} using jetty1:{}/{} and jetty2:{}/{}", COLLECTION, - jetty1.getNodeName(), jetty1.getLocalPort(), - jetty2.getNodeName(), jetty2.getLocalPort()); - } - - CollectionAdminRequest.createCollection(COLLECTION, "conf", 2, 2) - .setCreateNodeSet(jetty1.getNodeName()+","+jetty2.getNodeName()) - .setAutoAddReplicas(false) // NOTE: false - .process(cluster.getSolrClient()); - - cluster.waitForActiveCollection(COLLECTION, 2, 4); - - log.info("Modifying {} to use autoAddReplicas", COLLECTION); - new CollectionAdminRequest.AsyncCollectionAdminRequest(CollectionParams.CollectionAction.MODIFYCOLLECTION) { - @Override - public SolrParams getParams() { - ModifiableSolrParams params = (ModifiableSolrParams) super.getParams(); - params.set("collection", COLLECTION); - params.set("autoAddReplicas", true); - return params; - } - }.process(cluster.getSolrClient()); - - JettySolrRunner lostJetty = random().nextBoolean() ? jetty1 : jetty2; - String lostNodeName = lostJetty.getNodeName(); - List replacedHdfsReplicas = getReplacedSharedFsReplicas(COLLECTION, zkStateReader, lostNodeName); - - if (log.isInfoEnabled()) { - log.info("Stopping random node: {} / {}", lostNodeName, lostJetty.getLocalPort()); - } - lostJetty.stop(); - - cluster.waitForJettyToStop(lostJetty); - - waitForNodeLeave(lostNodeName); - - waitForState(COLLECTION + "=(2,4) w/o down replicas", - COLLECTION, clusterShapeNoDownReplicas(2,4), 90, TimeUnit.SECONDS); - checkSharedFsReplicasMovedCorrectly(replacedHdfsReplicas, zkStateReader, COLLECTION); - - if (log.isInfoEnabled()) { - log.info("Re-starting (same) random node: {} / {}", lostNodeName, lostJetty.getLocalPort()); - } - lostJetty.start(); - - waitForNodeLive(lostJetty); - - assertTrue("Timeout waiting for all live and active", - ClusterStateUtil.waitForAllActiveAndLiveReplicas(zkStateReader, 90000)); - } - - /** - * Test a specific sequence of problematic events: - *
    - *
  • create a collection with autoAddReplicas=false
    • - *
  • stop a nodeX in use by the collection
    • - *
  • re-start nodeX
    • - *
  • set autoAddReplicas=true
    • - *
  • re-stop nodeX
    • - *
- */ - @Test - @AwaitsFix(bugUrl="https://issues.apache.org/jira/browse/SOLR-13811") - public void testRapidStopStartStopWithPropChange() throws Exception { - - // This is the collection we'll be focused on in our testing... - final String COLLECTION = "test_stoptwice"; - // This is a collection we'll use as a "marker" to ensure we "wait" for the - // autoAddReplicas logic (via NodeLostTrigger) to kick in at least once before proceeding... - final String ALT_COLLECTION = "test_dummy"; - - final ZkStateReader zkStateReader = cluster.getSolrClient().getZkStateReader(); - final JettySolrRunner jetty1 = cluster.getJettySolrRunner(1); - final JettySolrRunner jetty2 = cluster.getJettySolrRunner(2); - - if (log.isInfoEnabled()) { - log.info("Creating {} using jetty1:{}/{} and jetty2:{}/{}", COLLECTION, - jetty1.getNodeName(), jetty1.getLocalPort(), - jetty2.getNodeName(), jetty2.getLocalPort()); - } - - CollectionAdminRequest.createCollection(COLLECTION, "conf", 2, 2) - .setCreateNodeSet(jetty1.getNodeName()+","+jetty2.getNodeName()) - .setAutoAddReplicas(false) // NOTE: false - .process(cluster.getSolrClient()); - - if (log.isInfoEnabled()) { - log.info("Creating {} using jetty1:{}/{} and jetty2:{}/{}", ALT_COLLECTION, - jetty1.getNodeName(), jetty1.getLocalPort(), - jetty2.getNodeName(), jetty2.getLocalPort()); - } - - CollectionAdminRequest.createCollection(ALT_COLLECTION, "conf", 2, 2) - .setCreateNodeSet(jetty1.getNodeName()+","+jetty2.getNodeName()) - .setAutoAddReplicas(true) // NOTE: true - .process(cluster.getSolrClient()); - - cluster.waitForActiveCollection(COLLECTION, 2, 4); - cluster.waitForActiveCollection(ALT_COLLECTION, 2, 4); - - JettySolrRunner lostJetty = random().nextBoolean() ? jetty1 : jetty2; - String lostNodeName = lostJetty.getNodeName(); - List replacedHdfsReplicas = getReplacedSharedFsReplicas(COLLECTION, zkStateReader, lostNodeName); - - if (log.isInfoEnabled()) { - log.info("Stopping random node: {} / {}", lostNodeName, lostJetty.getLocalPort()); - } - lostJetty.stop(); - - cluster.waitForJettyToStop(lostJetty); - waitForNodeLeave(lostNodeName); - - // ensure that our marker collection indicates that the autoAddReplicas logic - // has detected the down node and done some processing - waitForState(ALT_COLLECTION + "=(2,4) w/o down replicas", - ALT_COLLECTION, clusterShapeNoDownReplicas(2,4), 90, TimeUnit.SECONDS); - - waitForState(COLLECTION + "=(2,2)", COLLECTION, clusterShape(2, 2)); - - if (log.isInfoEnabled()) { - log.info("Re-starting (same) random node: {} / {}", lostNodeName, lostJetty.getLocalPort()); - } - lostJetty.start(); - // save time, don't bother waiting for lostJetty to start until after updating collection prop... - - log.info("Modifying {} to use autoAddReplicas", COLLECTION); - new CollectionAdminRequest.AsyncCollectionAdminRequest(CollectionParams.CollectionAction.MODIFYCOLLECTION) { - @Override - public SolrParams getParams() { - ModifiableSolrParams params = (ModifiableSolrParams) super.getParams(); - params.set("collection", COLLECTION); - params.set("autoAddReplicas", true); - return params; - } - }.process(cluster.getSolrClient()); - - // make sure lostJetty is fully up before stopping again... - waitForNodeLive(lostJetty); - - if (log.isInfoEnabled()) { - log.info("Re-Stopping (same) random node: {} / {}", lostNodeName, lostJetty.getLocalPort()); - } - lostJetty.stop(); - - cluster.waitForJettyToStop(lostJetty); - waitForNodeLeave(lostNodeName); - - // TODO: this is the problematic situation... - // wether or not NodeLostTrigger noticed that lostJetty was re-started and shutdown *again* - // and that the new auoAddReplicas=true since the last time lostJetty was shutdown is respected - waitForState(COLLECTION + "=(2,4) w/o down replicas", - COLLECTION, clusterShapeNoDownReplicas(2,4), 90, TimeUnit.SECONDS); - checkSharedFsReplicasMovedCorrectly(replacedHdfsReplicas, zkStateReader, COLLECTION); - - if (log.isInfoEnabled()) { - log.info("Re-Re-starting (same) random node: {} / {}", lostNodeName, lostJetty.getLocalPort()); - } - lostJetty.start(); - - waitForNodeLive(lostJetty); - - assertTrue("Timeout waiting for all live and active", - ClusterStateUtil.waitForAllActiveAndLiveReplicas(zkStateReader, 90000)); - } - - private void disableAutoAddReplicasInCluster() throws SolrServerException, IOException { - @SuppressWarnings({"rawtypes"}) - Map m = makeMap( - "action", CollectionParams.CollectionAction.CLUSTERPROP.toLower(), - "name", ZkStateReader.AUTO_ADD_REPLICAS, - "val", "false"); - @SuppressWarnings({"unchecked"}) - QueryRequest request = new QueryRequest(new MapSolrParams(m)); - request.setPath("/admin/collections"); - cluster.getSolrClient().request(request); - } - - private void enableAutoAddReplicasInCluster() throws SolrServerException, IOException { - @SuppressWarnings({"rawtypes"}) - Map m = makeMap( - "action", CollectionParams.CollectionAction.CLUSTERPROP.toLower(), - "name", ZkStateReader.AUTO_ADD_REPLICAS); - @SuppressWarnings({"unchecked"}) - QueryRequest request = new QueryRequest(new MapSolrParams(m)); - request.setPath("/admin/collections"); - cluster.getSolrClient().request(request); - } - - private void checkSharedFsReplicasMovedCorrectly(List replacedHdfsReplicas, ZkStateReader zkStateReader, String collection){ - DocCollection docCollection = zkStateReader.getClusterState().getCollection(collection); - for (Replica replica :replacedHdfsReplicas) { - boolean found = false; - String dataDir = replica.getStr("dataDir"); - String ulogDir = replica.getStr("ulogDir"); - for (Replica replica2 : docCollection.getReplicas()) { - if (dataDir.equals(replica2.getStr("dataDir")) && ulogDir.equals(replica2.getStr("ulogDir"))) { - found = true; - break; - } - } - if (!found) fail("Can not found a replica with same dataDir and ulogDir as " + replica + " from:" + docCollection.getReplicas()); - } - } - - private List getReplacedSharedFsReplicas(String collection, ZkStateReader zkStateReader, String lostNodeName) { - List replacedHdfsReplicas = new ArrayList<>(); - for (Replica replica : zkStateReader.getClusterState().getCollection(collection).getReplicas()) { - String dataDir = replica.getStr("dataDir"); - if (replica.getNodeName().equals(lostNodeName) && dataDir != null) { - replacedHdfsReplicas.add(replica); - } - } - - return replacedHdfsReplicas; - } - - /** - * {@link MiniSolrCloudCluster#waitForNode} Doesn't check isRunning first, and we don't want to - * use {@link MiniSolrCloudCluster#waitForAllNodes} because we don't want to waste cycles checking - * nodes we aren't messing with - */ - private void waitForNodeLive(final JettySolrRunner jetty) - throws InterruptedException, TimeoutException, IOException { - if (log.isInfoEnabled()) { - log.info("waitForNodeLive: {}/{}", jetty.getNodeName(), jetty.getLocalPort()); - } - - TimeOut timeout = new TimeOut(30, TimeUnit.SECONDS, TimeSource.NANO_TIME); - while(!timeout.hasTimedOut()) { - if (jetty.isRunning()) { - break; - } - try { - Thread.sleep(100); - } catch (InterruptedException e) { - // ignore - } - } - if (timeout.hasTimedOut()) { - throw new TimeoutException("Waiting for Jetty to stop timed out"); - } - cluster.waitForNode(jetty, 30); - } - - private void waitForNodeLeave(String lostNodeName) throws InterruptedException, TimeoutException { - log.info("waitForNodeLeave: {}", lostNodeName); - ZkStateReader reader = cluster.getSolrClient().getZkStateReader(); - reader.waitForLiveNodes(30, TimeUnit.SECONDS, (o, n) -> !n.contains(lostNodeName)); - } - - - private static CollectionStatePredicate clusterShapeNoDownReplicas(final int expectedShards, - final int expectedReplicas) { - return (liveNodes, collectionState) - -> (clusterShape(expectedShards, expectedReplicas).matches(liveNodes, collectionState) - && collectionState.getReplicas().size() == expectedReplicas); - } - -} diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoAddReplicasPlanActionTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoAddReplicasPlanActionTest.java deleted file mode 100644 index 081b89e8e47..00000000000 --- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoAddReplicasPlanActionTest.java +++ /dev/null @@ -1,263 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.cloud.autoscaling; - -import java.util.Collections; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Optional; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; - -import org.apache.solr.client.solrj.SolrRequest; -import org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventType; -import org.apache.solr.client.solrj.embedded.JettySolrRunner; -import org.apache.solr.client.solrj.impl.CloudSolrClient; -import org.apache.solr.client.solrj.request.CollectionAdminRequest; -import org.apache.solr.client.solrj.request.V2Request; -import org.apache.solr.cloud.CloudDescriptor; -import org.apache.solr.cloud.CloudTestUtils.AutoScalingRequest; -import org.apache.solr.cloud.SolrCloudTestCase; -import org.apache.solr.common.SolrException; -import org.apache.solr.common.cloud.ClusterStateUtil; -import org.apache.solr.common.cloud.ZkStateReader; -import org.apache.solr.common.params.CollectionParams; -import org.apache.solr.common.params.ModifiableSolrParams; -import org.apache.solr.common.params.SolrParams; -import org.apache.solr.common.util.NamedList; -import org.apache.solr.common.util.SuppressForbidden; -import org.junit.After; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; - -public class AutoAddReplicasPlanActionTest extends SolrCloudTestCase{ - - @BeforeClass - public static void setupCluster() throws Exception { - System.setProperty("solr.httpclient.retries", "4"); - System.setProperty("solr.retries.on.forward", "1"); - System.setProperty("solr.retries.to.followers", "1"); - - } - - @Before - public void beforeTest() throws Exception { - configureCluster(3) - .addConfig("conf", configset("cloud-minimal")) - .configure(); - - new V2Request.Builder("/cluster") - .withMethod(SolrRequest.METHOD.POST) - .withPayload("{set-obj-property:{defaults : {cluster: {useLegacyReplicaAssignment:true}}}}") - .build() - .process(cluster.getSolrClient()); - } - - @After - public void afterTest() throws Exception { - shutdownCluster(); - } - - @Test - //Commented out 11-Dec-2018 @AwaitsFix(bugUrl="https://issues.apache.org/jira/browse/SOLR-13028") - public void testSimple() throws Exception { - JettySolrRunner jetty1 = cluster.getJettySolrRunner(0); - JettySolrRunner jetty2 = cluster.getJettySolrRunner(1); - JettySolrRunner jetty3 = cluster.getJettySolrRunner(2); - - String collection1 = "testSimple1"; - String collection2 = "testSimple2"; - String collection3 = "testSimple3"; - CollectionAdminRequest.createCollection(collection1, "conf", 2, 2) - .setCreateNodeSet(jetty1.getNodeName()+","+jetty2.getNodeName()) - .setAutoAddReplicas(true) - .process(cluster.getSolrClient()); - CollectionAdminRequest.createCollection(collection2, "conf", 1, 2) - .setCreateNodeSet(jetty2.getNodeName()+","+jetty3.getNodeName()) - .setAutoAddReplicas(false) - .process(cluster.getSolrClient()); - // the number of cores in jetty1 (6) will be larger than jetty3 (1) - CollectionAdminRequest.createCollection(collection3, "conf", 3, 1) - .setCreateNodeSet(jetty1.getNodeName()) - .setAutoAddReplicas(false) - .process(cluster.getSolrClient()); - - cluster.waitForActiveCollection(collection1, 2, 4); - cluster.waitForActiveCollection(collection2, 1, 2); - cluster.waitForActiveCollection(collection3, 3, 3); - - // we remove the implicit created trigger, so the replicas won't be moved - String removeTriggerCommand = "{" + - "'remove-trigger' : {" + - "'name' : '.auto_add_replicas'," + - "'removeListeners': true" + - "}" + - "}"; - @SuppressWarnings({"rawtypes"}) - SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, removeTriggerCommand); - @SuppressWarnings({"rawtypes"}) - NamedList response = cluster.getSolrClient().request(req); - assertEquals(response.get("result").toString(), "success"); - - JettySolrRunner lostJetty = random().nextBoolean()? jetty1 : jetty2; - String lostNodeName = lostJetty.getNodeName(); - List cloudDescriptors = lostJetty.getCoreContainer().getCores().stream() - .map(solrCore -> solrCore.getCoreDescriptor().getCloudDescriptor()) - .collect(Collectors.toList()); - - ZkStateReader reader = cluster.getSolrClient().getZkStateReader(); - - lostJetty.stop(); - - cluster.waitForJettyToStop(lostJetty); - - reader.waitForLiveNodes(30, TimeUnit.SECONDS, missingLiveNode(lostNodeName)); - - - @SuppressWarnings({"rawtypes"}) - List operations = getOperations(jetty3, lostNodeName); - assertOperations(collection1, operations, lostNodeName, cloudDescriptors, null); - - lostJetty.start(); - cluster.waitForAllNodes(30); - - cluster.waitForActiveCollection(collection1, 2, 4); - cluster.waitForActiveCollection(collection2, 1, 2); - cluster.waitForActiveCollection(collection3, 3, 3); - - assertTrue("Timeout waiting for all live and active", ClusterStateUtil.waitForAllActiveAndLiveReplicas(cluster.getSolrClient().getZkStateReader(), 30000)); - - String setClusterPreferencesCommand = "{" + - "'set-cluster-preferences': [" + - "{'minimize': 'cores','precision': 0}]" + - "}"; - req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setClusterPreferencesCommand); - - // you can hit a stale connection from pool when restarting jetty - try (CloudSolrClient cloudClient = new CloudSolrClient.Builder(Collections.singletonList(cluster.getZkServer().getZkAddress()), - Optional.empty()) - .withSocketTimeout(45000).withConnectionTimeout(15000).build()) { - response = cloudClient.request(req); - } - - assertEquals(response.get("result").toString(), "success"); - - lostJetty = random().nextBoolean()? jetty1 : jetty2; - String lostNodeName2 = lostJetty.getNodeName(); - cloudDescriptors = lostJetty.getCoreContainer().getCores().stream() - .map(solrCore -> solrCore.getCoreDescriptor().getCloudDescriptor()) - .collect(Collectors.toList()); - - - - lostJetty.stop(); - - reader.waitForLiveNodes(30, TimeUnit.SECONDS, missingLiveNode(lostNodeName2)); - - try { - operations = getOperations(jetty3, lostNodeName2); - } catch (SolrException e) { - // we might get a stale connection from the pool after jetty restarts - operations = getOperations(jetty3, lostNodeName2); - } - - assertOperations(collection1, operations, lostNodeName2, cloudDescriptors, jetty3); - - lostJetty.start(); - cluster.waitForAllNodes(30); - - cluster.waitForActiveCollection(collection1, 2, 4); - cluster.waitForActiveCollection(collection2, 1, 2); - cluster.waitForActiveCollection(collection3, 3, 3); - - assertTrue("Timeout waiting for all live and active", ClusterStateUtil.waitForAllActiveAndLiveReplicas(cluster.getSolrClient().getZkStateReader(), 30000)); - - new CollectionAdminRequest.AsyncCollectionAdminRequest(CollectionParams.CollectionAction.MODIFYCOLLECTION) { - @Override - public SolrParams getParams() { - ModifiableSolrParams params = (ModifiableSolrParams) super.getParams(); - params.set("collection", collection1); - params.set("autoAddReplicas", false); - return params; - } - }.process(cluster.getSolrClient()); - lostJetty = jetty1; - String lostNodeName3 = lostJetty.getNodeName(); - - lostJetty.stop(); - - reader.waitForLiveNodes(30, TimeUnit.SECONDS, missingLiveNode(lostNodeName3)); - - operations = getOperations(jetty3, lostNodeName3); - assertNull(operations); - } - - @SuppressForbidden(reason = "Needs currentTimeMillis to create unique id") - @SuppressWarnings({"rawtypes"}) - private List getOperations(JettySolrRunner actionJetty, String lostNodeName) throws Exception { - try (AutoAddReplicasPlanAction action = new AutoAddReplicasPlanAction()) { - action.configure(actionJetty.getCoreContainer().getResourceLoader(), actionJetty.getCoreContainer().getZkController().getSolrCloudManager(), new HashMap<>()); - TriggerEvent lostNode = new NodeLostTrigger.NodeLostEvent(TriggerEventType.NODELOST, ".auto_add_replicas", Collections.singletonList(System.currentTimeMillis()), Collections.singletonList(lostNodeName), CollectionParams.CollectionAction.MOVEREPLICA.toLower()); - ActionContext context = new ActionContext(actionJetty.getCoreContainer().getZkController().getSolrCloudManager(), null, new HashMap<>()); - action.process(lostNode, context); - @SuppressWarnings({"unchecked", "rawtypes"}) - List operations = (List) context.getProperty("operations"); - return operations; - } - } - - private void assertOperations(String collection, - @SuppressWarnings({"rawtypes"})List operations, String lostNodeName, - List cloudDescriptors, JettySolrRunner destJetty) { - assertEquals("Replicas of " + collection + " is not fully moved, operations="+operations, - cloudDescriptors.stream().filter(cd -> cd.getCollectionName().equals(collection)).count(), operations.size()); - for (@SuppressWarnings({"rawtypes"})SolrRequest solrRequest : operations) { - assertTrue(solrRequest instanceof CollectionAdminRequest.MoveReplica); - SolrParams params = solrRequest.getParams(); - - assertEquals(params.get("collection"), collection); - - String replica = params.get("replica"); - boolean found = false; - Iterator it = cloudDescriptors.iterator(); - while (it.hasNext()) { - CloudDescriptor cd = it.next(); - if (cd.getCollectionName().equals(collection) && cd.getCoreNodeName().equals(replica)) { - found = true; - it.remove(); - break; - } - } - assertTrue("Can not find "+replica+ " in node " + lostNodeName, found); - - String targetNode = params.get("targetNode"); - assertFalse("Target node match the lost one " + lostNodeName, lostNodeName.equals(targetNode)); - if (destJetty != null) { - assertEquals("Target node is not as expectation", destJetty.getNodeName(), targetNode); - } - } - - for (CloudDescriptor cd : cloudDescriptors) { - if (cd.getCollectionName().equals(collection)) { - fail("Exist replica which is not moved " + cd); - } - } - } -} diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java deleted file mode 100644 index 6d83fe26001..00000000000 --- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java +++ /dev/null @@ -1,1114 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.cloud.autoscaling; - -import java.io.IOException; -import java.lang.invoke.MethodHandles; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; - -import org.apache.solr.client.solrj.SolrRequest; -import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.client.solrj.cloud.autoscaling.Policy; -import org.apache.solr.client.solrj.embedded.JettySolrRunner; -import org.apache.solr.client.solrj.impl.BaseHttpSolrClient; -import org.apache.solr.client.solrj.impl.CloudSolrClient; -import org.apache.solr.client.solrj.request.CollectionAdminRequest; -import org.apache.solr.client.solrj.response.CollectionAdminResponse; -import org.apache.solr.cloud.CloudTestUtils.AutoScalingRequest; -import org.apache.solr.cloud.SolrCloudTestCase; -import org.apache.solr.common.cloud.DocCollection; -import org.apache.solr.common.cloud.Replica; -import org.apache.solr.common.cloud.ZkNodeProps; -import org.apache.solr.common.params.AutoScalingParams; -import org.apache.solr.common.params.MapSolrParams; -import org.apache.solr.common.util.NamedList; -import org.apache.solr.common.util.TimeSource; -import org.apache.solr.common.util.Utils; -import org.apache.solr.util.TimeOut; -import org.apache.zookeeper.data.Stat; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import static org.apache.solr.client.solrj.cloud.autoscaling.Suggestion.Type.repair; -import static org.apache.solr.common.cloud.ZkStateReader.SOLR_AUTOSCALING_CONF_PATH; -import static org.apache.solr.common.util.Utils.getObjectByPath; - -/** - * Test for AutoScalingHandler - */ -public class AutoScalingHandlerTest extends SolrCloudTestCase { - final static String CONFIGSET_NAME = "conf"; - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - @BeforeClass - public static void setupCluster() throws Exception { - configureCluster(2) - .addConfig(CONFIGSET_NAME, configset("cloud-minimal")) - .configure(); - testAutoAddReplicas(); - } - - private static void testAutoAddReplicas() throws Exception { - TimeOut timeOut = new TimeOut(30, TimeUnit.SECONDS, TimeSource.NANO_TIME); - while (!timeOut.hasTimedOut()) { - byte[] data = zkClient().getData(SOLR_AUTOSCALING_CONF_PATH, null, null, true); - ZkNodeProps loaded = ZkNodeProps.load(data); - @SuppressWarnings({"rawtypes"}) - Map triggers = (Map) loaded.get("triggers"); - if (triggers != null && triggers.containsKey(".auto_add_replicas")) { - @SuppressWarnings({"unchecked"}) - Map autoAddReplicasTrigger = (Map) triggers.get(".auto_add_replicas"); - assertNotNull(autoAddReplicasTrigger); - @SuppressWarnings({"unchecked"}) - List> actions = (List>) autoAddReplicasTrigger.get("actions"); - assertNotNull(actions); - assertEquals(2, actions.size()); - assertEquals("auto_add_replicas_plan", actions.get(0).get("name").toString()); - assertEquals("solr.AutoAddReplicasPlanAction", actions.get(0).get("class").toString()); - break; - } else { - Thread.sleep(300); - } - } - if (timeOut.hasTimedOut()) { - fail("Timeout waiting for .auto_add_replicas being created"); - } - } - - @Before - public void beforeTest() throws Exception { - // clear any persisted auto scaling configuration - zkClient().setData(SOLR_AUTOSCALING_CONF_PATH, Utils.toJSON(new ZkNodeProps()), true); - } - - public void testSuggestionsWithPayload() throws Exception { - CloudSolrClient solrClient = cluster.getSolrClient(); - String COLLNAME = "testSuggestionsWithPayload.COLL"; - CollectionAdminResponse adminResponse = CollectionAdminRequest.createCollection(COLLNAME, CONFIGSET_NAME, 1, 2) - .process(solrClient); - cluster.waitForActiveCollection(COLLNAME, 1, 2); - DocCollection collection = solrClient.getClusterStateProvider().getCollection(COLLNAME); - Replica aReplica = collection.getReplicas().get(0); - - String configPayload = "{\n" + - " 'cluster-policy': [{'replica': 0, 'node': '_NODE'}]\n" + - "}"; - configPayload = configPayload.replaceAll("_NODE", aReplica.getNodeName()); - @SuppressWarnings({"rawtypes"}) - SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, "/suggestions", configPayload); - NamedList response = solrClient.request(req); - assertFalse(((Collection) response.get("suggestions")).isEmpty()); - String replicaName = response._getStr("suggestions[0]/operation/command/move-replica/replica", null); - boolean[] passed = new boolean[]{false}; - collection.forEachReplica((s, replica) -> { - if (replica.getName().equals(replicaName) && replica.getNodeName().equals(aReplica.getNodeName())) { - passed[0] = true; - } - }); - assertTrue(passed[0]); - - req = AutoScalingRequest.create(SolrRequest.METHOD.POST, "/suggestions", configPayload, new MapSolrParams(Collections.singletonMap("type", repair.name()))); - response = solrClient.request(req); - assertTrue(((Collection) response.get("suggestions")).isEmpty()); - - CollectionAdminRequest.deleteCollection(COLLNAME) - .process(cluster.getSolrClient()); - } - public void testDiagnosticsWithPayload() throws Exception { - CloudSolrClient solrClient = cluster.getSolrClient(); - String COLLNAME = "testDiagnosticsWithPayload.COLL"; - CollectionAdminResponse adminResponse = CollectionAdminRequest.createCollection(COLLNAME, CONFIGSET_NAME, 1, 2) - .process(solrClient); - cluster.waitForActiveCollection(COLLNAME, 1, 2); - DocCollection collection = solrClient.getClusterStateProvider().getCollection(COLLNAME); - Replica aReplica = collection.getReplicas().get(0); - - String configPayload = "{\n" + - " 'cluster-policy': [{'replica': 0, 'node': '_NODE'}]\n" + - "}"; - configPayload = configPayload.replaceAll("_NODE", aReplica.getNodeName()); - @SuppressWarnings({"rawtypes"}) - SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, "/diagnostics", configPayload); - NamedList response = solrClient.request(req); - assertEquals(response._getStr("diagnostics/violations[0]/node",null),response._getStr("diagnostics/violations[0]/node",null)); - CollectionAdminRequest.deleteCollection(COLLNAME) - .process(cluster.getSolrClient()); - } - - @Test - @SuppressWarnings({"unchecked"}) - public void testSuspendTrigger() throws Exception { - CloudSolrClient solrClient = cluster.getSolrClient(); - String suspendEachCommand = "{\n" + - "\t\"suspend-trigger\" : {\n" + - "\t\t\"name\" : \"" + Policy.EACH + "\"\n" + - "\t}\n" + - "}"; - String resumeEachCommand = "{\n" + - "\t\"resume-trigger\" : {\n" + - "\t\t\"name\" : \"" + Policy.EACH + "\"\n" + - "\t}\n" + - "}"; - // these should be no-ops because there are no triggers, and it should succeed - @SuppressWarnings({"rawtypes"}) - SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, suspendEachCommand); - NamedList response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - assertEquals(response.get("changed").toString(), "[]"); - req = AutoScalingRequest.create(SolrRequest.METHOD.POST, resumeEachCommand); - response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - assertEquals(response.get("changed").toString(), "[]"); - - String setTriggerCommand = "{" + - "'set-trigger' : {" + - "'name' : 'node_lost_trigger'," + - "'event' : 'nodeLost'," + - "'waitFor' : '10m'," + - "'enabled' : true}}"; - req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand); - response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - - setTriggerCommand = "{" + - "'set-trigger' : {" + - "'name' : 'node_added_trigger'," + - "'event' : 'nodeAdded'," + - "'waitFor' : '10m'," + - "'enabled' : true" + - "}" + - "}"; - req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand); - response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - - String suspendTriggerCommand = "{\n" + - "\t\"suspend-trigger\" : {\n" + - "\t\t\"name\" : \"node_lost_trigger\"\n" + - "\t}\n" + - "}"; - req = AutoScalingRequest.create(SolrRequest.METHOD.POST, suspendTriggerCommand); - response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - assertEquals(response.get("changed").toString(), "[node_lost_trigger]"); - - Stat stat = new Stat(); - byte[] data = zkClient().getData(SOLR_AUTOSCALING_CONF_PATH, null, stat, true); - ZkNodeProps loaded = ZkNodeProps.load(data); - Map triggers = (Map) loaded.get("triggers"); - assertNotNull(triggers); - assertEquals(2, countNotImplicitTriggers(triggers)); - assertTrue(triggers.containsKey("node_lost_trigger")); - assertTrue(triggers.containsKey("node_added_trigger")); - Map nodeLostTrigger = (Map) triggers.get("node_lost_trigger"); - assertEquals(4, nodeLostTrigger.size()); - assertEquals("false", nodeLostTrigger.get("enabled").toString()); - Map nodeAddedTrigger = (Map) triggers.get("node_added_trigger"); - assertEquals(4, nodeAddedTrigger.size()); - assertEquals("true", nodeAddedTrigger.get("enabled").toString()); - - suspendTriggerCommand = "{" + - "'suspend-trigger' : {" + - "'name' : '" + Policy.EACH + "'" + - "}" + - "}"; - req = AutoScalingRequest.create(SolrRequest.METHOD.POST, suspendTriggerCommand); - response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - List changed = (List) response.get("changed"); - assertEquals(1, changed.size()); - assertTrue(changed.contains("node_added_trigger")); - data = zkClient().getData(SOLR_AUTOSCALING_CONF_PATH, null, null, true); - loaded = ZkNodeProps.load(data); - triggers = (Map) loaded.get("triggers"); - assertNotNull(triggers); - assertEquals(2, countNotImplicitTriggers(triggers)); - nodeLostTrigger = (Map) triggers.get("node_lost_trigger"); - assertEquals(4, nodeLostTrigger.size()); - assertEquals("false", nodeLostTrigger.get("enabled").toString()); - nodeAddedTrigger = (Map) triggers.get("node_added_trigger"); - assertEquals(4, nodeAddedTrigger.size()); - assertEquals("false", nodeAddedTrigger.get("enabled").toString()); - - String resumeTriggerCommand = "{" + - "'resume-trigger' : {" + - "'name' : 'node_added_trigger'" + - "}" + - "}"; - req = AutoScalingRequest.create(SolrRequest.METHOD.POST, resumeTriggerCommand); - response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - changed = (List) response.get("changed"); - assertEquals(1, changed.size()); - assertTrue(changed.contains("node_added_trigger")); - data = zkClient().getData(SOLR_AUTOSCALING_CONF_PATH, null, null, true); - loaded = ZkNodeProps.load(data); - triggers = (Map) loaded.get("triggers"); - assertNotNull(triggers); - assertEquals(2, countNotImplicitTriggers(triggers)); - nodeLostTrigger = (Map) triggers.get("node_lost_trigger"); - assertEquals(4, nodeLostTrigger.size()); - assertEquals("false", nodeLostTrigger.get("enabled").toString()); - nodeAddedTrigger = (Map) triggers.get("node_added_trigger"); - assertEquals(4, nodeAddedTrigger.size()); - assertEquals("true", nodeAddedTrigger.get("enabled").toString()); - - resumeTriggerCommand = "{" + - "'resume-trigger' : {" + - "'name' : '" + Policy.EACH + "'" + - "}" + - "}"; - req = AutoScalingRequest.create(SolrRequest.METHOD.POST, resumeTriggerCommand); - response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - changed = (List) response.get("changed"); - assertEquals(1, changed.size()); - assertTrue(changed.contains("node_lost_trigger")); - data = zkClient().getData(SOLR_AUTOSCALING_CONF_PATH, null, null, true); - loaded = ZkNodeProps.load(data); - triggers = (Map) loaded.get("triggers"); - assertNotNull(triggers); - assertEquals(2, countNotImplicitTriggers(triggers)); - nodeLostTrigger = (Map) triggers.get("node_lost_trigger"); - assertEquals(4, nodeLostTrigger.size()); - assertEquals("true", nodeLostTrigger.get("enabled").toString()); - nodeAddedTrigger = (Map) triggers.get("node_added_trigger"); - assertEquals(4, nodeAddedTrigger.size()); - assertEquals("true", nodeAddedTrigger.get("enabled").toString()); - - suspendTriggerCommand = "{" + - "'suspend-trigger' : {" + - "'name' : 'node_lost_trigger'," + - "'timeout' : '1h'" + - "}" + - "}"; - req = AutoScalingRequest.create(SolrRequest.METHOD.POST, suspendTriggerCommand); - response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - changed = (List) response.get("changed"); - assertEquals(1, changed.size()); - assertTrue(changed.contains("node_lost_trigger")); - data = zkClient().getData(SOLR_AUTOSCALING_CONF_PATH, null, null, true); - loaded = ZkNodeProps.load(data); - triggers = (Map) loaded.get("triggers"); - assertNotNull(triggers); - assertEquals(2, countNotImplicitTriggers(triggers)); - nodeLostTrigger = (Map) triggers.get("node_lost_trigger"); - assertEquals(5, nodeLostTrigger.size()); - assertEquals("false", nodeLostTrigger.get("enabled").toString()); - assertTrue(nodeLostTrigger.containsKey("resumeAt")); - } - - @Test - @SuppressWarnings({"unchecked"}) - public void test() throws Exception { - CloudSolrClient solrClient = cluster.getSolrClient(); - String setTriggerCommand = "{" + - "'set-trigger' : {" + - "'name' : 'node_lost_trigger'," + - "'event' : 'nodeLost'," + - "'waitFor' : '10m'," + - "'enabled' : true," + - "'actions' : [" + - "{" + - "'name' : 'compute_plan'," + - "'class' : 'solr.ComputePlanAction'" + - "}]}}"; - @SuppressWarnings({"rawtypes"}) - SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand); - - NamedList response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - - byte[] data = zkClient().getData(SOLR_AUTOSCALING_CONF_PATH, null, null, true); - ZkNodeProps loaded = ZkNodeProps.load(data); - Map triggers = (Map) loaded.get("triggers"); - assertNotNull(triggers); - assertEquals(1, countNotImplicitTriggers(triggers)); - assertTrue(triggers.containsKey("node_lost_trigger")); - Map nodeLostTrigger = (Map) triggers.get("node_lost_trigger"); - assertEquals(4, nodeLostTrigger.size()); - List> actions = (List>) nodeLostTrigger.get("actions"); - assertNotNull(actions); - assertEquals(1, actions.size()); - assertEquals("600", nodeLostTrigger.get("waitFor").toString()); - - setTriggerCommand = "{" + - "'set-trigger' : {" + - "'name' : 'node_lost_trigger'," + - "'event' : 'nodeLost'," + - "'waitFor' : '20m'," + - "'enabled' : false" + - "}}"; - req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand); - response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - data = zkClient().getData(SOLR_AUTOSCALING_CONF_PATH, null, null, true); - loaded = ZkNodeProps.load(data); - triggers = (Map) loaded.get("triggers"); - assertNotNull(triggers); - assertEquals(1, countNotImplicitTriggers(triggers)); - assertTrue(triggers.containsKey("node_lost_trigger")); - nodeLostTrigger = (Map) triggers.get("node_lost_trigger"); - assertEquals(4, nodeLostTrigger.size()); - assertEquals("1200", nodeLostTrigger.get("waitFor").toString()); - assertEquals("false", nodeLostTrigger.get("enabled").toString()); - actions = (List>) nodeLostTrigger.get("actions"); - assertNotNull(actions); - assertEquals(2, actions.size()); - - String setListenerCommand = "{" + - "'set-listener' : " + - "{" + - "'name' : 'xyz'," + - "'trigger' : 'node_lost_trigger'," + - "'stage' : ['STARTED','ABORTED','SUCCEEDED']," + - "'beforeAction' : 'execute_plan'," + - "'class' : 'org.apache.solr.cloud.autoscaling.HttpTriggerListener'," + - "'url' : 'http://xyz.com/on_node_lost?node={$LOST_NODE_NAME}'" + - "}" + - "}"; - req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setListenerCommand); - response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - data = zkClient().getData(SOLR_AUTOSCALING_CONF_PATH, null, null, true); - loaded = ZkNodeProps.load(data); - Map listeners = (Map) loaded.get("listeners"); - assertNotNull(listeners); - assertEquals(2, listeners.size()); - assertTrue(listeners.containsKey("xyz")); - Map xyzListener = (Map) listeners.get("xyz"); - assertEquals(6, xyzListener.size()); - assertEquals("org.apache.solr.cloud.autoscaling.HttpTriggerListener", xyzListener.get("class").toString()); - - String removeTriggerCommand = "{" + - "'remove-trigger' : {" + - "'name' : 'node_lost_trigger'" + - "}" + - "}"; - req = AutoScalingRequest.create(SolrRequest.METHOD.POST, removeTriggerCommand); - try { - solrClient.request(req); - fail("expected exception"); - } catch (BaseHttpSolrClient.RemoteExecutionException e) { - // expected - assertTrue(String.valueOf(getObjectByPath(e.getMetaData(), - false, "error/details[0]/errorMessages[0]")).contains("Cannot remove trigger: node_lost_trigger because it has active listeners: [")); - } - - String removeListenerCommand = "{\n" + - "\t\"remove-listener\" : {\n" + - "\t\t\"name\" : \"xyz\"\n" + - "\t}\n" + - "}"; - req = AutoScalingRequest.create(SolrRequest.METHOD.POST, removeListenerCommand); - response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - data = zkClient().getData(SOLR_AUTOSCALING_CONF_PATH, null, null, true); - loaded = ZkNodeProps.load(data); - listeners = (Map) loaded.get("listeners"); - assertNotNull(listeners); - assertEquals(1, listeners.size()); - - removeTriggerCommand = "{" + - "'remove-trigger' : {" + - "'name' : 'node_lost_trigger'" + - "}" + - "}"; - req = AutoScalingRequest.create(SolrRequest.METHOD.POST, removeTriggerCommand); - response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - data = zkClient().getData(SOLR_AUTOSCALING_CONF_PATH, null, null, true); - loaded = ZkNodeProps.load(data); - triggers = (Map) loaded.get("triggers"); - assertNotNull(triggers); - assertEquals(0, countNotImplicitTriggers(triggers)); - - setListenerCommand = "{" + - "'set-listener' : {" + - "'name' : 'xyz'," + - "'trigger' : 'node_lost_trigger'," + - "'stage' : ['STARTED','ABORTED','SUCCEEDED']," + - "'beforeAction' : 'execute_plan'," + - "'class' : 'org.apache.solr.cloud.autoscaling.AutoScaling$HttpTriggerListener'," + - "'url' : 'http://xyz.com/on_node_lost?node={$LOST_NODE_NAME}'" + - "}" + - "}"; - req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setListenerCommand); - try { - solrClient.request(req); - fail("should have thrown Exception"); - } catch (BaseHttpSolrClient.RemoteSolrException e) { - // expected - assertTrue(String.valueOf(getObjectByPath(((BaseHttpSolrClient.RemoteExecutionException) e).getMetaData(), - false, "error/details[0]/errorMessages[0]")).contains("A trigger with the name node_lost_trigger does not exist")); - } - } - - @Test - public void testErrorHandling() throws Exception { - CloudSolrClient solrClient = cluster.getSolrClient(); - - String setClusterPolicyCommand = "{" + - " 'set-cluster-policy': [" + - " {'cores':'<10', 'node':'#ANY'}," + - " {'shard': '#EACH', 'node': '#ANY'}," + - " {'nodeRole':'overseer', 'replica':0}" + - " ]" + - "}"; - try { - @SuppressWarnings({"rawtypes"}) - SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setClusterPolicyCommand); - solrClient.request(req); - fail("expect exception"); - } catch (BaseHttpSolrClient.RemoteExecutionException e) { - String message = String.valueOf(getObjectByPath(e.getMetaData(), true, "error/details[0]/errorMessages[0]")); - assertTrue(message.contains("replica is required in")); - } - - } - - @Test - public void testValidation() throws Exception { - CloudSolrClient solrClient = cluster.getSolrClient(); - - // unknown trigger properties - String setTriggerCommand = "{" + - "'set-trigger' : {" + - "'name' : 'node_lost_trigger'," + - "'event' : 'nodeLost'," + - "'waitFor' : '10m'," + - "'enabled' : true," + - "'foo': 'bar'," + - "'actions' : [" + - "{" + - "'name' : 'compute_plan'," + - "'class' : 'solr.ComputePlanAction'" + - "}]}}"; - @SuppressWarnings({"rawtypes"}) - SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand); - - try { - solrClient.request(req); - fail("should have thrown Exception"); - } catch (BaseHttpSolrClient.RemoteSolrException e) { - // expected - assertTrue(String.valueOf(getObjectByPath(((BaseHttpSolrClient.RemoteExecutionException) e).getMetaData(), - false, "error/details[0]/errorMessages[0]")).contains("foo=unknown property")); - } - - // invalid trigger properties - setTriggerCommand = "{" + - "'set-trigger' : {" + - "'name' : 'search_rate_trigger'," + - "'event' : 'searchRate'," + - "'waitFor' : '10m'," + - "'enabled' : true," + - "'aboveRate': 'foo'," + - "'actions' : [" + - "{" + - "'name' : 'compute_plan'," + - "'class' : 'solr.ComputePlanAction'" + - "}]}}"; - req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand); - - try { - solrClient.request(req); - fail("should have thrown Exception"); - } catch (BaseHttpSolrClient.RemoteSolrException e) { - // expected - assertTrue(String.valueOf(getObjectByPath(((BaseHttpSolrClient.RemoteExecutionException) e).getMetaData(), - false, "error/details[0]/errorMessages[0]")).contains("aboveRate=Invalid configuration value: 'foo'")); - } - - // unknown trigger action properties - setTriggerCommand = "{" + - "'set-trigger' : {" + - "'name' : 'node_lost_trigger'," + - "'event' : 'nodeLost'," + - "'waitFor' : '10m'," + - "'enabled' : true," + - "'actions' : [" + - "{" + - "'name' : 'compute_plan'," + - "'foo' : 'bar'," + - "'class' : 'solr.ComputePlanAction'" + - "}]}}"; - req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand); - - try { - solrClient.request(req); - fail("should have thrown Exception"); - } catch (BaseHttpSolrClient.RemoteSolrException e) { - // expected - assertTrue(String.valueOf(getObjectByPath(((BaseHttpSolrClient.RemoteExecutionException) e).getMetaData(), - false, "error/details[0]/errorMessages[0]")).contains("foo=unknown property")); - } - - // unknown trigger listener properties - setTriggerCommand = "{" + - "'set-trigger' : {" + - "'name' : 'node_lost_trigger'," + - "'event' : 'nodeLost'," + - "'waitFor' : '10m'," + - "'enabled' : true," + - "'actions' : [" + - "{" + - "'name' : 'compute_plan'," + - "'class' : 'solr.ComputePlanAction'" + - "}]}}"; - req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand); - - NamedList response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - - String setListenerCommand = "{" + - "'set-listener' : " + - "{" + - "'name' : 'xyz'," + - "'trigger' : 'node_lost_trigger'," + - "'stage' : ['STARTED','ABORTED','SUCCEEDED']," + - "'foo' : 'bar'," + - "'beforeAction' : 'execute_plan'," + - "'class' : 'org.apache.solr.cloud.autoscaling.HttpTriggerListener'," + - "'url' : 'http://xyz.com/on_node_lost?node={$LOST_NODE_NAME}'" + - "}" + - "}"; - req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setListenerCommand); - try { - solrClient.request(req); - fail("should have thrown Exception"); - } catch (BaseHttpSolrClient.RemoteSolrException e) { - // expected - assertTrue(String.valueOf(getObjectByPath(((BaseHttpSolrClient.RemoteExecutionException) e).getMetaData(), - false, "error/details[0]/errorMessages[0]")).contains("foo=unknown property")); - } - } - - @Test - @SuppressWarnings({"unchecked"}) - public void testPolicyAndPreferences() throws Exception { - CloudSolrClient solrClient = cluster.getSolrClient(); - // add multiple policies - String setPolicyCommand = "{'set-policy': {" + - " 'xyz':[" + - " {'replica':'<2', 'shard': '#EACH', 'node': '#ANY'}," + - " {'nodeRole':'!overseer', 'replica':0}" + - " ]," + - " 'policy1':[" + - " {'cores':'<2', 'node':'#ANY'}," + - " {'replica':'<2', 'shard': '#EACH', 'node': '#ANY'}" + - " ]" + - "}}"; - @SuppressWarnings({"rawtypes"}) - SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setPolicyCommand); - NamedList response = null; - try { - solrClient.request(req); - fail("Adding a policy with 'cores' attribute should not have succeeded."); - } catch (BaseHttpSolrClient.RemoteExecutionException e) { - String message = e.getMetaData()._getStr("error/details[0]/errorMessages[0]",null); - - // expected - assertTrue(message.contains("cores is only allowed in 'cluster-policy'")); - } - - setPolicyCommand = "{'set-policy': {" + - " 'xyz':[" + - " {'replica':'<2', 'shard': '#EACH', 'node': '#ANY'}," + - " {'nodeRole':'!overseer', 'replica':0}" + - " ]," + - " 'policy1':[" + - " {'replica':'<2', 'shard': '#EACH', 'node': '#ANY'}" + - " ]" + - "}}"; - req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setPolicyCommand); - response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - - byte[] data = zkClient().getData(SOLR_AUTOSCALING_CONF_PATH, null, null, true); - ZkNodeProps loaded = ZkNodeProps.load(data); - Map policies = (Map) loaded.get("policies"); - assertNotNull(policies); - assertNotNull(policies.get("xyz")); - assertNotNull(policies.get("policy1")); - - // update default policy - setPolicyCommand = "{'set-policy': {" + - " 'xyz':[" + - " {'replica':'<2', 'shard': '#EACH', 'node': '#ANY'}" + - " ]" + - "}}"; - req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setPolicyCommand); - response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - data = zkClient().getData(SOLR_AUTOSCALING_CONF_PATH, null, null, true); - loaded = ZkNodeProps.load(data); - policies = (Map) loaded.get("policies"); - @SuppressWarnings({"rawtypes"}) - List conditions = (List) policies.get("xyz"); - assertEquals(1, conditions.size()); - - // remove policy - String removePolicyCommand = "{remove-policy : policy1}"; - req = AutoScalingRequest.create(SolrRequest.METHOD.POST, removePolicyCommand); - response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - data = zkClient().getData(SOLR_AUTOSCALING_CONF_PATH, null, null, true); - loaded = ZkNodeProps.load(data); - policies = (Map) loaded.get("policies"); - assertNull(policies.get("policy1")); - - // set preferences - String setPreferencesCommand = "{" + - " 'set-cluster-preferences': [" + - " {'minimize': 'cores', 'precision': 3}," + - " {'maximize': 'freedisk','precision': 100}," + - " {'minimize': 'sysLoadAvg','precision': 10}]" + - "}"; - req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setPreferencesCommand); - response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - data = zkClient().getData(SOLR_AUTOSCALING_CONF_PATH, null, null, true); - loaded = ZkNodeProps.load(data); - @SuppressWarnings({"rawtypes"}) - List preferences = (List) loaded.get("cluster-preferences"); - assertEquals(3, preferences.size()); - - // set preferences - setPreferencesCommand = "{" + - " 'set-cluster-preferences': [" + - " {'minimize': 'sysLoadAvg','precision': 10}]" + - "}"; - req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setPreferencesCommand); - response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - data = zkClient().getData(SOLR_AUTOSCALING_CONF_PATH, null, null, true); - loaded = ZkNodeProps.load(data); - preferences = (List) loaded.get("cluster-preferences"); - assertEquals(1, preferences.size()); - - String setClusterPolicyCommand = "{" + - " 'set-cluster-policy': [" + - " {'cores':'<10', 'node':'#ANY'}," + - " {'replica':'<2', 'shard': '#EACH', 'node': '#ANY'}," + - " {'nodeRole':'!overseer', 'replica':0}" + - " ]" + - "}"; - req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setClusterPolicyCommand); - response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - data = zkClient().getData(SOLR_AUTOSCALING_CONF_PATH, null, null, true); - loaded = ZkNodeProps.load(data); - @SuppressWarnings({"rawtypes"}) - List clusterPolicy = (List) loaded.get("cluster-policy"); - assertNotNull(clusterPolicy); - assertEquals(3, clusterPolicy.size()); - - setClusterPolicyCommand = "{" + - " 'set-cluster-policy': [" + - " {'cores':'<10', 'node':'#ANY'}," + - " {'replica':'<2', 'shard': '#EACH', 'node': '#ANY'}," + - " {'replica':0, put : on-each-node, nodeset:{'nodeRole':'overseer'} }" + - " ]" + - "}"; - req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setClusterPolicyCommand); - response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - data = zkClient().getData(SOLR_AUTOSCALING_CONF_PATH, null, null, true); - loaded = ZkNodeProps.load(data); - clusterPolicy = (List) loaded.get("cluster-policy"); - assertNotNull(clusterPolicy); - assertEquals(3, clusterPolicy.size()); - } - - @Test - // commented out on: 24-Dec-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 17-Aug-2018 - @SuppressWarnings({"unchecked", "rawtypes"}) - public void testReadApi() throws Exception { - CloudSolrClient solrClient = cluster.getSolrClient(); - // first trigger - String setTriggerCommand = "{" + - "'set-trigger' : {" + - "'name' : 'node_added_trigger1'," + - "'event' : 'nodeAdded'," + - "'waitFor' : '0s'," + - "'enabled' : true" + - "}}"; - SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand); - NamedList response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - - String setClusterPolicyCommand = "{" + - " 'set-cluster-policy': [" + - " {'cores':'<10', 'node':'#ANY'}," + - " {'replica':'<3', 'shard': '#EACH', 'node': '#ANY'}]" + - "}"; - req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setClusterPolicyCommand); - response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - - String setPreferencesCommand = "{" + - " 'set-cluster-preferences': [" + - " {'minimize': 'cores', 'precision': 3}," + - " {'maximize': 'freedisk','precision': 100}," + - " {'minimize': 'sysLoadAvg','precision': 10}," + - " {'minimize': 'heapUsage','precision': 10}]" + - "}"; - req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setPreferencesCommand); - response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - - String setPolicyCommand = "{'set-policy': {" + - " 'xyz':[{'replica':'<2', 'shard': '#EACH', 'node': '#ANY'}]," + - " 'policy1':[{'replica':'<2', 'shard': '#EACH', 'node': '#ANY'}]," + - " 'policy2':[{'replica':'<7', 'shard': '#EACH', 'node': '#ANY'}]}}"; - req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setPolicyCommand); - response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - - req = AutoScalingRequest.create(SolrRequest.METHOD.GET, null); - response = solrClient.request(req); - - Map triggers = (Map) response.get("triggers"); - assertNotNull(triggers); - assertEquals(1, countNotImplicitTriggers(triggers)); - assertTrue(triggers.containsKey("node_added_trigger1")); - Map node_added_trigger1 = (Map) triggers.get("node_added_trigger1"); - assertEquals(4, node_added_trigger1.size()); - assertEquals(0L, node_added_trigger1.get("waitFor")); - assertEquals(true, node_added_trigger1.get("enabled")); - assertEquals(2, ((List)node_added_trigger1.get("actions")).size()); - - List clusterPrefs = (List) response.get("cluster-preferences"); - assertNotNull(clusterPrefs); - assertEquals(4, clusterPrefs.size()); - - List clusterPolicy = (List) response.get("cluster-policy"); - assertNotNull(clusterPolicy); - assertEquals(2, clusterPolicy.size()); - - Map policies = (Map) response.get("policies"); - assertNotNull(policies); - assertEquals(3, policies.size()); - assertNotNull(policies.get("xyz")); - assertNotNull(policies.get("policy1")); - - req = AutoScalingRequest.create(SolrRequest.METHOD.GET, "/diagnostics", null); - response = solrClient.request(req); - - Map diagnostics = (Map) response.get("diagnostics"); - List sortedNodes = (List) response._get("diagnostics/sortedNodes", null); - assertNotNull(sortedNodes); - - assertEquals(2, sortedNodes.size()); - for (int i = 0; i < 2; i++) { - Map node = (Map) sortedNodes.get(i); - assertNotNull(node); - assertNotNull(node.get("node")); - assertNotNull(node.get("cores")); - assertEquals(0d, node.get("cores")); - assertNotNull(node.get("freedisk")); - assertNotNull(node.get("replicas")); - assertTrue(node.get("freedisk") instanceof Double); - assertNotNull(node.get("sysLoadAvg")); - assertTrue(node.get("sysLoadAvg") instanceof Double); - assertNotNull(node.get("heapUsage")); - assertTrue(node.get("heapUsage") instanceof Double); - } - - List> violations = (List>) diagnostics.get("violations"); - assertNotNull(violations); - assertEquals(0, violations.size()); - - violations = (List>) diagnostics.get("violations"); - assertNotNull(violations); - assertEquals(0, violations.size()); - - // temporarily increase replica limit in cluster policy so that we can create a collection with 6 replicas - String tempClusterPolicyCommand = "{" + - " 'set-cluster-policy': [" + - " {'cores':'<10', 'node':'#ANY'}," + - " {'replica':'<4', 'shard': '#EACH', 'node': '#ANY'}"+ - " ]" + - "}"; - req = AutoScalingRequest.create(SolrRequest.METHOD.POST, tempClusterPolicyCommand); - response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - - // lets create a collection which violates the rule replicas < 2 - CollectionAdminRequest.Create create = CollectionAdminRequest.Create.createCollection("readApiTestViolations", CONFIGSET_NAME, 1, 6); - CollectionAdminResponse adminResponse = create.process(solrClient); - cluster.waitForActiveCollection("readApiTestViolations", 1, 6); - assertTrue(adminResponse.isSuccess()); - - // reset the original cluster policy - req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setClusterPolicyCommand); - response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - - // get the diagnostics output again - req = AutoScalingRequest.create(SolrRequest.METHOD.GET, "/diagnostics", null); - response = solrClient.request(req); - diagnostics = (Map) response.get("diagnostics"); - sortedNodes = (List) diagnostics.get("sortedNodes"); - assertNotNull(sortedNodes); - - violations = (List>) diagnostics.get("violations"); - assertNotNull(violations); - assertEquals(2, violations.size()); - for (Map violation : violations) { - assertEquals("readApiTestViolations", violation.get("collection")); - assertEquals("shard1", violation.get("shard")); - assertEquals(1.0d, getObjectByPath(violation, true, "violation/delta")); - assertEquals(3l, getObjectByPath(violation, true, "violation/replica/NRT")); - assertNotNull(violation.get("clause")); - } - if (log.isInfoEnabled()) { - log.info("Before starting new jetty ,{}", cluster.getJettySolrRunners() - .stream() - .map(jettySolrRunner -> jettySolrRunner.getNodeName()).collect(Collectors.toList())); - } - JettySolrRunner runner1 = cluster.startJettySolrRunner(); - cluster.waitForAllNodes(30); - if (log.isInfoEnabled()) { - log.info("started new jetty {}", runner1.getNodeName()); - } - - response = waitForResponse(namedList -> { - List l = (List) namedList._get("diagnostics/liveNodes",null); - if (l != null && l.contains(runner1.getNodeName())) return true; - return false; - }, - AutoScalingRequest.create(SolrRequest.METHOD.GET, "/diagnostics", null), - 200, - 20, - runner1.getNodeName() + " could not come up "); - - req = AutoScalingRequest.create(SolrRequest.METHOD.GET, "/suggestions", null); - response = solrClient.request(req); - List l = (List) response.get("suggestions"); - assertNotNull(l); - assertEquals(2, l.size()); - for (int i = 0; i < l.size(); i++) { - Object suggestion = l.get(i); - assertEquals("violation", getObjectByPath(suggestion, true, "type")); - assertEquals("POST", getObjectByPath(suggestion, true, "operation/method")); - assertEquals("/c/readApiTestViolations", getObjectByPath(suggestion, true, "operation/path")); - String node = (String) getObjectByPath(suggestion, true, "operation/command/move-replica/targetNode"); - assertNotNull(node); - assertEquals(runner1.getNodeName(), node); - } - CollectionAdminRequest.deleteCollection("readApiTestViolations") - .process(cluster.getSolrClient()); - } - - @Test - public void testConcurrentUpdates() throws Exception { - int COUNT = 50; - CloudSolrClient solrClient = cluster.getSolrClient(); - CountDownLatch updateLatch = new CountDownLatch(COUNT * 2); - Runnable r = () -> { - for (int i = 0; i < COUNT; i++) { - String setTriggerCommand = "{" + - "'set-trigger' : {" + - "'name' : 'node_added_trigger1'," + - "'event' : 'nodeAdded'," + - "'waitFor' : '0s'," + - "'enabled' : true" + - "}}"; - @SuppressWarnings({"rawtypes"}) - SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand); - NamedList response = null; - try { - response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - } catch (Exception e) { - fail(e.toString()); - } finally { - updateLatch.countDown(); - } - } - }; - Thread t1 = new Thread(r); - Thread t2 = new Thread(r); - t1.start(); - t2.start(); - boolean await = updateLatch.await(60, TimeUnit.SECONDS); - assertTrue("not all updates executed in time, remaining=" + updateLatch.getCount(), await); - @SuppressWarnings({"rawtypes"}) - SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.GET, null); - NamedList response = solrClient.request(req); - - @SuppressWarnings({"rawtypes"}) - Map triggers = (Map) response.get("triggers"); - assertNotNull(triggers); - assertEquals(1, countNotImplicitTriggers(triggers)); - assertTrue(triggers.containsKey("node_added_trigger1")); - @SuppressWarnings({"rawtypes"}) - Map node_added_trigger1 = (Map) triggers.get("node_added_trigger1"); - assertEquals(4, node_added_trigger1.size()); - assertEquals(0L, node_added_trigger1.get("waitFor")); - assertEquals(true, node_added_trigger1.get("enabled")); - assertEquals(2, ((List)node_added_trigger1.get("actions")).size()); - - } - - private int countNotImplicitTriggers(@SuppressWarnings({"rawtypes"})Map triggers) { - if (triggers == null) return 0; - int count = 0; - for (Object trigger : triggers.keySet()) { - if (!trigger.toString().startsWith(".")) count++; - } - return count; - } - - @Test - public void testDeleteUsedPolicy() throws Exception { - CloudSolrClient solrClient = cluster.getSolrClient(); - // add multiple policies - String setPolicyCommand = "{'set-policy': {" + - " 'nodelete':[" + - " {'nodeRole':'overseer', 'replica':0}]}}"; - solrClient.request(AutoScalingRequest.create(SolrRequest.METHOD.POST, setPolicyCommand)); - CollectionAdminRequest.createCollection("COLL1", "conf", 1, 1) - .setPolicy("nodelete") - .process(cluster.getSolrClient()); - String removePolicyCommand = "{remove-policy : nodelete}"; - AutoScalingRequest.create(SolrRequest.METHOD.POST, removePolicyCommand); - try { - solrClient.request(AutoScalingRequest.create(SolrRequest.METHOD.POST, removePolicyCommand)); - fail("should have failed"); - } catch (BaseHttpSolrClient.RemoteExecutionException e) { - assertTrue(String.valueOf(getObjectByPath(e.getMetaData(), true, "error/details[0]/errorMessages[0]")) - .contains("is being used by collection")); - } catch (Exception e) { - fail("Only RemoteExecutionException expected"); - } - solrClient.request(CollectionAdminRequest.deleteCollection("COLL1")); - } - - @Test - public void testSetProperties() throws Exception { - CloudSolrClient solrClient = cluster.getSolrClient(); - String setPropertiesCommand = "{\n" + - "\t\"set-properties\" : {\n" + - "\t\t\"pqr\" : \"abc\"\n" + - "\t}\n" + - "}"; - solrClient.request(AutoScalingRequest.create(SolrRequest.METHOD.POST, setPropertiesCommand)); - @SuppressWarnings({"rawtypes"}) - SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.GET, null); - NamedList response = solrClient.request(req); - @SuppressWarnings({"rawtypes"}) - Map properties = (Map) response.get("properties"); - assertNotNull(properties); - assertEquals(1, properties.size()); - assertEquals("abc", properties.get("pqr")); - - setPropertiesCommand = "{\n" + - "\t\"set-properties\" : {\n" + - "\t\t\"xyz\" : 123\n" + - "\t}\n" + - "}"; - solrClient.request(AutoScalingRequest.create(SolrRequest.METHOD.POST, setPropertiesCommand)); - req = AutoScalingRequest.create(SolrRequest.METHOD.GET, null); - response = solrClient.request(req); - properties = (Map) response.get("properties"); - assertNotNull(properties); - assertEquals(2, properties.size()); - assertEquals("abc", properties.get("pqr")); - assertEquals(123L, properties.get("xyz")); - - setPropertiesCommand = "{\n" + - "\t\"set-properties\" : {\n" + - "\t\t\"xyz\" : 456\n" + - "\t}\n" + - "}"; - solrClient.request(AutoScalingRequest.create(SolrRequest.METHOD.POST, setPropertiesCommand)); - req = AutoScalingRequest.create(SolrRequest.METHOD.GET, null); - response = solrClient.request(req); - properties = (Map) response.get("properties"); - assertNotNull(properties); - assertEquals(2, properties.size()); - assertEquals("abc", properties.get("pqr")); - assertEquals(456L, properties.get("xyz")); - - setPropertiesCommand = "{\n" + - "\t\"set-properties\" : {\n" + - "\t\t\"xyz\" : null\n" + - "\t}\n" + - "}"; - solrClient.request(AutoScalingRequest.create(SolrRequest.METHOD.POST, setPropertiesCommand)); - req = AutoScalingRequest.create(SolrRequest.METHOD.GET, null); - response = solrClient.request(req); - properties = (Map) response.get("properties"); - assertNotNull(properties); - assertEquals(1, properties.size()); - assertEquals("abc", properties.get("pqr")); - - setPropertiesCommand = "{\n" + - "\t\"set-properties\" : {\n" + - "\t\t\"" + AutoScalingParams.TRIGGER_SCHEDULE_DELAY_SECONDS + "\" : 5\n" + - "\t\t\"" + AutoScalingParams.TRIGGER_COOLDOWN_PERIOD_SECONDS + "\" : 10\n" + - "\t\t\"" + AutoScalingParams.TRIGGER_CORE_POOL_SIZE + "\" : 10\n" + - "\t\t\"" + AutoScalingParams.ACTION_THROTTLE_PERIOD_SECONDS + "\" : 5\n" + - "\t}\n" + - "}"; - solrClient.request(AutoScalingRequest.create(SolrRequest.METHOD.POST, setPropertiesCommand)); - req = AutoScalingRequest.create(SolrRequest.METHOD.GET, null); - response = solrClient.request(req); - properties = (Map) response.get("properties"); - assertNotNull(properties); - assertEquals(5, properties.size()); - assertEquals("abc", properties.get("pqr")); - assertEquals(5L, properties.get(AutoScalingParams.TRIGGER_SCHEDULE_DELAY_SECONDS)); - assertEquals(10L, properties.get(AutoScalingParams.TRIGGER_COOLDOWN_PERIOD_SECONDS)); - assertEquals(10L, properties.get(AutoScalingParams.TRIGGER_CORE_POOL_SIZE)); - assertEquals(5L, properties.get(AutoScalingParams.ACTION_THROTTLE_PERIOD_SECONDS)); - } - - public void testUpdatePolicy() throws IOException, SolrServerException { - CloudSolrClient solrClient = cluster.getSolrClient(); - String setPropertiesCommand = "{'set-cluster-policy': [" + - "{'cores': '<4','node': '#ANY'}]}"; - solrClient.request(AutoScalingRequest.create(SolrRequest.METHOD.POST, setPropertiesCommand)); - @SuppressWarnings({"rawtypes"}) - SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.GET, null); - NamedList response = solrClient.request(req); - assertEquals("<4", response._get("cluster-policy[0]/cores", null)); - assertEquals("#ANY", response._get("cluster-policy[0]/node", null)); - setPropertiesCommand = "{'set-cluster-policy': [" + - "{'cores': '<3','node': '#ANY'}]}"; - solrClient.request(AutoScalingRequest.create(SolrRequest.METHOD.POST, setPropertiesCommand)); - req = AutoScalingRequest.create(SolrRequest.METHOD.GET, null); - response = solrClient.request(req); - assertEquals("<3", response._get("cluster-policy[0]/cores", null)); - assertEquals("#ANY", response._get("cluster-policy[0]/node", null)); - - } -} diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/CapturedEvent.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/CapturedEvent.java deleted file mode 100644 index d5c312757b4..00000000000 --- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/CapturedEvent.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.cloud.autoscaling; - -import java.util.HashMap; -import java.util.Map; - -import org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig; -import org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventProcessorStage; - -/** - * - */ -public class CapturedEvent { - public final AutoScalingConfig.TriggerListenerConfig config; - public final TriggerEventProcessorStage stage; - public final String actionName; - public final TriggerEvent event; - public final String message; - public final Map context = new HashMap<>(); - public final long timestamp; - - public CapturedEvent(long timestamp, ActionContext context, AutoScalingConfig.TriggerListenerConfig config, TriggerEventProcessorStage stage, String actionName, - TriggerEvent event, String message) { - if (context != null) { - context._forEachEntry((o, o2) -> CapturedEvent.this.context.put((String) o, o2)); - TriggerEvent.fixOps("properties." + TriggerEvent.REQUESTED_OPS, this.context); - TriggerEvent.fixOps("properties." + TriggerEvent.UNSUPPORTED_OPS, this.context); - } - this.config = config; - this.stage = stage; - this.actionName = actionName; - this.event = event; - this.message = message; - this.timestamp = timestamp; - } - - @Override - public String toString() { - return "CapturedEvent{" + - "timestamp=" + timestamp + - ", stage=" + stage + - ", actionName='" + actionName + '\'' + - ", event=" + event + - ", context=" + context + - ", config=" + config + - ", message='" + message + '\'' + - '}'; - } -} diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/ComputePlanActionTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/ComputePlanActionTest.java deleted file mode 100644 index 325b1cf97f8..00000000000 --- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/ComputePlanActionTest.java +++ /dev/null @@ -1,794 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.cloud.autoscaling; - -import org.apache.lucene.util.LuceneTestCase; -import org.apache.solr.client.solrj.SolrRequest; -import org.apache.solr.client.solrj.cloud.NodeStateProvider; -import org.apache.solr.client.solrj.cloud.SolrCloudManager; -import org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventType; -import org.apache.solr.client.solrj.embedded.JettySolrRunner; -import org.apache.solr.client.solrj.impl.CloudSolrClient; -import org.apache.solr.client.solrj.impl.SolrClientNodeStateProvider; -import org.apache.solr.client.solrj.request.CollectionAdminRequest; -import org.apache.solr.cloud.CloudTestUtils.AutoScalingRequest; -import org.apache.solr.cloud.SolrCloudTestCase; -import org.apache.solr.common.cloud.ClusterState; -import org.apache.solr.common.cloud.DocCollection; -import org.apache.solr.common.cloud.Replica; -import org.apache.solr.common.cloud.ZkStateReader; -import org.apache.solr.common.cloud.rule.ImplicitSnitch; -import org.apache.solr.common.params.AutoScalingParams; -import org.apache.solr.common.params.CollectionParams; -import org.apache.solr.common.params.SolrParams; -import org.apache.solr.common.util.NamedList; -import org.apache.solr.common.util.Pair; -import org.apache.solr.common.util.Utils; -import org.apache.solr.core.SolrResourceLoader; -import org.apache.solr.util.LogLevel; -import org.junit.*; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.lang.invoke.MethodHandles; -import java.util.*; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicReference; - -import static java.nio.charset.StandardCharsets.UTF_8; -import static org.apache.solr.common.params.CollectionParams.CollectionAction.MOVEREPLICA; - -/** - * Test for {@link ComputePlanAction} - */ -@LogLevel("org.apache.solr.cloud.autoscaling=DEBUG;org.apache.solr.cloud.Overseer=DEBUG;org.apache.solr.cloud.overseer=DEBUG;org.apache.solr.client.solrj.impl.SolrClientDataProvider=DEBUG;") -public class ComputePlanActionTest extends SolrCloudTestCase { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - private static final AtomicBoolean fired = new AtomicBoolean(false); - private static final int NODE_COUNT = 1; - private static CountDownLatch triggerFiredLatch = new CountDownLatch(1); - @SuppressWarnings({"rawtypes"}) - private static final AtomicReference actionContextPropsRef = new AtomicReference<>(); - private static final AtomicReference eventRef = new AtomicReference<>(); - private static SolrCloudManager cloudManager; - - @BeforeClass - public static void setupCluster() throws Exception { - configureCluster(NODE_COUNT) - .addConfig("conf", configset("cloud-minimal")) - .configure(); - } - - @Before - public void setUp() throws Exception { - super.setUp(); - - // remove everything from autoscaling.json in ZK - zkClient().setData(ZkStateReader.SOLR_AUTOSCALING_CONF_PATH, "{}".getBytes(UTF_8), true); - - if (cluster.getJettySolrRunners().size() > NODE_COUNT) { - // stop some to get to original state - int numJetties = cluster.getJettySolrRunners().size(); - for (int i = 0; i < numJetties - NODE_COUNT; i++) { - JettySolrRunner randomJetty = cluster.getRandomJetty(random()); - List jettySolrRunners = cluster.getJettySolrRunners(); - for (int i1 = 0; i1 < jettySolrRunners.size(); i1++) { - JettySolrRunner jettySolrRunner = jettySolrRunners.get(i1); - if (jettySolrRunner == randomJetty) { - JettySolrRunner j = cluster.stopJettySolrRunner(i1); - cluster.waitForJettyToStop(j); - break; - } - } - } - } - - cluster.deleteAllCollections(); - - CloudSolrClient solrClient = cluster.getSolrClient(); - - String setClusterPolicyCommand = "{" + - " 'set-cluster-policy': [" + - " {'cores':'<10', 'node':'#ANY'}," + - " {'replica':'<2', 'shard': '#EACH', 'node': '#ANY'}," + - " {'nodeRole':'overseer', 'replica':0}" + - " ]" + - "}"; - @SuppressWarnings({"rawtypes"}) - SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setClusterPolicyCommand); - NamedList response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - - String setClusterPreferencesCommand = "{" + - "'set-cluster-preferences': [" + - "{'minimize': 'cores'}," + - "{'maximize': 'freedisk','precision': 100}]" + - "}"; - req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setClusterPreferencesCommand); - response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - - cloudManager = cluster.getJettySolrRunner(0).getCoreContainer().getZkController().getSolrCloudManager(); - deleteChildrenRecursively(ZkStateReader.SOLR_AUTOSCALING_EVENTS_PATH); - deleteChildrenRecursively(ZkStateReader.SOLR_AUTOSCALING_TRIGGER_STATE_PATH); - deleteChildrenRecursively(ZkStateReader.SOLR_AUTOSCALING_NODE_LOST_PATH); - deleteChildrenRecursively(ZkStateReader.SOLR_AUTOSCALING_NODE_ADDED_PATH); - - reset(); - } - - private void reset() { - fired.set(false); - triggerFiredLatch = new CountDownLatch(1); - actionContextPropsRef.set(null); - eventRef.set(null); - AssertingTriggerAction.expectedNode = null; - } - - private void deleteChildrenRecursively(String path) throws Exception { - cloudManager.getDistribStateManager().removeRecursively(path, true, false); - } - - @After - public void printState() throws Exception { - log.debug("-------------_ FINAL STATE --------------"); - SolrCloudManager cloudManager = cluster.getJettySolrRunner(0).getCoreContainer().getZkController().getSolrCloudManager(); - for (String node: cloudManager.getClusterStateProvider().getLiveNodes()) { - Map values = cloudManager.getNodeStateProvider().getNodeValues(node, ImplicitSnitch.tags); - if (log.isDebugEnabled()) { - log.debug("* Node values: {}\n{}", node, Utils.toJSONString(values)); - } - } - if (log.isDebugEnabled()) { - log.debug("* Live nodes: {}", cloudManager.getClusterStateProvider().getLiveNodes()); - } - ClusterState state = cloudManager.getClusterStateProvider().getClusterState(); - if (log.isDebugEnabled()) { - state.forEachCollection(coll -> log.debug("* Collection {} state: {}", coll.getName(), coll)); - } - } - - @AfterClass - public static void cleanUpAfterClass() throws Exception { - cloudManager = null; - } - - @Test - @LuceneTestCase.AwaitsFix(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") - public void testNodeLost() throws Exception { - // let's start a node so that we have at least two - JettySolrRunner runner = cluster.startJettySolrRunner(); - String node = runner.getNodeName(); - AssertingTriggerAction.expectedNode = node; - - CloudSolrClient solrClient = cluster.getSolrClient(); - String setTriggerCommand = "{" + - "'set-trigger' : {" + - "'name' : 'node_lost_trigger'," + - "'event' : 'nodeLost'," + - "'waitFor' : '7s'," + - "'enabled' : true," + - "'actions' : [{'name':'compute_plan', 'class' : 'solr.ComputePlanAction'}," + - "{'name':'test','class':'" + ComputePlanActionTest.AssertingTriggerAction.class.getName() + "'}]" + - "}}"; - @SuppressWarnings({"rawtypes"}) - SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand); - NamedList response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - - CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection("testNodeLost", - "conf",1, 2); - create.process(solrClient); - - waitForState("Timed out waiting for replicas of new collection to be active", - "testNodeLost", clusterShape(1, 2)); - - ClusterState clusterState = cluster.getSolrClient().getZkStateReader().getClusterState(); - DocCollection collection = clusterState.getCollection("testNodeLost"); - List replicas = collection.getReplicas(node); - assertNotNull(replicas); - assertFalse(replicas.isEmpty()); - - // start another node because because when the other node goes away, the cluster policy requires only - // 1 replica per node and none on the overseer - JettySolrRunner node2 = cluster.startJettySolrRunner(); - cluster.waitForAllNodes(30); - assertTrue(node2.getNodeName() + "is not live yet", cluster.getSolrClient().getZkStateReader().getClusterState().liveNodesContain(node2.getNodeName()) ); - - // stop the original node - for (int i = 0; i < cluster.getJettySolrRunners().size(); i++) { - JettySolrRunner jettySolrRunner = cluster.getJettySolrRunners().get(i); - if (jettySolrRunner == runner) { - cluster.stopJettySolrRunner(i); - break; - } - } - log.info("Stopped_node : {}", node); - cluster.waitForAllNodes(30); - - assertTrue("Trigger was not fired even after 10 seconds", triggerFiredLatch.await(10, TimeUnit.SECONDS)); - assertTrue(fired.get()); - @SuppressWarnings({"rawtypes"}) - Map context = actionContextPropsRef.get(); - assertNotNull(context); - @SuppressWarnings({"unchecked", "rawtypes"}) - List operations = (List) context.get("operations"); - assertNotNull("The operations computed by ComputePlanAction should not be null , "+ getNodeStateProviderState() + eventRef.get(), operations); - assertEquals("ComputePlanAction should have computed exactly 1 operation", 1, operations.size()); - @SuppressWarnings({"rawtypes"}) - SolrRequest solrRequest = operations.get(0); - SolrParams params = solrRequest.getParams(); - assertEquals("Expected MOVEREPLICA action after adding node", MOVEREPLICA, CollectionParams.CollectionAction.get(params.get("action"))); - String replicaToBeMoved = params.get("replica"); - assertEquals("Unexpected node in computed operation", replicas.get(0).getName(), replicaToBeMoved); - - // shutdown the extra node that we had started - for (int i = 0; i < cluster.getJettySolrRunners().size(); i++) { - JettySolrRunner jettySolrRunner = cluster.getJettySolrRunners().get(i); - if (jettySolrRunner == node2) { - JettySolrRunner j = cluster.stopJettySolrRunner(i); - cluster.waitForJettyToStop(j); - break; - } - } - } - static String getNodeStateProviderState() { - String result = "SolrClientNodeStateProvider.DEBUG"; - if(SolrClientNodeStateProvider.INST != null) { - result+= Utils.toJSONString(SolrClientNodeStateProvider.INST); - } - return result; - - } - - // commented out on: 24-Dec-2018 @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2-Aug-2018 - public void testNodeWithMultipleReplicasLost() throws Exception { - // start 3 more nodes - cluster.startJettySolrRunner(); - cluster.startJettySolrRunner(); - cluster.startJettySolrRunner(); - - cluster.waitForAllNodes(30); - - CloudSolrClient solrClient = cluster.getSolrClient(); - String setTriggerCommand = "{" + - "'set-trigger' : {" + - "'name' : 'node_lost_trigger'," + - "'event' : 'nodeLost'," + - "'waitFor' : '1s'," + - "'enabled' : true," + - "'actions' : [{'name':'compute_plan', 'class' : 'solr.ComputePlanAction'}," + - "{'name':'test','class':'" + ComputePlanActionTest.AssertingTriggerAction.class.getName() + "'}]" + - "}}"; - @SuppressWarnings({"rawtypes"}) - SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand); - NamedList response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - - CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection("testNodeWithMultipleReplicasLost", - "conf", 2, 3); - create.process(solrClient); - - cluster.waitForActiveCollection("testNodeWithMultipleReplicasLost", 2, 6); - - waitForState("Timed out waiting for replicas of new collection to be active", - "testNodeWithMultipleReplicasLost", clusterShape(2, 6)); - - ClusterState clusterState = cluster.getSolrClient().getZkStateReader().getClusterState(); - DocCollection docCollection = clusterState.getCollection("testNodeWithMultipleReplicasLost"); - - // lets find a node with at least 2 replicas - String stoppedNodeName = null; - List replicasToBeMoved = null; - for (int i = 0; i < cluster.getJettySolrRunners().size(); i++) { - JettySolrRunner jettySolrRunner = cluster.getJettySolrRunners().get(i); - List replicas = docCollection.getReplicas(jettySolrRunner.getNodeName()); - if (replicas != null && replicas.size() == 2) { - stoppedNodeName = jettySolrRunner.getNodeName(); - replicasToBeMoved = replicas; - JettySolrRunner j = cluster.stopJettySolrRunner(i); - cluster.waitForJettyToStop(j); - break; - } - } - assertNotNull(stoppedNodeName); - - assertTrue("Trigger was not fired even after 5 seconds", triggerFiredLatch.await(15, TimeUnit.SECONDS)); - assertTrue(fired.get()); - - TriggerEvent triggerEvent = eventRef.get(); - assertNotNull(triggerEvent); - assertEquals(TriggerEventType.NODELOST, triggerEvent.getEventType()); - // TODO assertEquals(stoppedNodeName, triggerEvent.getProperty(TriggerEvent.NODE_NAME)); - - @SuppressWarnings({"rawtypes"}) - Map context = actionContextPropsRef.get(); - assertNotNull(context); - @SuppressWarnings({"unchecked", "rawtypes"}) - List operations = (List) context.get("operations"); - assertNotNull("The operations computed by ComputePlanAction should not be null "+ getNodeStateProviderState() + actionContextPropsRef.get(), operations); - if (log.isInfoEnabled()) { - operations.forEach(solrRequest -> log.info(solrRequest.getParams().toString())); - } - assertEquals("ComputePlanAction should have computed exactly 2 operation", 2, operations.size()); - - for (@SuppressWarnings({"rawtypes"})SolrRequest solrRequest : operations) { - SolrParams params = solrRequest.getParams(); - assertEquals("Expected MOVEREPLICA action after adding node", MOVEREPLICA, CollectionParams.CollectionAction.get(params.get("action"))); - String moved = params.get("replica"); - assertTrue(replicasToBeMoved.stream().anyMatch(replica -> replica.getName().equals(moved))); - } - } - - @Test - // commented out on: 17-Feb-2019 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 14-Oct-2018 - public void testNodeAdded() throws Exception { - CloudSolrClient solrClient = cluster.getSolrClient(); - String setTriggerCommand = "{" + - "'set-trigger' : {" + - "'name' : 'node_added_trigger'," + - "'event' : 'nodeAdded'," + - "'waitFor' : '1s'," + - "'enabled' : true," + - "'actions' : [{'name':'compute_plan', 'class' : 'solr.ComputePlanAction'}," + - "{'name':'test','class':'" + ComputePlanActionTest.AssertingTriggerAction.class.getName() + "'}]" + - "}}"; - @SuppressWarnings({"rawtypes"}) - SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand); - NamedList response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - - // the default policy limits 1 replica per node, we need more right now - String setClusterPolicyCommand = "{" + - " 'set-cluster-policy': [" + - " {'cores':'<10', 'node':'#ANY'}," + - " {'replica':'<3', 'shard': '#EACH', 'node': '#ANY'}," + - " {'nodeRole':'overseer', 'replica':0}" + - " ]" + - "}"; - req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setClusterPolicyCommand); - response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - - CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection("testNodeAdded", - "conf",1, 2); - create.process(solrClient); - - waitForState("Timed out waiting for replicas of new collection to be active", - "testNodeAdded", (liveNodes, collectionState) -> collectionState.getReplicas().stream().allMatch(replica -> replica.isActive(liveNodes))); - - // reset to the original policy which has only 1 replica per shard per node - setClusterPolicyCommand = "{" + - " 'set-cluster-policy': [" + - " {'cores':'<10', 'node':'#ANY'}," + - " {'replica':'<2', 'shard': '#EACH', 'node': '#ANY'}," + - " {'nodeRole':'overseer', 'replica':0}" + - " ]" + - "}"; - req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setClusterPolicyCommand); - response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - - // start a node so that the 'violation' created by the previous policy update is fixed - JettySolrRunner runner = cluster.startJettySolrRunner(); - assertTrue("Trigger was not fired even after 5 seconds", triggerFiredLatch.await(5, TimeUnit.SECONDS)); - assertTrue(fired.get()); - @SuppressWarnings({"rawtypes"}) - Map context = actionContextPropsRef.get(); - assertNotNull(context); - @SuppressWarnings({"unchecked", "rawtypes"}) - List operations = (List) context.get("operations"); - assertNotNull("The operations computed by ComputePlanAction should not be null" + getNodeStateProviderState() + context, operations); - assertEquals("ComputePlanAction should have computed exactly 1 operation", 1, operations.size()); - @SuppressWarnings({"rawtypes"}) - SolrRequest request = operations.get(0); - SolrParams params = request.getParams(); - assertEquals("Expected MOVEREPLICA action after adding node", MOVEREPLICA, CollectionParams.CollectionAction.get(params.get("action"))); - String nodeAdded = params.get("targetNode"); - assertEquals("Unexpected node in computed operation", runner.getNodeName(), nodeAdded); - } - - public static class AssertingTriggerAction implements TriggerAction { - static volatile String expectedNode; - - @Override - public void configure(SolrResourceLoader loader, SolrCloudManager cloudManager, Map properties) throws TriggerValidationException { - - } - - @Override - public void init() { - - } - - @Override - public String getName() { - return null; - } - - @Override - public void process(TriggerEvent event, ActionContext context) { - if (expectedNode != null) { - @SuppressWarnings({"rawtypes"}) - Collection nodes = (Collection) event.getProperty(TriggerEvent.NODE_NAMES); - if (nodes == null || !nodes.contains(expectedNode)) return;//this is not the event we are looking for - } - if (fired.compareAndSet(false, true)) { - eventRef.set(event); - actionContextPropsRef.set(context.getProperties()); - triggerFiredLatch.countDown(); - } - } - - @Override - public void close() throws IOException { - - } - } - - @Test - //2018-06-18 (commented) @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 09-Apr-2018 - public void testSelectedCollectionsByName() throws Exception { - String collectionsFilter = "'testSelected1,testSelected2'"; - testCollectionsPredicate(collectionsFilter, Collections.emptyMap()); - } - - @Test - //2018-06-18 (commented) @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 09-Apr-2018 - public void testSelectedCollectionsByPolicy() throws Exception { - CloudSolrClient solrClient = cluster.getSolrClient(); - String setSearchPolicyCommand = "{" + - " 'set-policy': {" + - " 'search': [" + - " {'replica':'<5', 'shard': '#EACH', 'node': '#ANY'}," + - " ]" + - "}}"; - @SuppressWarnings({"rawtypes"}) - SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setSearchPolicyCommand); - NamedList response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - - String collectionsFilter = "{'policy': 'search'}"; - Map createCollectionParameters = new HashMap<>(); - createCollectionParameters.put("testSelected1", "search"); - createCollectionParameters.put("testSelected2", "search"); - testCollectionsPredicate(collectionsFilter, createCollectionParameters); - } - - private void testCollectionsPredicate(String collectionsFilter, Map createCollectionParameters) throws Exception { - if (log.isInfoEnabled()) { - log.info("Found number of jetties: {}", cluster.getJettySolrRunners().size()); - } - // start 3 more nodes - cluster.startJettySolrRunner(); - cluster.startJettySolrRunner(); - cluster.startJettySolrRunner(); - - cluster.waitForAllNodes(30); - - CloudSolrClient solrClient = cluster.getSolrClient(); - String setTriggerCommand = "{" + - "'set-trigger' : {" + - "'name' : 'node_lost_trigger'," + - "'event' : 'nodeLost'," + - "'waitFor' : '1s'," + - "'enabled' : true," + - "'actions' : [{'name':'compute_plan', 'class' : 'solr.ComputePlanAction', 'collections' : " + collectionsFilter + "}," + - "{'name':'test','class':'" + ComputePlanActionTest.AssertingTriggerAction.class.getName() + "'}]" + - "}}"; - @SuppressWarnings({"rawtypes"}) - SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand); - NamedList response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - - CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection("testSelected1", - "conf", 2, 2); - if (createCollectionParameters.get("testSelected1") != null) { - create.setPolicy(createCollectionParameters.get("testSelected1")); - } - create.process(solrClient); - - create = CollectionAdminRequest.createCollection("testSelected2", - "conf", 2, 2); - if (createCollectionParameters.get("testSelected2") != null) { - create.setPolicy(createCollectionParameters.get("testSelected2")); - } - create.process(solrClient); - - create = CollectionAdminRequest.createCollection("testSelected3", - "conf", 2, 2); - if (createCollectionParameters.get("testSelected3") != null) { - create.setPolicy(createCollectionParameters.get("testSelected3")); - } - create.process(solrClient); - - cluster.waitForActiveCollection("testSelected1", 2, 4); - cluster.waitForActiveCollection("testSelected2", 2, 4); - cluster.waitForActiveCollection("testSelected3", 2, 4); - - waitForState("Timed out waiting for replicas of new collection to be active", - "testSelected1", clusterShape(2, 4)); - - waitForState("Timed out waiting for replicas of new collection to be active", - "testSelected2", clusterShape(2, 4)); - - waitForState("Timed out waiting for replicas of new collection to be active", - "testSelected3", clusterShape(2, 4)); - - // find a node that has replicas from all collections - SolrCloudManager cloudManager = cluster.getJettySolrRunner(0).getCoreContainer().getZkController().getSolrCloudManager(); - NodeStateProvider stateProvider = cloudManager.getNodeStateProvider(); - List nodes = new ArrayList<>(); - cloudManager.getClusterStateProvider().getLiveNodes().forEach(n -> { - Map>> map = stateProvider.getReplicaInfo(n, ImplicitSnitch.tags); - if (map.containsKey("testSelected3") && map.containsKey("testSelected2") && map.containsKey("testSelected1")) { - nodes.add(n); - } - }); - assertTrue(nodes.size() > 0); - // kill first such node - String node = nodes.get(0); - for (int i = 0; i < cluster.getJettySolrRunners().size(); i++) { - if (cluster.getJettySolrRunner(i).getNodeName().equals(node)) { - JettySolrRunner j = cluster.stopJettySolrRunner(i); - cluster.waitForJettyToStop(j); - break; - } - } - assertTrue("Trigger was not fired even after 5 seconds", triggerFiredLatch.await(5, TimeUnit.SECONDS)); - assertTrue(fired.get()); - @SuppressWarnings({"rawtypes"}) - Map context = actionContextPropsRef.get(); - assertNotNull(context); - @SuppressWarnings({"unchecked", "rawtypes"}) - List operations = (List) context.get("operations"); - assertNotNull("The operations computed by ComputePlanAction should not be null. " + getNodeStateProviderState() + context, operations); - assertEquals("ComputePlanAction should have computed exactly 2 operations", 2, operations.size()); - @SuppressWarnings({"rawtypes"}) - SolrRequest request = operations.get(0); - SolrParams params = request.getParams(); - assertEquals("Expected MOVEREPLICA action after adding node", MOVEREPLICA, CollectionParams.CollectionAction.get(params.get("action"))); - assertFalse("not expected testSelected3", "testSelected3".equals(params.get("collection"))); - request = operations.get(1); - params = request.getParams(); - assertEquals("Expected MOVEREPLICA action after adding node", MOVEREPLICA, CollectionParams.CollectionAction.get(params.get("action"))); - assertFalse("not expected testSelected3", "testSelected3".equals(params.get("collection"))); - } - - @Test - public void testNodeAddedTriggerWithAddReplicaPreferredOp_1Shard() throws Exception { - String collectionNamePrefix = "testNodeAddedTriggerWithAddReplicaPreferredOp_1Shard"; - int numShards = 1; - int numCollections = 5; - - nodeAddedTriggerWithAddReplicaPreferredOp(collectionNamePrefix, numShards, numCollections); - } - - @Test - public void testNodeAddedTriggerWithAddReplicaPreferredOpReplicaType_1Shard() throws Exception { - String collectionNamePrefix = "testNodeAddedTriggerWithAddReplicaPreferredOpReplicaType_1Shard"; - int numShards = 1; - int numCollections = 5; - - nodeAddedTriggerWithAddReplicaPreferredOpReplicaType(collectionNamePrefix, numShards, numCollections); - } - - @Test - // commented out on: 24-Dec-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 14-Oct-2018 - public void testNodeAddedTriggerWithAddReplicaPreferredOp_2Shard() throws Exception { - String collectionNamePrefix = "testNodeAddedTriggerWithAddReplicaPreferredOp_2Shard"; - int numShards = 2; - int numCollections = 5; - - nodeAddedTriggerWithAddReplicaPreferredOp(collectionNamePrefix, numShards, numCollections); - } - private void nodeAddedTriggerWithAddReplicaPreferredOp(String collectionNamePrefix, int numShards, int numCollections) throws Exception { - String setTriggerCommand = "{" + - "'set-trigger' : {" + - "'name' : 'node_added_trigger'," + - "'event' : 'nodeAdded'," + - "'waitFor' : '1s'," + - "'enabled' : true," + - "'" + AutoScalingParams.PREFERRED_OP + "':'addreplica'," + - "'actions' : [{'name':'compute_plan', 'class' : 'solr.ComputePlanAction'}," + - "{'name':'test','class':'" + AssertingTriggerAction.class.getName() + "'}]" + - "}}"; - - String setClusterPolicyCommand = "{" + - " 'set-cluster-policy': [" + - " {'cores':'<" + (1 + numCollections * numShards) + "', 'node':'#ANY'}," + - " {'replica':'<2', 'shard': '#EACH', 'node': '#ANY'}," + - " {'nodeRole':'overseer', 'replica':0}" + - " ]" + - "}"; - - nodeAddedTriggerWithAddReplicaPreferredOp(collectionNamePrefix, numShards, numCollections, setTriggerCommand, setClusterPolicyCommand); - } - - private void nodeAddedTriggerWithAddReplicaPreferredOpReplicaType(String collectionNamePrefix, int numShards, int numCollections) throws Exception { - String setTriggerCommand = "{" + - "'set-trigger' : {" + - "'name' : 'node_added_trigger'," + - "'event' : 'nodeAdded'," + - "'waitFor' : '1s'," + - "'enabled' : true," + - "'" + AutoScalingParams.PREFERRED_OP + "':'addreplica'," + - "'" + AutoScalingParams.REPLICA_TYPE + "':'" + Replica.Type.PULL + "'," + - "'actions' : [{'name':'compute_plan', 'class' : 'solr.ComputePlanAction'}," + - "{'name':'test','class':'" + AssertingTriggerAction.class.getName() + "'}]" + - "}}"; - - String setClusterPolicyCommand = "{" + - " 'set-cluster-policy': [" + - " {'cores':'<" + (1 + numCollections * numShards) + "', 'node':'#ANY'}," + - " {'replica':'<2', 'shard': '#EACH', 'node': '#ANY'}," + - " {'nodeRole':'overseer', 'replica':0}" + - " ]" + - "}"; - - nodeAddedTriggerWithAddReplicaPreferredOp(collectionNamePrefix, numShards, numCollections, setTriggerCommand, setClusterPolicyCommand, 0, 1, 0); - } - - private void nodeAddedTriggerWithAddReplicaPreferredOp(String collectionNamePrefix, int numShards, int numCollections, String setTriggerCommand, String setClusterPolicyCommand) throws Exception { - nodeAddedTriggerWithAddReplicaPreferredOp(collectionNamePrefix, numShards, numCollections, setTriggerCommand, setClusterPolicyCommand, 1, null, null); - } - private void nodeAddedTriggerWithAddReplicaPreferredOp(String collectionNamePrefix, int numShards, int numCollections, String setTriggerCommand, String setClusterPolicyCommand, Integer nNrtReplicas, Integer nTlogReplicas, Integer nPullReplicas) throws Exception { - CloudSolrClient solrClient = cluster.getSolrClient(); - @SuppressWarnings({"rawtypes"}) - SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand); - NamedList response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - - req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setClusterPolicyCommand); - response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - - - CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(collectionNamePrefix + "_0", - "conf", numShards, nNrtReplicas, nTlogReplicas, nPullReplicas); - create.process(solrClient); - - waitForState("Timed out waiting for replicas of new collection to be active", - collectionNamePrefix + "_0", (liveNodes, collectionState) -> - collectionState.getReplicas().stream().allMatch(replica -> replica.isActive(liveNodes))); - - JettySolrRunner newNode = cluster.startJettySolrRunner(); - cluster.waitForAllNodes(30); - assertTrue(triggerFiredLatch.await(30, TimeUnit.SECONDS)); - assertTrue(fired.get()); - @SuppressWarnings({"rawtypes"}) - Map actionContext = actionContextPropsRef.get(); - @SuppressWarnings({"rawtypes"}) - List operations = (List) actionContext.get("operations"); - assertNotNull(operations); - assertEquals(numShards, operations.size()); - Set affectedShards = new HashSet<>(2); - for (Object operation : operations) { - assertTrue(operation instanceof CollectionAdminRequest.AddReplica); - CollectionAdminRequest.AddReplica addReplica = (CollectionAdminRequest.AddReplica) operation; - assertEquals(newNode.getNodeName(), addReplica.getNode()); - assertEquals(collectionNamePrefix + "_0", addReplica.getCollection()); - affectedShards.add(addReplica.getShard()); - } - assertEquals(numShards, affectedShards.size()); - - for (int i = 1; i < numCollections; i++) { - create = CollectionAdminRequest.createCollection(collectionNamePrefix + "_" + i, - "conf", numShards, 2); - create.process(solrClient); - - waitForState("Timed out waiting for replicas of new collection to be active", - collectionNamePrefix + "_" + i, (liveNodes, collectionState) -> - collectionState.getReplicas().stream().allMatch(replica -> replica.isActive(liveNodes))); - } - - reset(); - - newNode = cluster.startJettySolrRunner(); - assertTrue(triggerFiredLatch.await(30, TimeUnit.SECONDS)); - assertTrue(fired.get()); - actionContext = actionContextPropsRef.get(); - operations = (List) actionContext.get("operations"); - assertNotNull(operations); - assertEquals(numCollections * numShards, operations.size()); - Set affectedCollections = new HashSet<>(numCollections); - affectedShards = new HashSet<>(numShards); - Set> affectedCollShards = new HashSet<>(numCollections * numShards); - for (Object operation : operations) { - assertTrue(operation instanceof CollectionAdminRequest.AddReplica); - CollectionAdminRequest.AddReplica addReplica = (CollectionAdminRequest.AddReplica) operation; - assertEquals(newNode.getNodeName(), addReplica.getNode()); - affectedCollections.add(addReplica.getCollection()); - affectedShards.add(addReplica.getShard()); - affectedCollShards.add(new Pair<>(addReplica.getCollection(), addReplica.getShard())); - } - assertEquals(numCollections, affectedCollections.size()); - assertEquals(numShards, affectedShards.size()); - assertEquals(numCollections * numShards, affectedCollShards.size()); - } - - @Test - // commented out on: 17-Feb-2019 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 14-Oct-2018 - public void testNodeLostTriggerWithDeleteNodePreferredOp() throws Exception { - String collectionNamePrefix = "testNodeLostTriggerWithDeleteNodePreferredOp"; - int numCollections = 1 + random().nextInt(3), numShards = 1 + random().nextInt(3); - - CloudSolrClient solrClient = cluster.getSolrClient(); - String setTriggerCommand = "{" + - "'set-trigger' : {" + - "'name' : 'node_lost_trigger'," + - "'event' : 'nodeLost'," + - "'waitFor' : '1s'," + - "'enabled' : true," + - "'" + AutoScalingParams.PREFERRED_OP + "':'deletenode'," + - "'actions' : [{'name':'compute_plan', 'class' : 'solr.ComputePlanAction'}," + - "{'name':'execute_plan','class':'solr.ExecutePlanAction'}" + - "{'name':'test','class':'" + AssertingTriggerAction.class.getName() + "'}]" + - "}}"; - @SuppressWarnings({"rawtypes"}) - SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand); - NamedList response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - - String setClusterPolicyCommand = "{" + - " 'set-cluster-policy': [" + - " {'cores':'<" + (1 + numCollections * numShards) + "', 'node':'#ANY'}," + - " {'replica':'<2', 'shard': '#EACH', 'node': '#ANY'}," + - " {'nodeRole':'overseer', 'replica':0}" + - " ]" + - "}"; - req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setClusterPolicyCommand); - response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - - JettySolrRunner newNode = cluster.startJettySolrRunner(); - // cache the node name because it won't be available once the node is shutdown - String newNodeName = newNode.getNodeName(); - - CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(collectionNamePrefix + "_0", - "conf", numShards, 2); - create.process(solrClient); - - waitForState("Timed out waiting for replicas of new collection to be active", - collectionNamePrefix + "_0", (liveNodes, collectionState) -> - collectionState.getReplicas().stream().allMatch(replica -> replica.isActive(liveNodes))); - - cluster.stopJettySolrRunner(newNode); - assertTrue(triggerFiredLatch.await(30, TimeUnit.SECONDS)); - assertTrue(fired.get()); - @SuppressWarnings({"rawtypes"}) - Map actionContext = actionContextPropsRef.get(); - @SuppressWarnings({"rawtypes"}) - List operations = (List) actionContext.get("operations"); - assertNotNull(operations); - assertEquals(1, operations.size()); - for (Object operation : operations) { - assertTrue(operation instanceof CollectionAdminRequest.DeleteNode); - CollectionAdminRequest.DeleteNode deleteNode = (CollectionAdminRequest.DeleteNode) operation; - SolrParams deleteNodeParams = deleteNode.getParams(); - assertEquals(newNodeName, deleteNodeParams.get("node")); - } - - waitForState("Timed out waiting for all shards to have only 1 replica", - collectionNamePrefix + "_0", clusterShape(numShards, numShards)); - } -} diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/ExecutePlanActionTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/ExecutePlanActionTest.java deleted file mode 100644 index 83aa3ff15c7..00000000000 --- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/ExecutePlanActionTest.java +++ /dev/null @@ -1,381 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.cloud.autoscaling; - -import java.lang.invoke.MethodHandles; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.stream.Collectors; - -import com.google.common.collect.Lists; -import org.apache.solr.client.solrj.SolrRequest; -import org.apache.solr.client.solrj.cloud.SolrCloudManager; -import org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventType; -import org.apache.solr.client.solrj.embedded.JettySolrRunner; -import org.apache.solr.client.solrj.impl.CloudSolrClient; -import org.apache.solr.client.solrj.request.CollectionAdminRequest; -import org.apache.solr.cloud.CloudTestUtils.AutoScalingRequest; -import org.apache.solr.cloud.CloudUtil; -import org.apache.solr.cloud.SolrCloudTestCase; -import org.apache.solr.common.cloud.ClusterState; -import org.apache.solr.common.cloud.DocCollection; -import org.apache.solr.common.cloud.Replica; -import org.apache.solr.common.cloud.ZkNodeProps; -import org.apache.solr.common.cloud.ZkStateReader; -import org.apache.solr.common.params.CollectionParams; -import org.apache.solr.common.util.NamedList; -import org.apache.solr.common.util.Utils; -import org.apache.solr.core.SolrResourceLoader; -import org.apache.solr.util.LogLevel; -import org.apache.solr.util.TestInjection; -import org.apache.zookeeper.data.Stat; -import org.junit.After; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import static org.apache.solr.common.cloud.ZkStateReader.SOLR_AUTOSCALING_CONF_PATH; - -/** - * Test for {@link ExecutePlanAction} - */ -@LogLevel("org.apache.solr.cloud.autoscaling=DEBUG") -public class ExecutePlanActionTest extends SolrCloudTestCase { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - private static final int NODE_COUNT = 2; - - private SolrResourceLoader loader; - private SolrCloudManager cloudManager; - - public static class StartAction extends TriggerActionBase { - - @Override - public void process(TriggerEvent event, ActionContext context) throws Exception { - startedProcessing.countDown(); - } - } - - private static CountDownLatch startedProcessing = new CountDownLatch(1); - - public static class FinishAction extends TriggerActionBase { - - @Override - public void process(TriggerEvent event, ActionContext context) throws Exception { - finishedProcessing.countDown(); - } - } - - private static CountDownLatch finishedProcessing = new CountDownLatch(1); - - @BeforeClass - public static void setupCluster() throws Exception { - - } - - @Before - public void setUp() throws Exception { - super.setUp(); - - configureCluster(NODE_COUNT) - .addConfig("conf", configset("cloud-minimal")) - .configure(); - - // clear any persisted auto scaling configuration - Stat stat = zkClient().setData(SOLR_AUTOSCALING_CONF_PATH, Utils.toJSON(new ZkNodeProps()), true); - - - cloudManager = cluster.getJettySolrRunner(0).getCoreContainer().getZkController().getSolrCloudManager(); - - finishedProcessing = new CountDownLatch(1); - startedProcessing = new CountDownLatch(1); - } - - - @After - public void tearDown() throws Exception { - shutdownCluster(); - super.tearDown(); - TestInjection.reset(); - } - - @Test - public void testExecute() throws Exception { - CloudSolrClient solrClient = cluster.getSolrClient(); - String collectionName = "testExecute"; - CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(collectionName, - "conf", 1, 2); - create.process(solrClient); - - cluster.waitForActiveCollection(collectionName, 1, 2); - - waitForState("Timed out waiting for replicas of new collection to be active", - collectionName, clusterShape(1, 2)); - - JettySolrRunner sourceNode = cluster.getRandomJetty(random()); - String sourceNodeName = sourceNode.getNodeName(); - ClusterState clusterState = solrClient.getZkStateReader().getClusterState(); - DocCollection docCollection = clusterState.getCollection(collectionName); - List replicas = docCollection.getReplicas(sourceNodeName); - assertNotNull(replicas); - assertFalse(replicas.isEmpty()); - - List otherJetties = cluster.getJettySolrRunners().stream() - .filter(jettySolrRunner -> jettySolrRunner != sourceNode).collect(Collectors.toList()); - assertFalse(otherJetties.isEmpty()); - JettySolrRunner survivor = otherJetties.get(0); - - try (ExecutePlanAction action = new ExecutePlanAction()) { - action.configure(loader, cloudManager, Collections.singletonMap("name", "execute_plan")); - - // used to signal if we found that ExecutePlanAction did in fact create the right znode before executing the operation - AtomicBoolean znodeCreated = new AtomicBoolean(false); - - CollectionAdminRequest.AsyncCollectionAdminRequest moveReplica = new CollectionAdminRequest.MoveReplica(collectionName, replicas.get(0).getName(), survivor.getNodeName()); - CollectionAdminRequest.AsyncCollectionAdminRequest mockRequest = new CollectionAdminRequest.AsyncCollectionAdminRequest(CollectionParams.CollectionAction.OVERSEERSTATUS) { - @Override - public void setAsyncId(String asyncId) { - super.setAsyncId(asyncId); - String parentPath = ZkStateReader.SOLR_AUTOSCALING_TRIGGER_STATE_PATH + "/xyz/execute_plan"; - try { - if (zkClient().exists(parentPath, true)) { - java.util.List children = zkClient().getChildren(parentPath, null, true); - if (!children.isEmpty()) { - String child = children.get(0); - byte[] data = zkClient().getData(parentPath + "/" + child, null, null, true); - @SuppressWarnings({"rawtypes"}) - Map m = (Map) Utils.fromJSON(data); - if (m.containsKey("requestid")) { - znodeCreated.set(m.get("requestid").equals(asyncId)); - } - } - } - } catch (Exception e) { - throw new RuntimeException(e); - } - - } - }; - List operations = Lists.asList(moveReplica, new CollectionAdminRequest.AsyncCollectionAdminRequest[]{mockRequest}); - NodeLostTrigger.NodeLostEvent nodeLostEvent = new NodeLostTrigger.NodeLostEvent - (TriggerEventType.NODELOST, "mock_trigger_name", - Collections.singletonList(cloudManager.getTimeSource().getTimeNs()), - Collections.singletonList(sourceNodeName), - CollectionParams.CollectionAction.MOVEREPLICA.toLower()); - ActionContext actionContext = new ActionContext(survivor.getCoreContainer().getZkController().getSolrCloudManager(), null, - new HashMap<>(Collections.singletonMap("operations", operations))); - action.process(nodeLostEvent, actionContext); - -// assertTrue("ExecutePlanAction should have stored the requestid in ZK before executing the request", znodeCreated.get()); - @SuppressWarnings({"unchecked"}) - List> responses = (List>) actionContext.getProperty("responses"); - assertNotNull(responses); - assertEquals(2, responses.size()); - NamedList response = responses.get(0); - assertNull(response.get("failure")); - assertNotNull(response.get("success")); - } - - waitForState("Timed out waiting for replicas of new collection to be active", - collectionName, clusterShape(1, 2)); - } - - @Test - public void testIntegration() throws Exception { - CloudSolrClient solrClient = cluster.getSolrClient(); - - String setTriggerCommand = "{" + - "'set-trigger' : {" + - "'name' : 'node_lost_trigger'," + - "'event' : 'nodeLost'," + - "'waitFor' : '1s'," + - "'enabled' : true," + - "'actions' : [{'name':'compute_plan', 'class' : 'solr.ComputePlanAction'}," + - "{'name':'execute_plan','class':'solr.ExecutePlanAction'}]" + - "}}"; - @SuppressWarnings({"rawtypes"}) - SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand); - NamedList response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - - String collectionName = "testIntegration"; - CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(collectionName, - "conf", 1, 2); - create.process(solrClient); - - cluster.waitForActiveCollection(collectionName, 1, 2); - - waitForState("Timed out waiting for replicas of new collection to be active", - collectionName, clusterShape(1, 2)); - - JettySolrRunner sourceNode = cluster.getRandomJetty(random()); - String sourceNodeName = sourceNode.getNodeName(); - ClusterState clusterState = solrClient.getZkStateReader().getClusterState(); - DocCollection docCollection = clusterState.getCollection(collectionName); - List replicas = docCollection.getReplicas(sourceNodeName); - assertNotNull(replicas); - assertFalse(replicas.isEmpty()); - - List otherJetties = cluster.getJettySolrRunners().stream() - .filter(jettySolrRunner -> jettySolrRunner != sourceNode).collect(Collectors.toList()); - assertFalse(otherJetties.isEmpty()); - JettySolrRunner survivor = otherJetties.get(0); - - for (int i = 0; i < cluster.getJettySolrRunners().size(); i++) { - JettySolrRunner runner = cluster.getJettySolrRunner(i); - if (runner == sourceNode) { - JettySolrRunner j = cluster.stopJettySolrRunner(i); - cluster.waitForJettyToStop(j); - } - } - - Thread.sleep(1000); - - waitForState("Timed out waiting for replicas of collection to be 2 again", - collectionName, clusterShape(1, 2)); - - clusterState = solrClient.getZkStateReader().getClusterState(); - docCollection = clusterState.getCollection(collectionName); - List replicasOnSurvivor = docCollection.getReplicas(survivor.getNodeName()); - assertNotNull(replicasOnSurvivor); - assertEquals(docCollection.toString(), 2, replicasOnSurvivor.size()); - } - - @Test - public void testTaskTimeout() throws Exception { - int DELAY = 2000; - boolean taskTimeoutFail = random().nextBoolean(); - TestInjection.delayInExecutePlanAction = DELAY; - CloudSolrClient solrClient = cluster.getSolrClient(); - String triggerName = "node_lost_trigger2"; - - String setTriggerCommand = "{" + - "'set-trigger' : {" + - "'name' : '" + triggerName + "'," + - "'event' : 'nodeLost'," + - "'waitFor' : '1s'," + - "'enabled' : true," + - "'actions' : [{'name':'compute_plan', 'class' : 'solr.ComputePlanAction'}," + - "{'name':'execute_plan','class':'solr.ExecutePlanAction', 'taskTimeoutSeconds' : '1','taskTimeoutFail':'" + taskTimeoutFail + "'}," + - "{'name':'finish','class':'" + FinishAction.class.getName() + "'}]" + - "}}"; - @SuppressWarnings({"rawtypes"}) - SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand); - NamedList response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - - String collectionName = "testTaskTimeout"; - CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(collectionName, - "conf", 1, 2); - create.process(solrClient); - - cluster.waitForActiveCollection(collectionName, 1, 2); - - waitForState("Timed out waiting for replicas of new collection to be active", - collectionName, clusterShape(1, 2)); - - JettySolrRunner sourceNode = cluster.getRandomJetty(random()); - - for (int i = 0; i < cluster.getJettySolrRunners().size(); i++) { - JettySolrRunner runner = cluster.getJettySolrRunner(i); - if (runner == sourceNode) { - JettySolrRunner j = cluster.stopJettySolrRunner(i); - cluster.waitForJettyToStop(j); - } - } - - boolean await = finishedProcessing.await(DELAY * 5, TimeUnit.MILLISECONDS); - if (taskTimeoutFail) { - assertFalse("finished processing event but should fail", await); - } else { - assertTrue("did not finish processing event in time", await); - } - String path = ZkStateReader.SOLR_AUTOSCALING_TRIGGER_STATE_PATH + "/" + triggerName + "/execute_plan"; - assertTrue(path + " does not exist", zkClient().exists(path, true)); - List requests = zkClient().getChildren(path, null, true); - assertFalse("some requests should be still present", requests.isEmpty()); - - // in either case the task will complete and move the replica as needed - waitForState("Timed out waiting for replicas of collection to be 2 again", - collectionName, clusterShape(1, 2)); - } - - @Test - public void testTaskFail() throws Exception { - TestInjection.failInExecutePlanAction = true; - CloudSolrClient solrClient = cluster.getSolrClient(); - String triggerName = "node_lost_trigger3"; - - String setTriggerCommand = "{" + - "'set-trigger' : {" + - "'name' : '" + triggerName + "'," + - "'event' : 'nodeLost'," + - "'waitFor' : '1s'," + - "'enabled' : true," + - "'actions' : [{'name':'start', 'class' : '" + StartAction.class.getName() + "'}," + - "{'name':'compute_plan','class':'solr.ComputePlanAction'}," + - "{'name':'execute_plan','class':'solr.ExecutePlanAction'}," + - "{'name':'finish','class':'" + FinishAction.class.getName() + "'}]" + - "}}"; - @SuppressWarnings({"rawtypes"}) - SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand); - NamedList response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - - String collectionName = "testTaskFail"; - CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(collectionName, - "conf", 1, 2); - create.process(solrClient); - - cluster.waitForActiveCollection(collectionName, 1, 2); - - waitForState("Timed out waiting for replicas of new collection to be active", - collectionName, clusterShape(1, 2)); - - // don't stop the jetty that runs our SolrCloudManager - JettySolrRunner runner = cluster.stopJettySolrRunner(1); - cluster.waitForJettyToStop(runner); - - boolean await = startedProcessing.await(10, TimeUnit.SECONDS); - assertTrue("did not start processing event in time", await); - await = finishedProcessing.await(2, TimeUnit.SECONDS); - assertFalse("finished processing event but should fail", await); - - String path = ZkStateReader.SOLR_AUTOSCALING_TRIGGER_STATE_PATH + "/" + triggerName + "/execute_plan"; - assertTrue(path + " does not exist", zkClient().exists(path, true)); - List requests = zkClient().getChildren(path, null, true); - assertTrue("there should be no requests pending but got " + requests, requests.isEmpty()); - - // the task never completed - we actually lost a replica - try { - CloudUtil.waitForState(cloudManager, collectionName, 5, TimeUnit.SECONDS, - CloudUtil.clusterShape(1, 2)); - fail("completed a task that should have failed"); - } catch (TimeoutException te) { - // expected - } - } -} diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/HdfsAutoAddReplicasIntegrationTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/HdfsAutoAddReplicasIntegrationTest.java deleted file mode 100644 index 7600bf46cc7..00000000000 --- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/HdfsAutoAddReplicasIntegrationTest.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.cloud.autoscaling; - -import com.carrotsearch.randomizedtesting.annotations.Nightly; -import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; -import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite; -import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.lucene.util.LuceneTestCase.Slow; -import org.apache.lucene.util.QuickPatchThreadsFilter; -import org.apache.lucene.util.TimeUnits; -import org.apache.solr.SolrIgnoredThreadsFilter; -import org.apache.solr.cloud.hdfs.HdfsTestUtil; -import org.apache.solr.util.BadHdfsThreadsFilter; -import org.junit.AfterClass; -import org.junit.BeforeClass; - -@Slow -@Nightly -@ThreadLeakFilters(defaultFilters = true, filters = { - SolrIgnoredThreadsFilter.class, - QuickPatchThreadsFilter.class, - BadHdfsThreadsFilter.class // hdfs currently leaks thread(s) -}) -@TimeoutSuite(millis = TimeUnits.HOUR) -public class HdfsAutoAddReplicasIntegrationTest extends AutoAddReplicasIntegrationTest { - private static MiniDFSCluster dfsCluster; - - @BeforeClass - public static void setupClass() throws Exception { - dfsCluster = HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath()); - } - - @AfterClass - public static void teardownClass() throws Exception { - try { - HdfsTestUtil.teardownClass(dfsCluster); - } finally { - dfsCluster = null; - } - } - - @Override - protected String getConfigSet() { - return "cloud-hdfs"; - } -} diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/HttpTriggerListenerTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/HttpTriggerListenerTest.java deleted file mode 100644 index a712aee6c00..00000000000 --- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/HttpTriggerListenerTest.java +++ /dev/null @@ -1,208 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.cloud.autoscaling; - -import javax.servlet.ServletException; -import javax.servlet.ServletInputStream; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; -import java.io.IOException; -import java.net.InetSocketAddress; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.Enumeration; -import java.util.List; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; - -import org.apache.solr.SolrTestCaseJ4; -import org.apache.solr.client.solrj.SolrRequest; -import org.apache.solr.client.solrj.impl.CloudSolrClient; -import org.apache.solr.cloud.CloudTestUtils.AutoScalingRequest; -import org.apache.solr.cloud.SolrCloudTestCase; -import org.apache.solr.common.util.NamedList; -import org.apache.solr.util.LogLevel; -import org.eclipse.jetty.server.Request; -import org.eclipse.jetty.server.Server; -import org.eclipse.jetty.server.handler.AbstractHandler; -import org.junit.After; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; - -/** - * - */ -@LogLevel("org.apache.solr.cloud.autoscaling=DEBUG") -@SolrTestCaseJ4.SuppressSSL -public class HttpTriggerListenerTest extends SolrCloudTestCase { - - private static CountDownLatch triggerFiredLatch; - - private MockService mockService; - - @BeforeClass - public static void setupCluster() throws Exception { - configureCluster(2) - .addConfig("conf", configset("cloud-minimal")) - .configure(); - } - - @Before - public void setupTest() throws Exception { - mockService = new MockService(); - mockService.start(); - triggerFiredLatch = new CountDownLatch(1); - } - - @After - public void teardownTest() throws Exception { - if (mockService != null) { - mockService.close(); - } - } - - @Test - public void testHttpListenerIntegration() throws Exception { - CloudSolrClient solrClient = cluster.getSolrClient(); - String setTriggerCommand = "{" + - "'set-trigger' : {" + - "'name' : 'node_added_trigger'," + - "'event' : 'nodeAdded'," + - "'waitFor' : '0s'," + - "'enabled' : true," + - "'actions' : [" + - "{'name':'test','class':'" + TestDummyAction.class.getName() + "'}" + - "]" + - "}}"; - @SuppressWarnings({"rawtypes"}) - SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand); - NamedList response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - - String setListenerCommand = "{" + - "'set-listener' : " + - "{" + - "'name' : 'foo'," + - "'trigger' : 'node_added_trigger'," + - "'stage' : ['STARTED','ABORTED','SUCCEEDED', 'FAILED']," + - "'beforeAction' : 'test'," + - "'afterAction' : ['test']," + - "'class' : '" + HttpTriggerListener.class.getName() + "'," + - "'url' : '" + mockService.server.getURI().toString() + "/${config.name:invalid}/${config.properties.beforeAction:invalid}/${stage}'," + - "'payload': 'actionName=${actionName}, source=${event.source}, type=${event.eventType}'," + - "'header.X-Foo' : '${config.name:invalid}'" + - "}" + - "}"; - req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setListenerCommand); - response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - - assertEquals(mockService.requests.toString(), 0, mockService.requests.size()); - - cluster.startJettySolrRunner(); - cluster.waitForAllNodes(30); - boolean await = triggerFiredLatch.await(20, TimeUnit.SECONDS); - assertTrue("The trigger did not fire at all", await); - - Thread.sleep(5000); - - assertEquals(mockService.requests.toString(), 4, mockService.requests.size()); - mockService.requests.forEach(s -> assertTrue(s.contains("Content-Type: application/json"))); - mockService.requests.forEach(s -> assertTrue(s.contains("X-Foo: foo"))); - mockService.requests.forEach(s -> assertTrue(s.contains("source=node_added_trigger"))); - mockService.requests.forEach(s -> assertTrue(s.contains("type=NODEADDED"))); - - String request = mockService.requests.get(0); - assertTrue(request, request.startsWith("/foo/test/STARTED")); - assertTrue(request, request.contains("actionName=,")); // empty actionName - - request = mockService.requests.get(1); - assertTrue(request, request.startsWith("/foo/test/BEFORE_ACTION")); - assertTrue(request, request.contains("actionName=test,")); // actionName - - request = mockService.requests.get(2); - assertTrue(request, request.startsWith("/foo/test/AFTER_ACTION")); - assertTrue(request, request.contains("actionName=test,")); // actionName - - request = mockService.requests.get(3); - assertTrue(request, request.startsWith("/foo/test/SUCCEEDED")); - assertTrue(request, request.contains("actionName=,")); // empty actionName - } - - public static class TestDummyAction extends TriggerActionBase { - - @Override - public void process(TriggerEvent event, ActionContext context) { - triggerFiredLatch.countDown(); - } - } - - private static class MockService extends Thread { - public final List requests = new ArrayList<>(); - private Server server; - - public void start() { - server = new Server(new InetSocketAddress("localhost", 0)); - server.setHandler(new AbstractHandler() { - @Override - public void handle(String s, Request request, HttpServletRequest httpServletRequest, HttpServletResponse httpServletResponse) throws IOException, ServletException { - StringBuilder stringBuilder = new StringBuilder(); - stringBuilder.append(httpServletRequest.getRequestURI()); - Enumeration headerNames = httpServletRequest.getHeaderNames(); - while (headerNames.hasMoreElements()) { - stringBuilder.append('\n'); - String name = headerNames.nextElement(); - stringBuilder.append(name); - stringBuilder.append(": "); - stringBuilder.append(httpServletRequest.getHeader(name)); - } - stringBuilder.append("\n\n"); - ServletInputStream is = request.getInputStream(); - byte[] httpInData = new byte[request.getContentLength()]; - int len = -1; - while ((len = is.read(httpInData)) != -1) { - stringBuilder.append(new String(httpInData, 0, len, StandardCharsets.UTF_8)); - } - requests.add(stringBuilder.toString()); - httpServletResponse.setStatus(HttpServletResponse.SC_OK); - request.setHandled(true); - } - }); - try { - server.start(); - for (int i = 0; i < 30; i++) { - Thread.sleep(1000); - if (server.isRunning()) { - break; - } - if (server.isFailed()) { - throw new Exception("MockService startup failed - the test will fail..."); - } - } - } catch (Exception e) { - throw new RuntimeException("Exception starting MockService", e); - } - } - - void close() throws Exception { - if (server != null) { - server.stop(); - } - } - } -} diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/IndexSizeTriggerMixedBoundsTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/IndexSizeTriggerMixedBoundsTest.java deleted file mode 100644 index 22bd537e4cf..00000000000 --- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/IndexSizeTriggerMixedBoundsTest.java +++ /dev/null @@ -1,374 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.cloud.autoscaling; - -import static org.apache.solr.common.cloud.ZkStateReader.SOLR_AUTOSCALING_CONF_PATH; - -import java.lang.invoke.MethodHandles; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; - -import org.apache.lucene.util.LuceneTestCase; -import org.apache.lucene.util.TestUtil; -import org.apache.solr.client.solrj.SolrClient; -import org.apache.solr.client.solrj.SolrRequest; -import org.apache.solr.client.solrj.cloud.SolrCloudManager; -import org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig; -import org.apache.solr.client.solrj.cloud.autoscaling.Suggester; -import org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventProcessorStage; -import org.apache.solr.client.solrj.request.CollectionAdminRequest; -import org.apache.solr.client.solrj.request.UpdateRequest; -import org.apache.solr.client.solrj.response.QueryResponse; -import org.apache.solr.cloud.CloudTestUtils.AutoScalingRequest; -import org.apache.solr.cloud.CloudUtil; -import org.apache.solr.cloud.SolrCloudTestCase; -import org.apache.solr.common.SolrInputDocument; -import org.apache.solr.common.cloud.ZkNodeProps; -import org.apache.solr.common.params.CollectionParams; -import org.apache.solr.common.params.CommonParams; -import org.apache.solr.common.params.UpdateParams; -import org.apache.solr.common.util.NamedList; -import org.apache.solr.common.util.Pair; -import org.apache.solr.common.util.Utils; -import org.apache.solr.core.SolrResourceLoader; -import org.apache.solr.util.LogLevel; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * - */ -@LogLevel("org.apache.solr.cloud.autoscaling=DEBUG") -@LuceneTestCase.Slow -public class IndexSizeTriggerMixedBoundsTest extends SolrCloudTestCase { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - private static SolrCloudManager cloudManager; - private static SolrClient solrClient; - - private static int SPEED = 1; - - static Map> listenerEvents = new ConcurrentHashMap<>(); - static CountDownLatch listenerCreated; - static CountDownLatch finished; - - @BeforeClass - public static void setupCluster() throws Exception { - configureCluster(2) - .addConfig("conf", configset("cloud-minimal")) - .configure(); - cloudManager = cluster.getJettySolrRunner(0).getCoreContainer().getZkController().getSolrCloudManager(); - solrClient = cluster.getSolrClient(); - } - - @Before - public void setDefaults() throws Exception { - cluster.deleteAllCollections(); - cloudManager.getDistribStateManager().setData(SOLR_AUTOSCALING_CONF_PATH, Utils.toJSON(new ZkNodeProps()), -1); - listenerEvents.clear(); - listenerCreated = new CountDownLatch(1); - finished = new CountDownLatch(1); - } - - public static class CapturingTriggerListener extends TriggerListenerBase { - @Override - public void configure(SolrResourceLoader loader, SolrCloudManager cloudManager, AutoScalingConfig.TriggerListenerConfig config) throws TriggerValidationException { - super.configure(loader, cloudManager, config); - listenerCreated.countDown(); - } - - @Override - public synchronized void onEvent(TriggerEvent event, TriggerEventProcessorStage stage, String actionName, - ActionContext context, Throwable error, String message) { - List lst = listenerEvents.computeIfAbsent(config.name, s -> new ArrayList<>()); - CapturedEvent ev = new CapturedEvent(cloudManager.getTimeSource().getTimeNs(), context, config, stage, actionName, event, message); - log.info("=======> {}", ev); - lst.add(ev); - } - } - - public static class FinishedProcessingListener extends TriggerListenerBase { - - @Override - public void onEvent(TriggerEvent event, TriggerEventProcessorStage stage, String actionName, ActionContext context, Throwable error, String message) throws Exception { - finished.countDown(); - } - } - - @Test - @SuppressWarnings({"unchecked"}) - public void testMixedBounds() throws Exception { - String collectionName = "testMixedBounds_collection"; - CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(collectionName, - "conf", 2, 2); - create.process(solrClient); - CloudUtil.waitForState(cloudManager, "failed to create " + collectionName, collectionName, - CloudUtil.clusterShape(2, 2, false, true)); - - for (int j = 0; j < 10; j++) { - UpdateRequest ureq = new UpdateRequest(); - ureq.setParam("collection", collectionName); - for (int i = 0; i < 100; i++) { - SolrInputDocument doc = new SolrInputDocument("id", "id-" + (i * 100) + "-" + j); - doc.addField("foo", TestUtil.randomSimpleString(random(), 130, 130)); - ureq.add(doc); - } - solrClient.request(ureq); - } - solrClient.commit(collectionName); - - // check the actual size of shard to set the threshold - QueryResponse rsp = solrClient.query(params(CommonParams.QT, "/admin/metrics", "group", "core")); - NamedList nl = rsp.getResponse(); - nl = (NamedList)nl.get("metrics"); - int maxSize = 0; - for (Iterator> it = nl.iterator(); it.hasNext(); ) { - Map.Entry e = it.next(); - NamedList metrics = (NamedList)e.getValue(); - Object o = metrics.get("INDEX.sizeInBytes"); - assertNotNull("INDEX.sizeInBytes missing: " + metrics, o); - assertTrue("not a number", o instanceof Number); - if (maxSize < ((Number)o).intValue()) { - maxSize = ((Number)o).intValue(); - } - } - assertTrue("maxSize should be non-zero", maxSize > 0); - - int aboveBytes = maxSize * 2 / 3; - - // need to wait for recovery after splitting - long waitForSeconds = 10 + random().nextInt(5); - - // the trigger is initially disabled so that we have time to add listeners - // and have them capture all events once the trigger is enabled - String setTriggerCommand = "{" + - "'set-trigger' : {" + - "'name' : 'index_size_trigger4'," + - "'event' : 'indexSize'," + - "'waitFor' : '" + waitForSeconds + "s'," + - // don't hit this limit when indexing - "'aboveDocs' : 10000," + - // hit this limit when deleting - "'belowDocs' : 100," + - // hit this limit when indexing - "'aboveBytes' : " + aboveBytes + "," + - // don't hit this limit when deleting - "'belowBytes' : 10," + - "'enabled' : false," + - "'actions' : [{'name' : 'compute_plan', 'class' : 'solr.ComputePlanAction'}," + - "{'name' : 'execute_plan', 'class' : '" + ExecutePlanAction.class.getName() + "'}]" + - "}}"; - @SuppressWarnings({"rawtypes"}) - SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand); - NamedList response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - - String setListenerCommand = "{" + - "'set-listener' : " + - "{" + - "'name' : 'capturing4'," + - "'trigger' : 'index_size_trigger4'," + - "'stage' : ['STARTED','ABORTED','SUCCEEDED','FAILED']," + - "'beforeAction' : ['compute_plan','execute_plan']," + - "'afterAction' : ['compute_plan','execute_plan']," + - "'class' : '" + CapturingTriggerListener.class.getName() + "'" + - "}" + - "}"; - req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setListenerCommand); - response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - - setListenerCommand = "{" + - "'set-listener' : " + - "{" + - "'name' : 'finished'," + - "'trigger' : 'index_size_trigger4'," + - "'stage' : ['SUCCEEDED']," + - "'class' : '" + FinishedProcessingListener.class.getName() + "'" + - "}" + - "}"; - req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setListenerCommand); - response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - - // now enable the trigger - String resumeTriggerCommand = "{" + - "'resume-trigger' : {" + - "'name' : 'index_size_trigger4'" + - "}" + - "}"; - log.info("-- resuming trigger"); - req = AutoScalingRequest.create(SolrRequest.METHOD.POST, resumeTriggerCommand); - response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - - boolean await = finished.await(90000 / SPEED, TimeUnit.MILLISECONDS); - assertTrue("did not finish processing in time", await); - log.info("-- suspending trigger"); - // suspend the trigger to avoid generating more events - String suspendTriggerCommand = "{" + - "'suspend-trigger' : {" + - "'name' : 'index_size_trigger4'" + - "}" + - "}"; - req = AutoScalingRequest.create(SolrRequest.METHOD.POST, suspendTriggerCommand); - response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - - assertEquals(1, listenerEvents.size()); - List events = listenerEvents.get("capturing4"); - assertNotNull("'capturing4' events not found", events); - assertEquals("events: " + events, 6, events.size()); - assertEquals(TriggerEventProcessorStage.STARTED, events.get(0).stage); - assertEquals(TriggerEventProcessorStage.BEFORE_ACTION, events.get(1).stage); - assertEquals(TriggerEventProcessorStage.AFTER_ACTION, events.get(2).stage); - assertEquals(TriggerEventProcessorStage.BEFORE_ACTION, events.get(3).stage); - assertEquals(TriggerEventProcessorStage.AFTER_ACTION, events.get(4).stage); - assertEquals(TriggerEventProcessorStage.SUCCEEDED, events.get(5).stage); - - // collection should have 2 inactive and 4 active shards - CloudUtil.waitForState(cloudManager, "failed to create " + collectionName, collectionName, - CloudUtil.clusterShape(6, 2, true, true)); - - // check ops - List ops = (List) events.get(4).event.getProperty(TriggerEvent.REQUESTED_OPS); - assertNotNull("should contain requestedOps", ops); - assertEquals("number of ops", 2, ops.size()); - boolean shard1 = false; - boolean shard2 = false; - for (TriggerEvent.Op op : ops) { - assertEquals(CollectionParams.CollectionAction.SPLITSHARD, op.getAction()); - Set> hints = (Set>)op.getHints().get(Suggester.Hint.COLL_SHARD); - assertNotNull("hints", hints); - assertEquals("hints", 1, hints.size()); - Pair p = hints.iterator().next(); - assertEquals(collectionName, p.first()); - if (p.second().equals("shard1")) { - shard1 = true; - } else if (p.second().equals("shard2")) { - shard2 = true; - } else { - fail("unexpected shard name " + p.second()); - } - } - assertTrue("shard1 should be split", shard1); - assertTrue("shard2 should be split", shard2); - - // now delete most of docs to trigger belowDocs condition - listenerEvents.clear(); - finished = new CountDownLatch(1); - - // suspend the trigger first so that we can safely delete all docs - suspendTriggerCommand = "{" + - "'suspend-trigger' : {" + - "'name' : 'index_size_trigger4'" + - "}" + - "}"; - req = AutoScalingRequest.create(SolrRequest.METHOD.POST, suspendTriggerCommand); - response = solrClient.request(req); - assertEquals("success", response.get("result").toString()); - - log.info("-- deleting documents"); - for (int j = 0; j < 10; j++) { - UpdateRequest ureq = new UpdateRequest(); - ureq.setParam("collection", collectionName); - for (int i = 0; i < 98; i++) { - ureq.deleteById("id-" + (i * 100) + "-" + j); - } - solrClient.request(ureq); - } - // make sure the actual index size is reduced by deletions, otherwise we may still violate aboveBytes - UpdateRequest ur = new UpdateRequest(); - ur.setParam(UpdateParams.COMMIT, "true"); - ur.setParam(UpdateParams.EXPUNGE_DELETES, "true"); - ur.setParam(UpdateParams.OPTIMIZE, "true"); - ur.setParam(UpdateParams.MAX_OPTIMIZE_SEGMENTS, "1"); - ur.setParam(UpdateParams.WAIT_SEARCHER, "true"); - ur.setParam(UpdateParams.OPEN_SEARCHER, "true"); - log.info("-- requesting optimize / expungeDeletes / commit"); - solrClient.request(ur, collectionName); - - // add some docs so that every shard gets an update - // we can reduce the number of docs here but this also works - for (int j = 0; j < 1; j++) { - UpdateRequest ureq = new UpdateRequest(); - ureq.setParam("collection", collectionName); - for (int i = 0; i < 98; i++) { - ureq.add("id", "id-" + (i * 100) + "-" + j); - } - solrClient.request(ureq); - } - - log.info("-- requesting commit"); - solrClient.commit(collectionName, true, true); - - // resume the trigger - log.info("-- resuming trigger"); - // resume trigger - req = AutoScalingRequest.create(SolrRequest.METHOD.POST, resumeTriggerCommand); - response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - - await = finished.await(90000 / SPEED, TimeUnit.MILLISECONDS); - assertTrue("did not finish processing in time", await); - log.info("-- suspending trigger"); - req = AutoScalingRequest.create(SolrRequest.METHOD.POST, suspendTriggerCommand); - response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - - assertEquals(1, listenerEvents.size()); - events = listenerEvents.get("capturing4"); - assertNotNull("'capturing4' events not found", events); - assertEquals("events: " + events, 6, events.size()); - assertEquals(TriggerEventProcessorStage.STARTED, events.get(0).stage); - assertEquals(TriggerEventProcessorStage.BEFORE_ACTION, events.get(1).stage); - assertEquals(TriggerEventProcessorStage.AFTER_ACTION, events.get(2).stage); - assertEquals(TriggerEventProcessorStage.BEFORE_ACTION, events.get(3).stage); - assertEquals(TriggerEventProcessorStage.AFTER_ACTION, events.get(4).stage); - assertEquals(TriggerEventProcessorStage.SUCCEEDED, events.get(5).stage); - - // check ops - ops = (List) events.get(4).event.getProperty(TriggerEvent.REQUESTED_OPS); - assertNotNull("should contain requestedOps", ops); - assertTrue("number of ops: " + ops, ops.size() > 0); - for (TriggerEvent.Op op : ops) { - assertEquals(CollectionParams.CollectionAction.MERGESHARDS, op.getAction()); - Set> hints = (Set>)op.getHints().get(Suggester.Hint.COLL_SHARD); - assertNotNull("hints", hints); - assertEquals("hints", 2, hints.size()); - Pair p = hints.iterator().next(); - assertEquals(collectionName, p.first()); - } - - // TODO: fix this once MERGESHARDS is supported - List unsupportedOps = (List)events.get(2).context.get("properties.unsupportedOps"); - assertNotNull("should have unsupportedOps", unsupportedOps); - assertEquals(unsupportedOps.toString() + "\n" + ops, ops.size(), unsupportedOps.size()); - unsupportedOps.forEach(op -> assertEquals(CollectionParams.CollectionAction.MERGESHARDS, op.getAction())); - } - -} diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/IndexSizeTriggerSizeEstimationTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/IndexSizeTriggerSizeEstimationTest.java deleted file mode 100644 index 528dd556d65..00000000000 --- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/IndexSizeTriggerSizeEstimationTest.java +++ /dev/null @@ -1,323 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.cloud.autoscaling; - -import static org.apache.solr.common.cloud.ZkStateReader.SOLR_AUTOSCALING_CONF_PATH; - -import java.lang.invoke.MethodHandles; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; - -import org.apache.lucene.util.LuceneTestCase; -import org.apache.solr.client.solrj.SolrClient; -import org.apache.solr.client.solrj.SolrRequest; -import org.apache.solr.client.solrj.cloud.SolrCloudManager; -import org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig; -import org.apache.solr.client.solrj.cloud.autoscaling.Suggester; -import org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventProcessorStage; -import org.apache.solr.client.solrj.request.CollectionAdminRequest; -import org.apache.solr.cloud.CloudTestUtils.AutoScalingRequest; -import org.apache.solr.cloud.CloudUtil; -import org.apache.solr.cloud.SolrCloudTestCase; -import org.apache.solr.cloud.autoscaling.sim.SimUtils; -import org.apache.solr.common.SolrInputDocument; -import org.apache.solr.common.cloud.DocCollection; -import org.apache.solr.common.cloud.Replica; -import org.apache.solr.common.cloud.ZkNodeProps; -import org.apache.solr.common.params.CollectionParams; -import org.apache.solr.common.util.NamedList; -import org.apache.solr.common.util.Pair; -import org.apache.solr.common.util.TimeSource; -import org.apache.solr.common.util.Utils; -import org.apache.solr.core.SolrResourceLoader; -import org.apache.solr.metrics.SolrCoreMetricManager; -import org.apache.solr.util.LogLevel; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * - */ -@LogLevel("org.apache.solr.cloud.autoscaling=DEBUG") -@LuceneTestCase.Slow -public class IndexSizeTriggerSizeEstimationTest extends SolrCloudTestCase { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - private static SolrCloudManager cloudManager; - private static SolrClient solrClient; - private static TimeSource timeSource; - - private static int SPEED = 1; - - static Map> listenerEvents = new ConcurrentHashMap<>(); - static CountDownLatch listenerCreated = new CountDownLatch(1); - static CountDownLatch finished = new CountDownLatch(1); - - @BeforeClass - public static void setupCluster() throws Exception { - configureCluster(2) - .addConfig("conf", configset("cloud-minimal")) - .configure(); - cloudManager = cluster.getJettySolrRunner(0).getCoreContainer().getZkController().getSolrCloudManager(); - solrClient = cluster.getSolrClient(); - timeSource = cloudManager.getTimeSource(); - } - - @After - public void restoreDefaults() throws Exception { - cluster.deleteAllCollections(); - cloudManager.getDistribStateManager().setData(SOLR_AUTOSCALING_CONF_PATH, Utils.toJSON(new ZkNodeProps()), -1); - cloudManager.getTimeSource().sleep(5000); - listenerEvents.clear(); - listenerCreated = new CountDownLatch(1); - finished = new CountDownLatch(1); - } - - @AfterClass - public static void teardown() throws Exception { - solrClient = null; - cloudManager = null; - } - - public static class CapturingTriggerListener extends TriggerListenerBase { - @Override - public void configure(SolrResourceLoader loader, SolrCloudManager cloudManager, AutoScalingConfig.TriggerListenerConfig config) throws TriggerValidationException { - super.configure(loader, cloudManager, config); - listenerCreated.countDown(); - } - - @Override - public synchronized void onEvent(TriggerEvent event, TriggerEventProcessorStage stage, String actionName, - ActionContext context, Throwable error, String message) { - List lst = listenerEvents.computeIfAbsent(config.name, s -> new ArrayList<>()); - CapturedEvent ev = new CapturedEvent(timeSource.getTimeNs(), context, config, stage, actionName, event, message); - log.info("=======> {}", ev); - lst.add(ev); - } - } - - public static class FinishedProcessingListener extends TriggerListenerBase { - - @Override - public void onEvent(TriggerEvent event, TriggerEventProcessorStage stage, String actionName, ActionContext context, Throwable error, String message) throws Exception { - finished.countDown(); - } - } - - @Test - public void testEstimatedIndexSize() throws Exception { - String collectionName = "testEstimatedIndexSize_collection"; - CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(collectionName, - "conf", 2, 2); - create.process(solrClient); - - CloudUtil.waitForState(cloudManager, "failed to create " + collectionName, collectionName, - CloudUtil.clusterShape(2, 2, false, true)); - - int NUM_DOCS = 20; - for (int i = 0; i < NUM_DOCS; i++) { - SolrInputDocument doc = new SolrInputDocument("id", "id-" + (i * 100)); - solrClient.add(collectionName, doc); - } - solrClient.commit(collectionName); - - // get the size of the leader's index - DocCollection coll = cloudManager.getClusterStateProvider().getCollection(collectionName); - Replica leader = coll.getSlice("shard1").getLeader(); - String replicaName = Utils.parseMetricsReplicaName(collectionName, leader.getCoreName()); - assertNotNull("replicaName could not be constructed from " + leader, replicaName); - final String registry = SolrCoreMetricManager.createRegistryName(true, collectionName, "shard1", replicaName, null); - Set tags = SimUtils.COMMON_REPLICA_TAGS.stream() - .map(s -> "metrics:" + registry + ":" + s).collect(Collectors.toSet()); - Map sizes = cloudManager.getNodeStateProvider().getNodeValues(leader.getNodeName(), tags); - String commitSizeTag = "metrics:" + registry + ":SEARCHER.searcher.indexCommitSize"; - String numDocsTag = "metrics:" + registry + ":SEARCHER.searcher.numDocs"; - String maxDocTag = "metrics:" + registry + ":SEARCHER.searcher.maxDoc"; - assertNotNull(sizes.toString(), sizes.get(commitSizeTag)); - assertNotNull(sizes.toString(), sizes.get(numDocsTag)); - assertNotNull(sizes.toString(), sizes.get(maxDocTag)); - long commitSize = ((Number)sizes.get(commitSizeTag)).longValue(); - long maxDoc = ((Number)sizes.get(maxDocTag)).longValue(); - long numDocs = ((Number)sizes.get(numDocsTag)).longValue(); - - assertEquals("maxDoc != numDocs", maxDoc, numDocs); - assertTrue("unexpected numDocs=" + numDocs, numDocs > NUM_DOCS / 3); - - long aboveBytes = commitSize * 9 / 10; - long waitForSeconds = 3 + random().nextInt(5); - String setTriggerCommand = "{" + - "'set-trigger' : {" + - "'name' : 'index_size_trigger7'," + - "'event' : 'indexSize'," + - "'waitFor' : '" + waitForSeconds + "s'," + - "'splitMethod' : 'link'," + - "'aboveBytes' : " + aboveBytes + "," + - "'enabled' : false," + - "'actions' : [{'name' : 'compute_plan', 'class' : 'solr.ComputePlanAction'}," + - "{'name' : 'execute_plan', 'class' : '" + ExecutePlanAction.class.getName() + "'}]" + - "}}"; - @SuppressWarnings({"rawtypes"}) - SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand); - NamedList response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - - String setListenerCommand = "{" + - "'set-listener' : " + - "{" + - "'name' : 'capturing7'," + - "'trigger' : 'index_size_trigger7'," + - "'stage' : ['STARTED','ABORTED','SUCCEEDED','FAILED']," + - "'beforeAction' : ['compute_plan','execute_plan']," + - "'afterAction' : ['compute_plan','execute_plan']," + - "'class' : '" + CapturingTriggerListener.class.getName() + "'" + - "}" + - "}"; - req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setListenerCommand); - response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - - setListenerCommand = "{" + - "'set-listener' : " + - "{" + - "'name' : 'finished'," + - "'trigger' : 'index_size_trigger7'," + - "'stage' : ['SUCCEEDED']," + - "'class' : '" + FinishedProcessingListener.class.getName() + "'" + - "}" + - "}"; - req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setListenerCommand); - response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - - // enable the trigger - String resumeTriggerCommand = "{" + - "'resume-trigger' : {" + - "'name' : 'index_size_trigger7'" + - "}" + - "}"; - req = AutoScalingRequest.create(SolrRequest.METHOD.POST, resumeTriggerCommand); - response = solrClient.request(req); - assertEquals("success", response.get("result").toString()); - - // aboveBytes was set to be slightly lower than the actual size of at least one shard, so - // we're expecting a SPLITSHARD - but with 'link' method the actual size of the resulting shards - // will likely not go down. However, the estimated size of the latest commit point will go down - // (see SOLR-12941). - - timeSource.sleep(TimeUnit.MILLISECONDS.convert(waitForSeconds + 1, TimeUnit.SECONDS)); - - boolean await = finished.await(90000 / SPEED, TimeUnit.MILLISECONDS); - assertTrue("did not finish processing in time", await); - // suspend the trigger - String suspendTriggerCommand = "{" + - "'suspend-trigger' : {" + - "'name' : 'index_size_trigger7'" + - "}" + - "}"; - req = AutoScalingRequest.create(SolrRequest.METHOD.POST, resumeTriggerCommand); - response = solrClient.request(req); - assertEquals("success", response.get("result").toString()); - - assertEquals(1, listenerEvents.size()); - List events = listenerEvents.get("capturing7"); - assertNotNull(listenerEvents.toString(), events); - assertFalse("empty events?", events.isEmpty()); - CapturedEvent ev = events.get(0); - @SuppressWarnings({"unchecked"}) - List ops = (List< TriggerEvent.Op>)ev.event.properties.get(TriggerEvent.REQUESTED_OPS); - assertNotNull("no requested ops in " + ev, ops); - assertFalse("empty list of ops in " + ev, ops.isEmpty()); - Set parentShards = new HashSet<>(); - ops.forEach(op -> { - assertTrue(op.toString(), op.getAction() == CollectionParams.CollectionAction.SPLITSHARD); - @SuppressWarnings({"unchecked"}) - Collection> hints = (Collection>)op.getHints().get(Suggester.Hint.COLL_SHARD); - assertNotNull("no hints in op " + op, hints); - hints.forEach(h -> parentShards.add(h.second())); - }); - - // allow for recovery of at least some sub-shards - timeSource.sleep(TimeUnit.MILLISECONDS.convert(waitForSeconds + 1, TimeUnit.SECONDS)); - - coll = cloudManager.getClusterStateProvider().getCollection(collectionName); - - int checkedSubShards = 0; - - for (String parentShard : parentShards) { - for (String subShard : Arrays.asList(parentShard + "_0", parentShard + "_1")) { - leader = coll.getSlice(subShard).getLeader(); - if (leader == null) { - // no leader yet - skip it - } - checkedSubShards++; - replicaName = Utils.parseMetricsReplicaName(collectionName, leader.getCoreName()); - assertNotNull("replicaName could not be constructed from " + leader, replicaName); - final String subregistry = SolrCoreMetricManager.createRegistryName(true, collectionName, subShard, replicaName, null); - Set subtags = SimUtils.COMMON_REPLICA_TAGS.stream() - .map(s -> "metrics:" + subregistry + ":" + s).collect(Collectors.toSet()); - sizes = cloudManager.getNodeStateProvider().getNodeValues(leader.getNodeName(), subtags); - commitSizeTag = "metrics:" + subregistry + ":SEARCHER.searcher.indexCommitSize"; - numDocsTag = "metrics:" + subregistry + ":SEARCHER.searcher.numDocs"; - maxDocTag = "metrics:" + subregistry + ":SEARCHER.searcher.maxDoc"; - assertNotNull(sizes.toString(), sizes.get(commitSizeTag)); - assertNotNull(sizes.toString(), sizes.get(numDocsTag)); - assertNotNull(sizes.toString(), sizes.get(maxDocTag)); - long subCommitSize = ((Number)sizes.get(commitSizeTag)).longValue(); - long subMaxDoc = ((Number)sizes.get(maxDocTag)).longValue(); - long subNumDocs = ((Number)sizes.get(numDocsTag)).longValue(); - assertTrue("subNumDocs=" + subNumDocs + " should be less than subMaxDoc=" + subMaxDoc + - " due to link split", subNumDocs < subMaxDoc); - assertTrue("subCommitSize=" + subCommitSize + " should be still greater than aboveBytes=" + aboveBytes + - " due to link split", subCommitSize > aboveBytes); - // calculate estimated size using the same formula - long estimatedSize = IndexSizeTrigger.estimatedSize(subMaxDoc, subNumDocs, subCommitSize); - assertTrue("estimatedSize=" + estimatedSize + " should be lower than aboveBytes=" + aboveBytes, - estimatedSize < aboveBytes); - } - } - - assertTrue("didn't find any leaders in new sub-shards", checkedSubShards > 0); - - // reset & resume - listenerEvents.clear(); - finished = new CountDownLatch(1); - req = AutoScalingRequest.create(SolrRequest.METHOD.POST, resumeTriggerCommand); - response = solrClient.request(req); - assertEquals("success", response.get("result").toString()); - timeSource.sleep(TimeUnit.MILLISECONDS.convert(waitForSeconds + 1, TimeUnit.SECONDS)); - - // estimated shard size should fall well below the aboveBytes, even though the real commitSize - // still remains larger due to the splitMethod=link side-effects - await = finished.await(10000 / SPEED, TimeUnit.MILLISECONDS); - assertFalse("should not fire the trigger again! " + listenerEvents, await); - - } -} diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/IndexSizeTriggerTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/IndexSizeTriggerTest.java deleted file mode 100644 index 705c0e51939..00000000000 --- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/IndexSizeTriggerTest.java +++ /dev/null @@ -1,795 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.cloud.autoscaling; - -import static org.apache.solr.common.cloud.ZkStateReader.SOLR_AUTOSCALING_CONF_PATH; - -import java.lang.invoke.MethodHandles; -import java.lang.reflect.Field; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicReference; - -import org.apache.lucene.util.LuceneTestCase; -import org.apache.solr.client.solrj.SolrClient; -import org.apache.solr.client.solrj.SolrRequest; -import org.apache.solr.client.solrj.cloud.SolrCloudManager; -import org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig; -import org.apache.solr.client.solrj.cloud.autoscaling.Suggester; -import org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventProcessorStage; -import org.apache.solr.client.solrj.request.CollectionAdminRequest; -import org.apache.solr.cloud.CloudTestUtils.AutoScalingRequest; -import org.apache.solr.cloud.CloudUtil; -import org.apache.solr.cloud.SolrCloudTestCase; -import org.apache.solr.cloud.autoscaling.sim.SimCloudManager; -import org.apache.solr.common.SolrInputDocument; -import org.apache.solr.common.cloud.ZkNodeProps; -import org.apache.solr.common.params.CollectionParams; -import org.apache.solr.common.params.CommonAdminParams; -import org.apache.solr.common.util.NamedList; -import org.apache.solr.common.util.Pair; -import org.apache.solr.common.util.TimeSource; -import org.apache.solr.common.util.Utils; -import org.apache.solr.core.SolrResourceLoader; -import org.apache.solr.update.SolrIndexSplitter; -import org.apache.solr.util.LogLevel; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * - */ -@LogLevel("org.apache.solr.cloud.autoscaling=DEBUG") -@LuceneTestCase.Slow -public class IndexSizeTriggerTest extends SolrCloudTestCase { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - private static SolrCloudManager cloudManager; - private static SolrClient solrClient; - private static TimeSource timeSource; - private static SolrResourceLoader loader; - - private static int SPEED; - - private AutoScaling.TriggerEventProcessor noFirstRunProcessor = event -> { - fail("Did not expect the processor to fire on first run! event=" + event); - return true; - }; - private static final long WAIT_FOR_DELTA_NANOS = TimeUnit.MILLISECONDS.toNanos(2); - - static Map> listenerEvents = new ConcurrentHashMap<>(); - static CountDownLatch listenerCreated = new CountDownLatch(1); - static CountDownLatch finished = new CountDownLatch(1); - static boolean realCluster; - - @BeforeClass - public static void setupCluster() throws Exception { - configureCluster(2) - .addConfig("conf", configset("cloud-minimal")) - .configure(); - realCluster = random().nextBoolean(); - if (realCluster) { - cloudManager = cluster.getJettySolrRunner(0).getCoreContainer().getZkController().getSolrCloudManager(); - solrClient = cluster.getSolrClient(); - loader = cluster.getJettySolrRunner(0).getCoreContainer().getResourceLoader(); - SPEED = 1; - } else { - SPEED = 50; - cloudManager = SimCloudManager.createCluster(2, TimeSource.get("simTime:" + SPEED)); - // wait for defaults to be applied - due to accelerated time sometimes we may miss this - cloudManager.getTimeSource().sleep(10000); - AutoScalingConfig cfg = cloudManager.getDistribStateManager().getAutoScalingConfig(); - assertFalse("autoscaling config is empty", cfg.isEmpty()); - solrClient = ((SimCloudManager)cloudManager).simGetSolrClient(); - loader = ((SimCloudManager) cloudManager).getLoader(); - } - timeSource = cloudManager.getTimeSource(); - } - - @After - public void restoreDefaults() throws Exception { - if (!realCluster) { - if (log.isInfoEnabled()) { - log.info(((SimCloudManager) cloudManager).dumpClusterState(true)); - } - ((SimCloudManager) cloudManager).getSimClusterStateProvider().simDeleteAllCollections(); - ((SimCloudManager) cloudManager).simClearSystemCollection(); - ((SimCloudManager) cloudManager).getSimClusterStateProvider().simResetLeaderThrottles(); - ((SimCloudManager) cloudManager).simRestartOverseer(null); - cloudManager.getTimeSource().sleep(500); - ((SimCloudManager) cloudManager).simResetOpCounts(); - } else { - cluster.deleteAllCollections(); - } - cloudManager.getDistribStateManager().setData(SOLR_AUTOSCALING_CONF_PATH, Utils.toJSON(new ZkNodeProps()), -1); - cloudManager.getTimeSource().sleep(5000); - listenerEvents.clear(); - listenerCreated = new CountDownLatch(1); - finished = new CountDownLatch(1); - } - - @AfterClass - public static void teardown() throws Exception { - if (cloudManager != null && !realCluster) { - cloudManager.close(); - } - solrClient = null; - cloudManager = null; - loader = null; - } - - @Test - public void testTrigger() throws Exception { - String collectionName = "testTrigger_collection"; - CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(collectionName, - "conf", 2, 2); - create.process(solrClient); - - if (SPEED == 1) { - cluster.waitForActiveCollection(collectionName, 2, 4); - } else { - CloudUtil.waitForState(cloudManager, "failed to create " + collectionName, collectionName, - CloudUtil.clusterShape(2, 2, false, true)); - } - - long waitForSeconds = 3 + random().nextInt(5); - Map props = createTriggerProps(waitForSeconds); - try (IndexSizeTrigger trigger = new IndexSizeTrigger("index_size_trigger1")) { - trigger.configure(loader, cloudManager, props); - trigger.init(); - trigger.setProcessor(noFirstRunProcessor); - trigger.run(); - - for (int i = 0; i < 25; i++) { - SolrInputDocument doc = new SolrInputDocument("id", "id-" + i); - solrClient.add(collectionName, doc); - } - solrClient.commit(collectionName); - - AtomicBoolean fired = new AtomicBoolean(false); - AtomicReference eventRef = new AtomicReference<>(); - trigger.setProcessor(event -> { - if (fired.compareAndSet(false, true)) { - eventRef.set(event); - long currentTimeNanos = timeSource.getTimeNs(); - long eventTimeNanos = event.getEventTime(); - long waitForNanos = TimeUnit.NANOSECONDS.convert(waitForSeconds, TimeUnit.SECONDS) - WAIT_FOR_DELTA_NANOS; - if (currentTimeNanos - eventTimeNanos <= waitForNanos) { - fail("processor was fired before the configured waitFor period: currentTimeNanos=" + currentTimeNanos + ", eventTimeNanos=" + eventTimeNanos + ",waitForNanos=" + waitForNanos); - } - } else { - fail("IndexSizeTrigger was fired more than once!"); - } - return true; - }); - trigger.run(); - TriggerEvent ev = eventRef.get(); - // waitFor delay - should not produce any event yet - assertNull("waitFor not elapsed but produced an event", ev); - timeSource.sleep(TimeUnit.MILLISECONDS.convert(waitForSeconds + 1, TimeUnit.SECONDS)); - trigger.run(); - ev = eventRef.get(); - assertNotNull("should have fired an event", ev); - @SuppressWarnings({"unchecked"}) - List ops = (List) ev.getProperty(TriggerEvent.REQUESTED_OPS); - assertNotNull("should contain requestedOps", ops); - assertEquals("number of ops: " + ops, 2, ops.size()); - boolean shard1 = false; - boolean shard2 = false; - for (TriggerEvent.Op op : ops) { - assertEquals(CollectionParams.CollectionAction.SPLITSHARD, op.getAction()); - @SuppressWarnings({"unchecked"}) - Set> hints = (Set>)op.getHints().get(Suggester.Hint.COLL_SHARD); - assertNotNull("hints", hints); - assertEquals("hints", 1, hints.size()); - Pair p = hints.iterator().next(); - assertEquals(collectionName, p.first()); - if (p.second().equals("shard1")) { - shard1 = true; - } else if (p.second().equals("shard2")) { - shard2 = true; - } else { - fail("unexpected shard name " + p.second()); - } - @SuppressWarnings({"unchecked"}) - Map params = (Map)op.getHints().get(Suggester.Hint.PARAMS); - assertNotNull("params are null: " + op, params); - - // verify default split configs - assertEquals("splitMethod: " + op, SolrIndexSplitter.SplitMethod.LINK.toLower(), - params.get(CommonAdminParams.SPLIT_METHOD)); - assertEquals("splitByPrefix: " + op, false, params.get(CommonAdminParams.SPLIT_BY_PREFIX)); - } - assertTrue("shard1 should be split", shard1); - assertTrue("shard2 should be split", shard2); - } - } - - public static class CapturingTriggerListener extends TriggerListenerBase { - @Override - public void configure(SolrResourceLoader loader, SolrCloudManager cloudManager, AutoScalingConfig.TriggerListenerConfig config) throws TriggerValidationException { - super.configure(loader, cloudManager, config); - listenerCreated.countDown(); - } - - @Override - public synchronized void onEvent(TriggerEvent event, TriggerEventProcessorStage stage, String actionName, - ActionContext context, Throwable error, String message) { - List lst = listenerEvents.computeIfAbsent(config.name, s -> new ArrayList<>()); - CapturedEvent ev = new CapturedEvent(timeSource.getTimeNs(), context, config, stage, actionName, event, message); - log.info("=======> {}", ev); - lst.add(ev); - } - } - - public static class FinishedProcessingListener extends TriggerListenerBase { - - @Override - public void onEvent(TriggerEvent event, TriggerEventProcessorStage stage, String actionName, ActionContext context, Throwable error, String message) throws Exception { - finished.countDown(); - } - } - - @Test - public void testSplitIntegration() throws Exception { - String collectionName = "testSplitIntegration_collection"; - CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(collectionName, - "conf", 2, 2); - create.process(solrClient); - - if (SPEED == 1) { - cluster.waitForActiveCollection(collectionName, 2, 4); - } else { - CloudUtil.waitForState(cloudManager, "failed to create " + collectionName, collectionName, - CloudUtil.clusterShape(2, 2, false, true)); - } - - long waitForSeconds = 6 + random().nextInt(5); - // add disabled trigger - String setTriggerCommand = "{" + - "'set-trigger' : {" + - "'name' : 'index_size_trigger2'," + - "'event' : 'indexSize'," + - "'waitFor' : '" + waitForSeconds + "s'," + - "'aboveDocs' : 10," + - "'belowDocs' : 4," + - "'enabled' : false," + - "'actions' : [{'name' : 'compute_plan', 'class' : 'solr.ComputePlanAction'}," + - "{'name' : 'execute_plan', 'class' : '" + ExecutePlanAction.class.getName() + "'}]" + - "}}"; - @SuppressWarnings({"rawtypes"}) - SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand); - NamedList response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - - String setListenerCommand = "{" + - "'set-listener' : " + - "{" + - "'name' : 'capturing2'," + - "'trigger' : 'index_size_trigger2'," + - "'stage' : ['STARTED','ABORTED','SUCCEEDED','FAILED']," + - "'beforeAction' : ['compute_plan','execute_plan']," + - "'afterAction' : ['compute_plan','execute_plan']," + - "'class' : '" + CapturingTriggerListener.class.getName() + "'" + - "}" + - "}"; - req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setListenerCommand); - response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - - setListenerCommand = "{" + - "'set-listener' : " + - "{" + - "'name' : 'finished'," + - "'trigger' : 'index_size_trigger2'," + - "'stage' : ['SUCCEEDED']," + - "'class' : '" + FinishedProcessingListener.class.getName() + "'" + - "}" + - "}"; - req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setListenerCommand); - response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - - - for (int i = 0; i < 50; i++) { - SolrInputDocument doc = new SolrInputDocument("id", "id-" + i); - solrClient.add(collectionName, doc); - } - solrClient.commit(collectionName); - - // enable the trigger - String resumeTriggerCommand = "{" + - "'resume-trigger' : {" + - "'name' : 'index_size_trigger2'" + - "}" + - "}"; - req = AutoScalingRequest.create(SolrRequest.METHOD.POST, resumeTriggerCommand); - response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - - timeSource.sleep(TimeUnit.MILLISECONDS.convert(waitForSeconds + 1, TimeUnit.SECONDS)); - - boolean await = finished.await(60000, TimeUnit.MILLISECONDS); - assertTrue("did not finish processing in time", await); - CloudUtil.waitForState(cloudManager, collectionName, 20, TimeUnit.SECONDS, CloudUtil.clusterShape(6, 2, true, true)); - assertEquals(1, listenerEvents.size()); - List events = listenerEvents.get("capturing2"); - assertNotNull("'capturing2' events not found", events); - assertEquals("events: " + events, 6, events.size()); - assertEquals(TriggerEventProcessorStage.STARTED, events.get(0).stage); - assertEquals(TriggerEventProcessorStage.BEFORE_ACTION, events.get(1).stage); - assertEquals(TriggerEventProcessorStage.AFTER_ACTION, events.get(2).stage); - assertEquals(TriggerEventProcessorStage.BEFORE_ACTION, events.get(3).stage); - assertEquals(TriggerEventProcessorStage.AFTER_ACTION, events.get(4).stage); - assertEquals(TriggerEventProcessorStage.SUCCEEDED, events.get(5).stage); - // check ops - @SuppressWarnings({"unchecked"}) - List ops = (List) events.get(4).event.getProperty(TriggerEvent.REQUESTED_OPS); - assertNotNull("should contain requestedOps", ops); - assertEquals("number of ops", 2, ops.size()); - boolean shard1 = false; - boolean shard2 = false; - for (TriggerEvent.Op op : ops) { - assertEquals(CollectionParams.CollectionAction.SPLITSHARD, op.getAction()); - @SuppressWarnings({"unchecked"}) - Set> hints = (Set>)op.getHints().get(Suggester.Hint.COLL_SHARD); - assertNotNull("hints", hints); - assertEquals("hints", 1, hints.size()); - Pair p = hints.iterator().next(); - assertEquals(collectionName, p.first()); - if (p.second().equals("shard1")) { - shard1 = true; - } else if (p.second().equals("shard2")) { - shard2 = true; - } else { - fail("unexpected shard name " + p.second()); - } - } - - - if (events.size() == 6) { - assertTrue("shard1 should be split", shard1); - assertTrue("shard2 should be split", shard2); - } else { - assertTrue("shard1 or shard2 should be split", shard1 || shard2); - } - - } - - @Test - public void testMergeIntegration() throws Exception { - String collectionName = "testMergeIntegration_collection"; - CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(collectionName, - "conf", 2, 2); - create.process(solrClient); - - if (SPEED == 1) { - cluster.waitForActiveCollection(collectionName, 2, 4); - } else { - CloudUtil.waitForState(cloudManager, "failed to create " + collectionName, collectionName, - CloudUtil.clusterShape(2, 2, false, true)); - } - - for (int i = 0; i < 20; i++) { - SolrInputDocument doc = new SolrInputDocument("id", "id-" + (i * 100)); - solrClient.add(collectionName, doc); - } - solrClient.commit(collectionName); - - long waitForSeconds = 3 + random().nextInt(5); - String setTriggerCommand = "{" + - "'set-trigger' : {" + - "'name' : 'index_size_trigger3'," + - "'event' : 'indexSize'," + - "'waitFor' : '" + waitForSeconds + "s'," + - "'aboveDocs' : 40," + - "'belowDocs' : 4," + - "'enabled' : false," + - "'actions' : [{'name' : 'compute_plan', 'class' : 'solr.ComputePlanAction'}," + - "{'name' : 'execute_plan', 'class' : '" + ExecutePlanAction.class.getName() + "'}]" + - "}}"; - @SuppressWarnings({"rawtypes"}) - SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand); - NamedList response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - - String setListenerCommand = "{" + - "'set-listener' : " + - "{" + - "'name' : 'capturing3'," + - "'trigger' : 'index_size_trigger3'," + - "'stage' : ['STARTED','ABORTED','SUCCEEDED','FAILED']," + - "'beforeAction' : ['compute_plan','execute_plan']," + - "'afterAction' : ['compute_plan','execute_plan']," + - "'class' : '" + CapturingTriggerListener.class.getName() + "'" + - "}" + - "}"; - req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setListenerCommand); - response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - - setListenerCommand = "{" + - "'set-listener' : " + - "{" + - "'name' : 'finished'," + - "'trigger' : 'index_size_trigger3'," + - "'stage' : ['SUCCEEDED']," + - "'class' : '" + FinishedProcessingListener.class.getName() + "'" + - "}" + - "}"; - req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setListenerCommand); - response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - - // delete some docs to trigger a merge - for (int i = 0; i < 15; i++) { - solrClient.deleteById(collectionName, "id-" + (i * 100)); - } - solrClient.commit(collectionName); - - // enable the trigger - String resumeTriggerCommand = "{" + - "'resume-trigger' : {" + - "'name' : 'index_size_trigger3'" + - "}" + - "}"; - req = AutoScalingRequest.create(SolrRequest.METHOD.POST, resumeTriggerCommand); - response = solrClient.request(req); - assertEquals("success", response.get("result").toString()); - - timeSource.sleep(TimeUnit.MILLISECONDS.convert(waitForSeconds + 1, TimeUnit.SECONDS)); - - boolean await = finished.await(90000 / SPEED, TimeUnit.MILLISECONDS); - assertTrue("did not finish processing in time", await); - assertEquals(1, listenerEvents.size()); - List events = listenerEvents.get("capturing3"); - assertNotNull("'capturing3' events not found", events); - assertEquals("events: " + events, 6, events.size()); - assertEquals(TriggerEventProcessorStage.STARTED, events.get(0).stage); - assertEquals(TriggerEventProcessorStage.BEFORE_ACTION, events.get(1).stage); - assertEquals(TriggerEventProcessorStage.AFTER_ACTION, events.get(2).stage); - assertEquals(TriggerEventProcessorStage.BEFORE_ACTION, events.get(3).stage); - assertEquals(TriggerEventProcessorStage.AFTER_ACTION, events.get(4).stage); - assertEquals(TriggerEventProcessorStage.SUCCEEDED, events.get(5).stage); - // check ops - @SuppressWarnings({"unchecked"}) - List ops = (List) events.get(4).event.getProperty(TriggerEvent.REQUESTED_OPS); - assertNotNull("should contain requestedOps", ops); - assertTrue("number of ops: " + ops, ops.size() > 0); - for (TriggerEvent.Op op : ops) { - assertEquals(CollectionParams.CollectionAction.MERGESHARDS, op.getAction()); - @SuppressWarnings({"unchecked"}) - Set> hints = (Set>)op.getHints().get(Suggester.Hint.COLL_SHARD); - assertNotNull("hints", hints); - assertEquals("hints", 2, hints.size()); - Pair p = hints.iterator().next(); - assertEquals(collectionName, p.first()); - } - - // TODO: fix this once MERGESHARDS is supported - @SuppressWarnings({"unchecked"}) - List unsupportedOps = (List)events.get(2).context.get("properties.unsupportedOps"); - assertNotNull("should have unsupportedOps", unsupportedOps); - assertEquals(unsupportedOps.toString() + "\n" + ops, ops.size(), unsupportedOps.size()); - unsupportedOps.forEach(op -> assertEquals(CollectionParams.CollectionAction.MERGESHARDS, op.getAction())); - } - - @Test - @SuppressWarnings({"unchecked"}) - public void testMaxOps() throws Exception { - String collectionName = "testMaxOps_collection"; - CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(collectionName, - "conf", 5, 2); - create.process(solrClient); - - CloudUtil.waitForState(cloudManager, "failed to create " + collectionName, collectionName, - CloudUtil.clusterShape(5, 2, false, true)); - - long waitForSeconds = 3 + random().nextInt(5); - // add disabled trigger - String setTriggerCommand = "{" + - "'set-trigger' : {" + - "'name' : 'index_size_trigger5'," + - "'event' : 'indexSize'," + - "'waitFor' : '" + waitForSeconds + "s'," + - "'aboveDocs' : 10," + - "'enabled' : false," + - "'actions' : [{'name' : 'compute_plan', 'class' : 'solr.ComputePlanAction'}]" + - "}}"; - @SuppressWarnings({"rawtypes"}) - SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand); - NamedList response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - - String setListenerCommand = "{" + - "'set-listener' : " + - "{" + - "'name' : 'capturing5'," + - "'trigger' : 'index_size_trigger5'," + - "'stage' : ['STARTED','ABORTED','SUCCEEDED','FAILED']," + - "'beforeAction' : ['compute_plan']," + - "'afterAction' : ['compute_plan']," + - "'class' : '" + CapturingTriggerListener.class.getName() + "'" + - "}" + - "}"; - req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setListenerCommand); - response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - - setListenerCommand = "{" + - "'set-listener' : " + - "{" + - "'name' : 'finished'," + - "'trigger' : 'index_size_trigger5'," + - "'stage' : ['SUCCEEDED']," + - "'class' : '" + FinishedProcessingListener.class.getName() + "'" + - "}" + - "}"; - req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setListenerCommand); - response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - - - for (int i = 0; i < 200; i++) { - SolrInputDocument doc = new SolrInputDocument("id", "id-" + i); - solrClient.add(collectionName, doc); - } - solrClient.commit(collectionName); - - // enable the trigger - String resumeTriggerCommand = "{" + - "'resume-trigger' : {" + - "'name' : 'index_size_trigger5'" + - "}" + - "}"; - req = AutoScalingRequest.create(SolrRequest.METHOD.POST, resumeTriggerCommand); - response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - - timeSource.sleep(TimeUnit.MILLISECONDS.convert(waitForSeconds + 1, TimeUnit.SECONDS)); - - boolean await = finished.await(60000 / SPEED, TimeUnit.MILLISECONDS); - assertTrue("did not finish processing in time", await); - - // suspend the trigger - String suspendTriggerCommand = "{" + - "'suspend-trigger' : {" + - "'name' : 'index_size_trigger5'" + - "}" + - "}"; - req = AutoScalingRequest.create(SolrRequest.METHOD.POST, suspendTriggerCommand); - response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - - assertEquals(1, listenerEvents.size()); - List events = listenerEvents.get("capturing5"); - assertNotNull("'capturing5' events not found", events); - assertEquals("events: " + events, 4, events.size()); - assertEquals(TriggerEventProcessorStage.STARTED, events.get(0).stage); - assertEquals(TriggerEventProcessorStage.BEFORE_ACTION, events.get(1).stage); - assertEquals(TriggerEventProcessorStage.AFTER_ACTION, events.get(2).stage); - assertEquals(TriggerEventProcessorStage.SUCCEEDED, events.get(3).stage); - // check ops - List ops = (List) events.get(2).event.getProperty(TriggerEvent.REQUESTED_OPS); - assertNotNull("should contain requestedOps", ops); - assertEquals("number of ops: " + ops, 5, ops.size()); - - listenerEvents.clear(); - finished = new CountDownLatch(1); - - setTriggerCommand = "{" + - "'set-trigger' : {" + - "'name' : 'index_size_trigger5'," + - "'event' : 'indexSize'," + - "'waitFor' : '" + waitForSeconds + "s'," + - "'aboveDocs' : 10," + - "'maxOps' : 3," + - "'enabled' : true," + - "'actions' : [{'name' : 'compute_plan', 'class' : 'solr.ComputePlanAction'}]" + - "}}"; - req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand); - response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - - await = finished.await(60000 / SPEED, TimeUnit.MILLISECONDS); - assertTrue("did not finish processing in time", await); - - // suspend the trigger - suspendTriggerCommand = "{" + - "'suspend-trigger' : {" + - "'name' : 'index_size_trigger5'" + - "}" + - "}"; - req = AutoScalingRequest.create(SolrRequest.METHOD.POST, suspendTriggerCommand); - response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - - assertEquals(1, listenerEvents.size()); - events = listenerEvents.get("capturing5"); - assertNotNull("'capturing5' events not found", events); - assertEquals("events: " + events, 4, events.size()); - assertEquals(TriggerEventProcessorStage.STARTED, events.get(0).stage); - assertEquals(TriggerEventProcessorStage.BEFORE_ACTION, events.get(1).stage); - assertEquals(TriggerEventProcessorStage.AFTER_ACTION, events.get(2).stage); - assertEquals(TriggerEventProcessorStage.SUCCEEDED, events.get(3).stage); - // check ops - ops = (List) events.get(2).event.getProperty(TriggerEvent.REQUESTED_OPS); - assertNotNull("should contain requestedOps", ops); - assertEquals("number of ops: " + ops, 3, ops.size()); - } - - //test that split parameters can be overridden - @Test - public void testSplitConfig() throws Exception { - String collectionName = "testSplitConfig_collection"; - CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(collectionName, - "conf", 2, 2); - create.process(solrClient); - CloudUtil.waitForState(cloudManager, "failed to create " + collectionName, collectionName, - CloudUtil.clusterShape(2, 2, false, true)); - - long waitForSeconds = 3 + random().nextInt(5); - Map props = createTriggerProps(waitForSeconds); - props.put(CommonAdminParams.SPLIT_METHOD, SolrIndexSplitter.SplitMethod.REWRITE.toLower()); - props.put(IndexSizeTrigger.SPLIT_BY_PREFIX, true); - - try (IndexSizeTrigger trigger = new IndexSizeTrigger("index_size_trigger6")) { - trigger.configure(loader, cloudManager, props); - trigger.init(); - trigger.setProcessor(noFirstRunProcessor); - trigger.run(); - - for (int i = 0; i < 25; i++) { - SolrInputDocument doc = new SolrInputDocument("id", "id-" + i); - solrClient.add(collectionName, doc); - } - solrClient.commit(collectionName); - - AtomicBoolean fired = new AtomicBoolean(false); - AtomicReference eventRef = new AtomicReference<>(); - trigger.setProcessor(event -> { - if (fired.compareAndSet(false, true)) { - eventRef.set(event); - long currentTimeNanos = timeSource.getTimeNs(); - long eventTimeNanos = event.getEventTime(); - long waitForNanos = TimeUnit.NANOSECONDS.convert(waitForSeconds, TimeUnit.SECONDS) - WAIT_FOR_DELTA_NANOS; - if (currentTimeNanos - eventTimeNanos <= waitForNanos) { - fail("processor was fired before the configured waitFor period: currentTimeNanos=" + currentTimeNanos + ", eventTimeNanos=" + eventTimeNanos + ",waitForNanos=" + waitForNanos); - } - } else { - fail("IndexSizeTrigger was fired more than once!"); - } - return true; - }); - trigger.run(); - TriggerEvent ev = eventRef.get(); - // waitFor delay - should not produce any event yet - assertNull("waitFor not elapsed but produced an event", ev); - timeSource.sleep(TimeUnit.MILLISECONDS.convert(waitForSeconds + 1, TimeUnit.SECONDS)); - trigger.run(); - ev = eventRef.get(); - assertNotNull("should have fired an event", ev); - @SuppressWarnings({"unchecked"}) - List ops = (List) ev.getProperty(TriggerEvent.REQUESTED_OPS); - assertNotNull("should contain requestedOps", ops); - assertEquals("number of ops: " + ops, 2, ops.size()); - boolean shard1 = false; - boolean shard2 = false; - for (TriggerEvent.Op op : ops) { - assertEquals(CollectionParams.CollectionAction.SPLITSHARD, op.getAction()); - @SuppressWarnings({"unchecked"}) - Set> hints = (Set>)op.getHints().get(Suggester.Hint.COLL_SHARD); - assertNotNull("hints", hints); - assertEquals("hints", 1, hints.size()); - Pair p = hints.iterator().next(); - assertEquals(collectionName, p.first()); - if (p.second().equals("shard1")) { - shard1 = true; - } else if (p.second().equals("shard2")) { - shard2 = true; - } else { - fail("unexpected shard name " + p.second()); - } - @SuppressWarnings({"unchecked"}) - Map params = (Map)op.getHints().get(Suggester.Hint.PARAMS); - assertNotNull("params are null: " + op, params); - - // verify overrides for split config - assertEquals("splitMethod: " + op, SolrIndexSplitter.SplitMethod.REWRITE.toLower(), - params.get(CommonAdminParams.SPLIT_METHOD)); - assertEquals("splitByPrefix: " + op, true, params.get(CommonAdminParams.SPLIT_BY_PREFIX)); - } - assertTrue("shard1 should be split", shard1); - assertTrue("shard2 should be split", shard2); - } - - } - - //validates that trigger configuration will fail for invalid split configs - @Test - public void testInvalidSplitConfig() throws Exception { - long waitForSeconds = 3 + random().nextInt(5); - Map props = createTriggerProps(waitForSeconds); - props.put(IndexSizeTrigger.SPLIT_BY_PREFIX, "hello"); - - try (IndexSizeTrigger trigger = new IndexSizeTrigger("index_size_trigger7")) { - trigger.configure(loader, cloudManager, props); - fail("Trigger configuration should have failed with invalid property."); - } catch (TriggerValidationException e) { - assertTrue(e.getDetails().containsKey(IndexSizeTrigger.SPLIT_BY_PREFIX)); - } - - props.put(IndexSizeTrigger.SPLIT_BY_PREFIX, true); - props.put(CommonAdminParams.SPLIT_METHOD, "hello"); - try (IndexSizeTrigger trigger = new IndexSizeTrigger("index_size_trigger8")) { - trigger.configure(loader, cloudManager, props); - fail("Trigger configuration should have failed with invalid property."); - } catch (TriggerValidationException e) { - assertTrue(e.getDetails().containsKey(IndexSizeTrigger.SPLIT_METHOD_PROP)); - } - } - - // make sure all defined properties are added to valid properties (SOLR-13264) - @Test - public void testValidProperties() throws Exception { - - final Set propFields = new HashSet<>(); - - final TriggerBase trigger = new IndexSizeTrigger("index_size_trigger"); - for (final Field field : trigger.getClass().getFields()) { - if (field.getName().endsWith("_PROP")) { - propFields.add(field.get(trigger).toString()); - } - } - propFields.removeAll(trigger.getValidProperties()); - - assertTrue("Invalid _PROP constants: "+propFields.toString(), propFields.isEmpty()); - } - - private Map createTriggerProps(long waitForSeconds) { - Map props = new HashMap<>(); - props.put("event", "indexSize"); - props.put("waitFor", waitForSeconds); - props.put("enabled", true); - props.put(IndexSizeTrigger.ABOVE_DOCS_PROP, 10); - props.put(IndexSizeTrigger.BELOW_DOCS_PROP, 2); - List> actions = new ArrayList<>(3); - Map map = new HashMap<>(2); - map.put("name", "compute_plan"); - map.put("class", "solr.ComputePlanAction"); - actions.add(map); - map = new HashMap<>(2); - map.put("name", "execute_plan"); - map.put("class", "solr.ExecutePlanAction"); - actions.add(map); - props.put("actions", actions); - return props; - } -} diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/MetricTriggerIntegrationTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/MetricTriggerIntegrationTest.java deleted file mode 100644 index e35a23052e1..00000000000 --- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/MetricTriggerIntegrationTest.java +++ /dev/null @@ -1,242 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.cloud.autoscaling; - -import java.lang.invoke.MethodHandles; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; - -import org.apache.solr.client.solrj.SolrRequest; -import org.apache.solr.client.solrj.cloud.SolrCloudManager; -import org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig; -import org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventProcessorStage; -import org.apache.solr.client.solrj.impl.CloudSolrClient; -import org.apache.solr.client.solrj.request.CollectionAdminRequest; -import org.apache.solr.cloud.CloudTestUtils; -import org.apache.solr.cloud.CloudTestUtils.AutoScalingRequest; -import org.apache.solr.cloud.SolrCloudTestCase; -import org.apache.solr.common.SolrInputDocument; -import org.apache.solr.common.cloud.DocCollection; -import org.apache.solr.common.cloud.Replica; -import org.apache.solr.common.util.NamedList; -import org.apache.solr.common.util.TimeSource; -import org.apache.solr.common.util.Utils; -import org.apache.solr.core.SolrResourceLoader; -import org.apache.solr.metrics.SolrCoreMetricManager; -import org.apache.solr.util.LogLevel; -import org.junit.BeforeClass; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import static org.apache.solr.cloud.autoscaling.TriggerIntegrationTest.WAIT_FOR_DELTA_NANOS; - -/** - * Integration test for {@link MetricTrigger} - */ -@LogLevel("org.apache.solr.cloud.autoscaling=DEBUG;org.apache.solr.client.solrj.cloud.autoscaling=DEBUG") -public class MetricTriggerIntegrationTest extends SolrCloudTestCase { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - private static final TimeSource timeSource = TimeSource.NANO_TIME; - - static final Map> listenerEvents = new HashMap<>(); - private static CountDownLatch triggerFiredLatch; - private static int waitForSeconds = 1; - - @BeforeClass - public static void setupCluster() throws Exception { - configureCluster(2) - .addConfig("conf", configset("cloud-minimal")) - .configure(); - - // disable .scheduled_maintenance (once it exists) - CloudTestUtils.waitForTriggerToBeScheduled(cluster.getOpenOverseer().getSolrCloudManager(), ".scheduled_maintenance"); - CloudTestUtils.suspendTrigger(cluster.getOpenOverseer().getSolrCloudManager(), ".scheduled_maintenance"); - - listenerEvents.clear(); - triggerFiredLatch = new CountDownLatch(1); - } - - @Test - // commented 4-Sep-2018 @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2-Aug-2018 - // commented out on: 24-Dec-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 14-Oct-2018 - public void testMetricTrigger() throws Exception { - String collectionName = "testMetricTrigger"; - CloudSolrClient solrClient = cluster.getSolrClient(); - CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(collectionName, - "conf", 2, 2); - create.process(solrClient); - solrClient.setDefaultCollection(collectionName); - - cluster.waitForActiveCollection(collectionName, 2, 4); - - DocCollection docCollection = solrClient.getZkStateReader().getClusterState().getCollection(collectionName); - String shardId = "shard1"; - Replica replica = docCollection.getSlice(shardId).getReplicas().iterator().next(); - String coreName = replica.getCoreName(); - String replicaName = Utils.parseMetricsReplicaName(collectionName, coreName); - long waitForSeconds = 2 + random().nextInt(5); - String registry = SolrCoreMetricManager.createRegistryName(true, collectionName, shardId, replicaName, null); - String tag = "metrics:" + registry + ":INDEX.sizeInBytes"; - - String setTriggerCommand = "{" + - "'set-trigger' : {" + - "'name' : 'metric_trigger'," + - "'event' : 'metric'," + - "'waitFor' : '" + waitForSeconds + "s'," + - "'enabled' : true," + - "'metric': '" + tag + "'" + - "'above' : 100.0," + - "'collection': '" + collectionName + "'" + - "'shard':'" + shardId + "'" + - "'actions' : [" + - "{'name':'compute','class':'" + ComputePlanAction.class.getName() + "'}," + - "{'name':'execute','class':'" + ExecutePlanAction.class.getName() + "'}," + - "{'name':'test','class':'" + MetricAction.class.getName() + "'}" + - "]" + - "}}"; - @SuppressWarnings({"rawtypes"}) - SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand); - NamedList response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - - String setListenerCommand1 = "{" + - "'set-listener' : " + - "{" + - "'name' : 'srt'," + - "'trigger' : 'metric_trigger'," + - "'stage' : ['FAILED','SUCCEEDED']," + - "'afterAction': ['compute', 'execute', 'test']," + - "'class' : '" + TestTriggerListener.class.getName() + "'" + - "}" + - "}"; - req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setListenerCommand1); - response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - - // start more nodes so that we have at least 4 - for (int i = cluster.getJettySolrRunners().size(); i < 4; i++) { - cluster.startJettySolrRunner(); - } - cluster.waitForAllNodes(10); - - List docs = new ArrayList<>(500); - for (int i = 0; i < 500; i++) { - docs.add(new SolrInputDocument("id", String.valueOf(i), "x_s", "x" + i)); - } - solrClient.add(docs); - solrClient.commit(); - - boolean await = triggerFiredLatch.await(20, TimeUnit.SECONDS); - assertTrue("The trigger did not fire at all", await); - // wait for listener to capture the SUCCEEDED stage - Thread.sleep(2000); - assertEquals(listenerEvents.toString(), 4, listenerEvents.get("srt").size()); - CapturedEvent ev = listenerEvents.get("srt").get(0); - long now = timeSource.getTimeNs(); - // verify waitFor - assertTrue(TimeUnit.SECONDS.convert(waitForSeconds, TimeUnit.NANOSECONDS) - WAIT_FOR_DELTA_NANOS <= now - ev.event.getEventTime()); - assertEquals(collectionName, ev.event.getProperties().get("collection")); - - // find a new replica and create its metric name - docCollection = solrClient.getZkStateReader().getClusterState().getCollection(collectionName); - replica = docCollection.getSlice(shardId).getReplicas().iterator().next(); - coreName = replica.getCoreName(); - replicaName = Utils.parseMetricsReplicaName(collectionName, coreName); - registry = SolrCoreMetricManager.createRegistryName(true, collectionName, shardId, replicaName, null); - tag = "metrics:" + registry + ":INDEX.sizeInBytes"; - - triggerFiredLatch = new CountDownLatch(1); - listenerEvents.clear(); - - setTriggerCommand = "{" + - "'set-trigger' : {" + - "'name' : 'metric_trigger'," + - "'event' : 'metric'," + - "'waitFor' : '" + waitForSeconds + "s'," + - "'enabled' : true," + - "'metric': '" + tag + "'," + - "'above' : 100.0," + - "'collection': '" + collectionName + "'," + - "'shard':'" + shardId + "'," + - "'preferredOperation':'addreplica'," + - "'actions' : [" + - "{'name':'compute','class':'" + ComputePlanAction.class.getName() + "'}," + - "{'name':'execute','class':'" + ExecutePlanAction.class.getName() + "'}," + - "{'name':'test','class':'" + MetricAction.class.getName() + "'}" + - "]" + - "}}"; - req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand); - response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - - await = triggerFiredLatch.await(20, TimeUnit.SECONDS); - assertTrue("The trigger did not fire at all", await); - // wait for listener to capture the SUCCEEDED stage - Thread.sleep(2000); - assertEquals(listenerEvents.toString(), 4, listenerEvents.get("srt").size()); - ev = listenerEvents.get("srt").get(0); - now = timeSource.getTimeNs(); - // verify waitFor - assertTrue(TimeUnit.SECONDS.convert(waitForSeconds, TimeUnit.NANOSECONDS) - WAIT_FOR_DELTA_NANOS <= now - ev.event.getEventTime()); - assertEquals(collectionName, ev.event.getProperties().get("collection")); - docCollection = solrClient.getZkStateReader().getClusterState().getCollection(collectionName); - assertEquals(5, docCollection.getReplicas().size()); - } - - public static class MetricAction extends TriggerActionBase { - - @Override - public void process(TriggerEvent event, ActionContext context) throws Exception { - try { - long currentTimeNanos = context.getCloudManager().getTimeSource().getTimeNs(); - long eventTimeNanos = event.getEventTime(); - long waitForNanos = TimeUnit.NANOSECONDS.convert(waitForSeconds, TimeUnit.SECONDS) - WAIT_FOR_DELTA_NANOS; - if (currentTimeNanos - eventTimeNanos <= waitForNanos) { - fail(event.source + " was fired before the configured waitFor period"); - } - triggerFiredLatch.countDown(); - } catch (Throwable t) { - log.debug("--throwable", t); - throw t; - } - } - } - - public static class TestTriggerListener extends TriggerListenerBase { - private TimeSource timeSource; - @Override - public void configure(SolrResourceLoader loader, SolrCloudManager cloudManager, AutoScalingConfig.TriggerListenerConfig config) throws TriggerValidationException { - super.configure(loader, cloudManager, config); - timeSource = cloudManager.getTimeSource(); - } - - @Override - public synchronized void onEvent(TriggerEvent event, TriggerEventProcessorStage stage, String actionName, - ActionContext context, Throwable error, String message) { - List lst = listenerEvents.computeIfAbsent(config.name, s -> new ArrayList<>()); - lst.add(new CapturedEvent(timeSource.getTimeNs(), context, config, stage, actionName, event, message)); - - } - } -} diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/MetricTriggerTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/MetricTriggerTest.java deleted file mode 100644 index ce00d393848..00000000000 --- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/MetricTriggerTest.java +++ /dev/null @@ -1,135 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.cloud.autoscaling; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import org.apache.solr.client.solrj.cloud.SolrCloudManager; -import org.apache.solr.client.solrj.impl.CloudSolrClient; -import org.apache.solr.client.solrj.impl.SolrClientCloudManager; -import org.apache.solr.client.solrj.request.CollectionAdminRequest; -import org.apache.solr.cloud.SolrCloudTestCase; -import org.apache.solr.cloud.ZkDistributedQueueFactory; -import org.apache.solr.common.cloud.SolrZkClient; -import org.apache.solr.common.util.Utils; -import org.apache.solr.core.CoreDescriptor; -import org.apache.solr.core.SolrResourceLoader; -import org.apache.solr.metrics.SolrCoreMetricManager; -import org.junit.BeforeClass; -import org.junit.Test; - -public class MetricTriggerTest extends SolrCloudTestCase { - - private AutoScaling.TriggerEventProcessor noFirstRunProcessor = event -> { - fail("Did not expect the listener to fire on first run!"); - return true; - }; - - @BeforeClass - public static void setupCluster() throws Exception { - configureCluster(1) - .addConfig("conf", configset("cloud-minimal")) - .configure(); - CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(DEFAULT_TEST_COLLECTION_NAME, - "conf", 1, 1); - CloudSolrClient solrClient = cluster.getSolrClient(); - create.process(solrClient); - cluster.waitForActiveCollection(DEFAULT_TEST_COLLECTION_NAME, 1, 1); - } - - @Test - public void test() throws Exception { - CoreDescriptor coreDescriptor = cluster.getJettySolrRunner(0).getCoreContainer().getCoreDescriptors().iterator().next(); - String shardId = coreDescriptor.getCloudDescriptor().getShardId(); - String coreName = coreDescriptor.getName(); - String replicaName = Utils.parseMetricsReplicaName(DEFAULT_TEST_COLLECTION_NAME, coreName); - long waitForSeconds = 2 + random().nextInt(5); - String registry = SolrCoreMetricManager.createRegistryName(true, DEFAULT_TEST_COLLECTION_NAME, shardId, replicaName, null); - String tag = "metrics:" + registry + ":ADMIN./admin/file.requests"; - - Map props = createTriggerProps(waitForSeconds, tag, 1.0d, null, DEFAULT_TEST_COLLECTION_NAME, null, null); - - final List events = new ArrayList<>(); - SolrZkClient zkClient = cluster.getSolrClient().getZkStateReader().getZkClient(); - SolrResourceLoader loader = cluster.getJettySolrRunner(0).getCoreContainer().getResourceLoader(); - try (SolrCloudManager cloudManager = new SolrClientCloudManager(new ZkDistributedQueueFactory(zkClient), cluster.getSolrClient())) { - try (MetricTrigger metricTrigger = new MetricTrigger("metricTrigger")) { - metricTrigger.configure(loader, cloudManager, props); - metricTrigger.setProcessor(noFirstRunProcessor); - metricTrigger.run(); - metricTrigger.setProcessor(event -> events.add(event)); - assertEquals(0, events.size()); - Thread.sleep(waitForSeconds * 1000 + 2000); - metricTrigger.run(); - assertEquals(1, events.size()); - } - - events.clear(); - tag = "metrics:" + registry + ":ADMIN./admin/file.handlerStart"; - props = createTriggerProps(waitForSeconds, tag, null, 100.0d, DEFAULT_TEST_COLLECTION_NAME, null, null); - try (MetricTrigger metricTrigger = new MetricTrigger("metricTrigger")) { - metricTrigger.configure(loader, cloudManager, props); - metricTrigger.setProcessor(noFirstRunProcessor); - metricTrigger.run(); - metricTrigger.setProcessor(event -> events.add(event)); - assertEquals(0, events.size()); - Thread.sleep(waitForSeconds * 1000 + 2000); - metricTrigger.run(); - assertEquals(1, events.size()); - } - } - } - - private Map createTriggerProps(long waitForSeconds, String metric, Double below, Double above, String collection, String shard, String node) { - Map props = new HashMap<>(); - props.put("metric", metric); - if (above != null) { - props.put("above", above); - } - if (below != null) { - props.put("below", below); - } - if (collection != null) { - props.put("collection", collection); - } - if (shard != null) { - props.put("shard", shard); - } - if (node != null) { - props.put("node", node); - } - props.put("event", "metric"); - props.put("waitFor", waitForSeconds); - props.put("enabled", true); - - List> actions = new ArrayList<>(3); - Map map = new HashMap<>(2); - map.put("name", "compute_plan"); - map.put("class", "solr.ComputePlanAction"); - actions.add(map); - map = new HashMap<>(2); - map.put("name", "execute_plan"); - map.put("class", "solr.ExecutePlanAction"); - actions.add(map); - props.put("actions", actions); - return props; - } -} diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/NodeAddedTriggerIntegrationTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/NodeAddedTriggerIntegrationTest.java deleted file mode 100644 index bbd24826b48..00000000000 --- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/NodeAddedTriggerIntegrationTest.java +++ /dev/null @@ -1,320 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.cloud.autoscaling; - -import java.lang.invoke.MethodHandles; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; - -import org.apache.solr.client.solrj.cloud.SolrCloudManager; -import org.apache.solr.client.solrj.embedded.JettySolrRunner; -import org.apache.solr.client.solrj.impl.CloudSolrClient; -import org.apache.solr.cloud.CloudTestUtils; -import org.apache.solr.cloud.Overseer; -import org.apache.solr.cloud.SolrCloudTestCase; -import org.apache.solr.common.cloud.ZkNodeProps; -import org.apache.solr.common.cloud.ZkStateReader; -import org.apache.solr.common.util.Utils; -import org.apache.solr.util.LogLevel; -import org.apache.solr.util.TimeOut; -import org.apache.zookeeper.data.Stat; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import static org.apache.solr.cloud.autoscaling.TriggerIntegrationTest.WAIT_FOR_DELTA_NANOS; -import static org.apache.solr.common.cloud.ZkStateReader.SOLR_AUTOSCALING_CONF_PATH; - -// TODO: this class shares duplicated code with NodeLostTriggerIntegrationTest ... merge? - -@LogLevel("org.apache.solr.cloud.autoscaling=DEBUG;org.apache.solr.client.solrj.cloud.autoscaling=DEBUG") -public class NodeAddedTriggerIntegrationTest extends SolrCloudTestCase { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - private static volatile CountDownLatch actionConstructorCalled; - private static volatile CountDownLatch actionInitCalled; - private static volatile CountDownLatch triggerFiredLatch; - private static volatile int waitForSeconds = 1; - private static volatile AtomicBoolean triggerFired; - private static volatile Set events = ConcurrentHashMap.newKeySet(); - private static volatile SolrCloudManager cloudManager; - - @After - public void after() throws Exception { - shutdownCluster(); - } - - @AfterClass - public static void cleanUpAfterClass() throws Exception { - cloudManager = null; - } - - private static CountDownLatch getTriggerFiredLatch() { - return triggerFiredLatch; - } - - @Before - public void setupTest() throws Exception { - configureCluster(2) - .addConfig("conf", configset("cloud-minimal")) - .configure(); - - final Overseer overseer = cluster.getOpenOverseer(); - assertNotNull(overseer); - cloudManager = overseer.getSolrCloudManager(); - assertNotNull(cloudManager); - - // disable .scheduled_maintenance (once it exists) - CloudTestUtils.waitForTriggerToBeScheduled(cloudManager, ".scheduled_maintenance"); - CloudTestUtils.suspendTrigger(cloudManager, ".scheduled_maintenance"); - - // aggressively remove all active scheduled triggers - final ScheduledTriggers scheduledTriggers = ((OverseerTriggerThread) overseer.getTriggerThread().getThread()).getScheduledTriggers(); - // TODO: is this really safe? is it possible overseer is still in process of adding some to schedule? - scheduledTriggers.removeAll(); - - // clear any persisted auto scaling configuration - Stat stat = zkClient().setData(SOLR_AUTOSCALING_CONF_PATH, Utils.toJSON(new ZkNodeProps()), true); - if (log.isInfoEnabled()) { - log.info("{} reset, new znode version {}", SOLR_AUTOSCALING_CONF_PATH, stat.getVersion()); - } - - cluster.getSolrClient().setDefaultCollection(null); - - waitForSeconds = 1 + random().nextInt(3); - actionConstructorCalled = new CountDownLatch(1); - actionInitCalled = new CountDownLatch(1); - triggerFiredLatch = new CountDownLatch(1); - triggerFired = new AtomicBoolean(false); - events.clear(); - - // clear any events or markers - // todo: consider the impact of such cleanup on regular cluster restarts - deleteChildrenRecursively(ZkStateReader.SOLR_AUTOSCALING_EVENTS_PATH); - deleteChildrenRecursively(ZkStateReader.SOLR_AUTOSCALING_TRIGGER_STATE_PATH); - deleteChildrenRecursively(ZkStateReader.SOLR_AUTOSCALING_NODE_LOST_PATH); - deleteChildrenRecursively(ZkStateReader.SOLR_AUTOSCALING_NODE_ADDED_PATH); - } - - private void deleteChildrenRecursively(String path) throws Exception { - cloudManager.getDistribStateManager().removeRecursively(path, true, false); - } - - @Test - @SuppressWarnings({"unchecked"}) - public void testNodeAddedTriggerRestoreState() throws Exception { - - final String triggerName = "node_added_restore_trigger"; - - // should be enough to ensure trigger doesn't fire any actions until we replace the trigger - waitForSeconds = 500000; - CloudTestUtils.assertAutoScalingRequest - (cloudManager, - "{" + - "'set-trigger' : {" + - "'name' : '"+triggerName+"'," + - "'event' : 'nodeAdded'," + - "'waitFor' : '"+waitForSeconds+"s'," + - "'enabled' : true," + - "'actions' : [{'name':'test','class':'" + TestTriggerAction.class.getName() + "'}]" + - "}}"); - - assertTrue("Trigger was not init()ed even after await()ing an excessive amount of time", - actionInitCalled.await(60, TimeUnit.SECONDS)); - - // start a new node - final JettySolrRunner newNode = cluster.startJettySolrRunner(); - final String nodeName = newNode.getNodeName(); - - // poll the internal state of the trigger until it run()s at least once and updates - // it's internal state to know the node we added is live - // - // (this should run roughly once a second) - (new TimeOut(30, TimeUnit.SECONDS, cloudManager.getTimeSource())) - .waitFor("initial trigger never ran to detect new live node", () -> - (((Collection) getTriggerState(triggerName).get("lastLiveNodes")) - .contains(nodeName))); - - // since we know the nodeAdded event has been detected, we can recored the current timestamp - // (relative to the cluster's time source) and later assert that (restored state) correctly - // tracked that the event happened prior to "now" - final long maxEventTimeNs = cloudManager.getTimeSource().getTimeNs(); - - // - // now replace the trigger with a new instance to test that the state gets copied over correctly - // - - // reset the actionInitCalled counter so we can confirm the second instances is inited - actionInitCalled = new CountDownLatch(1); - // use a low waitTime to ensure it processes the event quickly. - // (this updated property also ensures the set-trigger won't be treated as a No-Op) - waitForSeconds = 0 + random().nextInt(3); - CloudTestUtils.assertAutoScalingRequest - (cloudManager, - "{" + - "'set-trigger' : {" + - "'name' : '"+triggerName+"'," + - "'event' : 'nodeAdded'," + - "'waitFor' : '"+waitForSeconds+"s'," + - "'enabled' : true," + - "'actions' : [{'name':'test','class':'" + TestTriggerAction.class.getName() + "'}]" + - "}}"); - - assertTrue("Trigger was not init()ed even after await()ing an excessive amount of time", - actionInitCalled.await(60, TimeUnit.SECONDS)); - - // the trigger actions should now (eventually) record that the node was added - assertTrue("Second instance of our trigger never fired the action to process the event", - triggerFiredLatch.await(30, TimeUnit.SECONDS)); - - assertEquals("Wrong number of events recorded: " + events.toString(), - 1, events.size()); - - final TriggerEvent event = events.iterator().next(); - assertNotNull("null event???", event); - assertTrue("Event should have been a nodeAdded event: " + event.getClass(), - event instanceof NodeAddedTrigger.NodeAddedEvent); - - assertNotNull("event is missing NODE_NAMES: " + event, event.getProperty(TriggerEvent.NODE_NAMES)); - assertEquals("event has incorrect NODE_NAMES: " + event, - Collections.singletonList(nodeName), - event.getProperty(TriggerEvent.NODE_NAMES)); - - assertTrue("event TS is too late, should be before (max) expected TS @ " - + maxEventTimeNs + ": " + event, - event.getEventTime() < maxEventTimeNs); - - assertNotNull("event is missing EVENT_TIMES: " + event, event.getProperty(TriggerEvent.EVENT_TIMES)); - assertEquals("event has unexpeted number of EVENT_TIMES: " + event, - 1, ((Collection)event.getProperty(TriggerEvent.EVENT_TIMES)).size()); - assertEquals("event's TS doesn't match EVENT_TIMES: " + event, - event.getEventTime(), - ((Collection)event.getProperty(TriggerEvent.EVENT_TIMES)).iterator().next()); - } - - @Test - public void testNodeAddedTrigger() throws Exception { - CloudSolrClient solrClient = cluster.getSolrClient(); - CloudTestUtils.assertAutoScalingRequest - (cloudManager, - "{" + - "'set-trigger' : {" + - "'name' : 'node_added_trigger'," + - "'event' : 'nodeAdded'," + - "'waitFor' : '" + waitForSeconds + "s'," + - "'enabled' : true," + - "'actions' : [{'name':'test','class':'" + TestTriggerAction.class.getName() + "'}]" + - "}}"); - - if (!actionInitCalled.await(3, TimeUnit.SECONDS)) { - fail("The TriggerAction should have been created by now"); - } - - JettySolrRunner newNode = cluster.startJettySolrRunner(); - cluster.waitForAllNodes(15); - boolean await = triggerFiredLatch.await(20, TimeUnit.SECONDS); - assertTrue("The trigger did not fire at all", await); - assertTrue(triggerFired.get()); - NodeAddedTrigger.NodeAddedEvent nodeAddedEvent = (NodeAddedTrigger.NodeAddedEvent) events.iterator().next(); - assertNotNull(nodeAddedEvent); - @SuppressWarnings({"unchecked"}) - List nodeNames = (List) nodeAddedEvent.getProperty(TriggerEvent.NODE_NAMES); - assertTrue(nodeNames.contains(newNode.getNodeName())); - - // reset - actionConstructorCalled = new CountDownLatch(1); - actionInitCalled = new CountDownLatch(1); - - // update the trigger with exactly the same data - CloudTestUtils.assertAutoScalingRequest - (cloudManager, - "{" + - "'set-trigger' : {" + - "'name' : 'node_added_trigger'," + - "'event' : 'nodeAdded'," + - "'waitFor' : '" + waitForSeconds + "s'," + - "'enabled' : true," + - "'actions' : [{'name':'test','class':'" + TestTriggerAction.class.getName() + "'}]" + - "}}"); - - // this should be a no-op so the action should have been created but init should not be called - if (!actionConstructorCalled.await(3, TimeUnit.SECONDS)) { - fail("The TriggerAction should have been created by now"); - } - - assertFalse(actionInitCalled.await(2, TimeUnit.SECONDS)); - } - - public static class TestTriggerAction extends TriggerActionBase { - - public TestTriggerAction() { - actionConstructorCalled.countDown(); - } - - @Override - public void process(TriggerEvent event, ActionContext actionContext) { - try { - if (triggerFired.compareAndSet(false, true)) { - events.add(event); - long currentTimeNanos = actionContext.getCloudManager().getTimeSource().getTimeNs(); - long eventTimeNanos = event.getEventTime(); - long waitForNanos = TimeUnit.NANOSECONDS.convert(waitForSeconds, TimeUnit.SECONDS) - WAIT_FOR_DELTA_NANOS; - if (currentTimeNanos - eventTimeNanos <= waitForNanos) { - fail(event.source + " was fired before the configured waitFor period"); - } - getTriggerFiredLatch().countDown(); - } else { - fail(event.source + " was fired more than once!"); - } - } catch (Throwable t) { - log.debug("--throwable", t); - throw t; - } - } - - @Override - public void init() throws Exception { - log.info("TestTriggerAction init"); - actionInitCalled.countDown(); - super.init(); - } - } - - /** - * Helper method for getting a copy of the current (internal) trigger state of a scheduled trigger. - */ - private Map getTriggerState(final String name) { - final Overseer overseer = cluster.getOpenOverseer(); - final ScheduledTriggers scheduledTriggers = ((OverseerTriggerThread) overseer.getTriggerThread().getThread()).getScheduledTriggers(); - final AutoScaling.Trigger t = scheduledTriggers.getTrigger(name); - assertNotNull(name + " is not a currently scheduled trigger", t); - assertTrue(name + " is not a TriggerBase w/state: " + t.getClass(), - t instanceof TriggerBase); - return ((TriggerBase)t).deepCopyState(); - } -} diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/NodeAddedTriggerTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/NodeAddedTriggerTest.java deleted file mode 100644 index 88cdcc34b6f..00000000000 --- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/NodeAddedTriggerTest.java +++ /dev/null @@ -1,345 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.cloud.autoscaling; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicReference; - -import org.apache.solr.client.solrj.cloud.SolrCloudManager; -import org.apache.solr.client.solrj.embedded.JettySolrRunner; -import org.apache.solr.cloud.SolrCloudTestCase; -import org.apache.solr.common.cloud.ZkStateReader; -import org.apache.solr.core.CoreContainer; -import org.apache.solr.core.SolrResourceLoader; -import org.junit.After; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; - -/** - * Test for {@link NodeAddedTrigger} - */ -public class NodeAddedTriggerTest extends SolrCloudTestCase { - private static AtomicBoolean actionConstructorCalled = new AtomicBoolean(false); - private static AtomicBoolean actionInitCalled = new AtomicBoolean(false); - private static AtomicBoolean actionCloseCalled = new AtomicBoolean(false); - - private AutoScaling.TriggerEventProcessor noFirstRunProcessor = event -> { - fail("Did not expect the processor to fire on first run! event=" + event); - return true; - }; - - private static final long WAIT_FOR_DELTA_NANOS = TimeUnit.MILLISECONDS.toNanos(2); - - @BeforeClass - public static void setupCluster() throws Exception { - - } - - @Before - public void beforeTest() throws Exception { - actionConstructorCalled = new AtomicBoolean(false); - actionInitCalled = new AtomicBoolean(false); - actionCloseCalled = new AtomicBoolean(false); - configureCluster(1) - .addConfig("conf", configset("cloud-minimal")) - .configure(); - } - - @After - public void afterTest() throws Exception { - shutdownCluster(); - } - - @Test - public void testTrigger() throws Exception { - CoreContainer container = cluster.getJettySolrRunners().get(0).getCoreContainer(); - long waitForSeconds = 1 + random().nextInt(5); - Map props = createTriggerProps(waitForSeconds); - - try (NodeAddedTrigger trigger = new NodeAddedTrigger("node_added_trigger1")) { - final SolrCloudManager cloudManager = container.getZkController().getSolrCloudManager(); - trigger.configure(container.getResourceLoader(), cloudManager, props); - trigger.init(); - trigger.setProcessor(noFirstRunProcessor); - trigger.run(); - - JettySolrRunner newNode1 = cluster.startJettySolrRunner(); - JettySolrRunner newNode2 = cluster.startJettySolrRunner(); - - cluster.waitForAllNodes(30); - - AtomicBoolean fired = new AtomicBoolean(false); - AtomicReference eventRef = new AtomicReference<>(); - trigger.setProcessor(event -> { - if (fired.compareAndSet(false, true)) { - eventRef.set(event); - long currentTimeNanos = cloudManager.getTimeSource().getTimeNs(); - long eventTimeNanos = event.getEventTime(); - long waitForNanos = TimeUnit.NANOSECONDS.convert(waitForSeconds, TimeUnit.SECONDS) - WAIT_FOR_DELTA_NANOS; - if (currentTimeNanos - eventTimeNanos <= waitForNanos) { - fail("processor was fired before the configured waitFor period: currentTimeNanos=" + currentTimeNanos + ", eventTimeNanos=" + eventTimeNanos + ",waitForNanos=" + waitForNanos); - } - } else { - fail("NodeAddedTrigger was fired more than once!"); - } - return true; - }); - int counter = 0; - do { - trigger.run(); - Thread.sleep(1000); - if (counter++ > 10) { - fail("Newly added node was not discovered by trigger even after 10 seconds"); - } - } while (!fired.get()); - - TriggerEvent nodeAddedEvent = eventRef.get(); - assertNotNull(nodeAddedEvent); - @SuppressWarnings({"unchecked"}) - List nodeNames = (List)nodeAddedEvent.getProperty(TriggerEvent.NODE_NAMES); - assertTrue(nodeNames.contains(newNode1.getNodeName())); - assertTrue(nodeNames.contains(newNode2.getNodeName())); - } - - // clean nodeAdded markers - normally done by OverseerTriggerThread - container.getZkController().getSolrCloudManager().getDistribStateManager() - .removeRecursively(ZkStateReader.SOLR_AUTOSCALING_NODE_ADDED_PATH, true, false); - - // add a new node but remove it before the waitFor period expires - // and assert that the trigger doesn't fire at all - try (NodeAddedTrigger trigger = new NodeAddedTrigger("node_added_trigger2")) { - final SolrCloudManager cloudManager = container.getZkController().getSolrCloudManager(); - trigger.configure(container.getResourceLoader(), cloudManager, props); - trigger.init(); - final long waitTime = 2; - props.put("waitFor", waitTime); - trigger.setProcessor(noFirstRunProcessor); - trigger.run(); - - JettySolrRunner newNode = cluster.startJettySolrRunner(); - AtomicBoolean fired = new AtomicBoolean(false); - trigger.setProcessor(event -> { - if (fired.compareAndSet(false, true)) { - long currentTimeNanos = cloudManager.getTimeSource().getTimeNs(); - long eventTimeNanos = event.getEventTime(); - long waitForNanos = TimeUnit.NANOSECONDS.convert(waitForSeconds, TimeUnit.SECONDS) - WAIT_FOR_DELTA_NANOS; - if (currentTimeNanos - eventTimeNanos <= waitForNanos) { - fail("NodeAddedListener was fired before the configured waitFor period: currentTimeNanos=" + currentTimeNanos + ", eventTimeNanos=" + eventTimeNanos + ",waitForNanos=" + waitForNanos); - } - } else { - fail("NodeAddedTrigger was fired more than once!"); - } - return true; - }); - trigger.run(); // first run should detect the new node - newNode.stop(); // stop the new jetty - int counter = 0; - do { - trigger.run(); - Thread.sleep(1000); - if (counter++ > waitTime + 1) { // run it a little more than the wait time - break; - } - } while (true); - - // ensure the event was not fired - assertFalse(fired.get()); - } - } - - public void testActionLifecycle() throws Exception { - CoreContainer container = cluster.getJettySolrRunners().get(0).getCoreContainer(); - Map props = createTriggerProps(0); - @SuppressWarnings({"unchecked"}) - List> actions = (List>) props.get("actions"); - Map action = new HashMap<>(2); - action.put("name", "testActionInit"); - action.put("class", NodeAddedTriggerTest.AssertInitTriggerAction.class.getName()); - actions.add(action); - try (NodeAddedTrigger trigger = new NodeAddedTrigger("node_added_trigger")) { - trigger.configure(container.getResourceLoader(), container.getZkController().getSolrCloudManager(), props); - assertEquals(true, actionConstructorCalled.get()); - assertEquals(false, actionInitCalled.get()); - assertEquals(false, actionCloseCalled.get()); - trigger.init(); - assertEquals(true, actionInitCalled.get()); - assertEquals(false, actionCloseCalled.get()); - } - assertEquals(true, actionCloseCalled.get()); - } - - public static class AssertInitTriggerAction implements TriggerAction { - public AssertInitTriggerAction() { - actionConstructorCalled.set(true); - } - - @Override - public void configure(SolrResourceLoader loader, SolrCloudManager cloudManager, Map properties) throws TriggerValidationException { - - } - - @Override - public void init() { - actionInitCalled.compareAndSet(false, true); - } - - @Override - public String getName() { - return ""; - } - - @Override - public void process(TriggerEvent event, ActionContext actionContext) { - - } - - @Override - public void close() throws IOException { - actionCloseCalled.compareAndSet(false, true); - } - } - - @Test - public void testListenerAcceptance() throws Exception { - CoreContainer container = cluster.getJettySolrRunners().get(0).getCoreContainer(); - Map props = createTriggerProps(0); - try (NodeAddedTrigger trigger = new NodeAddedTrigger("node_added_trigger")) { - trigger.configure(container.getResourceLoader(), container.getZkController().getSolrCloudManager(), props); - trigger.init(); - trigger.setProcessor(noFirstRunProcessor); - trigger.run(); // starts tracking live nodes - - JettySolrRunner newNode = cluster.startJettySolrRunner(); - AtomicInteger callCount = new AtomicInteger(0); - AtomicBoolean fired = new AtomicBoolean(false); - - trigger.setProcessor(event -> { - if (callCount.incrementAndGet() < 2) { - return false; - } else { - fired.compareAndSet(false, true); - return true; - } - }); - - trigger.run(); // first run should detect the new node and fire immediately but listener isn't ready - assertEquals(1, callCount.get()); - assertFalse(fired.get()); - trigger.run(); // second run should again fire - assertEquals(2, callCount.get()); - assertTrue(fired.get()); - trigger.run(); // should not fire - assertEquals(2, callCount.get()); - } - } - - @Test - public void testRestoreState() throws Exception { - CoreContainer container = cluster.getJettySolrRunners().get(0).getCoreContainer(); - long waitForSeconds = 1 + random().nextInt(5); - Map props = createTriggerProps(waitForSeconds); - - // add a new node but update the trigger before the waitFor period expires - // and assert that the new trigger still fires - NodeAddedTrigger trigger = new NodeAddedTrigger("node_added_trigger"); - trigger.configure(container.getResourceLoader(), container.getZkController().getSolrCloudManager(), props); - trigger.setProcessor(noFirstRunProcessor); - trigger.run(); - - JettySolrRunner newNode = cluster.startJettySolrRunner(); - cluster.waitForAllNodes(30); - trigger.setProcessor(null); // the processor may get called for old nodes - trigger.run(); // this run should detect the new node - trigger.close(); // close the old trigger - - try (NodeAddedTrigger newTrigger = new NodeAddedTrigger("some_different_name")) { - newTrigger.configure(container.getResourceLoader(), container.getZkController().getSolrCloudManager(), props); - newTrigger.init(); - try { - newTrigger.restoreState(trigger); - fail("Trigger should only be able to restore state from an old trigger of the same name"); - } catch (AssertionError e) { - // expected - } - } - - try (NodeAddedTrigger newTrigger = new NodeAddedTrigger("node_added_trigger")) { - final SolrCloudManager cloudManager = container.getZkController().getSolrCloudManager(); - newTrigger.configure(container.getResourceLoader(), cloudManager, props); - newTrigger.init(); - AtomicBoolean stop = new AtomicBoolean(false); - AtomicReference eventRef = new AtomicReference<>(); - newTrigger.setProcessor(event -> { - //the processor may get called 2 times, for newly added node and initial nodes - long currentTimeNanos = cloudManager.getTimeSource().getTimeNs(); - long eventTimeNanos = event.getEventTime(); - long waitForNanos = TimeUnit.NANOSECONDS.convert(waitForSeconds, TimeUnit.SECONDS) - WAIT_FOR_DELTA_NANOS; - if (currentTimeNanos - eventTimeNanos <= waitForNanos) { - fail("NodeAddedListener was fired before the configured waitFor period: currentTimeNanos=" + currentTimeNanos + ", eventTimeNanos=" + eventTimeNanos + ",waitForNanos=" + waitForNanos); - } - @SuppressWarnings({"unchecked"}) - List nodeNames = (List) event.getProperty(NodeAddedTrigger.NodeAddedEvent.NODE_NAMES); - if (nodeNames.contains(newNode.getNodeName())) { - stop.set(true); - eventRef.set(event); - } - return true; - }); - newTrigger.restoreState(trigger); // restore state from the old trigger - int counter = 0; - do { - newTrigger.run(); - Thread.sleep(1000); - if (counter++ > 10) { - fail("Newly added node was not discovered by trigger even after 10 seconds"); - } - } while (!stop.get()); - - // ensure the event was fired - assertTrue(stop.get()); - TriggerEvent nodeAddedEvent = eventRef.get(); - assertNotNull(nodeAddedEvent); - } - } - - private Map createTriggerProps(long waitForSeconds) { - Map props = new HashMap<>(); - props.put("event", "nodeLost"); - props.put("waitFor", waitForSeconds); - props.put("enabled", true); - List> actions = new ArrayList<>(3); - Map map = new HashMap<>(2); - map.put("name", "compute_plan"); - map.put("class", "solr.ComputePlanAction"); - actions.add(map); - map = new HashMap<>(2); - map.put("name", "execute_plan"); - map.put("class", "solr.ExecutePlanAction"); - actions.add(map); - props.put("actions", actions); - return props; - } -} diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/NodeLostTriggerIntegrationTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/NodeLostTriggerIntegrationTest.java deleted file mode 100644 index ef52267bcaa..00000000000 --- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/NodeLostTriggerIntegrationTest.java +++ /dev/null @@ -1,356 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.cloud.autoscaling; - -import java.lang.invoke.MethodHandles; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; - -import org.apache.solr.client.solrj.cloud.SolrCloudManager; -import org.apache.solr.client.solrj.embedded.JettySolrRunner; -import org.apache.solr.client.solrj.impl.CloudSolrClient; -import org.apache.solr.client.solrj.request.CollectionAdminRequest; -import org.apache.solr.cloud.CloudTestUtils; -import org.apache.solr.cloud.Overseer; -import org.apache.solr.cloud.SolrCloudTestCase; -import org.apache.solr.common.cloud.ZkNodeProps; -import org.apache.solr.common.cloud.ZkStateReader; -import org.apache.solr.common.util.NamedList; -import org.apache.solr.common.util.Utils; -import org.apache.solr.util.LogLevel; -import org.apache.solr.util.TimeOut; -import org.apache.zookeeper.data.Stat; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import static org.apache.solr.cloud.autoscaling.TriggerIntegrationTest.WAIT_FOR_DELTA_NANOS; -import static org.apache.solr.common.cloud.ZkStateReader.SOLR_AUTOSCALING_CONF_PATH; - -// TODO: this class shares duplicated code with NodeAddedTriggerIntegrationTest ... merge? - -@LogLevel("org.apache.solr.cloud.autoscaling=DEBUG;org.apache.solr.client.solrj.cloud.autoscaling=DEBUG") -public class NodeLostTriggerIntegrationTest extends SolrCloudTestCase { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - private static volatile CountDownLatch actionConstructorCalled; - private static volatile CountDownLatch actionInitCalled; - private static volatile CountDownLatch triggerFiredLatch; - private static volatile int waitForSeconds = 1; - private static volatile AtomicBoolean triggerFired; - private static volatile Set events = ConcurrentHashMap.newKeySet(); - private static volatile SolrCloudManager cloudManager; - - private static CountDownLatch getTriggerFiredLatch() { - return triggerFiredLatch; - } - - @Before - public void setupTest() throws Exception { - - configureCluster(4) - .addConfig("conf", configset("cloud-minimal")) - .configure(); - - final Overseer overseer = cluster.getOpenOverseer(); - assertNotNull(overseer); - cloudManager = overseer.getSolrCloudManager(); - assertNotNull(cloudManager); - - // disable .scheduled_maintenance (once it exists) - CloudTestUtils.waitForTriggerToBeScheduled(cloudManager, ".scheduled_maintenance"); - CloudTestUtils.suspendTrigger(cloudManager, ".scheduled_maintenance"); - - // aggressively remove all active scheduled triggers - final ScheduledTriggers scheduledTriggers = ((OverseerTriggerThread) overseer.getTriggerThread().getThread()).getScheduledTriggers(); - // TODO: is this really safe? is it possible overseer is still in process of adding some to schedule? - scheduledTriggers.removeAll(); - - // clear any persisted auto scaling configuration - Stat stat = zkClient().setData(SOLR_AUTOSCALING_CONF_PATH, Utils.toJSON(new ZkNodeProps()), true); - if (log.isInfoEnabled()) { - log.info("{} reset, new znode version {}", SOLR_AUTOSCALING_CONF_PATH, stat.getVersion()); - } - - cluster.getSolrClient().setDefaultCollection(null); - - waitForSeconds = 1 + random().nextInt(3); - actionConstructorCalled = new CountDownLatch(1); - actionInitCalled = new CountDownLatch(1); - triggerFiredLatch = new CountDownLatch(1); - triggerFired = new AtomicBoolean(false); - events.clear(); - - // clear any events or markers - // todo: consider the impact of such cleanup on regular cluster restarts - deleteChildrenRecursively(ZkStateReader.SOLR_AUTOSCALING_EVENTS_PATH); - deleteChildrenRecursively(ZkStateReader.SOLR_AUTOSCALING_TRIGGER_STATE_PATH); - deleteChildrenRecursively(ZkStateReader.SOLR_AUTOSCALING_NODE_LOST_PATH); - deleteChildrenRecursively(ZkStateReader.SOLR_AUTOSCALING_NODE_ADDED_PATH); - } - - @After - public void cleanUpTest() throws Exception { - shutdownCluster(); - } - - @AfterClass - public static void cleanUpAfterClass() throws Exception { - cloudManager = null; - } - - private void deleteChildrenRecursively(String path) throws Exception { - cloudManager.getDistribStateManager().removeRecursively(path, true, false); - } - - @Test - @SuppressWarnings({"unchecked"}) - public void testNodeLostTriggerRestoreState() throws Exception { - - final String triggerName = "node_lost_restore_trigger"; - - // start a new node - final JettySolrRunner newNode = cluster.startJettySolrRunner(); - final String nodeName = newNode.getNodeName(); - - // should be enough to ensure trigger doesn't fire any actions until we replace the trigger - waitForSeconds = 500000; - CloudTestUtils.assertAutoScalingRequest - (cloudManager, - "{" + - "'set-trigger' : {" + - "'name' : '"+triggerName+"'," + - "'event' : 'nodeLost'," + - "'waitFor' : '"+waitForSeconds+"s'," + - "'enabled' : true," + - "'actions' : [{'name':'test','class':'" + TestTriggerAction.class.getName() + "'}]" + - "}}"); - - assertTrue("Trigger was not init()ed even after await()ing an excessive amount of time", - actionInitCalled.await(60, TimeUnit.SECONDS)); - - // poll the internal state of the trigger until it run()s at least once and updates - // it's internal state to know the node we added is live - // - // (this should run roughly once a second) - (new TimeOut(30, TimeUnit.SECONDS, cloudManager.getTimeSource())) - .waitFor("initial trigger never ran to detect new live node", () -> - (((Collection) getTriggerState(triggerName).get("lastLiveNodes")) - .contains(nodeName))); - - // kill our node - cluster.stopJettySolrRunner(newNode); - cluster.waitForJettyToStop(newNode); - - // poll the internal state of the trigger until it run()s at least once (more) and updates - // it's internal state to know the node we killed is no longer alive - // - // (this should run roughly once a second of simulated time) - (new TimeOut(30, TimeUnit.SECONDS, cloudManager.getTimeSource())) - .waitFor("initial trigger never ran to detect lost node", () -> - ! (((Collection) getTriggerState(triggerName).get("lastLiveNodes")) - .contains(nodeName))); - - // since we know the nodeLost event has been detected, we can recored the current timestamp - // (relative to the cluster's time source) and later assert that (restored state) correctly - // tracked that the event happened prior to "now" - final long maxEventTimeNs = cloudManager.getTimeSource().getTimeNs(); - - // even though our trigger has detected a lost node, the *action* we registered should not have - // been run yet, due to the large waitFor configuration... - assertEquals("initial trigger action should not have fired", false, triggerFired.get()); - assertEquals("initial trigger action latch should not have counted down", - 1, triggerFiredLatch.getCount()); - assertEquals("initial trigger action should not have recorded any events: " + events.toString(), - 0, events.size()); - - // - // now replace the trigger with a new instance to test that the state gets copied over correctly - // - - // reset the actionInitCalled counter so we can confirm the second instances is inited - actionInitCalled = new CountDownLatch(1); - // use a low waitTime to ensure it processes the event quickly. - // (this updated property also ensures the set-trigger won't be treated as a No-Op) - waitForSeconds = 0 + random().nextInt(3); - CloudTestUtils.assertAutoScalingRequest - (cloudManager, - "{" + - "'set-trigger' : {" + - "'name' : '"+triggerName+"'," + - "'event' : 'nodeLost'," + - "'waitFor' : '"+waitForSeconds+"s'," + - "'enabled' : true," + - "'actions' : [{'name':'test','class':'" + TestTriggerAction.class.getName() + "'}]" + - "}}"); - - assertTrue("Trigger was not init()ed even after await()ing an excessive amount of time", - actionInitCalled.await(60, TimeUnit.SECONDS)); - - // the trigger actions should now (eventually) record that the node is lost - assertTrue("Second instance of our trigger never fired the action to process the event", - triggerFiredLatch.await(30, TimeUnit.SECONDS)); - - assertEquals("Wrong number of events recorded: " + events.toString(), - 1, events.size()); - - final TriggerEvent event = events.iterator().next(); - assertNotNull("null event???", event); - assertTrue("Event should have been a nodeLost event: " + event.getClass(), - event instanceof NodeLostTrigger.NodeLostEvent); - - assertNotNull("event is missing NODE_NAMES: " + event, event.getProperty(TriggerEvent.NODE_NAMES)); - assertEquals("event has incorrect NODE_NAMES: " + event, - Collections.singletonList(nodeName), - event.getProperty(TriggerEvent.NODE_NAMES)); - - assertTrue("event TS is too late, should be before (max) expected TS @ " - + maxEventTimeNs + ": " + event, - event.getEventTime() < maxEventTimeNs); - - assertNotNull("event is missing EVENT_TIMES: " + event, event.getProperty(TriggerEvent.EVENT_TIMES)); - assertEquals("event has unexpeted number of EVENT_TIMES: " + event, - 1, ((Collection)event.getProperty(TriggerEvent.EVENT_TIMES)).size()); - assertEquals("event's TS doesn't match EVENT_TIMES: " + event, - event.getEventTime(), - ((Collection)event.getProperty(TriggerEvent.EVENT_TIMES)).iterator().next()); - } - - @Test - public void testNodeLostTrigger() throws Exception { - CloudSolrClient solrClient = cluster.getSolrClient(); - String setTriggerCommand = "{" + - "'set-trigger' : {" + - "'name' : 'node_lost_trigger'," + - "'event' : 'nodeLost'," + - "'waitFor' : '" + waitForSeconds + "s'," + - "'enabled' : true," + - "'actions' : [{'name':'test','class':'" + TestTriggerAction.class.getName() + "'}]" + - "}}"; - NamedList overSeerStatus = cluster.getSolrClient().request(CollectionAdminRequest.getOverseerStatus()); - String overseerLeader = (String) overSeerStatus.get("leader"); - int nonOverseerLeaderIndex = 0; - for (int i = 0; i < cluster.getJettySolrRunners().size(); i++) { - JettySolrRunner jetty = cluster.getJettySolrRunner(i); - if (!jetty.getNodeName().equals(overseerLeader)) { - nonOverseerLeaderIndex = i; - } - } - CloudTestUtils.assertAutoScalingRequest(cloudManager, setTriggerCommand); - - if (!actionInitCalled.await(3, TimeUnit.SECONDS)) { - fail("The TriggerAction should have been created by now"); - } - - triggerFired.set(false); - triggerFiredLatch = new CountDownLatch(1); - String lostNodeName = cluster.getJettySolrRunner(nonOverseerLeaderIndex).getNodeName(); - JettySolrRunner j = cluster.stopJettySolrRunner(nonOverseerLeaderIndex); - cluster.waitForJettyToStop(j); - boolean await = triggerFiredLatch.await(20, TimeUnit.SECONDS); - assertTrue("The trigger did not fire at all", await); - assertTrue(triggerFired.get()); - NodeLostTrigger.NodeLostEvent nodeLostEvent = (NodeLostTrigger.NodeLostEvent) events.iterator().next(); - assertNotNull(nodeLostEvent); - @SuppressWarnings({"unchecked"}) - List nodeNames = (List) nodeLostEvent.getProperty(TriggerEvent.NODE_NAMES); - assertTrue(nodeNames.contains(lostNodeName)); - - // reset - actionConstructorCalled = new CountDownLatch(1); - actionInitCalled = new CountDownLatch(1); - - // update the trigger with exactly the same data - CloudTestUtils.assertAutoScalingRequest - (cloudManager, - "{" + - "'set-trigger' : {" + - "'name' : 'node_lost_trigger'," + - "'event' : 'nodeLost'," + - "'waitFor' : '" + waitForSeconds + "s'," + - "'enabled' : true," + - "'actions' : [{'name':'test','class':'" + TestTriggerAction.class.getName() + "'}]" + - "}}"); - - // this should be a no-op so the action should have been created but init should not be called - if (!actionConstructorCalled.await(3, TimeUnit.SECONDS)) { - fail("The TriggerAction should have been created by now"); - } - - assertFalse(actionInitCalled.await(2, TimeUnit.SECONDS)); - } - - public static class TestTriggerAction extends TriggerActionBase { - - public TestTriggerAction() { - actionConstructorCalled.countDown(); - } - - @Override - public void process(TriggerEvent event, ActionContext actionContext) { - try { - if (triggerFired.compareAndSet(false, true)) { - events.add(event); - long currentTimeNanos = actionContext.getCloudManager().getTimeSource().getTimeNs(); - long eventTimeNanos = event.getEventTime(); - long waitForNanos = TimeUnit.NANOSECONDS.convert(waitForSeconds, TimeUnit.SECONDS) - WAIT_FOR_DELTA_NANOS; - if (currentTimeNanos - eventTimeNanos <= waitForNanos) { - fail(event.source + " was fired before the configured waitFor period"); - } - getTriggerFiredLatch().countDown(); - } else { - fail(event.source + " was fired more than once!"); - } - } catch (Throwable t) { - log.debug("--throwable", t); - throw t; - } - } - - @Override - public void init() throws Exception { - log.info("TestTriggerAction init"); - actionInitCalled.countDown(); - super.init(); - } - } - - /** - * Helper method for getting a copy of the current (internal) trigger state of a scheduled trigger. - */ - private Map getTriggerState(final String name) { - final Overseer overseer = cluster.getOpenOverseer(); - final ScheduledTriggers scheduledTriggers = ((OverseerTriggerThread) overseer.getTriggerThread().getThread()).getScheduledTriggers(); - final AutoScaling.Trigger t = scheduledTriggers.getTrigger(name); - assertNotNull(name + " is not a currently scheduled trigger", t); - assertTrue(name + " is not a TriggerBase w/state: " + t.getClass(), - t instanceof TriggerBase); - return ((TriggerBase)t).deepCopyState(); - } - -} diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/NodeLostTriggerTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/NodeLostTriggerTest.java deleted file mode 100644 index 5d417f987e1..00000000000 --- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/NodeLostTriggerTest.java +++ /dev/null @@ -1,394 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.cloud.autoscaling; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicReference; - -import org.apache.solr.client.solrj.cloud.SolrCloudManager; -import org.apache.solr.client.solrj.embedded.JettySolrRunner; -import org.apache.solr.cloud.SolrCloudTestCase; -import org.apache.solr.common.cloud.ZkStateReader; -import org.apache.solr.common.util.TimeSource; -import org.apache.solr.core.CoreContainer; -import org.apache.solr.core.SolrResourceLoader; -import org.apache.solr.util.TimeOut; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -/** - * Test for {@link NodeLostTrigger} - */ -public class NodeLostTriggerTest extends SolrCloudTestCase { - private static AtomicBoolean actionConstructorCalled = new AtomicBoolean(false); - private static AtomicBoolean actionInitCalled = new AtomicBoolean(false); - private static AtomicBoolean actionCloseCalled = new AtomicBoolean(false); - - private AutoScaling.TriggerEventProcessor noFirstRunProcessor = event -> { - fail("Did not expect the listener to fire on first run!"); - return true; - }; - - private static final long WAIT_FOR_DELTA_NANOS = TimeUnit.MILLISECONDS.toNanos(5); - - @After - public void tearDownCluster() throws Exception { - shutdownCluster(); - } - - @Before - public void beforeTest() throws Exception { - actionConstructorCalled = new AtomicBoolean(false); - actionInitCalled = new AtomicBoolean(false); - actionCloseCalled = new AtomicBoolean(false); - - configureCluster(3) - .addConfig("conf", configset("cloud-minimal")) - .configure(); - } - - @Test - public void testTrigger() throws Exception { - cluster.startJettySolrRunner(); - cluster.waitForAllNodes(30); - - CoreContainer container = cluster.getJettySolrRunners().get(0).getCoreContainer(); - long waitForSeconds = 1 + random().nextInt(5); - Map props = createTriggerProps(waitForSeconds); - - try (NodeLostTrigger trigger = new NodeLostTrigger("node_lost_trigger1")) { - final SolrCloudManager cloudManager = container.getZkController().getSolrCloudManager(); - trigger.configure(container.getResourceLoader(), cloudManager, props); - trigger.init(); - trigger.setProcessor(noFirstRunProcessor); - trigger.run(); - String lostNodeName1 = cluster.getJettySolrRunner(1).getNodeName(); - JettySolrRunner j = cluster.stopJettySolrRunner(1); - cluster.waitForJettyToStop(j); - String lostNodeName2 = cluster.getJettySolrRunner(1).getNodeName(); - j = cluster.stopJettySolrRunner(1); - cluster.waitForJettyToStop(j); - Thread.sleep(1000); - - AtomicBoolean fired = new AtomicBoolean(false); - AtomicReference eventRef = new AtomicReference<>(); - trigger.setProcessor(event -> { - if (fired.compareAndSet(false, true)) { - eventRef.set(event); - long currentTimeNanos = cloudManager.getTimeSource().getTimeNs(); - long eventTimeNanos = event.getEventTime(); - long waitForNanos = TimeUnit.NANOSECONDS.convert(waitForSeconds, TimeUnit.SECONDS) - WAIT_FOR_DELTA_NANOS; - if (currentTimeNanos - eventTimeNanos <= waitForNanos) { - fail("NodeLostListener was fired before the configured waitFor period: currentTimeNanos=" + currentTimeNanos + ", eventTimeNanos=" + eventTimeNanos + ",waitForNanos=" + waitForNanos); - } - } else { - fail("NodeLostListener was fired more than once!"); - } - return true; - }); - int counter = 0; - do { - trigger.run(); - Thread.sleep(1000); - if (counter++ > 10) { - fail("Lost node was not discovered by trigger even after 10 seconds"); - } - } while (!fired.get()); - - TriggerEvent nodeLostEvent = eventRef.get(); - assertNotNull(nodeLostEvent); - @SuppressWarnings({"unchecked"}) - List nodeNames = (List)nodeLostEvent.getProperty(TriggerEvent.NODE_NAMES); - assertTrue(nodeNames + " doesn't contain " + lostNodeName1, nodeNames.contains(lostNodeName1)); - assertTrue(nodeNames + " doesn't contain " + lostNodeName2, nodeNames.contains(lostNodeName2)); - - } - - // clean nodeLost markers - normally done by OverseerTriggerThread - container.getZkController().getSolrCloudManager().getDistribStateManager() - .removeRecursively(ZkStateReader.SOLR_AUTOSCALING_NODE_LOST_PATH, true, false); - - // remove a node but add it back before the waitFor period expires - // and assert that the trigger doesn't fire at all - try (NodeLostTrigger trigger = new NodeLostTrigger("node_lost_trigger2")) { - final SolrCloudManager cloudManager = container.getZkController().getSolrCloudManager(); - trigger.configure(container.getResourceLoader(), cloudManager, props); - final long waitTime = 2; - props.put("waitFor", waitTime); - trigger.init(); - trigger.setProcessor(noFirstRunProcessor); - trigger.run(); - - JettySolrRunner lostNode = cluster.getJettySolrRunner(1); - String lostNodeName = lostNode.getNodeName(); - lostNode.stop(); - AtomicBoolean fired = new AtomicBoolean(false); - AtomicReference eventRef = new AtomicReference<>(); - trigger.setProcessor(event -> { - if (fired.compareAndSet(false, true)) { - eventRef.set(event); - long currentTimeNanos = cloudManager.getTimeSource().getTimeNs(); - long eventTimeNanos = event.getEventTime(); - long waitForNanos = TimeUnit.NANOSECONDS.convert(waitTime, TimeUnit.SECONDS) - WAIT_FOR_DELTA_NANOS; - if (currentTimeNanos - eventTimeNanos <= waitForNanos) { - fail("NodeLostListener was fired before the configured waitFor period: currentTimeNanos=" + currentTimeNanos + ", eventTimeNanos=" + eventTimeNanos + ",waitForNanos=" + waitForNanos); - } - } else { - fail("NodeLostListener was fired more than once!"); - } - return true; - }); - trigger.run(); // first run should detect the lost node - int counter = 0; - do { - if (!container.getZkController().getZkStateReader().getClusterState().getLiveNodes().contains(lostNodeName)) { - break; - } - Thread.sleep(100); - if (counter++ > 20) { - fail("Live nodes not updated!"); - } - } while (true); - counter = 0; - lostNode.start(); - do { - trigger.run(); - Thread.sleep(1000); - if (counter++ > waitTime + 1) { // run it a little more than the wait time - break; - } - } while (true); - - // ensure the event was not fired - assertFalse("event was fired: " + eventRef.get(), fired.get()); - } - } - - public void testActionLifecycle() throws Exception { - CoreContainer container = cluster.getJettySolrRunners().get(0).getCoreContainer(); - Map props = createTriggerProps(0); - @SuppressWarnings({"unchecked"}) - List> actions = (List>) props.get("actions"); - Map action = new HashMap<>(2); - action.put("name", "testActionInit"); - action.put("class", AssertInitTriggerAction.class.getName()); - actions.add(action); - try (NodeLostTrigger trigger = new NodeLostTrigger("node_added_trigger")) { - trigger.configure(container.getResourceLoader(), container.getZkController().getSolrCloudManager(), props); - assertEquals(true, actionConstructorCalled.get()); - assertEquals(false, actionInitCalled.get()); - assertEquals(false, actionCloseCalled.get()); - trigger.init(); - assertEquals(true, actionInitCalled.get()); - assertEquals(false, actionCloseCalled.get()); - } - assertEquals(true, actionCloseCalled.get()); - } - - public static class AssertInitTriggerAction implements TriggerAction { - public AssertInitTriggerAction() { - actionConstructorCalled.set(true); - } - - @Override - public void configure(SolrResourceLoader loader, SolrCloudManager cloudManager, Map properties) throws TriggerValidationException { - - } - - @Override - public void init() { - actionInitCalled.compareAndSet(false, true); - } - - @Override - public String getName() { - return ""; - } - - @Override - public void process(TriggerEvent event, ActionContext actionContext) { - - } - - @Override - public void close() throws IOException { - actionCloseCalled.compareAndSet(false, true); - } - - } - - @Test - //28-June-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 16-Apr-2018 - public void testListenerAcceptance() throws Exception { - - CoreContainer container = cluster.getJettySolrRunners().get(0).getCoreContainer(); - Map props = createTriggerProps(0); - - try (NodeLostTrigger trigger = new NodeLostTrigger("node_added_trigger")) { - trigger.configure(container.getResourceLoader(), container.getZkController().getSolrCloudManager(), props); - trigger.init(); - trigger.setProcessor(noFirstRunProcessor); - - JettySolrRunner newNode = cluster.startJettySolrRunner(); - - cluster.waitForAllNodes(30); - - trigger.run(); // starts tracking live nodes - - // stop the newly created node - newNode.stop(); - cluster.waitForJettyToStop(newNode); - - AtomicInteger callCount = new AtomicInteger(0); - AtomicBoolean fired = new AtomicBoolean(false); - - trigger.setProcessor(event -> { - if (callCount.incrementAndGet() < 2) { - return false; - } else { - fired.compareAndSet(false, true); - return true; - } - }); - - Thread.sleep(1000); - - trigger.run(); // first run should detect the lost node and fire immediately but listener isn't ready - - TimeOut timeout = new TimeOut(5, TimeUnit.SECONDS, TimeSource.NANO_TIME); - timeout.waitFor("Timeout waiting for callCount to hit at least 1", () -> callCount.get() >= 1); - assertEquals(1, callCount.get()); - assertFalse(fired.get()); - trigger.run(); // second run should again fire - timeout = new TimeOut(5, TimeUnit.SECONDS, TimeSource.NANO_TIME); - timeout.waitFor("Timeout waiting for callCount to hit at least 2", () -> callCount.get() >= 2); - assertEquals(2, callCount.get()); - assertTrue(fired.get()); - trigger.run(); // should not fire - assertEquals(2, callCount.get()); - } - } - - @Test - public void testRestoreState() throws Exception { - CoreContainer container = cluster.getJettySolrRunners().get(0).getCoreContainer(); - long waitForSeconds = 1 + random().nextInt(5); - Map props = createTriggerProps(waitForSeconds); - - JettySolrRunner newNode = cluster.startJettySolrRunner(); - cluster.waitForAllNodes(30); - String lostNodeName = newNode.getNodeName(); - - // remove a node but update the trigger before the waitFor period expires - // and assert that the new trigger still fires - - NodeLostTrigger trigger = new NodeLostTrigger("node_lost_trigger"); - trigger.configure(container.getResourceLoader(), container.getZkController().getSolrCloudManager(), props); - trigger.init(); - trigger.setProcessor(noFirstRunProcessor); - trigger.run(); - - // stop the newly created node - List jettySolrRunners = cluster.getJettySolrRunners(); - for (int i = 0; i < jettySolrRunners.size(); i++) { - JettySolrRunner jettySolrRunner = jettySolrRunners.get(i); - if (newNode == jettySolrRunner) { - JettySolrRunner j = cluster.stopJettySolrRunner(i); - cluster.waitForJettyToStop(j); - break; - } - } - - trigger.run(); // this run should detect the lost node - trigger.close(); // close the old trigger - - try (NodeLostTrigger newTrigger = new NodeLostTrigger("some_different_name")) { - newTrigger.configure(container.getResourceLoader(), container.getZkController().getSolrCloudManager(), props); - newTrigger.init(); - try { - newTrigger.restoreState(trigger); - fail("Trigger should only be able to restore state from an old trigger of the same name"); - } catch (AssertionError e) { - // expected - } - } - - try (NodeLostTrigger newTrigger = new NodeLostTrigger("node_lost_trigger")) { - final SolrCloudManager cloudManager = container.getZkController().getSolrCloudManager(); - newTrigger.configure(container.getResourceLoader(), cloudManager, props); - newTrigger.init(); - AtomicBoolean fired = new AtomicBoolean(false); - AtomicReference eventRef = new AtomicReference<>(); - newTrigger.setProcessor(event -> { - if (fired.compareAndSet(false, true)) { - eventRef.set(event); - long currentTimeNanos = cloudManager.getTimeSource().getTimeNs(); - long eventTimeNanos = event.getEventTime(); - long waitForNanos = TimeUnit.NANOSECONDS.convert(waitForSeconds, TimeUnit.SECONDS) - WAIT_FOR_DELTA_NANOS; - if (currentTimeNanos - eventTimeNanos <= waitForNanos) { - fail("NodeLostListener was fired before the configured waitFor period: currentTimeNanos=" + currentTimeNanos + ", eventTimeNanos=" + eventTimeNanos + ",waitForNanos=" + waitForNanos); - } - } else { - fail("NodeLostListener was fired more than once!"); - } - return true; - }); - newTrigger.restoreState(trigger); // restore state from the old trigger - int counter = 0; - do { - newTrigger.run(); - Thread.sleep(1000); - if (counter++ > 10) { - fail("Lost node was not discovered by trigger even after 10 seconds"); - } - } while (!fired.get()); - - TriggerEvent nodeLostEvent = eventRef.get(); - assertNotNull(nodeLostEvent); - @SuppressWarnings({"unchecked"}) - List nodeNames = (List)nodeLostEvent.getProperty(TriggerEvent.NODE_NAMES); - assertTrue(nodeNames.contains(lostNodeName)); - } - } - - private Map createTriggerProps(long waitForSeconds) { - Map props = new HashMap<>(); - props.put("event", "nodeLost"); - props.put("waitFor", waitForSeconds); - props.put("enabled", true); - List> actions = new ArrayList<>(3); - Map map = new HashMap<>(2); - map.put("name", "compute_plan"); - map.put("class", "solr.ComputePlanAction"); - actions.add(map); - map = new HashMap<>(2); - map.put("name", "execute_plan"); - map.put("class", "solr.ExecutePlanAction"); - actions.add(map); - props.put("actions", actions); - return props; - } -} diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/NodeMarkersRegistrationTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/NodeMarkersRegistrationTest.java deleted file mode 100644 index 5ed30c04c8d..00000000000 --- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/NodeMarkersRegistrationTest.java +++ /dev/null @@ -1,356 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.cloud.autoscaling; - -import java.lang.invoke.MethodHandles; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.SortedSet; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.locks.ReentrantLock; - -import org.apache.solr.client.solrj.SolrRequest; -import org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventProcessorStage; -import org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventType; -import org.apache.solr.client.solrj.embedded.JettySolrRunner; -import org.apache.solr.client.solrj.impl.CloudSolrClient; -import org.apache.solr.client.solrj.request.CollectionAdminRequest; -import org.apache.solr.cloud.CloudTestUtils; -import org.apache.solr.cloud.CloudTestUtils.AutoScalingRequest; -import org.apache.solr.cloud.SolrCloudTestCase; -import org.apache.solr.common.cloud.LiveNodesListener; -import org.apache.solr.common.cloud.ZkStateReader; -import org.apache.solr.common.util.NamedList; -import org.apache.solr.common.util.TimeSource; -import org.apache.solr.common.util.Utils; -import org.apache.solr.util.LogLevel; -import org.apache.solr.util.TimeOut; -import org.apache.zookeeper.KeeperException; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import static org.apache.solr.cloud.autoscaling.OverseerTriggerThread.MARKER_ACTIVE; -import static org.apache.solr.cloud.autoscaling.OverseerTriggerThread.MARKER_INACTIVE; -import static org.apache.solr.cloud.autoscaling.OverseerTriggerThread.MARKER_STATE; - -@LogLevel("org.apache.solr.cloud.autoscaling=DEBUG;org.apache.solr.client.solrj.cloud.autoscaling=DEBUG") -public class NodeMarkersRegistrationTest extends SolrCloudTestCase { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - private static volatile CountDownLatch triggerFiredLatch; - private static volatile CountDownLatch listenerEventLatch; - private static Set events = ConcurrentHashMap.newKeySet(); - private volatile ZkStateReader zkStateReader; - private static final ReentrantLock lock = new ReentrantLock(); - - @Before - public void setupCluster() throws Exception { - configureCluster(2) - .addConfig("conf", configset("cloud-minimal")) - .configure(); - zkStateReader = cluster.getSolrClient().getZkStateReader(); - - // disable .scheduled_maintenance (once it exists) - CloudTestUtils.waitForTriggerToBeScheduled(cluster.getOpenOverseer().getSolrCloudManager(), ".scheduled_maintenance"); - CloudTestUtils.suspendTrigger(cluster.getOpenOverseer().getSolrCloudManager(), ".scheduled_maintenance"); - } - - @After - public void teardownCluster() throws Exception { - shutdownCluster(); - } - - private static CountDownLatch getTriggerFiredLatch() { - return triggerFiredLatch; - } - - //@AwaitsFix(bugUrl="https://issues.apache.org/jira/browse/SOLR-13376") - @Test - public void testNodeMarkersRegistration() throws Exception { - triggerFiredLatch = new CountDownLatch(1); - listenerEventLatch = new CountDownLatch(1); - TestLiveNodesListener listener = registerLiveNodesListener(); - - NamedList overSeerStatus = cluster.getSolrClient().request(CollectionAdminRequest.getOverseerStatus()); - String overseerLeader = (String) overSeerStatus.get("leader"); - int overseerLeaderIndex = 0; - for (int i = 0; i < cluster.getJettySolrRunners().size(); i++) { - JettySolrRunner jetty = cluster.getJettySolrRunner(i); - if (jetty.getNodeName().equals(overseerLeader)) { - overseerLeaderIndex = i; - break; - } - } - // add a nodes - JettySolrRunner node = cluster.startJettySolrRunner(); - cluster.waitForAllNodes(30); - if (!listener.onChangeLatch.await(10, TimeUnit.SECONDS)) { - fail("onChange listener didn't execute on cluster change"); - } - assertEquals(1, listener.addedNodes.size()); - assertTrue(listener.addedNodes.toString(), listener.addedNodes.contains(node.getNodeName())); - // verify that a znode doesn't exist (no trigger) - String pathAdded = ZkStateReader.SOLR_AUTOSCALING_NODE_ADDED_PATH + "/" + node.getNodeName(); - assertFalse("Path " + pathAdded + " was created but there are no nodeAdded triggers", zkClient().exists(pathAdded, true)); - listener.reset(); - - // stop overseer - log.info("====== KILL OVERSEER 1"); - JettySolrRunner j = cluster.stopJettySolrRunner(overseerLeaderIndex); - cluster.waitForJettyToStop(j); - if (!listener.onChangeLatch.await(10, TimeUnit.SECONDS)) { - fail("onChange listener didn't execute on cluster change"); - } - - assertEquals(0, listener.addedNodes.size()); - // wait until the new overseer is up - Thread.sleep(5000); - String newOverseerLeader; - do { - overSeerStatus = cluster.getSolrClient().request(CollectionAdminRequest.getOverseerStatus()); - newOverseerLeader = (String) overSeerStatus.get("leader"); - } while (newOverseerLeader == null || newOverseerLeader.equals(overseerLeader)); - - assertEquals(1, listener.lostNodes.size()); - assertEquals(overseerLeader, listener.lostNodes.iterator().next()); - - - String pathLost = ZkStateReader.SOLR_AUTOSCALING_NODE_LOST_PATH + "/" + overseerLeader; - - TimeOut timeout = new TimeOut(30, TimeUnit.SECONDS, TimeSource.NANO_TIME); - AtomicBoolean markerInactive = new AtomicBoolean(); - try { - timeout.waitFor("nodeLost marker to get inactive", () -> { - try { - if (!zkClient().exists(pathLost, true)) { - throw new RuntimeException("marker " + pathLost + " should exist!"); - } - Map markerData = Utils.getJson(zkClient(), pathLost, true); - markerInactive.set(markerData.getOrDefault(MARKER_STATE, MARKER_ACTIVE).equals(MARKER_INACTIVE)); - return markerInactive.get(); - } catch (KeeperException e) { - throw new RuntimeException(e); - } catch (InterruptedException e) { - return false; - } - }); - } catch (TimeoutException e) { - // okay - } - - // verify that the marker is inactive - the new overseer should deactivate markers once they are processed - assertTrue("Marker " + pathLost + " still active!", markerInactive.get()); - - listener.reset(); - - // set up triggers - CloudSolrClient solrClient = cluster.getSolrClient(); - - log.info("====== ADD TRIGGERS"); - String setTriggerCommand = "{" + - "'set-trigger' : {" + - "'name' : 'node_added_triggerMR'," + - "'event' : 'nodeAdded'," + - "'waitFor' : '1s'," + - "'enabled' : true," + - "'actions' : [{'name':'test','class':'" + TestEventMarkerAction.class.getName() + "'}]" + - "}}"; - @SuppressWarnings({"rawtypes"}) - SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand); - NamedList response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - - setTriggerCommand = "{" + - "'set-trigger' : {" + - "'name' : 'node_lost_triggerMR'," + - "'event' : 'nodeLost'," + - "'waitFor' : '1s'," + - "'enabled' : true," + - "'actions' : [{'name':'test','class':'" + TestEventMarkerAction.class.getName() + "'}]" + - "}}"; - req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand); - response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - - String setListener = "{\n" + - " \"set-listener\" : {\n" + - " \"name\" : \"listener_node_added_triggerMR\",\n" + - " \"trigger\" : \"node_added_triggerMR\",\n" + - " \"stage\" : \"STARTED\",\n" + - " \"class\" : \"" + AssertingListener.class.getName() + "\"\n" + - " }\n" + - "}"; - req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setListener); - response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - - overSeerStatus = cluster.getSolrClient().request(CollectionAdminRequest.getOverseerStatus()); - overseerLeader = (String) overSeerStatus.get("leader"); - overseerLeaderIndex = 0; - for (int i = 0; i < cluster.getJettySolrRunners().size(); i++) { - JettySolrRunner jetty = cluster.getJettySolrRunner(i); - if (jetty.getNodeName().equals(overseerLeader)) { - overseerLeaderIndex = i; - break; - } - } - - // create another node - log.info("====== ADD NODE 1"); - JettySolrRunner node1 = cluster.startJettySolrRunner(); - cluster.waitForAllNodes(30); - if (!listener.onChangeLatch.await(10, TimeUnit.SECONDS)) { - fail("onChange listener didn't execute on cluster change"); - } - assertEquals(1, listener.addedNodes.size()); - assertEquals(node1.getNodeName(), listener.addedNodes.iterator().next()); - // verify that a znode exists - pathAdded = ZkStateReader.SOLR_AUTOSCALING_NODE_ADDED_PATH + "/" + node1.getNodeName(); - assertTrue("Path " + pathAdded + " wasn't created", zkClient().exists(pathAdded, true)); - - listenerEventLatch.countDown(); // let the trigger thread continue - - assertTrue(triggerFiredLatch.await(10, TimeUnit.SECONDS)); - - // kill this node - listener.reset(); - events.clear(); - triggerFiredLatch = new CountDownLatch(1); - - String node1Name = node1.getNodeName(); - cluster.stopJettySolrRunner(node1); - if (!listener.onChangeLatch.await(10, TimeUnit.SECONDS)) { - fail("onChange listener didn't execute on cluster change"); - } - assertEquals(1, listener.lostNodes.size()); - assertEquals(node1Name, listener.lostNodes.iterator().next()); - // verify that a znode exists - String pathLost2 = ZkStateReader.SOLR_AUTOSCALING_NODE_LOST_PATH + "/" + node1Name; - assertTrue("Path " + pathLost2 + " wasn't created", zkClient().exists(pathLost2, true)); - - listenerEventLatch.countDown(); // let the trigger thread continue - - assertTrue(triggerFiredLatch.await(10, TimeUnit.SECONDS)); - - // triggers don't remove markers - assertTrue("Path " + pathLost2 + " should still exist", zkClient().exists(pathLost2, true)); - - listener.reset(); - events.clear(); - triggerFiredLatch = new CountDownLatch(1); - // kill overseer again - log.info("====== KILL OVERSEER 2"); - cluster.stopJettySolrRunner(overseerLeaderIndex); - if (!listener.onChangeLatch.await(10, TimeUnit.SECONDS)) { - fail("onChange listener didn't execute on cluster change"); - } - - - if (!triggerFiredLatch.await(20, TimeUnit.SECONDS)) { - fail("Trigger should have fired by now"); - } - assertEquals(1, events.size()); - TriggerEvent ev = events.iterator().next(); - @SuppressWarnings({"unchecked"}) - List nodeNames = (List) ev.getProperty(TriggerEvent.NODE_NAMES); - assertTrue(nodeNames.contains(overseerLeader)); - assertEquals(TriggerEventType.NODELOST, ev.getEventType()); - } - - private TestLiveNodesListener registerLiveNodesListener() { - TestLiveNodesListener listener = new TestLiveNodesListener(); - zkStateReader.registerLiveNodesListener(listener); - return listener; - } - - private static class TestLiveNodesListener implements LiveNodesListener { - Set lostNodes = ConcurrentHashMap.newKeySet(); - Set addedNodes = ConcurrentHashMap.newKeySet(); - CountDownLatch onChangeLatch = new CountDownLatch(1); - - public void reset() { - lostNodes.clear(); - addedNodes.clear(); - onChangeLatch = new CountDownLatch(1); - } - - @Override - public boolean onChange(SortedSet oldLiveNodes, SortedSet newLiveNodes) { - onChangeLatch.countDown(); - Set old = new HashSet<>(oldLiveNodes); - old.removeAll(newLiveNodes); - if (!old.isEmpty()) { - lostNodes.addAll(old); - } - newLiveNodes.removeAll(oldLiveNodes); - if (!newLiveNodes.isEmpty()) { - addedNodes.addAll(newLiveNodes); - } - return false; - } - } - - public static class TestEventMarkerAction extends TriggerActionBase { - - @Override - public void process(TriggerEvent event, ActionContext actionContext) { - boolean locked = lock.tryLock(); - if (!locked) { - log.info("We should never have a tryLock fail because actions are never supposed to be executed concurrently"); - return; - } - try { - events.add(event); - getTriggerFiredLatch().countDown(); - } catch (Throwable t) { - log.debug("--throwable", t); - throw t; - } finally { - lock.unlock(); - } - } - - @Override - public void init() throws Exception { - log.info("TestEventMarkerAction init"); - super.init(); - } - } - - public static class AssertingListener extends TriggerListenerBase { - @Override - public void onEvent(TriggerEvent event, TriggerEventProcessorStage stage, String actionName, ActionContext context, Throwable error, String message) throws Exception { - if (!Thread.currentThread().getName().startsWith("ScheduledTrigger")) { - // for future safety - throw new IllegalThreadStateException("AssertingListener should have been invoked by a thread from the scheduled trigger thread pool"); - } - log.debug(" --- listener fired for event: {}, stage: {}", event, stage); - listenerEventLatch.await(); - log.debug(" --- listener wait complete for event: {}, stage: {}", event, stage); - } - } -} diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/RestoreTriggerStateTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/RestoreTriggerStateTest.java deleted file mode 100644 index 7fa4dc74701..00000000000 --- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/RestoreTriggerStateTest.java +++ /dev/null @@ -1,168 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.cloud.autoscaling; - -import java.lang.invoke.MethodHandles; -import java.util.List; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; - -import org.apache.solr.client.solrj.SolrRequest; -import org.apache.solr.client.solrj.embedded.JettySolrRunner; -import org.apache.solr.client.solrj.impl.CloudSolrClient; -import org.apache.solr.client.solrj.request.CollectionAdminRequest; -import org.apache.solr.cloud.CloudTestUtils; -import org.apache.solr.cloud.CloudTestUtils.AutoScalingRequest; -import org.apache.solr.cloud.SolrCloudTestCase; -import org.apache.solr.common.util.NamedList; -import org.apache.solr.util.LogLevel; -import org.junit.BeforeClass; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import static org.apache.solr.cloud.autoscaling.TriggerIntegrationTest.WAIT_FOR_DELTA_NANOS; - -/** - * Integration test to ensure that triggers can restore state from ZooKeeper after overseer restart - * so that events detected before restart are not lost. - * - * Added in SOLR-10515 - */ -@LogLevel("org.apache.solr.cloud.autoscaling=DEBUG;org.apache.solr.client.solrj.cloud.autoscaling=DEBUG") -public class RestoreTriggerStateTest extends SolrCloudTestCase { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - private static CountDownLatch actionInitCalled; - private static CountDownLatch triggerFiredLatch; - private static AtomicBoolean triggerFired; - private static CountDownLatch actionConstructorCalled; - private static Set events = ConcurrentHashMap.newKeySet(); - private static int waitForSeconds = 1; - - @BeforeClass - public static void setupCluster() throws Exception { - configureCluster(2) - .addConfig("conf", configset("cloud-minimal")) - .configure(); - - // disable .scheduled_maintenance (once it exists) - CloudTestUtils.waitForTriggerToBeScheduled(cluster.getOpenOverseer().getSolrCloudManager(), ".scheduled_maintenance"); - CloudTestUtils.suspendTrigger(cluster.getOpenOverseer().getSolrCloudManager(), ".scheduled_maintenance"); - - actionInitCalled = new CountDownLatch(1); - triggerFiredLatch = new CountDownLatch(1); - actionConstructorCalled = new CountDownLatch(1); - triggerFired = new AtomicBoolean(); - } - - @Test - public void testEventFromRestoredState() throws Exception { - CloudSolrClient solrClient = cluster.getSolrClient(); - String setTriggerCommand = "{" + - "'set-trigger' : {" + - "'name' : 'node_added_triggerEFRS'," + - "'event' : 'nodeAdded'," + - "'waitFor' : '10s'," + - "'enabled' : true," + - "'actions' : [{'name':'test','class':'" + TestTriggerAction.class.getName() + "'}]" + - "}}"; - @SuppressWarnings({"rawtypes"}) - SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand); - NamedList response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - - if (!actionInitCalled.await(10, TimeUnit.SECONDS)) { - fail("The TriggerAction should have been created by now"); - } - - NamedList overSeerStatus = cluster.getSolrClient().request(CollectionAdminRequest.getOverseerStatus()); - String overseerLeader = (String) overSeerStatus.get("leader"); - int overseerLeaderIndex = 0; - for (int i = 0; i < cluster.getJettySolrRunners().size(); i++) { - JettySolrRunner jetty = cluster.getJettySolrRunner(i); - if (jetty.getNodeName().equals(overseerLeader)) { - overseerLeaderIndex = i; - break; - } - } - - events.clear(); - - JettySolrRunner newNode = cluster.startJettySolrRunner(); - cluster.waitForAllNodes(30); - boolean await = triggerFiredLatch.await(20, TimeUnit.SECONDS); - assertTrue("The trigger did not fire at all", await); - assertTrue(triggerFired.get()); - // reset - triggerFired.set(false); - triggerFiredLatch = new CountDownLatch(1); - NodeAddedTrigger.NodeAddedEvent nodeAddedEvent = (NodeAddedTrigger.NodeAddedEvent) events.iterator().next(); - assertNotNull(nodeAddedEvent); - @SuppressWarnings({"unchecked"}) - List nodeNames = (List) nodeAddedEvent.getProperty(TriggerEvent.NODE_NAMES); - assertTrue(nodeNames.contains(newNode.getNodeName())); - // add a second node - state of the trigger will change but it won't fire for waitFor sec. - JettySolrRunner newNode2 = cluster.startJettySolrRunner(); - Thread.sleep(10000); - // kill overseer leader - JettySolrRunner j = cluster.stopJettySolrRunner(overseerLeaderIndex); - cluster.waitForJettyToStop(j); - await = triggerFiredLatch.await(20, TimeUnit.SECONDS); - assertTrue("The trigger did not fire at all", await); - assertTrue(triggerFired.get()); - } - - public static class TestTriggerAction extends TriggerActionBase { - - public TestTriggerAction() { - actionConstructorCalled.countDown(); - } - - @Override - public void process(TriggerEvent event, ActionContext actionContext) { - try { - if (triggerFired.compareAndSet(false, true)) { - events.add(event); - long currentTimeNanos = actionContext.getCloudManager().getTimeSource().getTimeNs(); - long eventTimeNanos = event.getEventTime(); - long waitForNanos = TimeUnit.NANOSECONDS.convert(waitForSeconds, TimeUnit.SECONDS) - WAIT_FOR_DELTA_NANOS; - if (currentTimeNanos - eventTimeNanos <= waitForNanos) { - fail(event.source + " was fired before the configured waitFor period"); - } - triggerFiredLatch.countDown(); - } else { - fail(event.source + " was fired more than once!"); - } - } catch (Throwable t) { - log.debug("--throwable", t); - throw t; - } - } - - @Override - public void init() throws Exception { - log.info("TestTriggerAction init"); - actionInitCalled.countDown(); - super.init(); - } - } -} diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/ScheduledMaintenanceTriggerTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/ScheduledMaintenanceTriggerTest.java deleted file mode 100644 index 68808e19e47..00000000000 --- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/ScheduledMaintenanceTriggerTest.java +++ /dev/null @@ -1,384 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.cloud.autoscaling; - -import java.lang.invoke.MethodHandles; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; - -import org.apache.solr.client.solrj.SolrClient; -import org.apache.solr.client.solrj.SolrRequest; -import org.apache.solr.client.solrj.cloud.DistribStateManager; -import org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig; -import org.apache.solr.client.solrj.cloud.SolrCloudManager; -import org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventProcessorStage; -import org.apache.solr.client.solrj.embedded.JettySolrRunner; -import org.apache.solr.client.solrj.request.CollectionAdminRequest; -import org.apache.solr.cloud.CloudTestUtils.AutoScalingRequest; -import org.apache.solr.cloud.CloudUtil; -import org.apache.solr.cloud.SolrCloudTestCase; -import org.apache.solr.cloud.autoscaling.sim.SimCloudManager; -import org.apache.solr.common.cloud.ClusterState; -import org.apache.solr.common.cloud.ZkStateReader; -import org.apache.solr.common.params.AutoScalingParams; -import org.apache.solr.common.util.NamedList; -import org.apache.solr.common.util.TimeSource; -import org.apache.solr.common.util.Utils; -import org.apache.solr.core.SolrResourceLoader; -import org.apache.solr.util.LogLevel; -import org.apache.zookeeper.CreateMode; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * - */ -@LogLevel("org.apache.solr.cloud.autoscaling=DEBUG") -public class ScheduledMaintenanceTriggerTest extends SolrCloudTestCase { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - private static SolrCloudManager cloudManager; - private static SolrClient solrClient; - private static TimeSource timeSource; - - @BeforeClass - public static void setupCluster() throws Exception { - configureCluster(1) - .addConfig("conf", configset("cloud-minimal")) - .configure(); - if (random().nextBoolean()) { - cloudManager = cluster.getJettySolrRunner(0).getCoreContainer().getZkController().getSolrCloudManager(); - solrClient = cluster.getSolrClient(); - } else { - cloudManager = SimCloudManager.createCluster(1, TimeSource.get("simTime:50")); - // wait for defaults to be applied - due to accelerated time sometimes we may miss this - cloudManager.getTimeSource().sleep(10000); - AutoScalingConfig cfg = cloudManager.getDistribStateManager().getAutoScalingConfig(); - assertFalse("autoscaling config is empty", cfg.isEmpty()); - solrClient = ((SimCloudManager)cloudManager).simGetSolrClient(); - } - timeSource = cloudManager.getTimeSource(); - } - - @Before - public void initTest() throws Exception { - // disable .scheduled_maintenance - String suspendTriggerCommand = "{" + - "'suspend-trigger' : {'name' : '.scheduled_maintenance'}" + - "}"; - @SuppressWarnings({"rawtypes"}) - SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, suspendTriggerCommand); - NamedList response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - String setPropertiesCommand = "{" + - "'set-properties' : {" + - "'" + AutoScalingParams.TRIGGER_COOLDOWN_PERIOD_SECONDS + "': 1" + - "}" + - "}"; - response = solrClient.request(AutoScalingRequest.create(SolrRequest.METHOD.POST, setPropertiesCommand)); - assertEquals(response.get("result").toString(), "success"); - triggerFired = new CountDownLatch(1); - } - - private String addNode() throws Exception { - if (cloudManager instanceof SimCloudManager) { - return ((SimCloudManager) cloudManager).simAddNode(); - } else { - return cluster.startJettySolrRunner().getNodeName(); - } - } - - private void stopNode(String nodeName) throws Exception { - if (cloudManager instanceof SimCloudManager) { - ((SimCloudManager) cloudManager).simRemoveNode(nodeName, true); - } else { - for (JettySolrRunner jetty : cluster.getJettySolrRunners()) { - if (jetty.getNodeName().equals(nodeName)) { - cluster.stopJettySolrRunner(jetty); - break; - } - } - } - } - - @After - public void restoreDefaults() throws Exception { - @SuppressWarnings({"rawtypes"}) - SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, - "{'set-trigger' : " + AutoScaling.SCHEDULED_MAINTENANCE_TRIGGER_DSL + "}"); - NamedList response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - AutoScalingConfig autoScalingConfig = cloudManager.getDistribStateManager().getAutoScalingConfig(); - if (autoScalingConfig.getTriggerListenerConfigs().containsKey("foo")) { - String cmd = "{" + - "'remove-listener' : {'name' : 'foo'}" + - "}"; - response = solrClient.request(AutoScalingRequest.create(SolrRequest.METHOD.POST, cmd)); - assertEquals(response.get("result").toString(), "success"); - } - } - - @AfterClass - public static void teardown() throws Exception { - if (cloudManager instanceof SimCloudManager) { - cloudManager.close(); - } - solrClient = null; - cloudManager = null; - } - - @Test - public void testTriggerDefaults() throws Exception { - AutoScalingConfig autoScalingConfig = cloudManager.getDistribStateManager().getAutoScalingConfig(); - log.info("{}", autoScalingConfig); - AutoScalingConfig.TriggerConfig triggerConfig = autoScalingConfig.getTriggerConfigs().get(AutoScaling.SCHEDULED_MAINTENANCE_TRIGGER_NAME); - assertNotNull(triggerConfig); - assertEquals(3, triggerConfig.actions.size()); - assertTrue(triggerConfig.actions.get(0).actionClass.endsWith(InactiveShardPlanAction.class.getSimpleName())); - assertTrue(triggerConfig.actions.get(1).actionClass.endsWith(InactiveMarkersPlanAction.class.getSimpleName())); - assertTrue(triggerConfig.actions.get(2).actionClass.endsWith(ExecutePlanAction.class.getSimpleName())); - AutoScalingConfig.TriggerListenerConfig listenerConfig = autoScalingConfig.getTriggerListenerConfigs().get(AutoScaling.SCHEDULED_MAINTENANCE_TRIGGER_NAME + ".system"); - assertNotNull(listenerConfig); - assertEquals(AutoScaling.SCHEDULED_MAINTENANCE_TRIGGER_NAME, listenerConfig.trigger); - assertTrue(listenerConfig.listenerClass.endsWith(SystemLogListener.class.getSimpleName())); - } - - static Map> listenerEvents = new ConcurrentHashMap<>(); - static CountDownLatch listenerCreated = new CountDownLatch(1); - - public static class CapturingTriggerListener extends TriggerListenerBase { - @Override - public void configure(SolrResourceLoader loader, SolrCloudManager cloudManager, AutoScalingConfig.TriggerListenerConfig config) throws TriggerValidationException { - super.configure(loader, cloudManager, config); - listenerCreated.countDown(); - } - - @Override - public synchronized void onEvent(TriggerEvent event, TriggerEventProcessorStage stage, String actionName, - ActionContext context, Throwable error, String message) { - List lst = listenerEvents.computeIfAbsent(config.name, s -> new ArrayList<>()); - CapturedEvent ev = new CapturedEvent(timeSource.getTimeNs(), context, config, stage, actionName, event, message); - log.info("=======> {}", ev); - lst.add(ev); - } - } - - static CountDownLatch triggerFired; - - public static class TestTriggerAction extends TriggerActionBase { - - @Override - public void process(TriggerEvent event, ActionContext context) throws Exception { - if (context.getProperties().containsKey("inactive_shard_plan") || context.getProperties().containsKey("inactive_markers_plan")) { - triggerFired.countDown(); - } - } - } - - @Test - @AwaitsFix(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 17-Mar-2018 - @SuppressWarnings({"unchecked"}) - public void testInactiveShardCleanup() throws Exception { - String collection1 = getClass().getSimpleName() + "_collection1"; - CollectionAdminRequest.Create create1 = CollectionAdminRequest.createCollection(collection1, - "conf", 1, 1); - - create1.process(solrClient); - CloudUtil.waitForState(cloudManager, "failed to create " + collection1, collection1, - CloudUtil.clusterShape(1, 1)); - - // also create a very stale lock - Map lockData = new HashMap<>(); - lockData.put(ZkStateReader.STATE_TIMESTAMP_PROP, String.valueOf(cloudManager.getTimeSource().getEpochTimeNs() - - TimeUnit.NANOSECONDS.convert(48, TimeUnit.HOURS))); - String staleLockName = collection1 + "/staleShard-splitting"; - cloudManager.getDistribStateManager().makePath(ZkStateReader.COLLECTIONS_ZKNODE + "/" + - staleLockName, Utils.toJSON(lockData), CreateMode.EPHEMERAL, true); - - // expect two events - one for a very stale lock, one for the cleanup - triggerFired = new CountDownLatch(2); - - String setListenerCommand = "{" + - "'set-listener' : " + - "{" + - "'name' : 'foo'," + - "'trigger' : '" + AutoScaling.SCHEDULED_MAINTENANCE_TRIGGER_NAME + "'," + - "'stage' : ['STARTED','ABORTED','SUCCEEDED','FAILED']," + - "'beforeAction' : 'inactive_shard_plan'," + - "'afterAction' : 'inactive_shard_plan'," + - "'class' : '" + CapturingTriggerListener.class.getName() + "'" + - "}" + - "}"; - @SuppressWarnings({"rawtypes"}) - SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setListenerCommand); - NamedList response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - - String setTriggerCommand = "{" + - "'set-trigger' : {" + - "'name' : '" + AutoScaling.SCHEDULED_MAINTENANCE_TRIGGER_NAME + "'," + - "'event' : 'scheduled'," + - "'startTime' : 'NOW+10SECONDS'," + - "'every' : '+2SECONDS'," + // must be longer than the cooldown period - "'enabled' : true," + - "'actions' : [{'name' : 'inactive_shard_plan', 'class' : 'solr.InactiveShardPlanAction', 'ttl' : '20'}," + - "{'name' : 'execute_plan', 'class' : '" + ExecutePlanAction.class.getName() + "'}," + - "{'name' : 'test', 'class' : '" + TestTriggerAction.class.getName() + "'}]" + - "}}"; - req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand); - response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - - - boolean await = listenerCreated.await(10, TimeUnit.SECONDS); - assertTrue("listener not created in time", await); - - CollectionAdminRequest.SplitShard split1 = CollectionAdminRequest.splitShard(collection1) - .setShardName("shard1"); - split1.process(solrClient); - CloudUtil.waitForState(cloudManager, "failed to split " + collection1, collection1, - CloudUtil.clusterShape(3, 1, true, true)); - - - await = triggerFired.await(90, TimeUnit.SECONDS); - assertTrue("cleanup action didn't run", await); - - // cleanup should have occurred - assertFalse("no events captured!", listenerEvents.isEmpty()); - List events = new ArrayList<>(listenerEvents.get("foo")); - listenerEvents.clear(); - - assertFalse(events.isEmpty()); - CapturedEvent ce = null; - CapturedEvent staleLock = null; - for (CapturedEvent e : events) { - if (e.stage != TriggerEventProcessorStage.AFTER_ACTION) { - continue; - } - Map plan = (Map)e.context.get("properties.inactive_shard_plan"); - if (plan == null) { - continue; - } - if (plan.containsKey("cleanup")) { - ce = e; - } - // capture only the first - if (plan.containsKey("staleLocks") && staleLock == null) { - staleLock = e; - } - } - assertNotNull("missing cleanup event: " + events, ce); - assertNotNull("missing staleLocks event: " + events, staleLock); - - Map map = (Map)ce.context.get("properties.inactive_shard_plan"); - assertNotNull(map); - - Map> inactive = (Map>)map.get("inactive"); - assertEquals(1, inactive.size()); - assertNotNull(inactive.get(collection1)); - Map> cleanup = (Map>)map.get("cleanup"); - assertEquals(1, cleanup.size()); - assertNotNull(cleanup.get(collection1)); - - map = (Map)staleLock.context.get("properties.inactive_shard_plan"); - assertNotNull(map); - Map> locks = (Map>)map.get("staleLocks"); - assertNotNull(locks); - assertTrue("missing stale lock data: " + locks + "\nevents: " + events, locks.containsKey(staleLockName)); - - ClusterState state = cloudManager.getClusterStateProvider().getClusterState(); - - CloudUtil.clusterShape(2, 1).matches(state.getLiveNodes(), state.getCollection(collection1)); - } - - public static CountDownLatch getTriggerFired() { - return triggerFired; - } - - public static class TestTriggerAction2 extends TriggerActionBase { - - @Override - public void process(TriggerEvent event, ActionContext context) throws Exception { - getTriggerFired().countDown(); - } - } - - - @Test - public void testInactiveMarkersCleanup() throws Exception { - triggerFired = new CountDownLatch(1); - String setTriggerCommand = "{" + - "'set-trigger' : {" + - "'name' : 'trigger1'," + - "'event' : 'nodeAdded'," + - "'waitFor': '1s'" + - "'enabled' : true," + - "'actions' : [" + - "{'name' : 'test', 'class' : '" + TestTriggerAction2.class.getName() + "'}]" + - "}}"; - @SuppressWarnings({"rawtypes"}) - SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand); - NamedList response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - - setTriggerCommand = "{" + - "'set-trigger' : {" + - "'name' : '" + AutoScaling.SCHEDULED_MAINTENANCE_TRIGGER_NAME + "'," + - "'event' : 'scheduled'," + - "'startTime' : 'NOW+20SECONDS'," + - "'every' : '+2SECONDS'," + // must be longer than the cooldown period!! - "'enabled' : true," + - "'actions' : [{'name' : 'inactive_markers_plan', 'class' : 'solr.InactiveMarkersPlanAction', 'ttl' : '20'}," + - "{'name' : 'test', 'class' : '" + TestTriggerAction.class.getName() + "'}]" + - "}}"; - - req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand); - response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - - cloudManager.getTimeSource().sleep(5000); - - triggerFired = new CountDownLatch(1); - String node = addNode(); - - boolean await = triggerFired.await(30, TimeUnit.SECONDS); - assertTrue("trigger should have fired", await); - - triggerFired = new CountDownLatch(1); - - // should have a marker - DistribStateManager stateManager = cloudManager.getDistribStateManager(); - String nodeAddedPath = ZkStateReader.SOLR_AUTOSCALING_NODE_ADDED_PATH + "/" + node; - assertTrue("marker for nodeAdded doesn't exist", stateManager.hasData(nodeAddedPath)); - - // wait for the cleanup to fire - await = triggerFired.await(90, TimeUnit.SECONDS); - assertTrue("cleanup trigger should have fired", await); - assertFalse("marker for nodeAdded still exists", stateManager.hasData(nodeAddedPath)); - } -} diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/ScheduledTriggerIntegrationTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/ScheduledTriggerIntegrationTest.java deleted file mode 100644 index 0b812c2a1cc..00000000000 --- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/ScheduledTriggerIntegrationTest.java +++ /dev/null @@ -1,150 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.cloud.autoscaling; - -import java.lang.invoke.MethodHandles; -import java.util.Date; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicReference; - -import org.apache.solr.client.solrj.SolrRequest; -import org.apache.solr.client.solrj.embedded.JettySolrRunner; -import org.apache.solr.client.solrj.impl.CloudSolrClient; -import org.apache.solr.client.solrj.request.CollectionAdminRequest; -import org.apache.solr.cloud.CloudTestUtils; -import org.apache.solr.cloud.CloudTestUtils.AutoScalingRequest; -import org.apache.solr.cloud.SolrCloudTestCase; -import org.apache.solr.common.params.SolrParams; -import org.apache.solr.common.util.NamedList; -import org.apache.solr.util.LogLevel; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Integration test for {@link ScheduledTrigger} - */ -@LogLevel("org.apache.solr.cloud.autoscaling=DEBUG;org.apache.solr.client.solrj.cloud.autoscaling=DEBUG") -// 12-Jun-2018 @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 26-Mar-2018 -public class ScheduledTriggerIntegrationTest extends SolrCloudTestCase { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - private static CountDownLatch triggerFiredLatch; - private static final Set events = ConcurrentHashMap.newKeySet(); - private static final AtomicReference> actionContextPropertiesRef = new AtomicReference<>(); - - @Before - public void setupCluster() throws Exception { - configureCluster(2) - .addConfig("conf", configset("cloud-minimal")) - .configure(); - - // disable .scheduled_maintenance (once it exists) - CloudTestUtils.waitForTriggerToBeScheduled(cluster.getOpenOverseer().getSolrCloudManager(), ".scheduled_maintenance"); - CloudTestUtils.suspendTrigger(cluster.getOpenOverseer().getSolrCloudManager(), ".scheduled_maintenance"); - - triggerFiredLatch = new CountDownLatch(1); - } - - @After - public void afterTest() throws Exception { - shutdownCluster(); - events.clear(); - actionContextPropertiesRef.set(null); - } - - @Test - // commented 15-Sep-2018 @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2-Aug-2018 - // commented out on: 17-Feb-2019 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 14-Oct-2018 - public void testScheduledTrigger() throws Exception { - CloudSolrClient solrClient = cluster.getSolrClient(); - - // this collection will place 2 cores on 1st node and 1 core on 2nd node - String collectionName = "testScheduledTrigger"; - CollectionAdminRequest.createCollection(collectionName, 1, 3) - .process(solrClient); - - cluster.waitForActiveCollection(collectionName, 1, 3); - - // create a policy which allows only 1 core per node thereby creating a violation for the above collection - String setClusterPolicy = "{\n" + - " \"set-cluster-policy\" : [\n" + - " {\"cores\" : \"<2\", \"node\" : \"#EACH\"}\n" + - " ]\n" + - "}"; - @SuppressWarnings({"rawtypes"}) - SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setClusterPolicy); - NamedList response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - - // start a new node which can be used to balance the cluster as per policy - JettySolrRunner newNode = cluster.startJettySolrRunner(); - cluster.waitForAllNodes(30); - - String setTriggerCommand = "{" + - "'set-trigger' : {" + - "'name' : 'sched_trigger_integration1'," + - "'event' : 'scheduled'," + - "'startTime' : '" + new Date().toInstant().toString() + "'" + - "'every' : '+3SECONDS'" + - "'actions' : [" + - "{'name' : 'compute','class':'" + ComputePlanAction.class.getName() + "'}," + - "{'name' : 'execute','class':'" + ExecutePlanAction.class.getName() + "'}," + - "{'name' : 'recorder', 'class': '" + ContextPropertiesRecorderAction.class.getName() + "'}" + - "]}}"; - req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand); - response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - - assertTrue("ScheduledTrigger did not fire in time", triggerFiredLatch.await(45, TimeUnit.SECONDS)); - assertEquals(1, events.size()); - Map actionContextProps = actionContextPropertiesRef.get(); - assertNotNull(actionContextProps); - TriggerEvent event = events.iterator().next(); - @SuppressWarnings({"unchecked", "rawtypes"}) - List operations = (List) actionContextProps.get("operations"); - assertNotNull(operations); - assertEquals(1, operations.size()); - for (@SuppressWarnings({"rawtypes"})SolrRequest operation : operations) { - SolrParams params = operation.getParams(); - assertEquals(newNode.getNodeName(), params.get("targetNode")); - } - } - - public static class ContextPropertiesRecorderAction extends TriggerActionBase { - @Override - public void process(TriggerEvent event, ActionContext actionContext) { - actionContextPropertiesRef.set(actionContext.getProperties()); - try { - events.add(event); - triggerFiredLatch.countDown(); - } catch (Throwable t) { - log.debug("--throwable", t); - throw t; - } - } - } - -} diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/ScheduledTriggerTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/ScheduledTriggerTest.java deleted file mode 100644 index 84c6df9bd62..00000000000 --- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/ScheduledTriggerTest.java +++ /dev/null @@ -1,141 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.cloud.autoscaling; - -import java.time.Instant; -import java.time.format.DateTimeFormatter; -import java.time.format.DateTimeFormatterBuilder; -import java.time.temporal.ChronoField; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Date; -import java.util.HashMap; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.TimeZone; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicReference; - -import org.apache.solr.cloud.SolrCloudTestCase; -import org.apache.solr.core.CoreContainer; -import org.apache.solr.util.LogLevel; -import org.junit.BeforeClass; -import org.junit.Test; - -/** - * Test for {@link ScheduledTrigger} - */ -@LogLevel("org.apache.solr.cloud.autoscaling=DEBUG") -public class ScheduledTriggerTest extends SolrCloudTestCase { - - private AutoScaling.TriggerEventProcessor noFirstRunProcessor = event -> { - fail("Did not expect the listener to fire on first run!"); - return true; - }; - - @BeforeClass - public static void setupCluster() throws Exception { - configureCluster(1) - .addConfig("conf", configset("cloud-minimal")) - .configure(); - } - - @Test - @AwaitsFix(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018 - // this does not appear to be a good way to test this - public void testTrigger() throws Exception { - CoreContainer container = cluster.getJettySolrRunners().get(0).getCoreContainer(); - - Map properties = createTriggerProperties(new Date().toInstant().toString(), TimeZone.getDefault().getID()); - - scheduledTriggerTest(container, properties); - - TimeZone timeZone = TimeZone.getDefault(); - DateTimeFormatter dateTimeFormatter = new DateTimeFormatterBuilder() - .append(DateTimeFormatter.ISO_LOCAL_DATE).appendPattern("['T'[HH[:mm[:ss]]]]") //brackets mean optional - .parseDefaulting(ChronoField.HOUR_OF_DAY, 0) - .parseDefaulting(ChronoField.MINUTE_OF_HOUR, 0) - .parseDefaulting(ChronoField.SECOND_OF_MINUTE, 0) - .toFormatter(Locale.ROOT).withZone(timeZone.toZoneId()); - properties = createTriggerProperties(dateTimeFormatter.format(Instant.now()), timeZone.getID()); - scheduledTriggerTest(container, properties); - } - - @Test - public void testIgnoredEvent() throws Exception { - CoreContainer container = cluster.getJettySolrRunners().get(0).getCoreContainer(); - long threeDaysAgo = new Date().getTime() - TimeUnit.DAYS.toMillis(3); - Map properties = createTriggerProperties(new Date(threeDaysAgo).toInstant().toString(), - TimeZone.getDefault().getID(), - "+2DAYS", "+1HOUR"); - try (ScheduledTrigger scheduledTrigger = new ScheduledTrigger("sched1")) { - scheduledTrigger.configure(container.getResourceLoader(), container.getZkController().getSolrCloudManager(), properties); - scheduledTrigger.init(); - AtomicReference eventRef = new AtomicReference<>(); - scheduledTrigger.setProcessor(event -> { - eventRef.set(event); - return true; - }); - scheduledTrigger.run(); - assertTrue(eventRef.get().isIgnored()); - } - } - - private void scheduledTriggerTest(CoreContainer container, Map properties) throws Exception { - try (ScheduledTrigger scheduledTrigger = new ScheduledTrigger("sched1")) { - scheduledTrigger.configure(container.getResourceLoader(), container.getZkController().getSolrCloudManager(), properties); - scheduledTrigger.init(); - scheduledTrigger.setProcessor(noFirstRunProcessor); - scheduledTrigger.run(); - final List eventTimes = Collections.synchronizedList(new ArrayList<>()); - scheduledTrigger.setProcessor(event -> { - eventTimes.add(event.getEventTime()); - return true; - }); - for (int i = 0; i < 3; i++) { - Thread.sleep(3000); - scheduledTrigger.run(); - } - assertEquals(3, eventTimes.size()); - } - } - - private Map createTriggerProperties(String startTime, String timeZone) { - return createTriggerProperties(startTime, timeZone, "+3SECOND", "+2SECOND"); - } - - private Map createTriggerProperties(String startTime, String timeZone, String every, String graceTime) { - Map properties = new HashMap<>(); - properties.put("graceDuration", graceTime); - properties.put("startTime", startTime); - properties.put("timeZone", timeZone); - properties.put("every", every); - List> actions = new ArrayList<>(3); - Map map = new HashMap<>(2); - map.put("name", "compute_plan"); - map.put("class", "solr.ComputePlanAction"); - actions.add(map); - map = new HashMap<>(2); - map.put("name", "execute_plan"); - map.put("class", "solr.ExecutePlanAction"); - actions.add(map); - properties.put("actions", actions); - return properties; - } -} diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/SearchRateTriggerIntegrationTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/SearchRateTriggerIntegrationTest.java deleted file mode 100644 index 109f6980079..00000000000 --- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/SearchRateTriggerIntegrationTest.java +++ /dev/null @@ -1,747 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.cloud.autoscaling; - -import java.lang.invoke.MethodHandles; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; - -import com.carrotsearch.randomizedtesting.annotations.Nightly; -import com.google.common.util.concurrent.AtomicDouble; -import org.apache.lucene.util.LuceneTestCase; -import org.apache.solr.client.solrj.cloud.SolrCloudManager; -import org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig; -import org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventProcessorStage; -import org.apache.solr.client.solrj.impl.CloudSolrClient; -import org.apache.solr.client.solrj.request.CollectionAdminRequest; -import org.apache.solr.cloud.CloudTestUtils; -import org.apache.solr.cloud.CloudUtil; -import org.apache.solr.cloud.SolrCloudTestCase; -import org.apache.solr.common.MapWriter; -import org.apache.solr.common.cloud.Replica; -import org.apache.solr.common.cloud.ZkNodeProps; -import org.apache.solr.common.cloud.ZkStateReader; -import org.apache.solr.common.params.CollectionParams; -import org.apache.solr.common.params.CommonParams; -import org.apache.solr.common.params.SolrParams; -import org.apache.solr.common.util.NamedList; -import org.apache.solr.common.util.TimeSource; -import org.apache.solr.common.util.Utils; -import org.apache.solr.core.SolrResourceLoader; -import org.apache.solr.util.LogLevel; -import org.apache.zookeeper.data.Stat; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import static org.apache.solr.cloud.autoscaling.TriggerIntegrationTest.WAIT_FOR_DELTA_NANOS; -import static org.apache.solr.common.cloud.ZkStateReader.SOLR_AUTOSCALING_CONF_PATH; - -/** - * Integration test for {@link SearchRateTrigger} - */ -@LogLevel("org.apache.solr.cloud.autoscaling=DEBUG;org.apache.solr.client.solrj.cloud.autoscaling=DEBUG") -@LuceneTestCase.Slow -@Nightly // this test is too long for non nightly right now -public class SearchRateTriggerIntegrationTest extends SolrCloudTestCase { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - private static final TimeSource timeSource = TimeSource.NANO_TIME; - private static volatile CountDownLatch listenerCreated = new CountDownLatch(1); - private static volatile CountDownLatch listenerEventLatch = new CountDownLatch(0); - private static volatile Map> listenerEvents = new HashMap<>(); - private static volatile CountDownLatch finished = new CountDownLatch(1); - private static volatile CountDownLatch started = new CountDownLatch(1); - private static SolrCloudManager cloudManager; - - private int waitForSeconds; - - @BeforeClass - public static void setupCluster() throws Exception { - configureCluster(5) - .addConfig("conf", configset("cloud-minimal")) - .configure(); - - cloudManager = cluster.getOpenOverseer().getSolrCloudManager(); - - // disable .scheduled_maintenance (once it exists) - CloudTestUtils.waitForTriggerToBeScheduled(cloudManager, ".scheduled_maintenance"); - CloudTestUtils.suspendTrigger(cloudManager, ".scheduled_maintenance"); - - } - - @AfterClass - public static void cleanUpAfterClass() throws Exception { - cloudManager = null; - } - - @Before - public void beforeTest() throws Exception { - cluster.deleteAllCollections(); - // clear any persisted auto scaling configuration - Stat stat = zkClient().setData(SOLR_AUTOSCALING_CONF_PATH, Utils.toJSON(new ZkNodeProps()), true); - if (log.isInfoEnabled()) { - log.info("{} reset, new znode version {}", SOLR_AUTOSCALING_CONF_PATH, stat.getVersion()); - } - deleteChildrenRecursively(ZkStateReader.SOLR_AUTOSCALING_EVENTS_PATH); - deleteChildrenRecursively(ZkStateReader.SOLR_AUTOSCALING_TRIGGER_STATE_PATH); - deleteChildrenRecursively(ZkStateReader.SOLR_AUTOSCALING_NODE_LOST_PATH); - deleteChildrenRecursively(ZkStateReader.SOLR_AUTOSCALING_NODE_ADDED_PATH); - - finished = new CountDownLatch(1); - started = new CountDownLatch(1); - listenerCreated = new CountDownLatch(1); - listenerEvents = new HashMap<>(); - listenerEventLatch = new CountDownLatch(0); - - waitForSeconds = 3 + random().nextInt(5); - } - - private void deleteChildrenRecursively(String path) throws Exception { - cloudManager.getDistribStateManager().removeRecursively(path, true, false); - } - - @Test - @SuppressWarnings({"unchecked"}) - public void testAboveSearchRate() throws Exception { - CloudSolrClient solrClient = cluster.getSolrClient(); - String COLL1 = "aboveRate_collection"; - CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(COLL1, - "conf", 1, 2); - create.process(solrClient); - - CloudUtil.waitForState(cloudManager, COLL1, 60, TimeUnit.SECONDS, - CloudUtil.clusterShape(1, 2)); - - CloudTestUtils.assertAutoScalingRequest - (cloudManager, - // the trigger is initially disabled so that we have the time to set up listeners - // and generate the traffic - "{" + - "'set-trigger' : {" + - "'name' : 'search_rate_trigger1'," + - "'event' : 'searchRate'," + - "'waitFor' : '" + waitForSeconds + "s'," + - "'enabled' : false," + - "'collections' : '" + COLL1 + "'," + - "'aboveRate' : 1.0," + - "'belowRate' : 0.1," + - "'actions' : [" + - "{'name':'compute','class':'" + ComputePlanAction.class.getName() + "'}," + - "{'name':'execute','class':'" + ExecutePlanAction.class.getName() + "'}" + - "]" + - "}}"); - - CloudTestUtils.assertAutoScalingRequest - (cloudManager, - "{" + - "'set-listener' : " + - "{" + - "'name' : 'started'," + - "'trigger' : 'search_rate_trigger1'," + - "'stage' : ['STARTED']," + - "'class' : '" + StartedProcessingListener.class.getName() + "'" + - "}" + - "}"); - - CloudTestUtils.assertAutoScalingRequest - (cloudManager, - "{" + - "'set-listener' : " + - "{" + - "'name' : 'srt'," + - "'trigger' : 'search_rate_trigger1'," + - "'stage' : ['FAILED','SUCCEEDED']," + - "'afterAction': ['compute', 'execute']," + - "'class' : '" + CapturingTriggerListener.class.getName() + "'" + - "}" + - "}"); - listenerEventLatch = new CountDownLatch(3); - - CloudTestUtils.assertAutoScalingRequest - (cloudManager, - "{" + - "'set-listener' : " + - "{" + - "'name' : 'finished'," + - "'trigger' : 'search_rate_trigger1'," + - "'stage' : ['SUCCEEDED']," + - "'class' : '" + FinishedProcessingListener.class.getName() + "'" + - "}" + - "}"); - - SolrParams query = params(CommonParams.Q, "*:*"); - for (int i = 0; i < 500; i++) { - solrClient.query(COLL1, query); - } - - // enable the trigger - CloudTestUtils.assertAutoScalingRequest - (cloudManager, - "{" + - "'resume-trigger' : {" + - "'name' : 'search_rate_trigger1'" + - "}" + - "}"); - - assertTrue("The trigger did not start in a reasonable amount of time", - started.await(60, TimeUnit.SECONDS)); - - assertTrue("The trigger did not finish in a reasonable amount of time", - started.await(60, TimeUnit.SECONDS)); - - assertTrue("the listener should have recorded all events w/in a reasonable amount of time", - listenerEventLatch.await(60, TimeUnit.SECONDS)); - - // suspend the trigger - CloudTestUtils.assertAutoScalingRequest - (cloudManager, - "{" + - "'suspend-trigger' : {" + - "'name' : 'search_rate_trigger1'" + - "}" + - "}"); - - List events = listenerEvents.get("srt"); - assertEquals(listenerEvents.toString(), 3, events.size()); - assertEquals("AFTER_ACTION", events.get(0).stage.toString()); - assertEquals("compute", events.get(0).actionName); - assertEquals("AFTER_ACTION", events.get(1).stage.toString()); - assertEquals("execute", events.get(1).actionName); - assertEquals("SUCCEEDED", events.get(2).stage.toString()); - assertNull(events.get(2).actionName); - - CapturedEvent ev = events.get(0); - long now = timeSource.getTimeNs(); - // verify waitFor - assertTrue(TimeUnit.SECONDS.convert(waitForSeconds, TimeUnit.NANOSECONDS) - WAIT_FOR_DELTA_NANOS <= now - ev.event.getEventTime()); - Map nodeRates = (Map) ev.event.getProperties().get(SearchRateTrigger.HOT_NODES); - assertNotNull("nodeRates", nodeRates); - // no node violations because node rates weren't set in the config - assertTrue(nodeRates.toString(), nodeRates.isEmpty()); - List replicaRates = (List) ev.event.getProperties().get(SearchRateTrigger.HOT_REPLICAS); - assertNotNull("replicaRates", replicaRates); - assertTrue(replicaRates.toString(), replicaRates.size() > 0); - AtomicDouble totalReplicaRate = new AtomicDouble(); - replicaRates.forEach(r -> { - assertTrue(r.toString(), r.get("rate") != null); - totalReplicaRate.addAndGet((Double) r.get("rate")); - }); - Map shardRates = (Map) ev.event.getProperties().get(SearchRateTrigger.HOT_SHARDS); - assertNotNull("shardRates", shardRates); - assertEquals(shardRates.toString(), 1, shardRates.size()); - shardRates = (Map) shardRates.get(COLL1); - assertNotNull("shardRates", shardRates); - assertEquals(shardRates.toString(), 1, shardRates.size()); - AtomicDouble totalShardRate = new AtomicDouble(); - shardRates.forEach((s, r) -> totalShardRate.addAndGet((Double) r)); - Map collectionRates = (Map) ev.event.getProperties().get(SearchRateTrigger.HOT_COLLECTIONS); - assertNotNull("collectionRates", collectionRates); - assertEquals(collectionRates.toString(), 1, collectionRates.size()); - Double collectionRate = collectionRates.get(COLL1); - assertNotNull(collectionRate); - assertTrue(collectionRate > 5.0); - // two replicas - the trigger calculates average over all searchable replicas - assertEquals(collectionRate / 2, totalShardRate.get(), 5.0); - assertEquals(collectionRate, totalReplicaRate.get(), 5.0); - - // check operations - List ops = (List) ev.context.get("properties.operations"); - assertNotNull(ops); - assertTrue(ops.size() > 1); - for (MapWriter m : ops) { - assertEquals("ADDREPLICA", m._get("params.action",null)); - } - } - - @Test - @SuppressWarnings({"unchecked"}) - public void testBelowSearchRate() throws Exception { - CloudSolrClient solrClient = cluster.getSolrClient(); - String COLL1 = "belowRate_collection"; - // replicationFactor == 2 - CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(COLL1, - "conf", 1, 2); - create.process(solrClient); - CloudUtil.waitForState(cloudManager, COLL1, 60, TimeUnit.SECONDS, - CloudUtil.clusterShape(1, 2)); - - // add a couple of spare replicas above RF. Use different types. - // these additional replicas will be placed on other nodes in the cluster - solrClient.request(CollectionAdminRequest.addReplicaToShard(COLL1, "shard1", Replica.Type.NRT)); - solrClient.request(CollectionAdminRequest.addReplicaToShard(COLL1, "shard1", Replica.Type.TLOG)); - solrClient.request(CollectionAdminRequest.addReplicaToShard(COLL1, "shard1", Replica.Type.PULL)); - - CloudUtil.waitForState(cloudManager, COLL1, 60, TimeUnit.SECONDS, - CloudUtil.clusterShape(1, 5)); - - CloudTestUtils.assertAutoScalingRequest - (cloudManager, - "{" + - "'set-trigger' : {" + - "'name' : 'search_rate_trigger2'," + - "'event' : 'searchRate'," + - "'waitFor' : '" + waitForSeconds + "s'," + - "'enabled' : false," + - "'collections' : '" + COLL1 + "'," + - "'aboveRate' : 1.0," + - "'aboveNodeRate' : 1.0," + - // RecoveryStrategy calls /admin/ping, which calls /select so the rate may not be zero - // even when no external requests were made .. but it's hard to predict exactly - // what it will be. use an insanely high rate so all shards/nodes are suspect - // and produce an Op regardless of how much internal traffic is produced... - "'belowRate' : 1.0," + - "'belowNodeRate' : 1.0," + - // ...but do absolutely nothing to nodes except generate an 'NONE' Op - "'belowNodeOp' : 'none'," + - "'actions' : [" + - "{'name':'compute','class':'" + ComputePlanAction.class.getName() + "'}," + - "{'name':'execute','class':'" + ExecutePlanAction.class.getName() + "'}" + - "]" + - "}}"); - - CloudTestUtils.assertAutoScalingRequest - (cloudManager, - "{" + - "'set-listener' : " + - "{" + - "'name' : 'started'," + - "'trigger' : 'search_rate_trigger2'," + - "'stage' : ['STARTED']," + - "'class' : '" + StartedProcessingListener.class.getName() + "'" + - "}" + - "}"); - - CloudTestUtils.assertAutoScalingRequest - (cloudManager, - "{" + - "'set-listener' : " + - "{" + - "'name' : 'srt'," + - "'trigger' : 'search_rate_trigger2'," + - "'stage' : ['FAILED','SUCCEEDED']," + - "'afterAction': ['compute', 'execute']," + - "'class' : '" + CapturingTriggerListener.class.getName() + "'" + - "}" + - "}"); - listenerEventLatch = new CountDownLatch(3); - - CloudTestUtils.assertAutoScalingRequest - (cloudManager, - "{" + - "'set-listener' : " + - "{" + - "'name' : 'finished'," + - "'trigger' : 'search_rate_trigger2'," + - "'stage' : ['SUCCEEDED']," + - "'class' : '" + FinishedProcessingListener.class.getName() + "'" + - "}" + - "}"); - - // Explicitly Do Nothing Here - - // enable the trigger - final String resumeTriggerCommand = "{ 'resume-trigger' : { 'name' : 'search_rate_trigger2' } }"; - CloudTestUtils.assertAutoScalingRequest(cloudManager, resumeTriggerCommand); - - assertTrue("The trigger did not start in a reasonable amount of time", - started.await(60, TimeUnit.SECONDS)); - - assertTrue("The trigger did not finish in a reasonable amount of time", - started.await(60, TimeUnit.SECONDS)); - - assertTrue("the listener should have recorded all events w/in a reasonable amount of time", - listenerEventLatch.await(60, TimeUnit.SECONDS)); - - // suspend the trigger - final String suspendTriggerCommand = "{ 'suspend-trigger' : { 'name' : 'search_rate_trigger2' } }"; - CloudTestUtils.assertAutoScalingRequest(cloudManager, suspendTriggerCommand); - - List events = listenerEvents.get("srt"); - assertEquals(events.toString(), 3, events.size()); - CapturedEvent ev = events.get(0); - assertEquals(ev.toString(), "compute", ev.actionName); - List ops = (List)ev.event.getProperty(TriggerEvent.REQUESTED_OPS); - assertNotNull("there should be some requestedOps: " + ev.toString(), ops); - // 5 cold nodes, 3 cold replicas - assertEquals(ops.toString(), 5 + 3, ops.size()); - AtomicInteger coldNodes = new AtomicInteger(); - AtomicInteger coldReplicas = new AtomicInteger(); - ops.forEach(op -> { - if (op.getAction().equals(CollectionParams.CollectionAction.NONE)) { - coldNodes.incrementAndGet(); - } else if (op.getAction().equals(CollectionParams.CollectionAction.DELETEREPLICA)) { - coldReplicas.incrementAndGet(); - } else { - fail("unexpected op: " + op); - } - }); - assertEquals("cold nodes", 5, coldNodes.get()); - assertEquals("cold replicas", 3, coldReplicas.get()); - - // now the collection should be down to RF = 2 - CloudUtil.waitForState(cloudManager, COLL1, 60, TimeUnit.SECONDS, - CloudUtil.clusterShape(1, 2)); - - listenerEvents.clear(); - listenerEventLatch = new CountDownLatch(3); - finished = new CountDownLatch(1); - started = new CountDownLatch(1); - - // resume trigger - CloudTestUtils.assertAutoScalingRequest(cloudManager, resumeTriggerCommand); - - assertTrue("The trigger did not start in a reasonable amount of time", - started.await(60, TimeUnit.SECONDS)); - - assertTrue("The trigger did not finish in a reasonable amount of time", - started.await(60, TimeUnit.SECONDS)); - - assertTrue("the listener should have recorded all events w/in a reasonable amount of time", - listenerEventLatch.await(60, TimeUnit.SECONDS)); - - // suspend the trigger - CloudTestUtils.assertAutoScalingRequest(cloudManager, suspendTriggerCommand); - - // there should be only coldNode ops now, and no coldReplica ops since searchable RF == collection RF - - events = listenerEvents.get("srt"); - assertEquals(events.toString(), 3, events.size()); - - ev = events.get(0); - assertEquals(ev.toString(), "compute", ev.actionName); - ops = (List)ev.event.getProperty(TriggerEvent.REQUESTED_OPS); - assertNotNull("there should be some requestedOps: " + ev.toString(), ops); - assertEquals(ops.toString(), 2, ops.size()); - assertEquals(ops.toString(), CollectionParams.CollectionAction.NONE, ops.get(0).getAction()); - assertEquals(ops.toString(), CollectionParams.CollectionAction.NONE, ops.get(1).getAction()); - - - listenerEvents.clear(); - listenerEventLatch = new CountDownLatch(3); - finished = new CountDownLatch(1); - started = new CountDownLatch(1); - - log.info("## test single replicas."); - - // now allow single replicas - CloudTestUtils.assertAutoScalingRequest - (cloudManager, - "{" + - "'set-trigger' : {" + - "'name' : 'search_rate_trigger2'," + - "'event' : 'searchRate'," + - "'waitFor' : '" + waitForSeconds + "s'," + - "'enabled' : true," + - "'collections' : '" + COLL1 + "'," + - "'aboveRate' : 1.0," + - "'aboveNodeRate' : 1.0," + - "'belowRate' : 1.0," + // same excessively high values - "'belowNodeRate' : 1.0," + - "'minReplicas' : 1," + // NEW: force lower replicas - "'belowNodeOp' : 'none'," + // still do nothing to nodes - "'actions' : [" + - "{'name':'compute','class':'" + ComputePlanAction.class.getName() + "'}," + - "{'name':'execute','class':'" + ExecutePlanAction.class.getName() + "'}" + - "]" + - "}}"); - - assertTrue("The trigger did not start in a reasonable amount of time", - started.await(60, TimeUnit.SECONDS)); - - assertTrue("The trigger did not finish in a reasonable amount of time", - started.await(60, TimeUnit.SECONDS)); - - assertTrue("the listener should have recorded all events w/in a reasonable amount of time", - listenerEventLatch.await(60, TimeUnit.SECONDS)); - - // suspend the trigger - CloudTestUtils.assertAutoScalingRequest(cloudManager, suspendTriggerCommand); - - events = listenerEvents.get("srt"); - assertEquals(events.toString(), 3, events.size()); - - ev = events.get(0); - assertEquals(ev.toString(), "compute", ev.actionName); - ops = (List)ev.event.getProperty(TriggerEvent.REQUESTED_OPS); - assertNotNull("there should be some requestedOps: " + ev.toString(), ops); - - assertTrue(ops.toString(), ops.size() > 0); - AtomicInteger coldNodes2 = new AtomicInteger(); - ops.forEach(op -> { - if (op.getAction().equals(CollectionParams.CollectionAction.NONE)) { - coldNodes2.incrementAndGet(); - } else if (op.getAction().equals(CollectionParams.CollectionAction.DELETEREPLICA)) { - // ignore - } else { - fail("unexpected op: " + op); - } - }); - - assertEquals("coldNodes: " +ops.toString(), 2, coldNodes2.get()); - - // now the collection should be at RF == 1, with one additional PULL replica - CloudUtil.waitForState(cloudManager, COLL1, 60, TimeUnit.SECONDS, - CloudUtil.clusterShape(1, 1)); - } - - @Test - @AwaitsFix(bugUrl="https://issues.apache.org/jira/browse/SOLR-13163") - public void testDeleteNode() throws Exception { - CloudSolrClient solrClient = cluster.getSolrClient(); - String COLL1 = "deleteNode_collection"; - CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(COLL1, - "conf", 1, 2); - - create.process(solrClient); - CloudUtil.waitForState(cloudManager, COLL1, 60, TimeUnit.SECONDS, - CloudUtil.clusterShape(1, 2)); - - // add a couple of spare replicas above RF. Use different types to verify that only - // searchable replicas are considered - // these additional replicas will be placed on other nodes in the cluster - solrClient.request(CollectionAdminRequest.addReplicaToShard(COLL1, "shard1", Replica.Type.NRT)); - solrClient.request(CollectionAdminRequest.addReplicaToShard(COLL1, "shard1", Replica.Type.TLOG)); - solrClient.request(CollectionAdminRequest.addReplicaToShard(COLL1, "shard1", Replica.Type.PULL)); - - CloudUtil.waitForState(cloudManager, COLL1, 60, TimeUnit.SECONDS, - CloudUtil.clusterShape(1, 5)); - - CloudTestUtils.assertAutoScalingRequest - (cloudManager, - "{" + - "'set-trigger' : {" + - "'name' : 'search_rate_trigger3'," + - "'event' : 'searchRate'," + - "'waitFor' : '" + waitForSeconds + "s'," + - "'enabled' : false," + - "'collections' : '" + COLL1 + "'," + - "'aboveRate' : 1.0," + - "'aboveNodeRate' : 1.0," + - // RecoveryStrategy calls /admin/ping, which calls /select so the rate may not be zero - // even when no external requests were made .. but it's hard to predict exactly - // what it will be. use an insanely high rate so all shards/nodes are suspect - // and produce an Op regardless of how much internal traffic is produced... - "'belowRate' : 1.0," + - "'belowNodeRate' : 1.0," + - // ...our Ops should be to delete underutilised nodes... - "'belowNodeOp' : 'DELETENODE'," + - // ...allow deleting all spare replicas... - "'minReplicas' : 1," + - // ...and allow requesting all deletions in one event. - "'maxOps' : 10," + - "'actions' : [" + - "{'name':'compute','class':'" + ComputePlanAction.class.getName() + "'}," + - "{'name':'execute','class':'" + ExecutePlanAction.class.getName() + "'}" + - "]" + - "}}"); - - CloudTestUtils.assertAutoScalingRequest - (cloudManager, - "{" + - "'set-listener' : " + - "{" + - "'name' : 'started'," + - "'trigger' : 'search_rate_trigger3'," + - "'stage' : ['STARTED']," + - "'class' : '" + StartedProcessingListener.class.getName() + "'" + - "}" + - "}"); - - CloudTestUtils.assertAutoScalingRequest - (cloudManager, - "{" + - "'set-listener' : " + - "{" + - "'name' : 'srt'," + - "'trigger' : 'search_rate_trigger3'," + - "'stage' : ['FAILED','SUCCEEDED']," + - "'afterAction': ['compute', 'execute']," + - "'class' : '" + CapturingTriggerListener.class.getName() + "'" + - "}" + - "}"); - listenerEventLatch = new CountDownLatch(3); - - CloudTestUtils.assertAutoScalingRequest - (cloudManager, - "{" + - "'set-listener' : " + - "{" + - "'name' : 'finished'," + - "'trigger' : 'search_rate_trigger3'," + - "'stage' : ['SUCCEEDED']," + - "'class' : '" + FinishedProcessingListener.class.getName() + "'" + - "}" + - "}"); - - // Explicitly Do Nothing Here - - // enable the trigger - CloudTestUtils.assertAutoScalingRequest - (cloudManager, - "{" + - "'resume-trigger' : {" + - "'name' : 'search_rate_trigger3'" + - "}" + - "}"); - - assertTrue("The trigger did not start in a reasonable amount of time", - started.await(60, TimeUnit.SECONDS)); - - assertTrue("The trigger did not finish in a reasonable amount of time", - started.await(60, TimeUnit.SECONDS)); - - assertTrue("the listener should have recorded all events w/in a reasonable amount of time", - listenerEventLatch.await(60, TimeUnit.SECONDS)); - - // suspend the trigger - CloudTestUtils.assertAutoScalingRequest - (cloudManager, - "{" + - "'suspend-trigger' : {" + - "'name' : 'search_rate_trigger3'" + - "}" + - "}"); - - List events = listenerEvents.get("srt"); - assertEquals(events.toString(), 3, events.size()); - - CapturedEvent ev = events.get(0); - assertEquals(ev.toString(), "compute", ev.actionName); - @SuppressWarnings({"unchecked"}) - List ops = (List)ev.event.getProperty(TriggerEvent.REQUESTED_OPS); - assertNotNull("there should be some requestedOps: " + ev.toString(), ops); - // 4 DELETEREPLICA, 4 DELETENODE (minReplicas==1 & leader should be protected) - assertEquals(ops.toString(), 4 + 4, ops.size()); - // The above assert can fail with actual==9 because all 5 nodes are resulting in a DELETENODE - // Which is problemtatic for 2 reasons: - // 1) it means that the leader node has not been protected from the 'belowNodeOp':'DELETENODE' - // - definitely a bug that needs fixed - // 2) it suggests that minReplicas isn't being respected by 'belowNodeOp':'DELETENODE' - // - something that needs more rigerous testing - // - ie: if belowRate==0 && belowNodeRate==1 && minReplicas==2, will leader + 1 be protected? - // - // In general, to adequately trust testing of 'belowNodeOp':'DELETENODE' we should also test: - // - some nodes with multiple replicas of the shard to ensure best nodes are picked - // - node nodes hosting replicas of multiple shards/collection, only some of which are belowNodeRate - - - - AtomicInteger replicas = new AtomicInteger(); - AtomicInteger nodes = new AtomicInteger(); - ops.forEach(op -> { - if (op.getAction().equals(CollectionParams.CollectionAction.DELETEREPLICA)) { - replicas.incrementAndGet(); - } else if (op.getAction().equals(CollectionParams.CollectionAction.DELETENODE)) { - nodes.incrementAndGet(); - } else { - fail("unexpected op: " + op); - } - }); - assertEquals(ops.toString(), 4, replicas.get()); - assertEquals(ops.toString(), 4, nodes.get()); - // check status - ev = events.get(1); - assertEquals(ev.toString(), "execute", ev.actionName); - @SuppressWarnings({"unchecked"}) - List> responses = (List>)ev.context.get("properties.responses"); - assertNotNull(ev.toString(), responses); - assertEquals(responses.toString(), 8, responses.size()); - replicas.set(0); - nodes.set(0); - responses.forEach(m -> { - if (m.get("success") != null) { - replicas.incrementAndGet(); - } else if (m.get("status") != null) { - Object status = m.get("status"); - String state; - if (status instanceof Map) { - state = (String)((Map)status).get("state"); - } else if (status instanceof NamedList) { - state = (String)((NamedList)status).get("state"); - } else { - throw new IllegalArgumentException("unsupported status format: " + status.getClass().getName() + ", " + status); - } - if ("completed".equals(state)) { - nodes.incrementAndGet(); - } else { - fail("unexpected DELETENODE status: " + m); - } - } else { - fail("unexpected status: " + m); - } - }); - - assertEquals(responses.toString(), 4, replicas.get()); - assertEquals(responses.toString(), 4, nodes.get()); - - // we are left with one searchable replica - CloudUtil.waitForState(cloudManager, COLL1, 60, TimeUnit.SECONDS, - CloudUtil.clusterShape(1, 1)); - } - - public static class CapturingTriggerListener extends TriggerListenerBase { - @Override - public void configure(SolrResourceLoader loader, SolrCloudManager cloudManager, AutoScalingConfig.TriggerListenerConfig config) throws TriggerValidationException { - super.configure(loader, cloudManager, config); - listenerCreated.countDown(); - } - - @Override - public synchronized void onEvent(TriggerEvent event, TriggerEventProcessorStage stage, String actionName, - ActionContext context, Throwable error, String message) { - CapturedEvent ev = new CapturedEvent(timeSource.getTimeNs(), context, config, stage, actionName, event, message); - final CountDownLatch latch = listenerEventLatch; - synchronized (latch) { - if (0 == latch.getCount()) { - log.warn("Ignoring captured event since latch is 'full': {}", ev); - } else { - List lst = listenerEvents.computeIfAbsent(config.name, s -> new ArrayList<>()); - log.info("=======> {}", ev); - lst.add(ev); - latch.countDown(); - } - } - } - } - - public static class StartedProcessingListener extends TriggerListenerBase { - - @Override - public void onEvent(TriggerEvent event, TriggerEventProcessorStage stage, String actionName, ActionContext context, Throwable error, String message) throws Exception { - started.countDown(); - } - } - - public static class FinishedProcessingListener extends TriggerListenerBase { - - @Override - public void onEvent(TriggerEvent event, TriggerEventProcessorStage stage, String actionName, ActionContext context, Throwable error, String message) throws Exception { - finished.countDown(); - } - } - -} diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/SearchRateTriggerTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/SearchRateTriggerTest.java deleted file mode 100644 index 3b35f7e311a..00000000000 --- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/SearchRateTriggerTest.java +++ /dev/null @@ -1,357 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.cloud.autoscaling; - -import java.net.URL; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.TimeUnit; - -import com.codahale.metrics.MetricRegistry; -import com.google.common.util.concurrent.AtomicDouble; - -import org.apache.solr.client.solrj.cloud.NodeStateProvider; -import org.apache.solr.client.solrj.cloud.SolrCloudManager; -import org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventType; -import org.apache.solr.client.solrj.embedded.JettySolrRunner; -import org.apache.solr.client.solrj.impl.CloudSolrClient; -import org.apache.solr.client.solrj.impl.HttpSolrClient; -import org.apache.solr.client.solrj.impl.SolrClientCloudManager; -import org.apache.solr.client.solrj.impl.SolrClientNodeStateProvider; -import org.apache.solr.client.solrj.request.CollectionAdminRequest; -import org.apache.solr.cloud.CloudUtil; -import org.apache.solr.cloud.SolrCloudTestCase; -import org.apache.solr.cloud.ZkDistributedQueueFactory; -import org.apache.solr.common.cloud.Replica; -import org.apache.solr.common.cloud.SolrZkClient; -import org.apache.solr.common.params.AutoScalingParams; -import org.apache.solr.common.params.CollectionParams; -import org.apache.solr.common.params.CommonParams; -import org.apache.solr.common.params.SolrParams; -import org.apache.solr.common.util.TimeSource; -import org.apache.solr.core.CoreContainer; -import org.apache.solr.core.SolrResourceLoader; -import org.apache.solr.metrics.SolrMetricManager; -import org.apache.solr.util.TimeOut; -import org.junit.After; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; - -/** - * - */ -public class SearchRateTriggerTest extends SolrCloudTestCase { - private static final String PREFIX = SearchRateTriggerTest.class.getSimpleName() + "-"; - private static final String COLL1 = PREFIX + "collection1"; - private static final String COLL2 = PREFIX + "collection2"; - - private AutoScaling.TriggerEventProcessor noFirstRunProcessor = event -> { - fail("Did not expect the listener to fire on first run!"); - return true; - }; - - @BeforeClass - public static void setupCluster() throws Exception { - - } - - @Before - public void removeCollections() throws Exception { - configureCluster(4) - .addConfig("conf", configset("cloud-minimal")) - .configure(); - } - - @After - public void after() throws Exception { - shutdownCluster(); - } - - @Test - @AwaitsFix(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") - @SuppressWarnings({"unchecked"}) - public void testTrigger() throws Exception { - JettySolrRunner targetNode = cluster.getJettySolrRunner(0); - SolrZkClient zkClient = cluster.getSolrClient().getZkStateReader().getZkClient(); - SolrResourceLoader loader = targetNode.getCoreContainer().getResourceLoader(); - CoreContainer container = targetNode.getCoreContainer(); - SolrCloudManager cloudManager = new SolrClientCloudManager(new ZkDistributedQueueFactory(zkClient), cluster.getSolrClient()); - - CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(COLL1, - "conf", 2, 2); - CloudSolrClient solrClient = cluster.getSolrClient(); - create.process(solrClient); - create = CollectionAdminRequest.createCollection(COLL2, - "conf", 2, 2); - create.process(solrClient); - - CloudUtil.waitForState(cloudManager, COLL1, 60, TimeUnit.SECONDS, clusterShape(2, 2)); - CloudUtil.waitForState(cloudManager, COLL2, 60, TimeUnit.SECONDS, clusterShape(2, 2)); - - double rate = 1.0; - URL baseUrl = targetNode.getBaseUrl(); - long waitForSeconds = 5 + random().nextInt(5); - Map props = createTriggerProps(Arrays.asList(COLL1, COLL2), waitForSeconds, rate, -1); - final List events = new ArrayList<>(); - - try (SearchRateTrigger trigger = new SearchRateTrigger("search_rate_trigger")) { - trigger.configure(loader, cloudManager, props); - trigger.init(); - trigger.setProcessor(noFirstRunProcessor); - trigger.run(); - trigger.setProcessor(event -> events.add(event)); - - // generate replica traffic - String coreName = container.getLoadedCoreNames().iterator().next(); - String url = baseUrl.toString() + "/" + coreName; - try (HttpSolrClient simpleClient = new HttpSolrClient.Builder(url).build()) { - SolrParams query = params(CommonParams.Q, "*:*", CommonParams.DISTRIB, "false"); - for (int i = 0; i < 130; i++) { - simpleClient.query(query); - } - String registryCoreName = coreName.replaceFirst("_", ".").replaceFirst("_", "."); - SolrMetricManager manager = targetNode.getCoreContainer().getMetricManager(); - MetricRegistry registry = manager.registry("solr.core."+registryCoreName); - TimeOut timeOut = new TimeOut(10, TimeUnit.SECONDS, TimeSource.NANO_TIME); - // If we getting the rate too early, it will return 0 - timeOut.waitFor("Timeout waiting for rate is not zero", - () -> registry.timer("QUERY./select.requestTimes").getOneMinuteRate()!=0.0); - trigger.run(); - // waitFor delay - assertEquals(0, events.size()); - Thread.sleep(waitForSeconds * 1000); - // should generate replica event - trigger.run(); - assertEquals(1, events.size()); - TriggerEvent event = events.get(0); - assertEquals(TriggerEventType.SEARCHRATE, event.eventType); - List infos = (List)event.getProperty(SearchRateTrigger.HOT_REPLICAS); - assertEquals(1, infos.size()); - Replica info = infos.get(0); - assertEquals(coreName, info.getCoreName()); - assertTrue((Double)info.get(AutoScalingParams.RATE) > rate); - } - // close that jetty to remove the violation - alternatively wait for 1 min... - JettySolrRunner j = cluster.stopJettySolrRunner(1); - cluster.waitForJettyToStop(j); - events.clear(); - SolrParams query = params(CommonParams.Q, "*:*"); - for (int i = 0; i < 130; i++) { - solrClient.query(COLL1, query); - } - Thread.sleep(waitForSeconds * 1000); - trigger.run(); - // should generate collection event - assertEquals(1, events.size()); - TriggerEvent event = events.get(0); - Map hotCollections = (Map)event.getProperty(SearchRateTrigger.HOT_COLLECTIONS); - assertEquals(1, hotCollections.size()); - Double Rate = hotCollections.get(COLL1); - assertNotNull(Rate); - assertTrue(Rate > rate); - events.clear(); - - for (int i = 0; i < 150; i++) { - solrClient.query(COLL2, query); - solrClient.query(COLL1, query); - } - Thread.sleep(waitForSeconds * 1000); - trigger.run(); - // should generate collection event but not for COLL2 because of waitFor - assertEquals(1, events.size()); - event = events.get(0); - Map hotNodes = (Map)event.getProperty(SearchRateTrigger.HOT_NODES); - assertTrue("hotNodes", hotNodes.isEmpty()); - hotNodes.forEach((n, r) -> assertTrue(n, r > rate)); - hotCollections = (Map)event.getProperty(SearchRateTrigger.HOT_COLLECTIONS); - assertEquals(1, hotCollections.size()); - Rate = hotCollections.get(COLL1); - assertNotNull(Rate); - - events.clear(); - // assert that waitFor prevents new events from being generated - trigger.run(); - // should not generate any events - assertEquals(0, events.size()); - - Thread.sleep(waitForSeconds * 1000 * 2); - trigger.run(); - // should generate collection event - assertEquals(1, events.size()); - event = events.get(0); - hotCollections = (Map)event.getProperty(SearchRateTrigger.HOT_COLLECTIONS); - assertEquals(2, hotCollections.size()); - Rate = hotCollections.get(COLL1); - assertNotNull(Rate); - Rate = hotCollections.get(COLL2); - assertNotNull(Rate); - hotNodes = (Map)event.getProperty(SearchRateTrigger.HOT_NODES); - assertTrue("hotNodes", hotNodes.isEmpty()); - } - } - - private static final AtomicDouble mockRate = new AtomicDouble(); - - @Test - @SuppressWarnings({"unchecked"}) - public void testWaitForElapsed() throws Exception { - SolrResourceLoader loader = cluster.getJettySolrRunner(0).getCoreContainer().getResourceLoader(); - CloudSolrClient solrClient = cluster.getSolrClient(); - SolrZkClient zkClient = solrClient.getZkStateReader().getZkClient(); - SolrCloudManager cloudManager = new SolrClientCloudManager(new ZkDistributedQueueFactory(zkClient), solrClient) { - @Override - public NodeStateProvider getNodeStateProvider() { - return new SolrClientNodeStateProvider(solrClient) { - @Override - public Map getNodeValues(String node, Collection tags) { - Map values = super.getNodeValues(node, tags); - values.keySet().forEach(k -> { - values.replace(k, mockRate.get()); - }); - return values; - } - }; - } - }; - TimeSource timeSource = cloudManager.getTimeSource(); - CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(COLL1, - "conf", 2, 2); - create.process(solrClient); - CloudUtil.waitForState(cloudManager, COLL1, 60, TimeUnit.SECONDS, clusterShape(2, 4)); - - long waitForSeconds = 5 + random().nextInt(5); - Map props = createTriggerProps(Arrays.asList(COLL1, COLL2), waitForSeconds, 1.0, 0.1); - final List events = new ArrayList<>(); - - try (SearchRateTrigger trigger = new SearchRateTrigger("search_rate_trigger1")) { - trigger.configure(loader, cloudManager, props); - trigger.init(); - trigger.setProcessor(noFirstRunProcessor); - trigger.run(); - trigger.setProcessor(event -> events.add(event)); - - // set mock rates - mockRate.set(2.0); - TimeOut timeOut = new TimeOut(waitForSeconds + 2, TimeUnit.SECONDS, timeSource); - // simulate ScheduledTriggers - while (!timeOut.hasTimedOut()) { - trigger.run(); - timeSource.sleep(1000); - } - // violation persisted longer than waitFor - there should be events - assertTrue(events.toString(), events.size() > 0); - TriggerEvent event = events.get(0); - assertEquals(event.toString(), TriggerEventType.SEARCHRATE, event.eventType); - Map hotNodes, hotCollections, hotShards; - List hotReplicas; - hotNodes = (Map)event.properties.get(SearchRateTrigger.HOT_NODES); - hotCollections = (Map)event.properties.get(SearchRateTrigger.HOT_COLLECTIONS); - hotShards = (Map)event.properties.get(SearchRateTrigger.HOT_SHARDS); - hotReplicas = (List)event.properties.get(SearchRateTrigger.HOT_REPLICAS); - assertTrue("no hot nodes?", hotNodes.isEmpty()); - assertFalse("no hot collections?", hotCollections.isEmpty()); - assertFalse("no hot shards?", hotShards.isEmpty()); - assertFalse("no hot replicas?", hotReplicas.isEmpty()); - } - - mockRate.set(0.0); - events.clear(); - - try (SearchRateTrigger trigger = new SearchRateTrigger("search_rate_trigger2")) { - trigger.configure(loader, cloudManager, props); - trigger.init(); - trigger.setProcessor(noFirstRunProcessor); - trigger.run(); - trigger.setProcessor(event -> events.add(event)); - - mockRate.set(2.0); - trigger.run(); - // waitFor not elapsed - assertTrue(events.toString(), events.isEmpty()); - Thread.sleep(1000); - trigger.run(); - assertTrue(events.toString(), events.isEmpty()); - Thread.sleep(1000); - mockRate.set(0.0); - trigger.run(); - Thread.sleep(TimeUnit.MILLISECONDS.convert(waitForSeconds - 2, TimeUnit.SECONDS)); - trigger.run(); - - // violations persisted shorter than waitFor - there should be no events - assertTrue(events.toString(), events.isEmpty()); - - } - } - - @Test - public void testDefaultsAndBackcompat() throws Exception { - Map props = new HashMap<>(); - props.put("rate", 1.0); - props.put("collection", "test"); - SolrResourceLoader loader = cluster.getJettySolrRunner(0).getCoreContainer().getResourceLoader(); - SolrZkClient zkClient = cluster.getSolrClient().getZkStateReader().getZkClient(); - SolrCloudManager cloudManager = new SolrClientCloudManager(new ZkDistributedQueueFactory(zkClient), cluster.getSolrClient()); - try (SearchRateTrigger trigger = new SearchRateTrigger("search_rate_trigger2")) { - trigger.configure(loader, cloudManager, props); - Map config = trigger.getConfig(); - @SuppressWarnings({"unchecked"}) - Set collections = (Set)config.get(SearchRateTrigger.COLLECTIONS_PROP); - assertEquals(collections.toString(), 1, collections.size()); - assertEquals("test", collections.iterator().next()); - assertEquals("#ANY", config.get(AutoScalingParams.SHARD)); - assertEquals("#ANY", config.get(AutoScalingParams.NODE)); - assertEquals(1.0, config.get(SearchRateTrigger.ABOVE_RATE_PROP)); - assertEquals(-1.0, config.get(SearchRateTrigger.BELOW_RATE_PROP)); - assertEquals(SearchRateTrigger.DEFAULT_METRIC, config.get(SearchRateTrigger.METRIC_PROP)); - assertEquals(SearchRateTrigger.DEFAULT_MAX_OPS, config.get(SearchRateTrigger.MAX_OPS_PROP)); - assertNull(config.get(SearchRateTrigger.MIN_REPLICAS_PROP)); - assertEquals(CollectionParams.CollectionAction.ADDREPLICA, config.get(SearchRateTrigger.ABOVE_OP_PROP)); - assertEquals(CollectionParams.CollectionAction.MOVEREPLICA, config.get(SearchRateTrigger.ABOVE_NODE_OP_PROP)); - assertEquals(CollectionParams.CollectionAction.DELETEREPLICA, config.get(SearchRateTrigger.BELOW_OP_PROP)); - assertNull(config.get(SearchRateTrigger.BELOW_NODE_OP_PROP)); - } - } - - private Map createTriggerProps(List collections, long waitForSeconds, double aboveRate, double belowRate) { - Map props = new HashMap<>(); - props.put("aboveRate", aboveRate); - props.put("belowRate", belowRate); - props.put("event", "searchRate"); - props.put("waitFor", waitForSeconds); - props.put("enabled", true); - if (collections != null && !collections.isEmpty()) { - props.put("collections", String.join(",", collections)); - } - List> actions = new ArrayList<>(3); - Map map = new HashMap<>(2); - map.put("name", "compute_plan"); - map.put("class", "solr.ComputePlanAction"); - actions.add(map); - map = new HashMap<>(2); - map.put("name", "execute_plan"); - map.put("class", "solr.ExecutePlanAction"); - actions.add(map); - props.put("actions", actions); - return props; - } -} diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/SystemLogListenerTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/SystemLogListenerTest.java deleted file mode 100644 index 714bc3403fd..00000000000 --- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/SystemLogListenerTest.java +++ /dev/null @@ -1,293 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.cloud.autoscaling; - -import java.io.IOException; -import java.lang.invoke.MethodHandles; -import java.util.Collection; -import java.util.Map; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicReference; -import java.util.function.Supplier; - -import org.apache.solr.client.solrj.SolrRequest; -import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.client.solrj.embedded.JettySolrRunner; -import org.apache.solr.client.solrj.impl.CloudSolrClient; -import org.apache.solr.client.solrj.request.CollectionAdminRequest; -import org.apache.solr.client.solrj.response.QueryResponse; -import org.apache.solr.cloud.CloudTestUtils.AutoScalingRequest; -import org.apache.solr.cloud.SolrCloudTestCase; -import org.apache.solr.common.SolrDocument; -import org.apache.solr.common.SolrDocumentList; -import org.apache.solr.common.cloud.Replica; -import org.apache.solr.common.params.CollectionAdminParams; -import org.apache.solr.common.params.CommonParams; -import org.apache.solr.common.params.ModifiableSolrParams; -import org.apache.solr.common.util.NamedList; -import org.apache.solr.common.util.TimeSource; -import org.apache.solr.util.LogLevel; -import org.apache.solr.util.TimeOut; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Test for {@link SystemLogListener} - */ -@LogLevel("org.apache.solr.cloud.autoscaling=DEBUG") -public class SystemLogListenerTest extends SolrCloudTestCase { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - private static final AtomicBoolean fired = new AtomicBoolean(false); - private static final int NODE_COUNT = 3; - private static CountDownLatch triggerFiredLatch = new CountDownLatch(1); - @SuppressWarnings({"rawtypes"}) - private static final AtomicReference actionContextPropsRef = new AtomicReference<>(); - private static final AtomicReference eventRef = new AtomicReference<>(); - - public static class AssertingTriggerAction extends TriggerActionBase { - @Override - public void process(TriggerEvent event, ActionContext context) { - if (fired.compareAndSet(false, true)) { - eventRef.set(event); - actionContextPropsRef.set(context.getProperties()); - triggerFiredLatch.countDown(); - } - } - } - - public static class ErrorTriggerAction extends TriggerActionBase { - @Override - public void process(TriggerEvent event, ActionContext context) { - throw new RuntimeException("failure from ErrorTriggerAction"); - } - } - - @Before - public void setupCluster() throws Exception { - configureCluster(NODE_COUNT) - .addConfig("conf", configset("cloud-minimal")) - .configure(); - CollectionAdminRequest.createCollection(CollectionAdminParams.SYSTEM_COLL, null, 1, 3) - .process(cluster.getSolrClient()); - cluster.waitForActiveCollection(CollectionAdminParams.SYSTEM_COLL, 1, 3); - } - - @After - public void teardownCluster() throws Exception { - shutdownCluster(); - } - - @Test - public void test() throws Exception { - CloudSolrClient solrClient = cluster.getSolrClient(); - String setTriggerCommand = "{" + - "'set-trigger' : {" + - "'name' : 'node_lost_trigger'," + - "'event' : 'nodeLost'," + - "'waitFor' : '1s'," + - "'enabled' : true," + - "'actions' : [{'name':'compute_plan', 'class' : 'solr.ComputePlanAction'}," + - "{'name':'execute_plan','class':'solr.ExecutePlanAction'}," + - "{'name':'test','class':'" + AssertingTriggerAction.class.getName() + "'}," + - "{'name':'error','class':'" + ErrorTriggerAction.class.getName() + "'}]" + - "}}"; - @SuppressWarnings({"rawtypes"}) - SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand); - NamedList response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - - // remove default listener - String removeListenerCommand = "{\n" + - "\t\"remove-listener\" : {\n" + - "\t\t\"name\" : \"node_lost_trigger.system\"\n" + - "\t}\n" + - "}"; - req = AutoScalingRequest.create(SolrRequest.METHOD.POST, removeListenerCommand); - response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - - CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection("test", - "conf",3, 2); - create.process(solrClient); - - waitForState("Timed out waiting for replicas of new collection to be active", - "test", clusterShape(3, 6)); - - String setListenerCommand = "{" + - "'set-listener' : " + - "{" + - "'name' : 'foo'," + - "'trigger' : 'node_lost_trigger'," + - "'stage' : ['STARTED','ABORTED','SUCCEEDED', 'FAILED']," + - "'beforeAction' : ['compute_plan','execute_plan','test','error']," + - "'afterAction' : ['compute_plan','execute_plan','test','error']," + - "'class' : '" + SystemLogListener.class.getName() + "'" + - "}" + - "}"; - req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setListenerCommand); - response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - - // Stop a node (that's safe to stop for the purposes of this test) - final JettySolrRunner stoppedJetty = pickNodeToStop(); - if (log.isInfoEnabled()) { - log.info("Stopping node {}", stoppedJetty.getNodeName()); - } - cluster.stopJettySolrRunner(stoppedJetty); - cluster.waitForJettyToStop(stoppedJetty); - - assertTrue("Trigger was not fired ", triggerFiredLatch.await(60, TimeUnit.SECONDS)); - assertTrue(fired.get()); - @SuppressWarnings({"rawtypes"}) - Map context = actionContextPropsRef.get(); - assertNotNull(context); - - TimeOut timeout = new TimeOut(30, TimeUnit.SECONDS, TimeSource.NANO_TIME); - - ModifiableSolrParams query = new ModifiableSolrParams(); - query.add(CommonParams.Q, "type:" + SystemLogListener.DOC_TYPE); - query.add(CommonParams.SORT, "id asc"); - - try { - timeout.waitFor("", new Supplier() { - - @Override - public Boolean get() { - try { - cluster.getSolrClient().commit(CollectionAdminParams.SYSTEM_COLL, true, true); - - return cluster.getSolrClient().query(CollectionAdminParams.SYSTEM_COLL, query).getResults().size() == 9; - } catch (SolrServerException | IOException e) { - throw new RuntimeException(e); - } - } - }); - } catch (TimeoutException e) { - // fine - } - // make sure the event docs are replicated and committed - Thread.sleep(5000); - cluster.getSolrClient().commit(CollectionAdminParams.SYSTEM_COLL, true, true); - - - QueryResponse resp = cluster.getSolrClient().query(CollectionAdminParams.SYSTEM_COLL, query); - SolrDocumentList docs = resp.getResults(); - assertNotNull(docs); - assertEquals("wrong number of events added to .system: " + docs.toString(), - 9, docs.size()); - docs.forEach(doc -> assertCommonFields(doc)); - - // STARTED - SolrDocument doc = docs.get(0); - assertEquals("STARTED", doc.getFieldValue("stage_s")); - - // BEFORE_ACTION compute_plan - doc = docs.get(1); - assertEquals("BEFORE_ACTION", doc.getFieldValue("stage_s")); - assertEquals("compute_plan", doc.getFieldValue("action_s")); - - // AFTER_ACTION compute_plan - doc = docs.get(2); - assertEquals("AFTER_ACTION", doc.getFieldValue("stage_s")); - assertEquals("compute_plan", doc.getFieldValue("action_s")); - Collection vals = doc.getFieldValues("operations.params_ts"); - assertEquals(3, vals.size()); - for (Object val : vals) { - assertTrue(val.toString(), String.valueOf(val).contains("action=MOVEREPLICA")); - } - - // BEFORE_ACTION execute_plan - doc = docs.get(3); - assertEquals("BEFORE_ACTION", doc.getFieldValue("stage_s")); - assertEquals("execute_plan", doc.getFieldValue("action_s")); - vals = doc.getFieldValues("operations.params_ts"); - assertEquals(3, vals.size()); - - // AFTER_ACTION execute_plan - doc = docs.get(4); - assertEquals("AFTER_ACTION", doc.getFieldValue("stage_s")); - assertEquals("execute_plan", doc.getFieldValue("action_s")); - vals = doc.getFieldValues("operations.params_ts"); - assertNotNull(vals); - assertEquals(3, vals.size()); - vals = doc.getFieldValues("responses_ts"); - assertNotNull(vals); - assertEquals(3, vals.size()); - vals.forEach(s -> assertTrue(s.toString(), s.toString().startsWith("success MOVEREPLICA action completed successfully"))); - - // BEFORE_ACTION test - doc = docs.get(5); - assertEquals("BEFORE_ACTION", doc.getFieldValue("stage_s")); - assertEquals("test", doc.getFieldValue("action_s")); - - // AFTER_ACTION test - doc = docs.get(6); - assertEquals("AFTER_ACTION", doc.getFieldValue("stage_s")); - assertEquals("test", doc.getFieldValue("action_s")); - - // BEFORE_ACTION error - doc = docs.get(7); - assertEquals("BEFORE_ACTION", doc.getFieldValue("stage_s")); - assertEquals("error", doc.getFieldValue("action_s")); - - // FAILED error - doc = docs.get(8); - assertEquals("FAILED", doc.getFieldValue("stage_s")); - assertEquals("error", doc.getFieldValue("action_s")); - assertEquals("failure from ErrorTriggerAction", doc.getFieldValue("error.message_t")); - assertTrue(doc.getFieldValue("error.details_t").toString().contains("RuntimeException")); - } - - private void assertCommonFields(SolrDocument doc) { - assertEquals(SystemLogListener.class.getSimpleName(), doc.getFieldValue(SystemLogListener.SOURCE_FIELD)); - assertEquals(SystemLogListener.DOC_TYPE, doc.getFieldValue(CommonParams.TYPE)); - assertEquals("node_lost_trigger", doc.getFieldValue("event.source_s")); - assertNotNull(doc.getFieldValue("event.time_l")); - assertNotNull(doc.getFieldValue("timestamp")); - assertNotNull(doc.getFieldValue("event.property.nodeNames_ss")); - assertNotNull(doc.getFieldValue("event_str")); - assertEquals("NODELOST", doc.getFieldValue("event.type_s")); - } - - /** - * Helper method for picking a node that can safely be stoped - * @see SOLR-13050 - */ - private JettySolrRunner pickNodeToStop() throws Exception { - // first get the nodeName of the overser. - // stopping the overseer is not something we want to hassle with in this test - final String overseerNodeName = (String) cluster.getSolrClient().request - (CollectionAdminRequest.getOverseerStatus()).get("leader"); - - // now find a node that is *NOT* the overseer or the leader of a .system collection shard - for (Replica r : getCollectionState(CollectionAdminParams.SYSTEM_COLL).getReplicas()) { - if ( ! (r.getBool("leader", false) || r.getNodeName().equals(overseerNodeName) ) ) { - return cluster.getReplicaJetty(r); - } - } - fail("Couldn't find non-leader, non-overseer, replica of .system collection to kill"); - return null; - } - -} diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/TestPolicyCloud.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/TestPolicyCloud.java deleted file mode 100644 index eaca650ad84..00000000000 --- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/TestPolicyCloud.java +++ /dev/null @@ -1,542 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.cloud.autoscaling; - -import java.lang.invoke.MethodHandles; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.BiConsumer; - -import com.google.common.collect.ImmutableSet; -import org.apache.lucene.util.Constants; -import org.apache.lucene.util.LuceneTestCase; -import org.apache.solr.client.solrj.SolrRequest; -import org.apache.solr.client.solrj.cloud.DistributedQueueFactory; -import org.apache.solr.client.solrj.cloud.SolrCloudManager; -import org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig; -import org.apache.solr.client.solrj.cloud.autoscaling.Policy; -import org.apache.solr.client.solrj.cloud.autoscaling.Row; -import org.apache.solr.client.solrj.cloud.autoscaling.Variable.Type; -import org.apache.solr.client.solrj.embedded.JettySolrRunner; -import org.apache.solr.client.solrj.impl.BaseHttpSolrClient; -import org.apache.solr.client.solrj.impl.CloudSolrClient; -import org.apache.solr.client.solrj.impl.SolrClientCloudManager; -import org.apache.solr.client.solrj.impl.SolrClientNodeStateProvider; -import org.apache.solr.client.solrj.request.CollectionAdminRequest; -import org.apache.solr.cloud.CloudTestUtils.AutoScalingRequest; -import org.apache.solr.cloud.OverseerTaskProcessor; -import org.apache.solr.cloud.SolrCloudTestCase; -import org.apache.solr.cloud.ZkDistributedQueueFactory; -import org.apache.solr.common.cloud.CollectionStatePredicate; -import org.apache.solr.common.cloud.DocCollection; -import org.apache.solr.common.cloud.Replica; -import org.apache.solr.common.cloud.Slice; -import org.apache.solr.common.cloud.ZkStateReader; -import org.apache.solr.common.util.NamedList; -import org.apache.solr.common.util.TimeSource; -import org.apache.solr.common.util.Utils; -import org.apache.solr.util.TimeOut; -import org.junit.After; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.rules.ExpectedException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import static org.apache.solr.common.util.Utils.getObjectByPath; - -@LuceneTestCase.Slow -public class TestPolicyCloud extends SolrCloudTestCase { - - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - @org.junit.Rule - public ExpectedException expectedException = ExpectedException.none(); - - @BeforeClass - public static void setupCluster() throws Exception { - configureCluster(5) - .addConfig("conf", configset("cloud-minimal")) - .configure(); - } - - @Before - public void before() throws Exception { - // remove default policy - String commands = "{set-cluster-policy : []}"; - cluster.getSolrClient().request(AutoScalingRequest.create(SolrRequest.METHOD.POST, commands)); - } - - @After - public void after() throws Exception { - cluster.deleteAllCollections(); - cluster.getSolrClient().getZkStateReader().getZkClient().setData(ZkStateReader.SOLR_AUTOSCALING_CONF_PATH, - "{}".getBytes(StandardCharsets.UTF_8), true); - } - - public void testCreateCollection() throws Exception { - String commands = "{ set-cluster-policy: [ {cores: '0', node: '#ANY'} ] }"; // disallow replica placement anywhere - cluster.getSolrClient().request(AutoScalingRequest.create(SolrRequest.METHOD.POST, commands)); - String collectionName = "testCreateCollection"; - BaseHttpSolrClient.RemoteSolrException exp = expectThrows(BaseHttpSolrClient.RemoteSolrException.class, - () -> CollectionAdminRequest.createCollection(collectionName, "conf", 2, 1).process(cluster.getSolrClient())); - - assertTrue(exp.getMessage().contains("No node can satisfy the rules")); - assertTrue(exp.getMessage().contains("AutoScaling.error.diagnostics")); - - // wait for a while until we don't see the collection - TimeOut timeout = new TimeOut(30, TimeUnit.SECONDS, new TimeSource.NanoTimeSource()); - boolean removed = false; - while (! timeout.hasTimedOut()) { - timeout.sleep(100); - removed = !cluster.getSolrClient().getZkStateReader().getClusterState().hasCollection(collectionName); - if (removed) { - timeout.sleep(500); // just a bit of time so it's more likely other - // readers see on return - break; - } - } - if (!removed) { - fail("Collection should have been deleted from cluster state but still exists: " + collectionName); - } - - commands = "{ set-cluster-policy: [ {cores: '<2', node: '#ANY'} ] }"; - cluster.getSolrClient().request(AutoScalingRequest.create(SolrRequest.METHOD.POST, commands)); - CollectionAdminRequest.createCollection(collectionName, "conf", 2, 1).process(cluster.getSolrClient()); - - cluster.waitForActiveCollection(collectionName, 2, 2); - - } - - public void testDataProviderPerReplicaDetails() throws Exception { - CollectionAdminRequest.createCollection("perReplicaDataColl", "conf", 1, 5) - .process(cluster.getSolrClient()); - cluster.waitForActiveCollection("perReplicaDataColl", 1, 5); - DocCollection coll = getCollectionState("perReplicaDataColl"); - String autoScaleJson = "{" + - " 'cluster-preferences': [" + - " { maximize : freedisk , precision: 50}," + - " { minimize : cores, precision: 2}" + - " ]," + - " 'cluster-policy': [" + - " { replica : '0' , 'nodeRole': 'overseer'}," + - " { 'replica': '<2', 'shard': '#ANY', 'node': '#ANY'" + - " }" + - " ]," + - " 'policies': {" + - " 'policy1': [" + - " { 'replica': '<2', 'shard': '#EACH', 'node': '#ANY'}," + - " { 'replica': '<2', 'shard': '#EACH', 'sysprop.rack': 'rack1'}" + - " ]" + - " }" + - "}"; - @SuppressWarnings({"unchecked"}) - AutoScalingConfig config = new AutoScalingConfig((Map) Utils.fromJSONString(autoScaleJson)); - AtomicInteger count = new AtomicInteger(0); - try (SolrCloudManager cloudManager = new SolrClientCloudManager(new ZkDistributedQueueFactory(cluster.getZkClient()), cluster.getSolrClient())) { - String nodeName = cloudManager.getClusterStateProvider().getLiveNodes().iterator().next(); - SolrClientNodeStateProvider nodeStateProvider = (SolrClientNodeStateProvider) cloudManager.getNodeStateProvider(); - Map>> result = nodeStateProvider.getReplicaInfo(nodeName, Collections.singleton("UPDATE./update.requests")); - nodeStateProvider.forEachReplica(nodeName, replicaInfo -> { - if (replicaInfo.getProperties().containsKey("UPDATE./update.requests")) count.incrementAndGet(); - }); - assertTrue(count.get() > 0); - - Policy.Session session = config.getPolicy().createSession(cloudManager); - - for (Row row : session.getSortedNodes()) { - Object val = row.getVal(Type.TOTALDISK.tagName, null); - if (log.isInfoEnabled()) { - log.info("node: {} , totaldisk : {}, freedisk : {}", row.node, val, row.getVal("freedisk", null)); - } - assertTrue(val != null); - - } - - count .set(0); - for (Row row : session.getSortedNodes()) { - row.collectionVsShardVsReplicas.forEach((c, shardVsReplicas) -> shardVsReplicas.forEach((s, replicaInfos) -> { - for (Replica replicaInfo : replicaInfos) { - if (replicaInfo.getProperties().containsKey(Type.CORE_IDX.tagName)) count.incrementAndGet(); - } - })); - } - assertTrue(count.get() > 0); - } - } - - private static CollectionStatePredicate expectAllReplicasOnSpecificNode - (final String expectedNodeName, - final int expectedSliceCount, - final int expectedReplicaCount) { - - return (liveNodes, collection) -> { - if (null == collection || expectedSliceCount != collection.getSlices().size()) { - return false; - } - int actualReplicaCount = 0; - for (Slice slice : collection) { - for (Replica replica : slice) { - if ( ! (replica.isActive(liveNodes) - && expectedNodeName.equals(replica.getNodeName())) ) { - return false; - } - actualReplicaCount++; - } - } - return expectedReplicaCount == actualReplicaCount; - }; - } - - public void testCreateCollectionAddReplica() throws Exception { - final JettySolrRunner jetty = cluster.getRandomJetty(random()); - final String jettyNodeName = jetty.getNodeName(); - final int port = jetty.getLocalPort(); - - final String commands = "{set-policy :{c1 : [{replica:0 , shard:'#EACH', port: '!" + port + "'}]}}"; - cluster.getSolrClient().request(AutoScalingRequest.create(SolrRequest.METHOD.POST, commands)); - - final String collectionName = "testCreateCollectionAddReplica"; - log.info("Creating collection {}", collectionName); - CollectionAdminRequest.createCollection(collectionName, "conf", 1, 1) - .setPolicy("c1") - .process(cluster.getSolrClient()); - - waitForState("Should have found exactly one replica, only on expected jetty: " + - jettyNodeName + "/" + port, - collectionName, expectAllReplicasOnSpecificNode(jettyNodeName, 1, 1), - 120, TimeUnit.SECONDS); - - log.info("Adding replica to {}", collectionName); - CollectionAdminRequest.addReplicaToShard(collectionName, "shard1") - .process(cluster.getSolrClient()); - - waitForState("Should have found exactly two replicas, only on expected jetty: " + - jettyNodeName + "/" + port, - collectionName, expectAllReplicasOnSpecificNode(jettyNodeName, 1, 2), - 120, TimeUnit.SECONDS); - - } - - public void testCreateCollectionSplitShard() throws Exception { - - final List shuffledJetties = new ArrayList<>(cluster.getJettySolrRunners()); - Collections.shuffle(shuffledJetties, random()); - assertTrue(2 < shuffledJetties.size()); // sanity check test setup - - final JettySolrRunner firstNode = shuffledJetties.get(0); - final JettySolrRunner secondNode = shuffledJetties.get(1); - - final int firstNodePort = firstNode.getLocalPort(); - final int secondNodePort = secondNode.getLocalPort(); - assertNotEquals(firstNodePort, secondNodePort); - - final String commands = "{set-policy :{c1 : [{replica:1 , shard:'#EACH', port: '" + - firstNodePort + "'}, {replica:1, shard:'#EACH', port:'" + secondNodePort + "'}]}}"; - - final String firstNodeName = firstNode.getNodeName(); - final String secondNodeName = secondNode.getNodeName(); - assertNotEquals(firstNodeName, secondNodeName); - - final NamedList response = cluster.getSolrClient() - .request(AutoScalingRequest.create(SolrRequest.METHOD.POST, commands)); - assertEquals("success", response.get("result")); - - // through out the test, every shard shuld have 2 replicas, one on each of these two nodes - final Set expectedNodeNames = ImmutableSet.of(firstNodeName, secondNodeName); - - final String collectionName = "testCreateCollectionSplitShard"; - log.info("Creating collection {}", collectionName); - CollectionAdminRequest.createCollection(collectionName, "conf", 1, 2) - .setPolicy("c1") - .process(cluster.getSolrClient()); - - waitForState("Should have found exactly 1 slice w/2 live Replicas, one on each expected jetty: " + - firstNodeName + "/" + firstNodePort + " & " + secondNodeName + "/" + secondNodePort, - collectionName, (liveNodes, collection) -> { - // short circut if collection is deleted - // or we some how have the wrong number of slices - if (null == collection || 1 != collection.getSlices().size()) { - return false; - } - // Note: only 1 slices, but simpler to loop then extract... - for (Slice slice : collection.getSlices()) { - // short circut if our slice isn't active, or has wrong # replicas - if (Slice.State.ACTIVE != slice.getState() - || 2 != slice.getReplicas().size()) { - return false; - } - // make sure our replicas are fully live... - final List liveReplicas = slice.getReplicas - ((r) -> r.isActive(liveNodes)); - if (2 != liveReplicas.size()) { - return false; - } - // now the main check we care about: were the replicas split up on - // the expected nodes... - if (! expectedNodeNames.equals(ImmutableSet.of - (liveReplicas.get(0).getNodeName(), - liveReplicas.get(1).getNodeName()))) { - return false; - } - } - return true; - }); - - log.info("Splitting (single) Shard on collection {}", collectionName); - CollectionAdminRequest.splitShard(collectionName).setShardName("shard1") - .process(cluster.getSolrClient()); - - waitForState("Should have found exactly 3 shards (1 inactive) each w/two live Replicas, " + - "one on each expected jetty: " + - firstNodeName + "/" + firstNodePort + " & " + secondNodeName + "/" + secondNodePort, - collectionName, (liveNodes, collection) -> { - // short circut if collection is deleted - // or we some how have the wrong number of (active) slices - if (null == collection - || 3 != collection.getSlices().size() - || 2 != collection.getActiveSlices().size()) { - return false; - } - // Note: we're checking all slices, even the inactive (split) slice... - for (Slice slice : collection.getSlices()) { - // short circut if our slice has wrong # replicas - if (2 != slice.getReplicas().size()) { - return false; - } - // make sure our replicas are fully live... - final List liveReplicas = slice.getReplicas - ((r) -> r.isActive(liveNodes)); - if (2 != liveReplicas.size()) { - return false; - } - // now the main check we care about: were the replicas split up on - // the expected nodes... - if (! expectedNodeNames.equals(ImmutableSet.of - (liveReplicas.get(0).getNodeName(), - liveReplicas.get(1).getNodeName()))) { - return false; - } - } - return true; - }); - } - - public void testMetricsTag() throws Exception { - CloudSolrClient solrClient = cluster.getSolrClient(); - String setClusterPolicyCommand = "{" + - " 'set-cluster-policy': [" + - " {'cores':'<10', 'node':'#ANY'}," + - " {'replica':'<2', 'shard': '#EACH', 'node': '#ANY'}," + - " {'metrics:abc':'overseer', 'replica':0}" + - " ]" + - "}"; - @SuppressWarnings({"rawtypes"}) - SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setClusterPolicyCommand); - try { - solrClient.request(req); - fail("expected exception"); - } catch (BaseHttpSolrClient.RemoteExecutionException e) { - // expected - assertTrue(String.valueOf(getObjectByPath(e.getMetaData(), - false, "error/details[0]/errorMessages[0]")).contains("Invalid metrics: param in")); - } - setClusterPolicyCommand = "{" + - " 'set-cluster-policy': [" + - " {'cores':'<10', 'node':'#ANY'}," + - " {'replica':'<2', 'shard': '#EACH', 'node': '#ANY'}," + - " {'metrics:solr.node:ADMIN./admin/authorization.clientErrors:count':'>58768765', 'replica':0}" + - " ]" + - "}"; - req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setClusterPolicyCommand); - solrClient.request(req); - - final String collectionName = "metrics_tags"; - CollectionAdminRequest.createCollection(collectionName, "conf", 1, 1) - .process(cluster.getSolrClient()); - cluster.waitForActiveCollection(collectionName, 1, 1); - DocCollection collection = getCollectionState(collectionName); - DistributedQueueFactory queueFactory = new ZkDistributedQueueFactory(cluster.getZkClient()); - try (SolrCloudManager provider = new SolrClientCloudManager(queueFactory, solrClient)) { - List tags = Arrays.asList("metrics:solr.node:ADMIN./admin/authorization.clientErrors:count", - "metrics:solr.jvm:buffers.direct.Count"); - Map val = provider.getNodeStateProvider().getNodeValues(collection.getReplicas().get(0).getNodeName(), tags); - for (String tag : tags) { - assertNotNull("missing : " + tag, val.get(tag)); - } - val = provider.getNodeStateProvider().getNodeValues(collection.getReplicas().get(0).getNodeName(), Collections.singleton("diskType")); - - Set diskTypes = ImmutableSet.of("rotational", "ssd"); - assertTrue(diskTypes.contains(val.get("diskType"))); - } - } - - public void testCreateCollectionAddShardWithReplicaTypeUsingPolicy() throws Exception { - JettySolrRunner jetty = cluster.getJettySolrRunners().get(0); - String nrtNodeName = jetty.getNodeName(); - int nrtPort = jetty.getLocalPort(); - - jetty = cluster.getJettySolrRunners().get(1); - String pullNodeName = jetty.getNodeName(); - int pullPort = jetty.getLocalPort(); - - jetty = cluster.getJettySolrRunners().get(2); - String tlogNodeName = jetty.getNodeName(); - int tlogPort = jetty.getLocalPort(); - log.info("NRT {} PULL {} , TLOG {} ", nrtNodeName, pullNodeName, tlogNodeName); - - String commands = "{set-cluster-policy :[" + - "{replica:0 , shard:'#EACH', type: NRT, port: '!" + nrtPort + "'}" + - "{replica:0 , shard:'#EACH', type: PULL, port: '!" + pullPort + "'}" + - "{replica:0 , shard:'#EACH', type: TLOG, port: '!" + tlogPort + "'}" + - "]}"; - - - cluster.getSolrClient().request(AutoScalingRequest.create(SolrRequest.METHOD.POST, commands)); - Map json = Utils.getJson(cluster.getZkClient(), ZkStateReader.SOLR_AUTOSCALING_CONF_PATH, true); - assertEquals("full json:" + Utils.toJSONString(json), "!" + nrtPort, - Utils.getObjectByPath(json, true, "cluster-policy[0]/port")); - assertEquals("full json:" + Utils.toJSONString(json), "!" + pullPort, - Utils.getObjectByPath(json, true, "cluster-policy[1]/port")); - assertEquals("full json:" + Utils.toJSONString(json), "!" + tlogPort, - Utils.getObjectByPath(json, true, "cluster-policy[2]/port")); - - final String collectionName = "addshard_with_reptype_using_policy"; - CollectionAdminRequest.createCollectionWithImplicitRouter(collectionName, "conf", "s1", 1, 1, 1) - .process(cluster.getSolrClient()); - - cluster.waitForActiveCollection(collectionName, 1, 3); - - DocCollection coll = getCollectionState(collectionName); - - - BiConsumer verifyReplicas = (s, replica) -> { - switch (replica.getType()) { - case NRT: { - assertTrue("NRT replica should be in " + nrtNodeName, replica.getNodeName().equals(nrtNodeName)); - break; - } - case TLOG: { - assertTrue("TLOG replica should be in " + tlogNodeName, replica.getNodeName().equals(tlogNodeName)); - break; - } - case PULL: { - assertTrue("PULL replica should be in " + pullNodeName, replica.getNodeName().equals(pullNodeName)); - break; - } - } - - }; - coll.forEachReplica(verifyReplicas); - - CollectionAdminRequest.createShard(collectionName, "s3"). - process(cluster.getSolrClient()); - - cluster.waitForActiveCollection(collectionName, 2, 6); - - coll = getCollectionState(collectionName); - assertEquals(3, coll.getSlice("s3").getReplicas().size()); - coll.forEachReplica(verifyReplicas); - } - - public void testCreateCollectionAddShardUsingPolicy() throws Exception { - JettySolrRunner jetty = cluster.getRandomJetty(random()); - int port = jetty.getLocalPort(); - - String commands = "{set-policy :{c1 : [{replica:1 , shard:'#EACH', port: '" + port + "'}]}}"; - cluster.getSolrClient().request(AutoScalingRequest.create(SolrRequest.METHOD.POST, commands)); - Map json = Utils.getJson(cluster.getZkClient(), ZkStateReader.SOLR_AUTOSCALING_CONF_PATH, true); - assertEquals("full json:"+ Utils.toJSONString(json) , "#EACH", - Utils.getObjectByPath(json, true, "/policies/c1[0]/shard")); - - final String collectionName = "addshard_using_policy"; - CollectionAdminRequest.createCollectionWithImplicitRouter(collectionName, "conf", "s1,s2", 1) - .setPolicy("c1") - .process(cluster.getSolrClient()); - - cluster.waitForActiveCollection(collectionName, 2, 2); - DocCollection coll = getCollectionState(collectionName); - assertEquals("c1", coll.getPolicyName()); - assertEquals(2,coll.getReplicas().size()); - coll.forEachReplica((s, replica) -> assertEquals(jetty.getNodeName(), replica.getNodeName())); - - CollectionAdminRequest.createShard(collectionName, "s3").process(cluster.getSolrClient()); - - cluster.waitForActiveCollection(collectionName, 3, 3); - - coll = getCollectionState(collectionName); - assertEquals(1, coll.getSlice("s3").getReplicas().size()); - coll.getSlice("s3").forEach(replica -> assertEquals(jetty.getNodeName(), replica.getNodeName())); - } - - public void testDataProvider() throws Exception { - final String collectionName = "data_provider"; - CollectionAdminRequest.createCollectionWithImplicitRouter(collectionName, "conf", "shard1", 2) - .process(cluster.getSolrClient()); - - cluster.waitForActiveCollection(collectionName, 1, 2); - - DocCollection rulesCollection = getCollectionState(collectionName); - - try (SolrCloudManager cloudManager = new SolrClientCloudManager(new ZkDistributedQueueFactory(cluster.getZkClient()), cluster.getSolrClient())) { - Map val = cloudManager.getNodeStateProvider().getNodeValues(rulesCollection.getReplicas().get(0).getNodeName(), Arrays.asList( - "freedisk", - "cores", - "host", - "heapUsage", - "sysLoadAvg")); - assertNotNull(val.get("freedisk")); - assertNotNull(val.get("host")); - assertNotNull(val.get("heapUsage")); - assertNotNull(val.get("sysLoadAvg")); - assertTrue(((Number) val.get("cores")).intValue() > 0); - assertTrue("freedisk value is " + ((Number) val.get("freedisk")).doubleValue(), Double.compare(((Number) val.get("freedisk")).doubleValue(), 0.0d) > 0); - assertTrue("heapUsage value is " + ((Number) val.get("heapUsage")).doubleValue(), Double.compare(((Number) val.get("heapUsage")).doubleValue(), 0.0d) > 0); - if (!Constants.WINDOWS) { - // the system load average metrics is not available on windows platform - assertTrue("sysLoadAvg value is " + ((Number) val.get("sysLoadAvg")).doubleValue(), Double.compare(((Number) val.get("sysLoadAvg")).doubleValue(), 0.0d) > 0); - } - String overseerNode = OverseerTaskProcessor.getLeaderNode(cluster.getZkClient()); - cluster.getSolrClient().request(CollectionAdminRequest.addRole(overseerNode, "overseer")); - for (int i = 0; i < 10; i++) { - Map data = Utils.getJson(cluster.getZkClient(), ZkStateReader.ROLES, true); - if (i >= 9 && data.isEmpty()) { - throw new RuntimeException("NO overseer node created"); - } - Thread.sleep(100); - } - val = cloudManager.getNodeStateProvider().getNodeValues(overseerNode, Arrays.asList( - "nodeRole", - "ip_1", "ip_2", "ip_3", "ip_4", - "sysprop.java.version", - "sysprop.java.vendor")); - assertEquals("overseer", val.get("nodeRole")); - assertNotNull(val.get("ip_1")); - assertNotNull(val.get("ip_2")); - assertNotNull(val.get("ip_3")); - assertNotNull(val.get("ip_4")); - assertNotNull(val.get("sysprop.java.version")); - assertNotNull(val.get("sysprop.java.vendor")); - } - } -} diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerCooldownIntegrationTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerCooldownIntegrationTest.java deleted file mode 100644 index b7f40cf6e0b..00000000000 --- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerCooldownIntegrationTest.java +++ /dev/null @@ -1,228 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.cloud.autoscaling; - -import java.lang.invoke.MethodHandles; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; - -import org.apache.solr.client.solrj.SolrRequest; -import org.apache.solr.client.solrj.cloud.SolrCloudManager; -import org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig; -import org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventProcessorStage; -import org.apache.solr.client.solrj.embedded.JettySolrRunner; -import org.apache.solr.client.solrj.impl.CloudSolrClient; -import org.apache.solr.cloud.CloudTestUtils; -import org.apache.solr.cloud.CloudTestUtils.AutoScalingRequest; -import org.apache.solr.cloud.SolrCloudTestCase; -import org.apache.solr.common.params.AutoScalingParams; -import org.apache.solr.common.util.NamedList; -import org.apache.solr.common.util.TimeSource; -import org.apache.solr.core.SolrResourceLoader; -import org.apache.solr.util.LogLevel; -import org.junit.BeforeClass; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import static org.apache.solr.cloud.autoscaling.TriggerIntegrationTest.WAIT_FOR_DELTA_NANOS; - -@LogLevel("org.apache.solr.cloud.autoscaling=DEBUG;org.apache.solr.client.solrj.cloud.autoscaling=DEBUG") -public class TriggerCooldownIntegrationTest extends SolrCloudTestCase { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - private static final int waitForSeconds = 1; - - private static final Map> listenerEvents = new HashMap<>(); - private static CountDownLatch triggerFiredLatch = new CountDownLatch(1); - private static final AtomicBoolean triggerFired = new AtomicBoolean(); - - private static final void resetTriggerAndListenerState() { - // reset the trigger and captured events - listenerEvents.clear(); - triggerFiredLatch = new CountDownLatch(1); - triggerFired.compareAndSet(true, false); - } - - @BeforeClass - public static void setupCluster() throws Exception { - resetTriggerAndListenerState(); - configureCluster(2) - .addConfig("conf", configset("cloud-minimal")) - .configure(); - - // disable .scheduled_maintenance (once it exists) - CloudTestUtils.waitForTriggerToBeScheduled(cluster.getOpenOverseer().getSolrCloudManager(), ".scheduled_maintenance"); - CloudTestUtils.suspendTrigger(cluster.getOpenOverseer().getSolrCloudManager(), ".scheduled_maintenance"); - } - - @Test - public void testCooldown() throws Exception { - CloudSolrClient solrClient = cluster.getSolrClient(); - String setTriggerCommand = "{" + - "'set-trigger' : {" + - "'name' : 'node_added_cooldown_trigger'," + - "'event' : 'nodeAdded'," + - "'waitFor' : '" + waitForSeconds + "s'," + - "'enabled' : true," + - "'actions' : [" + - "{'name':'test','class':'" + TestTriggerAction.class.getName() + "'}" + - "]" + - "}}"; - @SuppressWarnings({"rawtypes"}) - SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand); - NamedList response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - - String setListenerCommand1 = "{" + - "'set-listener' : " + - "{" + - "'name' : 'bar'," + - "'trigger' : 'node_added_cooldown_trigger'," + - "'stage' : ['FAILED','SUCCEEDED', 'IGNORED']," + - "'class' : '" + TestTriggerListener.class.getName() + "'" + - "}" + - "}"; - req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setListenerCommand1); - response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - - JettySolrRunner newNode = cluster.startJettySolrRunner(); - cluster.waitForAllNodes(30); - boolean await = triggerFiredLatch.await(20, TimeUnit.SECONDS); - assertTrue("The trigger did not fire at all", await); - assertTrue(triggerFired.get()); - // wait for listener to capture the SUCCEEDED stage - Thread.sleep(1000); - - List capturedEvents = listenerEvents.get("bar"); - // we may get a few IGNORED events if other tests caused events within cooldown period - assertTrue(capturedEvents.toString(), capturedEvents.size() > 0); - long prevTimestamp = capturedEvents.get(capturedEvents.size() - 1).timestamp; - - resetTriggerAndListenerState(); - - JettySolrRunner newNode2 = cluster.startJettySolrRunner(); - await = triggerFiredLatch.await(20, TimeUnit.SECONDS); - assertTrue("The trigger did not fire at all", await); - // wait for listener to capture the SUCCEEDED stage - Thread.sleep(2000); - - // there must be at least one IGNORED event due to cooldown, and one SUCCEEDED event - capturedEvents = listenerEvents.get("bar"); - assertEquals(capturedEvents.toString(), 1, capturedEvents.size()); - CapturedEvent ev = capturedEvents.get(0); - assertEquals(ev.toString(), TriggerEventProcessorStage.SUCCEEDED, ev.stage); - // the difference between timestamps of the first SUCCEEDED and the last SUCCEEDED - // must be larger than cooldown period - assertTrue("timestamp delta is less than default cooldown period", ev.timestamp - prevTimestamp > TimeUnit.SECONDS.toNanos(ScheduledTriggers.DEFAULT_COOLDOWN_PERIOD_SECONDS)); - prevTimestamp = ev.timestamp; - - // this also resets the cooldown period - long modifiedCooldownPeriodSeconds = 7; - String setPropertiesCommand = "{\n" + - "\t\"set-properties\" : {\n" + - "\t\t\"" + AutoScalingParams.TRIGGER_COOLDOWN_PERIOD_SECONDS + "\" : " + modifiedCooldownPeriodSeconds + "\n" + - "\t}\n" + - "}"; - solrClient.request(AutoScalingRequest.create(SolrRequest.METHOD.POST, setPropertiesCommand)); - req = AutoScalingRequest.create(SolrRequest.METHOD.GET, null); - response = solrClient.request(req); - - resetTriggerAndListenerState(); - - JettySolrRunner newNode3 = cluster.startJettySolrRunner(); - await = triggerFiredLatch.await(20, TimeUnit.SECONDS); - assertTrue("The trigger did not fire at all", await); - triggerFiredLatch = new CountDownLatch(1); - triggerFired.compareAndSet(true, false); - // add another node - JettySolrRunner newNode4 = cluster.startJettySolrRunner(); - await = triggerFiredLatch.await(20, TimeUnit.SECONDS); - assertTrue("The trigger did not fire at all", await); - // wait for listener to capture the SUCCEEDED stage - Thread.sleep(2000); - - // there must be two SUCCEEDED (due to newNode3 and newNode4) and maybe some ignored events - capturedEvents = listenerEvents.get("bar"); - assertTrue(capturedEvents.toString(), capturedEvents.size() >= 2); - // first event should be SUCCEEDED - ev = capturedEvents.get(0); - assertEquals(ev.toString(), TriggerEventProcessorStage.SUCCEEDED, ev.stage); - - ev = capturedEvents.get(capturedEvents.size() - 1); - assertEquals(ev.toString(), TriggerEventProcessorStage.SUCCEEDED, ev.stage); - // the difference between timestamps of the first SUCCEEDED and the last SUCCEEDED - // must be larger than the modified cooldown period - assertTrue("timestamp delta is less than default cooldown period", ev.timestamp - prevTimestamp > TimeUnit.SECONDS.toNanos(modifiedCooldownPeriodSeconds)); - } - - public static class TestTriggerAction extends TriggerActionBase { - - public TestTriggerAction() { - // No-Op - } - - @Override - public void process(TriggerEvent event, ActionContext actionContext) { - try { - if (triggerFired.compareAndSet(false, true)) { - long currentTimeNanos = actionContext.getCloudManager().getTimeSource().getTimeNs(); - long eventTimeNanos = event.getEventTime(); - long waitForNanos = TimeUnit.NANOSECONDS.convert(waitForSeconds, TimeUnit.SECONDS) - WAIT_FOR_DELTA_NANOS; - if (currentTimeNanos - eventTimeNanos <= waitForNanos) { - fail(event.source + " was fired before the configured waitFor period"); - } - triggerFiredLatch.countDown(); - } else { - fail(event.source + " was fired more than once!"); - } - } catch (Throwable t) { - log.debug("--throwable", t); - throw t; - } - } - - @Override - public void init() throws Exception { - log.info("TestTriggerAction init"); - super.init(); - } - } - - public static class TestTriggerListener extends TriggerListenerBase { - private TimeSource timeSource; - @Override - public void configure(SolrResourceLoader loader, SolrCloudManager cloudManager, AutoScalingConfig.TriggerListenerConfig config) throws TriggerValidationException { - super.configure(loader, cloudManager, config); - timeSource = cloudManager.getTimeSource(); - } - - @Override - public synchronized void onEvent(TriggerEvent event, TriggerEventProcessorStage stage, String actionName, - ActionContext context, Throwable error, String message) { - List lst = listenerEvents.computeIfAbsent(config.name, s -> new ArrayList<>()); - lst.add(new CapturedEvent(timeSource.getTimeNs(), - context, config, stage, actionName, event, message)); - } - } -} diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerEventQueueTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerEventQueueTest.java deleted file mode 100644 index 9dc8169faf3..00000000000 --- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerEventQueueTest.java +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.cloud.autoscaling; - -import java.util.Collection; -import java.util.EnumMap; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import org.apache.solr.SolrTestCaseJ4; -import org.apache.solr.client.solrj.cloud.DistribStateManager; -import org.apache.solr.client.solrj.cloud.DistributedQueueFactory; -import org.apache.solr.client.solrj.cloud.SolrCloudManager; -import org.apache.solr.client.solrj.cloud.autoscaling.Suggester; -import org.apache.solr.cloud.autoscaling.sim.GenericDistributedQueueFactory; -import org.apache.solr.cloud.autoscaling.sim.SimDistribStateManager; -import org.apache.solr.common.params.CollectionParams; -import org.apache.solr.common.util.Pair; -import org.apache.solr.common.util.TimeSource; -import org.junit.Before; -import org.junit.Test; - -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -/** - * - */ -public class TriggerEventQueueTest extends SolrTestCaseJ4 { - - SolrCloudManager cloudManager; - - @Before - public void init() throws Exception { - assumeWorkingMockito(); - cloudManager = mock(SolrCloudManager.class); - DistribStateManager stateManager = new SimDistribStateManager(); - when(cloudManager.getDistribStateManager()).thenReturn(stateManager); - DistributedQueueFactory queueFactory = new GenericDistributedQueueFactory(stateManager); - when(cloudManager.getDistributedQueueFactory()).thenReturn(queueFactory); - when(cloudManager.getTimeSource()).thenReturn(TimeSource.NANO_TIME); - } - - @Test - @SuppressWarnings({"unchecked"}) - public void testSerialization() throws Exception { - TriggerEventQueue queue = new TriggerEventQueue(cloudManager, "test", null); - Map hotHosts = new HashMap<>(); - hotHosts.put("host1", 1); - hotHosts.put("host2", 1); - TriggerEvent ev = new MetricTrigger.MetricBreachedEvent("testTrigger", "testCollection", "shard1", - CollectionParams.CollectionAction.ADDREPLICA.toLower(), cloudManager.getTimeSource().getTimeNs(), - "foo", hotHosts); - queue.offerEvent(ev); - ev = queue.pollEvent(); - assertNotNull(ev); - Object ops = ev.getProperties().get(TriggerEvent.REQUESTED_OPS); - assertNotNull(ops); - assertTrue(ops.getClass().getName(), ops instanceof List); - List requestedOps = (List)ops; - assertEquals(requestedOps.toString(), 2, requestedOps.size()); - requestedOps.forEach(op -> { - assertTrue(op.getClass().getName(), op instanceof TriggerEvent.Op); - TriggerEvent.Op operation = (TriggerEvent.Op)op; - assertEquals(op.toString(), CollectionParams.CollectionAction.ADDREPLICA, operation.getAction()); - EnumMap hints = ((TriggerEvent.Op) op).getHints(); - assertEquals(hints.toString(), 2, hints.size()); - Object o = hints.get(Suggester.Hint.COLL_SHARD); - assertNotNull(Suggester.Hint.COLL_SHARD.toString(), o); - assertTrue(o.getClass().getName(), o instanceof Collection); - Collection col = (Collection)o; - assertEquals(col.toString(), 1, col.size()); - o = col.iterator().next(); - assertTrue(o.getClass().getName(), o instanceof Pair); - o = hints.get(Suggester.Hint.SRC_NODE); - assertNotNull(Suggester.Hint.SRC_NODE.toString(), o); - assertTrue(o.getClass().getName(), o instanceof Collection); - col = (Collection)o; - assertEquals(col.toString(), 1, col.size()); - o = col.iterator().next(); - assertTrue(o.getClass().getName(), o instanceof String); - }); - } -} diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerIntegrationTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerIntegrationTest.java deleted file mode 100644 index 73cfcfa139d..00000000000 --- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerIntegrationTest.java +++ /dev/null @@ -1,729 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.cloud.autoscaling; - -import java.lang.invoke.MethodHandles; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.locks.ReentrantLock; - -import org.apache.solr.client.solrj.SolrRequest; -import org.apache.solr.client.solrj.cloud.SolrCloudManager; -import org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig; -import org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventProcessorStage; -import org.apache.solr.client.solrj.embedded.JettySolrRunner; -import org.apache.solr.client.solrj.impl.CloudSolrClient; -import org.apache.solr.client.solrj.request.CollectionAdminRequest; -import org.apache.solr.client.solrj.response.CollectionAdminResponse; -import org.apache.solr.cloud.CloudTestUtils; -import org.apache.solr.cloud.CloudTestUtils.AutoScalingRequest; -import org.apache.solr.cloud.Overseer; -import org.apache.solr.cloud.SolrCloudTestCase; -import org.apache.solr.common.cloud.ZkNodeProps; -import org.apache.solr.common.cloud.ZkStateReader; -import org.apache.solr.common.util.NamedList; -import org.apache.solr.common.util.TimeSource; -import org.apache.solr.common.util.Utils; -import org.apache.solr.core.SolrResourceLoader; -import org.apache.solr.util.LogLevel; -import org.apache.zookeeper.data.Stat; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import static org.apache.solr.common.cloud.ZkStateReader.SOLR_AUTOSCALING_CONF_PATH; - -/** - * An end-to-end integration test for triggers - */ -@LogLevel("org.apache.solr.cloud.autoscaling=DEBUG;org.apache.solr.client.solrj.cloud.autoscaling=DEBUG") -public class TriggerIntegrationTest extends SolrCloudTestCase { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - private static final int NODE_COUNT = 2; - - private static volatile CountDownLatch actionConstructorCalled; - private static volatile CountDownLatch actionInitCalled; - private static volatile CountDownLatch triggerFiredLatch; - private static volatile int waitForSeconds = 1; - private static volatile CountDownLatch actionStarted; - private static volatile CountDownLatch actionInterrupted; - private static volatile CountDownLatch actionCompleted; - private static AtomicBoolean triggerFired; - private static Set events = ConcurrentHashMap.newKeySet(); - private static SolrCloudManager cloudManager; - - static final long WAIT_FOR_DELTA_NANOS = TimeUnit.MILLISECONDS.toNanos(5); - - @BeforeClass - public static void setupCluster() throws Exception { - configureCluster(NODE_COUNT) - .addConfig("conf", configset("cloud-minimal")) - .configure(); - - // disable .scheduled_maintenance (once it exists) - CloudTestUtils.waitForTriggerToBeScheduled(cluster.getOpenOverseer().getSolrCloudManager(), ".scheduled_maintenance"); - CloudTestUtils.suspendTrigger(cluster.getOpenOverseer().getSolrCloudManager(), ".scheduled_maintenance"); - } - - @AfterClass - public static void cleanUpAfterClass() throws Exception { - cloudManager = null; - } - - private static CountDownLatch getTriggerFiredLatch() { - return triggerFiredLatch; - } - - private static CountDownLatch getActionStarted() { - return actionStarted; - } - - private static CountDownLatch getActionInterrupted() { - return actionInterrupted; - } - - private static CountDownLatch getActionCompleted() { - return actionCompleted; - } - - @Before - public void setupTest() throws Exception { - SolrCloudTestCase.ensureRunningJettys(NODE_COUNT, 5); - - NamedList overSeerStatus = cluster.getSolrClient().request(CollectionAdminRequest.getOverseerStatus()); - String overseerLeader = (String) overSeerStatus.get("leader"); - int overseerLeaderIndex = 0; - for (int i = 0; i < cluster.getJettySolrRunners().size(); i++) { - JettySolrRunner jetty = cluster.getJettySolrRunner(i); - if (jetty.getNodeName().equals(overseerLeader)) { - overseerLeaderIndex = i; - break; - } - } - Overseer overseer = cluster.getJettySolrRunner(overseerLeaderIndex).getCoreContainer().getZkController().getOverseer(); - ScheduledTriggers scheduledTriggers = ((OverseerTriggerThread)overseer.getTriggerThread().getThread()).getScheduledTriggers(); - // aggressively remove all active scheduled triggers - scheduledTriggers.removeAll(); - - // clear any persisted auto scaling configuration - Stat stat = zkClient().setData(SOLR_AUTOSCALING_CONF_PATH, Utils.toJSON(new ZkNodeProps()), true); - if (log.isInfoEnabled()) { - log.info("{} reset, new znode version {}", SOLR_AUTOSCALING_CONF_PATH, stat.getVersion()); - } - - cluster.deleteAllCollections(); - cluster.getSolrClient().setDefaultCollection(null); - - // restart Overseer. Even though we reset the autoscaling config some already running - // trigger threads may still continue to execute and produce spurious events - JettySolrRunner j = cluster.stopJettySolrRunner(overseerLeaderIndex); - cluster.waitForJettyToStop(j); - Thread.sleep(5000); - - throttlingDelayMs.set(TimeUnit.SECONDS.toMillis(ScheduledTriggers.DEFAULT_ACTION_THROTTLE_PERIOD_SECONDS)); - waitForSeconds = 1 + random().nextInt(3); - actionConstructorCalled = new CountDownLatch(1); - actionInitCalled = new CountDownLatch(1); - triggerFiredLatch = new CountDownLatch(1); - triggerFired = new AtomicBoolean(false); - actionStarted = new CountDownLatch(1); - actionInterrupted = new CountDownLatch(1); - actionCompleted = new CountDownLatch(1); - events.clear(); - listenerEvents.clear(); - lastActionExecutedAt.set(0); - while (cluster.getJettySolrRunners().size() < 2) { - // perhaps a test stopped a node but didn't start it back - // lets start a node - cluster.startJettySolrRunner(); - } - cluster.waitForAllNodes(30); - cloudManager = cluster.getJettySolrRunner(0).getCoreContainer().getZkController().getSolrCloudManager(); - // clear any events or markers - // todo: consider the impact of such cleanup on regular cluster restarts - deleteChildrenRecursively(ZkStateReader.SOLR_AUTOSCALING_EVENTS_PATH); - deleteChildrenRecursively(ZkStateReader.SOLR_AUTOSCALING_TRIGGER_STATE_PATH); - deleteChildrenRecursively(ZkStateReader.SOLR_AUTOSCALING_NODE_LOST_PATH); - deleteChildrenRecursively(ZkStateReader.SOLR_AUTOSCALING_NODE_ADDED_PATH); - } - - private void deleteChildrenRecursively(String path) throws Exception { - cloudManager.getDistribStateManager().removeRecursively(path, true, false); - } - - @Test - // commented out on: 17-Feb-2019 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // annotated on: 24-Dec-2018 - public void testTriggerThrottling() throws Exception { - // for this test we want to create two triggers so we must assert that the actions were created twice - actionInitCalled = new CountDownLatch(2); - // similarly we want both triggers to fire - triggerFiredLatch = new CountDownLatch(2); - - CloudSolrClient solrClient = cluster.getSolrClient(); - - // first trigger - String setTriggerCommand = "{" + - "'set-trigger' : {" + - "'name' : 'node_added_trigger1'," + - "'event' : 'nodeAdded'," + - "'waitFor' : '0s'," + - "'enabled' : true," + - "'actions' : [{'name':'test','class':'" + ThrottlingTesterAction.class.getName() + "'}]" + - "}}"; - @SuppressWarnings({"rawtypes"}) - SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand); - NamedList response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - - // second trigger - setTriggerCommand = "{" + - "'set-trigger' : {" + - "'name' : 'node_added_trigger2'," + - "'event' : 'nodeAdded'," + - "'waitFor' : '0s'," + - "'enabled' : true," + - "'actions' : [{'name':'test','class':'" + ThrottlingTesterAction.class.getName() + "'}]" + - "}}"; - req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand); - response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - - // wait until the two instances of action are created - assertTrue("Two TriggerAction instances were not created "+ - "even after await()ing an excessive amount of time", - actionInitCalled.await(60, TimeUnit.SECONDS)); - - JettySolrRunner newNode = cluster.startJettySolrRunner(); - cluster.waitForAllNodes(30); - assertTrue("Both triggers did not fire event after await()ing an excessive amount of time", - triggerFiredLatch.await(60, TimeUnit.SECONDS)); - - // reset shared state - lastActionExecutedAt.set(0); - TriggerIntegrationTest.actionInitCalled = new CountDownLatch(2); - triggerFiredLatch = new CountDownLatch(2); - - setTriggerCommand = "{" + - "'set-trigger' : {" + - "'name' : 'node_lost_trigger1'," + - "'event' : 'nodeLost'," + - "'waitFor' : '0s'," + - "'enabled' : true," + - "'actions' : [{'name':'test','class':'" + ThrottlingTesterAction.class.getName() + "'}]" + - "}}"; - req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand); - response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - - setTriggerCommand = "{" + - "'set-trigger' : {" + - "'name' : 'node_lost_trigger2'," + - "'event' : 'nodeLost'," + - "'waitFor' : '0s'," + - "'enabled' : true," + - "'actions' : [{'name':'test','class':'" + ThrottlingTesterAction.class.getName() + "'}]" + - "}}"; - req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand); - response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - - // wait until the two instances of action are created - assertTrue("Two TriggerAction instances were not created "+ - "even after await()ing an excessive amount of time", - actionInitCalled.await(60, TimeUnit.SECONDS)); - - // stop the node we had started earlier - List jettySolrRunners = cluster.getJettySolrRunners(); - for (int i = 0; i < jettySolrRunners.size(); i++) { - JettySolrRunner jettySolrRunner = jettySolrRunners.get(i); - if (jettySolrRunner == newNode) { - JettySolrRunner j = cluster.stopJettySolrRunner(i); - cluster.waitForJettyToStop(j); - break; - } - } - - assertTrue("Both triggers did not fire event after await()ing an excessive amount of time", - triggerFiredLatch.await(60, TimeUnit.SECONDS)); - } - - static AtomicLong lastActionExecutedAt = new AtomicLong(0); - static AtomicLong throttlingDelayMs = new AtomicLong(TimeUnit.SECONDS.toMillis(ScheduledTriggers.DEFAULT_ACTION_THROTTLE_PERIOD_SECONDS)); - static ReentrantLock lock = new ReentrantLock(); - public static class ThrottlingTesterAction extends TestTriggerAction { - // nanos are very precise so we need a delta for comparison with ms - private static final long DELTA_MS = 2; - - // sanity check that an action instance is only invoked once - private final AtomicBoolean onlyOnce = new AtomicBoolean(false); - - @Override - public void process(TriggerEvent event, ActionContext actionContext) { - boolean locked = lock.tryLock(); - if (!locked) { - log.info("We should never have a tryLock fail because actions are never supposed to be executed concurrently"); - return; - } - try { - long currentTime = actionContext.getCloudManager().getTimeSource().getTimeNs(); - if (lastActionExecutedAt.get() != 0) { - long minDiff = TimeUnit.MILLISECONDS.toNanos(throttlingDelayMs.get() - DELTA_MS); - if (log.isInfoEnabled()) { - log.info("last action at {} current time = {}\nreal diff: {}\n min diff: {}" - , lastActionExecutedAt.get(), currentTime - , (currentTime - lastActionExecutedAt.get()) - , minDiff); - } - if (currentTime - lastActionExecutedAt.get() < minDiff) { - if (log.isInfoEnabled()) { - log.info("action executed again before minimum wait time from {}", event.getSource()); - } - fail("TriggerListener was fired before the throttling period"); - } - } - if (onlyOnce.compareAndSet(false, true)) { - if (log.isInfoEnabled()) { - log.info("action executed from {}", event.getSource()); - } - lastActionExecutedAt.set(currentTime); - getTriggerFiredLatch().countDown(); - } else { - if (log.isInfoEnabled()) { - log.info("action executed more than once from {}", event.getSource()); - } - fail("Trigger should not have fired more than once!"); - } - } finally { - lock.unlock(); - } - } - } - - @Test - public void testContinueTriggersOnOverseerRestart() throws Exception { - CollectionAdminRequest.OverseerStatus status = new CollectionAdminRequest.OverseerStatus(); - CloudSolrClient solrClient = cluster.getSolrClient(); - CollectionAdminResponse adminResponse = status.process(solrClient); - NamedList response = adminResponse.getResponse(); - String leader = (String) response.get("leader"); - JettySolrRunner overseerNode = null; - int index = -1; - List jettySolrRunners = cluster.getJettySolrRunners(); - for (int i = 0; i < jettySolrRunners.size(); i++) { - JettySolrRunner runner = jettySolrRunners.get(i); - if (runner.getNodeName().equals(leader)) { - overseerNode = runner; - index = i; - break; - } - } - assertNotNull(overseerNode); - - String setTriggerCommand = "{" + - "'set-trigger' : {" + - "'name' : 'node_added_triggerCTOOR'," + - "'event' : 'nodeAdded'," + - "'waitFor' : '" + waitForSeconds + "s'," + - "'enabled' : true," + - "'actions' : [{'name':'test','class':'" + TestTriggerAction.class.getName() + "'}]" + - "}}"; - @SuppressWarnings({"rawtypes"}) - SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand); - response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - assertTrue("Trigger was not init()ed even after await()ing an excessive amount of time", - actionInitCalled.await(60, TimeUnit.SECONDS)); - - // stop the overseer, somebody else will take over as the overseer - JettySolrRunner j = cluster.stopJettySolrRunner(index); - cluster.waitForJettyToStop(j); - Thread.sleep(10000); - JettySolrRunner newNode = cluster.startJettySolrRunner(); - cluster.waitForAllNodes(30); - assertTrue("trigger did not fire even after await()ing an excessive amount of time", - triggerFiredLatch.await(60, TimeUnit.SECONDS)); - assertTrue(triggerFired.get()); - NodeAddedTrigger.NodeAddedEvent nodeAddedEvent = (NodeAddedTrigger.NodeAddedEvent) events.iterator().next(); - assertNotNull(nodeAddedEvent); - @SuppressWarnings({"unchecked"}) - List nodeNames = (List)nodeAddedEvent.getProperty(TriggerEvent.NODE_NAMES); - assertTrue(nodeNames.contains(newNode.getNodeName())); - } - - public static class TestTriggerAction extends TriggerActionBase { - - public TestTriggerAction() { - actionConstructorCalled.countDown(); - } - - @Override - public void process(TriggerEvent event, ActionContext actionContext) { - try { - if (triggerFired.compareAndSet(false, true)) { - events.add(event); - long currentTimeNanos = actionContext.getCloudManager().getTimeSource().getTimeNs(); - long eventTimeNanos = event.getEventTime(); - long waitForNanos = TimeUnit.NANOSECONDS.convert(waitForSeconds, TimeUnit.SECONDS) - WAIT_FOR_DELTA_NANOS; - if (currentTimeNanos - eventTimeNanos <= waitForNanos) { - fail(event.source + " was fired before the configured waitFor period"); - } - getTriggerFiredLatch().countDown(); - } else { - fail(event.source + " was fired more than once!"); - } - } catch (Throwable t) { - log.debug("--throwable", t); - throw t; - } - } - - @Override - public void init() throws Exception { - log.info("TestTriggerAction init"); - actionInitCalled.countDown(); - super.init(); - } - } - - public static class TestEventQueueAction extends TriggerActionBase { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - public static volatile CountDownLatch stall = new CountDownLatch(0); - public TestEventQueueAction() { - log.info("TestEventQueueAction instantiated"); - } - - @Override - public void process(TriggerEvent event, ActionContext actionContext) { - // make a local copy of the latch so we're using it consistently even as test thread changes tings - final CountDownLatch stallLatch = stall; - log.info("processing: stall={} event={} ", stallLatch, event); - events.add(event); - getActionStarted().countDown(); - try { - if (stallLatch.await(60, TimeUnit.SECONDS)) { - log.info("Firing trigger event after await()ing 'stall' countdown"); - triggerFired.set(true); - } else { - log.error("Timed out await()ing 'stall' countdown"); - } - getActionCompleted().countDown(); - } catch (InterruptedException e) { - log.info("Interrupted"); - getActionInterrupted().countDown(); - } - } - - @Override - public void init() throws Exception { - log.info("TestEventQueueAction init"); - actionInitCalled.countDown(); - super.init(); - } - } - - @Test - public void testEventQueue() throws Exception { - waitForSeconds = 1; - CloudSolrClient solrClient = cluster.getSolrClient(); - String setTriggerCommand = "{" + - "'set-trigger' : {" + - "'name' : 'node_added_triggerEQ'," + - "'event' : 'nodeAdded'," + - "'waitFor' : '" + waitForSeconds + "s'," + - "'enabled' : true," + - "'actions' : [{'name':'test','class':'" + TestEventQueueAction.class.getName() + "'}]" + - "}}"; - NamedList overSeerStatus = cluster.getSolrClient().request(CollectionAdminRequest.getOverseerStatus()); - String overseerLeader = (String) overSeerStatus.get("leader"); - int overseerLeaderIndex = 0; - for (int i = 0; i < cluster.getJettySolrRunners().size(); i++) { - JettySolrRunner jetty = cluster.getJettySolrRunner(i); - if (jetty.getNodeName().equals(overseerLeader)) { - overseerLeaderIndex = i; - break; - } - } - @SuppressWarnings({"rawtypes"}) - SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand); - NamedList response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - - assertTrue("Trigger was not init()ed even after await()ing an excessive amount of time", - actionInitCalled.await(60, TimeUnit.SECONDS)); - - // setup the trigger action to stall so we can test interupting it w/overseer change - // NOTE: we will never release this latch, instead we expect the interupt on overseer shutdown - TestEventQueueAction.stall = new CountDownLatch(1); - - // add node to generate the event - JettySolrRunner newNode = cluster.startJettySolrRunner(); - cluster.waitForAllNodes(30); - assertTrue("Action did not start even after await()ing an excessive amount of time", - actionStarted.await(60, TimeUnit.SECONDS)); - - // event should be there - final TriggerEvent nodeAddedEvent = events.iterator().next(); - assertNotNull(nodeAddedEvent); - assertNotNull(nodeAddedEvent.getId()); - assertNotNull(nodeAddedEvent.getEventType()); - assertNotNull(nodeAddedEvent.getProperty(TriggerEventQueue.ENQUEUE_TIME)); - - // but action did not complete yet so the event is still enqueued - assertFalse(triggerFired.get()); - - // we know the event action has started, so we can re-set state for the next instance - // that will run after the overseer change - events.clear(); - actionStarted = new CountDownLatch(1); - TestEventQueueAction.stall = new CountDownLatch(0); // so replay won't wait - - // kill overseer leader - JettySolrRunner j = cluster.stopJettySolrRunner(overseerLeaderIndex); - cluster.waitForJettyToStop(j); - Thread.sleep(5000); - // new overseer leader should be elected and run triggers - assertTrue("Action was not interupted even after await()ing an excessive amount of time", - actionInterrupted.await(60, TimeUnit.SECONDS)); - // it should fire again from enqueued event - assertTrue("Action did not (re-)start even after await()ing an excessive amount of time", - actionStarted.await(60, TimeUnit.SECONDS)); - - final TriggerEvent replayedEvent = events.iterator().next(); - assertNotNull(replayedEvent); - - assertTrue("Action did not complete even after await()ing an excessive amount of time", - actionCompleted.await(60, TimeUnit.SECONDS)); - assertTrue(triggerFired.get()); - - assertEquals(nodeAddedEvent.getId(), replayedEvent.getId()); - assertEquals(nodeAddedEvent.getEventTime(), replayedEvent.getEventTime()); - assertEquals(nodeAddedEvent.getEventType(), replayedEvent.getEventType()); - assertEquals(nodeAddedEvent.getProperty(TriggerEventQueue.ENQUEUE_TIME), - replayedEvent.getProperty(TriggerEventQueue.ENQUEUE_TIME)); - assertEquals(Boolean.TRUE, replayedEvent.getProperty(TriggerEvent.REPLAYING)); - } - - static Map> listenerEvents = new HashMap<>(); - static List allListenerEvents = new ArrayList<>(); - static CountDownLatch listenerCreated = new CountDownLatch(1); - static boolean failDummyAction = false; - - public static class TestTriggerListener extends TriggerListenerBase { - private TimeSource timeSource; - @Override - public void configure(SolrResourceLoader loader, SolrCloudManager cloudManager, AutoScalingConfig.TriggerListenerConfig config) throws TriggerValidationException { - super.configure(loader, cloudManager, config); - listenerCreated.countDown(); - timeSource = cloudManager.getTimeSource(); - } - - @Override - public synchronized void onEvent(TriggerEvent event, TriggerEventProcessorStage stage, String actionName, - ActionContext context, Throwable error, String message) { - List lst = listenerEvents.computeIfAbsent(config.name, s -> new ArrayList<>()); - CapturedEvent ev = new CapturedEvent(timeSource.getTimeNs(), context, config, stage, actionName, event, message); - - lst.add(ev); - allListenerEvents.add(ev); - } - } - - public static class TestDummyAction extends TriggerActionBase { - - @Override - public void process(TriggerEvent event, ActionContext context) { - if (failDummyAction) { - throw new RuntimeException("failure"); - } - - } - } - - @Test - public void testListeners() throws Exception { - CloudSolrClient solrClient = cluster.getSolrClient(); - String setTriggerCommand = "{" + - "'set-trigger' : {" + - "'name' : 'node_added_triggerL'," + - "'event' : 'nodeAdded'," + - "'waitFor' : '" + waitForSeconds + "s'," + - "'enabled' : true," + - "'actions' : [" + - "{'name':'test','class':'" + TestTriggerAction.class.getName() + "'}," + - "{'name':'test1','class':'" + TestDummyAction.class.getName() + "'}," + - "]" + - "}}"; - @SuppressWarnings({"rawtypes"}) - SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand); - NamedList response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - - assertTrue("Trigger was not init()ed even after await()ing an excessive amount of time", - actionInitCalled.await(60, TimeUnit.SECONDS)); - - String setListenerCommand = "{" + - "'set-listener' : " + - "{" + - "'name' : 'foo'," + - "'trigger' : 'node_added_triggerL'," + - "'stage' : ['STARTED','ABORTED','SUCCEEDED', 'FAILED']," + - "'beforeAction' : 'test'," + - "'afterAction' : ['test', 'test1']," + - "'class' : '" + TestTriggerListener.class.getName() + "'" + - "}" + - "}"; - req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setListenerCommand); - response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - - String setListenerCommand1 = "{" + - "'set-listener' : " + - "{" + - "'name' : 'bar'," + - "'trigger' : 'node_added_triggerL'," + - "'stage' : ['FAILED','SUCCEEDED']," + - "'beforeAction' : ['test', 'test1']," + - "'afterAction' : 'test'," + - "'class' : '" + TestTriggerListener.class.getName() + "'" + - "}" + - "}"; - req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setListenerCommand1); - response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - - listenerEvents.clear(); - failDummyAction = false; - - JettySolrRunner newNode = cluster.startJettySolrRunner(); - assertTrue("trigger did not fire even after await()ing an excessive amount of time", - triggerFiredLatch.await(60, TimeUnit.SECONDS)); - assertTrue(triggerFired.get()); - - assertEquals("both listeners should have fired", 2, listenerEvents.size()); - - Thread.sleep(2000); - - // check foo events - List capturedEvents = listenerEvents.get("foo"); - assertNotNull("foo events: " + capturedEvents, capturedEvents); - assertEquals("foo events: " + capturedEvents, 5, capturedEvents.size()); - - assertEquals(TriggerEventProcessorStage.STARTED, capturedEvents.get(0).stage); - - assertEquals(TriggerEventProcessorStage.BEFORE_ACTION, capturedEvents.get(1).stage); - assertEquals("test", capturedEvents.get(1).actionName); - - assertEquals(TriggerEventProcessorStage.AFTER_ACTION, capturedEvents.get(2).stage); - assertEquals("test", capturedEvents.get(2).actionName); - - assertEquals(TriggerEventProcessorStage.AFTER_ACTION, capturedEvents.get(3).stage); - assertEquals("test1", capturedEvents.get(3).actionName); - - assertEquals(TriggerEventProcessorStage.SUCCEEDED, capturedEvents.get(4).stage); - - // check bar events - capturedEvents = listenerEvents.get("bar"); - assertNotNull("bar events", capturedEvents); - assertEquals("bar events", 4, capturedEvents.size()); - - assertEquals(TriggerEventProcessorStage.BEFORE_ACTION, capturedEvents.get(0).stage); - assertEquals("test", capturedEvents.get(0).actionName); - - assertEquals(TriggerEventProcessorStage.AFTER_ACTION, capturedEvents.get(1).stage); - assertEquals("test", capturedEvents.get(1).actionName); - - assertEquals(TriggerEventProcessorStage.BEFORE_ACTION, capturedEvents.get(2).stage); - assertEquals("test1", capturedEvents.get(2).actionName); - - assertEquals(TriggerEventProcessorStage.SUCCEEDED, capturedEvents.get(3).stage); - - // check global ordering of events (SOLR-12668) - int fooIdx = -1; - int barIdx = -1; - for (int i = 0; i < allListenerEvents.size(); i++) { - CapturedEvent ev = allListenerEvents.get(i); - if (ev.stage == TriggerEventProcessorStage.BEFORE_ACTION && ev.actionName.equals("test")) { - if (ev.config.name.equals("foo")) { - fooIdx = i; - } else if (ev.config.name.equals("bar")) { - barIdx = i; - } - } - } - assertTrue("fooIdx not found", fooIdx != -1); - assertTrue("barIdx not found", barIdx != -1); - assertTrue("foo fired later than bar: fooIdx=" + fooIdx + ", barIdx=" + barIdx, fooIdx < barIdx); - - // reset - triggerFired.set(false); - triggerFiredLatch = new CountDownLatch(1); - listenerEvents.clear(); - allListenerEvents.clear(); - failDummyAction = true; - - newNode = cluster.startJettySolrRunner(); - assertTrue("trigger did not fire event after await()ing an excessive amount of time", - triggerFiredLatch.await(60, TimeUnit.SECONDS)); - - Thread.sleep(2000); - - // check foo events - capturedEvents = listenerEvents.get("foo"); - assertNotNull("foo events: " + capturedEvents, capturedEvents); - assertEquals("foo events: " + capturedEvents, 4, capturedEvents.size()); - - assertEquals(TriggerEventProcessorStage.STARTED, capturedEvents.get(0).stage); - - assertEquals(TriggerEventProcessorStage.BEFORE_ACTION, capturedEvents.get(1).stage); - assertEquals("test", capturedEvents.get(1).actionName); - - assertEquals(TriggerEventProcessorStage.AFTER_ACTION, capturedEvents.get(2).stage); - assertEquals("test", capturedEvents.get(2).actionName); - - assertEquals(TriggerEventProcessorStage.FAILED, capturedEvents.get(3).stage); - assertEquals("test1", capturedEvents.get(3).actionName); - - // check bar events - capturedEvents = listenerEvents.get("bar"); - assertNotNull("bar events", capturedEvents); - assertEquals("bar events", 4, capturedEvents.size()); - - assertEquals(TriggerEventProcessorStage.BEFORE_ACTION, capturedEvents.get(0).stage); - assertEquals("test", capturedEvents.get(0).actionName); - - assertEquals(TriggerEventProcessorStage.AFTER_ACTION, capturedEvents.get(1).stage); - assertEquals("test", capturedEvents.get(1).actionName); - - assertEquals(TriggerEventProcessorStage.BEFORE_ACTION, capturedEvents.get(2).stage); - assertEquals("test1", capturedEvents.get(2).actionName); - - assertEquals(TriggerEventProcessorStage.FAILED, capturedEvents.get(3).stage); - assertEquals("test1", capturedEvents.get(3).actionName); - } -} diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerSetPropertiesIntegrationTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerSetPropertiesIntegrationTest.java deleted file mode 100644 index dbdff5107da..00000000000 --- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerSetPropertiesIntegrationTest.java +++ /dev/null @@ -1,275 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.cloud.autoscaling; - -import java.lang.invoke.MethodHandles; -import java.util.ArrayList; -import java.util.Collections; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ArrayBlockingQueue; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.CyclicBarrier; -import java.util.concurrent.Semaphore; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.atomic.AtomicReference; - -import org.apache.solr.client.solrj.cloud.SolrCloudManager; -import org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig; -import org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventType; -import org.apache.solr.client.solrj.embedded.JettySolrRunner; -import org.apache.solr.cloud.CloudTestUtils; -import org.apache.solr.cloud.SolrCloudTestCase; -import org.apache.solr.common.params.AutoScalingParams; -import org.apache.solr.core.SolrResourceLoader; -import org.apache.solr.util.LogLevel; -import org.junit.BeforeClass; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -@LogLevel("org.apache.solr.cloud.autoscaling=DEBUG;org.apache.solr.client.solrj.cloud.autoscaling=DEBUG") -public class TriggerSetPropertiesIntegrationTest extends SolrCloudTestCase { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - @BeforeClass - public static void setupCluster() throws Exception { - configureCluster(2) - .addConfig("conf", configset("cloud-minimal")) - .configure(); - - // disable .scheduled_maintenance (once it exists) - CloudTestUtils.waitForTriggerToBeScheduled(cluster.getOpenOverseer().getSolrCloudManager(), ".scheduled_maintenance"); - CloudTestUtils.suspendTrigger(cluster.getOpenOverseer().getSolrCloudManager(), ".scheduled_maintenance"); - } - - /** - * Test that we can add/remove triggers to a scheduler, and change the config on the fly, and still get - * expected behavior - */ - // commented out on: 17-Feb-2019 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // annotated on: 24-Dec-2018 - public void testSetProperties() throws Exception { - final JettySolrRunner runner = cluster.getJettySolrRunner(0); - final SolrResourceLoader resourceLoader = runner.getCoreContainer().getResourceLoader(); - final SolrCloudManager solrCloudManager = runner.getCoreContainer().getZkController().getSolrCloudManager(); - - try (ScheduledTriggers scheduledTriggers = new ScheduledTriggers(resourceLoader, solrCloudManager)) { - AutoScalingConfig config = new AutoScalingConfig(Collections.emptyMap()); - scheduledTriggers.setAutoScalingConfig(config); - - // Setup a trigger that records the timestamp of each time it was run - // we only need 2 timestamps for the test, so limit the queue and make the trigger a No-Op if full - final BlockingQueue timestamps = new ArrayBlockingQueue(2); - final AutoScaling.Trigger t1 = new MockTrigger(TriggerEventType.NODELOST, "mock-timestamper") { - @Override - public void run() { - if (log.isInfoEnabled()) { - log.info("Running {} in {}", this.getName(), Thread.currentThread().getName()); - } - timestamps.offer(solrCloudManager.getTimeSource().getTimeNs()); - } - }; - - if (log.isInfoEnabled()) { - log.info("Configuring simple scheduler and adding trigger: {}", t1.getName()); - } - t1.configure(resourceLoader, solrCloudManager, Collections.emptyMap()); - scheduledTriggers.add(t1); - - waitForAndDiffTimestamps("conf(default delay)", - ScheduledTriggers.DEFAULT_SCHEDULED_TRIGGER_DELAY_SECONDS, TimeUnit.SECONDS, - timestamps); - - if (log.isInfoEnabled()) { - log.info("Reconfiguing scheduler to use 4s delay and clearing queue for trigger: {}", t1.getName()); - } - config = config.withProperties(Collections.singletonMap - (AutoScalingParams.TRIGGER_SCHEDULE_DELAY_SECONDS, 4)); - scheduledTriggers.setAutoScalingConfig(config); - timestamps.clear(); - - waitForAndDiffTimestamps("conf(four sec delay)", - 4, TimeUnit.SECONDS, - timestamps); - - if (log.isInfoEnabled()) { - log.info("Removing trigger: {}", t1.getName()); - } - scheduledTriggers.remove(t1.getName()); - - log.info("Reconfiguing scheduler to use default props"); - config = config.withProperties(ScheduledTriggers.DEFAULT_PROPERTIES); - scheduledTriggers.setAutoScalingConfig(config); - - - assertTrue("Test sanity check, need default thread pool to be at least 3 so we can" + - "test lowering it by 2", ScheduledTriggers.DEFAULT_TRIGGER_CORE_POOL_SIZE >= 3); - final int numTriggers = ScheduledTriggers.DEFAULT_TRIGGER_CORE_POOL_SIZE; - final int reducedThreadPoolSize = numTriggers - 2; - - // Setup X instances of a trigger that: - // - records it's name as being run - // - skipping all remaining execution if it's name has already been recorded - // - records the name of the thread that ran it - // - blocks on a cyclic barrier untill at Y instances have run (to hog a thread) - // ...to test that the scheduler will add new threads as needed, up to the configured limit - // - // NOTE: the reason we need X unique instances is because the scheduler won't "re-run" a single - // trigger while a previouss "run" is still in process - final List triggerList = new ArrayList<>(numTriggers); - - // Use a cyclic barrier gated by an atomic ref so we can swap it out later - final AtomicReference latch = new AtomicReference<>(new CyclicBarrier(numTriggers)); - - // variables for tracking state as we go - // NOTE: all read/write must be gated by synchronizing on the barrier (ref), - // so we we can ensure we are reading a consistent view - final Set threadNames = Collections.synchronizedSet(new LinkedHashSet<>()); - final Set triggerNames = Collections.synchronizedSet(new LinkedHashSet<>()); - final AtomicLong fails = new AtomicLong(0); - - // Use a semaphore to track when each trigger *finishes* so our test thread - // can know when to check & clear the tracking state - final Semaphore completionSemaphore = new Semaphore(numTriggers); - - for (int i = 0; i < numTriggers; i++) { - AutoScaling.Trigger trigger = new MockTrigger(TriggerEventType.NODELOST, - "mock-blocking-trigger-" + i) { - @Override - public void run() { - if (log.isInfoEnabled()) { - log.info("Running {} in {}", this.getName(), Thread.currentThread().getName()); - } - CyclicBarrier barrier = null; - synchronized (latch) { - if (triggerNames.add(this.getName())) { - if (log.isInfoEnabled()) { - log.info("{}: No-Op since we've already recorded a run", this.getName()); - } - return; - } - threadNames.add(Thread.currentThread().getName()); - barrier = latch.get(); - } - - try { - if (log.isInfoEnabled()) { - log.info("{}: waiting on barrier to hog a thread", this.getName()); - } - barrier.await(30, TimeUnit.SECONDS); - completionSemaphore.release(); - } catch (Exception e) { - fails.incrementAndGet(); - log.error("{} : failure waiting on cyclic barrier: {}", this.getName(), e, e); - } - } - }; - - trigger.configure(resourceLoader, solrCloudManager, Collections.emptyMap()); - triggerList.add(trigger); - completionSemaphore.acquire(); - if (log.isInfoEnabled()) { - log.info("Adding trigger {} to scheduler", trigger.getName()); - } - scheduledTriggers.add(trigger); - } - - log.info("Waiting on semaphore for all triggers to signal completion..."); - assertTrue("Timed out waiting for semaphore count to be released", - completionSemaphore.tryAcquire(numTriggers, 60, TimeUnit.SECONDS)); - - synchronized (latch) { - assertEquals("Unexpected number of trigger names found: " + triggerNames.toString(), - numTriggers, triggerNames.size()); - assertEquals("Unexpected number of thread ames found: " + threadNames.toString(), - numTriggers, threadNames.size()); - assertEquals("Unexpected number of trigger fails recorded, check logs?", - 0, fails.get()); - - // before releasing the latch, clear the state and update our config to use a lower number of threads - log.info("Updating scheduler config to use {} threads", reducedThreadPoolSize); - config = config.withProperties(Collections.singletonMap(AutoScalingParams.TRIGGER_CORE_POOL_SIZE, - reducedThreadPoolSize)); - scheduledTriggers.setAutoScalingConfig(config); - - log.info("Updating cyclic barrier and clearing test state so triggers will 'run' again"); - latch.set(new CyclicBarrier(reducedThreadPoolSize)); - threadNames.clear(); - triggerNames.clear(); - } - - log.info("Waiting on semaphore for all triggers to signal completion..."); - assertTrue("Timed out waiting for semaphore count to be released", - completionSemaphore.tryAcquire(numTriggers, 60, TimeUnit.SECONDS)); - - synchronized (latch) { - assertEquals("Unexpected number of trigger names found: " + triggerNames.toString(), - numTriggers, triggerNames.size()); - assertEquals("Unexpected number of thread names found: " + threadNames.toString(), - reducedThreadPoolSize, threadNames.size()); - assertEquals("Unexpected number of trigger fails recorded, check logs?", - 0, fails.get()); - } - } - } - - - - - private static final void waitForAndDiffTimestamps(final String label, - final long minExpectedDelta, - final TimeUnit minExpectedDeltaUnit, - final BlockingQueue timestamps) { - try { - log.info("{}: Waiting for 2 timestamps to be recorded", label); - Long firstTs = timestamps.poll(minExpectedDelta * 3, minExpectedDeltaUnit); - assertNotNull(label + ": Couldn't get first timestampe after max allowed polling", firstTs); - Long secondTs = timestamps.poll(minExpectedDelta * 3, minExpectedDeltaUnit); - assertNotNull(label + ": Couldn't get second timestampe after max allowed polling", secondTs); - - final long deltaInNanos = secondTs - firstTs; - final long minExpectedDeltaInNanos = minExpectedDeltaUnit.toNanos(minExpectedDelta); - assertTrue(label + ": Delta between timestamps ("+secondTs+"ns - "+firstTs+"ns = "+deltaInNanos+"ns) is not " + - "at least as much as min expected delay: " + minExpectedDeltaInNanos + "ns", - deltaInNanos >= minExpectedDeltaInNanos); - } catch (InterruptedException e) { - log.error("{}: interupted", label, e); - fail(label + ": interupted:" + e.toString()); - } - } - - private static abstract class MockTrigger extends TriggerBase { - - public MockTrigger(TriggerEventType eventType, String name) { - super(eventType, name); - } - - @Override - protected Map getState() { - return Collections.emptyMap(); - } - - @Override - protected void setState(Map state) { /* No-Op */ } - - @Override - public void restoreState(AutoScaling.Trigger old) { /* No-Op */ } - } -} diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimSolrCloudTestCase.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimSolrCloudTestCase.java deleted file mode 100644 index 69180188ad9..00000000000 --- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimSolrCloudTestCase.java +++ /dev/null @@ -1,255 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.cloud.autoscaling.sim; - -import java.io.IOException; -import java.lang.invoke.MethodHandles; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.TimeUnit; -import java.util.function.Predicate; -import java.util.regex.Pattern; -import java.util.stream.Collectors; - -import org.apache.solr.SolrTestCaseJ4; -import org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig; -import org.apache.solr.cloud.CloudTestUtils; -import org.apache.solr.common.cloud.ClusterState; -import org.apache.solr.common.cloud.DocCollection; -import org.apache.solr.common.cloud.Replica; -import org.apache.solr.common.cloud.Slice; -import org.apache.solr.common.util.TimeSource; -import org.apache.solr.util.TimeOut; -import org.junit.After; -import org.junit.AfterClass; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Base class for simulated test cases. Tests that use this class should configure the simulated cluster - * in @BeforeClass like this: - * 
- *   @BeforeClass
- *   public static void setupCluster() throws Exception {
- *     cluster = configureCluster(5, TimeSource.get("simTime:50"));
- *   }
- *
- */ -public class SimSolrCloudTestCase extends SolrTestCaseJ4 { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - /** The cluster. */ - protected static SimCloudManager cluster; - protected static int clusterNodeCount = 0; - - protected static void configureCluster(int nodeCount, TimeSource timeSource) throws Exception { - cluster = SimCloudManager.createCluster(nodeCount, timeSource); - clusterNodeCount = nodeCount; - } - - @After - private void checkBackgroundTaskFailureCount() { - if (cluster != null) { - assertBackgroundTaskFailureCount(cluster); - } - } - - protected static void assertBackgroundTaskFailureCount(SimCloudManager c) { - assert null != c; - assertEquals("Cluster had background tasks submitted which failed", - 0, c.getBackgroundTaskFailureCount()); - } - - @AfterClass - public static void shutdownCluster() throws Exception { - if (cluster != null) { - try { - cluster.close(); - assertBackgroundTaskFailureCount(cluster); - } finally { - cluster = null; - } - } - } - - protected static void assertAutoscalingUpdateComplete() throws Exception { - (new TimeOut(30, TimeUnit.SECONDS, cluster.getTimeSource())) - .waitFor("OverseerTriggerThread never caught up to the latest znodeVersion", () -> { - try { - AutoScalingConfig autoscalingConfig = cluster.getDistribStateManager().getAutoScalingConfig(); - return autoscalingConfig.getZkVersion() == cluster.getOverseerTriggerThread().getProcessedZnodeVersion(); - } catch (Exception e) { - throw new RuntimeException("FAILED", e); - } - }); - } - - @Override - public void tearDown() throws Exception { - super.tearDown(); - if (cluster != null) { - if (log.isInfoEnabled()) { - log.info(cluster.dumpClusterState(false)); - } - } - } - - @Override - public void setUp() throws Exception { - super.setUp(); - } - - /* Cluster helper methods ************************************/ - - /** - * Get the collection state for a particular collection - */ - protected DocCollection getCollectionState(String collectionName) throws IOException { - return cluster.getClusterStateProvider().getClusterState().getCollection(collectionName); - } - - /** - * Get a (reproducibly) random shard from a {@link DocCollection} - */ - protected static Slice getRandomShard(DocCollection collection) { - List shards = new ArrayList<>(collection.getActiveSlices()); - if (shards.size() == 0) - fail("Couldn't get random shard for collection as it has no shards!\n" + collection.toString()); - Collections.shuffle(shards, random()); - return shards.get(0); - } - - /** - * Get a (reproducibly) random replica from a {@link Slice} - */ - protected static Replica getRandomReplica(Slice slice) { - List replicas = new ArrayList<>(slice.getReplicas()); - if (replicas.size() == 0) - fail("Couldn't get random replica from shard as it has no replicas!\n" + slice.toString()); - Collections.shuffle(replicas, random()); - return replicas.get(0); - } - - /** - * Get a (reproducibly) random replica from a {@link Slice} matching a predicate - */ - protected static Replica getRandomReplica(Slice slice, Predicate matchPredicate) { - List replicas = new ArrayList<>(slice.getReplicas()); - if (replicas.size() == 0) - fail("Couldn't get random replica from shard as it has no replicas!\n" + slice.toString()); - Collections.shuffle(replicas, random()); - for (Replica replica : replicas) { - if (matchPredicate.test(replica)) - return replica; - } - fail("Couldn't get random replica that matched conditions\n" + slice.toString()); - return null; // just to keep the compiler happy - fail will always throw an Exception - } - - /** - * Creates & executes an autoscaling request against the current cluster, asserting that - * the result is a success - * - * @param json The request to send - * @see CloudTestUtils#assertAutoScalingRequest - */ - public void assertAutoScalingRequest(final String json) throws IOException { - CloudTestUtils.assertAutoScalingRequest(cluster, json); - } - - /** - * Compare two ClusterState-s, filtering out simulation framework artifacts. - */ - public static void assertClusterStateEquals(ClusterState one, ClusterState two) { - assertEquals(one.getLiveNodes(), two.getLiveNodes()); - assertEquals(one.getCollectionsMap().keySet(), two.getCollectionsMap().keySet()); - one.forEachCollection(oneColl -> { - DocCollection twoColl = two.getCollection(oneColl.getName()); - Map oneSlices = oneColl.getSlicesMap(); - Map twoSlices = twoColl.getSlicesMap(); - assertEquals(oneSlices.keySet(), twoSlices.keySet()); - oneSlices.forEach((s, slice) -> { - Slice sTwo = twoSlices.get(s); - for (Replica oneReplica : slice.getReplicas()) { - Replica twoReplica = sTwo.getReplica(oneReplica.getName()); - assertNotNull(twoReplica); - assertReplicaEquals(oneReplica, twoReplica); - } - }); - }); - } - - // ignore these because SimCloudManager always modifies them - private static final Set IGNORE_REPLICA_PATTERNS = new HashSet<>(Arrays.asList( - Pattern.compile("QUERY\\..*"), - Pattern.compile("INDEX\\..*"), - Pattern.compile("UPDATE\\..*"), - Pattern.compile("SEARCHER\\..*") - )); - - private static final Predicate> REPLICA_FILTER_FUN = p -> { - for (Pattern pattern : IGNORE_REPLICA_PATTERNS) { - if (pattern.matcher(p.getKey()).matches()) { - return false; - } - } - return true; - }; - - public static void assertReplicaEquals(Replica one, Replica two) { - assertEquals(one.getName(), two.getName()); - assertEquals(one.getNodeName(), two.getNodeName()); - assertEquals(one.getState(), two.getState()); - assertEquals(one.getType(), two.getType()); - assertReplicaPropsEquals(one.getProperties(), two.getProperties()); - } - - @SuppressWarnings({"unchecked"}) - public static void assertReplicaInfoEquals(Replica one, Replica two) { - assertEquals(one.getName(), two.getName()); - assertEquals(one.getNodeName(), two.getNodeName()); - assertEquals(one.getState(), two.getState()); - assertEquals(one.getType(), two.getType()); - assertEquals(one.getCoreName(), two.getCoreName()); - assertEquals(one.getCollection(), two.getCollection()); - assertEquals(one.getShard(), two.getShard()); - assertEquals(one.isLeader(), two.isLeader()); - Map oneMap = new HashMap<>(); - Map twoMap = new HashMap<>(); - one.toMap(oneMap); - two.toMap(twoMap); - assertReplicaPropsEquals( - (Map)oneMap.get(one.getName()), - (Map)twoMap.get(two.getName())); - } - - public static void assertReplicaPropsEquals(Map propsOne, Map propsTwo) { - Map filteredPropsOne = propsOne.entrySet().stream() - .filter(REPLICA_FILTER_FUN) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - Map filteredPropsTwo = propsTwo.entrySet().stream() - .filter(REPLICA_FILTER_FUN) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - assertEquals(filteredPropsOne, filteredPropsTwo); - } -} diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimClusterStateProvider.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimClusterStateProvider.java deleted file mode 100644 index 2c82d0cf1c0..00000000000 --- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimClusterStateProvider.java +++ /dev/null @@ -1,227 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.cloud.autoscaling.sim; - -import java.lang.invoke.MethodHandles; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; - -import org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig; -import org.apache.solr.client.solrj.cloud.autoscaling.Preference; -import org.apache.solr.client.solrj.cloud.SolrCloudManager; -import org.apache.solr.client.solrj.embedded.JettySolrRunner; -import org.apache.solr.client.solrj.request.CollectionAdminRequest; -import org.apache.solr.cloud.SolrCloudTestCase; -import org.apache.solr.common.cloud.ClusterState; -import org.apache.solr.common.cloud.DocCollection; -import org.apache.solr.common.cloud.Replica; -import org.apache.solr.common.cloud.Slice; -import org.apache.solr.common.cloud.ZkStateReader; -import org.apache.solr.common.cloud.rule.ImplicitSnitch; -import org.apache.solr.common.params.CollectionAdminParams; -import org.apache.solr.common.util.TimeSource; -import org.apache.solr.common.util.Utils; -import org.apache.zookeeper.Watcher; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * This test compares the cluster state of a real cluster and a simulated one. - */ -public class TestSimClusterStateProvider extends SolrCloudTestCase { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - private static int NODE_COUNT = 3; - private static boolean simulated; - - private static SolrCloudManager cloudManager; - - private static Collection liveNodes; - private static Map clusterProperties; - private static AutoScalingConfig autoScalingConfig; - private static Map>>> replicas; - private static Map> nodeValues; - private static ClusterState realState; - - // set up a real cluster as the source of test data - @BeforeClass - public static void setupCluster() throws Exception { - simulated = random().nextBoolean(); - simulated = true; - log.info("####### Using simulated components? {}", simulated); - - configureCluster(NODE_COUNT) - .addConfig("conf", configset("cloud-minimal")) - .configure(); - CollectionAdminRequest.createCollection(CollectionAdminParams.SYSTEM_COLL, null, 1, 2, 0, 1) - .process(cluster.getSolrClient()); - init(); - } - - @AfterClass - public static void closeCloudManager() throws Exception { - if (simulated && cloudManager != null) { - cloudManager.close(); - } - cloudManager = null; - } - - private static void init() throws Exception { - SolrCloudManager realManager = cluster.getJettySolrRunner(cluster.getJettySolrRunners().size() - 1).getCoreContainer() - .getZkController().getSolrCloudManager(); - liveNodes = realManager.getClusterStateProvider().getLiveNodes(); - clusterProperties = realManager.getClusterStateProvider().getClusterProperties(); - autoScalingConfig = realManager.getDistribStateManager().getAutoScalingConfig(); - replicas = new HashMap<>(); - nodeValues = new HashMap<>(); - liveNodes.forEach(n -> { - replicas.put(n, realManager.getNodeStateProvider().getReplicaInfo(n, Collections.emptySet())); - nodeValues.put(n, realManager.getNodeStateProvider().getNodeValues(n, ImplicitSnitch.tags)); - }); - realState = realManager.getClusterStateProvider().getClusterState(); - - if (simulated) { - // initialize simulated provider - cloudManager = SimCloudManager.createCluster(realManager, null, TimeSource.get("simTime:10")); -// simCloudManager.getSimClusterStateProvider().simSetClusterProperties(clusterProperties); -// simCloudManager.getSimDistribStateManager().simSetAutoScalingConfig(autoScalingConfig); -// nodeValues.forEach((n, values) -> { -// try { -// simCloudManager.getSimNodeStateProvider().simSetNodeValues(n, values); -// } catch (InterruptedException e) { -// fail("Interrupted:" + e); -// } -// }); -// simCloudManager.getSimClusterStateProvider().simSetClusterState(realState); - ClusterState simState = cloudManager.getClusterStateProvider().getClusterState(); - assertClusterStateEquals(realState, simState); - } else { - cloudManager = realManager; - } - } - - private static void assertClusterStateEquals(ClusterState one, ClusterState two) { - assertEquals(one.getLiveNodes(), two.getLiveNodes()); - assertEquals(one.getCollectionsMap().keySet(), two.getCollectionsMap().keySet()); - one.forEachCollection(oneColl -> { - DocCollection twoColl = two.getCollection(oneColl.getName()); - Map oneSlices = oneColl.getSlicesMap(); - Map twoSlices = twoColl.getSlicesMap(); - assertEquals(oneSlices.keySet(), twoSlices.keySet()); - oneSlices.forEach((s, slice) -> { - Slice sTwo = twoSlices.get(s); - for (Replica oneReplica : slice.getReplicas()) { - Replica twoReplica = sTwo.getReplica(oneReplica.getName()); - assertNotNull(twoReplica); - SimSolrCloudTestCase.assertReplicaEquals(oneReplica, twoReplica); - } - }); - }); - } - - private String addNode() throws Exception { - JettySolrRunner solr = cluster.startJettySolrRunner(); - cluster.waitForAllNodes(30); - String nodeId = solr.getNodeName(); - if (simulated) { - ((SimCloudManager) cloudManager).getSimClusterStateProvider().simAddNode(nodeId); - } - return nodeId; - } - - private String deleteNode() throws Exception { - String nodeId = cluster.getJettySolrRunner(0).getNodeName(); - JettySolrRunner stoppedServer = cluster.stopJettySolrRunner(0); - cluster.waitForJettyToStop(stoppedServer); - if (simulated) { - ((SimCloudManager) cloudManager).getSimClusterStateProvider().simRemoveNode(nodeId); - } - return nodeId; - } - - private void setAutoScalingConfig(AutoScalingConfig cfg) throws Exception { - cluster.getJettySolrRunner(0).getCoreContainer().getZkController().getZkClient().setData(ZkStateReader.SOLR_AUTOSCALING_CONF_PATH, - Utils.toJSON(cfg), -1, true); - if (simulated) { - ((SimCloudManager) cloudManager).getSimDistribStateManager().simSetAutoScalingConfig(cfg); - } - } - - @Test - public void testAddRemoveNode() throws Exception { - Set lastNodes = new HashSet<>(cloudManager.getClusterStateProvider().getLiveNodes()); - List liveNodes = cloudManager.getDistribStateManager().listData(ZkStateReader.LIVE_NODES_ZKNODE); - assertEquals(lastNodes.size(), liveNodes.size()); - liveNodes.removeAll(lastNodes); - assertTrue(liveNodes.isEmpty()); - - String node = addNode(); - cloudManager.getTimeSource().sleep(2000); - assertFalse(lastNodes.contains(node)); - lastNodes = new HashSet<>(cloudManager.getClusterStateProvider().getLiveNodes()); - assertTrue(lastNodes.contains(node)); - liveNodes = cloudManager.getDistribStateManager().listData(ZkStateReader.LIVE_NODES_ZKNODE); - assertEquals(lastNodes.size(), liveNodes.size()); - liveNodes.removeAll(lastNodes); - assertTrue(liveNodes.isEmpty()); - - node = deleteNode(); - cloudManager.getTimeSource().sleep(2000); - assertTrue(lastNodes.contains(node)); - lastNodes = new HashSet<>(cloudManager.getClusterStateProvider().getLiveNodes()); - assertFalse(lastNodes.contains(node)); - liveNodes = cloudManager.getDistribStateManager().listData(ZkStateReader.LIVE_NODES_ZKNODE); - assertEquals(lastNodes.size(), liveNodes.size()); - liveNodes.removeAll(lastNodes); - assertTrue(liveNodes.isEmpty()); } - - @Test - public void testAutoScalingConfig() throws Exception { - final CountDownLatch triggered = new CountDownLatch(1); - Watcher w = ev -> { - if (triggered.getCount() == 0) { - fail("already triggered once!"); - } - triggered.countDown(); - }; - AutoScalingConfig cfg = cloudManager.getDistribStateManager().getAutoScalingConfig(w); - assertEquals(autoScalingConfig, cfg); - Preference p = new Preference(Collections.singletonMap("maximize", "freedisk")); - cfg = cfg.withPolicy(cfg.getPolicy().withClusterPreferences(Collections.singletonList(p))); - setAutoScalingConfig(cfg); - if (!triggered.await(10, TimeUnit.SECONDS)) { - fail("Watch should be triggered on update!"); - } - AutoScalingConfig cfg1 = cloudManager.getDistribStateManager().getAutoScalingConfig(null); - assertEquals(cfg, cfg1); - - // restore - setAutoScalingConfig(autoScalingConfig); - cfg1 = cloudManager.getDistribStateManager().getAutoScalingConfig(null); - assertEquals(autoScalingConfig, cfg1); - } -} diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimComputePlanAction.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimComputePlanAction.java deleted file mode 100644 index 40a43d566bb..00000000000 --- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimComputePlanAction.java +++ /dev/null @@ -1,401 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.cloud.autoscaling.sim; - -import java.io.IOException; -import java.lang.invoke.MethodHandles; -import java.util.Collection; -import java.util.List; -import java.util.Map; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicReference; - -import org.apache.solr.client.solrj.SolrClient; -import org.apache.solr.client.solrj.SolrRequest; -import org.apache.solr.client.solrj.SolrResponse; -import org.apache.solr.client.solrj.cloud.SolrCloudManager; -import org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventType; -import org.apache.solr.client.solrj.request.CollectionAdminRequest; -import org.apache.solr.cloud.CloudTestUtils.AutoScalingRequest; -import org.apache.solr.cloud.CloudUtil; -import org.apache.solr.cloud.autoscaling.ActionContext; -import org.apache.solr.cloud.autoscaling.ComputePlanAction; -import org.apache.solr.cloud.autoscaling.ScheduledTriggers; -import org.apache.solr.cloud.autoscaling.TriggerAction; -import org.apache.solr.cloud.autoscaling.TriggerEvent; -import org.apache.solr.cloud.autoscaling.TriggerValidationException; -import org.apache.solr.common.cloud.ClusterState; -import org.apache.solr.common.cloud.DocCollection; -import org.apache.solr.common.cloud.Replica; -import org.apache.solr.common.params.CollectionParams; -import org.apache.solr.common.params.SolrParams; -import org.apache.solr.common.util.NamedList; -import org.apache.solr.common.util.TimeSource; -import org.apache.solr.common.util.Utils; -import org.apache.solr.core.SolrResourceLoader; -import org.apache.solr.util.LogLevel; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import static org.apache.solr.common.params.CollectionParams.CollectionAction.MOVEREPLICA; - -/** - * Test for {@link ComputePlanAction} - */ -@LogLevel("org.apache.solr.cloud.autoscaling=DEBUG;org.apache.solr.cloud.Overseer=DEBUG;org.apache.solr.cloud.overseer=DEBUG;org.apache.solr.client.solrj.cloud.autoscaling=DEBUG;") -public class TestSimComputePlanAction extends SimSolrCloudTestCase { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - private static final AtomicBoolean fired = new AtomicBoolean(false); - private static final int NODE_COUNT = 1; - private static CountDownLatch triggerFiredLatch = new CountDownLatch(1); - @SuppressWarnings({"rawtypes"}) - private static final AtomicReference actionContextPropsRef = new AtomicReference<>(); - private static final AtomicReference eventRef = new AtomicReference<>(); - - @Before - public void init() throws Exception { - configureCluster(1, TimeSource.get("simTime:50")); - fired.set(false); - triggerFiredLatch = new CountDownLatch(1); - actionContextPropsRef.set(null); - - String setClusterPolicyCommand = "{" + - " 'set-cluster-policy': [" + - " {'cores':'<10', 'node':'#ANY'}," + - " {'replica':'<2', 'shard': '#EACH', 'node': '#ANY'}," + - " {'nodeRole':'overseer', 'replica':0}" + - " ]" + - "}"; - @SuppressWarnings({"rawtypes"}) - SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setClusterPolicyCommand); - SolrResponse rsp = cluster.request(req); - NamedList response = rsp.getResponse(); - assertEquals(response.get("result").toString(), "success"); - - String setClusterPreferencesCommand = "{" + - "'set-cluster-preferences': [" + - "{'minimize': 'cores'}," + - "{'maximize': 'freedisk','precision': 100}]" + - "}"; - req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setClusterPreferencesCommand); - rsp = cluster.request(req); - response = rsp.getResponse(); - assertEquals(response.get("result").toString(), "success"); - assertAutoscalingUpdateComplete(); - cluster.getTimeSource().sleep(TimeUnit.SECONDS.toMillis(ScheduledTriggers.DEFAULT_COOLDOWN_PERIOD_SECONDS)); - } - - @After - public void printState() throws Exception { - if (null == cluster) { - // test didn't init, nothing to do - return; - } - - if (log.isInfoEnabled()) { - log.info("-------------_ FINAL STATE --------------"); - log.info("* Node values: {}", Utils.toJSONString(cluster.getSimNodeStateProvider().simGetAllNodeValues())); // logOk - log.info("* Live nodes: {}", cluster.getClusterStateProvider().getLiveNodes()); // logOk - } - ClusterState state = cluster.getClusterStateProvider().getClusterState(); - if (log.isInfoEnabled()) { - for (String coll : cluster.getSimClusterStateProvider().simListCollections()) { - log.info("* Collection {} state: {}", coll, state.getCollection(coll)); // logOk - } - } - shutdownCluster(); - } - - @Test - @AwaitsFix(bugUrl = "https://issues.apache.org/jira/browse/SOLR-12028") // if you beast this, eventually you will see - // creation of 'testNodeLost' collection fail - // because shard1 elects no leader - public void testNodeLost() throws Exception { - // let's start a node so that we have at least two - String node = cluster.simAddNode(); - AssertingTriggerAction.expectedNode = node; - - SolrClient solrClient = cluster.simGetSolrClient(); - String setTriggerCommand = "{" + - "'set-trigger' : {" + - "'name' : 'node_lost_trigger'," + - "'event' : 'nodeLost'," + - "'waitFor' : '7s'," + - "'enabled' : true," + - "'actions' : [{'name':'compute_plan', 'class' : 'solr.ComputePlanAction'}," + - "{'name':'test','class':'" + TestSimComputePlanAction.AssertingTriggerAction.class.getName() + "'}]" + - "}}"; - @SuppressWarnings({"rawtypes"}) - SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand); - NamedList response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - - assertAutoscalingUpdateComplete(); - - CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection("testNodeLost", - "conf",1, 2); - create.process(solrClient); - - CloudUtil.waitForState(cluster, "Timed out waiting for replicas of new collection to be active", - "testNodeLost", CloudUtil.clusterShape(1, 2, false, true)); - - ClusterState clusterState = cluster.getClusterStateProvider().getClusterState(); - log.debug("-- cluster state: {}", clusterState); - DocCollection collection = clusterState.getCollection("testNodeLost"); - List replicas = collection.getReplicas(node); - assertNotNull(replicas); - assertFalse(replicas.isEmpty()); - - // start another node because because when the other node goes away, the cluster policy requires only - // 1 replica per node and none on the overseer - String node2 = cluster.simAddNode(); - assertTrue(node2 + "is not live yet", cluster.getClusterStateProvider().getClusterState().liveNodesContain(node2) ); - - // stop the original node - cluster.simRemoveNode(node, false); - log.info("Stopped_node : {}", node); - - assertTrue("Trigger was not fired even after 10 seconds", triggerFiredLatch.await(10, TimeUnit.SECONDS)); - assertTrue(fired.get()); - @SuppressWarnings({"rawtypes"}) - Map context = actionContextPropsRef.get(); - assertNotNull(context); - @SuppressWarnings({"unchecked", "rawtypes"}) - List operations = (List) context.get("operations"); - assertNotNull("The operations computed by ComputePlanAction should not be null , " + eventRef.get(), operations); - assertEquals("ComputePlanAction should have computed exactly 1 operation", 1, operations.size()); - @SuppressWarnings({"rawtypes"}) - SolrRequest solrRequest = operations.get(0); - SolrParams params = solrRequest.getParams(); - assertEquals("Expected MOVEREPLICA action after adding node", MOVEREPLICA, CollectionParams.CollectionAction.get(params.get("action"))); - String replicaToBeMoved = params.get("replica"); - assertEquals("Unexpected node in computed operation", replicas.get(0).getName(), replicaToBeMoved); - - // shutdown the extra node that we had started - cluster.simRemoveNode(node2, false); - } - - // TODO: AwaitsFix - some checks had to be ignore in this test - public void testNodeWithMultipleReplicasLost() throws Exception { - AssertingTriggerAction.expectedNode = null; - - // start 3 more nodes - cluster.simAddNode(); - cluster.simAddNode(); - cluster.simAddNode(); - - SolrClient solrClient = cluster.simGetSolrClient(); - String setTriggerCommand = "{" + - "'set-trigger' : {" + - "'name' : 'node_lost_trigger'," + - "'event' : 'nodeLost'," + - "'waitFor' : '1s'," + - "'enabled' : true," + - "'actions' : [{'name':'compute_plan', 'class' : 'solr.ComputePlanAction'}," + - "{'name':'test','class':'" + AssertingTriggerAction.class.getName() + "'}]" + - "}}"; - @SuppressWarnings({"rawtypes"}) - SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand); - NamedList response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - - assertAutoscalingUpdateComplete(); - - CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection("testNodeWithMultipleReplicasLost", - "conf",2, 3); - create.process(solrClient); - - CloudUtil.waitForState(cluster, "Timed out waiting for replicas of new collection to be active", - "testNodeWithMultipleReplicasLost", CloudUtil.clusterShape(2, 3, false, true)); - - ClusterState clusterState = cluster.getClusterStateProvider().getClusterState(); - log.debug("-- cluster state: {}", clusterState); - DocCollection docCollection = clusterState.getCollection("testNodeWithMultipleReplicasLost"); - - // lets find a node with at least 2 replicas - String stoppedNodeName = null; - List replicasToBeMoved = null; - for (String node : cluster.getClusterStateProvider().getLiveNodes()) { - List replicas = docCollection.getReplicas(node); - if (replicas != null && replicas.size() == 2) { - stoppedNodeName = node; - replicasToBeMoved = replicas; - cluster.simRemoveNode(node, false); - break; - } - } - assertNotNull(stoppedNodeName); - - assertTrue("Trigger was not fired even after 5 seconds", triggerFiredLatch.await(5, TimeUnit.SECONDS)); - assertTrue(fired.get()); - - TriggerEvent triggerEvent = eventRef.get(); - assertNotNull(triggerEvent); - assertEquals(TriggerEventType.NODELOST, triggerEvent.getEventType()); - // TODO assertEquals(stoppedNodeName, triggerEvent.getProperty(TriggerEvent.NODE_NAME)); - - @SuppressWarnings({"rawtypes"}) - Map context = actionContextPropsRef.get(); - assertNotNull(context); - @SuppressWarnings({"unchecked", "rawtypes"}) - List operations = (List) context.get("operations"); - assertNotNull("The operations computed by ComputePlanAction should not be null " + actionContextPropsRef.get() + "\nevent: " + eventRef.get(), operations); - if (log.isInfoEnabled()) { - operations.forEach(solrRequest -> log.info("{}", solrRequest.getParams())); - } - - // TODO: this can be 3! - // assertEquals("ComputePlanAction should have computed exactly 2 operation", 2, operations.size()); - - for (@SuppressWarnings({"rawtypes"})SolrRequest solrRequest : operations) { - SolrParams params = solrRequest.getParams(); - assertEquals("Expected MOVEREPLICA action after adding node", MOVEREPLICA, CollectionParams.CollectionAction.get(params.get("action"))); - String moved = params.get("replica"); - - // TODO: this can fail! - // assertTrue(replicasToBeMoved.stream().anyMatch(replica -> replica.getName().equals(moved))); - } - } - - @Test - //17-Aug-2018 commented @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 28-June-2018 - // commented out on: 24-Dec-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 14-Oct-2018 - public void testNodeAdded() throws Exception { - AssertingTriggerAction.expectedNode = null; - SolrClient solrClient = cluster.simGetSolrClient(); - String setTriggerCommand = "{" + - "'set-trigger' : {" + - "'name' : 'node_added_trigger'," + - "'event' : 'nodeAdded'," + - "'waitFor' : '1s'," + - "'enabled' : true," + - "'actions' : [{'name':'compute_plan', 'class' : 'solr.ComputePlanAction'}," + - "{'name':'test','class':'" + TestSimComputePlanAction.AssertingTriggerAction.class.getName() + "'}]" + - "}}"; - @SuppressWarnings({"rawtypes"}) - SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand); - NamedList response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - - // the default policy limits 1 replica per node, we need more right now - String setClusterPolicyCommand = "{" + - " 'set-cluster-policy': [" + - " {'cores':'<10', 'node':'#ANY'}," + - " {'replica':'<5', 'shard': '#EACH', 'node': '#ANY'}," + - " {'nodeRole':'overseer', 'replica':0}" + - " ]" + - "}"; - req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setClusterPolicyCommand); - response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - - assertAutoscalingUpdateComplete(); - - CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection("testNodeAdded", - "conf",1, 4); - create.process(solrClient); - - CloudUtil.waitForState(cluster, "Timed out waiting for replicas of new collection to be active", - "testNodeAdded", (liveNodes, collectionState) -> collectionState.getReplicas().stream().allMatch(replica -> replica.isActive(liveNodes))); - - // reset to the original policy which has only 1 replica per shard per node - setClusterPolicyCommand = "{" + - " 'set-cluster-policy': [" + - " {'cores':'<10', 'node':'#ANY'}," + - " {'replica':'<3', 'shard': '#EACH', 'node': '#ANY'}," + - " {'nodeRole':'overseer', 'replica':0}" + - " ]" + - "}"; - req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setClusterPolicyCommand); - response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - - assertAutoscalingUpdateComplete(); - - // start a node so that the 'violation' created by the previous policy update is fixed - String newNode = cluster.simAddNode(); - assertTrue("Trigger was not fired even after 5 seconds", triggerFiredLatch.await(5, TimeUnit.SECONDS)); - assertTrue(fired.get()); - @SuppressWarnings({"rawtypes"}) - Map context = actionContextPropsRef.get(); - assertNotNull(context); - if (log.isInfoEnabled()) { - log.info("Node values: {}", Utils.toJSONString(cluster.getSimNodeStateProvider().simGetAllNodeValues())); - log.info("Live nodes: {}, collection state: {}" - , cluster.getClusterStateProvider().getLiveNodes(), cluster.getClusterStateProvider().getClusterState().getCollection("testNodeAdded")); // logOk - } - - @SuppressWarnings({"unchecked", "rawtypes"}) - List operations = (List) context.get("operations"); - assertNotNull("The operations computed by ComputePlanAction should not be null" + context, operations); - - // TODO: can be 2! - // assertEquals("ComputePlanAction should have computed exactly 1 operation, but was: " + operations, 1, operations.size()); - - @SuppressWarnings({"rawtypes"}) - SolrRequest request = operations.get(0); - SolrParams params = request.getParams(); - assertEquals("Expected MOVEREPLICA action after adding node", MOVEREPLICA, CollectionParams.CollectionAction.get(params.get("action"))); - String nodeAdded = params.get("targetNode"); - assertEquals("Unexpected node in computed operation", newNode, nodeAdded); - } - - public static class AssertingTriggerAction implements TriggerAction { - static String expectedNode; - - @Override - public void configure(SolrResourceLoader loader, SolrCloudManager cloudManager, Map properties) throws TriggerValidationException { - - } - - @Override - public void init() { - - } - - @Override - public String getName() { - return null; - } - - @Override - public void process(TriggerEvent event, ActionContext context) { - if (expectedNode != null) { - @SuppressWarnings({"rawtypes"}) - Collection nodes = (Collection) event.getProperty(TriggerEvent.NODE_NAMES); - if (nodes == null || !nodes.contains(expectedNode)) return;//this is not the event we are looking for - } - if (fired.compareAndSet(false, true)) { - eventRef.set(event); - actionContextPropsRef.set(context.getProperties()); - triggerFiredLatch.countDown(); - } - } - - @Override - public void close() throws IOException { - - } - } -} diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimDistribStateManager.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimDistribStateManager.java deleted file mode 100644 index d6cef5b81a2..00000000000 --- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimDistribStateManager.java +++ /dev/null @@ -1,380 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.cloud.autoscaling.sim; - -import java.lang.invoke.MethodHandles; -import java.nio.charset.StandardCharsets; -import java.util.Arrays; -import java.util.List; -import java.util.Locale; -import java.util.NoSuchElementException; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; - -import org.apache.solr.SolrTestCaseJ4; -import org.apache.solr.client.solrj.cloud.autoscaling.AlreadyExistsException; -import org.apache.solr.client.solrj.cloud.autoscaling.BadVersionException; -import org.apache.solr.client.solrj.cloud.DistribStateManager; -import org.apache.solr.client.solrj.cloud.autoscaling.NotEmptyException; -import org.apache.solr.client.solrj.cloud.autoscaling.VersionedData; -import org.apache.solr.client.solrj.impl.ZkDistribStateManager; -import org.apache.solr.cloud.ZkTestServer; -import org.apache.solr.common.cloud.SolrZkClient; -import org.apache.zookeeper.CreateMode; -import org.apache.zookeeper.WatchedEvent; -import org.apache.zookeeper.Watcher; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * This test compares a ZK-based {@link DistribStateManager} to the simulated one. - */ -public class TestSimDistribStateManager extends SolrTestCaseJ4 { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - private DistribStateManager stateManager; - private ZkTestServer zkTestServer; - private SolrZkClient solrZkClient; - private boolean simulated; - private SimDistribStateManager.Node root; - - @Before - public void setup() throws Exception { - simulated = random().nextBoolean(); - if (simulated) { - root = SimDistribStateManager.createNewRootNode(); - } else { - zkTestServer = new ZkTestServer(createTempDir("zkDir")); - zkTestServer.run(); - } - reInit(); - } - - private void reInit() throws Exception { - if (stateManager != null) { - stateManager.close(); - } - if (simulated) { - stateManager = new SimDistribStateManager(root); - } else { - if (solrZkClient != null) { - solrZkClient.close(); - } - solrZkClient = new SolrZkClient(zkTestServer.getZkHost(), 30000); - stateManager = new ZkDistribStateManager(solrZkClient); - } - if (log.isInfoEnabled()) { - log.info("Using {}", stateManager.getClass().getName()); - } - } - - private DistribStateManager createDistribStateManager() { - if (simulated) { - return new SimDistribStateManager(root); - } else { - SolrZkClient cli = new SolrZkClient(zkTestServer.getZkHost(), 30000); - return new ZkDistribStateManager(cli); - } - } - - private void destroyDistribStateManager(DistribStateManager mgr) throws Exception { - mgr.close(); - if (mgr instanceof ZkDistribStateManager) { - ((ZkDistribStateManager)mgr).getZkClient().close(); - } - } - - @After - public void teardown() throws Exception { - if (solrZkClient != null) { - solrZkClient.close(); - solrZkClient = null; - } - if (zkTestServer != null) { - zkTestServer.shutdown(); - zkTestServer = null; - } - if (stateManager != null) { - stateManager.close(); - } - stateManager = null; - } - - @Test - public void testHasData() throws Exception { - assertFalse(stateManager.hasData("/hasData/foo")); - assertFalse(stateManager.hasData("/hasData/bar")); - try { - stateManager.createData("/hasData/foo", new byte[0], CreateMode.PERSISTENT); - fail("should have failed (parent /hasData doesn't exist)"); - } catch (NoSuchElementException e) { - // expected - } - stateManager.makePath("/hasData"); - stateManager.createData("/hasData/foo", new byte[0], CreateMode.PERSISTENT); - stateManager.createData("/hasData/bar", new byte[0], CreateMode.PERSISTENT); - assertTrue(stateManager.hasData("/hasData/foo")); - assertTrue(stateManager.hasData("/hasData/bar")); - } - - @Test - public void testRemoveData() throws Exception { - assertFalse(stateManager.hasData("/removeData/foo")); - assertFalse(stateManager.hasData("/removeData/foo/bar")); - assertFalse(stateManager.hasData("/removeData/baz")); - assertFalse(stateManager.hasData("/removeData/baz/1/2/3")); - stateManager.makePath("/removeData/foo/bar"); - stateManager.makePath("/removeData/baz/1/2/3"); - assertTrue(stateManager.hasData("/removeData/foo")); - assertTrue(stateManager.hasData("/removeData/foo/bar")); - assertTrue(stateManager.hasData("/removeData/baz/1/2/3")); - try { - stateManager.removeData("/removeData/foo", -1); - fail("should have failed (node has children)"); - } catch (NotEmptyException e) { - // expected - } - stateManager.removeData("/removeData/foo/bar", -1); - stateManager.removeData("/removeData/foo", -1); - // test recursive listing and removal - stateManager.removeRecursively("/removeData/baz/1", false, false); - assertFalse(stateManager.hasData("/removeData/baz/1/2")); - assertTrue(stateManager.hasData("/removeData/baz/1")); - // should silently ignore - stateManager.removeRecursively("/removeData/baz/1/2", true, true); - stateManager.removeRecursively("/removeData/baz/1", false, true); - assertFalse(stateManager.hasData("/removeData/baz/1")); - try { - stateManager.removeRecursively("/removeData/baz/1", false, true); - fail("should throw exception - missing path"); - } catch (NoSuchElementException e) { - // expected - } - stateManager.removeRecursively("/removeData", true, true); - assertFalse(stateManager.hasData("/removeData")); - } - - @Test - public void testListData() throws Exception { - assertFalse(stateManager.hasData("/listData/foo")); - assertFalse(stateManager.hasData("/listData/foo/bar")); - try { - stateManager.createData("/listData/foo/bar", new byte[0], CreateMode.PERSISTENT); - fail("should not succeed"); - } catch (NoSuchElementException e) { - // expected - } - try { - stateManager.listData("/listData/foo"); - fail("should not succeed"); - } catch (NoSuchElementException e) { - // expected - } - stateManager.makePath("/listData"); - List kids = stateManager.listData("/listData"); - assertEquals(0, kids.size()); - stateManager.makePath("/listData/foo"); - kids = stateManager.listData("/listData"); - assertEquals(1, kids.size()); - assertEquals("foo", kids.get(0)); - stateManager.createData("/listData/foo/bar", new byte[0], CreateMode.PERSISTENT); - stateManager.createData("/listData/foo/baz", new byte[0], CreateMode.PERSISTENT); - kids = stateManager.listData("/listData/foo"); - assertEquals(2, kids.size()); - assertTrue(kids.contains("bar")); - assertTrue(kids.contains("baz")); - try { - stateManager.createData("/listData/foo/bar", new byte[0], CreateMode.PERSISTENT); - fail("should not succeed"); - } catch (AlreadyExistsException e) { - // expected - } - } - - static final byte[] firstData = new byte[] { - (byte)0xca, (byte)0xfe, (byte)0xba, (byte)0xbe - }; - - static final byte[] secondData = new byte[] { - (byte)0xbe, (byte)0xba, (byte)0xfe, (byte)0xca - }; - - @Test - public void testCreateMode() throws Exception { - stateManager.makePath("/createMode"); - stateManager.createData("/createMode/persistent", firstData, CreateMode.PERSISTENT); - stateManager.createData("/createMode/persistent_seq", firstData, CreateMode.PERSISTENT); - for (int i = 0; i < 10; i++) { - stateManager.createData("/createMode/persistent_seq/data", firstData, CreateMode.PERSISTENT_SEQUENTIAL); - } - // check what happens with gaps - stateManager.createData("/createMode/persistent_seq/data", firstData, CreateMode.PERSISTENT_SEQUENTIAL); - stateManager.removeData("/createMode/persistent_seq/data" + String.format(Locale.ROOT, "%010d", 10), -1); - stateManager.createData("/createMode/persistent_seq/data", firstData, CreateMode.PERSISTENT_SEQUENTIAL); - - stateManager.createData("/createMode/ephemeral", firstData, CreateMode.EPHEMERAL); - stateManager.createData("/createMode/ephemeral_seq", firstData, CreateMode.PERSISTENT); - for (int i = 0; i < 10; i++) { - stateManager.createData("/createMode/ephemeral_seq/data", firstData, CreateMode.EPHEMERAL_SEQUENTIAL); - } - assertTrue(stateManager.hasData("/createMode")); - assertTrue(stateManager.hasData("/createMode/persistent")); - assertTrue(stateManager.hasData("/createMode/ephemeral")); - List kids = stateManager.listData("/createMode/persistent_seq"); - assertEquals(11, kids.size()); - kids = stateManager.listData("/createMode/ephemeral_seq"); - assertEquals(10, kids.size()); - for (int i = 0; i < 10; i++) { - assertTrue(stateManager.hasData("/createMode/persistent_seq/data" + String.format(Locale.ROOT, "%010d", i))); - } - assertFalse(stateManager.hasData("/createMode/persistent_seq/data" + String.format(Locale.ROOT, "%010d", 10))); - assertTrue(stateManager.hasData("/createMode/persistent_seq/data" + String.format(Locale.ROOT, "%010d", 11))); - - for (int i = 0; i < 10; i++) { - assertTrue(stateManager.hasData("/createMode/ephemeral_seq/data" + String.format(Locale.ROOT, "%010d", i))); - } - // check that ephemeral nodes disappear on disconnect - reInit(); - assertTrue(stateManager.hasData("/createMode/persistent")); - for (int i = 0; i < 10; i++) { - assertTrue(stateManager.hasData("/createMode/persistent_seq/data" + String.format(Locale.ROOT, "%010d", i))); - } - assertTrue(stateManager.hasData("/createMode/persistent_seq/data" + String.format(Locale.ROOT, "%010d", 11))); - - assertFalse(stateManager.hasData("/createMode/ephemeral")); - assertTrue(stateManager.hasData("/createMode/ephemeral_seq")); - kids = stateManager.listData("/createMode/ephemeral_seq"); - assertEquals(0, kids.size()); - } - - @Test - public void testCanCreateNodesWithDataAtTopLevel() throws Exception { - final String path = stateManager.createData("/topLevelNodeWithData", new String("helloworld").getBytes(StandardCharsets.UTF_8), CreateMode.PERSISTENT); - assertEquals("/topLevelNodeWithData", path); - } - - static class OnceWatcher implements Watcher { - CountDownLatch triggered = new CountDownLatch(1); - WatchedEvent event; - - @Override - public void process(WatchedEvent event) { - if (triggered.getCount() == 0) { - fail("Watch was already triggered once!"); - } - triggered.countDown(); - this.event = event; - } - } - - @Test - public void testGetSetRemoveData() throws Exception { - stateManager.makePath("/getData"); - stateManager.createData("/getData/persistentData", firstData, CreateMode.PERSISTENT); - OnceWatcher nodeWatcher = new OnceWatcher(); - VersionedData vd = stateManager.getData("/getData/persistentData", nodeWatcher); - assertNotNull(vd); - assertEquals(0, vd.getVersion()); - assertTrue(Arrays.equals(firstData, vd.getData())); - - // update data, test versioning - try { - stateManager.setData("/getData/persistentData", secondData, 1); - fail("should have failed"); - } catch (BadVersionException e) { - // expected - } - // watch should not have fired - assertEquals(1, nodeWatcher.triggered.getCount()); - - stateManager.setData("/getData/persistentData", secondData, 0); - if (!nodeWatcher.triggered.await(5, TimeUnit.SECONDS)) { - fail("Node watch should have fired!"); - } - // watch should not fire now because it needs to be reset - stateManager.setData("/getData/persistentData", secondData, -1); - - // create ephemeral node using another ZK connection - DistribStateManager ephemeralMgr = createDistribStateManager(); - ephemeralMgr.createData("/getData/ephemeralData", firstData, CreateMode.EPHEMERAL); - - nodeWatcher = new OnceWatcher(); - vd = stateManager.getData("/getData/ephemeralData", nodeWatcher); - destroyDistribStateManager(ephemeralMgr); - if (!nodeWatcher.triggered.await(5, TimeUnit.SECONDS)) { - fail("Node watch should have fired!"); - } - assertTrue(stateManager.hasData("/getData/persistentData")); - assertFalse(stateManager.hasData("/getData/ephemeralData")); - - nodeWatcher = new OnceWatcher(); - vd = stateManager.getData("/getData/persistentData", nodeWatcher); - // try wrong version - try { - stateManager.removeData("/getData/persistentData", vd.getVersion() - 1); - fail("should have failed"); - } catch (BadVersionException e) { - // expected - } - // watch should not have fired - assertEquals(1, nodeWatcher.triggered.getCount()); - - stateManager.removeData("/getData/persistentData", vd.getVersion()); - if (!nodeWatcher.triggered.await(5, TimeUnit.SECONDS)) { - fail("Node watch should have fired!"); - } - } - - @Test - public void testNewlyCreatedPathsStartWithVersionZero() throws Exception { - stateManager.makePath("/createdWithoutData"); - VersionedData vd = stateManager.getData("/createdWithoutData", null); - assertEquals(0, vd.getVersion()); - - stateManager.createData("/createdWithData", new String("helloworld").getBytes(StandardCharsets.UTF_8), CreateMode.PERSISTENT); - vd = stateManager.getData("/createdWithData"); - assertEquals(0, vd.getVersion()); - } - - @Test - public void testModifiedDataNodesGetUpdatedVersion() throws Exception { - stateManager.createData("/createdWithData", new String("foo").getBytes(StandardCharsets.UTF_8), CreateMode.PERSISTENT); - VersionedData vd = stateManager.getData("/createdWithData"); - assertEquals(0, vd.getVersion()); - - stateManager.setData("/createdWithData", new String("bar").getBytes(StandardCharsets.UTF_8), 0); - vd = stateManager.getData("/createdWithData"); - assertEquals(1, vd.getVersion()); - } - - // This is a little counterintuitive, so probably worth its own testcase so we don't break it accidentally. - @Test - public void testHasDataReturnsTrueForExistingButEmptyNodes() throws Exception { - stateManager.makePath("/nodeWithoutData"); - assertTrue(stateManager.hasData("/nodeWithoutData")); - } - - @Test - public void testMulti() throws Exception { - - } - -} diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimDistributedQueue.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimDistributedQueue.java deleted file mode 100644 index 4ac480b11a9..00000000000 --- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimDistributedQueue.java +++ /dev/null @@ -1,221 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.cloud.autoscaling.sim; - -import java.nio.charset.Charset; -import java.util.NoSuchElementException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; -import java.util.function.Predicate; - -import org.apache.solr.SolrTestCaseJ4; -import org.apache.solr.client.solrj.cloud.DistributedQueue; -import org.apache.solr.common.util.ExecutorUtil; -import org.apache.solr.common.util.SolrNamedThreadFactory; -import org.junit.After; -import org.junit.Test; - -/** - * - */ -public class TestSimDistributedQueue extends SolrTestCaseJ4 { - private static final Charset UTF8 = Charset.forName("UTF-8"); - protected ExecutorService executor = ExecutorUtil.newMDCAwareSingleThreadExecutor(new SolrNamedThreadFactory("sdqtest-")); - - @Test -// commented 20-July-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 05-Jul-2018 - public void testDistributedQueue() throws Exception { - String dqZNode = "/distqueue/test1"; - byte[] data = "hello world".getBytes(UTF8); - - DistributedQueue dq = makeDistributedQueue(dqZNode); - - // basic ops - assertNull(dq.poll()); - try { - dq.remove(); - fail("NoSuchElementException expected"); - } catch (NoSuchElementException expected) { - // expected - } - - dq.offer(data); - assertArrayEquals(dq.peek(500), data); - assertArrayEquals(dq.remove(), data); - assertNull(dq.poll()); - - dq.offer(data); - assertArrayEquals(dq.take(), data); // waits for data - assertNull(dq.poll()); - - dq.offer(data); - dq.peek(15000); // wait until data is definitely there before calling remove - assertArrayEquals(dq.remove(), data); - assertNull(dq.poll()); - - // should block until the background thread makes the offer - (new QueueChangerThread(dq, 1000)).start(); - assertNotNull(dq.peek(15000)); - assertNotNull(dq.remove()); - assertNull(dq.poll()); - - // timeout scenario ... background thread won't offer until long after the peek times out - QueueChangerThread qct = new QueueChangerThread(dq, 1000); - qct.start(); - assertNull(dq.peek(500)); - qct.join(); - } - - @Test - public void testDistributedQueueBlocking() throws Exception { - String dqZNode = "/distqueue/test2"; - String testData = "hello world"; - - DistributedQueue dq = makeDistributedQueue(dqZNode); - - assertNull(dq.peek()); - Future future = executor.submit(() -> new String(dq.peek(15000), UTF8)); - try { - future.get(1000, TimeUnit.MILLISECONDS); - fail("TimeoutException expected"); - } catch (TimeoutException expected) { - assertFalse(future.isDone()); - } - - dq.offer(testData.getBytes(UTF8)); - assertEquals(testData, future.get(1000, TimeUnit.MILLISECONDS)); - assertNotNull(dq.poll()); - - assertNull(dq.peek(100)); - - // Rerun the earlier test make sure updates are still seen, post reconnection. - future = executor.submit(() -> new String(dq.peek(15000), UTF8)); - try { - future.get(1000, TimeUnit.MILLISECONDS); - fail("TimeoutException expected"); - } catch (TimeoutException expected) { - assertFalse(future.isDone()); - } - - dq.offer(testData.getBytes(UTF8)); - assertEquals(testData, future.get(1000, TimeUnit.MILLISECONDS)); - assertNotNull(dq.poll()); - assertNull(dq.poll()); - } - - @Test - public void testLocallyOffer() throws Exception { - String dqZNode = "/distqueue/test3"; - DistributedQueue dq = makeDistributedQueue(dqZNode); - dq.peekElements(1, 1, s -> true); - for (int i = 0; i < 100; i++) { - byte[] data = String.valueOf(i).getBytes(UTF8); - dq.offer(data); - assertNotNull(dq.peek()); - dq.poll(); - dq.peekElements(1, 1, s -> true); - } - } - - - @Test - public void testPeekElements() throws Exception { - String dqZNode = "/distqueue/test4"; - byte[] data = "hello world".getBytes(UTF8); - - DistributedQueue dq = makeDistributedQueue(dqZNode); - - // Populate with data. - dq.offer(data); - dq.offer(data); - dq.offer(data); - - Predicate alwaysTrue = s -> true; - Predicate alwaysFalse = s -> false; - - // Should be able to get 0, 1, 2, or 3 instantly - for (int i = 0; i <= 3; ++i) { - assertEquals(i, dq.peekElements(i, 0, alwaysTrue).size()); - } - - // Asking for more should return only 3. - assertEquals(3, dq.peekElements(4, 0, alwaysTrue).size()); - - // If we filter everything out, we should block for the full time. - long start = System.nanoTime(); - assertEquals(0, dq.peekElements(4, 1000, alwaysFalse).size()); - assertTrue(System.nanoTime() - start >= TimeUnit.MILLISECONDS.toNanos(500)); - - // If someone adds a new matching element while we're waiting, we should return immediately. - executor.submit(() -> { - try { - Thread.sleep(500); - dq.offer(data); - } catch (Exception e) { - // ignore - } - }); - start = System.nanoTime(); - assertEquals(1, dq.peekElements(4, 2000, child -> { - // The 4th element in the queue will end with a "3". - return child.endsWith("3"); - }).size()); - long elapsed = System.nanoTime() - start; - assertTrue(elapsed < TimeUnit.MILLISECONDS.toNanos(1000)); - assertTrue(elapsed >= TimeUnit.MILLISECONDS.toNanos(250)); - } - - - protected DistributedQueue makeDistributedQueue(String dqZNode) throws Exception { - return new SimDistributedQueueFactory.SimDistributedQueue(dqZNode); - } - - private static class QueueChangerThread extends Thread { - - DistributedQueue dq; - long waitBeforeOfferMs; - - QueueChangerThread(DistributedQueue dq, long waitBeforeOfferMs) { - this.dq = dq; - this.waitBeforeOfferMs = waitBeforeOfferMs; - } - - public void run() { - try { - Thread.sleep(waitBeforeOfferMs); - dq.offer(getName().getBytes(UTF8)); - } catch (InterruptedException ie) { - // do nothing - } catch (Exception exc) { - throw new RuntimeException(exc); - } - } - } - - @Override - @After - public void tearDown() throws Exception { - try { - super.tearDown(); - } catch (Exception exc) { - } - ExecutorUtil.shutdownAndAwaitTermination(executor); - } - -} diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimExecutePlanAction.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimExecutePlanAction.java deleted file mode 100644 index 138ce8daa21..00000000000 --- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimExecutePlanAction.java +++ /dev/null @@ -1,224 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.cloud.autoscaling.sim; - -import java.lang.invoke.MethodHandles; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.stream.Collectors; - -import org.apache.solr.client.solrj.SolrClient; -import org.apache.solr.client.solrj.SolrRequest; -import org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventType; -import org.apache.solr.client.solrj.cloud.autoscaling.VersionedData; -import org.apache.solr.client.solrj.request.CollectionAdminRequest; -import org.apache.solr.cloud.CloudTestUtils.AutoScalingRequest; -import org.apache.solr.cloud.CloudUtil; -import org.apache.solr.cloud.autoscaling.ActionContext; -import org.apache.solr.cloud.autoscaling.ExecutePlanAction; -import org.apache.solr.cloud.autoscaling.NodeLostTrigger; -import org.apache.solr.common.cloud.ClusterState; -import org.apache.solr.common.cloud.DocCollection; -import org.apache.solr.common.cloud.Replica; -import org.apache.solr.common.cloud.ZkStateReader; -import org.apache.solr.common.params.CollectionParams; -import org.apache.solr.common.util.NamedList; -import org.apache.solr.common.util.TimeSource; -import org.apache.solr.common.util.Utils; -import org.apache.solr.util.LogLevel; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.google.common.collect.Lists; - -/** - * Test for {@link ExecutePlanAction} - */ -@LogLevel("org.apache.solr.cloud=DEBUG") -public class TestSimExecutePlanAction extends SimSolrCloudTestCase { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - private static final TimeSource SIM_TIME_SOURCE = TimeSource.get("simTime:50"); - private static final int NODE_COUNT = 2; - - @Before - public void setupCluster() throws Exception { - configureCluster(NODE_COUNT, SIM_TIME_SOURCE); - } - - @After - public void printState() throws Exception { - if (null == cluster) { - // test didn't init, nothing to do - return; - } - - if (log.isInfoEnabled()) { - log.info("-------------_ FINAL STATE --------------"); - log.info("* Node values: {}", Utils.toJSONString(cluster.getSimNodeStateProvider().simGetAllNodeValues())); // logOk - log.info("* Live nodes: {}", cluster.getClusterStateProvider().getLiveNodes()); // logOk - ClusterState state = cluster.getClusterStateProvider().getClusterState(); - for (String coll : cluster.getSimClusterStateProvider().simListCollections()) { - log.info("* Collection {} state: {}", coll, state.getCollection(coll)); // logOk - } - } - shutdownCluster(); - } - - @Test - // commented out on: 24-Dec-2018 @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 28-June-2018 - public void testExecute() throws Exception { - SolrClient solrClient = cluster.simGetSolrClient(); - String collectionName = "testExecute"; - CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(collectionName, - "conf", 1, 2); - create.process(solrClient); - - if (log.isInfoEnabled()) { - log.info("Collection ready after {} ms", CloudUtil.waitForState(cluster, collectionName, 120, TimeUnit.SECONDS, - CloudUtil.clusterShape(1, 2, false, true))); - } - - String sourceNodeName = cluster.getSimClusterStateProvider().simGetRandomNode(); - ClusterState clusterState = cluster.getClusterStateProvider().getClusterState(); - DocCollection docCollection = clusterState.getCollection(collectionName); - List replicas = docCollection.getReplicas(sourceNodeName); - assertNotNull(replicas); - assertFalse(replicas.isEmpty()); - - List otherNodes = cluster.getClusterStateProvider().getLiveNodes().stream() - .filter(node -> !node.equals(sourceNodeName)).collect(Collectors.toList()); - assertFalse(otherNodes.isEmpty()); - String survivor = otherNodes.get(0); - - try (ExecutePlanAction action = new ExecutePlanAction()) { - action.configure(cluster.getLoader(), cluster, Collections.singletonMap("name", "execute_plan")); - - // used to signal if we found that ExecutePlanAction did in fact create the right znode before executing the operation - AtomicBoolean znodeCreated = new AtomicBoolean(false); - - CollectionAdminRequest.AsyncCollectionAdminRequest moveReplica = new CollectionAdminRequest.MoveReplica(collectionName, replicas.get(0).getName(), survivor); - CollectionAdminRequest.AsyncCollectionAdminRequest mockRequest = new CollectionAdminRequest.AsyncCollectionAdminRequest(CollectionParams.CollectionAction.OVERSEERSTATUS) { - @Override - public void setAsyncId(String asyncId) { - super.setAsyncId(asyncId); - String parentPath = ZkStateReader.SOLR_AUTOSCALING_TRIGGER_STATE_PATH + "/xyz/execute_plan"; - try { - if (cluster.getDistribStateManager().hasData(parentPath)) { - java.util.List children = cluster.getDistribStateManager().listData(parentPath); - if (!children.isEmpty()) { - String child = children.get(0); - VersionedData data = cluster.getDistribStateManager().getData(parentPath + "/" + child); - @SuppressWarnings({"rawtypes"}) - Map m = (Map) Utils.fromJSON(data.getData()); - if (m.containsKey("requestid")) { - znodeCreated.set(m.get("requestid").equals(asyncId)); - } - } - } - } catch (Exception e) { - throw new RuntimeException(e); - } - - } - }; - List operations = Lists.asList(moveReplica, new CollectionAdminRequest.AsyncCollectionAdminRequest[]{mockRequest}); - NodeLostTrigger.NodeLostEvent nodeLostEvent = new NodeLostTrigger.NodeLostEvent(TriggerEventType.NODELOST, - "mock_trigger_name", Collections.singletonList(SIM_TIME_SOURCE.getTimeNs()), - Collections.singletonList(sourceNodeName), CollectionParams.CollectionAction.MOVEREPLICA.toLower()); - ActionContext actionContext = new ActionContext(cluster, null, - new HashMap<>(Collections.singletonMap("operations", operations))); - action.process(nodeLostEvent, actionContext); - -// assertTrue("ExecutePlanAction should have stored the requestid in ZK before executing the request", znodeCreated.get()); - @SuppressWarnings({"unchecked"}) - List> responses = (List>) actionContext.getProperty("responses"); - assertNotNull(responses); - assertEquals(2, responses.size()); - NamedList response = responses.get(0); - assertNull(response.get("failure")); - assertNotNull(response.get("success")); - } - - if (log.isInfoEnabled()) { - log.info("Collection ready after {} ms", CloudUtil.waitForState(cluster, collectionName, 300, TimeUnit.SECONDS, - CloudUtil.clusterShape(1, 2, false, true))); - } - } - - @Test - public void testIntegration() throws Exception { - SolrClient solrClient = cluster.simGetSolrClient(); - - String setTriggerCommand = "{" + - "'set-trigger' : {" + - "'name' : 'node_lost_trigger'," + - "'event' : 'nodeLost'," + - "'waitFor' : '1s'," + - "'enabled' : true," + - "'actions' : [{'name':'compute_plan', 'class' : 'solr.ComputePlanAction'}," + - "{'name':'execute_plan','class':'solr.ExecutePlanAction'}]" + - "}}"; - @SuppressWarnings({"rawtypes"}) - SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand); - NamedList response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - - assertAutoscalingUpdateComplete(); - - String collectionName = "testIntegration"; - CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(collectionName, - "conf", 1, 2); - create.process(solrClient); - - CloudUtil.waitForState(cluster, "Timed out waiting for replicas of new collection to be active", - collectionName, CloudUtil.clusterShape(1, 2, false, true)); - - String sourceNodeName = cluster.getSimClusterStateProvider().simGetRandomNode(); - ClusterState clusterState = cluster.getClusterStateProvider().getClusterState(); - DocCollection docCollection = clusterState.getCollection(collectionName); - List replicas = docCollection.getReplicas(sourceNodeName); - assertNotNull(replicas); - assertFalse(replicas.isEmpty()); - - List otherNodes = cluster.getClusterStateProvider().getLiveNodes().stream() - .filter(node -> !node.equals(sourceNodeName)).collect(Collectors.toList()); - assertFalse(otherNodes.isEmpty()); - String survivor = otherNodes.get(0); - - cluster.simRemoveNode(sourceNodeName, false); - - cluster.getTimeSource().sleep(3000); - - CloudUtil.waitForState(cluster, "Timed out waiting for replicas of collection to be 2 again", - collectionName, CloudUtil.clusterShape(1, 2, false, true)); - - clusterState = cluster.getClusterStateProvider().getClusterState(); - docCollection = clusterState.getCollection(collectionName); - List replicasOnSurvivor = docCollection.getReplicas(survivor); - assertNotNull(replicasOnSurvivor); - assertEquals(2, replicasOnSurvivor.size()); - } -} diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimExtremeIndexing.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimExtremeIndexing.java deleted file mode 100644 index 937b4fa9000..00000000000 --- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimExtremeIndexing.java +++ /dev/null @@ -1,147 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.cloud.autoscaling.sim; - -import java.lang.invoke.MethodHandles; -import java.util.Locale; -import java.util.concurrent.TimeUnit; - -import org.apache.solr.client.solrj.SolrClient; -import org.apache.solr.client.solrj.SolrRequest; -import org.apache.solr.client.solrj.request.CollectionAdminRequest; -import org.apache.solr.client.solrj.request.UpdateRequest; -import org.apache.solr.client.solrj.response.QueryResponse; -import org.apache.solr.cloud.CloudTestUtils.AutoScalingRequest; -import org.apache.solr.cloud.CloudUtil; -import org.apache.solr.cloud.autoscaling.ExecutePlanAction; -import org.apache.solr.common.SolrDocumentList; -import org.apache.solr.common.params.CommonParams; -import org.apache.solr.common.util.NamedList; -import org.apache.solr.common.util.TimeSource; -import org.apache.solr.util.LogLevel; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite; - -/** - * TODO: test can easily fail w/a count that is too low by a small amount (unrelated to BATCH_SIZE) - * TODO: test should not need arbitrary sleep calls if code + test are both working properly w/o concurrency bugs - */ -@org.apache.lucene.util.LuceneTestCase.AwaitsFix(bugUrl="https://issues.apache.org/jira/browse/SOLR-12923") -@TimeoutSuite(millis = 48 * 3600 * 1000) -@LogLevel("org.apache.solr.cloud.autoscaling=DEBUG;org.apache.solr.cloud.autoscaling.NodeLostTrigger=INFO;org.apache.client.solrj.cloud.autoscaling=DEBUG;org.apache.solr.cloud.autoscaling.ComputePlanAction=INFO;org.apache.solr.cloud.autoscaling.ExecutePlanAction=DEBUG;org.apache.solr.cloud.autoscaling.ScheduledTriggers=DEBUG") -//@LogLevel("org.apache.solr.cloud.autoscaling=DEBUG;org.apache.solr.cloud.autoscaling.NodeLostTrigger=INFO;org.apache.client.solrj.cloud.autoscaling=DEBUG;org.apache.solr.cloud.CloudTestUtils=TRACE") -public class TestSimExtremeIndexing extends SimSolrCloudTestCase { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - private static final int SPEED = 100; - // use higher speed for larger scale tests - // private static final int SPEED = 500; - private static final int NUM_NODES = 200; - - private static final long BATCH_SIZE = 200000; - - private static final long NUM_BATCHES = 5000; - // ... or use this for a 1 trillion docs test - // private static final long NUM_BATCHES = 5000000; - - // tweak this threshold to test the number of splits - private static final long ABOVE_SIZE = 20000000; - - // tweak this to allow more operations in one event - private static final int MAX_OPS = 100; - - - private static TimeSource timeSource; - private static SolrClient solrClient; - - @Before - public void setupCluster() throws Exception { - configureCluster(NUM_NODES, TimeSource.get("simTime:" + SPEED)); - timeSource = cluster.getTimeSource(); - solrClient = cluster.simGetSolrClient(); - cluster.simSetUseSystemCollection(false); - } - - @AfterClass - public static void tearDownCluster() throws Exception { - solrClient = null; - } - - @After - public void afterTest() throws Exception { - shutdownCluster(); - } - - @Test - public void testScaleUp() throws Exception { - String collectionName = "testScaleUp_collection"; - CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(collectionName, - "conf", 2, 2); - create.process(solrClient); - - CloudUtil.waitForState(cluster, collectionName, 90, TimeUnit.SECONDS, - CloudUtil.clusterShape(2, 2, false, true)); - - //long waitForSeconds = 3 + random().nextInt(5); - long waitForSeconds = 1; - String setTriggerCommand = "{" + - "'set-trigger' : {" + - "'name' : 'scaleUpTrigger'," + - "'event' : 'indexSize'," + - "'waitFor' : '" + waitForSeconds + "s'," + - "'aboveDocs' : " + ABOVE_SIZE + "," + - "'maxOps' : " + MAX_OPS + "," + - "'enabled' : true," + - "'actions' : [{'name' : 'compute_plan', 'class' : 'solr.ComputePlanAction'}," + - "{'name' : 'execute_plan', 'class' : '" + ExecutePlanAction.class.getName() + "'}]" + - "}}"; - @SuppressWarnings({"rawtypes"}) - SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand); - NamedList response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - - assertAutoscalingUpdateComplete(); - - long batchSize = BATCH_SIZE; - for (long i = 0; i < NUM_BATCHES; i++) { - addDocs(collectionName, i * batchSize, batchSize); - if (log.isInfoEnabled()) { - log.info(String.format(Locale.ROOT, "#### Total docs so far: %,d", ((i + 1) * batchSize))); // logOk - } - timeSource.sleep(waitForSeconds); - } - timeSource.sleep(60000); - QueryResponse rsp = solrClient.query(collectionName, params(CommonParams.Q, "*:*")); - SolrDocumentList docs = rsp.getResults(); - assertNotNull(docs); - assertEquals(docs.toString(), batchSize * NUM_BATCHES, docs.getNumFound()); - } - - private void addDocs(String collection, long start, long count) throws Exception { - UpdateRequest ureq = new UpdateRequest(); - ureq.setParam("collection", collection); - ureq.setDocIterator(new FakeDocIterator(start, count)); - solrClient.request(ureq); - } - -} diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimGenericDistributedQueue.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimGenericDistributedQueue.java deleted file mode 100644 index c4551aa7c5f..00000000000 --- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimGenericDistributedQueue.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.cloud.autoscaling.sim; - -import org.apache.solr.client.solrj.cloud.DistributedQueue; -import org.apache.solr.client.solrj.cloud.DistribStateManager; - -/** - * - */ -public class TestSimGenericDistributedQueue extends TestSimDistributedQueue { - DistribStateManager stateManager = new SimDistribStateManager(); - - @Override - protected DistributedQueue makeDistributedQueue(String dqZNode) throws Exception { - return new GenericDistributedQueue(stateManager, dqZNode); - } - - // commented 4-Sep-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 09-Aug-2018 - // commented out on: 17-Feb-2019 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 14-Oct-2018 - public void testDistributedQueue() throws Exception { - super.testDistributedQueue(); - } -} diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimLargeCluster.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimLargeCluster.java deleted file mode 100644 index edf2349c1a5..00000000000 --- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimLargeCluster.java +++ /dev/null @@ -1,870 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.cloud.autoscaling.sim; - -import java.lang.invoke.MethodHandles; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.stream.Collectors; - -import org.apache.commons.math3.stat.descriptive.SummaryStatistics; -import org.apache.lucene.util.TestUtil; -import org.apache.solr.client.solrj.SolrClient; -import org.apache.solr.client.solrj.cloud.autoscaling.Suggester; -import org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventProcessorStage; -import org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventType; -import org.apache.solr.client.solrj.cloud.autoscaling.Variable; -import org.apache.solr.client.solrj.request.CollectionAdminRequest; -import org.apache.solr.client.solrj.request.UpdateRequest; -import org.apache.solr.cloud.CloudTestUtils; -import org.apache.solr.cloud.CloudUtil; -import org.apache.solr.cloud.autoscaling.ActionContext; -import org.apache.solr.cloud.autoscaling.CapturedEvent; -import org.apache.solr.cloud.autoscaling.ComputePlanAction; -import org.apache.solr.cloud.autoscaling.ExecutePlanAction; -import org.apache.solr.cloud.autoscaling.SearchRateTrigger; -import org.apache.solr.cloud.autoscaling.TriggerActionBase; -import org.apache.solr.cloud.autoscaling.TriggerEvent; -import org.apache.solr.cloud.autoscaling.TriggerListenerBase; -import org.apache.solr.common.SolrInputDocument; -import org.apache.solr.common.cloud.ClusterState; -import org.apache.solr.common.cloud.DocCollection; -import org.apache.solr.common.cloud.Replica; -import org.apache.solr.common.params.CollectionParams; -import org.apache.solr.common.util.Pair; -import org.apache.solr.common.util.TimeSource; -import org.apache.solr.common.util.Utils; -import org.apache.solr.util.LogLevel; -import org.apache.solr.util.TimeOut; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * - */ -@LogLevel("org.apache.solr.cloud.autoscaling=DEBUG") -public class TestSimLargeCluster extends SimSolrCloudTestCase { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - public static final int SPEED = 100; - - public static final int NUM_NODES = 100; - - static final Map> listenerEvents = new ConcurrentHashMap<>(); - static final AtomicInteger triggerFinishedCount = new AtomicInteger(); - static final AtomicInteger triggerStartedCount = new AtomicInteger(); - static volatile CountDownLatch triggerStartedLatch; - static volatile CountDownLatch triggerFinishedLatch; - static volatile CountDownLatch listenerEventLatch; - static volatile int waitForSeconds; - - @After - public void tearDownTest() throws Exception { - shutdownCluster(); - } - - @Before - public void setupTest() throws Exception { - configureCluster(NUM_NODES, TimeSource.get("simTime:" + SPEED)); - - // disable metrics history collection - cluster.disableMetricsHistory(); - - // turn off the default policy to avoid slowdowns due to the costly #EQUAL rules - CloudTestUtils.assertAutoScalingRequest(cluster, "{'set-cluster-policy': []}"); - // disable .scheduled_maintenance (once it exists) - CloudTestUtils.waitForTriggerToBeScheduled(cluster, ".scheduled_maintenance"); - CloudTestUtils.suspendTrigger(cluster, ".scheduled_maintenance"); - // disable .auto_add_replicas (once it exists) - CloudTestUtils.waitForTriggerToBeScheduled(cluster, ".auto_add_replicas"); - CloudTestUtils.suspendTrigger(cluster, ".auto_add_replicas"); - cluster.getSimClusterStateProvider().createSystemCollection(); - - waitForSeconds = 5; - triggerStartedCount.set(0); - triggerFinishedCount.set(0); - triggerStartedLatch = new CountDownLatch(1); - triggerFinishedLatch = new CountDownLatch(1); - - // by default assume we want to allow a (virtually) unbounded amount of events, - // tests that expect a specific number can override - listenerEventLatch = new CountDownLatch(Integer.MAX_VALUE); - listenerEvents.clear(); - } - - public static class TestTriggerListener extends TriggerListenerBase { - @Override - public synchronized void onEvent(TriggerEvent event, TriggerEventProcessorStage stage, String actionName, - ActionContext context, Throwable error, String message) { - CapturedEvent ev = new CapturedEvent(cluster.getTimeSource().getTimeNs(), context, config, stage, actionName, event, message); - final CountDownLatch latch = listenerEventLatch; - synchronized (latch) { - if (0 == latch.getCount()) { - log.warn("Ignoring captured event since latch is 'full': {}", ev); - } else { - List lst = listenerEvents.computeIfAbsent(config.name, s -> new ArrayList<>()); - lst.add(ev); - latch.countDown(); - } - } - } - } - - public static class FinishTriggerAction extends TriggerActionBase { - @Override - public void process(TriggerEvent event, ActionContext context) throws Exception { - triggerFinishedCount.incrementAndGet(); - triggerFinishedLatch.countDown(); - } - } - - public static class StartTriggerAction extends TriggerActionBase { - @Override - public void process(TriggerEvent event, ActionContext context) throws Exception { - triggerStartedCount.incrementAndGet(); - triggerStartedLatch.countDown(); - } - } - - @Test - @AwaitsFix(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // this test hits a timeout easily - public void testBasic() throws Exception { - SolrClient solrClient = cluster.simGetSolrClient(); - assertAutoScalingRequest - ( "{" + - "'set-trigger' : {" + - "'name' : 'node_lost_trigger1'," + - "'event' : 'nodeLost'," + - "'waitFor' : '" + waitForSeconds + "s'," + - "'enabled' : true," + - "'actions' : [" + - "{'name':'start','class':'" + StartTriggerAction.class.getName() + "'}," + - "{'name':'compute','class':'" + ComputePlanAction.class.getName() + "'}," + - "{'name':'execute','class':'" + ExecutePlanAction.class.getName() + "'}," + - "{'name':'test','class':'" + FinishTriggerAction.class.getName() + "'}" + - "]" + - "}}"); - - assertAutoScalingRequest - ( "{" + - "'set-listener' : " + - "{" + - "'name' : 'foo'," + - "'trigger' : 'node_lost_trigger1'," + - "'stage' : ['STARTED','ABORTED','SUCCEEDED', 'FAILED']," + - "'beforeAction' : ['compute', 'execute']," + - "'afterAction' : ['compute', 'execute']," + - "'class' : '" + TestTriggerListener.class.getName() + "'" + - "}" + - "}"); - - assertAutoscalingUpdateComplete(); - - // pick a few random nodes - List nodes = new ArrayList<>(); - int limit = 75; - for (String node : cluster.getClusterStateProvider().getLiveNodes()) { - nodes.add(node); - if (nodes.size() > limit) { - break; - } - } - Collections.shuffle(nodes, random()); - // create collection on these nodes - String collectionName = "testBasic"; - CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(collectionName, - "conf", 5, 5, 5, 5); - create.setAutoAddReplicas(false); - create.setCreateNodeSet(String.join(",", nodes)); - create.process(solrClient); - - if (log.isInfoEnabled()) { - log.info("Ready after {} ms", CloudUtil.waitForState(cluster, collectionName, 30 * nodes.size(), TimeUnit.SECONDS, - CloudUtil.clusterShape(5, 15, false, true))); - } - - int KILL_NODES = 8; - // kill off a number of nodes - for (int i = 0; i < KILL_NODES; i++) { - cluster.simRemoveNode(nodes.get(i), false); - } - // should fully recover - if (log.isInfoEnabled()) { - log.info("Ready after {} ms", CloudUtil.waitForState(cluster, collectionName, 90 * KILL_NODES, TimeUnit.SECONDS, - CloudUtil.clusterShape(5, 15, false, true))); - log.info("OP COUNTS: {}", cluster.simGetOpCounts()); // logOk - } - long moveReplicaOps = cluster.simGetOpCount(CollectionParams.CollectionAction.MOVEREPLICA.name()); - - // simulate a number of flaky nodes - int FLAKY_NODES = 10; - int flakyReplicas = 0; - for (int cnt = 0; cnt < 10; cnt++) { - for (int i = KILL_NODES; i < KILL_NODES + FLAKY_NODES; i++) { - flakyReplicas += cluster.getSimClusterStateProvider().simGetReplicaInfos(nodes.get(i)) - .stream().filter(r -> r.getState().equals(Replica.State.ACTIVE)).count(); - cluster.simRemoveNode(nodes.get(i), false); - } - cluster.getTimeSource().sleep(TimeUnit.SECONDS.toMillis(waitForSeconds) * 2); - for (int i = KILL_NODES; i < KILL_NODES + FLAKY_NODES; i++) { - final String nodeId = nodes.get(i); - cluster.submit(() -> cluster.getSimClusterStateProvider().simRestoreNode(nodeId)); - } - } - - // wait until started == finished - TimeOut timeOut = new TimeOut(20 * waitForSeconds * NUM_NODES, TimeUnit.SECONDS, cluster.getTimeSource()); - while (!timeOut.hasTimedOut()) { - if (triggerStartedCount.get() == triggerFinishedCount.get()) { - break; - } - timeOut.sleep(1000); - } - if (timeOut.hasTimedOut()) { - fail("did not finish processing all events in time: started=" + triggerStartedCount.get() + ", finished=" + triggerFinishedCount.get()); - } - - if (log.isInfoEnabled()) { - log.info("Ready after {} ms", CloudUtil.waitForState(cluster, collectionName, 30 * nodes.size(), TimeUnit.SECONDS, - CloudUtil.clusterShape(5, 15, false, true))); - } - - long newMoveReplicaOps = cluster.simGetOpCount(CollectionParams.CollectionAction.MOVEREPLICA.name()); - if (log.isInfoEnabled()) { - log.info("==== Flaky replicas: {}. Additional MOVEREPLICA count: {}", flakyReplicas, (newMoveReplicaOps - moveReplicaOps)); - } - // flaky nodes lead to a number of MOVEREPLICA that is non-zero but lower than the number of flaky replicas - assertTrue("there should be new MOVERPLICA ops", newMoveReplicaOps - moveReplicaOps > 0); - assertTrue("there should be less than flakyReplicas=" + flakyReplicas + " MOVEREPLICA ops", - newMoveReplicaOps - moveReplicaOps < flakyReplicas); - } - - @Test - public void testCreateLargeSimCollections() throws Exception { - SolrClient solrClient = cluster.simGetSolrClient(); - - final int numCollections = atLeast(5); - for (int i = 0; i < numCollections; i++) { - // wide and shallow, or deep and narrow... - final int numShards = TestUtil.nextInt(random(), 5, 20); - final int nReps = TestUtil.nextInt(random(), 2, 25 - numShards); - final int tReps = TestUtil.nextInt(random(), 2, 25 - numShards); - final int pReps = TestUtil.nextInt(random(), 2, 25 - numShards); - final int repsPerShard = (nReps + tReps + pReps); - final int totalCores = repsPerShard * numShards; - final String name = "large_sim_collection" + i; - - final CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection - (name, "conf", numShards, nReps, tReps, pReps); - create.setAutoAddReplicas(false); - - log.info("CREATE: {}", create); - create.process(solrClient); - - // Since our current goal is to try and find situations where cores are just flat out missing - // no matter how long we wait, let's be excessive and generous in our timeout. - // (REMINDER: this uses the cluster's timesource, and ADDREPLICA has a hardcoded delay of 500ms) - CloudUtil.waitForState(cluster, name, 2 * totalCores, TimeUnit.SECONDS, - CloudUtil.clusterShape(numShards, repsPerShard, false, true)); - - final CollectionAdminRequest.Delete delete = CollectionAdminRequest.deleteCollection(name); - log.info("DELETE: {}", delete); - delete.process(solrClient); - } - } - - @Test - // impossible to complete due to the slowness of policy calculations - @AwaitsFix( bugUrl = "https://issues.apache.org/jira/browse/SOLR-14275") - public void testAddNode() throws Exception { - SolrClient solrClient = cluster.simGetSolrClient(); - assertAutoScalingRequest - ( "{" + - "'set-trigger' : {" + - "'name' : 'node_added_trigger2'," + - "'event' : 'nodeAdded'," + - "'waitFor' : '" + waitForSeconds + "s'," + - "'enabled' : true," + - "'actions' : [" + - "{'name':'start','class':'" + StartTriggerAction.class.getName() + "'}," + - "{'name':'compute','class':'" + ComputePlanAction.class.getName() + "'}," + - "{'name':'execute','class':'" + ExecutePlanAction.class.getName() + "'}," + - "{'name':'test','class':'" + FinishTriggerAction.class.getName() + "'}" + - "]" + - "}}"); - - assertAutoscalingUpdateComplete(); - - // create a collection with more than 1 replica per node - String collectionName = "testNodeAdded"; - CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(collectionName, - "conf", NUM_NODES / 10, NUM_NODES / 8, NUM_NODES / 8, NUM_NODES / 8); - create.setAutoAddReplicas(false); - create.process(solrClient); - - if (log.isInfoEnabled()) { - log.info("Ready after {} ms", CloudUtil.waitForState(cluster, collectionName, 90 * NUM_NODES, TimeUnit.SECONDS, - CloudUtil.clusterShape(NUM_NODES / 10, NUM_NODES / 8 * 3, false, true))); - } - - // start adding nodes - int numAddNode = NUM_NODES / 5; - List addNodesList = new ArrayList<>(numAddNode); - for (int i = 0; i < numAddNode; i++) { - addNodesList.add(cluster.simAddNode()); - } - // wait until at least one event is generated - assertTrue("Trigger did not start even after await()ing an excessive amount of time", - triggerStartedLatch.await(60, TimeUnit.SECONDS)); - - // wait until started == finished - TimeOut timeOut = new TimeOut(45 * waitForSeconds * NUM_NODES, TimeUnit.SECONDS, cluster.getTimeSource()); - while (!timeOut.hasTimedOut()) { - final int started = triggerStartedCount.get(); - final int finished = triggerFinishedCount.get(); - log.info("started={} =?= finished={}", started, finished); - if (triggerStartedCount.get() == triggerFinishedCount.get()) { - log.info("started == finished: {} == {}", started, finished); - break; - } - timeOut.sleep(1000); - } - if (timeOut.hasTimedOut()) { - fail("did not finish processing all events in time: started=" + triggerStartedCount.get() + ", finished=" + triggerFinishedCount.get()); - } - - List systemColl = cluster.simGetSystemCollection(); - int startedEventPos = -1; - for (int i = 0; i < systemColl.size(); i++) { - SolrInputDocument d = systemColl.get(i); - if (!"node_added_trigger2".equals(d.getFieldValue("event.source_s"))) { - continue; - } - if ("NODEADDED".equals(d.getFieldValue("event.type_s")) && - "STARTED".equals(d.getFieldValue("stage_s"))) { - startedEventPos = i; - break; - } - } - assertTrue("no STARTED event", startedEventPos > -1); - SolrInputDocument startedEvent = systemColl.get(startedEventPos); - int lastIgnoredPos = startedEventPos; - // make sure some replicas have been moved - long lastNumOps = cluster.simGetOpCount("MOVEREPLICA"); - log.info("1st check: lastNumOps (MOVEREPLICA) = {}", lastNumOps); - assertTrue("no MOVEREPLICA ops?", lastNumOps > 0); - - if (log.isInfoEnabled()) { - log.info("Ready after {} ms", CloudUtil.waitForState(cluster, collectionName, 20 * NUM_NODES, TimeUnit.SECONDS, - CloudUtil.clusterShape(NUM_NODES / 10, NUM_NODES / 8 * 3, false, true))); - } - - int count = 1000; - SolrInputDocument finishedEvent = null; - lastNumOps = cluster.simGetOpCount("MOVEREPLICA"); - log.info("2nd check: lastNumOps (MOVEREPLICA) = {}", lastNumOps); - while (count-- > 0) { - cluster.getTimeSource().sleep(10000); - - if (cluster.simGetOpCount("MOVEREPLICA") < 2) { - log.info("MOVEREPLICA < 2"); - continue; - } - - long currentNumOps = cluster.simGetOpCount("MOVEREPLICA"); - if (currentNumOps == lastNumOps) { - int size = systemColl.size() - 1; - for (int i = size; i > lastIgnoredPos; i--) { - SolrInputDocument d = systemColl.get(i); - if (!"node_added_trigger2".equals(d.getFieldValue("event.source_s"))) { - continue; - } - if ("SUCCEEDED".equals(d.getFieldValue("stage_s"))) { - finishedEvent = d; - log.info("finishedEvent = {}", finishedEvent); - break; - } - } - log.info("breaking because currentNumOps == lastNumOps == {}", currentNumOps); - break; - } else { - lastNumOps = currentNumOps; - } - } - - assertNotNull("did not finish processing changes", finishedEvent); - long delta = (Long) finishedEvent.getFieldValue("event.time_l") - (Long) startedEvent.getFieldValue("event.time_l"); - if (log.isInfoEnabled()) { - log.info("#### System stabilized after {} ms", TimeUnit.NANOSECONDS.toMillis(delta)); - } - assertTrue("unexpected number of MOVEREPLICA ops: " + cluster.simGetOpCount("MOVEREPLICA"), - cluster.simGetOpCount("MOVEREPLICA") > 1); - } - - @Test - public void testNodeLost() throws Exception { - doTestNodeLost(waitForSeconds, 5000, 0); - } - - // Renard R5 series - evenly covers a log10 range - private static final int[] renard5 = new int[] { - 1, 2, 3, 4, 6, - 10 - }; - private static final int[] renard5x = new int[] { - 1, 2, 3, 4, 6, - 10, 16, 25, 40, 63, - 100 - }; - private static final int[] renard5xx = new int[] { - 1, 2, 3, 4, 6, - 10, 16, 25, 40, 63, - 100, 158, 251, 398, 631, - 1000, 1585, 2512, 3981, 6310, - 10000 - }; - // Renard R10 series - private static final double[] renard10 = new double[] { - 1, 1.3, 1.6, 2, 2.5, 3.2, 4, 5, 6.3, 7.9, - 10 - }; - private static final double[] renard10x = new double[] { - 1, 1.3, 1.6, 2, 2.5, 3.2, 4, 5, 6.3, 7.9, - 10, 12.6, 15.8, 20, 25.1, 31.6, 39.8, 50.1, 63.1, 79.4, - 100 - }; - - private static final AtomicInteger ZERO = new AtomicInteger(0); - - //@Test - public void benchmarkNodeLost() throws Exception { - List results = new ArrayList<>(); - for (int wait : renard5x) { - for (int delay : renard5x) { - SummaryStatistics totalTime = new SummaryStatistics(); - SummaryStatistics ignoredOurEvents = new SummaryStatistics(); - SummaryStatistics ignoredOtherEvents = new SummaryStatistics(); - SummaryStatistics startedOurEvents = new SummaryStatistics(); - SummaryStatistics startedOtherEvents = new SummaryStatistics(); - for (int i = 0; i < 5; i++) { - if (cluster != null) { - cluster.close(); - } - setUp(); - setupTest(); - long total = doTestNodeLost(wait, delay * 1000, 0); - totalTime.addValue(total); - // get event counts - Map> counts = cluster.simGetEventCounts(); - Map map = counts.remove("node_lost_trigger"); - startedOurEvents.addValue(map.getOrDefault("STARTED", ZERO).get()); - ignoredOurEvents.addValue(map.getOrDefault("IGNORED", ZERO).get()); - int otherStarted = 0; - int otherIgnored = 0; - for (Map m : counts.values()) { - otherStarted += m.getOrDefault("STARTED", ZERO).get(); - otherIgnored += m.getOrDefault("IGNORED", ZERO).get(); - } - startedOtherEvents.addValue(otherStarted); - ignoredOtherEvents.addValue(otherIgnored); - } - results.add(String.format(Locale.ROOT, "%d\t%d\t%4.0f\t%4.0f\t%4.0f\t%4.0f\t%6.0f\t%6.0f\t%6.0f\t%6.0f\t%6.0f", - wait, delay, startedOurEvents.getMean(), ignoredOurEvents.getMean(), - startedOtherEvents.getMean(), ignoredOtherEvents.getMean(), - totalTime.getMin(), totalTime.getMax(), totalTime.getMean(), totalTime.getStandardDeviation(), totalTime.getVariance())); - } - } - log.info("===== RESULTS ======"); - log.info("waitFor\tdelay\tSTRT\tIGN\toSTRT\toIGN\tmin\tmax\tmean\tstdev\tvar"); - if (log.isInfoEnabled()) { - results.forEach(s -> log.info(s)); - } - } - - private long doTestNodeLost(int waitFor, long killDelay, int minIgnored) throws Exception { - SolrClient solrClient = cluster.simGetSolrClient(); - assertAutoScalingRequest - ( "{" + - "'set-trigger' : {" + - "'name' : 'node_lost_trigger3'," + - "'event' : 'nodeLost'," + - "'waitFor' : '" + waitFor + "s'," + - "'enabled' : true," + - "'actions' : [" + - "{'name':'start','class':'" + StartTriggerAction.class.getName() + "'}," + - "{'name':'compute','class':'" + ComputePlanAction.class.getName() + "'}," + - "{'name':'execute','class':'" + ExecutePlanAction.class.getName() + "'}," + - "{'name':'test','class':'" + FinishTriggerAction.class.getName() + "'}" + - "]" + - "}}"); - - assertAutoScalingRequest - ( "{" + - "'set-listener' : " + - "{" + - "'name' : 'failures'," + - "'trigger' : 'node_lost_trigger3'," + - "'stage' : ['FAILED']," + - "'class' : '" + TestTriggerListener.class.getName() + "'" + - "}" + - "}"); - - assertAutoscalingUpdateComplete(); - - // create a collection with 1 replica per node - String collectionName = "testNodeLost"; - CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(collectionName, - "conf", NUM_NODES / 5, NUM_NODES / 10); - create.setAutoAddReplicas(false); - create.process(solrClient); - - if (log.isInfoEnabled()) { - log.info("Ready after {} ms", CloudUtil.waitForState(cluster, collectionName, 60 * NUM_NODES, TimeUnit.SECONDS, - CloudUtil.clusterShape(NUM_NODES / 5, NUM_NODES / 10, false, true))); - } - - // start killing nodes - int numNodes = NUM_NODES / 5; - List nodes = new ArrayList<>(cluster.getLiveNodesSet().get()); - for (int i = 0; i < numNodes; i++) { - // this may also select a node where a replica is moved to, so the total number of - // MOVEREPLICA may vary - cluster.simRemoveNode(nodes.get(i), false); - cluster.getTimeSource().sleep(killDelay); - } - // wait for the trigger to fire and complete at least once - assertTrue("Trigger did not finish even after await()ing an excessive amount of time", - triggerFinishedLatch.await(60, TimeUnit.SECONDS)); - - List systemColl = cluster.simGetSystemCollection(); - int startedEventPos = -1; - - for (int i = 0; i < systemColl.size(); i++) { - SolrInputDocument d = systemColl.get(i); - if (!"node_lost_trigger3".equals(d.getFieldValue("event.source_s"))) { - continue; - } - if ("NODELOST".equals(d.getFieldValue("event.type_s")) && - "STARTED".equals(d.getFieldValue("stage_s"))) { - startedEventPos = i; - break; - } - - } - - // TODO we may not even have a .system collection because the message of node going down is interrupted on the executor - // by the OverseerTriggerThread executors being interrupted on Overseer restart - - if (systemColl.size() > 0) { - return 0; - } - assertTrue("no STARTED event: " + systemColl + ", " + - "waitFor=" + waitFor + ", killDelay=" + killDelay + ", minIgnored=" + minIgnored, - startedEventPos > -1); - SolrInputDocument startedEvent = systemColl.get(startedEventPos); - // we can expect some failures when target node in MOVEREPLICA has been killed - // between when the event processing started and the actual moment of MOVEREPLICA execution - // wait until started == (finished + failed) - TimeOut timeOut = new TimeOut(20 * waitFor * NUM_NODES, TimeUnit.SECONDS, cluster.getTimeSource()); - while (!timeOut.hasTimedOut()) { - if (triggerStartedCount.get() == triggerFinishedCount.get()) { - break; - } - if (log.isDebugEnabled()) { - log.debug("started={}, finished={}, failed={}", triggerStartedCount.get(), triggerFinishedCount.get() - , listenerEvents.size()); - } - timeOut.sleep(1000); - } - if (timeOut.hasTimedOut()) { - if (triggerStartedCount.get() > triggerFinishedCount.get() + listenerEvents.size()) { - fail("did not finish processing all events in time: started=" + triggerStartedCount.get() + ", finished=" + triggerFinishedCount.get() + - ", failed=" + listenerEvents.size()); - } - } - int ignored = 0; - int lastIgnoredPos = startedEventPos; - for (int i = startedEventPos + 1; i < systemColl.size(); i++) { - SolrInputDocument d = systemColl.get(i); - if (!"node_lost_trigger3".equals(d.getFieldValue("event.source_s"))) { - continue; - } - if ("NODELOST".equals(d.getFieldValue("event.type_s"))) { - if ("IGNORED".equals(d.getFieldValue("stage_s"))) { - ignored++; - lastIgnoredPos = i; - } - } - } - assertTrue("should be at least " + minIgnored + " IGNORED events, " + - "waitFor=" + waitFor + ", killDelay=" + killDelay + ", minIgnored=" + minIgnored, - ignored >= minIgnored); - // make sure some replicas have been moved - assertTrue("no MOVEREPLICA ops? " + - "waitFor=" + waitFor + ", killDelay=" + killDelay + ", minIgnored=" + minIgnored, - cluster.simGetOpCount("MOVEREPLICA") > 0); - - if (listenerEvents.isEmpty()) { - // no failed movements - verify collection shape - if (log.isInfoEnabled()) { - log.info("Ready after {} ms", CloudUtil.waitForState(cluster, collectionName, 20 * NUM_NODES, TimeUnit.SECONDS, - CloudUtil.clusterShape(NUM_NODES / 5, NUM_NODES / 10, false, true))); - } - } else { - cluster.getTimeSource().sleep(NUM_NODES * 100); - } - - int count = 50; - SolrInputDocument finishedEvent = null; - long lastNumOps = cluster.simGetOpCount("MOVEREPLICA"); - while (count-- > 0) { - cluster.getTimeSource().sleep(waitFor * 10000); - long currentNumOps = cluster.simGetOpCount("MOVEREPLICA"); - if (currentNumOps == lastNumOps) { - int size = systemColl.size() - 1; - for (int i = size; i > lastIgnoredPos; i--) { - SolrInputDocument d = systemColl.get(i); - if (!"node_lost_trigger3".equals(d.getFieldValue("event.source_s"))) { - continue; - } - if ("SUCCEEDED".equals(d.getFieldValue("stage_s"))) { - finishedEvent = d; - break; - } - } - break; - } else { - lastNumOps = currentNumOps; - } - } - - assertTrue("did not finish processing changes, " + - "waitFor=" + waitFor + ", killDelay=" + killDelay + ", minIgnored=" + minIgnored, - finishedEvent != null); - Long delta = 0L; - if (startedEvent != null) { - delta = (Long) finishedEvent.getFieldValue("event.time_l") - - (Long) startedEvent.getFieldValue("event.time_l"); - delta = TimeUnit.NANOSECONDS.toMillis(delta); - log.info("#### System stabilized after {} ms", delta); - } - long ops = cluster.simGetOpCount("MOVEREPLICA"); - long expectedMinOps = 40; - if (!listenerEvents.isEmpty()) { - expectedMinOps = 20; - } - assertTrue("unexpected number (" + expectedMinOps + ") of MOVEREPLICA ops: " + ops + ", " + - "waitFor=" + waitFor + ", killDelay=" + killDelay + ", minIgnored=" + minIgnored, - ops >= expectedMinOps); - return delta; - } - - @Test - @SuppressWarnings({"unchecked"}) - public void testSearchRate() throws Exception { - SolrClient solrClient = cluster.simGetSolrClient(); - String collectionName = "testSearchRate"; - CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(collectionName, - "conf", 2, 10); - create.process(solrClient); - - if (log.isInfoEnabled()) { - log.info("Ready after {} ms", CloudUtil.waitForState(cluster, collectionName, 300, TimeUnit.SECONDS, - CloudUtil.clusterShape(2, 10, false, true))); - } - - // collect the node names for shard1 - Set nodes = new HashSet<>(); - cluster.getSimClusterStateProvider().getClusterState().getCollection(collectionName) - .getSlice("shard1") - .getReplicas() - .forEach(r -> nodes.add(r.getNodeName())); - - String metricName = "QUERY./select.requestTimes:1minRate"; - // simulate search traffic - cluster.getSimClusterStateProvider().simSetShardValue(collectionName, "shard1", metricName, 40, false, true); - - // now define the trigger. doing it earlier may cause partial events to be generated (where only some - // nodes / replicas exceeded the threshold). - assertAutoScalingRequest - ( "{" + - "'set-trigger' : {" + - "'name' : 'search_rate_trigger'," + - "'event' : 'searchRate'," + - "'waitFor' : '" + waitForSeconds + "s'," + - "'aboveRate' : 1.0," + - "'aboveNodeRate' : 1.0," + - "'enabled' : true," + - "'actions' : [" + - "{'name':'compute','class':'" + ComputePlanAction.class.getName() + "'}," + - "{'name':'execute','class':'" + ExecutePlanAction.class.getName() + "'}," + - "{'name':'test','class':'" + FinishTriggerAction.class.getName() + "'}" + - "]" + - "}}"); - - - // we're going to expect our trigger listener to process exactly one captured event - listenerEventLatch = new CountDownLatch(1); - assertAutoScalingRequest - ( "{" + - "'set-listener' : " + - "{" + - "'name' : 'srt'," + - "'trigger' : 'search_rate_trigger'," + - "'stage' : ['FAILED','SUCCEEDED']," + - "'class' : '" + TestTriggerListener.class.getName() + "'" + - "}" + - "}"); - - assertAutoscalingUpdateComplete(); - - assertTrue("Trigger did not finish even after await()ing an excessive amount of time", - triggerFinishedLatch.await(60, TimeUnit.SECONDS)); - - assertTrue("The listener didn't record the event even after await()ing an excessive amount of time", - listenerEventLatch.await(60, TimeUnit.SECONDS)); - List events = listenerEvents.get("srt"); - assertNotNull("no srt events: " + listenerEvents.toString(), events); - assertEquals(events.toString(), 1, events.size()); - - CapturedEvent ev = events.get(0); - assertEquals(TriggerEventType.SEARCHRATE, ev.event.getEventType()); - Map m = (Map)ev.event.getProperty(SearchRateTrigger.HOT_NODES); - assertNotNull(m); - assertEquals(nodes.size(), m.size()); - assertEquals(nodes, m.keySet()); - m.forEach((k, v) -> assertEquals(4.0, v.doubleValue(), 0.01)); - List ops = (List)ev.event.getProperty(TriggerEvent.REQUESTED_OPS); - assertNotNull(ops); - assertEquals(ops.toString(), 1, ops.size()); - ops.forEach(op -> { - assertEquals(CollectionParams.CollectionAction.ADDREPLICA, op.getAction()); - assertEquals(1, op.getHints().size()); - Object o = op.getHints().get(Suggester.Hint.COLL_SHARD); - // this may be a pair or a HashSet of pairs with size 1 - Pair hint = null; - if (o instanceof Pair) { - hint = (Pair)o; - } else if (o instanceof Set) { - assertEquals("unexpected number of hints: " + o, 1, ((Set)o).size()); - o = ((Set)o).iterator().next(); - assertTrue("unexpected hint: " + o, o instanceof Pair); - hint = (Pair)o; - } else { - fail("unexpected hints: " + o); - } - assertNotNull(hint); - assertEquals(collectionName, hint.first()); - assertEquals("shard1", hint.second()); - }); - } - - @Test - public void testFreediskTracking() throws Exception { - int NUM_DOCS = 100000; - String collectionName = "testFreeDisk"; - SolrClient solrClient = cluster.simGetSolrClient(); - CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(collectionName, - "conf",2, 2); - create.process(solrClient); - - CloudUtil.waitForState(cluster, "Timed out waiting for replicas of new collection to be active", - collectionName, CloudUtil.clusterShape(2, 2, false, true)); - ClusterState clusterState = cluster.getClusterStateProvider().getClusterState(); - DocCollection coll = clusterState.getCollection(collectionName); - Set nodes = coll.getReplicas().stream() - .map(r -> r.getNodeName()) - .collect(Collectors.toSet()); - Map initialFreedisk = getFreeDiskPerNode(nodes); - - // test small updates - for (int i = 0; i < NUM_DOCS; i++) { - SolrInputDocument doc = new SolrInputDocument("id", "id-" + i); - solrClient.add(collectionName, doc); - } - Map updatedFreedisk = getFreeDiskPerNode(nodes); - double delta = getDeltaFreeDiskBytes(initialFreedisk, updatedFreedisk); - // 2 replicas - twice as much delta - assertEquals(SimClusterStateProvider.DEFAULT_DOC_SIZE_BYTES * NUM_DOCS * 2, delta, delta * 0.1); - - // test small deletes - delete half of docs - for (int i = 0; i < NUM_DOCS / 2; i++) { - solrClient.deleteById(collectionName, "id-" + i); - } - Map updatedFreedisk1 = getFreeDiskPerNode(nodes); - double delta1 = getDeltaFreeDiskBytes(initialFreedisk, updatedFreedisk1); - // 2 replicas but half the docs - assertEquals(SimClusterStateProvider.DEFAULT_DOC_SIZE_BYTES * NUM_DOCS * 2 / 2, delta1, delta1 * 0.1); - - // test bulk delete - solrClient.deleteByQuery(collectionName, "*:*"); - Map updatedFreedisk2 = getFreeDiskPerNode(nodes); - double delta2 = getDeltaFreeDiskBytes(initialFreedisk, updatedFreedisk2); - // 0 docs - initial freedisk - if (log.isInfoEnabled()) { - log.info(cluster.dumpClusterState(true)); - } - assertEquals(0.0, delta2, delta2 * 0.1); - - // test bulk update - UpdateRequest ureq = new UpdateRequest(); - ureq.setDocIterator(new FakeDocIterator(0, NUM_DOCS)); - ureq.process(solrClient, collectionName); - Map updatedFreedisk3 = getFreeDiskPerNode(nodes); - double delta3 = getDeltaFreeDiskBytes(initialFreedisk, updatedFreedisk3); - assertEquals(SimClusterStateProvider.DEFAULT_DOC_SIZE_BYTES * NUM_DOCS * 2, delta3, delta3 * 0.1); - } - - private double getDeltaFreeDiskBytes(Map initial, Map updated) { - double deltaGB = 0; - for (String node : initial.keySet()) { - double before = initial.get(node).doubleValue(); - double after = updated.get(node).doubleValue(); - assertTrue("freedisk after=" + after + " not smaller than before=" + before, after <= before); - deltaGB += before - after; - } - return deltaGB * 1024.0 * 1024.0 * 1024.0; - } - - private Map getFreeDiskPerNode(Collection nodes) throws Exception { - Map freediskPerNode = new HashMap<>(); - for (String node : nodes) { - Map values = cluster.getNodeStateProvider().getNodeValues(node, Arrays.asList(Variable.Type.FREEDISK.tagName)); - freediskPerNode.put(node, (Number) values.get(Variable.Type.FREEDISK.tagName)); - } - if (log.isInfoEnabled()) { - log.info("- freeDiskPerNode: {}", Utils.toJSONString(freediskPerNode)); - } - return freediskPerNode; - } -} diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimNodeAddedTrigger.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimNodeAddedTrigger.java deleted file mode 100644 index 759715425d6..00000000000 --- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimNodeAddedTrigger.java +++ /dev/null @@ -1,331 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.cloud.autoscaling.sim; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicReference; - -import org.apache.solr.client.solrj.cloud.SolrCloudManager; -import org.apache.solr.cloud.autoscaling.ActionContext; -import org.apache.solr.cloud.autoscaling.AutoScaling; -import org.apache.solr.cloud.autoscaling.NodeAddedTrigger; -import org.apache.solr.cloud.autoscaling.TriggerAction; -import org.apache.solr.cloud.autoscaling.TriggerEvent; -import org.apache.solr.cloud.autoscaling.TriggerValidationException; -import org.apache.solr.common.util.TimeSource; -import org.apache.solr.core.SolrResourceLoader; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -/** - * Test for {@link NodeAddedTrigger} - */ -public class TestSimNodeAddedTrigger extends SimSolrCloudTestCase { - private static AtomicBoolean actionConstructorCalled = new AtomicBoolean(false); - private static AtomicBoolean actionInitCalled = new AtomicBoolean(false); - private static AtomicBoolean actionCloseCalled = new AtomicBoolean(false); - - private AutoScaling.TriggerEventProcessor noFirstRunProcessor = event -> { - fail("Did not expect the listener to fire on first run!"); - return true; - }; - - private static int SPEED = 50; - - // currentTimeMillis is not as precise so to avoid false positives while comparing time of fire, we add some delta - private static final long WAIT_FOR_DELTA_NANOS = TimeUnit.MILLISECONDS.toNanos(2); - - private static TimeSource timeSource; - - @Before - public void beforeTest() throws Exception { - configureCluster(1, TimeSource.get("simTime:" + SPEED)); - timeSource = cluster.getTimeSource(); - - actionConstructorCalled = new AtomicBoolean(false); - actionInitCalled = new AtomicBoolean(false); - actionCloseCalled = new AtomicBoolean(false); - } - - @After - public void afterTest() throws Exception { - shutdownCluster(); - } - - @Test - public void testTrigger() throws Exception { - long waitForSeconds = 1 + random().nextInt(5); - Map props = createTriggerProps(waitForSeconds); - - try (NodeAddedTrigger trigger = new NodeAddedTrigger("node_added_trigger")) { - trigger.configure(cluster.getLoader(), cluster, props); - trigger.init(); - trigger.setProcessor(noFirstRunProcessor); - trigger.run(); - - String newNode1 = cluster.simAddNode(); - String newNode2 = cluster.simAddNode(); - AtomicBoolean fired = new AtomicBoolean(false); - AtomicReference eventRef = new AtomicReference<>(); - trigger.setProcessor(event -> { - if (fired.compareAndSet(false, true)) { - eventRef.set(event); - long currentTimeNanos = timeSource.getTimeNs(); - long eventTimeNanos = event.getEventTime(); - long waitForNanos = TimeUnit.NANOSECONDS.convert(waitForSeconds, TimeUnit.SECONDS) - WAIT_FOR_DELTA_NANOS; - if (currentTimeNanos - eventTimeNanos <= waitForNanos) { - fail("NodeAddedListener was fired before the configured waitFor period: currentTimeNanos=" + currentTimeNanos + ", eventTimeNanos=" + eventTimeNanos + ",waitForNanos=" + waitForNanos); - } - } else { - fail("NodeAddedTrigger was fired more than once!"); - } - return true; - }); - int counter = 0; - do { - trigger.run(); - timeSource.sleep(1000); - if (counter++ > 10) { - fail("Newly added node was not discovered by trigger even after 10 seconds"); - } - } while (!fired.get()); - - TriggerEvent nodeAddedEvent = eventRef.get(); - assertNotNull(nodeAddedEvent); - @SuppressWarnings({"unchecked"}) - List nodeNames = (List)nodeAddedEvent.getProperty(TriggerEvent.NODE_NAMES); - assertTrue(nodeNames.contains(newNode1)); - assertTrue(nodeNames.contains(newNode2)); - } - - // add a new node but remove it before the waitFor period expires - // and assert that the trigger doesn't fire at all - try (NodeAddedTrigger trigger = new NodeAddedTrigger("node_added_trigger")) { - trigger.configure(cluster.getLoader(), cluster, props); - trigger.init(); - final long waitTime = 2; - props.put("waitFor", waitTime); - trigger.setProcessor(noFirstRunProcessor); - trigger.run(); - - String newNode = cluster.simAddNode(); - AtomicBoolean fired = new AtomicBoolean(false); - trigger.setProcessor(event -> { - if (fired.compareAndSet(false, true)) { - long currentTimeNanos = timeSource.getTimeNs(); - long eventTimeNanos = event.getEventTime(); - long waitForNanos = TimeUnit.NANOSECONDS.convert(waitForSeconds, TimeUnit.SECONDS) - WAIT_FOR_DELTA_NANOS; - if (currentTimeNanos - eventTimeNanos <= waitForNanos) { - fail("NodeAddedListener was fired before the configured waitFor period: currentTimeNanos=" + currentTimeNanos + ", eventTimeNanos=" + eventTimeNanos + ",waitForNanos=" + waitForNanos); - } - } else { - fail("NodeAddedTrigger was fired more than once!"); - } - return true; - }); - trigger.run(); // first run should detect the new node - cluster.simRemoveNode(newNode, false); - int counter = 0; - do { - trigger.run(); - timeSource.sleep(1000); - if (counter++ > waitTime + 1) { // run it a little more than the wait time - break; - } - } while (true); - - // ensure the event was not fired - assertFalse(fired.get()); - } - } - - public void testActionLifecycle() throws Exception { - Map props = createTriggerProps(0); - @SuppressWarnings({"unchecked"}) - List> actions = (List>) props.get("actions"); - Map action = new HashMap<>(2); - action.put("name", "testActionInit"); - action.put("class", TestSimNodeAddedTrigger.AssertInitTriggerAction.class.getName()); - actions.add(action); - try (NodeAddedTrigger trigger = new NodeAddedTrigger("node_added_trigger")) { - trigger.configure(cluster.getLoader(), cluster, props); - assertEquals(true, actionConstructorCalled.get()); - assertEquals(false, actionInitCalled.get()); - assertEquals(false, actionCloseCalled.get()); - trigger.init(); - assertEquals(true, actionInitCalled.get()); - assertEquals(false, actionCloseCalled.get()); - } - assertEquals(true, actionCloseCalled.get()); - } - - public static class AssertInitTriggerAction implements TriggerAction { - public AssertInitTriggerAction() { - actionConstructorCalled.set(true); - } - - @Override - public void configure(SolrResourceLoader loader, SolrCloudManager cloudManager, Map properties) throws TriggerValidationException { - - } - - @Override - public void init() { - actionInitCalled.compareAndSet(false, true); - } - - @Override - public String getName() { - return ""; - } - - @Override - public void process(TriggerEvent event, ActionContext actionContext) { - - } - - @Override - public void close() throws IOException { - actionCloseCalled.compareAndSet(false, true); - } - } - - @Test - public void testListenerAcceptance() throws Exception { - Map props = createTriggerProps(0); - try (NodeAddedTrigger trigger = new NodeAddedTrigger("node_added_trigger")) { - trigger.configure(cluster.getLoader(), cluster, props); - trigger.init(); - trigger.setProcessor(noFirstRunProcessor); - trigger.run(); // starts tracking live nodes - - String newNode = cluster.simAddNode(); - AtomicInteger callCount = new AtomicInteger(0); - AtomicBoolean fired = new AtomicBoolean(false); - - trigger.setProcessor(event -> { - if (callCount.incrementAndGet() < 2) { - return false; - } else { - fired.compareAndSet(false, true); - return true; - } - }); - - trigger.run(); // first run should detect the new node and fire immediately but listener isn't ready - assertEquals(1, callCount.get()); - assertFalse(fired.get()); - trigger.run(); // second run should again fire - assertEquals(2, callCount.get()); - assertTrue(fired.get()); - trigger.run(); // should not fire - assertEquals(2, callCount.get()); - } - } - - @Test - public void testRestoreState() throws Exception { - long waitForSeconds = 1 + random().nextInt(5); - Map props = createTriggerProps(waitForSeconds); - - // add a new node but update the trigger before the waitFor period expires - // and assert that the new trigger still fires - NodeAddedTrigger trigger = new NodeAddedTrigger("node_added_trigger"); - trigger.configure(cluster.getLoader(), cluster, props); - trigger.init(); - trigger.setProcessor(noFirstRunProcessor); - trigger.run(); - - String newNode = cluster.simAddNode(); - trigger.run(); // this run should detect the new node - trigger.close(); // close the old trigger - - try (NodeAddedTrigger newTrigger = new NodeAddedTrigger("some_different_name")) { - newTrigger.configure(cluster.getLoader(), cluster, props); - trigger.init(); - try { - newTrigger.restoreState(trigger); - fail("Trigger should only be able to restore state from an old trigger of the same name"); - } catch (AssertionError e) { - // expected - } - } - - try (NodeAddedTrigger newTrigger = new NodeAddedTrigger("node_added_trigger")) { - newTrigger.configure(cluster.getLoader(), cluster, props); - newTrigger.init(); - AtomicBoolean fired = new AtomicBoolean(false); - AtomicReference eventRef = new AtomicReference<>(); - newTrigger.setProcessor(event -> { - if (fired.compareAndSet(false, true)) { - eventRef.set(event); - long currentTimeNanos = timeSource.getTimeNs(); - long eventTimeNanos = event.getEventTime(); - long waitForNanos = TimeUnit.NANOSECONDS.convert(waitForSeconds, TimeUnit.SECONDS) - WAIT_FOR_DELTA_NANOS; - if (currentTimeNanos - eventTimeNanos <= waitForNanos) { - fail("NodeAddedListener was fired before the configured waitFor period: currentTimeNanos=" + currentTimeNanos + ", eventTimeNanos=" + eventTimeNanos + ",waitForNanos=" + waitForNanos); - } - } else { - fail("NodeAddedTrigger was fired more than once!"); - } - return true; - }); - newTrigger.restoreState(trigger); // restore state from the old trigger - int counter = 0; - do { - newTrigger.run(); - timeSource.sleep(1000); - if (counter++ > 10) { - fail("Newly added node was not discovered by trigger even after 10 seconds"); - } - } while (!fired.get()); - - // ensure the event was fired - assertTrue(fired.get()); - TriggerEvent nodeAddedEvent = eventRef.get(); - assertNotNull(nodeAddedEvent); - //TODO assertEquals("", newNode.getNodeName(), nodeAddedEvent.getProperty(NodeAddedTrigger.NodeAddedEvent.NODE_NAME)); - } - } - - private Map createTriggerProps(long waitForSeconds) { - Map props = new HashMap<>(); - props.put("event", "nodeLost"); - props.put("waitFor", waitForSeconds); - props.put("enabled", true); - List> actions = new ArrayList<>(3); - Map map = new HashMap<>(2); - map.put("name", "compute_plan"); - map.put("class", "solr.ComputePlanAction"); - actions.add(map); - map = new HashMap<>(2); - map.put("name", "execute_plan"); - map.put("class", "solr.ExecutePlanAction"); - actions.add(map); - props.put("actions", actions); - return props; - } -} diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimNodeLostTrigger.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimNodeLostTrigger.java deleted file mode 100644 index f1d38aab8e5..00000000000 --- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimNodeLostTrigger.java +++ /dev/null @@ -1,350 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.cloud.autoscaling.sim; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicReference; - -import org.apache.solr.client.solrj.cloud.SolrCloudManager; -import org.apache.solr.cloud.autoscaling.ActionContext; -import org.apache.solr.cloud.autoscaling.AutoScaling; -import org.apache.solr.cloud.autoscaling.NodeLostTrigger; -import org.apache.solr.cloud.autoscaling.TriggerAction; -import org.apache.solr.cloud.autoscaling.TriggerEvent; -import org.apache.solr.cloud.autoscaling.TriggerValidationException; -import org.apache.solr.common.util.TimeSource; -import org.apache.solr.core.SolrResourceLoader; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -/** - * Test for {@link NodeLostTrigger} - */ -public class TestSimNodeLostTrigger extends SimSolrCloudTestCase { - private static AtomicBoolean actionConstructorCalled = new AtomicBoolean(false); - private static AtomicBoolean actionInitCalled = new AtomicBoolean(false); - private static AtomicBoolean actionCloseCalled = new AtomicBoolean(false); - - private AutoScaling.TriggerEventProcessor noFirstRunProcessor = event -> { - fail("Did not expect the listener to fire on first run!"); - return true; - }; - - private static final int SPEED = 50; - // use the same time source as the trigger - private static TimeSource timeSource; - // currentTimeMillis is not as precise so to avoid false positives while comparing time of fire, we add some delta - private static final long WAIT_FOR_DELTA_NANOS = TimeUnit.MILLISECONDS.toNanos(5); - - @Before - public void beforeTest() throws Exception { - configureCluster(5, TimeSource.get("simTime:" + SPEED)); - timeSource = cluster.getTimeSource(); - actionConstructorCalled = new AtomicBoolean(false); - actionInitCalled = new AtomicBoolean(false); - actionCloseCalled = new AtomicBoolean(false); - } - - @After - public void afterTest() throws Exception { - shutdownCluster(); - } - - @Test - public void testTrigger() throws Exception { - long waitForSeconds = 1 + random().nextInt(5); - Map props = createTriggerProps(waitForSeconds); - - try (NodeLostTrigger trigger = new NodeLostTrigger("node_lost_trigger")) { - trigger.configure(cluster.getLoader(), cluster, props); - trigger.setProcessor(noFirstRunProcessor); - trigger.run(); - Iterator it = cluster.getLiveNodesSet().get().iterator(); - String lostNodeName1 = it.next(); - String lostNodeName2 = it.next(); - cluster.simRemoveNode(lostNodeName1, false); - cluster.simRemoveNode(lostNodeName2, false); - timeSource.sleep(1000); - - AtomicBoolean fired = new AtomicBoolean(false); - AtomicReference eventRef = new AtomicReference<>(); - trigger.setProcessor(event -> { - if (fired.compareAndSet(false, true)) { - eventRef.set(event); - long currentTimeNanos = timeSource.getTimeNs(); - long eventTimeNanos = event.getEventTime(); - long waitForNanos = TimeUnit.NANOSECONDS.convert(waitForSeconds, TimeUnit.SECONDS) - WAIT_FOR_DELTA_NANOS; - if (currentTimeNanos - eventTimeNanos <= waitForNanos) { - fail("NodeLostListener was fired before the configured waitFor period: currentTimeNanos=" + currentTimeNanos + ", eventTimeNanos=" + eventTimeNanos + ",waitForNanos=" + waitForNanos); - } - } else { - fail("NodeLostListener was fired more than once!"); - } - return true; - }); - int counter = 0; - do { - trigger.run(); - timeSource.sleep(1000); - if (counter++ > 20) { - fail("Lost node was not discovered by trigger even after 10 seconds"); - } - } while (!fired.get()); - - TriggerEvent nodeLostEvent = eventRef.get(); - assertNotNull(nodeLostEvent); - @SuppressWarnings({"unchecked"}) - List nodeNames = (List)nodeLostEvent.getProperty(TriggerEvent.NODE_NAMES); - assertTrue(nodeNames + " doesn't contain " + lostNodeName1, nodeNames.contains(lostNodeName1)); - assertTrue(nodeNames + " doesn't contain " + lostNodeName2, nodeNames.contains(lostNodeName2)); - - } - - // remove a node but add it back before the waitFor period expires - // and assert that the trigger doesn't fire at all - try (NodeLostTrigger trigger = new NodeLostTrigger("node_lost_trigger")) { - trigger.configure(cluster.getLoader(), cluster, props); - final long waitTime = 2; - props.put("waitFor", waitTime); - trigger.setProcessor(noFirstRunProcessor); - trigger.run(); - - String lostNode = cluster.getSimClusterStateProvider().simGetRandomNode(); - cluster.simRemoveNode(lostNode, false); - AtomicBoolean fired = new AtomicBoolean(false); - trigger.setProcessor(event -> { - if (fired.compareAndSet(false, true)) { - long currentTimeNanos = timeSource.getTimeNs(); - long eventTimeNanos = event.getEventTime(); - long waitForNanos = TimeUnit.NANOSECONDS.convert(waitTime, TimeUnit.SECONDS) - WAIT_FOR_DELTA_NANOS; - if (currentTimeNanos - eventTimeNanos <= waitForNanos) { - fail("NodeLostListener was fired before the configured waitFor period: currentTimeNanos=" + currentTimeNanos + ", eventTimeNanos=" + eventTimeNanos + ",waitForNanos=" + waitForNanos); - } - } else { - fail("NodeLostListener was fired more than once!"); - } - return true; - }); - trigger.run(); // first run should detect the lost node - int counter = 0; - do { - if (cluster.getLiveNodesSet().get().size() == 2) { - break; - } - timeSource.sleep(100); - if (counter++ > 20) { - fail("Live nodes not updated!"); - } - } while (true); - counter = 0; - cluster.getSimClusterStateProvider().simRestoreNode(lostNode); - do { - trigger.run(); - timeSource.sleep(1000); - if (counter++ > waitTime + 1) { // run it a little more than the wait time - break; - } - } while (true); - - // ensure the event was not fired - assertFalse(fired.get()); - } - } - - public void testActionLifecycle() throws Exception { - Map props = createTriggerProps(0); - @SuppressWarnings({"unchecked"}) - List> actions = (List>) props.get("actions"); - Map action = new HashMap<>(2); - action.put("name", "testActionInit"); - action.put("class", AssertInitTriggerAction.class.getName()); - actions.add(action); - try (NodeLostTrigger trigger = new NodeLostTrigger("node_added_trigger")) { - trigger.configure(cluster.getLoader(), cluster, props); - assertEquals(true, actionConstructorCalled.get()); - assertEquals(false, actionInitCalled.get()); - assertEquals(false, actionCloseCalled.get()); - trigger.init(); - assertEquals(true, actionInitCalled.get()); - assertEquals(false, actionCloseCalled.get()); - } - assertEquals(true, actionCloseCalled.get()); - } - - public static class AssertInitTriggerAction implements TriggerAction { - public AssertInitTriggerAction() { - actionConstructorCalled.set(true); - } - - @Override - public void configure(SolrResourceLoader loader, SolrCloudManager cloudManager, Map properties) throws TriggerValidationException { - - } - - @Override - public void init() { - actionInitCalled.compareAndSet(false, true); - } - - @Override - public String getName() { - return ""; - } - - @Override - public void process(TriggerEvent event, ActionContext actionContext) { - - } - - @Override - public void close() throws IOException { - actionCloseCalled.compareAndSet(false, true); - } - } - - @Test - public void testListenerAcceptance() throws Exception { - Map props = createTriggerProps(0); - try (NodeLostTrigger trigger = new NodeLostTrigger("node_added_trigger")) { - trigger.configure(cluster.getLoader(), cluster, props); - trigger.setProcessor(noFirstRunProcessor); - - String newNode = cluster.simAddNode(); - - trigger.run(); // starts tracking live nodes - - // stop the newly created node - cluster.simRemoveNode(newNode, false); - - AtomicInteger callCount = new AtomicInteger(0); - AtomicBoolean fired = new AtomicBoolean(false); - - trigger.setProcessor(event -> { - if (callCount.incrementAndGet() < 2) { - return false; - } else { - fired.compareAndSet(false, true); - return true; - } - }); - - trigger.run(); // first run should detect the lost node and fire immediately but listener isn't ready - assertEquals(1, callCount.get()); - assertFalse(fired.get()); - trigger.run(); // second run should again fire - assertEquals(2, callCount.get()); - assertTrue(fired.get()); - trigger.run(); // should not fire - assertEquals(2, callCount.get()); - } - } - - @Test - public void testRestoreState() throws Exception { - long waitForSeconds = 1 + random().nextInt(5); - Map props = createTriggerProps(waitForSeconds); - - String newNode = cluster.simAddNode(); - - // remove a node but update the trigger before the waitFor period expires - // and assert that the new trigger still fires - - NodeLostTrigger trigger = new NodeLostTrigger("node_lost_trigger"); - trigger.configure(cluster.getLoader(), cluster, props); - trigger.setProcessor(noFirstRunProcessor); - trigger.run(); - - // stop the newly created node - cluster.simRemoveNode(newNode, false); - - trigger.run(); // this run should detect the lost node - trigger.close(); // close the old trigger - - try (NodeLostTrigger newTrigger = new NodeLostTrigger("some_different_name")) { - newTrigger.configure(cluster.getLoader(), cluster, props); - try { - newTrigger.restoreState(trigger); - fail("Trigger should only be able to restore state from an old trigger of the same name"); - } catch (AssertionError e) { - // expected - } - } - - try (NodeLostTrigger newTrigger = new NodeLostTrigger("node_lost_trigger")) { - newTrigger.configure(cluster.getLoader(), cluster, props); - AtomicBoolean fired = new AtomicBoolean(false); - AtomicReference eventRef = new AtomicReference<>(); - newTrigger.setProcessor(event -> { - if (fired.compareAndSet(false, true)) { - eventRef.set(event); - long currentTimeNanos = timeSource.getTimeNs(); - long eventTimeNanos = event.getEventTime(); - long waitForNanos = TimeUnit.NANOSECONDS.convert(waitForSeconds, TimeUnit.SECONDS) - WAIT_FOR_DELTA_NANOS; - if (currentTimeNanos - eventTimeNanos <= waitForNanos) { - fail("NodeLostListener was fired before the configured waitFor period: currentTimeNanos=" + currentTimeNanos + ", eventTimeNanos=" + eventTimeNanos + ",waitForNanos=" + waitForNanos); - } - } else { - fail("NodeLostListener was fired more than once!"); - } - return true; - }); - newTrigger.restoreState(trigger); // restore state from the old trigger - int counter = 0; - do { - newTrigger.run(); - timeSource.sleep(1000); - if (counter++ > 10) { - fail("Lost node was not discovered by trigger even after 10 seconds"); - } - } while (!fired.get()); - - TriggerEvent nodeLostEvent = eventRef.get(); - assertNotNull(nodeLostEvent); - @SuppressWarnings({"unchecked"}) - List nodeNames = (List)nodeLostEvent.getProperty(TriggerEvent.NODE_NAMES); - assertTrue(nodeNames.contains(newNode)); - } - } - - private Map createTriggerProps(long waitForSeconds) { - Map props = new HashMap<>(); - props.put("event", "nodeLost"); - props.put("waitFor", waitForSeconds); - props.put("enabled", true); - List> actions = new ArrayList<>(3); - Map map = new HashMap<>(2); - map.put("name", "compute_plan"); - map.put("class", "solr.ComputePlanAction"); - actions.add(map); - map = new HashMap<>(2); - map.put("name", "execute_plan"); - map.put("class", "solr.ExecutePlanAction"); - actions.add(map); - props.put("actions", actions); - return props; - } -} diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimPolicyCloud.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimPolicyCloud.java deleted file mode 100644 index 011ef3ae8a1..00000000000 --- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimPolicyCloud.java +++ /dev/null @@ -1,379 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.cloud.autoscaling.sim; - -import java.io.IOException; -import java.lang.invoke.MethodHandles; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.BiConsumer; - -import org.apache.lucene.util.Constants; -import org.apache.solr.client.solrj.SolrClient; -import org.apache.solr.client.solrj.SolrRequest; -import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig; -import org.apache.solr.client.solrj.cloud.autoscaling.Policy; -import org.apache.solr.client.solrj.cloud.autoscaling.Row; -import org.apache.solr.client.solrj.cloud.autoscaling.Variable.Type; -import org.apache.solr.client.solrj.request.CollectionAdminRequest; -import org.apache.solr.cloud.CloudTestUtils.AutoScalingRequest; -import org.apache.solr.cloud.CloudUtil; -import org.apache.solr.common.cloud.DocCollection; -import org.apache.solr.common.cloud.Replica; -import org.apache.solr.common.cloud.ZkStateReader; -import org.apache.solr.common.cloud.rule.ImplicitSnitch; -import org.apache.solr.common.util.NamedList; -import org.apache.solr.common.util.TimeSource; -import org.apache.solr.common.util.Utils; -import org.apache.solr.util.LogLevel; -import org.apache.zookeeper.KeeperException; -import org.junit.After; -import org.junit.Before; -import org.junit.rules.ExpectedException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -@LogLevel("org.apache.solr.cloud.autoscaling=DEBUG") -public class TestSimPolicyCloud extends SimSolrCloudTestCase { - - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - @org.junit.Rule - public ExpectedException expectedException = ExpectedException.none(); - - @Before - public void setupCluster() throws Exception { - configureCluster(5, TimeSource.get("simTime:50")); - // reset autoscaling policy to empty - String commands = "{set-cluster-policy : []}"; - cluster.simGetSolrClient().request(AutoScalingRequest.create(SolrRequest.METHOD.POST, commands)); - } - - @After - public void afterTest() throws Exception { - shutdownCluster(); - } - - public void testDataProviderPerReplicaDetails() throws Exception { - SolrClient solrClient = cluster.simGetSolrClient(); - CollectionAdminRequest.createCollection("perReplicaDataColl", "conf", 1, 5) - .process(solrClient); - - CloudUtil.waitForState(cluster, "Timeout waiting for collection to become active", "perReplicaDataColl", - CloudUtil.clusterShape(1, 5, false, true)); - DocCollection coll = getCollectionState("perReplicaDataColl"); - String autoScaleJson = "{" + - " 'cluster-preferences': [" + - " { maximize : freedisk , precision: 50}," + - " { minimize : cores, precision: 2}" + - " ]," + - " 'cluster-policy': [" + - " { replica : '0' , 'nodeRole': 'overseer'}," + - " { 'replica': '<2', 'shard': '#ANY', 'node': '#ANY'" + - " }" + - " ]," + - " 'policies': {" + - " 'policy1': [" + - " { 'replica': '<2', 'shard': '#EACH', 'node': '#ANY'}," + - " { 'replica': '<2', 'shard': '#EACH', 'sysprop.rack': 'rack1'}" + - " ]" + - " }" + - "}"; - @SuppressWarnings({"unchecked"}) - AutoScalingConfig config = new AutoScalingConfig((Map) Utils.fromJSONString(autoScaleJson)); - Policy.Session session = config.getPolicy().createSession(cluster); - - AtomicInteger count = new AtomicInteger(0); - for (Row row : session.getSortedNodes()) { - row.collectionVsShardVsReplicas.forEach((c, shardVsReplicas) -> shardVsReplicas.forEach((s, replicaInfos) -> { - for (Replica replicaInfo : replicaInfos) { - if (replicaInfo.getProperties().containsKey(Type.CORE_IDX.tagName)) count.incrementAndGet(); - } - })); - } - assertTrue(count.get() > 0); - - CollectionAdminRequest.deleteCollection("perReplicaDataColl").process(solrClient); - - } - - // commented out on: 17-Feb-2019 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // annotated on: 24-Dec-2018 - public void testCreateCollectionAddReplica() throws Exception { - SolrClient solrClient = cluster.simGetSolrClient(); - String nodeId = cluster.getSimClusterStateProvider().simGetRandomNode(); - - int port = (Integer)cluster.getSimNodeStateProvider().simGetNodeValue(nodeId, ImplicitSnitch.PORT); - - String commands = "{set-policy :{c1 : [{replica:0 , shard:'#EACH', port: '!" + port + "'}]}}"; - solrClient.request(AutoScalingRequest.create(SolrRequest.METHOD.POST, commands)); - - String collectionName = "testCreateCollectionAddReplica"; - CollectionAdminRequest.createCollection(collectionName, "conf", 1, 1) - .setPolicy("c1") - .process(solrClient); - CloudUtil.waitForState(cluster, collectionName, 120, TimeUnit.SECONDS, - CloudUtil.clusterShape(1, 1, false, true)); - - getCollectionState(collectionName).forEachReplica((s, replica) -> assertEquals(nodeId, replica.getNodeName())); - - CollectionAdminRequest.addReplicaToShard(collectionName, "shard1").process(solrClient); - CloudUtil.waitForState(cluster, - collectionName, 120l, TimeUnit.SECONDS, - (liveNodes, collectionState) -> collectionState.getReplicas().size() == 2); - - getCollectionState(collectionName).forEachReplica((s, replica) -> assertEquals(nodeId, replica.getNodeName())); - } - - public void testCreateCollectionSplitShard() throws Exception { - SolrClient solrClient = cluster.simGetSolrClient(); - String firstNode = cluster.getSimClusterStateProvider().simGetRandomNode(); - int firstNodePort = (Integer)cluster.getSimNodeStateProvider().simGetNodeValue(firstNode, ImplicitSnitch.PORT); - - String secondNode; - int secondNodePort; - while (true) { - secondNode = cluster.getSimClusterStateProvider().simGetRandomNode(); - secondNodePort = (Integer)cluster.getSimNodeStateProvider().simGetNodeValue(secondNode, ImplicitSnitch.PORT); - if (secondNodePort != firstNodePort) break; - } - - String commands = "{set-policy :{c1 : [{replica:1 , shard:'#EACH', port: '" + firstNodePort + "'}, {replica:1, shard:'#EACH', port:'" + secondNodePort + "'}]}}"; - NamedList response = solrClient.request(AutoScalingRequest.create(SolrRequest.METHOD.POST, commands)); - assertEquals("success", response.get("result")); - - String collectionName = "testCreateCollectionSplitShard"; - CollectionAdminRequest.createCollection(collectionName, "conf", 1, 2) - .setPolicy("c1") - .process(solrClient); - CloudUtil.waitForState(cluster, "Timeout waiting for collection to become active", collectionName, - CloudUtil.clusterShape(1, 2, false, true)); - - DocCollection docCollection = getCollectionState(collectionName); - List list = docCollection.getReplicas(firstNode); - int replicasOnNode1 = list != null ? list.size() : 0; - list = docCollection.getReplicas(secondNode); - int replicasOnNode2 = list != null ? list.size() : 0; - - assertEquals("Expected exactly one replica of collection on node with port: " + firstNodePort, 1, replicasOnNode1); - assertEquals("Expected exactly one replica of collection on node with port: " + secondNodePort, 1, replicasOnNode2); - - CollectionAdminRequest.splitShard(collectionName).setShardName("shard1").process(solrClient); - - CloudUtil.waitForState(cluster, "Timed out waiting to see 6 replicas for collection: " + collectionName, - collectionName, (liveNodes, collectionState) -> collectionState.getReplicas().size() == 6); - - docCollection = getCollectionState(collectionName); - list = docCollection.getReplicas(firstNode); - replicasOnNode1 = list != null ? list.size() : 0; - list = docCollection.getReplicas(secondNode); - replicasOnNode2 = list != null ? list.size() : 0; - - assertEquals("Expected exactly three replica of collection on node with port: " + firstNodePort, 3, replicasOnNode1); - assertEquals("Expected exactly three replica of collection on node with port: " + secondNodePort, 3, replicasOnNode2); - CollectionAdminRequest.deleteCollection(collectionName).process(solrClient); - - } - - public void testMetricsTag() throws Exception { - SolrClient solrClient = cluster.simGetSolrClient(); - String setClusterPolicyCommand = "{" + - " 'set-cluster-policy': [" + - " {'cores':'<10', 'node':'#ANY'}," + - " {'replica':'<2', 'shard': '#EACH', 'node': '#ANY'}," + - " {'metrics:abc':'overseer', 'replica':0}" + - " ]" + - "}"; - @SuppressWarnings({"rawtypes"}) - SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setClusterPolicyCommand); - try { - solrClient.request(req); - fail("expected exception"); - } catch (Exception e) { - // expected - assertTrue(e.toString().contains("Invalid metrics: param in")); - } - setClusterPolicyCommand = "{" + - " 'set-cluster-policy': [" + - " {'cores':'<10', 'node':'#ANY'}," + - " {'replica':'<2', 'shard': '#EACH', 'node': '#ANY'}," + - " {'metrics:solr.node:ADMIN./admin/authorization.clientErrors:count':'>58768765', 'replica':0}" + - " ]" + - "}"; - req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setClusterPolicyCommand); - solrClient.request(req); - - //org.eclipse.jetty.server.handler.DefaultHandler.2xx-responses - CollectionAdminRequest.createCollection("metricsTest", "conf", 1, 1) - .process(solrClient); - CloudUtil.waitForState(cluster, "Timeout waiting for collection to become active", "metricsTest", - CloudUtil.clusterShape(1, 1)); - - DocCollection collection = getCollectionState("metricsTest"); - List tags = Arrays.asList("metrics:solr.node:ADMIN./admin/authorization.clientErrors:count", - "metrics:solr.jvm:buffers.direct.Count"); - Map val = cluster.getNodeStateProvider().getNodeValues(collection.getReplicas().get(0).getNodeName(), tags); - for (String tag : tags) { - assertNotNull( "missing : "+ tag , val.get(tag)); - } - - - } - - public void testCreateCollectionAddShardWithReplicaTypeUsingPolicy() throws Exception { - SolrClient solrClient = cluster.simGetSolrClient(); - List nodes = new ArrayList<>(cluster.getClusterStateProvider().getLiveNodes()); - String nrtNodeName = nodes.get(0); - int nrtPort = (Integer)cluster.getSimNodeStateProvider().simGetNodeValue(nrtNodeName, ImplicitSnitch.PORT); - - - String pullNodeName = nodes.get(1); - int pullPort = (Integer)cluster.getSimNodeStateProvider().simGetNodeValue(pullNodeName, ImplicitSnitch.PORT); - - String tlogNodeName = nodes.get(2); - int tlogPort = (Integer)cluster.getSimNodeStateProvider().simGetNodeValue(tlogNodeName, ImplicitSnitch.PORT); - log.info("NRT {} PULL {} , TLOG {} ", nrtNodeName, pullNodeName, tlogNodeName); - - String commands = "{set-cluster-policy :[" + - "{replica:0 , shard:'#EACH', type: NRT, port: '!" + nrtPort + "'}" + - "{replica:0 , shard:'#EACH', type: PULL, port: '!" + pullPort + "'}" + - "{replica:0 , shard:'#EACH', type: TLOG, port: '!" + tlogPort + "'}" + - "]}"; - - - solrClient.request(AutoScalingRequest.create(SolrRequest.METHOD.POST, commands)); - Map json = Utils.getJson(cluster.getDistribStateManager(), ZkStateReader.SOLR_AUTOSCALING_CONF_PATH); - assertEquals("full json:" + Utils.toJSONString(json), "!" + nrtPort, - Utils.getObjectByPath(json, true, "cluster-policy[0]/port")); - assertEquals("full json:" + Utils.toJSONString(json), "!" + pullPort, - Utils.getObjectByPath(json, true, "cluster-policy[1]/port")); - assertEquals("full json:" + Utils.toJSONString(json), "!" + tlogPort, - Utils.getObjectByPath(json, true, "cluster-policy[2]/port")); - - CollectionAdminRequest.createCollectionWithImplicitRouter("policiesTest", "conf", "s1", 1, 1, 1) - .process(solrClient); - CloudUtil.waitForState(cluster, "Timeout waiting for collection to become active", "policiesTest", - CloudUtil.clusterShape(1, 3, false, true)); - - DocCollection coll = getCollectionState("policiesTest"); - - - BiConsumer verifyReplicas = (s, replica) -> { - switch (replica.getType()) { - case NRT: { - assertTrue("NRT replica should be in " + nrtNodeName, replica.getNodeName().equals(nrtNodeName)); - break; - } - case TLOG: { - assertTrue("TLOG replica should be in " + tlogNodeName, replica.getNodeName().equals(tlogNodeName)); - break; - } - case PULL: { - assertTrue("PULL replica should be in " + pullNodeName, replica.getNodeName().equals(pullNodeName)); - break; - } - } - - }; - coll.forEachReplica(verifyReplicas); - - CollectionAdminRequest.createShard("policiesTest", "s3"). - process(solrClient); - coll = getCollectionState("policiesTest"); - assertEquals(3, coll.getSlice("s3").getReplicas().size()); - coll.forEachReplica(verifyReplicas); - } - - public void testCreateCollectionAddShardUsingPolicy() throws Exception { - SolrClient solrClient = cluster.simGetSolrClient(); - String nodeId = cluster.getSimClusterStateProvider().simGetRandomNode(); - int port = (Integer)cluster.getSimNodeStateProvider().simGetNodeValue(nodeId, ImplicitSnitch.PORT); - - String commands = "{set-policy :{c1 : [{replica:1 , shard:'#EACH', port: '" + port + "'}]}}"; - solrClient.request(AutoScalingRequest.create(SolrRequest.METHOD.POST, commands)); - Map json = Utils.getJson(cluster.getDistribStateManager(), ZkStateReader.SOLR_AUTOSCALING_CONF_PATH); - assertEquals("full json:"+ Utils.toJSONString(json) , "#EACH", - Utils.getObjectByPath(json, true, "/policies/c1[0]/shard")); - CollectionAdminRequest.createCollectionWithImplicitRouter("policiesTest", "conf", "s1,s2", 1) - .setPolicy("c1") - .process(solrClient); - CloudUtil.waitForState(cluster, "Timeout waiting for collection to become active", "policiesTest", - CloudUtil.clusterShape(2, 1)); - - DocCollection coll = getCollectionState("policiesTest"); - assertEquals("c1", coll.getPolicyName()); - assertEquals(2,coll.getReplicas().size()); - coll.forEachReplica((s, replica) -> assertEquals(nodeId, replica.getNodeName())); - CollectionAdminRequest.createShard("policiesTest", "s3").process(solrClient); - CloudUtil.waitForState(cluster, "Timeout waiting for collection to become active", "policiesTest", - CloudUtil.clusterShape(3, 1)); - - coll = getCollectionState("policiesTest"); - assertEquals(1, coll.getSlice("s3").getReplicas().size()); - coll.getSlice("s3").forEach(replica -> assertEquals(nodeId, replica.getNodeName())); - } - - public void testDataProvider() throws IOException, SolrServerException, KeeperException, InterruptedException { - SolrClient solrClient = cluster.simGetSolrClient(); - CollectionAdminRequest.createCollectionWithImplicitRouter("policiesTest", "conf", "shard1", 2) - .process(solrClient); - CloudUtil.waitForState(cluster, "Timeout waiting for collection to become active", "policiesTest", - CloudUtil.clusterShape(1, 2, false, true)); - DocCollection rulesCollection = getCollectionState("policiesTest"); - - Map val = cluster.getNodeStateProvider().getNodeValues(rulesCollection.getReplicas().get(0).getNodeName(), Arrays.asList( - "freedisk", - "cores", - "heapUsage", - "sysLoadAvg")); - assertNotNull(val.get("freedisk")); - assertNotNull(val.get("heapUsage")); - assertNotNull(val.get("sysLoadAvg")); - assertTrue(((Number) val.get("cores")).intValue() > 0); - assertTrue("freedisk value is " + ((Number) val.get("freedisk")).doubleValue(), Double.compare(((Number) val.get("freedisk")).doubleValue(), 0.0d) > 0); - assertTrue("heapUsage value is " + ((Number) val.get("heapUsage")).doubleValue(), Double.compare(((Number) val.get("heapUsage")).doubleValue(), 0.0d) > 0); - if (!Constants.WINDOWS) { - // the system load average metrics is not available on windows platform - assertTrue("sysLoadAvg value is " + ((Number) val.get("sysLoadAvg")).doubleValue(), Double.compare(((Number) val.get("sysLoadAvg")).doubleValue(), 0.0d) > 0); - } - // simulator doesn't have Overseer, so just pick a random node - String overseerNode = cluster.getSimClusterStateProvider().simGetRandomNode(); - solrClient.request(CollectionAdminRequest.addRole(overseerNode, "overseer")); - for (int i = 0; i < 10; i++) { - Map data = Utils.getJson(cluster.getDistribStateManager(), ZkStateReader.ROLES); - if (i >= 9 && data.isEmpty()) { - throw new RuntimeException("NO overseer node created"); - } - cluster.getTimeSource().sleep(100); - } - val = cluster.getNodeStateProvider().getNodeValues(overseerNode, Arrays.asList( - "nodeRole", - "ip_1", "ip_2", "ip_3", "ip_4", - "sysprop.java.version", - "sysprop.java.vendor")); - assertEquals("overseer", val.get("nodeRole")); - assertNotNull(val.get("ip_1")); - assertNotNull(val.get("ip_2")); - assertNotNull(val.get("ip_3")); - assertNotNull(val.get("ip_4")); - assertNotNull(val.get("sysprop.java.version")); - assertNotNull(val.get("sysprop.java.vendor")); - } -} diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimScenario.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimScenario.java deleted file mode 100644 index 38e921aac4d..00000000000 --- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimScenario.java +++ /dev/null @@ -1,173 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.cloud.autoscaling.sim; - -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.PrintStream; -import java.nio.charset.Charset; -import java.util.List; -import java.util.Map; -import java.util.concurrent.TimeUnit; - -import org.apache.solr.client.solrj.cloud.autoscaling.Suggester; -import org.apache.solr.cloud.CloudUtil; -import org.apache.solr.common.util.Utils; -import org.apache.solr.util.LogLevel; -import org.junit.Test; - -/** - * - */ -@LogLevel("org.apache.solr.cloud.autoscaling=DEBUG") -public class TestSimScenario extends SimSolrCloudTestCase { - - // simple scenario to test .autoAddReplicas trigger - String autoAddReplicasScenario = - "# standard comment\n" + - "// java comment\n" + - "create_cluster numNodes=2 // inline comment\n" + - "load_autoscaling json={'cluster-policy'+:+[{'replica'+:+'<3',+'shard'+:+'#EACH',+'collection'+:+'testCollection','node':'#ANY'}]}&defaultWaitFor=10\n" + - "solr_request /admin/collections?action=CREATE&autoAddReplicas=true&name=testCollection&numShards=2&replicationFactor=2\n" + - "wait_collection collection=testCollection&shards=2&replicas=2\n" + - "event_listener trigger=.auto_add_replicas&stage=SUCCEEDED\n" + - "kill_nodes node=${_random_node_}\n" + - "wait_event trigger=.auto_add_replicas&wait=60\n" + - "wait_collection collection=testCollection&shards=2&replicas=2\n" + - "save_snapshot path=${snapshotPath}\n"; - - @Test - public void testAutoAddReplicas() throws Exception { - String snapshotPath = createTempDir() + "/snapshot"; - try (SimScenario scenario = SimScenario.load(autoAddReplicasScenario)) { - scenario.context.put("snapshotPath", snapshotPath); - scenario.run(); - } - SnapshotCloudManager snapshotCloudManager = SnapshotCloudManager.readSnapshot(new File(snapshotPath)); - CloudUtil.waitForState(snapshotCloudManager, "testCollection", 1, TimeUnit.SECONDS, - CloudUtil.clusterShape(2, 2)); - } - - String testSuggestionsScenario = - "create_cluster numNodes=2\n" + - "load_autoscaling json={'cluster-policy':[]}\n" + - "solr_request /admin/collections?action=CREATE&autoAddReplicas=true&name=testCollection&numShards=2&replicationFactor=2\n" + - "wait_collection collection=testCollection&shards=2&replicas=2\n" + - "ctx_set key=myNode&value=${_random_node_}\n" + - "solr_request /admin/collections?action=ADDREPLICA&collection=testCollection&shard=shard1&node=${myNode}\n" + - "solr_request /admin/collections?action=ADDREPLICA&collection=testCollection&shard=shard1&node=${myNode}\n" + - "loop_start iterations=${iterative}\n" + - " calculate_suggestions\n" + - " apply_suggestions\n" + - " solr_request /admin/collections?action=ADDREPLICA&collection=testCollection&shard=shard1&node=${myNode}\n" + - " solr_request /admin/collections?action=ADDREPLICA&collection=testCollection&shard=shard1&node=${myNode}\n" + - "loop_end\n" + - "loop_start iterations=${justCalc}\n" + - " calculate_suggestions\n" + - " save_snapshot path=${snapshotPath}/${_loop_iter_}\n" + - "loop_end\n" + - "dump redact=true"; - - @Test - public void testSuggestions() throws Exception { - String snapshotPath = createTempDir() + "/snapshot"; - try (SimScenario scenario = SimScenario.load(testSuggestionsScenario)) { - scenario.context.put("snapshotPath", snapshotPath); - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - PrintStream ps = new PrintStream(baos, true, Charset.forName("UTF-8")); - scenario.console = ps; - scenario.context.put("iterative", "0"); - scenario.context.put("justCalc", "1"); - scenario.run(); - @SuppressWarnings({"unchecked"}) - List suggestions = (List)scenario.context.get(SimScenario.SUGGESTIONS_CTX_PROP); - assertNotNull(suggestions); - assertEquals(suggestions.toString(), 1, suggestions.size()); - // reconstruct the snapshot from the dump - @SuppressWarnings({"unchecked"}) - Map snapshot = (Map)Utils.fromJSON(baos.toByteArray()); - @SuppressWarnings({"unchecked"}) - Map autoscalingState = (Map)snapshot.get(SnapshotCloudManager.AUTOSCALING_STATE_KEY); - assertNotNull(autoscalingState); - assertEquals(autoscalingState.toString(), 1, autoscalingState.size()); - assertTrue(autoscalingState.toString(), autoscalingState.containsKey("suggestions")); - @SuppressWarnings({"unchecked"}) - List> snapSuggestions = (List>)autoscalingState.get("suggestions"); - assertEquals(snapSuggestions.toString(), 1, snapSuggestions.size()); - // _loop_iter_ should be present and 0 (first iteration) - assertEquals("0", scenario.context.get(SimScenario.LOOP_ITER_PROP)); - } - // try looping more times - try (SimScenario scenario = SimScenario.load(testSuggestionsScenario)) { - scenario.context.put("iterative", "10"); - scenario.context.put("justCalc", "0"); - scenario.run(); - assertEquals("9", scenario.context.get(SimScenario.LOOP_ITER_PROP)); - } - - } - - String indexingScenario = - "create_cluster numNodes=100\n" + - "load_autoscaling json={'cluster-policy':[]}\n" + - "solr_request /admin/collections?action=CREATE&autoAddReplicas=true&name=testCollection&numShards=2&replicationFactor=2\n" + - "wait_collection collection=testCollection&shards=2&replicas=2\n" + - "solr_request /admin/autoscaling?httpMethod=POST&stream.body=" + - "{'set-trigger':{'name':'indexSizeTrigger','event':'indexSize','waitFor':'10s','aboveDocs':1000,'enabled':true,"+ - "'actions':[{'name':'compute_plan','class':'solr.ComputePlanAction'},{'name':'execute_plan','class':'solr.ExecutePlanAction'}]}}\n" + - "event_listener trigger=indexSizeTrigger&stage=SUCCEEDED\n" + - "index_docs collection=testCollection&numDocs=3000\n" + - "run\n" + - "wait_event trigger=indexSizeTrigger&wait=60\n" + - "assert condition=not_null&key=_trigger_event_indexSizeTrigger\n" + - "assert condition=equals&key=_trigger_event_indexSizeTrigger/eventType&expected=INDEXSIZE\n" + - "assert condition=equals&key=_trigger_event_indexSizeTrigger/properties/requestedOps[0]/action&expected=SPLITSHARD\n" + - "wait_collection collection=testCollection&shards=6&withInactive=true&requireLeaders=false&replicas=2"; - - @Test - public void testIndexing() throws Exception { - try (SimScenario scenario = SimScenario.load(indexingScenario)) { - scenario.run(); - } - } - - String splitShardScenario = - "create_cluster numNodes=2\n" + - "load_autoscaling json={'cluster-policy':[]}\n" + - "solr_request /admin/collections?action=CREATE&name=testCollection&numShards=2&replicationFactor=2\n" + - "wait_collection collection=testCollection&shards=2&replicas=2\n" + - "set_shard_metrics collection=testCollection&shard=shard1&INDEX.sizeInBytes=1000000000\n" + - "set_node_metrics nodeset=#ANY&freedisk=1.5\n" + - "solr_request /admin/collection?action=SPLITSHARD&collection=testCollection&shard=shard1&splitMethod=${method}\n" + - "wait_collection collection=testCollection&shards=4&&withInactive=true&replicas=2&requireLeaders=true\n" - ; - @Test - public void testSplitShard() throws Exception { - try (SimScenario scenario = SimScenario.load(splitShardScenario)) { - scenario.context.put("method", "REWRITE"); - scenario.run(); - } catch (Exception e) { - assertTrue(e.toString(), e.toString().contains("not enough free disk")); - } - try (SimScenario scenario = SimScenario.load(splitShardScenario)) { - scenario.context.put("method", "LINK"); - scenario.run(); - } catch (Exception e) { - fail("should have succeeded with method LINK, but failed: " + e.toString()); - } - } -} diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimTriggerIntegration.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimTriggerIntegration.java deleted file mode 100644 index c7b6153d6cc..00000000000 --- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimTriggerIntegration.java +++ /dev/null @@ -1,1551 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.cloud.autoscaling.sim; - -import java.io.IOException; -import java.lang.invoke.MethodHandles; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.SortedSet; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.locks.ReentrantLock; - -import org.apache.solr.client.solrj.SolrClient; -import org.apache.solr.client.solrj.cloud.SolrCloudManager; -import org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig; -import org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventProcessorStage; -import org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventType; -import org.apache.solr.client.solrj.request.CollectionAdminRequest; -import org.apache.solr.cloud.CloudTestUtils; -import org.apache.solr.cloud.CloudUtil; -import org.apache.solr.cloud.autoscaling.ActionContext; -import org.apache.solr.cloud.autoscaling.AutoScaling; -import org.apache.solr.cloud.autoscaling.CapturedEvent; -import org.apache.solr.cloud.autoscaling.ComputePlanAction; -import org.apache.solr.cloud.autoscaling.ExecutePlanAction; -import org.apache.solr.cloud.autoscaling.NodeAddedTrigger; -import org.apache.solr.cloud.autoscaling.NodeLostTrigger; -import org.apache.solr.cloud.autoscaling.ScheduledTriggers; -import org.apache.solr.cloud.autoscaling.SearchRateTrigger; -import org.apache.solr.cloud.autoscaling.TriggerActionBase; -import org.apache.solr.cloud.autoscaling.TriggerBase; -import org.apache.solr.cloud.autoscaling.TriggerEvent; -import org.apache.solr.cloud.autoscaling.TriggerEventQueue; -import org.apache.solr.cloud.autoscaling.TriggerListenerBase; -import org.apache.solr.cloud.autoscaling.TriggerValidationException; -import org.apache.solr.common.MapWriter; -import org.apache.solr.common.cloud.LiveNodesListener; -import org.apache.solr.common.cloud.Replica; -import org.apache.solr.common.cloud.ZkStateReader; -import org.apache.solr.common.util.TimeSource; -import org.apache.solr.common.util.Utils; -import org.apache.solr.core.SolrResourceLoader; -import org.apache.solr.util.LogLevel; -import org.apache.solr.util.TimeOut; -import org.apache.zookeeper.KeeperException; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.google.common.util.concurrent.AtomicDouble; - -import static org.apache.solr.cloud.autoscaling.OverseerTriggerThread.MARKER_ACTIVE; -import static org.apache.solr.cloud.autoscaling.OverseerTriggerThread.MARKER_INACTIVE; -import static org.apache.solr.cloud.autoscaling.OverseerTriggerThread.MARKER_STATE; - -/** - * An end-to-end integration test for triggers - */ -@LogLevel("org.apache.solr.cloud.autoscaling=DEBUG") -public class TestSimTriggerIntegration extends SimSolrCloudTestCase { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - public static final int SPEED = 50; - - private static volatile CountDownLatch actionConstructorCalled; - private static volatile CountDownLatch actionInitCalled; - private static volatile CountDownLatch triggerFiredLatch; - private static volatile int waitForSeconds = 1; - private static volatile CountDownLatch actionStarted; - private static volatile CountDownLatch actionInterrupted; - private static volatile CountDownLatch actionCompleted; - private static volatile CountDownLatch triggerStartedLatch; - private static volatile CountDownLatch triggerFinishedLatch; - private static volatile AtomicInteger triggerStartedCount; - private static volatile AtomicInteger triggerFinishedCount; - private static volatile AtomicBoolean triggerFired; - private static Set events = ConcurrentHashMap.newKeySet(); - - private static final long WAIT_FOR_DELTA_NANOS = TimeUnit.MILLISECONDS.toNanos(5); - - - @After - public void afterTest() throws Exception { - shutdownCluster(); - } - - private static CountDownLatch getTriggerFiredLatch() { - return triggerFiredLatch; - } - - private static CountDownLatch getActionStarted() { - return actionStarted; - } - - private static CountDownLatch getActionInterrupted() { - return actionInterrupted; - } - - private static CountDownLatch getActionCompleted() { - return actionCompleted; - } - - @Before - public void setupTest() throws Exception { - configureCluster(2, TimeSource.get("simTime:" + SPEED)); - - // disable .scheduled_maintenance (once it exists) - CloudTestUtils.waitForTriggerToBeScheduled(cluster, ".scheduled_maintenance"); - CloudTestUtils.suspendTrigger(cluster, ".scheduled_maintenance"); - - waitForSeconds = 1 + random().nextInt(3); - actionConstructorCalled = new CountDownLatch(1); - actionInitCalled = new CountDownLatch(1); - triggerFiredLatch = new CountDownLatch(1); - triggerFired = new AtomicBoolean(false); - actionStarted = new CountDownLatch(1); - actionInterrupted = new CountDownLatch(1); - actionCompleted = new CountDownLatch(1); - triggerStartedLatch = new CountDownLatch(1); - triggerFinishedLatch = new CountDownLatch(1); - triggerStartedCount = new AtomicInteger(); - triggerFinishedCount = new AtomicInteger(); - events.clear(); - listenerEvents.clear(); - allListenerEvents.clear(); - failDummyAction = false; - listenerCreated = new CountDownLatch(1); - listenerEventLatch = new CountDownLatch(0); - } - - @Test - public void testTriggerThrottling() throws Exception { - // for this test we want to create two triggers so we must assert that the actions were created twice - actionInitCalled = new CountDownLatch(2); - // similarly we want both triggers to fire - triggerFiredLatch = new CountDownLatch(2); - - SolrClient solrClient = cluster.simGetSolrClient(); - - // first trigger - assertAutoScalingRequest - ("{" + - "'set-trigger' : {" + - "'name' : 'node_added_trigger1'," + - "'event' : 'nodeAdded'," + - "'waitFor' : '0s'," + - "'enabled' : true," + - "'actions' : [{'name':'test','class':'" + ThrottlingTesterAction.class.getName() + "'}]" + - "}}"); - - assertAutoScalingRequest - ("{" + - "'set-trigger' : {" + - "'name' : 'node_added_trigger2'," + - "'event' : 'nodeAdded'," + - "'waitFor' : '0s'," + - "'enabled' : true," + - "'actions' : [{'name':'test','class':'" + ThrottlingTesterAction.class.getName() + "'}]" + - "}}"); - - assertAutoscalingUpdateComplete(); - - // wait until the two instances of action are created - assertTrue("Two TriggerAction instances were not created "+ - "even after await()ing an excessive amount of time", - actionInitCalled.await(60, TimeUnit.SECONDS)); - - String newNode = cluster.simAddNode(); - - assertTrue("Both triggers did not fire event after await()ing an excessive amount of time", - triggerFiredLatch.await(60, TimeUnit.SECONDS)); - - // reset shared state - lastActionExecutedAt.set(0); - actionInitCalled = new CountDownLatch(2); - triggerFiredLatch = new CountDownLatch(2); - - assertAutoScalingRequest - ("{" + - "'set-trigger' : {" + - "'name' : 'node_lost_trigger1'," + - "'event' : 'nodeLost'," + - "'waitFor' : '0s'," + - "'enabled' : true," + - "'actions' : [{'name':'test','class':'" + ThrottlingTesterAction.class.getName() + "'}]" + - "}}"); - - assertAutoScalingRequest - ("{" + - "'set-trigger' : {" + - "'name' : 'node_lost_trigger2'," + - "'event' : 'nodeLost'," + - "'waitFor' : '0s'," + - "'enabled' : true," + - "'actions' : [{'name':'test','class':'" + ThrottlingTesterAction.class.getName() + "'}]" + - "}}"); - - assertAutoscalingUpdateComplete(); - - // wait until the two instances of action are created - assertTrue("Two TriggerAction instances were not created "+ - "even after await()ing an excessive amount of time", - actionInitCalled.await(60, TimeUnit.SECONDS)); - - // stop the node we had started earlier - cluster.simRemoveNode(newNode, false); - - // AwaitsFix - maybe related to leaders not always getting elected in sim -// if (!triggerFiredLatch.await(34000 / SPEED, TimeUnit.MILLISECONDS)) { -// fail("Both triggers should have fired by now"); -// } - } - - static AtomicLong lastActionExecutedAt = new AtomicLong(0); - static ReentrantLock lock = new ReentrantLock(); - public static class ThrottlingTesterAction extends TestTriggerAction { - // nanos are very precise so we need a delta for comparison with ms - private static final long DELTA_MS = 2; - - // sanity check that an action instance is only invoked once - private final AtomicBoolean onlyOnce = new AtomicBoolean(false); - - @Override - public void process(TriggerEvent event, ActionContext actionContext) { - boolean locked = lock.tryLock(); - if (!locked) { - log.info("We should never have a tryLock fail because actions are never supposed to be executed concurrently"); - return; - } - try { - if (lastActionExecutedAt.get() != 0) { - if (log.isInfoEnabled()) { - log.info("last action at {} time = {}", lastActionExecutedAt.get(), cluster.getTimeSource().getTimeNs()); - } - if (TimeUnit.NANOSECONDS.toMillis(cluster.getTimeSource().getTimeNs() - lastActionExecutedAt.get()) < - TimeUnit.SECONDS.toMillis(ScheduledTriggers.DEFAULT_ACTION_THROTTLE_PERIOD_SECONDS) - DELTA_MS) { - if (log.isInfoEnabled()) { - log.info("action executed again before minimum wait time from {}", event.getSource()); - } - fail("TriggerListener was fired before the throttling period"); - } - } - if (onlyOnce.compareAndSet(false, true)) { - if (log.isInfoEnabled()) { - log.info("action executed from {}", event.getSource()); - } - lastActionExecutedAt.set(cluster.getTimeSource().getTimeNs()); - getTriggerFiredLatch().countDown(); - } else { - if (log.isInfoEnabled()) { - log.info("action executed more than once from {}", event.getSource()); - } - fail("Trigger should not have fired more than once!"); - } - } finally { - lock.unlock(); - } - } - } - - @Test - @SuppressWarnings({"unchecked"}) - public void testNodeLostTriggerRestoreState() throws Exception { - - final String triggerName = "node_lost_restore_trigger"; - - // should be enough to ensure trigger doesn't fire any actions until we replace the trigger - waitForSeconds = 500000; - assertAutoScalingRequest - ("{" + - "'set-trigger' : {" + - "'name' : '"+triggerName+"'," + - "'event' : 'nodeLost'," + - "'waitFor' : '"+waitForSeconds+"s'," + - "'enabled' : true," + - "'actions' : [{'name':'test','class':'" + TestTriggerAction.class.getName() + "'}]" + - "}}"); - - assertTrue("Trigger was not init()ed even after await()ing an excessive amount of time", - actionInitCalled.await(60, TimeUnit.SECONDS)); - - assertAutoscalingUpdateComplete(); - - // start a new node that we can kill later - final String nodeName = cluster.simAddNode(); - - // poll the internal state of the trigger until it run()s at least once and updates - // it's internal state to know the node we added is live - // - // (this should run roughly once a second of simulated time) - (new TimeOut(30, TimeUnit.SECONDS, cluster.getTimeSource())) - .waitFor("initial trigger never ran to detect new live node", () -> - (((Collection) getTriggerState(triggerName).get("lastLiveNodes")) - .contains(nodeName))); - - // kill our node - cluster.simRemoveNode(nodeName, false); - - // poll the internal state of the trigger until it run()s at least once (more) and updates - // it's internal state to know the node we killed is no longer alive - // - // (this should run roughly once a second of simulated time) - (new TimeOut(30, TimeUnit.SECONDS, cluster.getTimeSource())) - .waitFor("initial trigger never ran to detect lost node", () -> - ! (((Collection) getTriggerState(triggerName).get("lastLiveNodes")) - .contains(nodeName))); - (new TimeOut(30, TimeUnit.SECONDS, cluster.getTimeSource())) - .waitFor("initial trigger never ran to detect lost node", () -> - (((Map) getTriggerState(triggerName).get("nodeNameVsTimeRemoved")) - .containsKey(nodeName))); - - Map nodeNameVsTimeRemoved = (Map) getTriggerState(triggerName).get("nodeNameVsTimeRemoved"); - - // since we know the nodeLost event has been detected, we can recored the current timestamp - // (relative to the cluster's time source) and later assert that (restored state) correctly - // tracked that the event happened prior to "now" - final long maxEventTimeNs = cluster.getTimeSource().getTimeNs(); - - // even though our trigger has detected a lost node, the *action* we registered should not have - // been run yet, due to the large waitFor configuration... - assertEquals("initial trigger action should not have fired", false, triggerFired.get()); - assertEquals("initial trigger action latch should not have counted down", - 1, triggerFiredLatch.getCount()); - assertEquals("initial trigger action should not have recorded any events: " + events.toString(), - 0, events.size()); - - // - // now replace the trigger with a new instance to test that the state gets copied over correctly - // - - // reset the actionInitCalled counter so we can confirm the second instances is inited - actionInitCalled = new CountDownLatch(1); - // use a low waitTime to ensure it processes the event quickly. - // (this updated property also ensures the set-trigger won't be treated as a No-Op) - waitForSeconds = 0 + random().nextInt(3); - assertAutoScalingRequest - ("{" + - "'set-trigger' : {" + - "'name' : '"+triggerName+"'," + - "'event' : 'nodeLost'," + - "'waitFor' : '"+waitForSeconds+"s'," + - "'enabled' : true," + - "'actions' : [{'name':'test','class':'" + TestTriggerAction.class.getName() + "'}]" + - "}}"); - - assertTrue("Trigger was not init()ed even after await()ing an excessive amount of time", - actionInitCalled.await(60, TimeUnit.SECONDS)); - - // the trigger actions should now (eventually) record that the node is lost - assertTrue("Second instance of our trigger never fired the action to process the event", - triggerFiredLatch.await(30, TimeUnit.SECONDS)); - - final TriggerEvent event = assertSingleEvent(nodeName, maxEventTimeNs); - assertTrue("Event should have been a nodeLost event: " + event.getClass(), - event instanceof NodeLostTrigger.NodeLostEvent); - - // assert that the time nodes were removed was actually restored - NodeLostTrigger.NodeLostEvent nodeLostEvent = (NodeLostTrigger.NodeLostEvent) event; - List removedNodeNames = (List) nodeLostEvent.getProperty(TriggerEvent.NODE_NAMES); - List removedNodeTimes = (List) nodeLostEvent.getProperty(TriggerEvent.EVENT_TIMES); - - assertFalse("Empty removedNodeNames", removedNodeNames.isEmpty()); - assertEquals("Size of removedNodeNames and removedNodeTimes does not match", - removedNodeNames.size(), removedNodeTimes.size()); - for (int i = 0; i < removedNodeNames.size(); i++) { - String nn = removedNodeNames.get(i); - Long nt = removedNodeTimes.get(i); - assertEquals(nodeNameVsTimeRemoved.get(nn), nt); - } - } - - @Test - @SuppressWarnings({"unchecked"}) - public void testNodeAddedTriggerRestoreState() throws Exception { - - final String triggerName = "node_added_restore_trigger"; - - // should be enough to ensure trigger doesn't fire any actions until we replace the trigger - waitForSeconds = 500000; - assertAutoScalingRequest - ("{" + - "'set-trigger' : {" + - "'name' : '"+triggerName+"'," + - "'event' : 'nodeAdded'," + - "'waitFor' : '"+waitForSeconds+"s'," + - "'enabled' : true," + - "'actions' : [{'name':'test','class':'" + TestTriggerAction.class.getName() + "'}]" + - "}}"); - - assertAutoscalingUpdateComplete(); - - assertTrue("Trigger was not init()ed even after await()ing an excessive amount of time", - actionInitCalled.await(60, TimeUnit.SECONDS)); - - // start a new node - final String nodeName = cluster.simAddNode(); - - // poll the internal state of the trigger until it run()s at least once and updates - // it's internal state to know the node we added is live - // - // (this should run roughly once a second of simulated time) - (new TimeOut(30, TimeUnit.SECONDS, cluster.getTimeSource())) - .waitFor("initial trigger never ran to detect new live node", () -> - (((Collection) getTriggerState(triggerName).get("lastLiveNodes")) - .contains(nodeName))); - (new TimeOut(30, TimeUnit.SECONDS, cluster.getTimeSource())) - .waitFor("initial trigger never ran to detect new live node", () -> - (((Map) getTriggerState(triggerName).get("nodeNameVsTimeAdded")) - .containsKey(nodeName))); - - Map nodeNameVsTimeAdded = (Map) getTriggerState(triggerName).get("nodeNameVsTimeAdded"); - - - // since we know the nodeAddded event has been detected, we can recored the current timestamp - // (relative to the cluster's time source) and later assert that (restored state) correctly - // tracked that the event happened prior to "now" - final long maxEventTimeNs = cluster.getTimeSource().getTimeNs(); - - // even though our trigger has detected an added node, the *action* we registered should not have - // been run yet, due to the large waitFor configuration... - assertEquals("initial trigger action should not have fired", false, triggerFired.get()); - assertEquals("initial trigger action latch should not have counted down", - 1, triggerFiredLatch.getCount()); - assertEquals("initial trigger action should not have recorded any events: " + events.toString(), - 0, events.size()); - - // - // now replace the trigger with a new instance to test that the state gets copied over correctly - // - - // reset the actionInitCalled counter so we can confirm the second instances is inited - actionInitCalled = new CountDownLatch(1); - // use a low waitTime to ensure it processes the event quickly. - // (this updated property also ensures the set-trigger won't be treated as a No-Op) - waitForSeconds = 0 + random().nextInt(3); - assertAutoScalingRequest - ("{" + - "'set-trigger' : {" + - "'name' : '"+triggerName+"'," + - "'event' : 'nodeAdded'," + - "'waitFor' : '"+waitForSeconds+"s'," + - "'enabled' : true," + - "'actions' : [{'name':'test','class':'" + TestTriggerAction.class.getName() + "'}]" + - "}}"); - - assertAutoscalingUpdateComplete(); - - assertTrue("Trigger was not init()ed even after await()ing an excessive amount of time", - actionInitCalled.await(60, TimeUnit.SECONDS)); - - // the trigger actions should now (eventually) record that the new node is added - assertTrue("Second instance of our trigger never fired the action to process the event", - triggerFiredLatch.await(30, TimeUnit.SECONDS)); - - final TriggerEvent event = assertSingleEvent(nodeName, maxEventTimeNs); - assertTrue("Event should have been a nodeAdded event: " + event.getClass(), - event instanceof NodeAddedTrigger.NodeAddedEvent); - - // assert that the time nodes were added was actually restored - NodeAddedTrigger.NodeAddedEvent nodeAddedEvent = (NodeAddedTrigger.NodeAddedEvent) event; - List addedNodeNames = (List) nodeAddedEvent.getProperty(TriggerEvent.NODE_NAMES); - List addedNodeTimes = (List) nodeAddedEvent.getProperty(TriggerEvent.EVENT_TIMES); - - assertFalse("Empty addedNodeNames", addedNodeNames.isEmpty()); - assertEquals("Size of addedNodeNames and addedNodeTimes does not match", - addedNodeNames.size(), addedNodeTimes.size()); - for (int i = 0; i < addedNodeNames.size(); i++) { - String nn = addedNodeNames.get(i); - Long nt = addedNodeTimes.get(i); - assertEquals(nodeNameVsTimeAdded.get(nn), nt); - } - } - - @Test - public void testNodeAddedTrigger() throws Exception { - SolrClient solrClient = cluster.simGetSolrClient(); - assertAutoScalingRequest - ("{" + - "'set-trigger' : {" + - "'name' : 'node_added_trigger'," + - "'event' : 'nodeAdded'," + - "'waitFor' : '" + waitForSeconds + "s'," + - "'enabled' : true," + - "'actions' : [{'name':'test','class':'" + TestTriggerAction.class.getName() + "'}]" + - "}}"); - - assertAutoscalingUpdateComplete(); - - // wait until the two instances of action are created - assertTrue("TriggerAction was not created even after await()ing an excessive amount of time", - actionInitCalled.await(60, TimeUnit.SECONDS)); - - String newNode = cluster.simAddNode(); - - assertTrue("trigger did not fire event after await()ing an excessive amount of time", - triggerFiredLatch.await(60, TimeUnit.SECONDS)); - assertTrue(triggerFired.get()); - TriggerEvent nodeAddedEvent = events.iterator().next(); - assertNotNull(nodeAddedEvent); - @SuppressWarnings({"unchecked"}) - List nodeNames = (List)nodeAddedEvent.getProperty(TriggerEvent.NODE_NAMES); - assertTrue(nodeAddedEvent.toString(), nodeNames.contains(newNode)); - - // reset - actionConstructorCalled = new CountDownLatch(1); - actionInitCalled = new CountDownLatch(1); - - // update the trigger with exactly the same data - assertAutoScalingRequest - ("{" + - "'set-trigger' : {" + - "'name' : 'node_added_trigger'," + - "'event' : 'nodeAdded'," + - "'waitFor' : '" + waitForSeconds + "s'," + - "'enabled' : true," + - "'actions' : [{'name':'test','class':'" + TestTriggerAction.class.getName() + "'}]" + - "}}"); - - assertAutoscalingUpdateComplete(); - - // this should be a no-op so the action should have been created but init should not be called - assertTrue("TriggerAction was not created even after await()ing an excessive amount of time", - actionConstructorCalled.await(60, TimeUnit.SECONDS)); - // HACK: we are waiting a *short* amount of time and asserting that the init action was *not* called - assertFalse("init should not have been called on TriggerAction since update was No-Op", - actionInitCalled.await(3, TimeUnit.SECONDS)); - } - - @Test - public void testNodeLostTrigger() throws Exception { - SolrClient solrClient = cluster.simGetSolrClient(); - assertAutoScalingRequest - ("{" + - "'set-trigger' : {" + - "'name' : 'node_lost_trigger'," + - "'event' : 'nodeLost'," + - "'waitFor' : '" + waitForSeconds + "s'," + - "'enabled' : true," + - "'actions' : [{'name':'test','class':'" + TestTriggerAction.class.getName() + "'}]" + - "}}"); - - assertAutoscalingUpdateComplete(); - - assertTrue("Trigger was not init()ed even after await()ing an excessive amount of time", - actionInitCalled.await(60, TimeUnit.SECONDS)); - - String lostNodeName = cluster.getSimClusterStateProvider().simGetRandomNode(); - cluster.simRemoveNode(lostNodeName, false); - assertTrue("trigger did not fire event after await()ing an excessive amount of time", - triggerFiredLatch.await(60, TimeUnit.SECONDS)); - assertTrue(triggerFired.get()); - TriggerEvent nodeLostEvent = events.iterator().next(); - assertNotNull(nodeLostEvent); - @SuppressWarnings({"unchecked"}) - List nodeNames = (List)nodeLostEvent.getProperty(TriggerEvent.NODE_NAMES); - assertTrue(nodeNames.contains(lostNodeName)); - - // reset - actionConstructorCalled = new CountDownLatch(1); - actionInitCalled = new CountDownLatch(1); - - // update the trigger with exactly the same data - assertAutoScalingRequest - ("{" + - "'set-trigger' : {" + - "'name' : 'node_lost_trigger'," + - "'event' : 'nodeLost'," + - "'waitFor' : '" + waitForSeconds + "s'," + - "'enabled' : true," + - "'actions' : [{'name':'test','class':'" + TestTriggerAction.class.getName() + "'}]" + - "}}"); - - assertAutoscalingUpdateComplete(); - - // this should be a no-op so the action should have been created but init should not be called - assertTrue("TriggerAction was not created even after await()ing an excessive amount of time", - actionConstructorCalled.await(60, TimeUnit.SECONDS)); - // HACK: we are waiting a *short* amount of time and asserting that the init action was *not* called - assertFalse("init should not have been called on TriggerAction since update was No-Op", - actionInitCalled.await(3, TimeUnit.SECONDS)); - } - - // simulator doesn't support overseer functionality yet - /* - @Test - public void testContinueTriggersOnOverseerRestart() throws Exception { - CollectionAdminRequest.OverseerStatus status = new CollectionAdminRequest.OverseerStatus(); - CloudSolrClient solrClient = cluster.getSolrClient(); - CollectionAdminResponse adminResponse = status.process(solrClient); - NamedList response = adminResponse.getResponse(); - String leader = (String) response.get("leader"); - JettySolrRunner overseerNode = null; - int index = -1; - List jettySolrRunners = cluster.getJettySolrRunners(); - for (int i = 0; i < jettySolrRunners.size(); i++) { - JettySolrRunner runner = jettySolrRunners.get(i); - if (runner.getNodeName().equals(leader)) { - overseerNode = runner; - index = i; - break; - } - } - assertNotNull(overseerNode); - - String setTriggerCommand = "{" + - "'set-trigger' : {" + - "'name' : 'node_added_trigger'," + - "'event' : 'nodeAdded'," + - "'waitFor' : '" + waitForSeconds + "s'," + - "'enabled' : true," + - "'actions' : [{'name':'test','class':'" + TestTriggerAction.class.getName() + "'}]" + - "}}"; - SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand); - response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - if (!actionInitCalled.await(3, TimeUnit.SECONDS)) { - fail("The TriggerAction should have been created by now"); - } - - // stop the overseer, somebody else will take over as the overseer - cluster.stopJettySolrRunner(index); - Thread.sleep(10000); - JettySolrRunner newNode = cluster.startJettySolrRunner(); - boolean await = triggerFiredLatch.await(20, TimeUnit.SECONDS); - assertTrue("The trigger did not fire at all", await); - assertTrue(triggerFired.get()); - NodeAddedTrigger.NodeAddedEvent nodeAddedEvent = (NodeAddedTrigger.NodeAddedEvent) events.iterator().next(); - assertNotNull(nodeAddedEvent); - List nodeNames = (List)nodeAddedEvent.getProperty(TriggerEvent.NODE_NAMES); - assertTrue(nodeNames.contains(newNode.getNodeName())); - } - -*/ - - public static class TestTriggerAction extends TriggerActionBase { - - public TestTriggerAction() { - actionConstructorCalled.countDown(); - } - - @Override - public void process(TriggerEvent event, ActionContext actionContext) { - try { - if (triggerFired.compareAndSet(false, true)) { - events.add(event); - long currentTimeNanos = cluster.getTimeSource().getTimeNs(); - long eventTimeNanos = event.getEventTime(); - long waitForNanos = TimeUnit.NANOSECONDS.convert(waitForSeconds, TimeUnit.SECONDS) - WAIT_FOR_DELTA_NANOS; - if (currentTimeNanos - eventTimeNanos <= waitForNanos) { - fail(event.getSource() + " was fired before the configured waitFor period"); - } - getTriggerFiredLatch().countDown(); - } else { - fail(event.getSource() + " was fired more than once!"); - } - } catch (Throwable t) { - log.debug("--throwable", t); - throw t; - } - } - - @Override - public void init() throws Exception { - log.info("TestTriggerAction init"); - super.init(); - actionInitCalled.countDown(); - } - - } - - public static class TestEventQueueAction extends TriggerActionBase { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - public static volatile CountDownLatch stall = new CountDownLatch(0); - public TestEventQueueAction() { - log.info("TestEventQueueAction instantiated"); - } - - @Override - public void process(TriggerEvent event, ActionContext actionContext) { - // make a local copy of the latch so we're using it consistently even as test thread changes tings - final CountDownLatch stallLatch = stall; - log.info("processing: stall={} event={} ", stallLatch, event); - events.add(event); - getActionStarted().countDown(); - try { - if (stallLatch.await(60, TimeUnit.SECONDS)) { - log.info("Firing trigger event after await()ing 'stall' countdown"); - triggerFired.set(true); - } else { - log.error("Timed out await()ing 'stall' countdown"); - } - getActionCompleted().countDown(); - } catch (InterruptedException e) { - log.info("Interrupted"); - getActionInterrupted().countDown(); - } - } - - @Override - public void init() throws Exception { - log.info("TestEventQueueAction init"); - actionInitCalled.countDown(); - super.init(); - } - } - - @Test - public void testEventQueue() throws Exception { - waitForSeconds = 1; - SolrClient solrClient = cluster.simGetSolrClient(); - String overseerLeader = cluster.getSimClusterStateProvider().simGetRandomNode(); - - assertAutoScalingRequest - ("{" + - "'set-trigger' : {" + - "'name' : 'node_added_trigger1'," + - "'event' : 'nodeAdded'," + - "'waitFor' : '" + waitForSeconds + "s'," + - "'enabled' : true," + - "'actions' : [{'name':'test','class':'" + TestEventQueueAction.class.getName() + "'}]" + - "}}"); - - assertAutoscalingUpdateComplete(); - - assertTrue("Trigger was not init()ed even after await()ing an excessive amount of time", - actionInitCalled.await(60, TimeUnit.SECONDS)); - - // setup the trigger action to stall so we can test interupting it w/overseer change - // NOTE: we will never release this latch, instead we expect the interupt on overseer shutdown - TestEventQueueAction.stall = new CountDownLatch(1); - - // add node to generate the event - final String newNode = cluster.simAddNode(); - assertTrue("Action did not start even after await()ing an excessive amount of time", - actionStarted.await(60, TimeUnit.SECONDS)); - - // event should be there - final TriggerEvent nodeAddedEvent = events.iterator().next(); - assertNotNull(nodeAddedEvent); - assertNotNull(nodeAddedEvent.getId()); - assertNotNull(nodeAddedEvent.getEventType()); - assertNotNull(nodeAddedEvent.getProperty(TriggerEventQueue.ENQUEUE_TIME)); - - // but action did not complete yet (due to stall) so the event is still enqueued - assertFalse(triggerFired.get()); - - // we know the event action has started, so we can re-set state for the next instance - // that will run after the overseer change - events.clear(); - actionStarted = new CountDownLatch(1); - TestEventQueueAction.stall = new CountDownLatch(0); // so replay won't wait - - // kill overseer - cluster.simRestartOverseer(overseerLeader); - cluster.getTimeSource().sleep(5000); - - assertAutoscalingUpdateComplete(); - - // new overseer leader should be elected and run triggers - assertTrue("Action was not interupted even after await()ing an excessive amount of time", - actionInterrupted.await(60, TimeUnit.SECONDS)); - // it should fire again from enqueued event - assertTrue("Action did not (re-)start even after await()ing an excessive amount of time", - actionStarted.await(60, TimeUnit.SECONDS)); - - final TriggerEvent replayedEvent = events.iterator().next(); - assertNotNull(replayedEvent); - - assertTrue("Action did not complete even after await()ing an excessive amount of time", - actionCompleted.await(60, TimeUnit.SECONDS)); - assertTrue(triggerFired.get()); - - assertEquals(nodeAddedEvent.getId(), replayedEvent.getId()); - assertEquals(nodeAddedEvent.getEventTime(), replayedEvent.getEventTime()); - assertEquals(nodeAddedEvent.getEventType(), replayedEvent.getEventType()); - assertEquals(nodeAddedEvent.getProperty(TriggerEventQueue.ENQUEUE_TIME), - replayedEvent.getProperty(TriggerEventQueue.ENQUEUE_TIME)); - assertEquals(Boolean.TRUE, replayedEvent.getProperty(TriggerEvent.REPLAYING)); - - } - - @Test - public void testEventFromRestoredState() throws Exception { - SolrClient solrClient = cluster.simGetSolrClient(); - assertAutoScalingRequest - ("{" + - "'set-trigger' : {" + - "'name' : 'node_added_trigger'," + - "'event' : 'nodeAdded'," + - "'waitFor' : '10s'," + - "'enabled' : true," + - "'actions' : [{'name':'test','class':'" + TestTriggerAction.class.getName() + "'}]" + - "}}"); - - assertAutoscalingUpdateComplete(); - - assertTrue("Trigger was not init()ed even after await()ing an excessive amount of time", - actionInitCalled.await(60, TimeUnit.SECONDS)); - - events.clear(); - - String newNode = cluster.simAddNode(); - assertTrue("trigger did not fire event after await()ing an excessive amount of time", - triggerFiredLatch.await(60, TimeUnit.SECONDS)); - assertTrue(triggerFired.get()); - // reset - triggerFired.set(false); - triggerFiredLatch = new CountDownLatch(1); - TriggerEvent nodeAddedEvent = events.iterator().next(); - assertNotNull(nodeAddedEvent); - @SuppressWarnings({"unchecked"}) - List nodeNames = (List)nodeAddedEvent.getProperty(TriggerEvent.NODE_NAMES); - assertTrue(nodeNames.contains(newNode)); - // add a second node - state of the trigger will change but it won't fire for waitFor sec. - String newNode2 = cluster.simAddNode(); - cluster.getTimeSource().sleep(10000); - // kill overseer - cluster.simRestartOverseer(null); - assertTrue("trigger did not fire event after await()ing an excessive amount of time", - triggerFiredLatch.await(60, TimeUnit.SECONDS)); - assertTrue(triggerFired.get()); - } - - private static class TestLiveNodesListener implements LiveNodesListener { - Set lostNodes = new HashSet<>(); - Set addedNodes = new HashSet<>(); - CountDownLatch onChangeLatch = new CountDownLatch(1); - - public void reset() { - lostNodes.clear(); - addedNodes.clear(); - onChangeLatch = new CountDownLatch(1); - } - - @Override - public boolean onChange(SortedSet oldLiveNodes, SortedSet newLiveNodes) { - onChangeLatch.countDown(); - Set old = new HashSet<>(oldLiveNodes); - old.removeAll(newLiveNodes); - if (!old.isEmpty()) { - lostNodes.addAll(old); - } - newLiveNodes.removeAll(oldLiveNodes); - if (!newLiveNodes.isEmpty()) { - addedNodes.addAll(newLiveNodes); - } - return false; - } - } - - private TestLiveNodesListener registerLiveNodesListener() { - TestLiveNodesListener listener = new TestLiveNodesListener(); - cluster.getLiveNodesSet().registerLiveNodesListener(listener); - return listener; - } - - public static class TestEventMarkerAction extends TriggerActionBase { - - @Override - public void process(TriggerEvent event, ActionContext actionContext) { - boolean locked = lock.tryLock(); - if (!locked) { - log.info("We should never have a tryLock fail because actions are never supposed to be executed concurrently"); - return; - } - try { - events.add(event); - getTriggerFiredLatch().countDown(); - } catch (Throwable t) { - log.debug("--throwable", t); - throw t; - } finally { - lock.unlock(); - } - } - - @Override - public void init() throws Exception { - log.info("TestEventMarkerAction init"); - super.init(); - } - } - - public static class AssertingListener extends TriggerListenerBase { - @Override - public void onEvent(TriggerEvent event, TriggerEventProcessorStage stage, String actionName, ActionContext context, Throwable error, String message) throws Exception { - if (!Thread.currentThread().getName().startsWith("ScheduledTrigger")) { - // for future safety - throw new IllegalThreadStateException("AssertingListener should have been invoked by a thread from the scheduled trigger thread pool"); - } - log.debug(" --- listener fired for event: {}, stage: {}", event, stage); - listenerEventLatch.await(); - log.debug(" --- listener wait complete for event: {}, stage: {}", event, stage); - } - } - - @Test - public void testNodeMarkersRegistration() throws Exception { - triggerFiredLatch = new CountDownLatch(1); - listenerEventLatch = new CountDownLatch(1); - TestLiveNodesListener listener = registerLiveNodesListener(); - - SolrClient solrClient = cluster.simGetSolrClient(); - - // get overseer node - String overseerLeader = cluster.getSimClusterStateProvider().simGetOverseerLeader(); - - // add a node - String node = cluster.simAddNode(); - assertTrue("cluster onChange listener didn't execute even after await()ing an excessive amount of time", - listener.onChangeLatch.await(60, TimeUnit.SECONDS)); - assertEquals(1, listener.addedNodes.size()); - assertTrue(listener.addedNodes.toString(), listener.addedNodes.contains(node)); - // verify that a znode doesn't exist (no trigger) - String pathAdded = ZkStateReader.SOLR_AUTOSCALING_NODE_ADDED_PATH + "/" + node; - assertFalse("Path " + pathAdded + " was created but there are no nodeAdded triggers", - cluster.getDistribStateManager().hasData(pathAdded)); - listener.reset(); - // stop overseer - log.info("====== KILL OVERSEER 1"); - cluster.simRestartOverseer(overseerLeader); - assertAutoscalingUpdateComplete(); - - assertTrue("cluster onChange listener didn't execute even after await()ing an excessive amount of time", - listener.onChangeLatch.await(60, TimeUnit.SECONDS)); - - assertEquals(1, listener.lostNodes.size()); - assertEquals(overseerLeader, listener.lostNodes.iterator().next()); - assertEquals(0, listener.addedNodes.size()); - // wait until the new overseer is up - cluster.getTimeSource().sleep(5000); - - String pathLost = ZkStateReader.SOLR_AUTOSCALING_NODE_LOST_PATH + "/" + overseerLeader; - - TimeOut timeout = new TimeOut(30, TimeUnit.SECONDS, TimeSource.NANO_TIME); - AtomicBoolean markerInactive = new AtomicBoolean(); - timeout.waitFor("nodeLost marker to get inactive", () -> { - try { - if (!cluster.getDistribStateManager().hasData(pathLost)) { - throw new RuntimeException("marker " + pathLost + " should exist!"); - } - Map markerData = Utils.getJson(cluster.getDistribStateManager(), pathLost); - markerInactive.set(markerData.getOrDefault(MARKER_STATE, MARKER_ACTIVE).equals(MARKER_INACTIVE)); - return markerInactive.get(); - - } catch (IOException | KeeperException | InterruptedException e) { - e.printStackTrace(); - throw new RuntimeException(e); - } - }); - - // verify that the marker is inactive - the new overseer should deactivate markers once they are processed - assertTrue("Marker " + pathLost + " still active!", markerInactive.get()); - - listener.reset(); - - // set up triggers - - log.info("====== ADD TRIGGERS"); - assertAutoScalingRequest - ("{" + - "'set-trigger' : {" + - "'name' : 'node_added_triggerMR'," + - "'event' : 'nodeAdded'," + - "'waitFor' : '1s'," + - "'enabled' : true," + - "'actions' : [{'name':'test','class':'" + TestEventMarkerAction.class.getName() + "'}]" + - "}}"); - - assertAutoScalingRequest - ("{" + - "'set-trigger' : {" + - "'name' : 'node_lost_triggerMR'," + - "'event' : 'nodeLost'," + - "'waitFor' : '1s'," + - "'enabled' : true," + - "'actions' : [{'name':'test','class':'" + TestEventMarkerAction.class.getName() + "'}]" + - "}}"); - - assertAutoScalingRequest( - "{\n" + - " \"set-listener\" : {\n" + - " \"name\" : \"listener_node_added_triggerMR\",\n" + - " \"trigger\" : \"node_added_triggerMR\",\n" + - " \"stage\" : \"STARTED\",\n" + - " \"class\" : \"" + AssertingListener.class.getName() + "\"\n" + - " }\n" + - "}" - ); - assertAutoscalingUpdateComplete(); - - overseerLeader = cluster.getSimClusterStateProvider().simGetOverseerLeader(); - - // create another node - log.info("====== ADD NODE 1"); - String node1 = cluster.simAddNode(); - assertTrue("cluster onChange listener didn't execute even after await()ing an excessive amount of time", - listener.onChangeLatch.await(60, TimeUnit.SECONDS)); - assertEquals(1, listener.addedNodes.size()); - assertEquals(node1, listener.addedNodes.iterator().next()); - // verify that a znode exists - pathAdded = ZkStateReader.SOLR_AUTOSCALING_NODE_ADDED_PATH + "/" + node1; - assertTrue("Path " + pathAdded + " wasn't created", cluster.getDistribStateManager().hasData(pathAdded)); - - listenerEventLatch.countDown(); // let the trigger thread continue - - assertTrue(triggerFiredLatch.await(10, TimeUnit.SECONDS)); - - // kill this node - listener.reset(); - events.clear(); - triggerFiredLatch = new CountDownLatch(1); - - cluster.simRemoveNode(node1, true); - if (!listener.onChangeLatch.await(10, TimeUnit.SECONDS)) { - fail("onChange listener didn't execute on cluster change"); - } - assertEquals(1, listener.lostNodes.size()); - assertEquals(node1, listener.lostNodes.iterator().next()); - // verify that a znode exists - String pathLost2 = ZkStateReader.SOLR_AUTOSCALING_NODE_LOST_PATH + "/" + node1; - assertTrue("Path " + pathLost2 + " wasn't created", cluster.getDistribStateManager().hasData(pathLost2)); - - listenerEventLatch.countDown(); // let the trigger thread continue - - assertTrue(triggerFiredLatch.await(10, TimeUnit.SECONDS)); - - // triggers don't remove markers - assertTrue("Path " + pathLost2 + " should still exist", cluster.getDistribStateManager().hasData(pathLost2)); - - listener.reset(); - events.clear(); - triggerFiredLatch = new CountDownLatch(1); - // kill overseer again - log.info("====== KILL OVERSEER 2"); - cluster.simRemoveNode(overseerLeader, true); - if (!listener.onChangeLatch.await(10, TimeUnit.SECONDS)) { - fail("onChange listener didn't execute on cluster change"); - } - - - if (!triggerFiredLatch.await(20, TimeUnit.SECONDS)) { - fail("Trigger should have fired by now"); - } - assertEquals(1, events.size()); - TriggerEvent ev = events.iterator().next(); - @SuppressWarnings({"unchecked"}) - List nodeNames = (List) ev.getProperty(TriggerEvent.NODE_NAMES); - assertTrue(nodeNames.contains(overseerLeader)); - assertEquals(TriggerEventType.NODELOST, ev.getEventType()); - } - - static final Map> listenerEvents = new ConcurrentHashMap<>(); - static final List allListenerEvents = Collections.synchronizedList(new ArrayList<>()); - static volatile CountDownLatch listenerCreated = new CountDownLatch(1); - static volatile CountDownLatch listenerEventLatch = new CountDownLatch(0); - static volatile boolean failDummyAction = false; - - public static class TestTriggerListener extends TriggerListenerBase { - @Override - public void configure(SolrResourceLoader loader, SolrCloudManager cloudManager, AutoScalingConfig.TriggerListenerConfig config) throws TriggerValidationException { - super.configure(loader, cloudManager, config); - listenerCreated.countDown(); - } - - @Override - public synchronized void onEvent(TriggerEvent event, TriggerEventProcessorStage stage, String actionName, - ActionContext context, Throwable error, String message) { - CapturedEvent ev = new CapturedEvent(cluster.getTimeSource().getTimeNs(), context, config, stage, actionName, event, message); - final CountDownLatch latch = listenerEventLatch; - synchronized (latch) { - if (0 == latch.getCount()) { - log.warn("Ignoring captured event since latch is 'full': {}", ev); - } else { - List lst = listenerEvents.computeIfAbsent(config.name, s -> new ArrayList<>()); - lst.add(ev); - allListenerEvents.add(ev); - latch.countDown(); - } - } - } - } - - public static class TestDummyAction extends TriggerActionBase { - - @Override - public void process(TriggerEvent event, ActionContext context) { - if (failDummyAction) { - throw new RuntimeException("failure"); - } - - } - } - - @Test - public void testListeners() throws Exception { - listenerEventLatch = new CountDownLatch(4 + 5); - - SolrClient solrClient = cluster.simGetSolrClient(); - assertAutoScalingRequest - ("{" + - "'set-trigger' : {" + - "'name' : 'node_added_trigger'," + - "'event' : 'nodeAdded'," + - "'waitFor' : '" + waitForSeconds + "s'," + - "'enabled' : true," + - "'actions' : [" + - "{'name':'test','class':'" + TestTriggerAction.class.getName() + "'}," + - "{'name':'test1','class':'" + TestDummyAction.class.getName() + "'}," + - "]" + - "}}"); - - assertAutoScalingRequest - ("{" + - "'set-listener' : " + - "{" + - "'name' : 'foo'," + - "'trigger' : 'node_added_trigger'," + - "'stage' : ['STARTED','ABORTED','SUCCEEDED', 'FAILED']," + - "'beforeAction' : 'test'," + - "'afterAction' : ['test', 'test1']," + - "'class' : '" + TestTriggerListener.class.getName() + "'" + - "}" + - "}"); - - assertAutoScalingRequest - ("{" + - "'set-listener' : " + - "{" + - "'name' : 'bar'," + - "'trigger' : 'node_added_trigger'," + - "'stage' : ['FAILED','SUCCEEDED']," + - "'beforeAction' : ['test', 'test1']," + - "'afterAction' : 'test'," + - "'class' : '" + TestTriggerListener.class.getName() + "'" + - "}" + - "}"); - - assertAutoscalingUpdateComplete(); - assertTrue("The TriggerAction was not init'ed even after await()ing an excessive amount of time", - actionInitCalled.await(60, TimeUnit.SECONDS)); - - listenerEvents.clear(); - failDummyAction = false; - - String newNode = cluster.simAddNode(); - assertTrue("trigger did not fire event after await()ing an excessive amount of time", - triggerFiredLatch.await(60, TimeUnit.SECONDS)); - assertTrue(triggerFired.get()); - - assertTrue("the listeners didn't recorded all events even after await()ing an excessive amount of time", - listenerEventLatch.await(60, TimeUnit.SECONDS)); - assertEquals("at least 2 event types should have been recorded", 2, listenerEvents.size()); - - // check foo events - List testEvents = listenerEvents.get("foo"); - assertNotNull("foo events: " + testEvents, testEvents); - assertEquals("foo events: " + testEvents, 5, testEvents.size()); - - assertEquals(TriggerEventProcessorStage.STARTED, testEvents.get(0).stage); - - assertEquals(TriggerEventProcessorStage.BEFORE_ACTION, testEvents.get(1).stage); - assertEquals("test", testEvents.get(1).actionName); - - assertEquals(TriggerEventProcessorStage.AFTER_ACTION, testEvents.get(2).stage); - assertEquals("test", testEvents.get(2).actionName); - - assertEquals(TriggerEventProcessorStage.AFTER_ACTION, testEvents.get(3).stage); - assertEquals("test1", testEvents.get(3).actionName); - - assertEquals(TriggerEventProcessorStage.SUCCEEDED, testEvents.get(4).stage); - - // check bar events - testEvents = listenerEvents.get("bar"); - assertNotNull("bar events", testEvents); - assertEquals("bar events", 4, testEvents.size()); - - assertEquals(TriggerEventProcessorStage.BEFORE_ACTION, testEvents.get(0).stage); - assertEquals("test", testEvents.get(0).actionName); - - assertEquals(TriggerEventProcessorStage.AFTER_ACTION, testEvents.get(1).stage); - assertEquals("test", testEvents.get(1).actionName); - - assertEquals(TriggerEventProcessorStage.BEFORE_ACTION, testEvents.get(2).stage); - assertEquals("test1", testEvents.get(2).actionName); - - assertEquals(TriggerEventProcessorStage.SUCCEEDED, testEvents.get(3).stage); - - // check global ordering of events (SOLR-12668) - int fooIdx = -1; - int barIdx = -1; - for (int i = 0; i < allListenerEvents.size(); i++) { - CapturedEvent ev = allListenerEvents.get(i); - if (ev.stage == TriggerEventProcessorStage.BEFORE_ACTION && ev.actionName.equals("test")) { - if (ev.config.name.equals("foo")) { - fooIdx = i; - } else if (ev.config.name.equals("bar")) { - barIdx = i; - } - } - } - assertTrue("fooIdx not found", fooIdx != -1); - assertTrue("barIdx not found", barIdx != -1); - assertTrue("foo fired later than bar: fooIdx=" + fooIdx + ", barIdx=" + barIdx, fooIdx < barIdx); - - // reset - triggerFired.set(false); - triggerFiredLatch = new CountDownLatch(1); - listenerEvents.clear(); - failDummyAction = true; - listenerEventLatch = new CountDownLatch(4 + 4); // fewer total due to failDummyAction - - newNode = cluster.simAddNode(); - assertTrue("trigger did not fire event after await()ing an excessive amount of time", - triggerFiredLatch.await(60, TimeUnit.SECONDS)); - assertTrue(triggerFired.get()); - - assertTrue("the listeners didn't recorded all events even after await()ing an excessive amount of time", - listenerEventLatch.await(60, TimeUnit.SECONDS)); - assertEquals("at least 2 event types should have been recorded", 2, listenerEvents.size()); - - // check foo events - testEvents = listenerEvents.get("foo"); - assertNotNull("foo events: " + testEvents, testEvents); - assertEquals("foo events: " + testEvents, 4, testEvents.size()); - - assertEquals(TriggerEventProcessorStage.STARTED, testEvents.get(0).stage); - - assertEquals(TriggerEventProcessorStage.BEFORE_ACTION, testEvents.get(1).stage); - assertEquals("test", testEvents.get(1).actionName); - - assertEquals(TriggerEventProcessorStage.AFTER_ACTION, testEvents.get(2).stage); - assertEquals("test", testEvents.get(2).actionName); - - assertEquals(TriggerEventProcessorStage.FAILED, testEvents.get(3).stage); - assertEquals("test1", testEvents.get(3).actionName); - - // check bar events - testEvents = listenerEvents.get("bar"); - assertNotNull("bar events", testEvents); - assertEquals("bar events", 4, testEvents.size()); - - assertEquals(TriggerEventProcessorStage.BEFORE_ACTION, testEvents.get(0).stage); - assertEquals("test", testEvents.get(0).actionName); - - assertEquals(TriggerEventProcessorStage.AFTER_ACTION, testEvents.get(1).stage); - assertEquals("test", testEvents.get(1).actionName); - - assertEquals(TriggerEventProcessorStage.BEFORE_ACTION, testEvents.get(2).stage); - assertEquals("test1", testEvents.get(2).actionName); - - assertEquals(TriggerEventProcessorStage.FAILED, testEvents.get(3).stage); - assertEquals("test1", testEvents.get(3).actionName); - } - - @Test - public void testCooldown() throws Exception { - SolrClient solrClient = cluster.simGetSolrClient(); - failDummyAction = false; - listenerEventLatch = new CountDownLatch(1); - waitForSeconds = 1; - assertAutoScalingRequest - ("{" + - "'set-trigger' : {" + - "'name' : 'node_added_cooldown_trigger'," + - "'event' : 'nodeAdded'," + - "'waitFor' : '" + waitForSeconds + "s'," + - "'enabled' : true," + - "'actions' : [" + - "{'name':'test','class':'" + TestTriggerAction.class.getName() + "'}" + - "]" + - "}}"); - - assertAutoScalingRequest - ("{" + - "'set-listener' : " + - "{" + - "'name' : 'bar'," + - "'trigger' : 'node_added_cooldown_trigger'," + - "'stage' : ['FAILED','SUCCEEDED', 'IGNORED']," + - "'class' : '" + TestTriggerListener.class.getName() + "'" + - "}" + - "}"); - - assertAutoscalingUpdateComplete(); - assertTrue("The TriggerAction was not init'ed even after await()ing an excessive amount of time", - actionInitCalled.await(60, TimeUnit.SECONDS)); - - listenerCreated = new CountDownLatch(1); - listenerEvents.clear(); - - String newNode = cluster.simAddNode(); - assertTrue("trigger did not fire event after await()ing an excessive amount of time", - triggerFiredLatch.await(60, TimeUnit.SECONDS)); - assertTrue(triggerFired.get()); - assertTrue("the listener didn't recorded all events even after await()ing an excessive amount of time", - listenerEventLatch.await(60, TimeUnit.SECONDS)); - - List capturedEvents = listenerEvents.get("bar"); - assertNotNull("no events for 'bar'!", capturedEvents); - - assertEquals(capturedEvents.toString(), 1, capturedEvents.size()); - long prevTimestamp = capturedEvents.get(0).timestamp; - - // reset the trigger and captured events - listenerEventLatch = new CountDownLatch(1); - listenerEvents.clear(); - triggerFiredLatch = new CountDownLatch(1); - triggerFired.compareAndSet(true, false); - - String newNode2 = cluster.simAddNode(); - assertTrue("trigger did not fire event after await()ing an excessive amount of time", - triggerFiredLatch.await(60, TimeUnit.SECONDS)); - assertTrue("the listener didn't recorded all events even after await()ing an excessive amount of time", - listenerEventLatch.await(60, TimeUnit.SECONDS)); - - // there must be exactly one SUCCEEDED event - capturedEvents = listenerEvents.get("bar"); - assertNotNull(capturedEvents); - assertEquals(capturedEvents.toString(), 1, capturedEvents.size()); - CapturedEvent ev = capturedEvents.get(0); - assertEquals(ev.toString(), TriggerEventProcessorStage.SUCCEEDED, ev.stage); - // the difference between timestamps of the first SUCCEEDED and the last SUCCEEDED - // must be larger than cooldown period - assertTrue("timestamp delta is less than default cooldown period", ev.timestamp - prevTimestamp > TimeUnit.SECONDS.toNanos(ScheduledTriggers.DEFAULT_COOLDOWN_PERIOD_SECONDS)); - } - - public static class TestSearchRateAction extends TriggerActionBase { - - @Override - public void process(TriggerEvent event, ActionContext context) throws Exception { - try { - events.add(event); - long currentTimeNanos = cluster.getTimeSource().getTimeNs(); - long eventTimeNanos = event.getEventTime(); - long waitForNanos = TimeUnit.NANOSECONDS.convert(waitForSeconds, TimeUnit.SECONDS) - WAIT_FOR_DELTA_NANOS; - if (currentTimeNanos - eventTimeNanos <= waitForNanos) { - fail(event.getSource() + " was fired before the configured waitFor period"); - } - getTriggerFiredLatch().countDown(); - } catch (Throwable t) { - log.debug("--throwable", t); - throw t; - } - } - } - - public static class FinishTriggerAction extends TriggerActionBase { - @Override - public void process(TriggerEvent event, ActionContext context) throws Exception { - triggerFinishedCount.incrementAndGet(); - triggerFinishedLatch.countDown(); - } - } - - public static class StartTriggerAction extends TriggerActionBase { - @Override - public void process(TriggerEvent event, ActionContext context) throws Exception { - triggerStartedLatch.countDown(); - triggerStartedCount.incrementAndGet(); - } - } - - - - @Test - @SuppressWarnings({"unchecked"}) - public void testSearchRate() throws Exception { - SolrClient solrClient = cluster.simGetSolrClient(); - String COLL1 = "collection1"; - CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(COLL1, - "conf", 1, 2); - create.process(solrClient); - CloudUtil.waitForState(cluster, "searchRate testing collection creating", - COLL1, CloudUtil.clusterShape(1, 2, false, true)); - - listenerEventLatch = new CountDownLatch(4); - - assertAutoScalingRequest - ("{" + - "'set-trigger' : {" + - "'name' : 'search_rate_trigger'," + - "'event' : 'searchRate'," + - "'waitFor' : '" + waitForSeconds + "s'," + - "'enabled' : true," + - "'aboveRate' : 1.0," + - "'aboveNodeRate' : 1.0," + - "'actions' : [" + - "{'name':'start','class':'" + StartTriggerAction.class.getName() + "'}," + - "{'name':'compute','class':'" + ComputePlanAction.class.getName() + "'}," + - "{'name':'execute','class':'" + ExecutePlanAction.class.getName() + "'}," + - "{'name':'test','class':'" + TestSearchRateAction.class.getName() + "'}" + - "{'name':'finish','class':'" + FinishTriggerAction.class.getName() + "'}," + - "]" + - "}}"); - - assertAutoScalingRequest - ("{" + - "'set-listener' : " + - "{" + - "'name' : 'srt'," + - "'trigger' : 'search_rate_trigger'," + - "'stage' : ['FAILED','SUCCEEDED']," + - "'afterAction': ['compute', 'execute', 'test']," + - "'class' : '" + TestTriggerListener.class.getName() + "'" + - "}" + - "}"); - - assertAutoscalingUpdateComplete(); - - - cluster.getSimClusterStateProvider().simSetCollectionValue(COLL1, "QUERY./select.requestTimes:1minRate", 500, false, true); - - assertTrue("The trigger did not start even after await()ing an excessive amount of time", - triggerStartedLatch.await(60, TimeUnit.SECONDS)); - assertTrue("The trigger did not finish even after await()ing an excessive amount of time", - triggerFinishedLatch.await(60, TimeUnit.SECONDS)); - - assertTrue("the listener didn't recorded all events even after await()ing an excessive amount of time", - listenerEventLatch.await(60, TimeUnit.SECONDS)); - - List events = new ArrayList<>(listenerEvents.get("srt")); - assertNotNull("Could not find events for srt", events); - assertEquals(events.toString(), 4, events.size()); - assertEquals("AFTER_ACTION", events.get(0).stage.toString()); - assertEquals("compute", events.get(0).actionName); - assertEquals("AFTER_ACTION", events.get(1).stage.toString()); - assertEquals("execute", events.get(1).actionName); - assertEquals("AFTER_ACTION", events.get(2).stage.toString()); - assertEquals("test", events.get(2).actionName); - assertEquals("SUCCEEDED", events.get(3).stage.toString()); - assertNull(events.get(3).actionName); - - CapturedEvent ev = events.get(0); - long now = cluster.getTimeSource().getTimeNs(); - // verify waitFor - assertTrue(TimeUnit.SECONDS.convert(waitForSeconds, TimeUnit.NANOSECONDS) - WAIT_FOR_DELTA_NANOS <= now - ev.event.getEventTime()); - Map nodeRates = (Map)ev.event.getProperties().get(SearchRateTrigger.HOT_NODES); - assertNotNull("nodeRates", nodeRates); - assertTrue(nodeRates.toString(), nodeRates.size() > 0); - AtomicDouble totalNodeRate = new AtomicDouble(); - nodeRates.forEach((n, r) -> totalNodeRate.addAndGet(r)); - List replicaRates = (List)ev.event.getProperties().get(SearchRateTrigger.HOT_REPLICAS); - assertNotNull("replicaRates", replicaRates); - assertTrue(replicaRates.toString(), replicaRates.size() > 0); - AtomicDouble totalReplicaRate = new AtomicDouble(); - replicaRates.forEach(r -> { - assertTrue(r.toString(), r.get("rate") != null); - totalReplicaRate.addAndGet((Double)r.get("rate")); - }); - Map shardRates = (Map)ev.event.getProperties().get(SearchRateTrigger.HOT_SHARDS); - assertNotNull("shardRates", shardRates); - assertEquals(shardRates.toString(), 1, shardRates.size()); - shardRates = (Map)shardRates.get(COLL1); - assertNotNull("shardRates", shardRates); - assertEquals(shardRates.toString(), 1, shardRates.size()); - AtomicDouble totalShardRate = new AtomicDouble(); - shardRates.forEach((s, r) -> totalShardRate.addAndGet((Double)r)); - Map collectionRates = (Map)ev.event.getProperties().get(SearchRateTrigger.HOT_COLLECTIONS); - assertNotNull("collectionRates", collectionRates); - assertEquals(collectionRates.toString(), 1, collectionRates.size()); - Double collectionRate = collectionRates.get(COLL1); - assertNotNull(collectionRate); - assertTrue(collectionRate > 100.0); - assertTrue(totalNodeRate.get() > 100.0); - assertTrue(totalShardRate.get() > 100.0); - assertTrue(totalReplicaRate.get() > 100.0); - - // check operations - List ops = (List)ev.context.get("properties.operations"); - assertNotNull(ops); - assertTrue(ops.size() > 1); - for (MapWriter m : ops) { - assertEquals("ADDREPLICA", m._get("params.action", null)); - } - } - - /** - * Helper method for getting a copy of the current (internal) trigger state of a scheduled trigger. - */ - private Map getTriggerState(final String name) { - final AutoScaling.Trigger t = cluster.getOverseerTriggerThread().getScheduledTriggers().getTrigger(name); - assertNotNull(name + " is not a currently scheduled trigger", t); - assertTrue(name + " is not a TriggerBase w/state: " + t.getClass(), - t instanceof TriggerBase); - return ((TriggerBase)t).deepCopyState(); - } - - /** - * Helper method for making some common assertions about {@link #events}: - *
    - *
  • Exactly one event that is not null
    • - *
  • Event refers to exactly one expected {@link TriggerEvent#NODE_NAMES}
    • - *
  • Event has exactly one {@link TriggerEvent#EVENT_TIMES} (which matches {@link TriggerEvent#getEventTime}) which is less then the maxExpectedEventTimeNs
    • - *
- * @return the event found so that other assertions can be made - */ - private static TriggerEvent assertSingleEvent(final String expectedNodeName, - final long maxExpectedEventTimeNs) { - - assertEquals("Wrong number of events recorded: " + events.toString(), - 1, events.size()); - - final TriggerEvent event = events.iterator().next(); - assertNotNull("null event???", event); - assertNotNull("event is missing NODE_NAMES: " + event, event.getProperty(TriggerEvent.NODE_NAMES)); - assertEquals("event has incorrect NODE_NAMES: " + event, - Collections.singletonList(expectedNodeName), - event.getProperty(TriggerEvent.NODE_NAMES)); - - assertTrue("event TS is too late, should be before (max) expected TS @ " - + maxExpectedEventTimeNs + ": " + event, - event.getEventTime() < maxExpectedEventTimeNs); - - assertNotNull("event is missing EVENT_TIMES: " + event, event.getProperty(TriggerEvent.EVENT_TIMES)); - assertEquals("event has unexpeted number of EVENT_TIMES: " + event, - 1, ((Collection)event.getProperty(TriggerEvent.EVENT_TIMES)).size()); - assertEquals("event's TS doesn't match EVENT_TIMES: " + event, - event.getEventTime(), - ((Collection)event.getProperty(TriggerEvent.EVENT_TIMES)).iterator().next()); - return event; - } - -} diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimUtils.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimUtils.java deleted file mode 100644 index b2d4bea9d76..00000000000 --- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimUtils.java +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.cloud.autoscaling.sim; - -import org.apache.solr.SolrTestCaseJ4; -import org.apache.solr.client.solrj.SolrRequest; -import org.apache.solr.client.solrj.request.V2Request; -import org.apache.solr.common.params.SolrParams; -import org.junit.Test; - -/** - * - */ -public class TestSimUtils extends SolrTestCaseJ4 { - - @Test - public void testV2toV1() throws Exception { - // valid patterns - - V2Request req = new V2Request.Builder("/c/myCollection") - .withMethod(SolrRequest.METHOD.POST) - .withPayload("{'add-replica':{'shard':'shard2','node':'node1:1234','type':'TLOG'}}") - .withParams(params("foo", "bar")) - .build(); - SolrParams params = SimUtils.v2AdminRequestToV1Params(req); - assertEquals("/admin/collections", params.get("path")); - assertEquals("myCollection", params.get("collection")); - assertEquals("bar", params.get("foo")); - assertEquals("addreplica", params.get("action")); - assertEquals("shard2", params.get("shard")); - assertEquals("node1:1234", params.get("node")); - assertEquals("TLOG", params.get("type")); - - req = new V2Request.Builder("/c/myCollection/shards/shard1") - .withMethod(SolrRequest.METHOD.POST) - .withPayload("{'add-replica':{'shard':'shard2','node':'node1:1234','type':'TLOG'}}") - .build(); - params = SimUtils.v2AdminRequestToV1Params(req); - assertEquals("/admin/collections", params.get("path")); - assertEquals("myCollection", params.get("collection")); - // XXX should path parameters override the payload, or the other way around? - assertEquals("shard1", params.get("shard")); - - req = new V2Request.Builder("/c/myCollection/shards/shard1/core_node5") - .withMethod(SolrRequest.METHOD.POST) - .withPayload("{'deletereplica':{}}") - .build(); - params = SimUtils.v2AdminRequestToV1Params(req); - assertEquals("/admin/collections", params.get("path")); - assertEquals("myCollection", params.get("collection")); - // XXX should path parameters override the payload, or the other way around? - assertEquals("shard1", params.get("shard")); - assertEquals("core_node5", params.get("replica")); - - // invalid patterns - req = new V2Request.Builder("/invalid/myCollection") - .withMethod(SolrRequest.METHOD.POST) - .withPayload("{'add-replica':{'shard':'shard2','node':'node1:1234','type':'TLOG'}}") - .withParams(params("foo", "bar")) - .build(); - try { - params = SimUtils.v2AdminRequestToV1Params(req); - } catch (UnsupportedOperationException e) { - // expected - assertTrue(e.toString(), e.toString().contains("request path")); - } - - req = new V2Request.Builder("/collections/myCollection/foobar/xyz") - .withMethod(SolrRequest.METHOD.POST) - .withPayload("{'add-replica':{'shard':'shard2','node':'node1:1234','type':'TLOG'}}") - .withParams(params("foo", "bar")) - .build(); - try { - params = SimUtils.v2AdminRequestToV1Params(req); - } catch (UnsupportedOperationException e) { - // expected - assertTrue(e.toString(), e.toString().contains("expected 'shards'")); - } - } -} diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSnapshotCloudManager.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSnapshotCloudManager.java deleted file mode 100644 index 5b5e0c8fbb0..00000000000 --- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSnapshotCloudManager.java +++ /dev/null @@ -1,275 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.cloud.autoscaling.sim; - -import java.io.File; -import java.io.FileInputStream; -import java.lang.invoke.MethodHandles; -import java.nio.charset.Charset; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.TreeMap; -import java.util.function.Function; -import java.util.function.Predicate; -import java.util.regex.Pattern; -import java.util.stream.Collectors; - -import org.apache.commons.io.IOUtils; -import org.apache.solr.client.solrj.cloud.DistribStateManager; -import org.apache.solr.client.solrj.cloud.NodeStateProvider; -import org.apache.solr.client.solrj.cloud.SolrCloudManager; -import org.apache.solr.client.solrj.cloud.autoscaling.PolicyHelper; -import org.apache.solr.client.solrj.cloud.autoscaling.Suggester; -import org.apache.solr.client.solrj.cloud.autoscaling.Suggestion; -import org.apache.solr.client.solrj.cloud.autoscaling.VersionedData; -import org.apache.solr.client.solrj.request.CollectionAdminRequest; -import org.apache.solr.cloud.CloudTestUtils; -import org.apache.solr.cloud.SolrCloudTestCase; -import org.apache.solr.common.cloud.ClusterState; -import org.apache.solr.common.cloud.Replica; -import org.apache.solr.common.params.CollectionAdminParams; -import org.apache.solr.common.util.TimeSource; -import org.apache.solr.common.util.Utils; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * - */ -public class TestSnapshotCloudManager extends SolrCloudTestCase { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - private static int NODE_COUNT = 3; - - private static SolrCloudManager realManager; - - // set up a real cluster as the source of test data - @BeforeClass - public static void setupCluster() throws Exception { - configureCluster(NODE_COUNT) - .addConfig("conf", configset("cloud-minimal")) - .configure(); - CollectionAdminRequest.createCollection(CollectionAdminParams.SYSTEM_COLL, null, 1, 2, 0, 1) - .process(cluster.getSolrClient()); - CollectionAdminRequest.createCollection("coll1", null, 1, 1) - .process(cluster.getSolrClient()); - CollectionAdminRequest.createCollection("coll10", null, 1, 1) - .process(cluster.getSolrClient()); - realManager = cluster.getJettySolrRunner(cluster.getJettySolrRunners().size() - 1).getCoreContainer() - .getZkController().getSolrCloudManager(); - // disable .scheduled_maintenance (once it exists) - CloudTestUtils.waitForTriggerToBeScheduled(realManager, ".scheduled_maintenance"); - CloudTestUtils.suspendTrigger(realManager, ".scheduled_maintenance"); - } - - @AfterClass - public static void cleanUpAfterClass() throws Exception { - realManager = null; - } - - @Test - public void testSnapshots() throws Exception { - SnapshotCloudManager snapshotCloudManager = new SnapshotCloudManager(realManager, null); - Map snapshot = snapshotCloudManager.getSnapshot(true, false); - SnapshotCloudManager snapshotCloudManager1 = new SnapshotCloudManager(snapshot); - SimSolrCloudTestCase.assertClusterStateEquals(realManager.getClusterStateProvider().getClusterState(), snapshotCloudManager.getClusterStateProvider().getClusterState()); - SimSolrCloudTestCase.assertClusterStateEquals(realManager.getClusterStateProvider().getClusterState(), snapshotCloudManager1.getClusterStateProvider().getClusterState()); - // this will always fail because the metrics will be already different - // assertNodeStateProvider(realManager, snapshotCloudManager); - assertNodeStateProvider(snapshotCloudManager, snapshotCloudManager1); - assertDistribStateManager(snapshotCloudManager.getDistribStateManager(), snapshotCloudManager1.getDistribStateManager()); - } - - @Test - public void testPersistance() throws Exception { - Path tmpPath = createTempDir(); - File tmpDir = tmpPath.toFile(); - SnapshotCloudManager snapshotCloudManager = new SnapshotCloudManager(realManager, null); - snapshotCloudManager.saveSnapshot(tmpDir, true, false); - SnapshotCloudManager snapshotCloudManager1 = SnapshotCloudManager.readSnapshot(tmpDir); - SimSolrCloudTestCase.assertClusterStateEquals(snapshotCloudManager.getClusterStateProvider().getClusterState(), snapshotCloudManager1.getClusterStateProvider().getClusterState()); - assertNodeStateProvider(snapshotCloudManager, snapshotCloudManager1); - assertDistribStateManager(snapshotCloudManager.getDistribStateManager(), snapshotCloudManager1.getDistribStateManager()); - } - - @Test - public void testRedaction() throws Exception { - Path tmpPath = createTempDir(); - File tmpDir = tmpPath.toFile(); - Set redacted = new HashSet<>(realManager.getClusterStateProvider().getLiveNodes()); - try (SnapshotCloudManager snapshotCloudManager = new SnapshotCloudManager(realManager, null)) { - redacted.addAll(realManager.getClusterStateProvider().getClusterState().getCollectionStates().keySet()); - snapshotCloudManager.saveSnapshot(tmpDir, true, true); - } - for (String key : SnapshotCloudManager.REQUIRED_KEYS) { - File src = new File(tmpDir, key + ".json"); - assertTrue(src.toString() + " doesn't exist", src.exists()); - try (FileInputStream is = new FileInputStream(src)) { - String data = IOUtils.toString(is, Charset.forName("UTF-8")); - assertFalse("empty data in " + src, data.trim().isEmpty()); - for (String redactedName : redacted) { - assertFalse("redacted name " + redactedName + " found in " + src, data.contains(redactedName)); - } - } - } - } - - @Test - public void testComplexSnapshot() throws Exception { - File snapshotDir = new File(TEST_HOME(), "simSnapshot"); - SnapshotCloudManager snapshotCloudManager = SnapshotCloudManager.readSnapshot(snapshotDir); - assertEquals(48, snapshotCloudManager.getClusterStateProvider().getLiveNodes().size()); - assertEquals(16, snapshotCloudManager.getClusterStateProvider().getClusterState().getCollectionStates().size()); - try (SimCloudManager simCloudManager = SimCloudManager.createCluster(snapshotCloudManager, null, TimeSource.get("simTime:50"))) { - List suggestions = PolicyHelper.getSuggestions(simCloudManager.getDistribStateManager().getAutoScalingConfig(), simCloudManager); - //assertEquals(1, suggestions.size()); - if (suggestions.size() > 0) { - Suggester.SuggestionInfo suggestion = suggestions.get(0); - assertEquals(Suggestion.Type.improvement.toString(), suggestion.toMap(new HashMap<>()).get("type").toString()); - } - } - } - - @Test - public void testSimulatorFromSnapshot() throws Exception { - Path tmpPath = createTempDir(); - File tmpDir = tmpPath.toFile(); - SnapshotCloudManager snapshotCloudManager = new SnapshotCloudManager(realManager, null); - snapshotCloudManager.saveSnapshot(tmpDir, true, false); - SnapshotCloudManager snapshotCloudManager1 = SnapshotCloudManager.readSnapshot(tmpDir); - try (SimCloudManager simCloudManager = SimCloudManager.createCluster(snapshotCloudManager1, null, TimeSource.get("simTime:50"))) { - SimSolrCloudTestCase.assertClusterStateEquals(snapshotCloudManager.getClusterStateProvider().getClusterState(), simCloudManager.getClusterStateProvider().getClusterState()); - assertNodeStateProvider(snapshotCloudManager, simCloudManager, "freedisk"); - assertDistribStateManager(snapshotCloudManager.getDistribStateManager(), simCloudManager.getDistribStateManager()); - ClusterState state = simCloudManager.getClusterStateProvider().getClusterState(); - Replica r = state.getCollection(CollectionAdminParams.SYSTEM_COLL).getReplicas().get(0); - // get another node - String target = null; - for (String node : simCloudManager.getClusterStateProvider().getLiveNodes()) { - if (!node.equals(r.getNodeName())) { - target = node; - break; - } - } - if (target == null) { - fail("can't find suitable target node for replica " + r + ", liveNodes=" + simCloudManager.getClusterStateProvider().getLiveNodes()); - } - CollectionAdminRequest.MoveReplica moveReplica = CollectionAdminRequest - .moveReplica(CollectionAdminParams.SYSTEM_COLL, r.getName(), target); - log.info("################"); - simCloudManager.simGetSolrClient().request(moveReplica); - } - } - - @SuppressWarnings({"unchecked"}) - private static void assertNodeStateProvider(SolrCloudManager oneMgr, SolrCloudManager twoMgr, String... ignorableNodeValues) throws Exception { - NodeStateProvider one = oneMgr.getNodeStateProvider(); - NodeStateProvider two = twoMgr.getNodeStateProvider(); - for (String node : oneMgr.getClusterStateProvider().getLiveNodes()) { - Map oneVals = one.getNodeValues(node, SimUtils.COMMON_NODE_TAGS); - Map twoVals = two.getNodeValues(node, SimUtils.COMMON_NODE_TAGS); - oneVals = new TreeMap<>(Utils.getDeepCopy(oneVals, 10, false, true)); - twoVals = new TreeMap<>(Utils.getDeepCopy(twoVals, 10, false, true)); - if (ignorableNodeValues != null) { - for (String key : ignorableNodeValues) { - oneVals.remove(key); - twoVals.remove(key); - } - } - assertEquals(Utils.toJSONString(oneVals), Utils.toJSONString(twoVals)); - Map>> oneInfos = one.getReplicaInfo(node, SimUtils.COMMON_REPLICA_TAGS); - Map>> twoInfos = two.getReplicaInfo(node, SimUtils.COMMON_REPLICA_TAGS); - assertEquals("collections on node" + node, oneInfos.keySet(), twoInfos.keySet()); - oneInfos.forEach((coll, oneShards) -> { - Map> twoShards = twoInfos.get(coll); - assertEquals("shards on node " + node, oneShards.keySet(), twoShards.keySet()); - oneShards.forEach((shard, oneReplicas) -> { - List twoReplicas = twoShards.get(shard); - assertEquals("num replicas on node " + node, oneReplicas.size(), twoReplicas.size()); - Map oneMap = oneReplicas.stream() - .collect(Collectors.toMap(Replica::getName, Function.identity())); - Map twoMap = twoReplicas.stream() - .collect(Collectors.toMap(Replica::getName, Function.identity())); - assertEquals("replica coreNodeNames on node " + node, oneMap.keySet(), twoMap.keySet()); - oneMap.forEach((coreNode, oneReplica) -> { - Replica twoReplica = twoMap.get(coreNode); - SimSolrCloudTestCase.assertReplicaInfoEquals(oneReplica, twoReplica); - }); - }); - }); - } - } - - // ignore these because SimCloudManager always modifies them - private static final Set IGNORE_DISTRIB_STATE_PATTERNS = new HashSet<>(Arrays.asList( - Pattern.compile("/autoscaling/triggerState/.*"), - // some triggers may have run after the snapshot was taken - Pattern.compile("/autoscaling/events/.*"), - Pattern.compile("/clusterstate\\.json"), - Pattern.compile("/collections/[^/]+?/state.json"), - // depending on the startup sequence leaders may differ - Pattern.compile("/collections/[^/]+?/leader_elect/.*"), - Pattern.compile("/collections/[^/]+?/leaders/.*"), - Pattern.compile("/collections/[^/]+?/terms/.*"), - Pattern.compile("/overseer_elect/election/.*"), - Pattern.compile("/live_nodes/.*") - )); - - private static final Predicate STATE_FILTER_FUN = p -> { - for (Pattern pattern : IGNORE_DISTRIB_STATE_PATTERNS) { - if (pattern.matcher(p).matches()) { - return false; - } - } - return true; - }; - - private static void assertDistribStateManager(DistribStateManager one, DistribStateManager two) throws Exception { - List treeOne = new ArrayList<>(one.listTree("/").stream() - .filter(STATE_FILTER_FUN).collect(Collectors.toList())); - List treeTwo = new ArrayList<>(two.listTree("/").stream() - .filter(STATE_FILTER_FUN).collect(Collectors.toList())); - Collections.sort(treeOne); - Collections.sort(treeTwo); - if (!treeOne.equals(treeTwo)) { - List t1 = new ArrayList<>(treeOne); - t1.removeAll(treeTwo); - log.warn("Only in tree one: {}", t1); - List t2 = new ArrayList<>(treeTwo); - t2.removeAll(treeOne); - log.warn("Only in tree two: {}", t2); - } - assertEquals(treeOne, treeTwo); - for (String path : treeOne) { - VersionedData vd1 = one.getData(path); - VersionedData vd2 = two.getData(path); - assertEquals(path, vd1, vd2); - } - } - - -} diff --git a/solr/core/src/test/org/apache/solr/cloud/rule/RuleEngineTest.java b/solr/core/src/test/org/apache/solr/cloud/rule/RuleEngineTest.java index ac708b45e9f..0d05e3ef412 100644 --- a/solr/core/src/test/org/apache/solr/cloud/rule/RuleEngineTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/rule/RuleEngineTest.java @@ -29,7 +29,7 @@ import java.util.Set; import com.google.common.collect.ImmutableList; import org.apache.solr.SolrTestCaseJ4; -import org.apache.solr.client.solrj.cloud.autoscaling.DelegatingCloudManager; +import org.apache.solr.client.solrj.cloud.DelegatingCloudManager; import org.apache.solr.client.solrj.cloud.NodeStateProvider; import org.apache.solr.client.solrj.cloud.SolrCloudManager; import org.apache.solr.common.cloud.Replica; diff --git a/solr/core/src/test/org/apache/solr/cloud/rule/RulesTest.java b/solr/core/src/test/org/apache/solr/cloud/rule/RulesTest.java index 7401c8567b8..e9586a06b67 100644 --- a/solr/core/src/test/org/apache/solr/cloud/rule/RulesTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/rule/RulesTest.java @@ -17,7 +17,6 @@ package org.apache.solr.cloud.rule; import java.lang.invoke.MethodHandles; -import java.nio.charset.StandardCharsets; import java.nio.file.Path; import java.nio.file.Paths; import java.util.HashSet; @@ -28,18 +27,15 @@ import java.util.stream.Collectors; import org.apache.lucene.util.LuceneTestCase; import org.apache.solr.client.solrj.SolrClient; -import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.client.solrj.embedded.JettySolrRunner; import org.apache.solr.client.solrj.impl.BaseHttpSolrClient; import org.apache.solr.client.solrj.request.CollectionAdminRequest; import org.apache.solr.client.solrj.request.GenericSolrRequest; import org.apache.solr.client.solrj.response.SimpleSolrResponse; -import org.apache.solr.cloud.CloudTestUtils.AutoScalingRequest; import org.apache.solr.cloud.SolrCloudTestCase; import org.apache.solr.common.cloud.DocCollection; import org.apache.solr.common.cloud.Replica; import org.apache.solr.common.cloud.Slice; -import org.apache.solr.common.cloud.ZkStateReader; import org.apache.solr.common.params.ModifiableSolrParams; import org.junit.After; import org.junit.BeforeClass; @@ -71,9 +67,6 @@ public class RulesTest extends SolrCloudTestCase { @After public void removeCollections() throws Exception { cluster.deleteAllCollections(); - // clear any cluster policy test methods may have set - cluster.getSolrClient().getZkStateReader().getZkClient().setData(ZkStateReader.SOLR_AUTOSCALING_CONF_PATH, - "{}".getBytes(StandardCharsets.UTF_8), true); } @Test @@ -159,62 +152,6 @@ public class RulesTest extends SolrCloudTestCase { } - @Test - public void testPortRuleInPresenceOfClusterPolicy() throws Exception { - JettySolrRunner jetty = cluster.getRandomJetty(random()); - String port = Integer.toString(jetty.getLocalPort()); - - // this cluster policy prohibits having any replicas on a node with the above port - String setClusterPolicyCommand = "{" + - " 'set-cluster-policy': [" + - " {'replica': 0, 'port':'" + port + "'}" + - " ]" + - "}"; - @SuppressWarnings({"rawtypes"}) - SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setClusterPolicyCommand); - cluster.getSolrClient().request(req); - - // but this collection is created with a replica placement rule that says all replicas must be created - // on a node with above port (in direct conflict with the cluster policy) - String rulesColl = "portRuleColl2"; - CollectionAdminRequest.createCollectionWithImplicitRouter(rulesColl, "conf", "shard1", 2) - .setRule("port:" + port) - .setSnitch("class:ImplicitSnitch") - .process(cluster.getSolrClient()); - - waitForState("Collection should have followed port rule w/ImplicitSnitch, not cluster policy", - rulesColl, (liveNodes, rulesCollection) -> { - // first sanity check that the collection exists & the rules/snitch are listed - if (null == rulesCollection) { - return false; - } else { - @SuppressWarnings({"rawtypes"}) - List list = (List) rulesCollection.get("rule"); - if (null == list || 1 != list.size()) { - return false; - } - if (! port.equals(((Map) list.get(0)).get("port"))) { - return false; - } - list = (List) rulesCollection.get("snitch"); - if (null == list || 1 != list.size()) { - return false; - } - if (! "ImplicitSnitch".equals(((Map)list.get(0)).get("class"))) { - return false; - } - } - if (2 != rulesCollection.getReplicas().size()) { - return false; - } - // now sanity check that the rules were *obeyed* - // (and the contradictory policy was ignored) - return rulesCollection.getReplicas().stream().allMatch - (replica -> (replica.getNodeName().contains(port) && - replica.isActive(liveNodes))); - }); - } - @Test public void testPortRule() throws Exception { @@ -356,7 +293,6 @@ public class RulesTest extends SolrCloudTestCase { p.add("rule", "cores:<5"); p.add("rule", "node:*,replica:1"); p.add("rule", "freedisk:>"+minGB2); - p.add("autoAddReplicas", "true"); cluster.getSolrClient().request(new GenericSolrRequest(POST, COLLECTIONS_HANDLER_PATH, p)); waitForState("Should have found updated rules in DocCollection", @@ -378,9 +314,6 @@ public class RulesTest extends SolrCloudTestCase { if (! (">"+minGB2).equals(((Map) list.get(2)).get("freedisk"))) { return false; } - if (! "true".equals(String.valueOf(rulesCollection.getProperties().get("autoAddReplicas")))) { - return false; - } list = (List) rulesCollection.get("snitch"); if (null == list || 1 != list.size()) { return false; diff --git a/solr/core/src/test/org/apache/solr/handler/V2ApiIntegrationTest.java b/solr/core/src/test/org/apache/solr/handler/V2ApiIntegrationTest.java index 8e570e4645e..6fc1a8e1d9f 100644 --- a/solr/core/src/test/org/apache/solr/handler/V2ApiIntegrationTest.java +++ b/solr/core/src/test/org/apache/solr/handler/V2ApiIntegrationTest.java @@ -150,10 +150,10 @@ public class V2ApiIntegrationTest extends SolrCloudTestCase { public void testSetPropertyValidationOfCluster() throws IOException, SolrServerException { @SuppressWarnings({"rawtypes"}) NamedList resp = cluster.getSolrClient().request( - new V2Request.Builder("/cluster").withMethod(SolrRequest.METHOD.POST).withPayload("{set-property: {name: autoAddReplicas, val:false}}").build()); + new V2Request.Builder("/cluster").withMethod(SolrRequest.METHOD.POST).withPayload("{set-property: {name: maxCoresPerNode, val:42}}").build()); assertTrue(resp.toString().contains("status=0")); resp = cluster.getSolrClient().request( - new V2Request.Builder("/cluster").withMethod(SolrRequest.METHOD.POST).withPayload("{set-property: {name: autoAddReplicas, val:null}}").build()); + new V2Request.Builder("/cluster").withMethod(SolrRequest.METHOD.POST).withPayload("{set-property: {name: maxCoresPerNode, val:null}}").build()); assertTrue(resp.toString().contains("status=0")); } diff --git a/solr/core/src/test/org/apache/solr/handler/admin/AutoscalingHistoryHandlerTest.java b/solr/core/src/test/org/apache/solr/handler/admin/AutoscalingHistoryHandlerTest.java deleted file mode 100644 index 9886e8927e1..00000000000 --- a/solr/core/src/test/org/apache/solr/handler/admin/AutoscalingHistoryHandlerTest.java +++ /dev/null @@ -1,463 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.handler.admin; - -import java.lang.invoke.MethodHandles; -import java.util.Collection; -import java.util.Collections; -import java.util.Set; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; - -import org.apache.solr.client.solrj.SolrClient; -import org.apache.solr.client.solrj.SolrRequest; -import org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventProcessorStage; -import org.apache.solr.client.solrj.embedded.JettySolrRunner; -import org.apache.solr.client.solrj.impl.CloudSolrClient; -import org.apache.solr.client.solrj.request.CollectionAdminRequest; -import org.apache.solr.client.solrj.response.QueryResponse; -import org.apache.solr.cloud.CloudTestUtils.AutoScalingRequest; -import org.apache.solr.cloud.SolrCloudTestCase; -import org.apache.solr.cloud.autoscaling.ActionContext; -import org.apache.solr.cloud.autoscaling.SystemLogListener; -import org.apache.solr.cloud.autoscaling.TriggerActionBase; -import org.apache.solr.cloud.autoscaling.TriggerEvent; -import org.apache.solr.cloud.autoscaling.TriggerListenerBase; -import org.apache.solr.common.SolrDocument; -import org.apache.solr.common.SolrDocumentList; -import org.apache.solr.common.cloud.ClusterState; -import org.apache.solr.common.cloud.DocCollection; -import org.apache.solr.common.cloud.Replica; -import org.apache.solr.common.cloud.Slice; -import org.apache.solr.common.params.CollectionAdminParams; -import org.apache.solr.common.params.CommonParams; -import org.apache.solr.common.params.ModifiableSolrParams; -import org.apache.solr.common.util.NamedList; -import org.apache.solr.util.LogLevel; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -@LogLevel("org.apache.solr.cloud.autoscaling=DEBUG;org.apache.solr.cloud.Overseer=DEBUG;org.apache.solr.cloud.overseer=DEBUG;org.apache.solr.client.solrj.cloud.autoscaling=DEBUG") -public class AutoscalingHistoryHandlerTest extends SolrCloudTestCase { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - private static CountDownLatch actionFiredLatch; - private static CountDownLatch listenerFiredLatch; - private static CloudSolrClient solrClient; - private static String PREFIX = AutoscalingHistoryHandlerTest.class.getSimpleName(); - private static String COLL_NAME = PREFIX + "_collection"; - private static String systemCollNode; - - private static CountDownLatch getActionFiredLatch() { - return actionFiredLatch; - } - - private static CountDownLatch getListenerFiredLatch() { - return listenerFiredLatch; - } - - @BeforeClass - public static void setupCluster() throws Exception { - configureCluster(2) - .addConfig("conf", configset("cloud-minimal")) - .configure(); - solrClient = cluster.getSolrClient(); - // create the system collection and test collection on different nodes, to avoid - // any interference from .system replicas being moved around. - systemCollNode = cluster.getJettySolrRunner(0).getNodeName(); - CollectionAdminRequest.createCollection(CollectionAdminParams.SYSTEM_COLL, null, 1, 1) - .setCreateNodeSet(systemCollNode) - .process(solrClient); - cluster.waitForActiveCollection(CollectionAdminParams.SYSTEM_COLL, 1, 1); - Set otherNodes = cluster.getJettySolrRunners().stream().map(JettySolrRunner::getNodeName) - .collect(Collectors.toSet()); - otherNodes.remove(systemCollNode); - CollectionAdminRequest.createCollection(COLL_NAME, null, 1, 3) - .setCreateNodeSet(String.join(",", otherNodes)) - .process(solrClient); - cluster.waitForActiveCollection(COLL_NAME, 1, 3); - } - - @AfterClass - public static void releaseClient() throws Exception { - solrClient = null; - } - - public static class TesterListener extends TriggerListenerBase { - - @Override - public void onEvent(TriggerEvent event, TriggerEventProcessorStage stage, String actionName, ActionContext context, Throwable error, String message) throws Exception { - getListenerFiredLatch().countDown(); - } - } - - public static class TesterAction extends TriggerActionBase { - - @Override - public void process(TriggerEvent event, ActionContext context) { - getActionFiredLatch().countDown(); - } - } - - @Before - public void setupTest() throws Exception { - actionFiredLatch = new CountDownLatch(1); - listenerFiredLatch = new CountDownLatch(1); - - // change rules to create violations - String setClusterPolicyCommand = "{" + - " 'set-cluster-policy': [" + - " {'replica':'<2', 'shard': '#EACH', 'node': '#ANY'}" + - " ]" + - "}"; - @SuppressWarnings({"rawtypes"}) - SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setClusterPolicyCommand); - solrClient.request(req); - - - // first trigger - String setTriggerCommand = "{" + - "'set-trigger' : {" + - "'name' : '" + PREFIX + "_node_added_trigger'," + - "'event' : 'nodeAdded'," + - "'waitFor' : '0s'," + - "'enabled' : false," + - "'actions' : [" + - "{'name':'compute_plan','class':'solr.ComputePlanAction'}," + - "{'name':'execute_plan','class':'solr.ExecutePlanAction'}," + - "{'name':'test','class':'" + TesterAction.class.getName() + "'}" + - "]" + - "}}"; - req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand); - NamedList response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - - // second trigger - setTriggerCommand = "{" + - "'set-trigger' : {" + - "'name' : '" + PREFIX + "_node_lost_trigger'," + - "'event' : 'nodeLost'," + - "'waitFor' : '0s'," + - "'enabled' : false," + - "'actions' : [" + - "{'name':'compute_plan','class':'solr.ComputePlanAction'}," + - "{'name':'execute_plan','class':'solr.ExecutePlanAction'}," + - "{'name':'test','class':'" + TesterAction.class.getName() + "'}" + - "]" + - "}}"; - req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand); - response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - - // remove default listeners - String removeListenerCommand = "{\n" + - "\t\"remove-listener\" : {\n" + - "\t\t\"name\" : \"" + PREFIX + "_node_lost_trigger.system\"\n" + - "\t}\n" + - "}"; - req = AutoScalingRequest.create(SolrRequest.METHOD.POST, removeListenerCommand); - response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - removeListenerCommand = "{\n" + - "\t\"remove-listener\" : {\n" + - "\t\t\"name\" : \"" + PREFIX + "_node_added_trigger.system\"\n" + - "\t}\n" + - "}"; - req = AutoScalingRequest.create(SolrRequest.METHOD.POST, removeListenerCommand); - response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - // set up our own listeners - String setListenerCommand = "{" + - "'set-listener' : " + - "{" + - "'name' : 'node_added'," + - "'trigger' : '" + PREFIX + "_node_added_trigger'," + - "'stage' : ['STARTED','ABORTED','SUCCEEDED', 'FAILED']," + - "'beforeAction' : ['compute_plan','execute_plan','test']," + - "'afterAction' : ['compute_plan','execute_plan','test']," + - "'class' : '" + SystemLogListener.class.getName() + "'" + - "}" + - "}"; - req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setListenerCommand); - response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - setListenerCommand = "{" + - "'set-listener' : " + - "{" + - "'name' : 'node_added1'," + - "'trigger' : '" + PREFIX + "_node_added_trigger'," + - "'afterAction' : ['test']," + - "'class' : '" + TesterListener.class.getName() + "'" + - "}" + - "}"; - req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setListenerCommand); - response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - - setListenerCommand = "{" + - "'set-listener' : " + - "{" + - "'name' : 'node_lost'," + - "'trigger' : '" + PREFIX + "_node_lost_trigger'," + - "'stage' : ['STARTED','ABORTED','SUCCEEDED', 'FAILED']," + - "'beforeAction' : ['compute_plan','execute_plan','test']," + - "'afterAction' : ['compute_plan','execute_plan','test']," + - "'class' : '" + SystemLogListener.class.getName() + "'" + - "}" + - "}"; - req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setListenerCommand); - response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - setListenerCommand = "{" + - "'set-listener' : " + - "{" + - "'name' : 'node_lost1'," + - "'trigger' : '" + PREFIX + "_node_lost_trigger'," + - "'afterAction' : ['test']," + - "'class' : '" + TesterListener.class.getName() + "'" + - "}" + - "}"; - req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setListenerCommand); - response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - - // setup is complete, enable the triggers - String resumeTriggerCommand = "{" + - "'resume-trigger' : {" + - "'name' : '" + PREFIX + "_node_added_trigger'," + - "}" + - "}"; - req = AutoScalingRequest.create(SolrRequest.METHOD.POST, resumeTriggerCommand); - response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - resumeTriggerCommand = "{" + - "'resume-trigger' : {" + - "'name' : '" + PREFIX + "_node_lost_trigger'," + - "}" + - "}"; - req = AutoScalingRequest.create(SolrRequest.METHOD.POST, resumeTriggerCommand); - response = solrClient.request(req); - assertEquals(response.get("result").toString(), "success"); - - } - - private void resetLatches() { - actionFiredLatch = new CountDownLatch(1); - listenerFiredLatch = new CountDownLatch(1); - } - - @Test - public void testHistory() throws Exception { - waitForState("Timed out wait for collection be active", COLL_NAME, - clusterShape(1, 3)); - waitForState("Timed out wait for collection be active", CollectionAdminParams.SYSTEM_COLL, - clusterShape(1, 1)); - - log.info("### Start add node..."); - JettySolrRunner jetty = cluster.startJettySolrRunner(); - cluster.waitForAllNodes(30); - String nodeAddedName = jetty.getNodeName(); - log.info("### Added node {}", nodeAddedName); - boolean await = actionFiredLatch.await(60, TimeUnit.SECONDS); - assertTrue("action did not execute", await); - - await = listenerFiredLatch.await(60, TimeUnit.SECONDS); - assertTrue("listener did not execute", await); - - waitForRecovery(COLL_NAME); - - // commit on the history collection - Thread.sleep(5000); - log.info("### Commit .system"); - solrClient.commit(CollectionAdminParams.SYSTEM_COLL); - Thread.sleep(5000); - - // verify that new docs exist - ModifiableSolrParams query = params(CommonParams.Q, "type:" + SystemLogListener.DOC_TYPE, - CommonParams.FQ, "event.source_s:" + PREFIX + "_node_added_trigger"); - QueryResponse resp = solrClient.query(CollectionAdminParams.SYSTEM_COLL, query); - SolrDocumentList docs = resp.getResults(); - assertNotNull(docs); - - query = params(CommonParams.QT, CommonParams.AUTOSCALING_HISTORY_PATH, - AutoscalingHistoryHandler.TRIGGER_PARAM, PREFIX + "_node_added_trigger"); - docs = queryAndAssertDocs(query, solrClient, 8); - - query = params(CommonParams.QT, CommonParams.AUTOSCALING_HISTORY_PATH, - AutoscalingHistoryHandler.STAGE_PARAM, "STARTED"); - docs = solrClient.query(query).getResults(); - assertEquals(1, docs.size()); - assertEquals("NODEADDED", docs.get(0).getFieldValue("event.type_s")); - - query = params(CommonParams.QT, CommonParams.AUTOSCALING_HISTORY_PATH, - AutoscalingHistoryHandler.NODE_PARAM, nodeAddedName); - docs = queryAndAssertDocs(query, solrClient, 8); - for (SolrDocument doc : docs) { - assertTrue(doc.getFieldValues("event.property.nodeNames_ss").contains(nodeAddedName)); - } - - query = params(CommonParams.QT, CommonParams.AUTOSCALING_HISTORY_PATH, - AutoscalingHistoryHandler.ACTION_PARAM, "test"); - docs = solrClient.query(query).getResults(); - assertEquals(2, docs.size()); - assertEquals("BEFORE_ACTION", docs.get(0).getFieldValue("stage_s")); - assertEquals("AFTER_ACTION", docs.get(1).getFieldValue("stage_s")); - - query = params(CommonParams.QT, CommonParams.AUTOSCALING_HISTORY_PATH, - AutoscalingHistoryHandler.ACTION_PARAM, "test"); - docs = solrClient.query(query).getResults(); - assertEquals(2, docs.size()); - assertEquals("BEFORE_ACTION", docs.get(0).getFieldValue("stage_s")); - assertEquals("AFTER_ACTION", docs.get(1).getFieldValue("stage_s")); - - query = params(CommonParams.QT, CommonParams.AUTOSCALING_HISTORY_PATH, - AutoscalingHistoryHandler.COLLECTION_PARAM, COLL_NAME); - docs = queryAndAssertDocs(query, solrClient, 5); - assertEquals("AFTER_ACTION", docs.get(0).getFieldValue("stage_s")); - assertEquals("compute_plan", docs.get(0).getFieldValue("action_s")); - - // reset latches - resetLatches(); - - // kill a node where a replica exists - BUT not the Overseer - NamedList overSeerStatus = cluster.getSolrClient().request(CollectionAdminRequest.getOverseerStatus()); - String overseerLeader = (String) overSeerStatus.get("leader"); - ClusterState state = cluster.getSolrClient().getZkStateReader().getClusterState(); - DocCollection coll = state.getCollection(COLL_NAME); - DocCollection system = state.getCollectionOrNull(CollectionAdminParams.SYSTEM_COLL); - Set systemLeaderNodes; - if (system != null) { - systemLeaderNodes = system.getReplicas().stream() - .filter(r -> r.getBool("leader", false)) - .map(r -> r.getNodeName()) - .collect(Collectors.toSet()); - } else { - systemLeaderNodes = Collections.emptySet(); - } - String nodeToKill = null; - for (Replica r : coll.getReplicas()) { - if (r.isActive(state.getLiveNodes()) && - !r.getNodeName().equals(overseerLeader)) { - if (systemLeaderNodes.contains(r.getNodeName())) { - log.info("--skipping .system leader replica {}", r); - continue; - } - nodeToKill = r.getNodeName(); - break; - } - } - assertNotNull("no suitable node found", nodeToKill); - log.info("### Stopping node {}", nodeToKill); - for (int i = 0; i < cluster.getJettySolrRunners().size(); i++) { - if (cluster.getJettySolrRunner(i).getNodeName().equals(nodeToKill)) { - JettySolrRunner j = cluster.stopJettySolrRunner(i); - cluster.waitForJettyToStop(j); - break; - } - } - log.info("### Stopped node {}", nodeToKill); - await = actionFiredLatch.await(60, TimeUnit.SECONDS); - assertTrue("action did not execute", await); - - await = listenerFiredLatch.await(60, TimeUnit.SECONDS); - assertTrue("listener did not execute", await); - - // wait for recovery - waitForRecovery(COLL_NAME); - - Thread.sleep(5000); - // commit on the history collection - log.info("### Commit .system"); - solrClient.commit(CollectionAdminParams.SYSTEM_COLL); - Thread.sleep(5000); - - query = params(CommonParams.QT, CommonParams.AUTOSCALING_HISTORY_PATH, - AutoscalingHistoryHandler.TRIGGER_PARAM, PREFIX + "_node_lost_trigger"); - docs = solrClient.query(query).getResults(); - assertEquals(docs.toString(), 8, docs.size()); - - query = params(CommonParams.QT, CommonParams.AUTOSCALING_HISTORY_PATH, - AutoscalingHistoryHandler.TRIGGER_PARAM, PREFIX + "_node_lost_trigger", - AutoscalingHistoryHandler.COLLECTION_PARAM, COLL_NAME); - docs = queryAndAssertDocs(query, solrClient, 5); - } - - private SolrDocumentList queryAndAssertDocs(ModifiableSolrParams query, SolrClient client, int expected) throws Exception { - QueryResponse rsp = client.query(query); - SolrDocumentList docs = rsp.getResults(); - if (docs.size() != expected) { - log.info("History query: {}", query); - log.info("Wrong response: {}", rsp); - ModifiableSolrParams fullQuery = params(CommonParams.QT, CommonParams.AUTOSCALING_HISTORY_PATH); - if (log.isInfoEnabled()) { - log.info("Full response: {}", client.query(fullQuery)); - } - } - assertEquals("Wrong number of documents", expected, docs.size()); - return docs; - } - - private static void waitForRecovery(String collection) throws Exception { - log.info("Waiting for recovery of {}", collection); - boolean recovered = false; - boolean allActive = true; - boolean hasLeaders = true; - DocCollection collState = null; - for (int i = 0; i < 300; i++) { - ClusterState state = solrClient.getZkStateReader().getClusterState(); - collState = getCollectionState(collection); - log.debug("###### {}", collState); - Collection replicas = collState.getReplicas(); - allActive = true; - hasLeaders = true; - if (replicas != null && !replicas.isEmpty()) { - for (Replica r : replicas) { - if (state.getLiveNodes().contains(r.getNodeName())) { - if (!r.isActive(state.getLiveNodes())) { - log.info("Not active: {}", r); - allActive = false; - } - } else { - log.info("Replica no longer on a live node, ignoring: {}", r); - } - } - } else { - allActive = false; - } - for (Slice slice : collState.getSlices()) { - if (slice.getLeader() == null) { - hasLeaders = false; - } - } - if (allActive && hasLeaders) { - recovered = true; - break; - } else { - log.info("--- waiting, allActive={}, hasLeaders={}", allActive, hasLeaders); - Thread.sleep(1000); - } - } - assertTrue("replica never fully recovered: allActive=" + allActive + ", hasLeaders=" + hasLeaders + ", collState=" + collState, recovered); - - } - -} diff --git a/solr/core/src/test/org/apache/solr/handler/admin/MetricsHistoryHandlerTest.java b/solr/core/src/test/org/apache/solr/handler/admin/MetricsHistoryHandlerTest.java index 9c912cfcb6f..2c2ce888780 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/MetricsHistoryHandlerTest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/MetricsHistoryHandlerTest.java @@ -21,13 +21,12 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import org.apache.lucene.util.LuceneTestCase; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.cloud.SolrCloudManager; -import org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig; import org.apache.solr.client.solrj.request.CollectionAdminRequest; import org.apache.solr.cloud.CloudUtil; import org.apache.solr.cloud.SolrCloudTestCase; -import org.apache.solr.cloud.autoscaling.sim.SimCloudManager; import org.apache.solr.common.params.CollectionAdminParams; import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.util.Pair; @@ -45,13 +44,13 @@ import org.rrd4j.core.RrdDb; * */ @LogLevel("org.apache.solr.cloud=DEBUG") +@LuceneTestCase.Nightly public class MetricsHistoryHandlerTest extends SolrCloudTestCase { private volatile static SolrCloudManager cloudManager; private volatile static SolrMetricManager metricManager; private volatile static TimeSource timeSource; private volatile static SolrClient solrClient; - private volatile static boolean simulated; private volatile static int SPEED; private volatile static MetricsHistoryHandler handler; @@ -59,40 +58,21 @@ public class MetricsHistoryHandlerTest extends SolrCloudTestCase { @BeforeClass public static void beforeClass() throws Exception { - simulated = random().nextBoolean(); Map args = new HashMap<>(); args.put(MetricsHistoryHandler.SYNC_PERIOD_PROP, 1); args.put(MetricsHistoryHandler.COLLECT_PERIOD_PROP, 1); - if (simulated) { - SPEED = 50; - cloudManager = SimCloudManager.createCluster(1, TimeSource.get("simTime:" + SPEED)); - // wait for defaults to be applied - due to accelerated time sometimes we may miss this - cloudManager.getTimeSource().sleep(10000); - AutoScalingConfig cfg = cloudManager.getDistribStateManager().getAutoScalingConfig(); - assertFalse("autoscaling config is empty", cfg.isEmpty()); - metricManager = ((SimCloudManager)cloudManager).getMetricManager(); - solrClient = ((SimCloudManager)cloudManager).simGetSolrClient(); - // need to register the factory here, before we start the real cluster - metricsHandler = new MetricsHandler(metricManager); - SolrMetricsContext solrMetricsContext = new SolrMetricsContext(metricManager, SolrInfoBean.Group.node.toString(), ""); - handler = new MetricsHistoryHandler(cloudManager.getClusterStateProvider().getLiveNodes().iterator().next(), - metricsHandler, solrClient, cloudManager, args); - handler.initializeMetrics(solrMetricsContext, CommonParams.METRICS_HISTORY_PATH); - } configureCluster(1) .addConfig("conf", configset("cloud-minimal")) .configure(); - if (!simulated) { - cloudManager = cluster.getJettySolrRunner(0).getCoreContainer().getZkController().getSolrCloudManager(); - metricManager = cluster.getJettySolrRunner(0).getCoreContainer().getMetricManager(); - solrClient = cluster.getSolrClient(); - metricsHandler = new MetricsHandler(metricManager); - handler = new MetricsHistoryHandler(cluster.getJettySolrRunner(0).getNodeName(), metricsHandler, solrClient, cloudManager, args); - SolrMetricsContext solrMetricsContext = new SolrMetricsContext(metricManager, SolrInfoBean.Group.node.toString(), ""); - handler.initializeMetrics(solrMetricsContext, CommonParams.METRICS_HISTORY_PATH); - SPEED = 1; - } + cloudManager = cluster.getJettySolrRunner(0).getCoreContainer().getZkController().getSolrCloudManager(); + metricManager = cluster.getJettySolrRunner(0).getCoreContainer().getMetricManager(); + solrClient = cluster.getSolrClient(); + metricsHandler = new MetricsHandler(metricManager); + handler = new MetricsHistoryHandler(cluster.getJettySolrRunner(0).getNodeName(), metricsHandler, solrClient, cloudManager, args); + SolrMetricsContext solrMetricsContext = new SolrMetricsContext(metricManager, SolrInfoBean.Group.node.toString(), ""); + handler.initializeMetrics(solrMetricsContext, CommonParams.METRICS_HISTORY_PATH); + SPEED = 1; timeSource = cloudManager.getTimeSource(); // create .system collection @@ -108,9 +88,6 @@ public class MetricsHistoryHandlerTest extends SolrCloudTestCase { if (handler != null) { handler.close(); } - if (simulated) { - cloudManager.close(); - } handler = null; metricsHandler = null; cloudManager = null; diff --git a/solr/core/src/test/org/apache/solr/handler/admin/TestCollectionAPIs.java b/solr/core/src/test/org/apache/solr/handler/admin/TestCollectionAPIs.java index 617fc1d2a00..ff298aac55a 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/TestCollectionAPIs.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/TestCollectionAPIs.java @@ -167,8 +167,8 @@ public class TestCollectionAPIs extends SolrTestCaseJ4 { ); compareOutput(apiBag, "/collections/collName", POST, - "{modify : {rule : ['replica:*, cores:<5'], autoAddReplicas : false} }", null, - "{collection: collName, operation : modifycollection , autoAddReplicas : 'false', rule : [{replica: '*', cores : '<5' }]}" + "{modify : {rule : ['replica:*, cores:<5']} }", null, + "{collection: collName, operation : modifycollection , rule : [{replica: '*', cores : '<5' }]}" ); compareOutput(apiBag, "/cluster", POST, "{add-role : {role : overseer, node : 'localhost_8978'} }", null, diff --git a/solr/core/src/test/org/apache/solr/security/JWTAuthPluginIntegrationTest.java b/solr/core/src/test/org/apache/solr/security/JWTAuthPluginIntegrationTest.java index b35ed89cc57..5351968f1df 100644 --- a/solr/core/src/test/org/apache/solr/security/JWTAuthPluginIntegrationTest.java +++ b/solr/core/src/test/org/apache/solr/security/JWTAuthPluginIntegrationTest.java @@ -211,11 +211,11 @@ public class JWTAuthPluginIntegrationTest extends SolrCloudAuthTestCase { // Now update three documents assertAuthMetricsMinimums(1, 1, 0, 0, 0, 0); - assertPkiAuthMetricsMinimums(4, 4, 0, 0, 0, 0); + assertPkiAuthMetricsMinimums(2, 2, 0, 0, 0, 0); Pair result = post(baseUrl + "/" + COLLECTION + "/update?commit=true", "[{\"id\" : \"1\"}, {\"id\": \"2\"}, {\"id\": \"3\"}]", jwtTestToken); assertEquals(Integer.valueOf(200), result.second()); assertAuthMetricsMinimums(4, 4, 0, 0, 0, 0); - assertPkiAuthMetricsMinimums(4, 4, 0, 0, 0, 0); + assertPkiAuthMetricsMinimums(2, 2, 0, 0, 0, 0); // First a non distributed query result = get(baseUrl + "/" + COLLECTION + "/query?q=*:*&distrib=false", jwtTestToken); @@ -230,7 +230,7 @@ public class JWTAuthPluginIntegrationTest extends SolrCloudAuthTestCase { // Delete assertEquals(200, get(baseUrl + "/admin/collections?action=DELETE&name=" + COLLECTION, jwtTestToken).second().intValue()); assertAuthMetricsMinimums(11, 11, 0, 0, 0, 0); - assertPkiAuthMetricsMinimums(6, 6, 0, 0, 0, 0); + assertPkiAuthMetricsMinimums(4, 4, 0, 0, 0, 0); } private void getAndFail(String url, String token) { diff --git a/solr/core/src/test/org/apache/solr/security/hadoop/TestSolrCloudWithHadoopAuthPlugin.java b/solr/core/src/test/org/apache/solr/security/hadoop/TestSolrCloudWithHadoopAuthPlugin.java index 27f869e6c39..6538fd51504 100644 --- a/solr/core/src/test/org/apache/solr/security/hadoop/TestSolrCloudWithHadoopAuthPlugin.java +++ b/solr/core/src/test/org/apache/solr/security/hadoop/TestSolrCloudWithHadoopAuthPlugin.java @@ -70,13 +70,13 @@ public class TestSolrCloudWithHadoopAuthPlugin extends SolrCloudAuthTestCase { NUM_SHARDS, REPLICATION_FACTOR); create.process(solrClient); // The metrics counter for wrong credentials here really just means - assertAuthMetricsMinimums(6, 3, 0, 3, 0, 0); + assertAuthMetricsMinimums(4, 2, 0, 2, 0, 0); SolrInputDocument doc = new SolrInputDocument(); doc.setField("id", "1"); solrClient.add(collectionName, doc); solrClient.commit(collectionName); - assertAuthMetricsMinimums(10, 5, 0, 5, 0, 0); + assertAuthMetricsMinimums(8, 4, 0, 4, 0, 0); SolrQuery query = new SolrQuery(); query.setQuery("*:*"); @@ -88,5 +88,5 @@ public class TestSolrCloudWithHadoopAuthPlugin extends SolrCloudAuthTestCase { AbstractDistribZkTestBase.waitForCollectionToDisappear(collectionName, solrClient.getZkStateReader(), true, 330); // cookie was used to avoid re-authentication - assertAuthMetricsMinimums(13, 8, 0, 5, 0, 0); } + assertAuthMetricsMinimums(11, 7, 0, 4, 0, 0); } } diff --git a/solr/core/src/test/org/apache/solr/util/TestSolrCLIRunExample.java b/solr/core/src/test/org/apache/solr/util/TestSolrCLIRunExample.java index 0ef58512382..5721ecde8e9 100644 --- a/solr/core/src/test/org/apache/solr/util/TestSolrCLIRunExample.java +++ b/solr/core/src/test/org/apache/solr/util/TestSolrCLIRunExample.java @@ -40,16 +40,13 @@ import org.apache.commons.exec.ExecuteResultHandler; import org.apache.lucene.util.LuceneTestCase; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.client.solrj.SolrQuery; -import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.client.solrj.embedded.JettyConfig; import org.apache.solr.client.solrj.embedded.JettySolrRunner; import org.apache.solr.client.solrj.impl.CloudSolrClient; import org.apache.solr.client.solrj.impl.HttpSolrClient; import org.apache.solr.client.solrj.response.QueryResponse; -import org.apache.solr.cloud.CloudTestUtils.AutoScalingRequest; import org.apache.solr.cloud.MiniSolrCloudCluster; import org.apache.solr.common.SolrInputDocument; -import org.apache.solr.common.util.NamedList; import org.junit.After; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -514,120 +511,6 @@ public class TestSolrCLIRunExample extends SolrTestCaseJ4 { executor.execute(org.apache.commons.exec.CommandLine.parse("bin/solr stop -p " + bindPort)); } - @Test - public void testInteractiveSolrCloudExampleWithAutoScalingPolicy() throws Exception { - File solrHomeDir = new File(ExternalPaths.SERVER_HOME); - if (!solrHomeDir.isDirectory()) - fail(solrHomeDir.getAbsolutePath() + " not found and is required to run this test!"); - - Path tmpDir = createTempDir(); - File solrExampleDir = tmpDir.toFile(); - - File solrServerDir = solrHomeDir.getParentFile(); - - String[] toolArgs = new String[]{ - "-example", "cloud", - "-serverDir", solrServerDir.getAbsolutePath(), - "-exampleDir", solrExampleDir.getAbsolutePath() - }; - - int bindPort = -1; - try (ServerSocket socket = new ServerSocket(0)) { - bindPort = socket.getLocalPort(); - } - - String collectionName = "testCloudExamplePrompt1"; - - // this test only support launching one SolrCloud node due to how MiniSolrCloudCluster works - // and the need for setting the host and port system properties ... - String userInput = "1\n" + bindPort + "\n" + collectionName + "\n2\n2\n_default\n"; - - // simulate user input from stdin - InputStream userInputSim = new ByteArrayInputStream(userInput.getBytes(StandardCharsets.UTF_8)); - - // capture tool output to stdout - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - PrintStream stdoutSim = new PrintStream(baos, true, StandardCharsets.UTF_8.name()); - - RunExampleExecutor executor = new RunExampleExecutor(stdoutSim); - closeables.add(executor); - - SolrCLI.RunExampleTool tool = new SolrCLI.RunExampleTool(executor, userInputSim, stdoutSim); - try { - tool.runTool(SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(tool.getOptions()), toolArgs)); - } catch (Exception e) { - System.err.println("RunExampleTool failed due to: " + e + - "; stdout from tool prior to failure: " + baos.toString(StandardCharsets.UTF_8.name())); - throw e; - } - - String toolOutput = baos.toString(StandardCharsets.UTF_8.name()); - - // verify Solr is running on the expected port and verify the collection exists - String solrUrl = "http://localhost:" + bindPort + "/solr"; - String collectionListUrl = solrUrl + "/admin/collections?action=list"; - if (!SolrCLI.safeCheckCollectionExists(collectionListUrl, collectionName)) { - fail("After running Solr cloud example, test collection '" + collectionName + - "' not found in Solr at: " + solrUrl + "; tool output: " + toolOutput); - } - - // index some docs - to verify all is good for both shards - CloudSolrClient cloudClient = null; - - try { - cloudClient = getCloudSolrClient(executor.solrCloudCluster.getZkServer().getZkAddress()); - String setClusterPolicyCommand = "{" + - " 'set-cluster-policy': [" + - " {'cores':'<10', 'node':'#ANY'}," + - " {'replica':'<2', 'shard': '#EACH', 'node': '#ANY'}," + - " {'nodeRole':'overseer', 'replica':0}" + - " ]" + - "}"; - @SuppressWarnings({"rawtypes"}) - SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setClusterPolicyCommand); - NamedList response = cloudClient.request(req); - assertEquals(response.get("result").toString(), "success"); - SolrCLI.CreateCollectionTool createCollectionTool = new SolrCLI.CreateCollectionTool(stdoutSim); - String[] createArgs = new String[]{"create_collection", "-name", "newColl", "-configsetsDir", "_default", "-solrUrl", solrUrl}; - createCollectionTool.runTool( - SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(createCollectionTool.getOptions()), createArgs)); - solrUrl = "http://localhost:" + bindPort + "/solr"; - collectionListUrl = solrUrl + "/admin/collections?action=list"; - if (!SolrCLI.safeCheckCollectionExists(collectionListUrl, "newColl")) { - toolOutput = baos.toString(StandardCharsets.UTF_8.name()); - fail("After running Solr cloud example, test collection 'newColl' not found in Solr at: " + solrUrl + "; tool output: " + toolOutput); - } - } finally { - if (cloudClient != null) { - try { - cloudClient.close(); - } catch (Exception ignore) { - } - } - } - - File node1SolrHome = new File(solrExampleDir, "cloud/node1/solr"); - if (!node1SolrHome.isDirectory()) { - fail(node1SolrHome.getAbsolutePath()+" not found! run cloud example failed; tool output: "+toolOutput); - } - - // delete the collection - SolrCLI.DeleteTool deleteTool = new SolrCLI.DeleteTool(stdoutSim); - String[] deleteArgs = new String[] { "-name", collectionName, "-solrUrl", solrUrl }; - deleteTool.runTool( - SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(deleteTool.getOptions()), deleteArgs)); - deleteTool = new SolrCLI.DeleteTool(stdoutSim); - deleteArgs = new String[]{"-name", "newColl", "-solrUrl", solrUrl}; - deleteTool.runTool( - SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(deleteTool.getOptions()), deleteArgs)); - - // dump all the output written by the SolrCLI commands to stdout - //System.out.println(toolOutput); - - // stop the test instance - executor.execute(org.apache.commons.exec.CommandLine.parse("bin/solr stop -p "+bindPort)); - } - @Test public void testFailExecuteScript() throws Exception { File solrHomeDir = new File(ExternalPaths.SERVER_HOME); diff --git a/solr/core/src/test/org/apache/solr/util/TestUtils.java b/solr/core/src/test/org/apache/solr/util/TestUtils.java index 65a053641c6..4e34328d8cb 100644 --- a/solr/core/src/test/org/apache/solr/util/TestUtils.java +++ b/solr/core/src/test/org/apache/solr/util/TestUtils.java @@ -324,7 +324,6 @@ public class TestUtils extends SolrTestCaseJ4 { assertEquals("b1", Utils.getObjectByPath(sink, true, "k1/k11/a1")); sink = new HashMap<>(); - sink.put("autoAddReplicas", "false"); assertTrue(Utils.mergeJson(sink, (Map) Utils.fromJSONString("collectionDefaults:{numShards:3 , nrtReplicas:2}"))); assertEquals(3L, Utils.getObjectByPath(sink, true, ImmutableList.of(COLLECTION_DEF, NUM_SHARDS_PROP))); assertEquals(2L, Utils.getObjectByPath(sink, true, ImmutableList.of(COLLECTION_DEF, NRT_REPLICAS))); diff --git a/solr/solr-ref-guide/src/cluster-node-management.adoc b/solr/solr-ref-guide/src/cluster-node-management.adoc index 3b3de30a23a..b930db62f26 100644 --- a/solr/solr-ref-guide/src/cluster-node-management.adoc +++ b/solr/solr-ref-guide/src/cluster-node-management.adoc @@ -130,7 +130,7 @@ Add, edit or delete a cluster-wide property. === CLUSTERPROP Parameters `name`:: -The name of the property. Supported properties names are `autoAddReplicas`, `location`, `maxCoresPerNode`, `urlScheme` and `defaultShardPreferences`. Other properties can be set +The name of the property. Supported properties names are `location`, `maxCoresPerNode`, `urlScheme` and `defaultShardPreferences`. Other properties can be set (for example, if you need them for custom plugins) but they must begin with the prefix `ext.`. Unknown properties that don't begin with `ext.` will be rejected. `val`:: @@ -281,30 +281,6 @@ http://localhost:8983/solr/admin/collections?action=BALANCESHARDUNIQUE&collectio Examining the clusterstate after issuing this call should show exactly one replica in each shard that has this property. -[[utilizenode]] -== UTILIZENODE: Utilize a New Node - -This command can be used to move some replicas from the existing nodes to either a new node or a less loaded node to reduce the load on the existing node. - -This uses your autoscaling policies and preferences to identify which replica needs to be moved. It tries to fix any policy violations first and then it tries to move some load off of the most loaded nodes according to the preferences. - -`/admin/collections?action=UTILIZENODE&node=nodeName` - -=== UTILIZENODE Parameters - -`node`:: The name of the node that needs to be utilized. This parameter is required. - -[[replacenode]] -== REPLACENODE: Move All Replicas in a Node to Another - -This command recreates replicas in one node (the source) to another node(s) (the target). After each replica is copied, the replicas in the source node are deleted. - -For source replicas that are also shard leaders the operation will wait for the number of seconds set with the `timeout` parameter to make sure there's an active replica that can become a leader (either an existing replica becoming a leader or the new replica completing recovery and becoming a leader). - -The API uses the Autoscaling framework to find nodes that can satisfy the disk requirements for the new replicas but only when an Autoscaling policy is configured. Refer to <> section for more details. - -`/admin/collections?action=REPLACENODE&sourceNode=_source-node_&targetNode=_target-node_` - === REPLACENODE Parameters `sourceNode`:: diff --git a/solr/solr-ref-guide/src/collection-management.adoc b/solr/solr-ref-guide/src/collection-management.adoc index aa22dc3f253..b2bef1e4232 100644 --- a/solr/solr-ref-guide/src/collection-management.adoc +++ b/solr/solr-ref-guide/src/collection-management.adoc @@ -87,11 +87,6 @@ Please note that <> or retrieval by `property._name_=_value_`:: Set core property _name_ to _value_. See the section <> for details on supported properties and values. -`autoAddReplicas`:: -When set to `true`, enables automatic addition of replicas when the number of active replicas falls below the value set for `replicationFactor`. This may occur if a replica goes down, for example. The default is `false`, which means new replicas will not be added. -+ -While this parameter is provided as part of Solr's set of features to provide autoscaling of clusters, it is available even when you have not implemented any other part of autoscaling (such as a policy). See the section <> for more details about this option and how it can be used. - `async`:: Request ID to track this action which will be <>. @@ -101,15 +96,9 @@ Replica placement rules. See the section <> for details. -`policy`:: Name of the collection-level policy. See <> for details. - `waitForFinalState`:: If `true`, the request will complete only when all affected replicas become active. The default is `false`, which means that the API will return the status of the single action, which may be before the new replica is online and active. -`withCollection`:: -The name of the collection with which all replicas of this collection must be co-located. The collection must already exist and must have a single shard named `shard1`. -See <> for more details. - `alias`:: Starting with version 8.1 when a collection is created additionally an alias can be created that points to this collection. This parameter allows specifying the name of this alias, effectively combining @@ -235,7 +224,6 @@ At least one `_attribute_` parameter is required. The attributes that can be modified are: * replicationFactor -* autoAddReplicas * collection.configName * rule * snitch @@ -714,7 +702,7 @@ There's a number of optional parameters that determine the target collection lay are not specified in the request then their values are copied from the source collection. The following parameters are currently supported (described in details in the <> section): `numShards`, `replicationFactor`, `nrtReplicas`, `tlogReplicas`, `pullReplicas`, -`autoAddReplicas`, `shards`, `policy`, `createNodeSet`, `createNodeSet.shuffle`, `router.*`. +`shards`, `policy`, `createNodeSet`, `createNodeSet.shuffle`, `router.*`. `removeSource`:: Optional boolean. If true then after the processing is successfully finished the source collection will @@ -897,7 +885,6 @@ http://localhost:8983/solr/admin/collections?action=COLSTATUS&collection=getting "gettingstarted": { "znodeVersion": 16, "properties": { - "autoAddReplicas": "false", "nrtReplicas": "2", "pullReplicas": "0", "replicationFactor": "2", @@ -1041,7 +1028,6 @@ http://localhost:8983/solr/admin/collections?action=COLSTATUS&collection=getting "gettingstarted": { "znodeVersion": 33, "properties": { - "autoAddReplicas": "false", "nrtReplicas": "2", "pullReplicas": "0", "replicationFactor": "2", @@ -1289,9 +1275,6 @@ The number of TLOG replicas to create for this collection. This type of replica `pullReplicas`:: The number of PULL replicas to create for this collection. This type of replica does not maintain a transaction log and only updates its index via replication from a leader. This type is not eligible to become a leader and should not be the only type of replicas in the collection. See the section <> for more information about replica types. -`autoAddReplicas`:: -When set to `true`, enables auto addition of replicas on shared file systems. See the section <> for more details on settings and overrides. - `property._name_=_value_`:: Set core property _name_ to _value_. See the section <> for details on supported properties and values. diff --git a/solr/solr-ref-guide/src/colocating-collections.adoc b/solr/solr-ref-guide/src/colocating-collections.adoc deleted file mode 100644 index e8e9ce123a9..00000000000 --- a/solr/solr-ref-guide/src/colocating-collections.adoc +++ /dev/null @@ -1,75 +0,0 @@ -= Colocating Collections -:toclevels: 1 -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -Solr provides a way to colocate a collection with another so that cross-collection joins are always possible. - -The colocation guarantee applies to all future Collection operations made either via Collections API or by Autoscaling -actions. - -A collection may only be colocated with exactly one `withCollection`. However, arbitrarily many collections may be -_linked_ to the same `withCollection`. - -== Create a Colocated Collection -The Create Collection API supports a parameter named `withCollection` which can be used to specify a collection -with which the replicas of the newly created collection should be colocated. See <>. - -`/admin/collections?action=CREATE&name=techproducts&numShards=1&replicationFactor=2&withCollection=tech_categories` - -In the above example, all replicas of the `techproducts` collection will be colocated on a node with at least one -replica of the `tech_categories` collection. - -== Colocating Existing Collections -When collections already exist beforehand, the <> can be -used to set the `withCollection` parameter so that the two collections can be linked. This will *not* trigger -changes to the cluster automatically because moving a large number of replicas immediately might de-stabilize the system. -Instead, it is recommended that the Suggestions UI page should be consulted on the operations that can be performed -to change the cluster manually. - -Example: -`/admin/collections?action=MODIFYCOLLECTION&collection=techproducts&withCollection=tech_categories` - -== Deleting Colocated Collections -Deleting a collection which has been linked to another will fail unless the link itself is deleted first by using the -<> to un-set the `withCollection` attribute. - -Example: -`/admin/collections?action=MODIFYCOLLECTION&collection=techproducts&withCollection=` - -== Limitations and Caveats - -The collection being used as the `withCollection` must have one shard only and that shard should be named `shard1`. Note -that when using the default router, the shard name is always set to `shard1` but special care must be taken to name the -shard as `shard1` when using the implicit router. - -In case new replicas of the `withCollection` have to be added to maintain the colocation guarantees then the new replicas -will be of type `NRT` only. Automatically creating replicas of `TLOG` or `PULL` types is not supported. - -In case, replicas have to be moved from one node to another, perhaps in response to a node lost trigger, then the target -nodes will be chosen by preferring nodes that already have a replica of the `withCollection` so that the number of moves -is minimized. However, this also means that unless there are Autoscaling policy violations, Solr will continue to move -such replicas to already loaded nodes instead of preferring empty nodes. Therefore, it is advised to have policy rules -which can prevent such overloading by e.g., setting the maximum number of cores per node to a fixed value. - -Example: -`{'cores' : '<8', 'node' : '#ANY'}` - -The colocation guarantee is one-way only i.e., a collection 'X' colocated with 'Y' will always have one or more -replicas of 'Y' on any node that has a replica of 'X' but the reverse is not true. There may be nodes which have one or -more replicas of 'Y' but no replicas of 'X'. Such replicas of 'Y' will not be considered a violation of colocation -rules and will not be cleaned up automatically. diff --git a/solr/solr-ref-guide/src/major-changes-in-solr-7.adoc b/solr/solr-ref-guide/src/major-changes-in-solr-7.adoc index 9b235345032..e04f2cdbaef 100644 --- a/solr/solr-ref-guide/src/major-changes-in-solr-7.adoc +++ b/solr/solr-ref-guide/src/major-changes-in-solr-7.adoc @@ -47,8 +47,6 @@ Solr autoscaling is a new suite of features in Solr to make managing a SolrCloud At its core, Solr autoscaling provides users with a rule syntax to define preferences and policies for how to distribute nodes and shards in a cluster, with the goal of maintaining a balance in the cluster. As of Solr 7, Solr will take any policy or preference rules into account when determining where to place new shards and replicas created or moved with various Collections API commands. -See the section <> for details on the options available in 7.0. Expect more features to be released in subsequent 7.x releases in this area. - === Other Features & Enhancements // TODO 7.1 - update link to docs when complete diff --git a/solr/solr-ref-guide/src/major-changes-in-solr-8.adoc b/solr/solr-ref-guide/src/major-changes-in-solr-8.adoc index b5628ceb8ea..13184f86c08 100644 --- a/solr/solr-ref-guide/src/major-changes-in-solr-8.adoc +++ b/solr/solr-ref-guide/src/major-changes-in-solr-8.adoc @@ -256,8 +256,6 @@ curl -X POST -H 'Content-type:application/json' --data-binary ' ---- * A new command-line option is available via `bin/solr autoscaling` to calculate autoscaling policy suggestions and diagnostic information outside of the running Solr cluster. This option can use the existing autoscaling policy, or test the impact of a new one from a file located on the server filesystem. -+ -These options have been documented in the section <>. === Dependency Updates in 8.0 @@ -414,7 +412,7 @@ When upgrading to Solr 7.3, users should be aware of the following major changes * The behaviour of the autoscaling system will now pause all triggers from execution between the start of actions and the end of a cool down period. The triggers will resume after the cool down period expires. Previously, the cool down period was a fixed period started after actions for a trigger event completed and during this time all triggers continued to run but any events were rejected and tried later. -* The throttling mechanism used to limit the rate of autoscaling events processed has been removed. This deprecates the `actionThrottlePeriodSeconds` setting in the <> which is now non-operational. Use the `triggerCooldownPeriodSeconds` parameter instead to pause event processing. +* The throttling mechanism used to limit the rate of autoscaling events processed has been removed. This deprecates the `actionThrottlePeriodSeconds` setting in the Autoscaling API which is now non-operational. Use the `triggerCooldownPeriodSeconds` parameter instead to pause event processing. * The default value of `autoReplicaFailoverWaitAfterExpiration`, used with the AutoAddReplicas feature, has increased to 120 seconds from the previous default of 30 seconds. This affects how soon Solr adds new replicas to replace the replicas on nodes which have either crashed or shutdown. @@ -480,8 +478,6 @@ Existing users of this feature should not have to change anything. However, they ** Behavior: Changing the `autoAddReplicas` property from disabled (`false`) to enabled (`true`) using <> no longer replaces down replicas for the collection immediately. Instead, replicas are only added if a node containing them went down while `autoAddReplicas` was enabled. The parameters `autoReplicaFailoverBadNodeExpiration` and `autoReplicaFailoverWorkLoopDelay` are no longer used. ** Deprecations: Enabling/disabling autoAddReplicas cluster-wide with the API will be deprecated; use suspend/resume trigger APIs with `name=".auto_add_replicas"` instead. -+ -More information about the changes to this feature can be found in the section <>. *Metrics Reporters* diff --git a/solr/solr-ref-guide/src/major-changes-in-solr-9.adoc b/solr/solr-ref-guide/src/major-changes-in-solr-9.adoc index d2384324597..ff02e8fa1fc 100644 --- a/solr/solr-ref-guide/src/major-changes-in-solr-9.adoc +++ b/solr/solr-ref-guide/src/major-changes-in-solr-9.adoc @@ -108,6 +108,16 @@ _(raw; not yet edited)_ * SOLR-12823: Remove /clusterstate.json support, i.e. support for collections created with stateFormat=1 as well as support for Collection API MIGRATESTATEFORMAT action. Also removes support for cluster property `legacyCloud` (as if always false now). +* SOLR-14656: Autoscaling framework removed + This includes: + Autoscaling, policy, triggers etc. + withCollection handling + UTILIZENODE command + Sim framework + Suggestions tab in UI + Reference guide pages for autoscaling + autoAddReplicas feature + === Upgrade Prerequisites in Solr 9 * Upgrade all collections in stateFormat=1 to stateFormat=2 *before* upgrading to Solr 9, as Solr 9 does not support the diff --git a/solr/solr-ref-guide/src/metrics-history.adoc b/solr/solr-ref-guide/src/metrics-history.adoc index 5238642bc4e..992a36fc230 100644 --- a/solr/solr-ref-guide/src/metrics-history.adoc +++ b/solr/solr-ref-guide/src/metrics-history.adoc @@ -19,7 +19,7 @@ Solr collects long-term history of certain key metrics both in SolrCloud and in standalone mode. This information can be used for very simple monitoring and troubleshooting, but also some -SolrCloud components (e.g., autoscaling) can use this data for making informed decisions based on +SolrCloud components can use this data for making informed decisions based on long-term trends of selected metrics. [IMPORTANT] diff --git a/solr/solr-ref-guide/src/migrate-to-policy-rule.adoc b/solr/solr-ref-guide/src/migrate-to-policy-rule.adoc deleted file mode 100644 index 38b0cd6baee..00000000000 --- a/solr/solr-ref-guide/src/migrate-to-policy-rule.adoc +++ /dev/null @@ -1,198 +0,0 @@ -= Migrating Rule-Based Replica Rules to Autoscaling Policies -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -Creating rules for replica placement in a Solr cluster is now done with the <>. - -This document outlines how to migrate from the legacy <> to an <>. - -The autoscaling framework is designed to fully automate your cluster management. -However, if you do not want actions taken on your cluster in an automatic way, you can still use the framework to set rules and preferences. -With a set of rules and preferences in place, instead of taking action directly the system will suggest actions you can take manually. - -The section <> describes the capabilities of an autoscaling policy in detail. -Below we'll walk through a few examples to show how you would express the your legacy rules in the autoscaling syntax. -Every rule in the legacy rule-based replica framework can be expressed in the new syntax. - -== How Rules are Defined - -One key difference between the frameworks is the way rules are defined. - -With the rule-based replica placement framework, rules are defined with the Collections API at the time of collection creation. - -The autoscaling framework, however, has its own <>. -Policies can be configured for the entire cluster or for individual collections depending on your needs. - -The following is the legacy syntax for a rule that limits the cluster to one replica for each shard in any Solr node: - -[source,text] ----- -replica:<2,node:*,shard:** ----- - -The equivalent rule in the autoscaling policy is: - -[source,json] ----- -{"replica":"<2", "node":"#ANY", "shard":"#EACH"} ----- - -== Differences in Rule Syntaxes - -Many elements of defining rules are similar in both frameworks, but some elements are different. - -[[rule-operators1]] -=== Rule Operators - -All of the following operators can be directly used in the new policy syntax and they mean the same in both frameworks. - -* *equals (no operator required)*: `tag:x` means the value for a tag must be equal to `'x'`. -* *greater than (>)*: `tag:>x` means the tag value must be greater than `'x'`. In this case, `'x'` must be a number. -* *less than (<)*: `tag:>. - -The following examples are intended to help you translate your existing rules into new rules that fit the autoscaling framework. - -*Keep less than 2 replicas (at most 1 replica) of this collection on any node* - -For this rule, we define the `replica` condition with operators for "less than 2", and use a pre-defined tag named `node` to define nodes with any name. - -.Rule-based replica placement framework: -[source,text] ----- -replica:<2,node:* ----- - -.Autoscaling framework: -[source,json] ----- -{"replica":"<2","node":"#ANY"} ----- - -*For a given shard, keep less than 2 replicas on any node* - -For this rule, we use the `shard` condition to define any shard, the `replica` condition with operators for "less than 2", and finally a pre-defined tag named `node` to define nodes with any name. - -.Rule-based replica placement framework: -[source,text] ----- -shard:*,replica:<2,node:* ----- - -.Autoscaling framework: -[source,json] ----- -{"replica":"<2","shard":"#EACH", "node":"#ANY"} ----- - -*Assign all replicas in shard1 to rack 730* - -This rule limits the `shard` condition to 'shard1', but any number of replicas. We're also referencing a custom tag named `rack`. - -.Rule-based replica placement framework: -[source,text] ----- -shard:shard1,replica:*,rack:730 ----- - -.Autoscaling framework: -[source,json] ----- -{"replica":"#ALL", "shard":"shard1", "sysprop.rack":"730"} ----- - -In the rule-based replica placement framework, we needed to configure a custom Snitch which provides values for the tag `rack`. - -With the autoscaling framework, however, we need to start all nodes with a system property to define the rack values. For example, `bin/solr start -c -Drack=`. - -*Create replicas in nodes with less than 5 cores only* - -This rule uses the `replica` condition to define any number of replicas, but adds a pre-defined tag named `core` and uses operators for "less than 5". - -.Rule-based replica placement framework: -[source,text] ----- -cores:<5 ----- - -.Autoscaling framework: -[source,json] ----- -{"cores":"<5", "node":"#ANY"} ----- - -*Do not create any replicas in host 192.45.67.3* - -.legacy syntax: -[source,text] ----- -host:!192.45.67.3 ----- - -.autoscaling framework: -[source,json] ----- -{"replica": 0, "host":"192.45.67.3"} ----- diff --git a/solr/solr-ref-guide/src/replica-management.adoc b/solr/solr-ref-guide/src/replica-management.adoc index 5d110ae4fb0..6896995c03e 100644 --- a/solr/solr-ref-guide/src/replica-management.adoc +++ b/solr/solr-ref-guide/src/replica-management.adoc @@ -24,8 +24,6 @@ A replica is a physical copy of a shard. Add one or more replicas to a shard in a collection. The node name can be specified if the replica is to be created in a specific node. Otherwise, a set of nodes can be specified and the most suitable ones among them will be chosen to create the replica(s). -The API uses the Autoscaling framework to find nodes that can satisfy the disk requirements for the new replica(s) but only when an Autoscaling preferences or policy is configured. Refer to <> section for more details. - `/admin/collections?action=ADDREPLICA&collection=_collection_&shard=_shard_&node=_nodeName_` === ADDREPLICA Parameters diff --git a/solr/solr-ref-guide/src/rule-based-authorization-plugin.adoc b/solr/solr-ref-guide/src/rule-based-authorization-plugin.adoc index f75336aeb7b..28637700a6e 100644 --- a/solr/solr-ref-guide/src/rule-based-authorization-plugin.adoc +++ b/solr/solr-ref-guide/src/rule-based-authorization-plugin.adoc @@ -214,21 +214,6 @@ The predefined permission names (and their effects) are: * *config-read*: this permission is allowed to read a collection's configuration using the <>, the <>, and other APIs which modify `configoverlay.json`. Note that this allows configuration read permissions for _all_ collections. If read permissions should only be applied to specific collections, a custom permission would need to be created. * *metrics-read*: this permission allows access to Solr's <> * *metrics-history-read*: this permission allows access to Solr's <>, which provides long-term history for a select set of key Solr metrics. -* *autoscaling-read*: this permission allows users to read Solr's <> configuration. This covers all read-only autoscaling APIs, including: -** the "READ" API (`/solr/admin/autoscaling`) -** the Diagnostics API (`/solr/admin/autoscaling/diagnostics`) -** the Suggestions API (`/solr/admin/autoscaling/suggestions`) -** The History API (`/solr/admin/autoscaling/history`) -* *autoscaling-write*: this permission allows users to make changes to Solr's <> configuration. This covers all operations in the autoscaling Write API, including: -** set-cluster-preferences -** set-cluster-policy -** set-policy -** remove-policy -** set-trigger -** remove-trigger -** set-listener -** remove-listener -** set-properties * *core-admin-edit*: Core admin commands that can mutate the system state. * *core-admin-read*: Read operations on the core admin API * *collection-admin-edit*: this permission is allowed to edit a collection's configuration using the <>. Note that this allows configuration edit permissions for _all_ collections. If edit permissions should only be applied to specific collections, a custom permission would need to be created. Specifically, the following actions of the Collections API would be allowed: diff --git a/solr/solr-ref-guide/src/running-solr-on-hdfs.adoc b/solr/solr-ref-guide/src/running-solr-on-hdfs.adoc index e74b7a57bac..919a1fb9d94 100644 --- a/solr/solr-ref-guide/src/running-solr-on-hdfs.adoc +++ b/solr/solr-ref-guide/src/running-solr-on-hdfs.adoc @@ -178,42 +178,3 @@ If using Kerberos, you will need to add the three Kerberos related properties to ---- -// In Solr 8, this should be removed entirely; -// it's here now only for back-compat for existing users - -== Automatically Add Replicas in SolrCloud - -The ability to automatically add new replicas when the Overseer notices that a shard has gone down was previously only available to users running Solr in HDFS, but it is now available to all users via Solr's autoscaling framework. See the section <> for details on how to enable and disable this feature. - -[WARNING] -==== -The ability to enable or disable the autoAddReplicas feature with cluster properties has been deprecated and will be removed in a future version. All users of this feature who have previously used that approach are encouraged to change their configurations to use the autoscaling framework to ensure continued operation of this feature in their Solr installations. - -For users using this feature with the deprecated configuration, you can temporarily disable it cluster-wide by setting the cluster property `autoAddReplicas` to `false`, as in these examples: - -.V1 API -[source,bash] ----- -http://localhost:8983/solr/admin/collections?action=CLUSTERPROP&name=autoAddReplicas&val=false ----- - -.V2 API -[source,bash] ----- -curl -X POST -H 'Content-type: application/json' -d '{"set-property": {"name":"autoAddReplicas", "val":false}}' http://localhost:8983/api/cluster ----- - -Re-enable the feature by unsetting the `autoAddReplicas` cluster property. When no `val` parameter is provided, the cluster property is unset: - -.V1 API -[source,bash] ----- -http://localhost:8983/solr/admin/collections?action=CLUSTERPROP&name=autoAddReplicas ----- - -.V2 API -[source,bash] ----- -curl -X POST -H 'Content-type: application/json' -d '{"set-property": {"name":"autoAddReplicas"}}' http://localhost:8983/api/cluster ----- -==== diff --git a/solr/solr-ref-guide/src/shard-management.adoc b/solr/solr-ref-guide/src/shard-management.adoc index 228d5f94545..d06366673fa 100644 --- a/solr/solr-ref-guide/src/shard-management.adoc +++ b/solr/solr-ref-guide/src/shard-management.adoc @@ -34,9 +34,9 @@ The split is performed by dividing the original shard's hash range into two equa The newly created shards will have as many replicas as the parent shard, of the same replica types. -When using `splitMethod=rewrite` (default) you must ensure that the node running the leader of the parent shard has enough free disk space i.e., more than twice the index size, for the split to succeed. The API uses the Autoscaling framework to find nodes that can satisfy the disk requirements for the new replicas but only when an Autoscaling policy is configured. Refer to <> section for more details. +When using `splitMethod=rewrite` (default) you must ensure that the node running the leader of the parent shard has enough free disk space i.e., more than twice the index size, for the split to succeed. -Also, the first replicas of resulting sub-shards will always be placed on the shard leader node, which may cause Autoscaling policy violations that need to be resolved either automatically (when appropriate triggers are in use) or manually. +Also, the first replicas of resulting sub-shards will always be placed on the shard leader node. Shard splitting can be a long running process. In order to avoid timeouts, you should run this as an <>. @@ -209,8 +209,6 @@ Use SPLITSHARD for collections created with the 'compositeId' router (`router.ke The default values for `replicationFactor` or `nrtReplicas`, `tlogReplicas`, `pullReplicas` from the collection is used to determine the number of replicas to be created for the new shard. This can be customized by explicitly passing the corresponding parameters to the request. -The API uses the Autoscaling framework to find the best possible nodes in the cluster when an Autoscaling preferences or policy is configured. Refer to <> section for more details. - === CREATESHARD Parameters `collection`:: diff --git a/solr/solr-ref-guide/src/solr-upgrade-notes.adoc b/solr/solr-ref-guide/src/solr-upgrade-notes.adoc index f6ec8a40f45..ecf44893620 100644 --- a/solr/solr-ref-guide/src/solr-upgrade-notes.adoc +++ b/solr/solr-ref-guide/src/solr-upgrade-notes.adoc @@ -66,10 +66,8 @@ More information about this new feature is available in the section <>. -* The <> now supports a collection selector to identify collections based on collection properties to determine which collections should be operated on. +* The ComputePlan action now supports a collection selector to identify collections based on collection properties to determine which collections should be operated on. *Security* diff --git a/solr/solr-ref-guide/src/solrcloud-autoscaling-api.adoc b/solr/solr-ref-guide/src/solrcloud-autoscaling-api.adoc deleted file mode 100644 index 4d4ff58d972..00000000000 --- a/solr/solr-ref-guide/src/solrcloud-autoscaling-api.adoc +++ /dev/null @@ -1,808 +0,0 @@ -= Autoscaling API -:toclevels: 2 -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -The Autoscaling API is used to manage autoscaling policies, preferences, triggers, listeners and to get diagnostics on the state of the cluster. - -== Read API - -The autoscaling Read API is available at `/solr/admin/autoscaling` or `/api/cluster/autoscaling` (v2 API style). It returns information about the configured cluster preferences, cluster policy, collection-specific policies triggers and listeners. - -This API does not take any parameters. - -=== Read API Response - -The output will contain cluster preferences, cluster policy and collection specific policies. - -=== Examples using Read API - -*Output* - -[source,json] ----- -{ - "responseHeader": { - "status": 0, - "QTime": 2 - }, - "cluster-policy": [ - { - "replica": "<2", - "shard": "#EACH", - "node": "#ANY" - } - ], - "WARNING": "This response format is experimental. It is likely to change in the future." -} ----- - -== Diagnostics API - -The diagnostics API shows the violations, if any, of all conditions in the cluster and, if applicable, the collection-specific policy. It is available at the `/admin/autoscaling/diagnostics` path. - -This API does not take any parameters. - -=== Diagnostics API Response - -The output will contain `sortedNodes` which is a list of nodes in the cluster sorted according to overall load in descending order (as determined by the preferences) and `violations` which is a list of nodes along with the conditions that they violate. - -=== Examples Using Diagnostics API - -Here is an example with no violations but in the `sortedNodes` section, we can see that the first node is most loaded (according to number of cores): - -[source,json] ----- -{ - "responseHeader": { - "status": 0, - "QTime": 65 - }, - "diagnostics": { - "sortedNodes": [ - { - "node": "127.0.0.1:8983_solr", - "cores": 3 - }, - { - "node": "127.0.0.1:7574_solr", - "cores": 2 - } - ], - "violations": [] - }, - "WARNING": "This response format is experimental. It is likely to change in the future." -} ----- - -Suppose we added a condition to the cluster policy as follows: - -[source,json] ----- -{"replica": "<2", "shard": "#EACH", "node": "#ANY"} ----- - -However, since the first node in the first example had more than 1 replica for a shard already, then the diagnostics API will return: - -[source,json] ----- -{ - "responseHeader": { - "status": 0, - "QTime": 45 - }, - "diagnostics": { - "sortedNodes": [ - { - "node": "127.0.0.1:8983_solr", - "cores": 3 - }, - { - "node": "127.0.0.1:7574_solr", - "cores": 2 - } - ], - "violations": [ - { - "collection": "gettingstarted", - "shard": "shard1", - "node": "127.0.0.1:8983_solr", - "tagKey": "127.0.0.1:8983_solr", - "violation": { - "replica": "2", - "delta": 0 - }, - "clause": { - "replica": "<2", - "shard": "#EACH", - "node": "#ANY", - "collection": "gettingstarted" - } - } - ] - }, - "WARNING": "This response format is experimental. It is likely to change in the future." -} ----- - -In the above example the node with port 8983 has two replicas for `shard1` in violation of our policy. - -=== Inline Policy Configuration - -If there is no autoscaling policy configured or if you wish to use a configuration other than the default, it is possible to send the autoscaling policy JSON as an inline payload as follows: - -[source,bash] ----- - curl -X POST -H 'Content-type:application/json' -d '{ - "cluster-policy": [ - {"replica": 0, "put" : "on-each", "nodeset": {"port" : "7574"}}] - }' http://localhost:8983/api/cluster/autoscaling/diagnostics?omitHeader=true ----- - -*Output* -[source,json] ----- -{ - "diagnostics":{ - "sortedNodes":[{ - "node":"10.0.0.80:7574_solr", - "isLive":true, - "cores":2.0, - "freedisk":567.4989128112793, - "port":7574, - "totaldisk":1044.122688293457, - "replicas":{"mycoll":{ - "shard2":[{ - "core_node7":{ - "core":"mycoll_shard2_replica_n4", - "shard":"shard2", - "collection":"mycoll", - "node_name":"10.0.0.80:7574_solr", - "type":"NRT", - "base_url":"http://10.0.0.80:7574/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":6.426125764846802E-8}}], - "shard1":[{ - "core_node3":{ - "core":"mycoll_shard1_replica_n1", - "shard":"shard1", - "collection":"mycoll", - "node_name":"10.0.0.80:7574_solr", - "type":"NRT", - "base_url":"http://10.0.0.80:7574/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":6.426125764846802E-8}}]}}} - ,{ - "node":"10.0.0.80:8983_solr", - "isLive":true, - "cores":2.0, - "freedisk":567.498908996582, - "port":8983, - "totaldisk":1044.122688293457, - "replicas":{"mycoll":{ - "shard2":[{ - "core_node8":{ - "core":"mycoll_shard2_replica_n6", - "shard":"shard2", - "collection":"mycoll", - "node_name":"10.0.0.80:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://10.0.0.80:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":6.426125764846802E-8}}], - "shard1":[{ - "core_node5":{ - "core":"mycoll_shard1_replica_n2", - "shard":"shard1", - "collection":"mycoll", - "node_name":"10.0.0.80:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://10.0.0.80:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":6.426125764846802E-8}}]}}}], - "liveNodes":["10.0.0.80:7574_solr", - "10.0.0.80:8983_solr"], - "violations":[{ - "collection":"mycoll", - "tagKey":7574, - "violation":{ - "replica":{ - "NRT":2, - "count":2}, - "delta":2.0}, - "clause":{ - "replica":0, - "port":"7574", - "collection":"mycoll"}, - "violatingReplicas":[{ - "core_node7":{ - "core":"mycoll_shard2_replica_n4", - "shard":"shard2", - "collection":"mycoll", - "node_name":"10.0.0.80:7574_solr", - "type":"NRT", - "base_url":"http://10.0.0.80:7574/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":6.426125764846802E-8}} - ,{ - "core_node3":{ - "core":"mycoll_shard1_replica_n1", - "shard":"shard1", - "collection":"mycoll", - "node_name":"10.0.0.80:7574_solr", - "type":"NRT", - "base_url":"http://10.0.0.80:7574/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":6.426125764846802E-8}}]}], - "config":{ - "cluster-policy":[{ - "replica":0, - "port":"7574"}]}}, - "WARNING":"This response format is experimental. It is likely to change in the future."} ----- - -== Suggestions API -Suggestions are operations recommended by the system according to the policies and preferences the user has set. - -Suggestions are made only if there are `violations` to active policies. The `operation` section of the response uses the defined preferences to identify the target node. - -The API is available at `/admin/autoscaling/suggestions`. Here is an example output from a suggestion request: - -[source,json] ----- -{ - "responseHeader":{ - "status":0, - "QTime":101}, - "suggestions":[{ - "type":"violation", - "violation":{ - "collection":"mycoll", - "shard":"shard2", - "tagKey":"7574", - "violation":{ "delta":-1}, - "clause":{ - "replica":"0", - "shard":"#EACH", - "port":7574, - "collection":"mycoll"}}, - "operation":{ - "method":"POST", - "path":"/c/mycoll", - "command":{"move-replica":{ - "targetNode":"192.168.43.37:8983_solr", - "replica":"core_node7"}}}}, - { - "type":"violation", - "violation":{ - "collection":"mycoll", - "shard":"shard2", - "tagKey":"7574", - "violation":{ "delta":-1}, - "clause":{ - "replica":"0", - "shard":"#EACH", - "port":7574, - "collection":"mycoll"}}, - "operation":{ - "method":"POST", - "path":"/c/mycoll", - "command":{"move-replica":{ - "targetNode":"192.168.43.37:7575_solr", - "replica":"core_node15"}}}}], - "WARNING":"This response format is experimental. It is likely to change in the future."} ----- - -The suggested `operation` is an API call that can be invoked to remedy the current violation. - -The types of suggestions available are - -* `violation`: Fixes a violation to one or more policy rules -* `repair`: Add missing replicas -* `improvement`: move replicas around so that the load is more evenly balanced according to the autoscaling preferences - -By default, the suggestions API returns all of the above, in that order. However it is possible to fetch only certain types by adding a request parameter `type`. e.g: `type=violation&type=repair` - -=== Inline Policy Configuration - -If there is no autoscaling policy configured or if you wish to use a configuration other than the default, it is possible to send the autoscaling policy JSON as an inline payload as follows: - -[source,bash] ----- -curl -X POST -H 'Content-type:application/json' -d '{ - "cluster-policy": [ - {"replica": 0, "put" : "on-each-node", "nodeset": {"port" : "7574"}} - ] -}' http://localhost:8983/solr/admin/autoscaling/suggestions?omitHeader=true ----- - -*Output* -[source,json] ----- -{ - "suggestions":[{ - "type":"violation", - "violation":{ - "collection":"mycoll", - "tagKey":7574, - "violation":{ - "replica":{ - "NRT":2, - "count":2}, - "delta":2.0}, - "clause":{ - "replica":0, - "port":"7574", - "collection":"mycoll"}}, - "operation":{ - "method":"POST", - "path":"/c/mycoll", - "command":{"move-replica":{ - "targetNode":"10.0.0.80:8983_solr", - "inPlaceMove":"true", - "replica":"core_node8"}}}}, - { - "type":"violation", - "violation":{ - "collection":"mycoll", - "tagKey":7574, - "violation":{ - "replica":{ - "NRT":2, - "count":2}, - "delta":2.0}, - "clause":{ - "replica":0, - "port":"7574", - "collection":"mycoll"}}, - "operation":{ - "method":"POST", - "path":"/c/mycoll", - "command":{"move-replica":{ - "targetNode":"10.0.0.80:8983_solr", - "inPlaceMove":"true", - "replica":"core_node5"}}}}], - "WARNING":"This response format is experimental. It is likely to change in the future."} ----- - -== History API - -The history of autoscaling events is available at `/admin/autoscaling/history`. It returns information -about past autoscaling events and details about their processing. This history is kept in -the `.system` collection, and is populated by a trigger listener `SystemLogListener`. By default this -listener is added to all new triggers. - -History events are regular Solr documents so they can be also accessed directly by -searching on the `.system` collection. The history handler acts as a regular search handler, so all -query parameters supported by `/select` handler for that collection are supported here too. -However, the history handler makes this -process easier by offering a simpler syntax and knowledge of field names -used by `SystemLogListener` for serialization of event data. - -History documents contain the action context, if it was available, which gives -further insight into e.g., exact operations that were computed and/or executed. - -Specifically, the following query parameters can be used (they are turned into -filter queries, so an implicit AND is applied): - -`trigger`:: -The name of the trigger. - -`eventType`:: -The event type or trigger type (e.g., `nodeAdded`). - -`collection`:: -The name of the collection involved in event processing. - -`stage`:: -An event processing stage. - -`action`:: -A trigger action. - -`node`:: -A node name that the event refers to. - -`beforeAction`:: -A `beforeAction` stage. - -`afterAction`:: -An `afterAction` stage. - -// TODO someday add an input example also - -.Example output -[source,json] ----- -{ - "responseHeader": { - "status": 0, - "QTime": 64 - }, - "response": { - "numFound": 2, - "start": 0, - "docs": [ - { - "type": "autoscaling_event", - "source_s": "SystemLogListener", - "id": "15f53efdf4bT2qlmj80580yuu997vktddfob3", - "event.id_s": "14f0d67fe7b97d80T2qlmj80580yuu997vktddfob2", - "event.type_s": "NODELOST", - "event.source_s": ".auto_add_replicas", - "event.time_l": 1508941720006000000, - "timestamp": "2017-10-25T14:29:10.091Z", - "event.property.eventTimes_ss": [ - "1508941720006000000" - ], - "event.property._enqueue_time__ss": [ - "1508941750088000000" - ], - "event.property.nodeNames_ss": [ - "192.168.1.104:7574_solr" - ], - "stage_s": "STARTED", - "event_str": "{\n \"id\":\"14f0d67fe7b97d80T2qlmj80580yuu997vktddfob2\",\n \"source\":\".auto_add_replicas\",\n \"eventTime\":1508941720006000000,\n \"eventType\":\"NODELOST\",\n \"properties\":{\n \"eventTimes\":[1508941720006000000],\n \"_enqueue_time_\":1508941750088000000,\n \"nodeNames\":[\"192.168.1.104:7574_solr\"]}}", - "_version_": 1582240104552857600 - }, - { - "type": "autoscaling_event", - "source_s": "SystemLogListener", - "id": "15f53eff316T2qlmj80580yuu997vktddfob6", - "event.id_s": "14f0d67fe7b97d80T2qlmj80580yuu997vktddfob2", - "event.type_s": "NODELOST", - "event.source_s": ".auto_add_replicas", - "event.time_l": 1508941720006000000, - "timestamp": "2017-10-25T14:29:15.158Z", - "event.property.eventTimes_ss": [ - "1508941720006000000" - ], - "event.property._enqueue_time__ss": [ - "1508941750088000000" - ], - "event.property.nodeNames_ss": [ - "192.168.1.104:7574_solr" - ], - "stage_s": "SUCCEEDED", - "event_str": "{\n \"id\":\"14f0d67fe7b97d80T2qlmj80580yuu997vktddfob2\",\n \"source\":\".auto_add_replicas\",\n \"eventTime\":1508941720006000000,\n \"eventType\":\"NODELOST\",\n \"properties\":{\n \"eventTimes\":[1508941720006000000],\n \"_enqueue_time_\":1508941750088000000,\n \"nodeNames\":[\"192.168.1.104:7574_solr\"]}}", - "_version_": 1582240109859700736 - } - ] - } -} ----- - -== Write API - -The Write API is available at the same `/admin/autoscaling` and `/api/cluster/autoscaling` endpoints as the Read API but can only be used with the *POST* HTTP verb. - -The payload of the POST request is a JSON message with commands to set and remove components. Multiple commands can be specified together in the payload. The commands are executed in the order specified and the changes are atomic, i.e., either all succeed or none. - -=== Create and Modify Cluster Preferences - -Cluster preferences are specified as a list of sort preferences. Multiple sorting preferences can be specified and they are applied in the order they are set. - -They are defined using the `set-cluster-preferences` command. - -Each preference is a JSON map having the following syntax: - -`{'':'', 'precision':''}` - -See the section <> for details about the allowed values for the `sort_order`, `sort_param` and `precision` parameters. - -Changing the cluster preferences after the cluster is already built doesn't automatically reconfigure the cluster. However, all future cluster management operations will use the changed preferences. - -*Input* - -[source,json] ----- -{ -"set-cluster-preferences" : [ - {"minimize": "cores"} - ] -} ----- - -*Output* - -The output has a key named `result` which will return either `success` or `failure` depending on whether the command succeeded or failed. - -[source,json] ----- -{ - "responseHeader": { - "status": 0, - "QTime": 138 - }, - "result": "success", - "WARNING": "This response format is experimental. It is likely to change in the future." -} ----- - -==== Example Setting Cluster Preferences - -In this example we add cluster preferences that sort on three different parameters: - -[source,json] ----- -{ - "set-cluster-preferences": [ - { - "minimize": "cores", - "precision": 2 - }, - { - "maximize": "freedisk", - "precision": 100 - }, - { - "minimize": "sysLoadAvg", - "precision": 10 - } - ] -} ----- - -We can remove all cluster preferences by setting preferences to an empty list. -[source,json] ----- -{ - "set-cluster-preferences": [] -} ----- -[[cluster-specific-policies]] -=== Create and Modify Cluster Policies - -Cluster policies are set using the `set-cluster-policy` command. - -Like `set-cluster-preferences`, the policy definition is a JSON map defining the desired attributes and values. - -Refer to the <> section for details of the allowed values for each condition in the policy. - -*Input*: -[source,json] ----- -{ -"set-cluster-policy": [ - {"replica": "<2", "shard": "#EACH", "node": "#ANY"} - ] -} ----- - -*Output*: -[source,json] ----- -{ - "responseHeader": { - "status": 0, - "QTime": 47 - }, - "result": "success", - "WARNING": "This response format is experimental. It is likely to change in the future." -} ----- - -We can remove all cluster policy conditions by setting policy to an empty list. - -[source,json] ----- -{ - "set-cluster-policy": [] -} ----- - -Changing the cluster policy after the cluster is already built doesn't automatically reconfigure the cluster. However, all future cluster management operations will use the changed cluster policy. - -=== Create and Modify Collection-Specific Policy - -The `set-policy` command accepts a map of policy names to the list of conditions for that policy. Multiple named policies can be specified together. A named policy that does not exist already is created and if the named policy accepts already then it is replaced. - -Refer to the <> section for details of the allowed values for each condition in the policy. - -*Input* - -[source,json] ----- -{ -"set-policy": { - "policy1": [ - {"replica": "1", "shard": "#EACH", "nodeset":{"port": "8983"}} - ] - } -} ----- - -*Output* - -[source,json] ----- -{ - "responseHeader": { - "status": 0, - "QTime": 246 - }, - "result": "success", - "WARNING": "This response format is experimental. It is likely to change in the future." -} ----- - -Changing the policy after the collection is already built doesn't automatically reconfigure the collection. However, all future cluster management operations will use the changed policy. - -=== Remove a Collection-Specific Policy - -The `remove-policy` command accepts a policy name to be removed from Solr. The policy being removed must not be attached to any collection otherwise the command will fail. - -*Input* -[source,json] ----- -{"remove-policy": "policy1"} ----- - -*Output* -[source,json] ----- -{ - "responseHeader": { - "status": 0, - "QTime": 42 - }, - "result": "success", - "WARNING": "This response format is experimental. It is likely to change in the future." -} ----- - -If you attempt to remove a policy that is being used by a collection, this command will fail to delete the policy until the collection itself is deleted. - -=== Create or Update a Trigger - -The `set-trigger` command can be used to create a new trigger or overwrite an existing one. - -You can see the section <> for a full list of configuration options. - -.Creating a nodeAdded Trigger -[source,json] ----- -{ - "set-trigger": { - "name" : "node_added_trigger", - "event" : "nodeAdded", - "waitFor" : "1s" - } -} ----- - -.Updating Trigger with waitFor set to 5 seconds -[source,json] ----- -{ - "set-trigger": { - "name" : "node_added_trigger", - "event" : "nodeAdded", - "waitFor" : "5s", - } -} ----- - -.Creating a nodeLost Trigger -[source,json] ----- -{ - "set-trigger": { - "name" : "node_lost_trigger1", - "event" : "nodeLost", - "waitFor" : "60s", - } -} ----- - -=== Remove Trigger - -The `remove-trigger` command can be used to remove a trigger. It accepts a single parameter: the name of the trigger. - -.Removing the nodeLost Trigger -[source,json] ----- -{ - "remove-trigger": { - "name" : "node_lost_trigger1" - } -} ----- - -=== Create or Update a Trigger Listener - -The `set-listener` command can be used to create or modify a listener for a trigger. - -You can see the section <> for a full list of configuration options. - -.Creating a listener for the nodeAdded Trigger -[source,json] ----- -{ - "set-listener": { - "name": "foo", - "trigger": "node_added_trigger", - "stage": ["STARTED", "ABORTED", "SUCCEEDED", "FAILED"], - "class": "com.example.Listener" - } -} ----- - -=== Remove Trigger Listener - -The `remove-listener` command can be used to remove an existing listener. It accepts a single parameter: the name of the listener. - -.Removing the foo listener -[source,json] ----- -{ - "remove-listener": { - "name": "foo" - } -} ----- - -=== Change Autoscaling Properties - -The `set-properties` command can be used to change the default properties used by the Autoscaling framework. - -The following properties can be specified in the payload: - -`triggerScheduleDelaySeconds`:: -This is the delay in seconds between two executions of a trigger. Every trigger is scheduled using Java's ScheduledThreadPoolExecutor with this delay. The default is `1` second. - -`triggerCooldownPeriodSeconds`:: -Solr pauses all other triggers for this cool down period after a trigger fires so that the system can stabilize before running triggers again. The default is `5` seconds. - -`triggerCorePoolSize`:: -The core pool size of the `ScheduledThreadPoolExecutor` used to schedule triggers. The default is `4` threads. - -The command allows setting arbitrary properties in addition to the above properties. Such arbitrary properties can be useful in custom `TriggerAction` instances. - -.Change default `triggerScheduleDelaySeconds` -[source.json] ----- -{ - "set-properties": { - "triggerScheduleDelaySeconds": 8 - } -} ----- - -The `set-properties` command replaces older values if present. So using `set-properties` to set the same value twice will overwrite the old value. -If a property is not specified then it retains the last set value or the default, if no change was made. -A changed value can be unset by using a null value. - -.Revert changed value of `triggerScheduleDelaySeconds` to default -[source.json] ----- -{ - "set-properties": { - "triggerScheduleDelaySeconds": null - } -} ----- - -The changed values of these properties, if any, can be read using the Autoscaling <> in the `properties` section. diff --git a/solr/solr-ref-guide/src/solrcloud-autoscaling-auto-add-replicas.adoc b/solr/solr-ref-guide/src/solrcloud-autoscaling-auto-add-replicas.adoc deleted file mode 100644 index 622ce41e4e1..00000000000 --- a/solr/solr-ref-guide/src/solrcloud-autoscaling-auto-add-replicas.adoc +++ /dev/null @@ -1,73 +0,0 @@ -= SolrCloud Autoscaling Automatically Adding Replicas -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -Solr provides a way to automatically add replicas for a collection when the number of active replicas drops below -the replication factor specified at the time of the creation of the collection. - -== The autoAddReplicas Parameter - -The boolean `autoAddReplicas` parameter can be passed to the CREATE command of the Collection API to enable this feature for a given collection. - -.Create a collection with autoAddReplicas enabled -[source,text] -http://localhost:8983/solr/admin/collections?action=CREATE&name=my_collection&numShards=1&replicationFactor=5&autoAddReplicas=true - -The MODIFYCOLLECTION command can be used to enable or disable this feature for any collection. - -.Modify collection to disable autoAddReplicas -[source,text] -http://localhost:8983/solr/admin/collections?action=MODIFYCOLLECTION&collection=my_collection&autoAddReplicas=false - -== Implementation Using .autoAddReplicas Trigger - -A Trigger named `.autoAddReplicas` is automatically created whenever any collection has the autoAddReplicas feature enabled. - -Only one trigger is sufficient to serve all collections having this feature enabled. The `.autoAddReplicas` trigger watches for nodes that are lost from the cluster and uses the default `TriggerActions` to create new replicas to replace the ones which were hosted by the lost node. If the old node comes back online, it unloads the moved replicas and the node is free to host other replicas as and when required. - -Since the trigger provides the autoAddReplicas feature for all collections, the `suspend-trigger` and `resume-trigger` Autoscaling API commands can be used to disable and enable this feature for all collections in one API call. - -.Suspending autoAddReplicas for all collections -[source,json] ----- -{ - "suspend-trigger": { - "name" : ".autoAddReplicas" - } -} ----- - -.Resuming autoAddReplicas for all collections -[source,json] ----- -{ - "resume-trigger": { - "name" : ".autoAddReplicas" - } -} ----- - -== Using Cluster Property to Enable autoAddReplicas - -A cluster property, also named `autoAddReplicas`, can be set to `false` to disable this feature for all collections. -If this cluster property is missing or set to `true`, the autoAddReplicas is enabled for all collections. - -.Deprecation Warning -[WARNING] -==== -Using a cluster property to enable or disable autoAddReplicas is deprecated and only supported for back compatibility. Please use the `suspend-trigger` and `resume-trigger` API commands instead. -==== diff --git a/solr/solr-ref-guide/src/solrcloud-autoscaling-fault-tolerance.adoc b/solr/solr-ref-guide/src/solrcloud-autoscaling-fault-tolerance.adoc deleted file mode 100644 index 67f889c344b..00000000000 --- a/solr/solr-ref-guide/src/solrcloud-autoscaling-fault-tolerance.adoc +++ /dev/null @@ -1,59 +0,0 @@ -= SolrCloud Autoscaling Fault Tolerance -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -The autoscaling framework uses a few strategies to ensure it's able to still trigger actions in the event of unexpected changes to the system. - -== Node Added or Lost Markers -Since triggers execute on the node that runs the Overseer, should the Overseer node go down the `nodeLost` -event would be lost because there would be no mechanism to generate it. Similarly, if a node has -been added before the Overseer leader change was completed, the `nodeAdded` event would not be -generated. - -For this reason Solr implements additional mechanisms to ensure that these events are generated -reliably. - -With standard SolrCloud behavior, when a node joins a cluster its presence is marked as an ephemeral ZooKeeper path in the `/live_nodes/` ZooKeeper directory. Now an ephemeral path is also created under `/autoscaling/nodeAdded/`. -When a new instance of Overseer leader is started it will run the `nodeAdded` trigger (if it's configured) -and discover the presence of this ZooKeeper path, at which point it will remove it and generate a `nodeAdded` event. - -When a node leaves the cluster, up to three remaining nodes will try to create a persistent ZooKeeper path -`/autoscaling/nodeLost/` and eventually one of them succeeds. When a new instance of Overseer leader -is started it will run the `nodeLost` trigger (if it's configured) and discover the presence of this ZooKeeper -path, at which point it will remove it and generate a `nodeLost` event. - -== Trigger State Checkpointing -Triggers generate events based on their internal state. If the Overseer leader goes down while the trigger is -about to generate a new event, it's likely that the event would be lost because a new trigger instance -running on the new Overseer leader would start from a clean slate. - -For this reason, after each time a trigger is executed its internal state is persisted to ZooKeeper, and -on Overseer start its internal state is restored. - -== Trigger Event Queues -Autoscaling framework limits the rate at which events are processed using several different mechanisms. -One is the locking mechanism that prevents concurrent -processing of events, and another is a single-threaded executor that runs trigger actions. - -This means that the processing of an event may take significant time, and during this time it's possible that the -Overseer may go down. In order to avoid losing events that were already generated but not yet fully -processed, events are queued before processing is started. - -Separate ZooKeeper queues are created for each trigger, and events produced by triggers are put on these -per-trigger queues. When a new Overseer leader is started it will first check -these queues and process events accumulated there, and only then it will continue to run triggers -normally. Queued events that fail processing during this "replay" stage are discarded. diff --git a/solr/solr-ref-guide/src/solrcloud-autoscaling-listeners.adoc b/solr/solr-ref-guide/src/solrcloud-autoscaling-listeners.adoc deleted file mode 100644 index 6e6bfca7715..00000000000 --- a/solr/solr-ref-guide/src/solrcloud-autoscaling-listeners.adoc +++ /dev/null @@ -1,220 +0,0 @@ -= SolrCloud Autoscaling Listeners -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -Trigger Listeners allow users to configure additional behavior related to trigger events as they are being processed. - -For example, users may want to record autoscaling events to an external system, or notify an administrator when a -particular type of event occurs or when its processing reaches certain stage (e.g., failed). - -Listener configuration always refers to a specific trigger configuration because a listener is notified of -events generated by that specific trigger. Several (or none) named listeners can be registered for a trigger, -and they will be notified in the order in which they were defined. - -Listener configuration can specify what processing stages are of interest, and when an event enters this processing stage the listener will be notified. Currently the following stages are recognized: - -* STARTED - when an event has been generated by a trigger and its processing is starting. -* ABORTED - when event was being processed while the source trigger closed. -* BEFORE_ACTION - when a `TriggerAction` is about to be invoked. Action name and the current `ActionContext` are passed to the listener. -* AFTER_ACTION - after a `TriggerAction` has been successfully invoked. Action name, `ActionContext` and the list of action - names invoked so far are passed to the listener. -* FAILED - when event processing failed (or when a `TriggerAction` failed) -* SUCCEEDED - when event processing completes successfully - -Listener configuration can also specify what particular actions are of interest, both before and/or after they are invoked. - -== Listener Configuration -Currently the following listener configuration properties are supported: - -`name`:: -(string, required) A unique listener configuration name. - -`trigger`:: -(string, required) The name of an existing trigger configuration. - -`class`:: -(string, required) A listener implementation class name. - -`stage`:: -(list of strings, optional, ignored case) A list of processing stages that - this listener should be notified. Default is empty list. - -`beforeAction`:: -(list of strings, optional) A list of action names (as defined in trigger configuration) before -which the listener will be notified. Default is empty list. - -`afterAction`:: -(list of strings, optional) A list of action names after which the listener will be notified. -Default is empty list. - -TIP: Additional implementation-specific properties may be provided, depending on the listener implementation. - -Note: when both `stage` and `beforeAction` / `afterAction` lists are non-empty then the listener will be notified both -when a specified stage is entered and before / after specified actions. - -=== Managing Listener Configurations -Listener configurations are managed using the Autoscaling Write API, and using `set-listener` and `remove-listener` -commands. - -For example: - -[source,json] ----- -{ - "set-listener": { - "name": "foo", - "trigger": "node_lost_trigger", - "stage": ["STARTED", "ABORTED", "SUCCEEDED", "FAILED"], - "class": "solr.SystemLogListener" - } -} ----- - -[source,json] ----- -{ - "remove-listener": { - "name": "foo" - } -} ----- - -== Listener Implementations -Trigger listeners must implement the `TriggerListener` interface. Solr provides some -implementations of trigger listeners, which cover common use cases. These implementations are described below, together with their configuration parameters. - -=== SystemLogListener -This trigger listener sends trigger events and processing context as documents for indexing in -SolrCloud `.system` collection. - -When a trigger configuration is first created, a corresponding trigger listener configuration that -uses `SystemLogListener` is also automatically created, to make sure that all events and -actions related to the autoscaling framework are logged to the `.system` collection. - -Supported configuration properties: - -`collection`:: -(string, optional) Specifies the target collection where documents are sent. -Default value is `.system`. If the target collection is missing the listener will -silently discard events. -+ -NOTE: In rare situations when the target collection is in -an unstable state (e.g., when some leader replicas were just lost and the leader election hasn't -finished running yet), the listener may not be able to index some events. In such cases a -WARN message with the details of the event(s) will be added to the regular logs. - -`enabled`:: -(boolean, optional) Enables the listener when true. Default value is true. - -Documents created by this listener have several predefined fields: - -* `id` - time-based random id -* `type` - always set to `autoscaling_event` -* `source_s` - always set to `SystemLogListener` -* `timestamp` - current time when document was created -* `stage_s` - current stage of event processing -* `action_s` - current action name, if available -* `message_t` - optional additional message -* `error.message_t` - message from Throwable, if available -* `error.details_t` - stacktrace from Throwable, if available -* `before.actions_ss` - list of action names to be invoked so far -* `after.actions_ss` - list of action names that have been successfully invoked so far -* `event_str` - JSON representation of all event properties -* `context_str` - JSON representation of all `ActionContext` properties, if available - -The following fields are created using the information from trigger event: - -* `event.id_s` - event id -* `event.type_s` - event type -* `event.source_s` - event source (trigger name) -* `event.time_l` - Unix time when the event was created (may significantly differ from the time when it was actually -processed) -* `event.property.*` - additional fields that represent other arbitrary event properties. These fields use either -`_s` or `_ss` suffix depending on whether the property value is a collection (values inside collection are treated as -strings, there's no recursive flattening) - -The following configuration is used for the automatically created listener (in this case for a -trigger named `foo`): - -[source,json] ----- -{ - "name" : "foo.system", - "trigger" : "solr.SystemLogListener", - "stage" : ["STARTED", "ABORTED", "SUCCEEDED", "FAILED", "BEFORE_ACTION", "AFTER_ACTION"] -} ----- - -=== HttpTriggerListener -This listener uses HTTP POST to send a representation of the event and context to a specified URL. -The URL, payload, and headers may contain property substitution patterns, which are then replaced with values taken from the current event or context properties. - -Templates use the same syntax as property substitution in Solr configuration files, e.g., -`${foo.bar:baz}` means that the value of `foo.bar` property should be taken, and `baz` should be used -if the value is absent. - -Supported configuration properties: - -`url`:: -(string, required) A URL template. - -`payload`:: -(string, optional) A payload template. If absent, a JSON map of all properties listed above will be used. - -`contentType`:: -(string, optional) A payload content type. If absent then `application/json` will be used. - -`header.*`:: -(string, optional) A header template(s). The name of the property without "header." prefix defines the literal header name. - -`timeout`:: -(int, optional) Connection and socket timeout in milliseconds. Default is `60000` milliseconds (60 seconds). - -`followRedirects`:: -(boolean, optional) Allows following redirects. Default is `false`. - -The following properties are available in context and can be referenced from templates: - -* `config.*` - listener configuration properties -* `event.*` - current event properties -* `stage` - current stage of event processing -* `actionName` - optional current action name -* `context.*` - optional ActionContext properties -* `error` - optional error string (from Throwable.toString()) -* `message` - optional message - -.Example HttpTriggerListener -[source,json] ----- -{ - "name": "foo", - "trigger": "node_added_trigger", - "class": "solr.HttpTriggerListener", - "url": "http://foo.com/${config.name:invalidName}/${config.properties.xyz:invalidXyz}/${event.eventType}", - "xyz": "foobar", - "header.X-Trigger": "${config.trigger}", - "payload": "actionName=${actionName}, source=${event.source}, type=${event.eventType}", - "contentType": "text/plain", - "stage": ["STARTED", "ABORTED", "SUCCEEDED", "FAILED"], - "beforeAction": ["compute_plan", "execute_plan"], - "afterAction": ["compute_plan", "execute_plan"] -} ----- - -This configuration specifies that each time one of the listed stages is reached, or before and after each of the listed -actions is executed, the listener will send the templated payload to a URL that also depends on the configuration and the current event, -and with a custom header that indicates the trigger name. diff --git a/solr/solr-ref-guide/src/solrcloud-autoscaling-overview.adoc b/solr/solr-ref-guide/src/solrcloud-autoscaling-overview.adoc deleted file mode 100644 index 222ca62baa6..00000000000 --- a/solr/solr-ref-guide/src/solrcloud-autoscaling-overview.adoc +++ /dev/null @@ -1,116 +0,0 @@ -= Overview of SolrCloud Autoscaling -:toclevels: 1 -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -Autoscaling in Solr aims to provide good defaults so a SolrCloud cluster remains balanced and stable in the face of various cluster change events. This balance is achieved by satisfying a set of rules and sorting preferences to select the target of cluster management operations automatically on cluster events. - -A simple example is automatically adding a replica for a SolrCloud collection when a node containing an existing replica goes down. - -The goal of autoscaling in SolrCloud is to make cluster management easier, more automatic, and more intelligent. It aims to provide good defaults such that the cluster remains balanced and stable in the face of various events such as a node joining the cluster or leaving the cluster. This is achieved by satisfying a set of rules and sorting preferences that help Solr select the target of cluster management operations. - -There are three distinct problems that this feature solves: - -* When to run cluster management tasks? For example, we might want to add a replica when an existing replica is no longer alive. -* Which cluster management task to run? For example, do we add a new replica or should we move an existing one to a new node? -* How do we run the cluster management tasks so the cluster remains balanced and stable? - -Before we get into the details of how each of these problems are solved, let's take a quick look at the easiest way to setup autoscaling for your cluster. - -== Quick Start: Automatically Adding Replicas - -Say that we want to create a collection which always requires us to have three replicas available for each shard all the time. We can set the `replicationFactor=3` while creating the collection, but what happens if a node containing one or more of the replicas either crashed or was shutdown for maintenance? In such a case, we'd like to create additional replicas to replace the ones that are no longer available to preserve the original number of replicas. - -We have an easy way to enable this behavior without needing to understand the autoscaling features in depth. We can create a collection with such behavior by adding an additional parameter `autoAddReplicas=true` with the CREATE command of the Collection API. For example: - -[source,text] -/admin/collections?action=CREATE&name=_name_of_collection_&numShards=1&replicationFactor=3&autoAddReplicas=true - -A collection created with `autoAddReplicas=true` will be monitored by Solr such that if a node containing a replica of this collection goes down, Solr will add new replicas on other nodes after waiting for up to thirty seconds for the node to come back. - -You can see the section <> to learn more about how to enable or disable this feature as well as other details. - -The selection of the node that will host the new replica is made according to the default cluster preferences that we will learn more about in the next sections. - -== Cluster Preferences - -Cluster preferences allow you to tell Solr how to assess system load on each node. This information is used to guide selection of the node(s) on which cluster management operations will be performed. - -In general, when an operation increases replica counts, the *least loaded* <> will be chosen, and when the operation reduces replica counts, the *most loaded* qualified node will be chosen. - -The default cluster preferences are `[{minimize:cores},{maximize:freedisk}]`, which tells Solr to minimize the number of cores on all nodes and if number of cores are equal, maximize the free disk space available. In this case, the least loaded node is the one with the fewest cores or if two nodes have an equal number of cores, the node with the most free disk space. - -You can learn more about preferences in the section on <>. - -== Cluster Policy - -A cluster policy is a set of rules that a node, shard, or collection must satisfy before it can be chosen as the target of a cluster management operation. These rules are applied across the cluster regardless of the collection being managed. For example, the rule `{"cores":"<10", "node":"#ANY"}` means that any node must have less than 10 Solr cores in total, regardless of which collection they belong to. - -There are many metrics on which the rule can be based, e.g., system load average, heap usage, free disk space, etc. The full list of supported metrics can be found in the section describing <>. - -When a node, shard, or collection does not satisfy a policy rule, we call it a *violation*. By default, cluster management operations will fail if there is even one violation. You can allow operations to succeed in the face of a violation by marking the corresponding rule with <>. When you do this, Solr ensures that cluster management operations minimize the number of violations. - -The default cluster policy, if none is specified, is the following: - -[source,json] ----- -[ - { "replica" : "<2", "shard" : "#EACH", "node" : "#ANY", "strict" : false}, - { "replica" : "#EQUAL", "node" : "#ANY", "strict" : false}, - { "cores" : "#EQUAL", "node" : "#ANY", "strict" : false} -] ----- - -These rules mean that: - -* Each shard should not have more than one replica on the same node, if possible (strict: false). -* Each collection's replicas should be equally distributed among nodes. -* All cores should be equally distributed among nodes. - - -NOTE: You can remove this default policy by specifying an empty rule-set in the autoscaling -admin request, like this: `{set-cluster-policy : []}`. - -Solr also supports <>, which operate in tandem with the cluster policy. - -== Triggers - -Now that we have an idea about how cluster management operations use policies and preferences help Solr keep the cluster balanced and stable, we can talk about when to invoke such operations. - -Triggers are used to watch for events such as a node joining or leaving the cluster. When the event happens, the trigger executes a set of actions that compute and execute a *plan*, i.e., a set of operations to change the cluster so that the policy and preferences are respected. - -The `autoAddReplicas` parameter passed with the CREATE Collection API command in the <> section above automatically creates a trigger that watches for a node going away. When the trigger fires, it executes a set of actions that compute and execute a plan to move all replicas hosted by the lost node to new nodes in the cluster. The target nodes are chosen based on the policy and preferences. - -You can learn more about Triggers in the section <>. - -== Trigger Actions - -A trigger executes *actions* that tell Solr what to do in response to the trigger. Solr ships with two actions that are added to every trigger by default. The first is called the *ComputePlanAction* and the other is *ExecutePlanAction*. The former computes the cluster management operations necessary to stabilize the cluster and the latter executes them on the cluster. - -You can learn more about Trigger Actions in the section <>. - -== Listeners - -An Autoscaling *listener* can be attached to a trigger. Solr calls the listener each time the trigger fires as well as before and after the actions performed by the trigger. Listeners are useful as a call back mechanism to perform tasks such as logging or informing external systems about events. For example, a listener is automatically added by Solr to each trigger to log details of the trigger fire and actions to the `.system` collection. - -You can learn more about Listeners in the section <>. - -== Autoscaling APIs - -The autoscaling APIs available at `/admin/autoscaling` can be used to read and modify each of the components discussed above. - -You can learn more about these APIs in the section <>. diff --git a/solr/solr-ref-guide/src/solrcloud-autoscaling-policy-preferences.adoc b/solr/solr-ref-guide/src/solrcloud-autoscaling-policy-preferences.adoc deleted file mode 100644 index 38d5821737a..00000000000 --- a/solr/solr-ref-guide/src/solrcloud-autoscaling-policy-preferences.adoc +++ /dev/null @@ -1,638 +0,0 @@ -= Autoscaling Policy and Preferences -:toclevels: 2 -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -The autoscaling policy and preferences are a set of rules and sorting preferences that help Solr select the target of cluster management operations so the overall load on the cluster remains balanced. - -The configured autoscaling policy and preferences are used by <> in all contexts: manual, for example using `bin/solr` to create a collection; semi-automatic, via the <> or the Admin UI's <>; or fully automatic, via configured <>. - -See the section <> for an example of how policy and preferences affect replica placement. - -== Cluster Preferences Specification - -A preference is a hint to Solr on how to sort nodes based on their utilization. - -The default cluster preference is to sort by the total number of Solr cores (or replicas) hosted by a node, with a precision of 1. -Therefore, by default, when selecting a node to which to add a replica, Solr can apply the preferences and choose the node with the fewest cores. -In the case of a tie in the number of cores, available freedisk will be used to further sort nodes. - -More than one preference can be added to break ties. For example, we may choose to use free disk space to break ties if the number of cores on two nodes is the same. The node with the higher free disk space can be chosen as the target of the cluster operation. - -Each preference takes the following form: - -[source,json] -{"":"", "precision":""} - -`sort_order`:: -The value can be either `maximize` or `minimize`. Choose `minimize` to sort the nodes with least value as the least loaded. For example, `{"minimize":"cores"}` sorts the nodes with the least number of cores as the least loaded node. A sort order such as `{"maximize":"freedisk"}` sorts the nodes with maximum free disk space as the least loaded node. -+ -The objective of the system is to make every node the least loaded. So, in case of a `MOVEREPLICA` operation, it usually targets the _most loaded_ node and takes load off of it. In a sort of more loaded to less loaded, `minimize` is akin to sorting in descending order and `maximize` is akin to sorting in ascending order. -+ -This is a required parameter. - -`sort_param`:: -One and only one of the following supported parameters must be specified: - -. `cores`: The number of total Solr cores on a node. -. `freedisk`: The amount of free disk space for Solr's data home directory. This is always in gigabytes. -. `sysLoadAvg`: The system load average on a node as reported by the Metrics API under the key `solr.jvm/os.systemLoadAverage`. This is always a double value between 0 and 1 and the higher the value, the more loaded the node is. -. `heapUsage`: The heap usage of a node as reported by the Metrics API under the key `solr.jvm/memory.heap.usage`. This is always a double value between 0 and 1 and the higher the value, the more loaded the node is. - -`precision`:: -Precision tells the system the minimum (absolute) difference between 2 values to treat them as distinct values. -+ -For example, a precision of 10 for `freedisk` means that two nodes whose free disk space is within 10GB of each other should be treated as equal for the purpose of sorting. This helps create ties without which specifying multiple preferences is not useful. This is an optional parameter whose value must be a positive integer. The maximum value of `precision` must be less than the maximum value of the `sort_value`, if any. - -See the section <> for details on how to manage cluster preferences with the API. - -=== Examples of Cluster Preferences - -==== Default Preferences -The following shows the default cluster preferences. This is applied automatically by Solr when no explicit cluster preferences have been set using the <>. - -[source,json] -[ - {"minimize":"cores"} -] - -==== Minimize Cores; Maximize Free Disk -In this example, we want to minimize the number of Solr cores and in case of a tie, maximize the amount of free disk space on each node. - -[source,json] -[ - {"minimize" : "cores"}, - {"maximize" : "freedisk"} -] - -==== Add Precision to Free Disk; Minimize System Load -In this example, we add a precision to the `freedisk` parameter so that nodes with free disk space within 10GB of each other are considered equal. In such a case, the tie is broken by minimizing `sysLoadAvg`. - -[source,json] -[ - {"minimize" : "cores"}, - {"maximize" : "freedisk", "precision" : 10}, - {"minimize" : "sysLoadAvg"} -] - -== Policy Specification - -A policy is a hard rule to be satisfied by each node. If a node does not satisfy the rule then it is called a *violation*. Solr ensures that the number of violations are minimized while invoking any cluster management operations. - -=== Policy Rule Structure - -==== Rule Types - -Policy rules can be either global or per-collection: - -* *Global rules* constrain the number of cores per node or node group. This type of rule applies to cores from all collections hosted on the specified node(s). As a result, <>, which are associated with individual collections, may not contain global rules. -* *Per-collection rules* constrain the number of replicas per node or node group. - -Global rules have three parts: - -* <> -* <> (`"cores": "..."`) -* <> (optional) - -Per-collection rules have five parts: - -* <> -* <> -* <> (`"replica": "..."`) -* <> (optional) -* `put` (optional) specifies how to place these replicas on the selected nodes. All the selected nodes are considered as one bucket by default. `"put" : "on-each-node"` treats each selected node as a bucket - -==== Node Selector - -A node selector is specified using the `node` `nodeset` attribute. This is used to filter the set of nodes where this rules needs to be applied - -examples - -[source,json] -{ "replica" : "<2", "node":"#ANY"} - -[source,json] -//place 3 replicas in the group of nodes node-name1, node-name2 -{ "replica" : "3", "nodeset":["node-name1","node-name2"]} - -[source,json] -{ "nodeset":{"":""}} - -The property names can be one of: `node`, `host`, `sysprop.*`, `freedisk`, `ip_*`, `nodeRole`, `heapUsage`, `metrics.*`. - -when using the `nodeset` attribute, an optional attribute `put` can be used to specify how to distribute the replicas in that node set. - -example: _put one replica on each node with a system property zone=east_ -[source,json] -{ "replica":1, "put" :"on-each-node", "nodeset":{"sysprop.zone":"east"}} - -example: _put a total of 2 replicas on the set of nodes with property zone=east_ -[source,json] -{ "replica":2, "put" :"on-each-node", "nodeset":{"sysprop.zone":"east"}} - - - -Rule evaluation is restricted to node(s) matching the value of one of the following attributes: <>, <>, <>, <>, or <>. For replica/core count constraints other than `#EQUAL`, a condition specified in one of the following attributes may instead be used to select nodes: <>, <>, <>, <>, <>, or <>. - -Except for `node`, the attributes above cause selected nodes to be partitioned into node groups. A node group is referred to as a "bucket". Those attributes usable with the `#EQUAL` directive may define buckets either via the special function <> or an <> `["value1", ...]` (a subset of all possible values); in both cases, each node is placed in the bucket corresponding to the matching attribute value. - -The `node` attribute always places each selected node into its own bucket, regardless of the attribute value's form (`#ANY`, `node-name`, or `["node1-name", ...]`). - -Replica and core count constraints, described below, are evaluated against the total number in each bucket. - -==== Core Count Constraint - -The `cores` attribute value can be specified in one of the following forms: - -* <>: distribute all cores equally across all the <>. -* a constraint on the core count on each <>; see <>. - -==== Replica Selector and Rule Evaluation Context - -Rule evaluation can be restricted to replicas that meet any combination of conditions specified with the following attributes: - -* <>: The replica is of a shard belonging to the collection specified in the attribute value. (Not usable with <>.) -* <>: The replica is of the shard named in the attribute value. -* <>: The replica has the specified replica type (`NRT`, `TLOG`, or `PULL`). - -If none of the above attributes is specified, then the rule is evaluated separately for each collection against all types of replicas of all shards. - -Specifying <> as the `shard` attribute value causes the rule to be evaluated separately for each shard of each collection. - -==== Replica Count Constraint - -The `replica` attribute value can be specified in one of the following forms: - -* <>: All <> will be placed on the <>. -* <>: Distribute <> equally across all the <>. -* a constraint on the replica count on each <>; see <>. - -==== Specifying Replica and Core Count Constraints - -<> (`"replica":"..."`) and <> (`"cores":"..."`) allow specification of acceptable counts for replicas (cores tied to a collection) and cores (regardless of the collection to which they belong), respectively. - -You can specify one of the following as the value of a `replica` and `cores` policy rule attribute: - -* an exact integer (e.g., `2`) -* an exclusive lower integer bound (e.g., `>0`) -* an exclusive upper integer bound (e.g., `<3`) -* a decimal value, interpreted as an acceptable range of core counts, from the floor of the value to the ceiling of the value, with the system preferring the rounded value (e.g., `1.6`: `1` or `2` is acceptable, and `2` is preferred) -* a <> of acceptable replica/core counts, as inclusive lower and upper integer bounds separated by a hyphen (e.g., `3-5`) -* a percentage (e.g., `33%`), which is multiplied at runtime either by the number of <> (for a `replica` constraint) or the number of cores in the cluster (for a `cores` constraint). This value is then interpreted as described above for a literal decimal value. - -NOTE: Using an exact integer value for count constraints is of limited utility, since collection or cluster changes could quickly invalidate them. For example, attempting to add a third replica to each shard of a collection on a two-node cluster with policy rule `{"replica":1, "shard":"#EACH", "node":"#ANY"}` would cause a violation, since at least one node would have to host more than one replica. Percentage rules are less brittle. Rewriting the rule as `{"replica":"50%", "shard":"#EACH", "node":"#ANY"}` eliminates the violation: `50% of 3 replicas = 1.5 replicas per node`, meaning that it's acceptable for a node to host either one or two replicas of each shard. - -=== Policy Rule Attributes - -==== Rule Strictness - -This attribute is usable in all rules: - -`strict`:: -An optional boolean value. The default is `true`. If true, the rule must be satisfied; if the rule is not satisfied, the resulting violation will cause the cluster management operation to fail. If false, Solr tries to satisfy the rule on a best effort basis, but if no node can satisfy the rule, the cluster management operation will not fail, and any node may be chosen. If multiple rules declared to be `strict:false` can not be satisfied by some nodes, then a node will be chosen such that the number of such violations is minimized. - -==== Global Rule Attributes - -[[cores-attribute]] -`cores`:: -The number of cores that must exist to satisfy the rule. This is a required attribute for <>. The <> must also be specified, and the only other allowed attribute is the optional <>. See <> for possible attribute values. - -==== Per-collection Rule Attributes - -The following attributes are usable with <>, in addition to the attributes in the <> section below: - -[[collection-attribute]] -`collection`:: -The name of the collection to which the policy rule should apply. If omitted, the rule applies to all collections. This attribute is optional. - -[[shard-attribute]] -`shard`:: -The name of the shard to which the policy rule should apply. If omitted, the rule is applied for all shards in the collection. It supports the special function <> which means that the rule is applied for each shard in the collection. - -[[type-attribute]] -`type`:: -The type of the replica to which the policy rule should apply. If omitted, the rule is applied for all replica types of this collection/shard. The allowed values are `NRT`, `TLOG` and `PULL` - -[[replica-attribute]] -`replica`:: -The number of replicas that must exist to satisfy the rule. This is a required attribute for <>. See <> for possible attribute values. - -==== Node Selection Attributes - -One and only one of the following attributes can be specified in addition to the above attributes. See the <> section for more information: - -[[node-attribute]] -`node`:: -The name of the node to which the rule should apply. The <> or the <> or the <> may be used in this attribute's value. - -[[port-attribute]] -`port`:: -The port of the node to which the rule should apply. The <> or the <> may be used in this attribute's value. - -[[freedisk-attribute]] -`freedisk`:: -The free disk space in gigabytes of the node. This must be a positive 64-bit integer value, or a <>. If a percentage is specified, either an upper or lower bound may also be specified using the `<` or `>` operators, respectively, e.g., `>50%`, `<25%`. - -[[host-attribute]] -`host`:: -The host name of the node. - -[[sysLoadAvg-attribute]] -`sysLoadAvg`:: -The system load average of the node as reported by the Metrics API under the key `solr.jvm/os.systemLoadAverage`. This is floating point value between 0 and 1. - -[[heapUsage-attribute]] -`heapUsage`:: -The heap usage of the node as reported by the Metrics API under the key `solr.jvm/memory.heap.usage`. This is floating point value between 0 and 1. - -[[nodeRole-attribute]] -`nodeRole`:: -The role of the node. The only supported value currently is `overseer`. - -[[ip-attributes]] -`ip_1, ip_2, ip_3, ip_4`:: -The least significant to most significant segments of IP address. For example, for an IP address `192.168.1.2`, `"ip_1":"2", "ip_2":"1", "ip_3":"168", "ip_4":"192"`. The <> may be used in any of these attributes' values. - -[[sysprop-attribute]] -`sysprop.`:: -Any arbitrary system property set on the node on startup. The <> or the <> may be used in this attribute's value. - -[[metrics-attribute]] -`metrics:`:: -Any arbitrary metric. For example, `metrics:solr.node:CONTAINER.fs.totalSpace`. Refer to the `key` parameter in the <> section. - -[[diskType-attribute]] -`diskType`:: -The type of disk drive being used for Solr's `coreRootDirectory`. The only two supported values are `rotational` and `ssd`. Refer to `coreRootDirectory` parameter in the <> section. The <> or the <> may be used in this attribute's value. -+ -Its value is fetched from the Metrics API with the key named `solr.node:CONTAINER.fs.coreRoot.spins`. The disk type is auto-detected by Lucene using various heuristics and it is not guaranteed to be correct across all platforms or operating systems. Refer to the <> section for more details. - -=== Policy Operators - -Each attribute in the policy may specify one of the following operators along with the value. - -* No operator means equality -* `<`: Less than -* `>`: Greater than -* [[not-operator]]`!`: Not -* [[range-operator]]Range operator `(-)`: a value such as `"3-5"` means a value between 3 to 5 (inclusive). This is only supported in the <> and <> attributes. -* [[array-operator]]Array operator `[]`: e.g., `sysprop.zone= ["east","west","apac"]`. This is equivalent to having multiple rules with each of these values. This can be used in the following attributes: -** <> -** <> -** <> -** <> -** <> - -==== Special Functions - -This supports values calculated at the time of execution. - -* [[percentage-function]]`%` : A certain percentage of the value. This is supported by the following attributes: -** <> -** <> -** <> -* [[any-function]]`#ANY`: Applies to the <> only. This means the rule applies to any node. -* [[all-function]]`#ALL`: Applies to the <> only. This means all replicas that meet the rule condition. -* [[each-function]]`#EACH`: Applies to the <> (meaning the rule should be evaluated separately for each shard), and to the attributes used to define the buckets for the <> (meaning all possible values for the bucket-defining attribute). -* [[equal-function]]`#EQUAL`: Applies to the <> and <> attributes only. This means an equal number of replicas/cores in each bucket. The buckets can be defined using the below attributes with a value that can either be <> or a list specified with the <>: -** <> \<- <>, i.e., those with the <>, may only specify this attribute -** <> -** <> -** <> -** <> - - -=== Examples of Policy Rules - -==== Limit Replica Placement - -Do not place more than one replica of the same shard on the same node. The rule is evaluated separately for <> shard in each collection. The rule is applied to <> node. - -[source,json] -{"replica": "<2", "shard": "#EACH", "node": "#ANY"} - -==== Limit Cores per Node - -Do not place more than 10 cores in <> node. This rule can only be added to the cluster policy because it is a <>. - -[source,json] -{"cores": "<10", "node": "#ANY"} - -==== Place Replicas Based on Port - -Place exactly 1 replica of <> shard of collection `xyz` on a node running on port `8983`. - -[source,json] -{"replica": 1, "shard": "#EACH", "collection": "xyz", "nodeset": {"port": "8983"}} - -==== Place Replicas Based on a System Property - -Place <> replicas on nodes with system property `availability_zone=us-east-1a`. - -[source,json] -{"replica": "#ALL", "nodeset": {"sysprop.availability_zone": "us-east-1a"}} - -==== Use Percentage - -Place a maximum of (roughly) a third of the replicas of <> shard in <> node. In the following example, the value of `replica` is computed in real time as a percentage of the replicas of <> shard of each collection: - -[source,json] -{"replica": "33%", "shard": "#EACH", "node": "#ANY"} - -If the number of replicas in a shard is `2`, `33% of 2 = 0.66`. This means a node may have a maximum of `1` and a minimum of `0` replicas of each shard. - -It is possible to get the same effect by hard coding the value of `replica` as a decimal value: - -[source,json] -{"replica": 0.66, "shard": "#EACH", "node": "#ANY"} - -or using the <>: - -[source,json] -{"replica": "0-1", "shard": "#EACH", "node": "#ANY"} - -==== Multiple Percentage Rules - -Distribute replicas of <> shard of each collection across datacenters `east` and `west` at a `1:2` ratio: - -[source,json] ----- -{"replica": "33%", "shard": "#EACH", "nodeset":{ "sysprop.zone": "east"}} -{"replica": "66%", "shard": "#EACH", "nodeset":{"sysprop.zone": "west"}} ----- - -For the above rules to work, all nodes must the started with a system property called `"zone"` - -==== Distribute Replicas Equally in Each Zone - -For <> shard of each collection, distribute replicas equally across the `east` and `west` zones. - -[source,json] -{"replica": "#EQUAL", "shard": "#EACH", "nodeset":[{"sysprop.zone": "east"},{"sysprop.zone": "west"}]}} - - -==== Place Replicas Based on Node Role - -Do not place any replica on any node that has the overseer role. Note that the role is added by the `addRole` collection API. It is *not* automatically the node which is currently the overseer. - -[source,json] -{"replica": 0, "put" :"on-each-node", "nodeset":{ "nodeRole": "overseer"}} - -==== Place Replicas Based on Free Disk - -Place <> replicas in nodes where <> is greater than 500GB. - -[source,json] -{"replica": "#ALL", "nodeset":{ "freedisk": ">500"}} - -Keep all replicas in nodes where <> percentage is greater than `50%`. - -[source,json] -{"replica": "#ALL", "nodeset":{"freedisk": ">50%"}} - -==== Try to Place Replicas Based on Free Disk - -When possible, place <> replicas in nodes where <> is greater than 500GB. Here we use the <> attribute to signal that this rule is to be honored on a best effort basis. - -[source,json] -{"replica": "#ALL", "nodeset":{ "freedisk": ">500"}, "strict": false} - -==== Place All Replicas of Type TLOG on Nodes with SSD Drives - -[source,json] -{"replica": "#ALL", "type": "TLOG", "nodeset": {"diskType": "ssd"}} - -==== Place All Replicas of Type PULL on Nodes with Rotational Disk Drives - -[source,json] -{"replica": "#ALL", "type": "PULL", "nodeset" : {"diskType": "rotational"}} - -[[collection-specific-policy]] -== Defining Collection-Specific Policies - -By default, the cluster policy, if it exists, is used automatically for all collections in the cluster. However, we can create named policies that can be attached to a collection at the time of its creation by specifying the policy name along with a `policy` parameter. - -When a collection-specific policy is used, the rules in that policy are *appended* to the rules in the cluster policy and the combination of both are used. Therefore, it is recommended that you do not add rules to collection-specific policy that conflict with the ones in the cluster policy. Doing so will disqualify all nodes in the cluster from matching all criteria and make the policy useless. - -It is possible to override rules specified in the cluster policy using collection-specific policy. For example, if a rule `{replica:'<3', node:'#ANY'}` is present in the cluster policy and the collection-specific policy has a rule `{replica:'<4', node:'#ANY'}`, the cluster policy is ignored in favor of the collection policy. - -Some attributes such as `cores` can only be used in the cluster policy. See the section <> for details. - -To create a new named policy, use the <>. Once you have a named policy, you can specify the `policy=` parameter to the CREATE command of the Collection API: - -[source,text] -/admin/collections?action=CREATE&name=coll1&numShards=1&replicationFactor=2&policy=policy1 - -The above CREATE collection command will associate a policy named `policy1` with the collection named `coll1`. Only a single policy may be associated with a collection. - -== Example: Manual Collection Creation with a Policy - -The starting state for this example is a Solr cluster with 3 nodes: "nodeA", "nodeB", and "nodeC". An existing 2-shard `FirstCollection` with a `replicationFactor` of 1 has one replica on "nodeB" and one on "nodeC". The default Autoscaling preferences are in effect: - -[source,json] -[ {"minimize": "cores"} ] - -The configured policy rule allows at most 1 core per node: - -[source,json] -[ {"cores": "<2", "node": "#ANY"} ] - -We now issue a CREATE command for a `SecondCollection` with two shards and a `replicationFactor` of 1: - -[source,text] ----- -http://localhost:8983/solr/admin/collections?action=CREATE&name=SecondCollection&numShards=2&replicationFactor=1 ----- - -For each of the two replicas to be created, each Solr node is tested, in order from least to most loaded: would all policy rules be satisfied if a replica were placed there using an ADDREPLICA sub-command? - -* ADDREPLICA for `shard1`: According to the Autoscaling preferences, the least loaded node is the one with the fewest cores: "nodeA", because it hosts no cores, while the other two nodes each host one core. The test to place a replica here succeeds, because doing so causes no policy violations, since the core count after adding the replica would not exceed the configured maximum of 1. Because "nodeA" can host the first shard's replica, Solr skips testing of the other two nodes. -* ADDREPLICA for `shard2`: After placing the `shard1` replica, all nodes would be equally loaded, since each would have one core. The test to place the `shard2` replica fails on each node, because placement would push the node over its maximum core count. This causes a policy violation. - -Since there is no node that can host a replica for `shard2` without causing a violation, the overall CREATE command fails. Let's try again after increasing the maximum core count on all nodes to 2: - -[source,json] -[ {"cores": "<3", "node": "#ANY"} ] - -After re-issuing the `SecondCollection` CREATE command, the replica for `shard1` will be placed on "nodeA": it's least loaded, so is tested first, and no policy violation will result from placement there. The `shard2` replica could be placed on any of the 3 nodes, since they're all equally loaded, and the chosen node will remain below its maximum core count after placement. The CREATE command succeeds. - -== Testing Autoscaling Configuration and Suggestions -It's not always easy to predict the impact of autoscaling configuration changes on the -cluster layout. Starting with release 8.1 Solr provides a tool for assessing the impact of -such changes without affecting the state of the target cluster. - -This testing tool is a part of `bin/solr autoscaling` command. In addition to other -options that provide detailed status of the current cluster layout the following options -specifically allow users to test new autoscaling configurations and run "what if" scenarios: - -`-a `:: -JSON file containing autoscaling configuration to test. This file needs to be in the same -format as the result of the `/solr/admin/autoscaling` call. If this parameter is missing then the -currently deployed autoscaling configuration is used. - -`-simulate`:: -Simulate the effects of applying all autoscaling suggestions on the cluster layout. NOTE: this does not -affect in any way the actual cluster - this option uses the simulation framework to calculate the -new layout without actually making the changes. Calculations are performed in the tool's JVM so they don't -affect the performance of the running cluster either. This process is repeated several times until a limit -is reached or there are no more suggestions left to apply (although unresolved violations may still remain!) - -`-i `:: -Number of iterations of the simulation loop. Default is 10. - -Results of the simulation contain the initial suggestions, suggestions at each step of the -simulation and the final simulated state of the cluster. - -=== Simulation Scenario Tool -The autoscaling command-line tool supports also the execution of end-to-end simulation scenarios consisting of -several cluster- and collection-level operations and events. - -This tool can be invoked using `bin/solr autoscaling -scenario `. All other command-line options are ignored in this mode. - -The file describing a scenario to test uses a simple plain text (UTF-8 encoded) line-oriented format, where -each line of text uses the following syntax: - -[source,text] ----- -line := command whitespace params | '#' -params := [ path, '?' ] key, '=', value { '&', key, '=', value } * ----- - -Keys and values additionally use www-urlencoded format to avoid meta-characters and non-ascii characters. - -The `params` part of the line closely follows a regular Solr parameter representation on purpose - in many cases -the content of this part of the command is passed directly to the respective collection- or cluster-level API. - -==== Scenario Context -Scenario has a context, which is simply a map of key-value pairs. Before executing each command the context is -updated to contain the current values for the following properties: - -* `_random_node_` - randomly selected node name, or null if no node is live -* `_overseer_leader_` - node name of the current Overseer leader node, or absent if there's no Overseer -* `_live_nodes_` - a list of current live nodes, or absent if there are no live nodes -* `_collections_` - a list of existing collections, or absent if there are no collections (or no live nodes) -* `_suggestions_` - a list of autoscaling suggestions generated using CREATE_SUGGESTIONS command. -* `_responses_` - a list of SolrResponse-s resulting from SOLR_REQUEST commands. -* `_loop_iter_` - current loop iteration (as a string), or absent outside of loop. -* `_trigger_event_` - last trigger event captured by WAIT_EVENT - -Command parameters support variable expansion using string values from the current context (non-string values, including numeric, are ignored) -and from system properties, with the context values taking precedence if set. - -For example, assuming a system property is set 'foo=bar', the following command will load a snapshot from -`/tmp/bar`: -[source,text] ----- -load_snapshot path=/tmp/${foo} ----- - -==== Scenario Commands -The following commands are supported (command names are case insensitive, but parameter names are not): - -* `create_cluster numNodes=N[&disableMetricsHistory=false&timeSourcee=simTime:50]` - create a simulated cluster with N nodes -* `load_snapshot (path=/some/path | zkHost=ZK_CONNECT_STRING)` - create a simulated cluster from an autoscaling snapshot or from a live cluster. -* `save_snapshot path=/some/path[&redact=false]` - save an autoscaling snapshot of the current simulated cluster state. -* `calculate_suggestions` - calculate autoscaling suggestions based on the current cluster state and the policy. -* `apply_suggestions` - apply previously calculated suggestions. -* `kill_nodes (numNodes=N | nodes=node1,node2,...)` - kill a number of randomly selected nodes, or specific nodes. -* `add_nodes numNodes=N` - add a number of new nodes. -* `load_autoscaling (path=/some/path | json={...}` - load `autoscaling.json` config from a path or from the supplied JSON string, and apply this config to the simulated cluster. -* `loop_start [iterations=N]`, `loop_end` - iterate commands enclosed in `loop_start` / `loop_end` N times, or until a loop abort is requested. -* `set_op_delays op1=delayMs1&op2=delayMs2...` - set operation delays for specific collection commands to simulate slow execution. -* `solr_request /admin/handler?httpMethod=POST&stream.body={'json':'body'}&other=params` - execute one of SolrRequest types supported by `SimCloudManager`. -* `run [time=60000]` - run the simulator for some time, allowing background tasks to execute (e.g., trigger event processing). -* `wait_collection collection=test&shards=N&replicas=M[&withInactive=false&requireLeaders=true&wait=90]` - wait until the collection shape matches the criteria or the wait time elapses (in which case an error is thrown). -* `event_listener trigger=triggerName&stage=SUCCEEDED[&beforeAction=foo | &afterAction=bar]` - prepare to listen for a specific trigger event. -* `wait_event trigger=triggerName[&wait=90]` - wait until an event specified in `event_listener` is captured or a wait time elapses (in which cases an error is thrown). -* `ctx_set key=myKey&value=myValue` - set a key / value pair in the scenario's context. -* `ctx_remove key=myKey` - remove a key / value pair from the scenario's context. -* `dump [redact=false&withData=false&withStats=false&withSuggestions=false&withDiagnostics=false&withNodeState=false&withClusterState=false&withManagerState=false]` - dump the simulator state to the console. -* `set_node_metrics nodeset=node1,node2...&aKey1=aValue1&aKey2=aValue2...` - set node metrics. -* `set_shard_metrics collection=test&shard=shard1[&delta=false÷=false]&aKey1=aValue1&aKey2=aValue2...` - set per-shard metrics, optionally expressed as delta change from existing values and optionally with the values divided across existing replicas for a shard. -* `index_docs numDocs=NNN[&start=XXX]` - simulate bulk indexing of a large number of documents. -* `assert condition=(equals | not_equals | null | not_null)&(key=objectPath | value=myValue)[&expected=value]` - assert a condition. When `key` is specified then it can be an object path to complex values present in the scenario's context. - -==== Example Scenarios -Example scenario testing the behavior of `.autoAddReplicas` trigger: -[source,text] ----- -# standard comment -// java comment -create_cluster numNodes=2 // inline comment -// load autoscaling config from a JSON string. Notice that the value must be URL-encoded -load_autoscaling json={'cluster-policy'+:+[{'replica'+:+'<3',+'shard'+:+'#EACH',+'collection'+:+'testCollection','node':'#ANY'}]}&defaultWaitFor=10 -solr_request /admin/collections?action=CREATE&autoAddReplicas=true&name=testCollection&numShards=2&replicationFactor=2 -wait_collection collection=testCollection&shards=2&replicas=2 -// prepare a listener for trigger events and the processing state SUCCEEDED -event_listener trigger=.auto_add_replicas&stage=SUCCEEDED -// kill a random node -kill_nodes node=${_random_node_} -// wait for the listener to capture the event -wait_event trigger=.auto_add_replicas&wait=60 -// the collection should have the same shape as before -wait_collection collection=testCollection&shards=2&replicas=2 -save_snapshot path=${snapshotPath} ----- - -Example scenario testing the behavior of `indexSize` trigger. Notice the use of POST SolrRequest and the use of -`assert` command with an object path: - -[source,text] ----- -create_cluster numNodes=100 -solr_request /admin/collections?action=CREATE&autoAddReplicas=true&name=testCollection&numShards=2&replicationFactor=2 -wait_collection collection=testCollection&shards=2&replicas=2 -// example of defining a trigger config -solr_request /admin/autoscaling?httpMethod=POST&stream.body={'set-trigger':{'name':'indexSizeTrigger','event':'indexSize','waitFor':'10s','aboveDocs':1000,'enabled':true,'actions':[{'name':'compute_plan','class':'solr.ComputePlanAction'},{'name':'execute_plan','class':'solr.ExecutePlanAction'}]}} -// prepare an event listener -event_listener trigger=indexSizeTrigger&stage=SUCCEEDED -// add documents -index_docs collection=testCollection&numDocs=3000 -// run for 60 sec -run -// wait for a trigger event (as defined in the listener) -wait_event trigger=indexSizeTrigger&wait=60 -// even is stored in the context -assert condition=not_null&key=_trigger_event_indexSizeTrigger -assert condition=equals&key=_trigger_event_indexSizeTrigger/eventType&expected=INDEXSIZE -assert condition=equals&key=_trigger_event_indexSizeTrigger/properties/requestedOps[0]/action&expected=SPLITSHARD -wait_collection collection=testCollection&shards=6&withInactive=true&requireLeaders=false&replicas=2 ----- - -Example scenario where context variables are used for conditional execution of loops. Depending on the value of -`iterative` and `justCalc` the two loops will execute 0 or more times. Notice also how the scenario picks up -a random node to consistently add replicas to it. - -[source,text] ----- -create_cluster numNodes=2 -solr_request /admin/collections?action=CREATE&autoAddReplicas=true&name=testCollection&numShards=2&replicationFactor=2 -wait_collection collection=testCollection&shards=2&replicas=2 -ctx_set key=myNode&value=${_random_node_} -solr_request /admin/collections?action=ADDREPLICA&collection=testCollection&shard=shard1&node=${myNode} -solr_request /admin/collections?action=ADDREPLICA&collection=testCollection&shard=shard1&node=${myNode} -loop_start iterations=${iterative} - calculate_suggestions - apply_suggestions - solr_request /admin/collections?action=ADDREPLICA&collection=testCollection&shard=shard1&node=${myNode} - solr_request /admin/collections?action=ADDREPLICA&collection=testCollection&shard=shard1&node=${myNode} -loop_end -loop_start iterations=${justCalc} - calculate_suggestions -loop_end -dump redact=true ----- diff --git a/solr/solr-ref-guide/src/solrcloud-autoscaling-trigger-actions.adoc b/solr/solr-ref-guide/src/solrcloud-autoscaling-trigger-actions.adoc deleted file mode 100644 index 5bf63f665c2..00000000000 --- a/solr/solr-ref-guide/src/solrcloud-autoscaling-trigger-actions.adoc +++ /dev/null @@ -1,145 +0,0 @@ -= SolrCloud Autoscaling Trigger Actions -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -`TriggerAction` implementations process events generated by triggers in order to ensure the cluster's -health and good use of resources. - -Currently two implementations are provided: `ComputePlanAction` and `ExecutePlanAction`. - -== Compute Plan Action - -The `ComputePlanAction` uses the policy and preferences to calculate the optimal set of Collection API -commands which can re-balance the cluster in response to trigger events. - -The following parameters are configurable: - -`collections`:: -A comma-separated list of collection names, or a selector on collection properties that can be used to filter collections for which the plan is computed. -+ -If a non-empty list or selector is specified then the computed operations will only calculate collection operations that affect -matched collections and ignore any other collection operations for collections -not listed here. This does not affect non-collection operations. -+ -A collection selector is of the form `collections: {key1: value1, key2: value2, ...}` where the key can be any collection property such as `name`, `policy`, `numShards`, etc. -The value must match exactly and all specified properties must match for a collection to match. -+ -A collection selector is useful in a cluster where collections are added and removed frequently and where selecting only collections that -use a specific autoscaling policy is useful. - -Example configurations: - -[source,json] ----- -{ - "set-trigger" : { - "name" : "node_added_trigger", - "event" : "nodeAdded", - "waitFor" : "1s", - "enabled" : true, - "actions" : [ - { - "name" : "compute_plan", - "class" : "solr.ComputePlanAction", - "collections" : "test1,test2" - }, - { - "name" : "execute_plan", - "class" : "solr.ExecutePlanAction" - } - ] - } -} ----- - -In this example only collections `test1` and `test2` will be potentially -replicated / moved to an added node, other collections will be ignored even -if they cause policy violations. - -[source,json] ----- -{ - "set-trigger" : { - "name" : "node_added_trigger", - "event" : "nodeAdded", - "waitFor" : "1s", - "enabled" : true, - "actions" : [ - { - "name" : "compute_plan", - "class" : "solr.ComputePlanAction", - "collections" : {"policy": "my_policy"} - }, - { - "name" : "execute_plan", - "class" : "solr.ExecutePlanAction" - } - ] - } -} ----- - -In this example only collections which use the `my_policy` as their autoscaling policy will be potentially replicated / moved to an added node, other collections will be ignored even if they cause policy violations. - -[source,json] ----- -{ - "set-trigger" : { - "name" : "node_added_trigger", - "event" : "nodeAdded", - "waitFor" : "1s", - "enabled" : true, - "actions" : [ - { - "name" : "compute_plan", - "class" : "solr.ComputePlanAction", - "collections" : {"policy": "my_policy", "numShards" : "4"} - }, - { - "name" : "execute_plan", - "class" : "solr.ExecutePlanAction" - } - ] - } -} ----- - -In this example only collections which use the `my_policy` as their autoscaling policy and that have `numShards` equal to `4` will be potentially replicated / moved to an added node, other collections will be ignored even if they cause policy violations. - -== Execute Plan Action - -The `ExecutePlanAction` executes the Collection API commands emitted by the `ComputePlanAction` against -the cluster using SolrJ. It executes the commands serially, waiting for each of them to succeed before -continuing with the next one. - -Currently, it has the following configurable parameters: - -`taskTimeoutSeconds`:: -Default value of this parameter is 120 seconds. This value defines how long the action will wait for a -command to complete its execution. If a timeout is reached while the command is still running then -the command status is provisionally considered a success but a warning is logged, unless `taskTimeoutFail` -is set to true. - -`taskTimeoutFail`:: -Boolean with a default value of false. If this value is true then a timeout in command processing will be -marked as failure and an exception will be thrown. - -If the Overseer node fails while `ExecutePlanAction` is running, -then the new Overseer node will run the chain of actions for the same event again after waiting for any -running Collection API operations belonging to the event to complete. - -Please see <> for more details on fault tolerance within the autoscaling framework. diff --git a/solr/solr-ref-guide/src/solrcloud-autoscaling-triggers.adoc b/solr/solr-ref-guide/src/solrcloud-autoscaling-triggers.adoc deleted file mode 100644 index 9a0b8238f94..00000000000 --- a/solr/solr-ref-guide/src/solrcloud-autoscaling-triggers.adoc +++ /dev/null @@ -1,627 +0,0 @@ -= SolrCloud Autoscaling Triggers -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -Triggers are used in autoscaling to watch for cluster events such as nodes joining or leaving, search rate, index rate, on a schedule, or any other metric breaching a threshold. - -Trigger implementations verify the state of resources that they monitor. When they detect a -change that merits attention they generate _events_, which are then queued and processed by configured -`TriggerAction` implementations. This usually involves computing and executing a plan to do something (e.g., move replicas). Solr provides predefined implementations of triggers for <>. - -Triggers execute on the node that runs `Overseer`. They are scheduled to run periodically, at a default interval of 1 second between each execution (although it's important to note that not every execution of a trigger produces events). - -== Event Types -Currently the following event types (and corresponding trigger implementations) are defined: - -* `nodeAdded`: generated when a node joins the cluster. See <>. -* `nodeLost`: generated when a node leaves the cluster. See <> and <>. -* `metric`: generated when the configured metric crosses a configured lower or upper threshold value. See <>. -* `indexSize`: generated when a shard size (defined as index size in bytes or number of documents) -exceeds upper or lower threshold values. See <>. -* `searchRate`: generated when the search rate exceeds configured upper or lower thresholds. See <>. -* `scheduled`: generated according to a scheduled time period such as every 24 hours, etc. See <>. - -Events are not necessarily generated immediately after the corresponding state change occurred; the -maximum rate of events is controlled by the `waitFor` configuration parameter (see <> below for more explanation). - -The following properties are common to all event types: - -`id`:: (string) A unique time-based event id. - -`eventType`:: (string) The type of event. - -`source`:: (string) The name of the trigger that produced this event. - -`eventTime`:: (long) Unix time when the condition that caused this event occurred. For example, for a -`nodeAdded` event this will be the time when the node was added and not when the event was actually -generated, which may significantly differ due to the rate limits set by `waitFor`. - -`properties`:: (map, optional) Any additional properties. Currently includes e.g., `nodeNames` property that -indicates the nodes that were lost or added. - -== Trigger Configuration -Trigger configurations are managed using the <> with the commands `<>`, `<>`, -`suspend-trigger`, and `resume-trigger`. - -=== Trigger Properties - -Trigger configuration consists of the following properties: - -`name`:: (string, required) A unique trigger configuration name. - -`event`:: (string, required) One of the predefined event types (`nodeAdded` or `nodeLost`). - -`actions`:: (list of action configs, optional) An ordered list of actions to execute when event is fired. - -`waitFor`:: (string, optional) The time to wait between generating new events, as an integer number immediately -followed by unit symbol, one of `s` (seconds), `m` (minutes), or `h` (hours). Default is `0s`. A condition must -persist at least for the `waitFor` period to generate an event. - -`enabled`:: (boolean, optional) When `true` the trigger is enabled. Default is `true`. - -Additional implementation-specific properties may be provided, as described in the sections for individual triggers below. - -=== Action Properties - -Action configuration consists of the following properties: - -`name`:: (string, required) A unique name of the action configuration. - -`class`:: (string, required) The action implementation class. - -Additional implementation-specific properties may be provided, as described in the sections for individual triggers below. - -If the `actions` configuration is omitted, then by default, the `ComputePlanAction` and the `ExecutePlanAction` are automatically added to the trigger configuration. - -=== Example Trigger Configuration - -This simple example shows the configuration for adding (or updating) a trigger for `nodeAdded` events. - -[source,json] ----- -{ - "set-trigger": { - "name" : "node_added_trigger", - "event" : "nodeAdded", - "waitFor" : "1s", - "enabled" : true, - "actions" : [ - { - "name" : "compute_plan", - "class": "solr.ComputePlanAction" - }, - { - "name" : "custom_action", - "class": "com.example.CustomAction" - }, - { - "name" : "execute_plan", - "class": "solr.ExecutePlanAction" - } - ] - } -} ----- - -This trigger configuration will compute and execute a plan to allocate the resources available on the new node. A custom action could also be used to possibly modify the plan. - -== Available Triggers - -As described earlier, there are several triggers available to watch for events. - -=== Node Added Trigger - -The `NodeAddedTrigger` generates `nodeAdded` events when a node joins the cluster. It can be used to either move replicas -from other nodes to the new node or to add new replicas. - -In addition to the parameters described at <>, this trigger supports one more parameter: - -`preferredOperation`:: (string, optional, defaults to `movereplica`) The operation to be performed in response to an event generated by this trigger. By default, replicas will be moved from other nodes to the added node. The only other supported value is `addreplica` which adds more replicas of the existing collections on the new node. - -`replicaType`:: (string, optional, defaults to `NRT`) The replica type that will be used to add replicas, in response to an event generated by this trigger, when "preferredOperation" is "ADDREPLICA". By default, the replica(s) will be of type `NRT`. The only other supported values are `PULL` and `TLOG`, which will add more replicas of the specified type to the existing collections on the new node. - -.Example: Node Added Trigger to move replicas to new node -[source,json] ----- -{ - "set-trigger": { - "name": "node_added_trigger", - "event": "nodeAdded", - "waitFor": "5s" - } -} ----- - -.Example: Node Added Trigger to add replicas on new node with replica type PULL -[source,json] ----- -{ - "set-trigger": { - "name": "node_added_trigger", - "event": "nodeAdded", - "waitFor": "5s", - "preferredOperation": "ADDREPLICA", - "replicaType": "PULL" - } -} ----- - -=== Node Lost Trigger - -The `NodeLostTrigger` generates `nodeLost` events when a node leaves the cluster. It can be used to either move replicas -that were hosted by the lost node to other nodes or to delete them from the cluster. - -In addition to the parameters described at <>, this trigger supports the one more parameter: - -`preferredOperation`:: (string, optional, defaults to `MOVEREPLICA`) The operation to be performed in response to an event generated by this trigger. By default, replicas will be moved from the lost nodes to the other nodes in the cluster. The only other supported value is `DELETENODE` which deletes all information about replicas that were hosted by the lost node. - -.Example: Node Lost Trigger to move replicas to new node -[source,json] ----- -{ - "set-trigger": { - "name": "node_lost_trigger", - "event": "nodeLost", - "waitFor": "120s" - } -} ----- - -.Example: Node Lost Trigger to delete replicas -[source,json] ----- -{ - "set-trigger": { - "name": "node_lost_trigger", - "event": "nodeLost", - "waitFor": "120s", - "preferredOperation": "DELETENODE" - } -} ----- - -TIP: It is recommended that the value of `waitFor` configuration for the node lost trigger be larger than 1 minute so that large full garbage collection pauses do not cause this trigger to generate events and needlessly move or delete replicas in the cluster. - -=== Auto Add Replicas Trigger - -When a collection has the parameter `autoAddReplicas` set to true then a trigger configuration named `.auto_add_replicas` is automatically created to watch for nodes going away. This trigger produces `nodeLost` events, -which are then processed by configured actions (usually resulting in computing and executing a plan -to add replicas on the live nodes to maintain the expected replication factor). - -Refer to the section <> to learn more about how the `.autoAddReplicas` trigger works. - -If you would like to change the value of `.autoAddReplicas` trigger, you need to call the autoscaling API and use the `set-trigger` command to add a value for `waitFor`. - -.Example: Updating Trigger with wait for 5 seconds ----- -{ - "set-trigger": { - "name": ".auto_add_replicas", - "event": "nodeLost, - "waitFor": "5s", - "enabled": true, - "actions": [ - { - "name": "auto_add_replicas_plan", - "class": "solr.AutoAddReplicasPlanAction" - }, - { - "name": "execute_plan", - "class": "solr.ExecutePlanAction" - } - ] - } -} ----- - -TIP: See < Element>> for more details about how to work with `solr.xml`. - -=== Metric Trigger - -The metric trigger can be used to monitor any metric exposed by the <>. It supports lower and upper threshold configurations as well as optional filters to limit operation to specific collection, shards, and nodes. - -In addition to the parameters described at <>, this trigger supports the following parameters: - -`metrics`:: -(string, required) The metric property name to be watched in the format metrics:__group__:__prefix__, e.g., `metrics:solr.node:CONTAINER.fs.coreRoot.usableSpace`. - -`below`:: -(double, optional) The lower threshold for the metric value. The trigger produces a metric breached event if the metric's value falls below this value. - -`above`:: -(double, optional) The upper threshold for the metric value. The trigger produces a metric breached event if the metric's value crosses above this value. - -`collection`:: -(string, optional) The collection used to limit the nodes on which the given metric is watched. When the metric is breached, trigger actions will limit operations to this collection only. - -`shard`:: -(string, optional) The shard used to limit the nodes on which the given metric is watched. When the metric is breached, trigger actions will limit operations to this shard only. - -`node`:: -(string, optional) The node on which the given metric is watched. Trigger actions will operate on this node only. - -`preferredOperation`:: -(string, optional, defaults to `MOVEREPLICA`) The operation to be performed in response to an event generated by this trigger. By default, replicas will be moved from the hot node to others. The only other supported value is `ADDREPLICA` which adds more replicas if the metric is breached. - -.Example: a metric trigger that fires when total usable space on a node having replicas of "mycollection" falls below 100GB -[source,json] ----- -{ - "set-trigger": { - "name": "metric_trigger", - "event": "metric", - "waitFor": "5s", - "metrics": "metric:solr.node:CONTAINER.fs.coreRoot.usableSpace", - "below": 107374182400, - "collection": "mycollection" - } -} ----- - -=== Index Size Trigger -This trigger can be used for monitoring the size of collection shards, measured either by the -number of documents in a shard or the physical size of the shard's index in bytes. - -When either of the upper thresholds is exceeded for a particular shard the trigger will generate -an event with a (configurable) requested operation to perform on the offending shards - by default -this is a SPLITSHARD operation. - -Similarly, when either of the lower thresholds is exceeded the trigger will generate an -event with a (configurable) requested operation to perform on two of the smallest -shards. By default this is a MERGESHARDS operation, and is currently ignored because -that operation is not yet implemented (see https://issues.apache.org/jira/browse/SOLR-9407[SOLR-9407]). - -When `splitMethod=link` is used the resulting sub-shards will initially have nearly the same size -as the parent shard due to the hard-linking of parent index files, and will differ just in the lists of -deleted documents. In order to correctly recognize the effectively reduced index size an estimate -is calculated using a simple formula: `indexCommitSize * numDocs / maxDoc`. This value is then -compared with `aboveBytes` and `belowBytes` limits. - -Additionally, monitoring can be restricted to a list of collections; by default -all collections are monitored. - -In addition to the parameters described at <>, this trigger supports the following configuration parameters (all thresholds are exclusive): - -`aboveBytes`:: -An upper threshold in bytes. This value is compared to the `SEARCHER.searcher.indexCommitSize` metric, which -reports the size of the latest commit point (ignoring any data related to earlier commit points, which may be -still present for replication or snapshot purposes). See also the note above how this value is used with -`splitMethod=link`. - -`belowBytes`:: -A lower threshold in bytes. Note that this value should be at least 2x smaller than -`aboveBytes` - -`aboveDocs`:: -An upper threshold expressed as the number of documents. This value is compared with `SEARCHER.searcher.numDocs` metric. -+ -NOTE: Due to the way Lucene indexes work, a shard may exceed the `aboveBytes` threshold -on disk even if the number of documents is relatively small, because replaced and deleted documents keep -occupying disk space until they are actually removed during Lucene index merging. - -`belowDocs`:: -A lower threshold expressed as the number of documents. - -`aboveOp`:: -The operation to request when an upper threshold is exceeded. If not specified the -default value is `SPLITSHARD`. - -`belowOp`:: -The operation to request when a lower threshold is exceeded. If not specified -the default value is `MERGESHARDS` (but see the note above). - -`collections`:: -A comma-separated list of collection names that this trigger should monitor. If not -specified or empty all collections are monitored. - -`maxOps`:: -Maximum number of operations requested in a single event. This property limits the speed of -changes in a highly dynamic situation, which may lead to more serious threshold violations, -but it also limits the maximum load on the cluster that the large number of requested -operations may cause. The default value is 10. - -`splitMethod`:: -One of the supported methods for index splitting to use. Default value is `rewrite`, which is -slow and puts a high CPU load on the shard leader but results in optimized sub-shard indexes. -The `link` method is much faster and puts very little load on the shard leader but results in -indexes that are initially as large as the parent shard's index, which slows down replication and -may lead to excessive initial disk space consumption on replicas. - -`splitFuzz`:: -A float value (default is 0.0f, must be smaller than 0.5f) that allows to vary the sub-shard ranges -by this percentage of total shard range, odd shards being larger and even shards being smaller. -Non-zero values are useful for large indexes with aggressively growing size, as they help to prevent -avalanches of split shard requests when the total size of the index -reaches even multiples of the maximum shard size thresholds. - -`splitByPrefix`:: -A boolean value (default is false) that specifies whether the aboveOp shard split should try to -calculate sub-shard hash ranges according to document prefixes, or do a traditional shard split (i.e. -split the hash range into n sub-ranges). - -Events generated by this trigger contain additional details about the shards -that exceeded thresholds and the types of violations (upper / lower bounds, bytes / docs metrics). - -.Example: Index Size Trigger -This configuration specifies an index size trigger that monitors collections "test1" and "test2", -with both bytes (1GB) and number of docs (1 million) upper limits, and a custom `belowOp` -operation `NONE` (which still can be monitored and acted upon by an appropriate trigger listener): - -[source,json] ----- -{ - "set-trigger": { - "name" : "index_size_trigger", - "event" : "indexSize", - "collections" : "test1,test2", - "aboveBytes" : 1000000000, - "aboveDocs" : 1000000000, - "belowBytes" : 200000, - "belowDocs" : 200000, - "belowOp" : "NONE", - "waitFor" : "1m", - "enabled" : true, - "actions" : [ - { - "name" : "compute_plan", - "class": "solr.ComputePlanAction" - }, - { - "name" : "execute_plan", - "class": "solr.ExecutePlanAction" - } - ] - } -} ----- - -=== Search Rate Trigger - -The search rate trigger can be used for monitoring search rates in a selected -collection (1-min average rate by default), and request that either replicas be moved from -"hot nodes" to different nodes, or new replicas be added to "hot shards" to reduce the -per-replica search rate for a collection or shard with hot spots. - -Similarly, if the search rate falls below a threshold then the trigger may request that some -replicas are deleted from "cold" shards. It can also optionally issue node-level action requests -when a cumulative node-level rate falls below a threshold. - -Per-shard rates are calculated as arithmetic average of rates of all searchable replicas in a given shard. -This method was chosen to avoid generating false events when a simple client keeps sending requests -to a single specific replica (because adding or removing other replicas can't solve this situation, -only proper load balancing can - either by using `CloudSolrClient` or another load-balancing client). - -This trigger calculates node-level cumulative rates using per-replica rates reported by -replicas that are part of monitored collections / shards on each node. This means that it may report -some nodes as "cold" (underutilized) because it ignores other, perhaps more active, replicas -belonging to other collections. Also, nodes that don't host any of the monitored replicas or -those that are explicitly excluded by `node` configuration property won't be reported at all. - -.Calculating `waitFor` -[CAUTION] -==== -Special care should be taken when configuring the `waitFor` property. By default the trigger -monitors a 1-minute average search rate of a replica. Changes to the number of replicas that should in turn -change per-replica search rates may be requested and executed relatively quickly if the -`waitFor` is set to comparable values of 1 min or shorter. - -However, the metric value, being a moving average, will always lag behind the new "momentary" rate after the changes. This in turn means that the monitored metric may not change sufficiently enough to prevent the -trigger from firing again, because it will continue to measure the average rate as still violating -the threshold for some time after the change was executed. As a result the trigger may keep -requesting that even more replicas be added (or removed) and thus it may "overshoot" the optimal number of replicas. - -For this reason it's recommended to always set `waitFor` to values several -times longer than the time constant of the used metric. For example, with the default 1-minute average the -`waitFor` should be set to at least `2m` (2 minutes) or more. -==== - -In addition to the parameters described at <>, this trigger supports the following configuration properties: - -`collections`:: -(string, optional) A comma-separated list of collection names to monitor, or any collection if empty or not set. - -`shard`:: -(string, optional) A shard name within the collection (requires `collections` to be set to exactly one name), or any shard if empty. - -`node`:: -(string, optional) A node name to monitor, or any if empty. - -`metric`:: -(string, optional) A metric name that represents the search rate. The default is `QUERY./select.requestTimes:1minRate`. This name has to identify a single numeric metric value, and it may use the colon syntax for selecting one property of -a complex metric. This value is collected from all replicas for a shard, and then an arithmetic average is calculated -per shard to determine shard-level violations. - -`maxOps`:: -(integer, optional) The maximum number of `ADDREPLICA` or `DELETEREPLICA` operations -requested in a single autoscaling event. The default value is `3` and it helps to smooth out -the changes to the number of replicas during periods of large search rate fluctuations. - -`minReplicas`:: -(integer, optional) The minimum acceptable number of searchable replicas (i.e., replicas other -than `PULL` type). The trigger will not generate any `DELETEREPLICA` requests when the number of -searchable replicas in a shard reaches this threshold. -+ -When this value is not set (the default) -the `replicationFactor` property of the collection is used, and if that property is not set then -the value is set to `1`. Note also that shard leaders are never deleted. - -`aboveRate`:: -(float) The upper bound for the request rate metric value. At least one of -`aboveRate` or `belowRate` must be set. - -`belowRate`:: -(float) The lower bound for the request rate metric value. At least one of -`aboveRate` or `belowRate` must be set. - -`aboveNodeRate`:: -(float) The upper bound for the total request rate metric value per node. If not -set then cumulative per-node rates will be ignored. - -`belowNodeRate`:: -(float) The lower bound for the total request rate metric value per node. If not -set then cumulative per-node rates will be ignored. - -`aboveOp`:: -(string, optional) A collection action to request when the upper threshold for a shard is -exceeded. Default action is `ADDREPLICA` and the trigger will request from 1 up to `maxOps` operations -per shard per event, proportionally to how much the rate is exceeded. This property can be set to 'NONE' -to effectively disable the action but still report it to the listeners. - -`aboveNodeOp`:: -(string, optional) The collection action to request when the upper threshold for a node (`aboveNodeRate`) is exceeded. -Default action is `MOVEREPLICA`, and the trigger will request 1 replica operation per hot node per event. -If both `aboveOp` and `aboveNodeOp` operations are to be requested then `aboveNodeOp` operations are -always requested first, and only if no `aboveOp` (shard level) operations are to be requested (because `aboveOp` -operations will change node-level rates anyway). This property can be set to 'NONE' to effectively disable -the action but still report it to the listeners. - -`belowOp`:: -(string, optional) The collection action to request when the lower threshold for a shard is -exceeded. Default action is `DELETEREPLICA`, and the trigger will request at most `maxOps` replicas -to be deleted from eligible cold shards. This property can be set to 'NONE' -to effectively disable the action but still report it to the listeners. - -`belowNodeOp`:: -(string, optional) The action to request when the lower threshold for a node (`belowNodeRate`) is exceeded. -Default action is null (not set) and the condition is ignored, because in many cases the -trigger will monitor only some selected resources (replicas from selected -collections or shards) so setting this by default to e.g., `DELETENODE` could interfere with -these non-monitored resources. The trigger will request 1 operation per cold node per event. -If both `belowOp` and `belowNodeOp` operations are requested then `belowOp` operations are -always requested first. - -.Example: -A search rate trigger that monitors collection "test" and adds new replicas if 5-minute -average request rate of "/select" handler exceeds 100 requests/sec, and the condition persists -for over 20 minutes. If the rate falls below 0.01 and persists for 20 min the trigger will -request not only replica deletions (leaving at most 1 replica per shard) but also it may -request node deletion. - -[source,json] ----- -{ - "set-trigger": { - "name" : "search_rate_trigger", - "event" : "searchRate", - "collections" : "test", - "metric" : "QUERY./select.requestTimes:5minRate", - "aboveRate" : 100.0, - "belowRate" : 0.01, - "belowNodeRate" : 0.01, - "belowNodeOp" : "DELETENODE", - "minReplicas" : 1, - "waitFor" : "20m", - "enabled" : true, - "actions" : [ - { - "name" : "compute_plan", - "class": "solr.ComputePlanAction" - }, - { - "name" : "execute_plan", - "class": "solr.ExecutePlanAction" - } - ] - } -} ----- - -[[scheduledtrigger]] -=== Scheduled Trigger - -The Scheduled trigger generates events according to a fixed rate schedule. - -In addition to the parameters described at <>, this trigger supports the following configuration: - -`startTime`:: -(string, required) The start date/time of the schedule. This should either be a DateMath string e.g., 'NOW', or be an ISO-8601 date time string (the same standard used during search and indexing in Solr, which defaults to UTC), or be specified without the trailing 'Z' accompanied with the `timeZone` parameter. For example, each of the following values are acceptable: -* `2018-01-31T15:30:00Z`: ISO-8601 date time string. The trailing `Z` signals that the time is in UTC -* `NOW+5MINUTES`: Solr's date math string -* `2018-01-31T15:30:00`: No trailing 'Z' signals that the `timeZone` parameter must be specified to avoid ambiguity - -`every`:: -(string, required) A positive Solr date math string which is added to the `startTime` or the last run time to arrive at the next scheduled time. - -`graceTime`:: -(string, optional) A positive Solr date math string. This is the additional grace time over the scheduled time within which the trigger is allowed to generate an event. - -`timeZone`:: -(string, optional) A time zone string which is used for calculating the scheduled times. - -`preferredOperation`:: -(string, optional, defaults to `MOVEREPLICA`) The preferred operation to perform in response to an event generated by this trigger. The only supported values are `MOVEREPLICA` or `ADDREPLICA`. - -This trigger applies the `every` date math expression on the `startTime` or the last event time to derive the next scheduled time and if current time is greater than next scheduled time but within `graceTime` then an event is generated. - -Apart from the common event properties described in the <> section, the trigger adds an additional `actualEventTime` event property which has the actual event time as opposed to the scheduled time. - -For example, if the scheduled time was `2018-01-31T15:30:00Z` and grace time was `+15MINUTES` then an event may be fired at `2018-01-31T15:45:00Z`. Such an event will have `eventTime` as `2018-01-31T15:30:00Z`, the scheduled time, but the `actualEventTime` property will have a value of `2018-01-31T15:45:00Z`, the actual time. - -.Frequently scheduled events and trigger starvation -[CAUTION] -==== -Be cautious with scheduled triggers that are set to run as or more frequently than the trigger cooldown period (defaults to 5 seconds). - -Solr pauses all triggers for a cooldown period after a trigger fires so that the system has some time to stabilize. An aggressive scheduled trigger can starve all other triggers from -ever executing if a new scheduled event is ready as soon as the cooldown period is over. The same starvation scenario can happen to the scheduled trigger as well. - -Solr randomizes the order in which the triggers are resumed after the cooldown period to mitigate this problem. However, it is recommended that scheduled triggers -are not used with low `every` values and an external scheduling process such as cron be used for such cases instead. -==== - -== Default Triggers -A fresh installation of SolrCloud always creates some default triggers. If these triggers are missing (e.g., they were -deleted) they are re-created on any autoscaling configuration change or Overseer restart. These triggers can be -suspended if their functionality somehow interferes with other configuration but they can't be permanently deleted. - -=== Auto-add Replicas Trigger -The default configuration and functionality of this trigger is described in detail in the -section titled <>. - -=== Scheduled Maintenance Trigger -This is a <> named `.scheduled_maintenance` and it's configured to run once per day. -It executes the following actions: - -==== `solr.InactiveShardPlanAction` -This action checks existing collections for any shards in `INACTIVE` state, which indicates that they -are the original parent shards remaining after a successful `SPLITSHARD` operation. - -These shards are not immediately deleted because shard splitting is a complex operation that may fail in -non-obvious ways, so keeping the original parent shard gives users a chance to recover from potential failures. - -However, keeping these shards indefinitely doesn't make sense either because they still use system -resources (their Solr cores are still being loaded, and their indexes still occupy disk space). -This scheduled action is responsible for removing such inactive parent shards after their -time-to-live expires. By default the TTL is set to 48 hours after the shard state was set to -`INACTIVE`. When this TTL elapses this scheduled action requests that the shard be deleted, which is then -executed by `solr.ExecutePlanAction` that is configured for this trigger. - -==== `solr.InactiveMarkersPlanAction` -When a node is lost or added an event is generated - but if the lost node was the one running -Overseer leader such event may not be properly processed by the triggers (which run in the Overseer leader context). -For this reason a special marker is created in ZooKeeper so that when the next Overseer leader is elected the -triggers will be able to learn about and process these past events. - -Triggers don't delete these markers once they are done processing (because several triggers may need them and e.g., -scheduled triggers may run at arbitrary times with arbitrary delays) so Solr needs a mechanism to clean up -old markers for such events so that they don't accumulate over time. This trigger action performs the clean-up -- it deletes markers older than the configured time-to-live (by default it's 48 hours). - -=== `solr.ExecutePlanAction` -This action simply executes any collection admin requests generated by other -actions - in particular, in the default configuration it executes `DELETESHARD` requests produced by -`solr.InactiveShardPlanAction`, as described above. diff --git a/solr/solr-ref-guide/src/solrcloud-autoscaling.adoc b/solr/solr-ref-guide/src/solrcloud-autoscaling.adoc deleted file mode 100644 index 5f9f1487037..00000000000 --- a/solr/solr-ref-guide/src/solrcloud-autoscaling.adoc +++ /dev/null @@ -1,35 +0,0 @@ -= SolrCloud Autoscaling -:page-children: solrcloud-autoscaling-overview, solrcloud-autoscaling-policy-preferences, solrcloud-autoscaling-triggers, solrcloud-autoscaling-trigger-actions, solrcloud-autoscaling-listeners, solrcloud-autoscaling-auto-add-replicas, solrcloud-autoscaling-fault-tolerance, solrcloud-autoscaling-api, migrate-to-policy-rule -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -[.lead] -The goal of autoscaling is to make SolrCloud cluster management easier by providing a way for changes to the cluster to be more automatic and more intelligent. - -Autoscaling includes an API to manage cluster-wide and collection-specific policies and preferences and a rules syntax to define the guidelines for your cluster. Also included are features to utilize the policies and preferences so they perform actions automatically when certain conditions are met. - -The following sections describe the autoscaling features of SolrCloud: - -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> diff --git a/solr/solr-ref-guide/src/solrcloud.adoc b/solr/solr-ref-guide/src/solrcloud.adoc index 909a8787f2c..73f5e60339d 100644 --- a/solr/solr-ref-guide/src/solrcloud.adoc +++ b/solr/solr-ref-guide/src/solrcloud.adoc @@ -1,5 +1,5 @@ = SolrCloud -:page-children: getting-started-with-solrcloud, how-solrcloud-works, solrcloud-resilience, solrcloud-configuration-and-parameters, rule-based-replica-placement, cross-data-center-replication-cdcr, solrcloud-autoscaling, colocating-collections +:page-children: getting-started-with-solrcloud, how-solrcloud-works, solrcloud-resilience, solrcloud-configuration-and-parameters, rule-based-replica-placement, cross-data-center-replication-cdcr // Licensed to the Apache Software Foundation (ASF) under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information @@ -44,6 +44,4 @@ In this section, we'll cover everything you need to know about using Solr in Sol ** <> ** <> * <> -* <> -* <> -* <> +* <> \ No newline at end of file diff --git a/solr/solr-ref-guide/src/suggestions-screen.adoc b/solr/solr-ref-guide/src/suggestions-screen.adoc deleted file mode 100644 index 1936c136143..00000000000 --- a/solr/solr-ref-guide/src/suggestions-screen.adoc +++ /dev/null @@ -1,40 +0,0 @@ -= Suggestions Screen -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -[.lead] -The Suggestions screen shows violations to an <> that exist in the system, and allows you to take action to correct the violations. - -This screen is a visual representation of the output of the <>. - -When there are no violations or other suggestions, the screen will appear somewhat blank: - -image::images/suggestions-screen/no-violations.png[] - -When the system is in violation of an aspect of a policy, each violation will be shown, as in this screenshot: - -image::images/suggestions-screen/violations.png[] - -A line is shown for each violation. In this case, we have defined a policy where no replica can exist on a node that has less than 500Gb of available disk space. In this example, 4 replicas in our sample cluster violates this rule. - -In the "Action" column, the green button allows you to execute the recommended change to allow the system to return to compliance with the policy. If you hover your mouse over this button, you will see the recommended Collections API command: - -image::images/suggestions-screen/violations-with-action.png[] - -In this case, the recommendation is to issue a MOVEREPLICA command to move this replica to a node with more available disk space. - -NOTE: Since autoscaling features are only available in SolrCloud mode, this screen will only appear when running Solr in SolrCloud mode. diff --git a/solr/solr-ref-guide/src/using-the-solr-administration-user-interface.adoc b/solr/solr-ref-guide/src/using-the-solr-administration-user-interface.adoc index b05bb84c40f..a74b3edbf00 100644 --- a/solr/solr-ref-guide/src/using-the-solr-administration-user-interface.adoc +++ b/solr/solr-ref-guide/src/using-the-solr-administration-user-interface.adoc @@ -1,5 +1,5 @@ = Using the Solr Administration User Interface -:page-children: overview-of-the-solr-admin-ui, logging, cloud-screens, collections-core-admin, java-properties, thread-dump, suggestions-screen, collection-specific-tools, core-specific-tools +:page-children: overview-of-the-solr-admin-ui, logging, cloud-screens, collections-core-admin, java-properties, thread-dump, collection-specific-tools, core-specific-tools // Licensed to the Apache Software Foundation (ASF) under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information @@ -26,7 +26,6 @@ The <>* explains how to get management information about each core. * *<>* shows the Java information about each core. * *<>* lets you see detailed information about each thread, along with state information. -* *<>* displays the state of the system with regard to the autoscaling policies that are in place. * *<>* is a section explaining additional screens available for each collection. // TODO: SOLR-10655 BEGIN: refactor this into a 'collection-screens-list.include.adoc' file for reuse diff --git a/solr/solr-ref-guide/src/v2-api.adoc b/solr/solr-ref-guide/src/v2-api.adoc index 8d895fe6933..2ef42bb1b74 100644 --- a/solr/solr-ref-guide/src/v2-api.adoc +++ b/solr/solr-ref-guide/src/v2-api.adoc @@ -126,9 +126,6 @@ Example of introspect for a POST API: `\http://localhost:8983/api/c/gettingstart "documentation":"https://lucene.apache.org/solr/guide/rule-based-replica-placement.html", "description":"Details of the snitch provider", "items":{"type":"string"}}, - "autoAddReplicas":{ - "type":"boolean", - "description":"When set to true, enables auto addition of replicas on shared file systems (such as HDFS). See https://lucene.apache.org/solr/guide/running-solr-on-hdfs.html for more details on settings and overrides."}, "replicationFactor":{ "type":"string", "description":"The number of replicas to be created for each shard. Replicas are physical copies of each shard, acting as failover for the shard. Note that changing this value on an existing collection does not automatically add more replicas to the collection. However, it will allow add-replica commands to succeed."}}}}}], @@ -152,7 +149,7 @@ For the "gettingstarted" collection, set the replication factor and whether to a [source,bash] ---- $ curl http://localhost:8983/api/c/gettingstarted -H 'Content-type:application/json' -d ' -{ modify: { replicationFactor: "3", autoAddReplicas: false } }' +{ modify: { replicationFactor: "3" } }' {"responseHeader":{"status":0,"QTime":842}} ---- @@ -171,7 +168,7 @@ Set a cluster property: [source,bash] ---- $ curl http://localhost:8983/api/cluster -H 'Content-type: application/json' -d ' -{ set-property: { name: autoAddReplicas, val: "false" } }' +{ set-property: { name: maxCoresPerNode, val: "100" } }' {"responseHeader":{"status":0,"QTime":4}} ---- diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/AlreadyExistsException.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/AlreadyExistsException.java similarity index 94% rename from solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/AlreadyExistsException.java rename to solr/solrj/src/java/org/apache/solr/client/solrj/cloud/AlreadyExistsException.java index e29bd6f0231..ef9ee0c63d1 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/AlreadyExistsException.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/AlreadyExistsException.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.solr.client.solrj.cloud.autoscaling; +package org.apache.solr.client.solrj.cloud; /** * diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/BadVersionException.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/BadVersionException.java similarity index 95% rename from solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/BadVersionException.java rename to solr/solrj/src/java/org/apache/solr/client/solrj/cloud/BadVersionException.java index 757d9793db3..742f77141dc 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/BadVersionException.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/BadVersionException.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.solr.client.solrj.cloud.autoscaling; +package org.apache.solr.client.solrj.cloud; /** * diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/DelegatingCloudManager.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/DelegatingCloudManager.java new file mode 100644 index 00000000000..98dee9633e3 --- /dev/null +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/DelegatingCloudManager.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.client.solrj.cloud; + +import java.io.IOException; +import java.util.Map; + +import org.apache.solr.client.solrj.SolrRequest; +import org.apache.solr.client.solrj.SolrResponse; +import org.apache.solr.client.solrj.impl.ClusterStateProvider; +import org.apache.solr.common.util.ObjectCache; +import org.apache.solr.common.util.TimeSource; + +/** + * Base class for overriding some behavior of {@link SolrCloudManager}. + */ +public class DelegatingCloudManager implements SolrCloudManager { + protected final SolrCloudManager delegate; + private ObjectCache objectCache = new ObjectCache(); + private TimeSource timeSource = TimeSource.NANO_TIME; + + public DelegatingCloudManager(SolrCloudManager delegate) { + this.delegate = delegate; + } + + @Override + public ClusterStateProvider getClusterStateProvider() { + return delegate.getClusterStateProvider(); + } + + @Override + public NodeStateProvider getNodeStateProvider() { + return delegate.getNodeStateProvider(); + } + + @Override + public DistribStateManager getDistribStateManager() { + return delegate.getDistribStateManager(); + } + + @Override + public DistributedQueueFactory getDistributedQueueFactory() { + return delegate.getDistributedQueueFactory(); + } + + @Override + public ObjectCache getObjectCache() { + return delegate == null ? objectCache : delegate.getObjectCache(); + } + + @Override + public boolean isClosed() { + return delegate.isClosed(); + } + + @Override + public TimeSource getTimeSource() { + return delegate == null ? timeSource : delegate.getTimeSource(); + } + + @Override + public SolrResponse request(@SuppressWarnings({"rawtypes"})SolrRequest req) throws IOException { + return delegate.request(req); + } + + @Override + public byte[] httpRequest(String url, SolrRequest.METHOD method, Map headers, String payload, int timeout, boolean followRedirects) throws IOException { + return delegate.httpRequest(url, method, headers, payload, timeout, followRedirects); + } + + @Override + public void close() throws IOException { + delegate.close(); + } +} diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/DelegatingClusterStateProvider.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/DelegatingClusterStateProvider.java new file mode 100644 index 00000000000..656c068be15 --- /dev/null +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/DelegatingClusterStateProvider.java @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.client.solrj.cloud; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.solr.client.solrj.impl.ClusterStateProvider; +import org.apache.solr.common.cloud.ClusterState; +import org.apache.solr.common.cloud.DocCollection; + +/** + * Base class for overriding some behavior of {@link ClusterStateProvider} + */ +public class DelegatingClusterStateProvider implements ClusterStateProvider { + protected ClusterStateProvider delegate; + + public DelegatingClusterStateProvider(ClusterStateProvider delegate) { + this.delegate = delegate; + } + + @Override + public ClusterState.CollectionRef getState(String collection) { + if (delegate != null) { + return delegate.getState(collection); + } else { + return null; + } + } + + @Override + public Set getLiveNodes() { + if (delegate != null) { + return delegate.getLiveNodes(); + } else { + return Collections.emptySet(); + } + } + + @Override + public List resolveAlias(String alias) { + if (delegate != null) { + return delegate.resolveAlias(alias); + } else { + return Collections.singletonList(alias); + } + } + + @Override + public Map getAliasProperties(String alias) { + if (delegate != null) { + return delegate.getAliasProperties(alias); + } else { + return Collections.emptyMap(); + } + } + + @Override + public String resolveSimpleAlias(String alias) throws IllegalArgumentException { + if (delegate != null) { + return delegate.resolveSimpleAlias(alias); + } else { + return alias; + } + } + + @Override + public ClusterState getClusterState() throws IOException { + if (delegate != null) { + return delegate.getClusterState(); + } else { + return null; + } + } + + @Override + public Map getClusterProperties() { + if (delegate != null) { + return delegate.getClusterProperties(); + } else { + return Collections.emptyMap(); + } + } + + @Override + public String getPolicyNameByCollection(String coll) { + if (delegate != null) { + return delegate.getPolicyNameByCollection(coll); + } else { + return null; + } + } + + @Override + public DocCollection getCollection(String name) throws IOException { + ClusterState cs = getClusterState(); + return cs == null ? null : cs.getCollectionOrNull(name); + } + + @Override + public void connect() { + if (delegate != null) { + delegate.connect(); + } + } + + @Override + public void close() throws IOException { + if (delegate != null) { + delegate.close(); + } + } +} diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/DistribStateManager.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/DistribStateManager.java index f811c5a56eb..cac6cd2412a 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/DistribStateManager.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/DistribStateManager.java @@ -24,11 +24,6 @@ import java.util.LinkedList; import java.util.List; import java.util.NoSuchElementException; -import org.apache.solr.client.solrj.cloud.autoscaling.AlreadyExistsException; -import org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig; -import org.apache.solr.client.solrj.cloud.autoscaling.BadVersionException; -import org.apache.solr.client.solrj.cloud.autoscaling.NotEmptyException; -import org.apache.solr.client.solrj.cloud.autoscaling.VersionedData; import org.apache.solr.common.SolrCloseable; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; @@ -77,12 +72,6 @@ public interface DistribStateManager extends SolrCloseable { List multi(final Iterable ops) throws BadVersionException, NoSuchElementException, AlreadyExistsException, IOException, KeeperException, InterruptedException; - AutoScalingConfig getAutoScalingConfig(Watcher watcher) throws InterruptedException, IOException; - - default AutoScalingConfig getAutoScalingConfig() throws InterruptedException, IOException { - return getAutoScalingConfig(null); - } - /** * List a subtree including the root path, using breadth-first traversal. * @param root root path diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/NotEmptyException.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/NotEmptyException.java similarity index 94% rename from solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/NotEmptyException.java rename to solr/solrj/src/java/org/apache/solr/client/solrj/cloud/NotEmptyException.java index 1480cd911e1..f75fe0269c0 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/NotEmptyException.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/NotEmptyException.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.solr.client.solrj.cloud.autoscaling; +package org.apache.solr.client.solrj.cloud; /** * diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/VersionedData.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/VersionedData.java similarity index 97% rename from solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/VersionedData.java rename to solr/solrj/src/java/org/apache/solr/client/solrj/cloud/VersionedData.java index 010beb70805..711411d4a24 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/VersionedData.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/VersionedData.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.solr.client.solrj.cloud.autoscaling; +package org.apache.solr.client.solrj.cloud; import java.io.IOException; import java.util.Arrays; diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/AddReplicaSuggester.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/AddReplicaSuggester.java deleted file mode 100644 index 58ddb5cac1f..00000000000 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/AddReplicaSuggester.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.client.solrj.cloud.autoscaling; - -import java.util.Collections; -import java.util.List; -import java.util.Set; - -import org.apache.solr.client.solrj.SolrRequest; -import org.apache.solr.client.solrj.request.CollectionAdminRequest; -import org.apache.solr.common.cloud.Replica; -import org.apache.solr.common.params.CollectionParams; -import org.apache.solr.common.util.Pair; - -import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDREPLICA; - -class AddReplicaSuggester extends Suggester { - - @SuppressWarnings({"rawtypes"}) - SolrRequest init() { - SolrRequest operation = tryEachNode(true); - if (operation == null) operation = tryEachNode(false); - return operation; - } - - @SuppressWarnings({"rawtypes"}) - SolrRequest tryEachNode(boolean strict) { - @SuppressWarnings({"unchecked"}) - Set> shards = (Set>) hints.getOrDefault(Hint.COLL_SHARD, Collections.emptySet()); - if (shards.isEmpty()) { - throw new RuntimeException("add-replica requires 'collection' and 'shard'"); - } - for (Pair shard : shards) { - Replica.Type type = Replica.Type.get((String) hints.get(Hint.REPLICATYPE)); - //iterate through nodes and identify the least loaded - List leastSeriousViolation = null; - Row bestNode = null; - for (int i = getMatrix().size() - 1; i >= 0; i--) { - Row row = getMatrix().get(i); - if (!isNodeSuitableForReplicaAddition(row, null)) continue; - Row tmpRow = row.addReplica(shard.first(), shard.second(), type, strict); - List errs = testChangedMatrix(strict, tmpRow.session); - if (!containsNewErrors(errs)) { - if ((errs.isEmpty() && isLessDeviant()) ||//there are no violations but this is deviating less - isLessSerious(errs, leastSeriousViolation)) {//there are errors , but this has less serious violation - leastSeriousViolation = errs; - bestNode = tmpRow; - } - } - } - - if (bestNode != null) {// there are no rule violations - this.session = bestNode.session; - return CollectionAdminRequest - .addReplicaToShard(shard.first(), shard.second()) - .setType(type) - .setNode(bestNode.node); - } - } - - return null; - } - - - @Override - public CollectionParams.CollectionAction getAction() { - return ADDREPLICA; - } -} \ No newline at end of file diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/AutoScalingConfig.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/AutoScalingConfig.java deleted file mode 100644 index 41ed24d1b05..00000000000 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/AutoScalingConfig.java +++ /dev/null @@ -1,616 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.client.solrj.cloud.autoscaling; - -import java.io.IOException; -import java.lang.invoke.MethodHandles; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.EnumSet; -import java.util.LinkedHashMap; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Set; -import java.util.Objects; - -import org.apache.solr.common.MapWriter; -import org.apache.solr.common.params.AutoScalingParams; -import org.apache.solr.common.util.Utils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import static java.util.stream.Collectors.collectingAndThen; -import static java.util.stream.Collectors.toList; - -/** - * Bean representation of autoscaling.json, which parses data - * lazily. - */ -public class AutoScalingConfig implements MapWriter { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - private final Map jsonMap; - private final boolean empty; - - private Policy policy; - private Map triggers; - private Map listeners; - private Map properties; - - private final int zkVersion; - - /** - * Bean representation of trigger listener config. - */ - public static class TriggerListenerConfig implements MapWriter { - public final String name; - public final String trigger; - public final EnumSet stages = EnumSet.noneOf(TriggerEventProcessorStage.class); - public final String listenerClass; - public final Set beforeActions; - public final Set afterActions; - public final Map properties; - - public TriggerListenerConfig(String name, Map properties) { - this.name = name; - if (properties == null) { - this.properties = Collections.emptyMap(); - } else { - this.properties = Collections.unmodifiableMap(new LinkedHashMap<>(properties)); - } - trigger = (String)this.properties.get(AutoScalingParams.TRIGGER); - List stageNames = getList(AutoScalingParams.STAGE, this.properties); - for (Object stageName : stageNames) { - try { - TriggerEventProcessorStage stage = TriggerEventProcessorStage.valueOf(String.valueOf(stageName).toUpperCase(Locale.ROOT)); - stages.add(stage); - } catch (Exception e) { - log.warn("Invalid stage name '{}' for '{}' in listener config, skipping it in: {}", - stageName, name, properties); - } - } - listenerClass = (String)this.properties.get(AutoScalingParams.CLASS); - Set bActions = new LinkedHashSet<>(); - getList(AutoScalingParams.BEFORE_ACTION, this.properties).forEach(o -> bActions.add(String.valueOf(o))); - beforeActions = Collections.unmodifiableSet(bActions); - Set aActions = new LinkedHashSet<>(); - getList(AutoScalingParams.AFTER_ACTION, this.properties).forEach(o -> aActions.add(String.valueOf(o))); - afterActions = Collections.unmodifiableSet(aActions); - } - - @Override - public void writeMap(EntryWriter ew) throws IOException { - // don't write duplicate entries - skip explicit fields if their values - // are already contained in properties -// if (!properties.containsKey(AutoScalingParams.NAME)) { -// ew.put(AutoScalingParams.NAME, name); -// } - if (!properties.containsKey(AutoScalingParams.CLASS)) { - ew.put(AutoScalingParams.CLASS, listenerClass); - } - if (!properties.containsKey(AutoScalingParams.TRIGGER)) { - ew.put(AutoScalingParams.TRIGGER, trigger); - } - if (!properties.containsKey(AutoScalingParams.STAGE)) { - ew.put(AutoScalingParams.STAGE, stages); - } - if (!properties.containsKey(AutoScalingParams.BEFORE_ACTION)) { - ew.put(AutoScalingParams.BEFORE_ACTION, beforeActions); - } - if (!properties.containsKey(AutoScalingParams.AFTER_ACTION)) { - ew.put(AutoScalingParams.AFTER_ACTION, afterActions); - } - // forEach doesn't allow throwing exceptions... - for (Map.Entry entry : properties.entrySet()) { - ew.put(entry.getKey(), entry.getValue()); - } - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - - TriggerListenerConfig that = (TriggerListenerConfig) o; - - if (name != null ? !name.equals(that.name) : that.name != null) return false; - if (trigger != null ? !trigger.equals(that.trigger) : that.trigger != null) return false; - if (!stages.equals(that.stages)) return false; - if (listenerClass != null ? !listenerClass.equals(that.listenerClass) : that.listenerClass != null) return false; - if (!beforeActions.equals(that.beforeActions)) return false; - if (!afterActions.equals(that.afterActions)) return false; - return properties.equals(that.properties); - } - - @Override - public int hashCode() { - return Objects.hash(name, trigger, listenerClass); - } - - @Override - public String toString() { - return Utils.toJSONString(this); - } - } - - /** - * Bean representation of trigger config. - */ - public static class TriggerConfig implements MapWriter { - /** Trigger name. */ - public final String name; - /** Trigger event type. */ - public final TriggerEventType event; - /** Enabled flag. */ - public final boolean enabled; - /** List of configured actions, never null. */ - public final List actions; - /** Map of additional trigger properties, never null. */ - public final Map properties; - - public TriggerConfig(String name, Map properties) { - this.name = name; - if (properties != null) { - this.properties = Collections.unmodifiableMap(new LinkedHashMap<>(properties)); - } else { - this.properties = Collections.emptyMap(); - } - String event = (String) this.properties.get(AutoScalingParams.EVENT); - if (event != null) { - TriggerEventType type = null; - try { - type = TriggerEventType.valueOf(event.toUpperCase(Locale.ROOT)); - } catch (Exception e) { - } - if (type == null) { - this.event = TriggerEventType.INVALID; - } else { - this.event = type; - } - } else { - this.event = TriggerEventType.INVALID; - } - enabled = Boolean.parseBoolean(String.valueOf(this.properties.getOrDefault("enabled", "true"))); - - @SuppressWarnings({"unchecked"}) - List> newActions = (List>)this.properties.get("actions"); - if (newActions != null) { - this.actions = newActions.stream().map(ActionConfig::new).collect(collectingAndThen(toList(), Collections::unmodifiableList)); - } else { - this.actions = Collections.emptyList(); - } - } - - /** - * Create a copy of this config with specified enabled flag. - * @param enabled true when enabled, false otherwise. - * @return modified copy of the configuration - */ - public TriggerConfig withEnabled(boolean enabled) { - Map props = new LinkedHashMap<>(properties); - props.put(AutoScalingParams.ENABLED, String.valueOf(enabled)); - return new TriggerConfig(name, props); - } - - /** - * Create a copy of this config with specified property. - * @param key property name - * @param value property value - * @return modified copy of the configuration - */ - public TriggerConfig withProperty(String key, Object value) { - Map props = new LinkedHashMap<>(properties); - props.put(key, String.valueOf(value)); - return new TriggerConfig(name, props); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - - TriggerConfig that = (TriggerConfig) o; - - if (name != null ? !name.equals(that.name) : that.name != null) return false; - if (event != that.event) return false; - return properties.equals(that.properties); - } - - @Override - public int hashCode() { - return Objects.hash(name); - } - - @Override - public void writeMap(EntryWriter ew) throws IOException { -// if (!properties.containsKey(AutoScalingParams.NAME)) { -// ew.put(AutoScalingParams.NAME, name); -// } - if (!properties.containsKey(AutoScalingParams.EVENT)) { - ew.put(AutoScalingParams.EVENT, event.toString()); - } - // forEach doesn't allow throwing exceptions... - for (Map.Entry entry : properties.entrySet()) { - ew.put(entry.getKey(), entry.getValue()); - } - } - - @Override - public String toString() { - return Utils.toJSONString(this); - } - } - - /** - * Bean representation of trigger action configuration. - */ - public static class ActionConfig implements MapWriter { - /** Action name. */ - public final String name; - /** Class name of action implementation. */ - public final String actionClass; - /** Additional action properties. */ - public final Map properties; - - /** - * Construct from a JSON map. - * @param properties JSON map with properties - selected properties will be - * used for setting the values of name and - * actionClass. - */ - public ActionConfig(Map properties) { - if (properties != null) { - this.properties = Collections.unmodifiableMap(new LinkedHashMap<>(properties)); - } else { - this.properties = Collections.emptyMap(); - } - this.name = (String)this.properties.get(AutoScalingParams.NAME); - this.actionClass = (String)this.properties.get(AutoScalingParams.CLASS); - } - - @Override - public void writeMap(EntryWriter ew) throws IOException { - // forEach doesn't allow throwing exceptions... - for (Map.Entry entry : properties.entrySet()) { - ew.put(entry.getKey(), entry.getValue()); - } - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - - ActionConfig that = (ActionConfig) o; - - return properties.equals(that.properties); - } - - @Override - public int hashCode() { - return Objects.hash(properties); - } - - @Override - public String toString() { - return Utils.toJSONString(this); - } - } - - /** - * Construct from bytes that represent a UTF-8 JSON string. - * @param utf8 config data - */ - @SuppressWarnings({"unchecked"}) - public AutoScalingConfig(byte[] utf8) { - this(utf8 != null && utf8.length > 0 ? (Map)Utils.fromJSON(utf8) : Collections.emptyMap()); - } - - /** - * Construct from a JSON map representation. - * @param jsonMap JSON map representation of the config. Note that this map is evaluated lazily, and - * outside modifications may cause unpredictable behavior. - */ - public AutoScalingConfig(Map jsonMap) { - this.jsonMap = jsonMap; - int version = 0; - if (jsonMap.containsKey(AutoScalingParams.ZK_VERSION)) { - try { - version = (Integer)jsonMap.get(AutoScalingParams.ZK_VERSION); - } catch (Exception e) { - // ignore - } - } - zkVersion = version; - jsonMap.remove(AutoScalingParams.ZK_VERSION); - empty = jsonMap.isEmpty(); - } - - public AutoScalingConfig(Policy policy, Map triggerConfigs, Map listenerConfigs, Map properties, int zkVersion) { - this.policy = policy; - this.triggers = triggerConfigs != null ? Collections.unmodifiableMap(new LinkedHashMap<>(triggerConfigs)) : null; - this.listeners = listenerConfigs != null ? Collections.unmodifiableMap(new LinkedHashMap<>(listenerConfigs)) : null; - this.jsonMap = null; - this.properties = properties != null ? Collections.unmodifiableMap(new LinkedHashMap<>(properties)) : null; - this.zkVersion = zkVersion; - this.empty = policy == null && - (triggerConfigs == null || triggerConfigs.isEmpty()) && - (listenerConfigs == null || listenerConfigs.isEmpty()); - } - - /** - * Return true if the source autoscaling.json was empty, false otherwise. - */ - public boolean isEmpty() { - return empty; - } - - /** - * Get {@link Policy} configuration. - */ - public Policy getPolicy() { - if (policy == null) { - if (jsonMap != null) { - policy = new Policy(jsonMap, zkVersion); - } else { - policy = new Policy(); - } - } - return policy; - } - - /** - * Get trigger configurations. - */ - @SuppressWarnings({"unchecked"}) - public Map getTriggerConfigs() { - if (triggers == null) { - if (jsonMap != null) { - Map trigMap = (Map)jsonMap.get("triggers"); - if (trigMap == null) { - triggers = Collections.emptyMap(); - } else { - Map newTriggers = new LinkedHashMap<>(trigMap.size()); - for (Map.Entry entry : trigMap.entrySet()) { - newTriggers.put(entry.getKey(), new TriggerConfig(entry.getKey(), (Map)entry.getValue())); - } - triggers = Collections.unmodifiableMap(newTriggers); - } - } else { - triggers = Collections.emptyMap(); - } - } - return triggers; - } - - /** - * Check whether triggers for specific event type exist. - * @param types list of event types - * @return true if there's at least one trigger matching at least one event type, - * false otherwise, - */ - public boolean hasTriggerForEvents(TriggerEventType... types) { - if (types == null || types.length == 0) { - return false; - } - for (TriggerConfig config : getTriggerConfigs().values()) { - for (TriggerEventType type : types) { - if (config.event.equals(type)) { - return true; - } - } - } - return false; - } - - /** - * Get listener configurations. - */ - @SuppressWarnings({"unchecked"}) - public Map getTriggerListenerConfigs() { - if (listeners == null) { - if (jsonMap != null) { - Map map = (Map)jsonMap.get("listeners"); - if (map == null) { - listeners = Collections.emptyMap(); - } else { - Map newListeners = new LinkedHashMap<>(map.size()); - for (Map.Entry entry : map.entrySet()) { - newListeners.put(entry.getKey(), new TriggerListenerConfig(entry.getKey(), (Map)entry.getValue())); - } - this.listeners = Collections.unmodifiableMap(newListeners); - } - } else { - listeners = Collections.emptyMap(); - } - } - return listeners; - } - - public Map getProperties() { - if (properties == null) { - if (jsonMap != null) { - @SuppressWarnings({"unchecked"}) - Map map = (Map) jsonMap.get("properties"); - if (map == null) { - this.properties = Collections.emptyMap(); - } else { - this.properties = new LinkedHashMap<>(map); - } - } else { - this.properties = Collections.emptyMap(); - } - } - return properties; - } - - /** - * Create a copy of the config with replaced properties. - * @param properties the new properties map - * @return modified copy of the configuration - */ - public AutoScalingConfig withProperties(Map properties) { - return new AutoScalingConfig(policy, getTriggerConfigs(), getTriggerListenerConfigs(), properties, zkVersion); - } - - /** - * Create a copy of the config with replaced policy. - * @param policy new policy - * @return modified copy of the configuration - */ - public AutoScalingConfig withPolicy(Policy policy) { - return new AutoScalingConfig(policy, getTriggerConfigs(), getTriggerListenerConfigs(), getProperties(), zkVersion); - } - - /** - * Create a copy of the config with replaced trigger configurations. - * @param configs new trigger configurations - * @return modified copy of the configuration - */ - public AutoScalingConfig withTriggerConfigs(Map configs) { - return new AutoScalingConfig(getPolicy(), configs, getTriggerListenerConfigs(), getProperties(), zkVersion); - } - - /** - * Create a copy of the config with replaced trigger configuration - * @param config new trigger configuration - * @return modified copy of the configuration - */ - public AutoScalingConfig withTriggerConfig(TriggerConfig config) { - Map configs = new LinkedHashMap<>(getTriggerConfigs()); - configs.put(config.name, config); - return withTriggerConfigs(configs); - } - - /** - * Create a copy of the config without a trigger configuration. - * @param name trigger configuration name - * @return modified copy of the configuration, even if the specified config name didn't exist. - */ - public AutoScalingConfig withoutTriggerConfig(String name) { - Map configs = new LinkedHashMap<>(getTriggerConfigs()); - configs.remove(name); - return withTriggerConfigs(configs); - } - - /** - * Create a copy of the config with replaced trigger listener configurations. - * @param configs new trigger listener configurations - * @return modified copy of the configuration - */ - public AutoScalingConfig withTriggerListenerConfigs(Map configs) { - return new AutoScalingConfig(getPolicy(), getTriggerConfigs(), configs, getProperties(), zkVersion); - } - - /** - * Create a copy of the config with replaced trigger listener configuration. - * @param config new trigger listener configuration - * @return modified copy of the configuration - */ - public AutoScalingConfig withTriggerListenerConfig(TriggerListenerConfig config) { - Map configs = new LinkedHashMap<>(getTriggerListenerConfigs()); - configs.put(config.name, config); - return withTriggerListenerConfigs(configs); - } - - /** - * Create a copy of the config without a trigger listener configuration. - * @param name trigger listener configuration name - * @return modified copy of the configuration, even if the specified config name didn't exist. - */ - public AutoScalingConfig withoutTriggerListenerConfig(String name) { - Map configs = new LinkedHashMap<>(getTriggerListenerConfigs()); - configs.remove(name); - return withTriggerListenerConfigs(configs); - } - - @Override - public Object clone() { - if (jsonMap != null) { - return new AutoScalingConfig(jsonMap); - } else { - return new AutoScalingConfig(getPolicy(), getTriggerConfigs(), getTriggerListenerConfigs(), getProperties(), zkVersion); - } - } - - /** - * Return the znode version that was used to create this configuration. - */ - public int getZkVersion() { - return zkVersion; - } - - @Override - public void writeMap(EntryWriter ew) throws IOException { - Policy policy = getPolicy(); - // properties of Policy are expected at top level - policy.writeMap(ew); - - ew.put("triggers", getTriggerConfigs()); - ew.put("listeners", getTriggerListenerConfigs()); - ew.put("properties", getProperties()); - } - - public String toString() { - return Utils.toJSONString(this); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - - AutoScalingConfig that = (AutoScalingConfig) o; - - if (!getPolicy().equals(that.getPolicy())) return false; - if (!getTriggerConfigs().equals(that.getTriggerConfigs())) return false; - if (!getTriggerListenerConfigs().equals(that.getTriggerListenerConfigs())) return false; - return getProperties().equals(that.getProperties()); - } - - @Override - public int hashCode() { - return Objects.hash(getPolicy()); - } - - private static List getList(String key, Map properties) { - return getList(key, properties, null); - } - - @SuppressWarnings({"unchecked", "rawtypes"}) - private static List getList(String key, Map properties, List defaultList) { - if (defaultList == null) { - defaultList = Collections.emptyList(); - } - Object o = properties.get(key); - if (o == null) { - return defaultList; - } - if (o instanceof List) { - return (List)o; - } else if (o instanceof Collection) { - return new ArrayList<>((Collection) o); - } else { - return Collections.singletonList(String.valueOf(o)); - } - } - -} diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Cell.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Cell.java deleted file mode 100644 index e2225415dd6..00000000000 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Cell.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.client.solrj.cloud.autoscaling; - -import java.io.IOException; -import java.util.HashMap; - -import org.apache.solr.client.solrj.cloud.autoscaling.Variable.Type; -import org.apache.solr.common.MapWriter; -import org.apache.solr.common.util.Utils; - -/**Each instance represents an attribute that is being tracked by the framework such as , freedisk, cores etc - * - */ -public class Cell implements MapWriter { - final int index; - final Type type; - final String name; - Object val, approxVal; - Row row; - - public Cell(int index, String name, Object val, Object approxVal, Type type, Row row) { - this.index = index; - this.name = name; - this.val = val; - this.approxVal = approxVal; - this.type = type; - this.row = row; - } - - @Override - public void writeMap(EntryWriter ew) throws IOException { - ew.put(name, val); - } - public Row getRow(){ - return row; - } - - @Override - public String toString() { - return Utils.toJSONString(this.toMap(new HashMap<>())); - } - - public Cell copy() { - return new Cell(index, name, val, approxVal, this.type, row); - } - - public String getName() { - return name; - } - - public Object getValue() { - return val; - } - - public Object getApproxValue() { - return approxVal; - } -} diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Clause.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Clause.java deleted file mode 100644 index 06f49e6801c..00000000000 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Clause.java +++ /dev/null @@ -1,839 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.client.solrj.cloud.autoscaling; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.Set; -import java.util.function.Function; -import java.util.stream.Collectors; - -import org.apache.solr.client.solrj.cloud.autoscaling.Variable.Type; -import org.apache.solr.common.MapWriter; -import org.apache.solr.common.cloud.Replica; -import org.apache.solr.common.util.StrUtils; -import org.apache.solr.common.util.Utils; - -import static java.util.Collections.singletonMap; -import static org.apache.solr.client.solrj.cloud.autoscaling.Operand.EQUAL; -import static org.apache.solr.client.solrj.cloud.autoscaling.Operand.GREATER_THAN; -import static org.apache.solr.client.solrj.cloud.autoscaling.Operand.LESS_THAN; -import static org.apache.solr.client.solrj.cloud.autoscaling.Operand.NOT_EQUAL; -import static org.apache.solr.client.solrj.cloud.autoscaling.Operand.RANGE_EQUAL; -import static org.apache.solr.client.solrj.cloud.autoscaling.Operand.WILDCARD; -import static org.apache.solr.client.solrj.cloud.autoscaling.Policy.ANY; -import static org.apache.solr.common.params.CoreAdminParams.COLLECTION; -import static org.apache.solr.common.params.CoreAdminParams.NODE; -import static org.apache.solr.common.params.CoreAdminParams.REPLICA; -import static org.apache.solr.common.params.CoreAdminParams.SHARD; -import static org.apache.solr.common.util.StrUtils.formatString; -import static org.apache.solr.common.util.Utils.toJSONString; - -/** - * Represents a set of conditions in the policy - */ -public class Clause implements MapWriter, Comparable { - public static final String NODESET = "nodeset"; - static final Set IGNORE_TAGS = new HashSet<>(Arrays.asList(REPLICA, COLLECTION, SHARD, "strict", "type", "put", NODESET)); - - private final int hashCode; - final boolean hasComputedValue; - final Map original; - final Clause derivedFrom; - boolean nodeSetPresent = false; - Condition collection, shard, replica, tag, globalTag; - final Replica.Type type; - Put put; - boolean strict; - - protected Clause(Clause clause, Function computedValueEvaluator) { - this.original = clause.original; - this.hashCode = original.hashCode(); - this.type = clause.type; - this.put = clause.put; - this.nodeSetPresent = clause.nodeSetPresent; - this.collection = clause.collection; - this.shard = clause.shard; - this.tag = evaluateValue(clause.tag, computedValueEvaluator); - this.replica = evaluateValue(clause.replica, computedValueEvaluator); - this.globalTag = evaluateValue(clause.globalTag, computedValueEvaluator); - this.hasComputedValue = clause.hasComputedValue; - this.strict = clause.strict; - derivedFrom = clause.derivedFrom; - } - - // internal use only - Clause(Map original, Condition tag, Condition globalTag, boolean isStrict, Put put, boolean nodeSetPresent) { - this.hashCode = original.hashCode(); - this.original = original; - this.tag = tag; - this.globalTag = globalTag; - this.globalTag.clause = this; - this.type = null; - this.hasComputedValue = false; - this.strict = isStrict; - derivedFrom = null; - this.put = put; - this.nodeSetPresent = nodeSetPresent; - } - - @SuppressWarnings({"unchecked"}) - private Clause(Map m) { - derivedFrom = (Clause) m.remove(Clause.class.getName()); - this.original = Utils.getDeepCopy(m, 10); - this.hashCode = original.hashCode(); - String type = (String) m.get("type"); - this.type = type == null || ANY.equals(type) ? null : Replica.Type.valueOf(type.toUpperCase(Locale.ROOT)); - String put = (String) m.getOrDefault("put", m.containsKey(NODESET)? Put.ON_ALL.val: null ); - if (put != null) { - this.put = Put.get(put); - if (this.put == null) throwExp(m, "invalid value for put : {0}", put); - } - - strict = Boolean.parseBoolean(String.valueOf(m.getOrDefault("strict", "true"))); - Optional globalTagName = m.keySet().stream().filter(Policy.GLOBAL_ONLY_TAGS::contains).findFirst(); - if (globalTagName.isPresent()) { - globalTag = parse(globalTagName.get(), m); - if (m.size() > 3) { - throw new RuntimeException("Only one extra tag supported for the tag " + globalTagName.get() + " in " + toJSONString(m)); - } - tag = parse(m.keySet().stream() - .filter(s -> (!globalTagName.get().equals(s) && !IGNORE_TAGS.contains(s))) - .findFirst().get(), m); - } else { - collection = parse(COLLECTION, m); - shard = parse(SHARD, m); - if (m.get(REPLICA) == null) { - throw new IllegalArgumentException(formatString("'replica' is required in {0}", toJSONString(m))); - } - this.replica = parse(REPLICA, m); - if (replica.op == WILDCARD) throw new IllegalArgumentException("replica val cannot be null" + toJSONString(m)); - - this.nodeSetPresent = parseNodeset(m); - m.forEach((s, o) -> parseCondition(s, o, m)); - } - if (tag == null) - throw new RuntimeException("Invalid op, must have one and only one tag other than collection, shard,replica " + toJSONString(m)); - if (tag.name.startsWith(Clause.METRICS_PREFIX)) { - List ss = StrUtils.splitSmart(tag.name, ':'); - if (ss.size() < 3 || ss.size() > 4) { - throw new RuntimeException("Invalid metrics: param in " + toJSONString(m) + " must have at 2 or 3 segments after 'metrics:' separated by ':'"); - } - } - doPostValidate(collection, shard, replica, tag, globalTag); - hasComputedValue = hasComputedValue(); - } - - private boolean parseNodeset(Map m) { - if (!m.containsKey(NODESET)) return false; - Object o = m.get(NODESET); - if (o instanceof Map) { - String key = validateObjectInNodeset(m, (Map) o); - parseCondition(key, o, m); - } else if (o instanceof List) { - @SuppressWarnings({"rawtypes"}) - List l = (List) o; - if(l.size()<2) throwExp(m, "nodeset [] must have atleast 2 items"); - if( checkMapArray(l, m)) return true; - for (Object it : l) { - if (it instanceof String) continue; - else throwExp(m, "nodeset :[]must have only string values"); - } - parseCondition("node", o, m); - } else { - throwExp(m, "invalid value for nodeset, must be an object or a list of String"); - } - return true; - } - - private String validateObjectInNodeset(@SuppressWarnings({"rawtypes"})Map m, - @SuppressWarnings({"rawtypes"})Map map) { - if (map.size() != 1) { - throwExp(m, "nodeset must only have one and only one key"); - } - String key = (String) map.keySet().iterator().next(); - Object val = map.get(key); - if(val instanceof String && ((String )val).trim().charAt(0) == '#'){ - throwExp(m, formatString("computed value {0} not allowed in nodeset", val)); - } - return key; - } - - private boolean checkMapArray(@SuppressWarnings({"rawtypes"})List l, Map m) { - @SuppressWarnings({"rawtypes"}) - List maps = null; - for (Object o : l) { - if (o instanceof Map) { - if (maps == null) maps = new ArrayList<>(); - maps.add((Map) o); - } - } - String key = null; - if (maps != null) { - if (maps.size() != l.size()) throwExp(m, "all elements of nodeset must be Objects"); - List tags = new ArrayList<>(maps.size()); - for (@SuppressWarnings({"rawtypes"})Map map : maps) { - String s = validateObjectInNodeset(m, map); - if(key == null) key = s; - if(!Objects.equals(key, s)){ - throwExp(m, "all element must have same key"); - } - tags.add(parse(s, m)); - } - if(this.put == Put.ON_EACH) throwExp(m, "cannot use put: ''on-each-node'' with an array value in nodeset "); - this.tag = new Condition(key, tags,Operand.IN, null,this); - return true; - } - return false; - - } - - public Condition getThirdTag() { - return globalTag == null ? tag : globalTag; - } - - private void doPostValidate(Condition... conditions) { - for (Condition condition : conditions) { - if (condition == null) continue; - String err = condition.varType.postValidate(condition); - if (err != null) { - throw new IllegalArgumentException(formatString("Error in clause : {0}, caused by : {1}", toJSONString(original), err)); - } - } - } - - @SuppressWarnings({"unchecked"}) - public static Clause create(String json) { - return create((Map) Utils.fromJSONString(json)); - } - - public static Clause create(Map m) { - Clause clause = new Clause(m); - return clause.hasComputedValue() ? - clause : - clause.getSealedClause(null); - } - - public static String parseString(Object val) { - if (val instanceof Condition) val = ((Condition) val).val; - return val == null ? null : String.valueOf(val); - } - - public Condition getCollection() { - return collection; - } - - public Condition getShard() { - return shard; - } - - public Condition getReplica() { - return replica; - } - - public Condition getTag() { - return tag; - } - - public Condition getGlobalTag() { - return globalTag; - } - - Condition evaluateValue(Condition condition, Function computedValueEvaluator) { - if (condition == null) return null; - if (condition.computedType == null) return condition; - Object val = computedValueEvaluator.apply(condition); - val = condition.op.readRuleValue(new Condition(condition.name, val, condition.op, null, null)); - return new Condition(condition.name, val, condition.op, null, this); - } - - public boolean doesOverride(Clause that) { - return (collection.equals(that.collection) && - tag.name.equals(that.tag.name)); - - } - - public boolean isPerCollectiontag() { - return globalTag == null; - } - - @SuppressWarnings({"unchecked"}) - void parseCondition(String s, Object o, @SuppressWarnings({"rawtypes"})Map m) { - if (IGNORE_TAGS.contains(s)) return; - if (tag != null) { - throwExp(m, "Only one tag other than collection, shard, replica is possible"); - } - tag = parse(s, o instanceof Map? (Map) o : singletonMap(s, o)); - } - - private int compareTypes(Replica.Type t1, Replica.Type t2) { - if (t1 == null && t2 == null) return 0; - if (t1 != null && t2 == null) return -1; - if (t1 == null) return 1; - return 0; - } - - private boolean hasComputedValue() { - if (replica != null && replica.computedType != null) return true; - if (tag != null && tag.computedType != null) return true; - if (globalTag != null && globalTag.computedType != null) return true; - return false; - - } - - @Override - public int compareTo(Clause that) { - int v = Integer.compare(this.tag.op.priority, that.tag.op.priority); - if (v != 0) return v; - if (this.isPerCollectiontag() && that.isPerCollectiontag()) { - v = Integer.compare(this.replica.op.priority, that.replica.op.priority); - if (v == 0) {// higher the number of replicas , harder to satisfy - Double thisVal = this.replica.val instanceof RangeVal ? ((RangeVal) this.replica.val).max.doubleValue() : (Double) this.replica.val; - Double thatVal = that.replica.val instanceof RangeVal ? ((RangeVal) that.replica.val).max.doubleValue() : (Double) that.replica.val; - v = Preference.compareWithTolerance(thisVal, thatVal, 1); - v = this.replica.op == LESS_THAN ? v : v * -1; - } - if (v == 0) v = compareTypes(this.type, that.type); - return v; - } else { - return 0; - } - } - - void addTags(Collection params) { - if (globalTag != null && !params.contains(globalTag.name)) params.add(globalTag.name); - if (tag != null && !params.contains(tag.name)) params.add(tag.name); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (!(o instanceof Clause)) return false; - Clause that = (Clause) o; - return Objects.equals(this.original, that.original); - } - - //replica value is zero - boolean isReplicaZero() { - return replica != null && replica.getOperand() == Operand.EQUAL && replica.val instanceof Double && - Preference.compareWithTolerance(0d, (Double) replica.val, 1) == 0; - } - - public SealedClause getSealedClause(Function computedValueEvaluator) { - return this instanceof SealedClause ? - (SealedClause) this : - new SealedClause(this, computedValueEvaluator); - } - - Condition parse(String s, Map m) { - - Object expectedVal = null; - ComputedType computedType = null; - Object val = m.get(s); - Type varType = VariableBase.getTagType(s); - if (varType.meta.isHidden()) { - throwExp(m, "''{0}'' is not allowed", varType.tagName); - } - try { - String conditionName = s.trim(); - Operand operand = null; - if (val == null) { - operand = WILDCARD; - expectedVal = Policy.ANY; - } else if (val instanceof List) { - if (!varType.meta.supportArrayVals()) { - throwExp(m, "array values are not supported for {0}", conditionName); - } - expectedVal = readListVal(m, (List) val, varType, conditionName); - operand = Operand.IN; - } else if (val instanceof String) { - String strVal = ((String) val).trim(); - val = strVal; - operand = getOperand(strVal); - strVal = strVal.substring(Operand.EQUAL == operand || WILDCARD == operand ? 0 : 1); - for (ComputedType t : ComputedType.values()) { - String changedVal = t.match(strVal); - if (changedVal != null) { - computedType = t; - strVal = changedVal; - if (varType == null || !varType.supportedComputedTypes.contains(computedType)) { - throwExp(m, "''{0}'' is not allowed for variable : ''{1}''", t, conditionName); - } - } - } - if (computedType == null && ((String) val).charAt(0) == '#' && !varType.wildCards.contains(val)) { - throwExp(m, "''{0}'' is not an allowed value for ''{1}'', supported value is : {2} ", val, conditionName, varType.wildCards); - - } - operand = varType == null ? operand : varType.getOperand(operand, strVal, computedType); - expectedVal = validate(s, new Condition(s, strVal, operand, computedType, null), true); - - } else if (val instanceof Number) { - operand = Operand.EQUAL; - operand = varType.getOperand(operand, val, null); - expectedVal = validate(s, new Condition(s, val, operand, null, null), true); - } - return new Condition(conditionName, expectedVal, operand, computedType, this); - - } catch (IllegalArgumentException iae) { - throw iae; - } catch (Exception e) { - throwExp(m, " Invalid tag : {0} "+ e.getMessage(), s); - return null; - } - } - - public static void throwExp(@SuppressWarnings({"rawtypes"})Map clause, String msg, Object... args) { - throw new IllegalArgumentException("syntax error in clause :" + toJSONString(clause) + " , msg: " + formatString(msg, args)); - } - - @SuppressWarnings({"unchecked", "rawtypes"}) - private static List readListVal(Map m, List val, Type varType, String conditionName) { - List list = val; - list = (List) list.stream() - .map(it -> varType.validate(conditionName, it, true)) - .map(it -> { - if (it instanceof String) { - String trim = ((String) it).trim(); - if (trim.isEmpty()) - throw new IllegalArgumentException(formatString("{0} cannot have an empty string value in clause : {1}", - conditionName, toJSONString(m))); - return trim; - } else return it; - }).filter(it -> it == null ? false : true) - .collect(Collectors.toList()); - if (list.isEmpty()) - throw new IllegalArgumentException(formatString("{0} cannot have an empty list value in clause : {1}", - conditionName, toJSONString(m))); - for (Object o : list) { - if (o instanceof String) { - if (getOperand((String) o) != EQUAL) { - throw new IllegalArgumentException(formatString("No operators are supported in collection values in condition : {0} in clause : {1}", - conditionName, toJSONString(m))); - } - } - } - if (list.size() < 2) { - throw new IllegalArgumentException(formatString("Array should have more than one value in condition : {0} in clause : {1}", - conditionName, toJSONString(m))); - - } - return list; - } - - private static Operand getOperand(String strVal) { - Operand operand; - if (Policy.ANY.equals(strVal) || Policy.EACH.equals(strVal)) operand = WILDCARD; - else if (strVal.startsWith(NOT_EQUAL.operand)) operand = NOT_EQUAL; - else if (strVal.startsWith(GREATER_THAN.operand)) operand = GREATER_THAN; - else if (strVal.startsWith(LESS_THAN.operand)) operand = LESS_THAN; - else operand = Operand.EQUAL; - return operand; - } - - - private boolean isRowPass(ComputedValueEvaluator eval, Object t, Row row) { - eval.node = row.node; - if (t instanceof Condition) { - Condition tag = (Condition) t; - if (tag.computedType != null) tag = evaluateValue(tag, eval); - return tag.isPass(row); - } else { - return t.equals(row.getVal(tag.name)); - } - } - - List testGroupNodes(Policy.Session session, double[] deviations) { - //e.g: {replica:'#EQUAL', shard:'#EACH', sysprop.zone:'#EACH'} - ComputedValueEvaluator eval = new ComputedValueEvaluator(session); - eval.collName = (String) collection.getValue(); - Violation.Ctx ctx = new Violation.Ctx(this, session.matrix, eval); - - @SuppressWarnings({"rawtypes"}) - Set tags = getUniqueTags(session, eval); - if (tags.isEmpty()) return Collections.emptyList(); - - Set shards = getShardNames(session, eval); - - for (String s : shards) { - final ReplicaCount replicaCount = new ReplicaCount(); - eval.shardName = s; - - for (Object tag : tags) { - replicaCount.reset(); - for (Row row : session.matrix) { - if(!isRowPass(eval, tag, row)) continue; - addReplicaCountsForNode(eval, replicaCount, row); - } - - SealedClause sealedClause = this.getSealedClause(eval); - if (!sealedClause.replica.isPass(replicaCount)) { - ReplicaCount replicaCountCopy = replicaCount.copy(); - Violation violation = new Violation(sealedClause, - eval.collName, - eval.shardName, - null, - replicaCountCopy, - sealedClause.getReplica().replicaCountDelta(replicaCountCopy), - tag); - ctx.resetAndAddViolation(tag, replicaCountCopy, violation); - sealedClause.addViolatingReplicasForGroup(sealedClause.tag, eval, ctx, this.tag.name, tag, violation, session.matrix); - if (!this.strict && deviations != null) { - this.tag.varType.computeDeviation(session, deviations, replicaCount, sealedClause); - } - } else { - if (replica.op == RANGE_EQUAL) this.tag.varType.computeDeviation(session, deviations, replicaCount, sealedClause); - } - } - } - return ctx.allViolations; - } - - @SuppressWarnings({"unchecked", "rawtypes"}) - private Set getUniqueTags(Policy.Session session, ComputedValueEvaluator eval) { - Set tags = new HashSet(); - - if(nodeSetPresent) { - if (tag.val instanceof List && ((List) tag.val).get(0) instanceof Condition) { - tags.addAll((List) tag.val); - } else { - tags.add(tag); - } - return tags; - } - if(tag.op == WILDCARD){ - for (Row row : session.matrix) { - eval.node = row.node; - Condition tag = this.tag; - if (tag.computedType != null) tag = evaluateValue(tag, eval); - Object val = row.getVal(tag.name); - if (val != null) { - if (tag.op == LESS_THAN || tag.op == GREATER_THAN) { - tags.add(this.tag); - } else if (tag.isPass(val)) { - tags.add(val); - } - } - } - - } else { - - if (tag.op == LESS_THAN || tag.op == GREATER_THAN || tag.op == RANGE_EQUAL || tag.op == NOT_EQUAL) { - tags.add(tag); // eg: freedisk > 100 - } else if (tag.val instanceof Collection) { - tags.add(tag); //e: sysprop.zone:[east,west] - } else { - tags.add(tag.val);// - } - } - return tags; - } - - void addViolatingReplicasForGroup(Condition tag, - ComputedValueEvaluator eval, - Violation.Ctx ctx, String tagName, Object tagVal, - Violation violation, - List nodes) { - if (tag.varType.addViolatingReplicas(ctx)) return; - for (Row row : nodes) { - boolean isPass; - if (tagVal instanceof Condition) { - Condition condition = (Condition) tagVal; - if(condition.computedType != null){ - eval.node = row.node; - Object val = eval.apply(condition); - condition = new Condition(condition.name, val,condition.op, null, condition.clause); - } - isPass = condition.isPass(row); - } - else isPass = tagVal.equals(row.getVal(tagName)); - if (isPass) { - row.forEachReplica(eval.collName, ri -> { - if (Policy.ANY.equals(eval.shardName) - || eval.shardName.equals(ri.getShard())) - violation.addReplica(new Violation.ReplicaInfoAndErr(ri).withDelta(tag.delta(row.getVal(tag.name)))); - }); - } - } - - } - - public static long addReplicaCountsForNode = 0; - public static long addReplicaCountsForNodeCacheMiss = 0; - public static final String PERSHARD_REPLICAS = Clause.class.getSimpleName() + ".perShardReplicas"; - - private void addReplicaCountsForNode(ComputedValueEvaluator computedValueEvaluator, ReplicaCount replicaCount, Row node) { - addReplicaCountsForNode++; - - ReplicaCount rc = node.computeCacheIfAbsent(computedValueEvaluator.collName, computedValueEvaluator.shardName, PERSHARD_REPLICAS, - this, o -> { - addReplicaCountsForNodeCacheMiss++; - ReplicaCount result = new ReplicaCount(); - node.forEachReplica((String) collection.getValue(), ri -> { - if (Policy.ANY.equals(computedValueEvaluator.shardName) - || computedValueEvaluator.shardName.equals(ri.getShard())) - result.increment(ri); - }); - return result; - }); - if (rc != null) - replicaCount.increment(rc); - - - } - - List testPerNode(Policy.Session session, double[] deviations) { - ComputedValueEvaluator eval = new ComputedValueEvaluator(session); - eval.collName = (String) collection.getValue(); - Violation.Ctx ctx = new Violation.Ctx(this, session.matrix, eval); - Set shards = getShardNames(session, eval); - for (String s : shards) { - final ReplicaCount replicaCount = new ReplicaCount(); - eval.shardName = s; - for (Row row : session.matrix) { - replicaCount.reset(); - eval.node = row.node; - Condition tag = this.tag; - if (tag.computedType != null) { - tag = evaluateValue(tag, eval); - } - if (!tag.isPass(row)) continue; - addReplicaCountsForNode(eval, replicaCount, row); - SealedClause sealedClause = this.getSealedClause(eval); - if (!sealedClause.replica.isPass(replicaCount)) { - ReplicaCount replicaCountCopy = replicaCount.copy(); - Violation violation = new Violation(sealedClause, - eval.collName, - eval.shardName, - eval.node, - replicaCountCopy, - sealedClause.getReplica().replicaCountDelta(replicaCountCopy), - eval.node); - ctx.resetAndAddViolation(row.node, replicaCountCopy, violation); - sealedClause.addViolatingReplicasForGroup(sealedClause.tag, eval, ctx, NODE, row.node, violation, - Collections.singletonList(row)); - if (!this.strict && deviations != null) { - tag.varType.computeDeviation(session, deviations, replicaCount, sealedClause); - } - } else { - if (replica.op == RANGE_EQUAL) tag.varType.computeDeviation(session, deviations, replicaCount, sealedClause); - } - } - } - return ctx.allViolations; - } - - private Set getShardNames(Policy.Session session, - ComputedValueEvaluator eval) { - Set shards = new HashSet<>(); - if (isShardAbsent()) { - shards.add(Policy.ANY); //consider the entire collection is a single shard - } else { - for (Row row : session.matrix) { - row.forEachShard(eval.collName, (shard, r) -> { - if (this.shard.isPass(shard)) shards.add(shard); // add relevant shards - }); - } - } - return shards; - } - - boolean isShardAbsent() { - return Policy.ANY.equals(shard.val); - } - - public List test(Policy.Session session, double[] deviations) { - if (isPerCollectiontag()) { - if(nodeSetPresent) { - if(put == Put.ON_EACH){ - return testPerNode(session, deviations) ; - } else { - return testGroupNodes(session, deviations); - } - } - - return tag.varType == Type.NODE || - (tag.varType.meta.isNodeSpecificVal() && replica.computedType == null) ? - testPerNode(session, deviations) : - testGroupNodes(session, deviations); - } else { - ComputedValueEvaluator computedValueEvaluator = new ComputedValueEvaluator(session); - Violation.Ctx ctx = new Violation.Ctx(this, session.matrix, computedValueEvaluator); - for (Row r : session.matrix) { - computedValueEvaluator.node = r.node; - SealedClause sealedClause = getSealedClause(computedValueEvaluator); - // check only live nodes - if (r.isLive() && !sealedClause.getGlobalTag().isPass(r)) { - ctx.resetAndAddViolation(r.node, null, new Violation(sealedClause, null, null, r.node, r.getVal(sealedClause.globalTag.name), - sealedClause.globalTag.delta(r.getVal(globalTag.name)), r.node)); - addViolatingReplicasForGroup(sealedClause.globalTag, computedValueEvaluator, ctx, Type.CORES.tagName, r.node, ctx.currentViolation, session.matrix); - - } - } - return ctx.allViolations; - - } - } - - - public boolean isMatch(Replica r, String collection, String shard) { - if (type != null && r.getType() != type) return false; - if (r.getCollection().equals(collection)) { - if (this.shard == null || this.shard.val.equals(Policy.ANY)) return true; - else if (this.shard.val.equals(Policy.EACH) && r.getShard().equals(shard)) return true; - else return this.shard.val.equals(r.getShard()) && r.getShard().equals(shard); - } - return false; - } - - boolean matchShard(String replicaShard, String shardInContext) { - if (shard == null || shard.val.equals(ANY)) return true; - if (shard.val.equals(Policy.EACH) && replicaShard.equals(shardInContext)) return true; - if (shard.val.equals(replicaShard)) return true; - return false; - } - - public boolean isStrict() { - return strict; - } - - @Override - public String toString() { - return toJSONString(original); - } - - @Override - public void writeMap(EntryWriter ew) throws IOException { - for (Map.Entry e : original.entrySet()) ew.put(e.getKey(), e.getValue()); - } - - enum TestStatus { - NOT_APPLICABLE, FAIL, PASS - } - - public static class ComputedValueEvaluator implements Function { - final Policy.Session session; - String collName = null; - String shardName = null; - String node = null; - - public ComputedValueEvaluator(Policy.Session session) { - this.session = session; - } - - @Override - public Object apply(Condition computedCondition) { - return computedCondition.varType.computeValue(session, computedCondition, collName, shardName, node); - } - - } - - /** - * @param name name of the condition - * @param val value of the condition - * @param isRuleVal is this provided in the rule - * @return actual validated value - */ - public static Object validate(String name, Object val, boolean isRuleVal) { - if (val == null) return null; - Type info = VariableBase.getTagType(name); - if (info == null) throw new RuntimeException("Unknown type :" + name); - return info.validate(name, val, isRuleVal); - } - - public static Long parseLong(String name, Object val) { - if (val == null) return null; - if (val instanceof Long) return (Long) val; - Number num = null; - if (val instanceof String) { - try { - num = Long.parseLong(((String) val).trim()); - } catch (NumberFormatException e) { - try { - num = Double.parseDouble((String) val); - } catch (NumberFormatException e1) { - throw new RuntimeException(name + ": " + val + "not a valid number", e); - } - } - - } else if (val instanceof Number) { - num = (Number) val; - } - - if (num != null) { - return num.longValue(); - } - throw new RuntimeException(name + ": " + val + "not a valid number"); - } - - @Override - public int hashCode() { - return hashCode; - } - - public static Double parseDouble(String name, Object val) { - if (val == null) return null; - if (val instanceof RangeVal) val = ((RangeVal) val).actual; - if (val instanceof Double) return (Double) val; - Number num = null; - if (val instanceof String) { - try { - num = Double.parseDouble((String) val); - } catch (NumberFormatException e) { - throw new RuntimeException(name + ": " + val + "not a valid number", e); - } - - } else if (val instanceof Number) { - num = (Number) val; - } - - if (num != null) { - return num.doubleValue(); - } - throw new RuntimeException(name + ": " + val + "not a valid number"); - } - - public static final String METRICS_PREFIX = "metrics:"; - - enum Put { - ON_ALL(""), ON_EACH("on-each-node"); - - public final String val; - - Put(String s) { - this.val = s; - } - - public static Put get(String s) { - for (Put put : values()) { - if (put.val.equals(s)) return put; - } - return null; - } - } - -} diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ComputedType.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ComputedType.java deleted file mode 100644 index 73bca1f5d11..00000000000 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ComputedType.java +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.client.solrj.cloud.autoscaling; - -public enum ComputedType { - NULL(), - EQUAL() { - @Override - public String wrap(String value) { - return "#EQUAL"; - } - - @Override - public String match(String val) { - if ("#EQUAL".equals(val)) return "1"; - return null; - } - - }, - ALL() { - @Override - public String wrap(String value) { - return "#ALL"; - } - - @Override - public String match(String val) { - if ("#ALL".equals(val)) return "1"; - return null; - } - - }, - PERCENT { - @Override - public String wrap(String value) { - return value + "%"; - } - - @Override - public String match(String val) { - if (val != null && !val.isEmpty() && val.charAt(val.length() - 1) == '%') { - String newVal = val.substring(0, val.length() - 1); - double d; - try { - d = Double.parseDouble(newVal); - } catch (NumberFormatException e) { - throw new IllegalArgumentException("Invalid percentage value : " + val); - } - if (d < 0 || d > 100) { - throw new IllegalArgumentException("Percentage value must lie between [1 -100] : provided value : " + val); - } - return newVal; - } else { - return null; - } - } - - @Override - public Object compute(Object val, Condition c) { - if (val == null || Clause.parseDouble(c.name, val) == 0) return 0d; - return Clause.parseDouble(c.name, val) * Clause.parseDouble(c.name, c.val).doubleValue() / 100; - } - - @Override - public String toString() { - return "%"; - } - }; - - // return null if there is no match. return a modified string - // if there is a match - public String match(String val) { - return null; - } - - public String wrap(String value) { - return value; - } - - public Object compute(Object val, Condition c) { - return val; - } - -} diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Condition.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Condition.java deleted file mode 100644 index dd590871871..00000000000 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Condition.java +++ /dev/null @@ -1,125 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.client.solrj.cloud.autoscaling; - -import java.io.IOException; -import java.util.Objects; - -import org.apache.solr.common.MapWriter; - -import static org.apache.solr.client.solrj.cloud.autoscaling.Operand.GREATER_THAN; -import static org.apache.solr.client.solrj.cloud.autoscaling.Operand.LESS_THAN; - -public class Condition implements MapWriter { - final String name; - final Object val; - final Variable.Type varType; - final ComputedType computedType; - final Operand op; - Clause clause; - - Condition(String name, Object val, Operand op, ComputedType computedType, Clause parent) { - this.name = name; - this.val = val; - this.op = op; - varType = VariableBase.getTagType(name); - this.computedType = computedType; - this.clause = parent; - } - - @Override - public void writeMap(EntryWriter ew) throws IOException { - String value = op.wrap(val); - if (computedType != null) value = computedType.wrap(value); - ew.put(name, value); - } - - @Override - public String toString() { - return jsonStr(); - } - - public Clause getClause() { - return clause; - } - - boolean isPass(Object inputVal) { - return isPass(inputVal, null); - } - - boolean isPass(Object inputVal, Row row) { - if (computedType != null) { - throw new IllegalStateException("This is supposed to be called only from a Condition with no computed value or a SealedCondition"); - - } - if (inputVal instanceof ReplicaCount) inputVal = ((ReplicaCount) inputVal).getVal(getClause().type); - return varType.match(inputVal, op, val, name, row); - } - - - boolean isPass(Row row) { - return isPass(row.getVal(name), row); - } - - @Override - public int hashCode() { - return Objects.hash(name, val, op); - } - - @Override - public boolean equals(Object that) { - if (that instanceof Condition) { - Condition c = (Condition) that; - return Objects.equals(c.name, name) && Objects.equals(c.val, val) && c.op == op; - } - return false; - } - - public Double replicaCountDelta(Object val) { - if (val instanceof ReplicaCount) val = ((ReplicaCount) val).getVal(getClause().type); - return op.delta(this.val, val); - } - - public Double delta(Object val) { - if (this.val instanceof String) { - if (op == LESS_THAN || op == GREATER_THAN) { - return op - .opposite(getClause().isReplicaZero() && this == getClause().tag) - .delta(Clause.parseDouble(name, this.val), Clause.parseDouble(name, val)); - } else { - return 0d; - } - } else { - return op - .opposite(getClause().isReplicaZero() && this == getClause().getTag()) - .delta(this.val, val); - } - } - - public String getName() { - return name; - } - - public Object getValue() { - return val; - } - - public Operand getOperand() { - return op; - } -} diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/CoresVariable.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/CoresVariable.java deleted file mode 100644 index 2e2c9119d47..00000000000 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/CoresVariable.java +++ /dev/null @@ -1,127 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.client.solrj.cloud.autoscaling; - -import java.util.Collection; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.Consumer; - -import org.apache.solr.common.cloud.Replica; - -import static org.apache.solr.common.params.CollectionParams.CollectionAction.MOVEREPLICA; - -public class CoresVariable extends VariableBase { - public CoresVariable(Type type) { - super(type); - } - - @Override - public Object validate(String name, Object val, boolean isRuleVal) { - return VariableBase.getOperandAdjustedValue(super.validate(name, val, isRuleVal), val); - } - - public boolean addViolatingReplicas(Violation.Ctx ctx) { - for (Row row : ctx.allRows) { - if (row.node.equals(ctx.currentViolation.node)) { - row.forEachReplica(replicaInfo -> ctx.currentViolation - .addReplica(new Violation.ReplicaInfoAndErr(replicaInfo) - .withDelta(ctx.currentViolation.replicaCountDelta))); - } - } - return true; - - } - - @Override - public void getSuggestions(Suggestion.Ctx ctx) { - if (ctx.violation == null || ctx.violation.replicaCountDelta == 0) return; - if (ctx.violation.replicaCountDelta > 0) {//there are more replicas than necessary - for (int i = 0; i < Math.abs(ctx.violation.replicaCountDelta); i++) { - if (!ctx.needMore()) return; - Suggester suggester = ctx.session.getSuggester(MOVEREPLICA) - .hint(Suggester.Hint.SRC_NODE, ctx.violation.node); - if (ctx.addSuggestion(suggester) == null) break; - } - } - } - - @Override - public void projectAddReplica(Cell cell, Replica ri, Consumer ops, boolean strictMode) { - cell.val = cell.val == null ? 0 : ((Number) cell.val).doubleValue() + 1; - } - - @Override - public void projectRemoveReplica(Cell cell, Replica ri, Consumer opCollector) { - cell.val = cell.val == null ? 0 : ((Number) cell.val).doubleValue() - 1; - } - - @Override - public Object computeValue(Policy.Session session, Condition condition, String collection, String shard, String node) { - if (condition.computedType == ComputedType.EQUAL) { - AtomicInteger liveNodes = new AtomicInteger(0); - int coresCount = getTotalCores(session, liveNodes); - int numBuckets = condition.clause.tag.op == Operand.IN ? - ((Collection) condition.clause.tag.val).size() : - liveNodes.get(); - return numBuckets == 0 || coresCount == 0 ? 0d : (double) coresCount / (double) numBuckets; - } else if (condition.computedType == ComputedType.PERCENT) { - return ComputedType.PERCENT.compute(getTotalCores(session, new AtomicInteger()), condition); - } else { - throw new IllegalArgumentException("Invalid computed type in " + condition); - } - } - - static final String TOTALCORES = CoresVariable.class.getSimpleName() + ".totalcores"; - private int getTotalCores(Policy.Session session, AtomicInteger liveNodes) { - int coresCount = 0; - for (Row row : session.matrix) { - if (!row.isLive) continue; - liveNodes.incrementAndGet(); - Integer res = row.computeCacheIfAbsent(TOTALCORES, o -> { - int[] result = new int[1]; - row.forEachReplica(replicaInfo -> result[0]++); - return result[0]; - }); - if (res != null) - coresCount += res; - - - } - return coresCount; - } - - @Override - public String postValidate(Condition condition) { - Condition nodeTag = condition.getClause().getTag(); - if (nodeTag.varType != Type.NODE) return "'cores' attribute can only be used with 'node' attribute"; - if (condition.computedType == ComputedType.EQUAL) { - if (nodeTag.name.equals("node") && (nodeTag.op == Operand.WILDCARD || nodeTag.op == Operand.IN)) { - return null; - } else { - return "cores: '#EQUAL' can be used only with node: '#ANY', node :[....]"; - } - } else { - return ReplicaVariable.checkNonEqualOp(condition); - } - } - - @Override - public Operand getOperand(Operand expected, Object strVal, ComputedType computedType) { - return ReplicaVariable.checkForRangeOperand(expected, strVal, computedType); - } -} diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/DelegatingCloudManager.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/DelegatingCloudManager.java deleted file mode 100644 index aa0e62e2080..00000000000 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/DelegatingCloudManager.java +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.client.solrj.cloud.autoscaling; - -import java.io.IOException; -import java.util.Map; - -import org.apache.solr.client.solrj.SolrRequest; -import org.apache.solr.client.solrj.SolrResponse; -import org.apache.solr.client.solrj.cloud.DistribStateManager; -import org.apache.solr.client.solrj.cloud.DistributedQueueFactory; -import org.apache.solr.client.solrj.cloud.NodeStateProvider; -import org.apache.solr.client.solrj.cloud.SolrCloudManager; -import org.apache.solr.client.solrj.impl.ClusterStateProvider; -import org.apache.solr.common.util.ObjectCache; -import org.apache.solr.common.util.TimeSource; - -/** - * Base class for overriding some behavior of {@link SolrCloudManager}. - */ -public class DelegatingCloudManager implements SolrCloudManager { - protected final SolrCloudManager delegate; - private ObjectCache objectCache = new ObjectCache(); - private TimeSource timeSource = TimeSource.NANO_TIME; - - public DelegatingCloudManager(SolrCloudManager delegate) { - this.delegate = delegate; - } - - @Override - public ClusterStateProvider getClusterStateProvider() { - return delegate.getClusterStateProvider(); - } - - @Override - public NodeStateProvider getNodeStateProvider() { - return delegate.getNodeStateProvider(); - } - - @Override - public DistribStateManager getDistribStateManager() { - return delegate.getDistribStateManager(); - } - - @Override - public DistributedQueueFactory getDistributedQueueFactory() { - return delegate.getDistributedQueueFactory(); - } - - @Override - public ObjectCache getObjectCache() { - return delegate == null ? objectCache : delegate.getObjectCache(); - } - - @Override - public boolean isClosed() { - return delegate.isClosed(); - } - - @Override - public TimeSource getTimeSource() { - return delegate == null ? timeSource : delegate.getTimeSource(); - } - - @Override - public SolrResponse request(@SuppressWarnings({"rawtypes"})SolrRequest req) throws IOException { - return delegate.request(req); - } - - @Override - public byte[] httpRequest(String url, SolrRequest.METHOD method, Map headers, String payload, int timeout, boolean followRedirects) throws IOException { - return delegate.httpRequest(url, method, headers, payload, timeout, followRedirects); - } - - @Override - public void close() throws IOException { - delegate.close(); - } -} diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/DelegatingClusterStateProvider.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/DelegatingClusterStateProvider.java deleted file mode 100644 index 827b198e7b8..00000000000 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/DelegatingClusterStateProvider.java +++ /dev/null @@ -1,130 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.client.solrj.cloud.autoscaling; - -import java.io.IOException; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import org.apache.solr.client.solrj.impl.ClusterStateProvider; -import org.apache.solr.common.cloud.ClusterState; -import org.apache.solr.common.cloud.DocCollection; - -/** - * Base class for overriding some behavior of {@link ClusterStateProvider} - */ -public class DelegatingClusterStateProvider implements ClusterStateProvider { - protected ClusterStateProvider delegate; - - public DelegatingClusterStateProvider(ClusterStateProvider delegate) { - this.delegate = delegate; - } - - @Override - public ClusterState.CollectionRef getState(String collection) { - if (delegate != null) { - return delegate.getState(collection); - } else { - return null; - } - } - - @Override - public Set getLiveNodes() { - if (delegate != null) { - return delegate.getLiveNodes(); - } else { - return Collections.emptySet(); - } - } - - @Override - public List resolveAlias(String alias) { - if (delegate != null) { - return delegate.resolveAlias(alias); - } else { - return Collections.singletonList(alias); - } - } - - @Override - public Map getAliasProperties(String alias) { - if (delegate != null) { - return delegate.getAliasProperties(alias); - } else { - return Collections.emptyMap(); - } - } - - @Override - public String resolveSimpleAlias(String alias) throws IllegalArgumentException { - if (delegate != null) { - return delegate.resolveSimpleAlias(alias); - } else { - return alias; - } - } - - @Override - public ClusterState getClusterState() throws IOException { - if (delegate != null) { - return delegate.getClusterState(); - } else { - return null; - } - } - - @Override - public Map getClusterProperties() { - if (delegate != null) { - return delegate.getClusterProperties(); - } else { - return Collections.emptyMap(); - } - } - - @Override - public String getPolicyNameByCollection(String coll) { - if (delegate != null) { - return delegate.getPolicyNameByCollection(coll); - } else { - return null; - } - } - - @Override - public DocCollection getCollection(String name) throws IOException { - ClusterState cs = getClusterState(); - return cs == null ? null : cs.getCollectionOrNull(name); - } - - @Override - public void connect() { - if (delegate != null) { - delegate.connect(); - } - } - - @Override - public void close() throws IOException { - if (delegate != null) { - delegate.close(); - } - } -} diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/DelegatingDistribStateManager.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/DelegatingDistribStateManager.java deleted file mode 100644 index 40655fb1b16..00000000000 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/DelegatingDistribStateManager.java +++ /dev/null @@ -1,107 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.client.solrj.cloud.autoscaling; - -import java.io.IOException; -import java.util.List; -import java.util.NoSuchElementException; - -import org.apache.solr.client.solrj.cloud.DistribStateManager; -import org.apache.zookeeper.CreateMode; -import org.apache.zookeeper.KeeperException; -import org.apache.zookeeper.Op; -import org.apache.zookeeper.OpResult; -import org.apache.zookeeper.Watcher; - -public class DelegatingDistribStateManager implements DistribStateManager { - private final DistribStateManager delegate; - - public DelegatingDistribStateManager(DistribStateManager delegate) { - this.delegate = delegate; - } - - @Override - public boolean hasData(String path) throws IOException, KeeperException, InterruptedException { - return delegate.hasData(path); - } - - @Override - public List listData(String path) throws NoSuchElementException, IOException, KeeperException, InterruptedException { - return delegate.listData(path); - } - - @Override - public List listData(String path, Watcher watcher) throws NoSuchElementException, IOException, KeeperException, InterruptedException { - return delegate.listData(path, watcher); - } - - @Override - public VersionedData getData(String path, Watcher watcher) throws NoSuchElementException, IOException, KeeperException, InterruptedException { - return delegate.getData(path, watcher); - } - - @Override - public VersionedData getData(String path) throws NoSuchElementException, IOException, KeeperException, InterruptedException { - return delegate.getData(path); - } - - @Override - public void makePath(String path) throws AlreadyExistsException, IOException, KeeperException, InterruptedException { - delegate.makePath(path); - } - - @Override - public void makePath(String path, byte[] data, CreateMode createMode, boolean failOnExists) throws AlreadyExistsException, IOException, KeeperException, InterruptedException { - delegate.makePath(path, data, createMode, failOnExists); - } - - @Override - public String createData(String path, byte[] data, CreateMode mode) throws AlreadyExistsException, IOException, KeeperException, InterruptedException { - return delegate.createData(path, data, mode); - } - - @Override - public void removeData(String path, int version) throws NoSuchElementException, NotEmptyException, IOException, BadVersionException, KeeperException, InterruptedException { - delegate.removeData(path, version); - } - - @Override - public void setData(String path, byte[] data, int version) throws BadVersionException, NoSuchElementException, IOException, KeeperException, InterruptedException { - delegate.setData(path, data, version); - } - - @Override - public List multi(Iterable ops) throws BadVersionException, NoSuchElementException, AlreadyExistsException, IOException, KeeperException, InterruptedException { - return delegate.multi(ops); - } - - @Override - public AutoScalingConfig getAutoScalingConfig(Watcher watcher) throws InterruptedException, IOException { - return delegate.getAutoScalingConfig(watcher); - } - - @Override - public AutoScalingConfig getAutoScalingConfig() throws InterruptedException, IOException { - return delegate.getAutoScalingConfig(); - } - - @Override - public void close() throws IOException { - delegate.close(); - } -} diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/DelegatingNodeStateProvider.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/DelegatingNodeStateProvider.java deleted file mode 100644 index 2a30d37a2e8..00000000000 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/DelegatingNodeStateProvider.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.client.solrj.cloud.autoscaling; - -import java.io.IOException; -import java.util.Collection; -import java.util.List; -import java.util.Map; - -import org.apache.solr.client.solrj.cloud.NodeStateProvider; -import org.apache.solr.common.cloud.Replica; - -/** - * Base class for overriding some behavior of {@link NodeStateProvider}. - */ -public class DelegatingNodeStateProvider implements NodeStateProvider { - private final NodeStateProvider delegate; - - public DelegatingNodeStateProvider(NodeStateProvider delegate) { - this.delegate = delegate; - } - - @Override - public Map getNodeValues(String node, Collection tags) { - return delegate.getNodeValues(node, tags); - } - - @Override - public Map>> getReplicaInfo(String node, Collection keys) { - return delegate.getReplicaInfo(node, keys); - } - - @Override - public void close() throws IOException { - delegate.close(); - } - - @Override - public boolean isClosed() { - return delegate.isClosed(); - } -} diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/DeleteNodeSuggester.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/DeleteNodeSuggester.java deleted file mode 100644 index 62bfbdef982..00000000000 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/DeleteNodeSuggester.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.client.solrj.cloud.autoscaling; - -import java.util.Set; - -import org.apache.solr.client.solrj.SolrRequest; -import org.apache.solr.client.solrj.request.CollectionAdminRequest; -import org.apache.solr.common.params.CollectionParams; - -/** - * This suggester produces a DELETENODE request using provided {@link org.apache.solr.client.solrj.cloud.autoscaling.Suggester.Hint#SRC_NODE}. - */ -class DeleteNodeSuggester extends Suggester { - - @Override - public CollectionParams.CollectionAction getAction() { - return CollectionParams.CollectionAction.DELETENODE; - } - - @Override - @SuppressWarnings({"rawtypes"}) - SolrRequest init() { - @SuppressWarnings({"unchecked"}) - Set srcNodes = (Set) hints.get(Hint.SRC_NODE); - if (srcNodes.isEmpty()) { - throw new RuntimeException("delete-node requires 'src_node' hint"); - } - if (srcNodes.size() > 1) { - throw new RuntimeException("delete-node requires exactly one 'src_node' hint"); - } - return CollectionAdminRequest.deleteNode(srcNodes.iterator().next()); - } -} diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/DeleteReplicaSuggester.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/DeleteReplicaSuggester.java deleted file mode 100644 index 20f38275007..00000000000 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/DeleteReplicaSuggester.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.client.solrj.cloud.autoscaling; - -import java.util.Collections; -import java.util.Set; - -import org.apache.solr.client.solrj.SolrRequest; -import org.apache.solr.client.solrj.request.CollectionAdminRequest; -import org.apache.solr.common.params.CollectionParams; -import org.apache.solr.common.util.Pair; - -/** - * This suggester produces a DELETEREPLICA request using provided {@link org.apache.solr.client.solrj.cloud.autoscaling.Suggester.Hint#COLL_SHARD} and - * {@link org.apache.solr.client.solrj.cloud.autoscaling.Suggester.Hint#NUMBER} hints to specify the collection, shard and number of replicas to delete. - */ -class DeleteReplicaSuggester extends Suggester { - - @Override - public CollectionParams.CollectionAction getAction() { - return CollectionParams.CollectionAction.DELETEREPLICA; - } - - @Override - @SuppressWarnings({"rawtypes"}) - SolrRequest init() { - @SuppressWarnings({"unchecked"}) - Set> shards = (Set>) hints.getOrDefault(Hint.COLL_SHARD, Collections.emptySet()); - if (shards.isEmpty()) { - throw new RuntimeException("delete-replica requires 'collection' and 'shard'"); - } - if (shards.size() > 1) { - throw new RuntimeException("delete-replica requires exactly one pair of 'collection' and 'shard'"); - } - Pair collShard = shards.iterator().next(); - @SuppressWarnings({"unchecked"}) - Set counts = (Set) hints.getOrDefault(Hint.NUMBER, Collections.emptySet()); - Integer count = null; - if (!counts.isEmpty()) { - if (counts.size() > 1) { - throw new RuntimeException("delete-replica allows at most one number hint specifying the number of replicas to delete"); - } - Number n = counts.iterator().next(); - count = n.intValue(); - } - @SuppressWarnings({"unchecked"}) - Set replicas = (Set) hints.getOrDefault(Hint.REPLICA, Collections.emptySet()); - String replica = null; - if (!replicas.isEmpty()) { - if (replicas.size() > 1) { - throw new RuntimeException("delete-replica allows at most one 'replica' hint"); - } - replica = replicas.iterator().next(); - } - if (replica == null && count == null) { - throw new RuntimeException("delete-replica requires either 'replica' or 'number' hint"); - } - if (replica != null) { - return CollectionAdminRequest.deleteReplica(collShard.first(), collShard.second(), replica); - } else { - return CollectionAdminRequest.deleteReplica(collShard.first(), collShard.second(), count); - } - } -} diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/FreeDiskVariable.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/FreeDiskVariable.java deleted file mode 100644 index f9e647d9178..00000000000 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/FreeDiskVariable.java +++ /dev/null @@ -1,176 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.client.solrj.cloud.autoscaling; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.Comparator; -import java.util.List; -import java.util.concurrent.atomic.AtomicReference; -import java.util.function.Consumer; -import java.util.stream.Collectors; - -import org.apache.solr.client.solrj.cloud.autoscaling.Suggester.Hint; -import org.apache.solr.common.cloud.Replica; -import org.apache.solr.common.util.Pair; - -import static org.apache.solr.client.solrj.cloud.autoscaling.Suggestion.suggestNegativeViolations; -import static org.apache.solr.client.solrj.cloud.autoscaling.Variable.Type.CORE_IDX; -import static org.apache.solr.client.solrj.cloud.autoscaling.Variable.Type.FREEDISK; -import static org.apache.solr.client.solrj.cloud.autoscaling.Variable.Type.TOTALDISK; -import static org.apache.solr.common.cloud.rule.ImplicitSnitch.DISK; -import static org.apache.solr.common.params.CollectionParams.CollectionAction.MOVEREPLICA; - -public class FreeDiskVariable extends VariableBase { - - public FreeDiskVariable(Type type) { - super(type); - } - - @Override - public Object convertVal(Object val) { - Number value = (Number) super.validate(FREEDISK.tagName, val, false); - if (value != null) { - value = value.doubleValue() / 1024.0d / 1024.0d / 1024.0d; - } - return value; - } - - @Override - public Object computeValue(Policy.Session session, Condition condition, String collection, String shard, String node) { - if (condition.computedType == ComputedType.PERCENT) { - Row r = session.getNode(node); - if (r == null) return 0d; - return ComputedType.PERCENT.compute(r.getVal(TOTALDISK.tagName), condition); - } - throw new IllegalArgumentException("Unsupported type " + condition.computedType); - } - - - - @Override - public int compareViolation(Violation v1, Violation v2) { - //TODO use tolerance compare - return Double.compare( - v1.getViolatingReplicas().stream().mapToDouble(v -> v.delta == null ? 0 : v.delta).max().orElse(0d), - v2.getViolatingReplicas().stream().mapToDouble(v3 -> v3.delta == null ? 0 : v3.delta).max().orElse(0d)); - } - - @Override - public void computeDeviation(Policy.Session session, double[] deviation, ReplicaCount replicaCount, - SealedClause sealedClause) { - if (deviation == null) return; - for (Row node : session.matrix) { - Object val = node.getVal(sealedClause.tag.name); - Double delta = sealedClause.tag.delta(val); - if (delta != null) { - deviation[0] += Math.abs(delta); - } - } - } - - @Override - public void getSuggestions(Suggestion.Ctx ctx) { - if (ctx.violation == null) return; - if (ctx.violation.replicaCountDelta > 0) { - List matchingNodes = ctx.session.matrix.stream().filter( - row -> ctx.violation.getViolatingReplicas() - .stream() - .anyMatch(p -> row.node.equals(p.replicaInfo.getNodeName()))) - .sorted(Comparator.comparing(r -> ((Double) r.getVal(DISK, 0d)))) - .collect(Collectors.toList()); - - - for (Row node : matchingNodes) { - //lets try to start moving the smallest cores off of the node - ArrayList replicas = new ArrayList<>(); - node.forEachReplica(replicas::add); - replicas.sort((r1, r2) -> { - Long s1 = Clause.parseLong(CORE_IDX.tagName, r1.getProperties().get(CORE_IDX.tagName)); - Long s2 = Clause.parseLong(CORE_IDX.tagName, r2.getProperties().get(CORE_IDX.tagName)); - if (s1 != null && s2 != null) return s1.compareTo(s2); - return 0; - }); - double currentDelta = ctx.violation.getClause().tag.delta(node.getVal(DISK)); - for (Replica replica : replicas) { - if (currentDelta < 1) break; - if (replica.getProperties().get(CORE_IDX.tagName) == null) continue; - Suggester suggester = ctx.session.getSuggester(MOVEREPLICA) - .hint(Hint.COLL_SHARD, new Pair<>(replica.getCollection(), replica.getShard())) - .hint(Hint.SRC_NODE, node.node) - .forceOperation(true); - ctx.addSuggestion(suggester); - currentDelta -= Clause.parseLong(CORE_IDX.tagName, replica.get(CORE_IDX.tagName)); - } - } - } else if (ctx.violation.replicaCountDelta < 0) { - suggestNegativeViolations(ctx, shards -> getSortedShards(ctx.session.matrix, shards, ctx.violation.coll)); - } - } - - - static List getSortedShards(List matrix, Collection shardSet, String coll) { - return shardSet.stream() - .map(shard1 -> { - AtomicReference> result = new AtomicReference<>(); - for (Row node : matrix) { - node.forEachShard(coll, (s, ri) -> { - if (result.get() != null) return; - if (s.equals(shard1) && ri.size() > 0) { - Number sz = ((Number) ri.get(0).get(CORE_IDX.tagName)); - if (sz != null) result.set(new Pair<>(shard1, sz.longValue())); - } - }); - } - return result.get() == null ? new Pair<>(shard1, 0L) : result.get(); - }) - .sorted(Comparator.comparingLong(Pair::second)) - .map(Pair::first) - .collect(Collectors.toList()); - - } - - //When a replica is added, freedisk should be incremented - @Override - public void projectAddReplica(Cell cell, Replica ri, Consumer ops, boolean strictMode) { - //go through other replicas of this shard and copy the index size value into this - for (Row row : cell.getRow().session.matrix) { - row.forEachReplica(replicaInfo -> { - if (ri != replicaInfo && - ri.getCollection().equals(replicaInfo.getCollection()) && - ri.getShard().equals(replicaInfo.getShard()) && - ri.get(CORE_IDX.tagName) == null && - replicaInfo.get(CORE_IDX.tagName) != null) { - ri.getProperties().put(CORE_IDX.tagName, validate(CORE_IDX.tagName, replicaInfo.get(CORE_IDX.tagName), false)); - } - }); - } - Double idxSize = (Double) validate(CORE_IDX.tagName, ri.get(CORE_IDX.tagName), false); - if (idxSize == null) return; - Double currFreeDisk = cell.val == null ? 0.0d : (Double) cell.val; - cell.val = currFreeDisk - idxSize; - } - - @Override - public void projectRemoveReplica(Cell cell, Replica ri, Consumer opCollector) { - Double idxSize = (Double) validate(CORE_IDX.tagName, ri.get(CORE_IDX.tagName), false); - if (idxSize == null) return; - Double currFreeDisk = cell.val == null ? 0.0d : (Double) cell.val; - cell.val = currFreeDisk + idxSize; - } -} diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/MoveReplicaSuggester.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/MoveReplicaSuggester.java deleted file mode 100644 index f88d9d4b741..00000000000 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/MoveReplicaSuggester.java +++ /dev/null @@ -1,115 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.client.solrj.cloud.autoscaling; - -import java.lang.invoke.MethodHandles; -import java.util.Comparator; -import java.util.List; - -import org.apache.solr.client.solrj.SolrRequest; -import org.apache.solr.client.solrj.request.CollectionAdminRequest; -import org.apache.solr.common.cloud.Replica; -import org.apache.solr.common.params.CollectionParams; -import org.apache.solr.common.util.Pair; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import static org.apache.solr.common.params.CollectionParams.CollectionAction.MOVEREPLICA; - -public class MoveReplicaSuggester extends Suggester { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - @Override - @SuppressWarnings({"rawtypes"}) - SolrRequest init() { - SolrRequest operation = tryEachNode(true); - if (operation == null) operation = tryEachNode(false); - return operation; - } - - @SuppressWarnings({"rawtypes"}) - SolrRequest tryEachNode(boolean strict) { - //iterate through elements and identify the least loaded - List leastSeriousViolation = null; - Row bestSrcRow = null; - Row bestTargetRow = null; - Replica sourceReplicaInfo = null; - List> validReplicas = getValidReplicas(true, true, -1); - validReplicas.sort(leaderLast); - for (int i1 = 0; i1 < validReplicas.size(); i1++) { - lastBestDeviation = null; - Pair fromReplica = validReplicas.get(i1); - Row fromRow = fromReplica.second(); - Replica ri = fromReplica.first(); - if (ri == null) continue; - final int i = session.indexOf(fromRow.node); - int stopAt = force ? 0 : i; - Row targetRow = null; - for (int j = session.matrix.size() - 1; j >= stopAt; j--) { - targetRow = session.matrix.get(j); - if (targetRow.node.equals(fromRow.node)) continue; - if (!isNodeSuitableForReplicaAddition(targetRow, fromRow)) continue; - targetRow = targetRow.addReplica(ri.getCollection(), ri.getShard(), ri.getType(), strict); // add replica to target first - Row srcRowModified = targetRow.session.getNode(fromRow.node).removeReplica(ri.getCollection(), ri.getShard(), ri.getType());//then remove replica from source node - List errs = testChangedMatrix(strict, srcRowModified.session); - Policy.Session tmpSession = srcRowModified.session; - - if (!containsNewErrors(errs) && - isLessSerious(errs, leastSeriousViolation) && - (force || (tmpSession.indexOf(srcRowModified.node) < tmpSession.indexOf(targetRow.node)))) { - - int result = -1; - if (!force && srcRowModified.isLive && targetRow.isLive) { - result = tmpSession.getPolicy().getClusterPreferences().get(0).compare(srcRowModified, tmpSession.getNode(targetRow.node), true); - if (result == 0) result = tmpSession.getPolicy().getClusterPreferences().get(0).compare(srcRowModified, tmpSession.getNode(targetRow.node), false); - } - - if (result <= 0) { - leastSeriousViolation = errs; - bestSrcRow = srcRowModified; - sourceReplicaInfo = ri; - bestTargetRow = targetRow; - } - } - } - } - if (bestSrcRow != null) { - this.session = bestSrcRow.session; - return new CollectionAdminRequest.MoveReplica( - sourceReplicaInfo.getCollection(), - sourceReplicaInfo.getName(), - bestTargetRow.node); - } - return null; - } - - static Comparator> leaderLast = (r1, r2) -> { - int leaderCompare = Boolean.compare(r1.first().isLeader(), r2.first().isLeader()); - if ( leaderCompare != 0 ) { - return leaderCompare; - } else { - return r1.first().getName().compareTo(r2.first().getName()); - } - }; - - - @Override - public CollectionParams.CollectionAction getAction() { - return MOVEREPLICA; - } -} diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/NodeVariable.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/NodeVariable.java deleted file mode 100644 index 48dcd5e8691..00000000000 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/NodeVariable.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.client.solrj.cloud.autoscaling; - -import org.apache.solr.common.util.Pair; - -import static org.apache.solr.client.solrj.cloud.autoscaling.Policy.ANY; -import static org.apache.solr.common.params.CollectionParams.CollectionAction.MOVEREPLICA; - -public class NodeVariable extends VariableBase { - public NodeVariable(Type type) { - super(type); - } - - @Override - public void getSuggestions(Suggestion.Ctx ctx) { - if (ctx.violation == null || ctx.violation.replicaCountDelta == 0) return; - if (ctx.violation.replicaCountDelta > 0) {//there are more replicas than necessary - for (int i = 0; i < Math.abs(ctx.violation.replicaCountDelta); i++) { - Suggester suggester = ctx.session.getSuggester(MOVEREPLICA) - .forceOperation(true) - .hint(Suggester.Hint.SRC_NODE, ctx.violation.node) - .hint(ctx.violation.shard.equals(ANY) ? Suggester.Hint.COLL : Suggester.Hint.COLL_SHARD, - ctx.violation.shard.equals(ANY) ? ctx.violation.coll : new Pair<>(ctx.violation.coll, ctx.violation.shard)); - if(ctx.addSuggestion(suggester) == null) break; - } - } - } -} \ No newline at end of file diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/NoneSuggester.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/NoneSuggester.java deleted file mode 100644 index 51bfc70e426..00000000000 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/NoneSuggester.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.client.solrj.cloud.autoscaling; - -import org.apache.solr.client.solrj.SolrRequest; - -public class NoneSuggester extends Suggester { - - public static NoneSuggester get(Policy.Session session) { - NoneSuggester suggester = new NoneSuggester(); - suggester._init(session); - return suggester; - } - - @Override - @SuppressWarnings({"rawtypes"}) - SolrRequest init() { - return null; - } - - @Override - @SuppressWarnings({"rawtypes"}) - public SolrRequest getSuggestion() { - return null; - } -} diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Operand.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Operand.java deleted file mode 100644 index dac28c654b1..00000000000 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Operand.java +++ /dev/null @@ -1,210 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.client.solrj.cloud.autoscaling; - -import java.util.List; -import java.util.Objects; - -import org.apache.solr.client.solrj.cloud.autoscaling.Clause.TestStatus; - -import static org.apache.solr.client.solrj.cloud.autoscaling.Clause.TestStatus.FAIL; -import static org.apache.solr.client.solrj.cloud.autoscaling.Clause.TestStatus.NOT_APPLICABLE; -import static org.apache.solr.client.solrj.cloud.autoscaling.Clause.TestStatus.PASS; -import static org.apache.solr.client.solrj.cloud.autoscaling.Policy.ANY; - - -public enum Operand { - WILDCARD(ANY, Integer.MAX_VALUE) { - @Override - public TestStatus match(Object ruleVal, Object testVal) { - return testVal == null ? NOT_APPLICABLE : PASS; - } - - }, - - RANGE_EQUAL("", 0) { - @Override - public TestStatus match(Object ruleVal, Object testVal) { - return ((RangeVal) ruleVal).match((Number) testVal) ? PASS : FAIL; - } - - @Override - public Double delta(Object expected, Object actual) { - return ((RangeVal) expected).delta(((Number) actual).doubleValue()); - } - - @Override - public Object readRuleValue(Condition condition) { - if (condition.val instanceof String) { - String strVal = ((String) condition.val).trim(); - int hyphenIdx = strVal.indexOf('-'); - if (hyphenIdx > 0) { - String minS = strVal.substring(0, hyphenIdx).trim(); - String maxS = strVal.substring(hyphenIdx + 1, strVal.length()).trim(); - return new RangeVal( - (Number) condition.varType.validate(condition.name, minS, true), - (Number) condition.varType.validate(condition.name, maxS, true), - null - ); - - } - - } - - - Number num = (Number) condition.varType.validate(condition.name, condition.val, true); - return new RangeVal(Math.floor(num.doubleValue()), Math.ceil(num.doubleValue()), num); - } - }, - EQUAL("", 0) { - @Override - public double _delta(double expected, double actual) { - return actual - expected; - } - }, - IN("", 0) { - @Override - public TestStatus match(Object ruleVal, Object testVal) { - @SuppressWarnings({"rawtypes"}) - List l = (List) ruleVal; - return (l.contains(testVal)) ? PASS: FAIL; - } - }, - RANGE_NOT_EQUAL("", 2) { - @Override - public TestStatus match(Object ruleVal, Object testVal) { - return ((RangeVal) ruleVal).match((Number) testVal) ? FAIL : PASS; - } - - @Override - public Object readRuleValue(Condition condition) { - return RANGE_EQUAL.readRuleValue(condition); - } - - }, - NOT_EQUAL("!", 2) { - @Override - public TestStatus match(Object ruleVal, Object testVal) { - if(testVal == null) return PASS; - return super.match(ruleVal, testVal) == PASS ? FAIL : PASS; - } - - @Override - public double _delta(double expected, double actual) { - return expected - actual; - } - - }, - GREATER_THAN(">", 1) { - @Override - public TestStatus match(Object ruleVal, Object testVal) { - if (testVal == null) return NOT_APPLICABLE; - if (ruleVal instanceof String) ruleVal = Clause.parseDouble("", ruleVal); - if (ruleVal instanceof Double) { - return Double.compare(Clause.parseDouble("", testVal), (Double) ruleVal) == -1 ? FAIL : PASS; - } - return getLong(testVal) > getLong(ruleVal) ? PASS: FAIL ; - } - - @Override - public String wrap(Object val) { - return ">" + (((Number) val).doubleValue() - 1); - } - - @Override - public Operand opposite(boolean flag) { - return flag ? LESS_THAN : GREATER_THAN; - } - - @Override - protected double _delta(double expected, double actual) { - return actual > expected ? 0 : expected - actual; - } - }, - LESS_THAN("<", 2) { - @Override - public TestStatus match(Object ruleVal, Object testVal) { - if (testVal == null) return NOT_APPLICABLE; - if (ruleVal instanceof String) ruleVal = Clause.parseDouble("", ruleVal); - if (ruleVal instanceof Double) { - return Double.compare(Clause.parseDouble("", testVal), (Double) ruleVal) == 1 ? FAIL : PASS; - } - return getLong(testVal) < getLong(ruleVal) ? PASS: FAIL ; - } - - @Override - public String wrap(Object val) { - return "<" + (((Number) val).doubleValue() + 1); - } - - @Override - protected double _delta(double expected, double actual) { - return actual < expected ? 0 : actual - expected; - } - - @Override - public Operand opposite(boolean flag) { - return flag ? GREATER_THAN : this; - } - }; - - public Operand opposite(boolean flag) { - return this; - } - public final String operand; - final int priority; - - Operand(String val, int priority) { - this.operand = val; - this.priority = priority; - } - - public TestStatus match(Object ruleVal, Object testVal) { - return Objects.equals(ruleVal, testVal) ? PASS : FAIL; - } - - Long getLong(Object o) { - if (o instanceof Long) return (Long) o; - if(o instanceof Number ) return ((Number) o).longValue(); - return Long.parseLong(String.valueOf(o)); - - } - - public Double delta(Object expected, Object actual) { - if (expected instanceof Number && actual instanceof Number) { - Double expectedL = ((Number) expected).doubleValue(); - Double actualL = ((Number) actual).doubleValue(); - return _delta(expectedL, actualL); - } else { - return 0d; - } - - } - - protected double _delta(double expected, double actual) { - return 0; - } - - public String wrap(Object val) { - return operand + val.toString(); - } - - public Object readRuleValue(Condition condition) { - return condition.val; - } -} diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Policy.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Policy.java deleted file mode 100644 index 896a283a74c..00000000000 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Policy.java +++ /dev/null @@ -1,792 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.client.solrj.cloud.autoscaling; - -import java.io.IOException; -import java.io.PrintWriter; -import java.io.StringWriter; -import java.lang.invoke.MethodHandles; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.SortedSet; -import java.util.TreeSet; -import java.util.function.Supplier; -import java.util.stream.Collectors; - -import org.apache.solr.client.solrj.cloud.NodeStateProvider; -import org.apache.solr.client.solrj.cloud.SolrCloudManager; -import org.apache.solr.client.solrj.cloud.autoscaling.Variable.Type; -import org.apache.solr.client.solrj.impl.ClusterStateProvider; -import org.apache.solr.common.IteratorWriter; -import org.apache.solr.common.MapWriter; -import org.apache.solr.common.cloud.ClusterState; -import org.apache.solr.common.cloud.rule.ImplicitSnitch; -import org.apache.solr.common.params.CollectionAdminParams; -import org.apache.solr.common.params.CollectionParams.CollectionAction; -import org.apache.solr.common.util.Pair; -import org.apache.solr.common.util.StrUtils; -import org.apache.solr.common.util.Utils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import static java.util.Collections.emptyList; -import static java.util.Collections.emptyMap; -import static java.util.stream.Collectors.collectingAndThen; -import static java.util.stream.Collectors.toList; -import static org.apache.solr.client.solrj.cloud.autoscaling.Variable.Type.NODE; -import static org.apache.solr.client.solrj.cloud.autoscaling.Variable.Type.WITH_COLLECTION; - -/*The class that reads, parses and applies policies specified in - * autoscaling.json - * - * Create one instance of this class per unique autoscaling.json. - * This is immutable and is thread-safe - * - * Create a fresh new session for each use - * - */ -public class Policy implements MapWriter { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - public static final String POLICY = "policy"; - public static final String EACH = "#EACH"; - public static final String ANY = "#ANY"; - public static final String POLICIES = "policies"; - public static final String CLUSTER_POLICY = "cluster-policy"; - public static final String CLUSTER_PREFERENCES = "cluster-preferences"; - public static final Set GLOBAL_ONLY_TAGS = Set.of("cores", CollectionAdminParams.WITH_COLLECTION); - @SuppressWarnings({"unchecked"}) - public static final List DEFAULT_PREFERENCES = Collections.unmodifiableList( - Arrays.asList( - // NOTE - if you change this, make sure to update the solrcloud-autoscaling-overview.adoc which - // lists the default preferences - new Preference((Map) Utils.fromJSONString("{minimize : cores, precision:1}")), - new Preference((Map) Utils.fromJSONString("{maximize : freedisk}")))); - - public static final List> DEFAULT_CLUSTER_POLICY_JSON = Collections.unmodifiableList( - Arrays.asList( - Utils.makeMap("replica","<2", "shard","#EACH", "node", "#ANY", "strict", "false"), - Utils.makeMap("replica", "#EQUAL", "node", "#ANY", "strict", "false"), - Utils.makeMap("cores", "#EQUAL", "node","#ANY", "strict", "false") - ) - ); - - public static final List DEFAULT_CLUSTER_POLICY = DEFAULT_CLUSTER_POLICY_JSON.stream() - .map(Clause::create) - .collect(collectingAndThen(toList(), Collections::unmodifiableList)); - - /** - * These parameters are always fetched for all nodes regardless of whether they are used in preferences or not - */ - private static final List DEFAULT_PARAMS_OF_INTEREST = Arrays.asList(ImplicitSnitch.DISK, ImplicitSnitch.CORES); - - private final Map> policies; - private final List clusterPolicy; - private final List clusterPreferences; - private final List> params; - private final List perReplicaAttributes; - private final int zkVersion; - /** - * True if cluster policy, preferences and custom policies are all non-existent - */ - private final boolean empty; - /** - * True if cluster preferences was originally empty, false otherwise. It is used to figure out if - * the current preferences were implicitly added or not. - */ - private final boolean emptyPreferences; - - /** - * True if cluster policy was originally empty, false otherwise. It is used to figure out if the - * current policy was implicitly added or not. - */ - final boolean emptyClusterPolicy; - - public Policy() { - this(Collections.emptyMap()); - } - - public Policy(Map jsonMap) { - this(jsonMap, 0); - } - @SuppressWarnings("unchecked") - public Policy(Map jsonMap, int version) { - this.empty = jsonMap.get(CLUSTER_PREFERENCES) == null && jsonMap.get(CLUSTER_POLICY) == null && jsonMap.get(POLICIES) == null; - this.zkVersion = version; - int[] idx = new int[1]; - List initialClusterPreferences = ((List>) jsonMap.getOrDefault(CLUSTER_PREFERENCES, emptyList())).stream() - .map(m -> new Preference(m, idx[0]++)) - .collect(toList()); - for (int i = 0; i < initialClusterPreferences.size() - 1; i++) { - Preference preference = initialClusterPreferences.get(i); - preference.next = initialClusterPreferences.get(i + 1); - } - emptyPreferences = initialClusterPreferences.isEmpty(); - if (emptyPreferences) { - initialClusterPreferences.addAll(DEFAULT_PREFERENCES); - } - this.clusterPreferences = Collections.unmodifiableList(initialClusterPreferences); - final SortedSet paramsOfInterest = new TreeSet<>(DEFAULT_PARAMS_OF_INTEREST); - clusterPreferences.forEach(preference -> paramsOfInterest.add(preference.name.toString())); - List newParams = new ArrayList<>(paramsOfInterest); - - // if json map has CLUSTER_POLICY and even if its size is 0, we consider it as a custom cluster policy - // and do not add the implicit policy clauses - emptyClusterPolicy = !jsonMap.containsKey(CLUSTER_POLICY); - - clusterPolicy = ((List>) jsonMap.getOrDefault(CLUSTER_POLICY, DEFAULT_CLUSTER_POLICY_JSON)).stream() - .map(Clause::create) - .filter(clause -> { - clause.addTags(newParams); - return true; - }) - .collect(collectingAndThen(toList(), Collections::unmodifiableList)); - - for (String newParam : new ArrayList<>(newParams)) { - Type t = VariableBase.getTagType(newParam); - if(t != null && !t.associatedPerNodeValues.isEmpty()) { - for (String s : t.associatedPerNodeValues) { - if(!newParams.contains(s)) newParams.add(s); - } - } - } - - this.policies = Collections.unmodifiableMap( - clausesFromMap((Map>>) jsonMap.getOrDefault(POLICIES, emptyMap()), newParams)); - List> params = newParams.stream() - .map(s -> new Pair<>(s, VariableBase.getTagType(s))) - .collect(toList()); - //let this be there always, there is no extra cost - params.add(new Pair<>(WITH_COLLECTION.tagName, WITH_COLLECTION)); - this.params = Collections.unmodifiableList(params); - perReplicaAttributes = readPerReplicaAttrs(); - } - - private List readPerReplicaAttrs() { - return this.params.stream() - .map(s -> s.second().perReplicaValue) - .filter(Objects::nonNull) - .collect(Collectors.toList()); - } - - private Policy(Map> policies, List clusterPolicy, List clusterPreferences, int version) { - this.empty = policies == null && clusterPolicy == null && clusterPreferences == null; - this.zkVersion = version; - this.policies = policies != null ? Collections.unmodifiableMap(policies) : Collections.emptyMap(); - this.emptyClusterPolicy = clusterPolicy == null; - this.clusterPolicy = clusterPolicy != null ? Collections.unmodifiableList(clusterPolicy) : Collections.emptyList(); - this.emptyPreferences = clusterPreferences == null; - this.clusterPreferences = emptyPreferences ? DEFAULT_PREFERENCES : Collections.unmodifiableList(clusterPreferences); - this.params = Collections.unmodifiableList( - buildParams(this.clusterPreferences, this.clusterPolicy, this.policies).stream() - .map(s -> new Pair<>(s, VariableBase.getTagType(s))) - .collect(toList()) - ); - perReplicaAttributes = readPerReplicaAttrs(); - } - - private List buildParams(List preferences, List policy, Map> policies) { - final SortedSet paramsOfInterest = new TreeSet<>(); - preferences.forEach(p -> { - if (paramsOfInterest.contains(p.name.name())) { - throw new RuntimeException(p.name + " is repeated"); - } - paramsOfInterest.add(p.name.toString()); - }); - List newParams = new ArrayList<>(paramsOfInterest); - policy.forEach(c -> c.addTags(newParams)); - policies.values().forEach(clauses -> clauses.forEach(c -> c.addTags(newParams))); - return newParams; - } - - public Policy withPolicies(Map> policies) { - return new Policy(policies, clusterPolicy, clusterPreferences, 0); - } - - public Policy withClusterPreferences(List clusterPreferences) { - return new Policy(policies, clusterPolicy, clusterPreferences, 0); - } - - public Policy withClusterPolicy(List clusterPolicy) { - return new Policy(policies, clusterPolicy, clusterPreferences, 0); - } - - public Policy withParams(List params) { - return new Policy(policies, clusterPolicy, clusterPreferences, 0); - } - - public List getClusterPolicy() { - return Collections.unmodifiableList(clusterPolicy); - } - - public List getClusterPreferences() { - return Collections.unmodifiableList(clusterPreferences); - } - - @Override - public void writeMap(EntryWriter ew) throws IOException { - // if we were initially empty then we don't want to persist any implicitly added - // policy or preferences - if (empty) return; - - if (!policies.isEmpty()) { - ew.put(POLICIES, (MapWriter) ew1 -> { - for (Map.Entry> e : policies.entrySet()) { - ew1.put(e.getKey(), e.getValue()); - } - }); - } - if (!emptyPreferences && !clusterPreferences.isEmpty()) { - ew.put(CLUSTER_PREFERENCES, (IteratorWriter) iw -> { - for (Preference p : clusterPreferences) iw.add(p); - }); - } - if (!emptyClusterPolicy) { - ew.put(CLUSTER_POLICY, (IteratorWriter) iw -> { - for (Clause c : clusterPolicy) { - iw.add(c); - } - }); - } - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - - Policy policy = (Policy) o; - - if (!getPolicies().equals(policy.getPolicies())) return false; - if (!getClusterPolicy().equals(policy.getClusterPolicy())) return false; - return getClusterPreferences().equals(policy.getClusterPreferences()); - } - - @Override - public int hashCode() { return Objects.hash(getPolicies()); } - - public static Map> clausesFromMap(Map>> map, List newParams) { - Map> newPolicies = new HashMap<>(); - map.forEach((s, l1) -> - newPolicies.put(s, l1.stream() - .map(Clause::create) - .filter(clause -> { - if (!clause.isPerCollectiontag()) - throw new RuntimeException(clause.getGlobalTag().name + " is only allowed in 'cluster-policy'"); - clause.addTags(newParams); - return true; - }) - .sorted() - .collect(collectingAndThen(toList(), Collections::unmodifiableList)))); - return newPolicies; - } - - static void setApproxValuesAndSortNodes(List clusterPreferences, List matrix) { - List matrixCopy = new ArrayList<>(matrix); - List deadNodes = null; - Iterator it =matrix.iterator(); - while (it.hasNext()){ - Row row = it.next(); - if(!row.isLive){ - if(deadNodes == null) deadNodes = new ArrayList<>(); - deadNodes.add(row); - it.remove(); - } - } - - if (!clusterPreferences.isEmpty()) { - //this is to set the approximate value according to the precision - ArrayList tmpMatrix = new ArrayList<>(matrix); - Row[] lastComparison = new Row[2]; - for (Preference p : clusterPreferences) { - try { - tmpMatrix.sort((r1, r2) -> { - lastComparison[0] = r1; - lastComparison[1] = r2; - return p.compare(r1, r2, false); - }); - } catch (Exception e) { - try { - @SuppressWarnings({"rawtypes"}) - Map m = Collections.singletonMap("diagnostics", (MapWriter) ew -> { - PolicyHelper.writeNodes(ew, matrixCopy); - ew.put("config", matrix.get(0).session.getPolicy()); - }); - StringWriter exc = new StringWriter(); - e.printStackTrace(new PrintWriter(exc)); - log.error("Exception during matrix sorting! prefs = {}, recent r1 = {}, r2 = {}, matrix = {}, exception={}", - clusterPreferences, - lastComparison[0].node, - lastComparison[1].node, - Utils.writeJson(m, new StringWriter(), true).toString(), exc.toString()); // logOk - } catch (IOException e1) { - // - } - throw new RuntimeException(e.getMessage()); - } - p.setApproxVal(tmpMatrix); - } - // the tmpMatrix was needed only to set the approximate values, now we sort the real matrix - // recursing through each preference - matrix.sort((Row r1, Row r2) -> { - int result = clusterPreferences.get(0).compare(r1, r2, true); - if (result == 0) result = clusterPreferences.get(0).compare(r1, r2, false); - return result; - }); - - if(deadNodes != null){ - for (Row deadNode : deadNodes) { - matrix.add(0, deadNode); - } - } - } - } - - /** - * Insert the collection name into the clauses where collection is not specified - */ - static List insertColl(String coll, Collection conditions) { - return conditions.stream() - .filter(Clause::isPerCollectiontag) - .map(clause -> { - Map copy = new LinkedHashMap<>(clause.original); - if (!copy.containsKey("collection")) { - copy.put("collection", coll); - copy.put(Clause.class.getName(), clause); - } - return Clause.create(copy); - }) - .filter(it -> (it.getCollection().isPass(coll))) - .collect(Collectors.toList()); - - } - - public Session createSession(SolrCloudManager cloudManager) { - return createSession(cloudManager, null); - } - - public Session createSession(SolrCloudManager cloudManager, Transaction tx) { - return new Session(cloudManager, this, tx); - } - - public enum SortParam { - freedisk(0, Integer.MAX_VALUE), cores(0, Integer.MAX_VALUE), heapUsage(0, Integer.MAX_VALUE), sysLoadAvg(0, 100); - - public final int min, max; - - SortParam(int min, int max) { - this.min = min; - this.max = max; - } - - static SortParam get(String m) { - for (SortParam p : values()) if (p.name().equals(m)) return p; - throw new RuntimeException(StrUtils.formatString("Invalid sort {0} Sort must be on one of these {1}", m, Arrays.asList(values()))); - } - } - - enum Sort { - maximize(1), minimize(-1); - final int sortval; - - Sort(int i) { - sortval = i; - } - - static Sort get(Map m) { - if (m.containsKey(maximize.name()) && m.containsKey(minimize.name())) { - throw new RuntimeException("Cannot have both 'maximize' and 'minimize'"); - } - if (m.containsKey(maximize.name())) return maximize; - if (m.containsKey(minimize.name())) return minimize; - throw new RuntimeException("must have either 'maximize' or 'minimize'"); - } - } - - public static List mergePolicies(String coll, - List collPolicy, - List globalPolicy) { - - List merged = insertColl(coll, collPolicy); - List global = insertColl(coll, globalPolicy); - merged.addAll(global.stream() - .filter(clusterPolicyClause -> merged.stream().noneMatch(perCollPolicy -> perCollPolicy.doesOverride(clusterPolicyClause))) - .collect(Collectors.toList())); - return merged; - } - - static class Transaction { - private final Policy policy; - private boolean open = false; - private Session firstSession; - private Session currentSession; - - - public Transaction(Policy config) { - this.policy = config; - - } - - public Session open(SolrCloudManager cloudManager) { - firstSession = currentSession = policy.createSession(cloudManager, Transaction.this); - open = true; - return firstSession; - } - - - public boolean isOpen() { - return open; - } - - List close() { - if (!open) throw new RuntimeException("Already closed"); - open = false; - return currentSession.getViolations(); - } - - public Session getCurrentSession() { - return currentSession; - } - - void updateSession(Session session) { - currentSession = session; - } - } - - private static final Map> ops = new HashMap<>(); - - static { - ops.put(CollectionAction.ADDREPLICA, AddReplicaSuggester::new); - ops.put(CollectionAction.DELETEREPLICA, DeleteReplicaSuggester::new); - ops.put(CollectionAction.DELETENODE, DeleteNodeSuggester::new); - ops.put(CollectionAction.MOVEREPLICA, MoveReplicaSuggester::new); - ops.put(CollectionAction.SPLITSHARD, SplitShardSuggester::new); - ops.put(CollectionAction.MERGESHARDS, () -> new UnsupportedSuggester(CollectionAction.MERGESHARDS)); - ops.put(CollectionAction.NONE, () -> new UnsupportedSuggester(CollectionAction.NONE)); - } - - public Map> getPolicies() { - return policies; - } - - public List> getParams() { - return Collections.unmodifiableList(params); - } - - public List getParamNames() { - return params.stream().map(Pair::first).collect(toList()); - } - - public List getPerReplicaAttributes() { - return Collections.unmodifiableList(perReplicaAttributes); - } - - public int getZkVersion() { - return zkVersion; - } - - /** - * Compares two {@link Row} loads according to a policy. - * - * @param r1 the first {@link Row} to compare - * @param r2 the second {@link Row} to compare - * @return the value {@code 0} if r1 and r2 are equally loaded - * a value {@code -1} if r1 is more loaded than r2 - * a value {@code 1} if r1 is less loaded than r2 - */ - static int compareRows(Row r1, Row r2, Policy policy) { - return policy.clusterPreferences.get(0).compare(r1, r2, true); - } - - @Override - public String toString() { - return Utils.toJSONString(this); - } - - public boolean isEmpty() { - return empty; - } - - /** - * @return true if no preferences were specified by the user, false otherwise - */ - public boolean hasEmptyPreferences() { - return emptyPreferences; - } - - /** - * @return true if no cluster policy was specified by the user, false otherwise - */ - public boolean hasEmptyClusterPolicy() { - return emptyClusterPolicy; - } - - /*This stores the logical state of the system, given a policy and - * a cluster state. - * - */ - public static class Session implements MapWriter { - final List nodes; - final SolrCloudManager cloudManager; - final List matrix; - final NodeStateProvider nodeStateProvider; - final Set collections; - final Policy policy; - List expandedClauses; - List violations = new ArrayList<>(); - Transaction transaction; - - - /** - * This constructor creates a Session from the current Zookeeper collection, replica and node states. - */ - Session(SolrCloudManager cloudManager, Policy policy, Transaction transaction) { - collections = new HashSet<>(); - this.transaction = transaction; - this.policy = policy; - ClusterState state = null; - this.nodeStateProvider = cloudManager.getNodeStateProvider(); - try { - state = cloudManager.getClusterStateProvider().getClusterState(); - log.trace("-- session created with cluster state: {}", state); - } catch (Exception e) { - log.trace("-- session created, can't obtain cluster state", e); - } - this.nodes = new ArrayList<>(cloudManager.getClusterStateProvider().getLiveNodes()); - this.cloudManager = cloudManager; - for (String node : nodes) { - collections.addAll(nodeStateProvider.getReplicaInfo(node, Collections.emptyList()).keySet()); - } - - expandedClauses = policy.getClusterPolicy().stream() - .filter(clause -> !clause.isPerCollectiontag()) - .collect(Collectors.toList()); - - if (nodes.size() > 0) { - //if any collection has 'withCollection' irrespective of the node, the NodeStateProvider returns a map value - Map vals = nodeStateProvider.getNodeValues(nodes.get(0), Collections.singleton("withCollection")); - if (!vals.isEmpty() && vals.get("withCollection") != null) { - @SuppressWarnings({"unchecked"}) - Map withCollMap = (Map) vals.get("withCollection"); - if (!withCollMap.isEmpty()) { - @SuppressWarnings({"unchecked"}) - Clause withCollClause = new Clause((Map)Utils.fromJSONString("{withCollection:'*' , node: '#ANY'}") , - new Condition(NODE.tagName, "#ANY", Operand.EQUAL, null, null), - new Condition(WITH_COLLECTION.tagName,"*" , Operand.EQUAL, null, null), true, null, false - ); - expandedClauses.add(withCollClause); - } - } - } - - ClusterStateProvider stateProvider = cloudManager.getClusterStateProvider(); - for (String c : collections) { - addClausesForCollection(policy, expandedClauses, stateProvider, c); - } - - Collections.sort(expandedClauses); - - matrix = new ArrayList<>(nodes.size()); - for (String node : nodes) matrix.add(new Row(node, policy.getParams(), policy.getPerReplicaAttributes(), this)); - applyRules(); - } - - /** - * Creates a new Session and updates the Rows in the internal matrix to reference this session. - */ - private Session(List nodes, SolrCloudManager cloudManager, - List matrix, Set collections, List expandedClauses, - NodeStateProvider nodeStateProvider, Policy policy, Transaction transaction) { - this.transaction = transaction; - this.policy = policy; - this.nodes = nodes; - this.cloudManager = cloudManager; - this.collections = collections; - this.matrix = matrix; - this.expandedClauses = expandedClauses; - this.nodeStateProvider = nodeStateProvider; - for (Row row : matrix) row.session = this; - } - - /** - * Given a session (this one), creates a new one for placement simulations that retains all the relevant information, - * whether or not that info already made it to Zookeeper. - */ - public Session cloneToNewSession(SolrCloudManager cloudManager) { - NodeStateProvider nodeStateProvider = cloudManager.getNodeStateProvider(); - ClusterStateProvider clusterStateProvider = cloudManager.getClusterStateProvider(); - - List nodes = new ArrayList<>(clusterStateProvider.getLiveNodes()); - - // Copy all collections from old session, even those not yet in ZK state - Set collections = new HashSet<>(this.collections); - - // (shallow) copy the expanded clauses - List expandedClauses = new ArrayList<>(this.expandedClauses); - - List matrix = new ArrayList<>(nodes.size()); - Map copyNodes = new HashMap<>(); - for (Row oldRow: this.matrix) { - copyNodes.put(oldRow.node, oldRow.copy()); - } - for (String node : nodes) { - // Do we have a row for that node in this session? If yes, reuse without trying to fetch from cluster state (latest changes might not be there) - Row newRow = copyNodes.get(node); - if (newRow == null) { - // Dealing with a node that doesn't exist in this Session. Need to create related data from scratch. - // We pass null for the Session in purpose. The current (this) session in not the correct one for this Row. - // The correct session will be set when we build the new Session instance at the end of this method. - newRow = new Row(node, this.policy.getParams(), this.policy.getPerReplicaAttributes(), null, nodeStateProvider, cloudManager); - // Get info for collections on that node - Set collectionsOnNewNode = nodeStateProvider.getReplicaInfo(node, Collections.emptyList()).keySet(); - collections.addAll(collectionsOnNewNode); - - // Adjust policies to take into account new collections - for (String collection : collectionsOnNewNode) { - // We pass this.policy but it is not modified so will not impact this session being cloned - addClausesForCollection(this.policy, expandedClauses, clusterStateProvider, collection); - } - } - matrix.add(newRow); - } - - if (nodes.size() > 0) { - //if any collection has 'withCollection' irrespective of the node, the NodeStateProvider returns a map value - Map vals = nodeStateProvider.getNodeValues(nodes.get(0), Collections.singleton("withCollection")); - if (!vals.isEmpty() && vals.get("withCollection") != null) { - @SuppressWarnings({"unchecked"}) - Map withCollMap = (Map) vals.get("withCollection"); - if (!withCollMap.isEmpty()) { - @SuppressWarnings({"unchecked"}) - Clause withCollClause = new Clause((Map)Utils.fromJSONString("{withCollection:'*' , node: '#ANY'}") , - new Condition(NODE.tagName, "#ANY", Operand.EQUAL, null, null), - new Condition(WITH_COLLECTION.tagName,"*" , Operand.EQUAL, null, null), true, null, false - ); - expandedClauses.add(withCollClause); - } - } - } - - Collections.sort(expandedClauses); - - Session newSession = new Session(nodes, cloudManager, matrix, collections, expandedClauses, - nodeStateProvider, this.policy, this.transaction); - newSession.applyRules(); - - return newSession; - } - - void addClausesForCollection(ClusterStateProvider stateProvider, String collection) { - addClausesForCollection(policy, expandedClauses, stateProvider, collection); - } - - public static void addClausesForCollection(Policy policy, List clauses, ClusterStateProvider stateProvider, String collectionName) { - String p = stateProvider.getPolicyNameByCollection(collectionName); - if (p != null) { - List perCollPolicy = policy.getPolicies().get(p); - if (perCollPolicy == null) { - return; - } - } - clauses.addAll(mergePolicies(collectionName, policy.getPolicies().getOrDefault(p, emptyList()), policy.getClusterPolicy())); - } - - Session copy() { - return new Session(nodes, cloudManager, getMatrixCopy(), new HashSet<>(), expandedClauses, nodeStateProvider, policy, transaction); - } - - public Row getNode(String node) { - for (Row row : matrix) if (row.node.equals(node)) return row; - return null; - } - - List getMatrixCopy() { - return matrix.stream() - .map(row -> row.copy()) - .collect(Collectors.toList()); - } - - public Policy getPolicy() { - return policy; - - } - - /** - * Apply the preferences and conditions - */ - void applyRules() { - sortNodes(); - - for (Clause clause : expandedClauses) { - List errs = clause.test(this, null); - violations.addAll(errs); - } - } - - void sortNodes() { - setApproxValuesAndSortNodes(policy.getClusterPreferences(), matrix); - } - - - public List getViolations() { - return violations; - } - - public Suggester getSuggester(CollectionAction action) { - Suggester op = ops.get(action).get(); - if (op == null) throw new UnsupportedOperationException(action.toString() + "is not supported"); - op._init(this); - return op; - } - - @Override - public void writeMap(EntryWriter ew) throws IOException { - for (Row row : matrix) { - ew.put(row.node, row); - } - } - - @Override - public String toString() { - return Utils.toJSONString(toMap(new LinkedHashMap<>())); - } - - public List getSortedNodes() { - return Collections.unmodifiableList(matrix); - } - - public NodeStateProvider getNodeStateProvider() { - return nodeStateProvider; - } - - public int indexOf(String node) { - for (int i = 0; i < matrix.size(); i++) if (matrix.get(i).node.equals(node)) return i; - throw new RuntimeException("NO such node found " + node); - } - } -} diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/PolicyHelper.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/PolicyHelper.java deleted file mode 100644 index 3f3a300c0de..00000000000 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/PolicyHelper.java +++ /dev/null @@ -1,733 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.client.solrj.cloud.autoscaling; - - -import java.io.IOException; -import java.io.StringWriter; -import java.lang.invoke.MethodHandles; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.EnumMap; -import java.util.HashMap; -import java.util.HashSet; -import java.util.IdentityHashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.BiPredicate; - -import org.apache.solr.client.solrj.SolrRequest; -import org.apache.solr.client.solrj.cloud.DistribStateManager; -import org.apache.solr.client.solrj.cloud.SolrCloudManager; -import org.apache.solr.client.solrj.cloud.autoscaling.Suggester.Hint; -import org.apache.solr.client.solrj.impl.ClusterStateProvider; -import org.apache.solr.common.ConditionalMapWriter; -import org.apache.solr.common.IteratorWriter; -import org.apache.solr.common.MapWriter; -import org.apache.solr.common.SolrException; -import org.apache.solr.common.cloud.DocCollection; -import org.apache.solr.common.cloud.Replica; -import org.apache.solr.common.cloud.ReplicaPosition; -import org.apache.solr.common.params.SolrParams; -import org.apache.solr.common.util.Pair; -import org.apache.solr.common.util.TimeSource; -import org.apache.solr.common.util.Utils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import static java.util.Collections.emptyMap; -import static java.util.Collections.singletonList; -import static java.util.concurrent.TimeUnit.MILLISECONDS; -import static org.apache.solr.client.solrj.cloud.autoscaling.Suggestion.Type.improvement; -import static org.apache.solr.client.solrj.cloud.autoscaling.Suggestion.Type.repair; -import static org.apache.solr.client.solrj.cloud.autoscaling.Suggestion.Type.unresolved_violation; -import static org.apache.solr.client.solrj.cloud.autoscaling.Suggestion.Type.violation; -import static org.apache.solr.client.solrj.cloud.autoscaling.Variable.Type.FREEDISK; -import static org.apache.solr.common.ConditionalMapWriter.dedupeKeyPredicate; -import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDREPLICA; -import static org.apache.solr.common.params.CollectionParams.CollectionAction.MOVEREPLICA; -import static org.apache.solr.common.params.CoreAdminParams.NODE; -import static org.apache.solr.common.util.Utils.handleExp; -import static org.apache.solr.common.util.Utils.time; -import static org.apache.solr.common.util.Utils.timeElapsed; - -public class PolicyHelper { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - private static final String POLICY_MAPPING_KEY = "PolicyHelper.policyMapping"; - - @SuppressWarnings({"unchecked"}) - private static ThreadLocal> getPolicyMapping(SolrCloudManager cloudManager) { - return (ThreadLocal>) cloudManager.getObjectCache() - .computeIfAbsent(POLICY_MAPPING_KEY, k -> new ThreadLocal<>()); - } - - public static List getReplicaLocations(String collName, AutoScalingConfig autoScalingConfig, - SolrCloudManager cloudManager, - Map optionalPolicyMapping, - List shardNames, - int nrtReplicas, - int tlogReplicas, - int pullReplicas, - List nodesList) { - List positions = new ArrayList<>(); - ThreadLocal> policyMapping = getPolicyMapping(cloudManager); - ClusterStateProvider stateProvider = new DelegatingClusterStateProvider(cloudManager.getClusterStateProvider()) { - @Override - public String getPolicyNameByCollection(String coll) { - return policyMapping.get() != null && policyMapping.get().containsKey(coll) ? - optionalPolicyMapping.get(coll) : - delegate.getPolicyNameByCollection(coll); - } - }; - SolrCloudManager delegatingManager = new DelegatingCloudManager(cloudManager) { - @Override - public ClusterStateProvider getClusterStateProvider() { - return stateProvider; - } - - @Override - public DistribStateManager getDistribStateManager() { - if (autoScalingConfig != null) { - return new DelegatingDistribStateManager(null) { - @Override - public AutoScalingConfig getAutoScalingConfig() { - return autoScalingConfig; - } - }; - } else { - return super.getDistribStateManager(); - } - } - }; - - policyMapping.set(optionalPolicyMapping); - SessionWrapper sessionWrapper = null; - - try { - try { - SESSION_WRAPPPER_REF.set(sessionWrapper = getSession(delegatingManager)); - } catch (Exception e) { - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "unable to get autoscaling policy session", e); - } - - Policy.Session origSession = sessionWrapper.session; - // new session needs to be created to avoid side-effects from per-collection policies - // TODO: refactor so cluster state cache is separate from storage of policies to avoid per cluster vs per collection interactions - // Need a Session that has all previous history of the original session, NOT filtered by what's present or not in Zookeeper - // (as does constructor Session(SolrCloudManager, Policy, Transaction)). - Policy.Session newSession = origSession.cloneToNewSession(delegatingManager); - - Map diskSpaceReqd = new HashMap<>(); - try { - DocCollection coll = cloudManager.getClusterStateProvider().getCollection(collName); - if (coll != null) { - for (String shardName : shardNames) { - Replica ldr = coll.getLeader(shardName); - if (ldr != null && cloudManager.getClusterStateProvider().getLiveNodes().contains(ldr.getNodeName())) { - Map>> details = cloudManager.getNodeStateProvider().getReplicaInfo(ldr.getNodeName(), - Collections.singleton(FREEDISK.perReplicaValue)); - Replica replicaInfo = details.getOrDefault(collName, emptyMap()).getOrDefault(shardName, singletonList(null)).get(0); - if (replicaInfo != null) { - Object idxSz = replicaInfo.getProperties().get(FREEDISK.perReplicaValue); - if (idxSz != null) { - diskSpaceReqd.put(shardName, 1.5 * (Double) Variable.Type.FREEDISK.validate(null, idxSz, false)); - } - } - } - - } - } - } catch (IOException e) { - log.warn("Exception while reading disk free metric values for nodes to be used for collection: {}", collName, e); - } - - - Map typeVsCount = new EnumMap<>(Replica.Type.class); - typeVsCount.put(Replica.Type.NRT, nrtReplicas); - typeVsCount.put(Replica.Type.TLOG, tlogReplicas); - typeVsCount.put(Replica.Type.PULL, pullReplicas); - for (String shardName : shardNames) { - int idx = 0; - for (Map.Entry e : typeVsCount.entrySet()) { - for (int i = 0; i < e.getValue(); i++) { - Suggester suggester = newSession.getSuggester(ADDREPLICA) - .hint(Hint.REPLICATYPE, e.getKey()) - .hint(Hint.COLL_SHARD, new Pair<>(collName, shardName)); - if (nodesList != null) { - for (String nodeName : nodesList) { - suggester = suggester.hint(Hint.TARGET_NODE, nodeName); - } - } - if (diskSpaceReqd.get(shardName) != null) { - suggester.hint(Hint.MINFREEDISK, diskSpaceReqd.get(shardName)); - } - @SuppressWarnings({"rawtypes"}) - SolrRequest op = suggester.getSuggestion(); - if (op == null) { - String errorId = "AutoScaling.error.diagnostics." + System.nanoTime(); - Policy.Session sessionCopy = suggester.session; - log.error("errorId : {} {}", errorId - , handleExp(log, "", () -> Utils.writeJson(getDiagnostics(sessionCopy), new StringWriter(), true).toString())); // logOk - - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, " No node can satisfy the rules " + - Utils.toJSONString(Utils.getDeepCopy(newSession.expandedClauses, 4, true) + " More details from logs in node : " - + Utils.getMDCNode() + ", errorId : " + errorId)); - } - newSession = suggester.getSession(); - positions.add(new ReplicaPosition(shardName, ++idx, e.getKey(), op.getParams().get(NODE))); - } - } - } - - // We're happy with the updated session based on the original one, so let's update what the wrapper would hand - // to the next computation that wants a session. - sessionWrapper.update(newSession); - } finally { - policyMapping.remove(); - // We mark the wrapper (and its session) as being available to others. - if (sessionWrapper != null) { - sessionWrapper.returnSession(); - } - } - return positions; - } - - - public static final int SESSION_EXPIRY = 180; // 3 minutes - - public static MapWriter getDiagnostics(Policy policy, SolrCloudManager cloudManager) { - Policy.Session session = policy.createSession(cloudManager); - return getDiagnostics(session); - } - - public static MapWriter getDiagnostics(Policy.Session session) { - List sorted = session.getSortedNodes(); - return ew -> { - writeNodes(ew, sorted); - ew.put("liveNodes", session.cloudManager.getClusterStateProvider().getLiveNodes()) - .put("violations", session.getViolations()) - .put("config", session.getPolicy()); - }; - } - - static void writeNodes(MapWriter.EntryWriter ew, List sorted) throws IOException { - Set alreadyWritten = new HashSet<>(); - BiPredicate p = dedupeKeyPredicate(alreadyWritten) - .and(ConditionalMapWriter.NON_NULL_VAL) - .and((s, o) -> !(o instanceof Map) || !((Map) o).isEmpty()); - ew.put("sortedNodes", (IteratorWriter) iw -> { - for (Row row : sorted) { - iw.add((MapWriter) ew1 -> { - alreadyWritten.clear(); - ew1.put("node", row.node, p). - put("isLive", row.isLive, p); - for (Cell cell : row.getCells()) - ew1.put(cell.name, cell.val, p); - ew1.put("replicas", row.collectionVsShardVsReplicas); - }); - } - }); - } - public static List getSuggestions(AutoScalingConfig autoScalingConf, - SolrCloudManager cloudManager, SolrParams params) { - return getSuggestions(autoScalingConf, cloudManager, 20, 10, params); - } - - public static List getSuggestions(AutoScalingConfig autoScalingConf, - SolrCloudManager cloudManager) { - return getSuggestions(autoScalingConf, cloudManager, 20, 10, null); - } - - - public static List getSuggestions(AutoScalingConfig autoScalingConf, - SolrCloudManager cloudManager, int max, int timeoutInSecs, SolrParams params) { - Policy policy = autoScalingConf.getPolicy(); - Suggestion.Ctx ctx = new Suggestion.Ctx(); - ctx.endTime = cloudManager.getTimeSource().getTimeNs() + TimeUnit.SECONDS.toNanos(timeoutInSecs); - ctx.max = max; - ctx.session = policy.createSession(cloudManager); - String[] t = params == null ? null : params.getParams("type"); - List types = t == null? Collections.emptyList(): Arrays.asList(t); - - if(types.isEmpty() || types.contains(violation.name())) { - List violations = ctx.session.getViolations(); - for (Violation violation : violations) { - violation.getClause().getThirdTag().varType.getSuggestions(ctx.setViolation(violation)); - ctx.violation = null; - } - - for (Violation current : ctx.session.getViolations()) { - for (Violation old : violations) { - if (!ctx.needMore()) return ctx.getSuggestions(); - if (current.equals(old)) { - //could not be resolved - ctx.suggestions.add(new Suggester.SuggestionInfo(current, null, unresolved_violation)); - break; - } - } - } - } - - if(types.isEmpty() || types.contains(repair.name())) { - if (ctx.needMore()) { - try { - addMissingReplicas(cloudManager, ctx); - } catch (IOException e) { - log.error("Unable to fetch cluster state", e); - } - } - } - - if(types.isEmpty() || types.contains(improvement.name())) { - if (ctx.needMore()) { - suggestOptimizations(ctx, Math.min(ctx.max - ctx.getSuggestions().size(), 10)); - } - } - return ctx.getSuggestions(); - } - - private static void addMissingReplicas(SolrCloudManager cloudManager, Suggestion.Ctx ctx) throws IOException { - cloudManager.getClusterStateProvider().getClusterState().forEachCollection(coll -> coll.forEach(slice -> { - if (!ctx.needMore()) return; - ReplicaCount replicaCount = new ReplicaCount(); - slice.forEach(replica -> { - if (replica.getState() == Replica.State.ACTIVE || replica.getState() == Replica.State.RECOVERING) { - replicaCount.increment(replica.getType()); - } - }); - addMissingReplicas(replicaCount, coll, slice.getName(), Replica.Type.NRT, ctx); - addMissingReplicas(replicaCount, coll, slice.getName(), Replica.Type.PULL, ctx); - addMissingReplicas(replicaCount, coll, slice.getName(), Replica.Type.TLOG, ctx); - } - )); - } - - @SuppressWarnings({"unchecked"}) - private static void addMissingReplicas(ReplicaCount count, DocCollection coll, String shard, Replica.Type type, Suggestion.Ctx ctx) { - int delta = count.delta(coll.getExpectedReplicaCount(type, 0), type); - for (; ; ) { - if (!ctx.needMore()) return; - if (delta >= 0) break; - @SuppressWarnings({"rawtypes"}) - SolrRequest suggestion = ctx.addSuggestion( - ctx.session.getSuggester(ADDREPLICA) - .hint(Hint.REPLICATYPE, type) - .hint(Hint.COLL_SHARD, new Pair(coll.getName(), shard)), Suggestion.Type.repair); - if (suggestion == null) return; - delta++; - } - } - - - private static void suggestOptimizations(Suggestion.Ctx ctx, int count) { - int maxTotalSuggestions = ctx.getSuggestions().size() + count; - List matrix = ctx.session.matrix; - if (matrix.isEmpty()) return; - for (int i = 0; i < matrix.size(); i++) { - if (ctx.getSuggestions().size() >= maxTotalSuggestions || ctx.hasTimedOut()) break; - Row row = matrix.get(i); - Map> collVsShards = new HashMap<>(); - row.forEachReplica(ri -> collVsShards.computeIfAbsent(ri.getCollection(), s -> new HashSet<>()).add(ri.getShard())); - for (Map.Entry> e : collVsShards.entrySet()) { - e.setValue(FreeDiskVariable.getSortedShards(Collections.singletonList(row), e.getValue(), e.getKey())); - } - for (Map.Entry> e : collVsShards.entrySet()) { - if (!ctx.needMore()) return; - if (ctx.getSuggestions().size() >= maxTotalSuggestions || ctx.hasTimedOut()) break; - for (String shard : e.getValue()) { - Suggester suggester = ctx.session.getSuggester(MOVEREPLICA) - .hint(Hint.COLL_SHARD, new Pair<>(e.getKey(), shard)) - .hint(Hint.SRC_NODE, row.node); - ctx.addSuggestion(suggester, Suggestion.Type.improvement); - if (ctx.getSuggestions().size() >= maxTotalSuggestions) break; - } - } - } - } - - - /** - * Use this to dump the state of a system and to generate a testcase - */ - public static void logState(SolrCloudManager cloudManager, Suggester suggester) { - if (log.isTraceEnabled()) { - try { - if (log.isTraceEnabled()) { - log.trace("LOGSTATE: {}", - Utils.writeJson(loggingInfo(cloudManager.getDistribStateManager().getAutoScalingConfig().getPolicy(), cloudManager, suggester), - new StringWriter(), true)); - } - } catch (Exception e) { - throw new RuntimeException(e); - } - } - } - - - static MapWriter loggingInfo(Policy policy, SolrCloudManager cloudManager, Suggester suggester) { - return ew -> { - ew.put("diagnostics", getDiagnostics(policy, - cloudManager)); - if (suggester != null) { - ew.put("suggester", suggester); - } - }; - } - - public enum Status { - /** - * A command is actively using and modifying the session to compute placements - */ - COMPUTING, - /** - * A command is not done yet processing its changes but no longer updates or even uses the session - */ - EXECUTING - } - - /** - * This class stores sessions for sharing purposes. If a process requires a session to - * compute operations: - *
    - *
  1. see if there is an available non expired session in the cache,
    2. - *
  2. if yes, borrow it.
    3. - *
  3. if no, create a new one and borrow it.
    4. - *
  4. after computing (update) operations are done, {@link #returnSession(SessionWrapper)} back to the cache so it's - * again available for borrowing.
    5. - *
  5. after all borrowers are done computing then executing with the session, {@link #release(SessionWrapper)} it, - * which removes it from the cache.
    6. - *
- */ - static class SessionRef { - /** - * Lock protecting access to {@link #sessionWrapperSet} and to {@link #creationsInProgress} - */ - private final Object lockObj = new Object(); - - /** - * Sessions currently in use in {@link Status#COMPUTING} or {@link Status#EXECUTING} states. As soon as all - * uses of a session are over, that session is removed from this set. Sessions not actively in use are NOT kept around. - * - *

Access should only be done under the protection of {@link #lockObj}

- */ - private Set sessionWrapperSet = Collections.newSetFromMap(new IdentityHashMap<>()); - - - /** - * Number of sessions currently being created but not yet present in {@link #sessionWrapperSet}. - * - *

Access should only be done under the protection of {@link #lockObj}

- */ - private int creationsInProgress = 0; - - public SessionRef() { - } - - // used only by tests - boolean isEmpty() { - synchronized (lockObj) { - return sessionWrapperSet.isEmpty(); - } - } - - /** - * All operations suggested by the current session object - * is complete. Do not even cache anything - */ - private void release(SessionWrapper sessionWrapper) { - boolean present; - synchronized (lockObj) { - present = sessionWrapperSet.remove(sessionWrapper); - } - if (!present) { - log.warn("released session {} not found in session set", sessionWrapper.getCreateTime()); - } else { - if (log.isDebugEnabled()) { - TimeSource timeSource = sessionWrapper.session.cloudManager.getTimeSource(); - log.debug("final release, session {} lived a total of {}ms, ", sessionWrapper.getCreateTime(), - timeElapsed(timeSource, TimeUnit.MILLISECONDS.convert(sessionWrapper.getCreateTime(), - TimeUnit.NANOSECONDS), MILLISECONDS)); // logOk - } - } - } - - /** - * Computing is over for this session and it may contain a new session with new state - * The session can be used by others while the caller is performing operations - */ - private void returnSession(SessionWrapper sessionWrapper) { - boolean present; - synchronized (lockObj) { - sessionWrapper.status = Status.EXECUTING; - present = sessionWrapperSet.contains(sessionWrapper); - - // wake up single thread waiting for a session return (ok if not woken up, wait is short) - // Important to wake up a single one, otherwise of multiple waiting threads, all but one will immediately create new sessions - lockObj.notify(); - } - - // Logging - if (present) { - if (log.isDebugEnabled()) { - log.debug("returnSession {}", sessionWrapper.getCreateTime()); - } - } else { - log.warn("returning unknown session {} ", sessionWrapper.getCreateTime()); - } - } - - /** - *

Method returning an available session that can be used for {@link Status#COMPUTING}, either from the - * {@link #sessionWrapperSet} cache or by creating a new one. The status of the returned session is set to {@link Status#COMPUTING}.

- * - * Some waiting is done in two cases: - *
    - *
  • A candidate session is present in {@link #sessionWrapperSet} but is still {@link Status#COMPUTING}, a random wait - * is observed to see if the session gets freed to save a session creation and allow session reuse,
    • - *
  • It is necessary to create a new session but there are already sessions in the process of being created, a - * random wait is observed (if no waiting already occurred waiting for a session to become free) before creation - * takes place, just in case one of the created sessions got used then {@link #returnSession(SessionWrapper)} in the meantime.
    • - *
- * - * The random wait prevents the "thundering herd" effect when all threads needing a session at the same time create a new - * one even though some differentiated waits could have led to better reuse and less session creations. - * - * @param allowWait usually true except in tests that know there's no point in waiting because nothing - * will happen... - */ - public SessionWrapper get(SolrCloudManager cloudManager, boolean allowWait) throws IOException, InterruptedException { - TimeSource timeSource = cloudManager.getTimeSource(); - long oldestUpdateTimeNs = TimeUnit.SECONDS.convert(timeSource.getTimeNs(), TimeUnit.NANOSECONDS) - SESSION_EXPIRY; - int zkVersion = cloudManager.getDistribStateManager().getAutoScalingConfig().getZkVersion(); - - synchronized (lockObj) { - SessionWrapper sw = getAvailableSession(zkVersion, oldestUpdateTimeNs); - - // Best case scenario: an available session - if (sw != null) { - if (log.isDebugEnabled()) { - log.debug("reusing session {}", sw.getCreateTime()); - } - return sw; - } - - // Wait for a while before deciding what to do if waiting could help... - if ((creationsInProgress != 0 || hasCandidateSession(zkVersion, oldestUpdateTimeNs)) && allowWait) { - // Either an existing session might be returned and become usable while we wait, or a session in the process of being - // created might finish creation, be used then returned and become usable. So we wait. - // wait 1 to 10 secs. Random to help spread wakeups. - long waitForMs = (long) (Math.random() * 9 * 1000) + 1000; - - if (log.isDebugEnabled()) { - log.debug("No sessions are available, all busy COMPUTING (or {} creations in progress). starting wait of {}ms", - creationsInProgress, waitForMs); - } - long waitStart = time(timeSource, MILLISECONDS); - try { - lockObj.wait(waitForMs); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - if (log.isDebugEnabled()) { - log.debug("out of waiting. wait of {}ms, actual time elapsed {}ms", waitForMs, timeElapsed(timeSource, waitStart, MILLISECONDS)); - } - - // We've waited, now we can either reuse immediately an available session, or immediately create a new one - sw = getAvailableSession(zkVersion, oldestUpdateTimeNs); - - // Second best case scenario: an available session - if (sw != null) { - if (log.isDebugEnabled()) { - log.debug("reusing session {} after wait", sw.getCreateTime()); - } - return sw; - } - } - - // We're going to create a new Session OUTSIDE of the critical section because session creation can take quite some time - creationsInProgress++; - } - - SessionWrapper newSessionWrapper = null; - try { - if (log.isDebugEnabled()) { - log.debug("Creating a new session"); - } - Policy.Session session = cloudManager.getDistribStateManager().getAutoScalingConfig().getPolicy().createSession(cloudManager); - newSessionWrapper = new SessionWrapper(session, this); - if (log.isDebugEnabled()) { - log.debug("New session created, {}", newSessionWrapper.getCreateTime()); - } - return newSessionWrapper; - } finally { - synchronized (lockObj) { - creationsInProgress--; - - if (newSessionWrapper != null) { - // Session created successfully - sessionWrapperSet.add(newSessionWrapper); - } - } - } - } - - /** - * Returns an available session from the cache (the best one once cache strategies are defined), or null if no session - * from the cache is available (i.e. all are still COMPUTING, are too old, wrong zk version or the cache is empty).

- * This method must be called while holding the monitor on {@link #lockObj}.

- * The method updates the session status to computing. - */ - private SessionWrapper getAvailableSession(int zkVersion, long oldestUpdateTimeNs) { - for (SessionWrapper sw : sessionWrapperSet) { - if (sw.status == Status.EXECUTING && sw.getLastUpdateTime() >= oldestUpdateTimeNs && sw.zkVersion == zkVersion) { - sw.status = Status.COMPUTING; - return sw; - } - } - return null; - } - - /** - * Returns true if there's a session in the cache that could be returned (if it was free). This is required to - * know if there's any point in waiting or if a new session should better be created right away. - */ - private boolean hasCandidateSession(int zkVersion, long oldestUpdateTimeNs) { - for (SessionWrapper sw : sessionWrapperSet) { - if (sw.getLastUpdateTime() >= oldestUpdateTimeNs && sw.zkVersion == zkVersion) { - return true; - } - } - return false; - } - } - - /** - * How to get a shared Policy Session - * 1) call {@link #getSession(SolrCloudManager)} - * 2) compute all suggestions - * 3) call {@link SessionWrapper#returnSession(Policy.Session)} - * 4) perform all suggestions - * 5) call {@link SessionWrapper#release()} - */ - public static SessionWrapper getSession(SolrCloudManager cloudManager) throws IOException, InterruptedException { - return getSession(cloudManager, true); - } - - static SessionWrapper getSession(SolrCloudManager cloudManager, boolean allowWait) throws IOException, InterruptedException { - SessionRef sessionRef = (SessionRef) cloudManager.getObjectCache().computeIfAbsent(SessionRef.class.getName(), s -> new SessionRef()); - return sessionRef.get(cloudManager, allowWait); - } - - /** - * Use this to get the last used session wrapper in this thread - * - * @param clear whether to unset the threadlocal or not - */ - public static SessionWrapper getLastSessionWrapper(boolean clear) { - SessionWrapper wrapper = SESSION_WRAPPPER_REF.get(); - if (clear) SESSION_WRAPPPER_REF.remove(); - return wrapper; - - } - - - static ThreadLocal SESSION_WRAPPPER_REF = new ThreadLocal<>(); - - - public static class SessionWrapper { - private final long createTime; - private long lastUpdateTime; - private Policy.Session session; - public Status status; - private final SessionRef ref; - /** - * Number of commands currently using the session in {@link Status#EXECUTING}. There is one additional command - * using the session and updating it if {@link #status} is {@link Status#COMPUTING} - */ - private final AtomicInteger refCount = new AtomicInteger(); - public final long zkVersion; - - /** - * Nanoseconds (since/to some arbitrary time) when the session got created. Also used in logs (only in logs!) to identify the session. - */ - public long getCreateTime() { - return createTime; - } - - public long getLastUpdateTime() { - return lastUpdateTime; - } - - public SessionWrapper(Policy.Session session, SessionRef ref) { - createTime = session.cloudManager.getTimeSource().getTimeNs(); - lastUpdateTime = createTime; - this.session = session; - this.status = Status.COMPUTING; // Created for being used, so COMPUTING right away - this.ref = ref; - this.zkVersion = session.getPolicy().getZkVersion(); - } - - public Policy.Session get() { - return session; - } - - public void update(Policy.Session session) { - // JMM multithreaded access issue on lastUpdateTime. - this.lastUpdateTime = session.cloudManager.getTimeSource().getTimeNs(); - this.session = session; - } - - public int getRefCount() { - return refCount.get(); - } - - /** - * return this for later use and update the session with the latest state - * ensure that this is done after computing the suggestions - */ - public void returnSession(Policy.Session session) { - if (this.status != Status.COMPUTING) { - log.warn("returning session {} not in state COMPUTING", this.getCreateTime()); - } - - this.update(session); - this.returnSession(); - } - - /** - * return this for later use without updating the internal Session for cases where it's easier to update separately - */ - public void returnSession() { - refCount.incrementAndGet(); - ref.returnSession(this); - } - - //all ops are executed now it can be destroyed - public void release() { - if (refCount.decrementAndGet() <= 0) { - ref.release(this); - } - } - } -} diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Preference.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Preference.java deleted file mode 100644 index 18f39a18d11..00000000000 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Preference.java +++ /dev/null @@ -1,157 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.client.solrj.cloud.autoscaling; - -import java.io.IOException; -import java.lang.invoke.MethodHandles; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Objects; - -import org.apache.solr.common.MapWriter; -import org.apache.solr.common.util.StrUtils; -import org.apache.solr.common.util.Utils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class Preference implements MapWriter { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - final Policy.SortParam name; - Integer precision; - final Policy.Sort sort; - Preference next; - final int idx; - @SuppressWarnings({"rawtypes"}) - private final Map original; - - public Preference(Map m) { - this(m, 0); - } - - public Preference(Map m, int idx) { - this.idx = idx; - this.original = Utils.getDeepCopy(m, 3); - sort = Policy.Sort.get(m); - name = Policy.SortParam.get(m.get(sort.name()).toString()); - Object p = m.getOrDefault("precision", 0); - precision = p instanceof Number ? ((Number) p).intValue() : Integer.parseInt(p.toString()); - if (precision < 0) { - throw new RuntimeException("precision must be a positive value "); - } - if (precision < name.min || precision > name.max) { - throw new RuntimeException(StrUtils.formatString("invalid precision value {0} , must lie between {1} and {2}", - precision, name.min, name.max)); - } - - } - - // there are 2 modes of compare. - // recursive, it uses the precision to tie & when there is a tie use the next preference to compare - // in non-recursive mode, precision is not taken into consideration and sort is done on actual value - int compare(Row r1, Row r2, boolean useApprox) { - Object o1 = useApprox ? r1.cells[idx].approxVal : r1.cells[idx].val; - Object o2 = useApprox ? r2.cells[idx].approxVal : r2.cells[idx].val; - int result = 0; - if (o1 instanceof Long && o2 instanceof Long) result = ((Long) o1).compareTo((Long) o2); - else if (o1 instanceof Double && o2 instanceof Double) { - result = compareWithTolerance((Double) o1, (Double) o2, useApprox ? 1f : 0.01f); - } else if (!o1.getClass().getName().equals(o2.getClass().getName())) { - throw new RuntimeException("Unable to compare " + o1 + " of type: " + o1.getClass().getName() + " from " + r1.cells[idx].toString() + " and " + o2 + " of type: " + o2.getClass().getName() + " from " + r2.cells[idx].toString()); - } - return result == 0 ? - (next == null ? 0 : - next.compare(r1, r2, useApprox)) : sort.sortval * result; - } - - static int compareWithTolerance(Double o1, Double o2, float percentage) { - if (percentage == 0) return o1.compareTo(o2); - if (o1.equals(o2)) return 0; - double delta = Math.abs(o1 - o2); - if ((100 * delta / o1) < percentage) return 0; - return o1.compareTo(o2); - } - - //sets the new value according to precision in val_ - void setApproxVal(List tmpMatrix) { - Object prevVal = null; - for (Row row : tmpMatrix) { - if (!row.isLive) { - continue; - } - if (prevVal == null) {//this is the first - prevVal = row.cells[idx].approxVal = row.cells[idx].val; - } else { - double prevD = ((Number) prevVal).doubleValue(); - double currD = ((Number) row.cells[idx].val).doubleValue(); - if (Math.abs(prevD - currD) >= precision) { - prevVal = row.cells[idx].approxVal = row.cells[idx].val; - } else { - prevVal = row.cells[idx].approxVal = prevVal; - } - } - } - } - - @Override - public void writeMap(EntryWriter ew) throws IOException { - for (Object o : original.entrySet()) { - @SuppressWarnings({"rawtypes"}) - Map.Entry e = (Map.Entry) o; - ew.put(String.valueOf(e.getKey()), e.getValue()); - } - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - - Preference that = (Preference) o; - - if (idx != that.idx) return false; - if (getName() != that.getName()) return false; - if (precision != null ? !precision.equals(that.precision) : that.precision != null) return false; - if (sort != that.sort) return false; - if (next != null ? !next.equals(that.next) : that.next != null) return false; - return original.equals(that.original); - } - - @Override - public int hashCode() { - return Objects.hash(getName(), precision, sort, idx); - } - - public Policy.SortParam getName() { - return name; - } - - @Override - public String toString() { - return Utils.toJSONString(this); - } - - /** - * @return an unmodifiable copy of the original map from which this object was constructed - */ - @SuppressWarnings({"unchecked", "rawtypes"}) - public Map getOriginal() { - return Collections.unmodifiableMap(original); - } -} diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/RangeVal.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/RangeVal.java deleted file mode 100644 index 11c5ab3f973..00000000000 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/RangeVal.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.client.solrj.cloud.autoscaling; - -import java.io.IOException; - -import org.apache.solr.common.MapWriter; - -class RangeVal implements MapWriter { - final Number min, max, actual; - - RangeVal(Number min, Number max, Number actual) { - this.min = min; - this.max = max; - this.actual = actual; - } - - public boolean match(Number testVal) { - if (testVal == null) return false; - return Double.compare(testVal.doubleValue(), min.doubleValue()) >= 0 && - Double.compare(testVal.doubleValue(), max.doubleValue()) <= 0; - } - - public Double realDelta(double v) { - if (actual != null) return v - actual.doubleValue(); - else return delta(v); - } - - public Double delta(double v) { - if (v >= max.doubleValue()) return v - max.doubleValue(); - if (v <= min.doubleValue()) return v - min.doubleValue(); - return 0d; - } - - @Override - public String toString() { - return jsonStr(); - } - - @Override - public void writeMap(EntryWriter ew) throws IOException { - ew.put("min", min) - .put("max", max) - .putIfNotNull("actual", actual); - } -} diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ReplicaCount.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ReplicaCount.java deleted file mode 100644 index c41e48ca593..00000000000 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ReplicaCount.java +++ /dev/null @@ -1,134 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.client.solrj.cloud.autoscaling; - -import java.io.IOException; -import java.util.List; -import java.util.Objects; - -import org.apache.solr.common.MapWriter; -import org.apache.solr.common.cloud.Replica; -import org.apache.solr.common.util.Utils; - -class ReplicaCount implements MapWriter { - long nrt, tlog, pull; - - public ReplicaCount() { - nrt = tlog = pull = 0; - } - - public ReplicaCount(long nrt, long tlog, long pull) { - this.nrt = nrt; - this.tlog = tlog; - this.pull = pull; - } - - public long total() { - return nrt + tlog + pull; - } - - @Override - public void writeMap(EntryWriter ew) throws IOException { - if (nrt > 0) ew.put(Replica.Type.NRT.name(), nrt); - if (pull > 0) ew.put(Replica.Type.PULL.name(), pull); - if (tlog > 0) ew.put(Replica.Type.TLOG.name(), tlog); - ew.put("count", total()); - } - - public Long getVal(Replica.Type type) { - if (type == null) return total(); - switch (type) { - case NRT: - return nrt; - case PULL: - return pull; - case TLOG: - return tlog; - } - return total(); - } - - public void increment(List infos) { - if (infos == null) return; - for (Replica info : infos) { - increment(info); - } - } - - void increment(Replica info) { - increment(info.getType()); - } - - void increment(ReplicaCount count) { - nrt += count.nrt; - pull += count.pull; - tlog += count.tlog; - } - - - public void increment(Replica.Type type) { - switch (type) { - case NRT: - nrt++; - break; - case PULL: - pull++; - break; - case TLOG: - tlog++; - break; - default: - nrt++; - } - } - - @Override - public boolean equals(Object obj) { - if (obj instanceof ReplicaCount) { - ReplicaCount that = (ReplicaCount) obj; - return that.nrt == this.nrt && that.tlog == this.tlog && that.pull == this.pull; - - } - return false; - } - - @Override - public int hashCode() { - return Objects.hash(nrt, tlog, pull); - } - - @Override - public String toString() { - return Utils.toJSONString(this); - } - - public ReplicaCount copy() { - return new ReplicaCount(nrt, tlog, pull); - } - - public void reset() { - nrt = tlog = pull = 0; - } - - public int delta(int expectedReplicaCount, Replica.Type type) { - if (type == Replica.Type.NRT) return (int) (nrt - expectedReplicaCount); - if (type == Replica.Type.PULL) return (int) (pull - expectedReplicaCount); - if (type == Replica.Type.TLOG) return (int) (tlog - expectedReplicaCount); - throw new RuntimeException("NO type"); - } -} diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ReplicaVariable.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ReplicaVariable.java deleted file mode 100644 index 5163d3a0c20..00000000000 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ReplicaVariable.java +++ /dev/null @@ -1,176 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.client.solrj.cloud.autoscaling; - -import java.util.Collection; -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -import org.apache.solr.common.util.StrUtils; - -class ReplicaVariable extends VariableBase { - - public ReplicaVariable(Type type) { - super(type); - } - - public static final String REPLICASCOUNT = "relevantReplicas"; - - - - - static int getRelevantReplicasCount(Policy.Session session, Condition cv, String collection, String shard) { - int totalReplicasOfInterest = 0; - Clause clause = cv.getClause(); - for (Row row : session.matrix) { - Integer perShardCount = row.computeCacheIfAbsent(collection, shard, REPLICASCOUNT, cv.clause, o -> { - int[] result = new int[1]; - row.forEachReplica(collection, replicaInfo -> { - if (clause.isMatch(replicaInfo, collection, shard)) - result[0]++; - }); - return result[0]; - }); - if (perShardCount != null) - totalReplicasOfInterest += perShardCount; - } - return totalReplicasOfInterest; - } - - @Override - public Object validate(String name, Object val, boolean isRuleVal) { - return getOperandAdjustedValue(super.validate(name, val, isRuleVal), val); - } - - - - @Override - public Operand getOperand(Operand expected, Object strVal, ComputedType computedType) { - if (computedType == ComputedType.ALL) return expected; - return checkForRangeOperand(expected, strVal, computedType); - } - - static Operand checkForRangeOperand(Operand expected, Object strVal, ComputedType computedType) { - if (strVal instanceof String) { - String s = ((String) strVal).trim(); - int hyphenIdx = s.indexOf('-'); - if (hyphenIdx > 0) { - if (hyphenIdx == s.length() - 1) { - throw new IllegalArgumentException("bad range input :" + expected); - } - if (expected == Operand.EQUAL) return Operand.RANGE_EQUAL; - if (expected == Operand.NOT_EQUAL) return Operand.RANGE_NOT_EQUAL; - } - - } - - if (expected == Operand.EQUAL && (computedType != null || !isIntegerEquivalent(strVal))) { - return Operand.RANGE_EQUAL; - } - if (expected == Operand.NOT_EQUAL && (computedType != null || !isIntegerEquivalent(strVal))) - return Operand.RANGE_NOT_EQUAL; - - return expected; - } - - @Override - public String postValidate(Condition condition) { - Object val = condition.clause.getThirdTag().val; - boolean isNodesetObjectList = condition.clause.nodeSetPresent && (val instanceof List) && ((List)val).get(0) instanceof Condition ; - if(condition.clause.nodeSetPresent ){ - if(condition.computedType == ComputedType.EQUAL){ - if(!isNodesetObjectList) return " 'nodeset' must have an array value when 'replica': '#EQUAL` is used"; - } else { - if(isNodesetObjectList){ - return "cannot use array value for nodeset if replica : '#EQUAL' is not used"; - } - - } - - } - - if (condition.computedType == ComputedType.EQUAL) { - if (condition.getClause().tag != null && - (condition.getClause().tag.op == Operand.WILDCARD || condition.getClause().tag.op == Operand.IN)) { - return null; - } else { - return "'replica': '#EQUAL` must be used with 'node':'#ANY'"; - } - } else if (condition.computedType == ComputedType.ALL) { - if(isNodesetObjectList) return "replica: '#ALL' cannot be used with a list of values in nodeset"; - if (condition.getClause().tag != null && (condition.getClause().getTag().op == Operand.IN || - condition.getClause().getTag().op == Operand.WILDCARD)) { - return StrUtils.formatString("array value or wild card cannot be used for tag {0} with replica : '#ALL'", - condition.getClause().tag.getName()); - } - } else { - return checkNonEqualOp(condition); - } - - return null; - } - - static String checkNonEqualOp(Condition condition) { - if (condition.computedType == null && - condition.val instanceof RangeVal && - condition.op != Operand.RANGE_EQUAL) { - return "non-integer values cannot have any other operators"; - } - - if(condition.computedType == ComputedType.PERCENT && condition.op != Operand.RANGE_EQUAL){ - return "percentage values cannot have any other operators"; - } - return null; - } - - @Override - public Object computeValue(Policy.Session session, Condition cv, String collection, String shard, String node) { - if (cv.computedType == ComputedType.ALL) - return Double.valueOf(getRelevantReplicasCount(session, cv, collection, shard)); - if (cv.computedType == ComputedType.EQUAL) { - int relevantReplicasCount = getRelevantReplicasCount(session, cv, collection, shard); - double bucketsCount = getNumBuckets(session, cv.getClause()); - if (relevantReplicasCount == 0 || bucketsCount == 0) return 0; - return (double) relevantReplicasCount / bucketsCount; - } else if (cv.computedType == ComputedType.PERCENT) { - return ComputedType.PERCENT.compute(getRelevantReplicasCount(session, cv, collection, shard), cv); - } else { - throw new IllegalArgumentException("Unsupported type " + cv.computedType); - - } - } - - @SuppressWarnings({"unchecked", "rawtypes"}) - private int getNumBuckets(Policy.Session session, Clause clause) { - if (clause.getTag().getOperand() == Operand.IN) { - return ((Collection) clause.getTag().val).size(); - } else if (clause.getTag().getOperand() == Operand.WILDCARD) { - if (clause.getTag().varType == Type.NODE) return session.matrix.size(); - Set uniqueVals = new HashSet(); - for (Row matrix : session.matrix) { - Object val = matrix.getVal(clause.getTag().name); - if (val != null) uniqueVals.add(val); - } - return uniqueVals.size(); - } else { - throw new IllegalArgumentException("Invalid operand for the tag in " + clause); - } - - } -} \ No newline at end of file diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Row.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Row.java deleted file mode 100644 index 3010accb60f..00000000000 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Row.java +++ /dev/null @@ -1,402 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.client.solrj.cloud.autoscaling; - -import java.io.IOException; -import java.lang.invoke.MethodHandles; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Random; -import java.util.Set; -import java.util.concurrent.atomic.AtomicLong; -import java.util.function.BiConsumer; -import java.util.function.Consumer; -import java.util.function.Function; -import java.util.stream.Collectors; - -import org.apache.solr.client.solrj.cloud.NodeStateProvider; -import org.apache.solr.client.solrj.cloud.SolrCloudManager; -import org.apache.solr.common.MapWriter; -import org.apache.solr.common.cloud.Replica; -import org.apache.solr.common.util.Pair; -import org.apache.solr.common.util.Utils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import static org.apache.solr.common.params.CoreAdminParams.NODE; - -/** - * Each instance represents a node in the cluster - */ -public class Row implements MapWriter { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - public final String node; - final Cell[] cells; - //this holds the details of each replica in the node - public Map>> collectionVsShardVsReplicas; - - boolean anyValueMissing = false; - boolean isLive = true; - Policy.Session session; - @SuppressWarnings({"rawtypes"}) - Map globalCache; - @SuppressWarnings({"rawtypes"}) - Map perCollCache; - - public Row(String node, List> params, List perReplicaAttributes, Policy.Session session) { - this(node, params, perReplicaAttributes, session, session.nodeStateProvider, session.cloudManager); - } - - /** - * Constructor that allows explicitly passing a {@link NodeStateProvider} and a {@link SolrCloudManager} in order not to - * use those obtained through the passed session. - *

Note the resulting row has a {@link Policy.Session} that may not be consistent with the rest of the Row's state. When rows are copied - * as part of a {@link Policy.Session} copy, the copied rows' sessions are eventually updated in - * {@link org.apache.solr.client.solrj.cloud.autoscaling.Policy.Session#Session(List, SolrCloudManager, List, Set, List, NodeStateProvider, Policy, Policy.Transaction)} - * once the new {@link Policy.Session} instance is available.

- */ - @SuppressWarnings({"rawtypes"}) - Row(String node, List> params, List perReplicaAttributes, Policy.Session session, - NodeStateProvider nsp, SolrCloudManager cloudManager) { - this.session = session; - collectionVsShardVsReplicas = nsp.getReplicaInfo(node, perReplicaAttributes); - if (collectionVsShardVsReplicas == null) collectionVsShardVsReplicas = new HashMap<>(); - this.node = node; - cells = new Cell[params.size()]; - isLive = cloudManager.getClusterStateProvider().getLiveNodes().contains(node); - List paramNames = params.stream().map(Pair::first).collect(Collectors.toList()); - Map vals = isLive ? nsp.getNodeValues(node, paramNames) : Collections.emptyMap(); - for (int i = 0; i < params.size(); i++) { - Pair pair = params.get(i); - cells[i] = new Cell(i, pair.first(), Clause.validate(pair.first(), vals.get(pair.first()), false), null, pair.second(), this); - if (NODE.equals(pair.first())) cells[i].val = node; - if (cells[i].val == null) anyValueMissing = true; - } - this.globalCache = new HashMap(); - this.perCollCache = new HashMap(); - isAlreadyCopied = true; - } - - public static final Map cacheStats = new HashMap<>(); - - static class CacheEntry implements MapWriter { - AtomicLong hits = new AtomicLong(), misses = new AtomicLong(); - - @Override - public void writeMap(EntryWriter ew) throws IOException { - ew.put("hits", hits.get()); - ew.put("misses", misses.get()); - } - - public static boolean hit(String cacheName) { -// getCacheEntry(cacheName).hits.incrementAndGet(); - return true; - } - - private static CacheEntry getCacheEntry(String cacheName) { - CacheEntry cacheEntry = cacheStats.get(cacheName); - if (cacheEntry == null) { - cacheStats.put(cacheName, cacheEntry = new CacheEntry()); - } - return cacheEntry; - } - - public static boolean miss(String cacheName) { - getCacheEntry(cacheName).misses.incrementAndGet(); - return true; - } - } - - - public void forEachShard(String collection, BiConsumer> consumer) { - collectionVsShardVsReplicas - .getOrDefault(collection, Collections.emptyMap()) - .forEach(consumer); - } - - - @SuppressWarnings({"unchecked"}) - public R computeCacheIfAbsent(String cacheName, Function supplier) { - R result = (R) globalCache.get(cacheName); - if (result != null) { - assert CacheEntry.hit(cacheName); - return result; - } else { - assert CacheEntry.miss(cacheName); - globalCache.put(cacheName, result = supplier.apply(cacheName)); - return result; - } - } - - @SuppressWarnings({"unchecked", "rawtypes"}) - public R computeCacheIfAbsent(String coll, String shard, String cacheName, Object key, Function supplier) { - Map collMap = (Map) this.perCollCache.get(coll); - if (collMap == null) this.perCollCache.put(coll, collMap = new HashMap()); - Map shardMap = (Map) collMap.get(shard); - if (shardMap == null) collMap.put(shard, shardMap = new HashMap()); - Map cacheNameMap = (Map) shardMap.get(cacheName); - if (cacheNameMap == null) shardMap.put(cacheName, cacheNameMap = new HashMap()); - R result = (R) cacheNameMap.get(key); - if (result == null) { - CacheEntry.miss(cacheName); - cacheNameMap.put(key, result = supplier.apply(key)); - return result; - } else { - CacheEntry.hit(cacheName); - return result; - } - } - - - public Row(String node, Cell[] cells, boolean anyValueMissing, - @SuppressWarnings({"rawtypes"}) Map>> collectionVsShardVsReplicas, boolean isLive, Policy.Session session, - @SuppressWarnings({"rawtypes"}) Map perRowCache, - @SuppressWarnings({"rawtypes"})Map globalCache) { - this.session = session; - this.node = node; - this.isLive = isLive; - this.cells = new Cell[cells.length]; - for (int i = 0; i < this.cells.length; i++) { - this.cells[i] = cells[i].copy(); - this.cells[i].row = this; - } - this.anyValueMissing = anyValueMissing; - this.collectionVsShardVsReplicas = collectionVsShardVsReplicas; - this.perCollCache = perRowCache; - this.globalCache = globalCache; - } - - @Override - public void writeMap(EntryWriter ew) throws IOException { - ew.put(NODE, node); - ew.put("replicas", collectionVsShardVsReplicas); - ew.put("isLive", isLive); - ew.put("attributes", Arrays.asList(cells)); - } - - Row copy() { - return new Row(node, cells, anyValueMissing, collectionVsShardVsReplicas, isLive, session, this.globalCache, this.perCollCache); - } - - Object getVal(String name) { - if (NODE.equals(name)) return this.node; - for (Cell cell : cells) if (cell.name.equals(name)) return cell.val; - return null; - } - - public Object getVal(String name, Object def) { - for (Cell cell : cells) - if (cell.name.equals(name)) { - return cell.val == null ? def : cell.val; - } - return def; - } - - @Override - public String toString() { - return jsonStr(); - } - - public Row addReplica(String coll, String shard, Replica.Type type) { - return addReplica(coll, shard, type, 0, true); - } - - public Row addReplica(String coll, String shard, Replica.Type type, boolean strictMode) { - return addReplica(coll, shard, type, 0, strictMode); - } - - /** - * this simulates adding a replica of a certain coll+shard to node. as a result of adding a replica , - * values of certain attributes will be modified, in this node as well as other nodes. Please note that - * the state of the current session is kept intact while this operation is being performed - * - * @param coll collection name - * @param shard shard name - * @param type replica type - * @param recursionCount the number of times we have recursed to add more replicas - * @param strictMode whether suggester is operating in strict mode or not - */ - Row addReplica(String coll, String shard, Replica.Type type, int recursionCount, boolean strictMode) { - if (recursionCount > 3) { - log.error("more than 3 levels of recursion ", new RuntimeException()); - return this; - } - lazyCopyReplicas(coll, shard); - List furtherOps = new LinkedList<>(); - Consumer opCollector = it -> furtherOps.add(it); - Row row = null; - row = session.copy().getNode(this.node); - if (row == null) throw new RuntimeException("couldn't get a row"); - row.lazyCopyReplicas(coll, shard); - Map> c = row.collectionVsShardVsReplicas.computeIfAbsent(coll, k -> new HashMap<>()); - List replicas = c.computeIfAbsent(shard, k -> new ArrayList<>()); - String replicaname = "SYNTHETIC." + new Random().nextInt(1000) + 1000; - Replica ri = new Replica(replicaname, this.node, coll, shard, replicaname, - Replica.State.ACTIVE, type != null ? type : Replica.Type.NRT, Collections.emptyMap()); - replicas.add(ri); - for (Cell cell : row.cells) { - cell.type.projectAddReplica(cell, ri, opCollector, strictMode); - } - for (OperationInfo op : furtherOps) { - if (op.isAdd) { - row = row.session.getNode(op.node).addReplica(op.coll, op.shard, op.type, recursionCount + 1, strictMode); - } else { - row.session.getNode(op.node).removeReplica(op.coll, op.shard, op.type, recursionCount + 1); - } - } - - return row; - } - - boolean isAlreadyCopied = false; - - @SuppressWarnings({"unchecked", "rawtypes"}) - private void lazyCopyReplicas(String coll, String shard) { - globalCache = new HashMap(); - Map cacheCopy = new HashMap<>(perCollCache); - cacheCopy.remove(coll);//todo optimize at shard level later - perCollCache = cacheCopy; - if (isAlreadyCopied) return;//caches need to be invalidated but the rest can remain as is - - Map>> replicasCopy = new HashMap<>(collectionVsShardVsReplicas); - Map> oneColl = replicasCopy.get(coll); - if (oneColl != null) { - replicasCopy.put(coll, Utils.getDeepCopy(oneColl, 2)); - } - collectionVsShardVsReplicas = replicasCopy; - isAlreadyCopied = true; - } - - boolean hasColl(String coll) { - return collectionVsShardVsReplicas.containsKey(coll); - } - - @SuppressWarnings({"unchecked"}) - public void createCollShard(Pair collShard) { - Map> shardInfo = collectionVsShardVsReplicas.computeIfAbsent(collShard.first(), Utils.NEW_HASHMAP_FUN); - if (collShard.second() != null) shardInfo.computeIfAbsent(collShard.second(), Utils.NEW_ARRAYLIST_FUN); - } - - - static class OperationInfo { - final String coll, shard, node, cellName; - final boolean isAdd;// true =addReplica, false=removeReplica - final Replica.Type type; - - - OperationInfo(String coll, String shard, String node, String cellName, boolean isAdd, Replica.Type type) { - this.coll = coll; - this.shard = shard; - this.node = node; - this.cellName = cellName; - this.isAdd = isAdd; - this.type = type; - } - } - - - public Replica getReplica(String coll, String shard, Replica.Type type) { - Map> c = collectionVsShardVsReplicas.get(coll); - if (c == null) return null; - List r = c.get(shard); - if (r == null) return null; - int idx = -1; - for (int i = 0; i < r.size(); i++) { - Replica info = r.get(i); - if (type == null || info.getType() == type) { - idx = i; - break; - } - } - if (idx == -1) return null; - return r.get(idx); - } - - public Row removeReplica(String coll, String shard, Replica.Type type) { - return removeReplica(coll, shard, type, 0); - - } - - // this simulates removing a replica from a node - public Row removeReplica(String coll, String shard, Replica.Type type, int recursionCount) { - if (recursionCount > 3) { - log.error("more than 3 levels of recursion ", new RuntimeException()); - return this; - } - List furtherOps = new LinkedList<>(); - Consumer opCollector = it -> furtherOps.add(it); - Row row = session.copy().getNode(this.node); - row.lazyCopyReplicas(coll, shard); - Map> c = row.collectionVsShardVsReplicas.get(coll); - if (c == null) return null; - List r = c.get(shard); - if (r == null) return null; - int idx = -1; - for (int i = 0; i < r.size(); i++) { - Replica info = r.get(i); - if (type == null || info.getType() == type) { - idx = i; - break; - } - } - if (idx == -1) return null; - Replica removed = r.remove(idx); - for (Cell cell : row.cells) { - cell.type.projectRemoveReplica(cell, removed, opCollector); - } - return row; - - } - - public Cell[] getCells() { - return cells; - } - - public boolean isLive() { - return isLive; - } - - public void forEachReplica(Consumer consumer) { - forEachReplica(collectionVsShardVsReplicas, consumer); - } - - public void forEachReplica(String coll, Consumer consumer) { - collectionVsShardVsReplicas.getOrDefault(coll, Collections.emptyMap()).forEach((shard, replicaInfos) -> { - for (Replica replicaInfo : replicaInfos) { - consumer.accept(replicaInfo); - } - }); - } - - public static void forEachReplica(Map>> collectionVsShardVsReplicas, Consumer consumer) { - collectionVsShardVsReplicas.forEach((coll, shardVsReplicas) -> shardVsReplicas - .forEach((shard, replicaInfos) -> { - for (int i = 0; i < replicaInfos.size(); i++) { - Replica r = replicaInfos.get(i); - consumer.accept(r); - } - })); - } -} diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/SealedClause.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/SealedClause.java deleted file mode 100644 index 495bcb4d4de..00000000000 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/SealedClause.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.client.solrj.cloud.autoscaling; - -import java.util.function.Function; - -/** - * This clause is an instance with no conditions with computed value. every value is computed just in time - */ -public class SealedClause extends Clause { - SealedClause(Clause clause, Function computedValueEvaluator) { - super(clause, computedValueEvaluator); - } -} diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/SplitShardSuggester.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/SplitShardSuggester.java deleted file mode 100644 index b483033c009..00000000000 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/SplitShardSuggester.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.client.solrj.cloud.autoscaling; - -import java.util.Collections; -import java.util.Map; -import java.util.Set; - -import org.apache.solr.client.solrj.SolrRequest; -import org.apache.solr.client.solrj.request.CollectionAdminRequest; -import org.apache.solr.common.params.CollectionParams; -import org.apache.solr.common.params.CommonAdminParams; -import org.apache.solr.common.util.Pair; - -/** - * This suggester produces a SPLITSHARD request using provided {@link org.apache.solr.client.solrj.cloud.autoscaling.Suggester.Hint#COLL_SHARD} value. - */ -class SplitShardSuggester extends Suggester { - - @Override - public CollectionParams.CollectionAction getAction() { - return CollectionParams.CollectionAction.SPLITSHARD; - } - - @Override - @SuppressWarnings({"rawtypes"}) - SolrRequest init() { - @SuppressWarnings({"unchecked"}) - Set> shards = (Set>) hints.getOrDefault(Hint.COLL_SHARD, Collections.emptySet()); - if (shards.isEmpty()) { - throw new RuntimeException("split-shard requires 'collection' and 'shard'"); - } - if (shards.size() > 1) { - throw new RuntimeException("split-shard requires exactly one pair of 'collection' and 'shard'"); - } - Pair collShard = shards.iterator().next(); - @SuppressWarnings({"unchecked"}) - Map params = (Map)hints.getOrDefault(Hint.PARAMS, Collections.emptyMap()); - Float splitFuzz = (Float)params.get(CommonAdminParams.SPLIT_FUZZ); - CollectionAdminRequest.SplitShard req = CollectionAdminRequest.splitShard(collShard.first()).setShardName(collShard.second()); - if (splitFuzz != null) { - req.setSplitFuzz(splitFuzz); - } - String splitMethod = (String)params.get(CommonAdminParams.SPLIT_METHOD); - if (splitMethod != null) { - req.setSplitMethod(splitMethod); - } - Boolean splitByPrefix = (Boolean)params.get(CommonAdminParams.SPLIT_BY_PREFIX); - if (splitByPrefix != null) { - req.setSplitByPrefix(splitByPrefix); - } - return req; - } -} diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggester.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggester.java deleted file mode 100644 index 889feddc244..00000000000 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggester.java +++ /dev/null @@ -1,530 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.client.solrj.cloud.autoscaling; - -import java.io.IOException; -import java.lang.invoke.MethodHandles; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.EnumMap; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.Set; -import java.util.function.Consumer; -import java.util.function.Predicate; - -import org.apache.solr.client.solrj.SolrRequest; -import org.apache.solr.client.solrj.impl.ClusterStateProvider; -import org.apache.solr.common.ConditionalMapWriter; -import org.apache.solr.common.MapWriter; -import org.apache.solr.common.SolrException; -import org.apache.solr.common.cloud.DocCollection; -import org.apache.solr.common.cloud.Replica; -import org.apache.solr.common.cloud.rule.ImplicitSnitch; -import org.apache.solr.common.params.CollectionParams; -import org.apache.solr.common.util.Pair; -import org.apache.solr.common.util.Utils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import static org.apache.solr.client.solrj.cloud.autoscaling.Variable.Type.FREEDISK; -import static org.apache.solr.common.params.CollectionAdminParams.WITH_COLLECTION; - -/* A suggester is capable of suggesting a collection operation - * given a particular session. Before it suggests a new operation, - * it ensures that , - * a) load is reduced on the most loaded node - * b) it causes no new violations - * - */ -public abstract class Suggester implements MapWriter { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - protected final EnumMap hints = new EnumMap<>(Hint.class); - Policy.Session session; - @SuppressWarnings({"rawtypes"}) - SolrRequest operation; - boolean force; - protected List originalViolations = new ArrayList<>(); - private boolean isInitialized = false; - LinkedHashMap deviations, lastBestDeviation; - - - void _init(Policy.Session session) { - this.session = session.copy(); - } - - boolean isLessDeviant() { - if (lastBestDeviation == null && deviations == null) return false; - if (deviations == null) return true; - if (lastBestDeviation == null) return false; - if (lastBestDeviation.size() < deviations.size()) return true; - for (Map.Entry currentDeviation : deviations.entrySet()) { - double[] lastDeviation = lastBestDeviation.get(currentDeviation.getKey()); - if (lastDeviation == null) return false; - int result = Preference.compareWithTolerance(currentDeviation.getValue()[0], - lastDeviation[0], 1); - if (result < 0) return true; - if (result > 0) return false; - } - return false; - } - @SuppressWarnings({"unchecked", "rawtypes"}) - public Suggester hint(Hint hint, Object value) { - hint.validator.accept(value); - if (hint.multiValued) { - Collection values = value instanceof Collection ? (Collection) value : Collections.singletonList(value); - ((Set) hints.computeIfAbsent(hint, h -> new HashSet<>())).addAll(values); - } else { - if (value == null) { - hints.put(hint, null); - } else { - if ((value instanceof Map) || (value instanceof Number)) { - hints.put(hint, value); - } else { - hints.put(hint, String.valueOf(value)); - } - } - } - return this; - } - - public CollectionParams.CollectionAction getAction() { - return null; - } - - /** - * Normally, only less loaded nodes are used for moving replicas. If this is a violation and a MOVE must be performed, - * set the flag to true. - */ - public Suggester forceOperation(boolean force) { - this.force = force; - return this; - } - - protected boolean isNodeSuitableForReplicaAddition(Row targetRow, Row srcRow) { - if (!targetRow.isLive) return false; - if (!isAllowed(targetRow.node, Hint.TARGET_NODE)) return false; - if (!isAllowed(targetRow.getVal(ImplicitSnitch.DISK), Hint.MINFREEDISK)) return false; - - if (srcRow != null) {// if the src row has the same violation it's not - for (Violation v1 : originalViolations) { - if (!v1.getClause().getThirdTag().varType.meta.isNodeSpecificVal()) continue; - if (v1.getClause().hasComputedValue) continue; - if (targetRow.node.equals(v1.node)) { - for (Violation v2 : originalViolations) { - if (srcRow.node.equals(v2.node)) { - if (v1.getClause().equals(v2.getClause())) - return false; - } - } - } - } - } - - return true; - } - - @SuppressWarnings({"rawtypes"}) - abstract SolrRequest init(); - - @SuppressWarnings({"unchecked", "rawtypes"}) - public SolrRequest getSuggestion() { - if (!isInitialized) { - Set collections = (Set) hints.getOrDefault(Hint.COLL, Collections.emptySet()); - Set> s = (Set>) hints.getOrDefault(Hint.COLL_SHARD, Collections.emptySet()); - if (!collections.isEmpty() || !s.isEmpty()) { - HashSet> collectionShardPairs = new HashSet<>(s); - collections.forEach(c -> collectionShardPairs.add(new Pair<>(c, null))); - collections.forEach(c -> { - try { - getWithCollection(c).ifPresent(withCollection -> collectionShardPairs.add(new Pair<>(withCollection, null))); - } catch (IOException e) { - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, - "Exception while fetching 'withCollection' attribute for collection: " + c, e); - } - }); - s.forEach(kv -> { - try { - getWithCollection(kv.first()).ifPresent(withCollection -> collectionShardPairs.add(new Pair<>(withCollection, null))); - } catch (IOException e) { - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, - "Exception while fetching 'withCollection' attribute for collection: " + kv.first(), e); - } - }); - setupCollection(collectionShardPairs); - Collections.sort(session.expandedClauses); - } - Set srcNodes = (Set) hints.get(Hint.SRC_NODE); - if (srcNodes != null && !srcNodes.isEmpty()) { - // the source node is dead so live nodes may not have it - for (String srcNode : srcNodes) { - if (session.matrix.stream().noneMatch(row -> row.node.equals(srcNode))) { - session.matrix.add(new Row(srcNode, session.getPolicy().getParams(), session.getPolicy().getPerReplicaAttributes(), session)); - } - } - } - session.applyRules(); - originalViolations.addAll(session.getViolations()); - this.operation = init(); - isInitialized = true; - } - if (operation != null && session.transaction != null && session.transaction.isOpen()) { - session.transaction.updateSession(session); - } - return operation; - } - - protected Optional getWithCollection(String collectionName) throws IOException { - DocCollection collection = session.cloudManager.getClusterStateProvider().getCollection(collectionName); - if (collection != null) { - return Optional.ofNullable(collection.getStr(WITH_COLLECTION)); - } else { - return Optional.empty(); - } - } - - private void setupCollection(HashSet> collectionShardPairs) { - ClusterStateProvider stateProvider = session.cloudManager.getClusterStateProvider(); - for (Pair shard : collectionShardPairs) { - // if this is not a known collection from the existing clusterstate, - // then add it - if (session.matrix.stream().noneMatch(row -> row.hasColl(shard.first()))) { - session.addClausesForCollection(stateProvider, shard.first()); - } - for (Row row : session.matrix) row.createCollShard(shard); - } - } - - - public Policy.Session getSession() { - return session; - } - - List getMatrix() { - return session.matrix; - - } - - public static class SuggestionInfo implements MapWriter { - Suggestion.Type type; - Violation violation; - @SuppressWarnings({"rawtypes"}) - SolrRequest operation; - - public SuggestionInfo(Violation violation, @SuppressWarnings({"rawtypes"})SolrRequest op, Suggestion.Type type) { - this.violation = violation; - this.operation = op; - this.type = type; - } - - @SuppressWarnings({"rawtypes"}) - public SolrRequest getOperation() { - return operation; - } - - public Violation getViolation() { - return violation; - } - - @Override - public void writeMap(EntryWriter ew) throws IOException { - ew.put("type", type.name()); - if(violation!= null) ew.put("violation", - new ConditionalMapWriter(violation, - (k, v) -> !"violatingReplicas".equals(k))); - ew.put("operation", operation); - } - - @Override - public String toString() { - return Utils.toJSONString(this); - } - } - - //check if the fresh set of violations is less serious than the last set of violations - boolean isLessSerious(List fresh, List old) { - if (old == null || fresh.size() < old.size()) return true; - if (fresh.size() == old.size()) { - for (int i = 0; i < fresh.size(); i++) { - Violation freshViolation = fresh.get(i); - Violation oldViolation = null; - for (Violation v : old) {//look for exactly same clause being violated - if (v.equals(freshViolation)) oldViolation = v; - } - if (oldViolation == null) {//if no match, look for similar violation - for (Violation v : old) { - if (v.isSimilarViolation(freshViolation)) oldViolation = v; - } - } - - if (oldViolation != null && freshViolation.isLessSerious(oldViolation)) return true; - } - } - return false; - } - - boolean containsNewErrors(List violations) { - boolean isTxOpen = session.transaction != null && session.transaction.isOpen(); - if (violations.size() > originalViolations.size()) return true; - for (Violation v : violations) { - //the computed value can change over time. So it's better to evaluate it in the end - if (isTxOpen && v.getClause().hasComputedValue) continue; - int idx = originalViolations.indexOf(v); - if (idx < 0 || originalViolations.get(idx).isLessSerious(v)) return true; - } - return false; - } - - List> getValidReplicas(boolean sortDesc, boolean isSource, int until) { - List> allPossibleReplicas = new ArrayList<>(); - - if (sortDesc) { - if (until == -1) until = getMatrix().size(); - for (int i = 0; i < until; i++) addReplicaToList(getMatrix().get(i), isSource, allPossibleReplicas); - } else { - if (until == -1) until = 0; - for (int i = getMatrix().size() - 1; i >= until; i--) - addReplicaToList(getMatrix().get(i), isSource, allPossibleReplicas); - } - return allPossibleReplicas; - } - - void addReplicaToList(Row r, boolean isSource, List> replicaList) { - if (!isAllowed(r.node, isSource ? Hint.SRC_NODE : Hint.TARGET_NODE)) return; - for (Map.Entry>> e : r.collectionVsShardVsReplicas.entrySet()) { - if (!isAllowed(e.getKey(), Hint.COLL)) continue; - for (Map.Entry> shard : e.getValue().entrySet()) { - if (!isAllowed(new Pair<>(e.getKey(), shard.getKey()), Hint.COLL_SHARD)) continue;//todo fix - if (shard.getValue() == null || shard.getValue().isEmpty()) continue; - for (Replica replicaInfo : shard.getValue()) { - if (replicaInfo.getName().startsWith("SYNTHETIC.")) continue; - replicaList.add(new Pair<>(shard.getValue().get(0), r)); - break; - } - } - } - } - - List testChangedMatrix(boolean executeInStrictMode, Policy.Session session) { - if (this.deviations != null) this.lastBestDeviation = this.deviations; - this.deviations = null; - Policy.setApproxValuesAndSortNodes(session.getPolicy().getClusterPreferences(), session.matrix); - List errors = new ArrayList<>(); - for (Clause clause : session.expandedClauses) { - Clause originalClause = clause.derivedFrom == null ? clause : clause.derivedFrom; - if (this.deviations == null) this.deviations = new LinkedHashMap<>(); - this.deviations.put(originalClause, new double[1]); - List errs = clause.test(session, this.deviations == null ? null : this.deviations.get(originalClause)); - if (!errs.isEmpty() && - (executeInStrictMode || clause.strict)) errors.addAll(errs); - } - session.violations = errors; - if (!errors.isEmpty()) deviations = null; - return errors; - } - - - protected boolean isAllowed(Object v, Hint hint) { - Object hintVal = hints.get(hint); - if (hintVal == null) return true; - if (hint.multiValued) { - @SuppressWarnings({"rawtypes"}) - Set set = (Set) hintVal; - return set == null || set.contains(v); - } else { - return hintVal == null || hint.valueValidator.test(new Pair<>(hintVal, v)); - } - } - - public enum Hint { - COLL(true), - // collection shard pair - // this should be a Pair , (collection,shard) - COLL_SHARD(true, v -> { - @SuppressWarnings({"rawtypes"}) - Collection c = v instanceof Collection ? (Collection) v : Collections.singleton(v); - for (Object o : c) { - if (!(o instanceof Pair)) { - throw new RuntimeException("COLL_SHARD hint must use a Pair"); - } - @SuppressWarnings({"rawtypes"}) - Pair p = (Pair) o; - if (p.first() == null || p.second() == null) { - throw new RuntimeException("Both collection and shard must not be null"); - } - } - - }) { - @Override - public Object parse(Object v) { - if (v instanceof Map) { - @SuppressWarnings({"rawtypes"}) - Map map = (Map) v; - return Pair.parse(map); - } - return super.parse(v); - } - }, - SRC_NODE(true), - TARGET_NODE(true), - REPLICATYPE(false, o -> { - if (!(o instanceof Replica.Type)) { - throw new RuntimeException("REPLICATYPE hint must use a ReplicaType"); - } - }), - - MINFREEDISK(false, o -> { - if (!(o instanceof Number)) throw new RuntimeException("MINFREEDISK hint must be a number"); - }, hintValVsActual -> { - Double hintFreediskInGb = (Double) FREEDISK.validate(null, hintValVsActual.first(), false); - Double actualFreediskInGb = (Double) FREEDISK.validate(null, hintValVsActual.second(), false); - if (actualFreediskInGb == null) return false; - return actualFreediskInGb > hintFreediskInGb; - }), - NUMBER(true, o -> { - if (!(o instanceof Number)) throw new RuntimeException("NUMBER hint must be a number"); - }), - PARAMS(false, o -> { - if (!(o instanceof Map)) { - throw new RuntimeException("PARAMS hint must be a Map"); - } - }), - REPLICA(true); - - public final boolean multiValued; - public final Consumer validator; - public final Predicate> valueValidator; - - Hint(boolean multiValued) { - this(multiValued, v -> { - @SuppressWarnings({"rawtypes"}) - Collection c = v instanceof Collection ? (Collection) v : Collections.singleton(v); - for (Object o : c) { - if (!(o instanceof String)) throw new RuntimeException("hint must be of type String"); - } - }); - } - - Hint(boolean multiValued, Consumer c) { - this(multiValued, c, equalsPredicate); - } - - Hint(boolean multiValued, Consumer c, Predicate> testval) { - this.multiValued = multiValued; - this.validator = c; - this.valueValidator = testval; - } - - public static Hint get(String s) { - for (Hint hint : values()) { - if (hint.name().equals(s)) return hint; - } - return null; - } - - public Object parse(Object v) { - return v; - } - - - } - - static Predicate> equalsPredicate = valPair -> Objects.equals(valPair.first(), valPair.second()); - - @Override - public String toString() { - return jsonStr(); - } - - @Override - public void writeMap(EntryWriter ew) throws IOException { - ew.put("action", String.valueOf(getAction())); - ew.put("hints", (MapWriter) ew1 -> hints.forEach((hint, o) -> ew1.putNoEx(hint.toString(), o))); - } - - @SuppressWarnings({"rawtypes"}) - protected Collection setupWithCollectionTargetNodes(Set collections, Set> s, String withCollection) { - Collection originalTargetNodesCopy = null; - if (withCollection != null) { - if (log.isDebugEnabled()) { - HashSet set = new HashSet<>(collections); - s.forEach(kv -> set.add(kv.first())); - log.debug("Identified withCollection = {} for collection: {}", withCollection, set); - } - - originalTargetNodesCopy = Utils.getDeepCopy((Collection) hints.get(Hint.TARGET_NODE), 10, true); - - Set withCollectionNodes = new HashSet<>(); - - for (Row row : getMatrix()) { - row.forEachReplica(r -> { - if (withCollection.equals(r.getCollection()) && - "shard1".equals(r.getShard())) { - withCollectionNodes.add(r.getNodeName()); - } - }); - } - - if (originalTargetNodesCopy != null && !originalTargetNodesCopy.isEmpty()) { - // find intersection of the set of target nodes with the set of 'withCollection' nodes - @SuppressWarnings({"unchecked"}) - Set set = (Set) hints.computeIfAbsent(Hint.TARGET_NODE, h -> new HashSet<>()); - set.retainAll(withCollectionNodes); - if (set.isEmpty()) { - // no nodes common between the sets, we have no choice but to restore the original target node hint - hints.put(Hint.TARGET_NODE, originalTargetNodesCopy); - } - } else if (originalTargetNodesCopy == null) { - hints.put(Hint.TARGET_NODE, withCollectionNodes); - } - } - return originalTargetNodesCopy; - } - - protected String findWithCollection(Set collections, Set> s) { - List withCollections = new ArrayList<>(1); - collections.forEach(c -> { - try { - getWithCollection(c).ifPresent(withCollections::add); - } catch (IOException e) { - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, - "Exception while fetching 'withCollection' attribute for collection: " + c, e); - } - }); - s.forEach(kv -> { - try { - getWithCollection(kv.first()).ifPresent(withCollections::add); - } catch (IOException e) { - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, - "Exception while fetching 'withCollection' attribute for collection: " + kv.first(), e); - } - }); - - if (withCollections.size() > 1) { - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, - "The number of 'withCollection' attributes should be exactly 1 for any policy but found: " + withCollections); - } - return withCollections.isEmpty() ? null : withCollections.get(0); - } -} diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggestion.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggestion.java deleted file mode 100644 index e0d0a457e81..00000000000 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggestion.java +++ /dev/null @@ -1,133 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.client.solrj.cloud.autoscaling; - -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.function.Function; - -import org.apache.solr.client.solrj.SolrRequest; -import org.apache.solr.client.solrj.V2RequestSupport; -import org.apache.solr.client.solrj.cloud.autoscaling.Violation.ReplicaInfoAndErr; -import org.apache.solr.common.util.Pair; - -import static org.apache.solr.common.params.CollectionParams.CollectionAction.MOVEREPLICA; - -public class Suggestion { - - public enum Type { - violation, repair, improvement, unresolved_violation; - } - - static class Ctx { - long endTime = -1; - int max = Integer.MAX_VALUE; - public Policy.Session session; - public Violation violation; - List suggestions = new ArrayList<>(); - @SuppressWarnings({"rawtypes"}) - SolrRequest addSuggestion(Suggester suggester) { - return addSuggestion(suggester, Type.violation); - } - - @SuppressWarnings({"rawtypes"}) - SolrRequest addSuggestion(Suggester suggester, Type type) { - @SuppressWarnings({"rawtypes"}) - SolrRequest op = suggester.getSuggestion(); - if (op != null) { - session = suggester.getSession(); - suggestions.add(new Suggester.SuggestionInfo(violation, - ((V2RequestSupport) op.setUseV2(true)).getV2Request(), type)); - } - return op; - } - - - public Ctx setViolation(Violation violation) { - this.violation = violation; - return this; - } - - public List getSuggestions() { - return suggestions; - } - - public boolean hasTimedOut() { - return session.cloudManager.getTimeSource().getTimeNs() >= endTime; - } - - public boolean needMore() { - return suggestions.size() < max && !hasTimedOut(); - } - } - - static void suggestNegativeViolations(Suggestion.Ctx ctx, Function, List> shardSorter) { - if (ctx.violation.coll == null) return; - Set shardSet = new HashSet<>(); - if (!ctx.needMore()) return; - for (Row node : ctx.session.matrix) - node.forEachShard(ctx.violation.coll, (s, ri) -> { - if (Policy.ANY.equals(ctx.violation.shard) || s.equals(ctx.violation.shard)) shardSet.add(s); - }); - //Now, sort shards based on their index size ascending - List shards = shardSorter.apply(shardSet); - outer: - for (int i = 0; i < 5; i++) { - if (!ctx.needMore()) break; - int totalSuggestions = 0; - for (String shard : shards) { - Suggester suggester = ctx.session.getSuggester(MOVEREPLICA) - .hint(Suggester.Hint.COLL_SHARD, new Pair<>(ctx.violation.coll, shard)) - .forceOperation(true); - @SuppressWarnings({"rawtypes"}) - SolrRequest op = ctx.addSuggestion(suggester); - if (op == null) continue; - totalSuggestions++; - boolean violationStillExists = false; - for (Violation violation : suggester.session.getViolations()) { - if (violation.getClause().original == ctx.violation.getClause().original) { - violationStillExists = true; - break; - } - } - if (!violationStillExists) break outer; - } - if (totalSuggestions == 0) break; - } - } - - - static void suggestPositiveViolations(Ctx ctx) { - if (ctx.violation == null) return; - Double currentDelta = ctx.violation.replicaCountDelta; - for (ReplicaInfoAndErr e : ctx.violation.getViolatingReplicas()) { - if (!ctx.needMore()) break; - if (currentDelta <= 0) break; - Suggester suggester = ctx.session.getSuggester(MOVEREPLICA) - .forceOperation(true) - .hint(Suggester.Hint.COLL_SHARD, new Pair<>(e.replicaInfo.getCollection(), e.replicaInfo.getShard())) - .hint(Suggester.Hint.SRC_NODE, e.replicaInfo.getNodeName()); - if (ctx.addSuggestion(suggester) != null) { - currentDelta--; - } - } - } - -} diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/TriggerEventProcessorStage.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/TriggerEventProcessorStage.java deleted file mode 100644 index f8648e12289..00000000000 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/TriggerEventProcessorStage.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.client.solrj.cloud.autoscaling; - -/** - * Enum that represents the stages of trigger event processing. - */ -public enum TriggerEventProcessorStage { - STARTED, - ABORTED, - SUCCEEDED, - FAILED, - BEFORE_ACTION, - AFTER_ACTION, - IGNORED -} diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/TriggerEventType.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/TriggerEventType.java deleted file mode 100644 index a983bf02602..00000000000 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/TriggerEventType.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.client.solrj.cloud.autoscaling; - -/** - * Enum that represents trigger event types. - */ -public enum TriggerEventType { - NODEADDED, - NODELOST, - REPLICALOST, - MANUAL, - SCHEDULED, - SEARCHRATE, - INDEXRATE, - INVALID, - METRIC, - INDEXSIZE -} diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/UnsupportedSuggester.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/UnsupportedSuggester.java deleted file mode 100644 index 0ba49ba178f..00000000000 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/UnsupportedSuggester.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.client.solrj.cloud.autoscaling; - -import java.lang.invoke.MethodHandles; - -import org.apache.solr.client.solrj.SolrRequest; -import org.apache.solr.common.params.CollectionParams; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * This suggester simply logs the request but does not produce any suggestions. - */ -public class UnsupportedSuggester extends Suggester { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - private final CollectionParams.CollectionAction action; - - public static UnsupportedSuggester get(Policy.Session session, CollectionParams.CollectionAction action) { - UnsupportedSuggester suggester = new UnsupportedSuggester(action); - suggester._init(session); - return suggester; - } - - public UnsupportedSuggester(CollectionParams.CollectionAction action) { - this.action = action; - } - - @Override - public CollectionParams.CollectionAction getAction() { - return action; - } - - @Override - @SuppressWarnings({"rawtypes"}) - SolrRequest init() { - log.warn("Unsupported suggester for action {} with hings {} - no suggestion available", action, hints); - return null; - } - - @Override - @SuppressWarnings({"rawtypes"}) - public SolrRequest getSuggestion() { - return null; - } -} diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Variable.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Variable.java deleted file mode 100644 index 86af567b5cb..00000000000 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Variable.java +++ /dev/null @@ -1,410 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.client.solrj.cloud.autoscaling; - -import java.lang.annotation.ElementType; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; -import java.lang.annotation.Target; -import java.util.HashMap; -import java.util.Map; -import java.util.Set; -import java.util.function.Consumer; - -import org.apache.solr.common.cloud.Replica; -import org.apache.solr.common.cloud.rule.ImplicitSnitch; - -import static java.util.Collections.emptySet; -import static java.util.Collections.unmodifiableMap; - - -/** - * A Variable Type used in Autoscaling policy rules. Each variable type may have unique implementation - * of functionalities - */ -public interface Variable { - String NULL = ""; - String coreidxsize = "INDEX.sizeInGB"; - - default boolean match(Object inputVal, Operand op, Object val, String name, Row row) { - return op.match(val, validate(name, inputVal, false)) == Clause.TestStatus.PASS; - } - - default Object convertVal(Object val) { - return val; - } - - default void projectAddReplica(Cell cell, Replica ri, Consumer opCollector, boolean strictMode) { - } - - default boolean addViolatingReplicas(Violation.Ctx ctx) { - return false; - } - - void getSuggestions(Suggestion.Ctx ctx); - - /**When a non constant value is used in a variable, the actual value needs to be computed at the runtime - * - */ - default Object computeValue(Policy.Session session, Condition condition, String collection, String shard, String node) { - return condition.val; - } - - default void computeDeviation(Policy.Session session, double[] deviations, ReplicaCount replicaCount, SealedClause sealedClause) { - if (deviations != null) { - Number actualCount = replicaCount.getVal(sealedClause.type); - if(sealedClause.replica.val instanceof RangeVal) { - Double realDelta = ((RangeVal) sealedClause.replica.val).realDelta(actualCount.doubleValue()); - realDelta = sealedClause.isReplicaZero() ? -1 * realDelta : realDelta; - deviations[0] += Math.abs(realDelta); - } - } - } - - int compareViolation(Violation v1, Violation v2); - - default void projectRemoveReplica(Cell cell, Replica ri, Consumer opCollector) { - } - - default String postValidate(Condition condition) { - return null; - } - - default Operand getOperand(Operand expected, Object strVal, ComputedType computedType) { - return expected; - } - - Object validate(String name, Object val, boolean isRuleVal); - - /** - * Type details of each variable in policies - */ - public enum Type implements Variable { - @Meta(name = "withCollection", - type = String.class, - isNodeSpecificVal = true, - implementation = WithCollectionVariable.class) - WITH_COLLECTION(), - - @Meta(name = "collection", - type = String.class) - COLL, - @Meta( - name = "shard", - type = String.class, - wildCards = {Policy.EACH, Policy.ANY}) - SHARD(), - - @Meta(name = "replica", - type = Double.class, - min = 0, max = -1, - implementation = ReplicaVariable.class, - computedValues = {ComputedType.EQUAL, ComputedType.PERCENT, ComputedType.ALL}) - REPLICA, - @Meta(name = ImplicitSnitch.PORT, - type = Long.class, - min = 1, - max = 65535, - supportArrayVals = true, - wildCards = Policy.EACH - ) - PORT, - @Meta(name = "ip_1", - type = Long.class, - min = 0, - max = 255, - supportArrayVals = true, - wildCards = Policy.EACH) - IP_1, - @Meta(name = "ip_2", - type = Long.class, - min = 0, - max = 255, - supportArrayVals = true, - wildCards = Policy.EACH) - IP_2, - @Meta(name = "ip_3", - type = Long.class, - min = 0, - max = 255, - supportArrayVals = true, - wildCards = Policy.EACH) - IP_3, - @Meta(name = "ip_4", - type = Long.class, - min = 0, - max = 255, - supportArrayVals = true, - wildCards = Policy.EACH) - IP_4, - @Meta(name = ImplicitSnitch.DISK, - type = Double.class, - min = 0, - isNodeSpecificVal = true, - associatedPerReplicaValue = Variable.coreidxsize, - associatedPerNodeValue = "totaldisk", - implementation = FreeDiskVariable.class, - computedValues = ComputedType.PERCENT) - FREEDISK, - - @Meta(name = "totaldisk", - type = Double.class, - isHidden = true, implementation = VariableBase.TotalDiskVariable.class) - TOTALDISK, - - @Meta(name = Variable.coreidxsize, - type = Double.class, - isNodeSpecificVal = true, - isHidden = true, - min = 0, - implementation = VariableBase.CoreIndexSizeVariable.class, - metricsKey = "INDEX.sizeInBytes") - CORE_IDX, - @Meta(name = ImplicitSnitch.NODEROLE, - type = String.class, - enumVals = "overseer") - NODE_ROLE, - - @Meta(name = ImplicitSnitch.CORES, - type = Double.class, - min = 0, max = -1, - computedValues = {ComputedType.EQUAL, ComputedType.PERCENT}, - implementation = CoresVariable.class) - CORES, - - @Meta(name = ImplicitSnitch.SYSLOADAVG, - type = Double.class, - min = 0, - max = 100, - isNodeSpecificVal = true) - SYSLOADAVG, - - @Meta(name = ImplicitSnitch.HEAPUSAGE, - type = Double.class, - min = 0, - isNodeSpecificVal = true) - HEAPUSAGE, - @Meta(name = "NUMBER", - type = Long.class, - min = 0) - NUMBER, - - @Meta(name = "host", - type = String.class, - wildCards = Policy.EACH, - supportArrayVals = true) - HOST, - - @Meta(name = "STRING", - type = String.class, - wildCards = Policy.EACH, - supportArrayVals = true - ) - SYSPROP, - - @Meta(name = "node", - type = String.class, - isNodeSpecificVal = true, - wildCards = {Policy.ANY, Policy.EACH}, - implementation = NodeVariable.class, - supportArrayVals = true) - NODE, - - @Meta(name = "LAZY", - type = void.class, - implementation = VariableBase.LazyVariable.class) - LAZY, - - @Meta(name = ImplicitSnitch.DISKTYPE, - type = String.class, - enumVals = {"ssd", "rotational"}, - implementation = VariableBase.DiskTypeVariable.class, - supportArrayVals = true) - DISKTYPE; - - public final String tagName; - @SuppressWarnings({"rawtypes"}) - public final Class type; - public Meta meta; - - public final Set vals; - public final Number min; - public final Number max; - public final Boolean additive; - public final Set wildCards; - public final String perReplicaValue; - public final Set associatedPerNodeValues; - public final String metricsAttribute; - public final Set supportedComputedTypes; - final Variable impl; - - - Type() { - try { - meta = Type.class.getField(name()).getAnnotation(Meta.class); - if (meta == null) { - throw new RuntimeException("Invalid type, should have a @Meta annotation " + name()); - } - } catch (NoSuchFieldException e) { - //cannot happen - } - impl = VariableBase.loadImpl(meta, this); - - this.tagName = meta.name(); - this.type = meta.type(); - - this.vals = readSet(meta.enumVals()); - this.max = readNum(meta.max()); - this.min = readNum(meta.min()); - this.perReplicaValue = readStr(meta.associatedPerReplicaValue()); - this.associatedPerNodeValues = readSet(meta.associatedPerNodeValue()); - this.additive = meta.isAdditive(); - this.metricsAttribute = readStr(meta.metricsKey()); - this.supportedComputedTypes = meta.computedValues()[0] == ComputedType.NULL ? - emptySet() : - Set.of(meta.computedValues()); - this.wildCards = readSet(meta.wildCards()); - - } - - public String getTagName() { - return meta.name(); - } - - private String readStr(String s) { - return NULL.equals(s) ? null : s; - } - - private Number readNum(double v) { - return v == -1 ? null : - (Number) validate(null, v, true); - } - - Set readSet(String[] vals) { - if (NULL.equals(vals[0])) return emptySet(); - return Set.of(vals); - } - - @Override - public void getSuggestions(Suggestion.Ctx ctx) { - impl.getSuggestions(ctx); - } - - @Override - public boolean addViolatingReplicas(Violation.Ctx ctx) { - return impl.addViolatingReplicas(ctx); - } - - public Operand getOperand(Operand expected, Object val, ComputedType computedType) { - return impl.getOperand(expected, val, computedType); - } - - - public Object convertVal(Object val) { - return impl.convertVal(val); - } - - public String postValidate(Condition condition) { - return impl.postValidate(condition); - } - - public Object validate(String name, Object val, boolean isRuleVal) { - return impl.validate(name, val, isRuleVal); - } - - /** - * Simulate a replica addition to a node in the cluster - */ - public void projectAddReplica(Cell cell, Replica ri, Consumer opCollector, boolean strictMode) { - impl.projectAddReplica(cell, ri, opCollector, strictMode); - } - - public void projectRemoveReplica(Cell cell, Replica ri, Consumer opCollector) { - impl.projectRemoveReplica(cell, ri, opCollector); - } - - @Override - public int compareViolation(Violation v1, Violation v2) { - return impl.compareViolation(v1, v2); - } - - @Override - public Object computeValue(Policy.Session session, Condition condition, String collection, String shard, String node) { - return impl.computeValue(session, condition, collection, shard, node); - } - - @Override - public void computeDeviation(Policy.Session session, double[] deviations, ReplicaCount replicaCount, SealedClause sealedClause) { - impl.computeDeviation(session, deviations, replicaCount, sealedClause); - } - - - @Override - public boolean match(Object inputVal, Operand op, Object val, String name, Row row) { - return impl.match(inputVal, op, val, name, row); - } - - private static final Map typeByNameMap; - static { - HashMap m = new HashMap<>(); - for (Type t : Type.values()) { - m.put(t.tagName, t); - } - typeByNameMap = unmodifiableMap(m); - } - static Type get(String name) { - return typeByNameMap.get(name); - } - } - - @Target(ElementType.FIELD) - @Retention(RetentionPolicy.RUNTIME) - @interface Meta { - String name(); - - @SuppressWarnings({"rawtypes"}) - Class type(); - - String[] associatedPerNodeValue() default NULL; - - String associatedPerReplicaValue() default NULL; - - String[] enumVals() default NULL; - - String[] wildCards() default NULL; - - boolean isNodeSpecificVal() default false; - - boolean isHidden() default false; - - boolean isAdditive() default true; - - double min() default -1d; - - double max() default -1d; - - boolean supportArrayVals() default false; - - String metricsKey() default NULL; - - @SuppressWarnings({"rawtypes"}) - Class implementation() default void.class; - - ComputedType[] computedValues() default ComputedType.NULL; - } -} diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/VariableBase.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/VariableBase.java deleted file mode 100644 index 746faca3565..00000000000 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/VariableBase.java +++ /dev/null @@ -1,213 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.client.solrj.cloud.autoscaling; - - -import java.util.ArrayList; - -import org.apache.solr.common.cloud.rule.ImplicitSnitch; -import org.apache.solr.common.util.StrUtils; - -import static org.apache.solr.client.solrj.cloud.autoscaling.Clause.parseString; -import static org.apache.solr.client.solrj.cloud.autoscaling.Suggestion.suggestNegativeViolations; -import static org.apache.solr.client.solrj.cloud.autoscaling.Suggestion.suggestPositiveViolations; -import static org.apache.solr.client.solrj.cloud.autoscaling.Variable.Type.FREEDISK; - -public class VariableBase implements Variable { - final Type varType; - - public VariableBase(Type type) { - this.varType = type; - } - - @Override - public void getSuggestions(Suggestion.Ctx ctx) { - if (ctx.violation == null) return; - if (ctx.violation.replicaCountDelta > 0) { - suggestPositiveViolations(ctx); - } else { - suggestNegativeViolations(ctx, ArrayList::new); - } - } - - @Override - public String postValidate(Condition condition) { - if(Clause.IGNORE_TAGS.contains(condition.getName())) return null; - if(condition.getOperand() == Operand.WILDCARD && condition.clause.nodeSetPresent){ - return "#EACH not supported in tags in nodeset"; - } - return null; - } - - static Object getOperandAdjustedValue(Object val, Object original) { - if (original instanceof Condition) { - Condition condition = (Condition) original; - if (condition.computedType == null && isIntegerEquivalent(val)) { - if (condition.op == Operand.LESS_THAN) { - //replica : '<3' - val = val instanceof Long ? - (Long) val - 1 : - (Double) val - 1; - } else if (condition.op == Operand.GREATER_THAN) { - //replica : '>4' - val = val instanceof Long ? - (Long) val + 1 : - (Double) val + 1; - } - } - } - return val; - } - - static boolean isIntegerEquivalent(Object val) { - if (val instanceof Number) { - Number number = (Number) val; - return Math.ceil(number.doubleValue()) == Math.floor(number.doubleValue()); - } else if (val instanceof String) { - try { - double dval = Double.parseDouble((String) val); - return Math.ceil(dval) == Math.floor(dval); - } catch (NumberFormatException e) { - return false; - } - } else { - return false; - } - - } - - public static Type getTagType(String name) { - Type info = Type.get(name); - if (info == null && name.startsWith(ImplicitSnitch.SYSPROP)) info = Type.SYSPROP; - if (info == null && name.startsWith(Clause.METRICS_PREFIX)) info = Type.LAZY; - return info; - } - - @SuppressWarnings({"unchecked"}) - static Variable loadImpl(Meta meta, Type t) { - @SuppressWarnings({"rawtypes"}) - Class implementation = meta.implementation(); - if (implementation == void.class) implementation = VariableBase.class; - try { - return (Variable) implementation.getConstructor(Type.class).newInstance(t); - } catch (Exception e) { - throw new RuntimeException("Unable to instantiate: " + implementation.getName(), e); - } - } - - @Override - public int compareViolation(Violation v1, Violation v2) { - if (v2.replicaCountDelta == null || v1.replicaCountDelta == null) return 0; - if (Math.abs(v1.replicaCountDelta) == Math.abs(v2.replicaCountDelta)) return 0; - return Math.abs(v1.replicaCountDelta) < Math.abs(v2.replicaCountDelta) ? -1 : 1; - } - - @Override - public Object validate(String name, Object val, boolean isRuleVal) { - if (val instanceof Condition) { - Condition condition = (Condition) val; - val = condition.op.readRuleValue(condition); - if (val != condition.val) return val; - } - if (!isRuleVal && "".equals(val) && this.varType.type != String.class) val = -1; - if (name == null) name = this.varType.tagName; - if (varType.type == Double.class) { - Double num = Clause.parseDouble(name, val); - if (isRuleVal) { - if (varType.min != null) - if (Double.compare(num, varType.min.doubleValue()) == -1) - throw new RuntimeException(name + ": " + val + " must be greater than " + varType.min); - if (varType.max != null) - if (Double.compare(num, varType.max.doubleValue()) == 1) - throw new RuntimeException(name + ": " + val + " must be less than " + varType.max); - } - return num; - } else if (varType.type == Long.class) { - Long num = Clause.parseLong(name, val); - if (isRuleVal) { - if (varType.min != null) - if (num < varType.min.longValue()) - throw new RuntimeException(name + ": " + val + " must be greater than " + varType.min); - if (varType.max != null) - if (num > varType.max.longValue()) - throw new RuntimeException(name + ": " + val + " must be less than " + varType.max); - } - return num; - } else if (varType.type == String.class) { - if (isRuleVal && !varType.vals.isEmpty() && !varType.vals.contains(val)) - throw new RuntimeException(name + ": " + val + " must be one of " + StrUtils.join(varType.vals, ',')); - return val; - } else { - throw new RuntimeException("Invalid type "); - } - } - - public static class TotalDiskVariable extends VariableBase { - public TotalDiskVariable(Type type) { - super(type); - } - - @Override - public Object convertVal(Object val) { - return FREEDISK.convertVal(val); - } - } - - public static class CoreIndexSizeVariable extends VariableBase { - public CoreIndexSizeVariable(Type type) { - super(type); - } - - @Override - public Object convertVal(Object val) { - return FREEDISK.convertVal(val); - } - } - - public static class LazyVariable extends VariableBase { - public LazyVariable(Type type) { - super(type); - } - - @Override - public Object validate(String name, Object val, boolean isRuleVal) { - return parseString(val); - } - - @Override - public boolean match(Object inputVal, Operand op, Object val, String name, Row row) { - return op.match(parseString(val), parseString(inputVal)) == Clause.TestStatus.PASS; - } - - @Override - public void getSuggestions(Suggestion.Ctx ctx) { - suggestPositiveViolations(ctx); - } - } - - public static class DiskTypeVariable extends VariableBase { - public DiskTypeVariable(Type type) { - super(type); - } - - @Override - public void getSuggestions(Suggestion.Ctx ctx) { - suggestPositiveViolations(ctx); - } - } -} diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Violation.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Violation.java deleted file mode 100644 index 41221a280ae..00000000000 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Violation.java +++ /dev/null @@ -1,182 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.client.solrj.cloud.autoscaling; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Objects; -import java.util.function.Function; - -import org.apache.solr.common.IteratorWriter; -import org.apache.solr.common.MapWriter; -import org.apache.solr.common.cloud.Replica; -import org.apache.solr.common.util.Utils; - -public class Violation implements MapWriter { - final String shard, coll, node; - final Object actualVal; - Double replicaCountDelta;//how many extra replicas - final Object tagKey; - private final int hash; - private final Clause clause; - private List replicaInfoAndErrs = new ArrayList<>(); - - Violation(SealedClause clause, String coll, String shard, String node, Object actualVal, Double replicaCountDelta, Object tagKey) { - this.clause = clause; - this.shard = shard; - this.coll = coll; - this.node = node; - this.replicaCountDelta = replicaCountDelta; - this.actualVal = actualVal; - this.tagKey = tagKey; - hash = ("" + coll + " " + shard + " " + node + " " + String.valueOf(tagKey) + " " + Utils.toJSONString(getClause().toMap(new HashMap<>()))).hashCode(); - } - - public Violation addReplica(ReplicaInfoAndErr r) { - replicaInfoAndErrs.add(r); - return this; - } - - public List getViolatingReplicas() { - return replicaInfoAndErrs; - } - - public Clause getClause() { - return clause; - } - - public boolean matchShard(String shard) { - if (getClause().getShard().op == Operand.WILDCARD) return true; - return this.shard == null || this.shard.equals(shard); - } - - //if the delta is lower , this violation is less serious - public boolean isLessSerious(Violation that) { - return this.getClause().getTag().varType.compareViolation(this, that) < 0; - } - - @Override - public int hashCode() { - return hash; - } - - public boolean isSimilarViolation(Violation that) { - if (Objects.equals(this.shard, that.shard) && - Objects.equals(this.coll, that.coll) && - Objects.equals(this.node, that.node)) { - if (this.clause.isPerCollectiontag() && that.clause.isPerCollectiontag()) { - return Objects.equals(this.clause.tag.getName(), that.clause.tag.getName()); - } else if (!this.clause.isPerCollectiontag() && !that.clause.isPerCollectiontag()) { - return Objects.equals(this.clause.globalTag.getName(), that.clause.globalTag.getName()) - && Objects.equals(this.node, that.node); - } else { - return false; - } - } else { - return false; - } - - } - - @Override - public boolean equals(Object that) { - if (that instanceof Violation) { - Violation v = (Violation) that; - return Objects.equals(this.shard, v.shard) && - Objects.equals(this.coll, v.coll) && -// Objects.equals(this.node, v.node) && - Objects.equals(this.clause, v.clause) - ; - } - return false; - } - - static class ReplicaInfoAndErr implements MapWriter{ - final Replica replicaInfo; - Double delta; - - ReplicaInfoAndErr(Replica replicaInfo) { - this.replicaInfo = replicaInfo; - } - - public ReplicaInfoAndErr withDelta(Double delta) { - this.delta = delta; - return this; - } - - @Override - public void writeMap(EntryWriter ew) throws IOException { - ew.put("replica", replicaInfo); - ew.putIfNotNull("delta", delta); - } - } - - @Override - public String toString() { - return Utils.toJSONString(Utils.getDeepCopy(toMap(new LinkedHashMap<>()), 5)); - } - - @Override - public void writeMap(EntryWriter ew) throws IOException { - ew.putIfNotNull("collection", coll); - if (!Policy.ANY.equals(shard)) ew.putIfNotNull("shard", shard); - ew.putIfNotNull("node", node); - ew.putIfNotNull("tagKey", tagKey); - ew.putIfNotNull("violation", (MapWriter) ew1 -> { - if (getClause().isPerCollectiontag()) ew1.put("replica", actualVal); - else ew1.put(clause.tag.name, String.valueOf(actualVal)); - ew1.putIfNotNull("delta", replicaCountDelta); - }); - ew.put("clause", getClause()); - if (!replicaInfoAndErrs.isEmpty()) { - ew.put("violatingReplicas", (IteratorWriter) iw -> { - for (ReplicaInfoAndErr replicaInfoAndErr : replicaInfoAndErrs) { - iw.add(replicaInfoAndErr.replicaInfo); - } - }); - } - } - - static class Ctx { - final Function evaluator; - Object tagKey; - Clause clause; - ReplicaCount count; - Violation currentViolation; - List allRows; - List allViolations = new ArrayList<>(); - - public Ctx(Clause clause, List allRows, Function evaluator) { - this.allRows = allRows; - this.clause = clause; - this.evaluator = evaluator; - } - - public Ctx resetAndAddViolation(Object tagKey, ReplicaCount count, Violation currentViolation) { - this.tagKey = tagKey; - this.count = count; - this.currentViolation = currentViolation; - allViolations.add(currentViolation); - this.clause = currentViolation.getClause(); - return this; - } - } -} diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/WithCollectionVariable.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/WithCollectionVariable.java deleted file mode 100644 index f0a9e0493e0..00000000000 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/WithCollectionVariable.java +++ /dev/null @@ -1,170 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.client.solrj.cloud.autoscaling; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.function.Consumer; - -import org.apache.solr.common.cloud.Replica; -import org.apache.solr.common.util.Pair; - -import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDREPLICA; -import static org.apache.solr.common.params.CollectionParams.CollectionAction.MOVEREPLICA; - -/** - * Implements the 'withCollection' variable type - */ -public class WithCollectionVariable extends VariableBase { - - public WithCollectionVariable(Type type) { - super(type); - } - - @Override - public boolean match(Object inputVal, Operand op, Object val, String name, Row row) { - @SuppressWarnings({"unchecked"}) - Map withCollectionMap = (Map) inputVal; - if (withCollectionMap == null || withCollectionMap.isEmpty()) return true; - - Set uniqueColls = new HashSet<>(); - row.forEachReplica(replicaInfo -> uniqueColls.add(replicaInfo.getCollection())); - - for (Map.Entry e : withCollectionMap.entrySet()) { - if (uniqueColls.contains(e.getKey()) && !uniqueColls.contains(e.getValue())) return false; - } - - return true; - } - - public void projectAddReplica(Cell cell, Replica ri, Consumer opCollector, boolean strictMode) { - if (strictMode) { - // we do not want to add a replica of the 'withCollection' in strict mode - return; - } - - @SuppressWarnings({"unchecked"}) - Map withCollectionMap = (Map) cell.val; - if (withCollectionMap == null || withCollectionMap.isEmpty()) return; - - Set uniqueColls = new HashSet<>(); - Row row = cell.row; - row.forEachReplica(replicaInfo -> uniqueColls.add(replicaInfo.getCollection())); - - for (Map.Entry e : withCollectionMap.entrySet()) { - if (uniqueColls.contains(e.getKey()) && !uniqueColls.contains(e.getValue())) { - String withCollection = e.getValue(); - - opCollector.accept(new Row.OperationInfo(withCollection, "shard1", row.node, cell.name, true, Replica.Type.NRT)); - } - } - } - - @Override - public int compareViolation(Violation v1, Violation v2) { - return Integer.compare(v1.getViolatingReplicas().size(), v2.getViolatingReplicas().size()); - } - - public boolean addViolatingReplicas(Violation.Ctx ctx) { - String node = ctx.currentViolation.node; - for (Row row : ctx.allRows) { - if (node.equals(row.node)) { - @SuppressWarnings({"unchecked"}) - Map withCollectionMap = (Map) row.getVal("withCollection"); - if (withCollectionMap != null) { - row.forEachReplica(r -> { - String withCollection = withCollectionMap.get(r.getCollection()); - if (withCollection != null) { - // test whether this row has at least 1 replica of withCollection, else there is a violation - Set uniqueCollections = new HashSet<>(); - row.forEachReplica(replicaInfo -> uniqueCollections.add(replicaInfo.getCollection())); - if (!uniqueCollections.contains(withCollection)) { - ctx.currentViolation.addReplica(new Violation.ReplicaInfoAndErr(r).withDelta(1.0d)); - } - } - }); - ctx.currentViolation.replicaCountDelta = (double) ctx.currentViolation.getViolatingReplicas().size(); - } - } - } - return true; - } - - @Override - public void getSuggestions(Suggestion.Ctx ctx) { - if (ctx.violation.getViolatingReplicas().isEmpty()) return; - - Map nodeValues = ctx.session.nodeStateProvider.getNodeValues(ctx.violation.node, Collections.singleton("withCollection")); - @SuppressWarnings({"unchecked"}) - Map withCollectionsMap = (Map) nodeValues.get("withCollection"); - if (withCollectionsMap == null) return; - - Set uniqueCollections = new HashSet<>(); - for (Violation.ReplicaInfoAndErr replicaInfoAndErr : ctx.violation.getViolatingReplicas()) { - uniqueCollections.add(replicaInfoAndErr.replicaInfo.getCollection()); - } - - collectionLoop: - for (String collection : uniqueCollections) { - String withCollection = withCollectionsMap.get(collection); - if (withCollection == null) continue; - - // can we find a node from which we can move a replica of the `withCollection` - // without creating another violation? - for (Row row : ctx.session.matrix) { - if (ctx.violation.node.equals(row.node)) continue; // filter the violating node - - Set hostedCollections = new HashSet<>(); - row.forEachReplica(replicaInfo -> hostedCollections.add(replicaInfo.getCollection())); - - if (hostedCollections.contains(withCollection) && !hostedCollections.contains(collection)) { - // find the candidate replicas that we can move - List movableReplicas = new ArrayList<>(); - row.forEachReplica(replicaInfo -> { - if (replicaInfo.getCollection().equals(withCollection)) { - movableReplicas.add(replicaInfo); - } - }); - - for (Replica toMove : movableReplicas) { - // candidate source node for a move replica operation - Suggester suggester = ctx.session.getSuggester(MOVEREPLICA) - .forceOperation(true) - .hint(Suggester.Hint.COLL_SHARD, new Pair<>(withCollection, "shard1")) - .hint(Suggester.Hint.SRC_NODE, row.node) - .hint(Suggester.Hint.REPLICA, toMove.getName()) - .hint(Suggester.Hint.TARGET_NODE, ctx.violation.node); - if (ctx.addSuggestion(suggester) != null) - continue collectionLoop; // one suggestion is enough for this collection - } - } - } - - // we could not find a valid move, so we suggest adding a replica - Suggester suggester = ctx.session.getSuggester(ADDREPLICA) - .forceOperation(true) - .hint(Suggester.Hint.COLL_SHARD, new Pair<>(withCollection, "shard1")) - .hint(Suggester.Hint.TARGET_NODE, ctx.violation.node); - ctx.addSuggestion(suggester); - } - } -} diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/package-info.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/package-info.java deleted file mode 100644 index 620f57d3eb9..00000000000 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/package-info.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * Common classes for autoscaling parsing filtering nodes and sorting - */ - -package org.apache.solr.client.solrj.cloud.autoscaling; - diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java index 1ce3beee8ae..1c0c225de6a 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java @@ -34,9 +34,6 @@ import java.util.stream.Collectors; import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.cloud.NodeStateProvider; -import org.apache.solr.client.solrj.cloud.autoscaling.Row; -import org.apache.solr.client.solrj.cloud.autoscaling.Variable.Type; -import org.apache.solr.client.solrj.cloud.autoscaling.VariableBase; import org.apache.solr.client.solrj.request.GenericSolrRequest; import org.apache.solr.client.solrj.response.SimpleSolrResponse; import org.apache.solr.common.MapWriter; @@ -60,15 +57,12 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import static java.util.Collections.emptyMap; -import static org.apache.solr.client.solrj.cloud.autoscaling.Clause.METRICS_PREFIX; -import static org.apache.solr.client.solrj.cloud.autoscaling.Variable.Type.FREEDISK; -import static org.apache.solr.client.solrj.cloud.autoscaling.Variable.Type.TOTALDISK; -import static org.apache.solr.client.solrj.cloud.autoscaling.Variable.Type.WITH_COLLECTION; /** * The real {@link NodeStateProvider}, which communicates with Solr via SolrJ. */ public class SolrClientNodeStateProvider implements NodeStateProvider, MapWriter { + public static final String METRICS_PREFIX = "metrics:"; private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); //only for debugging public static SolrClientNodeStateProvider INST; @@ -128,40 +122,40 @@ public class SolrClientNodeStateProvider implements NodeStateProvider, MapWriter public Map getNodeValues(String node, Collection tags) { Map tagVals = fetchTagValues(node, tags); nodeVsTags.put(node, tagVals); - if (tags.contains(WITH_COLLECTION.tagName)) { - tagVals.put(WITH_COLLECTION.tagName, withCollectionsMap); - } return tagVals; } protected Map fetchTagValues(String node, Collection tags) { - AutoScalingSnitch snitch = new AutoScalingSnitch(); + MetricsFetchingSnitch snitch = new MetricsFetchingSnitch(); ClientSnitchCtx ctx = new ClientSnitchCtx(null, node, snitchSession, solrClient); snitch.getTags(node, new HashSet<>(tags), ctx); return ctx.getTags(); } public void forEachReplica(String node, Consumer consumer){ - Row.forEachReplica(nodeVsCollectionVsShardVsReplicaInfo.get(node), consumer); + forEachReplica(nodeVsCollectionVsShardVsReplicaInfo.get(node), consumer); } - + public static void forEachReplica(Map>> collectionVsShardVsReplicas, Consumer consumer) { + collectionVsShardVsReplicas.forEach((coll, shardVsReplicas) -> shardVsReplicas + .forEach((shard, replicaInfos) -> { + for (int i = 0; i < replicaInfos.size(); i++) { + Replica r = replicaInfos.get(i); + consumer.accept(r); + } + })); + } @Override public Map>> getReplicaInfo(String node, Collection keys) { @SuppressWarnings({"unchecked"}) Map>> result = nodeVsCollectionVsShardVsReplicaInfo.computeIfAbsent(node, Utils.NEW_HASHMAP_FUN); if (!keys.isEmpty()) { Map> metricsKeyVsTagReplica = new HashMap<>(); - Row.forEachReplica(result, r -> { + forEachReplica(result, r -> { for (String key : keys) { if (r.getProperties().containsKey(key)) continue;// it's already collected String perReplicaMetricsKey = "solr.core." + r.getCollection() + "." + r.getShard() + "." + Utils.parseMetricsReplicaName(r.getCollection(), r.getCoreName()) + ":"; - Type tagType = VariableBase.getTagType(key); String perReplicaValue = key; - if (tagType != null) { - perReplicaValue = tagType.metricsAttribute; - perReplicaValue = perReplicaValue == null ? key : perReplicaValue; - } perReplicaMetricsKey += perReplicaValue; metricsKeyVsTagReplica.put(perReplicaMetricsKey, new Pair<>(key, r)); } @@ -171,8 +165,6 @@ public class SolrClientNodeStateProvider implements NodeStateProvider, MapWriter Map tagValues = fetchReplicaMetrics(node, metricsKeyVsTagReplica); tagValues.forEach((k, o) -> { Pair p = metricsKeyVsTagReplica.get(k); - Type validator = VariableBase.getTagType(p.first()); - if (validator != null) o = validator.convertVal(o); if (p.second() != null) p.second().getProperties().put(p.first(), o); }); @@ -183,7 +175,7 @@ public class SolrClientNodeStateProvider implements NodeStateProvider, MapWriter protected Map fetchReplicaMetrics(String node, Map> metricsKeyVsTagReplica) { Map collect = metricsKeyVsTagReplica.entrySet().stream() - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getKey)); + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getKey)); ClientSnitchCtx ctx = new ClientSnitchCtx(null, null, emptyMap(), solrClient); fetchReplicaMetrics(node, ctx, collect); return ctx.getTags(); @@ -216,8 +208,8 @@ public class SolrClientNodeStateProvider implements NodeStateProvider, MapWriter } - //uses metrics API to get node information - static class AutoScalingSnitch extends ImplicitSnitch { + // uses metrics API to get node information + static class MetricsFetchingSnitch extends ImplicitSnitch { @Override protected void getRemoteInfo(String solrNode, Set requestedTags, SnitchContext ctx) { if (!((ClientSnitchCtx)ctx).isNodeAlive(solrNode)) return; @@ -252,7 +244,7 @@ public class SolrClientNodeStateProvider implements NodeStateProvider, MapWriter groups.add("solr.node"); prefixes.add("CONTAINER.fs.usableSpace"); } - if (requestedTags.contains(TOTALDISK.tagName)) { + if (requestedTags.contains (Variable.TOTALDISK.tagName)) { groups.add("solr.node"); prefixes.add("CONTAINER.fs.totalSpace"); } @@ -278,13 +270,13 @@ public class SolrClientNodeStateProvider implements NodeStateProvider, MapWriter SimpleSolrResponse rsp = snitchContext.invokeWithRetry(solrNode, CommonParams.METRICS_PATH, params); NamedList metrics = (NamedList) rsp.nl.get("metrics"); - if (requestedTags.contains(FREEDISK.tagName)) { + if (requestedTags.contains(Variable.FREEDISK.tagName)) { Object n = Utils.getObjectByPath(metrics, true, "solr.node/CONTAINER.fs.usableSpace"); - if (n != null) ctx.getTags().put(FREEDISK.tagName, FREEDISK.convertVal(n)); + if (n != null) ctx.getTags().put(Variable.FREEDISK.tagName, Variable.FREEDISK.convertVal(n)); } - if (requestedTags.contains(TOTALDISK.tagName)) { + if (requestedTags.contains(Variable.TOTALDISK.tagName)) { Object n = Utils.getObjectByPath(metrics, true, "solr.node/CONTAINER.fs.totalSpace"); - if (n != null) ctx.getTags().put(TOTALDISK.tagName, TOTALDISK.convertVal(n)); + if (n != null) ctx.getTags().put(Variable.TOTALDISK.tagName, Variable.TOTALDISK.convertVal(n)); } if (requestedTags.contains(CORES)) { NamedList node = (NamedList) metrics.get("solr.node"); @@ -315,7 +307,7 @@ public class SolrClientNodeStateProvider implements NodeStateProvider, MapWriter } static class ClientSnitchCtx - extends SnitchContext { + extends SnitchContext { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); ZkClientClusterStateProvider zkClientClusterStateProvider; @@ -395,4 +387,37 @@ public class SolrClientNodeStateProvider implements NodeStateProvider, MapWriter } } + + public enum Variable { + FREEDISK("freedisk", null, Double.class), + TOTALDISK("totaldisk", null, Double.class), + CORE_IDX("INDEX.sizeInGB", "INDEX.sizeInBytes", Double.class) + ; + + + public final String tagName, metricsAttribute; + @SuppressWarnings("rawtypes") + public final Class type; + + + @SuppressWarnings("rawtypes") + Variable(String tagName, String metricsAttribute, Class type) { + this.tagName = tagName; + this.metricsAttribute = metricsAttribute; + this.type = type; + } + + public Object convertVal(Object val) { + if(val instanceof String) { + return Double.valueOf((String)val); + } else if (val instanceof Number) { + Number num = (Number)val; + return num.doubleValue(); + + } else { + throw new IllegalArgumentException("Unknown type : "+val); + } + + } + } } diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/ZkDistribStateManager.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/ZkDistribStateManager.java index b15445b75ec..d2c3f40c501 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/ZkDistribStateManager.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/ZkDistribStateManager.java @@ -18,21 +18,16 @@ package org.apache.solr.client.solrj.impl; import java.io.IOException; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.NoSuchElementException; -import org.apache.solr.client.solrj.cloud.autoscaling.AlreadyExistsException; -import org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig; -import org.apache.solr.client.solrj.cloud.autoscaling.BadVersionException; +import org.apache.solr.client.solrj.cloud.AlreadyExistsException; +import org.apache.solr.client.solrj.cloud.BadVersionException; import org.apache.solr.client.solrj.cloud.DistribStateManager; -import org.apache.solr.client.solrj.cloud.autoscaling.NotEmptyException; -import org.apache.solr.client.solrj.cloud.autoscaling.VersionedData; +import org.apache.solr.client.solrj.cloud.NotEmptyException; +import org.apache.solr.client.solrj.cloud.VersionedData; import org.apache.solr.common.AlreadyClosedException; import org.apache.solr.common.cloud.SolrZkClient; -import org.apache.solr.common.cloud.ZkStateReader; -import org.apache.solr.common.params.AutoScalingParams; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.Op; @@ -40,8 +35,6 @@ import org.apache.zookeeper.OpResult; import org.apache.zookeeper.Watcher; import org.apache.zookeeper.data.Stat; -import static org.apache.solr.common.util.Utils.fromJSON; - /** * Implementation of {@link DistribStateManager} that uses Zookeeper. */ @@ -180,25 +173,6 @@ public class ZkDistribStateManager implements DistribStateManager { } } - @Override - @SuppressWarnings({"unchecked"}) - public AutoScalingConfig getAutoScalingConfig(Watcher watcher) throws InterruptedException, IOException { - Map map = new HashMap<>(); - Stat stat = new Stat(); - try { - byte[] bytes = zkClient.getData(ZkStateReader.SOLR_AUTOSCALING_CONF_PATH, watcher, stat, true); - if (bytes != null && bytes.length > 0) { - map = (Map) fromJSON(bytes); - } - } catch (KeeperException.NoNodeException e) { - // ignore - } catch (KeeperException e) { - throw new IOException(e); - } - map.put(AutoScalingParams.ZK_VERSION, stat.getVersion()); - return new AutoScalingConfig(map); - } - @Override public void close() throws IOException { diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java index bbfe4747707..e0955c2fe31 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java @@ -55,10 +55,8 @@ import org.apache.solr.common.params.ShardParams; import org.apache.solr.common.params.SolrParams; import org.apache.solr.common.util.NamedList; -import static org.apache.solr.client.solrj.cloud.autoscaling.Policy.POLICY; import static org.apache.solr.common.cloud.DocCollection.RULE; import static org.apache.solr.common.cloud.DocCollection.SNITCH; -import static org.apache.solr.common.cloud.ZkStateReader.AUTO_ADD_REPLICAS; import static org.apache.solr.common.cloud.ZkStateReader.NRT_REPLICAS; import static org.apache.solr.common.cloud.ZkStateReader.PULL_REPLICAS; import static org.apache.solr.common.cloud.ZkStateReader.READ_ONLY; @@ -88,8 +86,6 @@ public abstract class CollectionAdminRequest RULE, SNITCH, REPLICATION_FACTOR, - AUTO_ADD_REPLICAS, - POLICY, COLL_CONF, WITH_COLLECTION, COLOCATED_WITH, @@ -443,10 +439,8 @@ public abstract class CollectionAdminRequest protected Integer tlogReplicas; protected Properties properties; - protected Boolean autoAddReplicas; protected String alias; protected String[] rule , snitch; - protected String withCollection; /** Constructor intended for typical use cases */ protected Create(String collection, String config, Integer numShards, Integer numNrtReplicas, Integer numTlogReplicas, Integer numPullReplicas) { // TODO: maybe add other constructors @@ -476,7 +470,6 @@ public abstract class CollectionAdminRequest public Create setCreateNodeSet(String nodeSet) { this.createNodeSet = nodeSet; return this; } public Create setRouterName(String routerName) { this.routerName = routerName; return this; } public Create setRouterField(String routerField) { this.routerField = routerField; return this; } - public Create setAutoAddReplicas(boolean autoAddReplicas) { this.autoAddReplicas = autoAddReplicas; return this; } public Create setNrtReplicas(Integer nrtReplicas) { this.nrtReplicas = nrtReplicas; return this;} public Create setTlogReplicas(Integer tlogReplicas) { this.tlogReplicas = tlogReplicas; return this;} public Create setPullReplicas(Integer pullReplicas) { this.pullReplicas = pullReplicas; return this;} @@ -498,7 +491,6 @@ public abstract class CollectionAdminRequest public Integer getReplicationFactor() { return getNumNrtReplicas(); } public Integer getNumNrtReplicas() { return nrtReplicas; } - public Boolean getAutoAddReplicas() { return autoAddReplicas; } public Integer getNumTlogReplicas() {return tlogReplicas;} public Integer getNumPullReplicas() {return pullReplicas;} @@ -562,9 +554,6 @@ public abstract class CollectionAdminRequest if (nrtReplicas != null) { params.set( ZkStateReader.NRT_REPLICAS, nrtReplicas); } - if (autoAddReplicas != null) { - params.set(ZkStateReader.AUTO_ADD_REPLICAS, autoAddReplicas); - } if (properties != null) { addProperties(params, properties); } @@ -576,8 +565,6 @@ public abstract class CollectionAdminRequest } if (rule != null) params.set(DocCollection.RULE, rule); if (snitch != null) params.set(DocCollection.SNITCH, snitch); - params.setNonNull(POLICY, policy); - params.setNonNull(WITH_COLLECTION, withCollection); params.setNonNull(ALIAS, alias); return params; } @@ -586,15 +573,6 @@ public abstract class CollectionAdminRequest this.policy = policy; return this; } - - public String getWithCollection() { - return withCollection; - } - - public Create setWithCollection(String withCollection) { - this.withCollection = withCollection; - return this; - } } /** @@ -687,19 +665,6 @@ public abstract class CollectionAdminRequest return params; } - } - public static class UtilizeNode extends AsyncCollectionAdminRequest { - protected String node; - - public UtilizeNode(String node) { - super(CollectionAction.UTILIZENODE); - this.node = node; - } - @Override - public SolrParams getParams() { - return ((ModifiableSolrParams) super.getParams()).set(CoreAdminParams.NODE, node); - } - } public static MoveReplica moveReplica(String collection, String replica, String targetNode) { @@ -1079,7 +1044,6 @@ public abstract class CollectionAdminRequest protected Integer nrtReplicas; protected Integer tlogReplicas; protected Integer pullReplicas; - protected Boolean autoAddReplicas; protected Optional createNodeSet = Optional.empty(); protected Optional createNodeSetShuffle = Optional.empty(); protected Properties properties; @@ -1139,9 +1103,6 @@ public abstract class CollectionAdminRequest public Integer getPullReplicas() { return pullReplicas; } public Restore setPullReplicas(Integer pullReplicas) { this.pullReplicas = pullReplicas; return this; } - public Boolean getAutoAddReplicas() { return autoAddReplicas; } - public Restore setAutoAddReplicas(boolean autoAddReplicas) { this.autoAddReplicas = autoAddReplicas; return this; } - public Properties getProperties() { return properties; } @@ -1172,9 +1133,6 @@ public abstract class CollectionAdminRequest if (tlogReplicas != null) { params.set(ZkStateReader.TLOG_REPLICAS, tlogReplicas); } - if (autoAddReplicas != null) { - params.set(ZkStateReader.AUTO_ADD_REPLICAS, autoAddReplicas); - } if (properties != null) { addProperties(params, properties); } diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionApiMapping.java b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionApiMapping.java index 5df18e02c47..721dbd1c099 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionApiMapping.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionApiMapping.java @@ -207,11 +207,6 @@ public class CollectionApiMapping { POST, null, "set-obj-property", null), - UTILIZE_NODE(CLUSTER_CMD, - POST, - UTILIZENODE, - "utilize-node",null), - BACKUP_COLLECTION(COLLECTIONS_COMMANDS, POST, BACKUP, diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/DocCollection.java b/solr/solrj/src/java/org/apache/solr/common/cloud/DocCollection.java index 3a4a28a8fb9..9552c619ce9 100644 --- a/solr/solrj/src/java/org/apache/solr/common/cloud/DocCollection.java +++ b/solr/solrj/src/java/org/apache/solr/common/cloud/DocCollection.java @@ -29,10 +29,8 @@ import java.util.Set; import java.util.function.BiConsumer; import java.util.function.BiPredicate; -import org.apache.solr.client.solrj.cloud.autoscaling.Policy; import org.noggit.JSONWriter; -import static org.apache.solr.common.cloud.ZkStateReader.AUTO_ADD_REPLICAS; import static org.apache.solr.common.cloud.ZkStateReader.NRT_REPLICAS; import static org.apache.solr.common.cloud.ZkStateReader.PULL_REPLICAS; import static org.apache.solr.common.cloud.ZkStateReader.READ_ONLY; @@ -64,8 +62,6 @@ public class DocCollection extends ZkNodeProps implements Iterable { private final Integer numNrtReplicas; private final Integer numTlogReplicas; private final Integer numPullReplicas; - private final Boolean autoAddReplicas; - private final String policy; private final Boolean readOnly; public DocCollection(String name, Map slices, Map props, DocRouter router) { @@ -92,9 +88,6 @@ public class DocCollection extends ZkNodeProps implements Iterable { this.numNrtReplicas = (Integer) verifyProp(props, NRT_REPLICAS, 0); this.numTlogReplicas = (Integer) verifyProp(props, TLOG_REPLICAS, 0); this.numPullReplicas = (Integer) verifyProp(props, PULL_REPLICAS, 0); - Boolean autoAddReplicas = (Boolean) verifyProp(props, AUTO_ADD_REPLICAS); - this.policy = (String) props.get(Policy.POLICY); - this.autoAddReplicas = autoAddReplicas == null ? Boolean.FALSE : autoAddReplicas; Boolean readOnly = (Boolean) verifyProp(props, READ_ONLY); this.readOnly = readOnly == null ? Boolean.FALSE : readOnly; @@ -147,7 +140,6 @@ public class DocCollection extends ZkNodeProps implements Iterable { case PULL_REPLICAS: case TLOG_REPLICAS: return Integer.parseInt(o.toString()); - case AUTO_ADD_REPLICAS: case READ_ONLY: return Boolean.parseBoolean(o.toString()); case "snitch": @@ -247,10 +239,6 @@ public class DocCollection extends ZkNodeProps implements Iterable { return replicationFactor; } - public boolean getAutoAddReplicas() { - return autoAddReplicas; - } - public DocRouter getRouter() { return router; } @@ -394,13 +382,6 @@ public class DocCollection extends ZkNodeProps implements Iterable { return numPullReplicas; } - /** - * @return the policy associated with this collection if any - */ - public String getPolicyName() { - return policy; - } - public int getExpectedReplicaCount(Replica.Type type, int def) { Integer result = null; if (type == Replica.Type.NRT) result = numNrtReplicas; diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java index 51392592b43..d49a39c0a16 100644 --- a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java +++ b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java @@ -23,7 +23,6 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.EnumSet; -import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; @@ -45,13 +44,11 @@ import java.util.function.Predicate; import java.util.function.UnaryOperator; import java.util.stream.Collectors; -import org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig; import org.apache.solr.common.AlreadyClosedException; import org.apache.solr.common.Callable; import org.apache.solr.common.SolrCloseable; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException.ErrorCode; -import org.apache.solr.common.params.AutoScalingParams; import org.apache.solr.common.params.CollectionAdminParams; import org.apache.solr.common.params.CoreAdminParams; import org.apache.solr.common.util.ExecutorUtil; @@ -116,16 +113,10 @@ public class ZkStateReader implements SolrCloseable { public static final String COLLECTION_PROPS_ZKNODE = "collectionprops.json"; public static final String REJOIN_AT_HEAD_PROP = "rejoinAtHead"; public static final String SOLR_SECURITY_CONF_PATH = "/security.json"; - public static final String SOLR_AUTOSCALING_CONF_PATH = "/autoscaling.json"; - public static final String SOLR_AUTOSCALING_EVENTS_PATH = "/autoscaling/events"; - public static final String SOLR_AUTOSCALING_TRIGGER_STATE_PATH = "/autoscaling/triggerState"; - public static final String SOLR_AUTOSCALING_NODE_ADDED_PATH = "/autoscaling/nodeAdded"; - public static final String SOLR_AUTOSCALING_NODE_LOST_PATH = "/autoscaling/nodeLost"; public static final String SOLR_PKGS_PATH = "/packages.json"; public static final String DEFAULT_SHARD_PREFERENCES = "defaultShardPreferences"; public static final String REPLICATION_FACTOR = "replicationFactor"; - public static final String AUTO_ADD_REPLICAS = "autoAddReplicas"; public static final String MAX_CORES_PER_NODE = "maxCoresPerNode"; public static final String PULL_REPLICAS = "pullReplicas"; public static final String NRT_REPLICAS = "nrtReplicas"; @@ -217,40 +208,6 @@ public class ZkStateReader implements SolrCloseable { private Future collectionPropsCacheCleaner; // only kept to identify if the cleaner has already been started. - /** - * Get current {@link AutoScalingConfig}. - * - * @return current configuration from autoscaling.json. NOTE: - * this data is retrieved from ZK on each call. - */ - public AutoScalingConfig getAutoScalingConfig() throws KeeperException, InterruptedException { - return getAutoScalingConfig(null); - } - - /** - * Get current {@link AutoScalingConfig}. - * - * @param watcher optional {@link Watcher} to set on a znode to watch for config changes. - * @return current configuration from autoscaling.json. NOTE: - * this data is retrieved from ZK on each call. - */ - @SuppressWarnings({"unchecked"}) - public AutoScalingConfig getAutoScalingConfig(Watcher watcher) throws KeeperException, InterruptedException { - Stat stat = new Stat(); - - Map map = new HashMap<>(); - try { - byte[] bytes = zkClient.getData(SOLR_AUTOSCALING_CONF_PATH, watcher, stat, true); - if (bytes != null && bytes.length > 0) { - map = (Map) fromJSON(bytes); - } - } catch (KeeperException.NoNodeException e) { - // ignore - } - map.put(AutoScalingParams.ZK_VERSION, stat.getVersion()); - return new AutoScalingConfig(map); - } - private static class CollectionWatch { int coreRefCount = 0; @@ -264,7 +221,6 @@ public class ZkStateReader implements SolrCloseable { public static final Set KNOWN_CLUSTER_PROPS = Set.of( URL_SCHEME, - AUTO_ADD_REPLICAS, CoreAdminParams.BACKUP_LOCATION, DEFAULT_SHARD_PREFERENCES, MAX_CORES_PER_NODE, diff --git a/solr/solrj/src/java/org/apache/solr/common/params/AutoScalingParams.java b/solr/solrj/src/java/org/apache/solr/common/params/AutoScalingParams.java deleted file mode 100644 index e626fef1827..00000000000 --- a/solr/solrj/src/java/org/apache/solr/common/params/AutoScalingParams.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.common.params; - -/** - * Requests parameters for autoscaling. - */ -public interface AutoScalingParams { - - // parameters - String DIAGNOSTICS = "diagnostics"; - String SUGGESTIONS = "suggestions"; - String NAME = "name"; - String TRIGGER = "trigger"; - String EVENT = "event"; - String ACTIONS = "actions"; - String WAIT_FOR = "waitFor"; - String LOWER_BOUND = "lowerBound"; - String UPPER_BOUND = "upperBound"; - String STAGE = "stage"; - String CLASS = "class"; - String ENABLED = "enabled"; - String RESUME_AT = "resumeAt"; - String BEFORE_ACTION = "beforeAction"; - String AFTER_ACTION = "afterAction"; - String TIMEOUT = "timeout"; - String COLLECTION = "collection"; - String SHARD = "shard"; - String REPLICA = "replica"; - String NODE = "node"; - String HANDLER = "handler"; - String RATE = "rate"; - String REMOVE_LISTENERS = "removeListeners"; - String ZK_VERSION = "zkVersion"; - String METRIC = "metric"; - String ABOVE = "above"; - String BELOW = "below"; - String PREFERRED_OP = "preferredOperation"; - String REPLICA_TYPE = "replicaType"; - - // commands - String CMD_SET_TRIGGER = "set-trigger"; - String CMD_REMOVE_TRIGGER = "remove-trigger"; - String CMD_SET_LISTENER = "set-listener"; - String CMD_REMOVE_LISTENER = "remove-listener"; - String CMD_SUSPEND_TRIGGER = "suspend-trigger"; - String CMD_RESUME_TRIGGER = "resume-trigger"; - String CMD_SET_POLICY = "set-policy"; - String CMD_REMOVE_POLICY = "remove-policy"; - String CMD_SET_CLUSTER_PREFERENCES = "set-cluster-preferences"; - String CMD_SET_CLUSTER_POLICY = "set-cluster-policy"; - String CMD_SET_PROPERTIES = "set-properties"; - - // properties - String TRIGGER_SCHEDULE_DELAY_SECONDS = "triggerScheduleDelaySeconds"; - String TRIGGER_COOLDOWN_PERIOD_SECONDS = "triggerCooldownPeriodSeconds"; - String TRIGGER_CORE_POOL_SIZE = "triggerCorePoolSize"; - String MAX_COMPUTE_OPERATIONS = "maxComputeOperations"; - - @Deprecated - String ACTION_THROTTLE_PERIOD_SECONDS = "actionThrottlePeriodSeconds"; -} diff --git a/solr/solrj/src/java/org/apache/solr/common/params/CollectionAdminParams.java b/solr/solrj/src/java/org/apache/solr/common/params/CollectionAdminParams.java index 5b6af2e3074..3fc11616f37 100644 --- a/solr/solrj/src/java/org/apache/solr/common/params/CollectionAdminParams.java +++ b/solr/solrj/src/java/org/apache/solr/common/params/CollectionAdminParams.java @@ -104,12 +104,6 @@ public interface CollectionAdminParams { */ String CLUSTER = "cluster"; - /** - * This cluster property decides whether Solr should use the legacy round-robin replica placement strategy - * or the autoscaling policy based strategy to assign replicas to nodes. The default is false. - */ - String USE_LEGACY_REPLICA_ASSIGNMENT = "useLegacyReplicaAssignment"; - /** * When creating a collection create also a specified alias. */ diff --git a/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java b/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java index fce970fa340..f55cb70e2f5 100644 --- a/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java +++ b/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java @@ -116,7 +116,6 @@ public interface CollectionParams { CREATESNAPSHOT(true, LockLevel.COLLECTION), DELETESNAPSHOT(true, LockLevel.COLLECTION), LISTSNAPSHOTS(false, LockLevel.NONE), - UTILIZENODE(false, LockLevel.NONE), //only for testing. it just waits for specified time // these are not exposed via collection API commands // but the overseer is aware of these tasks diff --git a/solr/solrj/src/java/org/apache/solr/common/params/CommonParams.java b/solr/solrj/src/java/org/apache/solr/common/params/CommonParams.java index c817abe2c4a..6f1d4e8783b 100644 --- a/solr/solrj/src/java/org/apache/solr/common/params/CommonParams.java +++ b/solr/solrj/src/java/org/apache/solr/common/params/CommonParams.java @@ -189,10 +189,6 @@ public interface CommonParams { String SYSTEM_INFO_PATH = "/admin/info/system"; String METRICS_PATH = "/admin/metrics"; String METRICS_HISTORY_PATH = "/admin/metrics/history"; - String AUTOSCALING_PATH = "/admin/autoscaling"; - String AUTOSCALING_HISTORY_PATH = "/admin/autoscaling/history"; - String AUTOSCALING_DIAGNOSTICS_PATH = "/admin/autoscaling/diagnostics"; - String AUTOSCALING_SUGGESTIONS_PATH = "/admin/autoscaling/suggestions"; String STATUS = "status"; @@ -208,11 +204,7 @@ public interface CommonParams { AUTHC_PATH, AUTHZ_PATH, METRICS_PATH, - METRICS_HISTORY_PATH, - AUTOSCALING_PATH, - AUTOSCALING_HISTORY_PATH, - AUTOSCALING_DIAGNOSTICS_PATH, - AUTOSCALING_SUGGESTIONS_PATH); + METRICS_HISTORY_PATH); String APISPEC_LOCATION = "apispec/"; String INTROSPECT = "/_introspect"; diff --git a/solr/solrj/src/java/org/apache/solr/common/util/Utils.java b/solr/solrj/src/java/org/apache/solr/common/util/Utils.java index 82d84c15812..331048070fc 100644 --- a/solr/solrj/src/java/org/apache/solr/common/util/Utils.java +++ b/solr/solrj/src/java/org/apache/solr/common/util/Utils.java @@ -64,7 +64,7 @@ import org.apache.http.client.HttpClient; import org.apache.http.client.methods.HttpGet; import org.apache.http.util.EntityUtils; import org.apache.solr.client.solrj.cloud.DistribStateManager; -import org.apache.solr.client.solrj.cloud.autoscaling.VersionedData; +import org.apache.solr.client.solrj.cloud.VersionedData; import org.apache.solr.client.solrj.impl.BinaryRequestWriter; import org.apache.solr.common.IteratorWriter; import org.apache.solr.common.LinkedHashMapWriter; diff --git a/solr/solrj/src/resources/apispec/autoscaling.Commands.json b/solr/solrj/src/resources/apispec/autoscaling.Commands.json deleted file mode 100644 index bcbab89a0aa..00000000000 --- a/solr/solrj/src/resources/apispec/autoscaling.Commands.json +++ /dev/null @@ -1,199 +0,0 @@ -{ - "documentation": "https://lucene.apache.org/solr/guide/solrcloud-autoscaling-api.html", - "description": "The Scaling API provides API for adding cluster level scaling rules, triggers and event listeners", - "methods": [ - "GET", - "POST" - ], - "url": { - "paths": [ - "/cluster/autoscaling", - "/cluster/autoscaling/diagnostics", - "/cluster/autoscaling/suggestions" - ] - }, - "commands": { - "set-trigger": { - "type": "object", - "description": "The set-trigger command allows you to add and update triggers on various system metrics", - "properties": { - "name": { - "type": "string", - "description": "The name of the trigger" - }, - "event": { - "type": "string", - "description": "The event type on which to set a trigger" - }, - "waitFor": { - "type": "string", - "description": "The amount of time to wait after the trigger condition is satisfied before trigger is activated" - }, - "lowerBound": { - "type": "number", - "description": "The lower bound of the condition below which the trigger is activated" - }, - "upperBound": { - "type": "number", - "description": "The upper bound of the condition below which the trigger is activated" - }, - "enabled": { - "type": "boolean", - "description": "The state of the trigger" - }, - "actions": { - "type": "array", - "description": "The actions to be performed in sequence when the trigger is activated", - "items": { - "type": "object" - } - } - }, - "required": [ - "name", - "event" - ] - }, - "remove-trigger": { - "description": "Remove a trigger", - "type": "object", - "properties": { - "name": { - "type": "string", - "description": "The name of the trigger to be removed" - }, - "removeListeners": { - "type": "boolean", - "description": "If true, all listeners of this triggers are deleted together with the trigger" - } - }, - "required": [ - "name" - ] - }, - "set-listener": { - "description": "The set-listener command lets you add a listener to a trigger", - "type": "object", - "properties": { - "name": { - "type": "string", - "description": "The name of the listener" - }, - "trigger": { - "type": "string", - "description": "The name of the trigger to listen to" - }, - "stage": { - "type": "array", - "description": "The stage of the trigger for which to listen", - "items": { - "type": "string" - } - }, - "beforeAction": { - "type": "array", - "description": "The name of the action before which the listener should be notified", - "items": { - "type": "string" - } - }, - "afterAction": { - "type": "array", - "description": "The name of the action after which the listener should be notified", - "items": { - "type": "string" - } - }, - "class": { - "type": "string", - "description": "The listener class to be notified once the given stage of the given trigger is activated" - } - }, - "required": [ - "name", - "trigger", - "class" - ], - "additionalProperties": true - }, - "remove-listener": { - "description": "Remove a listener", - "type": "object", - "properties": { - "name": { - "type": "string", - "description": "The name of the listener to be removed" - } - }, - "required": [ - "name" - ] - }, - "suspend-trigger": { - "description": "Pauses a trigger until an explicit resume is invoked or if the optional timeout expires", - "type": "object", - "properties": { - "name": { - "type": "string", - "description": "The name of the trigger to be suspended or '#EACH' to suspend all triggers" - }, - "timeout": { - "type": "string", - "description": "Optional timeout after which all triggers are resumed automatically" - } - }, - "required": [ - "name" - ] - }, - "resume-trigger": { - "description": "Resumes a suspended trigger", - "type": "object", - "properties": { - "name": { - "type": "string", - "description": "The name of the trigger to be resumed or '#EACH' to resume all triggers" - } - }, - "required": [ - "name" - ] - }, - "set-policy": { - "type": "object", - "description": "The set-policy command allows you to add and update policies that apply to collections", - /* "patternProperties": { - "^.+$": { - "type": "array" - } - },*/ - "additionalProperties": true - }, - "set-cluster-policy": { - "type": "array", - "description": "The set-cluster-policy command allows you to add and update cluster-level policy that acts as the base for all collection level policies, if any" - }, - "set-cluster-preferences": { - "type": "array", - "description": "The set-cluster-preferences command allows you to add and update cluster-level preferences that are used to sort nodes for selection in cluster operations" - }, - "remove-policy": { - "description": "Remove a policy", - "type": "object", - "properties": { - "name": { - "type": "string", - "description": "The name of the policy to be removed" - } - }, - "required": [ - "name" - ] - }, - "set-properties": { - "type": "object", - "description": "The set-properties command allows you to add and update properties used by autoscaling framework itself", - "additionalProperties": true - } - } -} diff --git a/solr/solrj/src/resources/apispec/autoscaling.history.json b/solr/solrj/src/resources/apispec/autoscaling.history.json deleted file mode 100644 index dadd6cb9a36..00000000000 --- a/solr/solrj/src/resources/apispec/autoscaling.history.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "documentation": "https://lucene.apache.org/solr/guide/solrcloud-autoscaling-api.html", - "description": "Autoscaling history handler allows retrieving past autoscaling events recorded in the .system collection.", - "methods": [ - "GET" - ], - "url": { - "paths": [ - "/cluster/autoscaling/history" - ], - "params": { - "collection": { - "type": "string", - "description": "Collection where autoscaling events are stored, '.system' by default.", - "default": ".system" - }, - "q": { - "type": "string", - "description": "Arbitrary query to limit the selected events.", - "default": "*:*" - }, - "sort": { - "type": "string", - "description": "Sorting criteria for returned events.", - "default": "id asc" - }, - "action": { - "type": "string", - "description": "Trigger action name to select" - }, - "message": { - "type": "string", - "description": "Event message to select" - }, - "trigger": { - "type": "string", - "description": "Trigger name to select" - }, - "type": { - "type": "string", - "description": "Event type to select" - }, - "node": { - "type": "string", - "description": "Node name to select" - }, - "stage": { - "type": "string", - "description": "Processing stage to select" - }, - "beforeAction": { - "type": "string", - "description": "Action name to select events before the action was executed" - }, - "afterAction": { - "type": "string", - "description": "Action name to select events after the action was executed" - } - } - } -} diff --git a/solr/solrj/src/resources/apispec/cluster.Commands.json b/solr/solrj/src/resources/apispec/cluster.Commands.json index b72b67cabb2..04428b3fa0a 100644 --- a/solr/solrj/src/resources/apispec/cluster.Commands.json +++ b/solr/solrj/src/resources/apispec/cluster.Commands.json @@ -78,9 +78,6 @@ "urlScheme": { "type": "string" }, - "autoAddReplicas": { - "type": "boolean" - }, "maxCoresPerNode": { "type": "boolean" }, @@ -144,28 +141,6 @@ } } } - }, - "utilize-node": { - "type": "object", - "documentation": "https://lucene.apache.org/solr/guide/cluster-node-management.html#utilizenode", - "description": "use a replica to reduce load", - "properties": { - "node": { - "type": "string", - "description": "The name of the node" - }, - "async": { - "type": [ - "string", - "boolean", - "null" - ], - "description": "The value of the property. If the value is empty or null, the property is unset." - } - }, - "required": [ - "name" - ] } } } diff --git a/solr/solrj/src/resources/apispec/collections.Commands.json b/solr/solrj/src/resources/apispec/collections.Commands.json index 2eb700f4c42..c3f1491494b 100644 --- a/solr/solrj/src/resources/apispec/collections.Commands.json +++ b/solr/solrj/src/resources/apispec/collections.Commands.json @@ -76,12 +76,6 @@ "type": "boolean", "description": "Controls whether or not the shard-replicas created for this collection will be assigned to the nodes specified by the nodeSet property in a sequential manner, or if the list of nodes should be shuffled prior to creating individual replicas. A 'false' value makes the results of a collection creation predictable and gives more exact control over the location of the individual shard-replicas, but 'true' can be a better choice for ensuring replicas are distributed evenly across nodes. This property is ignored if nodeSet is not also specified." }, - "autoAddReplicas": { - "type": "boolean", - "description": "When set to true, enables auto addition of replicas when the number of active replicas falls below the value set for replicationFactor.", - "documentation": "https://lucene.apache.org/solr/guide/solrcloud-autoscaling-auto-add-replicas.html", - "default": "false" - }, "rule": { "type": "array", "documentation": "https://lucene.apache.org/solr/guide/rule-based-replica-placement.html", @@ -98,11 +92,6 @@ "type": "string" } }, - "policy": { - "type": "string", - "documentation": "https://lucene.apache.org/solr/guide/solrcloud-autoscaling-policy-preferences.html", - "description": "Name of the collection-level policy" - }, "properties": { "type": "object", "documentation": "https://lucene.apache.org/solr/guide/defining-core-properties.html", diff --git a/solr/solrj/src/resources/apispec/collections.collection.Commands.modify.json b/solr/solrj/src/resources/apispec/collections.collection.Commands.modify.json index 4206b332e3b..2afae173cea 100644 --- a/solr/solrj/src/resources/apispec/collections.collection.Commands.modify.json +++ b/solr/solrj/src/resources/apispec/collections.collection.Commands.modify.json @@ -19,17 +19,6 @@ "type": "string" } }, - "policy": { - "type": "string", - "documentation": "https://lucene.apache.org/solr/guide/solrcloud-autoscaling-policy-preferences.html", - "description": "Name of the collection-level policy" - }, - "autoAddReplicas": { - "type": "boolean", - "description": "When set to true, enables auto addition of replicas when the number of active replicas falls below the value set for replicationFactor.", - "documentation": "https://lucene.apache.org/solr/guide/solrcloud-autoscaling-auto-add-replicas.html", - "default": "false" - }, "replicationFactor": { "type": "integer", "description": "The number of replicas to be created for each shard. Replicas are physical copies of each shard, acting as failover for the shard. Note that changing this value on an existing collection does not automatically add more replicas to the collection. However, it will allow add-replica commands to succeed." diff --git a/solr/solrj/src/test-files/solrj/solr/autoscaling/testAddMissingReplica.json b/solr/solrj/src/test-files/solrj/solr/autoscaling/testAddMissingReplica.json deleted file mode 100644 index 6c202a51d89..00000000000 --- a/solr/solrj/src/test-files/solrj/solr/autoscaling/testAddMissingReplica.json +++ /dev/null @@ -1,122 +0,0 @@ -{ - "responseHeader":{ - "status":0, - "QTime":23}, - "diagnostics":{ - "sortedNodes":[{ - "node":"10.0.0.80:7575_solr", - "isLive":true, - "cores":1.0, - "freedisk":673.2483978271484, - "totaldisk":1037.938980102539, - "replicas":{"gettingstarted":{"shard1":[{ - "core_node4":{ - "core":"gettingstarted_shard1_replica_n1", - "shard":"shard1", - "collection":"gettingstarted", - "node_name":"10.0.0.80:7575_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://10.0.0.80:7575/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":6.426125764846802E-8}}]}}} - ,{ - "node":"10.0.0.80:8983_solr", - "isLive":true, - "cores":1.0, - "freedisk":673.2483940124512, - "totaldisk":1037.938980102539, - "replicas":{"gettingstarted":{"shard2":[{ - "core_node8":{ - "core":"gettingstarted_shard2_replica_n5", - "shard":"shard2", - "collection":"gettingstarted", - "node_name":"10.0.0.80:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://10.0.0.80:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":6.426125764846802E-8}}]}}} - ,{ - "node":"10.0.0.80:8984_solr", - "isLive":true, - "cores":1.0, - "freedisk":673.2483901977539, - "totaldisk":1037.938980102539, - "replicas":{"gettingstarted":{"shard1":[{ - "core_node6":{ - "core":"gettingstarted_shard1_replica_n2", - "shard":"shard1", - "collection":"gettingstarted", - "node_name":"10.0.0.80:8984_solr", - "type":"NRT", - "base_url":"http://10.0.0.80:8984/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":6.426125764846802E-8}}]}}}], - "liveNodes":["10.0.0.80:7575_solr", - "10.0.0.80:8983_solr", - "10.0.0.80:8984_solr"], - "violations":[], - "config":{ - "cluster-preferences":[{ - "minimize":"cores", - "precision":1} - ,{ - "maximize":"freedisk"}]}}, - - "cluster":{ - "collections":{ - "gettingstarted":{ - "pullReplicas":"0", - "replicationFactor":"2", - "shards":{ - "shard1":{ - "range":"80000000-ffffffff", - "state":"active", - "replicas":{ - "core_node4":{ - "core":"gettingstarted_shard1_replica_n1", - "base_url":"http://10.0.0.80:7575/solr", - "node_name":"10.0.0.80:7575_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node6":{ - "core":"gettingstarted_shard1_replica_n2", - "base_url":"http://10.0.0.80:8984/solr", - "node_name":"10.0.0.80:8984_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}}, - "shard2":{ - "range":"0-7fffffff", - "state":"active", - "replicas":{ - "core_node7":{ - "core":"gettingstarted_shard2_replica_n3", - "base_url":"http://10.0.0.80:7574/solr", - "node_name":"10.0.0.80:7574_solr", - "state":"down", - "type":"NRT", - "force_set_state":"false"}, - "core_node8":{ - "core":"gettingstarted_shard2_replica_n5", - "base_url":"http://10.0.0.80:8983/solr", - "node_name":"10.0.0.80:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}}, - "router":{"name":"compositeId"}, - "autoAddReplicas":"false", - "nrtReplicas":"2", - "tlogReplicas":"0", - "znodeVersion":12, - "configName":"gettingstarted"}}, - "live_nodes":["10.0.0.80:8983_solr", - "10.0.0.80:7575_solr", - "10.0.0.80:8984_solr"]}} diff --git a/solr/solrj/src/test-files/solrj/solr/autoscaling/testAddTooManyPerPolicy.json b/solr/solrj/src/test-files/solrj/solr/autoscaling/testAddTooManyPerPolicy.json deleted file mode 100644 index 23ab471f9b6..00000000000 --- a/solr/solrj/src/test-files/solrj/solr/autoscaling/testAddTooManyPerPolicy.json +++ /dev/null @@ -1,129 +0,0 @@ -{ - "diagnostics": { - "sortedNodes": [ - { - "node": "127.0.0.1:61737_solr", - "isLive": true, - "cores": 2.0, - "freedisk": 184.94042205810547, - "totaldisk": 465.62699127197266, - "replicas": { - "TooManyPerPolicy": { - "shard1": [ - { - "core_node3": { - "core": "TooManyPerPolicy_shard1_replica_n1", - "shard": "shard1", - "collection": "TooManyPerPolicy", - "node_name": "127.0.0.1:61737_solr", - "type": "NRT", - "leader": "true", - "base_url": "http://127.0.0.1:61737/solr", - "state": "active", - "force_set_state": "false", - "INDEX.sizeInGB": 6.426125764846802E-8 - } - }, - { - "core_node8": { - "core": "TooManyPerPolicy_shard1_replica_n7", - "shard": "shard1", - "collection": "TooManyPerPolicy", - "node_name": "127.0.0.1:61737_solr", - "type": "NRT", - "base_url": "http://127.0.0.1:61737/solr", - "state": "down", - "force_set_state": "false", - "INDEX.sizeInGB": 6.426125764846802E-8 - }}]}}}, - { - "node": "127.0.0.1:61738_solr", - "isLive": true, - "cores": 1.0, - "freedisk": 184.94042205810547, - "totaldisk": 465.62699127197266, - "replicas": { - "TooManyPerPolicy": { - "shard3": [ - { - "core_node6": { - "core": "TooManyPerPolicy_shard3_replica_n4", - "shard": "shard3", - "collection": "TooManyPerPolicy", - "node_name": "127.0.0.1:61738_solr", - "type": "NRT", - "leader": "true", - "base_url": "http://127.0.0.1:61738/solr", - "state": "active", - "force_set_state": "false", - "INDEX.sizeInGB": 6.426125764846802E-8 - }}]}}}, - { - "node": "127.0.0.1:61739_solr", - "isLive": true, - "cores": 1.0, - "freedisk": 184.94042205810547, - "totaldisk": 465.62699127197266, - "replicas": { - "TooManyPerPolicy": { - "shard2": [ - { - "core_node5": { - "core": "TooManyPerPolicy_shard2_replica_n2", - "shard": "shard2", - "collection": "TooManyPerPolicy", - "node_name": "127.0.0.1:61739_solr", - "type": "NRT", - "leader": "true", - "base_url": "http://127.0.0.1:61739/solr", - "state": "active", - "force_set_state": "false", - "INDEX.sizeInGB": 6.426125764846802E-8 - }}]}}} - ], - "liveNodes": [ - "127.0.0.1:61738_solr", - "127.0.0.1:61739_solr", - "127.0.0.1:61737_solr" - ], - "violations": [ - { - "collection": "TooManyPerPolicy", - "node": "127.0.0.1:61737_solr", - "tagKey": "127.0.0.1:61737_solr", - "violation": { - "replica": {"NRT": 2, "count": 2}, - "delta": 1.0}, - "clause": { - "replica": "<2", "shard": "#ANY", "node": "#ANY", "strict": true, "collection": "TooManyPerPolicy"}, - "violatingReplicas": [ - { - "core_node3": { - "core": "TooManyPerPolicy_shard1_replica_n1", - "shard": "shard1", - "collection": "TooManyPerPolicy", - "node_name": "127.0.0.1:61737_solr", - "type": "NRT", - "leader": "true", - "base_url": "http://127.0.0.1:61737/solr", - "state": "active", - "force_set_state": "false", - "INDEX.sizeInGB": 6.426125764846802E-8}}, - { - "core_node8": { - "core": "TooManyPerPolicy_shard1_replica_n7", - "shard": "shard1", - "collection": "TooManyPerPolicy", - "node_name": "127.0.0.1:61737_solr", - "type": "NRT", - "base_url": "http://127.0.0.1:61737/solr", - "state": "down", - "force_set_state": "false", - "INDEX.sizeInGB": 6.426125764846802E-8 - }}]}], - "config": { - "cluster-preferences": [ - {"minimize": "cores", "precision": 1}, - {"maximize": "freedisk"}], - "cluster-policy": [ - {"replica": "<2", "shard": "#ANY", "node": "#ANY", "strict": true}]}}} \ No newline at end of file diff --git a/solr/solrj/src/test-files/solrj/solr/autoscaling/testAutoScalingHandlerFailure.json b/solr/solrj/src/test-files/solrj/solr/autoscaling/testAutoScalingHandlerFailure.json deleted file mode 100644 index 97368c2302e..00000000000 --- a/solr/solrj/src/test-files/solrj/solr/autoscaling/testAutoScalingHandlerFailure.json +++ /dev/null @@ -1,141 +0,0 @@ -{"diagnostics":{ - "sortedNodes":[{ - "node":"127.0.0.1:63191_solr", - "isLive":true, - "cores":3.0, - "freedisk":680.908073425293, - "heapUsage":24.97510064011647, - "sysLoadAvg":272.75390625, - "totaldisk":1037.938980102539, - "replicas":{"readApiTestViolations":{"shard1":[{"core_node5":{ - "core":"readApiTestViolations_shard1_replica_n2", - "leader":"true", - "base_url":"https://127.0.0.1:63191/solr", - "node_name":"127.0.0.1:63191_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "INDEX.sizeInGB":6.426125764846802E-8, - "shard":"shard1", - "collection":"readApiTestViolations"}}, - {"core_node7":{ - "core":"readApiTestViolations_shard1_replica_n4", - "base_url":"https://127.0.0.1:63191/solr", - "node_name":"127.0.0.1:63191_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "INDEX.sizeInGB":6.426125764846802E-8, - "shard":"shard1", - "collection":"readApiTestViolations"}}, - {"core_node12":{ - "core":"readApiTestViolations_shard1_replica_n10", - "base_url":"https://127.0.0.1:63191/solr", - "node_name":"127.0.0.1:63191_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "INDEX.sizeInGB":6.426125764846802E-8, - "shard":"shard1", - "collection":"readApiTestViolations"}}]}}}, - { - "node":"127.0.0.1:63192_solr", - "isLive":true, - "cores":3.0, - "freedisk":680.908073425293, - "heapUsage":24.98878807983566, - "sysLoadAvg":272.75390625, - "totaldisk":1037.938980102539, - "replicas":{"readApiTestViolations":{"shard1":[{"core_node3":{ - "core":"readApiTestViolations_shard1_replica_n1", - "base_url":"https://127.0.0.1:63192/solr", - "node_name":"127.0.0.1:63192_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "INDEX.sizeInGB":6.426125764846802E-8, - "shard":"shard1", - "collection":"readApiTestViolations"}}, - {"core_node9":{ - "core":"readApiTestViolations_shard1_replica_n6", - "base_url":"https://127.0.0.1:63192/solr", - "node_name":"127.0.0.1:63192_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "INDEX.sizeInGB":6.426125764846802E-8, - "shard":"shard1", - "collection":"readApiTestViolations"}}, - {"core_node11":{ - "core":"readApiTestViolations_shard1_replica_n8", - "base_url":"https://127.0.0.1:63192/solr", - "node_name":"127.0.0.1:63192_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "INDEX.sizeInGB":6.426125764846802E-8, - "shard":"shard1", - "collection":"readApiTestViolations"}}]}}}, - { - "node":"127.0.0.1:63219_solr", - "isLive":true, - "cores":0.0, - "freedisk":680.908073425293, - "heapUsage":24.98878807983566, - "sysLoadAvg":272.75390625, - "totaldisk":1037.938980102539, - "replicas":{}}], - "liveNodes":["127.0.0.1:63191_solr", - "127.0.0.1:63192_solr", - "127.0.0.1:63219_solr"], - "violations":[{ - "collection":"readApiTestViolations", - "shard":"shard1", - "node":"127.0.0.1:63191_solr", - "violation":{ - "replica":{ - "NRT":3, - "count":3}, - "delta":2.0}, - "clause":{ - "replica":"<3", - "shard":"#EACH", - "node":"#ANY", - "collection":"readApiTestViolations"}}, - { - "collection":"readApiTestViolations", - "shard":"shard1", - "node":"127.0.0.1:63192_solr", - "violation":{ - "replica":{ - "NRT":3, - "count":3}, - "delta":2.0}, - "clause":{ - "replica":"<2", - "shard":"#EACH", - "node":"#ANY", - "collection":"readApiTestViolations"}}], - "config":{ - "cluster-preferences":[{ - "minimize":"cores", - "precision":3}, - { - "maximize":"freedisk", - "precision":100}, - { - "minimize":"sysLoadAvg", - "precision":10}, - { - "minimize":"heapUsage", - "precision":10}], - "cluster-policy":[{ - "cores":"<10", - "node":"#ANY"}, - { - "replica":"<2", - "shard":"#EACH", - "node":"#ANY"}, - { - "nodeRole":"overseer", - "replica":0}]}}} \ No newline at end of file diff --git a/solr/solrj/src/test-files/solrj/solr/autoscaling/testAutoscalingPreferencesUsedWithNoPolicy.json b/solr/solrj/src/test-files/solrj/solr/autoscaling/testAutoscalingPreferencesUsedWithNoPolicy.json deleted file mode 100644 index 4974eefc1e5..00000000000 --- a/solr/solrj/src/test-files/solrj/solr/autoscaling/testAutoscalingPreferencesUsedWithNoPolicy.json +++ /dev/null @@ -1,52 +0,0 @@ -{ - "liveNodes":["node1:8983", - "node2:8984", - "node3:8985"], - "replicaInfo":{"node1:8983":{"c1":{ - "s1":[{"r1":{ - "type":"NRT", - "INDEX.sizeInGB":"1100"}}, - {"r2":{"type":"NRT"}}], - "s2":[{"r1":{ - "type":"NRT", - "INDEX.sizeInGB":"1100"}}, - {"r2":{"type":"NRT"}}]}}}, - "nodeValues":{ - "node1:8983":{ - "cores":4, - "freedisk":300, - "totaldisk":4700, - "port":8983}, - "node2:8984":{ - "cores":0, - "freedisk":1000, - "totaldisk":1200, - "port":8984}, - "node3:8985":{ - "cores":0, - "freedisk":1651, - "totaldisk":1700, - "port":8985}}, -"clusterstate":{"c1":{ - "router":{"name":"compositeId"}, - "shards":{ - "s1":{"replicas":{ - "r1":{ - "type":"NRT", - "node_name":"node1:8983", - "state":"active", - "leader":"true"}, - "r2":{ - "type":"NRT", - "node_name":"node1:8983", - "state":"active"}}}, - "s2":{"replicas":{ - "r1":{ - "type":"NRT", - "node_name":"node1:8983", - "state":"active", - "leader":"true"}, - "r2":{ - "type":"NRT", - "node_name":"node1:8983", - "state":"active"}}}}}}} diff --git a/solr/solrj/src/test-files/solrj/solr/autoscaling/testComputePlanAfterNodeAdded.json b/solr/solrj/src/test-files/solrj/solr/autoscaling/testComputePlanAfterNodeAdded.json deleted file mode 100644 index 2171c38a205..00000000000 --- a/solr/solrj/src/test-files/solrj/solr/autoscaling/testComputePlanAfterNodeAdded.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "liveNodes":["127.0.0.1:51078_solr", - "127.0.0.1:51147_solr"], - "replicaInfo":{ - "127.0.0.1:51147_solr":{}, - "127.0.0.1:51078_solr":{"testNodeAdded":{"shard1":[{"core_node3":{"type":"NRT"}}, - {"core_node4":{"type":"NRT"}}]}}}, - "nodeValues":{ - "127.0.0.1:51147_solr":{ - "node":"127.0.0.1:51147_solr", - "cores":0, - "freedisk":880.5428657531738}, - "127.0.0.1:51078_solr":{ - "node":"127.0.0.1:51078_solr", - "cores":2, - "freedisk":880.5428695678711}}} \ No newline at end of file diff --git a/solr/solrj/src/test-files/solrj/solr/autoscaling/testCoresSuggestions.json b/solr/solrj/src/test-files/solrj/solr/autoscaling/testCoresSuggestions.json deleted file mode 100644 index a6901db839f..00000000000 --- a/solr/solrj/src/test-files/solrj/solr/autoscaling/testCoresSuggestions.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "liveNodes":["10.0.0.6:7574_solr", - "10.0.0.6:8983_solr"], - "replicaInfo":{ - "10.0.0.6:7574_solr":{}, - "10.0.0.6:8983_solr":{"mycoll1":{ - "shard1":[{"core_node1":{"type":"NRT"}}], - "shard2":[{"core_node2":{"type":"NRT"}}], - "shard3":[{"core_node3":{"type":"NRT"}}], - "shard4":[{"core_node4":{"type":"NRT"}}]}}}, - "nodeValues":{ - "10.0.0.6:7574_solr":{ - "node":"10.0.0.6:7574_solr", - "cores":0}, - "10.0.0.6:8983_solr":{ - "node":"10.0.0.6:8983_solr", - "cores":4}}} diff --git a/solr/solrj/src/test-files/solrj/solr/autoscaling/testCreateCollectionWithEmptyPolicy.json b/solr/solrj/src/test-files/solrj/solr/autoscaling/testCreateCollectionWithEmptyPolicy.json deleted file mode 100644 index 1d9a2b44440..00000000000 --- a/solr/solrj/src/test-files/solrj/solr/autoscaling/testCreateCollectionWithEmptyPolicy.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "diagnostics":{ - "sortedNodes":[{ - "node":"127.0.0.1:49469_solr", - "isLive":true, - "cores":0.0, - "freedisk":672.6827087402344, - "totaldisk":1037.938980102539, - "replicas":{}} - ,{ - "node":"127.0.0.1:49470_solr", - "isLive":true, - "cores":0.0, - "freedisk":672.6827087402344, - "totaldisk":1037.938980102539, - "replicas":{}}], - "liveNodes":["127.0.0.1:49469_solr", - "127.0.0.1:49470_solr"], - "violations":[], - "config":{}}} \ No newline at end of file diff --git a/solr/solrj/src/test-files/solrj/solr/autoscaling/testDiskSpaceHint.json b/solr/solrj/src/test-files/solrj/solr/autoscaling/testDiskSpaceHint.json deleted file mode 100644 index edfcf7fa257..00000000000 --- a/solr/solrj/src/test-files/solrj/solr/autoscaling/testDiskSpaceHint.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "liveNodes":["127.0.0.1:51078_solr", - "127.0.0.1:51147_solr"], - "replicaInfo":{ - "127.0.0.1:51147_solr":{}, - "127.0.0.1:51078_solr":{"testNodeAdded":{"shard1":[{"core_node3":{"type":"NRT"}}, - {"core_node4":{"type":"NRT"}}]}}}, - "nodeValues":{ - "127.0.0.1:51147_solr":{ - "node":"127.0.0.1:51147_solr", - "cores":0, - "freedisk":100}, - "127.0.0.1:51078_solr":{ - "node":"127.0.0.1:51078_solr", - "cores":2, - "freedisk":200}}} \ No newline at end of file diff --git a/solr/solrj/src/test-files/solrj/solr/autoscaling/testEmptyCollection.json b/solr/solrj/src/test-files/solrj/solr/autoscaling/testEmptyCollection.json deleted file mode 100644 index 0da18b4bd63..00000000000 --- a/solr/solrj/src/test-files/solrj/solr/autoscaling/testEmptyCollection.json +++ /dev/null @@ -1,26 +0,0 @@ -{"clusterstate":{"test_empty_collection":{ - "pullReplicas":0, - "replicationFactor":1, - "shards":{ - "shard2":{ - "range":"0-7fffffff", - "state":"active", - "replicas":{}}, - "shard1_1":{ - "range":"c0000000-ffffffff", - "state":"active", - "replicas":{}, - "stateTimestamp":"1540603288721383849"}, - "shard1_0":{ - "range":"80000000-bfffffff", - "state":"active", - "replicas":{}, - "stateTimestamp":"1540603288721468797"}}, - "router":{ - "name":"compositeId", - "field":"shard_s"}, - "autoAddReplicas":"false", - "nrtReplicas":1, - "tlogReplicas":0}}, - - "replicaInfo":{}} diff --git a/solr/solrj/src/test-files/solrj/solr/autoscaling/testEqualOnNonNode.json b/solr/solrj/src/test-files/solrj/solr/autoscaling/testEqualOnNonNode.json deleted file mode 100644 index c2dc13d84c8..00000000000 --- a/solr/solrj/src/test-files/solrj/solr/autoscaling/testEqualOnNonNode.json +++ /dev/null @@ -1,83 +0,0 @@ -[{"coll1":{ - "router":{"name":"compositeId"}, - "shards":{ - "shard1":{ - "range":"80000000-ffffffff", - "replicas":{ - "r1":{//east - "core":"r1", - "base_url":"http://10.0.0.4:8983/solr", - "node_name":"node1", - "state":"active"}, - "r2":{//west - "core":"r2", - "base_url":"http://10.0.0.4:7574/solr", - "node_name":"node2", - "state":"active"}}}, - "shard2":{ - "range":"0-7fffffff", - "replicas":{ - "r3":{//east - "core":"r3", - "base_url":"http://10.0.0.4:8983/solr", - "node_name":"node1", - "state":"active"}, - "r4":{//west - "core":"r4", - "base_url":"http://10.0.0.4:8987/solr", - "node_name":"node4", - "state":"active"}, - "r6":{//east - "core":"r6", - "base_url":"http://10.0.0.4:8989/solr", - "node_name":"node3", - "state":"active"}, - "r5":{//east - "core":"r5", - "base_url":"http://10.0.0.4:8983/solr", - "node_name":"node1", - "state":"active"}}}}}}, - {"nodeValues":{ - "node1":{ - "cores":3, - "freedisk":700, - "totaldisk":1000, - "sysprop.zone":"east"}, - "node2":{ - "cores":1, - "freedisk":900, - "totaldisk":1000, - "sysprop.zone":"west"}, - "node3":{ - "cores":1, - "freedisk":900, - "totaldisk":1000, - "sysprop.zone":"east"}, - "node4":{ - "cores":1, - "freedisk":900, - "totaldisk":1000, - "sysprop.zone":"west"}, - "node5":{ - "cores":0, - "freedisk":1000, - "totaldisk":1000, - "sysprop.zone":"west"}}, - "replicaValues":[{ - "INDEX.sizeInGB":100, - "core":"r1"}, - { - "INDEX.sizeInGB":100, - "core":"r2"}, - { - "INDEX.sizeInGB":100, - "core":"r3"}, - { - "INDEX.sizeInGB":100, - "core":"r4"}, - { - "INDEX.sizeInGB":100, - "core":"r5"}, - { - "INDEX.sizeInGB":100, - "core":"r6"}]}] \ No newline at end of file diff --git a/solr/solrj/src/test-files/solrj/solr/autoscaling/testFreeDiskDeviation.json b/solr/solrj/src/test-files/solrj/solr/autoscaling/testFreeDiskDeviation.json deleted file mode 100644 index 10e367000a7..00000000000 --- a/solr/solrj/src/test-files/solrj/solr/autoscaling/testFreeDiskDeviation.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "liveNodes": [ - "node1", - "node2", - "node3" - ], - "replicaInfo": { - "node1": { - "mycoll1": { - "shard3": [{"r3": {"type": "NRT", "INDEX.sizeInGB": 700}}], - "shard4": [{"r4": {"type": "NRT", "INDEX.sizeInGB": 400}}] - } - }, - "node2": { - "mycoll1": { - "shard1": [{"r1": {"type": "NRT", "INDEX.sizeInGB": 450}}], - "shard2": [{"r2": {"type": "NRT", "INDEX.sizeInGB": 750}}] - } - }, - "node3": { - "mycoll2": { - "shard1": [{"r1": {"type": "NRT", "INDEX.sizeInGB": 250}}] - } - } - }, - "nodeValues": { - "node1": {"node": "node1", "cores": 2, "freedisk": 900}, - "node2": {"node": "node2", "cores": 2, "freedisk": 800}, - "node3": {"node": "node3", "cores": 1, "freedisk": 1200} - }, - "config": { - "cluster-policy": [{"replica":"<2", "shard":"#EACH", "node":"#ANY"}, - {"replica": "#ALL", "freedisk": ">700", "strict": false}] - } -} \ No newline at end of file diff --git a/solr/solrj/src/test-files/solrj/solr/autoscaling/testFreeDiskSuggestions.json b/solr/solrj/src/test-files/solrj/solr/autoscaling/testFreeDiskSuggestions.json deleted file mode 100644 index 2b8897bb21b..00000000000 --- a/solr/solrj/src/test-files/solrj/solr/autoscaling/testFreeDiskSuggestions.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "liveNodes":["node1", - "node2"], - "replicaInfo":{ - "node1":{}, - "node2":{"mycoll1":{ - "shard1":[{"r1":{ - "type":"NRT", - "INDEX.sizeInGB":900}}], - "shard2":[{"r2":{ - "type":"NRT", - "INDEX.sizeInGB":300}}], - "shard3":[{"r3":{ - "type":"NRT", - "INDEX.sizeInGB":200}}], - "shard4":[{"r4":{ - "type":"NRT", - "INDEX.sizeInGB":100}}]}}}, - "nodeValues":{ - "node1":{ - "node":"node1", - "cores":0, - "freedisk":2000}, - "node2":{ - "node":"node2", - "cores":4, - "freedisk":500}}} \ No newline at end of file diff --git a/solr/solrj/src/test-files/solrj/solr/autoscaling/testFreediskPercentage.json b/solr/solrj/src/test-files/solrj/solr/autoscaling/testFreediskPercentage.json deleted file mode 100644 index 174f862d4d6..00000000000 --- a/solr/solrj/src/test-files/solrj/solr/autoscaling/testFreediskPercentage.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "liveNodes":["node1:8983", - "node2:8984", - "node3:8985"], - "replicaInfo":{"node1:8983":{"c1":{ - "s1":[{"r1":{"type":"NRT"}}, - {"r2":{"type":"NRT"}}], - "s2":[{"r1":{"type":"NRT"}}, - {"r2":{"type":"NRT"}}]}}}, - "nodeValues":{ - "node1:8983":{ - "cores":4, - "freedisk":230, - "totaldisk":800, - "port":8983}, - "node2:8984":{ - "cores":0, - "freedisk":1000, - "totaldisk":1200, - "port":8984}, - "node3:8985":{ - "cores":0, - "freedisk":1500, - "totaldisk":1700, - "port":8985}}} \ No newline at end of file diff --git a/solr/solrj/src/test-files/solrj/solr/autoscaling/testHostAttribute.json b/solr/solrj/src/test-files/solrj/solr/autoscaling/testHostAttribute.json deleted file mode 100644 index 7afd7ade732..00000000000 --- a/solr/solrj/src/test-files/solrj/solr/autoscaling/testHostAttribute.json +++ /dev/null @@ -1,119 +0,0 @@ -{"diagnostics":{ - "sortedNodes":[{ - "node":"127.0.0.191:63191_solr", - "isLive":true, - "cores":3.0, - "host":"127.0.0.191", - "sysprop.zone":"east", - "freedisk":1727.1459312438965, - "heapUsage":24.97510064011647, - "sysLoadAvg":272.75390625, - "totaldisk":1037.938980102539, - "replicas":{"zonesTest":{"shard1":[{"core_node5":{ - "core":"zonesTest_shard1_replica_n2", - "leader":"true", - "base_url":"https://127.0.0.191:63191/solr", - "node_name":"127.0.0.191:63191_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "INDEX.sizeInGB":6.426125764846802E-8, - "shard":"shard1", - "collection":"zonesTest"}}, - {"core_node7":{ - "core":"zonesTest_shard1_replica_n4", - "base_url":"https://127.0.0.191:63191/solr", - "node_name":"127.0.0.191:63191_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "INDEX.sizeInGB":6.426125764846802E-8, - "shard":"shard1", - "collection":"zonesTest"}}, - {"core_node12":{ - "core":"zonesTest_shard1_replica_n10", - "base_url":"https://127.0.0.191:63191/solr", - "node_name":"127.0.0.191:63191_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "INDEX.sizeInGB":6.426125764846802E-8, - "shard":"shard1", - "collection":"zonesTest"}}]}}}, - { - "node":"127.0.0.191:63192_solr", - "isLive":true, - "cores":3.0, - "host":"127.0.0.191", - "sysprop.zone":"east", - "freedisk":1727.1459312438965, - "heapUsage":24.98878807983566, - "sysLoadAvg":272.75390625, - "totaldisk":1037.938980102539, - "replicas":{"zonesTest":{"shard2":[{"core_node3":{ - "core":"zonesTest_shard1_replica_n1", - "base_url":"https://127.0.0.191:63192/solr", - "node_name":"127.0.0.191:63192_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "INDEX.sizeInGB":6.426125764846802E-8, - "shard":"shard2", - "collection":"zonesTest"}}, - {"core_node9":{ - "core":"zonesTest_shard1_replica_n6", - "base_url":"https://127.0.0.191:63192/solr", - "node_name":"127.0.0.191:63192_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "INDEX.sizeInGB":6.426125764846802E-8, - "shard":"shard2", - "collection":"zonesTest"}}, - {"core_node11":{ - "core":"zonesTest_shard1_replica_n8", - "base_url":"https://127.0.0.191:63192/solr", - "node_name":"127.0.0.191:63192_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "INDEX.sizeInGB":6.426125764846802E-8, - "shard":"shard2", - "collection":"zonesTest"}}]}}}, - { - "node":"127.0.0.219:63219_solr", - "isLive":true, - "cores":0.0, - "host":"127.0.0.219", - "sysprop.zone":"west", - "freedisk":1768.6174201965332, - "heapUsage":24.98878807983566, - "sysLoadAvg":272.75390625, - "totaldisk":1037.938980102539, - "replicas":{}}, - { - "node":"127.0.0.219:63229_solr", - "isLive":true, - "cores":0.0, - "host":"127.0.0.219", - "sysprop.zone":"west", - "freedisk":1768.6174201965332, - "heapUsage":24.98878807983566, - "sysLoadAvg":272.75390625, - "totaldisk":1037.938980102539, - "replicas":{}}], - "liveNodes":["127.0.0.191:63191_solr", - "127.0.0.191:63192_solr", - "127.0.0.219:63219_solr", - "127.0.0.219:63229_solr"], - "config":{ - "cluster-preferences":[{ - "minimize":"cores", - "precision":1}, - {"maximize":"freedisk", - "precision":100}], - "cluster-policy":[{ - "replica":"<3", - "shard":"#EACH", - "host":["127.0.0.191", - "127.0.0.219:63219"]}]}}} \ No newline at end of file diff --git a/solr/solrj/src/test-files/solrj/solr/autoscaling/testInfiniteLoop.json b/solr/solrj/src/test-files/solrj/solr/autoscaling/testInfiniteLoop.json deleted file mode 100644 index dbc425d5f66..00000000000 --- a/solr/solrj/src/test-files/solrj/solr/autoscaling/testInfiniteLoop.json +++ /dev/null @@ -1,13173 +0,0 @@ -{ - "diagnostics":{ - "sortedNodes":[{ - "node":"NODE_S_A0a3cb04c83b693d80:8983_solr", - "isLive":true, - "cores":20.0, - "freedisk":693.357551574707, - "totaldisk":999.755859375, - "replicas":{ - "COLL_S_R":{"shard30":[{ - "core_node61":{ - "core":"COLL_S_R_shard30_replica_n58", - "shard":"shard30", - "collection":"COLL_S_R", - "node_name":"NODE_S_A0a3cb04c83b693d80:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0a3cb04c83b693d80:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.0037675881758332253}}]}, - "COLL_S_M_1":{"shard9":[{ - "core_node18":{ - "core":"COLL_S_M_1_shard9_replica_n16", - "shard":"shard9", - "collection":"COLL_S_M_1", - "node_name":"NODE_S_A0a3cb04c83b693d80:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0a3cb04c83b693d80:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":6.426125764846802E-8}}]}, - "COLL_E_V2_4":{"shard2":[{ - "core_node8":{ - "core":"COLL_E_V2_4_shard2_replica_n6", - "shard":"shard2", - "collection":"COLL_E_V2_4", - "node_name":"NODE_S_A0a3cb04c83b693d80:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0a3cb04c83b693d80:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.23103012703359127}}]}, - "COLL_S_E_15":{ - "shard13":[{ - "core_node53":{ - "core":"COLL_S_E_15_shard13_replica_n50", - "shard":"shard13", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A0a3cb04c83b693d80:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0a3cb04c83b693d80:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}}], - "shard4":[{ - "core_node17":{ - "core":"COLL_S_E_15_shard4_replica_n14", - "shard":"shard4", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A0a3cb04c83b693d80:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0a3cb04c83b693d80:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}}], - "shard18":[{ - "core_node71":{ - "core":"COLL_S_E_15_shard18_replica_n68", - "shard":"shard18", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A0a3cb04c83b693d80:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0a3cb04c83b693d80:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}}], - "shard9":[{ - "core_node35":{ - "core":"COLL_S_E_15_shard9_replica_n32", - "shard":"shard9", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A0a3cb04c83b693d80:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0a3cb04c83b693d80:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}}]}, - "COLL_P_V_E_v2":{"shard3":[{ - "core_node11":{ - "core":"COLL_P_V_E_v2_shard3_replica_n8", - "shard":"shard3", - "collection":"COLL_P_V_E_v2", - "node_name":"NODE_S_A0a3cb04c83b693d80:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A0a3cb04c83b693d80:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":8.222954365424812}}]}, - "COLL_S_H_7":{"shard2":[{ - "core_node5":{ - "core":"COLL_S_H_7_shard2_replica_n2", - "shard":"shard2", - "collection":"COLL_S_H_7", - "node_name":"NODE_S_A0a3cb04c83b693d80:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0a3cb04c83b693d80:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.005126317963004112}}]}, - "COLL_E_v2_5":{ - "shard21":[{ - "core_node85":{ - "core":"COLL_E_v2_5_shard21_replica_n82", - "shard":"shard21", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A0a3cb04c83b693d80:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0a3cb04c83b693d80:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":135.4880904313177}}], - "shard12":[{ - "core_node236":{ - "core":"COLL_E_v2_5_shard12_replica_n235", - "shard":"shard12", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A0a3cb04c83b693d80:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0a3cb04c83b693d80:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":133.1140805343166}}]}, - "S_B_2":{"shard5":[{ - "core_node11":{ - "core":"S_B_2_shard5_replica_n8", - "shard":"shard5", - "collection":"S_B_2", - "node_name":"NODE_S_A0a3cb04c83b693d80:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0a3cb04c83b693d80:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.03135890234261751}}]}, - "COLL_S_R_aggr":{"shard1":[{ - "core_node3":{ - "core":"COLL_S_R_aggr_shard1_replica_n1", - "shard":"shard1", - "collection":"COLL_S_R_aggr", - "node_name":"NODE_S_A0a3cb04c83b693d80:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0a3cb04c83b693d80:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.7853863816708326}}]}, - "COLL_E_V2_S_A_3":{"shard2":[{ - "core_node8":{ - "core":"COLL_E_V2_S_A_3_shard2_replica_n6", - "shard":"shard2", - "collection":"COLL_E_V2_S_A_3", - "node_name":"NODE_S_A0a3cb04c83b693d80:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0a3cb04c83b693d80:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":24.86383740324527}}]}, - "COLL_S_J_H_9":{"shard2":[{ - "core_node8":{ - "core":"COLL_S_J_H_9_shard2_replica_n6", - "shard":"shard2", - "collection":"COLL_S_J_H_9", - "node_name":"NODE_S_A0a3cb04c83b693d80:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0a3cb04c83b693d80:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.005902050994336605}}]}, - "logs":{"shard1":[{ - "core_node5":{ - "core":"logs_shard1_replica_n2", - "shard":"shard1", - "collection":"logs", - "node_name":"NODE_S_A0a3cb04c83b693d80:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0a3cb04c83b693d80:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.012885616160929203}}]}, - "COLL_N_Q_C_T_11":{"shard6":[{ - "core_node13":{ - "core":"COLL_N_Q_C_T_11_shard6_replica_n7", - "shard":"shard6", - "collection":"COLL_N_Q_C_T_11", - "node_name":"NODE_S_A0a3cb04c83b693d80:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0a3cb04c83b693d80:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.0060289520770311356}}]}, - "COLL_PV_E_v2_5":{"shard1":[{ - "core_node5":{ - "core":"COLL_PV_E_v2_5_shard1_replica_n2", - "shard":"shard1", - "collection":"COLL_PV_E_v2_5", - "node_name":"NODE_S_A0a3cb04c83b693d80:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A0a3cb04c83b693d80:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":3.0124771054834127}}]}, - "COLL_S_E_15_signals":{"shard1":[{ - "core_node3":{ - "core":"COLL_S_E_15_signals_shard1_replica_n1", - "shard":"shard1", - "collection":"COLL_S_E_15_signals", - "node_name":"NODE_S_A0a3cb04c83b693d80:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0a3cb04c83b693d80:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":5.913432687520981E-5}}]}, - "COLL_R_P_S_A_22":{"shard30":[{ - "core_node61":{ - "core":"COLL_R_P_S_A_22_shard30_replica_n58", - "shard":"shard30", - "collection":"COLL_R_P_S_A_22", - "node_name":"NODE_S_A0a3cb04c83b693d80:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0a3cb04c83b693d80:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.42820744402706623}}]}}} - ,{ - "node":"NODE_S_A0d1445647aaae8dd1:8983_solr", - "isLive":true, - "cores":19.0, - "freedisk":596.0926208496094, - "totaldisk":999.755859375, - "replicas":{ - "COLL_S_R":{"shard32":[{ - "core_node65":{ - "core":"COLL_S_R_shard32_replica_n62", - "shard":"shard32", - "collection":"COLL_S_R", - "node_name":"NODE_S_A0d1445647aaae8dd1:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0d1445647aaae8dd1:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.003853907808661461}}]}, - "COLL_S_M_1":{"shard4":[{ - "core_node9":{ - "core":"COLL_S_M_1_shard4_replica_n6", - "shard":"shard4", - "collection":"COLL_S_M_1", - "node_name":"NODE_S_A0d1445647aaae8dd1:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0d1445647aaae8dd1:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":6.426125764846802E-8}}]}, - "COLL_E_V2_4":{"shard2":[{ - "core_node7":{ - "core":"COLL_E_V2_4_shard2_replica_n4", - "shard":"shard2", - "collection":"COLL_E_V2_4", - "node_name":"NODE_S_A0d1445647aaae8dd1:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A0d1445647aaae8dd1:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.23189987149089575}}]}, - "COLL_S_E_15":{ - "shard14":[{ - "core_node57":{ - "core":"COLL_S_E_15_shard14_replica_n54", - "shard":"shard14", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A0d1445647aaae8dd1:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A0d1445647aaae8dd1:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}}], - "shard5":[{ - "core_node21":{ - "core":"COLL_S_E_15_shard5_replica_n18", - "shard":"shard5", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A0d1445647aaae8dd1:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A0d1445647aaae8dd1:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}}], - "shard1":[{ - "core_node3":{ - "core":"COLL_S_E_15_shard1_replica_n1", - "shard":"shard1", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A0d1445647aaae8dd1:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0d1445647aaae8dd1:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}}], - "shard10":[{ - "core_node39":{ - "core":"COLL_S_E_15_shard10_replica_n36", - "shard":"shard10", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A0d1445647aaae8dd1:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0d1445647aaae8dd1:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}}]}, - "COLL_P_V_E_v2":{"shard1":[{ - "core_node3":{ - "core":"COLL_P_V_E_v2_shard1_replica_n1", - "shard":"shard1", - "collection":"COLL_P_V_E_v2", - "node_name":"NODE_S_A0d1445647aaae8dd1:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A0d1445647aaae8dd1:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":8.172482682392001}}]}, - "audit_logs":{"shard1":[{ - "core_node3":{ - "core":"audit_logs_shard1_replica_n1", - "shard":"shard1", - "collection":"audit_logs", - "node_name":"NODE_S_A0d1445647aaae8dd1:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A0d1445647aaae8dd1:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.30385160446167E-7}}]}, - "COLL_S_H_7":{"shard7":[{ - "core_node15":{ - "core":"COLL_S_H_7_shard7_replica_n12", - "shard":"shard7", - "collection":"COLL_S_H_7", - "node_name":"NODE_S_A0d1445647aaae8dd1:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0d1445647aaae8dd1:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.005527845583856106}}]}, - "COLL_E_v2_5":{ - "shard27":[{ - "core_node109":{ - "core":"COLL_E_v2_5_shard27_replica_n106", - "shard":"shard27", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A0d1445647aaae8dd1:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0d1445647aaae8dd1:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":137.78214563801885}}], - "shard19":[{ - "core_node77":{ - "core":"COLL_E_v2_5_shard19_replica_n74", - "shard":"shard19", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A0d1445647aaae8dd1:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0d1445647aaae8dd1:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":130.64438762888312}}], - "shard11":[{ - "core_node45":{ - "core":"COLL_E_v2_5_shard11_replica_n42", - "shard":"shard11", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A0d1445647aaae8dd1:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A0d1445647aaae8dd1:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":126.09197529405355}}]}, - "system_metrics":{"shard2":[{ - "core_node8":{ - "core":"system_metrics_shard2_replica_n6", - "shard":"shard2", - "collection":"system_metrics", - "node_name":"NODE_S_A0d1445647aaae8dd1:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A0d1445647aaae8dd1:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}}]}, - "S_B_2":{"shard9":[{ - "core_node18":{ - "core":"S_B_2_shard9_replica_n16", - "shard":"shard9", - "collection":"S_B_2", - "node_name":"NODE_S_A0d1445647aaae8dd1:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0d1445647aaae8dd1:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.015032877214252949}}]}, - "COLL_S_E_15_signals_aggr":{"shard2":[{ - "core_node7":{ - "core":"COLL_S_E_15_signals_aggr_shard2_replica_n4", - "shard":"shard2", - "collection":"COLL_S_E_15_signals_aggr", - "node_name":"NODE_S_A0d1445647aaae8dd1:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A0d1445647aaae8dd1:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":6.426125764846802E-8}}]}, - "COLL_S_J_H_9":{"shard2":[{ - "core_node7":{ - "core":"COLL_S_J_H_9_shard2_replica_n4", - "shard":"shard2", - "collection":"COLL_S_J_H_9", - "node_name":"NODE_S_A0d1445647aaae8dd1:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A0d1445647aaae8dd1:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.0059021394699811935}}]}, - "COLL_S_E_15_signals":{"shard1":[{ - "core_node5":{ - "core":"COLL_S_E_15_signals_shard1_replica_n2", - "shard":"shard1", - "collection":"COLL_S_E_15_signals", - "node_name":"NODE_S_A0d1445647aaae8dd1:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A0d1445647aaae8dd1:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":5.915109068155289E-5}}]}, - "COLL_R_P_S_A_22":{"shard32":[{ - "core_node65":{ - "core":"COLL_R_P_S_A_22_shard32_replica_n62", - "shard":"shard32", - "collection":"COLL_R_P_S_A_22", - "node_name":"NODE_S_A0d1445647aaae8dd1:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0d1445647aaae8dd1:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.5012764362618327}}]}}} - ,{ - "node":"NODE_S_A0ee1c34fb3e9c261d:8983_solr", - "isLive":true, - "cores":18.0, - "freedisk":694.8752136230469, - "totaldisk":999.755859375, - "replicas":{ - "COLL_S_R":{"shard26":[{ - "core_node53":{ - "core":"COLL_S_R_shard26_replica_n50", - "shard":"shard26", - "collection":"COLL_S_R", - "node_name":"NODE_S_A0ee1c34fb3e9c261d:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0ee1c34fb3e9c261d:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.0038864407688379288}}]}, - "COLL_S_M_1":{"shard2":[{ - "core_node5":{ - "core":"COLL_S_M_1_shard2_replica_n2", - "shard":"shard2", - "collection":"COLL_S_M_1", - "node_name":"NODE_S_A0ee1c34fb3e9c261d:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0ee1c34fb3e9c261d:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":6.426125764846802E-8}}]}, - "COLL_E_V2_4":{"shard1":[{ - "core_node3":{ - "core":"COLL_E_V2_4_shard1_replica_n1", - "shard":"shard1", - "collection":"COLL_E_V2_4", - "node_name":"NODE_S_A0ee1c34fb3e9c261d:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A0ee1c34fb3e9c261d:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.23009658977389336}}]}, - "COLL_S_E_15":{ - "shard2":[{ - "core_node7":{ - "core":"COLL_S_E_15_shard2_replica_n4", - "shard":"shard2", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A0ee1c34fb3e9c261d:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A0ee1c34fb3e9c261d:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}}], - "shard15":[{ - "core_node61":{ - "core":"COLL_S_E_15_shard15_replica_n58", - "shard":"shard15", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A0ee1c34fb3e9c261d:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0ee1c34fb3e9c261d:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}}], - "shard6":[{ - "core_node25":{ - "core":"COLL_S_E_15_shard6_replica_n22", - "shard":"shard6", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A0ee1c34fb3e9c261d:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0ee1c34fb3e9c261d:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}}], - "shard11":[{ - "core_node43":{ - "core":"COLL_S_E_15_shard11_replica_n40", - "shard":"shard11", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A0ee1c34fb3e9c261d:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A0ee1c34fb3e9c261d:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}}]}, - "COLL_S_H_7":{"shard4":[{ - "core_node9":{ - "core":"COLL_S_H_7_shard4_replica_n6", - "shard":"shard4", - "collection":"COLL_S_H_7", - "node_name":"NODE_S_A0ee1c34fb3e9c261d:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0ee1c34fb3e9c261d:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.005278228782117367}}]}, - "COLL_E_v2_5":{ - "shard31":[{ - "core_node125":{ - "core":"COLL_E_v2_5_shard31_replica_n122", - "shard":"shard31", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A0ee1c34fb3e9c261d:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A0ee1c34fb3e9c261d:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":136.22203192673624}}], - "shard32":[{ - "core_node127":{ - "core":"COLL_E_v2_5_shard32_replica_n124", - "shard":"shard32", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A0ee1c34fb3e9c261d:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A0ee1c34fb3e9c261d:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":135.84658553358167}}]}, - "system_metrics":{"shard2":[{ - "core_node7":{ - "core":"system_metrics_shard2_replica_n4", - "shard":"shard2", - "collection":"system_metrics", - "node_name":"NODE_S_A0ee1c34fb3e9c261d:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0ee1c34fb3e9c261d:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}}]}, - "S_B_2":{"shard7":[{ - "core_node15":{ - "core":"S_B_2_shard7_replica_n12", - "shard":"shard7", - "collection":"S_B_2", - "node_name":"NODE_S_A0ee1c34fb3e9c261d:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0ee1c34fb3e9c261d:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.051517823711037636}}]}, - "COLL_S_R_aggr":{"shard1":[{ - "core_node5":{ - "core":"COLL_S_R_aggr_shard1_replica_n2", - "shard":"shard1", - "collection":"COLL_S_R_aggr", - "node_name":"NODE_S_A0ee1c34fb3e9c261d:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A0ee1c34fb3e9c261d:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.7883908534422517}}]}, - "COLL_S_J_H_9":{"shard1":[{ - "core_node5":{ - "core":"COLL_S_J_H_9_shard1_replica_n2", - "shard":"shard1", - "collection":"COLL_S_J_H_9", - "node_name":"NODE_S_A0ee1c34fb3e9c261d:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A0ee1c34fb3e9c261d:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.005922678858041763}}]}, - "logs":{"shard1":[{ - "core_node3":{ - "core":"logs_shard1_replica_n1", - "shard":"shard1", - "collection":"logs", - "node_name":"NODE_S_A0ee1c34fb3e9c261d:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A0ee1c34fb3e9c261d:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.012863828800618649}}]}, - "COLL_N_Q_C_T_11":{"shard3":[{ - "core_node9":{ - "core":"COLL_N_Q_C_T_11_shard3_replica_n4", - "shard":"shard3", - "collection":"COLL_N_Q_C_T_11", - "node_name":"NODE_S_A0ee1c34fb3e9c261d:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0ee1c34fb3e9c261d:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.006221611984074116}}]}, - "COLL_PV_E_v2_5":{"shard3":[{ - "core_node11":{ - "core":"COLL_PV_E_v2_5_shard3_replica_n8", - "shard":"shard3", - "collection":"COLL_PV_E_v2_5", - "node_name":"NODE_S_A0ee1c34fb3e9c261d:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A0ee1c34fb3e9c261d:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":3.029745533131063}}]}, - "COLL_R_P_S_A_22":{"shard26":[{ - "core_node53":{ - "core":"COLL_R_P_S_A_22_shard26_replica_n50", - "shard":"shard26", - "collection":"COLL_R_P_S_A_22", - "node_name":"NODE_S_A0ee1c34fb3e9c261d:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0ee1c34fb3e9c261d:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.4816165193915367}}]}}} - ,{ - "node":"NODE_S_A0dc21b7f9ef048b48:8983_solr", - "isLive":true, - "cores":18.0, - "freedisk":713.6261863708496, - "totaldisk":999.755859375, - "replicas":{ - "COLL_S_R":{"shard33":[{ - "core_node66":{ - "core":"COLL_S_R_shard33_replica_n64", - "shard":"shard33", - "collection":"COLL_S_R", - "node_name":"NODE_S_A0dc21b7f9ef048b48:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0dc21b7f9ef048b48:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.003936124965548515}}]}, - "COLL_S_M_1":{"shard5":[{ - "core_node11":{ - "core":"COLL_S_M_1_shard5_replica_n8", - "shard":"shard5", - "collection":"COLL_S_M_1", - "node_name":"NODE_S_A0dc21b7f9ef048b48:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0dc21b7f9ef048b48:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":6.426125764846802E-8}}]}, - "COLL_E_V2_4":{"shard1":[{ - "core_node5":{ - "core":"COLL_E_V2_4_shard1_replica_n2", - "shard":"shard1", - "collection":"COLL_E_V2_4", - "node_name":"NODE_S_A0dc21b7f9ef048b48:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0dc21b7f9ef048b48:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.23144119326025248}}]}, - "COLL_S_E_15":{ - "shard3":[{ - "core_node13":{ - "core":"COLL_S_E_15_shard3_replica_n10", - "shard":"shard3", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A0dc21b7f9ef048b48:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0dc21b7f9ef048b48:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}}], - "shard17":[{ - "core_node67":{ - "core":"COLL_S_E_15_shard17_replica_n64", - "shard":"shard17", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A0dc21b7f9ef048b48:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0dc21b7f9ef048b48:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}}], - "shard8":[{ - "core_node31":{ - "core":"COLL_S_E_15_shard8_replica_n28", - "shard":"shard8", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A0dc21b7f9ef048b48:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0dc21b7f9ef048b48:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}}], - "shard12":[{ - "core_node49":{ - "core":"COLL_S_E_15_shard12_replica_n46", - "shard":"shard12", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A0dc21b7f9ef048b48:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0dc21b7f9ef048b48:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}}]}, - "COLL_S_H_7":{"shard6":[{ - "core_node13":{ - "core":"COLL_S_H_7_shard6_replica_n10", - "shard":"shard6", - "collection":"COLL_S_H_7", - "node_name":"NODE_S_A0dc21b7f9ef048b48:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0dc21b7f9ef048b48:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.005341590382158756}}]}, - "COLL_E_v2_5":{ - "shard7":[{ - "core_node228":{ - "core":"COLL_E_v2_5_shard7_replica_n227", - "shard":"shard7", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A0dc21b7f9ef048b48:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0dc21b7f9ef048b48:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":125.11620672326535}}], - "shard23":[{ - "core_node232":{ - "core":"COLL_E_v2_5_shard23_replica_n231", - "shard":"shard23", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A0dc21b7f9ef048b48:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0dc21b7f9ef048b48:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":137.6997431376949}}]}, - "system_metrics":{"shard1":[{ - "core_node3":{ - "core":"system_metrics_shard1_replica_n1", - "shard":"shard1", - "collection":"system_metrics", - "node_name":"NODE_S_A0dc21b7f9ef048b48:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0dc21b7f9ef048b48:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}}]}, - "S_B_2":{"shard3":[{ - "core_node7":{ - "core":"S_B_2_shard3_replica_n4", - "shard":"shard3", - "collection":"S_B_2", - "node_name":"NODE_S_A0dc21b7f9ef048b48:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0dc21b7f9ef048b48:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.0767898540943861}}]}, - "COLL_S_E_15_signals_aggr":{"shard2":[{ - "core_node8":{ - "core":"COLL_S_E_15_signals_aggr_shard2_replica_n6", - "shard":"shard2", - "collection":"COLL_S_E_15_signals_aggr", - "node_name":"NODE_S_A0dc21b7f9ef048b48:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0dc21b7f9ef048b48:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":6.426125764846802E-8}}]}, - "COLL_E_V2_S_A_3":{"shard1":[{ - "core_node5":{ - "core":"COLL_E_V2_S_A_3_shard1_replica_n2", - "shard":"shard1", - "collection":"COLL_E_V2_S_A_3", - "node_name":"NODE_S_A0dc21b7f9ef048b48:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A0dc21b7f9ef048b48:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":22.407991461455822}}]}, - "COLL_S_J_H_9":{"shard1":[{ - "core_node3":{ - "core":"COLL_S_J_H_9_shard1_replica_n1", - "shard":"shard1", - "collection":"COLL_S_J_H_9", - "node_name":"NODE_S_A0dc21b7f9ef048b48:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0dc21b7f9ef048b48:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.0059427423402667046}}]}, - "COLL_N_Q_C_T_11":{"shard4":[{ - "core_node10":{ - "core":"COLL_N_Q_C_T_11_shard4_replica_n5", - "shard":"shard4", - "collection":"COLL_N_Q_C_T_11", - "node_name":"NODE_S_A0dc21b7f9ef048b48:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0dc21b7f9ef048b48:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.0060385772958397865}}]}, - "COLL_S_E_15_signals":{"shard2":[{ - "core_node8":{ - "core":"COLL_S_E_15_signals_shard2_replica_n6", - "shard":"shard2", - "collection":"COLL_S_E_15_signals", - "node_name":"NODE_S_A0dc21b7f9ef048b48:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0dc21b7f9ef048b48:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":2.8850510716438293E-5}}]}, - "COLL_R_P_S_A_22":{"shard33":[{ - "core_node66":{ - "core":"COLL_R_P_S_A_22_shard33_replica_n64", - "shard":"shard33", - "collection":"COLL_R_P_S_A_22", - "node_name":"NODE_S_A0dc21b7f9ef048b48:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0dc21b7f9ef048b48:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.45377735421061516}}]}}} - ,{ - "node":"NODE_S_A0ccef54efe3178758:8983_solr", - "isLive":true, - "cores":18.0, - "freedisk":722.821891784668, - "totaldisk":999.755859375, - "replicas":{ - "COLL_S_R":{"shard28":[{ - "core_node57":{ - "core":"COLL_S_R_shard28_replica_n54", - "shard":"shard28", - "collection":"COLL_S_R", - "node_name":"NODE_S_A0ccef54efe3178758:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0ccef54efe3178758:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.0038556959480047226}}]}, - "COLL_S_M_1":{"shard7":[{ - "core_node15":{ - "core":"COLL_S_M_1_shard7_replica_n12", - "shard":"shard7", - "collection":"COLL_S_M_1", - "node_name":"NODE_S_A0ccef54efe3178758:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0ccef54efe3178758:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":6.426125764846802E-8}}]}, - "COLL_S_E_15":{ - "shard2":[{ - "core_node9":{ - "core":"COLL_S_E_15_shard2_replica_n6", - "shard":"shard2", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A0ccef54efe3178758:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0ccef54efe3178758:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}}], - "shard16":[{ - "core_node63":{ - "core":"COLL_S_E_15_shard16_replica_n60", - "shard":"shard16", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A0ccef54efe3178758:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0ccef54efe3178758:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}}], - "shard7":[{ - "core_node27":{ - "core":"COLL_S_E_15_shard7_replica_n24", - "shard":"shard7", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A0ccef54efe3178758:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0ccef54efe3178758:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}}], - "shard11":[{ - "core_node45":{ - "core":"COLL_S_E_15_shard11_replica_n42", - "shard":"shard11", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A0ccef54efe3178758:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0ccef54efe3178758:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}}]}, - "COLL_P_V_E_v2":{"shard5":[{ - "core_node21":{ - "core":"COLL_P_V_E_v2_shard5_replica_n18", - "shard":"shard5", - "collection":"COLL_P_V_E_v2", - "node_name":"NODE_S_A0ccef54efe3178758:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A0ccef54efe3178758:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":8.201123727485538}}]}, - "audit_logs":{"shard1":[{ - "core_node5":{ - "core":"audit_logs_shard1_replica_n2", - "shard":"shard1", - "collection":"audit_logs", - "node_name":"NODE_S_A0ccef54efe3178758:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0ccef54efe3178758:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.30385160446167E-7}}]}, - "COLL_S_H_7":{"shard8":[{ - "core_node17":{ - "core":"COLL_S_H_7_shard8_replica_n14", - "shard":"shard8", - "collection":"COLL_S_H_7", - "node_name":"NODE_S_A0ccef54efe3178758:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0ccef54efe3178758:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.005281115882098675}}]}, - "COLL_E_v2_5":{ - "shard28":[{ - "core_node113":{ - "core":"COLL_E_v2_5_shard28_replica_n110", - "shard":"shard28", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A0ccef54efe3178758:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0ccef54efe3178758:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":132.98001556191593}}], - "shard20":[{ - "core_node81":{ - "core":"COLL_E_v2_5_shard20_replica_n78", - "shard":"shard20", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A0ccef54efe3178758:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0ccef54efe3178758:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":131.2389001082629}}]}, - "S_B_2":{"shard1":[{ - "core_node3":{ - "core":"S_B_2_shard1_replica_n1", - "shard":"shard1", - "collection":"S_B_2", - "node_name":"NODE_S_A0ccef54efe3178758:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0ccef54efe3178758:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.03542718291282654}}]}, - "COLL_S_R_aggr":{"shard2":[{ - "core_node7":{ - "core":"COLL_S_R_aggr_shard2_replica_n4", - "shard":"shard2", - "collection":"COLL_S_R_aggr", - "node_name":"NODE_S_A0ccef54efe3178758:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0ccef54efe3178758:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.7941454965621233}}]}, - "COLL_S_E_15_signals_aggr":{"shard1":[{ - "core_node3":{ - "core":"COLL_S_E_15_signals_aggr_shard1_replica_n1", - "shard":"shard1", - "collection":"COLL_S_E_15_signals_aggr", - "node_name":"NODE_S_A0ccef54efe3178758:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0ccef54efe3178758:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":6.426125764846802E-8}}]}, - "logs":{"shard2":[{ - "core_node8":{ - "core":"logs_shard2_replica_n6", - "shard":"shard2", - "collection":"logs", - "node_name":"NODE_S_A0ccef54efe3178758:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0ccef54efe3178758:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.012702811509370804}}]}, - "COLL_N_Q_C_T_11":{"shard5":[{ - "core_node12":{ - "core":"COLL_N_Q_C_T_11_shard5_replica_n6", - "shard":"shard5", - "collection":"COLL_N_Q_C_T_11", - "node_name":"NODE_S_A0ccef54efe3178758:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0ccef54efe3178758:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.0061960527673363686}}]}, - "COLL_PV_E_v2_5":{"shard2":[{ - "core_node7":{ - "core":"COLL_PV_E_v2_5_shard2_replica_n4", - "shard":"shard2", - "collection":"COLL_PV_E_v2_5", - "node_name":"NODE_S_A0ccef54efe3178758:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A0ccef54efe3178758:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":3.0324512012302876}}]}, - "COLL_R_P_S_A_22":{"shard28":[{ - "core_node57":{ - "core":"COLL_R_P_S_A_22_shard28_replica_n54", - "shard":"shard28", - "collection":"COLL_R_P_S_A_22", - "node_name":"NODE_S_A0ccef54efe3178758:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0ccef54efe3178758:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.45370675902813673}}]}}} - ,{ - "node":"NODE_S_A077430d186f0e1de2:8983_solr", - "isLive":true, - "cores":17.0, - "freedisk":714.6234321594238, - "totaldisk":999.755859375, - "replicas":{ - "system_metrics":{"shard1":[{ - "core_node5":{ - "core":"system_metrics_shard1_replica_n2", - "shard":"shard1", - "collection":"system_metrics", - "node_name":"NODE_S_A077430d186f0e1de2:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A077430d186f0e1de2:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}}]}, - "S_B_2":{"shard8":[{ - "core_node17":{ - "core":"S_B_2_shard8_replica_n14", - "shard":"shard8", - "collection":"S_B_2", - "node_name":"NODE_S_A077430d186f0e1de2:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A077430d186f0e1de2:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.03129926137626171}}]}, - "COLL_S_R":{"shard31":[{ - "core_node63":{ - "core":"COLL_S_R_shard31_replica_n60", - "shard":"shard31", - "collection":"COLL_S_R", - "node_name":"NODE_S_A077430d186f0e1de2:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A077430d186f0e1de2:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.0038539329543709755}}]}, - "COLL_S_M_1":{"shard6":[{ - "core_node13":{ - "core":"COLL_S_M_1_shard6_replica_n10", - "shard":"shard6", - "collection":"COLL_S_M_1", - "node_name":"NODE_S_A077430d186f0e1de2:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A077430d186f0e1de2:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":6.426125764846802E-8}}]}, - "COLL_S_E_15":{ - "shard13":[{ - "core_node51":{ - "core":"COLL_S_E_15_shard13_replica_n48", - "shard":"shard13", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A077430d186f0e1de2:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A077430d186f0e1de2:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}}], - "shard4":[{ - "core_node15":{ - "core":"COLL_S_E_15_shard4_replica_n12", - "shard":"shard4", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A077430d186f0e1de2:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A077430d186f0e1de2:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}}], - "shard17":[{ - "core_node69":{ - "core":"COLL_S_E_15_shard17_replica_n66", - "shard":"shard17", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A077430d186f0e1de2:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A077430d186f0e1de2:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}}], - "shard8":[{ - "core_node33":{ - "core":"COLL_S_E_15_shard8_replica_n30", - "shard":"shard8", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A077430d186f0e1de2:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A077430d186f0e1de2:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}}]}, - "COLL_E_V2_S_A_3":{"shard2":[{ - "core_node7":{ - "core":"COLL_E_V2_S_A_3_shard2_replica_n4", - "shard":"shard2", - "collection":"COLL_E_V2_S_A_3", - "node_name":"NODE_S_A077430d186f0e1de2:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A077430d186f0e1de2:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":24.63952654134482}}]}, - "subreddit_experiment_results":{"shard26":[{ - "core_node53":{ - "core":"subreddit_experiment_results_shard26_replica_n50", - "shard":"shard26", - "collection":"subreddit_experiment_results", - "node_name":"NODE_S_A077430d186f0e1de2:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A077430d186f0e1de2:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.0044707562774419785}}]}, - "audit_logs":{"shard2":[{ - "core_node7":{ - "core":"audit_logs_shard2_replica_n4", - "shard":"shard2", - "collection":"audit_logs", - "node_name":"NODE_S_A077430d186f0e1de2:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A077430d186f0e1de2:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.30385160446167E-7}}]}, - "logs":{"shard2":[{ - "core_node7":{ - "core":"logs_shard2_replica_n4", - "shard":"shard2", - "collection":"logs", - "node_name":"NODE_S_A077430d186f0e1de2:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A077430d186f0e1de2:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.012706683948636055}}]}, - "COLL_N_Q_C_T_11":{"shard8":[{ - "core_node17":{ - "core":"COLL_N_Q_C_T_11_shard8_replica_n14", - "shard":"shard8", - "collection":"COLL_N_Q_C_T_11", - "node_name":"NODE_S_A077430d186f0e1de2:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A077430d186f0e1de2:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.006143262609839439}}]}, - "COLL_S_H_7":{"shard5":[{ - "core_node11":{ - "core":"COLL_S_H_7_shard5_replica_n8", - "shard":"shard5", - "collection":"COLL_S_H_7", - "node_name":"NODE_S_A077430d186f0e1de2:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A077430d186f0e1de2:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.005316532216966152}}]}, - "COLL_R_P_S_A_22":{"shard31":[{ - "core_node63":{ - "core":"COLL_R_P_S_A_22_shard31_replica_n60", - "shard":"shard31", - "collection":"COLL_R_P_S_A_22", - "node_name":"NODE_S_A077430d186f0e1de2:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A077430d186f0e1de2:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.49999847542494535}}]}, - "COLL_E_v2_5":{ - "shard5":[{ - "core_node19":{ - "core":"COLL_E_v2_5_shard5_replica_n16", - "shard":"shard5", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A077430d186f0e1de2:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A077430d186f0e1de2:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":127.56859436631203}}], - "shard21":[{ - "core_node83":{ - "core":"COLL_E_v2_5_shard21_replica_n80", - "shard":"shard21", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A077430d186f0e1de2:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A077430d186f0e1de2:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":132.22010754607618}}]}}} - ,{ - "node":"NODE_S_A0afcb9535d2619dac:8983_solr", - "isLive":true, - "cores":16.0, - "freedisk":740.1097831726074, - "totaldisk":999.755859375, - "replicas":{ - "S_B_2":{"shard4":[{ - "core_node9":{ - "core":"S_B_2_shard4_replica_n6", - "shard":"shard4", - "collection":"S_B_2", - "node_name":"NODE_S_A0afcb9535d2619dac:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0afcb9535d2619dac:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.02314597088843584}}]}, - "COLL_S_R":{"shard29":[{ - "core_node59":{ - "core":"COLL_S_R_shard29_replica_n56", - "shard":"shard29", - "collection":"COLL_S_R", - "node_name":"NODE_S_A0afcb9535d2619dac:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0afcb9535d2619dac:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.0038082310929894447}}]}, - "COLL_S_M_1":{"shard8":[{ - "core_node17":{ - "core":"COLL_S_M_1_shard8_replica_n14", - "shard":"shard8", - "collection":"COLL_S_M_1", - "node_name":"NODE_S_A0afcb9535d2619dac:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0afcb9535d2619dac:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":6.426125764846802E-8}}]}, - "COLL_S_E_15":{ - "shard15":[{ - "core_node59":{ - "core":"COLL_S_E_15_shard15_replica_n56", - "shard":"shard15", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A0afcb9535d2619dac:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A0afcb9535d2619dac:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}}], - "shard1":[{ - "core_node5":{ - "core":"COLL_S_E_15_shard1_replica_n2", - "shard":"shard1", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A0afcb9535d2619dac:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A0afcb9535d2619dac:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}}], - "shard6":[{ - "core_node23":{ - "core":"COLL_S_E_15_shard6_replica_n20", - "shard":"shard6", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A0afcb9535d2619dac:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A0afcb9535d2619dac:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}}], - "shard10":[{ - "core_node41":{ - "core":"COLL_S_E_15_shard10_replica_n38", - "shard":"shard10", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A0afcb9535d2619dac:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A0afcb9535d2619dac:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}}]}, - "subreddit_experiment_results":{ - "shard18":[{ - "core_node37":{ - "core":"subreddit_experiment_results_shard18_replica_n34", - "shard":"shard18", - "collection":"subreddit_experiment_results", - "node_name":"NODE_S_A0afcb9535d2619dac:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0afcb9535d2619dac:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.0045873550698161125}}], - "shard7":[{ - "core_node15":{ - "core":"subreddit_experiment_results_shard7_replica_n12", - "shard":"shard7", - "collection":"subreddit_experiment_results", - "node_name":"NODE_S_A0afcb9535d2619dac:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0afcb9535d2619dac:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.004490339197218418}}]}, - "audit_logs":{"shard2":[{ - "core_node8":{ - "core":"audit_logs_shard2_replica_n6", - "shard":"shard2", - "collection":"audit_logs", - "node_name":"NODE_S_A0afcb9535d2619dac:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0afcb9535d2619dac:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.30385160446167E-7}}]}, - "COLL_N_Q_C_T_11":{ - "shard2":[{ - "core_node8":{ - "core":"COLL_N_Q_C_T_11_shard2_replica_n2", - "shard":"shard2", - "collection":"COLL_N_Q_C_T_11", - "node_name":"NODE_S_A0afcb9535d2619dac:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0afcb9535d2619dac:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.006244504824280739}}], - "shard1":[{ - "core_node3":{ - "core":"COLL_N_Q_C_T_11_shard1_replica_n1", - "shard":"shard1", - "collection":"COLL_N_Q_C_T_11", - "node_name":"NODE_S_A0afcb9535d2619dac:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0afcb9535d2619dac:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.006021473556756973}}]}, - "COLL_S_H_7":{"shard9":[{ - "core_node18":{ - "core":"COLL_S_H_7_shard9_replica_n16", - "shard":"shard9", - "collection":"COLL_S_H_7", - "node_name":"NODE_S_A0afcb9535d2619dac:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0afcb9535d2619dac:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.005475449375808239}}]}, - "COLL_R_P_S_A_22":{"shard29":[{ - "core_node59":{ - "core":"COLL_R_P_S_A_22_shard29_replica_n56", - "shard":"shard29", - "collection":"COLL_R_P_S_A_22", - "node_name":"NODE_S_A0afcb9535d2619dac:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0afcb9535d2619dac:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.5458483807742596}}]}, - "COLL_E_v2_5":{ - "shard15":[{ - "core_node61":{ - "core":"COLL_E_v2_5_shard15_replica_n58", - "shard":"shard15", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A0afcb9535d2619dac:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A0afcb9535d2619dac:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":134.34092226438224}}], - "shard7":[{ - "core_node29":{ - "core":"COLL_E_v2_5_shard7_replica_n26", - "shard":"shard7", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A0afcb9535d2619dac:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A0afcb9535d2619dac:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":124.55378058645874}}]}}} - ,{ - "node":"NODE_S_A09aede91f0e8c147c:8983_solr", - "isLive":true, - "cores":15.0, - "freedisk":721.5844802856445, - "totaldisk":999.755859375, - "replicas":{ - "S_B_2":{"shard2":[{ - "core_node5":{ - "core":"S_B_2_shard2_replica_n2", - "shard":"shard2", - "collection":"S_B_2", - "node_name":"NODE_S_A09aede91f0e8c147c:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A09aede91f0e8c147c:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.030836801044642925}}]}, - "COLL_S_R":{"shard23":[{ - "core_node47":{ - "core":"COLL_S_R_shard23_replica_n44", - "shard":"shard23", - "collection":"COLL_S_R", - "node_name":"NODE_S_A09aede91f0e8c147c:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A09aede91f0e8c147c:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.003840106539428234}}]}, - "COLL_S_M_1":{"shard1":[{ - "core_node3":{ - "core":"COLL_S_M_1_shard1_replica_n1", - "shard":"shard1", - "collection":"COLL_S_M_1", - "node_name":"NODE_S_A09aede91f0e8c147c:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A09aede91f0e8c147c:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":6.426125764846802E-8}}]}, - "COLL_S_E_15_signals_aggr":{"shard1":[{ - "core_node5":{ - "core":"COLL_S_E_15_signals_aggr_shard1_replica_n2", - "shard":"shard1", - "collection":"COLL_S_E_15_signals_aggr", - "node_name":"NODE_S_A09aede91f0e8c147c:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A09aede91f0e8c147c:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":6.426125764846802E-8}}]}, - "COLL_S_E_15":{ - "shard14":[{ - "core_node55":{ - "core":"COLL_S_E_15_shard14_replica_n52", - "shard":"shard14", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A09aede91f0e8c147c:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A09aede91f0e8c147c:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}}], - "shard5":[{ - "core_node19":{ - "core":"COLL_S_E_15_shard5_replica_n16", - "shard":"shard5", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A09aede91f0e8c147c:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A09aede91f0e8c147c:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}}], - "shard18":[{ - "core_node72":{ - "core":"COLL_S_E_15_shard18_replica_n70", - "shard":"shard18", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A09aede91f0e8c147c:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A09aede91f0e8c147c:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}}], - "shard9":[{ - "core_node37":{ - "core":"COLL_S_E_15_shard9_replica_n34", - "shard":"shard9", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A09aede91f0e8c147c:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A09aede91f0e8c147c:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}}]}, - "COLL_P_V_E_v2":{"shard3":[{ - "core_node13":{ - "core":"COLL_P_V_E_v2_shard3_replica_n10", - "shard":"shard3", - "collection":"COLL_P_V_E_v2", - "node_name":"NODE_S_A09aede91f0e8c147c:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A09aede91f0e8c147c:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":8.193078896962106}}]}, - "COLL_N_Q_C_T_11":{"shard7":[{ - "core_node15":{ - "core":"COLL_N_Q_C_T_11_shard7_replica_n11", - "shard":"shard7", - "collection":"COLL_N_Q_C_T_11", - "node_name":"NODE_S_A09aede91f0e8c147c:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A09aede91f0e8c147c:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.00618187990039587}}]}, - "COLL_S_H_7":{"shard1":[{ - "core_node3":{ - "core":"COLL_S_H_7_shard1_replica_n1", - "shard":"shard1", - "collection":"COLL_S_H_7", - "node_name":"NODE_S_A09aede91f0e8c147c:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A09aede91f0e8c147c:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.0052086347714066505}}]}, - "COLL_PV_E_v2_5":{"shard1":[{ - "core_node3":{ - "core":"COLL_PV_E_v2_5_shard1_replica_n1", - "shard":"shard1", - "collection":"COLL_PV_E_v2_5", - "node_name":"NODE_S_A09aede91f0e8c147c:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A09aede91f0e8c147c:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":3.0264872340485454}}]}, - "COLL_R_P_S_A_22":{"shard23":[{ - "core_node47":{ - "core":"COLL_R_P_S_A_22_shard23_replica_n44", - "shard":"shard23", - "collection":"COLL_R_P_S_A_22", - "node_name":"NODE_S_A09aede91f0e8c147c:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A09aede91f0e8c147c:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.4307612795382738}}]}, - "COLL_E_v2_5":{ - "shard35":[{ - "core_node141":{ - "core":"COLL_E_v2_5_shard35_replica_n138", - "shard":"shard35", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A09aede91f0e8c147c:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A09aede91f0e8c147c:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":128.57825357280672}}], - "shard23":[{ - "core_node396":{ - "core":"COLL_E_v2_5_shard23_replica_n395", - "shard":"shard23", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A09aede91f0e8c147c:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A09aede91f0e8c147c:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":137.7283339947462}}]}}} - ,{ - "node":"NODE_S_A024a9c138d972d437:8983_solr", - "isLive":true, - "cores":14.0, - "freedisk":725.4795761108398, - "totaldisk":999.755859375, - "replicas":{ - "S_B_2":{"shard6":[{ - "core_node13":{ - "core":"S_B_2_shard6_replica_n10", - "shard":"shard6", - "collection":"S_B_2", - "node_name":"NODE_S_A024a9c138d972d437:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A024a9c138d972d437:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.010442202910780907}}]}, - "COLL_S_R_aggr":{"shard2":[{ - "core_node8":{ - "core":"COLL_S_R_aggr_shard2_replica_n6", - "shard":"shard2", - "collection":"COLL_S_R_aggr", - "node_name":"NODE_S_A024a9c138d972d437:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A024a9c138d972d437:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.7956495881080627}}]}, - "COLL_S_R":{"shard27":[{ - "core_node55":{ - "core":"COLL_S_R_shard27_replica_n52", - "shard":"shard27", - "collection":"COLL_S_R", - "node_name":"NODE_S_A024a9c138d972d437:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A024a9c138d972d437:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.0038878321647644043}}]}, - "COLL_S_M_1":{"shard3":[{ - "core_node7":{ - "core":"COLL_S_M_1_shard3_replica_n4", - "shard":"shard3", - "collection":"COLL_S_M_1", - "node_name":"NODE_S_A024a9c138d972d437:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A024a9c138d972d437:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":6.426125764846802E-8}}]}, - "COLL_S_E_15":{ - "shard3":[{ - "core_node11":{ - "core":"COLL_S_E_15_shard3_replica_n8", - "shard":"shard3", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A024a9c138d972d437:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A024a9c138d972d437:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}}], - "shard16":[{ - "core_node65":{ - "core":"COLL_S_E_15_shard16_replica_n62", - "shard":"shard16", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A024a9c138d972d437:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A024a9c138d972d437:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}}], - "shard7":[{ - "core_node29":{ - "core":"COLL_S_E_15_shard7_replica_n26", - "shard":"shard7", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A024a9c138d972d437:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A024a9c138d972d437:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}}], - "shard12":[{ - "core_node47":{ - "core":"COLL_S_E_15_shard12_replica_n44", - "shard":"shard12", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A024a9c138d972d437:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A024a9c138d972d437:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}}]}, - "COLL_N_Q_C_T_11":{"shard9":[{ - "core_node18":{ - "core":"COLL_N_Q_C_T_11_shard9_replica_n16", - "shard":"shard9", - "collection":"COLL_N_Q_C_T_11", - "node_name":"NODE_S_A024a9c138d972d437:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A024a9c138d972d437:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.006099941208958626}}]}, - "COLL_S_H_7":{"shard3":[{ - "core_node7":{ - "core":"COLL_S_H_7_shard3_replica_n4", - "shard":"shard3", - "collection":"COLL_S_H_7", - "node_name":"NODE_S_A024a9c138d972d437:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A024a9c138d972d437:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.005192643962800503}}]}, - "COLL_S_E_15_signals":{"shard2":[{ - "core_node7":{ - "core":"COLL_S_E_15_signals_shard2_replica_n4", - "shard":"shard2", - "collection":"COLL_S_E_15_signals", - "node_name":"NODE_S_A024a9c138d972d437:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A024a9c138d972d437:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":2.8833746910095215E-5}}]}, - "COLL_R_P_S_A_22":{"shard27":[{ - "core_node55":{ - "core":"COLL_R_P_S_A_22_shard27_replica_n52", - "shard":"shard27", - "collection":"COLL_R_P_S_A_22", - "node_name":"NODE_S_A024a9c138d972d437:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A024a9c138d972d437:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.46627365704625845}}]}, - "COLL_E_v2_5":{ - "shard2":[{ - "core_node7":{ - "core":"COLL_E_v2_5_shard2_replica_n4", - "shard":"shard2", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A024a9c138d972d437:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A024a9c138d972d437:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":138.75423685833812}}], - "shard3":[{ - "core_node11":{ - "core":"COLL_E_v2_5_shard3_replica_n8", - "shard":"shard3", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A024a9c138d972d437:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A024a9c138d972d437:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":134.08093044813722}}]}}} - ,{ - "node":"NODE_S_A0ebcce85ef9c13308:8983_solr", - "isLive":true, - "cores":11.0, - "freedisk":617.0940132141113, - "totaldisk":999.51171875, - "replicas":{ - "query2query":{ - "shard11":[{ - "core_node23":{ - "core":"query2query_shard11_replica_n20", - "shard":"shard11", - "collection":"query2query", - "node_name":"NODE_S_A0ebcce85ef9c13308:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0ebcce85ef9c13308:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.43222586531192064}}], - "shard8":[{ - "core_node17":{ - "core":"query2query_shard8_replica_n14", - "shard":"shard8", - "collection":"query2query", - "node_name":"NODE_S_A0ebcce85ef9c13308:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0ebcce85ef9c13308:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.4294911604374647}}]}, - "COLL_S_R":{"shard24":[{ - "core_node49":{ - "core":"COLL_S_R_shard24_replica_n46", - "shard":"shard24", - "collection":"COLL_S_R", - "node_name":"NODE_S_A0ebcce85ef9c13308:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0ebcce85ef9c13308:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.0038131549954414368}}]}, - "Q2Q_postsandsubs":{ - "shard2":[{ - "core_node5":{ - "core":"Q2Q_postsandsubs_shard2_replica_n2", - "shard":"shard2", - "collection":"Q2Q_postsandsubs", - "node_name":"NODE_S_A0ebcce85ef9c13308:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0ebcce85ef9c13308:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.020282848738133907}}], - "shard11":[{ - "core_node23":{ - "core":"Q2Q_postsandsubs_shard11_replica_n20", - "shard":"shard11", - "collection":"Q2Q_postsandsubs", - "node_name":"NODE_S_A0ebcce85ef9c13308:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0ebcce85ef9c13308:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.0202587079256773}}]}, - "subreddit_experiment_results":{ - "shard13":[{ - "core_node27":{ - "core":"subreddit_experiment_results_shard13_replica_n24", - "shard":"shard13", - "collection":"subreddit_experiment_results", - "node_name":"NODE_S_A0ebcce85ef9c13308:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0ebcce85ef9c13308:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.004564180038869381}}], - "shard12":[{ - "core_node25":{ - "core":"subreddit_experiment_results_shard12_replica_n22", - "shard":"shard12", - "collection":"subreddit_experiment_results", - "node_name":"NODE_S_A0ebcce85ef9c13308:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0ebcce85ef9c13308:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.004466232843697071}}]}, - "COLL_R_P_S_A_22":{"shard24":[{ - "core_node49":{ - "core":"COLL_R_P_S_A_22_shard24_replica_n46", - "shard":"shard24", - "collection":"COLL_R_P_S_A_22", - "node_name":"NODE_S_A0ebcce85ef9c13308:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0ebcce85ef9c13308:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.45443680975586176}}]}, - "COLL_E_v2_5":{ - "shard14":[{ - "core_node152":{ - "core":"COLL_E_v2_5_shard14_replica_n151", - "shard":"shard14", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A0ebcce85ef9c13308:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A0ebcce85ef9c13308:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":126.04772036429495}}], - "shard4":[{ - "core_node200":{ - "core":"COLL_E_v2_5_shard4_replica_n199", - "shard":"shard4", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A0ebcce85ef9c13308:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0ebcce85ef9c13308:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":127.18770047463477}}], - "shard8":[{ - "core_node172":{ - "core":"COLL_E_v2_5_shard8_replica_n171", - "shard":"shard8", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A0ebcce85ef9c13308:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0ebcce85ef9c13308:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":127.62798370420933}}]}}} - ,{ - "node":"NODE_S_A0f66f4bec1ce8fc09:8983_solr", - "isLive":true, - "cores":11.0, - "freedisk":715.2999267578125, - "totaldisk":999.51171875, - "replicas":{ - "query2query":{ - "shard7":[{ - "core_node15":{ - "core":"query2query_shard7_replica_n12", - "shard":"shard7", - "collection":"query2query", - "node_name":"NODE_S_A0f66f4bec1ce8fc09:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0f66f4bec1ce8fc09:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.4308900246396661}}], - "shard12":[{ - "core_node25":{ - "core":"query2query_shard12_replica_n22", - "shard":"shard12", - "collection":"query2query", - "node_name":"NODE_S_A0f66f4bec1ce8fc09:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0f66f4bec1ce8fc09:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.43196454364806414}}]}, - "COLL_S_R":{"shard25":[{ - "core_node51":{ - "core":"COLL_S_R_shard25_replica_n48", - "shard":"shard25", - "collection":"COLL_S_R", - "node_name":"NODE_S_A0f66f4bec1ce8fc09:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0f66f4bec1ce8fc09:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.003865146078169346}}]}, - "Q2Q_postsandsubs":{ - "shard1":[{ - "core_node3":{ - "core":"Q2Q_postsandsubs_shard1_replica_n1", - "shard":"shard1", - "collection":"Q2Q_postsandsubs", - "node_name":"NODE_S_A0f66f4bec1ce8fc09:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0f66f4bec1ce8fc09:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.01946087647229433}}], - "shard12":[{ - "core_node25":{ - "core":"Q2Q_postsandsubs_shard12_replica_n22", - "shard":"shard12", - "collection":"Q2Q_postsandsubs", - "node_name":"NODE_S_A0f66f4bec1ce8fc09:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0f66f4bec1ce8fc09:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.018688012845814228}}]}, - "COLL_P_V_E_v2":{"shard4":[{ - "core_node17":{ - "core":"COLL_P_V_E_v2_shard4_replica_n14", - "shard":"shard4", - "collection":"COLL_P_V_E_v2", - "node_name":"NODE_S_A0f66f4bec1ce8fc09:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0f66f4bec1ce8fc09:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":8.185325101017952}}]}, - "subreddit_experiment_results":{ - "shard17":[{ - "core_node35":{ - "core":"subreddit_experiment_results_shard17_replica_n32", - "shard":"shard17", - "collection":"subreddit_experiment_results", - "node_name":"NODE_S_A0f66f4bec1ce8fc09:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0f66f4bec1ce8fc09:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.0044752322137355804}}], - "shard8":[{ - "core_node17":{ - "core":"subreddit_experiment_results_shard8_replica_n14", - "shard":"shard8", - "collection":"subreddit_experiment_results", - "node_name":"NODE_S_A0f66f4bec1ce8fc09:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0f66f4bec1ce8fc09:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.004475797526538372}}]}, - "COLL_R_P_S_A_22":{"shard25":[{ - "core_node51":{ - "core":"COLL_R_P_S_A_22_shard25_replica_n48", - "shard":"shard25", - "collection":"COLL_R_P_S_A_22", - "node_name":"NODE_S_A0f66f4bec1ce8fc09:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0f66f4bec1ce8fc09:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.45042258128523827}}]}, - "COLL_E_v2_5":{ - "shard19":[{ - "core_node196":{ - "core":"COLL_E_v2_5_shard19_replica_n195", - "shard":"shard19", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A0f66f4bec1ce8fc09:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A0f66f4bec1ce8fc09:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":131.19690136332065}}], - "shard8":[{ - "core_node150":{ - "core":"COLL_E_v2_5_shard8_replica_n149", - "shard":"shard8", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A0f66f4bec1ce8fc09:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A0f66f4bec1ce8fc09:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":127.70166133996099}}]}}} - ,{ - "node":"NODE_S_A02f6288aa3fc9eb3d:8983_solr", - "isLive":true, - "cores":10.0, - "freedisk":598.8659477233887, - "totaldisk":999.51171875, - "replicas":{ - "query2query":{ - "shard10":[{ - "core_node21":{ - "core":"query2query_shard10_replica_n18", - "shard":"shard10", - "collection":"query2query", - "node_name":"NODE_S_A02f6288aa3fc9eb3d:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A02f6288aa3fc9eb3d:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.429669008590281}}], - "shard9":[{ - "core_node19":{ - "core":"query2query_shard9_replica_n16", - "shard":"shard9", - "collection":"query2query", - "node_name":"NODE_S_A02f6288aa3fc9eb3d:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A02f6288aa3fc9eb3d:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.4358885111287236}}]}, - "COLL_S_R":{"shard17":[{ - "core_node35":{ - "core":"COLL_S_R_shard17_replica_n32", - "shard":"shard17", - "collection":"COLL_S_R", - "node_name":"NODE_S_A02f6288aa3fc9eb3d:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A02f6288aa3fc9eb3d:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.0038110455498099327}}]}, - "Q2Q_postsandsubs":{ - "shard4":[{ - "core_node9":{ - "core":"Q2Q_postsandsubs_shard4_replica_n6", - "shard":"shard4", - "collection":"Q2Q_postsandsubs", - "node_name":"NODE_S_A02f6288aa3fc9eb3d:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A02f6288aa3fc9eb3d:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.02034578751772642}}], - "shard9":[{ - "core_node19":{ - "core":"Q2Q_postsandsubs_shard9_replica_n16", - "shard":"shard9", - "collection":"Q2Q_postsandsubs", - "node_name":"NODE_S_A02f6288aa3fc9eb3d:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A02f6288aa3fc9eb3d:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.01951229851692915}}]}, - "COLL_P_V_E_v2":{"shard6":[{ - "core_node23":{ - "core":"COLL_P_V_E_v2_shard6_replica_n20", - "shard":"shard6", - "collection":"COLL_P_V_E_v2", - "node_name":"NODE_S_A02f6288aa3fc9eb3d:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A02f6288aa3fc9eb3d:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":8.19284622464329}}]}, - "COLL_R_P_S_A_22":{"shard17":[{ - "core_node35":{ - "core":"COLL_R_P_S_A_22_shard17_replica_n32", - "shard":"shard17", - "collection":"COLL_R_P_S_A_22", - "node_name":"NODE_S_A02f6288aa3fc9eb3d:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A02f6288aa3fc9eb3d:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.4581049457192421}}]}, - "COLL_E_v2_5":{ - "shard17":[{ - "core_node214":{ - "core":"COLL_E_v2_5_shard17_replica_n213", - "shard":"shard17", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A02f6288aa3fc9eb3d:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A02f6288aa3fc9eb3d:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":128.23563138395548}}], - "shard18":[{ - "core_node242":{ - "core":"COLL_E_v2_5_shard18_replica_n241", - "shard":"shard18", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A02f6288aa3fc9eb3d:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A02f6288aa3fc9eb3d:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":129.97117522824556}}], - "shard33":[{ - "core_node188":{ - "core":"COLL_E_v2_5_shard33_replica_n187", - "shard":"shard33", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A02f6288aa3fc9eb3d:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A02f6288aa3fc9eb3d:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":132.64680305775255}}]}}} - ,{ - "node":"NODE_S_A0cf36cc11a8447fac:8983_solr", - "isLive":true, - "cores":10.0, - "freedisk":597.7325668334961, - "totaldisk":999.51171875, - "replicas":{ - "query2query":{ - "shard14":[{ - "core_node29":{ - "core":"query2query_shard14_replica_n26", - "shard":"shard14", - "collection":"query2query", - "node_name":"NODE_S_A0cf36cc11a8447fac:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0cf36cc11a8447fac:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.4342966331169009}}], - "shard5":[{ - "core_node11":{ - "core":"query2query_shard5_replica_n8", - "shard":"shard5", - "collection":"query2query", - "node_name":"NODE_S_A0cf36cc11a8447fac:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0cf36cc11a8447fac:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.43006310891360044}}]}, - "COLL_S_R":{"shard22":[{ - "core_node45":{ - "core":"COLL_S_R_shard22_replica_n42", - "shard":"shard22", - "collection":"COLL_S_R", - "node_name":"NODE_S_A0cf36cc11a8447fac:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0cf36cc11a8447fac:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.003885832615196705}}]}, - "Q2Q_postsandsubs":{"shard14":[{ - "core_node29":{ - "core":"Q2Q_postsandsubs_shard14_replica_n26", - "shard":"shard14", - "collection":"Q2Q_postsandsubs", - "node_name":"NODE_S_A0cf36cc11a8447fac:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0cf36cc11a8447fac:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.01883882563561201}}]}, - "subreddit_experiment_results":{ - "shard4":[{ - "core_node9":{ - "core":"subreddit_experiment_results_shard4_replica_n6", - "shard":"shard4", - "collection":"subreddit_experiment_results", - "node_name":"NODE_S_A0cf36cc11a8447fac:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0cf36cc11a8447fac:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.004524897783994675}}], - "shard21":[{ - "core_node43":{ - "core":"subreddit_experiment_results_shard21_replica_n40", - "shard":"shard21", - "collection":"subreddit_experiment_results", - "node_name":"NODE_S_A0cf36cc11a8447fac:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0cf36cc11a8447fac:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.004553055390715599}}]}, - "COLL_R_P_S_A_22":{"shard22":[{ - "core_node45":{ - "core":"COLL_R_P_S_A_22_shard22_replica_n42", - "shard":"shard22", - "collection":"COLL_R_P_S_A_22", - "node_name":"NODE_S_A0cf36cc11a8447fac:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0cf36cc11a8447fac:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.4733404340222478}}]}, - "COLL_E_v2_5":{ - "shard14":[{ - "core_node174":{ - "core":"COLL_E_v2_5_shard14_replica_n173", - "shard":"shard14", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A0cf36cc11a8447fac:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0cf36cc11a8447fac:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":133.26378810219467}}], - "shard5":[{ - "core_node202":{ - "core":"COLL_E_v2_5_shard5_replica_n201", - "shard":"shard5", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A0cf36cc11a8447fac:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0cf36cc11a8447fac:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":128.80492931138724}}], - "shard34":[{ - "core_node226":{ - "core":"COLL_E_v2_5_shard34_replica_n225", - "shard":"shard34", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A0cf36cc11a8447fac:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0cf36cc11a8447fac:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":138.11697097029537}}]}}} - ,{ - "node":"NODE_S_A061d3ac3c71d42de2:8983_solr", - "isLive":true, - "cores":10.0, - "freedisk":600.650707244873, - "totaldisk":999.51171875, - "replicas":{ - "query2query":{ - "shard3":[{ - "core_node7":{ - "core":"query2query_shard3_replica_n4", - "shard":"shard3", - "collection":"query2query", - "node_name":"NODE_S_A061d3ac3c71d42de2:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A061d3ac3c71d42de2:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.430878022685647}}], - "shard16":[{ - "core_node33":{ - "core":"query2query_shard16_replica_n30", - "shard":"shard16", - "collection":"query2query", - "node_name":"NODE_S_A061d3ac3c71d42de2:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A061d3ac3c71d42de2:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.4258228335529566}}]}, - "COLL_S_R":{"shard16":[{ - "core_node33":{ - "core":"COLL_S_R_shard16_replica_n30", - "shard":"shard16", - "collection":"COLL_S_R", - "node_name":"NODE_S_A061d3ac3c71d42de2:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A061d3ac3c71d42de2:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.0038702944293618202}}]}, - "Q2Q_postsandsubs":{ - "shard17":[{ - "core_node35":{ - "core":"Q2Q_postsandsubs_shard17_replica_n32", - "shard":"shard17", - "collection":"Q2Q_postsandsubs", - "node_name":"NODE_S_A061d3ac3c71d42de2:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A061d3ac3c71d42de2:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.020032744854688644}}], - "shard20":[{ - "core_node41":{ - "core":"Q2Q_postsandsubs_shard20_replica_n38", - "shard":"shard20", - "collection":"Q2Q_postsandsubs", - "node_name":"NODE_S_A061d3ac3c71d42de2:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A061d3ac3c71d42de2:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.02001035585999489}}]}, - "subreddit_experiment_results":{"shard25":[{ - "core_node51":{ - "core":"subreddit_experiment_results_shard25_replica_n48", - "shard":"shard25", - "collection":"subreddit_experiment_results", - "node_name":"NODE_S_A061d3ac3c71d42de2:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A061d3ac3c71d42de2:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.004562162794172764}}]}, - "COLL_R_P_S_A_22":{"shard16":[{ - "core_node33":{ - "core":"COLL_R_P_S_A_22_shard16_replica_n30", - "shard":"shard16", - "collection":"COLL_R_P_S_A_22", - "node_name":"NODE_S_A061d3ac3c71d42de2:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A061d3ac3c71d42de2:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.46511795837432146}}]}, - "COLL_E_v2_5":{ - "shard24":[{ - "core_node180":{ - "core":"COLL_E_v2_5_shard24_replica_n179", - "shard":"shard24", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A061d3ac3c71d42de2:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A061d3ac3c71d42de2:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":132.03363047912717}}], - "shard13":[{ - "core_node208":{ - "core":"COLL_E_v2_5_shard13_replica_n207", - "shard":"shard13", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A061d3ac3c71d42de2:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A061d3ac3c71d42de2:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":134.98152516968548}}], - "shard30":[{ - "core_node160":{ - "core":"COLL_E_v2_5_shard30_replica_n159", - "shard":"shard30", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A061d3ac3c71d42de2:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A061d3ac3c71d42de2:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":130.25954080559313}}]}}} - ,{ - "node":"NODE_S_A08c1ec5957d8a5d44:8983_solr", - "isLive":true, - "cores":10.0, - "freedisk":724.6202278137207, - "totaldisk":999.51171875, - "replicas":{ - "query2query":{ - "shard15":[{ - "core_node31":{ - "core":"query2query_shard15_replica_n28", - "shard":"shard15", - "collection":"query2query", - "node_name":"NODE_S_A08c1ec5957d8a5d44:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A08c1ec5957d8a5d44:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.4261511182412505}}], - "shard4":[{ - "core_node9":{ - "core":"query2query_shard4_replica_n6", - "shard":"shard4", - "collection":"query2query", - "node_name":"NODE_S_A08c1ec5957d8a5d44:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A08c1ec5957d8a5d44:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.4257269874215126}}]}, - "COLL_S_R":{"shard19":[{ - "core_node39":{ - "core":"COLL_S_R_shard19_replica_n36", - "shard":"shard19", - "collection":"COLL_S_R", - "node_name":"NODE_S_A08c1ec5957d8a5d44:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A08c1ec5957d8a5d44:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.003870014101266861}}]}, - "Q2Q_postsandsubs":{"shard15":[{ - "core_node31":{ - "core":"Q2Q_postsandsubs_shard15_replica_n28", - "shard":"shard15", - "collection":"Q2Q_postsandsubs", - "node_name":"NODE_S_A08c1ec5957d8a5d44:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A08c1ec5957d8a5d44:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.020441552624106407}}]}, - "COLL_P_V_E_v2":{"shard4":[{ - "core_node15":{ - "core":"COLL_P_V_E_v2_shard4_replica_n12", - "shard":"shard4", - "collection":"COLL_P_V_E_v2", - "node_name":"NODE_S_A08c1ec5957d8a5d44:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A08c1ec5957d8a5d44:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":8.152389544062316}}]}, - "subreddit_experiment_results":{ - "shard3":[{ - "core_node7":{ - "core":"subreddit_experiment_results_shard3_replica_n4", - "shard":"shard3", - "collection":"subreddit_experiment_results", - "node_name":"NODE_S_A08c1ec5957d8a5d44:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A08c1ec5957d8a5d44:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.004642372019588947}}], - "shard22":[{ - "core_node45":{ - "core":"subreddit_experiment_results_shard22_replica_n42", - "shard":"shard22", - "collection":"subreddit_experiment_results", - "node_name":"NODE_S_A08c1ec5957d8a5d44:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A08c1ec5957d8a5d44:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.004599213600158691}}]}, - "COLL_R_P_S_A_22":{"shard19":[{ - "core_node39":{ - "core":"COLL_R_P_S_A_22_shard19_replica_n36", - "shard":"shard19", - "collection":"COLL_R_P_S_A_22", - "node_name":"NODE_S_A08c1ec5957d8a5d44:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A08c1ec5957d8a5d44:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.41446082293987274}}]}, - "COLL_E_v2_5":{ - "shard16":[{ - "core_node176":{ - "core":"COLL_E_v2_5_shard16_replica_n175", - "shard":"shard16", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A08c1ec5957d8a5d44:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A08c1ec5957d8a5d44:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":132.18093105591834}}], - "shard22":[{ - "core_node156":{ - "core":"COLL_E_v2_5_shard22_replica_n155", - "shard":"shard22", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A08c1ec5957d8a5d44:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A08c1ec5957d8a5d44:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":133.10871708858758}}]}}} - ,{ - "node":"NODE_S_A084aef0a141a43572:8983_solr", - "isLive":true, - "cores":10.0, - "freedisk":733.0749816894531, - "totaldisk":999.51171875, - "replicas":{ - "query2query":{ - "shard2":[{ - "core_node5":{ - "core":"query2query_shard2_replica_n2", - "shard":"shard2", - "collection":"query2query", - "node_name":"NODE_S_A084aef0a141a43572:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A084aef0a141a43572:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.4285783916711807}}], - "shard17":[{ - "core_node35":{ - "core":"query2query_shard17_replica_n32", - "shard":"shard17", - "collection":"query2query", - "node_name":"NODE_S_A084aef0a141a43572:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A084aef0a141a43572:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.4329432938247919}}]}, - "COLL_S_R":{"shard20":[{ - "core_node41":{ - "core":"COLL_S_R_shard20_replica_n38", - "shard":"shard20", - "collection":"COLL_S_R", - "node_name":"NODE_S_A084aef0a141a43572:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A084aef0a141a43572:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.0037896251305937767}}]}, - "Q2Q_postsandsubs":{ - "shard16":[{ - "core_node33":{ - "core":"Q2Q_postsandsubs_shard16_replica_n30", - "shard":"shard16", - "collection":"Q2Q_postsandsubs", - "node_name":"NODE_S_A084aef0a141a43572:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A084aef0a141a43572:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.0185230216011405}}], - "shard21":[{ - "core_node42":{ - "core":"Q2Q_postsandsubs_shard21_replica_n40", - "shard":"shard21", - "collection":"Q2Q_postsandsubs", - "node_name":"NODE_S_A084aef0a141a43572:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A084aef0a141a43572:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.019887510687112808}}]}, - "subreddit_experiment_results":{ - "shard15":[{ - "core_node31":{ - "core":"subreddit_experiment_results_shard15_replica_n28", - "shard":"shard15", - "collection":"subreddit_experiment_results", - "node_name":"NODE_S_A084aef0a141a43572:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A084aef0a141a43572:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.004504849202930927}}], - "shard10":[{ - "core_node21":{ - "core":"subreddit_experiment_results_shard10_replica_n18", - "shard":"shard10", - "collection":"subreddit_experiment_results", - "node_name":"NODE_S_A084aef0a141a43572:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A084aef0a141a43572:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.00449906662106514}}]}, - "COLL_R_P_S_A_22":{"shard20":[{ - "core_node41":{ - "core":"COLL_R_P_S_A_22_shard20_replica_n38", - "shard":"shard20", - "collection":"COLL_R_P_S_A_22", - "node_name":"NODE_S_A084aef0a141a43572:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A084aef0a141a43572:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.5044156881049275}}]}, - "COLL_E_v2_5":{ - "shard30":[{ - "core_node184":{ - "core":"COLL_E_v2_5_shard30_replica_n183", - "shard":"shard30", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A084aef0a141a43572:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A084aef0a141a43572:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":132.65275741927326}}], - "shard22":[{ - "core_node178":{ - "core":"COLL_E_v2_5_shard22_replica_n177", - "shard":"shard22", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A084aef0a141a43572:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A084aef0a141a43572:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":132.18785760644823}}]}}} - ,{ - "node":"NODE_S_A0f096665ad4a3bc58:8983_solr", - "isLive":true, - "cores":10.0, - "freedisk":731.5768585205078, - "totaldisk":999.51171875, - "replicas":{ - "query2query":{ - "shard18":[{ - "core_node37":{ - "core":"query2query_shard18_replica_n34", - "shard":"shard18", - "collection":"query2query", - "node_name":"NODE_S_A0f096665ad4a3bc58:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0f096665ad4a3bc58:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.4305841075256467}}], - "shard1":[{ - "core_node3":{ - "core":"query2query_shard1_replica_n1", - "shard":"shard1", - "collection":"query2query", - "node_name":"NODE_S_A0f096665ad4a3bc58:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0f096665ad4a3bc58:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.42587023694068193}}]}, - "COLL_S_R":{"shard21":[{ - "core_node43":{ - "core":"COLL_S_R_shard21_replica_n40", - "shard":"shard21", - "collection":"COLL_S_R", - "node_name":"NODE_S_A0f096665ad4a3bc58:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0f096665ad4a3bc58:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.0038251150399446487}}]}, - "Q2Q_postsandsubs":{"shard13":[{ - "core_node27":{ - "core":"Q2Q_postsandsubs_shard13_replica_n24", - "shard":"shard13", - "collection":"Q2Q_postsandsubs", - "node_name":"NODE_S_A0f096665ad4a3bc58:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0f096665ad4a3bc58:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.02031210344284773}}]}, - "subreddit_experiment_results":{ - "shard24":[{ - "core_node49":{ - "core":"subreddit_experiment_results_shard24_replica_n46", - "shard":"shard24", - "collection":"subreddit_experiment_results", - "node_name":"NODE_S_A0f096665ad4a3bc58:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0f096665ad4a3bc58:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.004560600966215134}}], - "shard1":[{ - "core_node3":{ - "core":"subreddit_experiment_results_shard1_replica_n1", - "shard":"shard1", - "collection":"subreddit_experiment_results", - "node_name":"NODE_S_A0f096665ad4a3bc58:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0f096665ad4a3bc58:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.0045231664553284645}}]}, - "COLL_PV_E_v2_5":{"shard2":[{ - "core_node9":{ - "core":"COLL_PV_E_v2_5_shard2_replica_n6", - "shard":"shard2", - "collection":"COLL_PV_E_v2_5", - "node_name":"NODE_S_A0f096665ad4a3bc58:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0f096665ad4a3bc58:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":3.0214350102469325}}]}, - "COLL_R_P_S_A_22":{"shard21":[{ - "core_node43":{ - "core":"COLL_R_P_S_A_22_shard21_replica_n40", - "shard":"shard21", - "collection":"COLL_R_P_S_A_22", - "node_name":"NODE_S_A0f096665ad4a3bc58:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0f096665ad4a3bc58:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.4323454797267914}}]}, - "COLL_E_v2_5":{ - "shard29":[{ - "core_node222":{ - "core":"COLL_E_v2_5_shard29_replica_n221", - "shard":"shard29", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A0f096665ad4a3bc58:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0f096665ad4a3bc58:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":131.6502357730642}}], - "shard6":[{ - "core_node170":{ - "core":"COLL_E_v2_5_shard6_replica_n169", - "shard":"shard6", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A0f096665ad4a3bc58:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A0f096665ad4a3bc58:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":131.79092547670007}}]}}} - ,{ - "node":"NODE_S_A006b71d81aedd8577:8983_solr", - "isLive":true, - "cores":9.0, - "freedisk":603.3365592956543, - "totaldisk":999.51171875, - "replicas":{ - "query2query":{"shard20":[{ - "core_node41":{ - "core":"query2query_shard20_replica_n38", - "shard":"shard20", - "collection":"query2query", - "node_name":"NODE_S_A006b71d81aedd8577:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A006b71d81aedd8577:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.43263052497059107}}]}, - "COLL_S_R":{"shard14":[{ - "core_node29":{ - "core":"COLL_S_R_shard14_replica_n26", - "shard":"shard14", - "collection":"COLL_S_R", - "node_name":"NODE_S_A006b71d81aedd8577:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A006b71d81aedd8577:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.003805852495133877}}]}, - "Q2Q_postsandsubs":{ - "shard6":[{ - "core_node13":{ - "core":"Q2Q_postsandsubs_shard6_replica_n10", - "shard":"shard6", - "collection":"Q2Q_postsandsubs", - "node_name":"NODE_S_A006b71d81aedd8577:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A006b71d81aedd8577:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.01943675335496664}}], - "shard7":[{ - "core_node15":{ - "core":"Q2Q_postsandsubs_shard7_replica_n12", - "shard":"shard7", - "collection":"Q2Q_postsandsubs", - "node_name":"NODE_S_A006b71d81aedd8577:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A006b71d81aedd8577:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.020343396812677383}}]}, - "COLL_P_V_E_v2":{"shard2":[{ - "core_node9":{ - "core":"COLL_P_V_E_v2_shard2_replica_n6", - "shard":"shard2", - "collection":"COLL_P_V_E_v2", - "node_name":"NODE_S_A006b71d81aedd8577:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A006b71d81aedd8577:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":8.209227412007749}}]}, - "COLL_R_P_S_A_22":{"shard14":[{ - "core_node29":{ - "core":"COLL_R_P_S_A_22_shard14_replica_n26", - "shard":"shard14", - "collection":"COLL_R_P_S_A_22", - "node_name":"NODE_S_A006b71d81aedd8577:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A006b71d81aedd8577:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.4834298687055707}}]}, - "COLL_E_v2_5":{ - "shard25":[{ - "core_node218":{ - "core":"COLL_E_v2_5_shard25_replica_n217", - "shard":"shard25", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A006b71d81aedd8577:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A006b71d81aedd8577:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":133.02660531457514}}], - "shard1":[{ - "core_node166":{ - "core":"COLL_E_v2_5_shard1_replica_n165", - "shard":"shard1", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A006b71d81aedd8577:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A006b71d81aedd8577:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":127.43107242323458}}], - "shard11":[{ - "core_node194":{ - "core":"COLL_E_v2_5_shard11_replica_n193", - "shard":"shard11", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A006b71d81aedd8577:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A006b71d81aedd8577:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":126.14626492932439}}]}}} - ,{ - "node":"NODE_S_A014292d062d5088da:8983_solr", - "isLive":true, - "cores":8.0, - "freedisk":724.7788581848145, - "totaldisk":999.51171875, - "replicas":{ - "query2query":{"shard21":[{ - "core_node42":{ - "core":"query2query_shard21_replica_n40", - "shard":"shard21", - "collection":"query2query", - "node_name":"NODE_S_A014292d062d5088da:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A014292d062d5088da:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.42733157239854336}}]}, - "COLL_S_R":{"shard13":[{ - "core_node27":{ - "core":"COLL_S_R_shard13_replica_n24", - "shard":"shard13", - "collection":"COLL_S_R", - "node_name":"NODE_S_A014292d062d5088da:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A014292d062d5088da:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.00382127333432436}}]}, - "Q2Q_postsandsubs":{ - "shard5":[{ - "core_node11":{ - "core":"Q2Q_postsandsubs_shard5_replica_n8", - "shard":"shard5", - "collection":"Q2Q_postsandsubs", - "node_name":"NODE_S_A014292d062d5088da:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A014292d062d5088da:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.019260264933109283}}], - "shard8":[{ - "core_node17":{ - "core":"Q2Q_postsandsubs_shard8_replica_n14", - "shard":"shard8", - "collection":"Q2Q_postsandsubs", - "node_name":"NODE_S_A014292d062d5088da:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A014292d062d5088da:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.01941517647355795}}]}, - "COLL_P_V_E_v2":{"shard2":[{ - "core_node7":{ - "core":"COLL_P_V_E_v2_shard2_replica_n4", - "shard":"shard2", - "collection":"COLL_P_V_E_v2", - "node_name":"NODE_S_A014292d062d5088da:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A014292d062d5088da:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":8.212146437726915}}]}, - "COLL_R_P_S_A_22":{"shard13":[{ - "core_node27":{ - "core":"COLL_R_P_S_A_22_shard13_replica_n24", - "shard":"shard13", - "collection":"COLL_R_P_S_A_22", - "node_name":"NODE_S_A014292d062d5088da:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A014292d062d5088da:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.4219518853351474}}]}, - "COLL_E_v2_5":{ - "shard10":[{ - "core_node238":{ - "core":"COLL_E_v2_5_shard10_replica_n237", - "shard":"shard10", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A014292d062d5088da:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A014292d062d5088da:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":133.62058495730162}}], - "shard12":[{ - "core_node240":{ - "core":"COLL_E_v2_5_shard12_replica_n239", - "shard":"shard12", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A014292d062d5088da:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A014292d062d5088da:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":131.8720874004066}}]}}} - ,{ - "node":"NODE_S_A06032b38152c0f019:8983_solr", - "isLive":true, - "cores":8.0, - "freedisk":734.5076332092285, - "totaldisk":999.51171875, - "replicas":{ - "query2query":{ - "shard13":[{ - "core_node27":{ - "core":"query2query_shard13_replica_n24", - "shard":"shard13", - "collection":"query2query", - "node_name":"NODE_S_A06032b38152c0f019:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A06032b38152c0f019:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.4311547642573714}}], - "shard6":[{ - "core_node13":{ - "core":"query2query_shard6_replica_n10", - "shard":"shard6", - "collection":"query2query", - "node_name":"NODE_S_A06032b38152c0f019:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A06032b38152c0f019:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.42810762021690607}}]}, - "COLL_S_R":{"shard15":[{ - "core_node31":{ - "core":"COLL_S_R_shard15_replica_n28", - "shard":"shard15", - "collection":"COLL_S_R", - "node_name":"NODE_S_A06032b38152c0f019:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A06032b38152c0f019:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.0038444409146904945}}]}, - "Q2Q_postsandsubs":{ - "shard3":[{ - "core_node7":{ - "core":"Q2Q_postsandsubs_shard3_replica_n4", - "shard":"shard3", - "collection":"Q2Q_postsandsubs", - "node_name":"NODE_S_A06032b38152c0f019:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A06032b38152c0f019:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.020358268171548843}}], - "shard10":[{ - "core_node21":{ - "core":"Q2Q_postsandsubs_shard10_replica_n18", - "shard":"shard10", - "collection":"Q2Q_postsandsubs", - "node_name":"NODE_S_A06032b38152c0f019:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A06032b38152c0f019:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.02053267415612936}}]}, - "COLL_R_P_S_A_22":{"shard15":[{ - "core_node31":{ - "core":"COLL_R_P_S_A_22_shard15_replica_n28", - "shard":"shard15", - "collection":"COLL_R_P_S_A_22", - "node_name":"NODE_S_A06032b38152c0f019:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A06032b38152c0f019:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.5062682097777724}}]}, - "COLL_E_v2_5":{ - "shard24":[{ - "core_node182":{ - "core":"COLL_E_v2_5_shard24_replica_n181", - "shard":"shard24", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A06032b38152c0f019:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A06032b38152c0f019:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":132.00969661120325}}], - "shard6":[{ - "core_node162":{ - "core":"COLL_E_v2_5_shard6_replica_n161", - "shard":"shard6", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A06032b38152c0f019:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A06032b38152c0f019:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":131.38960415963084}}]}}} - ,{ - "node":"NODE_S_A04a75744569a27ccf:8983_solr", - "isLive":true, - "cores":8.0, - "freedisk":728.2148056030273, - "totaldisk":999.51171875, - "replicas":{ - "query2query":{"shard19":[{ - "core_node39":{ - "core":"query2query_shard19_replica_n36", - "shard":"shard19", - "collection":"query2query", - "node_name":"NODE_S_A04a75744569a27ccf:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A04a75744569a27ccf:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.42521866224706173}}]}, - "COLL_S_R":{"shard18":[{ - "core_node37":{ - "core":"COLL_S_R_shard18_replica_n34", - "shard":"shard18", - "collection":"COLL_S_R", - "node_name":"NODE_S_A04a75744569a27ccf:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A04a75744569a27ccf:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.003841894678771496}}]}, - "Q2Q_postsandsubs":{ - "shard18":[{ - "core_node37":{ - "core":"Q2Q_postsandsubs_shard18_replica_n34", - "shard":"shard18", - "collection":"Q2Q_postsandsubs", - "node_name":"NODE_S_A04a75744569a27ccf:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A04a75744569a27ccf:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.019906101748347282}}], - "shard19":[{ - "core_node39":{ - "core":"Q2Q_postsandsubs_shard19_replica_n36", - "shard":"shard19", - "collection":"Q2Q_postsandsubs", - "node_name":"NODE_S_A04a75744569a27ccf:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A04a75744569a27ccf:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.019286231137812138}}]}, - "subreddit_experiment_results":{"shard27":[{ - "core_node54":{ - "core":"subreddit_experiment_results_shard27_replica_n52", - "shard":"shard27", - "collection":"subreddit_experiment_results", - "node_name":"NODE_S_A04a75744569a27ccf:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A04a75744569a27ccf:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.004555700346827507}}]}, - "COLL_R_P_S_A_22":{"shard18":[{ - "core_node37":{ - "core":"COLL_R_P_S_A_22_shard18_replica_n34", - "shard":"shard18", - "collection":"COLL_R_P_S_A_22", - "node_name":"NODE_S_A04a75744569a27ccf:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A04a75744569a27ccf:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.4519655341282487}}]}, - "COLL_E_v2_5":{ - "shard17":[{ - "core_node212":{ - "core":"COLL_E_v2_5_shard17_replica_n211", - "shard":"shard17", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A04a75744569a27ccf:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A04a75744569a27ccf:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":135.40381868369877}}], - "shard33":[{ - "core_node186":{ - "core":"COLL_E_v2_5_shard33_replica_n185", - "shard":"shard33", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A04a75744569a27ccf:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A04a75744569a27ccf:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":134.5945677505806}}]}}} - ,{ - "node":"NODE_S_A0e48ab035b869305e:8983_solr", - "isLive":true, - "cores":8.0, - "freedisk":735.5891609191895, - "totaldisk":999.51171875, - "replicas":{ - "recall_cache":{"shard1":[{ - "core_node5":{ - "core":"recall_cache_shard1_replica_n2", - "shard":"shard1", - "collection":"recall_cache", - "node_name":"NODE_S_A0e48ab035b869305e:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0e48ab035b869305e:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.0050263796001672745}}]}, - "COLL_S_R":{"shard12":[{ - "core_node25":{ - "core":"COLL_S_R_shard12_replica_n22", - "shard":"shard12", - "collection":"COLL_S_R", - "node_name":"NODE_S_A0e48ab035b869305e:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0e48ab035b869305e:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.0038190074265003204}}]}, - "subreddit_experiment_results":{ - "shard19":[{ - "core_node39":{ - "core":"subreddit_experiment_results_shard19_replica_n36", - "shard":"shard19", - "collection":"subreddit_experiment_results", - "node_name":"NODE_S_A0e48ab035b869305e:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0e48ab035b869305e:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.00449786800891161}}], - "shard6":[{ - "core_node13":{ - "core":"subreddit_experiment_results_shard6_replica_n10", - "shard":"shard6", - "collection":"subreddit_experiment_results", - "node_name":"NODE_S_A0e48ab035b869305e:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0e48ab035b869305e:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.004472509957849979}}]}, - "COLL_PV_E_v2_5":{"shard3":[{ - "core_node12":{ - "core":"COLL_PV_E_v2_5_shard3_replica_n10", - "shard":"shard3", - "collection":"COLL_PV_E_v2_5", - "node_name":"NODE_S_A0e48ab035b869305e:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0e48ab035b869305e:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":3.0129052978008986}}]}, - "COLL_R_P_S_A_22":{"shard12":[{ - "core_node25":{ - "core":"COLL_R_P_S_A_22_shard12_replica_n22", - "shard":"shard12", - "collection":"COLL_R_P_S_A_22", - "node_name":"NODE_S_A0e48ab035b869305e:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0e48ab035b869305e:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.5445274133235216}}]}, - "COLL_E_v2_5":{ - "shard36":[{ - "core_node358":{ - "core":"COLL_E_v2_5_shard36_replica_n357", - "shard":"shard36", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A0e48ab035b869305e:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0e48ab035b869305e:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":133.00134566705674}}], - "shard4":[{ - "core_node388":{ - "core":"COLL_E_v2_5_shard4_replica_n387", - "shard":"shard4", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A0e48ab035b869305e:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A0e48ab035b869305e:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":127.21125747542828}}]}}} - ,{ - "node":"NODE_S_A0d1f4439b8e8878e8:8983_solr", - "isLive":true, - "cores":7.0, - "freedisk":711.224853515625, - "totaldisk":999.51171875, - "replicas":{ - "COLL_S_R":{"shard11":[{ - "core_node23":{ - "core":"COLL_S_R_shard11_replica_n20", - "shard":"shard11", - "collection":"COLL_S_R", - "node_name":"NODE_S_A0d1f4439b8e8878e8:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0d1f4439b8e8878e8:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.0038332296535372734}}]}, - "COLL_E_V2_S_A_3":{"shard1":[{ - "core_node10":{ - "core":"COLL_E_V2_S_A_3_shard1_replica_n9", - "shard":"shard1", - "collection":"COLL_E_V2_S_A_3", - "node_name":"NODE_S_A0d1f4439b8e8878e8:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0d1f4439b8e8878e8:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":25.122611593455076}}]}, - "subreddit_experiment_results":{ - "shard5":[{ - "core_node11":{ - "core":"subreddit_experiment_results_shard5_replica_n8", - "shard":"shard5", - "collection":"subreddit_experiment_results", - "node_name":"NODE_S_A0d1f4439b8e8878e8:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0d1f4439b8e8878e8:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.004589034244418144}}], - "shard20":[{ - "core_node41":{ - "core":"subreddit_experiment_results_shard20_replica_n38", - "shard":"shard20", - "collection":"subreddit_experiment_results", - "node_name":"NODE_S_A0d1f4439b8e8878e8:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0d1f4439b8e8878e8:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.004457270726561546}}]}, - "COLL_R_P_S_A_22":{"shard11":[{ - "core_node23":{ - "core":"COLL_R_P_S_A_22_shard11_replica_n20", - "shard":"shard11", - "collection":"COLL_R_P_S_A_22", - "node_name":"NODE_S_A0d1f4439b8e8878e8:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0d1f4439b8e8878e8:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.4702793164178729}}]}, - "COLL_E_v2_5":{ - "shard35":[{ - "core_node374":{ - "core":"COLL_E_v2_5_shard35_replica_n373", - "shard":"shard35", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A0d1f4439b8e8878e8:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0d1f4439b8e8878e8:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":129.69269095920026}}], - "shard27":[{ - "core_node402":{ - "core":"COLL_E_v2_5_shard27_replica_n401", - "shard":"shard27", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A0d1f4439b8e8878e8:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A0d1f4439b8e8878e8:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":132.8685013735667}}]}}} - ,{ - "node":"NODE_S_A0b7fb52d8d0a2ecde:8983_solr", - "isLive":true, - "cores":7.0, - "freedisk":725.6816101074219, - "totaldisk":999.51171875, - "replicas":{ - "recall_cache":{"shard1":[{ - "core_node6":{ - "core":"recall_cache_shard1_replica_n4", - "shard":"shard1", - "collection":"recall_cache", - "node_name":"NODE_S_A0b7fb52d8d0a2ecde:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A0b7fb52d8d0a2ecde:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.005021187476813793}}]}, - "COLL_S_R":{"shard10":[{ - "core_node21":{ - "core":"COLL_S_R_shard10_replica_n18", - "shard":"shard10", - "collection":"COLL_S_R", - "node_name":"NODE_S_A0b7fb52d8d0a2ecde:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0b7fb52d8d0a2ecde:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.003915207460522652}}]}, - "subreddit_experiment_results":{ - "shard16":[{ - "core_node33":{ - "core":"subreddit_experiment_results_shard16_replica_n30", - "shard":"shard16", - "collection":"subreddit_experiment_results", - "node_name":"NODE_S_A0b7fb52d8d0a2ecde:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0b7fb52d8d0a2ecde:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.004510008729994297}}], - "shard9":[{ - "core_node19":{ - "core":"subreddit_experiment_results_shard9_replica_n16", - "shard":"shard9", - "collection":"subreddit_experiment_results", - "node_name":"NODE_S_A0b7fb52d8d0a2ecde:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0b7fb52d8d0a2ecde:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.0044971127063035965}}]}, - "COLL_R_P_S_A_22":{"shard10":[{ - "core_node21":{ - "core":"COLL_R_P_S_A_22_shard10_replica_n18", - "shard":"shard10", - "collection":"COLL_R_P_S_A_22", - "node_name":"NODE_S_A0b7fb52d8d0a2ecde:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0b7fb52d8d0a2ecde:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.45318685937672853}}]}, - "COLL_E_v2_5":{ - "shard31":[{ - "core_node362":{ - "core":"COLL_E_v2_5_shard31_replica_n361", - "shard":"shard31", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A0b7fb52d8d0a2ecde:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0b7fb52d8d0a2ecde:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":135.04639122448862}}], - "shard34":[{ - "core_node384":{ - "core":"COLL_E_v2_5_shard34_replica_n383", - "shard":"shard34", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A0b7fb52d8d0a2ecde:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A0b7fb52d8d0a2ecde:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":138.16951165627688}}]}}} - ,{ - "node":"NODE_S_A0d391cf8eeaca59cd:8983_solr", - "isLive":true, - "cores":7.0, - "freedisk":725.0959548950195, - "totaldisk":999.51171875, - "replicas":{ - "recall_cache":{"shard1":[{ - "core_node3":{ - "core":"recall_cache_shard1_replica_n1", - "shard":"shard1", - "collection":"recall_cache", - "node_name":"NODE_S_A0d391cf8eeaca59cd:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A0d391cf8eeaca59cd:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.005028365179896355}}]}, - "COLL_S_R":{"shard9":[{ - "core_node19":{ - "core":"COLL_S_R_shard9_replica_n16", - "shard":"shard9", - "collection":"COLL_S_R", - "node_name":"NODE_S_A0d391cf8eeaca59cd:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0d391cf8eeaca59cd:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.003799574449658394}}]}, - "subreddit_experiment_results":{ - "shard2":[{ - "core_node5":{ - "core":"subreddit_experiment_results_shard2_replica_n2", - "shard":"shard2", - "collection":"subreddit_experiment_results", - "node_name":"NODE_S_A0d391cf8eeaca59cd:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0d391cf8eeaca59cd:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.004513397812843323}}], - "shard23":[{ - "core_node47":{ - "core":"subreddit_experiment_results_shard23_replica_n44", - "shard":"shard23", - "collection":"subreddit_experiment_results", - "node_name":"NODE_S_A0d391cf8eeaca59cd:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0d391cf8eeaca59cd:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.0045642731711268425}}]}, - "COLL_R_P_S_A_22":{"shard9":[{ - "core_node19":{ - "core":"COLL_R_P_S_A_22_shard9_replica_n16", - "shard":"shard9", - "collection":"COLL_R_P_S_A_22", - "node_name":"NODE_S_A0d391cf8eeaca59cd:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0d391cf8eeaca59cd:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.46696313098073006}}]}, - "COLL_E_v2_5":{ - "shard2":[{ - "core_node378":{ - "core":"COLL_E_v2_5_shard2_replica_n377", - "shard":"shard2", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A0d391cf8eeaca59cd:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A0d391cf8eeaca59cd:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":139.2840456981212}}], - "shard25":[{ - "core_node360":{ - "core":"COLL_E_v2_5_shard25_replica_n359", - "shard":"shard25", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A0d391cf8eeaca59cd:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A0d391cf8eeaca59cd:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":134.43318709544837}}]}}} - ,{ - "node":"NODE_S_A0aaaf34e6b281b24c:8983_solr", - "isLive":true, - "cores":6.0, - "freedisk":735.5332298278809, - "totaldisk":999.51171875, - "replicas":{ - "COLL_S_R":{"shard8":[{ - "core_node17":{ - "core":"COLL_S_R_shard8_replica_n14", - "shard":"shard8", - "collection":"COLL_S_R", - "node_name":"NODE_S_A0aaaf34e6b281b24c:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0aaaf34e6b281b24c:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.003879970870912075}}]}, - "subreddit_experiment_results":{ - "shard14":[{ - "core_node29":{ - "core":"subreddit_experiment_results_shard14_replica_n26", - "shard":"shard14", - "collection":"subreddit_experiment_results", - "node_name":"NODE_S_A0aaaf34e6b281b24c:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0aaaf34e6b281b24c:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.004518838599324226}}], - "shard11":[{ - "core_node23":{ - "core":"subreddit_experiment_results_shard11_replica_n20", - "shard":"shard11", - "collection":"subreddit_experiment_results", - "node_name":"NODE_S_A0aaaf34e6b281b24c:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0aaaf34e6b281b24c:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.0045999325811862946}}]}, - "COLL_R_P_S_A_22":{"shard8":[{ - "core_node17":{ - "core":"COLL_R_P_S_A_22_shard8_replica_n14", - "shard":"shard8", - "collection":"COLL_R_P_S_A_22", - "node_name":"NODE_S_A0aaaf34e6b281b24c:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0aaaf34e6b281b24c:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.5148522546514869}}]}, - "COLL_E_v2_5":{ - "shard16":[{ - "core_node380":{ - "core":"COLL_E_v2_5_shard16_replica_n379", - "shard":"shard16", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A0aaaf34e6b281b24c:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A0aaaf34e6b281b24c:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":130.66182304918766}}], - "shard32":[{ - "core_node364":{ - "core":"COLL_E_v2_5_shard32_replica_n363", - "shard":"shard32", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A0aaaf34e6b281b24c:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0aaaf34e6b281b24c:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":132.6516477856785}}]}}} - ,{ - "node":"NODE_S_A0bdba3561ff291bc4:8983_solr", - "isLive":true, - "cores":5.0, - "freedisk":673.0733299255371, - "totaldisk":999.51171875, - "replicas":{ - "COLL_S_R":{"shard7":[{ - "core_node15":{ - "core":"COLL_S_R_shard7_replica_n12", - "shard":"shard7", - "collection":"COLL_S_R", - "node_name":"NODE_S_A0bdba3561ff291bc4:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0bdba3561ff291bc4:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.003801817074418068}}]}, - "COLL_P_V_E_v2":{"shard6":[{ - "core_node24":{ - "core":"COLL_P_V_E_v2_shard6_replica_n22", - "shard":"shard6", - "collection":"COLL_P_V_E_v2", - "node_name":"NODE_S_A0bdba3561ff291bc4:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0bdba3561ff291bc4:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":8.199045146815479}}]}, - "COLL_R_P_S_A_22":{"shard7":[{ - "core_node15":{ - "core":"COLL_R_P_S_A_22_shard7_replica_n12", - "shard":"shard7", - "collection":"COLL_R_P_S_A_22", - "node_name":"NODE_S_A0bdba3561ff291bc4:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0bdba3561ff291bc4:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.4553129719570279}}]}, - "COLL_E_v2_5":{ - "shard3":[{ - "core_node404":{ - "core":"COLL_E_v2_5_shard3_replica_n403", - "shard":"shard3", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A0bdba3561ff291bc4:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A0bdba3561ff291bc4:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":133.96376375574619}}], - "shard28":[{ - "core_node376":{ - "core":"COLL_E_v2_5_shard28_replica_n375", - "shard":"shard28", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A0bdba3561ff291bc4:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A0bdba3561ff291bc4:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":134.145930592902}}]}}} - ,{ - "node":"NODE_S_A086a3a5ff0617c80f:8983_solr", - "isLive":true, - "cores":5.0, - "freedisk":723.7366943359375, - "totaldisk":999.51171875, - "replicas":{ - "COLL_S_R":{"shard6":[{ - "core_node13":{ - "core":"COLL_S_R_shard6_replica_n10", - "shard":"shard6", - "collection":"COLL_S_R", - "node_name":"NODE_S_A086a3a5ff0617c80f:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A086a3a5ff0617c80f:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.0038511399179697037}}]}, - "COLL_P_V_E_v2":{"shard1":[{ - "core_node5":{ - "core":"COLL_P_V_E_v2_shard1_replica_n2", - "shard":"shard1", - "collection":"COLL_P_V_E_v2", - "node_name":"NODE_S_A086a3a5ff0617c80f:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A086a3a5ff0617c80f:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":8.208250655792654}}]}, - "COLL_R_P_S_A_22":{"shard6":[{ - "core_node13":{ - "core":"COLL_R_P_S_A_22_shard6_replica_n10", - "shard":"shard6", - "collection":"COLL_R_P_S_A_22", - "node_name":"NODE_S_A086a3a5ff0617c80f:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A086a3a5ff0617c80f:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.48864932265132666}}]}, - "COLL_E_v2_5":{ - "shard36":[{ - "core_node400":{ - "core":"COLL_E_v2_5_shard36_replica_n399", - "shard":"shard36", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A086a3a5ff0617c80f:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A086a3a5ff0617c80f:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":132.73244500439614}}], - "shard10":[{ - "core_node370":{ - "core":"COLL_E_v2_5_shard10_replica_n369", - "shard":"shard10", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A086a3a5ff0617c80f:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A086a3a5ff0617c80f:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":134.23334093671292}}]}}} - ,{ - "node":"NODE_S_A09f108fb78272a03d:8983_solr", - "isLive":true, - "cores":5.0, - "freedisk":724.1397323608398, - "totaldisk":999.51171875, - "replicas":{ - "COLL_S_R":{"shard2":[{ - "core_node4":{ - "core":"COLL_S_R_shard2_replica_n2", - "shard":"shard2", - "collection":"COLL_S_R", - "node_name":"NODE_S_A09f108fb78272a03d:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A09f108fb78272a03d:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.0037746401503682137}}]}, - "COLL_P_V_E_v2":{"shard5":[{ - "core_node19":{ - "core":"COLL_P_V_E_v2_shard5_replica_n16", - "shard":"shard5", - "collection":"COLL_P_V_E_v2", - "node_name":"NODE_S_A09f108fb78272a03d:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A09f108fb78272a03d:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":9.87934261187911}}]}, - "COLL_R_P_S_A_22":{"shard2":[{ - "core_node5":{ - "core":"COLL_R_P_S_A_22_shard2_replica_n2", - "shard":"shard2", - "collection":"COLL_R_P_S_A_22", - "node_name":"NODE_S_A09f108fb78272a03d:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A09f108fb78272a03d:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.4274335531517863}}]}, - "COLL_E_v2_5":{ - "shard29":[{ - "core_node392":{ - "core":"COLL_E_v2_5_shard29_replica_n391", - "shard":"shard29", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A09f108fb78272a03d:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A09f108fb78272a03d:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":131.70282221771777}}], - "shard20":[{ - "core_node246":{ - "core":"COLL_E_v2_5_shard20_replica_n245", - "shard":"shard20", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A09f108fb78272a03d:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A09f108fb78272a03d:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":133.18864511698484}}]}}} - ,{ - "node":"NODE_S_A013eaa884d3c59fab:8983_solr", - "isLive":true, - "cores":4.0, - "freedisk":734.8396911621094, - "totaldisk":999.51171875, - "replicas":{ - "COLL_S_R":{"shard5":[{ - "core_node11":{ - "core":"COLL_S_R_shard5_replica_n8", - "shard":"shard5", - "collection":"COLL_S_R", - "node_name":"NODE_S_A013eaa884d3c59fab:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A013eaa884d3c59fab:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.0038577821105718613}}]}, - "COLL_R_P_S_A_22":{"shard5":[{ - "core_node11":{ - "core":"COLL_R_P_S_A_22_shard5_replica_n8", - "shard":"shard5", - "collection":"COLL_R_P_S_A_22", - "node_name":"NODE_S_A013eaa884d3c59fab:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A013eaa884d3c59fab:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.42266264744102955}}]}, - "COLL_E_v2_5":{ - "shard15":[{ - "core_node368":{ - "core":"COLL_E_v2_5_shard15_replica_n367", - "shard":"shard15", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A013eaa884d3c59fab:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A013eaa884d3c59fab:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":128.1388803133741}}], - "shard9":[{ - "core_node398":{ - "core":"COLL_E_v2_5_shard9_replica_n397", - "shard":"shard9", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A013eaa884d3c59fab:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A013eaa884d3c59fab:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":135.6189723983407}}]}}} - ,{ - "node":"NODE_S_A03baeedfd28ba05ba:8983_solr", - "isLive":true, - "cores":4.0, - "freedisk":734.3327178955078, - "totaldisk":999.51171875, - "replicas":{ - "COLL_S_R":{"shard4":[{ - "core_node9":{ - "core":"COLL_S_R_shard4_replica_n6", - "shard":"shard4", - "collection":"COLL_S_R", - "node_name":"NODE_S_A03baeedfd28ba05ba:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A03baeedfd28ba05ba:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.003800010308623314}}]}, - "COLL_R_P_S_A_22":{"shard4":[{ - "core_node9":{ - "core":"COLL_R_P_S_A_22_shard4_replica_n6", - "shard":"shard4", - "collection":"COLL_R_P_S_A_22", - "node_name":"NODE_S_A03baeedfd28ba05ba:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A03baeedfd28ba05ba:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.41693424154073}}]}, - "COLL_E_v2_5":{ - "shard13":[{ - "core_node390":{ - "core":"COLL_E_v2_5_shard13_replica_n389", - "shard":"shard13", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A03baeedfd28ba05ba:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A03baeedfd28ba05ba:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":135.04321804642677}}], - "shard26":[{ - "core_node366":{ - "core":"COLL_E_v2_5_shard26_replica_n365", - "shard":"shard26", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A03baeedfd28ba05ba:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A03baeedfd28ba05ba:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":129.58171366155148}}]}}} - ,{ - "node":"NODE_S_A082e313208af7da04:8983_solr", - "isLive":true, - "cores":4.0, - "freedisk":731.822093963623, - "totaldisk":999.51171875, - "replicas":{ - "COLL_S_R":{"shard1":[{ - "core_node3":{ - "core":"COLL_S_R_shard1_replica_n1", - "shard":"shard1", - "collection":"COLL_S_R", - "node_name":"NODE_S_A082e313208af7da04:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A082e313208af7da04:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.003850843757390976}}]}, - "COLL_R_P_S_A_22":{"shard1":[{ - "core_node3":{ - "core":"COLL_R_P_S_A_22_shard1_replica_n1", - "shard":"shard1", - "collection":"COLL_R_P_S_A_22", - "node_name":"NODE_S_A082e313208af7da04:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A082e313208af7da04:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.46176901180297136}}]}, - "COLL_E_v2_5":{ - "shard18":[{ - "core_node244":{ - "core":"COLL_E_v2_5_shard18_replica_n243", - "shard":"shard18", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A082e313208af7da04:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A082e313208af7da04:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":130.08673685323447}}], - "shard9":[{ - "core_node386":{ - "core":"COLL_E_v2_5_shard9_replica_n385", - "shard":"shard9", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A082e313208af7da04:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A082e313208af7da04:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":136.76205699518323}}]}}} - ,{ - "node":"NODE_S_A0a0ffe317a4f85c55:8983_solr", - "isLive":true, - "cores":4.0, - "freedisk":743.7088737487793, - "totaldisk":999.51171875, - "replicas":{ - "COLL_S_R":{"shard3":[{ - "core_node7":{ - "core":"COLL_S_R_shard3_replica_n5", - "shard":"shard3", - "collection":"COLL_S_R", - "node_name":"NODE_S_A0a0ffe317a4f85c55:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0a0ffe317a4f85c55:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.003859194926917553}}]}, - "COLL_R_P_S_A_22":{"shard3":[{ - "core_node7":{ - "core":"COLL_R_P_S_A_22_shard3_replica_n4", - "shard":"shard3", - "collection":"COLL_R_P_S_A_22", - "node_name":"NODE_S_A0a0ffe317a4f85c55:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0a0ffe317a4f85c55:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.41597359627485275}}]}, - "COLL_E_v2_5":{ - "shard26":[{ - "core_node248":{ - "core":"COLL_E_v2_5_shard26_replica_n247", - "shard":"shard26", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A0a0ffe317a4f85c55:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0a0ffe317a4f85c55:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":127.8478315602988}}], - "shard1":[{ - "core_node394":{ - "core":"COLL_E_v2_5_shard1_replica_n393", - "shard":"shard1", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A0a0ffe317a4f85c55:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A0a0ffe317a4f85c55:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":127.3860678607598}}]}}}], - "liveNodes":["NODE_S_A0bdba3561ff291bc4:8983_solr", - "NODE_S_A0a0ffe317a4f85c55:8983_solr", - "NODE_S_A0ebcce85ef9c13308:8983_solr", - "NODE_S_A0a3cb04c83b693d80:8983_solr", - "NODE_S_A06032b38152c0f019:8983_solr", - "NODE_S_A0ccef54efe3178758:8983_solr", - "NODE_S_A024a9c138d972d437:8983_solr", - "NODE_S_A086a3a5ff0617c80f:8983_solr", - "NODE_S_A0dc21b7f9ef048b48:8983_solr", - "NODE_S_A006b71d81aedd8577:8983_solr", - "NODE_S_A013eaa884d3c59fab:8983_solr", - "NODE_S_A0aaaf34e6b281b24c:8983_solr", - "NODE_S_A09f108fb78272a03d:8983_solr", - "NODE_S_A084aef0a141a43572:8983_solr", - "NODE_S_A0ee1c34fb3e9c261d:8983_solr", - "NODE_S_A03baeedfd28ba05ba:8983_solr", - "NODE_S_A014292d062d5088da:8983_solr", - "NODE_S_A0d1445647aaae8dd1:8983_solr", - "NODE_S_A0b7fb52d8d0a2ecde:8983_solr", - "NODE_S_A02f6288aa3fc9eb3d:8983_solr", - "NODE_S_A09aede91f0e8c147c:8983_solr", - "NODE_S_A0f66f4bec1ce8fc09:8983_solr", - "NODE_S_A0afcb9535d2619dac:8983_solr", - "NODE_S_A0e48ab035b869305e:8983_solr", - "NODE_S_A0d1f4439b8e8878e8:8983_solr", - "NODE_S_A0cf36cc11a8447fac:8983_solr", - "NODE_S_A04a75744569a27ccf:8983_solr", - "NODE_S_A077430d186f0e1de2:8983_solr", - "NODE_S_A08c1ec5957d8a5d44:8983_solr", - "NODE_S_A061d3ac3c71d42de2:8983_solr", - "NODE_S_A082e313208af7da04:8983_solr", - "NODE_S_A0d391cf8eeaca59cd:8983_solr", - "NODE_S_A0f096665ad4a3bc58:8983_solr"], - "violations":[{ - "node":"NODE_S_A0a3cb04c83b693d80:8983_solr", - "tagKey":"NODE_S_A0a3cb04c83b693d80:8983_solr", - "violation":{ - "node":"20.0", - "delta":9.0}, - "clause":{ - "cores":"#EQUAL", - "node":"#ANY"}, - "violatingReplicas":[{ - "core_node61":{ - "core":"COLL_S_R_shard30_replica_n58", - "shard":"shard30", - "collection":"COLL_S_R", - "node_name":"NODE_S_A0a3cb04c83b693d80:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0a3cb04c83b693d80:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.0037675881758332253}} - ,{ - "core_node18":{ - "core":"COLL_S_M_1_shard9_replica_n16", - "shard":"shard9", - "collection":"COLL_S_M_1", - "node_name":"NODE_S_A0a3cb04c83b693d80:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0a3cb04c83b693d80:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":6.426125764846802E-8}} - ,{ - "core_node8":{ - "core":"COLL_E_V2_4_shard2_replica_n6", - "shard":"shard2", - "collection":"COLL_E_V2_4", - "node_name":"NODE_S_A0a3cb04c83b693d80:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0a3cb04c83b693d80:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.23103012703359127}} - ,{ - "core_node53":{ - "core":"COLL_S_E_15_shard13_replica_n50", - "shard":"shard13", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A0a3cb04c83b693d80:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0a3cb04c83b693d80:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}} - ,{ - "core_node17":{ - "core":"COLL_S_E_15_shard4_replica_n14", - "shard":"shard4", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A0a3cb04c83b693d80:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0a3cb04c83b693d80:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}} - ,{ - "core_node71":{ - "core":"COLL_S_E_15_shard18_replica_n68", - "shard":"shard18", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A0a3cb04c83b693d80:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0a3cb04c83b693d80:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}} - ,{ - "core_node35":{ - "core":"COLL_S_E_15_shard9_replica_n32", - "shard":"shard9", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A0a3cb04c83b693d80:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0a3cb04c83b693d80:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}} - ,{ - "core_node11":{ - "core":"COLL_P_V_E_v2_shard3_replica_n8", - "shard":"shard3", - "collection":"COLL_P_V_E_v2", - "node_name":"NODE_S_A0a3cb04c83b693d80:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A0a3cb04c83b693d80:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":8.222954365424812}} - ,{ - "core_node5":{ - "core":"COLL_S_H_7_shard2_replica_n2", - "shard":"shard2", - "collection":"COLL_S_H_7", - "node_name":"NODE_S_A0a3cb04c83b693d80:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0a3cb04c83b693d80:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.005126317963004112}} - ,{ - "core_node85":{ - "core":"COLL_E_v2_5_shard21_replica_n82", - "shard":"shard21", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A0a3cb04c83b693d80:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0a3cb04c83b693d80:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":135.4880904313177}} - ,{ - "core_node236":{ - "core":"COLL_E_v2_5_shard12_replica_n235", - "shard":"shard12", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A0a3cb04c83b693d80:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0a3cb04c83b693d80:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":133.1140805343166}} - ,{ - "core_node11":{ - "core":"S_B_2_shard5_replica_n8", - "shard":"shard5", - "collection":"S_B_2", - "node_name":"NODE_S_A0a3cb04c83b693d80:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0a3cb04c83b693d80:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.03135890234261751}} - ,{ - "core_node3":{ - "core":"COLL_S_R_aggr_shard1_replica_n1", - "shard":"shard1", - "collection":"COLL_S_R_aggr", - "node_name":"NODE_S_A0a3cb04c83b693d80:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0a3cb04c83b693d80:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.7853863816708326}} - ,{ - "core_node8":{ - "core":"COLL_E_V2_S_A_3_shard2_replica_n6", - "shard":"shard2", - "collection":"COLL_E_V2_S_A_3", - "node_name":"NODE_S_A0a3cb04c83b693d80:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0a3cb04c83b693d80:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":24.86383740324527}} - ,{ - "core_node8":{ - "core":"COLL_S_J_H_9_shard2_replica_n6", - "shard":"shard2", - "collection":"COLL_S_J_H_9", - "node_name":"NODE_S_A0a3cb04c83b693d80:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0a3cb04c83b693d80:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.005902050994336605}} - ,{ - "core_node5":{ - "core":"logs_shard1_replica_n2", - "shard":"shard1", - "collection":"logs", - "node_name":"NODE_S_A0a3cb04c83b693d80:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0a3cb04c83b693d80:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.012885616160929203}} - ,{ - "core_node13":{ - "core":"COLL_N_Q_C_T_11_shard6_replica_n7", - "shard":"shard6", - "collection":"COLL_N_Q_C_T_11", - "node_name":"NODE_S_A0a3cb04c83b693d80:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0a3cb04c83b693d80:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.0060289520770311356}} - ,{ - "core_node5":{ - "core":"COLL_PV_E_v2_5_shard1_replica_n2", - "shard":"shard1", - "collection":"COLL_PV_E_v2_5", - "node_name":"NODE_S_A0a3cb04c83b693d80:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A0a3cb04c83b693d80:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":3.0124771054834127}} - ,{ - "core_node3":{ - "core":"COLL_S_E_15_signals_shard1_replica_n1", - "shard":"shard1", - "collection":"COLL_S_E_15_signals", - "node_name":"NODE_S_A0a3cb04c83b693d80:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0a3cb04c83b693d80:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":5.913432687520981E-5}} - ,{ - "core_node61":{ - "core":"COLL_R_P_S_A_22_shard30_replica_n58", - "shard":"shard30", - "collection":"COLL_R_P_S_A_22", - "node_name":"NODE_S_A0a3cb04c83b693d80:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0a3cb04c83b693d80:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.42820744402706623}}]}, - { - "node":"NODE_S_A0d1445647aaae8dd1:8983_solr", - "tagKey":"NODE_S_A0d1445647aaae8dd1:8983_solr", - "violation":{ - "node":"19.0", - "delta":8.0}, - "clause":{ - "cores":"#EQUAL", - "node":"#ANY"}, - "violatingReplicas":[{ - "core_node65":{ - "core":"COLL_S_R_shard32_replica_n62", - "shard":"shard32", - "collection":"COLL_S_R", - "node_name":"NODE_S_A0d1445647aaae8dd1:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0d1445647aaae8dd1:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.003853907808661461}} - ,{ - "core_node9":{ - "core":"COLL_S_M_1_shard4_replica_n6", - "shard":"shard4", - "collection":"COLL_S_M_1", - "node_name":"NODE_S_A0d1445647aaae8dd1:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0d1445647aaae8dd1:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":6.426125764846802E-8}} - ,{ - "core_node7":{ - "core":"COLL_E_V2_4_shard2_replica_n4", - "shard":"shard2", - "collection":"COLL_E_V2_4", - "node_name":"NODE_S_A0d1445647aaae8dd1:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A0d1445647aaae8dd1:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.23189987149089575}} - ,{ - "core_node57":{ - "core":"COLL_S_E_15_shard14_replica_n54", - "shard":"shard14", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A0d1445647aaae8dd1:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A0d1445647aaae8dd1:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}} - ,{ - "core_node21":{ - "core":"COLL_S_E_15_shard5_replica_n18", - "shard":"shard5", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A0d1445647aaae8dd1:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A0d1445647aaae8dd1:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}} - ,{ - "core_node3":{ - "core":"COLL_S_E_15_shard1_replica_n1", - "shard":"shard1", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A0d1445647aaae8dd1:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0d1445647aaae8dd1:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}} - ,{ - "core_node39":{ - "core":"COLL_S_E_15_shard10_replica_n36", - "shard":"shard10", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A0d1445647aaae8dd1:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0d1445647aaae8dd1:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}} - ,{ - "core_node3":{ - "core":"COLL_P_V_E_v2_shard1_replica_n1", - "shard":"shard1", - "collection":"COLL_P_V_E_v2", - "node_name":"NODE_S_A0d1445647aaae8dd1:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A0d1445647aaae8dd1:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":8.172482682392001}} - ,{ - "core_node3":{ - "core":"audit_logs_shard1_replica_n1", - "shard":"shard1", - "collection":"audit_logs", - "node_name":"NODE_S_A0d1445647aaae8dd1:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A0d1445647aaae8dd1:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.30385160446167E-7}} - ,{ - "core_node15":{ - "core":"COLL_S_H_7_shard7_replica_n12", - "shard":"shard7", - "collection":"COLL_S_H_7", - "node_name":"NODE_S_A0d1445647aaae8dd1:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0d1445647aaae8dd1:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.005527845583856106}} - ,{ - "core_node109":{ - "core":"COLL_E_v2_5_shard27_replica_n106", - "shard":"shard27", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A0d1445647aaae8dd1:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0d1445647aaae8dd1:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":137.78214563801885}} - ,{ - "core_node77":{ - "core":"COLL_E_v2_5_shard19_replica_n74", - "shard":"shard19", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A0d1445647aaae8dd1:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0d1445647aaae8dd1:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":130.64438762888312}} - ,{ - "core_node45":{ - "core":"COLL_E_v2_5_shard11_replica_n42", - "shard":"shard11", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A0d1445647aaae8dd1:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A0d1445647aaae8dd1:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":126.09197529405355}} - ,{ - "core_node8":{ - "core":"system_metrics_shard2_replica_n6", - "shard":"shard2", - "collection":"system_metrics", - "node_name":"NODE_S_A0d1445647aaae8dd1:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A0d1445647aaae8dd1:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}} - ,{ - "core_node18":{ - "core":"S_B_2_shard9_replica_n16", - "shard":"shard9", - "collection":"S_B_2", - "node_name":"NODE_S_A0d1445647aaae8dd1:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0d1445647aaae8dd1:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.015032877214252949}} - ,{ - "core_node7":{ - "core":"COLL_S_E_15_signals_aggr_shard2_replica_n4", - "shard":"shard2", - "collection":"COLL_S_E_15_signals_aggr", - "node_name":"NODE_S_A0d1445647aaae8dd1:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A0d1445647aaae8dd1:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":6.426125764846802E-8}} - ,{ - "core_node7":{ - "core":"COLL_S_J_H_9_shard2_replica_n4", - "shard":"shard2", - "collection":"COLL_S_J_H_9", - "node_name":"NODE_S_A0d1445647aaae8dd1:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A0d1445647aaae8dd1:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.0059021394699811935}} - ,{ - "core_node5":{ - "core":"COLL_S_E_15_signals_shard1_replica_n2", - "shard":"shard1", - "collection":"COLL_S_E_15_signals", - "node_name":"NODE_S_A0d1445647aaae8dd1:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A0d1445647aaae8dd1:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":5.915109068155289E-5}} - ,{ - "core_node65":{ - "core":"COLL_R_P_S_A_22_shard32_replica_n62", - "shard":"shard32", - "collection":"COLL_R_P_S_A_22", - "node_name":"NODE_S_A0d1445647aaae8dd1:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0d1445647aaae8dd1:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.5012764362618327}}]}, - { - "node":"NODE_S_A0ee1c34fb3e9c261d:8983_solr", - "tagKey":"NODE_S_A0ee1c34fb3e9c261d:8983_solr", - "violation":{ - "node":"18.0", - "delta":7.0}, - "clause":{ - "cores":"#EQUAL", - "node":"#ANY"}, - "violatingReplicas":[{ - "core_node53":{ - "core":"COLL_S_R_shard26_replica_n50", - "shard":"shard26", - "collection":"COLL_S_R", - "node_name":"NODE_S_A0ee1c34fb3e9c261d:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0ee1c34fb3e9c261d:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.0038864407688379288}} - ,{ - "core_node5":{ - "core":"COLL_S_M_1_shard2_replica_n2", - "shard":"shard2", - "collection":"COLL_S_M_1", - "node_name":"NODE_S_A0ee1c34fb3e9c261d:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0ee1c34fb3e9c261d:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":6.426125764846802E-8}} - ,{ - "core_node3":{ - "core":"COLL_E_V2_4_shard1_replica_n1", - "shard":"shard1", - "collection":"COLL_E_V2_4", - "node_name":"NODE_S_A0ee1c34fb3e9c261d:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A0ee1c34fb3e9c261d:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.23009658977389336}} - ,{ - "core_node7":{ - "core":"COLL_S_E_15_shard2_replica_n4", - "shard":"shard2", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A0ee1c34fb3e9c261d:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A0ee1c34fb3e9c261d:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}} - ,{ - "core_node61":{ - "core":"COLL_S_E_15_shard15_replica_n58", - "shard":"shard15", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A0ee1c34fb3e9c261d:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0ee1c34fb3e9c261d:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}} - ,{ - "core_node25":{ - "core":"COLL_S_E_15_shard6_replica_n22", - "shard":"shard6", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A0ee1c34fb3e9c261d:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0ee1c34fb3e9c261d:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}} - ,{ - "core_node43":{ - "core":"COLL_S_E_15_shard11_replica_n40", - "shard":"shard11", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A0ee1c34fb3e9c261d:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A0ee1c34fb3e9c261d:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}} - ,{ - "core_node9":{ - "core":"COLL_S_H_7_shard4_replica_n6", - "shard":"shard4", - "collection":"COLL_S_H_7", - "node_name":"NODE_S_A0ee1c34fb3e9c261d:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0ee1c34fb3e9c261d:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.005278228782117367}} - ,{ - "core_node125":{ - "core":"COLL_E_v2_5_shard31_replica_n122", - "shard":"shard31", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A0ee1c34fb3e9c261d:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A0ee1c34fb3e9c261d:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":136.22203192673624}} - ,{ - "core_node127":{ - "core":"COLL_E_v2_5_shard32_replica_n124", - "shard":"shard32", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A0ee1c34fb3e9c261d:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A0ee1c34fb3e9c261d:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":135.84658553358167}} - ,{ - "core_node7":{ - "core":"system_metrics_shard2_replica_n4", - "shard":"shard2", - "collection":"system_metrics", - "node_name":"NODE_S_A0ee1c34fb3e9c261d:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0ee1c34fb3e9c261d:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}} - ,{ - "core_node15":{ - "core":"S_B_2_shard7_replica_n12", - "shard":"shard7", - "collection":"S_B_2", - "node_name":"NODE_S_A0ee1c34fb3e9c261d:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0ee1c34fb3e9c261d:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.051517823711037636}} - ,{ - "core_node5":{ - "core":"COLL_S_R_aggr_shard1_replica_n2", - "shard":"shard1", - "collection":"COLL_S_R_aggr", - "node_name":"NODE_S_A0ee1c34fb3e9c261d:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A0ee1c34fb3e9c261d:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.7883908534422517}} - ,{ - "core_node5":{ - "core":"COLL_S_J_H_9_shard1_replica_n2", - "shard":"shard1", - "collection":"COLL_S_J_H_9", - "node_name":"NODE_S_A0ee1c34fb3e9c261d:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A0ee1c34fb3e9c261d:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.005922678858041763}} - ,{ - "core_node3":{ - "core":"logs_shard1_replica_n1", - "shard":"shard1", - "collection":"logs", - "node_name":"NODE_S_A0ee1c34fb3e9c261d:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A0ee1c34fb3e9c261d:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.012863828800618649}} - ,{ - "core_node9":{ - "core":"COLL_N_Q_C_T_11_shard3_replica_n4", - "shard":"shard3", - "collection":"COLL_N_Q_C_T_11", - "node_name":"NODE_S_A0ee1c34fb3e9c261d:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0ee1c34fb3e9c261d:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.006221611984074116}} - ,{ - "core_node11":{ - "core":"COLL_PV_E_v2_5_shard3_replica_n8", - "shard":"shard3", - "collection":"COLL_PV_E_v2_5", - "node_name":"NODE_S_A0ee1c34fb3e9c261d:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A0ee1c34fb3e9c261d:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":3.029745533131063}} - ,{ - "core_node53":{ - "core":"COLL_R_P_S_A_22_shard26_replica_n50", - "shard":"shard26", - "collection":"COLL_R_P_S_A_22", - "node_name":"NODE_S_A0ee1c34fb3e9c261d:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0ee1c34fb3e9c261d:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.4816165193915367}}]}, - { - "node":"NODE_S_A0dc21b7f9ef048b48:8983_solr", - "tagKey":"NODE_S_A0dc21b7f9ef048b48:8983_solr", - "violation":{ - "node":"18.0", - "delta":7.0}, - "clause":{ - "cores":"#EQUAL", - "node":"#ANY"}, - "violatingReplicas":[{ - "core_node66":{ - "core":"COLL_S_R_shard33_replica_n64", - "shard":"shard33", - "collection":"COLL_S_R", - "node_name":"NODE_S_A0dc21b7f9ef048b48:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0dc21b7f9ef048b48:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.003936124965548515}} - ,{ - "core_node11":{ - "core":"COLL_S_M_1_shard5_replica_n8", - "shard":"shard5", - "collection":"COLL_S_M_1", - "node_name":"NODE_S_A0dc21b7f9ef048b48:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0dc21b7f9ef048b48:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":6.426125764846802E-8}} - ,{ - "core_node5":{ - "core":"COLL_E_V2_4_shard1_replica_n2", - "shard":"shard1", - "collection":"COLL_E_V2_4", - "node_name":"NODE_S_A0dc21b7f9ef048b48:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0dc21b7f9ef048b48:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.23144119326025248}} - ,{ - "core_node13":{ - "core":"COLL_S_E_15_shard3_replica_n10", - "shard":"shard3", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A0dc21b7f9ef048b48:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0dc21b7f9ef048b48:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}} - ,{ - "core_node67":{ - "core":"COLL_S_E_15_shard17_replica_n64", - "shard":"shard17", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A0dc21b7f9ef048b48:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0dc21b7f9ef048b48:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}} - ,{ - "core_node31":{ - "core":"COLL_S_E_15_shard8_replica_n28", - "shard":"shard8", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A0dc21b7f9ef048b48:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0dc21b7f9ef048b48:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}} - ,{ - "core_node49":{ - "core":"COLL_S_E_15_shard12_replica_n46", - "shard":"shard12", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A0dc21b7f9ef048b48:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0dc21b7f9ef048b48:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}} - ,{ - "core_node13":{ - "core":"COLL_S_H_7_shard6_replica_n10", - "shard":"shard6", - "collection":"COLL_S_H_7", - "node_name":"NODE_S_A0dc21b7f9ef048b48:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0dc21b7f9ef048b48:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.005341590382158756}} - ,{ - "core_node228":{ - "core":"COLL_E_v2_5_shard7_replica_n227", - "shard":"shard7", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A0dc21b7f9ef048b48:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0dc21b7f9ef048b48:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":125.11620672326535}} - ,{ - "core_node232":{ - "core":"COLL_E_v2_5_shard23_replica_n231", - "shard":"shard23", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A0dc21b7f9ef048b48:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0dc21b7f9ef048b48:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":137.6997431376949}} - ,{ - "core_node3":{ - "core":"system_metrics_shard1_replica_n1", - "shard":"shard1", - "collection":"system_metrics", - "node_name":"NODE_S_A0dc21b7f9ef048b48:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0dc21b7f9ef048b48:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}} - ,{ - "core_node7":{ - "core":"S_B_2_shard3_replica_n4", - "shard":"shard3", - "collection":"S_B_2", - "node_name":"NODE_S_A0dc21b7f9ef048b48:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0dc21b7f9ef048b48:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.0767898540943861}} - ,{ - "core_node8":{ - "core":"COLL_S_E_15_signals_aggr_shard2_replica_n6", - "shard":"shard2", - "collection":"COLL_S_E_15_signals_aggr", - "node_name":"NODE_S_A0dc21b7f9ef048b48:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0dc21b7f9ef048b48:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":6.426125764846802E-8}} - ,{ - "core_node5":{ - "core":"COLL_E_V2_S_A_3_shard1_replica_n2", - "shard":"shard1", - "collection":"COLL_E_V2_S_A_3", - "node_name":"NODE_S_A0dc21b7f9ef048b48:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A0dc21b7f9ef048b48:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":22.407991461455822}} - ,{ - "core_node3":{ - "core":"COLL_S_J_H_9_shard1_replica_n1", - "shard":"shard1", - "collection":"COLL_S_J_H_9", - "node_name":"NODE_S_A0dc21b7f9ef048b48:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0dc21b7f9ef048b48:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.0059427423402667046}} - ,{ - "core_node10":{ - "core":"COLL_N_Q_C_T_11_shard4_replica_n5", - "shard":"shard4", - "collection":"COLL_N_Q_C_T_11", - "node_name":"NODE_S_A0dc21b7f9ef048b48:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0dc21b7f9ef048b48:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.0060385772958397865}} - ,{ - "core_node8":{ - "core":"COLL_S_E_15_signals_shard2_replica_n6", - "shard":"shard2", - "collection":"COLL_S_E_15_signals", - "node_name":"NODE_S_A0dc21b7f9ef048b48:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0dc21b7f9ef048b48:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":2.8850510716438293E-5}} - ,{ - "core_node66":{ - "core":"COLL_R_P_S_A_22_shard33_replica_n64", - "shard":"shard33", - "collection":"COLL_R_P_S_A_22", - "node_name":"NODE_S_A0dc21b7f9ef048b48:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0dc21b7f9ef048b48:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.45377735421061516}}]}, - { - "node":"NODE_S_A0ccef54efe3178758:8983_solr", - "tagKey":"NODE_S_A0ccef54efe3178758:8983_solr", - "violation":{ - "node":"18.0", - "delta":7.0}, - "clause":{ - "cores":"#EQUAL", - "node":"#ANY"}, - "violatingReplicas":[{ - "core_node57":{ - "core":"COLL_S_R_shard28_replica_n54", - "shard":"shard28", - "collection":"COLL_S_R", - "node_name":"NODE_S_A0ccef54efe3178758:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0ccef54efe3178758:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.0038556959480047226}} - ,{ - "core_node15":{ - "core":"COLL_S_M_1_shard7_replica_n12", - "shard":"shard7", - "collection":"COLL_S_M_1", - "node_name":"NODE_S_A0ccef54efe3178758:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0ccef54efe3178758:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":6.426125764846802E-8}} - ,{ - "core_node9":{ - "core":"COLL_S_E_15_shard2_replica_n6", - "shard":"shard2", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A0ccef54efe3178758:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0ccef54efe3178758:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}} - ,{ - "core_node63":{ - "core":"COLL_S_E_15_shard16_replica_n60", - "shard":"shard16", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A0ccef54efe3178758:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0ccef54efe3178758:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}} - ,{ - "core_node27":{ - "core":"COLL_S_E_15_shard7_replica_n24", - "shard":"shard7", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A0ccef54efe3178758:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0ccef54efe3178758:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}} - ,{ - "core_node45":{ - "core":"COLL_S_E_15_shard11_replica_n42", - "shard":"shard11", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A0ccef54efe3178758:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0ccef54efe3178758:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}} - ,{ - "core_node21":{ - "core":"COLL_P_V_E_v2_shard5_replica_n18", - "shard":"shard5", - "collection":"COLL_P_V_E_v2", - "node_name":"NODE_S_A0ccef54efe3178758:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A0ccef54efe3178758:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":8.201123727485538}} - ,{ - "core_node5":{ - "core":"audit_logs_shard1_replica_n2", - "shard":"shard1", - "collection":"audit_logs", - "node_name":"NODE_S_A0ccef54efe3178758:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0ccef54efe3178758:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.30385160446167E-7}} - ,{ - "core_node17":{ - "core":"COLL_S_H_7_shard8_replica_n14", - "shard":"shard8", - "collection":"COLL_S_H_7", - "node_name":"NODE_S_A0ccef54efe3178758:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0ccef54efe3178758:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.005281115882098675}} - ,{ - "core_node113":{ - "core":"COLL_E_v2_5_shard28_replica_n110", - "shard":"shard28", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A0ccef54efe3178758:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0ccef54efe3178758:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":132.98001556191593}} - ,{ - "core_node81":{ - "core":"COLL_E_v2_5_shard20_replica_n78", - "shard":"shard20", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A0ccef54efe3178758:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0ccef54efe3178758:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":131.2389001082629}} - ,{ - "core_node3":{ - "core":"S_B_2_shard1_replica_n1", - "shard":"shard1", - "collection":"S_B_2", - "node_name":"NODE_S_A0ccef54efe3178758:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0ccef54efe3178758:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.03542718291282654}} - ,{ - "core_node7":{ - "core":"COLL_S_R_aggr_shard2_replica_n4", - "shard":"shard2", - "collection":"COLL_S_R_aggr", - "node_name":"NODE_S_A0ccef54efe3178758:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0ccef54efe3178758:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.7941454965621233}} - ,{ - "core_node3":{ - "core":"COLL_S_E_15_signals_aggr_shard1_replica_n1", - "shard":"shard1", - "collection":"COLL_S_E_15_signals_aggr", - "node_name":"NODE_S_A0ccef54efe3178758:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0ccef54efe3178758:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":6.426125764846802E-8}} - ,{ - "core_node8":{ - "core":"logs_shard2_replica_n6", - "shard":"shard2", - "collection":"logs", - "node_name":"NODE_S_A0ccef54efe3178758:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0ccef54efe3178758:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.012702811509370804}} - ,{ - "core_node12":{ - "core":"COLL_N_Q_C_T_11_shard5_replica_n6", - "shard":"shard5", - "collection":"COLL_N_Q_C_T_11", - "node_name":"NODE_S_A0ccef54efe3178758:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0ccef54efe3178758:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.0061960527673363686}} - ,{ - "core_node7":{ - "core":"COLL_PV_E_v2_5_shard2_replica_n4", - "shard":"shard2", - "collection":"COLL_PV_E_v2_5", - "node_name":"NODE_S_A0ccef54efe3178758:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A0ccef54efe3178758:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":3.0324512012302876}} - ,{ - "core_node57":{ - "core":"COLL_R_P_S_A_22_shard28_replica_n54", - "shard":"shard28", - "collection":"COLL_R_P_S_A_22", - "node_name":"NODE_S_A0ccef54efe3178758:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0ccef54efe3178758:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.45370675902813673}}]}, - { - "node":"NODE_S_A077430d186f0e1de2:8983_solr", - "tagKey":"NODE_S_A077430d186f0e1de2:8983_solr", - "violation":{ - "node":"17.0", - "delta":6.0}, - "clause":{ - "cores":"#EQUAL", - "node":"#ANY"}, - "violatingReplicas":[{ - "core_node5":{ - "core":"system_metrics_shard1_replica_n2", - "shard":"shard1", - "collection":"system_metrics", - "node_name":"NODE_S_A077430d186f0e1de2:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A077430d186f0e1de2:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}} - ,{ - "core_node17":{ - "core":"S_B_2_shard8_replica_n14", - "shard":"shard8", - "collection":"S_B_2", - "node_name":"NODE_S_A077430d186f0e1de2:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A077430d186f0e1de2:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.03129926137626171}} - ,{ - "core_node63":{ - "core":"COLL_S_R_shard31_replica_n60", - "shard":"shard31", - "collection":"COLL_S_R", - "node_name":"NODE_S_A077430d186f0e1de2:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A077430d186f0e1de2:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.0038539329543709755}} - ,{ - "core_node13":{ - "core":"COLL_S_M_1_shard6_replica_n10", - "shard":"shard6", - "collection":"COLL_S_M_1", - "node_name":"NODE_S_A077430d186f0e1de2:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A077430d186f0e1de2:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":6.426125764846802E-8}} - ,{ - "core_node51":{ - "core":"COLL_S_E_15_shard13_replica_n48", - "shard":"shard13", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A077430d186f0e1de2:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A077430d186f0e1de2:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}} - ,{ - "core_node15":{ - "core":"COLL_S_E_15_shard4_replica_n12", - "shard":"shard4", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A077430d186f0e1de2:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A077430d186f0e1de2:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}} - ,{ - "core_node69":{ - "core":"COLL_S_E_15_shard17_replica_n66", - "shard":"shard17", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A077430d186f0e1de2:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A077430d186f0e1de2:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}} - ,{ - "core_node33":{ - "core":"COLL_S_E_15_shard8_replica_n30", - "shard":"shard8", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A077430d186f0e1de2:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A077430d186f0e1de2:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}} - ,{ - "core_node7":{ - "core":"COLL_E_V2_S_A_3_shard2_replica_n4", - "shard":"shard2", - "collection":"COLL_E_V2_S_A_3", - "node_name":"NODE_S_A077430d186f0e1de2:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A077430d186f0e1de2:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":24.63952654134482}} - ,{ - "core_node53":{ - "core":"subreddit_experiment_results_shard26_replica_n50", - "shard":"shard26", - "collection":"subreddit_experiment_results", - "node_name":"NODE_S_A077430d186f0e1de2:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A077430d186f0e1de2:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.0044707562774419785}} - ,{ - "core_node7":{ - "core":"audit_logs_shard2_replica_n4", - "shard":"shard2", - "collection":"audit_logs", - "node_name":"NODE_S_A077430d186f0e1de2:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A077430d186f0e1de2:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.30385160446167E-7}} - ,{ - "core_node7":{ - "core":"logs_shard2_replica_n4", - "shard":"shard2", - "collection":"logs", - "node_name":"NODE_S_A077430d186f0e1de2:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A077430d186f0e1de2:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.012706683948636055}} - ,{ - "core_node17":{ - "core":"COLL_N_Q_C_T_11_shard8_replica_n14", - "shard":"shard8", - "collection":"COLL_N_Q_C_T_11", - "node_name":"NODE_S_A077430d186f0e1de2:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A077430d186f0e1de2:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.006143262609839439}} - ,{ - "core_node11":{ - "core":"COLL_S_H_7_shard5_replica_n8", - "shard":"shard5", - "collection":"COLL_S_H_7", - "node_name":"NODE_S_A077430d186f0e1de2:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A077430d186f0e1de2:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.005316532216966152}} - ,{ - "core_node63":{ - "core":"COLL_R_P_S_A_22_shard31_replica_n60", - "shard":"shard31", - "collection":"COLL_R_P_S_A_22", - "node_name":"NODE_S_A077430d186f0e1de2:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A077430d186f0e1de2:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.49999847542494535}} - ,{ - "core_node19":{ - "core":"COLL_E_v2_5_shard5_replica_n16", - "shard":"shard5", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A077430d186f0e1de2:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A077430d186f0e1de2:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":127.56859436631203}} - ,{ - "core_node83":{ - "core":"COLL_E_v2_5_shard21_replica_n80", - "shard":"shard21", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A077430d186f0e1de2:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A077430d186f0e1de2:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":132.22010754607618}}]}, - { - "node":"NODE_S_A0afcb9535d2619dac:8983_solr", - "tagKey":"NODE_S_A0afcb9535d2619dac:8983_solr", - "violation":{ - "node":"16.0", - "delta":5.0}, - "clause":{ - "cores":"#EQUAL", - "node":"#ANY"}, - "violatingReplicas":[{ - "core_node9":{ - "core":"S_B_2_shard4_replica_n6", - "shard":"shard4", - "collection":"S_B_2", - "node_name":"NODE_S_A0afcb9535d2619dac:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0afcb9535d2619dac:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.02314597088843584}} - ,{ - "core_node59":{ - "core":"COLL_S_R_shard29_replica_n56", - "shard":"shard29", - "collection":"COLL_S_R", - "node_name":"NODE_S_A0afcb9535d2619dac:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0afcb9535d2619dac:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.0038082310929894447}} - ,{ - "core_node17":{ - "core":"COLL_S_M_1_shard8_replica_n14", - "shard":"shard8", - "collection":"COLL_S_M_1", - "node_name":"NODE_S_A0afcb9535d2619dac:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0afcb9535d2619dac:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":6.426125764846802E-8}} - ,{ - "core_node59":{ - "core":"COLL_S_E_15_shard15_replica_n56", - "shard":"shard15", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A0afcb9535d2619dac:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A0afcb9535d2619dac:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}} - ,{ - "core_node5":{ - "core":"COLL_S_E_15_shard1_replica_n2", - "shard":"shard1", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A0afcb9535d2619dac:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A0afcb9535d2619dac:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}} - ,{ - "core_node23":{ - "core":"COLL_S_E_15_shard6_replica_n20", - "shard":"shard6", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A0afcb9535d2619dac:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A0afcb9535d2619dac:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}} - ,{ - "core_node41":{ - "core":"COLL_S_E_15_shard10_replica_n38", - "shard":"shard10", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A0afcb9535d2619dac:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A0afcb9535d2619dac:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}} - ,{ - "core_node37":{ - "core":"subreddit_experiment_results_shard18_replica_n34", - "shard":"shard18", - "collection":"subreddit_experiment_results", - "node_name":"NODE_S_A0afcb9535d2619dac:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0afcb9535d2619dac:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.0045873550698161125}} - ,{ - "core_node15":{ - "core":"subreddit_experiment_results_shard7_replica_n12", - "shard":"shard7", - "collection":"subreddit_experiment_results", - "node_name":"NODE_S_A0afcb9535d2619dac:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0afcb9535d2619dac:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.004490339197218418}} - ,{ - "core_node8":{ - "core":"audit_logs_shard2_replica_n6", - "shard":"shard2", - "collection":"audit_logs", - "node_name":"NODE_S_A0afcb9535d2619dac:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0afcb9535d2619dac:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.30385160446167E-7}} - ,{ - "core_node8":{ - "core":"COLL_N_Q_C_T_11_shard2_replica_n2", - "shard":"shard2", - "collection":"COLL_N_Q_C_T_11", - "node_name":"NODE_S_A0afcb9535d2619dac:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0afcb9535d2619dac:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.006244504824280739}} - ,{ - "core_node3":{ - "core":"COLL_N_Q_C_T_11_shard1_replica_n1", - "shard":"shard1", - "collection":"COLL_N_Q_C_T_11", - "node_name":"NODE_S_A0afcb9535d2619dac:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0afcb9535d2619dac:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.006021473556756973}} - ,{ - "core_node18":{ - "core":"COLL_S_H_7_shard9_replica_n16", - "shard":"shard9", - "collection":"COLL_S_H_7", - "node_name":"NODE_S_A0afcb9535d2619dac:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0afcb9535d2619dac:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.005475449375808239}} - ,{ - "core_node59":{ - "core":"COLL_R_P_S_A_22_shard29_replica_n56", - "shard":"shard29", - "collection":"COLL_R_P_S_A_22", - "node_name":"NODE_S_A0afcb9535d2619dac:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0afcb9535d2619dac:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.5458483807742596}} - ,{ - "core_node61":{ - "core":"COLL_E_v2_5_shard15_replica_n58", - "shard":"shard15", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A0afcb9535d2619dac:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A0afcb9535d2619dac:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":134.34092226438224}} - ,{ - "core_node29":{ - "core":"COLL_E_v2_5_shard7_replica_n26", - "shard":"shard7", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A0afcb9535d2619dac:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A0afcb9535d2619dac:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":124.55378058645874}}]}, - { - "node":"NODE_S_A09aede91f0e8c147c:8983_solr", - "tagKey":"NODE_S_A09aede91f0e8c147c:8983_solr", - "violation":{ - "node":"15.0", - "delta":4.0}, - "clause":{ - "cores":"#EQUAL", - "node":"#ANY"}, - "violatingReplicas":[{ - "core_node5":{ - "core":"S_B_2_shard2_replica_n2", - "shard":"shard2", - "collection":"S_B_2", - "node_name":"NODE_S_A09aede91f0e8c147c:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A09aede91f0e8c147c:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.030836801044642925}} - ,{ - "core_node47":{ - "core":"COLL_S_R_shard23_replica_n44", - "shard":"shard23", - "collection":"COLL_S_R", - "node_name":"NODE_S_A09aede91f0e8c147c:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A09aede91f0e8c147c:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.003840106539428234}} - ,{ - "core_node3":{ - "core":"COLL_S_M_1_shard1_replica_n1", - "shard":"shard1", - "collection":"COLL_S_M_1", - "node_name":"NODE_S_A09aede91f0e8c147c:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A09aede91f0e8c147c:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":6.426125764846802E-8}} - ,{ - "core_node5":{ - "core":"COLL_S_E_15_signals_aggr_shard1_replica_n2", - "shard":"shard1", - "collection":"COLL_S_E_15_signals_aggr", - "node_name":"NODE_S_A09aede91f0e8c147c:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A09aede91f0e8c147c:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":6.426125764846802E-8}} - ,{ - "core_node55":{ - "core":"COLL_S_E_15_shard14_replica_n52", - "shard":"shard14", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A09aede91f0e8c147c:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A09aede91f0e8c147c:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}} - ,{ - "core_node19":{ - "core":"COLL_S_E_15_shard5_replica_n16", - "shard":"shard5", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A09aede91f0e8c147c:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A09aede91f0e8c147c:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}} - ,{ - "core_node72":{ - "core":"COLL_S_E_15_shard18_replica_n70", - "shard":"shard18", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A09aede91f0e8c147c:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A09aede91f0e8c147c:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}} - ,{ - "core_node37":{ - "core":"COLL_S_E_15_shard9_replica_n34", - "shard":"shard9", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A09aede91f0e8c147c:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A09aede91f0e8c147c:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}} - ,{ - "core_node13":{ - "core":"COLL_P_V_E_v2_shard3_replica_n10", - "shard":"shard3", - "collection":"COLL_P_V_E_v2", - "node_name":"NODE_S_A09aede91f0e8c147c:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A09aede91f0e8c147c:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":8.193078896962106}} - ,{ - "core_node15":{ - "core":"COLL_N_Q_C_T_11_shard7_replica_n11", - "shard":"shard7", - "collection":"COLL_N_Q_C_T_11", - "node_name":"NODE_S_A09aede91f0e8c147c:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A09aede91f0e8c147c:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.00618187990039587}} - ,{ - "core_node3":{ - "core":"COLL_S_H_7_shard1_replica_n1", - "shard":"shard1", - "collection":"COLL_S_H_7", - "node_name":"NODE_S_A09aede91f0e8c147c:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A09aede91f0e8c147c:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.0052086347714066505}} - ,{ - "core_node3":{ - "core":"COLL_PV_E_v2_5_shard1_replica_n1", - "shard":"shard1", - "collection":"COLL_PV_E_v2_5", - "node_name":"NODE_S_A09aede91f0e8c147c:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A09aede91f0e8c147c:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":3.0264872340485454}} - ,{ - "core_node47":{ - "core":"COLL_R_P_S_A_22_shard23_replica_n44", - "shard":"shard23", - "collection":"COLL_R_P_S_A_22", - "node_name":"NODE_S_A09aede91f0e8c147c:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A09aede91f0e8c147c:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.4307612795382738}} - ,{ - "core_node141":{ - "core":"COLL_E_v2_5_shard35_replica_n138", - "shard":"shard35", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A09aede91f0e8c147c:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A09aede91f0e8c147c:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":128.57825357280672}} - ,{ - "core_node396":{ - "core":"COLL_E_v2_5_shard23_replica_n395", - "shard":"shard23", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A09aede91f0e8c147c:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A09aede91f0e8c147c:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":137.7283339947462}}]}, - { - "node":"NODE_S_A024a9c138d972d437:8983_solr", - "tagKey":"NODE_S_A024a9c138d972d437:8983_solr", - "violation":{ - "node":"14.0", - "delta":3.0}, - "clause":{ - "cores":"#EQUAL", - "node":"#ANY"}, - "violatingReplicas":[{ - "core_node13":{ - "core":"S_B_2_shard6_replica_n10", - "shard":"shard6", - "collection":"S_B_2", - "node_name":"NODE_S_A024a9c138d972d437:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A024a9c138d972d437:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.010442202910780907}} - ,{ - "core_node8":{ - "core":"COLL_S_R_aggr_shard2_replica_n6", - "shard":"shard2", - "collection":"COLL_S_R_aggr", - "node_name":"NODE_S_A024a9c138d972d437:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A024a9c138d972d437:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.7956495881080627}} - ,{ - "core_node55":{ - "core":"COLL_S_R_shard27_replica_n52", - "shard":"shard27", - "collection":"COLL_S_R", - "node_name":"NODE_S_A024a9c138d972d437:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A024a9c138d972d437:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.0038878321647644043}} - ,{ - "core_node7":{ - "core":"COLL_S_M_1_shard3_replica_n4", - "shard":"shard3", - "collection":"COLL_S_M_1", - "node_name":"NODE_S_A024a9c138d972d437:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A024a9c138d972d437:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":6.426125764846802E-8}} - ,{ - "core_node11":{ - "core":"COLL_S_E_15_shard3_replica_n8", - "shard":"shard3", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A024a9c138d972d437:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A024a9c138d972d437:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}} - ,{ - "core_node65":{ - "core":"COLL_S_E_15_shard16_replica_n62", - "shard":"shard16", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A024a9c138d972d437:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A024a9c138d972d437:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}} - ,{ - "core_node29":{ - "core":"COLL_S_E_15_shard7_replica_n26", - "shard":"shard7", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A024a9c138d972d437:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A024a9c138d972d437:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}} - ,{ - "core_node47":{ - "core":"COLL_S_E_15_shard12_replica_n44", - "shard":"shard12", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A024a9c138d972d437:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A024a9c138d972d437:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}} - ,{ - "core_node18":{ - "core":"COLL_N_Q_C_T_11_shard9_replica_n16", - "shard":"shard9", - "collection":"COLL_N_Q_C_T_11", - "node_name":"NODE_S_A024a9c138d972d437:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A024a9c138d972d437:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.006099941208958626}} - ,{ - "core_node7":{ - "core":"COLL_S_H_7_shard3_replica_n4", - "shard":"shard3", - "collection":"COLL_S_H_7", - "node_name":"NODE_S_A024a9c138d972d437:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A024a9c138d972d437:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.005192643962800503}} - ,{ - "core_node7":{ - "core":"COLL_S_E_15_signals_shard2_replica_n4", - "shard":"shard2", - "collection":"COLL_S_E_15_signals", - "node_name":"NODE_S_A024a9c138d972d437:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A024a9c138d972d437:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":2.8833746910095215E-5}} - ,{ - "core_node55":{ - "core":"COLL_R_P_S_A_22_shard27_replica_n52", - "shard":"shard27", - "collection":"COLL_R_P_S_A_22", - "node_name":"NODE_S_A024a9c138d972d437:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A024a9c138d972d437:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.46627365704625845}} - ,{ - "core_node7":{ - "core":"COLL_E_v2_5_shard2_replica_n4", - "shard":"shard2", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A024a9c138d972d437:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A024a9c138d972d437:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":138.75423685833812}} - ,{ - "core_node11":{ - "core":"COLL_E_v2_5_shard3_replica_n8", - "shard":"shard3", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A024a9c138d972d437:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A024a9c138d972d437:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":134.08093044813722}}]}, - { - "node":"NODE_S_A006b71d81aedd8577:8983_solr", - "tagKey":"NODE_S_A006b71d81aedd8577:8983_solr", - "violation":{ - "node":"9.0", - "delta":-1.0}, - "clause":{ - "cores":"#EQUAL", - "node":"#ANY"}, - "violatingReplicas":[{ - "core_node41":{ - "core":"query2query_shard20_replica_n38", - "shard":"shard20", - "collection":"query2query", - "node_name":"NODE_S_A006b71d81aedd8577:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A006b71d81aedd8577:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.43263052497059107}} - ,{ - "core_node29":{ - "core":"COLL_S_R_shard14_replica_n26", - "shard":"shard14", - "collection":"COLL_S_R", - "node_name":"NODE_S_A006b71d81aedd8577:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A006b71d81aedd8577:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.003805852495133877}} - ,{ - "core_node13":{ - "core":"Q2Q_postsandsubs_shard6_replica_n10", - "shard":"shard6", - "collection":"Q2Q_postsandsubs", - "node_name":"NODE_S_A006b71d81aedd8577:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A006b71d81aedd8577:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.01943675335496664}} - ,{ - "core_node15":{ - "core":"Q2Q_postsandsubs_shard7_replica_n12", - "shard":"shard7", - "collection":"Q2Q_postsandsubs", - "node_name":"NODE_S_A006b71d81aedd8577:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A006b71d81aedd8577:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.020343396812677383}} - ,{ - "core_node9":{ - "core":"COLL_P_V_E_v2_shard2_replica_n6", - "shard":"shard2", - "collection":"COLL_P_V_E_v2", - "node_name":"NODE_S_A006b71d81aedd8577:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A006b71d81aedd8577:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":8.209227412007749}} - ,{ - "core_node29":{ - "core":"COLL_R_P_S_A_22_shard14_replica_n26", - "shard":"shard14", - "collection":"COLL_R_P_S_A_22", - "node_name":"NODE_S_A006b71d81aedd8577:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A006b71d81aedd8577:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.4834298687055707}} - ,{ - "core_node218":{ - "core":"COLL_E_v2_5_shard25_replica_n217", - "shard":"shard25", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A006b71d81aedd8577:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A006b71d81aedd8577:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":133.02660531457514}} - ,{ - "core_node166":{ - "core":"COLL_E_v2_5_shard1_replica_n165", - "shard":"shard1", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A006b71d81aedd8577:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A006b71d81aedd8577:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":127.43107242323458}} - ,{ - "core_node194":{ - "core":"COLL_E_v2_5_shard11_replica_n193", - "shard":"shard11", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A006b71d81aedd8577:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A006b71d81aedd8577:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":126.14626492932439}}]}, - { - "node":"NODE_S_A014292d062d5088da:8983_solr", - "tagKey":"NODE_S_A014292d062d5088da:8983_solr", - "violation":{ - "node":"8.0", - "delta":-2.0}, - "clause":{ - "cores":"#EQUAL", - "node":"#ANY"}, - "violatingReplicas":[{ - "core_node42":{ - "core":"query2query_shard21_replica_n40", - "shard":"shard21", - "collection":"query2query", - "node_name":"NODE_S_A014292d062d5088da:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A014292d062d5088da:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.42733157239854336}} - ,{ - "core_node27":{ - "core":"COLL_S_R_shard13_replica_n24", - "shard":"shard13", - "collection":"COLL_S_R", - "node_name":"NODE_S_A014292d062d5088da:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A014292d062d5088da:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.00382127333432436}} - ,{ - "core_node11":{ - "core":"Q2Q_postsandsubs_shard5_replica_n8", - "shard":"shard5", - "collection":"Q2Q_postsandsubs", - "node_name":"NODE_S_A014292d062d5088da:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A014292d062d5088da:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.019260264933109283}} - ,{ - "core_node17":{ - "core":"Q2Q_postsandsubs_shard8_replica_n14", - "shard":"shard8", - "collection":"Q2Q_postsandsubs", - "node_name":"NODE_S_A014292d062d5088da:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A014292d062d5088da:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.01941517647355795}} - ,{ - "core_node7":{ - "core":"COLL_P_V_E_v2_shard2_replica_n4", - "shard":"shard2", - "collection":"COLL_P_V_E_v2", - "node_name":"NODE_S_A014292d062d5088da:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A014292d062d5088da:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":8.212146437726915}} - ,{ - "core_node27":{ - "core":"COLL_R_P_S_A_22_shard13_replica_n24", - "shard":"shard13", - "collection":"COLL_R_P_S_A_22", - "node_name":"NODE_S_A014292d062d5088da:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A014292d062d5088da:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.4219518853351474}} - ,{ - "core_node238":{ - "core":"COLL_E_v2_5_shard10_replica_n237", - "shard":"shard10", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A014292d062d5088da:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A014292d062d5088da:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":133.62058495730162}} - ,{ - "core_node240":{ - "core":"COLL_E_v2_5_shard12_replica_n239", - "shard":"shard12", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A014292d062d5088da:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A014292d062d5088da:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":131.8720874004066}}]}, - { - "node":"NODE_S_A06032b38152c0f019:8983_solr", - "tagKey":"NODE_S_A06032b38152c0f019:8983_solr", - "violation":{ - "node":"8.0", - "delta":-2.0}, - "clause":{ - "cores":"#EQUAL", - "node":"#ANY"}, - "violatingReplicas":[{ - "core_node27":{ - "core":"query2query_shard13_replica_n24", - "shard":"shard13", - "collection":"query2query", - "node_name":"NODE_S_A06032b38152c0f019:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A06032b38152c0f019:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.4311547642573714}} - ,{ - "core_node13":{ - "core":"query2query_shard6_replica_n10", - "shard":"shard6", - "collection":"query2query", - "node_name":"NODE_S_A06032b38152c0f019:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A06032b38152c0f019:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.42810762021690607}} - ,{ - "core_node31":{ - "core":"COLL_S_R_shard15_replica_n28", - "shard":"shard15", - "collection":"COLL_S_R", - "node_name":"NODE_S_A06032b38152c0f019:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A06032b38152c0f019:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.0038444409146904945}} - ,{ - "core_node7":{ - "core":"Q2Q_postsandsubs_shard3_replica_n4", - "shard":"shard3", - "collection":"Q2Q_postsandsubs", - "node_name":"NODE_S_A06032b38152c0f019:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A06032b38152c0f019:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.020358268171548843}} - ,{ - "core_node21":{ - "core":"Q2Q_postsandsubs_shard10_replica_n18", - "shard":"shard10", - "collection":"Q2Q_postsandsubs", - "node_name":"NODE_S_A06032b38152c0f019:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A06032b38152c0f019:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.02053267415612936}} - ,{ - "core_node31":{ - "core":"COLL_R_P_S_A_22_shard15_replica_n28", - "shard":"shard15", - "collection":"COLL_R_P_S_A_22", - "node_name":"NODE_S_A06032b38152c0f019:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A06032b38152c0f019:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.5062682097777724}} - ,{ - "core_node182":{ - "core":"COLL_E_v2_5_shard24_replica_n181", - "shard":"shard24", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A06032b38152c0f019:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A06032b38152c0f019:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":132.00969661120325}} - ,{ - "core_node162":{ - "core":"COLL_E_v2_5_shard6_replica_n161", - "shard":"shard6", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A06032b38152c0f019:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A06032b38152c0f019:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":131.38960415963084}}]}, - { - "node":"NODE_S_A04a75744569a27ccf:8983_solr", - "tagKey":"NODE_S_A04a75744569a27ccf:8983_solr", - "violation":{ - "node":"8.0", - "delta":-2.0}, - "clause":{ - "cores":"#EQUAL", - "node":"#ANY"}, - "violatingReplicas":[{ - "core_node39":{ - "core":"query2query_shard19_replica_n36", - "shard":"shard19", - "collection":"query2query", - "node_name":"NODE_S_A04a75744569a27ccf:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A04a75744569a27ccf:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.42521866224706173}} - ,{ - "core_node37":{ - "core":"COLL_S_R_shard18_replica_n34", - "shard":"shard18", - "collection":"COLL_S_R", - "node_name":"NODE_S_A04a75744569a27ccf:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A04a75744569a27ccf:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.003841894678771496}} - ,{ - "core_node37":{ - "core":"Q2Q_postsandsubs_shard18_replica_n34", - "shard":"shard18", - "collection":"Q2Q_postsandsubs", - "node_name":"NODE_S_A04a75744569a27ccf:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A04a75744569a27ccf:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.019906101748347282}} - ,{ - "core_node39":{ - "core":"Q2Q_postsandsubs_shard19_replica_n36", - "shard":"shard19", - "collection":"Q2Q_postsandsubs", - "node_name":"NODE_S_A04a75744569a27ccf:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A04a75744569a27ccf:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.019286231137812138}} - ,{ - "core_node54":{ - "core":"subreddit_experiment_results_shard27_replica_n52", - "shard":"shard27", - "collection":"subreddit_experiment_results", - "node_name":"NODE_S_A04a75744569a27ccf:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A04a75744569a27ccf:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.004555700346827507}} - ,{ - "core_node37":{ - "core":"COLL_R_P_S_A_22_shard18_replica_n34", - "shard":"shard18", - "collection":"COLL_R_P_S_A_22", - "node_name":"NODE_S_A04a75744569a27ccf:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A04a75744569a27ccf:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.4519655341282487}} - ,{ - "core_node212":{ - "core":"COLL_E_v2_5_shard17_replica_n211", - "shard":"shard17", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A04a75744569a27ccf:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A04a75744569a27ccf:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":135.40381868369877}} - ,{ - "core_node186":{ - "core":"COLL_E_v2_5_shard33_replica_n185", - "shard":"shard33", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A04a75744569a27ccf:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A04a75744569a27ccf:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":134.5945677505806}}]}, - { - "node":"NODE_S_A0e48ab035b869305e:8983_solr", - "tagKey":"NODE_S_A0e48ab035b869305e:8983_solr", - "violation":{ - "node":"8.0", - "delta":-2.0}, - "clause":{ - "cores":"#EQUAL", - "node":"#ANY"}, - "violatingReplicas":[{ - "core_node5":{ - "core":"recall_cache_shard1_replica_n2", - "shard":"shard1", - "collection":"recall_cache", - "node_name":"NODE_S_A0e48ab035b869305e:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0e48ab035b869305e:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.0050263796001672745}} - ,{ - "core_node25":{ - "core":"COLL_S_R_shard12_replica_n22", - "shard":"shard12", - "collection":"COLL_S_R", - "node_name":"NODE_S_A0e48ab035b869305e:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0e48ab035b869305e:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.0038190074265003204}} - ,{ - "core_node39":{ - "core":"subreddit_experiment_results_shard19_replica_n36", - "shard":"shard19", - "collection":"subreddit_experiment_results", - "node_name":"NODE_S_A0e48ab035b869305e:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0e48ab035b869305e:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.00449786800891161}} - ,{ - "core_node13":{ - "core":"subreddit_experiment_results_shard6_replica_n10", - "shard":"shard6", - "collection":"subreddit_experiment_results", - "node_name":"NODE_S_A0e48ab035b869305e:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0e48ab035b869305e:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.004472509957849979}} - ,{ - "core_node12":{ - "core":"COLL_PV_E_v2_5_shard3_replica_n10", - "shard":"shard3", - "collection":"COLL_PV_E_v2_5", - "node_name":"NODE_S_A0e48ab035b869305e:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0e48ab035b869305e:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":3.0129052978008986}} - ,{ - "core_node25":{ - "core":"COLL_R_P_S_A_22_shard12_replica_n22", - "shard":"shard12", - "collection":"COLL_R_P_S_A_22", - "node_name":"NODE_S_A0e48ab035b869305e:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0e48ab035b869305e:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.5445274133235216}} - ,{ - "core_node358":{ - "core":"COLL_E_v2_5_shard36_replica_n357", - "shard":"shard36", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A0e48ab035b869305e:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0e48ab035b869305e:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":133.00134566705674}} - ,{ - "core_node388":{ - "core":"COLL_E_v2_5_shard4_replica_n387", - "shard":"shard4", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A0e48ab035b869305e:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A0e48ab035b869305e:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":127.21125747542828}}]}, - { - "node":"NODE_S_A0d1f4439b8e8878e8:8983_solr", - "tagKey":"NODE_S_A0d1f4439b8e8878e8:8983_solr", - "violation":{ - "node":"7.0", - "delta":-3.0}, - "clause":{ - "cores":"#EQUAL", - "node":"#ANY"}, - "violatingReplicas":[{ - "core_node23":{ - "core":"COLL_S_R_shard11_replica_n20", - "shard":"shard11", - "collection":"COLL_S_R", - "node_name":"NODE_S_A0d1f4439b8e8878e8:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0d1f4439b8e8878e8:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.0038332296535372734}} - ,{ - "core_node10":{ - "core":"COLL_E_V2_S_A_3_shard1_replica_n9", - "shard":"shard1", - "collection":"COLL_E_V2_S_A_3", - "node_name":"NODE_S_A0d1f4439b8e8878e8:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0d1f4439b8e8878e8:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":25.122611593455076}} - ,{ - "core_node11":{ - "core":"subreddit_experiment_results_shard5_replica_n8", - "shard":"shard5", - "collection":"subreddit_experiment_results", - "node_name":"NODE_S_A0d1f4439b8e8878e8:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0d1f4439b8e8878e8:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.004589034244418144}} - ,{ - "core_node41":{ - "core":"subreddit_experiment_results_shard20_replica_n38", - "shard":"shard20", - "collection":"subreddit_experiment_results", - "node_name":"NODE_S_A0d1f4439b8e8878e8:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0d1f4439b8e8878e8:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.004457270726561546}} - ,{ - "core_node23":{ - "core":"COLL_R_P_S_A_22_shard11_replica_n20", - "shard":"shard11", - "collection":"COLL_R_P_S_A_22", - "node_name":"NODE_S_A0d1f4439b8e8878e8:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0d1f4439b8e8878e8:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.4702793164178729}} - ,{ - "core_node374":{ - "core":"COLL_E_v2_5_shard35_replica_n373", - "shard":"shard35", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A0d1f4439b8e8878e8:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0d1f4439b8e8878e8:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":129.69269095920026}} - ,{ - "core_node402":{ - "core":"COLL_E_v2_5_shard27_replica_n401", - "shard":"shard27", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A0d1f4439b8e8878e8:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A0d1f4439b8e8878e8:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":132.8685013735667}}]}, - { - "node":"NODE_S_A0b7fb52d8d0a2ecde:8983_solr", - "tagKey":"NODE_S_A0b7fb52d8d0a2ecde:8983_solr", - "violation":{ - "node":"7.0", - "delta":-3.0}, - "clause":{ - "cores":"#EQUAL", - "node":"#ANY"}, - "violatingReplicas":[{ - "core_node6":{ - "core":"recall_cache_shard1_replica_n4", - "shard":"shard1", - "collection":"recall_cache", - "node_name":"NODE_S_A0b7fb52d8d0a2ecde:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A0b7fb52d8d0a2ecde:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.005021187476813793}} - ,{ - "core_node21":{ - "core":"COLL_S_R_shard10_replica_n18", - "shard":"shard10", - "collection":"COLL_S_R", - "node_name":"NODE_S_A0b7fb52d8d0a2ecde:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0b7fb52d8d0a2ecde:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.003915207460522652}} - ,{ - "core_node33":{ - "core":"subreddit_experiment_results_shard16_replica_n30", - "shard":"shard16", - "collection":"subreddit_experiment_results", - "node_name":"NODE_S_A0b7fb52d8d0a2ecde:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0b7fb52d8d0a2ecde:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.004510008729994297}} - ,{ - "core_node19":{ - "core":"subreddit_experiment_results_shard9_replica_n16", - "shard":"shard9", - "collection":"subreddit_experiment_results", - "node_name":"NODE_S_A0b7fb52d8d0a2ecde:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0b7fb52d8d0a2ecde:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.0044971127063035965}} - ,{ - "core_node21":{ - "core":"COLL_R_P_S_A_22_shard10_replica_n18", - "shard":"shard10", - "collection":"COLL_R_P_S_A_22", - "node_name":"NODE_S_A0b7fb52d8d0a2ecde:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0b7fb52d8d0a2ecde:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.45318685937672853}} - ,{ - "core_node362":{ - "core":"COLL_E_v2_5_shard31_replica_n361", - "shard":"shard31", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A0b7fb52d8d0a2ecde:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0b7fb52d8d0a2ecde:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":135.04639122448862}} - ,{ - "core_node384":{ - "core":"COLL_E_v2_5_shard34_replica_n383", - "shard":"shard34", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A0b7fb52d8d0a2ecde:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A0b7fb52d8d0a2ecde:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":138.16951165627688}}]}, - { - "node":"NODE_S_A0d391cf8eeaca59cd:8983_solr", - "tagKey":"NODE_S_A0d391cf8eeaca59cd:8983_solr", - "violation":{ - "node":"7.0", - "delta":-3.0}, - "clause":{ - "cores":"#EQUAL", - "node":"#ANY"}, - "violatingReplicas":[{ - "core_node3":{ - "core":"recall_cache_shard1_replica_n1", - "shard":"shard1", - "collection":"recall_cache", - "node_name":"NODE_S_A0d391cf8eeaca59cd:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A0d391cf8eeaca59cd:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.005028365179896355}} - ,{ - "core_node19":{ - "core":"COLL_S_R_shard9_replica_n16", - "shard":"shard9", - "collection":"COLL_S_R", - "node_name":"NODE_S_A0d391cf8eeaca59cd:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0d391cf8eeaca59cd:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.003799574449658394}} - ,{ - "core_node5":{ - "core":"subreddit_experiment_results_shard2_replica_n2", - "shard":"shard2", - "collection":"subreddit_experiment_results", - "node_name":"NODE_S_A0d391cf8eeaca59cd:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0d391cf8eeaca59cd:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.004513397812843323}} - ,{ - "core_node47":{ - "core":"subreddit_experiment_results_shard23_replica_n44", - "shard":"shard23", - "collection":"subreddit_experiment_results", - "node_name":"NODE_S_A0d391cf8eeaca59cd:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0d391cf8eeaca59cd:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.0045642731711268425}} - ,{ - "core_node19":{ - "core":"COLL_R_P_S_A_22_shard9_replica_n16", - "shard":"shard9", - "collection":"COLL_R_P_S_A_22", - "node_name":"NODE_S_A0d391cf8eeaca59cd:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0d391cf8eeaca59cd:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.46696313098073006}} - ,{ - "core_node378":{ - "core":"COLL_E_v2_5_shard2_replica_n377", - "shard":"shard2", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A0d391cf8eeaca59cd:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A0d391cf8eeaca59cd:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":139.2840456981212}} - ,{ - "core_node360":{ - "core":"COLL_E_v2_5_shard25_replica_n359", - "shard":"shard25", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A0d391cf8eeaca59cd:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A0d391cf8eeaca59cd:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":134.43318709544837}}]}, - { - "node":"NODE_S_A0aaaf34e6b281b24c:8983_solr", - "tagKey":"NODE_S_A0aaaf34e6b281b24c:8983_solr", - "violation":{ - "node":"6.0", - "delta":-4.0}, - "clause":{ - "cores":"#EQUAL", - "node":"#ANY"}, - "violatingReplicas":[{ - "core_node17":{ - "core":"COLL_S_R_shard8_replica_n14", - "shard":"shard8", - "collection":"COLL_S_R", - "node_name":"NODE_S_A0aaaf34e6b281b24c:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0aaaf34e6b281b24c:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.003879970870912075}} - ,{ - "core_node29":{ - "core":"subreddit_experiment_results_shard14_replica_n26", - "shard":"shard14", - "collection":"subreddit_experiment_results", - "node_name":"NODE_S_A0aaaf34e6b281b24c:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0aaaf34e6b281b24c:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.004518838599324226}} - ,{ - "core_node23":{ - "core":"subreddit_experiment_results_shard11_replica_n20", - "shard":"shard11", - "collection":"subreddit_experiment_results", - "node_name":"NODE_S_A0aaaf34e6b281b24c:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0aaaf34e6b281b24c:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.0045999325811862946}} - ,{ - "core_node17":{ - "core":"COLL_R_P_S_A_22_shard8_replica_n14", - "shard":"shard8", - "collection":"COLL_R_P_S_A_22", - "node_name":"NODE_S_A0aaaf34e6b281b24c:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0aaaf34e6b281b24c:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.5148522546514869}} - ,{ - "core_node380":{ - "core":"COLL_E_v2_5_shard16_replica_n379", - "shard":"shard16", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A0aaaf34e6b281b24c:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A0aaaf34e6b281b24c:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":130.66182304918766}} - ,{ - "core_node364":{ - "core":"COLL_E_v2_5_shard32_replica_n363", - "shard":"shard32", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A0aaaf34e6b281b24c:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0aaaf34e6b281b24c:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":132.6516477856785}}]}, - { - "node":"NODE_S_A0bdba3561ff291bc4:8983_solr", - "tagKey":"NODE_S_A0bdba3561ff291bc4:8983_solr", - "violation":{ - "node":"5.0", - "delta":-5.0}, - "clause":{ - "cores":"#EQUAL", - "node":"#ANY"}, - "violatingReplicas":[{ - "core_node15":{ - "core":"COLL_S_R_shard7_replica_n12", - "shard":"shard7", - "collection":"COLL_S_R", - "node_name":"NODE_S_A0bdba3561ff291bc4:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0bdba3561ff291bc4:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.003801817074418068}} - ,{ - "core_node24":{ - "core":"COLL_P_V_E_v2_shard6_replica_n22", - "shard":"shard6", - "collection":"COLL_P_V_E_v2", - "node_name":"NODE_S_A0bdba3561ff291bc4:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0bdba3561ff291bc4:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":8.199045146815479}} - ,{ - "core_node15":{ - "core":"COLL_R_P_S_A_22_shard7_replica_n12", - "shard":"shard7", - "collection":"COLL_R_P_S_A_22", - "node_name":"NODE_S_A0bdba3561ff291bc4:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0bdba3561ff291bc4:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.4553129719570279}} - ,{ - "core_node404":{ - "core":"COLL_E_v2_5_shard3_replica_n403", - "shard":"shard3", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A0bdba3561ff291bc4:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A0bdba3561ff291bc4:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":133.96376375574619}} - ,{ - "core_node376":{ - "core":"COLL_E_v2_5_shard28_replica_n375", - "shard":"shard28", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A0bdba3561ff291bc4:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A0bdba3561ff291bc4:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":134.145930592902}}]}, - { - "node":"NODE_S_A086a3a5ff0617c80f:8983_solr", - "tagKey":"NODE_S_A086a3a5ff0617c80f:8983_solr", - "violation":{ - "node":"5.0", - "delta":-5.0}, - "clause":{ - "cores":"#EQUAL", - "node":"#ANY"}, - "violatingReplicas":[{ - "core_node13":{ - "core":"COLL_S_R_shard6_replica_n10", - "shard":"shard6", - "collection":"COLL_S_R", - "node_name":"NODE_S_A086a3a5ff0617c80f:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A086a3a5ff0617c80f:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.0038511399179697037}} - ,{ - "core_node5":{ - "core":"COLL_P_V_E_v2_shard1_replica_n2", - "shard":"shard1", - "collection":"COLL_P_V_E_v2", - "node_name":"NODE_S_A086a3a5ff0617c80f:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A086a3a5ff0617c80f:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":8.208250655792654}} - ,{ - "core_node13":{ - "core":"COLL_R_P_S_A_22_shard6_replica_n10", - "shard":"shard6", - "collection":"COLL_R_P_S_A_22", - "node_name":"NODE_S_A086a3a5ff0617c80f:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A086a3a5ff0617c80f:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.48864932265132666}} - ,{ - "core_node400":{ - "core":"COLL_E_v2_5_shard36_replica_n399", - "shard":"shard36", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A086a3a5ff0617c80f:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A086a3a5ff0617c80f:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":132.73244500439614}} - ,{ - "core_node370":{ - "core":"COLL_E_v2_5_shard10_replica_n369", - "shard":"shard10", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A086a3a5ff0617c80f:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A086a3a5ff0617c80f:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":134.23334093671292}}]}, - { - "node":"NODE_S_A09f108fb78272a03d:8983_solr", - "tagKey":"NODE_S_A09f108fb78272a03d:8983_solr", - "violation":{ - "node":"5.0", - "delta":-5.0}, - "clause":{ - "cores":"#EQUAL", - "node":"#ANY"}, - "violatingReplicas":[{ - "core_node4":{ - "core":"COLL_S_R_shard2_replica_n2", - "shard":"shard2", - "collection":"COLL_S_R", - "node_name":"NODE_S_A09f108fb78272a03d:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A09f108fb78272a03d:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.0037746401503682137}} - ,{ - "core_node19":{ - "core":"COLL_P_V_E_v2_shard5_replica_n16", - "shard":"shard5", - "collection":"COLL_P_V_E_v2", - "node_name":"NODE_S_A09f108fb78272a03d:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A09f108fb78272a03d:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":9.87934261187911}} - ,{ - "core_node5":{ - "core":"COLL_R_P_S_A_22_shard2_replica_n2", - "shard":"shard2", - "collection":"COLL_R_P_S_A_22", - "node_name":"NODE_S_A09f108fb78272a03d:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A09f108fb78272a03d:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.4274335531517863}} - ,{ - "core_node392":{ - "core":"COLL_E_v2_5_shard29_replica_n391", - "shard":"shard29", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A09f108fb78272a03d:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A09f108fb78272a03d:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":131.70282221771777}} - ,{ - "core_node246":{ - "core":"COLL_E_v2_5_shard20_replica_n245", - "shard":"shard20", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A09f108fb78272a03d:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A09f108fb78272a03d:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":133.18864511698484}}]}, - { - "node":"NODE_S_A013eaa884d3c59fab:8983_solr", - "tagKey":"NODE_S_A013eaa884d3c59fab:8983_solr", - "violation":{ - "node":"4.0", - "delta":-6.0}, - "clause":{ - "cores":"#EQUAL", - "node":"#ANY"}, - "violatingReplicas":[{ - "core_node11":{ - "core":"COLL_S_R_shard5_replica_n8", - "shard":"shard5", - "collection":"COLL_S_R", - "node_name":"NODE_S_A013eaa884d3c59fab:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A013eaa884d3c59fab:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.0038577821105718613}} - ,{ - "core_node11":{ - "core":"COLL_R_P_S_A_22_shard5_replica_n8", - "shard":"shard5", - "collection":"COLL_R_P_S_A_22", - "node_name":"NODE_S_A013eaa884d3c59fab:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A013eaa884d3c59fab:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.42266264744102955}} - ,{ - "core_node368":{ - "core":"COLL_E_v2_5_shard15_replica_n367", - "shard":"shard15", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A013eaa884d3c59fab:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A013eaa884d3c59fab:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":128.1388803133741}} - ,{ - "core_node398":{ - "core":"COLL_E_v2_5_shard9_replica_n397", - "shard":"shard9", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A013eaa884d3c59fab:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A013eaa884d3c59fab:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":135.6189723983407}}]}, - { - "node":"NODE_S_A03baeedfd28ba05ba:8983_solr", - "tagKey":"NODE_S_A03baeedfd28ba05ba:8983_solr", - "violation":{ - "node":"4.0", - "delta":-6.0}, - "clause":{ - "cores":"#EQUAL", - "node":"#ANY"}, - "violatingReplicas":[{ - "core_node9":{ - "core":"COLL_S_R_shard4_replica_n6", - "shard":"shard4", - "collection":"COLL_S_R", - "node_name":"NODE_S_A03baeedfd28ba05ba:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A03baeedfd28ba05ba:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.003800010308623314}} - ,{ - "core_node9":{ - "core":"COLL_R_P_S_A_22_shard4_replica_n6", - "shard":"shard4", - "collection":"COLL_R_P_S_A_22", - "node_name":"NODE_S_A03baeedfd28ba05ba:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A03baeedfd28ba05ba:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.41693424154073}} - ,{ - "core_node390":{ - "core":"COLL_E_v2_5_shard13_replica_n389", - "shard":"shard13", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A03baeedfd28ba05ba:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A03baeedfd28ba05ba:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":135.04321804642677}} - ,{ - "core_node366":{ - "core":"COLL_E_v2_5_shard26_replica_n365", - "shard":"shard26", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A03baeedfd28ba05ba:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A03baeedfd28ba05ba:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":129.58171366155148}}]}, - { - "node":"NODE_S_A082e313208af7da04:8983_solr", - "tagKey":"NODE_S_A082e313208af7da04:8983_solr", - "violation":{ - "node":"4.0", - "delta":-6.0}, - "clause":{ - "cores":"#EQUAL", - "node":"#ANY"}, - "violatingReplicas":[{ - "core_node3":{ - "core":"COLL_S_R_shard1_replica_n1", - "shard":"shard1", - "collection":"COLL_S_R", - "node_name":"NODE_S_A082e313208af7da04:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A082e313208af7da04:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.003850843757390976}} - ,{ - "core_node3":{ - "core":"COLL_R_P_S_A_22_shard1_replica_n1", - "shard":"shard1", - "collection":"COLL_R_P_S_A_22", - "node_name":"NODE_S_A082e313208af7da04:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A082e313208af7da04:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.46176901180297136}} - ,{ - "core_node244":{ - "core":"COLL_E_v2_5_shard18_replica_n243", - "shard":"shard18", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A082e313208af7da04:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A082e313208af7da04:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":130.08673685323447}} - ,{ - "core_node386":{ - "core":"COLL_E_v2_5_shard9_replica_n385", - "shard":"shard9", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A082e313208af7da04:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A082e313208af7da04:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":136.76205699518323}}]}, - { - "node":"NODE_S_A0a0ffe317a4f85c55:8983_solr", - "tagKey":"NODE_S_A0a0ffe317a4f85c55:8983_solr", - "violation":{ - "node":"4.0", - "delta":-6.0}, - "clause":{ - "cores":"#EQUAL", - "node":"#ANY"}, - "violatingReplicas":[{ - "core_node7":{ - "core":"COLL_S_R_shard3_replica_n5", - "shard":"shard3", - "collection":"COLL_S_R", - "node_name":"NODE_S_A0a0ffe317a4f85c55:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0a0ffe317a4f85c55:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.003859194926917553}} - ,{ - "core_node7":{ - "core":"COLL_R_P_S_A_22_shard3_replica_n4", - "shard":"shard3", - "collection":"COLL_R_P_S_A_22", - "node_name":"NODE_S_A0a0ffe317a4f85c55:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0a0ffe317a4f85c55:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.41597359627485275}} - ,{ - "core_node248":{ - "core":"COLL_E_v2_5_shard26_replica_n247", - "shard":"shard26", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A0a0ffe317a4f85c55:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0a0ffe317a4f85c55:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":127.8478315602988}} - ,{ - "core_node394":{ - "core":"COLL_E_v2_5_shard1_replica_n393", - "shard":"shard1", - "collection":"COLL_E_v2_5", - "node_name":"NODE_S_A0a0ffe317a4f85c55:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A0a0ffe317a4f85c55:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":127.3860678607598}}]}, - { - "collection":"query2query", - "node":"NODE_S_A0ebcce85ef9c13308:8983_solr", - "tagKey":"NODE_S_A0ebcce85ef9c13308:8983_solr", - "violation":{ - "replica":{ - "NRT":2, - "count":2}, - "delta":1.0}, - "clause":{ - "replica":"#EQUAL", - "node":"#ANY", - "collection":"query2query"}, - "violatingReplicas":[{ - "core_node23":{ - "core":"query2query_shard11_replica_n20", - "shard":"shard11", - "collection":"query2query", - "node_name":"NODE_S_A0ebcce85ef9c13308:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0ebcce85ef9c13308:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.43222586531192064}} - ,{ - "core_node17":{ - "core":"query2query_shard8_replica_n14", - "shard":"shard8", - "collection":"query2query", - "node_name":"NODE_S_A0ebcce85ef9c13308:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0ebcce85ef9c13308:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.4294911604374647}}]}, - { - "collection":"query2query", - "node":"NODE_S_A0f66f4bec1ce8fc09:8983_solr", - "tagKey":"NODE_S_A0f66f4bec1ce8fc09:8983_solr", - "violation":{ - "replica":{ - "NRT":2, - "count":2}, - "delta":1.0}, - "clause":{ - "replica":"#EQUAL", - "node":"#ANY", - "collection":"query2query"}, - "violatingReplicas":[{ - "core_node15":{ - "core":"query2query_shard7_replica_n12", - "shard":"shard7", - "collection":"query2query", - "node_name":"NODE_S_A0f66f4bec1ce8fc09:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0f66f4bec1ce8fc09:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.4308900246396661}} - ,{ - "core_node25":{ - "core":"query2query_shard12_replica_n22", - "shard":"shard12", - "collection":"query2query", - "node_name":"NODE_S_A0f66f4bec1ce8fc09:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0f66f4bec1ce8fc09:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.43196454364806414}}]}, - { - "collection":"query2query", - "node":"NODE_S_A02f6288aa3fc9eb3d:8983_solr", - "tagKey":"NODE_S_A02f6288aa3fc9eb3d:8983_solr", - "violation":{ - "replica":{ - "NRT":2, - "count":2}, - "delta":1.0}, - "clause":{ - "replica":"#EQUAL", - "node":"#ANY", - "collection":"query2query"}, - "violatingReplicas":[{ - "core_node21":{ - "core":"query2query_shard10_replica_n18", - "shard":"shard10", - "collection":"query2query", - "node_name":"NODE_S_A02f6288aa3fc9eb3d:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A02f6288aa3fc9eb3d:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.429669008590281}} - ,{ - "core_node19":{ - "core":"query2query_shard9_replica_n16", - "shard":"shard9", - "collection":"query2query", - "node_name":"NODE_S_A02f6288aa3fc9eb3d:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A02f6288aa3fc9eb3d:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.4358885111287236}}]}, - { - "collection":"query2query", - "node":"NODE_S_A0cf36cc11a8447fac:8983_solr", - "tagKey":"NODE_S_A0cf36cc11a8447fac:8983_solr", - "violation":{ - "replica":{ - "NRT":2, - "count":2}, - "delta":1.0}, - "clause":{ - "replica":"#EQUAL", - "node":"#ANY", - "collection":"query2query"}, - "violatingReplicas":[{ - "core_node29":{ - "core":"query2query_shard14_replica_n26", - "shard":"shard14", - "collection":"query2query", - "node_name":"NODE_S_A0cf36cc11a8447fac:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0cf36cc11a8447fac:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.4342966331169009}} - ,{ - "core_node11":{ - "core":"query2query_shard5_replica_n8", - "shard":"shard5", - "collection":"query2query", - "node_name":"NODE_S_A0cf36cc11a8447fac:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0cf36cc11a8447fac:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.43006310891360044}}]}, - { - "collection":"query2query", - "node":"NODE_S_A061d3ac3c71d42de2:8983_solr", - "tagKey":"NODE_S_A061d3ac3c71d42de2:8983_solr", - "violation":{ - "replica":{ - "NRT":2, - "count":2}, - "delta":1.0}, - "clause":{ - "replica":"#EQUAL", - "node":"#ANY", - "collection":"query2query"}, - "violatingReplicas":[{ - "core_node7":{ - "core":"query2query_shard3_replica_n4", - "shard":"shard3", - "collection":"query2query", - "node_name":"NODE_S_A061d3ac3c71d42de2:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A061d3ac3c71d42de2:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.430878022685647}} - ,{ - "core_node33":{ - "core":"query2query_shard16_replica_n30", - "shard":"shard16", - "collection":"query2query", - "node_name":"NODE_S_A061d3ac3c71d42de2:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A061d3ac3c71d42de2:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.4258228335529566}}]}, - { - "collection":"query2query", - "node":"NODE_S_A08c1ec5957d8a5d44:8983_solr", - "tagKey":"NODE_S_A08c1ec5957d8a5d44:8983_solr", - "violation":{ - "replica":{ - "NRT":2, - "count":2}, - "delta":1.0}, - "clause":{ - "replica":"#EQUAL", - "node":"#ANY", - "collection":"query2query"}, - "violatingReplicas":[{ - "core_node31":{ - "core":"query2query_shard15_replica_n28", - "shard":"shard15", - "collection":"query2query", - "node_name":"NODE_S_A08c1ec5957d8a5d44:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A08c1ec5957d8a5d44:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.4261511182412505}} - ,{ - "core_node9":{ - "core":"query2query_shard4_replica_n6", - "shard":"shard4", - "collection":"query2query", - "node_name":"NODE_S_A08c1ec5957d8a5d44:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A08c1ec5957d8a5d44:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.4257269874215126}}]}, - { - "collection":"query2query", - "node":"NODE_S_A084aef0a141a43572:8983_solr", - "tagKey":"NODE_S_A084aef0a141a43572:8983_solr", - "violation":{ - "replica":{ - "NRT":2, - "count":2}, - "delta":1.0}, - "clause":{ - "replica":"#EQUAL", - "node":"#ANY", - "collection":"query2query"}, - "violatingReplicas":[{ - "core_node5":{ - "core":"query2query_shard2_replica_n2", - "shard":"shard2", - "collection":"query2query", - "node_name":"NODE_S_A084aef0a141a43572:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A084aef0a141a43572:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.4285783916711807}} - ,{ - "core_node35":{ - "core":"query2query_shard17_replica_n32", - "shard":"shard17", - "collection":"query2query", - "node_name":"NODE_S_A084aef0a141a43572:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A084aef0a141a43572:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.4329432938247919}}]}, - { - "collection":"query2query", - "node":"NODE_S_A0f096665ad4a3bc58:8983_solr", - "tagKey":"NODE_S_A0f096665ad4a3bc58:8983_solr", - "violation":{ - "replica":{ - "NRT":2, - "count":2}, - "delta":1.0}, - "clause":{ - "replica":"#EQUAL", - "node":"#ANY", - "collection":"query2query"}, - "violatingReplicas":[{ - "core_node37":{ - "core":"query2query_shard18_replica_n34", - "shard":"shard18", - "collection":"query2query", - "node_name":"NODE_S_A0f096665ad4a3bc58:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0f096665ad4a3bc58:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.4305841075256467}} - ,{ - "core_node3":{ - "core":"query2query_shard1_replica_n1", - "shard":"shard1", - "collection":"query2query", - "node_name":"NODE_S_A0f096665ad4a3bc58:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0f096665ad4a3bc58:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.42587023694068193}}]}, - { - "collection":"query2query", - "node":"NODE_S_A06032b38152c0f019:8983_solr", - "tagKey":"NODE_S_A06032b38152c0f019:8983_solr", - "violation":{ - "replica":{ - "NRT":2, - "count":2}, - "delta":1.0}, - "clause":{ - "replica":"#EQUAL", - "node":"#ANY", - "collection":"query2query"}, - "violatingReplicas":[{ - "core_node27":{ - "core":"query2query_shard13_replica_n24", - "shard":"shard13", - "collection":"query2query", - "node_name":"NODE_S_A06032b38152c0f019:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A06032b38152c0f019:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.4311547642573714}} - ,{ - "core_node13":{ - "core":"query2query_shard6_replica_n10", - "shard":"shard6", - "collection":"query2query", - "node_name":"NODE_S_A06032b38152c0f019:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A06032b38152c0f019:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.42810762021690607}}]}, - { - "collection":"Q2Q_postsandsubs", - "node":"NODE_S_A0ebcce85ef9c13308:8983_solr", - "tagKey":"NODE_S_A0ebcce85ef9c13308:8983_solr", - "violation":{ - "replica":{ - "NRT":2, - "count":2}, - "delta":1.0}, - "clause":{ - "replica":"#EQUAL", - "node":"#ANY", - "collection":"Q2Q_postsandsubs"}, - "violatingReplicas":[{ - "core_node5":{ - "core":"Q2Q_postsandsubs_shard2_replica_n2", - "shard":"shard2", - "collection":"Q2Q_postsandsubs", - "node_name":"NODE_S_A0ebcce85ef9c13308:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0ebcce85ef9c13308:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.020282848738133907}} - ,{ - "core_node23":{ - "core":"Q2Q_postsandsubs_shard11_replica_n20", - "shard":"shard11", - "collection":"Q2Q_postsandsubs", - "node_name":"NODE_S_A0ebcce85ef9c13308:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0ebcce85ef9c13308:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.0202587079256773}}]}, - { - "collection":"Q2Q_postsandsubs", - "node":"NODE_S_A0f66f4bec1ce8fc09:8983_solr", - "tagKey":"NODE_S_A0f66f4bec1ce8fc09:8983_solr", - "violation":{ - "replica":{ - "NRT":2, - "count":2}, - "delta":1.0}, - "clause":{ - "replica":"#EQUAL", - "node":"#ANY", - "collection":"Q2Q_postsandsubs"}, - "violatingReplicas":[{ - "core_node3":{ - "core":"Q2Q_postsandsubs_shard1_replica_n1", - "shard":"shard1", - "collection":"Q2Q_postsandsubs", - "node_name":"NODE_S_A0f66f4bec1ce8fc09:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0f66f4bec1ce8fc09:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.01946087647229433}} - ,{ - "core_node25":{ - "core":"Q2Q_postsandsubs_shard12_replica_n22", - "shard":"shard12", - "collection":"Q2Q_postsandsubs", - "node_name":"NODE_S_A0f66f4bec1ce8fc09:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0f66f4bec1ce8fc09:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.018688012845814228}}]}, - { - "collection":"Q2Q_postsandsubs", - "node":"NODE_S_A02f6288aa3fc9eb3d:8983_solr", - "tagKey":"NODE_S_A02f6288aa3fc9eb3d:8983_solr", - "violation":{ - "replica":{ - "NRT":2, - "count":2}, - "delta":1.0}, - "clause":{ - "replica":"#EQUAL", - "node":"#ANY", - "collection":"Q2Q_postsandsubs"}, - "violatingReplicas":[{ - "core_node9":{ - "core":"Q2Q_postsandsubs_shard4_replica_n6", - "shard":"shard4", - "collection":"Q2Q_postsandsubs", - "node_name":"NODE_S_A02f6288aa3fc9eb3d:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A02f6288aa3fc9eb3d:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.02034578751772642}} - ,{ - "core_node19":{ - "core":"Q2Q_postsandsubs_shard9_replica_n16", - "shard":"shard9", - "collection":"Q2Q_postsandsubs", - "node_name":"NODE_S_A02f6288aa3fc9eb3d:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A02f6288aa3fc9eb3d:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.01951229851692915}}]}, - { - "collection":"Q2Q_postsandsubs", - "node":"NODE_S_A061d3ac3c71d42de2:8983_solr", - "tagKey":"NODE_S_A061d3ac3c71d42de2:8983_solr", - "violation":{ - "replica":{ - "NRT":2, - "count":2}, - "delta":1.0}, - "clause":{ - "replica":"#EQUAL", - "node":"#ANY", - "collection":"Q2Q_postsandsubs"}, - "violatingReplicas":[{ - "core_node35":{ - "core":"Q2Q_postsandsubs_shard17_replica_n32", - "shard":"shard17", - "collection":"Q2Q_postsandsubs", - "node_name":"NODE_S_A061d3ac3c71d42de2:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A061d3ac3c71d42de2:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.020032744854688644}} - ,{ - "core_node41":{ - "core":"Q2Q_postsandsubs_shard20_replica_n38", - "shard":"shard20", - "collection":"Q2Q_postsandsubs", - "node_name":"NODE_S_A061d3ac3c71d42de2:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A061d3ac3c71d42de2:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.02001035585999489}}]}, - { - "collection":"Q2Q_postsandsubs", - "node":"NODE_S_A084aef0a141a43572:8983_solr", - "tagKey":"NODE_S_A084aef0a141a43572:8983_solr", - "violation":{ - "replica":{ - "NRT":2, - "count":2}, - "delta":1.0}, - "clause":{ - "replica":"#EQUAL", - "node":"#ANY", - "collection":"Q2Q_postsandsubs"}, - "violatingReplicas":[{ - "core_node33":{ - "core":"Q2Q_postsandsubs_shard16_replica_n30", - "shard":"shard16", - "collection":"Q2Q_postsandsubs", - "node_name":"NODE_S_A084aef0a141a43572:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A084aef0a141a43572:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.0185230216011405}} - ,{ - "core_node42":{ - "core":"Q2Q_postsandsubs_shard21_replica_n40", - "shard":"shard21", - "collection":"Q2Q_postsandsubs", - "node_name":"NODE_S_A084aef0a141a43572:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A084aef0a141a43572:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.019887510687112808}}]}, - { - "collection":"Q2Q_postsandsubs", - "node":"NODE_S_A006b71d81aedd8577:8983_solr", - "tagKey":"NODE_S_A006b71d81aedd8577:8983_solr", - "violation":{ - "replica":{ - "NRT":2, - "count":2}, - "delta":1.0}, - "clause":{ - "replica":"#EQUAL", - "node":"#ANY", - "collection":"Q2Q_postsandsubs"}, - "violatingReplicas":[{ - "core_node13":{ - "core":"Q2Q_postsandsubs_shard6_replica_n10", - "shard":"shard6", - "collection":"Q2Q_postsandsubs", - "node_name":"NODE_S_A006b71d81aedd8577:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A006b71d81aedd8577:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.01943675335496664}} - ,{ - "core_node15":{ - "core":"Q2Q_postsandsubs_shard7_replica_n12", - "shard":"shard7", - "collection":"Q2Q_postsandsubs", - "node_name":"NODE_S_A006b71d81aedd8577:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A006b71d81aedd8577:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.020343396812677383}}]}, - { - "collection":"Q2Q_postsandsubs", - "node":"NODE_S_A014292d062d5088da:8983_solr", - "tagKey":"NODE_S_A014292d062d5088da:8983_solr", - "violation":{ - "replica":{ - "NRT":2, - "count":2}, - "delta":1.0}, - "clause":{ - "replica":"#EQUAL", - "node":"#ANY", - "collection":"Q2Q_postsandsubs"}, - "violatingReplicas":[{ - "core_node11":{ - "core":"Q2Q_postsandsubs_shard5_replica_n8", - "shard":"shard5", - "collection":"Q2Q_postsandsubs", - "node_name":"NODE_S_A014292d062d5088da:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A014292d062d5088da:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.019260264933109283}} - ,{ - "core_node17":{ - "core":"Q2Q_postsandsubs_shard8_replica_n14", - "shard":"shard8", - "collection":"Q2Q_postsandsubs", - "node_name":"NODE_S_A014292d062d5088da:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A014292d062d5088da:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.01941517647355795}}]}, - { - "collection":"Q2Q_postsandsubs", - "node":"NODE_S_A06032b38152c0f019:8983_solr", - "tagKey":"NODE_S_A06032b38152c0f019:8983_solr", - "violation":{ - "replica":{ - "NRT":2, - "count":2}, - "delta":1.0}, - "clause":{ - "replica":"#EQUAL", - "node":"#ANY", - "collection":"Q2Q_postsandsubs"}, - "violatingReplicas":[{ - "core_node7":{ - "core":"Q2Q_postsandsubs_shard3_replica_n4", - "shard":"shard3", - "collection":"Q2Q_postsandsubs", - "node_name":"NODE_S_A06032b38152c0f019:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A06032b38152c0f019:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.020358268171548843}} - ,{ - "core_node21":{ - "core":"Q2Q_postsandsubs_shard10_replica_n18", - "shard":"shard10", - "collection":"Q2Q_postsandsubs", - "node_name":"NODE_S_A06032b38152c0f019:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A06032b38152c0f019:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.02053267415612936}}]}, - { - "collection":"Q2Q_postsandsubs", - "node":"NODE_S_A04a75744569a27ccf:8983_solr", - "tagKey":"NODE_S_A04a75744569a27ccf:8983_solr", - "violation":{ - "replica":{ - "NRT":2, - "count":2}, - "delta":1.0}, - "clause":{ - "replica":"#EQUAL", - "node":"#ANY", - "collection":"Q2Q_postsandsubs"}, - "violatingReplicas":[{ - "core_node37":{ - "core":"Q2Q_postsandsubs_shard18_replica_n34", - "shard":"shard18", - "collection":"Q2Q_postsandsubs", - "node_name":"NODE_S_A04a75744569a27ccf:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A04a75744569a27ccf:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.019906101748347282}} - ,{ - "core_node39":{ - "core":"Q2Q_postsandsubs_shard19_replica_n36", - "shard":"shard19", - "collection":"Q2Q_postsandsubs", - "node_name":"NODE_S_A04a75744569a27ccf:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A04a75744569a27ccf:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.019286231137812138}}]}, - { - "collection":"COLL_S_E_15", - "node":"NODE_S_A0a3cb04c83b693d80:8983_solr", - "tagKey":"NODE_S_A0a3cb04c83b693d80:8983_solr", - "violation":{ - "replica":{ - "NRT":4, - "count":4}, - "delta":2.0}, - "clause":{ - "replica":"#EQUAL", - "node":"#ANY", - "collection":"COLL_S_E_15"}, - "violatingReplicas":[{ - "core_node53":{ - "core":"COLL_S_E_15_shard13_replica_n50", - "shard":"shard13", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A0a3cb04c83b693d80:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0a3cb04c83b693d80:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}} - ,{ - "core_node17":{ - "core":"COLL_S_E_15_shard4_replica_n14", - "shard":"shard4", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A0a3cb04c83b693d80:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0a3cb04c83b693d80:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}} - ,{ - "core_node71":{ - "core":"COLL_S_E_15_shard18_replica_n68", - "shard":"shard18", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A0a3cb04c83b693d80:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0a3cb04c83b693d80:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}} - ,{ - "core_node35":{ - "core":"COLL_S_E_15_shard9_replica_n32", - "shard":"shard9", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A0a3cb04c83b693d80:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0a3cb04c83b693d80:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}}]}, - { - "collection":"COLL_S_E_15", - "node":"NODE_S_A0d1445647aaae8dd1:8983_solr", - "tagKey":"NODE_S_A0d1445647aaae8dd1:8983_solr", - "violation":{ - "replica":{ - "NRT":4, - "count":4}, - "delta":2.0}, - "clause":{ - "replica":"#EQUAL", - "node":"#ANY", - "collection":"COLL_S_E_15"}, - "violatingReplicas":[{ - "core_node57":{ - "core":"COLL_S_E_15_shard14_replica_n54", - "shard":"shard14", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A0d1445647aaae8dd1:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A0d1445647aaae8dd1:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}} - ,{ - "core_node21":{ - "core":"COLL_S_E_15_shard5_replica_n18", - "shard":"shard5", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A0d1445647aaae8dd1:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A0d1445647aaae8dd1:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}} - ,{ - "core_node3":{ - "core":"COLL_S_E_15_shard1_replica_n1", - "shard":"shard1", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A0d1445647aaae8dd1:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0d1445647aaae8dd1:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}} - ,{ - "core_node39":{ - "core":"COLL_S_E_15_shard10_replica_n36", - "shard":"shard10", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A0d1445647aaae8dd1:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0d1445647aaae8dd1:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}}]}, - { - "collection":"COLL_S_E_15", - "node":"NODE_S_A0ee1c34fb3e9c261d:8983_solr", - "tagKey":"NODE_S_A0ee1c34fb3e9c261d:8983_solr", - "violation":{ - "replica":{ - "NRT":4, - "count":4}, - "delta":2.0}, - "clause":{ - "replica":"#EQUAL", - "node":"#ANY", - "collection":"COLL_S_E_15"}, - "violatingReplicas":[{ - "core_node7":{ - "core":"COLL_S_E_15_shard2_replica_n4", - "shard":"shard2", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A0ee1c34fb3e9c261d:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A0ee1c34fb3e9c261d:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}} - ,{ - "core_node61":{ - "core":"COLL_S_E_15_shard15_replica_n58", - "shard":"shard15", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A0ee1c34fb3e9c261d:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0ee1c34fb3e9c261d:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}} - ,{ - "core_node25":{ - "core":"COLL_S_E_15_shard6_replica_n22", - "shard":"shard6", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A0ee1c34fb3e9c261d:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0ee1c34fb3e9c261d:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}} - ,{ - "core_node43":{ - "core":"COLL_S_E_15_shard11_replica_n40", - "shard":"shard11", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A0ee1c34fb3e9c261d:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A0ee1c34fb3e9c261d:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}}]}, - { - "collection":"COLL_S_E_15", - "node":"NODE_S_A0dc21b7f9ef048b48:8983_solr", - "tagKey":"NODE_S_A0dc21b7f9ef048b48:8983_solr", - "violation":{ - "replica":{ - "NRT":4, - "count":4}, - "delta":2.0}, - "clause":{ - "replica":"#EQUAL", - "node":"#ANY", - "collection":"COLL_S_E_15"}, - "violatingReplicas":[{ - "core_node13":{ - "core":"COLL_S_E_15_shard3_replica_n10", - "shard":"shard3", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A0dc21b7f9ef048b48:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0dc21b7f9ef048b48:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}} - ,{ - "core_node67":{ - "core":"COLL_S_E_15_shard17_replica_n64", - "shard":"shard17", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A0dc21b7f9ef048b48:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0dc21b7f9ef048b48:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}} - ,{ - "core_node31":{ - "core":"COLL_S_E_15_shard8_replica_n28", - "shard":"shard8", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A0dc21b7f9ef048b48:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0dc21b7f9ef048b48:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}} - ,{ - "core_node49":{ - "core":"COLL_S_E_15_shard12_replica_n46", - "shard":"shard12", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A0dc21b7f9ef048b48:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0dc21b7f9ef048b48:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}}]}, - { - "collection":"COLL_S_E_15", - "node":"NODE_S_A0ccef54efe3178758:8983_solr", - "tagKey":"NODE_S_A0ccef54efe3178758:8983_solr", - "violation":{ - "replica":{ - "NRT":4, - "count":4}, - "delta":2.0}, - "clause":{ - "replica":"#EQUAL", - "node":"#ANY", - "collection":"COLL_S_E_15"}, - "violatingReplicas":[{ - "core_node9":{ - "core":"COLL_S_E_15_shard2_replica_n6", - "shard":"shard2", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A0ccef54efe3178758:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0ccef54efe3178758:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}} - ,{ - "core_node63":{ - "core":"COLL_S_E_15_shard16_replica_n60", - "shard":"shard16", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A0ccef54efe3178758:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0ccef54efe3178758:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}} - ,{ - "core_node27":{ - "core":"COLL_S_E_15_shard7_replica_n24", - "shard":"shard7", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A0ccef54efe3178758:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0ccef54efe3178758:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}} - ,{ - "core_node45":{ - "core":"COLL_S_E_15_shard11_replica_n42", - "shard":"shard11", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A0ccef54efe3178758:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0ccef54efe3178758:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}}]}, - { - "collection":"COLL_S_E_15", - "node":"NODE_S_A077430d186f0e1de2:8983_solr", - "tagKey":"NODE_S_A077430d186f0e1de2:8983_solr", - "violation":{ - "replica":{ - "NRT":4, - "count":4}, - "delta":2.0}, - "clause":{ - "replica":"#EQUAL", - "node":"#ANY", - "collection":"COLL_S_E_15"}, - "violatingReplicas":[{ - "core_node51":{ - "core":"COLL_S_E_15_shard13_replica_n48", - "shard":"shard13", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A077430d186f0e1de2:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A077430d186f0e1de2:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}} - ,{ - "core_node15":{ - "core":"COLL_S_E_15_shard4_replica_n12", - "shard":"shard4", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A077430d186f0e1de2:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A077430d186f0e1de2:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}} - ,{ - "core_node69":{ - "core":"COLL_S_E_15_shard17_replica_n66", - "shard":"shard17", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A077430d186f0e1de2:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A077430d186f0e1de2:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}} - ,{ - "core_node33":{ - "core":"COLL_S_E_15_shard8_replica_n30", - "shard":"shard8", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A077430d186f0e1de2:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A077430d186f0e1de2:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}}]}, - { - "collection":"COLL_S_E_15", - "node":"NODE_S_A0afcb9535d2619dac:8983_solr", - "tagKey":"NODE_S_A0afcb9535d2619dac:8983_solr", - "violation":{ - "replica":{ - "NRT":4, - "count":4}, - "delta":2.0}, - "clause":{ - "replica":"#EQUAL", - "node":"#ANY", - "collection":"COLL_S_E_15"}, - "violatingReplicas":[{ - "core_node59":{ - "core":"COLL_S_E_15_shard15_replica_n56", - "shard":"shard15", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A0afcb9535d2619dac:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A0afcb9535d2619dac:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}} - ,{ - "core_node5":{ - "core":"COLL_S_E_15_shard1_replica_n2", - "shard":"shard1", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A0afcb9535d2619dac:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A0afcb9535d2619dac:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}} - ,{ - "core_node23":{ - "core":"COLL_S_E_15_shard6_replica_n20", - "shard":"shard6", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A0afcb9535d2619dac:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A0afcb9535d2619dac:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}} - ,{ - "core_node41":{ - "core":"COLL_S_E_15_shard10_replica_n38", - "shard":"shard10", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A0afcb9535d2619dac:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A0afcb9535d2619dac:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}}]}, - { - "collection":"COLL_S_E_15", - "node":"NODE_S_A09aede91f0e8c147c:8983_solr", - "tagKey":"NODE_S_A09aede91f0e8c147c:8983_solr", - "violation":{ - "replica":{ - "NRT":4, - "count":4}, - "delta":2.0}, - "clause":{ - "replica":"#EQUAL", - "node":"#ANY", - "collection":"COLL_S_E_15"}, - "violatingReplicas":[{ - "core_node55":{ - "core":"COLL_S_E_15_shard14_replica_n52", - "shard":"shard14", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A09aede91f0e8c147c:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A09aede91f0e8c147c:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}} - ,{ - "core_node19":{ - "core":"COLL_S_E_15_shard5_replica_n16", - "shard":"shard5", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A09aede91f0e8c147c:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A09aede91f0e8c147c:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}} - ,{ - "core_node72":{ - "core":"COLL_S_E_15_shard18_replica_n70", - "shard":"shard18", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A09aede91f0e8c147c:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A09aede91f0e8c147c:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}} - ,{ - "core_node37":{ - "core":"COLL_S_E_15_shard9_replica_n34", - "shard":"shard9", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A09aede91f0e8c147c:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A09aede91f0e8c147c:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}}]}, - { - "collection":"COLL_S_E_15", - "node":"NODE_S_A024a9c138d972d437:8983_solr", - "tagKey":"NODE_S_A024a9c138d972d437:8983_solr", - "violation":{ - "replica":{ - "NRT":4, - "count":4}, - "delta":2.0}, - "clause":{ - "replica":"#EQUAL", - "node":"#ANY", - "collection":"COLL_S_E_15"}, - "violatingReplicas":[{ - "core_node11":{ - "core":"COLL_S_E_15_shard3_replica_n8", - "shard":"shard3", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A024a9c138d972d437:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A024a9c138d972d437:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}} - ,{ - "core_node65":{ - "core":"COLL_S_E_15_shard16_replica_n62", - "shard":"shard16", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A024a9c138d972d437:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A024a9c138d972d437:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}} - ,{ - "core_node29":{ - "core":"COLL_S_E_15_shard7_replica_n26", - "shard":"shard7", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A024a9c138d972d437:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A024a9c138d972d437:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}} - ,{ - "core_node47":{ - "core":"COLL_S_E_15_shard12_replica_n44", - "shard":"shard12", - "collection":"COLL_S_E_15", - "node_name":"NODE_S_A024a9c138d972d437:8983_solr", - "type":"NRT", - "base_url":"http://NODE_S_A024a9c138d972d437:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":1.257285475730896E-7}}]}, - { - "collection":"COLL_S_E_15", - "node":"NODE_S_A0ebcce85ef9c13308:8983_solr", - "tagKey":"NODE_S_A0ebcce85ef9c13308:8983_solr", - "violation":{ - "replica":{ - "count":0}, - "delta":-1.0}, - "clause":{ - "replica":"#EQUAL", - "node":"#ANY", - "collection":"COLL_S_E_15"}}, - { - "collection":"COLL_S_E_15", - "node":"NODE_S_A0f66f4bec1ce8fc09:8983_solr", - "tagKey":"NODE_S_A0f66f4bec1ce8fc09:8983_solr", - "violation":{ - "replica":{ - "count":0}, - "delta":-1.0}, - "clause":{ - "replica":"#EQUAL", - "node":"#ANY", - "collection":"COLL_S_E_15"}}, - { - "collection":"COLL_S_E_15", - "node":"NODE_S_A02f6288aa3fc9eb3d:8983_solr", - "tagKey":"NODE_S_A02f6288aa3fc9eb3d:8983_solr", - "violation":{ - "replica":{ - "count":0}, - "delta":-1.0}, - "clause":{ - "replica":"#EQUAL", - "node":"#ANY", - "collection":"COLL_S_E_15"}}, - { - "collection":"COLL_S_E_15", - "node":"NODE_S_A0cf36cc11a8447fac:8983_solr", - "tagKey":"NODE_S_A0cf36cc11a8447fac:8983_solr", - "violation":{ - "replica":{ - "count":0}, - "delta":-1.0}, - "clause":{ - "replica":"#EQUAL", - "node":"#ANY", - "collection":"COLL_S_E_15"}}, - { - "collection":"COLL_S_E_15", - "node":"NODE_S_A061d3ac3c71d42de2:8983_solr", - "tagKey":"NODE_S_A061d3ac3c71d42de2:8983_solr", - "violation":{ - "replica":{ - "count":0}, - "delta":-1.0}, - "clause":{ - "replica":"#EQUAL", - "node":"#ANY", - "collection":"COLL_S_E_15"}}, - { - "collection":"COLL_S_E_15", - "node":"NODE_S_A08c1ec5957d8a5d44:8983_solr", - "tagKey":"NODE_S_A08c1ec5957d8a5d44:8983_solr", - "violation":{ - "replica":{ - "count":0}, - "delta":-1.0}, - "clause":{ - "replica":"#EQUAL", - "node":"#ANY", - "collection":"COLL_S_E_15"}}, - { - "collection":"COLL_S_E_15", - "node":"NODE_S_A084aef0a141a43572:8983_solr", - "tagKey":"NODE_S_A084aef0a141a43572:8983_solr", - "violation":{ - "replica":{ - "count":0}, - "delta":-1.0}, - "clause":{ - "replica":"#EQUAL", - "node":"#ANY", - "collection":"COLL_S_E_15"}}, - { - "collection":"COLL_S_E_15", - "node":"NODE_S_A0f096665ad4a3bc58:8983_solr", - "tagKey":"NODE_S_A0f096665ad4a3bc58:8983_solr", - "violation":{ - "replica":{ - "count":0}, - "delta":-1.0}, - "clause":{ - "replica":"#EQUAL", - "node":"#ANY", - "collection":"COLL_S_E_15"}}, - { - "collection":"COLL_S_E_15", - "node":"NODE_S_A006b71d81aedd8577:8983_solr", - "tagKey":"NODE_S_A006b71d81aedd8577:8983_solr", - "violation":{ - "replica":{ - "count":0}, - "delta":-1.0}, - "clause":{ - "replica":"#EQUAL", - "node":"#ANY", - "collection":"COLL_S_E_15"}}, - { - "collection":"COLL_S_E_15", - "node":"NODE_S_A014292d062d5088da:8983_solr", - "tagKey":"NODE_S_A014292d062d5088da:8983_solr", - "violation":{ - "replica":{ - "count":0}, - "delta":-1.0}, - "clause":{ - "replica":"#EQUAL", - "node":"#ANY", - "collection":"COLL_S_E_15"}}, - { - "collection":"COLL_S_E_15", - "node":"NODE_S_A06032b38152c0f019:8983_solr", - "tagKey":"NODE_S_A06032b38152c0f019:8983_solr", - "violation":{ - "replica":{ - "count":0}, - "delta":-1.0}, - "clause":{ - "replica":"#EQUAL", - "node":"#ANY", - "collection":"COLL_S_E_15"}}, - { - "collection":"COLL_S_E_15", - "node":"NODE_S_A04a75744569a27ccf:8983_solr", - "tagKey":"NODE_S_A04a75744569a27ccf:8983_solr", - "violation":{ - "replica":{ - "count":0}, - "delta":-1.0}, - "clause":{ - "replica":"#EQUAL", - "node":"#ANY", - "collection":"COLL_S_E_15"}}, - { - "collection":"COLL_S_E_15", - "node":"NODE_S_A0e48ab035b869305e:8983_solr", - "tagKey":"NODE_S_A0e48ab035b869305e:8983_solr", - "violation":{ - "replica":{ - "count":0}, - "delta":-1.0}, - "clause":{ - "replica":"#EQUAL", - "node":"#ANY", - "collection":"COLL_S_E_15"}}, - { - "collection":"COLL_S_E_15", - "node":"NODE_S_A0d1f4439b8e8878e8:8983_solr", - "tagKey":"NODE_S_A0d1f4439b8e8878e8:8983_solr", - "violation":{ - "replica":{ - "count":0}, - "delta":-1.0}, - "clause":{ - "replica":"#EQUAL", - "node":"#ANY", - "collection":"COLL_S_E_15"}}, - { - "collection":"COLL_S_E_15", - "node":"NODE_S_A0b7fb52d8d0a2ecde:8983_solr", - "tagKey":"NODE_S_A0b7fb52d8d0a2ecde:8983_solr", - "violation":{ - "replica":{ - "count":0}, - "delta":-1.0}, - "clause":{ - "replica":"#EQUAL", - "node":"#ANY", - "collection":"COLL_S_E_15"}}, - { - "collection":"COLL_S_E_15", - "node":"NODE_S_A0d391cf8eeaca59cd:8983_solr", - "tagKey":"NODE_S_A0d391cf8eeaca59cd:8983_solr", - "violation":{ - "replica":{ - "count":0}, - "delta":-1.0}, - "clause":{ - "replica":"#EQUAL", - "node":"#ANY", - "collection":"COLL_S_E_15"}}, - { - "collection":"COLL_S_E_15", - "node":"NODE_S_A0aaaf34e6b281b24c:8983_solr", - "tagKey":"NODE_S_A0aaaf34e6b281b24c:8983_solr", - "violation":{ - "replica":{ - "count":0}, - "delta":-1.0}, - "clause":{ - "replica":"#EQUAL", - "node":"#ANY", - "collection":"COLL_S_E_15"}}, - { - "collection":"COLL_S_E_15", - "node":"NODE_S_A0bdba3561ff291bc4:8983_solr", - "tagKey":"NODE_S_A0bdba3561ff291bc4:8983_solr", - "violation":{ - "replica":{ - "count":0}, - "delta":-1.0}, - "clause":{ - "replica":"#EQUAL", - "node":"#ANY", - "collection":"COLL_S_E_15"}}, - { - "collection":"COLL_S_E_15", - "node":"NODE_S_A086a3a5ff0617c80f:8983_solr", - "tagKey":"NODE_S_A086a3a5ff0617c80f:8983_solr", - "violation":{ - "replica":{ - "count":0}, - "delta":-1.0}, - "clause":{ - "replica":"#EQUAL", - "node":"#ANY", - "collection":"COLL_S_E_15"}}, - { - "collection":"COLL_S_E_15", - "node":"NODE_S_A09f108fb78272a03d:8983_solr", - "tagKey":"NODE_S_A09f108fb78272a03d:8983_solr", - "violation":{ - "replica":{ - "count":0}, - "delta":-1.0}, - "clause":{ - "replica":"#EQUAL", - "node":"#ANY", - "collection":"COLL_S_E_15"}}, - { - "collection":"COLL_S_E_15", - "node":"NODE_S_A013eaa884d3c59fab:8983_solr", - "tagKey":"NODE_S_A013eaa884d3c59fab:8983_solr", - "violation":{ - "replica":{ - "count":0}, - "delta":-1.0}, - "clause":{ - "replica":"#EQUAL", - "node":"#ANY", - "collection":"COLL_S_E_15"}}, - { - "collection":"COLL_S_E_15", - "node":"NODE_S_A03baeedfd28ba05ba:8983_solr", - "tagKey":"NODE_S_A03baeedfd28ba05ba:8983_solr", - "violation":{ - "replica":{ - "count":0}, - "delta":-1.0}, - "clause":{ - "replica":"#EQUAL", - "node":"#ANY", - "collection":"COLL_S_E_15"}}, - { - "collection":"COLL_S_E_15", - "node":"NODE_S_A082e313208af7da04:8983_solr", - "tagKey":"NODE_S_A082e313208af7da04:8983_solr", - "violation":{ - "replica":{ - "count":0}, - "delta":-1.0}, - "clause":{ - "replica":"#EQUAL", - "node":"#ANY", - "collection":"COLL_S_E_15"}}, - { - "collection":"COLL_S_E_15", - "node":"NODE_S_A0a0ffe317a4f85c55:8983_solr", - "tagKey":"NODE_S_A0a0ffe317a4f85c55:8983_solr", - "violation":{ - "replica":{ - "count":0}, - "delta":-1.0}, - "clause":{ - "replica":"#EQUAL", - "node":"#ANY", - "collection":"COLL_S_E_15"}}, - { - "collection":"subreddit_experiment_results", - "node":"NODE_S_A0afcb9535d2619dac:8983_solr", - "tagKey":"NODE_S_A0afcb9535d2619dac:8983_solr", - "violation":{ - "replica":{ - "NRT":2, - "count":2}, - "delta":1.0}, - "clause":{ - "replica":"#EQUAL", - "node":"#ANY", - "collection":"subreddit_experiment_results"}, - "violatingReplicas":[{ - "core_node37":{ - "core":"subreddit_experiment_results_shard18_replica_n34", - "shard":"shard18", - "collection":"subreddit_experiment_results", - "node_name":"NODE_S_A0afcb9535d2619dac:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0afcb9535d2619dac:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.0045873550698161125}} - ,{ - "core_node15":{ - "core":"subreddit_experiment_results_shard7_replica_n12", - "shard":"shard7", - "collection":"subreddit_experiment_results", - "node_name":"NODE_S_A0afcb9535d2619dac:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0afcb9535d2619dac:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.004490339197218418}}]}, - { - "collection":"subreddit_experiment_results", - "node":"NODE_S_A0ebcce85ef9c13308:8983_solr", - "tagKey":"NODE_S_A0ebcce85ef9c13308:8983_solr", - "violation":{ - "replica":{ - "NRT":2, - "count":2}, - "delta":1.0}, - "clause":{ - "replica":"#EQUAL", - "node":"#ANY", - "collection":"subreddit_experiment_results"}, - "violatingReplicas":[{ - "core_node27":{ - "core":"subreddit_experiment_results_shard13_replica_n24", - "shard":"shard13", - "collection":"subreddit_experiment_results", - "node_name":"NODE_S_A0ebcce85ef9c13308:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0ebcce85ef9c13308:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.004564180038869381}} - ,{ - "core_node25":{ - "core":"subreddit_experiment_results_shard12_replica_n22", - "shard":"shard12", - "collection":"subreddit_experiment_results", - "node_name":"NODE_S_A0ebcce85ef9c13308:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0ebcce85ef9c13308:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.004466232843697071}}]}, - { - "collection":"subreddit_experiment_results", - "node":"NODE_S_A0f66f4bec1ce8fc09:8983_solr", - "tagKey":"NODE_S_A0f66f4bec1ce8fc09:8983_solr", - "violation":{ - "replica":{ - "NRT":2, - "count":2}, - "delta":1.0}, - "clause":{ - "replica":"#EQUAL", - "node":"#ANY", - "collection":"subreddit_experiment_results"}, - "violatingReplicas":[{ - "core_node35":{ - "core":"subreddit_experiment_results_shard17_replica_n32", - "shard":"shard17", - "collection":"subreddit_experiment_results", - "node_name":"NODE_S_A0f66f4bec1ce8fc09:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0f66f4bec1ce8fc09:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.0044752322137355804}} - ,{ - "core_node17":{ - "core":"subreddit_experiment_results_shard8_replica_n14", - "shard":"shard8", - "collection":"subreddit_experiment_results", - "node_name":"NODE_S_A0f66f4bec1ce8fc09:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0f66f4bec1ce8fc09:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.004475797526538372}}]}, - { - "collection":"subreddit_experiment_results", - "node":"NODE_S_A0cf36cc11a8447fac:8983_solr", - "tagKey":"NODE_S_A0cf36cc11a8447fac:8983_solr", - "violation":{ - "replica":{ - "NRT":2, - "count":2}, - "delta":1.0}, - "clause":{ - "replica":"#EQUAL", - "node":"#ANY", - "collection":"subreddit_experiment_results"}, - "violatingReplicas":[{ - "core_node9":{ - "core":"subreddit_experiment_results_shard4_replica_n6", - "shard":"shard4", - "collection":"subreddit_experiment_results", - "node_name":"NODE_S_A0cf36cc11a8447fac:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0cf36cc11a8447fac:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.004524897783994675}} - ,{ - "core_node43":{ - "core":"subreddit_experiment_results_shard21_replica_n40", - "shard":"shard21", - "collection":"subreddit_experiment_results", - "node_name":"NODE_S_A0cf36cc11a8447fac:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0cf36cc11a8447fac:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.004553055390715599}}]}, - { - "collection":"subreddit_experiment_results", - "node":"NODE_S_A08c1ec5957d8a5d44:8983_solr", - "tagKey":"NODE_S_A08c1ec5957d8a5d44:8983_solr", - "violation":{ - "replica":{ - "NRT":2, - "count":2}, - "delta":1.0}, - "clause":{ - "replica":"#EQUAL", - "node":"#ANY", - "collection":"subreddit_experiment_results"}, - "violatingReplicas":[{ - "core_node7":{ - "core":"subreddit_experiment_results_shard3_replica_n4", - "shard":"shard3", - "collection":"subreddit_experiment_results", - "node_name":"NODE_S_A08c1ec5957d8a5d44:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A08c1ec5957d8a5d44:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.004642372019588947}} - ,{ - "core_node45":{ - "core":"subreddit_experiment_results_shard22_replica_n42", - "shard":"shard22", - "collection":"subreddit_experiment_results", - "node_name":"NODE_S_A08c1ec5957d8a5d44:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A08c1ec5957d8a5d44:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.004599213600158691}}]}, - { - "collection":"subreddit_experiment_results", - "node":"NODE_S_A084aef0a141a43572:8983_solr", - "tagKey":"NODE_S_A084aef0a141a43572:8983_solr", - "violation":{ - "replica":{ - "NRT":2, - "count":2}, - "delta":1.0}, - "clause":{ - "replica":"#EQUAL", - "node":"#ANY", - "collection":"subreddit_experiment_results"}, - "violatingReplicas":[{ - "core_node31":{ - "core":"subreddit_experiment_results_shard15_replica_n28", - "shard":"shard15", - "collection":"subreddit_experiment_results", - "node_name":"NODE_S_A084aef0a141a43572:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A084aef0a141a43572:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.004504849202930927}} - ,{ - "core_node21":{ - "core":"subreddit_experiment_results_shard10_replica_n18", - "shard":"shard10", - "collection":"subreddit_experiment_results", - "node_name":"NODE_S_A084aef0a141a43572:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A084aef0a141a43572:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.00449906662106514}}]}, - { - "collection":"subreddit_experiment_results", - "node":"NODE_S_A0f096665ad4a3bc58:8983_solr", - "tagKey":"NODE_S_A0f096665ad4a3bc58:8983_solr", - "violation":{ - "replica":{ - "NRT":2, - "count":2}, - "delta":1.0}, - "clause":{ - "replica":"#EQUAL", - "node":"#ANY", - "collection":"subreddit_experiment_results"}, - "violatingReplicas":[{ - "core_node49":{ - "core":"subreddit_experiment_results_shard24_replica_n46", - "shard":"shard24", - "collection":"subreddit_experiment_results", - "node_name":"NODE_S_A0f096665ad4a3bc58:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0f096665ad4a3bc58:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.004560600966215134}} - ,{ - "core_node3":{ - "core":"subreddit_experiment_results_shard1_replica_n1", - "shard":"shard1", - "collection":"subreddit_experiment_results", - "node_name":"NODE_S_A0f096665ad4a3bc58:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0f096665ad4a3bc58:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.0045231664553284645}}]}, - { - "collection":"subreddit_experiment_results", - "node":"NODE_S_A0e48ab035b869305e:8983_solr", - "tagKey":"NODE_S_A0e48ab035b869305e:8983_solr", - "violation":{ - "replica":{ - "NRT":2, - "count":2}, - "delta":1.0}, - "clause":{ - "replica":"#EQUAL", - "node":"#ANY", - "collection":"subreddit_experiment_results"}, - "violatingReplicas":[{ - "core_node39":{ - "core":"subreddit_experiment_results_shard19_replica_n36", - "shard":"shard19", - "collection":"subreddit_experiment_results", - "node_name":"NODE_S_A0e48ab035b869305e:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0e48ab035b869305e:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.00449786800891161}} - ,{ - "core_node13":{ - "core":"subreddit_experiment_results_shard6_replica_n10", - "shard":"shard6", - "collection":"subreddit_experiment_results", - "node_name":"NODE_S_A0e48ab035b869305e:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0e48ab035b869305e:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.004472509957849979}}]}, - { - "collection":"subreddit_experiment_results", - "node":"NODE_S_A0d1f4439b8e8878e8:8983_solr", - "tagKey":"NODE_S_A0d1f4439b8e8878e8:8983_solr", - "violation":{ - "replica":{ - "NRT":2, - "count":2}, - "delta":1.0}, - "clause":{ - "replica":"#EQUAL", - "node":"#ANY", - "collection":"subreddit_experiment_results"}, - "violatingReplicas":[{ - "core_node11":{ - "core":"subreddit_experiment_results_shard5_replica_n8", - "shard":"shard5", - "collection":"subreddit_experiment_results", - "node_name":"NODE_S_A0d1f4439b8e8878e8:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0d1f4439b8e8878e8:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.004589034244418144}} - ,{ - "core_node41":{ - "core":"subreddit_experiment_results_shard20_replica_n38", - "shard":"shard20", - "collection":"subreddit_experiment_results", - "node_name":"NODE_S_A0d1f4439b8e8878e8:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0d1f4439b8e8878e8:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.004457270726561546}}]}, - { - "collection":"subreddit_experiment_results", - "node":"NODE_S_A0b7fb52d8d0a2ecde:8983_solr", - "tagKey":"NODE_S_A0b7fb52d8d0a2ecde:8983_solr", - "violation":{ - "replica":{ - "NRT":2, - "count":2}, - "delta":1.0}, - "clause":{ - "replica":"#EQUAL", - "node":"#ANY", - "collection":"subreddit_experiment_results"}, - "violatingReplicas":[{ - "core_node33":{ - "core":"subreddit_experiment_results_shard16_replica_n30", - "shard":"shard16", - "collection":"subreddit_experiment_results", - "node_name":"NODE_S_A0b7fb52d8d0a2ecde:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0b7fb52d8d0a2ecde:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.004510008729994297}} - ,{ - "core_node19":{ - "core":"subreddit_experiment_results_shard9_replica_n16", - "shard":"shard9", - "collection":"subreddit_experiment_results", - "node_name":"NODE_S_A0b7fb52d8d0a2ecde:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0b7fb52d8d0a2ecde:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.0044971127063035965}}]}, - { - "collection":"subreddit_experiment_results", - "node":"NODE_S_A0d391cf8eeaca59cd:8983_solr", - "tagKey":"NODE_S_A0d391cf8eeaca59cd:8983_solr", - "violation":{ - "replica":{ - "NRT":2, - "count":2}, - "delta":1.0}, - "clause":{ - "replica":"#EQUAL", - "node":"#ANY", - "collection":"subreddit_experiment_results"}, - "violatingReplicas":[{ - "core_node5":{ - "core":"subreddit_experiment_results_shard2_replica_n2", - "shard":"shard2", - "collection":"subreddit_experiment_results", - "node_name":"NODE_S_A0d391cf8eeaca59cd:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0d391cf8eeaca59cd:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.004513397812843323}} - ,{ - "core_node47":{ - "core":"subreddit_experiment_results_shard23_replica_n44", - "shard":"shard23", - "collection":"subreddit_experiment_results", - "node_name":"NODE_S_A0d391cf8eeaca59cd:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0d391cf8eeaca59cd:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.0045642731711268425}}]}, - { - "collection":"subreddit_experiment_results", - "node":"NODE_S_A0aaaf34e6b281b24c:8983_solr", - "tagKey":"NODE_S_A0aaaf34e6b281b24c:8983_solr", - "violation":{ - "replica":{ - "NRT":2, - "count":2}, - "delta":1.0}, - "clause":{ - "replica":"#EQUAL", - "node":"#ANY", - "collection":"subreddit_experiment_results"}, - "violatingReplicas":[{ - "core_node29":{ - "core":"subreddit_experiment_results_shard14_replica_n26", - "shard":"shard14", - "collection":"subreddit_experiment_results", - "node_name":"NODE_S_A0aaaf34e6b281b24c:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0aaaf34e6b281b24c:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.004518838599324226}} - ,{ - "core_node23":{ - "core":"subreddit_experiment_results_shard11_replica_n20", - "shard":"shard11", - "collection":"subreddit_experiment_results", - "node_name":"NODE_S_A0aaaf34e6b281b24c:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0aaaf34e6b281b24c:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.0045999325811862946}}]}, - { - "collection":"COLL_N_Q_C_T_11", - "node":"NODE_S_A0afcb9535d2619dac:8983_solr", - "tagKey":"NODE_S_A0afcb9535d2619dac:8983_solr", - "violation":{ - "replica":{ - "NRT":2, - "count":2}, - "delta":1.0}, - "clause":{ - "replica":"#EQUAL", - "node":"#ANY", - "collection":"COLL_N_Q_C_T_11"}, - "violatingReplicas":[{ - "core_node8":{ - "core":"COLL_N_Q_C_T_11_shard2_replica_n2", - "shard":"shard2", - "collection":"COLL_N_Q_C_T_11", - "node_name":"NODE_S_A0afcb9535d2619dac:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0afcb9535d2619dac:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.006244504824280739}} - ,{ - "core_node3":{ - "core":"COLL_N_Q_C_T_11_shard1_replica_n1", - "shard":"shard1", - "collection":"COLL_N_Q_C_T_11", - "node_name":"NODE_S_A0afcb9535d2619dac:8983_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://NODE_S_A0afcb9535d2619dac:8983/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":0.006021473556756973}}]}], - "config":{ - "cluster-preferences":[ - {"maximize":"freedisk", "precision" : 30.0}, - {"minimize":"cores"} - ], - "cluster-policy":[ - {"replica":"<2", "shard":"#EACH", "node":"#ANY"} - ,{"replica":"#EQUAL", "node":"#ANY", "strict" : false} - - ]}}, - - "cluster":{ - "collections":{ - "query2query":{ - "pullReplicas":"0", - "replicationFactor":"1", - "shards":{ - "shard1":{ - "range":"80000000-8c2fffff", - "state":"active", - "replicas":{"core_node3":{ - "core":"query2query_shard1_replica_n1", - "base_url":"http://NODE_S_A0f096665ad4a3bc58:8983/solr", - "node_name":"NODE_S_A0f096665ad4a3bc58:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard2":{ - "range":"8c300000-9860ffff", - "state":"active", - "replicas":{"core_node5":{ - "core":"query2query_shard2_replica_n2", - "base_url":"http://NODE_S_A084aef0a141a43572:8983/solr", - "node_name":"NODE_S_A084aef0a141a43572:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard3":{ - "range":"98610000-a491ffff", - "state":"active", - "replicas":{"core_node7":{ - "core":"query2query_shard3_replica_n4", - "base_url":"http://NODE_S_A061d3ac3c71d42de2:8983/solr", - "node_name":"NODE_S_A061d3ac3c71d42de2:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard4":{ - "range":"a4920000-b0c2ffff", - "state":"active", - "replicas":{"core_node9":{ - "core":"query2query_shard4_replica_n6", - "base_url":"http://NODE_S_A08c1ec5957d8a5d44:8983/solr", - "node_name":"NODE_S_A08c1ec5957d8a5d44:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard5":{ - "range":"b0c30000-bcf2ffff", - "state":"active", - "replicas":{"core_node11":{ - "core":"query2query_shard5_replica_n8", - "base_url":"http://NODE_S_A0cf36cc11a8447fac:8983/solr", - "node_name":"NODE_S_A0cf36cc11a8447fac:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard6":{ - "range":"bcf30000-c923ffff", - "state":"active", - "replicas":{"core_node13":{ - "core":"query2query_shard6_replica_n10", - "base_url":"http://NODE_S_A06032b38152c0f019:8983/solr", - "node_name":"NODE_S_A06032b38152c0f019:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard7":{ - "range":"c9240000-d554ffff", - "state":"active", - "replicas":{"core_node15":{ - "core":"query2query_shard7_replica_n12", - "base_url":"http://NODE_S_A0f66f4bec1ce8fc09:8983/solr", - "node_name":"NODE_S_A0f66f4bec1ce8fc09:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard8":{ - "range":"d5550000-e185ffff", - "state":"active", - "replicas":{"core_node17":{ - "core":"query2query_shard8_replica_n14", - "base_url":"http://NODE_S_A0ebcce85ef9c13308:8983/solr", - "node_name":"NODE_S_A0ebcce85ef9c13308:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard9":{ - "range":"e1860000-edb5ffff", - "state":"active", - "replicas":{"core_node19":{ - "core":"query2query_shard9_replica_n16", - "base_url":"http://NODE_S_A02f6288aa3fc9eb3d:8983/solr", - "node_name":"NODE_S_A02f6288aa3fc9eb3d:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard10":{ - "range":"edb60000-f9e6ffff", - "state":"active", - "replicas":{"core_node21":{ - "core":"query2query_shard10_replica_n18", - "base_url":"http://NODE_S_A02f6288aa3fc9eb3d:8983/solr", - "node_name":"NODE_S_A02f6288aa3fc9eb3d:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard11":{ - "range":"f9e70000-617ffff", - "state":"active", - "replicas":{"core_node23":{ - "core":"query2query_shard11_replica_n20", - "base_url":"http://NODE_S_A0ebcce85ef9c13308:8983/solr", - "node_name":"NODE_S_A0ebcce85ef9c13308:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard12":{ - "range":"6180000-1248ffff", - "state":"active", - "replicas":{"core_node25":{ - "core":"query2query_shard12_replica_n22", - "base_url":"http://NODE_S_A0f66f4bec1ce8fc09:8983/solr", - "node_name":"NODE_S_A0f66f4bec1ce8fc09:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard13":{ - "range":"12490000-1e78ffff", - "state":"active", - "replicas":{"core_node27":{ - "core":"query2query_shard13_replica_n24", - "base_url":"http://NODE_S_A06032b38152c0f019:8983/solr", - "node_name":"NODE_S_A06032b38152c0f019:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard14":{ - "range":"1e790000-2aa9ffff", - "state":"active", - "replicas":{"core_node29":{ - "core":"query2query_shard14_replica_n26", - "base_url":"http://NODE_S_A0cf36cc11a8447fac:8983/solr", - "node_name":"NODE_S_A0cf36cc11a8447fac:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard15":{ - "range":"2aaa0000-36daffff", - "state":"active", - "replicas":{"core_node31":{ - "core":"query2query_shard15_replica_n28", - "base_url":"http://NODE_S_A08c1ec5957d8a5d44:8983/solr", - "node_name":"NODE_S_A08c1ec5957d8a5d44:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard16":{ - "range":"36db0000-430bffff", - "state":"active", - "replicas":{"core_node33":{ - "core":"query2query_shard16_replica_n30", - "base_url":"http://NODE_S_A061d3ac3c71d42de2:8983/solr", - "node_name":"NODE_S_A061d3ac3c71d42de2:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard17":{ - "range":"430c0000-4f3bffff", - "state":"active", - "replicas":{"core_node35":{ - "core":"query2query_shard17_replica_n32", - "base_url":"http://NODE_S_A084aef0a141a43572:8983/solr", - "node_name":"NODE_S_A084aef0a141a43572:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard18":{ - "range":"4f3c0000-5b6cffff", - "state":"active", - "replicas":{"core_node37":{ - "core":"query2query_shard18_replica_n34", - "base_url":"http://NODE_S_A0f096665ad4a3bc58:8983/solr", - "node_name":"NODE_S_A0f096665ad4a3bc58:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard19":{ - "range":"5b6d0000-679dffff", - "state":"active", - "replicas":{"core_node39":{ - "core":"query2query_shard19_replica_n36", - "base_url":"http://NODE_S_A04a75744569a27ccf:8983/solr", - "node_name":"NODE_S_A04a75744569a27ccf:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard20":{ - "range":"679e0000-73ceffff", - "state":"active", - "replicas":{"core_node41":{ - "core":"query2query_shard20_replica_n38", - "base_url":"http://NODE_S_A006b71d81aedd8577:8983/solr", - "node_name":"NODE_S_A006b71d81aedd8577:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard21":{ - "range":"73cf0000-7fffffff", - "state":"active", - "replicas":{"core_node42":{ - "core":"query2query_shard21_replica_n40", - "base_url":"http://NODE_S_A014292d062d5088da:8983/solr", - "node_name":"NODE_S_A014292d062d5088da:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}}, - "router":{"name":"compositeId"}, - "autoAddReplicas":"false", - "nrtReplicas":"1", - "tlogReplicas":"0", - "znodeVersion":69, - "configName":"query2query"}, - "COLL_S_R":{ - "pullReplicas":"0", - "replicationFactor":"1", - "shards":{ - "shard1":{ - "range":"80000000-87c0ffff", - "state":"active", - "replicas":{"core_node3":{ - "core":"COLL_S_R_shard1_replica_n1", - "base_url":"http://NODE_S_A082e313208af7da04:8983/solr", - "node_name":"NODE_S_A082e313208af7da04:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard2":{ - "range":"87c10000-8f82ffff", - "state":"active", - "replicas":{"core_node4":{ - "core":"COLL_S_R_shard2_replica_n2", - "base_url":"http://NODE_S_A09f108fb78272a03d:8983/solr", - "node_name":"NODE_S_A09f108fb78272a03d:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard3":{ - "range":"8f830000-9744ffff", - "state":"active", - "replicas":{"core_node7":{ - "core":"COLL_S_R_shard3_replica_n5", - "base_url":"http://NODE_S_A0a0ffe317a4f85c55:8983/solr", - "node_name":"NODE_S_A0a0ffe317a4f85c55:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard4":{ - "range":"97450000-9f06ffff", - "state":"active", - "replicas":{"core_node9":{ - "core":"COLL_S_R_shard4_replica_n6", - "base_url":"http://NODE_S_A03baeedfd28ba05ba:8983/solr", - "node_name":"NODE_S_A03baeedfd28ba05ba:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard5":{ - "range":"9f070000-a6c8ffff", - "state":"active", - "replicas":{"core_node11":{ - "core":"COLL_S_R_shard5_replica_n8", - "base_url":"http://NODE_S_A013eaa884d3c59fab:8983/solr", - "node_name":"NODE_S_A013eaa884d3c59fab:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard6":{ - "range":"a6c90000-ae8affff", - "state":"active", - "replicas":{"core_node13":{ - "core":"COLL_S_R_shard6_replica_n10", - "base_url":"http://NODE_S_A086a3a5ff0617c80f:8983/solr", - "node_name":"NODE_S_A086a3a5ff0617c80f:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard7":{ - "range":"ae8b0000-b64cffff", - "state":"active", - "replicas":{"core_node15":{ - "core":"COLL_S_R_shard7_replica_n12", - "base_url":"http://NODE_S_A0bdba3561ff291bc4:8983/solr", - "node_name":"NODE_S_A0bdba3561ff291bc4:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard8":{ - "range":"b64d0000-be0effff", - "state":"active", - "replicas":{"core_node17":{ - "core":"COLL_S_R_shard8_replica_n14", - "base_url":"http://NODE_S_A0aaaf34e6b281b24c:8983/solr", - "node_name":"NODE_S_A0aaaf34e6b281b24c:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard9":{ - "range":"be0f0000-c5d0ffff", - "state":"active", - "replicas":{"core_node19":{ - "core":"COLL_S_R_shard9_replica_n16", - "base_url":"http://NODE_S_A0d391cf8eeaca59cd:8983/solr", - "node_name":"NODE_S_A0d391cf8eeaca59cd:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard10":{ - "range":"c5d10000-cd92ffff", - "state":"active", - "replicas":{"core_node21":{ - "core":"COLL_S_R_shard10_replica_n18", - "base_url":"http://NODE_S_A0b7fb52d8d0a2ecde:8983/solr", - "node_name":"NODE_S_A0b7fb52d8d0a2ecde:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard11":{ - "range":"cd930000-d554ffff", - "state":"active", - "replicas":{"core_node23":{ - "core":"COLL_S_R_shard11_replica_n20", - "base_url":"http://NODE_S_A0d1f4439b8e8878e8:8983/solr", - "node_name":"NODE_S_A0d1f4439b8e8878e8:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard12":{ - "range":"d5550000-dd16ffff", - "state":"active", - "replicas":{"core_node25":{ - "core":"COLL_S_R_shard12_replica_n22", - "base_url":"http://NODE_S_A0e48ab035b869305e:8983/solr", - "node_name":"NODE_S_A0e48ab035b869305e:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard13":{ - "range":"dd170000-e4d8ffff", - "state":"active", - "replicas":{"core_node27":{ - "core":"COLL_S_R_shard13_replica_n24", - "base_url":"http://NODE_S_A014292d062d5088da:8983/solr", - "node_name":"NODE_S_A014292d062d5088da:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard14":{ - "range":"e4d90000-ec9affff", - "state":"active", - "replicas":{"core_node29":{ - "core":"COLL_S_R_shard14_replica_n26", - "base_url":"http://NODE_S_A006b71d81aedd8577:8983/solr", - "node_name":"NODE_S_A006b71d81aedd8577:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard15":{ - "range":"ec9b0000-f45cffff", - "state":"active", - "replicas":{"core_node31":{ - "core":"COLL_S_R_shard15_replica_n28", - "base_url":"http://NODE_S_A06032b38152c0f019:8983/solr", - "node_name":"NODE_S_A06032b38152c0f019:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard16":{ - "range":"f45d0000-fc1effff", - "state":"active", - "replicas":{"core_node33":{ - "core":"COLL_S_R_shard16_replica_n30", - "base_url":"http://NODE_S_A061d3ac3c71d42de2:8983/solr", - "node_name":"NODE_S_A061d3ac3c71d42de2:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard17":{ - "range":"fc1f0000-3dfffff", - "state":"active", - "replicas":{"core_node35":{ - "core":"COLL_S_R_shard17_replica_n32", - "base_url":"http://NODE_S_A02f6288aa3fc9eb3d:8983/solr", - "node_name":"NODE_S_A02f6288aa3fc9eb3d:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard18":{ - "range":"3e00000-ba1ffff", - "state":"active", - "replicas":{"core_node37":{ - "core":"COLL_S_R_shard18_replica_n34", - "base_url":"http://NODE_S_A04a75744569a27ccf:8983/solr", - "node_name":"NODE_S_A04a75744569a27ccf:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard19":{ - "range":"ba20000-1363ffff", - "state":"active", - "replicas":{"core_node39":{ - "core":"COLL_S_R_shard19_replica_n36", - "base_url":"http://NODE_S_A08c1ec5957d8a5d44:8983/solr", - "node_name":"NODE_S_A08c1ec5957d8a5d44:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard20":{ - "range":"13640000-1b25ffff", - "state":"active", - "replicas":{"core_node41":{ - "core":"COLL_S_R_shard20_replica_n38", - "base_url":"http://NODE_S_A084aef0a141a43572:8983/solr", - "node_name":"NODE_S_A084aef0a141a43572:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard21":{ - "range":"1b260000-22e7ffff", - "state":"active", - "replicas":{"core_node43":{ - "core":"COLL_S_R_shard21_replica_n40", - "base_url":"http://NODE_S_A0f096665ad4a3bc58:8983/solr", - "node_name":"NODE_S_A0f096665ad4a3bc58:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard22":{ - "range":"22e80000-2aa9ffff", - "state":"active", - "replicas":{"core_node45":{ - "core":"COLL_S_R_shard22_replica_n42", - "base_url":"http://NODE_S_A0cf36cc11a8447fac:8983/solr", - "node_name":"NODE_S_A0cf36cc11a8447fac:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard23":{ - "range":"2aaa0000-326bffff", - "state":"active", - "replicas":{"core_node47":{ - "core":"COLL_S_R_shard23_replica_n44", - "base_url":"http://NODE_S_A09aede91f0e8c147c:8983/solr", - "node_name":"NODE_S_A09aede91f0e8c147c:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard24":{ - "range":"326c0000-3a2dffff", - "state":"active", - "replicas":{"core_node49":{ - "core":"COLL_S_R_shard24_replica_n46", - "base_url":"http://NODE_S_A0ebcce85ef9c13308:8983/solr", - "node_name":"NODE_S_A0ebcce85ef9c13308:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard25":{ - "range":"3a2e0000-41efffff", - "state":"active", - "replicas":{"core_node51":{ - "core":"COLL_S_R_shard25_replica_n48", - "base_url":"http://NODE_S_A0f66f4bec1ce8fc09:8983/solr", - "node_name":"NODE_S_A0f66f4bec1ce8fc09:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard26":{ - "range":"41f00000-49b1ffff", - "state":"active", - "replicas":{"core_node53":{ - "core":"COLL_S_R_shard26_replica_n50", - "base_url":"http://NODE_S_A0ee1c34fb3e9c261d:8983/solr", - "node_name":"NODE_S_A0ee1c34fb3e9c261d:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard27":{ - "range":"49b20000-5173ffff", - "state":"active", - "replicas":{"core_node55":{ - "core":"COLL_S_R_shard27_replica_n52", - "base_url":"http://NODE_S_A024a9c138d972d437:8983/solr", - "node_name":"NODE_S_A024a9c138d972d437:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard28":{ - "range":"51740000-5935ffff", - "state":"active", - "replicas":{"core_node57":{ - "core":"COLL_S_R_shard28_replica_n54", - "base_url":"http://NODE_S_A0ccef54efe3178758:8983/solr", - "node_name":"NODE_S_A0ccef54efe3178758:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard29":{ - "range":"59360000-60f7ffff", - "state":"active", - "replicas":{"core_node59":{ - "core":"COLL_S_R_shard29_replica_n56", - "base_url":"http://NODE_S_A0afcb9535d2619dac:8983/solr", - "node_name":"NODE_S_A0afcb9535d2619dac:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard30":{ - "range":"60f80000-68b9ffff", - "state":"active", - "replicas":{"core_node61":{ - "core":"COLL_S_R_shard30_replica_n58", - "base_url":"http://NODE_S_A0a3cb04c83b693d80:8983/solr", - "node_name":"NODE_S_A0a3cb04c83b693d80:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard31":{ - "range":"68ba0000-707bffff", - "state":"active", - "replicas":{"core_node63":{ - "core":"COLL_S_R_shard31_replica_n60", - "base_url":"http://NODE_S_A077430d186f0e1de2:8983/solr", - "node_name":"NODE_S_A077430d186f0e1de2:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard32":{ - "range":"707c0000-783dffff", - "state":"active", - "replicas":{"core_node65":{ - "core":"COLL_S_R_shard32_replica_n62", - "base_url":"http://NODE_S_A0d1445647aaae8dd1:8983/solr", - "node_name":"NODE_S_A0d1445647aaae8dd1:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard33":{ - "range":"783e0000-7fffffff", - "state":"active", - "replicas":{"core_node66":{ - "core":"COLL_S_R_shard33_replica_n64", - "base_url":"http://NODE_S_A0dc21b7f9ef048b48:8983/solr", - "node_name":"NODE_S_A0dc21b7f9ef048b48:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}}, - "router":{"name":"compositeId"}, - "autoAddReplicas":"false", - "nrtReplicas":"1", - "tlogReplicas":"0", - "znodeVersion":7, - "configName":"COLL_S_R"}, - "COLL_S_M_1":{ - "pullReplicas":"0", - "replicationFactor":"1", - "shards":{ - "shard1":{ - "range":"80000000-9c70ffff", - "state":"active", - "replicas":{"core_node3":{ - "core":"COLL_S_M_1_shard1_replica_n1", - "base_url":"http://NODE_S_A09aede91f0e8c147c:8983/solr", - "node_name":"NODE_S_A09aede91f0e8c147c:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard2":{ - "range":"9c710000-b8e2ffff", - "state":"active", - "replicas":{"core_node5":{ - "core":"COLL_S_M_1_shard2_replica_n2", - "base_url":"http://NODE_S_A0ee1c34fb3e9c261d:8983/solr", - "node_name":"NODE_S_A0ee1c34fb3e9c261d:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard3":{ - "range":"b8e30000-d554ffff", - "state":"active", - "replicas":{"core_node7":{ - "core":"COLL_S_M_1_shard3_replica_n4", - "base_url":"http://NODE_S_A024a9c138d972d437:8983/solr", - "node_name":"NODE_S_A024a9c138d972d437:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard4":{ - "range":"d5550000-f1c6ffff", - "state":"active", - "replicas":{"core_node9":{ - "core":"COLL_S_M_1_shard4_replica_n6", - "base_url":"http://NODE_S_A0d1445647aaae8dd1:8983/solr", - "node_name":"NODE_S_A0d1445647aaae8dd1:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard5":{ - "range":"f1c70000-e37ffff", - "state":"active", - "replicas":{"core_node11":{ - "core":"COLL_S_M_1_shard5_replica_n8", - "base_url":"http://NODE_S_A0dc21b7f9ef048b48:8983/solr", - "node_name":"NODE_S_A0dc21b7f9ef048b48:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard6":{ - "range":"e380000-2aa9ffff", - "state":"active", - "replicas":{"core_node13":{ - "core":"COLL_S_M_1_shard6_replica_n10", - "base_url":"http://NODE_S_A077430d186f0e1de2:8983/solr", - "node_name":"NODE_S_A077430d186f0e1de2:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard7":{ - "range":"2aaa0000-471bffff", - "state":"active", - "replicas":{"core_node15":{ - "core":"COLL_S_M_1_shard7_replica_n12", - "base_url":"http://NODE_S_A0ccef54efe3178758:8983/solr", - "node_name":"NODE_S_A0ccef54efe3178758:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard8":{ - "range":"471c0000-638dffff", - "state":"active", - "replicas":{"core_node17":{ - "core":"COLL_S_M_1_shard8_replica_n14", - "base_url":"http://NODE_S_A0afcb9535d2619dac:8983/solr", - "node_name":"NODE_S_A0afcb9535d2619dac:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard9":{ - "range":"638e0000-7fffffff", - "state":"active", - "replicas":{"core_node18":{ - "core":"COLL_S_M_1_shard9_replica_n16", - "base_url":"http://NODE_S_A0a3cb04c83b693d80:8983/solr", - "node_name":"NODE_S_A0a3cb04c83b693d80:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}}, - "router":{"name":"compositeId"}, - "autoAddReplicas":"false", - "nrtReplicas":"1", - "tlogReplicas":"0", - "znodeVersion":348, - "configName":"COLL_S_M_1"}, - "COLL_E_V2_4":{ - "pullReplicas":"0", - "replicationFactor":"2", - "shards":{ - "shard1":{ - "range":"80000000-ffffffff", - "state":"active", - "replicas":{ - "core_node3":{ - "core":"COLL_E_V2_4_shard1_replica_n1", - "base_url":"http://NODE_S_A0ee1c34fb3e9c261d:8983/solr", - "node_name":"NODE_S_A0ee1c34fb3e9c261d:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node5":{ - "core":"COLL_E_V2_4_shard1_replica_n2", - "base_url":"http://NODE_S_A0dc21b7f9ef048b48:8983/solr", - "node_name":"NODE_S_A0dc21b7f9ef048b48:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard2":{ - "range":"0-7fffffff", - "state":"active", - "replicas":{ - "core_node7":{ - "core":"COLL_E_V2_4_shard2_replica_n4", - "base_url":"http://NODE_S_A0d1445647aaae8dd1:8983/solr", - "node_name":"NODE_S_A0d1445647aaae8dd1:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node8":{ - "core":"COLL_E_V2_4_shard2_replica_n6", - "base_url":"http://NODE_S_A0a3cb04c83b693d80:8983/solr", - "node_name":"NODE_S_A0a3cb04c83b693d80:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}}, - "router":{"name":"compositeId"}, - "autoAddReplicas":"false", - "nrtReplicas":"1", - "tlogReplicas":"0", - "znodeVersion":226, - "configName":"COLL_E_V2_4"}, - "Q2Q_postsandsubs":{ - "pullReplicas":"0", - "replicationFactor":"1", - "shards":{ - "shard1":{ - "range":"80000000-8c2fffff", - "state":"active", - "replicas":{"core_node3":{ - "core":"Q2Q_postsandsubs_shard1_replica_n1", - "base_url":"http://NODE_S_A0f66f4bec1ce8fc09:8983/solr", - "node_name":"NODE_S_A0f66f4bec1ce8fc09:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard2":{ - "range":"8c300000-9860ffff", - "state":"active", - "replicas":{"core_node5":{ - "core":"Q2Q_postsandsubs_shard2_replica_n2", - "base_url":"http://NODE_S_A0ebcce85ef9c13308:8983/solr", - "node_name":"NODE_S_A0ebcce85ef9c13308:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard3":{ - "range":"98610000-a491ffff", - "state":"active", - "replicas":{"core_node7":{ - "core":"Q2Q_postsandsubs_shard3_replica_n4", - "base_url":"http://NODE_S_A06032b38152c0f019:8983/solr", - "node_name":"NODE_S_A06032b38152c0f019:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard4":{ - "range":"a4920000-b0c2ffff", - "state":"active", - "replicas":{"core_node9":{ - "core":"Q2Q_postsandsubs_shard4_replica_n6", - "base_url":"http://NODE_S_A02f6288aa3fc9eb3d:8983/solr", - "node_name":"NODE_S_A02f6288aa3fc9eb3d:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard5":{ - "range":"b0c30000-bcf2ffff", - "state":"active", - "replicas":{"core_node11":{ - "core":"Q2Q_postsandsubs_shard5_replica_n8", - "base_url":"http://NODE_S_A014292d062d5088da:8983/solr", - "node_name":"NODE_S_A014292d062d5088da:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard6":{ - "range":"bcf30000-c923ffff", - "state":"active", - "replicas":{"core_node13":{ - "core":"Q2Q_postsandsubs_shard6_replica_n10", - "base_url":"http://NODE_S_A006b71d81aedd8577:8983/solr", - "node_name":"NODE_S_A006b71d81aedd8577:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard7":{ - "range":"c9240000-d554ffff", - "state":"active", - "replicas":{"core_node15":{ - "core":"Q2Q_postsandsubs_shard7_replica_n12", - "base_url":"http://NODE_S_A006b71d81aedd8577:8983/solr", - "node_name":"NODE_S_A006b71d81aedd8577:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard8":{ - "range":"d5550000-e185ffff", - "state":"active", - "replicas":{"core_node17":{ - "core":"Q2Q_postsandsubs_shard8_replica_n14", - "base_url":"http://NODE_S_A014292d062d5088da:8983/solr", - "node_name":"NODE_S_A014292d062d5088da:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard9":{ - "range":"e1860000-edb5ffff", - "state":"active", - "replicas":{"core_node19":{ - "core":"Q2Q_postsandsubs_shard9_replica_n16", - "base_url":"http://NODE_S_A02f6288aa3fc9eb3d:8983/solr", - "node_name":"NODE_S_A02f6288aa3fc9eb3d:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard10":{ - "range":"edb60000-f9e6ffff", - "state":"active", - "replicas":{"core_node21":{ - "core":"Q2Q_postsandsubs_shard10_replica_n18", - "base_url":"http://NODE_S_A06032b38152c0f019:8983/solr", - "node_name":"NODE_S_A06032b38152c0f019:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard11":{ - "range":"f9e70000-617ffff", - "state":"active", - "replicas":{"core_node23":{ - "core":"Q2Q_postsandsubs_shard11_replica_n20", - "base_url":"http://NODE_S_A0ebcce85ef9c13308:8983/solr", - "node_name":"NODE_S_A0ebcce85ef9c13308:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard12":{ - "range":"6180000-1248ffff", - "state":"active", - "replicas":{"core_node25":{ - "core":"Q2Q_postsandsubs_shard12_replica_n22", - "base_url":"http://NODE_S_A0f66f4bec1ce8fc09:8983/solr", - "node_name":"NODE_S_A0f66f4bec1ce8fc09:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard13":{ - "range":"12490000-1e78ffff", - "state":"active", - "replicas":{"core_node27":{ - "core":"Q2Q_postsandsubs_shard13_replica_n24", - "base_url":"http://NODE_S_A0f096665ad4a3bc58:8983/solr", - "node_name":"NODE_S_A0f096665ad4a3bc58:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard14":{ - "range":"1e790000-2aa9ffff", - "state":"active", - "replicas":{"core_node29":{ - "core":"Q2Q_postsandsubs_shard14_replica_n26", - "base_url":"http://NODE_S_A0cf36cc11a8447fac:8983/solr", - "node_name":"NODE_S_A0cf36cc11a8447fac:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard15":{ - "range":"2aaa0000-36daffff", - "state":"active", - "replicas":{"core_node31":{ - "core":"Q2Q_postsandsubs_shard15_replica_n28", - "base_url":"http://NODE_S_A08c1ec5957d8a5d44:8983/solr", - "node_name":"NODE_S_A08c1ec5957d8a5d44:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard16":{ - "range":"36db0000-430bffff", - "state":"active", - "replicas":{"core_node33":{ - "core":"Q2Q_postsandsubs_shard16_replica_n30", - "base_url":"http://NODE_S_A084aef0a141a43572:8983/solr", - "node_name":"NODE_S_A084aef0a141a43572:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard17":{ - "range":"430c0000-4f3bffff", - "state":"active", - "replicas":{"core_node35":{ - "core":"Q2Q_postsandsubs_shard17_replica_n32", - "base_url":"http://NODE_S_A061d3ac3c71d42de2:8983/solr", - "node_name":"NODE_S_A061d3ac3c71d42de2:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard18":{ - "range":"4f3c0000-5b6cffff", - "state":"active", - "replicas":{"core_node37":{ - "core":"Q2Q_postsandsubs_shard18_replica_n34", - "base_url":"http://NODE_S_A04a75744569a27ccf:8983/solr", - "node_name":"NODE_S_A04a75744569a27ccf:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard19":{ - "range":"5b6d0000-679dffff", - "state":"active", - "replicas":{"core_node39":{ - "core":"Q2Q_postsandsubs_shard19_replica_n36", - "base_url":"http://NODE_S_A04a75744569a27ccf:8983/solr", - "node_name":"NODE_S_A04a75744569a27ccf:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard20":{ - "range":"679e0000-73ceffff", - "state":"active", - "replicas":{"core_node41":{ - "core":"Q2Q_postsandsubs_shard20_replica_n38", - "base_url":"http://NODE_S_A061d3ac3c71d42de2:8983/solr", - "node_name":"NODE_S_A061d3ac3c71d42de2:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard21":{ - "range":"73cf0000-7fffffff", - "state":"active", - "replicas":{"core_node42":{ - "core":"Q2Q_postsandsubs_shard21_replica_n40", - "base_url":"http://NODE_S_A084aef0a141a43572:8983/solr", - "node_name":"NODE_S_A084aef0a141a43572:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}}, - "router":{"name":"compositeId"}, - "autoAddReplicas":"false", - "nrtReplicas":"1", - "tlogReplicas":"0", - "znodeVersion":70, - "configName":"Q2Q_postsandsubs"}, - "COLL_S_E_15":{ - "pullReplicas":"0", - "replicationFactor":"2", - "shards":{ - "shard1":{ - "range":"80000000-8e37ffff", - "state":"active", - "replicas":{ - "core_node3":{ - "core":"COLL_S_E_15_shard1_replica_n1", - "base_url":"http://NODE_S_A0d1445647aaae8dd1:8983/solr", - "node_name":"NODE_S_A0d1445647aaae8dd1:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node5":{ - "core":"COLL_S_E_15_shard1_replica_n2", - "base_url":"http://NODE_S_A0afcb9535d2619dac:8983/solr", - "node_name":"NODE_S_A0afcb9535d2619dac:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}}, - "shard2":{ - "range":"8e380000-9c70ffff", - "state":"active", - "replicas":{ - "core_node7":{ - "core":"COLL_S_E_15_shard2_replica_n4", - "base_url":"http://NODE_S_A0ee1c34fb3e9c261d:8983/solr", - "node_name":"NODE_S_A0ee1c34fb3e9c261d:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node9":{ - "core":"COLL_S_E_15_shard2_replica_n6", - "base_url":"http://NODE_S_A0ccef54efe3178758:8983/solr", - "node_name":"NODE_S_A0ccef54efe3178758:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard3":{ - "range":"9c710000-aaa9ffff", - "state":"active", - "replicas":{ - "core_node11":{ - "core":"COLL_S_E_15_shard3_replica_n8", - "base_url":"http://NODE_S_A024a9c138d972d437:8983/solr", - "node_name":"NODE_S_A024a9c138d972d437:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node13":{ - "core":"COLL_S_E_15_shard3_replica_n10", - "base_url":"http://NODE_S_A0dc21b7f9ef048b48:8983/solr", - "node_name":"NODE_S_A0dc21b7f9ef048b48:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard4":{ - "range":"aaaa0000-b8e2ffff", - "state":"active", - "replicas":{ - "core_node15":{ - "core":"COLL_S_E_15_shard4_replica_n12", - "base_url":"http://NODE_S_A077430d186f0e1de2:8983/solr", - "node_name":"NODE_S_A077430d186f0e1de2:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node17":{ - "core":"COLL_S_E_15_shard4_replica_n14", - "base_url":"http://NODE_S_A0a3cb04c83b693d80:8983/solr", - "node_name":"NODE_S_A0a3cb04c83b693d80:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard5":{ - "range":"b8e30000-c71bffff", - "state":"active", - "replicas":{ - "core_node19":{ - "core":"COLL_S_E_15_shard5_replica_n16", - "base_url":"http://NODE_S_A09aede91f0e8c147c:8983/solr", - "node_name":"NODE_S_A09aede91f0e8c147c:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node21":{ - "core":"COLL_S_E_15_shard5_replica_n18", - "base_url":"http://NODE_S_A0d1445647aaae8dd1:8983/solr", - "node_name":"NODE_S_A0d1445647aaae8dd1:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}}, - "shard6":{ - "range":"c71c0000-d554ffff", - "state":"active", - "replicas":{ - "core_node23":{ - "core":"COLL_S_E_15_shard6_replica_n20", - "base_url":"http://NODE_S_A0afcb9535d2619dac:8983/solr", - "node_name":"NODE_S_A0afcb9535d2619dac:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node25":{ - "core":"COLL_S_E_15_shard6_replica_n22", - "base_url":"http://NODE_S_A0ee1c34fb3e9c261d:8983/solr", - "node_name":"NODE_S_A0ee1c34fb3e9c261d:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard7":{ - "range":"d5550000-e38dffff", - "state":"active", - "replicas":{ - "core_node27":{ - "core":"COLL_S_E_15_shard7_replica_n24", - "base_url":"http://NODE_S_A0ccef54efe3178758:8983/solr", - "node_name":"NODE_S_A0ccef54efe3178758:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node29":{ - "core":"COLL_S_E_15_shard7_replica_n26", - "base_url":"http://NODE_S_A024a9c138d972d437:8983/solr", - "node_name":"NODE_S_A024a9c138d972d437:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}}, - "shard8":{ - "range":"e38e0000-f1c6ffff", - "state":"active", - "replicas":{ - "core_node31":{ - "core":"COLL_S_E_15_shard8_replica_n28", - "base_url":"http://NODE_S_A0dc21b7f9ef048b48:8983/solr", - "node_name":"NODE_S_A0dc21b7f9ef048b48:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node33":{ - "core":"COLL_S_E_15_shard8_replica_n30", - "base_url":"http://NODE_S_A077430d186f0e1de2:8983/solr", - "node_name":"NODE_S_A077430d186f0e1de2:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}}, - "shard9":{ - "range":"f1c70000-ffffffff", - "state":"active", - "replicas":{ - "core_node35":{ - "core":"COLL_S_E_15_shard9_replica_n32", - "base_url":"http://NODE_S_A0a3cb04c83b693d80:8983/solr", - "node_name":"NODE_S_A0a3cb04c83b693d80:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node37":{ - "core":"COLL_S_E_15_shard9_replica_n34", - "base_url":"http://NODE_S_A09aede91f0e8c147c:8983/solr", - "node_name":"NODE_S_A09aede91f0e8c147c:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}}, - "shard10":{ - "range":"0-e37ffff", - "state":"active", - "replicas":{ - "core_node39":{ - "core":"COLL_S_E_15_shard10_replica_n36", - "base_url":"http://NODE_S_A0d1445647aaae8dd1:8983/solr", - "node_name":"NODE_S_A0d1445647aaae8dd1:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node41":{ - "core":"COLL_S_E_15_shard10_replica_n38", - "base_url":"http://NODE_S_A0afcb9535d2619dac:8983/solr", - "node_name":"NODE_S_A0afcb9535d2619dac:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}}, - "shard11":{ - "range":"e380000-1c70ffff", - "state":"active", - "replicas":{ - "core_node43":{ - "core":"COLL_S_E_15_shard11_replica_n40", - "base_url":"http://NODE_S_A0ee1c34fb3e9c261d:8983/solr", - "node_name":"NODE_S_A0ee1c34fb3e9c261d:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node45":{ - "core":"COLL_S_E_15_shard11_replica_n42", - "base_url":"http://NODE_S_A0ccef54efe3178758:8983/solr", - "node_name":"NODE_S_A0ccef54efe3178758:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard12":{ - "range":"1c710000-2aa9ffff", - "state":"active", - "replicas":{ - "core_node47":{ - "core":"COLL_S_E_15_shard12_replica_n44", - "base_url":"http://NODE_S_A024a9c138d972d437:8983/solr", - "node_name":"NODE_S_A024a9c138d972d437:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node49":{ - "core":"COLL_S_E_15_shard12_replica_n46", - "base_url":"http://NODE_S_A0dc21b7f9ef048b48:8983/solr", - "node_name":"NODE_S_A0dc21b7f9ef048b48:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard13":{ - "range":"2aaa0000-38e2ffff", - "state":"active", - "replicas":{ - "core_node51":{ - "core":"COLL_S_E_15_shard13_replica_n48", - "base_url":"http://NODE_S_A077430d186f0e1de2:8983/solr", - "node_name":"NODE_S_A077430d186f0e1de2:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node53":{ - "core":"COLL_S_E_15_shard13_replica_n50", - "base_url":"http://NODE_S_A0a3cb04c83b693d80:8983/solr", - "node_name":"NODE_S_A0a3cb04c83b693d80:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard14":{ - "range":"38e30000-471bffff", - "state":"active", - "replicas":{ - "core_node55":{ - "core":"COLL_S_E_15_shard14_replica_n52", - "base_url":"http://NODE_S_A09aede91f0e8c147c:8983/solr", - "node_name":"NODE_S_A09aede91f0e8c147c:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node57":{ - "core":"COLL_S_E_15_shard14_replica_n54", - "base_url":"http://NODE_S_A0d1445647aaae8dd1:8983/solr", - "node_name":"NODE_S_A0d1445647aaae8dd1:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}}, - "shard15":{ - "range":"471c0000-5554ffff", - "state":"active", - "replicas":{ - "core_node59":{ - "core":"COLL_S_E_15_shard15_replica_n56", - "base_url":"http://NODE_S_A0afcb9535d2619dac:8983/solr", - "node_name":"NODE_S_A0afcb9535d2619dac:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node61":{ - "core":"COLL_S_E_15_shard15_replica_n58", - "base_url":"http://NODE_S_A0ee1c34fb3e9c261d:8983/solr", - "node_name":"NODE_S_A0ee1c34fb3e9c261d:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard16":{ - "range":"55550000-638dffff", - "state":"active", - "replicas":{ - "core_node63":{ - "core":"COLL_S_E_15_shard16_replica_n60", - "base_url":"http://NODE_S_A0ccef54efe3178758:8983/solr", - "node_name":"NODE_S_A0ccef54efe3178758:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node65":{ - "core":"COLL_S_E_15_shard16_replica_n62", - "base_url":"http://NODE_S_A024a9c138d972d437:8983/solr", - "node_name":"NODE_S_A024a9c138d972d437:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}}, - "shard17":{ - "range":"638e0000-71c6ffff", - "state":"active", - "replicas":{ - "core_node67":{ - "core":"COLL_S_E_15_shard17_replica_n64", - "base_url":"http://NODE_S_A0dc21b7f9ef048b48:8983/solr", - "node_name":"NODE_S_A0dc21b7f9ef048b48:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node69":{ - "core":"COLL_S_E_15_shard17_replica_n66", - "base_url":"http://NODE_S_A077430d186f0e1de2:8983/solr", - "node_name":"NODE_S_A077430d186f0e1de2:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}}, - "shard18":{ - "range":"71c70000-7fffffff", - "state":"active", - "replicas":{ - "core_node71":{ - "core":"COLL_S_E_15_shard18_replica_n68", - "base_url":"http://NODE_S_A0a3cb04c83b693d80:8983/solr", - "node_name":"NODE_S_A0a3cb04c83b693d80:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node72":{ - "core":"COLL_S_E_15_shard18_replica_n70", - "base_url":"http://NODE_S_A09aede91f0e8c147c:8983/solr", - "node_name":"NODE_S_A09aede91f0e8c147c:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}}}, - "router":{"name":"compositeId"}, - "autoAddReplicas":"false", - "nrtReplicas":"1", - "tlogReplicas":"0", - "znodeVersion":1106, - "configName":"COLL_S_E_15"}, - "COLL_P_V_E_v2":{ - "pullReplicas":"0", - "replicationFactor":"2", - "shards":{ - "shard1":{ - "range":"80000000-aaa9ffff", - "state":"active", - "replicas":{ - "core_node3":{ - "core":"COLL_P_V_E_v2_shard1_replica_n1", - "base_url":"http://NODE_S_A0d1445647aaae8dd1:8983/solr", - "node_name":"NODE_S_A0d1445647aaae8dd1:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node5":{ - "core":"COLL_P_V_E_v2_shard1_replica_n2", - "base_url":"http://NODE_S_A086a3a5ff0617c80f:8983/solr", - "node_name":"NODE_S_A086a3a5ff0617c80f:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard2":{ - "range":"aaaa0000-d554ffff", - "state":"active", - "replicas":{ - "core_node7":{ - "core":"COLL_P_V_E_v2_shard2_replica_n4", - "base_url":"http://NODE_S_A014292d062d5088da:8983/solr", - "node_name":"NODE_S_A014292d062d5088da:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node9":{ - "core":"COLL_P_V_E_v2_shard2_replica_n6", - "base_url":"http://NODE_S_A006b71d81aedd8577:8983/solr", - "node_name":"NODE_S_A006b71d81aedd8577:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard3":{ - "range":"d5550000-ffffffff", - "state":"active", - "replicas":{ - "core_node11":{ - "core":"COLL_P_V_E_v2_shard3_replica_n8", - "base_url":"http://NODE_S_A0a3cb04c83b693d80:8983/solr", - "node_name":"NODE_S_A0a3cb04c83b693d80:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node13":{ - "core":"COLL_P_V_E_v2_shard3_replica_n10", - "base_url":"http://NODE_S_A09aede91f0e8c147c:8983/solr", - "node_name":"NODE_S_A09aede91f0e8c147c:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard4":{ - "range":"0-2aa9ffff", - "state":"active", - "replicas":{ - "core_node15":{ - "core":"COLL_P_V_E_v2_shard4_replica_n12", - "base_url":"http://NODE_S_A08c1ec5957d8a5d44:8983/solr", - "node_name":"NODE_S_A08c1ec5957d8a5d44:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node17":{ - "core":"COLL_P_V_E_v2_shard4_replica_n14", - "base_url":"http://NODE_S_A0f66f4bec1ce8fc09:8983/solr", - "node_name":"NODE_S_A0f66f4bec1ce8fc09:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard5":{ - "range":"2aaa0000-5554ffff", - "state":"active", - "replicas":{ - "core_node19":{ - "core":"COLL_P_V_E_v2_shard5_replica_n16", - "base_url":"http://NODE_S_A09f108fb78272a03d:8983/solr", - "node_name":"NODE_S_A09f108fb78272a03d:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node21":{ - "core":"COLL_P_V_E_v2_shard5_replica_n18", - "base_url":"http://NODE_S_A0ccef54efe3178758:8983/solr", - "node_name":"NODE_S_A0ccef54efe3178758:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}}, - "shard6":{ - "range":"55550000-7fffffff", - "state":"active", - "replicas":{ - "core_node23":{ - "core":"COLL_P_V_E_v2_shard6_replica_n20", - "base_url":"http://NODE_S_A02f6288aa3fc9eb3d:8983/solr", - "node_name":"NODE_S_A02f6288aa3fc9eb3d:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node24":{ - "core":"COLL_P_V_E_v2_shard6_replica_n22", - "base_url":"http://NODE_S_A0bdba3561ff291bc4:8983/solr", - "node_name":"NODE_S_A0bdba3561ff291bc4:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}}, - "router":{"name":"compositeId"}, - "autoAddReplicas":"false", - "nrtReplicas":"2", - "tlogReplicas":"0", - "znodeVersion":8, - "configName":"COLL_P_V_E_v2"}, - "audit_logs":{ - "pullReplicas":"0", - "replicationFactor":"2", - "shards":{ - "shard1":{ - "range":"80000000-ffffffff", - "state":"active", - "replicas":{ - "core_node3":{ - "core":"audit_logs_shard1_replica_n1", - "base_url":"http://NODE_S_A0d1445647aaae8dd1:8983/solr", - "node_name":"NODE_S_A0d1445647aaae8dd1:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node5":{ - "core":"audit_logs_shard1_replica_n2", - "base_url":"http://NODE_S_A0ccef54efe3178758:8983/solr", - "node_name":"NODE_S_A0ccef54efe3178758:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard2":{ - "range":"0-7fffffff", - "state":"active", - "replicas":{ - "core_node7":{ - "core":"audit_logs_shard2_replica_n4", - "base_url":"http://NODE_S_A077430d186f0e1de2:8983/solr", - "node_name":"NODE_S_A077430d186f0e1de2:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node8":{ - "core":"audit_logs_shard2_replica_n6", - "base_url":"http://NODE_S_A0afcb9535d2619dac:8983/solr", - "node_name":"NODE_S_A0afcb9535d2619dac:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}}, - "router":{"name":"compositeId"}, - "autoAddReplicas":"false", - "nrtReplicas":"1", - "tlogReplicas":"0", - "znodeVersion":286, - "configName":"audit_logs"}, - "COLL_S_H_7":{ - "pullReplicas":"0", - "replicationFactor":"1", - "shards":{ - "shard1":{ - "range":"80000000-9c70ffff", - "state":"active", - "replicas":{"core_node3":{ - "core":"COLL_S_H_7_shard1_replica_n1", - "base_url":"http://NODE_S_A09aede91f0e8c147c:8983/solr", - "node_name":"NODE_S_A09aede91f0e8c147c:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard2":{ - "range":"9c710000-b8e2ffff", - "state":"active", - "replicas":{"core_node5":{ - "core":"COLL_S_H_7_shard2_replica_n2", - "base_url":"http://NODE_S_A0a3cb04c83b693d80:8983/solr", - "node_name":"NODE_S_A0a3cb04c83b693d80:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard3":{ - "range":"b8e30000-d554ffff", - "state":"active", - "replicas":{"core_node7":{ - "core":"COLL_S_H_7_shard3_replica_n4", - "base_url":"http://NODE_S_A024a9c138d972d437:8983/solr", - "node_name":"NODE_S_A024a9c138d972d437:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard4":{ - "range":"d5550000-f1c6ffff", - "state":"active", - "replicas":{"core_node9":{ - "core":"COLL_S_H_7_shard4_replica_n6", - "base_url":"http://NODE_S_A0ee1c34fb3e9c261d:8983/solr", - "node_name":"NODE_S_A0ee1c34fb3e9c261d:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard5":{ - "range":"f1c70000-e37ffff", - "state":"active", - "replicas":{"core_node11":{ - "core":"COLL_S_H_7_shard5_replica_n8", - "base_url":"http://NODE_S_A077430d186f0e1de2:8983/solr", - "node_name":"NODE_S_A077430d186f0e1de2:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard6":{ - "range":"e380000-2aa9ffff", - "state":"active", - "replicas":{"core_node13":{ - "core":"COLL_S_H_7_shard6_replica_n10", - "base_url":"http://NODE_S_A0dc21b7f9ef048b48:8983/solr", - "node_name":"NODE_S_A0dc21b7f9ef048b48:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard7":{ - "range":"2aaa0000-471bffff", - "state":"active", - "replicas":{"core_node15":{ - "core":"COLL_S_H_7_shard7_replica_n12", - "base_url":"http://NODE_S_A0d1445647aaae8dd1:8983/solr", - "node_name":"NODE_S_A0d1445647aaae8dd1:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard8":{ - "range":"471c0000-638dffff", - "state":"active", - "replicas":{"core_node17":{ - "core":"COLL_S_H_7_shard8_replica_n14", - "base_url":"http://NODE_S_A0ccef54efe3178758:8983/solr", - "node_name":"NODE_S_A0ccef54efe3178758:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard9":{ - "range":"638e0000-7fffffff", - "state":"active", - "replicas":{"core_node18":{ - "core":"COLL_S_H_7_shard9_replica_n16", - "base_url":"http://NODE_S_A0afcb9535d2619dac:8983/solr", - "node_name":"NODE_S_A0afcb9535d2619dac:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}}, - "router":{"name":"compositeId"}, - "autoAddReplicas":"false", - "nrtReplicas":"1", - "tlogReplicas":"0", - "znodeVersion":349, - "configName":"COLL_S_H_7"}, - "COLL_E_v2_5":{ - "pullReplicas":"0", - "replicationFactor":"2", - "shards":{ - "shard1":{ - "range":"80000000-871bffff", - "state":"active", - "replicas":{ - "core_node166":{ - "core":"COLL_E_v2_5_shard1_replica_n165", - "base_url":"http://NODE_S_A006b71d81aedd8577:8983/solr", - "node_name":"NODE_S_A006b71d81aedd8577:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node394":{ - "core":"COLL_E_v2_5_shard1_replica_n393", - "base_url":"http://NODE_S_A0a0ffe317a4f85c55:8983/solr", - "node_name":"NODE_S_A0a0ffe317a4f85c55:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}}, - "shard2":{ - "range":"871c0000-8e37ffff", - "state":"active", - "replicas":{ - "core_node7":{ - "core":"COLL_E_v2_5_shard2_replica_n4", - "base_url":"http://NODE_S_A024a9c138d972d437:8983/solr", - "node_name":"NODE_S_A024a9c138d972d437:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node378":{ - "core":"COLL_E_v2_5_shard2_replica_n377", - "base_url":"http://NODE_S_A0d391cf8eeaca59cd:8983/solr", - "node_name":"NODE_S_A0d391cf8eeaca59cd:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}}, - "shard3":{ - "range":"8e380000-9554ffff", - "state":"active", - "replicas":{ - "core_node11":{ - "core":"COLL_E_v2_5_shard3_replica_n8", - "base_url":"http://NODE_S_A024a9c138d972d437:8983/solr", - "node_name":"NODE_S_A024a9c138d972d437:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node404":{ - "core":"COLL_E_v2_5_shard3_replica_n403", - "base_url":"http://NODE_S_A0bdba3561ff291bc4:8983/solr", - "node_name":"NODE_S_A0bdba3561ff291bc4:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}}, - "shard4":{ - "range":"95550000-9c70ffff", - "state":"active", - "replicas":{ - "core_node200":{ - "core":"COLL_E_v2_5_shard4_replica_n199", - "base_url":"http://NODE_S_A0ebcce85ef9c13308:8983/solr", - "node_name":"NODE_S_A0ebcce85ef9c13308:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node388":{ - "core":"COLL_E_v2_5_shard4_replica_n387", - "base_url":"http://NODE_S_A0e48ab035b869305e:8983/solr", - "node_name":"NODE_S_A0e48ab035b869305e:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}}, - "shard5":{ - "range":"9c710000-a38dffff", - "state":"active", - "replicas":{ - "core_node19":{ - "core":"COLL_E_v2_5_shard5_replica_n16", - "base_url":"http://NODE_S_A077430d186f0e1de2:8983/solr", - "node_name":"NODE_S_A077430d186f0e1de2:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node202":{ - "core":"COLL_E_v2_5_shard5_replica_n201", - "base_url":"http://NODE_S_A0cf36cc11a8447fac:8983/solr", - "node_name":"NODE_S_A0cf36cc11a8447fac:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard6":{ - "range":"a38e0000-aaa9ffff", - "state":"active", - "replicas":{ - "core_node162":{ - "core":"COLL_E_v2_5_shard6_replica_n161", - "base_url":"http://NODE_S_A06032b38152c0f019:8983/solr", - "node_name":"NODE_S_A06032b38152c0f019:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node170":{ - "core":"COLL_E_v2_5_shard6_replica_n169", - "base_url":"http://NODE_S_A0f096665ad4a3bc58:8983/solr", - "node_name":"NODE_S_A0f096665ad4a3bc58:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}}, - "shard7":{ - "range":"aaaa0000-b1c6ffff", - "state":"active", - "replicas":{ - "core_node29":{ - "core":"COLL_E_v2_5_shard7_replica_n26", - "base_url":"http://NODE_S_A0afcb9535d2619dac:8983/solr", - "node_name":"NODE_S_A0afcb9535d2619dac:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node228":{ - "core":"COLL_E_v2_5_shard7_replica_n227", - "base_url":"http://NODE_S_A0dc21b7f9ef048b48:8983/solr", - "node_name":"NODE_S_A0dc21b7f9ef048b48:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard8":{ - "range":"b1c70000-b8e2ffff", - "state":"active", - "replicas":{ - "core_node150":{ - "core":"COLL_E_v2_5_shard8_replica_n149", - "base_url":"http://NODE_S_A0f66f4bec1ce8fc09:8983/solr", - "node_name":"NODE_S_A0f66f4bec1ce8fc09:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node172":{ - "core":"COLL_E_v2_5_shard8_replica_n171", - "base_url":"http://NODE_S_A0ebcce85ef9c13308:8983/solr", - "node_name":"NODE_S_A0ebcce85ef9c13308:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard9":{ - "range":"b8e30000-bfffffff", - "state":"active", - "replicas":{ - "core_node386":{ - "core":"COLL_E_v2_5_shard9_replica_n385", - "base_url":"http://NODE_S_A082e313208af7da04:8983/solr", - "node_name":"NODE_S_A082e313208af7da04:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node398":{ - "core":"COLL_E_v2_5_shard9_replica_n397", - "base_url":"http://NODE_S_A013eaa884d3c59fab:8983/solr", - "node_name":"NODE_S_A013eaa884d3c59fab:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}}, - "shard10":{ - "range":"c0000000-c71bffff", - "state":"active", - "replicas":{ - "core_node238":{ - "core":"COLL_E_v2_5_shard10_replica_n237", - "base_url":"http://NODE_S_A014292d062d5088da:8983/solr", - "node_name":"NODE_S_A014292d062d5088da:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node370":{ - "core":"COLL_E_v2_5_shard10_replica_n369", - "base_url":"http://NODE_S_A086a3a5ff0617c80f:8983/solr", - "node_name":"NODE_S_A086a3a5ff0617c80f:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}}, - "shard11":{ - "range":"c71c0000-ce37ffff", - "state":"active", - "replicas":{ - "core_node45":{ - "core":"COLL_E_v2_5_shard11_replica_n42", - "base_url":"http://NODE_S_A0d1445647aaae8dd1:8983/solr", - "node_name":"NODE_S_A0d1445647aaae8dd1:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node194":{ - "core":"COLL_E_v2_5_shard11_replica_n193", - "base_url":"http://NODE_S_A006b71d81aedd8577:8983/solr", - "node_name":"NODE_S_A006b71d81aedd8577:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard12":{ - "range":"ce380000-d554ffff", - "state":"active", - "replicas":{ - "core_node236":{ - "core":"COLL_E_v2_5_shard12_replica_n235", - "base_url":"http://NODE_S_A0a3cb04c83b693d80:8983/solr", - "node_name":"NODE_S_A0a3cb04c83b693d80:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node240":{ - "core":"COLL_E_v2_5_shard12_replica_n239", - "base_url":"http://NODE_S_A014292d062d5088da:8983/solr", - "node_name":"NODE_S_A014292d062d5088da:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}}, - "shard13":{ - "range":"d5550000-dc70ffff", - "state":"active", - "replicas":{ - "core_node208":{ - "core":"COLL_E_v2_5_shard13_replica_n207", - "base_url":"http://NODE_S_A061d3ac3c71d42de2:8983/solr", - "node_name":"NODE_S_A061d3ac3c71d42de2:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node390":{ - "core":"COLL_E_v2_5_shard13_replica_n389", - "base_url":"http://NODE_S_A03baeedfd28ba05ba:8983/solr", - "node_name":"NODE_S_A03baeedfd28ba05ba:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}}, - "shard14":{ - "range":"dc710000-e38dffff", - "state":"active", - "replicas":{ - "core_node152":{ - "core":"COLL_E_v2_5_shard14_replica_n151", - "base_url":"http://NODE_S_A0ebcce85ef9c13308:8983/solr", - "node_name":"NODE_S_A0ebcce85ef9c13308:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node174":{ - "core":"COLL_E_v2_5_shard14_replica_n173", - "base_url":"http://NODE_S_A0cf36cc11a8447fac:8983/solr", - "node_name":"NODE_S_A0cf36cc11a8447fac:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard15":{ - "range":"e38e0000-eaa9ffff", - "state":"active", - "replicas":{ - "core_node61":{ - "core":"COLL_E_v2_5_shard15_replica_n58", - "base_url":"http://NODE_S_A0afcb9535d2619dac:8983/solr", - "node_name":"NODE_S_A0afcb9535d2619dac:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node368":{ - "core":"COLL_E_v2_5_shard15_replica_n367", - "base_url":"http://NODE_S_A013eaa884d3c59fab:8983/solr", - "node_name":"NODE_S_A013eaa884d3c59fab:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard16":{ - "range":"eaaa0000-f1c6ffff", - "state":"active", - "replicas":{ - "core_node176":{ - "core":"COLL_E_v2_5_shard16_replica_n175", - "base_url":"http://NODE_S_A08c1ec5957d8a5d44:8983/solr", - "node_name":"NODE_S_A08c1ec5957d8a5d44:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node380":{ - "core":"COLL_E_v2_5_shard16_replica_n379", - "base_url":"http://NODE_S_A0aaaf34e6b281b24c:8983/solr", - "node_name":"NODE_S_A0aaaf34e6b281b24c:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}}, - "shard17":{ - "range":"f1c70000-f8e2ffff", - "state":"active", - "replicas":{ - "core_node212":{ - "core":"COLL_E_v2_5_shard17_replica_n211", - "base_url":"http://NODE_S_A04a75744569a27ccf:8983/solr", - "node_name":"NODE_S_A04a75744569a27ccf:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node214":{ - "core":"COLL_E_v2_5_shard17_replica_n213", - "base_url":"http://NODE_S_A02f6288aa3fc9eb3d:8983/solr", - "node_name":"NODE_S_A02f6288aa3fc9eb3d:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard18":{ - "range":"f8e30000-ffffffff", - "state":"active", - "replicas":{ - "core_node242":{ - "core":"COLL_E_v2_5_shard18_replica_n241", - "base_url":"http://NODE_S_A02f6288aa3fc9eb3d:8983/solr", - "node_name":"NODE_S_A02f6288aa3fc9eb3d:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node244":{ - "core":"COLL_E_v2_5_shard18_replica_n243", - "base_url":"http://NODE_S_A082e313208af7da04:8983/solr", - "node_name":"NODE_S_A082e313208af7da04:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}}, - "shard19":{ - "range":"0-71bffff", - "state":"active", - "replicas":{ - "core_node77":{ - "core":"COLL_E_v2_5_shard19_replica_n74", - "base_url":"http://NODE_S_A0d1445647aaae8dd1:8983/solr", - "node_name":"NODE_S_A0d1445647aaae8dd1:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node196":{ - "core":"COLL_E_v2_5_shard19_replica_n195", - "base_url":"http://NODE_S_A0f66f4bec1ce8fc09:8983/solr", - "node_name":"NODE_S_A0f66f4bec1ce8fc09:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}}, - "shard20":{ - "range":"71c0000-e37ffff", - "state":"active", - "replicas":{ - "core_node81":{ - "core":"COLL_E_v2_5_shard20_replica_n78", - "base_url":"http://NODE_S_A0ccef54efe3178758:8983/solr", - "node_name":"NODE_S_A0ccef54efe3178758:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node246":{ - "core":"COLL_E_v2_5_shard20_replica_n245", - "base_url":"http://NODE_S_A09f108fb78272a03d:8983/solr", - "node_name":"NODE_S_A09f108fb78272a03d:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}}, - "shard21":{ - "range":"e380000-1554ffff", - "state":"active", - "replicas":{ - "core_node83":{ - "core":"COLL_E_v2_5_shard21_replica_n80", - "base_url":"http://NODE_S_A077430d186f0e1de2:8983/solr", - "node_name":"NODE_S_A077430d186f0e1de2:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node85":{ - "core":"COLL_E_v2_5_shard21_replica_n82", - "base_url":"http://NODE_S_A0a3cb04c83b693d80:8983/solr", - "node_name":"NODE_S_A0a3cb04c83b693d80:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard22":{ - "range":"15550000-1c70ffff", - "state":"active", - "replicas":{ - "core_node156":{ - "core":"COLL_E_v2_5_shard22_replica_n155", - "base_url":"http://NODE_S_A08c1ec5957d8a5d44:8983/solr", - "node_name":"NODE_S_A08c1ec5957d8a5d44:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node178":{ - "core":"COLL_E_v2_5_shard22_replica_n177", - "base_url":"http://NODE_S_A084aef0a141a43572:8983/solr", - "node_name":"NODE_S_A084aef0a141a43572:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}}, - "shard23":{ - "range":"1c710000-238dffff", - "state":"active", - "replicas":{ - "core_node232":{ - "core":"COLL_E_v2_5_shard23_replica_n231", - "base_url":"http://NODE_S_A0dc21b7f9ef048b48:8983/solr", - "node_name":"NODE_S_A0dc21b7f9ef048b48:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node396":{ - "core":"COLL_E_v2_5_shard23_replica_n395", - "base_url":"http://NODE_S_A09aede91f0e8c147c:8983/solr", - "node_name":"NODE_S_A09aede91f0e8c147c:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}}, - "shard24":{ - "range":"238e0000-2aa9ffff", - "state":"active", - "replicas":{ - "core_node180":{ - "core":"COLL_E_v2_5_shard24_replica_n179", - "base_url":"http://NODE_S_A061d3ac3c71d42de2:8983/solr", - "node_name":"NODE_S_A061d3ac3c71d42de2:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node182":{ - "core":"COLL_E_v2_5_shard24_replica_n181", - "base_url":"http://NODE_S_A06032b38152c0f019:8983/solr", - "node_name":"NODE_S_A06032b38152c0f019:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}}, - "shard25":{ - "range":"2aaa0000-31c6ffff", - "state":"active", - "replicas":{ - "core_node218":{ - "core":"COLL_E_v2_5_shard25_replica_n217", - "base_url":"http://NODE_S_A006b71d81aedd8577:8983/solr", - "node_name":"NODE_S_A006b71d81aedd8577:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node360":{ - "core":"COLL_E_v2_5_shard25_replica_n359", - "base_url":"http://NODE_S_A0d391cf8eeaca59cd:8983/solr", - "node_name":"NODE_S_A0d391cf8eeaca59cd:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}}, - "shard26":{ - "range":"31c70000-38e2ffff", - "state":"active", - "replicas":{ - "core_node248":{ - "core":"COLL_E_v2_5_shard26_replica_n247", - "base_url":"http://NODE_S_A0a0ffe317a4f85c55:8983/solr", - "node_name":"NODE_S_A0a0ffe317a4f85c55:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node366":{ - "core":"COLL_E_v2_5_shard26_replica_n365", - "base_url":"http://NODE_S_A03baeedfd28ba05ba:8983/solr", - "node_name":"NODE_S_A03baeedfd28ba05ba:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}}, - "shard27":{ - "range":"38e30000-3fffffff", - "state":"active", - "replicas":{ - "core_node109":{ - "core":"COLL_E_v2_5_shard27_replica_n106", - "base_url":"http://NODE_S_A0d1445647aaae8dd1:8983/solr", - "node_name":"NODE_S_A0d1445647aaae8dd1:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node402":{ - "core":"COLL_E_v2_5_shard27_replica_n401", - "base_url":"http://NODE_S_A0d1f4439b8e8878e8:8983/solr", - "node_name":"NODE_S_A0d1f4439b8e8878e8:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}}, - "shard28":{ - "range":"40000000-471bffff", - "state":"active", - "replicas":{ - "core_node113":{ - "core":"COLL_E_v2_5_shard28_replica_n110", - "base_url":"http://NODE_S_A0ccef54efe3178758:8983/solr", - "node_name":"NODE_S_A0ccef54efe3178758:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node376":{ - "core":"COLL_E_v2_5_shard28_replica_n375", - "base_url":"http://NODE_S_A0bdba3561ff291bc4:8983/solr", - "node_name":"NODE_S_A0bdba3561ff291bc4:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}}, - "shard29":{ - "range":"471c0000-4e37ffff", - "state":"active", - "replicas":{ - "core_node222":{ - "core":"COLL_E_v2_5_shard29_replica_n221", - "base_url":"http://NODE_S_A0f096665ad4a3bc58:8983/solr", - "node_name":"NODE_S_A0f096665ad4a3bc58:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node392":{ - "core":"COLL_E_v2_5_shard29_replica_n391", - "base_url":"http://NODE_S_A09f108fb78272a03d:8983/solr", - "node_name":"NODE_S_A09f108fb78272a03d:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}}, - "shard30":{ - "range":"4e380000-5554ffff", - "state":"active", - "replicas":{ - "core_node160":{ - "core":"COLL_E_v2_5_shard30_replica_n159", - "base_url":"http://NODE_S_A061d3ac3c71d42de2:8983/solr", - "node_name":"NODE_S_A061d3ac3c71d42de2:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node184":{ - "core":"COLL_E_v2_5_shard30_replica_n183", - "base_url":"http://NODE_S_A084aef0a141a43572:8983/solr", - "node_name":"NODE_S_A084aef0a141a43572:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}}, - "shard31":{ - "range":"55550000-5c70ffff", - "state":"active", - "replicas":{ - "core_node125":{ - "core":"COLL_E_v2_5_shard31_replica_n122", - "base_url":"http://NODE_S_A0ee1c34fb3e9c261d:8983/solr", - "node_name":"NODE_S_A0ee1c34fb3e9c261d:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node362":{ - "core":"COLL_E_v2_5_shard31_replica_n361", - "base_url":"http://NODE_S_A0b7fb52d8d0a2ecde:8983/solr", - "node_name":"NODE_S_A0b7fb52d8d0a2ecde:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard32":{ - "range":"5c710000-638dffff", - "state":"active", - "replicas":{ - "core_node127":{ - "core":"COLL_E_v2_5_shard32_replica_n124", - "base_url":"http://NODE_S_A0ee1c34fb3e9c261d:8983/solr", - "node_name":"NODE_S_A0ee1c34fb3e9c261d:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node364":{ - "core":"COLL_E_v2_5_shard32_replica_n363", - "base_url":"http://NODE_S_A0aaaf34e6b281b24c:8983/solr", - "node_name":"NODE_S_A0aaaf34e6b281b24c:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard33":{ - "range":"638e0000-6aa9ffff", - "state":"active", - "replicas":{ - "core_node186":{ - "core":"COLL_E_v2_5_shard33_replica_n185", - "base_url":"http://NODE_S_A04a75744569a27ccf:8983/solr", - "node_name":"NODE_S_A04a75744569a27ccf:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node188":{ - "core":"COLL_E_v2_5_shard33_replica_n187", - "base_url":"http://NODE_S_A02f6288aa3fc9eb3d:8983/solr", - "node_name":"NODE_S_A02f6288aa3fc9eb3d:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard34":{ - "range":"6aaa0000-71c6ffff", - "state":"active", - "replicas":{ - "core_node226":{ - "core":"COLL_E_v2_5_shard34_replica_n225", - "base_url":"http://NODE_S_A0cf36cc11a8447fac:8983/solr", - "node_name":"NODE_S_A0cf36cc11a8447fac:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node384":{ - "core":"COLL_E_v2_5_shard34_replica_n383", - "base_url":"http://NODE_S_A0b7fb52d8d0a2ecde:8983/solr", - "node_name":"NODE_S_A0b7fb52d8d0a2ecde:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}}, - "shard35":{ - "range":"71c70000-78e2ffff", - "state":"active", - "replicas":{ - "core_node141":{ - "core":"COLL_E_v2_5_shard35_replica_n138", - "base_url":"http://NODE_S_A09aede91f0e8c147c:8983/solr", - "node_name":"NODE_S_A09aede91f0e8c147c:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node374":{ - "core":"COLL_E_v2_5_shard35_replica_n373", - "base_url":"http://NODE_S_A0d1f4439b8e8878e8:8983/solr", - "node_name":"NODE_S_A0d1f4439b8e8878e8:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard36":{ - "range":"78e30000-7fffffff", - "state":"active", - "replicas":{ - "core_node358":{ - "core":"COLL_E_v2_5_shard36_replica_n357", - "base_url":"http://NODE_S_A0e48ab035b869305e:8983/solr", - "node_name":"NODE_S_A0e48ab035b869305e:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node400":{ - "core":"COLL_E_v2_5_shard36_replica_n399", - "base_url":"http://NODE_S_A086a3a5ff0617c80f:8983/solr", - "node_name":"NODE_S_A086a3a5ff0617c80f:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}}}, - "router":{"name":"compositeId"}, - "autoAddReplicas":"false", - "nrtReplicas":"1", - "tlogReplicas":"0", - "znodeVersion":124109, - "configName":"COLL_E_v2_5"}, - "system_metrics":{ - "pullReplicas":"0", - "replicationFactor":"2", - "shards":{ - "shard1":{ - "range":"80000000-ffffffff", - "state":"active", - "replicas":{ - "core_node3":{ - "core":"system_metrics_shard1_replica_n1", - "base_url":"http://NODE_S_A0dc21b7f9ef048b48:8983/solr", - "node_name":"NODE_S_A0dc21b7f9ef048b48:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node5":{ - "core":"system_metrics_shard1_replica_n2", - "base_url":"http://NODE_S_A077430d186f0e1de2:8983/solr", - "node_name":"NODE_S_A077430d186f0e1de2:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}}, - "shard2":{ - "range":"0-7fffffff", - "state":"active", - "replicas":{ - "core_node7":{ - "core":"system_metrics_shard2_replica_n4", - "base_url":"http://NODE_S_A0ee1c34fb3e9c261d:8983/solr", - "node_name":"NODE_S_A0ee1c34fb3e9c261d:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node8":{ - "core":"system_metrics_shard2_replica_n6", - "base_url":"http://NODE_S_A0d1445647aaae8dd1:8983/solr", - "node_name":"NODE_S_A0d1445647aaae8dd1:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}}}, - "router":{"name":"compositeId"}, - "autoAddReplicas":"false", - "nrtReplicas":"1", - "tlogReplicas":"0", - "znodeVersion":202, - "configName":"system_metrics"}, - "S_B_2":{ - "pullReplicas":"0", - "replicationFactor":"1", - "shards":{ - "shard1":{ - "range":"80000000-9c70ffff", - "state":"active", - "replicas":{"core_node3":{ - "core":"S_B_2_shard1_replica_n1", - "base_url":"http://NODE_S_A0ccef54efe3178758:8983/solr", - "node_name":"NODE_S_A0ccef54efe3178758:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard2":{ - "range":"9c710000-b8e2ffff", - "state":"active", - "replicas":{"core_node5":{ - "core":"S_B_2_shard2_replica_n2", - "base_url":"http://NODE_S_A09aede91f0e8c147c:8983/solr", - "node_name":"NODE_S_A09aede91f0e8c147c:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard3":{ - "range":"b8e30000-d554ffff", - "state":"active", - "replicas":{"core_node7":{ - "core":"S_B_2_shard3_replica_n4", - "base_url":"http://NODE_S_A0dc21b7f9ef048b48:8983/solr", - "node_name":"NODE_S_A0dc21b7f9ef048b48:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard4":{ - "range":"d5550000-f1c6ffff", - "state":"active", - "replicas":{"core_node9":{ - "core":"S_B_2_shard4_replica_n6", - "base_url":"http://NODE_S_A0afcb9535d2619dac:8983/solr", - "node_name":"NODE_S_A0afcb9535d2619dac:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard5":{ - "range":"f1c70000-e37ffff", - "state":"active", - "replicas":{"core_node11":{ - "core":"S_B_2_shard5_replica_n8", - "base_url":"http://NODE_S_A0a3cb04c83b693d80:8983/solr", - "node_name":"NODE_S_A0a3cb04c83b693d80:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard6":{ - "range":"e380000-2aa9ffff", - "state":"active", - "replicas":{"core_node13":{ - "core":"S_B_2_shard6_replica_n10", - "base_url":"http://NODE_S_A024a9c138d972d437:8983/solr", - "node_name":"NODE_S_A024a9c138d972d437:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard7":{ - "range":"2aaa0000-471bffff", - "state":"active", - "replicas":{"core_node15":{ - "core":"S_B_2_shard7_replica_n12", - "base_url":"http://NODE_S_A0ee1c34fb3e9c261d:8983/solr", - "node_name":"NODE_S_A0ee1c34fb3e9c261d:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard8":{ - "range":"471c0000-638dffff", - "state":"active", - "replicas":{"core_node17":{ - "core":"S_B_2_shard8_replica_n14", - "base_url":"http://NODE_S_A077430d186f0e1de2:8983/solr", - "node_name":"NODE_S_A077430d186f0e1de2:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard9":{ - "range":"638e0000-7fffffff", - "state":"active", - "replicas":{"core_node18":{ - "core":"S_B_2_shard9_replica_n16", - "base_url":"http://NODE_S_A0d1445647aaae8dd1:8983/solr", - "node_name":"NODE_S_A0d1445647aaae8dd1:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}}, - "router":{"name":"compositeId"}, - "autoAddReplicas":"false", - "nrtReplicas":"1", - "tlogReplicas":"0", - "znodeVersion":348, - "configName":"S_B_2"}, - "COLL_S_R_aggr":{ - "pullReplicas":"0", - "replicationFactor":"2", - "shards":{ - "shard1":{ - "range":"80000000-ffffffff", - "state":"active", - "replicas":{ - "core_node3":{ - "core":"COLL_S_R_aggr_shard1_replica_n1", - "base_url":"http://NODE_S_A0a3cb04c83b693d80:8983/solr", - "node_name":"NODE_S_A0a3cb04c83b693d80:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node5":{ - "core":"COLL_S_R_aggr_shard1_replica_n2", - "base_url":"http://NODE_S_A0ee1c34fb3e9c261d:8983/solr", - "node_name":"NODE_S_A0ee1c34fb3e9c261d:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}}, - "shard2":{ - "range":"0-7fffffff", - "state":"active", - "replicas":{ - "core_node7":{ - "core":"COLL_S_R_aggr_shard2_replica_n4", - "base_url":"http://NODE_S_A0ccef54efe3178758:8983/solr", - "node_name":"NODE_S_A0ccef54efe3178758:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node8":{ - "core":"COLL_S_R_aggr_shard2_replica_n6", - "base_url":"http://NODE_S_A024a9c138d972d437:8983/solr", - "node_name":"NODE_S_A024a9c138d972d437:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}}}, - "router":{"name":"compositeId"}, - "autoAddReplicas":"false", - "nrtReplicas":"1", - "tlogReplicas":"0", - "znodeVersion":357, - "configName":"COLL_S_R_aggr"}, - "recall_cache":{ - "pullReplicas":"0", - "replicationFactor":"3", - "shards":{"shard1":{ - "range":"80000000-7fffffff", - "state":"active", - "replicas":{ - "core_node3":{ - "core":"recall_cache_shard1_replica_n1", - "base_url":"http://NODE_S_A0d391cf8eeaca59cd:8983/solr", - "node_name":"NODE_S_A0d391cf8eeaca59cd:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node5":{ - "core":"recall_cache_shard1_replica_n2", - "base_url":"http://NODE_S_A0e48ab035b869305e:8983/solr", - "node_name":"NODE_S_A0e48ab035b869305e:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node6":{ - "core":"recall_cache_shard1_replica_n4", - "base_url":"http://NODE_S_A0b7fb52d8d0a2ecde:8983/solr", - "node_name":"NODE_S_A0b7fb52d8d0a2ecde:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}}}, - "router":{"name":"compositeId"}, - "autoAddReplicas":"false", - "nrtReplicas":"1", - "tlogReplicas":"0", - "znodeVersion":24, - "configName":"recall_cache"}, - "COLL_S_E_15_signals_aggr":{ - "pullReplicas":"0", - "replicationFactor":"2", - "shards":{ - "shard1":{ - "range":"80000000-ffffffff", - "state":"active", - "replicas":{ - "core_node3":{ - "core":"COLL_S_E_15_signals_aggr_shard1_replica_n1", - "base_url":"http://NODE_S_A0ccef54efe3178758:8983/solr", - "node_name":"NODE_S_A0ccef54efe3178758:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node5":{ - "core":"COLL_S_E_15_signals_aggr_shard1_replica_n2", - "base_url":"http://NODE_S_A09aede91f0e8c147c:8983/solr", - "node_name":"NODE_S_A09aede91f0e8c147c:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}}, - "shard2":{ - "range":"0-7fffffff", - "state":"active", - "replicas":{ - "core_node7":{ - "core":"COLL_S_E_15_signals_aggr_shard2_replica_n4", - "base_url":"http://NODE_S_A0d1445647aaae8dd1:8983/solr", - "node_name":"NODE_S_A0d1445647aaae8dd1:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node8":{ - "core":"COLL_S_E_15_signals_aggr_shard2_replica_n6", - "base_url":"http://NODE_S_A0dc21b7f9ef048b48:8983/solr", - "node_name":"NODE_S_A0dc21b7f9ef048b48:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}}, - "router":{"name":"compositeId"}, - "autoAddReplicas":"false", - "nrtReplicas":"1", - "tlogReplicas":"0", - "znodeVersion":213, - "configName":"COLL_S_E_15_signals_aggr"}, - "COLL_E_V2_S_A_3":{ - "pullReplicas":"0", - "replicationFactor":"2", - "shards":{ - "shard1":{ - "range":"80000000-ffffffff", - "state":"active", - "replicas":{ - "core_node5":{ - "core":"COLL_E_V2_S_A_3_shard1_replica_n2", - "base_url":"http://NODE_S_A0dc21b7f9ef048b48:8983/solr", - "node_name":"NODE_S_A0dc21b7f9ef048b48:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node10":{ - "core":"COLL_E_V2_S_A_3_shard1_replica_n9", - "base_url":"http://NODE_S_A0d1f4439b8e8878e8:8983/solr", - "node_name":"NODE_S_A0d1f4439b8e8878e8:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard2":{ - "range":"0-7fffffff", - "state":"active", - "replicas":{ - "core_node7":{ - "core":"COLL_E_V2_S_A_3_shard2_replica_n4", - "base_url":"http://NODE_S_A077430d186f0e1de2:8983/solr", - "node_name":"NODE_S_A077430d186f0e1de2:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node8":{ - "core":"COLL_E_V2_S_A_3_shard2_replica_n6", - "base_url":"http://NODE_S_A0a3cb04c83b693d80:8983/solr", - "node_name":"NODE_S_A0a3cb04c83b693d80:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}}, - "router":{"name":"compositeId"}, - "autoAddReplicas":"false", - "nrtReplicas":"1", - "tlogReplicas":"0", - "znodeVersion":522, - "configName":"COLL_E_V2_S_A_3"}, - "COLL_N_Q_C_T_11":{ - "pullReplicas":"0", - "replicationFactor":"1", - "shards":{ - "shard1":{ - "range":"80000000-9c70ffff", - "state":"active", - "replicas":{"core_node3":{ - "core":"COLL_N_Q_C_T_11_shard1_replica_n1", - "base_url":"http://NODE_S_A0afcb9535d2619dac:8983/solr", - "node_name":"NODE_S_A0afcb9535d2619dac:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard2":{ - "range":"9c710000-b8e2ffff", - "state":"active", - "replicas":{"core_node8":{ - "core":"COLL_N_Q_C_T_11_shard2_replica_n2", - "base_url":"http://NODE_S_A0afcb9535d2619dac:8983/solr", - "node_name":"NODE_S_A0afcb9535d2619dac:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard3":{ - "range":"b8e30000-d554ffff", - "state":"active", - "replicas":{"core_node9":{ - "core":"COLL_N_Q_C_T_11_shard3_replica_n4", - "base_url":"http://NODE_S_A0ee1c34fb3e9c261d:8983/solr", - "node_name":"NODE_S_A0ee1c34fb3e9c261d:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard4":{ - "range":"d5550000-f1c6ffff", - "state":"active", - "replicas":{"core_node10":{ - "core":"COLL_N_Q_C_T_11_shard4_replica_n5", - "base_url":"http://NODE_S_A0dc21b7f9ef048b48:8983/solr", - "node_name":"NODE_S_A0dc21b7f9ef048b48:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard5":{ - "range":"f1c70000-e37ffff", - "state":"active", - "replicas":{"core_node12":{ - "core":"COLL_N_Q_C_T_11_shard5_replica_n6", - "base_url":"http://NODE_S_A0ccef54efe3178758:8983/solr", - "node_name":"NODE_S_A0ccef54efe3178758:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard6":{ - "range":"e380000-2aa9ffff", - "state":"active", - "replicas":{"core_node13":{ - "core":"COLL_N_Q_C_T_11_shard6_replica_n7", - "base_url":"http://NODE_S_A0a3cb04c83b693d80:8983/solr", - "node_name":"NODE_S_A0a3cb04c83b693d80:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard7":{ - "range":"2aaa0000-471bffff", - "state":"active", - "replicas":{"core_node15":{ - "core":"COLL_N_Q_C_T_11_shard7_replica_n11", - "base_url":"http://NODE_S_A09aede91f0e8c147c:8983/solr", - "node_name":"NODE_S_A09aede91f0e8c147c:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard8":{ - "range":"471c0000-638dffff", - "state":"active", - "replicas":{"core_node17":{ - "core":"COLL_N_Q_C_T_11_shard8_replica_n14", - "base_url":"http://NODE_S_A077430d186f0e1de2:8983/solr", - "node_name":"NODE_S_A077430d186f0e1de2:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard9":{ - "range":"638e0000-7fffffff", - "state":"active", - "replicas":{"core_node18":{ - "core":"COLL_N_Q_C_T_11_shard9_replica_n16", - "base_url":"http://NODE_S_A024a9c138d972d437:8983/solr", - "node_name":"NODE_S_A024a9c138d972d437:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}}, - "router":{"name":"compositeId"}, - "autoAddReplicas":"false", - "nrtReplicas":"1", - "tlogReplicas":"0", - "znodeVersion":135, - "configName":"COLL_N_Q_C_T_11"}, - "subreddit_experiment_results":{ - "pullReplicas":"0", - "replicationFactor":"1", - "shards":{ - "shard1":{ - "range":"80000000-897affff", - "state":"active", - "replicas":{"core_node3":{ - "core":"subreddit_experiment_results_shard1_replica_n1", - "base_url":"http://NODE_S_A0f096665ad4a3bc58:8983/solr", - "node_name":"NODE_S_A0f096665ad4a3bc58:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard2":{ - "range":"897b0000-92f5ffff", - "state":"active", - "replicas":{"core_node5":{ - "core":"subreddit_experiment_results_shard2_replica_n2", - "base_url":"http://NODE_S_A0d391cf8eeaca59cd:8983/solr", - "node_name":"NODE_S_A0d391cf8eeaca59cd:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard3":{ - "range":"92f60000-9c70ffff", - "state":"active", - "replicas":{"core_node7":{ - "core":"subreddit_experiment_results_shard3_replica_n4", - "base_url":"http://NODE_S_A08c1ec5957d8a5d44:8983/solr", - "node_name":"NODE_S_A08c1ec5957d8a5d44:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard4":{ - "range":"9c710000-a5ecffff", - "state":"active", - "replicas":{"core_node9":{ - "core":"subreddit_experiment_results_shard4_replica_n6", - "base_url":"http://NODE_S_A0cf36cc11a8447fac:8983/solr", - "node_name":"NODE_S_A0cf36cc11a8447fac:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard5":{ - "range":"a5ed0000-af67ffff", - "state":"active", - "replicas":{"core_node11":{ - "core":"subreddit_experiment_results_shard5_replica_n8", - "base_url":"http://NODE_S_A0d1f4439b8e8878e8:8983/solr", - "node_name":"NODE_S_A0d1f4439b8e8878e8:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard6":{ - "range":"af680000-b8e2ffff", - "state":"active", - "replicas":{"core_node13":{ - "core":"subreddit_experiment_results_shard6_replica_n10", - "base_url":"http://NODE_S_A0e48ab035b869305e:8983/solr", - "node_name":"NODE_S_A0e48ab035b869305e:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard7":{ - "range":"b8e30000-c25dffff", - "state":"active", - "replicas":{"core_node15":{ - "core":"subreddit_experiment_results_shard7_replica_n12", - "base_url":"http://NODE_S_A0afcb9535d2619dac:8983/solr", - "node_name":"NODE_S_A0afcb9535d2619dac:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard8":{ - "range":"c25e0000-cbd9ffff", - "state":"active", - "replicas":{"core_node17":{ - "core":"subreddit_experiment_results_shard8_replica_n14", - "base_url":"http://NODE_S_A0f66f4bec1ce8fc09:8983/solr", - "node_name":"NODE_S_A0f66f4bec1ce8fc09:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard9":{ - "range":"cbda0000-d554ffff", - "state":"active", - "replicas":{"core_node19":{ - "core":"subreddit_experiment_results_shard9_replica_n16", - "base_url":"http://NODE_S_A0b7fb52d8d0a2ecde:8983/solr", - "node_name":"NODE_S_A0b7fb52d8d0a2ecde:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard10":{ - "range":"d5550000-decfffff", - "state":"active", - "replicas":{"core_node21":{ - "core":"subreddit_experiment_results_shard10_replica_n18", - "base_url":"http://NODE_S_A084aef0a141a43572:8983/solr", - "node_name":"NODE_S_A084aef0a141a43572:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard11":{ - "range":"ded00000-e84affff", - "state":"active", - "replicas":{"core_node23":{ - "core":"subreddit_experiment_results_shard11_replica_n20", - "base_url":"http://NODE_S_A0aaaf34e6b281b24c:8983/solr", - "node_name":"NODE_S_A0aaaf34e6b281b24c:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard12":{ - "range":"e84b0000-f1c6ffff", - "state":"active", - "replicas":{"core_node25":{ - "core":"subreddit_experiment_results_shard12_replica_n22", - "base_url":"http://NODE_S_A0ebcce85ef9c13308:8983/solr", - "node_name":"NODE_S_A0ebcce85ef9c13308:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard13":{ - "range":"f1c70000-fb41ffff", - "state":"active", - "replicas":{"core_node27":{ - "core":"subreddit_experiment_results_shard13_replica_n24", - "base_url":"http://NODE_S_A0ebcce85ef9c13308:8983/solr", - "node_name":"NODE_S_A0ebcce85ef9c13308:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard14":{ - "range":"fb420000-4bcffff", - "state":"active", - "replicas":{"core_node29":{ - "core":"subreddit_experiment_results_shard14_replica_n26", - "base_url":"http://NODE_S_A0aaaf34e6b281b24c:8983/solr", - "node_name":"NODE_S_A0aaaf34e6b281b24c:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard15":{ - "range":"4bd0000-e37ffff", - "state":"active", - "replicas":{"core_node31":{ - "core":"subreddit_experiment_results_shard15_replica_n28", - "base_url":"http://NODE_S_A084aef0a141a43572:8983/solr", - "node_name":"NODE_S_A084aef0a141a43572:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard16":{ - "range":"e380000-17b3ffff", - "state":"active", - "replicas":{"core_node33":{ - "core":"subreddit_experiment_results_shard16_replica_n30", - "base_url":"http://NODE_S_A0b7fb52d8d0a2ecde:8983/solr", - "node_name":"NODE_S_A0b7fb52d8d0a2ecde:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard17":{ - "range":"17b40000-212effff", - "state":"active", - "replicas":{"core_node35":{ - "core":"subreddit_experiment_results_shard17_replica_n32", - "base_url":"http://NODE_S_A0f66f4bec1ce8fc09:8983/solr", - "node_name":"NODE_S_A0f66f4bec1ce8fc09:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard18":{ - "range":"212f0000-2aa9ffff", - "state":"active", - "replicas":{"core_node37":{ - "core":"subreddit_experiment_results_shard18_replica_n34", - "base_url":"http://NODE_S_A0afcb9535d2619dac:8983/solr", - "node_name":"NODE_S_A0afcb9535d2619dac:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard19":{ - "range":"2aaa0000-3424ffff", - "state":"active", - "replicas":{"core_node39":{ - "core":"subreddit_experiment_results_shard19_replica_n36", - "base_url":"http://NODE_S_A0e48ab035b869305e:8983/solr", - "node_name":"NODE_S_A0e48ab035b869305e:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard20":{ - "range":"34250000-3da0ffff", - "state":"active", - "replicas":{"core_node41":{ - "core":"subreddit_experiment_results_shard20_replica_n38", - "base_url":"http://NODE_S_A0d1f4439b8e8878e8:8983/solr", - "node_name":"NODE_S_A0d1f4439b8e8878e8:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard21":{ - "range":"3da10000-471bffff", - "state":"active", - "replicas":{"core_node43":{ - "core":"subreddit_experiment_results_shard21_replica_n40", - "base_url":"http://NODE_S_A0cf36cc11a8447fac:8983/solr", - "node_name":"NODE_S_A0cf36cc11a8447fac:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard22":{ - "range":"471c0000-5096ffff", - "state":"active", - "replicas":{"core_node45":{ - "core":"subreddit_experiment_results_shard22_replica_n42", - "base_url":"http://NODE_S_A08c1ec5957d8a5d44:8983/solr", - "node_name":"NODE_S_A08c1ec5957d8a5d44:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard23":{ - "range":"50970000-5a11ffff", - "state":"active", - "replicas":{"core_node47":{ - "core":"subreddit_experiment_results_shard23_replica_n44", - "base_url":"http://NODE_S_A0d391cf8eeaca59cd:8983/solr", - "node_name":"NODE_S_A0d391cf8eeaca59cd:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard24":{ - "range":"5a120000-638dffff", - "state":"active", - "replicas":{"core_node49":{ - "core":"subreddit_experiment_results_shard24_replica_n46", - "base_url":"http://NODE_S_A0f096665ad4a3bc58:8983/solr", - "node_name":"NODE_S_A0f096665ad4a3bc58:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard25":{ - "range":"638e0000-6d08ffff", - "state":"active", - "replicas":{"core_node51":{ - "core":"subreddit_experiment_results_shard25_replica_n48", - "base_url":"http://NODE_S_A061d3ac3c71d42de2:8983/solr", - "node_name":"NODE_S_A061d3ac3c71d42de2:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard26":{ - "range":"6d090000-7683ffff", - "state":"active", - "replicas":{"core_node53":{ - "core":"subreddit_experiment_results_shard26_replica_n50", - "base_url":"http://NODE_S_A077430d186f0e1de2:8983/solr", - "node_name":"NODE_S_A077430d186f0e1de2:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard27":{ - "range":"76840000-7fffffff", - "state":"active", - "replicas":{"core_node54":{ - "core":"subreddit_experiment_results_shard27_replica_n52", - "base_url":"http://NODE_S_A04a75744569a27ccf:8983/solr", - "node_name":"NODE_S_A04a75744569a27ccf:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}}, - "router":{"name":"compositeId"}, - "autoAddReplicas":"false", - "nrtReplicas":"1", - "tlogReplicas":"0", - "znodeVersion":87, - "configName":"subreddit_experiment_results"}, - "logs":{ - "pullReplicas":"0", - "replicationFactor":"2", - "shards":{ - "shard1":{ - "range":"80000000-ffffffff", - "state":"active", - "replicas":{ - "core_node3":{ - "core":"logs_shard1_replica_n1", - "base_url":"http://NODE_S_A0ee1c34fb3e9c261d:8983/solr", - "node_name":"NODE_S_A0ee1c34fb3e9c261d:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node5":{ - "core":"logs_shard1_replica_n2", - "base_url":"http://NODE_S_A0a3cb04c83b693d80:8983/solr", - "node_name":"NODE_S_A0a3cb04c83b693d80:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard2":{ - "range":"0-7fffffff", - "state":"active", - "replicas":{ - "core_node7":{ - "core":"logs_shard2_replica_n4", - "base_url":"http://NODE_S_A077430d186f0e1de2:8983/solr", - "node_name":"NODE_S_A077430d186f0e1de2:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node8":{ - "core":"logs_shard2_replica_n6", - "base_url":"http://NODE_S_A0ccef54efe3178758:8983/solr", - "node_name":"NODE_S_A0ccef54efe3178758:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}}, - "router":{"name":"compositeId"}, - "autoAddReplicas":"false", - "nrtReplicas":"1", - "tlogReplicas":"0", - "znodeVersion":4954, - "configName":"logs"}, - "COLL_S_J_H_9":{ - "pullReplicas":"0", - "replicationFactor":"2", - "shards":{ - "shard1":{ - "range":"80000000-ffffffff", - "state":"active", - "replicas":{ - "core_node3":{ - "core":"COLL_S_J_H_9_shard1_replica_n1", - "base_url":"http://NODE_S_A0dc21b7f9ef048b48:8983/solr", - "node_name":"NODE_S_A0dc21b7f9ef048b48:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node5":{ - "core":"COLL_S_J_H_9_shard1_replica_n2", - "base_url":"http://NODE_S_A0ee1c34fb3e9c261d:8983/solr", - "node_name":"NODE_S_A0ee1c34fb3e9c261d:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}}, - "shard2":{ - "range":"0-7fffffff", - "state":"active", - "replicas":{ - "core_node7":{ - "core":"COLL_S_J_H_9_shard2_replica_n4", - "base_url":"http://NODE_S_A0d1445647aaae8dd1:8983/solr", - "node_name":"NODE_S_A0d1445647aaae8dd1:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node8":{ - "core":"COLL_S_J_H_9_shard2_replica_n6", - "base_url":"http://NODE_S_A0a3cb04c83b693d80:8983/solr", - "node_name":"NODE_S_A0a3cb04c83b693d80:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}}, - "router":{"name":"compositeId"}, - "autoAddReplicas":"false", - "nrtReplicas":"1", - "tlogReplicas":"0", - "znodeVersion":231, - "configName":"COLL_S_J_H_9"}, - "COLL_S_E_15_signals":{ - "pullReplicas":"0", - "replicationFactor":"2", - "shards":{ - "shard1":{ - "range":"80000000-ffffffff", - "state":"active", - "replicas":{ - "core_node3":{ - "core":"COLL_S_E_15_signals_shard1_replica_n1", - "base_url":"http://NODE_S_A0a3cb04c83b693d80:8983/solr", - "node_name":"NODE_S_A0a3cb04c83b693d80:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node5":{ - "core":"COLL_S_E_15_signals_shard1_replica_n2", - "base_url":"http://NODE_S_A0d1445647aaae8dd1:8983/solr", - "node_name":"NODE_S_A0d1445647aaae8dd1:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}}, - "shard2":{ - "range":"0-7fffffff", - "state":"active", - "replicas":{ - "core_node7":{ - "core":"COLL_S_E_15_signals_shard2_replica_n4", - "base_url":"http://NODE_S_A024a9c138d972d437:8983/solr", - "node_name":"NODE_S_A024a9c138d972d437:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node8":{ - "core":"COLL_S_E_15_signals_shard2_replica_n6", - "base_url":"http://NODE_S_A0dc21b7f9ef048b48:8983/solr", - "node_name":"NODE_S_A0dc21b7f9ef048b48:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}}, - "router":{"name":"compositeId"}, - "autoAddReplicas":"false", - "nrtReplicas":"1", - "tlogReplicas":"0", - "znodeVersion":289, - "configName":"COLL_S_E_15_signals"}, - "COLL_PV_E_v2_5":{ - "pullReplicas":"0", - "replicationFactor":"2", - "shards":{ - "shard1":{ - "range":"80000000-d554ffff", - "state":"active", - "replicas":{ - "core_node3":{ - "core":"COLL_PV_E_v2_5_shard1_replica_n1", - "base_url":"http://NODE_S_A09aede91f0e8c147c:8983/solr", - "node_name":"NODE_S_A09aede91f0e8c147c:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node5":{ - "core":"COLL_PV_E_v2_5_shard1_replica_n2", - "base_url":"http://NODE_S_A0a3cb04c83b693d80:8983/solr", - "node_name":"NODE_S_A0a3cb04c83b693d80:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}}, - "shard2":{ - "range":"d5550000-2aa9ffff", - "state":"active", - "replicas":{ - "core_node7":{ - "core":"COLL_PV_E_v2_5_shard2_replica_n4", - "base_url":"http://NODE_S_A0ccef54efe3178758:8983/solr", - "node_name":"NODE_S_A0ccef54efe3178758:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node9":{ - "core":"COLL_PV_E_v2_5_shard2_replica_n6", - "base_url":"http://NODE_S_A0f096665ad4a3bc58:8983/solr", - "node_name":"NODE_S_A0f096665ad4a3bc58:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard3":{ - "range":"2aaa0000-7fffffff", - "state":"active", - "replicas":{ - "core_node11":{ - "core":"COLL_PV_E_v2_5_shard3_replica_n8", - "base_url":"http://NODE_S_A0ee1c34fb3e9c261d:8983/solr", - "node_name":"NODE_S_A0ee1c34fb3e9c261d:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node12":{ - "core":"COLL_PV_E_v2_5_shard3_replica_n10", - "base_url":"http://NODE_S_A0e48ab035b869305e:8983/solr", - "node_name":"NODE_S_A0e48ab035b869305e:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}}, - "router":{"name":"compositeId"}, - "autoAddReplicas":"false", - "nrtReplicas":"2", - "tlogReplicas":"0", - "znodeVersion":6, - "configName":"COLL_PV_E_v2_5"}, - "COLL_R_P_S_A_22":{ - "pullReplicas":"0", - "replicationFactor":"1", - "shards":{ - "shard1":{ - "range":"80000000-87c0ffff", - "state":"active", - "replicas":{"core_node3":{ - "core":"COLL_R_P_S_A_22_shard1_replica_n1", - "base_url":"http://NODE_S_A082e313208af7da04:8983/solr", - "node_name":"NODE_S_A082e313208af7da04:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard2":{ - "range":"87c10000-8f82ffff", - "state":"active", - "replicas":{"core_node5":{ - "core":"COLL_R_P_S_A_22_shard2_replica_n2", - "base_url":"http://NODE_S_A09f108fb78272a03d:8983/solr", - "node_name":"NODE_S_A09f108fb78272a03d:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard3":{ - "range":"8f830000-9744ffff", - "state":"active", - "replicas":{"core_node7":{ - "core":"COLL_R_P_S_A_22_shard3_replica_n4", - "base_url":"http://NODE_S_A0a0ffe317a4f85c55:8983/solr", - "node_name":"NODE_S_A0a0ffe317a4f85c55:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard4":{ - "range":"97450000-9f06ffff", - "state":"active", - "replicas":{"core_node9":{ - "core":"COLL_R_P_S_A_22_shard4_replica_n6", - "base_url":"http://NODE_S_A03baeedfd28ba05ba:8983/solr", - "node_name":"NODE_S_A03baeedfd28ba05ba:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard5":{ - "range":"9f070000-a6c8ffff", - "state":"active", - "replicas":{"core_node11":{ - "core":"COLL_R_P_S_A_22_shard5_replica_n8", - "base_url":"http://NODE_S_A013eaa884d3c59fab:8983/solr", - "node_name":"NODE_S_A013eaa884d3c59fab:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard6":{ - "range":"a6c90000-ae8affff", - "state":"active", - "replicas":{"core_node13":{ - "core":"COLL_R_P_S_A_22_shard6_replica_n10", - "base_url":"http://NODE_S_A086a3a5ff0617c80f:8983/solr", - "node_name":"NODE_S_A086a3a5ff0617c80f:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard7":{ - "range":"ae8b0000-b64cffff", - "state":"active", - "replicas":{"core_node15":{ - "core":"COLL_R_P_S_A_22_shard7_replica_n12", - "base_url":"http://NODE_S_A0bdba3561ff291bc4:8983/solr", - "node_name":"NODE_S_A0bdba3561ff291bc4:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard8":{ - "range":"b64d0000-be0effff", - "state":"active", - "replicas":{"core_node17":{ - "core":"COLL_R_P_S_A_22_shard8_replica_n14", - "base_url":"http://NODE_S_A0aaaf34e6b281b24c:8983/solr", - "node_name":"NODE_S_A0aaaf34e6b281b24c:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard9":{ - "range":"be0f0000-c5d0ffff", - "state":"active", - "replicas":{"core_node19":{ - "core":"COLL_R_P_S_A_22_shard9_replica_n16", - "base_url":"http://NODE_S_A0d391cf8eeaca59cd:8983/solr", - "node_name":"NODE_S_A0d391cf8eeaca59cd:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard10":{ - "range":"c5d10000-cd92ffff", - "state":"active", - "replicas":{"core_node21":{ - "core":"COLL_R_P_S_A_22_shard10_replica_n18", - "base_url":"http://NODE_S_A0b7fb52d8d0a2ecde:8983/solr", - "node_name":"NODE_S_A0b7fb52d8d0a2ecde:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard11":{ - "range":"cd930000-d554ffff", - "state":"active", - "replicas":{"core_node23":{ - "core":"COLL_R_P_S_A_22_shard11_replica_n20", - "base_url":"http://NODE_S_A0d1f4439b8e8878e8:8983/solr", - "node_name":"NODE_S_A0d1f4439b8e8878e8:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard12":{ - "range":"d5550000-dd16ffff", - "state":"active", - "replicas":{"core_node25":{ - "core":"COLL_R_P_S_A_22_shard12_replica_n22", - "base_url":"http://NODE_S_A0e48ab035b869305e:8983/solr", - "node_name":"NODE_S_A0e48ab035b869305e:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard13":{ - "range":"dd170000-e4d8ffff", - "state":"active", - "replicas":{"core_node27":{ - "core":"COLL_R_P_S_A_22_shard13_replica_n24", - "base_url":"http://NODE_S_A014292d062d5088da:8983/solr", - "node_name":"NODE_S_A014292d062d5088da:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard14":{ - "range":"e4d90000-ec9affff", - "state":"active", - "replicas":{"core_node29":{ - "core":"COLL_R_P_S_A_22_shard14_replica_n26", - "base_url":"http://NODE_S_A006b71d81aedd8577:8983/solr", - "node_name":"NODE_S_A006b71d81aedd8577:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard15":{ - "range":"ec9b0000-f45cffff", - "state":"active", - "replicas":{"core_node31":{ - "core":"COLL_R_P_S_A_22_shard15_replica_n28", - "base_url":"http://NODE_S_A06032b38152c0f019:8983/solr", - "node_name":"NODE_S_A06032b38152c0f019:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard16":{ - "range":"f45d0000-fc1effff", - "state":"active", - "replicas":{"core_node33":{ - "core":"COLL_R_P_S_A_22_shard16_replica_n30", - "base_url":"http://NODE_S_A061d3ac3c71d42de2:8983/solr", - "node_name":"NODE_S_A061d3ac3c71d42de2:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard17":{ - "range":"fc1f0000-3dfffff", - "state":"active", - "replicas":{"core_node35":{ - "core":"COLL_R_P_S_A_22_shard17_replica_n32", - "base_url":"http://NODE_S_A02f6288aa3fc9eb3d:8983/solr", - "node_name":"NODE_S_A02f6288aa3fc9eb3d:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard18":{ - "range":"3e00000-ba1ffff", - "state":"active", - "replicas":{"core_node37":{ - "core":"COLL_R_P_S_A_22_shard18_replica_n34", - "base_url":"http://NODE_S_A04a75744569a27ccf:8983/solr", - "node_name":"NODE_S_A04a75744569a27ccf:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard19":{ - "range":"ba20000-1363ffff", - "state":"active", - "replicas":{"core_node39":{ - "core":"COLL_R_P_S_A_22_shard19_replica_n36", - "base_url":"http://NODE_S_A08c1ec5957d8a5d44:8983/solr", - "node_name":"NODE_S_A08c1ec5957d8a5d44:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard20":{ - "range":"13640000-1b25ffff", - "state":"active", - "replicas":{"core_node41":{ - "core":"COLL_R_P_S_A_22_shard20_replica_n38", - "base_url":"http://NODE_S_A084aef0a141a43572:8983/solr", - "node_name":"NODE_S_A084aef0a141a43572:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard21":{ - "range":"1b260000-22e7ffff", - "state":"active", - "replicas":{"core_node43":{ - "core":"COLL_R_P_S_A_22_shard21_replica_n40", - "base_url":"http://NODE_S_A0f096665ad4a3bc58:8983/solr", - "node_name":"NODE_S_A0f096665ad4a3bc58:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard22":{ - "range":"22e80000-2aa9ffff", - "state":"active", - "replicas":{"core_node45":{ - "core":"COLL_R_P_S_A_22_shard22_replica_n42", - "base_url":"http://NODE_S_A0cf36cc11a8447fac:8983/solr", - "node_name":"NODE_S_A0cf36cc11a8447fac:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard23":{ - "range":"2aaa0000-326bffff", - "state":"active", - "replicas":{"core_node47":{ - "core":"COLL_R_P_S_A_22_shard23_replica_n44", - "base_url":"http://NODE_S_A09aede91f0e8c147c:8983/solr", - "node_name":"NODE_S_A09aede91f0e8c147c:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard24":{ - "range":"326c0000-3a2dffff", - "state":"active", - "replicas":{"core_node49":{ - "core":"COLL_R_P_S_A_22_shard24_replica_n46", - "base_url":"http://NODE_S_A0ebcce85ef9c13308:8983/solr", - "node_name":"NODE_S_A0ebcce85ef9c13308:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard25":{ - "range":"3a2e0000-41efffff", - "state":"active", - "replicas":{"core_node51":{ - "core":"COLL_R_P_S_A_22_shard25_replica_n48", - "base_url":"http://NODE_S_A0f66f4bec1ce8fc09:8983/solr", - "node_name":"NODE_S_A0f66f4bec1ce8fc09:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard26":{ - "range":"41f00000-49b1ffff", - "state":"active", - "replicas":{"core_node53":{ - "core":"COLL_R_P_S_A_22_shard26_replica_n50", - "base_url":"http://NODE_S_A0ee1c34fb3e9c261d:8983/solr", - "node_name":"NODE_S_A0ee1c34fb3e9c261d:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard27":{ - "range":"49b20000-5173ffff", - "state":"active", - "replicas":{"core_node55":{ - "core":"COLL_R_P_S_A_22_shard27_replica_n52", - "base_url":"http://NODE_S_A024a9c138d972d437:8983/solr", - "node_name":"NODE_S_A024a9c138d972d437:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard28":{ - "range":"51740000-5935ffff", - "state":"active", - "replicas":{"core_node57":{ - "core":"COLL_R_P_S_A_22_shard28_replica_n54", - "base_url":"http://NODE_S_A0ccef54efe3178758:8983/solr", - "node_name":"NODE_S_A0ccef54efe3178758:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard29":{ - "range":"59360000-60f7ffff", - "state":"active", - "replicas":{"core_node59":{ - "core":"COLL_R_P_S_A_22_shard29_replica_n56", - "base_url":"http://NODE_S_A0afcb9535d2619dac:8983/solr", - "node_name":"NODE_S_A0afcb9535d2619dac:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard30":{ - "range":"60f80000-68b9ffff", - "state":"active", - "replicas":{"core_node61":{ - "core":"COLL_R_P_S_A_22_shard30_replica_n58", - "base_url":"http://NODE_S_A0a3cb04c83b693d80:8983/solr", - "node_name":"NODE_S_A0a3cb04c83b693d80:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard31":{ - "range":"68ba0000-707bffff", - "state":"active", - "replicas":{"core_node63":{ - "core":"COLL_R_P_S_A_22_shard31_replica_n60", - "base_url":"http://NODE_S_A077430d186f0e1de2:8983/solr", - "node_name":"NODE_S_A077430d186f0e1de2:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard32":{ - "range":"707c0000-783dffff", - "state":"active", - "replicas":{"core_node65":{ - "core":"COLL_R_P_S_A_22_shard32_replica_n62", - "base_url":"http://NODE_S_A0d1445647aaae8dd1:8983/solr", - "node_name":"NODE_S_A0d1445647aaae8dd1:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard33":{ - "range":"783e0000-7fffffff", - "state":"active", - "replicas":{"core_node66":{ - "core":"COLL_R_P_S_A_22_shard33_replica_n64", - "base_url":"http://NODE_S_A0dc21b7f9ef048b48:8983/solr", - "node_name":"NODE_S_A0dc21b7f9ef048b48:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}}, - "router":{"name":"compositeId"}, - "autoAddReplicas":"false", - "nrtReplicas":"1", - "tlogReplicas":"0", - "znodeVersion":7, - "configName":"COLL_R_P_S_A_22"}}, - "live_nodes":["NODE_S_A0bdba3561ff291bc4:8983_solr", - "NODE_S_A0a0ffe317a4f85c55:8983_solr", - "NODE_S_A0ebcce85ef9c13308:8983_solr", - "NODE_S_A0a3cb04c83b693d80:8983_solr", - "NODE_S_A0ccef54efe3178758:8983_solr", - "NODE_S_A06032b38152c0f019:8983_solr", - "NODE_S_A024a9c138d972d437:8983_solr", - "NODE_S_A0dc21b7f9ef048b48:8983_solr", - "NODE_S_A086a3a5ff0617c80f:8983_solr", - "NODE_S_A006b71d81aedd8577:8983_solr", - "NODE_S_A013eaa884d3c59fab:8983_solr", - "NODE_S_A0aaaf34e6b281b24c:8983_solr", - "NODE_S_A09f108fb78272a03d:8983_solr", - "NODE_S_A084aef0a141a43572:8983_solr", - "NODE_S_A0ee1c34fb3e9c261d:8983_solr", - "NODE_S_A03baeedfd28ba05ba:8983_solr", - "NODE_S_A014292d062d5088da:8983_solr", - "NODE_S_A0d1445647aaae8dd1:8983_solr", - "NODE_S_A0b7fb52d8d0a2ecde:8983_solr", - "NODE_S_A02f6288aa3fc9eb3d:8983_solr", - "NODE_S_A09aede91f0e8c147c:8983_solr", - "NODE_S_A0f66f4bec1ce8fc09:8983_solr", - "NODE_S_A0afcb9535d2619dac:8983_solr", - "NODE_S_A0e48ab035b869305e:8983_solr", - "NODE_S_A0d1f4439b8e8878e8:8983_solr", - "NODE_S_A0cf36cc11a8447fac:8983_solr", - "NODE_S_A04a75744569a27ccf:8983_solr", - "NODE_S_A077430d186f0e1de2:8983_solr", - "NODE_S_A08c1ec5957d8a5d44:8983_solr", - "NODE_S_A061d3ac3c71d42de2:8983_solr", - "NODE_S_A082e313208af7da04:8983_solr", - "NODE_S_A0d391cf8eeaca59cd:8983_solr", - "NODE_S_A0f096665ad4a3bc58:8983_solr"]}} diff --git a/solr/solrj/src/test-files/solrj/solr/autoscaling/testMoveReplicaSuggester.json b/solr/solrj/src/test-files/solrj/solr/autoscaling/testMoveReplicaSuggester.json deleted file mode 100644 index e45c87f3f69..00000000000 --- a/solr/solrj/src/test-files/solrj/solr/autoscaling/testMoveReplicaSuggester.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "liveNodes":["10.0.0.6:7574_solr", - "10.0.0.6:8983_solr"], - "replicaInfo":{ - "10.0.0.6:7574_solr":{}, - "10.0.0.6:8983_solr":{"mycoll1":{ - "shard2":[{"core_node2":{"type":"NRT"}}], - "shard1":[{"core_node1":{"type":"NRT"}}]}}}, - "nodeValues":{ - "10.0.0.6:7574_solr":{ - "node":"10.0.0.6:7574_solr", - "cores":0}, - "10.0.0.6:8983_solr":{ - "node":"10.0.0.6:8983_solr", - "cores":2}}} \ No newline at end of file diff --git a/solr/solrj/src/test-files/solrj/solr/autoscaling/testMoveReplicasInMultipleCollections.json b/solr/solrj/src/test-files/solrj/solr/autoscaling/testMoveReplicasInMultipleCollections.json deleted file mode 100644 index b515050d19e..00000000000 --- a/solr/solrj/src/test-files/solrj/solr/autoscaling/testMoveReplicasInMultipleCollections.json +++ /dev/null @@ -1,86 +0,0 @@ -{"collection1":{ - "pullReplicas":"0", - "replicationFactor":"2", - "shards":{ - "shard1":{ - "range":"80000000-ffffffff", - "state":"active", - "replicas":{ - "core_node1":{ - "core":"collection1_shard1_replica_n1", - "base_url":"http://127.0.0.1:51650/solr", - "node_name":"node1", - "state":"active", - "type":"NRT", - "leader":"true"}, - "core_node6":{ - "core":"collection1_shard1_replica_n3", - "base_url":"http://127.0.0.1:51651/solr", - "node_name":"node3", - "state":"active", - "type":"NRT"}}}, - "shard2":{ - "range":"0-7fffffff", - "state":"active", - "replicas":{ - "core_node3":{ - "core":"collection1_shard2_replica_n1", - "base_url":"http://127.0.0.1:51650/solr", - "node_name":"node1", - "state":"active", - "type":"NRT", - "leader":"true"}, - "core_node5":{ - "core":"collection1_shard2_replica_n3", - "base_url":"http://127.0.0.1:51651/solr", - "node_name":"node3", - "state":"active", - "type":"NRT"}}}}, - "router":{ - "name":"compositeId"}, - "autoAddReplicas":"true", - "nrtReplicas":"2", - "tlogReplicas":"0"}, - "collection2":{ - "pullReplicas":"0", - "replicationFactor":"2", - "shards":{ - "shard1":{ - "range":"80000000-ffffffff", - "state":"active", - "replicas":{ - "core_node1":{ - "core":"collection2_shard1_replica_n1", - "base_url":"http://127.0.0.1:51649/solr", - "node_name":"node2", - "state":"active", - "type":"NRT"}, - "core_node2":{ - "core":"collection2_shard1_replica_n2", - "base_url":"http://127.0.0.1:51651/solr", - "node_name":"node3", - "state":"active", - "type":"NRT", - "leader":"true"}}}, - "shard2":{ - "range":"0-7fffffff", - "state":"active", - "replicas":{ - "core_node3":{ - "core":"collection2_shard2_replica_n1", - "base_url":"http://127.0.0.1:51649/solr", - "node_name":"node2", - "state":"active", - "type":"NRT"}, - "core_node4":{ - "core":"collection2_shard2_replica_n2", - "base_url":"http://127.0.0.1:51651/solr", - "node_name":"node3", - "state":"active", - "type":"NRT", - "leader":"true"}}}}, - "router":{ - "name":"compositeId"}, - "autoAddReplicas":"true", - "nrtReplicas":"2", - "tlogReplicas":"0"}} diff --git a/solr/solrj/src/test-files/solrj/solr/autoscaling/testPolicy.json b/solr/solrj/src/test-files/solrj/solr/autoscaling/testPolicy.json deleted file mode 100644 index 373607ee6a5..00000000000 --- a/solr/solrj/src/test-files/solrj/solr/autoscaling/testPolicy.json +++ /dev/null @@ -1,41 +0,0 @@ -{"gettingstarted":{ - "router":{"name":"compositeId"}, - "shards":{ - "shard1":{ - "range":"80000000-ffffffff", - "replicas":{ - "r1":{ - "core":"r1", - "base_url":"http://10.0.0.4:8983/solr", - "node_name":"node1", - "state":"active", - "leader":"true"}, - "r2":{ - "core":"r2", - "base_url":"http://10.0.0.4:7574/solr", - "node_name":"node2", - "state":"active"}}}, - "shard2":{ - "range":"0-7fffffff", - "replicas":{ - "r3":{ - "core":"r3", - "base_url":"http://10.0.0.4:8983/solr", - "node_name":"node1", - "state":"active", - "leader":"true"}, - "r4":{ - "core":"r4", - "base_url":"http://10.0.0.4:8987/solr", - "node_name":"node4", - "state":"active"}, - "r6":{ - "core":"r6", - "base_url":"http://10.0.0.4:8989/solr", - "node_name":"node3", - "state":"active"}, - "r5":{ - "core":"r5", - "base_url":"http://10.0.0.4:8983/solr", - "node_name":"node1", - "state":"active"}}}}}} \ No newline at end of file diff --git a/solr/solrj/src/test-files/solrj/solr/autoscaling/testPortSuggestions.json b/solr/solrj/src/test-files/solrj/solr/autoscaling/testPortSuggestions.json deleted file mode 100644 index f518de1ad55..00000000000 --- a/solr/solrj/src/test-files/solrj/solr/autoscaling/testPortSuggestions.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "liveNodes":["node1:8983", - "node2:8984", - "node3:8985"], - "replicaInfo":{"node1:8983":{"c1":{ - "s1":[{"r1":{"type":"NRT"}}, - {"r2":{"type":"NRT"}}], - "s2":[{"r1":{"type":"NRT"}}, - {"r2":{"type":"NRT"}}]}}}, - "nodeValues":{ - "node1:8983":{ - "cores":4, - "freedisk":334, - "port":8983}, - "node2:8984":{ - "cores":0, - "freedisk":1000, - "port":8984}, - "node3:8985":{ - "cores":0, - "freedisk":1500, - "port":8985}}} \ No newline at end of file diff --git a/solr/solrj/src/test-files/solrj/solr/autoscaling/testReplicaCountSuggestions.json b/solr/solrj/src/test-files/solrj/solr/autoscaling/testReplicaCountSuggestions.json deleted file mode 100644 index e45c87f3f69..00000000000 --- a/solr/solrj/src/test-files/solrj/solr/autoscaling/testReplicaCountSuggestions.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "liveNodes":["10.0.0.6:7574_solr", - "10.0.0.6:8983_solr"], - "replicaInfo":{ - "10.0.0.6:7574_solr":{}, - "10.0.0.6:8983_solr":{"mycoll1":{ - "shard2":[{"core_node2":{"type":"NRT"}}], - "shard1":[{"core_node1":{"type":"NRT"}}]}}}, - "nodeValues":{ - "10.0.0.6:7574_solr":{ - "node":"10.0.0.6:7574_solr", - "cores":0}, - "10.0.0.6:8983_solr":{ - "node":"10.0.0.6:8983_solr", - "cores":2}}} \ No newline at end of file diff --git a/solr/solrj/src/test-files/solrj/solr/autoscaling/testReplicaPercentage.json b/solr/solrj/src/test-files/solrj/solr/autoscaling/testReplicaPercentage.json deleted file mode 100644 index a6b38010066..00000000000 --- a/solr/solrj/src/test-files/solrj/solr/autoscaling/testReplicaPercentage.json +++ /dev/null @@ -1,46 +0,0 @@ -[{ - "liveNodes":["10.0.0.6:7574_solr", - "10.0.0.6:8983_solr"], - "replicaInfo":{ - "10.0.0.6:7574_solr":{}, - "10.0.0.6:8983_solr":{"mycoll1":{"shard1":[{"core_node1":{"type":"NRT"}}, - {"core_node2":{"type":"NRT"}}, - {"core_node3":{"type":"NRT"}}]}}}, - "nodeValues":{ - "10.0.0.6:7574_solr":{ - "node":"10.0.0.6:7574_solr", - "cores":0}, - "10.0.0.6:8983_solr":{ - "node":"10.0.0.6:8983_solr", - "cores":3}}}, -{ - "liveNodes":["10.0.0.6:7574_solr", - "10.0.0.6:8983_solr"], - "replicaInfo":{ - "10.0.0.6:7574_solr":{}, - "10.0.0.6:8983_solr":{"mycoll1":{ - "shard2":[{"core_node2":{"type":"NRT"}}], - "shard1":[{"core_node1":{"type":"NRT"}}]}}}, - "nodeValues":{ - "10.0.0.6:7574_solr":{ - "node":"10.0.0.6:7574_solr", - "cores":0}, - "10.0.0.6:8983_solr":{ - "node":"10.0.0.6:8983_solr", - "cores":2}}}, -{ - "liveNodes":["10.0.0.6:7574_solr", - "10.0.0.6:8983_solr"], - "replicaInfo":{ - "10.0.0.6:7574_solr":{}, - "10.0.0.6:8983_solr":{"mycoll1":{ - "shard1":[{"core_node3":{"type":"PULL"}}], - "shard3":[{"core_node2":{"type":"TLOG"}}], - "shard2":[{"core_node1":{"type":"TLOG"}}]}}}, - "nodeValues":{ - "10.0.0.6:7574_solr":{ - "node":"10.0.0.6:7574_solr", - "cores":0}, - "10.0.0.6:8983_solr":{ - "node":"10.0.0.6:8983_solr", - "cores":2}}}] \ No newline at end of file diff --git a/solr/solrj/src/test-files/solrj/solr/autoscaling/testReplicaZonesPercentage.json b/solr/solrj/src/test-files/solrj/solr/autoscaling/testReplicaZonesPercentage.json deleted file mode 100644 index eb4e3a80933..00000000000 --- a/solr/solrj/src/test-files/solrj/solr/autoscaling/testReplicaZonesPercentage.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "liveNodes":["10.0.0.6:7574_solr", - "10.0.0.6:8983_solr"], - "replicaInfo":{ - "10.0.0.6:7574_solr":{}, - "10.0.0.6:8983_solr":{}}, - "nodeValues":{ - "10.0.0.6:7574_solr":{ - "node":"10.0.0.6:7574_solr", - "cores":0, - "sysprop.az":"west"}, - "10.0.0.6:8983_solr":{ - "node":"10.0.0.6:8983_solr", - "cores":0, - "sysprop.az":"east"}}} \ No newline at end of file diff --git a/solr/solrj/src/test-files/solrj/solr/autoscaling/testScheduledTriggerFailure.json b/solr/solrj/src/test-files/solrj/solr/autoscaling/testScheduledTriggerFailure.json deleted file mode 100644 index 9347494b27e..00000000000 --- a/solr/solrj/src/test-files/solrj/solr/autoscaling/testScheduledTriggerFailure.json +++ /dev/null @@ -1,52 +0,0 @@ -{ - "liveNodes":["127.0.0.1:49221_solr", - "127.0.0.1:49210_solr"], - "suggester":{ - "action":"MOVEREPLICA", - "hints":{}}, - "replicaInfo":{ - "127.0.0.1:49210_solr":{"testScheduledTrigger":{"shard1":[{"core_node3":{ - "base_url":"http://127.0.0.1:49210/solr", - "node_name":"127.0.0.1:49210_solr", - "core":"testScheduledTrigger_shard1_replica_n1", - "state":"active", - "type":"NRT", - "INDEX.sizeInBytes":6.426125764846802E-8, - "shard":"shard1", - "collection":"testScheduledTrigger"}}, - {"core_node6":{ - "base_url":"http://127.0.0.1:49210/solr", - "node_name":"127.0.0.1:49210_solr", - "core":"testScheduledTrigger_shard1_replica_n4", - "state":"active", - "type":"NRT", - "INDEX.sizeInBytes":6.426125764846802E-8, - "shard":"shard1", - "collection":"testScheduledTrigger"}}]}}, - "127.0.0.1:49221_solr":{"testScheduledTrigger":{"shard1":[{"core_node5":{ - "core":"testScheduledTrigger_shard1_replica_n2", - "leader":"true", - "INDEX.sizeInBytes":6.426125764846802E-8, - "base_url":"http://127.0.0.1:49221/solr", - "node_name":"127.0.0.1:49221_solr", - "state":"active", - "type":"NRT", - "shard":"shard1", - "collection":"testScheduledTrigger"}}]}}}, - "nodeValues":{ - "127.0.0.1:49210_solr":{ - "node":"127.0.0.1:49210_solr", - "cores":2, - "freedisk":197.39717864990234}, - "127.0.0.1:49221_solr":{ - "node":"127.0.0.1:49221_solr", - "cores":1, - "freedisk":197.39717864990234}}, - "autoscalingJson":{ - "cluster-preferences":[{ - "minimize":"cores", - "precision":1}, - {"maximize":"freedisk"}], - "cluster-policy":[{ - "cores":"<3", - "node":"#EACH"}]}} \ No newline at end of file diff --git a/solr/solrj/src/test-files/solrj/solr/autoscaling/testSortError.json b/solr/solrj/src/test-files/solrj/solr/autoscaling/testSortError.json deleted file mode 100644 index 434e6277dc2..00000000000 --- a/solr/solrj/src/test-files/solrj/solr/autoscaling/testSortError.json +++ /dev/null @@ -1,225 +0,0 @@ -[{"node":"solr-01:8983_solr", - "replicas":{}, - "isLive":true, - "attributes":[{"cores":2}, - {"freedisk":1734.5261459350586}, - {"sysLoadAvg":35.0}, - {"node":"solr-01:8983_solr"}]}, - { - "node":"solr-07:8983_solr", - "replicas":{}, - "isLive":true, - "attributes":[{"cores":3}, - {"freedisk":1721.5669250488281}, - {"sysLoadAvg":10.0}, - {"node":"solr-07:8983_solr"}]}, - { - "node":"solr-08:8983_solr", - "isLive":true, - "attributes":[{"cores":3}, - {"freedisk":1764.9518203735352}, - {"sysLoadAvg":330.0}, - {"node":"solr-08:8983_solr"}]}, - { - "node":"solr-25:8983_solr", - "isLive":true, - "attributes":[{"cores":3}, - {"freedisk":1779.7792778015137}, - {"sysLoadAvg":304.0}, - {"node":"solr-25:8983_solr"}]}, - { - "node":"solr-15:8983_solr", - "isLive":true, - "attributes":[{"cores":3}, - {"freedisk":1697.5930519104004}, - {"sysLoadAvg":277.0}, - {"node":"solr-15:8983_solr"}]}, - { - "node":"solr-13:8983_solr", - "isLive":true, - "attributes":[{"cores":2}, - {"freedisk":1755.1909484863281}, - {"sysLoadAvg":265.0}, - {"node":"solr-13:8983_solr"}]}, - { - "node":"solr-14:8983_solr", - "isLive":true, - "attributes":[{"cores":3}, - {"freedisk":1757.6035423278809}, - {"sysLoadAvg":61.0}, - {"node":"solr-14:8983_solr"}]}, - { - "node":"solr-16:8983_solr", - "isLive":true, - "attributes":[{"cores":3}, - {"freedisk":1746.081386566162}, - {"sysLoadAvg":260.0}, - {"node":"solr-16:8983_solr"}]}, - { - "node":"solr-04:8983_solr", - "isLive":true, - "attributes":[{"cores":2}, - {"freedisk":1708.7230529785156}, - {"sysLoadAvg":216.0}, - {"node":"solr-04:8983_solr"}]}, - { - "node":"solr-06:8983_solr", - "isLive":true, - "attributes":[{"cores":3}, - {"freedisk":1688.3182678222656}, - {"sysLoadAvg":385.0}, - {"node":"solr-06:8983_solr"}]}, - { - "node":"solr-02:8983_solr", - "isLive":true, - "attributes":[{"cores":6}, - {"freedisk":1778.226963043213}, - {"sysLoadAvg":369.0}, - {"node":"solr-02:8983_solr"}]}, - { - "node":"solr-05:8983_solr", - "isLive":true, - "attributes":[{"cores":3}, - {"freedisk":1741.9401931762695}, - {"sysLoadAvg":354.0}, - {"node":"solr-05:8983_solr"}]}, - { - "node":"solr-23:8983_solr", - "isLive":true, - "attributes":[{"cores":3}, - {"freedisk":1718.854579925537}, - {"sysLoadAvg":329.0}, - {"node":"solr-23:8983_solr"}]}, - { - "node":"solr-24:8983_solr", - "isLive":true, - "attributes":[{"cores":3}, - {"freedisk":1733.6669311523438}, - {"sysLoadAvg":327.0}, - {"node":"solr-24:8983_solr"}]}, - { - "node":"solr-09:8983_solr", - "isLive":true, - "attributes":[{"cores":3}, - {"freedisk":1714.6191711425781}, - {"sysLoadAvg":278.0}, - {"node":"solr-09:8983_solr"}]}, - { - "node":"solr-10:8983_solr", - "isLive":true, - "attributes":[{"cores":3}, - {"freedisk":1755.3038482666016}, - {"sysLoadAvg":266.0}, - {"node":"solr-10:8983_solr"}]}, - { - "node":"solr-28:8983_solr", - "isLive":false, - "attributes":[{"cores":3}, - {"freedisk":1691.3830909729004}, - {"sysLoadAvg":261.0}, - {"node":"solr-28:8983_solr"}]}, - { - "node":"solr-29:8983_solr", - "isLive":true, - "attributes":[{"cores":2}, - {"freedisk":1706.797966003418}, - {"sysLoadAvg":252.99999999999997}, - {"node":"solr-29:8983_solr"}]}, - { - "node":"solr-32:8983_solr", - "isLive":true, - "attributes":[{"cores":3}, - {"freedisk":1762.432300567627}, - {"sysLoadAvg":221.0}, - {"node":"solr-32:8983_solr"}]}, - { - "node":"solr-21:8983_solr", - "isLive":true, - "attributes":[{"cores":3}, - {"freedisk":1760.9801979064941}, - {"sysLoadAvg":213.0}, - {"node":"solr-21:8983_solr"}]}, - { - "node":"solr-22:8983_solr", - "isLive":true, - "attributes":[{"cores":3}, - {"freedisk":1780.5297241210938}, - {"sysLoadAvg":209.0}, - {"node":"solr-22:8983_solr"}]}, - { - "node":"solr-31:8983_solr", - "isLive":true, - "attributes":[{"cores":3}, - {"freedisk":1700.1481628417969}, - {"sysLoadAvg":211.0}, - {"node":"solr-31:8983_solr"}]}, - { - "node":"solr-33:8983_solr", - "isLive":false, - "attributes":[{"cores":3}, - {"freedisk":1748.1132926940918}, - {"sysLoadAvg":199.0}, - {"node":"solr-33:8983_solr"}]}, - { - "node":"solr-36:8983_solr", - "isLive":true, - "attributes":[{"cores":3}, - {"freedisk":1776.197639465332}, - {"sysLoadAvg":193.0}, - {"node":"solr-36:8983_solr"}]}, - { - "node":"solr-35:8983_solr", - "isLive":true, - "attributes":[{"cores":3}, - {"freedisk":1746.7729606628418}, - {"sysLoadAvg":191.0}, - {"node":"solr-35:8983_solr"}]}, - { - "node":"solr-12:8983_solr", - "isLive":true, - "attributes":[{"cores":3}, - {"freedisk":1713.287540435791}, - {"sysLoadAvg":175.0}, - {"node":"solr-12:8983_solr"}]}, - { - "node":"solr-11:8983_solr", - "isLive":true, - "attributes":[{"cores":3}, - {"freedisk":1736.784511566162}, - {"sysLoadAvg":169.0}, - {"node":"solr-11:8983_solr"}]}, - { - "node":"solr-35:8983_solr", - "isLive":true, - "attributes":[{"cores":3}, - {"freedisk":1766.9416885375977}, - {"sysLoadAvg":155.0}, - {"node":"solr-35:8983_solr"}]}, - { - "node":"solr-17:8983_solr", - "isLive":true, - "attributes":[{"cores":3}, - {"freedisk":1764.3425407409668}, - {"sysLoadAvg":139.0}, - {"node":"solr-17:8983_solr"}]}, - { - "node":"solr-18:8983_solr", - "isLive":true, - "attributes":[{"cores":2}, - {"freedisk":1757.0613975524902}, - {"sysLoadAvg":132.0}, - {"node":"solr-18:8983_solr"}]}, - { - "node":"solr-20:8983_solr", - "isLive":false, - "attributes":[{"cores":3}, - {"freedisk":1747.4205322265625}, - {"sysLoadAvg":126.0}, - {"node":"solr-20:8983_solr"}]}, - { - "node":"solr-27:8983_solr", - "isLive":true, - "attributes":[{"cores":4}, - {"freedisk":1721.0442085266113}, - {"sysLoadAvg":118.0}, - {"node":"solr-27:8983_solr"}]}] \ No newline at end of file diff --git a/solr/solrj/src/test-files/solrj/solr/autoscaling/testSuggestionsRebalance2.json b/solr/solrj/src/test-files/solrj/solr/autoscaling/testSuggestionsRebalance2.json deleted file mode 100644 index d9763030da3..00000000000 --- a/solr/solrj/src/test-files/solrj/solr/autoscaling/testSuggestionsRebalance2.json +++ /dev/null @@ -1,131 +0,0 @@ -{ - "diagnostics":{ - "sortedNodes":[ - { - "node":"10.0.0.79:7574_solr", - "isLive":true, - "cores":4.0, - "freedisk":140.8341178894043, - "totaldisk":233.5667953491211, - "replicas":{ - "gettingstarted":{ - "shard2":[{ - "core_node7":{ - "core":"gettingstarted_shard2_replica_n4", - "shard":"shard2", - "collection":"gettingstarted", - "node_name":"10.0.0.79:7574_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://10.0.0.79:7574/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":6.426125764846802E-8}}], - "shard1":[{ - "core_node3":{ - "core":"gettingstarted_shard1_replica_n1", - "shard":"shard1", - "collection":"gettingstarted", - "node_name":"10.0.0.79:7574_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://10.0.0.79:7574/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":6.426125764846802E-8}}]}, - "go":{ - "shard2":[{ - "core_node7":{ - "core":"go_shard2_replica_n4", - "shard":"shard2", - "collection":"go", - "node_name":"10.0.0.79:7574_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://10.0.0.79:7574/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":6.426125764846802E-8}}], - "shard1":[{ - "core_node3":{ - "core":"go_shard1_replica_n1", - "shard":"shard1", - "collection":"go", - "node_name":"10.0.0.79:7574_solr", - "type":"NRT", - "leader":"true", - "base_url":"http://10.0.0.79:7574/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":6.426125764846802E-8}}]}}} - ,{ - "node":"10.0.0.79:8984_solr", - "isLive":true, - "cores":4.0, - "freedisk":140.8341178894043, - "totaldisk":233.5667953491211, - "replicas":{ - "gettingstarted":{ - "shard2":[{ - "core_node8":{ - "core":"gettingstarted_shard2_replica_n6", - "shard":"shard2", - "collection":"gettingstarted", - "node_name":"10.0.0.79:8984_solr", - "type":"NRT", - "base_url":"http://10.0.0.79:8984/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":6.426125764846802E-8}}], - "shard1":[{ - "core_node5":{ - "core":"gettingstarted_shard1_replica_n2", - "shard":"shard1", - "collection":"gettingstarted", - "node_name":"10.0.0.79:8984_solr", - "type":"NRT", - "base_url":"http://10.0.0.79:8984/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":6.426125764846802E-8}}]}, - "go":{ - "shard2":[{ - "core_node8":{ - "core":"go_shard2_replica_n6", - "shard":"shard2", - "collection":"go", - "node_name":"10.0.0.79:8984_solr", - "type":"NRT", - "base_url":"http://10.0.0.79:8984/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":6.426125764846802E-8}}], - "shard1":[{ - "core_node5":{ - "core":"go_shard1_replica_n2", - "shard":"shard1", - "collection":"go", - "node_name":"10.0.0.79:8984_solr", - "type":"NRT", - "base_url":"http://10.0.0.79:8984/solr", - "state":"active", - "force_set_state":"false", - "INDEX.sizeInGB":6.426125764846802E-8}}]}}} - ,{ - "node":"10.0.0.79:8983_solr", - "isLive":true, - "cores":0.0, - "freedisk":140.8341178894043, - "totaldisk":233.5667953491211, - "replicas":{}}], - "liveNodes":["10.0.0.79:7574_solr", - "10.0.0.79:8983_solr", - "10.0.0.79:8984_solr"], - "violations":[], - "config":{ - "cluster-preferences":[{ - "minimize":"cores", - "precision":1} - ,{ - "maximize":"freedisk"}], - "cluster-policy": []}}} \ No newline at end of file diff --git a/solr/solrj/src/test-files/solrj/solr/autoscaling/testSuggestionsRebalanceOnly.json b/solr/solrj/src/test-files/solrj/solr/autoscaling/testSuggestionsRebalanceOnly.json deleted file mode 100644 index ce0b682312b..00000000000 --- a/solr/solrj/src/test-files/solrj/solr/autoscaling/testSuggestionsRebalanceOnly.json +++ /dev/null @@ -1,105 +0,0 @@ -{"diagnostics":{ - "sortedNodes":[{ - "node":"127.0.0.1:63191_solr", - "isLive":true, - "cores":3.0, - "sysprop.zone":"east", - "freedisk":1727.1459312438965, - "heapUsage":24.97510064011647, - "sysLoadAvg":272.75390625, - "totaldisk":1037.938980102539, - "replicas":{"zonesTest":{"shard1":[{"core_node5":{ - "core":"zonesTest_shard1_replica_n2", - "leader":"true", - "base_url":"https://127.0.0.1:63191/solr", - "node_name":"127.0.0.1:63191_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "INDEX.sizeInGB":6.426125764846802E-8, - "shard":"shard1", - "collection":"zonesTest"}}, - {"core_node7":{ - "core":"zonesTest_shard1_replica_n4", - "base_url":"https://127.0.0.1:63191/solr", - "node_name":"127.0.0.1:63191_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "INDEX.sizeInGB":6.426125764846802E-8, - "shard":"shard1", - "collection":"zonesTest"}}, - {"core_node12":{ - "core":"zonesTest_shard1_replica_n10", - "base_url":"https://127.0.0.1:63191/solr", - "node_name":"127.0.0.1:63191_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "INDEX.sizeInGB":6.426125764846802E-8, - "shard":"shard1", - "collection":"zonesTest"}}]}}}, - { - "node":"127.0.0.1:63192_solr", - "isLive":true, - "cores":3.0, - "sysprop.zone":"east", - "freedisk":1727.1459312438965, - "heapUsage":24.98878807983566, - "sysLoadAvg":272.75390625, - "totaldisk":1037.938980102539, - "replicas":{"zonesTest":{"shard2":[{"core_node3":{ - "core":"zonesTest_shard1_replica_n1", - "base_url":"https://127.0.0.1:63192/solr", - "node_name":"127.0.0.1:63192_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "INDEX.sizeInGB":6.426125764846802E-8, - "shard":"shard2", - "collection":"zonesTest"}}, - {"core_node9":{ - "core":"zonesTest_shard1_replica_n6", - "base_url":"https://127.0.0.1:63192/solr", - "node_name":"127.0.0.1:63192_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "INDEX.sizeInGB":6.426125764846802E-8, - "shard":"shard2", - "collection":"zonesTest"}}, - {"core_node11":{ - "core":"zonesTest_shard1_replica_n8", - "base_url":"https://127.0.0.1:63192/solr", - "node_name":"127.0.0.1:63192_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "INDEX.sizeInGB":6.426125764846802E-8, - "shard":"shard2", - "collection":"zonesTest"}}]}}}, - { - "node":"127.0.0.1:63219_solr", - "isLive":true, - "cores":0.0, - "sysprop.zone":"west", - "freedisk":1768.6174201965332, - "heapUsage":24.98878807983566, - "sysLoadAvg":272.75390625, - "totaldisk":1037.938980102539, - "replicas":{}}, - { - "node":"127.0.0.1:63229_solr", - "isLive":true, - "cores":0.0, - "sysprop.zone":"west", - "freedisk":1768.6174201965332, - "heapUsage":24.98878807983566, - "sysLoadAvg":272.75390625, - "totaldisk":1037.938980102539, - "replicas":{}}], - "liveNodes":["127.0.0.1:63191_solr", - "127.0.0.1:63192_solr", - "127.0.0.1:63219_solr", - "127.0.0.1:63229_solr"] -}} \ No newline at end of file diff --git a/solr/solrj/src/test-files/solrj/solr/autoscaling/testSysPropSuggestions.json b/solr/solrj/src/test-files/solrj/solr/autoscaling/testSysPropSuggestions.json deleted file mode 100644 index d1d6cad04d4..00000000000 --- a/solr/solrj/src/test-files/solrj/solr/autoscaling/testSysPropSuggestions.json +++ /dev/null @@ -1,127 +0,0 @@ -{"diagnostics":{ - "sortedNodes":[{ - "node":"127.0.0.1:63191_solr", - "isLive":true, - "cores":3.0, - "sysprop.zone":"east", - "freedisk":1727.1459312438965, - "heapUsage":24.97510064011647, - "sysLoadAvg":272.75390625, - "totaldisk":1037.938980102539, - "replicas":{"zonesTest":{"shard1":[{"core_node5":{ - "core":"zonesTest_shard1_replica_n2", - "leader":"true", - "base_url":"https://127.0.0.1:63191/solr", - "node_name":"127.0.0.1:63191_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "INDEX.sizeInGB":6.426125764846802E-8, - "shard":"shard1", - "collection":"zonesTest"}}, - {"core_node7":{ - "core":"zonesTest_shard1_replica_n4", - "base_url":"https://127.0.0.1:63191/solr", - "node_name":"127.0.0.1:63191_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "INDEX.sizeInGB":6.426125764846802E-8, - "shard":"shard1", - "collection":"zonesTest"}}, - {"core_node12":{ - "core":"zonesTest_shard1_replica_n10", - "base_url":"https://127.0.0.1:63191/solr", - "node_name":"127.0.0.1:63191_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "INDEX.sizeInGB":6.426125764846802E-8, - "shard":"shard1", - "collection":"zonesTest"}}]}}}, - { - "node":"127.0.0.1:63192_solr", - "isLive":true, - "cores":3.0, - "sysprop.zone":"east", - "freedisk":1727.1459312438965, - "heapUsage":24.98878807983566, - "sysLoadAvg":272.75390625, - "totaldisk":1037.938980102539, - "replicas":{"zonesTest":{"shard2":[{"core_node3":{ - "core":"zonesTest_shard1_replica_n1", - "base_url":"https://127.0.0.1:63192/solr", - "node_name":"127.0.0.1:63192_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "INDEX.sizeInGB":6.426125764846802E-8, - "shard":"shard2", - "collection":"zonesTest"}}, - {"core_node9":{ - "core":"zonesTest_shard1_replica_n6", - "base_url":"https://127.0.0.1:63192/solr", - "node_name":"127.0.0.1:63192_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "INDEX.sizeInGB":6.426125764846802E-8, - "shard":"shard2", - "collection":"zonesTest"}}, - {"core_node11":{ - "core":"zonesTest_shard1_replica_n8", - "base_url":"https://127.0.0.1:63192/solr", - "node_name":"127.0.0.1:63192_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "INDEX.sizeInGB":6.426125764846802E-8, - "shard":"shard2", - "collection":"zonesTest"}}]}}}, - { - "node":"127.0.0.1:63219_solr", - "isLive":true, - "cores":0.0, - "sysprop.zone":"west", - "freedisk":1768.6174201965332, - "heapUsage":24.98878807983566, - "sysLoadAvg":272.75390625, - "totaldisk":1037.938980102539, - "replicas":{}}, - { - "node":"127.0.0.1:63229_solr", - "isLive":true, - "cores":0.0, - "sysprop.zone":"west", - "freedisk":1768.6174201965332, - "heapUsage":24.98878807983566, - "sysLoadAvg":272.75390625, - "totaldisk":1037.938980102539, - "replicas":{}}], - "liveNodes":["127.0.0.1:63191_solr", - "127.0.0.1:63192_solr", - "127.0.0.1:63219_solr", - "127.0.0.1:63229_solr"], - "config":{ - "cluster-preferences":[{ - "minimize":"cores", - "precision":1}, - { - "maximize":"freedisk", - "precision":100}, - { - "minimize":"sysLoadAvg", - "precision":10}], - "cluster-policy":[ - { - "replica":"<3", - "shard":"#EACH", - "sysprop.zone":"east"}, - { - "replica":"<3", - "shard":"#EACH", - "sysprop.zone":"west"} - - - - ]}}} \ No newline at end of file diff --git a/solr/solrj/src/test-files/solrj/solr/autoscaling/testSyspropSuggestions1.json b/solr/solrj/src/test-files/solrj/solr/autoscaling/testSyspropSuggestions1.json deleted file mode 100644 index 085fe6041c2..00000000000 --- a/solr/solrj/src/test-files/solrj/solr/autoscaling/testSyspropSuggestions1.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "liveNodes":["node1", - "node2", - "node3"], - "replicaInfo":{"node1":{"c1":{ - "s1":[{ - "r1":{"type":"NRT"}, - "r2":{"type":"NRT"}}], - "s2":[{ - "r1":{"type":"NRT"}, - "r2":{"type":"NRT"}}]}}}, - "nodeValues":{ - "node1":{ - "cores":2, - "freedisk":334, - "sysprop.fs":"slowdisk"}, - "node2":{ - "cores":2, - "freedisk":749, - "sysprop.fs":"slowdisk"}, - "node3":{ - "cores":0, - "freedisk":262, - "sysprop.fs":"ssd"}}} \ No newline at end of file diff --git a/solr/solrj/src/test-files/solrj/solr/autoscaling/testUnresolvedSuggestion.json b/solr/solrj/src/test-files/solrj/solr/autoscaling/testUnresolvedSuggestion.json deleted file mode 100644 index e22b65262c2..00000000000 --- a/solr/solrj/src/test-files/solrj/solr/autoscaling/testUnresolvedSuggestion.json +++ /dev/null @@ -1,211 +0,0 @@ -{ - "diagnostics": { - "sortedNodes": [ - { - "node": "10.0.0.80:7574_solr", - "isLive": true, - "cores": 2.0, - "freedisk": 661.6284255981445, - "totaldisk": 1037.938980102539, - "replicas": { - "go": { - "shard2": [ - { - "core_node7": { - "core": "go_shard2_replica_n4", - "shard": "shard2", - "collection": "go", - "node_name": "10.0.0.80:7574_solr", - "type": "NRT", - "base_url": "http://10.0.0.80:7574/solr", - "state": "active", - "force_set_state": "false", - "INDEX.sizeInGB": 6.426125764846802E-8 - } - }, - { - "core_node9": { - "core": "go_shard2_replica_n6", - "shard": "shard2", - "collection": "go", - "node_name": "10.0.0.80:7574_solr", - "type": "NRT", - "leader": "true", - "base_url": "http://10.0.0.80:7574/solr", - "state": "active", - "force_set_state": "false", - "INDEX.sizeInGB": 6.426125764846802E-8 - } - } - ] - } - } - }, - { - "node": "10.0.0.80:8983_solr", - "isLive": true, - "cores": 2.0, - "freedisk": 661.6284217834473, - "totaldisk": 1037.938980102539, - "replicas": { - "go": { - "shard3": [ - { - "core_node11": { - "core": "go_shard3_replica_n8", - "shard": "shard3", - "collection": "go", - "node_name": "10.0.0.80:8983_solr", - "type": "NRT", - "leader": "true", - "base_url": "http://10.0.0.80:8983/solr", - "state": "active", - "force_set_state": "false", - "INDEX.sizeInGB": 6.426125764846802E-8 - } - } - ], - "shard1": [ - { - "core_node5": { - "core": "go_shard1_replica_n2", - "shard": "shard1", - "collection": "go", - "node_name": "10.0.0.80:8983_solr", - "type": "NRT", - "leader": "true", - "base_url": "http://10.0.0.80:8983/solr", - "state": "active", - "force_set_state": "false", - "INDEX.sizeInGB": 6.426125764846802E-8 - } - } - ] - } - } - }, - { - "node": "10.0.0.80:8984_solr", - "isLive": true, - "cores": 2.0, - "freedisk": 661.6284217834473, - "totaldisk": 1037.938980102539, - "replicas": { - "go": { - "shard3": [ - { - "core_node12": { - "core": "go_shard3_replica_n10", - "shard": "shard3", - "collection": "go", - "node_name": "10.0.0.80:8984_solr", - "type": "NRT", - "base_url": "http://10.0.0.80:8984/solr", - "state": "active", - "force_set_state": "false", - "INDEX.sizeInGB": 6.426125764846802E-8 - } - } - ], - "shard1": [ - { - "core_node3": { - "core": "go_shard1_replica_n1", - "shard": "shard1", - "collection": "go", - "node_name": "10.0.0.80:8984_solr", - "type": "NRT", - "base_url": "http://10.0.0.80:8984/solr", - "state": "active", - "force_set_state": "false", - "INDEX.sizeInGB": 6.426125764846802E-8 - } - } - ] - } - } - } - ], - "liveNodes": [ - "10.0.0.80:7574_solr", - "10.0.0.80:8983_solr", - "10.0.0.80:8984_solr" - ], - "config": { - "cluster-preferences": [{"minimize": "cores"}], - "cluster-policy": [{"replica": "<2", "node": "#ANY"}] - } - }, - "cluster":{ - "collections":{ - "go":{ - "pullReplicas":"0", - "replicationFactor":"2", - "shards":{ - "shard1":{ - "range":"80000000-d554ffff", - "state":"active", - "replicas":{ - "core_node3":{ - "core":"go_shard1_replica_n1", - "base_url":"http://10.0.0.80:8984/solr", - "node_name":"10.0.0.80:8984_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node5":{ - "core":"go_shard1_replica_n2", - "base_url":"http://10.0.0.80:8983/solr", - "node_name":"10.0.0.80:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard2":{ - "range":"d5550000-2aa9ffff", - "state":"active", - "replicas":{ - "core_node7":{ - "core":"go_shard2_replica_n4", - "base_url":"http://10.0.0.80:7574/solr", - "node_name":"10.0.0.80:7574_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}, - "core_node9":{ - "core":"go_shard2_replica_n6", - "base_url":"http://10.0.0.80:7574/solr", - "node_name":"10.0.0.80:7574_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}}}, - "shard3":{ - "range":"2aaa0000-7fffffff", - "state":"active", - "replicas":{ - "core_node11":{ - "core":"go_shard3_replica_n8", - "base_url":"http://10.0.0.80:8983/solr", - "node_name":"10.0.0.80:8983_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false", - "leader":"true"}, - "core_node12":{ - "core":"go_shard3_replica_n10", - "base_url":"http://10.0.0.80:8984/solr", - "node_name":"10.0.0.80:8984_solr", - "state":"active", - "type":"NRT", - "force_set_state":"false"}}}}, - "router":{"name":"compositeId"}, - "autoAddReplicas":"false", - "nrtReplicas":"2", - "tlogReplicas":"0", - "znodeVersion":9, - "configName":"go"}}, - "live_nodes":["10.0.0.80:8983_solr", - "10.0.0.80:7574_solr", - "10.0.0.80:8984_solr"]} -} diff --git a/solr/solrj/src/test-files/solrj/solr/autoscaling/testUtilizeNodeFailure.json b/solr/solrj/src/test-files/solrj/solr/autoscaling/testUtilizeNodeFailure.json deleted file mode 100644 index 350957c81fe..00000000000 --- a/solr/solrj/src/test-files/solrj/solr/autoscaling/testUtilizeNodeFailure.json +++ /dev/null @@ -1,69 +0,0 @@ -{ - "liveNodes":["127.0.0.1:50417_solr", - "127.0.0.1:50418_solr", - "127.0.0.1:50419_solr", - "127.0.0.1:50420_solr", - "127.0.0.1:50443_solr"], - "suggester":{ - "action":"MOVEREPLICA", - "hints":{"TARGET_NODE":["127.0.0.1:50443_solr"]}}, - "replicaInfo":{ - "127.0.0.1:50418_solr":{"utilizenodecoll":{"shard2":[{"core_node7":{ - "core":"utilizenodecoll_shard2_replica_n4", - "leader":"true", - "INDEX.sizeInBytes":6.426125764846802E-8, - "base_url":"http://127.0.0.1:50418/solr", - "node_name":"127.0.0.1:50418_solr", - "state":"active", - "type":"NRT", - "shard":"shard2", - "collection":"utilizenodecoll"}}]}}, - "127.0.0.1:50417_solr":{"utilizenodecoll":{"shard2":[{"core_node8":{ - "base_url":"http://127.0.0.1:50417/solr", - "node_name":"127.0.0.1:50417_solr", - "core":"utilizenodecoll_shard2_replica_n6", - "state":"active", - "type":"NRT", - "INDEX.sizeInBytes":6.426125764846802E-8, - "shard":"shard2", - "collection":"utilizenodecoll"}}]}}, - "127.0.0.1:50419_solr":{"utilizenodecoll":{"shard1":[{"core_node5":{ - "base_url":"http://127.0.0.1:50419/solr", - "node_name":"127.0.0.1:50419_solr", - "core":"utilizenodecoll_shard1_replica_n2", - "state":"active", - "type":"NRT", - "INDEX.sizeInBytes":6.426125764846802E-8, - "shard":"shard1", - "collection":"utilizenodecoll"}}]}}, - "127.0.0.1:50420_solr":{"utilizenodecoll":{"shard1":[{"core_node3":{ - "core":"utilizenodecoll_shard1_replica_n1", - "leader":"true", - "INDEX.sizeInBytes":6.426125764846802E-8, - "base_url":"http://127.0.0.1:50420/solr", - "node_name":"127.0.0.1:50420_solr", - "state":"active", - "type":"NRT", - "shard":"shard1", - "collection":"utilizenodecoll"}}]}}, - "127.0.0.1:50443_solr":{}}, - "nodeValues":{ - "127.0.0.1:50418_solr":{ - "cores":1, - "freedisk":187.70782089233398}, - "127.0.0.1:50417_solr":{ - "cores":1, - "freedisk":187.70782089233398}, - "127.0.0.1:50419_solr":{ - "cores":1, - "freedisk":187.70782089233398}, - "127.0.0.1:50420_solr":{ - "cores":1, - "freedisk":187.70782089233398}, - "127.0.0.1:50443_solr":{ - "cores":0, - "freedisk":187.70782089233398}}, - "autoscalingJson":{"cluster-preferences":[{ - "minimize":"cores", - "precision":1}, - {"maximize":"freedisk"}]}} \ No newline at end of file diff --git a/solr/solrj/src/test-files/solrj/solr/autoscaling/testUtilizeNodeFailure2.json b/solr/solrj/src/test-files/solrj/solr/autoscaling/testUtilizeNodeFailure2.json deleted file mode 100644 index 2d6c384a6a3..00000000000 --- a/solr/solrj/src/test-files/solrj/solr/autoscaling/testUtilizeNodeFailure2.json +++ /dev/null @@ -1,66 +0,0 @@ -{ - "liveNodes":["127.0.0.1:51075_solr", - "127.0.0.1:51076_solr", - "127.0.0.1:51077_solr", - "127.0.0.1:51097_solr"], - "suggester":{ - "action":"MOVEREPLICA", - "hints":{"TARGET_NODE":["127.0.0.1:51097_solr"]}}, - "replicaInfo":{ - "127.0.0.1:51076_solr":{"utilizenodecoll":{"shard1":[{"core_node5":{ - "base_url":"https://127.0.0.1:51076/solr", - "node_name":"127.0.0.1:51076_solr", - "core":"utilizenodecoll_shard1_replica_n2", - "state":"active", - "type":"NRT", - "INDEX.sizeInBytes":6.426125764846802E-8, - "shard":"shard1", - "collection":"utilizenodecoll"}}]}}, - "127.0.0.1:51077_solr":{"utilizenodecoll":{ - "shard2":[{"core_node8":{ - "base_url":"https://127.0.0.1:51077/solr", - "node_name":"127.0.0.1:51077_solr", - "core":"utilizenodecoll_shard2_replica_n6", - "state":"active", - "type":"NRT", - "INDEX.sizeInBytes":6.426125764846802E-8, - "shard":"shard2", - "collection":"utilizenodecoll"}}], - "shard1":[{"core_node3":{ - "core":"utilizenodecoll_shard1_replica_n1", - "leader":"true", - "INDEX.sizeInBytes":6.426125764846802E-8, - "base_url":"https://127.0.0.1:51077/solr", - "node_name":"127.0.0.1:51077_solr", - "state":"active", - "type":"NRT", - "shard":"shard1", - "collection":"utilizenodecoll"}}]}}, - "127.0.0.1:51097_solr":{}, - "127.0.0.1:51075_solr":{"utilizenodecoll":{"shard2":[{"core_node7":{ - "core":"utilizenodecoll_shard2_replica_n4", - "leader":"true", - "INDEX.sizeInBytes":6.426125764846802E-8, - "base_url":"https://127.0.0.1:51075/solr", - "node_name":"127.0.0.1:51075_solr", - "state":"active", - "type":"NRT", - "shard":"shard2", - "collection":"utilizenodecoll"}}]}}}, - "nodeValues":{ - "127.0.0.1:51076_solr":{ - "cores":1, - "freedisk":188.7262191772461}, - "127.0.0.1:51077_solr":{ - "cores":2, - "freedisk":188.7262191772461}, - "127.0.0.1:51097_solr":{ - "cores":0, - "freedisk":188.7262191772461}, - "127.0.0.1:51075_solr":{ - "cores":1, - "freedisk":188.7262191772461}}, - "autoscalingJson":{"cluster-preferences":[{ - "minimize":"cores", - "precision":1}, - {"maximize":"freedisk"}]}} \ No newline at end of file diff --git a/solr/solrj/src/test-files/solrj/solr/autoscaling/testViolationOutput.json b/solr/solrj/src/test-files/solrj/solr/autoscaling/testViolationOutput.json deleted file mode 100644 index f518de1ad55..00000000000 --- a/solr/solrj/src/test-files/solrj/solr/autoscaling/testViolationOutput.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "liveNodes":["node1:8983", - "node2:8984", - "node3:8985"], - "replicaInfo":{"node1:8983":{"c1":{ - "s1":[{"r1":{"type":"NRT"}}, - {"r2":{"type":"NRT"}}], - "s2":[{"r1":{"type":"NRT"}}, - {"r2":{"type":"NRT"}}]}}}, - "nodeValues":{ - "node1:8983":{ - "cores":4, - "freedisk":334, - "port":8983}, - "node2:8984":{ - "cores":0, - "freedisk":1000, - "port":8984}, - "node3:8985":{ - "cores":0, - "freedisk":1500, - "port":8985}}} \ No newline at end of file diff --git a/solr/solrj/src/test-files/solrj/solr/autoscaling/testWithCollection.json b/solr/solrj/src/test-files/solrj/solr/autoscaling/testWithCollection.json deleted file mode 100644 index d1719982177..00000000000 --- a/solr/solrj/src/test-files/solrj/solr/autoscaling/testWithCollection.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "comments_coll":{ - "router":{"name":"compositeId"}, - "shards":{}, - "withCollection":"articles_coll"}, - "articles_coll":{ - "router":{"name":"compositeId"}, - "shards":{"shard1":{ - "range":"80000000-ffffffff", - "replicas":{ - "r1":{ - "core":"r1", - "base_url":"http://10.0.0.4:8983/solr", - "node_name":"node1", - "state":"active", - "leader":"true"}, - "r2":{ - "core":"r2", - "base_url":"http://10.0.0.4:7574/solr", - "node_name":"node2", - "state":"active"}}}}}} \ No newline at end of file diff --git a/solr/solrj/src/test-files/solrj/solr/autoscaling/testWithCollectionMoveReplica.json b/solr/solrj/src/test-files/solrj/solr/autoscaling/testWithCollectionMoveReplica.json deleted file mode 100644 index 469eef15156..00000000000 --- a/solr/solrj/src/test-files/solrj/solr/autoscaling/testWithCollectionMoveReplica.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "comments_coll":{ - "router":{"name":"compositeId"}, - "shards":{"shard1":{ - "range":"80000000-ffffffff", - "replicas":{"r1":{ - "core":"r1", - "base_url":"http://10.0.0.4:8983/solr", - "node_name":"node1", - "state":"active", - "leader":"true"}}}}, - "withCollection":"articles_coll"}, - "articles_coll":{ - "router":{"name":"compositeId"}, - "shards":{"shard1":{ - "range":"80000000-ffffffff", - "replicas":{ - "r1":{ - "core":"r1", - "base_url":"http://10.0.0.4:8983/solr", - "node_name":"node1", - "state":"active", - "leader":"true"}, - "r2":{ - "core":"r2", - "base_url":"http://10.0.0.4:7574/solr", - "node_name":"node2", - "state":"active"}}}}}} \ No newline at end of file diff --git a/solr/solrj/src/test-files/solrj/solr/autoscaling/testWithCollectionMoveVsAddSuggestions.json b/solr/solrj/src/test-files/solrj/solr/autoscaling/testWithCollectionMoveVsAddSuggestions.json deleted file mode 100644 index 0d99d4a4167..00000000000 --- a/solr/solrj/src/test-files/solrj/solr/autoscaling/testWithCollectionMoveVsAddSuggestions.json +++ /dev/null @@ -1,49 +0,0 @@ -{ - "articles_coll":{ - "router":{"name":"compositeId"}, - "shards":{"shard1":{ - "range":"80000000-ffffffff", - "replicas":{ - "r1":{ - "core":"r1", - "base_url":"http://10.0.0.4:8983/solr", - "node_name":"node1", - "state":"active", - "leader":"true"}, - "r2":{ - "core":"r2", - "base_url":"http://10.0.0.4:7574/solr", - "node_name":"node2", - "state":"active"}, - "r3":{ - "core":"r3", - "base_url":"http://10.0.0.4:7579/solr", - "node_name":"node6", - "state":"active"}}}}}, - "comments_coll":{ - "withCollection":"articles_coll", - "router":{"name":"compositeId"}, - "shards":{"shard1":{ - "range":"80000000-ffffffff", - "replicas":{ - "r1":{ - "core":"r1", - "base_url":"http://10.0.0.4:7576/solr", - "node_name":"node3", - "state":"active", - "leader":"true"}, - "r2":{ - "core":"r2", - "base_url":"http://10.0.0.4:7577/solr", - "node_name":"node4", - "state":"active"}, - "r3":{ - "core":"r3", - "base_url":"http://10.0.0.4:7578/solr", - "node_name":"node5", - "state":"active"}, - "r4":{ - "core":"r4", - "base_url":"http://10.0.0.4:7579/solr", - "node_name":"node6", - "state":"active"}}}}}} \ No newline at end of file diff --git a/solr/solrj/src/test-files/solrj/solr/autoscaling/testWithCollectionSuggestions.json b/solr/solrj/src/test-files/solrj/solr/autoscaling/testWithCollectionSuggestions.json deleted file mode 100644 index c4a29db003b..00000000000 --- a/solr/solrj/src/test-files/solrj/solr/autoscaling/testWithCollectionSuggestions.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "articles_coll":{ - "router":{"name":"compositeId"}, - "shards":{"shard1":{}}}, - "comments_coll":{ - "withCollection":"articles_coll", - "router":{"name":"compositeId"}, - "shards":{"shard1":{ - "range":"80000000-ffffffff", - "replicas":{ - "r1":{ - "core":"r1", - "base_url":"http://10.0.0.4:8983/solr", - "node_name":"node1", - "state":"active", - "leader":"true"}, - "r2":{ - "core":"r2", - "base_url":"http://10.0.0.4:7574/solr", - "node_name":"node2", - "state":"active"}}}}}} \ No newline at end of file diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/ConditionTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/ConditionTest.java deleted file mode 100644 index 2b455f12e11..00000000000 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/ConditionTest.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.client.solrj.cloud.autoscaling; - -import static org.junit.Assert.*; - -import org.junit.Test; - -public class ConditionTest { - @Test - public void testEqualsHashCode() { - assertHashMatchesEquals("equals should match hash (names are equal)", - new Condition("node", null, null, null, null), - new Condition("node", null, null, null, null)); - assertHashMatchesEquals("equals should match hash (names aren't equal)", - new Condition("node", null, null, null, null), - new Condition("host", null, null, null, null)); - assertHashMatchesEquals("equals should match hash (values are equal)", - new Condition("node", "localhost", null, null, null), - new Condition("node", "localhost", null, null, null)); - assertHashMatchesEquals("equals should match hash (values aren't equal)", - new Condition("node", "localhost", null, null, null), - new Condition("node", "lucene.apache.org", null, null, null)); - assertHashMatchesEquals("equals should match hash (operands are equal)", - new Condition("node", null, Operand.EQUAL, null, null), - new Condition("node", null, Operand.EQUAL, null, null)); - assertHashMatchesEquals("equals should match hash (operands aren't equal)", - new Condition("node", null, Operand.EQUAL, null, null), - new Condition("node", null, Operand.NOT_EQUAL, null, null)); - - Condition condition = new Condition("host", "localhost", Operand.EQUAL, null, null); - assertHashMatchesEquals("equals should match hash when compared to self", condition, condition); - assertTrue("equals should be true when compared to self", condition.equals(condition)); - } - - @Test - public void testEqualsInvertible() { - assertEqualsInvertible("equals should be invertible (names are equal)", - new Condition("node", null, null, null, null), - new Condition("node", null, null, null, null)); - assertEqualsInvertible("equals should be invertible (names aren't equal)", - new Condition("node", null, null, null, null), - new Condition("host", null, null, null, null)); - assertEqualsInvertible("equals should be invertible (values are equal)", - new Condition("node", "localhost", null, null, null), - new Condition("node", "localhost", null, null, null)); - assertEqualsInvertible("equals should be invertible (values aren't equal)", - new Condition("node", "localhost", null, null, null), - new Condition("node", "lucene.apache.org", null, null, null)); - assertEqualsInvertible("equals should be invertible (operands are equal)", - new Condition("node", null, Operand.EQUAL, null, null), - new Condition("node", null, Operand.EQUAL, null, null)); - assertEqualsInvertible("equals should be invertible (operands aren't equal)", - new Condition("node", null, Operand.EQUAL, null, null), - new Condition("node", null, Operand.NOT_EQUAL, null, null)); - } - - private void assertEqualsInvertible(String message, Condition a, Condition b) { - assertEquals(message, a != null && a.equals(b), b != null && b.equals(a)); - } - - private void assertHashMatchesEquals(String message, Condition a, Condition b) { - assertTrue(message, (a.hashCode() == b.hashCode()) || (!a.equals(b) && !b.equals(a))); - } -} diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/MoveReplicaSuggesterTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/MoveReplicaSuggesterTest.java deleted file mode 100644 index af60cd8e30c..00000000000 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/MoveReplicaSuggesterTest.java +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.client.solrj.cloud.autoscaling; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Map; - -import org.apache.solr.SolrTestCaseJ4; -import org.apache.solr.common.cloud.Replica; -import org.apache.solr.common.cloud.ZkStateReader; -import org.apache.solr.common.util.Pair; -import org.junit.Test; - -import static org.apache.solr.client.solrj.cloud.autoscaling.MoveReplicaSuggester.leaderLast; - -public class MoveReplicaSuggesterTest extends SolrTestCaseJ4 { - private String CORE = "collection_shard1_replica_n1"; - private String COLLECTION = "collection"; - private String SHARD = "shard1"; - private Map IS_LEADER = Collections.singletonMap(ZkStateReader.LEADER_PROP, true); - private String NODE = "127.0.0.1:8080_solr"; - private Replica.Type REPLICA_TYPE = Replica.Type.NRT; - private Row ROW = null; - - private Replica REPLICA_INFO_ONE = new Replica("core_node1", NODE, COLLECTION, SHARD, CORE, Replica.State.ACTIVE, REPLICA_TYPE, IS_LEADER); - private Replica REPLICA_INFO_TWO = new Replica("core_node2", NODE, COLLECTION, SHARD, CORE, Replica.State.ACTIVE, REPLICA_TYPE, null); - private Replica REPLICA_INFO_THREE = new Replica("core_node3", NODE, COLLECTION, SHARD, CORE, Replica.State.ACTIVE, REPLICA_TYPE, IS_LEADER); - private Replica REPLICA_INFO_FOUR = new Replica("core_node4", NODE, COLLECTION, SHARD, CORE, Replica.State.ACTIVE, REPLICA_TYPE, null); - - private Pair PAIR_ONE = new Pair<>(REPLICA_INFO_ONE, ROW); - private Pair PAIR_TWO = new Pair<>(REPLICA_INFO_TWO, ROW); - private Pair PAIR_THREE = new Pair<>(REPLICA_INFO_THREE, ROW); - private Pair PAIR_FOUR = new Pair<>(REPLICA_INFO_FOUR, ROW); - - @Test - public void assertLeaderProperties() { - assertTrue(REPLICA_INFO_ONE.isLeader()); - assertFalse(REPLICA_INFO_TWO.isLeader()); - assertTrue(REPLICA_INFO_THREE.isLeader()); - assertFalse(REPLICA_INFO_FOUR.isLeader()); - } - - @Test - public void sortReplicasValidate() { - List> validReplicas = new ArrayList>() { - { - add(PAIR_ONE); - add(PAIR_FOUR); - add(PAIR_TWO); - } - }; - if (random().nextBoolean()) { - Collections.shuffle(validReplicas, random()); - } - validReplicas.sort(leaderLast); - - assertFalse(isReplicaLeader(validReplicas, 0)); - assertFalse(isReplicaLeader(validReplicas, 1)); - assertTrue(isReplicaLeader(validReplicas, 2)); - } - - @Test - public void sortReplicasValidateLeadersMultipleLeadersComeLast() { - List> validReplicas = new ArrayList>() { - { - add(PAIR_THREE); - add(PAIR_ONE); - add(PAIR_FOUR); - add(PAIR_TWO); - } - }; - if (random().nextBoolean()) { - Collections.shuffle(validReplicas, random()); - } - validReplicas.sort(leaderLast); - - assertFalse(isReplicaLeader(validReplicas, 0)); - assertFalse(isReplicaLeader(validReplicas, 1)); - assertTrue(isReplicaLeader(validReplicas, 2)); - assertTrue(isReplicaLeader(validReplicas, 3)); - } - - private boolean isReplicaLeader(List> replicas, int index) { - return replicas.get(index).first().isLeader(); - } - -} diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java b/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java deleted file mode 100644 index cc049a3dbee..00000000000 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java +++ /dev/null @@ -1,3358 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.client.solrj.cloud.autoscaling; - - -import java.io.IOException; -import java.io.StringWriter; -import java.lang.invoke.MethodHandles; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicLong; -import java.util.stream.Collectors; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Sets; -import org.apache.solr.SolrTestCaseJ4; -import org.apache.solr.client.solrj.SolrRequest; -import org.apache.solr.client.solrj.SolrResponse; -import org.apache.solr.client.solrj.V2RequestSupport; -import org.apache.solr.client.solrj.cloud.DistribStateManager; -import org.apache.solr.client.solrj.cloud.DistributedQueueFactory; -import org.apache.solr.client.solrj.cloud.NodeStateProvider; -import org.apache.solr.client.solrj.cloud.SolrCloudManager; -import org.apache.solr.client.solrj.cloud.autoscaling.Suggester.Hint; -import org.apache.solr.client.solrj.impl.ClusterStateProvider; -import org.apache.solr.client.solrj.impl.SolrClientNodeStateProvider; -import org.apache.solr.client.solrj.request.CollectionAdminRequest; -import org.apache.solr.cloud.api.collections.Assign; -import org.apache.solr.common.MapWriter; -import org.apache.solr.common.cloud.ClusterState; -import org.apache.solr.common.cloud.DocCollection; -import org.apache.solr.common.cloud.DocRouter; -import org.apache.solr.common.cloud.Replica; -import org.apache.solr.common.cloud.ReplicaPosition; -import org.apache.solr.common.cloud.ZkStateReader; -import org.apache.solr.common.cloud.rule.ImplicitSnitch; -import org.apache.solr.common.params.CollectionAdminParams; -import org.apache.solr.common.params.CollectionParams; -import org.apache.solr.common.params.CollectionParams.CollectionAction; -import org.apache.solr.common.params.SolrParams; -import org.apache.solr.common.util.JsonTextWriter; -import org.apache.solr.common.util.NamedList; -import org.apache.solr.common.util.ObjectCache; -import org.apache.solr.common.util.Pair; -import org.apache.solr.common.util.SolrJSONWriter; -import org.apache.solr.common.util.TimeSource; -import org.apache.solr.common.util.Utils; -import org.apache.solr.common.util.ValidatingJsonMap; -import org.apache.solr.response.JSONWriter; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import static java.nio.charset.StandardCharsets.UTF_8; -import static org.apache.solr.client.solrj.cloud.autoscaling.Policy.CLUSTER_POLICY; -import static org.apache.solr.client.solrj.cloud.autoscaling.Policy.CLUSTER_PREFERENCES; -import static org.apache.solr.client.solrj.cloud.autoscaling.TestPolicy2.loadFromResource; -import static org.apache.solr.client.solrj.cloud.autoscaling.Variable.Type.CORES; -import static org.apache.solr.client.solrj.cloud.autoscaling.Variable.Type.FREEDISK; -import static org.apache.solr.client.solrj.cloud.autoscaling.Variable.Type.REPLICA; -import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDREPLICA; -import static org.apache.solr.common.params.CollectionParams.CollectionAction.MOVEREPLICA; - -public class TestPolicy extends SolrTestCaseJ4 { - boolean useNodeset ; - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - public TestPolicy(){ - useNodeset = true; - } - @SuppressWarnings({"unchecked"}) - static Suggester createSuggester(SolrCloudManager cloudManager, - @SuppressWarnings({"rawtypes"})Map jsonObj, Suggester seed) throws IOException, InterruptedException { - Policy.Session session = null; - if (seed != null) session = seed.session; - else { - session = cloudManager.getDistribStateManager().getAutoScalingConfig().getPolicy().createSession(cloudManager); - } - - @SuppressWarnings({"rawtypes"}) - Map m = (Map) jsonObj.get("suggester"); - Suggester result = session.getSuggester(CollectionParams.CollectionAction.get((String) m.get("action"))); - m = (Map) m.get("hints"); - m.forEach((k, v) -> { - Hint hint = Hint.get(k.toString()); - result.hint(hint, hint.parse(v)); - }); - return result; - } - - static SolrCloudManager createCloudManager(@SuppressWarnings({"rawtypes"})Map jsonObj) { - return cloudManagerWithData(jsonObj); - } - - public static String clusterState = Utils.toJSONString(loadFromResource("testPolicy.json")); - - public static Map>> getReplicaDetails(String node, - @SuppressWarnings({"rawtypes"})Map clusterState) { - ValidatingJsonMap m = ValidatingJsonMap - .getDeepCopy(clusterState, 6, true); - Map>> result = new LinkedHashMap<>(); - - m.forEach((collName, o) -> { - ValidatingJsonMap coll = (ValidatingJsonMap) o; - coll.getMap("shards").forEach((shard, o1) -> { - ValidatingJsonMap sh = (ValidatingJsonMap) o1; - sh.getMap("replicas").forEach((replicaName, o2) -> { - ValidatingJsonMap r = (ValidatingJsonMap) o2; - String node_name = (String) r.get("node_name"); - if (!node_name.equals(node)) return; - Map> shardVsReplicaStats = result.computeIfAbsent(collName, k -> new HashMap<>()); - List replicaInfos = shardVsReplicaStats.computeIfAbsent(shard, k -> new ArrayList<>()); - replicaInfos.add(new Replica(replicaName, node, collName, shard, (String) r.get("core"), - Replica.State.ACTIVE, Replica.Type.get((String) r.get(ZkStateReader.REPLICA_TYPE)), r)); - }); - }); - }); - return result; - } - - - public void testWithCollection() { - @SuppressWarnings({"unchecked"}) - ClusterState clusterState = ClusterState.createFromCollectionMap(1, - (Map) loadFromResource("testWithCollection.json"), - ImmutableSet.of("node1", "node2", "node3", "node4", "node5")); - DelegatingClusterStateProvider clusterStateProvider = new DelegatingClusterStateProvider(null) { - @Override - public ClusterState getClusterState() throws IOException { - return clusterState; - } - - @Override - public Set getLiveNodes() { - return clusterState.getLiveNodes(); - } - }; - - SolrClientNodeStateProvider solrClientNodeStateProvider = new SolrClientNodeStateProvider(null) { - @Override - protected Map fetchTagValues(String node, Collection tags) { - Map result = new HashMap<>(); - AtomicInteger cores = new AtomicInteger(); - forEachReplica(node, replicaInfo -> cores.incrementAndGet()); - if (tags.contains(ImplicitSnitch.CORES)) result.put(ImplicitSnitch.CORES, cores.get()); - if (tags.contains(ImplicitSnitch.DISK)) result.put(ImplicitSnitch.DISK, 100); - return result; - } - - @Override - protected Map fetchReplicaMetrics(String solrNode, Map> metricsKeyVsTagReplica) { - //e.g: solr.core.perReplicaDataColl.shard1.replica_n4:INDEX.sizeInBytes - Map result = new HashMap<>(); - metricsKeyVsTagReplica.forEach((k, v) -> { - if (k.endsWith(":INDEX.sizeInBytes")) result.put(k, 100); - }); - return result; - } - - @Override - protected ClusterStateProvider getClusterStateProvider() { - return clusterStateProvider; - } - }; - @SuppressWarnings({"rawtypes"}) - Map m = solrClientNodeStateProvider.getNodeValues("node1", ImmutableSet.of("cores", "withCollection")); - assertNotNull(m.get("withCollection")); - - @SuppressWarnings({"rawtypes"}) - Map policies = (Map) Utils.fromJSONString("{" + - " 'cluster-preferences': [" + - " { 'minimize': 'cores'}," + - " { 'maximize': 'freedisk', 'precision': 50}" + - " ]," + - " 'cluster-policy': [" + - " { 'replica': 0, 'nodeRole': 'overseer'}" + - " { 'replica': '<2', 'shard': '#EACH', 'node': '#ANY'}," + - " ]" + - "}"); - @SuppressWarnings({"unchecked"}) - AutoScalingConfig config = new AutoScalingConfig(policies); - Policy policy = config.getPolicy(); - Policy.Session session = policy.createSession(new DelegatingCloudManager(null) { - @Override - public ClusterStateProvider getClusterStateProvider() { - return clusterStateProvider; - } - - @Override - public NodeStateProvider getNodeStateProvider() { - return solrClientNodeStateProvider; - } - }); - Suggester suggester = session.getSuggester(CollectionAction.ADDREPLICA); - suggester.hint(Hint.COLL_SHARD, new Pair<>("comments_coll", "shard1")); - @SuppressWarnings({"rawtypes"}) - SolrRequest op = suggester.getSuggestion(); - assertNotNull(op); - Set nodes = new HashSet<>(2); - nodes.add(op.getParams().get("node")); - session = suggester.getSession(); - suggester = session.getSuggester(ADDREPLICA); - suggester.hint(Hint.COLL_SHARD, new Pair<>("comments_coll", "shard1")); - op = suggester.getSuggestion(); - assertNotNull(op); - nodes.add(op.getParams().get("node")); - assertEquals(2, nodes.size()); - assertTrue("node1 should have been selected by add replica", nodes.contains("node1")); - assertTrue("node2 should have been selected by add replica", nodes.contains("node2")); - - session = suggester.getSession(); - suggester = session.getSuggester(MOVEREPLICA); - suggester.hint(Hint.COLL_SHARD, new Pair<>("comments_coll", "shard1")); - op = suggester.getSuggestion(); - assertNull(op); - } - - public void testWithCollectionSuggestions() { - @SuppressWarnings({"unchecked"}) - ClusterState clusterState = - ClusterState.createFromCollectionMap(1, - (Map) loadFromResource("testWithCollectionSuggestions.json"), - ImmutableSet.of("node1", "node2", "node3", "node4", "node5")); - DelegatingClusterStateProvider clusterStateProvider = new DelegatingClusterStateProvider(null) { - @Override - public ClusterState getClusterState() throws IOException { - return clusterState; - } - - @Override - public Set getLiveNodes() { - return clusterState.getLiveNodes(); - } - }; - - SolrClientNodeStateProvider solrClientNodeStateProvider = new SolrClientNodeStateProvider(null) { - @Override - protected Map fetchTagValues(String node, Collection tags) { - Map result = new HashMap<>(); - AtomicInteger cores = new AtomicInteger(); - forEachReplica(node, replicaInfo -> cores.incrementAndGet()); - if (tags.contains(ImplicitSnitch.CORES)) result.put(ImplicitSnitch.CORES, cores.get()); - if (tags.contains(ImplicitSnitch.DISK)) result.put(ImplicitSnitch.DISK, 100); - return result; - } - - @Override - protected Map fetchReplicaMetrics(String solrNode, Map> metricsKeyVsTagReplica) { - //e.g: solr.core.perReplicaDataColl.shard1.replica_n4:INDEX.sizeInBytes - Map result = new HashMap<>(); - metricsKeyVsTagReplica.forEach((k, v) -> { - if (k.endsWith(":INDEX.sizeInBytes")) result.put(k, 100); - }); - return result; - } - - @Override - protected ClusterStateProvider getClusterStateProvider() { - return clusterStateProvider; - } - }; - @SuppressWarnings({"rawtypes"}) - Map m = solrClientNodeStateProvider.getNodeValues("node1", ImmutableSet.of("cores", "withCollection")); - assertNotNull(m.get("withCollection")); - - @SuppressWarnings({"rawtypes"}) - Map policies = (Map) Utils.fromJSONString("{" + - " 'cluster-preferences': [" + - " { 'maximize': 'freedisk', 'precision': 50}," + - " { 'minimize': 'cores'}" + - " ]," + - " 'cluster-policy': [" + - " { 'replica': 0, 'nodeRole': 'overseer'}" + - " { 'replica': '<2', 'shard': '#EACH', 'node': '#ANY'}," + - " ]" + - "}"); - - @SuppressWarnings({"unchecked"}) - List l = PolicyHelper.getSuggestions(new AutoScalingConfig(policies), - new DelegatingCloudManager(null) { - @Override - public ClusterStateProvider getClusterStateProvider() { - return clusterStateProvider; - } - - @Override - public NodeStateProvider getNodeStateProvider() { - return solrClientNodeStateProvider; - } - }); - assertNotNull(l); - assertEquals(2, l.size()); - - // collect the set of nodes to which replicas are being added - Set nodes = new HashSet<>(2); - - assertEquals(1.0d, l.get(0)._get("violation/violation/delta", null)); - assertEquals("POST", l.get(0)._get("operation/method", null)); - assertEquals("/c/articles_coll/shards", l.get(0)._get("operation/path", null)); - assertNotNull(l.get(0)._get("operation/command/add-replica", null)); - nodes.add((String) l.get(0)._get("operation/command/add-replica/node", null)); - - assertEquals(1.0d, l.get(1)._get("violation/violation/delta", null)); - assertEquals("POST", l.get(1)._get("operation/method", null)); - assertEquals("/c/articles_coll/shards", l.get(1)._get("operation/path", null)); - assertNotNull(l.get(1)._get("operation/command/add-replica", null)); - nodes.add((String) l.get(1)._get("operation/command/add-replica/node", null)); - - assertEquals(2, nodes.size()); - assertTrue(nodes.contains("node1")); - assertTrue(nodes.contains("node2")); - } - - public void testWithCollectionMoveVsAddSuggestions() throws IOException { - @SuppressWarnings({"unchecked"}) - ClusterState clusterState = ClusterState.createFromCollectionMap(1, - (Map) loadFromResource("testWithCollectionMoveVsAddSuggestions.json"), - ImmutableSet.of("node1", "node2", "node3", "node4", "node5", "node6")); - DelegatingClusterStateProvider clusterStateProvider = new DelegatingClusterStateProvider(null) { - @Override - public ClusterState getClusterState() { - return clusterState; - } - - @Override - public Set getLiveNodes() { - return clusterState.getLiveNodes(); - } - }; - - SolrClientNodeStateProvider solrClientNodeStateProvider = new SolrClientNodeStateProvider(null) { - @Override - protected Map fetchTagValues(String node, Collection tags) { - Map result = new HashMap<>(); - AtomicInteger cores = new AtomicInteger(); - forEachReplica(node, replicaInfo -> cores.incrementAndGet()); - if (tags.contains(ImplicitSnitch.CORES)) result.put(ImplicitSnitch.CORES, cores.get()); - if (tags.contains(ImplicitSnitch.DISK)) result.put(ImplicitSnitch.DISK, 100); - return result; - } - - @Override - protected Map fetchReplicaMetrics(String solrNode, Map> metricsKeyVsTagReplica) { - //e.g: solr.core.perReplicaDataColl.shard1.replica_n4:INDEX.sizeInBytes - Map result = new HashMap<>(); - metricsKeyVsTagReplica.forEach((k, v) -> { - if (k.endsWith(":INDEX.sizeInBytes")) result.put(k, 100); - }); - return result; - } - - @Override - protected ClusterStateProvider getClusterStateProvider() { - return clusterStateProvider; - } - }; - @SuppressWarnings({"rawtypes"}) - Map m = solrClientNodeStateProvider.getNodeValues("node1", ImmutableSet.of("cores", "withCollection")); - assertNotNull(m.get("withCollection")); - - @SuppressWarnings({"rawtypes"}) - Map policies = (Map) Utils.fromJSONString("{" + - " 'cluster-preferences': [" + - " { 'maximize': 'freedisk', 'precision': 50}," + - " { 'minimize': 'cores'}" + - " ]," + - " 'cluster-policy': [" + - " { 'replica': 0, 'nodeRole': 'overseer'}" + - " { 'replica': '<2', 'shard': '#EACH', 'node': '#ANY'}," + - " ]" + - "}"); - - @SuppressWarnings({"unchecked"}) - List l = PolicyHelper.getSuggestions(new AutoScalingConfig(policies), - new DelegatingCloudManager(null) { - @Override - public ClusterStateProvider getClusterStateProvider() { - return clusterStateProvider; - } - - @Override - public NodeStateProvider getNodeStateProvider() { - return solrClientNodeStateProvider; - } - }); - assertNotNull(l); - assertEquals(3, l.size()); - - // collect the set of nodes to which replicas are being added - Set nodes = new HashSet<>(2); - - int numMoves = 0, numAdds = 0; - Set addNodes = new HashSet<>(); - Set targetNodes = new HashSet<>(); - Set movedReplicas = new HashSet<>(); - for (Suggester.SuggestionInfo suggestionInfo : l) { - assertEquals("POST", suggestionInfo._get("operation/method", null)); - if (suggestionInfo._get("operation/command/add-replica", null) != null) { - numAdds++; - assertEquals(1.0d, suggestionInfo._get("violation/violation/delta", null)); - assertEquals("/c/articles_coll/shards", suggestionInfo._get("operation/path", null)); - addNodes.add((String) suggestionInfo._get("operation/command/add-replica/node", null)); - } else if (suggestionInfo._get("operation/command/move-replica", null) != null) { - numMoves++; - assertEquals("/c/articles_coll", suggestionInfo._get("operation/path", null)); - targetNodes.add((String) suggestionInfo._get("operation/command/move-replica/targetNode", null)); - movedReplicas.add((String) suggestionInfo._get("operation/command/move-replica/replica", null)); - } else { - fail("Unexpected operation type suggested for suggestion: " + suggestionInfo); - } - } - - assertEquals(2, targetNodes.size()); - assertEquals(1, addNodes.size()); - assertEquals(2, movedReplicas.size()); - Set allTargetNodes = new HashSet<>(targetNodes); - allTargetNodes.addAll(addNodes); - assertEquals(3, allTargetNodes.size()); - assertTrue(allTargetNodes.contains("node3")); - assertTrue(allTargetNodes.contains("node4")); - assertTrue(allTargetNodes.contains("node5")); - } - - public void testWithCollectionMoveReplica() { - @SuppressWarnings({"unchecked"}) - ClusterState clusterState = ClusterState.createFromCollectionMap(1, - (Map) loadFromResource("testWithCollectionMoveReplica.json"), - ImmutableSet.of("node2", "node3", "node4", "node5")); - DelegatingClusterStateProvider clusterStateProvider = new DelegatingClusterStateProvider(null) { - @Override - public ClusterState getClusterState() throws IOException { - return clusterState; - } - - @Override - public Set getLiveNodes() { - return clusterState.getLiveNodes(); - } - }; - - SolrClientNodeStateProvider solrClientNodeStateProvider = new SolrClientNodeStateProvider(null) { - @Override - protected Map fetchTagValues(String node, Collection tags) { - Map result = new HashMap<>(); - AtomicInteger cores = new AtomicInteger(); - forEachReplica(node, replicaInfo -> cores.incrementAndGet()); - if (tags.contains(ImplicitSnitch.CORES)) result.put(ImplicitSnitch.CORES, cores.get()); - if (tags.contains(ImplicitSnitch.DISK)) result.put(ImplicitSnitch.DISK, 100); - return result; - } - - @Override - protected Map fetchReplicaMetrics(String solrNode, Map> metricsKeyVsTagReplica) { - //e.g: solr.core.perReplicaDataColl.shard1.replica_n4:INDEX.sizeInBytes - Map result = new HashMap<>(); - metricsKeyVsTagReplica.forEach((k, v) -> { - if (k.endsWith(":INDEX.sizeInBytes")) result.put(k, 100); - }); - return result; - } - - @Override - protected ClusterStateProvider getClusterStateProvider() { - return clusterStateProvider; - } - }; - @SuppressWarnings({"rawtypes"}) - Map m = solrClientNodeStateProvider.getNodeValues("node1", ImmutableSet.of("cores", "withCollection")); - assertNotNull(m.get("withCollection")); - - @SuppressWarnings({"rawtypes"}) - Map policies = (Map) Utils.fromJSONString("{" + - " 'cluster-preferences': [" + - " { 'minimize': 'cores'}," + - " { 'maximize': 'freedisk', 'precision': 50}" + - " ]," + - " 'cluster-policy': [" + - " { 'replica': 0, 'nodeRole': 'overseer'}" + - " { 'replica': '<2', 'shard': '#EACH', 'node': '#ANY'}," + - " ]" + - "}"); - @SuppressWarnings({"unchecked"}) - AutoScalingConfig config = new AutoScalingConfig(policies); - Policy policy = config.getPolicy(); - Policy.Session session = policy.createSession(new DelegatingCloudManager(null) { - @Override - public ClusterStateProvider getClusterStateProvider() { - return clusterStateProvider; - } - - @Override - public NodeStateProvider getNodeStateProvider() { - return solrClientNodeStateProvider; - } - }); - Suggester suggester = session.getSuggester(CollectionAction.MOVEREPLICA); - suggester.hint(Hint.COLL_SHARD, new Pair<>("comments_coll", "shard1")); - suggester.hint(Hint.SRC_NODE, "node1"); - @SuppressWarnings({"rawtypes"}) - SolrRequest op = suggester.getSuggestion(); - assertNotNull(op); - assertEquals("node2 should have been selected by move replica", "node2", - op.getParams().get("targetNode")); - - session = suggester.getSession(); - suggester = session.getSuggester(MOVEREPLICA); - suggester.hint(Hint.COLL_SHARD, new Pair<>("comments_coll", "shard1")); - suggester.hint(Hint.SRC_NODE, "node1"); - op = suggester.getSuggestion(); - assertNull(op); - } - - public void testValidate() { - expectError("replica", -1, "must be greater than"); - expectError("replica", "hello", "not a valid number"); - assertEquals(1d, Clause.validate("replica", "1", true)); - assertEquals("c", Clause.validate("collection", "c", true)); - assertEquals("s", Clause.validate("shard", "s", true)); - assertEquals("overseer", Clause.validate("nodeRole", "overseer", true)); - - expectError("nodeRole", "wrong", "must be one of"); - - expectError("sysLoadAvg", "101", "must be less than "); - expectError("sysLoadAvg", 101, "must be less than "); - expectError("sysLoadAvg", "-1", "must be greater than"); - expectError("sysLoadAvg", -1, "must be greater than"); - - assertEquals(12.46d, Clause.validate("sysLoadAvg", "12.46", true)); - assertEquals(12.46, Clause.validate("sysLoadAvg", 12.46d, true)); - - - expectError("ip_1", "300", "must be less than "); - expectError("ip_1", 300, "must be less than "); - expectError("ip_1", "-1", "must be greater than"); - expectError("ip_1", -1, "must be greater than"); - - assertEquals(1L, Clause.validate("ip_1", "1", true)); - - expectError("heapUsage", "-1", "must be greater than"); - expectError("heapUsage", -1, "must be greater than"); - assertEquals(69.9d, Clause.validate("heapUsage", "69.9", true)); - assertEquals(69.9d, Clause.validate("heapUsage", 69.9d, true)); - - expectError("port", "70000", "must be less than "); - expectError("port", 70000, "must be less than "); - expectError("port", "0", "must be greater than"); - expectError("port", 0, "must be greater than"); - - expectError("cores", "-1", "must be greater than"); - - assertEquals(Operand.EQUAL, REPLICA.getOperand(Operand.EQUAL, "2.0", null)); - assertEquals(Operand.NOT_EQUAL, REPLICA.getOperand(Operand.NOT_EQUAL, "2.0", null)); - assertEquals(Operand.EQUAL, REPLICA.getOperand(Operand.EQUAL, "2", null)); - assertEquals(Operand.NOT_EQUAL, REPLICA.getOperand(Operand.NOT_EQUAL, "2", null)); - assertEquals(Operand.RANGE_EQUAL, REPLICA.getOperand(Operand.EQUAL, "2.1", null)); - assertEquals(Operand.RANGE_EQUAL, REPLICA.getOperand(Operand.EQUAL, "2.01", null)); - - Clause clause = Clause.create("{replica: '1.23', node:'#ANY'}"); - assertTrue(clause.getReplica().isPass(2)); - assertTrue(clause.getReplica().isPass(1)); - assertFalse(clause.getReplica().isPass(0)); - assertFalse(clause.getReplica().isPass(3)); - - clause = Clause.create("{replica: '<1.23', node:'#ANY'}"); - assertTrue(clause.getReplica().isPass(1)); - assertTrue(clause.getReplica().isPass(0)); - assertFalse(clause.getReplica().isPass(2)); - - expectThrows(IllegalArgumentException.class, - () -> Clause.create("{replica: '!1.23', node:'#ANY'}")); - - - clause = Clause.create("{replica: 1.23, node:'#ANY'}"); - assertTrue(clause.getReplica().isPass(2)); - assertTrue(clause.getReplica().isPass(1)); - assertFalse(clause.getReplica().isPass(0)); - assertFalse(clause.getReplica().isPass(3)); - - clause = Clause.create("{replica: '33%', node:'#ANY'}"); - assertEquals(Operand.RANGE_EQUAL, clause.getReplica().op); - clause = clause.getSealedClause(condition -> { - if (condition.name.equals("replica")) { - return 2.0d; - } - throw new RuntimeException(""); - }); - assertTrue(clause.getReplica().isPass(2)); - - clause = Clause.create("{replica: '3 - 5', node:'#ANY'}"); - assertEquals(Operand.RANGE_EQUAL, clause.getReplica().getOperand()); - RangeVal range = (RangeVal) clause.getReplica().getValue(); - assertEquals(3.0, range.min); - assertEquals(5.0, range.max); - assertTrue(clause.replica.isPass(3)); - assertTrue(clause.replica.isPass(4)); - assertTrue(clause.replica.isPass(5)); - assertFalse(clause.replica.isPass(6)); - assertFalse(clause.replica.isPass(2)); - - assertEquals(Double.valueOf(1.0), clause.replica.delta(6)); - assertEquals(Double.valueOf(-1.0), clause.replica.delta(2)); - assertEquals(Double.valueOf(0.0), clause.replica.delta(4)); - - expectThrows(IllegalArgumentException.class, - () -> Clause.create("{replica: '-33%', node:'#ANY'}")); - expectThrows(IllegalArgumentException.class, - () -> Clause.create("{replica: 'x%', node:'#ANY'}")); - expectThrows(IllegalArgumentException.class, - () -> Clause.create("{replica: '20%-33%', node:'#ANY'}")); - - clause = Clause.create("{replica: '#EQUAL', shard:'#EACH', node:'#ANY'}"); - assertEquals(Operand.RANGE_EQUAL, clause.replica.op); - clause = Clause.create("{replica: '#EQUAL', node:'#ANY'}"); - assertEquals(Operand.RANGE_EQUAL, clause.replica.op); - expectThrows(IllegalArgumentException.class, - () -> Clause.create("{replica: '#EQUAL', node:'node_1'}")); - clause = Clause.create("{replica : 0, freedisk:'<20%'}"); - assertEquals(clause.tag.computedType, ComputedType.PERCENT); - assertEquals(clause.tag.op, Operand.LESS_THAN); - expectThrows(IllegalArgumentException.class, - () -> Clause.create("{replica : 0, INDEX.sizeInGB:'>300'}")); - expectThrows(IllegalArgumentException.class, - () -> Clause.create("{replica:'<3', shard: '#ANV', node:'#ANY'}")); - expectThrows(IllegalArgumentException.class, - () -> Clause.create("{replica:'<3', shard: '#EACH', node:'#E4CH'}")); - try { - Clause.create("{replica:0, 'ip_1':'<30%'}"); - fail("Expected exception"); - } catch (Exception e) { - assertTrue(e.getMessage().contains("'%' is not allowed for variable : 'ip_1'")); - } - - clause = Clause.create("{replica: '#ALL', freedisk:'>20%'}"); - clause = Clause.create("{replica: '#ALL', sysprop.zone :'west'}"); - expectThrows(IllegalArgumentException.class, - () -> Clause.create("{replica: [3,4] , freedisk:'>20'}")); - clause = Clause.create("{replica: 3 , port:[8983, 7574]}"); - assertEquals(Operand.IN, clause.tag.op); - expectThrows(IllegalArgumentException.class, - () -> Clause.create("{replica: 3 , sysprop.zone :['east', ' ', 'west']}")); - expectThrows(IllegalArgumentException.class, - () -> Clause.create("{replica: 3 , sysprop.zone :[]}")); - expectThrows(IllegalArgumentException.class, - () -> Clause.create("{replica: 3 , sysprop.zone :['!east','west']}")); - expectThrows(IllegalArgumentException.class, - () -> Clause.create("{replica: '#ALL' , shard: '#EACH' , sysprop.zone:[east, west]}")); - expectThrows(IllegalArgumentException.class, - () -> Clause.create("{replica: '#ALL' , shard: '#EACH' , sysprop.zone:'#EACH'}")); - clause = Clause.create("{replica: '#EQUAL' , shard: '#EACH' , sysprop.zone:[east, west]}"); - assertEquals(ComputedType.EQUAL, clause.replica.computedType); - assertEquals(Operand.IN, clause.tag.op); - expectThrows(IllegalArgumentException.class, - () -> Clause.create("{replica: '#EQUAL' , shard: '#EACH' , sysprop.zone:[east]}")); - - clause = Clause.create("{cores: '#EQUAL' , node:'#ANY'}"); - assertEquals(ComputedType.EQUAL, clause.globalTag.computedType); - expectThrows(IllegalArgumentException.class, - () -> Clause.create("{cores: '#EQUAL' , node:'node1'}")); - - clause = Clause.create("{cores: '#EQUAL' , node:[node1 , node2 , node3]}"); - assertEquals(Operand.IN, clause.getTag().op); - assertEquals(ComputedType.EQUAL, clause.getGlobalTag().computedType); - - clause = Clause.create("{cores: '3-5' , node:'#ANY'}"); - assertEquals(Operand.RANGE_EQUAL, clause.globalTag.op); - assertEquals(3.0d, ((RangeVal) clause.globalTag.val).min.doubleValue(), 0.001); - assertEquals(5.0d, ((RangeVal) clause.globalTag.val).max.doubleValue(), 0.001); - - clause = Clause.create("{cores: 1.66 , node:'#ANY'}"); - assertEquals(Operand.RANGE_EQUAL, clause.globalTag.op); - assertEquals(1.0d, ((RangeVal) clause.globalTag.val).min.doubleValue(), 0.001); - assertEquals(2.0d, ((RangeVal) clause.globalTag.val).max.doubleValue(), 0.001); - assertEquals(1.66d, ((RangeVal) clause.globalTag.val).actual.doubleValue(), 0.001); - - expectThrows(IllegalArgumentException.class, - () -> Clause.create("{cores:5, sysprop.zone : west}")); - - clause = Clause.create("{cores: '14%' , node:'#ANY'}"); - assertEquals(ComputedType.PERCENT, clause.getGlobalTag().computedType); - - clause = Clause.create("{cores: '14%' , node:[node1, node2, node3]}"); - assertEquals(Operand.IN, clause.getTag().op); - - expectThrows(IllegalArgumentException.class, - () -> Clause.create("{replica: '!14%' , node:'#ANY'}")); - - expectThrows(IllegalArgumentException.class, - () -> Clause.create("{cores: '!14%' , node:[node1, node2, node3]}")); - - expectThrows(IllegalArgumentException.class, - () -> Clause.create("{cores: '!1.66' , node:'#ANY'}")); - expectThrows(IllegalArgumentException.class, - () -> Clause.create("{replica: '<14%' , node:'#ANY'}")); - expectThrows(IllegalArgumentException.class, - () -> Clause.create("{replica: '>14%' , node:'#ANY'}")); - - expectThrows(IllegalArgumentException.class, - () -> Clause.create("{cores: '>14%' , node:'#ANY'}")); - clause = Clause.create("{replica:1, nodeset : {sysprop.zone : east}}"); - assertEquals(Variable.Type.SYSPROP, clause.tag.varType); - clause =Clause.create("{replica:1, nodeset : [node1, node2, node3]}"); - assertEquals(Variable.Type.NODE, clause.tag.varType); - assertEquals(Operand.IN, clause.tag.op); - - expectThrows(IllegalArgumentException.class, - () -> Clause.create("{replica:1, nodeset : {sysprop.zone : '#EACH'}}")); - - expectThrows(IllegalArgumentException.class, - () -> Clause.create("{replica:1, nodeset : {host : '#EACH'}}")); - - expectThrows(IllegalArgumentException.class, - () -> Clause.create("{replica:1, node: n1, nodeset : {sysprop.zone : east}}")); - - IllegalArgumentException exp = expectThrows(IllegalArgumentException.class, - () -> Clause.create("{replica:1, nodeset : {sysprop.zone : east , port: 8983 }}")); - assertTrue(exp.getMessage().contains("nodeset must only have one and only one key")); - clause = Clause.create("{'replica': '#ALL', 'nodeset': {'freedisk': '>700'}, 'strict': false}"); - assertEquals(clause.put, Clause.Put.ON_ALL); - assertEquals(Operand.GREATER_THAN , clause.tag.op); - clause = Clause.create("{'replica': '#ALL', put: on-each-node, 'nodeset': {sysprop.zone : east}}"); - assertEquals(clause.put, Clause.Put.ON_EACH); - exp = expectThrows(IllegalArgumentException.class, ()-> Clause.create("{'replica': '#ALL', put: on-Each, 'nodeset': {sysprop.zone : east}}")); - assertTrue(exp.getMessage().contains("invalid value for put : on-Each")); - - clause = Clause.create("{replica : '#EQUAL', nodeset : [{sysprop.zone : east}, {sysprop.zone : west}]}"); - assertTrue(((List)clause.tag.val).get(0) instanceof Condition); - assertTrue( Utils.fromJSONString(Utils.toJSONString(clause)) instanceof Map); - exp = expectThrows(IllegalArgumentException.class, ()-> Clause.create("{replica : '#EQUAL', nodeset : [{sysprop.zone : east}, {port : '8983'}]}")); - assertTrue(exp.getMessage().contains("all element must have same key")); - exp = expectThrows(IllegalArgumentException.class, ()-> Clause.create("{replica : '#EQUAL', nodeset : [{sysprop.zone : east}, {sysprop.zone : '#EACH'}]}")); - assertTrue(exp.getMessage().contains("computed value #EACH not allowed in nodeset")); - exp = expectThrows(IllegalArgumentException.class, ()-> Clause.create("{replica : '#EQUAL', nodeset : {sysprop.zone : east}}")); - assertTrue(exp.getMessage().contains("'nodeset' must have an array value when 'replica': '#EQUAL` is used")); - exp = expectThrows(IllegalArgumentException.class, ()-> Clause.create("{replica : '#ALL', nodeset : [{sysprop.zone : east}, {sysprop.zone : west}]}")); - assertTrue(exp.getMessage().contains("cannot use array value for nodeset if replica : '#EQUAL' is not used")); - exp = expectThrows(IllegalArgumentException.class, ()-> Clause.create("{replica : '50%', nodeset : [{sysprop.zone : east}, {sysprop.zone : west}]}")); - assertTrue(exp.getMessage().contains("cannot use array value for nodeset if replica : '#EQUAL' is not used")); - exp = expectThrows(IllegalArgumentException.class, ()-> Clause.create("{replica : 3, nodeset : [{sysprop.zone : east}, {sysprop.zone : west}]}")); - assertTrue(exp.getMessage().contains("cannot use array value for nodeset if replica : '#EQUAL' is not used")); - exp = expectThrows(IllegalArgumentException.class, ()-> Clause.create("{replica : '#EQUAL', put: on-each-node, nodeset : [{sysprop.zone : east}, {sysprop.zone : west}]}")); - assertTrue(exp.getMessage().contains("cannot use put: 'on-each-node' with an array value in nodeset ")); - } - - - public void testEqualFunction() { - - String clusterStateStr = "{" + - " 'coll1': {" + - " 'router': {" + - " 'name': 'compositeId'" + - " }," + - " 'shards': {" + - " 'shard1': {" + - " 'range': '80000000-ffffffff'," + - " 'replicas': {" + - " 'r1': {" + - " 'core': 'r1'," + - " 'base_url': 'http://10.0.0.4:8983/solr'," + - " 'node_name': 'node1'," + - " 'state': 'active'," + - " 'leader': 'true'" + - " }," + - " 'r2': {" + - " 'core': 'r2'," + - " 'base_url': 'http://10.0.0.4:7574/solr'," + - " 'node_name': 'node2'," + - " 'state': 'active'" + - " }" + - " }" + - " }," + - " 'shard2': {" + - " 'range': '0-7fffffff'," + - " 'replicas': {" + - " 'r3': {" + - " 'core': 'r3'," + - " 'base_url': 'http://10.0.0.4:8983/solr'," + - " 'node_name': 'node1'," + - " 'state': 'active'," + - " 'leader': 'true'" + - " }," + - " 'r4': {" + - " 'core': 'r4'," + - " 'base_url': 'http://10.0.0.4:8987/solr'," + - " 'node_name': 'node4'," + - " 'state': 'active'" + - " }," + - " 'r6': {" + - " 'core': 'r6'," + - " 'base_url': 'http://10.0.0.4:8989/solr'," + - " 'node_name': 'node3'," + - " 'state': 'active'" + - " }," + - " 'r5': {" + - " 'core': 'r5'," + - " 'base_url': 'http://10.0.0.4:8983/solr'," + - " 'node_name': 'node1'," + - " 'state': 'active'" + - " }" + - " }" + - " }" + - " }" + - " }" + - "}"; - - - ClusterState clusterState = ClusterState.createFromJson(1, clusterStateStr.getBytes(UTF_8), - ImmutableSet.of("node1", "node2", "node3", "node4", "node5")); - DelegatingClusterStateProvider clusterStateProvider = new DelegatingClusterStateProvider(null) { - @Override - public ClusterState getClusterState() throws IOException { - return clusterState; - } - - @Override - public Set getLiveNodes() { - return clusterState.getLiveNodes(); - } - }; - - SolrClientNodeStateProvider solrClientNodeStateProvider = new SolrClientNodeStateProvider(null) { - @Override - protected Map fetchTagValues(String node, Collection tags) { - Map result = new HashMap<>(); - AtomicInteger cores = new AtomicInteger(); - forEachReplica(node, replicaInfo -> cores.incrementAndGet()); - if (tags.contains(ImplicitSnitch.CORES)) result.put(ImplicitSnitch.CORES, cores.get()); - if (tags.contains(ImplicitSnitch.DISK)) result.put(ImplicitSnitch.DISK, 100); - return result; - } - - @Override - protected Map fetchReplicaMetrics(String node, Map> metricsKeyVsTagReplica) { - //e.g: solr.core.perReplicaDataColl.shard1.replica_n4:INDEX.sizeInBytes - Map result = new HashMap<>(); - metricsKeyVsTagReplica.forEach((k, v) -> { - if (k.endsWith(":INDEX.sizeInBytes")) result.put(k, 100); - }); - - return result; - } - - @Override - protected ClusterStateProvider getClusterStateProvider() { - return clusterStateProvider; - } - }; - - @SuppressWarnings({"rawtypes"}) - Map policies = (Map) Utils.fromJSONString("{" + - " 'cluster-preferences': [" + - " { 'minimize': 'cores', 'precision': 50}" + - " ]," + - " 'cluster-policy': [" + - " { 'replica': '#EQUAL', 'node': '#ANY'}," + - " ]" + - "}"); - @SuppressWarnings({"unchecked"}) - AutoScalingConfig config = new AutoScalingConfig(policies); - Policy policy = config.getPolicy(); - Policy.Session session = policy.createSession(new DelegatingCloudManager(null) { - @Override - public ClusterStateProvider getClusterStateProvider() { - return clusterStateProvider; - } - - @Override - public NodeStateProvider getNodeStateProvider() { - return solrClientNodeStateProvider; - } - }); - List violations = session.getViolations(); - assertEquals(2, violations.size()); - for (Violation violation : violations) { - if (violation.node.equals("node1")) { - RangeVal val = (RangeVal) violation.getClause().replica.val; - assertEquals(1.0d, val.min.doubleValue(), 0.01); - assertEquals(2.0, val.max.doubleValue(), 0.01); - assertEquals(1.2d, val.actual.doubleValue(), 0.01d); - assertEquals(1, violation.replicaCountDelta.doubleValue(), 0.01); - assertEquals(3, violation.getViolatingReplicas().size()); - Set expected = ImmutableSet.of("r1", "r3", "r5"); - for (Violation.ReplicaInfoAndErr replicaInfoAndErr : violation.getViolatingReplicas()) { - assertTrue(expected.contains(replicaInfoAndErr.replicaInfo.getCoreName())); - } - } else if (violation.node.equals("node5")) { - assertEquals(-1, violation.replicaCountDelta.doubleValue(), 0.01); - - } else { - fail(); - } - } -// Violation violation = violations.get(0); -// assertEquals("node1", violation.node); - - - } - - private static void expectError(String name, Object val, String msg) { - Exception e = expectThrows(Exception.class, () -> Clause.validate(name, val, true)); - assertTrue("expected exception containing " + msg, e.getMessage().contains(msg)); - } - - public void testOperands() { - Clause c = Clause.create("{replica:'<2', node:'#ANY'}"); - assertFalse(c.replica.isPass(3)); - assertFalse(c.replica.isPass(2)); - assertTrue(c.replica.isPass(1)); - assertEquals("{\"replica\":\"<2.0\"}", c.replica.toString()); - - c = Clause.create("{replica:'>2', node:'#ANY'}"); - assertTrue(c.replica.isPass(3)); - assertFalse(c.replica.isPass(2)); - assertFalse(c.replica.isPass(1)); - assertEquals("{\"replica\":\">2.0\"}", c.replica.toString()); - - - c = Clause.create("{replica:0, nodeRole:'!overseer'}"); - assertTrue(c.tag.isPass("OVERSEER")); - assertFalse(c.tag.isPass("overseer")); - - c = Clause.create("{replica:0, sysLoadAvg:'<12.7'}"); - assertTrue(c.tag.isPass("12.6")); - assertTrue(c.tag.isPass(12.6d)); - assertFalse(c.tag.isPass("12.9")); - assertFalse(c.tag.isPass(12.9d)); - - c = Clause.create("{replica:0, sysLoadAvg:'>12.7'}"); - assertTrue(c.tag.isPass("12.8")); - assertTrue(c.tag.isPass(12.8d)); - assertFalse(c.tag.isPass("12.6")); - assertFalse(c.tag.isPass(12.6d)); - - c = Clause.create("{replica:0, 'metrics:x:y:z':'>12.7'}"); - assertTrue(c.tag.val instanceof String); - assertTrue(c.tag.isPass("12.8")); - assertTrue(c.tag.isPass(12.8d)); - assertFalse(c.tag.isPass("12.6")); - assertFalse(c.tag.isPass(12.6d)); - - c = Clause.create("{replica: '<3', sysprop.zone : [east, west]}"); - assertTrue(c.tag.isPass("east")); - assertTrue(c.tag.isPass("west")); - assertFalse(c.tag.isPass("south")); - - } - - public void testNodeLost() { - String dataproviderdata = " {'liveNodes':[" + - " '127.0.0.1:65417_solr'," + - " '127.0.0.1:65434_solr']," + - " 'replicaInfo':{" + - " '127.0.0.1:65427_solr':{'testNodeLost':{'shard1':[{'core_node2':{type: NRT}}]}}," + - " '127.0.0.1:65417_solr':{'testNodeLost':{'shard1':[{'core_node1':{type: NRT}}]}}," + - " '127.0.0.1:65434_solr':{}}," + - " 'nodeValues':{" + - " '127.0.0.1:65417_solr':{" + - " 'node':'127.0.0.1:65417_solr'," + - " 'cores':1," + - " 'freedisk':884.7097854614258}," + - " '127.0.0.1:65434_solr':{" + - " 'node':'127.0.0.1:65434_solr'," + - " 'cores':0," + - " 'freedisk':884.7097854614258}}}"; - String autoScalingjson = "{" + - " 'cluster-policy':[" + - " {" + - " 'cores':'<10'," + - " 'node':'#ANY'}," + - " {" + - " 'replica':'<2'," + - " 'shard':'#EACH'," + - " 'node':'#ANY'}," + - " {" + - " 'nodeRole':'overseer'," + - " 'replica':0}]," + - " 'cluster-preferences':[" + - " {" + - " 'minimize':'cores'," + - " 'precision':3}," + - " {" + - " 'maximize':'freedisk'," + - " 'precision':100}]}"; - - @SuppressWarnings({"unchecked"}) - Policy policy = new Policy((Map) Utils.fromJSONString(autoScalingjson)); - Policy.Session session = policy.createSession(cloudManagerWithData(dataproviderdata)); - @SuppressWarnings({"rawtypes"}) - SolrRequest op = session.getSuggester(MOVEREPLICA).hint(Hint.SRC_NODE, "127.0.0.1:65427_solr").getSuggestion(); - assertNotNull(op); - assertEquals("127.0.0.1:65434_solr", op.getParams().get("targetNode")); - } - - @SuppressWarnings({"unchecked", "rawtypes"}) - public void testNodeLostMultipleReplica() { - String nodeValues = " {" + - " 'node4':{" + - " 'node':'10.0.0.4:8987_solr'," + - " 'cores':1," + - " 'freedisk':884.7097854614258}," + - " 'node3':{" + - " 'node':'10.0.0.4:8989_solr'," + - " 'cores':1," + - " 'freedisk':884.7097854614258}," + - " 'node2':{" + - " 'node':'10.0.0.4:7574_solr'," + - " 'cores':1," + - " 'freedisk':884.7097854614258}," + - "}"; - - SolrCloudManager provider = getSolrCloudManager((Map) Utils.fromJSONString(nodeValues), clusterState); - Map policies = (Map) Utils.fromJSONString("{" + - " 'cluster-preferences': [" + - " { 'maximize': 'freedisk', 'precision': 50}," + - " { 'minimize': 'cores', 'precision': 50}" + - " ]," + - " 'cluster-policy': [" + - " { 'replica': 0, 'nodeRole': 'overseer'}" + - " { 'replica': '<2', 'shard': '#EACH', 'node': '#ANY'}," + - " ]" + - "}"); - - if(useNodeset) { - policies = (Map) Utils.fromJSONString("{" + - " 'cluster-preferences': [" + - " { 'maximize': 'freedisk', 'precision': 50}," + - " { 'minimize': 'cores', 'precision': 50}" + - " ]," + - " 'cluster-policy': [" + - " { 'replica': 0, nodeset:{'nodeRole': 'overseer'}}" + - " { 'replica': '<2', 'shard': '#EACH', 'node': '#ANY'}," + - " ]" + - "}"); - } - AutoScalingConfig config = new AutoScalingConfig(policies); - Policy policy = config.getPolicy(); - Policy.Session session = policy.createSession(provider); - Suggester suggester = session.getSuggester(MOVEREPLICA) - .hint(Hint.SRC_NODE, "node1"); - - SolrRequest operation = suggester.getSuggestion(); - assertNotNull(operation); - assertEquals("node2", operation.getParams().get("targetNode")); - - session = suggester.getSession(); - suggester = session.getSuggester(MOVEREPLICA) - .hint(Hint.SRC_NODE, "node1"); - operation = suggester.getSuggestion(); - assertNotNull(operation); - assertEquals("node3", operation.getParams().get("targetNode")); - - session = suggester.getSession(); - suggester = session.getSuggester(MOVEREPLICA) - .hint(Hint.SRC_NODE, "node1"); - operation = suggester.getSuggestion(); - assertNull(operation); - - // lets change the policy such that all replicas that were on node1 - // can now fit on node2 - policies = (Map) Utils.fromJSONString("{" + - " 'cluster-preferences': [" + - " { 'maximize': 'freedisk', 'precision': 50}," + - " { 'minimize': 'cores', 'precision': 50}" + - " ]," + - " 'cluster-policy': [" + - " { 'replica': 0, 'nodeRole': 'overseer'}" + - " { 'replica': '<3', 'shard': '#EACH', 'node': '#ANY'}," + - " ]" + - "}"); - if(useNodeset){ - policies = (Map) Utils.fromJSONString("{" + - " 'cluster-preferences': [" + - " { 'maximize': 'freedisk', 'precision': 50}," + - " { 'minimize': 'cores', 'precision': 50}" + - " ]," + - " 'cluster-policy': [" + - " { 'replica': 0, nodeset: {'nodeRole': 'overseer'}}" + - " { 'replica': '<3', 'shard': '#EACH', 'node': '#ANY'}," + - " ]" + - "}"); - - } - config = new AutoScalingConfig(policies); - policy = config.getPolicy(); - session = null; - for (String expectedReplica : new String[] { "r1", "r3", "r5", null }) { - if (session == null) { - session = policy.createSession(provider); - } else { - session = suggester.getSession(); - } - suggester = session.getSuggester(MOVEREPLICA) - .hint(Hint.SRC_NODE, "node1"); - operation = suggester.getSuggestion(); - if (expectedReplica == null) { - assertNull(operation); - } else { - assertNotNull(operation); - assertEquals("node2", operation.getParams().get("targetNode")); - assertEquals(expectedReplica, operation.getParams().get("replica")); - } - } - - // now lets change the policy such that a node can have 2 shard2 replicas - policies = (Map) Utils.fromJSONString("{" + - " 'cluster-preferences': [" + - " { 'maximize': 'freedisk', 'precision': 50}," + - " { 'minimize': 'cores', 'precision': 50}" + - " ]," + - " 'cluster-policy': [" + - " { 'replica': 0, 'nodeRole': 'overseer'}" + - " { 'replica': '<2', 'shard': 'shard1', 'node': '#ANY'}," + - " { 'replica': '<3', 'shard': 'shard2', 'node': '#ANY'}," + - " ]" + - "}"); - if(useNodeset){ - policies = (Map) Utils.fromJSONString("{" + - " 'cluster-preferences': [" + - " { 'maximize': 'freedisk', 'precision': 50}," + - " { 'minimize': 'cores', 'precision': 50}" + - " ]," + - " 'cluster-policy': [" + - " { 'replica': 0, nodeset:{'nodeRole': 'overseer'}}" + - " { 'replica': '<2', 'shard': 'shard1', 'node': '#ANY'}," + - " { 'replica': '<3', 'shard': 'shard2', 'node': '#ANY'}," + - " ]" + - "}"); - - } - config = new AutoScalingConfig(policies); - policy = config.getPolicy(); - - session = null; - final String[] expectedReplica = new String[] { "r1", "r3", "r5" }; - final String[] expectedTargetNode = new String[] { "node3", "node3", "node2" }; - for (int ii = 0; ii < expectedReplica.length; ++ii) { - if (session == null) { - session = policy.createSession(provider); - } else { - session = suggester.getSession(); - } - suggester = session.getSuggester(MOVEREPLICA) - .hint(Hint.SRC_NODE, "node1"); - operation = suggester.getSuggestion(); - assertNotNull(operation); - assertEquals(expectedTargetNode[ii], operation.getParams().get("targetNode")); - assertEquals(expectedReplica[ii], operation.getParams().get("replica")); - } - } - - private static SolrCloudManager cloudManagerWithData(String data) { - return cloudManagerWithData((Map) Utils.fromJSONString(data)); - } - - @SuppressWarnings({"unchecked", "rawtypes"}) - static SolrCloudManager cloudManagerWithData(Map m) { - Map replicaInfo = (Map) m.get("replicaInfo"); - replicaInfo.forEach((node, val) -> { - Map m1 = (Map) val; - m1.forEach((coll, val2) -> { - Map m2 = (Map) val2; - m2.forEach((shard, val3) -> { - List l3 = (List) val3; - for (int i = 0; i < l3.size(); i++) { - Object o = l3.get(i); - Map m3 = (Map) o; - String name = m3.keySet().iterator().next().toString(); - m3 = (Map) m3.get(name); - Replica.Type type = Replica.Type.get((String) m3.get("type")); - l3.set(i, new Replica(name, (String) node, coll.toString(), shard.toString(), name, Replica.State.ACTIVE, type, m3)); - } - }); - - }); - }); - @SuppressWarnings({"unchecked"}) - AutoScalingConfig asc = m.containsKey("autoscalingJson") ? new AutoScalingConfig((Map) m.get("autoscalingJson")) : null; - return new DelegatingCloudManager(null) { - - @Override - public DistribStateManager getDistribStateManager() { - return new DelegatingDistribStateManager(null) { - @Override - public AutoScalingConfig getAutoScalingConfig() { - return asc; - } - }; - } - - @Override - public ClusterStateProvider getClusterStateProvider() { - return new DelegatingClusterStateProvider(null) { - @Override - public ClusterState getClusterState() throws IOException { - return ClusterState.createFromCollectionMap(0, new HashMap<>(), getLiveNodes()); - } - - @Override - @SuppressWarnings({"unchecked"}) - public Set getLiveNodes() { - return new HashSet<>((Collection) m.get("liveNodes")); - } - }; - } - - @Override - public NodeStateProvider getNodeStateProvider() { - return new DelegatingNodeStateProvider(null) { - @Override - public Map getNodeValues(String node, Collection tags) { - @SuppressWarnings({"unchecked"}) - Map result = (Map) Utils.getObjectByPath(m, false, Arrays.asList("nodeValues", node)); - return result == null ? new HashMap<>() : result; - } - - @Override - public Map>> getReplicaInfo(String node, Collection keys) { - @SuppressWarnings({"unchecked"}) - Map>> result = (Map>>) Utils.getObjectByPath(m, false, Arrays.asList("replicaInfo", node)); - return result == null ? new HashMap<>() : result; - } - }; - } - }; - } - - @SuppressWarnings({"unchecked", "rawtypes"}) - public void testPolicyWithReplicaType() { - Map policies = (Map) Utils.fromJSONString("{" + - " 'cluster-preferences': [" + - " { 'maximize': 'freedisk', 'precision': 50}," + - " { 'minimize': 'cores', 'precision': 50}" + - " ]," + - " 'cluster-policy': [" + - " { 'replica': 0, 'nodeRole': 'overseer'}" + - " { 'replica': '<2', 'shard': '#EACH', 'node': '#ANY'}," + - " { 'replica': 0, 'shard': '#EACH', sysprop.fs : '!ssd', type : TLOG }" + - " { 'replica': 0, 'shard': '#EACH', sysprop.fs : '!slowdisk' , type : PULL }" + - " ]" + - "}"); - if(useNodeset){ - policies = (Map) Utils.fromJSONString("{" + - " 'cluster-preferences': [" + - " { 'maximize': 'freedisk', 'precision': 50}," + - " { 'minimize': 'cores', 'precision': 50}" + - " ]," + - " 'cluster-policy': [" + - " { 'replica': 0, nodeset : {'nodeRole': 'overseer'}}" + - " { 'replica': '<2', 'shard': '#EACH', 'node': '#ANY'}," + - " { 'replica': 0, 'shard': '#EACH', nodeset : { sysprop.fs : '!ssd'}, type : TLOG }" + - " { 'replica': 0, 'shard': '#EACH', put:'on-each-node' nodeset : {sysprop.fs : '!slowdisk'} , type : PULL }" + - " ]" + - "}"); - - } - Map nodeValues = (Map) Utils.fromJSONString("{" + - "node1:{cores:12, freedisk: 334, heapUsage:10480, rack: rack4, sysprop.fs: slowdisk}," + - "node2:{cores:4, freedisk: 749, heapUsage:6873, rack: rack3, sysprop.fs: unknown }," + - "node3:{cores:7, freedisk: 262, heapUsage:7834, rack: rack2, sysprop.fs : ssd}," + - "node4:{cores:8, freedisk: 375, heapUsage:16900, nodeRole:overseer, rack: rack1, sysprop.fs: unknown}" + - "}"); - Policy policy = new Policy(policies); - Suggester suggester = policy.createSession(getSolrCloudManager(nodeValues, clusterState)) - .getSuggester(ADDREPLICA) - .hint(Hint.COLL_SHARD, new Pair("newColl", "shard1")) - .hint(Hint.REPLICATYPE, Replica.Type.PULL); - SolrRequest op = suggester.getSuggestion(); - assertNotNull(op); - assertEquals(Replica.Type.PULL.name(), op.getParams().get("type")); - assertEquals("PULL type node must be in 'slowdisk' node", "node1", op.getParams().get("node")); - - suggester = suggester.getSession() - .getSuggester(ADDREPLICA) - .hint(Hint.COLL_SHARD, new Pair<>("newColl", "shard2")) - .hint(Hint.REPLICATYPE, Replica.Type.PULL); - op = suggester.getSuggestion(); - assertNotNull(op); - assertEquals(Replica.Type.PULL.name(), op.getParams().get("type")); - assertEquals("PULL type node must be in 'slowdisk' node", "node1", op.getParams().get("node")); - - suggester = suggester.getSession() - .getSuggester(ADDREPLICA) - .hint(Hint.COLL_SHARD, new Pair("newColl", "shard1")) - .hint(Hint.REPLICATYPE, Replica.Type.TLOG); - op = suggester.getSuggestion(); - assertNotNull(op); - assertEquals(Replica.Type.TLOG.name(), op.getParams().get("type")); - assertEquals("TLOG type node must be in 'ssd' node", "node3", op.getParams().get("node")); - - suggester = suggester.getSession() - .getSuggester(ADDREPLICA) - .hint(Hint.COLL_SHARD, new Pair("newColl", "shard2")) - .hint(Hint.REPLICATYPE, Replica.Type.TLOG); - op = suggester.getSuggestion(); - assertNotNull(op); - assertEquals(Replica.Type.TLOG.name(), op.getParams().get("type")); - assertEquals("TLOG type node must be in 'ssd' node", "node3", op.getParams().get("node")); - - suggester = suggester.getSession() - .getSuggester(ADDREPLICA) - .hint(Hint.COLL_SHARD, new Pair<>("newColl", "shard2")) - .hint(Hint.REPLICATYPE, Replica.Type.TLOG); - op = suggester.getSuggestion(); - assertNull("No node should qualify for this", op); - - } - - - public void testMoveReplicasInMultipleCollections() throws IOException { - @SuppressWarnings({"unchecked", "rawtypes"}) - Map nodeValues = (Map) Utils.fromJSONString("{" + - "node1:{cores:2}," + - "node3:{cores:4}" + - "node2:{cores:2}" + - "}"); - Policy policy = new Policy(new HashMap<>()); - @SuppressWarnings({"unchecked"}) - Suggester suggester = policy.createSession(getSolrCloudManager(nodeValues, - (Map) loadFromResource("testMoveReplicasInMultipleCollections.json"))) - .getSuggester(MOVEREPLICA) - .hint(Hint.COLL, "collection1") - .hint(Hint.COLL, "collection2") - .hint(Suggester.Hint.SRC_NODE, "node2") - .forceOperation(true); - @SuppressWarnings({"rawtypes"}) - SolrRequest op = suggester.getSuggestion(); - assertNotNull(op); - assertEquals("collection2", op.getParams().get("collection")); - assertEquals("node1", op.getParams().get("targetNode")); - String coreNodeName = op.getParams().get("replica"); - assertTrue(coreNodeName.equals("core_node3") || coreNodeName.equals("core_node1")); - - suggester = suggester.getSession() - .getSuggester(MOVEREPLICA) - .hint(Hint.COLL, "collection1") - .hint(Hint.COLL, "collection2") - .hint(Suggester.Hint.SRC_NODE, "node2") - .forceOperation(true); - op = suggester.getSuggestion(); - assertNotNull(op); - assertEquals("collection2", op.getParams().get("collection")); - assertEquals("node1", op.getParams().get("targetNode")); - coreNodeName = op.getParams().get("replica"); - assertTrue(coreNodeName.equals("core_node3") || coreNodeName.equals("core_node1")); - - suggester = suggester.getSession() - .getSuggester(MOVEREPLICA) - .hint(Hint.COLL, "collection1") - .hint(Hint.COLL, "collection2") - .hint(Suggester.Hint.SRC_NODE, "node2"); - op = suggester.getSuggestion(); - assertNull(op); - } - - - public void testMultipleCollections() { - @SuppressWarnings({"rawtypes"}) - Map policies = (Map) Utils.fromJSONString("{" + - " 'cluster-preferences': [" + - " { 'maximize': 'freedisk', 'precision': 50}," + - " { 'minimize': 'cores', 'precision': 1}" + - " ]," + - " 'cluster-policy': [" + - " { 'replica': 0, 'nodeRole': 'overseer'}" + - " { 'replica': '<2', 'shard': '#EACH', 'node': '#ANY', 'collection':'newColl'}," + - " { 'replica': '<2', 'shard': '#EACH', 'node': '#ANY', 'collection':'newColl2', type : PULL}," + - " { 'replica': '<3', 'shard': '#EACH', 'node': '#ANY', 'collection':'newColl2'}," + - " { 'replica': 0, 'shard': '#EACH', sysprop.fs : '!ssd', type : TLOG }" + - " { 'replica': 0, 'shard': '#EACH', sysprop.fs : '!slowdisk' , type : PULL }" + - " ]" + - "}"); - if(useNodeset){ - policies = (Map) Utils.fromJSONString("{" + - " 'cluster-preferences': [" + - " { 'maximize': 'freedisk', 'precision': 50}," + - " { 'minimize': 'cores', 'precision': 1}" + - " ]," + - " 'cluster-policy': [" + - " { 'replica': 0, nodeset : {'nodeRole': 'overseer'}}" + - " { 'replica': '<2', 'shard': '#EACH', 'node': '#ANY', 'collection':'newColl'}," + - " { 'replica': '<2', 'shard': '#EACH', 'node': '#ANY', 'collection':'newColl2', type : PULL}," + - " { 'replica': '<3', 'shard': '#EACH', 'node': '#ANY', 'collection':'newColl2'}," + - " { 'replica': 0, 'shard': '#EACH', put: on-each-node , nodeset:{ sysprop.fs : '!ssd'}, type : TLOG }" + - " { 'replica': 0, 'shard': '#EACH', put: on-each-node ,nodeset : {sysprop.fs : '!slowdisk'} , type : PULL }" + - " ]" + - "}"); - - } - @SuppressWarnings({"unchecked", "rawtypes"}) - Map nodeValues = (Map) Utils.fromJSONString("{" + - "node1:{cores:12, freedisk: 334, heapUsage:10480, rack: rack4, sysprop.fs: slowdisk}," + - "node2:{cores:4, freedisk: 749, heapUsage:6873, rack: rack3, sysprop.fs: unknown}," + - "node3:{cores:7, freedisk: 262, heapUsage:7834, rack: rack2, sysprop.fs : ssd}," + - "node4:{cores:8, freedisk: 375, heapUsage:16900, nodeRole:overseer, rack: rack1, sysprop.fs: unknown}" + - "}"); - @SuppressWarnings({"unchecked"}) - Policy policy = new Policy(policies); - Suggester suggester = policy.createSession(getSolrCloudManager(nodeValues, clusterState)) - .getSuggester(ADDREPLICA) - .hint(Hint.REPLICATYPE, Replica.Type.PULL) - .hint(Hint.COLL_SHARD, new Pair<>("newColl", "shard1")) - .hint(Hint.COLL_SHARD, new Pair<>("newColl2", "shard1")); - @SuppressWarnings({"rawtypes"}) - SolrRequest op; - int countOp = 0; - int countNewCollOp = 0; - int countNewColl2Op = 0; - while ((op = suggester.getSuggestion()) != null) { - countOp++; - assertEquals(Replica.Type.PULL.name(), op.getParams().get("type")); - String collection = op.getParams().get("collection"); - assertTrue("Collection for replica is not as expected " + collection, collection.equals("newColl") || collection.equals("newColl2")); - if (collection.equals("newColl")) countNewCollOp++; - else countNewColl2Op++; - assertEquals("PULL type node must be in 'slowdisk' node, countOp : " + countOp, "node1", op.getParams().get("node")); - suggester = suggester.getSession().getSuggester(ADDREPLICA) - .hint(Hint.REPLICATYPE, Replica.Type.PULL) - .hint(Hint.COLL_SHARD, new Pair<>("newColl", "shard1")) - .hint(Hint.COLL_SHARD, new Pair<>("newColl2", "shard1")); - } - assertEquals(2, countOp); - assertEquals(1, countNewCollOp); - assertEquals(1, countNewColl2Op); - - countOp = 0; - countNewCollOp = 0; - countNewColl2Op = 0; - suggester = suggester.getSession() - .getSuggester(ADDREPLICA) - .hint(Hint.COLL_SHARD, new Pair<>("newColl", "shard2")) - .hint(Hint.COLL_SHARD, new Pair<>("newColl2", "shard2")) - .hint(Hint.REPLICATYPE, Replica.Type.TLOG); - while ((op = suggester.getSuggestion()) != null) { - countOp++; - assertEquals(Replica.Type.TLOG.name(), op.getParams().get("type")); - String collection = op.getParams().get("collection"); - assertTrue("Collection for replica is not as expected " + collection, collection.equals("newColl") || collection.equals("newColl2")); - if (collection.equals("newColl")) countNewCollOp++; - else countNewColl2Op++; - assertEquals("TLOG type node must be in 'ssd' node", "node3", op.getParams().get("node")); - suggester = suggester.getSession() - .getSuggester(ADDREPLICA) - .hint(Hint.COLL_SHARD, new Pair<>("newColl", "shard2")) - .hint(Hint.COLL_SHARD, new Pair<>("newColl2", "shard2")) - .hint(Hint.REPLICATYPE, Replica.Type.TLOG); - } - assertEquals(3, countOp); - assertEquals(1, countNewCollOp); - assertEquals(2, countNewColl2Op); - } - - public void testRow() { - Policy policy = new Policy(); - Policy.Session session = policy.createSession(new DelegatingCloudManager(null) { - @Override - public NodeStateProvider getNodeStateProvider() { - return new DelegatingNodeStateProvider(null) { - @Override - public Map>> getReplicaInfo(String node, Collection keys) { - @SuppressWarnings({"unchecked"}) - Map>> o = (Map>>) Utils.fromJSONString("{c1: {s0:[{}]}}"); - Utils.setObjectByPath(o, "c1/s0[0]", new Replica("r0", "nodex", "c1", "s0", "c1.s0", Replica.State.ACTIVE, Replica.Type.NRT, new HashMap<>())); - return o; - } - - @Override - public Map getNodeValues(String node, Collection tags) { - return Utils.makeMap("node", "nodex", "cores", 1); - } - }; - } - - @Override - public ClusterStateProvider getClusterStateProvider() { - return new DelegatingClusterStateProvider(null) { - @Override - public String getPolicyNameByCollection(String coll) { - return null; - } - - @Override - public Set getLiveNodes() { - return Collections.singleton("nodex"); - } - }; - } - }); - - Row row = session.getNode("nodex"); - Row r1 = row.addReplica("c1", "s1", Replica.Type.NRT); - Row r2 = r1.addReplica("c1", "s1", Replica.Type.NRT); - assertEquals(1, r1.collectionVsShardVsReplicas.get("c1").get("s1").size()); - assertEquals(2, r2.collectionVsShardVsReplicas.get("c1").get("s1").size()); - assertTrue(r2.collectionVsShardVsReplicas.get("c1").get("s1").get(0) instanceof Replica); - assertTrue(r2.collectionVsShardVsReplicas.get("c1").get("s1").get(1) instanceof Replica); - } - - public void testMerge() { - - @SuppressWarnings({"rawtypes"}) - Map map = (Map) Utils.fromJSONString("{" + - " 'cluster-preferences': [" + - " { 'maximize': 'freedisk', 'precision': 50}," + - " { 'minimize': 'cores', 'precision': 50}" + - " ]," + - " 'cluster-policy': [" + - " { 'replica': 0, 'nodeRole': 'overseer'}," + - " { 'replica': '<2', 'shard': '#EACH', 'node': '#ANY'}" + - " ]," + - " 'policies': {" + - " 'policy1': [" + - " { 'replica': '1', 'sysprop.fs': 'ssd', 'shard': '#EACH'}," + - " { 'replica': '<2', 'shard': '#ANY', 'node': '#ANY'}," + - " { 'replica': '<2', 'shard': '#EACH', 'sysprop.rack': 'rack1'}" + - " ]" + - " }" + - "}"); - if(useNodeset){ - map = (Map) Utils.fromJSONString("{" + - " 'cluster-preferences': [" + - " { 'maximize': 'freedisk', 'precision': 50}," + - " { 'minimize': 'cores', 'precision': 50}" + - " ]," + - " 'cluster-policy': [" + - " { 'replica': 0, nodeset: {'nodeRole': 'overseer'}}," + - " { 'replica': '<2', 'shard': '#EACH', 'node': '#ANY'}" + - " ]," + - " 'policies': {" + - " 'policy1': [" + - " { 'replica': '1', nodeset:{ 'sysprop.fs': 'ssd'}, 'shard': '#EACH'}," + - " { 'replica': '<2', 'shard': '#ANY', 'node': '#ANY'}," + - " { 'replica': '<2', 'shard': '#EACH',nodeset:{ 'sysprop.rack': 'rack1'}}" + - " ]" + - " }" + - "}"); - - } - @SuppressWarnings({"unchecked"}) - Policy policy = new Policy(map); - List clauses = Policy.mergePolicies("mycoll", policy.getPolicies().get("policy1"), policy.getClusterPolicy()); - Collections.sort(clauses); - assertEquals(clauses.size(), 4); - assertEquals("1", String.valueOf(clauses.get(0).original.get("replica"))); - assertEquals("0", String.valueOf(clauses.get(1).original.get("replica"))); - assertEquals("#ANY", clauses.get(3).original.get("shard")); - assertEquals("rack1", clauses.get(2).tag.val); - assertEquals("overseer",clauses.get(1).tag.val); - } - - public void testConditionsSort() { - String rules = "{" + - " 'cluster-policy':[" + - " { 'nodeRole':'overseer', replica: 0, 'strict':false}," + - " { 'replica':'<1', 'node':'node3', 'shard':'#EACH'}," + - " { 'replica':'<2', 'node':'#ANY', 'shard':'#EACH'}," + - " { 'replica':1, 'sysprop.rack':'rack1'}]" + - " }"; - if(useNodeset){ - rules = "{" + - " 'cluster-policy':[" + - " { 'nodeRole':'overseer', replica: 0, 'strict':false}," + - " { 'replica':'<1', 'node':'node3', 'shard':'#EACH'}," + - " { 'replica':'<2', 'node':'#ANY', 'shard':'#EACH'}," + - " { 'replica':1, nodeset: {'sysprop.rack':'rack1'}}]" + - " }"; - - } - @SuppressWarnings({"unchecked"}) - Policy p = new Policy((Map) Utils.fromJSONString(rules)); - List clauses = new ArrayList<>(p.getClusterPolicy()); - Collections.sort(clauses); - assertEquals("nodeRole", clauses.get(1).tag.getName()); - assertEquals("sysprop.rack", clauses.get(0).tag.getName()); - } - - @SuppressWarnings({"unchecked", "rawtypes"}) - public void testRules() { - String rules = "{" + - "cluster-policy:[" + - "{nodeRole:'overseer',replica : 0 , strict:false}," + - "{replica:'<1',node:node3}," + - "{replica:'<2',node:'#ANY', shard:'#EACH'}]," + - " cluster-preferences:[" + - "{minimize:cores , precision:2}," + - "{maximize:freedisk, precision:50}, " + - "{minimize:heapUsage, precision:1000}]}"; - if(useNodeset){ - rules = "{" + - "cluster-policy:[" + - "{nodeset:{nodeRole:'overseer'},replica : 0 , strict:false}," + - "{replica:'<1',node:node3}," + - "{replica:'<2',node:'#ANY', shard:'#EACH'}]," + - " cluster-preferences:[" + - "{minimize:cores , precision:2}," + - "{maximize:freedisk, precision:50}, " + - "{minimize:heapUsage, precision:1000}]}"; - } - - Map nodeValues = (Map) Utils.fromJSONString("{" + - "node1:{cores:12, freedisk: 334, heapUsage:10480}," + - "node2:{cores:4, freedisk: 749, heapUsage:6873}," + - "node3:{cores:7, freedisk: 262, heapUsage:7834}," + - "node4:{cores:8, freedisk: 375, heapUsage:16900, nodeRole:overseer}" + - "}"); - - Policy policy = new Policy((Map) Utils.fromJSONString(rules)); - Policy.Session session; - session = policy.createSession(getSolrCloudManager(nodeValues, clusterState)); - - List l = session.getSortedNodes(); - assertEquals("node1", l.get(0).node); - assertEquals("node3", l.get(1).node); - assertEquals("node4", l.get(2).node); - assertEquals("node2", l.get(3).node); - - - List violations = session.getViolations(); - assertEquals(3, violations.size()); - assertTrue(violations.stream().anyMatch(violation -> "node3".equals(violation.getClause().tag.getValue()))); - assertTrue(violations.stream().anyMatch(violation -> "nodeRole".equals(violation.getClause().tag.getName()))); - assertTrue(violations.stream().anyMatch(violation -> (violation.getClause().replica.getOperand() == Operand.LESS_THAN && "node".equals(violation.getClause().tag.getName())))); - - Suggester suggester = session.getSuggester(ADDREPLICA) - .hint(Hint.COLL_SHARD, new Pair<>("gettingstarted", "r1")); - SolrParams operation = suggester.getSuggestion().getParams(); - assertEquals("node2", operation.get("node")); - - nodeValues = (Map) Utils.fromJSONString("{" + - "node1:{cores:12, freedisk: 334, heapUsage:10480}," + - "node2:{cores:4, freedisk: 749, heapUsage:6873}," + - "node3:{cores:7, freedisk: 262, heapUsage:7834}," + - "node5:{cores:0, freedisk: 895, heapUsage:17834}," + - "node4:{cores:8, freedisk: 375, heapUsage:16900, nodeRole:overseer}" + - "}"); - session = policy.createSession(getSolrCloudManager(nodeValues, clusterState)); - SolrRequest opReq = session.getSuggester(MOVEREPLICA) - .hint(Hint.TARGET_NODE, "node5") - .getSuggestion(); - assertNotNull(opReq); - assertEquals("node5", opReq.getParams().get("targetNode")); - } - - @Test - public void testSessionCaching() throws IOException, InterruptedException { -// PolicyHelper.SessionRef ref1 = new PolicyHelper.SessionRef(); - String autoScalingjson = " '{cluster-policy':[" + - " { 'cores':'<10', 'node':'#ANY'}," + - " { 'replica':'<2', 'shard':'#EACH', 'node':'#ANY'}," + - " { 'nodeRole':'overseer','replica':0}]," + - " 'cluster-preferences':[{'minimize':'cores'}]}"; - @SuppressWarnings({"unchecked"}) - Policy policy = new Policy((Map) Utils.fromJSONString(autoScalingjson)); -// PolicyHelper.SESSION_REF.set(ref1); - String nodeValues = " {" + - " 'node4':{" + - " 'node':'10.0.0.4:8987_solr'," + - " 'cores':1," + - " 'freedisk':884.7097854614258}," + - " 'node3':{" + - " 'node':'10.0.0.4:8989_solr'," + - " 'cores':1," + - " 'freedisk':884.7097854614258}," + - " 'node2':{" + - " 'node':'10.0.0.4:7574_solr'," + - " 'cores':1," + - " 'freedisk':884.7097854614258}," + - "}"; - - - @SuppressWarnings({"rawtypes"}) - Map policies = (Map) Utils.fromJSONString("{" + - " 'cluster-preferences': [" + - " { 'maximize': 'freedisk', 'precision': 50}," + - " { 'minimize': 'cores', 'precision': 50}" + - " ]," + - " 'cluster-policy': [" + - " { 'replica': 0, 'nodeRole': 'overseer'}" + - " { 'replica': '<2', 'shard': '#EACH', 'node': '#ANY'}," + - " ]" + - "}"); - @SuppressWarnings({"unchecked"}) - AutoScalingConfig config = new AutoScalingConfig(policies); - @SuppressWarnings({"unchecked", "rawtypes"}) - final SolrCloudManager solrCloudManager = new DelegatingCloudManager(getSolrCloudManager((Map) Utils.fromJSONString(nodeValues), - clusterState)) { - @Override - public DistribStateManager getDistribStateManager() { - return delegatingDistribStateManager(config); - } - }; - - List locations = PolicyHelper.getReplicaLocations("c", config, solrCloudManager, null, - Arrays.asList("s1", "s2"), 1, 0, 0, - null); - - PolicyHelper.SessionRef sessionRef = (PolicyHelper.SessionRef) solrCloudManager.getObjectCache().get(PolicyHelper.SessionRef.class.getName()); - assertNotNull(sessionRef); - PolicyHelper.SessionWrapper sessionWrapper = PolicyHelper.getLastSessionWrapper(true); - - - Policy.Session session = sessionWrapper.get(); - assertNotNull(session); - assertTrue(session.getPolicy() == config.getPolicy()); - assertEquals(sessionWrapper.status, PolicyHelper.Status.EXECUTING); - sessionWrapper.release(); - assertTrue(sessionRef.isEmpty()); - PolicyHelper.SessionWrapper s1 = PolicyHelper.getSession(solrCloudManager); - PolicyHelper.SessionWrapper[] s2 = new PolicyHelper.SessionWrapper[1]; - AtomicLong secondTime = new AtomicLong(); - Thread thread = new Thread(() -> { - try { - s2[0] = PolicyHelper.getSession(solrCloudManager); - secondTime.set(System.nanoTime()); - } catch (Exception e) { - throw new RuntimeException(e); - } - }); - thread.start(); - Thread.sleep(50); - long beforeReturn = System.nanoTime(); - s1.returnSession(s1.get()); - assertEquals(1, s1.getRefCount()); - thread.join(); - assertNotNull(s2[0]); - assertTrue(secondTime.get() > beforeReturn); - assertTrue(s1.getCreateTime() == s2[0].getCreateTime()); - - s2[0].returnSession(s2[0].get()); - assertEquals(2, s1.getRefCount()); - - s2[0].release(); - assertFalse(sessionRef.isEmpty()); - s1.release(); - assertTrue(sessionRef.isEmpty()); - - - } - - @Test - public void testMultiSessionsCache() throws IOException, InterruptedException { - @SuppressWarnings({"rawtypes", "unchecked"}) - Map nodeValues = (Map) Utils.fromJSONString(" {" + - " 'node1':{ 'node':'10.0.0.4:8987_solr', 'cores':1 }," + - " 'node2':{ 'node':'10.0.0.4:8989_solr', 'cores':1 }," + - " 'node3':{ 'node':'10.0.0.4:7574_solr', 'cores':1 }" + - "}"); - - @SuppressWarnings({"rawtypes"}) - Map policies = (Map) Utils.fromJSONString("{ 'cluster-preferences': [{ 'minimize': 'cores', 'precision': 1}]}"); - - @SuppressWarnings({"unchecked"}) - AutoScalingConfig config = new AutoScalingConfig(policies); - final SolrCloudManager solrCloudManager = new DelegatingCloudManager(getSolrCloudManager(nodeValues, clusterState)) { - @Override - public DistribStateManager getDistribStateManager() { - return delegatingDistribStateManager(config); - } - }; - - PolicyHelper.SessionWrapper s1 = PolicyHelper.getSession(solrCloudManager); - // Must skip the wait time otherwise test takes a few seconds to run (and s1 is not returned now anyway so no point waiting). - PolicyHelper.SessionWrapper s2 = PolicyHelper.getSession(solrCloudManager, false); - // Got two sessions, they are different - assertNotSame(s1, s2); - - // Done COMPUTING with first session, it can be reused - s1.returnSession(s1.get()); - - PolicyHelper.SessionWrapper s3 = PolicyHelper.getSession(solrCloudManager); - // First session indeed reused when a new session is requested - assertSame(s3, s1); - - // Done COMPUTING with second session, it can be reused - s2.returnSession(s2.get()); - - PolicyHelper.SessionWrapper s4 = PolicyHelper.getSession(solrCloudManager); - // Second session indeed reused when a new session is requested - assertSame(s4, s2); - - s4.returnSession(s4.get()); - s4.release(); - - s2.release(); - - s3.returnSession(s3.get()); - s3.release(); - - PolicyHelper.SessionRef sessionRef = (PolicyHelper.SessionRef) solrCloudManager.getObjectCache().get(PolicyHelper.SessionRef.class.getName()); - - // First session not yet released so is still in the cache - assertFalse(sessionRef.isEmpty()); - - s1.release(); - - assertTrue(sessionRef.isEmpty()); - } - - /** - * Verify number of sessions allocated when parallel session requests arrive is reasonable. - * Test takes about 3 seconds to run. - */ - @Test - @Slow - public void testMultiThreadedSessionsCache() throws IOException, InterruptedException { - @SuppressWarnings({"rawtypes", "unchecked"}) - Map nodeValues = (Map) Utils.fromJSONString(" {" + - " 'node1':{ 'node':'10.0.0.4:8987_solr', 'cores':1 }," + - " 'node2':{ 'node':'10.0.0.4:8989_solr', 'cores':1 }," + - " 'node3':{ 'node':'10.0.0.4:7574_solr', 'cores':1 }" + - "}"); - - @SuppressWarnings({"rawtypes"}) - Map policies = (Map) Utils.fromJSONString("{ 'cluster-preferences': [{ 'minimize': 'cores', 'precision': 1}]}"); - - @SuppressWarnings({"unchecked"}) - AutoScalingConfig config = new AutoScalingConfig(policies); - final SolrCloudManager solrCloudManager = new DelegatingCloudManager(getSolrCloudManager(nodeValues, clusterState)) { - @Override - public DistribStateManager getDistribStateManager() { - return delegatingDistribStateManager(config); - } - }; - - final Set seenSessions = Sets.newHashSet(); - final AtomicInteger completedThreads = new AtomicInteger(0); - - final int COUNT_THREADS = 100; - Thread[] threads = new Thread[COUNT_THREADS]; - - for (int i = 0; i < COUNT_THREADS; i++) { - threads[i] = new Thread(() -> { - try { - // This thread requests a session, computes using it for 25ms then returns is, executes for 1000ms more, - // releases the sessions and finishes. - PolicyHelper.SessionWrapper session = PolicyHelper.getSession(solrCloudManager); - synchronized (seenSessions) { - seenSessions.add(session); - } - Thread.sleep(25); - session.returnSession(session.get()); - Thread.sleep(1000); - session.release(); - - completedThreads.incrementAndGet(); - } catch (InterruptedException | IOException ignored) { - } - }); - threads[i].start(); - } - - for (int i = 0; i < COUNT_THREADS; i++) { - threads[i].join(12000); - } - - assertEquals(COUNT_THREADS, completedThreads.get()); - // The value asserted below is somewhat arbitrary. Running locally usually uses up to 5 sessions, so hopefully 30 is - // safe. Idea is to verify we do not allocate a high number of sessions even if many concurrent session - // requests arrive at the same time. The session computing time is short in purpose. If it were long, it would be - // expected for more sessions to be needed. - // Note we joined with all the threads having updated seenSessions so no need to synchronize ("All actions in a thread - // happen before any other thread successfully returns from a join() on that thread" - JSR-133) - assertTrue("Too many (>=30) sessions created: " + seenSessions.size(), seenSessions.size() < 30); - - PolicyHelper.SessionRef sessionRef = (PolicyHelper.SessionRef) solrCloudManager.getObjectCache().get(PolicyHelper.SessionRef.class.getName()); - assertTrue(sessionRef.isEmpty()); - } - - private DistribStateManager delegatingDistribStateManager(AutoScalingConfig config) { - return new DelegatingDistribStateManager(null) { - @Override - public AutoScalingConfig getAutoScalingConfig() { - return config; - } - }; - } - - public void testNegativeConditions() { - String autoscaleJson = "{" + - " 'cluster-policy':[" + - " {'replica':'<4','shard':'#EACH','node':'#ANY'}," + - " { 'replica': 0, 'sysprop.fs': '!ssd', 'shard': '#EACH'}," +//negative greedy condition - " {'nodeRole':'overseer','replica':'0'}]," + - " 'cluster-preferences':[" + - " {'minimize':'cores', 'precision':3}," + - " {'maximize':'freedisk','precision':100}]}"; - @SuppressWarnings({"unchecked", "rawtypes"}) - Map nodeValues = (Map) Utils.fromJSONString("{" + - "node1:{cores:12, freedisk: 334, heapUsage:10480, rack: rack4, sysprop.fs: slowdisk}," + - "node2:{cores:4, freedisk: 749, heapUsage:6873, rack: rack3, sysprop.fs: slowdisk}," + - "node3:{cores:7, freedisk: 262, heapUsage:7834, rack: rack2, sysprop.fs : ssd}," + - "node4:{cores:8, freedisk: 375, heapUsage:16900, nodeRole:overseer, rack: rack1, sysprop.fs: slowdisk}" + - "}"); - @SuppressWarnings({"unchecked"}) - Policy policy = new Policy((Map) Utils.fromJSONString(autoscaleJson)); - SolrCloudManager cloudManager = getSolrCloudManager(nodeValues, clusterState); - Policy.Session session = policy.createSession(cloudManager); - for (int i = 0; i < 3; i++) { - Suggester suggester = session.getSuggester(ADDREPLICA); - @SuppressWarnings({"rawtypes"}) - SolrRequest op = suggester - .hint(Hint.COLL_SHARD, new Pair<>("newColl", "shard1")) - .getSuggestion(); - assertNotNull(op); - assertEquals("node3", op.getParams().get("node")); - session = suggester.getSession(); - } - - } - - public void testGreedyConditions() { - String autoscaleJson = "{" + - " 'cluster-policy':[" + - " {'cores':'<10','node':'#ANY'}," + - " {'replica':'<3','shard':'#EACH','node':'#ANY'}," + - " { 'replica': 2, 'sysprop.fs': 'ssd', 'shard': '#EACH'}," +//greedy condition - " {'nodeRole':'overseer','replica':'0'}]," + - " 'cluster-preferences':[" + - " {'minimize':'cores', 'precision':3}," + - " {'maximize':'freedisk','precision':100}]}"; - if(useNodeset){ - autoscaleJson = "{" + - " 'cluster-policy':[" + - " {'cores':'<10','node':'#ANY'}," + - " {'replica':'<3','shard':'#EACH','node':'#ANY'}," + - " { 'replica': 2, nodeset: {'sysprop.fs': 'ssd'}, 'shard': '#EACH'}," +//greedy condition - " {nodeset:{'nodeRole':'overseer'},'replica':'0'}]," + - " 'cluster-preferences':[" + - " {'minimize':'cores', 'precision':3}," + - " {'maximize':'freedisk','precision':100}]}"; - - } - @SuppressWarnings({"unchecked", "rawtypes"}) - Map nodeValues = (Map) Utils.fromJSONString("{" + - "node1:{cores:12, freedisk: 334, heapUsage:10480, rack: rack4}," + - "node2:{cores:4, freedisk: 749, heapUsage:6873, rack: rack3}," + - "node3:{cores:7, freedisk: 262, heapUsage:7834, rack: rack2, sysprop.fs : ssd}," + - "node4:{cores:8, freedisk: 375, heapUsage:16900, nodeRole:overseer, rack: rack1}" + - "}"); - - @SuppressWarnings({"unchecked"}) - Policy policy = new Policy((Map) Utils.fromJSONString(autoscaleJson)); - SolrCloudManager cloudManager = getSolrCloudManager(nodeValues, clusterState); - Policy.Session session = policy.createSession(cloudManager); - Suggester suggester = session.getSuggester(ADDREPLICA); - @SuppressWarnings({"rawtypes"}) - SolrRequest op = suggester - .hint(Hint.COLL_SHARD, new Pair<>("newColl", "shard1")) - .getSuggestion(); - assertNotNull(op); - assertEquals("node3", op.getParams().get("node")); - suggester = suggester - .getSession() - .getSuggester(ADDREPLICA) - .hint(Hint.COLL_SHARD, new Pair<>("newColl", "shard1")); - op = suggester.getSuggestion(); - assertNotNull(op); - assertEquals("node3", op.getParams().get("node")); - - suggester = suggester - .getSession() - .getSuggester(ADDREPLICA) - .hint(Hint.COLL_SHARD, new Pair<>("newColl", "shard1")); - op = suggester.getSuggestion(); - assertNotNull(op); - assertEquals("node2", op.getParams().get("node")); - } - - @SuppressWarnings({"unchecked", "rawtypes"}) - public void testMoveReplica() { - String autoscaleJson = "{" + - " 'cluster-policy':[" + - " {'cores':'<10','node':'#ANY'}," + - " {'replica':'<3','shard':'#EACH','node':'#ANY'}," + - " {'nodeRole':'overseer','replica':'0'}]," + - " 'cluster-preferences':[" + - " {'minimize':'cores', 'precision':3}," + - " {'maximize':'freedisk','precision':100}]}"; - - if(useNodeset){ - autoscaleJson = "{" + - " 'cluster-policy':[" + - " {'cores':'<10','node':'#ANY'}," + - " {'replica':'<3','shard':'#EACH','node':'#ANY'}," + - " {nodeset: {'nodeRole':'overseer'},'replica':'0'}]," + - " 'cluster-preferences':[" + - " {'minimize':'cores', 'precision':3}," + - " {'maximize':'freedisk','precision':100}]}"; - - } - - Map replicaInfoMap = (Map) Utils.fromJSONString("{ '127.0.0.1:60099_solr':{}," + - " '127.0.0.1:60089_solr':{'compute_plan_action_test':{'shard1':[" + - " {'core_node1':{}}," + - " {'core_node2':{}}]}}}"); - Map m = (Map) Utils.getObjectByPath(replicaInfoMap, false, "127.0.0.1:60089_solr/compute_plan_action_test"); - m.put("shard1", Arrays.asList( - new Replica("core_node1", "127.0.0.1:60089_solr", "compute_plan_action_test", "shard1", "core_node1", - Replica.State.ACTIVE, Replica.Type.NRT, Collections.emptyMap()), - new Replica("core_node2", "127.0.0.1:60089_solr", "compute_plan_action_test", "shard1", "core_node2", - Replica.State.ACTIVE, Replica.Type.NRT, Collections.emptyMap()))); - - @SuppressWarnings({"unchecked", "rawtypes"}) - Map> tagsMap = (Map) Utils.fromJSONString("{" + - " '127.0.0.1:60099_solr':{" + - " 'cores':0," + - " 'freedisk':918005641216}," + - " '127.0.0.1:60089_solr':{" + - " 'cores':2," + - " 'freedisk':918005641216}}"); - - @SuppressWarnings({"unchecked"}) - Policy policy = new Policy((Map) Utils.fromJSONString(autoscaleJson)); - Policy.Session session = policy.createSession(new DelegatingCloudManager(null) { - @Override - @SuppressWarnings({"unchecked"}) - public ClusterStateProvider getClusterStateProvider() { - return new DelegatingClusterStateProvider(null) { - @Override - public Set getLiveNodes() { - return replicaInfoMap.keySet(); - } - - }; - } - - @Override - public NodeStateProvider getNodeStateProvider() { - return new DelegatingNodeStateProvider(null) { - @Override - public Map getNodeValues(String node, Collection tags) { - return tagsMap.get(node); - } - - @Override - @SuppressWarnings({"unchecked"}) - public Map>> getReplicaInfo(String node, Collection keys) { - return (Map>>) replicaInfoMap.get(node); - } - }; - } - }); - Suggester suggester = session.getSuggester(MOVEREPLICA) - .hint(Hint.TARGET_NODE, "127.0.0.1:60099_solr"); - @SuppressWarnings({"rawtypes"}) - SolrRequest op = suggester.getSuggestion(); - assertNotNull("expect a non null operation", op); - } - - public void testOtherTag() { - String rules = "{" + - "'cluster-preferences':[" + - "{'minimize':'cores','precision':2}," + - "{'maximize':'freedisk','precision':50}," + - "{'minimize':'heapUsage','precision':1000}" + - "]," + - "'cluster-policy':[" + - "{replica:0, 'nodeRole':'overseer','strict':false}," + - "{'replica':'<1','node':'node3'}," + - "{'replica':'<2','node':'#ANY','shard':'#EACH'}" + - "]," + - "'policies':{" + - "'p1':[" + - "{replica:0, 'nodeRole':'overseer','strict':false}," + - "{'replica':'<1','node':'node3'}," + - "{'replica':'<2','node':'#ANY','shard':'#EACH'}," + - "{'replica':'<3','shard':'#EACH','sysprop.rack':'#EACH'}" + - "]" + - "}" + - "}"; - - if(useNodeset){ - rules = "{" + - "'cluster-preferences':[" + - "{'minimize':'cores','precision':2}," + - "{'maximize':'freedisk','precision':50}," + - "{'minimize':'heapUsage','precision':1000}" + - "]," + - "'cluster-policy':[" + - "{replica:0, nodeset:{'nodeRole':'overseer'},'strict':false}," + - "{'replica':'<1','node':'node3'}," + - "{'replica':'<2','node':'#ANY','shard':'#EACH'}" + - "]," + - "'policies':{" + - "'p1':[" + - "{replica:0, nodeset:{'nodeRole':'overseer'},'strict':false}," + - "{'replica':'<1','node':'node3'}," + - "{'replica':'<2','node':'#ANY','shard':'#EACH'}," + - "{'replica':'<3','shard':'#EACH', nodeset : { 'sysprop.rack':[rack1, rack2, rack3, rack4]}}" + - "]" + - "}" + - "}"; - - - } - - @SuppressWarnings({"unchecked", "rawtypes"}) - Map nodeValues = (Map) Utils.fromJSONString("{" + - "node1:{cores:12, freedisk: 334, heapUsage:10480, rack: rack4}," + - "node2:{cores:4, freedisk: 749, heapUsage:6873, rack: rack3}," + - "node3:{cores:7, freedisk: 262, heapUsage:7834, rack: rack2}," + - "node4:{cores:8, freedisk: 375, heapUsage:16900, nodeRole:overseer, sysprop.rack: rack1}" + - "}"); - @SuppressWarnings({"unchecked"}) - Policy policy = new Policy((Map) Utils.fromJSONString(rules)); - SolrCloudManager cloudManager = getSolrCloudManager(nodeValues, clusterState); - SolrCloudManager cdp = new DelegatingCloudManager(null) { - @Override - public NodeStateProvider getNodeStateProvider() { - return new DelegatingNodeStateProvider(null) { - @Override - public Map getNodeValues(String node, Collection tags) { - return cloudManager.getNodeStateProvider().getNodeValues(node, tags); - } - - @Override - public Map>> getReplicaInfo(String node, Collection keys) { - return cloudManager.getNodeStateProvider().getReplicaInfo(node, keys); - } - }; - } - - @Override - public ClusterStateProvider getClusterStateProvider() { - return new DelegatingClusterStateProvider(null) { - @Override - public Set getLiveNodes() { - return cloudManager.getClusterStateProvider().getLiveNodes(); - } - - @Override - public String getPolicyNameByCollection(String coll) { - return "p1"; - } - }; - } - - - }; - Policy.Session session = policy.createSession(cdp); - - CollectionAdminRequest.AddReplica op = (CollectionAdminRequest.AddReplica) session - .getSuggester(ADDREPLICA) - .hint(Hint.COLL_SHARD, new Pair<>("newColl", "s1")).getSuggestion(); - assertNotNull(op); - assertEquals("node2", op.getNode()); - } - @SuppressWarnings({"rawtypes"}) - static SolrCloudManager getSolrCloudManager(final Map nodeValues, String clusterS) { - return getSolrCloudManager(nodeValues,(Map) Utils.fromJSONString(clusterS)); - - } - private static SolrCloudManager getSolrCloudManager(@SuppressWarnings({"rawtypes"})final Map nodeValues, - @SuppressWarnings({"rawtypes"})Map clusterS) { - return new SolrCloudManager() { - ObjectCache objectCache = new ObjectCache(); - - @Override - public ObjectCache getObjectCache() { - return objectCache; - } - - @Override - public TimeSource getTimeSource() { - return TimeSource.NANO_TIME; - } - - @Override - public void close() { - - } - - @Override - public ClusterStateProvider getClusterStateProvider() { - return new DelegatingClusterStateProvider(null) { - @Override - public Set getLiveNodes() { - return nodeValues.keySet(); - } - - }; - } - - @Override - public NodeStateProvider getNodeStateProvider() { - return new DelegatingNodeStateProvider(null) { - @Override - public Map getNodeValues(String node, Collection tags) { - Map result = new LinkedHashMap<>(); - tags.stream().forEach(s -> result.put(s, nodeValues.get(node).get(s))); - return result; - } - - @Override - public Map>> getReplicaInfo(String node, Collection keys) { - return getReplicaDetails(node, clusterS); - } - }; - } - - @Override - public DistribStateManager getDistribStateManager() { - return null; - } - - @Override - public DistributedQueueFactory getDistributedQueueFactory() { - return null; - } - - @Override - public SolrResponse request(@SuppressWarnings({"rawtypes"})SolrRequest req) { - return null; - } - - @Override - public byte[] httpRequest(String url, SolrRequest.METHOD method, Map headers, String payload, int timeout, boolean followRedirects) { - return new byte[0]; - } - }; - } - - public void testEmptyClusterState() { - String autoScaleJson = " {'cluster-policy':[], 'policies':{'c1':[{" + - " 'replica':1," + - " 'shard':'#EACH'," + - " 'port':'50096'}]}}"; - @SuppressWarnings({"unchecked", "rawtypes"}) - Map nodeValues = (Map) Utils.fromJSONString("{" + - " '127.0.0.1:50097_solr':{" + - " 'cores':0," + - " 'port':'50097'}," + - " '127.0.0.1:50096_solr':{" + - " 'cores':0," + - " 'port':'50096'}}"); - SolrCloudManager dataProvider = new DelegatingCloudManager(null) { - @Override - public ClusterStateProvider getClusterStateProvider() { - return new DelegatingClusterStateProvider(null) { - @Override - public Set getLiveNodes() { - return new HashSet<>(Arrays.asList("127.0.0.1:50097_solr", "127.0.0.1:50096_solr")); - } - }; - } - - @Override - public NodeStateProvider getNodeStateProvider() { - return new DelegatingNodeStateProvider(null) { - @Override - public Map getNodeValues(String node, Collection keys) { - Map result = new LinkedHashMap<>(); - keys.stream().forEach(s -> result.put(s, nodeValues.get(node).get(s))); - return result; - } - - @Override - public Map>> getReplicaInfo(String node, Collection keys) { - return getReplicaDetails(node, (Map)Utils.fromJSONString(clusterState)); - } - }; - } - }; - @SuppressWarnings({"unchecked"}) - List locations = PolicyHelper.getReplicaLocations( - "newColl", new AutoScalingConfig((Map) Utils.fromJSONString(autoScaleJson)), - dataProvider, Collections.singletonMap("newColl", "c1"), Arrays.asList("shard1", "shard2"), 1, 0, 0, null); - - assertTrue(locations.stream().allMatch(it -> it.node.equals("127.0.0.1:50096_solr"))); - } - - public void testMultiReplicaPlacement() { - String autoScaleJson = "{" + - " cluster-preferences: [" + - " { maximize : freedisk , precision: 50}," + - " { minimize : cores, precision: 2}" + - " ]," + - " cluster-policy: [" + - " { replica : '0' , nodeRole: overseer}," + - " { replica: '<2', shard: '#ANY', node: '#ANY'" + - " }" + - " ]," + - " policies: {" + - " policy1: [" + - " { replica: '<2', shard: '#EACH', node: '#ANY'}," + - " { replica: '<2', shard: '#EACH', sysprop.rack: rack1}" + - " ]" + - " }" + - "}"; - if(useNodeset){ - autoScaleJson = "{" + - " cluster-preferences: [" + - " { maximize : freedisk , precision: 50}," + - " { minimize : cores, precision: 2}" + - " ]," + - " cluster-policy: [" + - " { replica : '0' , nodeset: {nodeRole: overseer}}," + - " { replica: '<2', shard: '#ANY', node: '#ANY'" + - " }" + - " ]," + - " policies: {" + - " policy1: [" + - " { replica: '<2', shard: '#EACH', node: '#ANY'}," + - " { replica: '<2', shard: '#EACH', nodeset:{ sysprop.rack: rack1}}" + - " ]" + - " }" + - "}"; - - } - - - @SuppressWarnings({"unchecked", "rawtypes"}) - Map nodeValues = (Map) Utils.fromJSONString("{" + - "node1:{cores:12, freedisk: 334, heap:10480, sysprop.rack:rack3}," + - "node2:{cores:4, freedisk: 749, heap:6873, sysprop.fs : ssd, sysprop.rack:rack1}," + - "node3:{cores:7, freedisk: 262, heap:7834, sysprop.rack:rack4}," + - "node4:{cores:0, freedisk: 900, heap:16900, nodeRole:overseer, sysprop.rack:rack2}" + - "}"); - - SolrCloudManager cloudManager = new DelegatingCloudManager(null) { - @Override - public NodeStateProvider getNodeStateProvider() { - return new DelegatingNodeStateProvider(null) { - @Override - public Map getNodeValues(String node, Collection keys) { - Map result = new LinkedHashMap<>(); - keys.stream().forEach(s -> result.put(s, nodeValues.get(node).get(s))); - return result; - } - - @Override - public Map>> getReplicaInfo(String node, Collection keys) { - return getReplicaDetails(node, (Map)Utils.fromJSONString(clusterState)); - } - }; - } - - @Override - public ClusterStateProvider getClusterStateProvider() { - return new DelegatingClusterStateProvider(null) { - @Override - public Set getLiveNodes() { - return new HashSet<>(Arrays.asList("node1", "node2", "node3", "node4")); - } - }; - } - }; - @SuppressWarnings({"unchecked"}) - List locations = PolicyHelper.getReplicaLocations( - "newColl", new AutoScalingConfig((Map) Utils.fromJSONString(autoScaleJson)), - cloudManager, Collections.singletonMap("newColl", "policy1"), Arrays.asList("shard1", "shard2"), 3, 0, 0, null); - assertTrue(locations.stream().allMatch(it -> ImmutableList.of("node2", "node1", "node3").contains(it.node))); - } - - public void testMoveReplicaSuggester() { - String autoScalingjson = " '{cluster-policy':[" + - "{'cores':'<10', 'node':'#ANY'}," + - "{'replica':'<2', 'shard':'#EACH','node':'#ANY'}]," + - "'cluster-preferences':[{'minimize':'cores'}]}"; - @SuppressWarnings({"unchecked"}) - Policy policy = new Policy((Map) Utils.fromJSONString(autoScalingjson)); - Policy.Session session = policy.createSession(cloudManagerWithData((Map) loadFromResource("testMoveReplicaSuggester.json"))); - Suggester suggester = session.getSuggester(MOVEREPLICA) - .hint(Hint.TARGET_NODE, "10.0.0.6:7574_solr"); - @SuppressWarnings({"rawtypes"}) - SolrRequest op = suggester.getSuggestion(); - assertNotNull(op); - suggester = suggester.getSession() - .getSuggester(MOVEREPLICA) - .hint(Hint.TARGET_NODE, "10.0.0.6:7574_solr"); - op = suggester.getSuggestion(); - assertNull(op); - - suggester = suggester.getSession() - .getSuggester(MOVEREPLICA) - .forceOperation(true) - .hint(Hint.TARGET_NODE, "10.0.0.6:8983_solr"); - op = suggester.getSuggestion(); - assertNull(op); - } - - public void testComputePlanAfterNodeAdded() { - String autoScalingjson = "cluster-preferences:[" + - " {minimize : cores}," + - " {'maximize':freedisk , precision:100}], " + - " cluster-policy:[{cores:'<10',node:'#ANY'}," + - " {replica:'<2', shard:'#EACH',node:'#ANY'}," + - " { nodeRole:overseer,replica:0}]}"; - if(useNodeset){ - autoScalingjson = "cluster-preferences:[" + - " {minimize : cores}," + - " {'maximize':freedisk , precision:100}], " + - " cluster-policy:[{cores:'<10',node:'#ANY'}," + - " {replica:'<2', shard:'#EACH',node:'#ANY'}," + - " {nodeset:{ nodeRole:overseer},replica:0}]}"; - - } - @SuppressWarnings({"unchecked"}) - Policy policy = new Policy((Map) Utils.fromJSONString(autoScalingjson)); - Policy.Session session = policy.createSession(cloudManagerWithData((Map) loadFromResource("testComputePlanAfterNodeAdded.json"))); - Suggester suggester = session.getSuggester(CollectionParams.CollectionAction.MOVEREPLICA) - .hint(Hint.TARGET_NODE, "127.0.0.1:51147_solr"); - @SuppressWarnings({"rawtypes"}) - SolrRequest op = suggester.getSuggestion(); - log.info("{}", op); - assertNotNull("operation expected ", op); - } - - public void testReplicaCountSuggestions() { - String autoScalingjson = " { cluster-policy:[" + - " { cores :'<10', node :'#ANY'}," + - " { replica :'<2', node:'#ANY'}," + - " { nodeRole : overseer, replica :0}]," + - " cluster-preferences :[{ minimize : cores }]}"; - if(useNodeset){ - autoScalingjson = " { cluster-policy:[" + - " { cores :'<10', node :'#ANY'}," + - " { replica :'<2', node:'#ANY'}," + - " { nodeset:{nodeRole : overseer}, replica :0}]," + - " cluster-preferences :[{ minimize : cores }]}"; - - } - @SuppressWarnings({"unchecked"}) - List l = PolicyHelper.getSuggestions(new AutoScalingConfig((Map) Utils.fromJSONString(autoScalingjson)), - cloudManagerWithData((Map) loadFromResource("testReplicaCountSuggestions.json"))); - assertFalse(l.isEmpty()); - - assertEquals(1.0d, l.get(0)._get( "violation/violation/delta",null)); - assertEquals("POST", l.get(0)._get("operation/method",null)); - assertEquals("/c/mycoll1", l.get(0)._get( "operation/path",null)); - assertNotNull(l.get(0)._get("operation/command/move-replica", null)); - assertEquals("10.0.0.6:7574_solr", l.get(0)._get( "operation/command/move-replica/targetNode",null)); - /* - * one of the two cores on 10.0.0.6:8983_solr should move to 10.0.0.6:7574_solr and - * (everything else being equal) core_node1 is chosen ahead of core_node2 based on its name - */ - assertEquals("core_node1", l.get(0)._get("operation/command/move-replica/replica", null)); - } - - - @SuppressWarnings({"unchecked", "rawtypes"}) - public void testReplicaPercentage() { - List l = (List) loadFromResource("testReplicaPercentage.json"); - String autoScalingjson = " { cluster-policy:[" + - " { replica :'51%', shard:'#EACH', node:'#ANY'}]," + - " cluster-preferences :[{ minimize : cores }]}"; - - - AutoScalingConfig autoScalingConfig = new AutoScalingConfig((Map) Utils.fromJSONString(autoScalingjson)); - Policy.Session session = autoScalingConfig.getPolicy().createSession(cloudManagerWithData(l.get(0))); - List violations = session.getViolations(); - assertEquals(2, violations.size()); - for (Violation violation : violations) { - if (violation.node.equals("10.0.0.6:8983_solr")) { - assertEquals(1.0d, violation.replicaCountDelta, 0.01); - assertEquals(1.53d, ((RangeVal) violation.getClause().getReplica().val).actual); - } else if (violation.node.equals("10.0.0.6:7574_solr")) { - assertEquals(-1.0d, violation.replicaCountDelta, 0.01); - } - - } - - - session = autoScalingConfig.getPolicy().createSession(cloudManagerWithData(Utils.getDeepCopy(l.get(1), 6))); - violations = session.getViolations(); - assertEquals(0, violations.size()); - autoScalingjson = " { cluster-policy:[" + - " { replica :'51%', shard: '#EACH' , node:'#ANY'}]," + - " cluster-preferences :[{ minimize : cores }]}"; - autoScalingConfig = new AutoScalingConfig((Map) Utils.fromJSONString(autoScalingjson)); - session = autoScalingConfig.getPolicy().createSession(cloudManagerWithData(l.get(1))); - violations = session.getViolations(); - assertEquals(0, violations.size()); - autoScalingjson = " { cluster-policy:[" + - " { replica :'50%',node:'#ANY' , type: TLOG } ,{ replica :'50%',node:'#ANY' , type: PULL } ]," + - " cluster-preferences :[{ minimize : cores }]}"; - autoScalingConfig = new AutoScalingConfig((Map) Utils.fromJSONString(autoScalingjson)); - session = autoScalingConfig.getPolicy().createSession(cloudManagerWithData(l.get(2))); - violations = session.getViolations(); - assertEquals(2, violations.size()); - - } - - @SuppressWarnings({"unchecked"}) - public void testReplicaZonesPercentage() { - String autoScalingjson = " { cluster-policy:[" + - " { replica :'33%', shard: '#EACH', sysprop.az : east}," + - " { replica :'67%', shard: '#EACH', sysprop.az : west}" + - " ]," + - " cluster-preferences :[{ minimize : cores }]}"; - - String COLL_NAME = "percentColl"; - AutoScalingConfig autoScalingConfig = new AutoScalingConfig((Map) Utils.fromJSONString(autoScalingjson)); - - Policy.Transaction txn = new Policy.Transaction(autoScalingConfig.getPolicy()); - txn.open(cloudManagerWithData((Map) loadFromResource("testReplicaZonesPercentage.json"))); - - List nodes = new ArrayList<>(); - - int westCount = 0, eastCount = 0; - for (int i = 0; i < 12; i++) { - @SuppressWarnings({"rawtypes"}) - SolrRequest suggestion = txn.getCurrentSession() - .getSuggester(ADDREPLICA) - .hint(Hint.COLL_SHARD, new Pair<>(COLL_NAME, "shard1")) - .getSuggestion(); - assertNotNull(suggestion); - String node = suggestion.getParams().get("node"); - nodes.add(node); - if ("10.0.0.6:8983_solr".equals(node)) eastCount++; - if ("10.0.0.6:7574_solr".equals(node)) westCount++; - if (i % 3 == 1) assertEquals("10.0.0.6:8983_solr", node); - else assertEquals("10.0.0.6:7574_solr", node); - } - assertEquals(8, westCount); - assertEquals(4, eastCount); - - List violations = txn.close(); - assertTrue(violations.isEmpty()); - Policy.Session latestSession = txn.getCurrentSession(); - assertEquals("10.0.0.6:7574_solr", latestSession.matrix.get(0).node); - AtomicInteger count = new AtomicInteger(); - latestSession.matrix.get(0).forEachReplica(replicaInfo -> count.incrementAndGet()); - assertEquals(8, count.get()); - - assertEquals("10.0.0.6:8983_solr", latestSession.matrix.get(1).node); - count.set(0); - latestSession.matrix.get(1).forEachReplica(replicaInfo -> count.incrementAndGet()); - assertEquals(4, count.get()); - - } - - @SuppressWarnings({"unchecked"}) - public void testFreeDiskDeviation() { - @SuppressWarnings({"rawtypes"}) - Map map = (Map) loadFromResource("testFreeDiskDeviation.json"); - AutoScalingConfig cfg = new AutoScalingConfig((Map) map.get("config")); - if(useNodeset){ - cfg = new AutoScalingConfig((Map) Utils.fromJSONString("{" + - " 'cluster-policy': [{'replica':'<2', 'shard':'#EACH', 'node':'#ANY'}," + - " {'replica': '#ALL', 'nodeset': {'freedisk': '>700'}, 'strict': false}]" + - " }")); - } - SolrCloudManager scm = cloudManagerWithData(map); - Suggester suggester = cfg.getPolicy() - .createSession(scm) - .getSuggester(ADDREPLICA); - - MapWriter v2Request = (MapWriter) ((V2RequestSupport) suggester - .hint(Hint.COLL_SHARD, new Pair<>("mycoll2", "shard1")) - .getSuggestion() - .setUseV2(true)) - .getV2Request(); - assertEquals("/c/mycoll2/shards", v2Request._get("path",null)); - assertEquals("add-replica", v2Request._get("command[0]/key",null)); - assertEquals("node1", v2Request._get("command/add-replica/node",null)); - - - suggester = suggester.getSession() - .getSuggester(ADDREPLICA); - v2Request = (MapWriter) ((V2RequestSupport) suggester - .hint(Hint.COLL_SHARD, new Pair<>("mycoll2", "shard1")) - .getSuggestion() - .setUseV2(true)) - .getV2Request(); - - assertEquals("/c/mycoll2/shards", v2Request._get("path",null)); - assertEquals("add-replica", v2Request._get("command[0]/key",null)); - assertEquals("node2", v2Request._get("command/add-replica/node",null)); - - - } - - - @SuppressWarnings({"unchecked"}) - public void testFreeDiskSuggestions() { - String autoScalingjson = " { cluster-policy:[" + - " { replica :'0', freedisk:'<1000'}," + - " { nodeRole : overseer, replica :0}]," + - " cluster-preferences :[{ minimize : cores, precision : 2 }]}"; - if(useNodeset){ - autoScalingjson = " { cluster-policy:[" + - " { replica :'0', put:on-each-node , nodeset:{ freedisk:'<1000'}}," + - " { replica :0, put : on-each-node , nodeset : {nodeRole : overseer}}]," + - " cluster-preferences :[{ minimize : cores, precision : 2 }]}"; - } - AutoScalingConfig cfg = new AutoScalingConfig((Map) Utils.fromJSONString(autoScalingjson)); - List violations = cfg.getPolicy().createSession(cloudManagerWithData((Map) loadFromResource("testFreeDiskSuggestions.json"))).getViolations(); - assertEquals(1, violations.size()); - assertEquals(4, violations.get(0).getViolatingReplicas().size()); - assertEquals(4, violations.get(0).replicaCountDelta, 0.1); - for (Violation.ReplicaInfoAndErr r : violations.get(0).getViolatingReplicas()) { - assertEquals(500d, r.delta, 0.1); - - } - - List l = PolicyHelper.getSuggestions(cfg, cloudManagerWithData((Map) loadFromResource("testFreeDiskSuggestions.json"))); - assertEquals(3, l.size()); - assertEquals("r4", l.get(0)._get("operation/command/move-replica/replica", null)); - assertEquals("node1", l.get(0)._get("operation/command/move-replica/targetNode", null)); - - assertEquals("r3", l.get(1)._get("operation/command/move-replica/replica", null)); - assertEquals("node1", l.get(1)._get("operation/command/move-replica/targetNode", null)); - - assertEquals("r2", l.get(2)._get("operation/command/move-replica/replica", null)); - assertEquals("node1", l.get(2)._get("operation/command/move-replica/targetNode", null)); - - - autoScalingjson = " { cluster-policy:[" + - " { replica :'#ALL', freedisk:'>1000'}," + - " { nodeRole : overseer, replica :0}]," + - " cluster-preferences :[{ minimize : cores, precision : 2 }]}"; - if(useNodeset){ - autoScalingjson = " { cluster-policy:[" + - " { replica :'#ALL', nodeset:{ freedisk:'>1000'}}," + - " { replica :0 , put: on-each-node , nodeset : {nodeRole : overseer}}]," + - " cluster-preferences :[{ minimize : cores, precision : 2 }]}"; - } - cfg = new AutoScalingConfig((Map) Utils.fromJSONString(autoScalingjson)); - violations = cfg.getPolicy().createSession(cloudManagerWithData((Map) loadFromResource("testFreeDiskSuggestions.json"))).getViolations(); - assertEquals(1, violations.size()); - assertEquals(-4, violations.get(0).replicaCountDelta, 0.1); - assertEquals(1, violations.size()); - assertEquals(0, violations.get(0).getViolatingReplicas().size()); - - l = PolicyHelper.getSuggestions(cfg, cloudManagerWithData((Map) loadFromResource("testFreeDiskSuggestions.json"))); - assertEquals(3, l.size()); - assertEquals("r4", l.get(0)._get("operation/command/move-replica/replica", null)); - assertEquals("node1", l.get(0)._get("operation/command/move-replica/targetNode", null)); - - assertEquals("r3", l.get(1)._get("operation/command/move-replica/replica", null)); - assertEquals("node1", l.get(1)._get("operation/command/move-replica/targetNode", null)); - - assertEquals("r2", l.get(2)._get("operation/command/move-replica/replica", null)); - assertEquals("node1", l.get(2)._get("operation/command/move-replica/targetNode", null)); - - - } - - - public void testCoresSuggestions() { - String autoScalingjson = " { cluster-policy:[" + - " { cores :'<3', node :'#ANY'}]," + - " cluster-preferences :[{ minimize : cores }]}"; - @SuppressWarnings({"unchecked"}) - AutoScalingConfig cfg = new AutoScalingConfig((Map) Utils.fromJSONString(autoScalingjson)); - List violations = cfg.getPolicy().createSession(cloudManagerWithData((Map) loadFromResource("testCoresSuggestions.json"))).getViolations(); - assertFalse(violations.isEmpty()); - assertEquals(2L, violations.get(0).replicaCountDelta.longValue()); - - List l = PolicyHelper.getSuggestions(cfg, - cloudManagerWithData((Map) loadFromResource("testCoresSuggestions.json"))); - assertEquals(2, l.size()); - for (Suggester.SuggestionInfo suggestionInfo : l) { - assertEquals("10.0.0.6:7574_solr", suggestionInfo._get("operation/command/move-replica/targetNode", null)); - assertEquals("POST", suggestionInfo._get("operation/method", null)); - assertEquals("/c/mycoll1", suggestionInfo._get("operation/path", null)); - } - - } - - public void testSyspropSuggestions1() { - String autoScalingjson = "{" + - " 'cluster-preferences': [" + - " { 'maximize': 'freedisk', 'precision': 50}," + - " { 'minimize': 'cores', 'precision': 3}" + - " ]," + - " 'cluster-policy': [" + - " { 'replica': '1', shard:'#EACH', sysprop.fs : 'ssd'}" + - " ]" + - "}"; - if(useNodeset){ - autoScalingjson = "{" + - " 'cluster-preferences': [" + - " { 'maximize': 'freedisk', 'precision': 50}," + - " { 'minimize': 'cores', 'precision': 3}" + - " ]," + - " 'cluster-policy': [" + - " { 'replica': '1', shard:'#EACH', nodeset:{ sysprop.fs : 'ssd'}}" + - " ]" + - "}"; - } - - - @SuppressWarnings({"unchecked"}) - AutoScalingConfig cfg = new AutoScalingConfig((Map) Utils.fromJSONString(autoScalingjson)); - List violations = cfg.getPolicy().createSession(cloudManagerWithData((Map) loadFromResource("testSyspropSuggestions1.json"))).getViolations(); - assertEquals("expected 2 violations", 2, violations.size()); - List suggestions = PolicyHelper.getSuggestions(cfg, cloudManagerWithData((Map) loadFromResource("testSyspropSuggestions1.json"))); - assertEquals(2, suggestions.size()); - for (Suggester.SuggestionInfo suggestion : suggestions) { - suggestion._get("operation/move-replica/targetNode", null); - } - } - - public void testPortSuggestions() { - String autoScalingjson = "{" + - " 'cluster-preferences': [" + - " { 'maximize': 'freedisk', 'precision': 50}," + - " { 'minimize': 'cores', 'precision': 3}" + - " ]," + - " 'cluster-policy': [" + - " { 'replica': 0, shard:'#EACH', port : '8983'}" + - " ]" + - "}"; - - if(useNodeset){ - autoScalingjson = "{" + - " 'cluster-preferences': [" + - " { 'maximize': 'freedisk', 'precision': 50}," + - " { 'minimize': 'cores', 'precision': 3}" + - " ]," + - " 'cluster-policy': [" + - " { 'replica': 0, shard:'#EACH', nodeset :{ port : '8983'}}" + - " ]" + - "}"; - - } - @SuppressWarnings({"unchecked"}) - AutoScalingConfig cfg = new AutoScalingConfig((Map) Utils.fromJSONString(autoScalingjson)); - List violations = cfg.getPolicy().createSession(cloudManagerWithData((Map) loadFromResource("testPortSuggestions.json"))).getViolations(); - assertEquals(2, violations.size()); - List suggestions = PolicyHelper.getSuggestions(cfg, cloudManagerWithData((Map) loadFromResource("testPortSuggestions.json"))); - assertEquals(4, suggestions.size()); - for (Suggester.SuggestionInfo suggestionInfo : suggestions) { - assertEquals(suggestionInfo.operation.getPath(), "/c/c1"); - } - } - - public void testDiskSpaceHint() { - String autoScalingjson = "cluster-preferences:[" + - " {minimize : cores}]" + - " cluster-policy:[{cores:'<10',node:'#ANY'}," + - " {replica:'<2', shard:'#EACH',node:'#ANY'}," + - " { nodeRole:overseer,replica:0}]}"; - @SuppressWarnings({"unchecked"}) - Policy policy = new Policy((Map) Utils.fromJSONString(autoScalingjson)); - Policy.Session session = policy.createSession(cloudManagerWithData((Map) loadFromResource("testDiskSpaceHint.json"))); - Suggester suggester = session.getSuggester(CollectionAction.ADDREPLICA) - .hint(Hint.COLL_SHARD, new Pair<>("coll1", "shard1")) - .hint(Hint.MINFREEDISK, 150); - CollectionAdminRequest.AddReplica op = (CollectionAdminRequest.AddReplica) suggester.getSuggestion(); - - assertEquals("127.0.0.1:51078_solr", op.getNode()); - - suggester = session.getSuggester(CollectionAction.ADDREPLICA) - .hint(Hint.COLL_SHARD, new Pair<>("coll1", "shard1")); - op = (CollectionAdminRequest.AddReplica) suggester.getSuggestion(); - - assertEquals("127.0.0.1:51147_solr", op.getNode()); - } - - public void testDiskSpaceReqd() { - String autoScaleJson = "{" + - " cluster-preferences: [" + - " { minimize : cores, precision: 2}" + - " ]," + - " cluster-policy: [" + - " { replica : '0' , nodeRole: overseer}" + - - " ]" + - "}"; - - - @SuppressWarnings({"unchecked", "rawtypes"}) - Map nodeValues = (Map) Utils.fromJSONString("{" + - "node1:{cores:12, freedisk: 334, heap:10480, sysprop.rack:rack3}," + - "node2:{cores:4, freedisk: 262, heap:6873, sysprop.fs : ssd, sysprop.rack:rack1}," + - "node3:{cores:7, freedisk: 749, heap:7834, sysprop.rack:rack4}," + - "node4:{cores:0, freedisk: 900, heap:16900, nodeRole:overseer, sysprop.rack:rack2}" + - "}"); - - SolrCloudManager cloudManager = new DelegatingCloudManager(null) { - @Override - public NodeStateProvider getNodeStateProvider() { - return new DelegatingNodeStateProvider(null) { - @Override - public Map getNodeValues(String node, Collection keys) { - Map result = new LinkedHashMap<>(); - keys.stream().forEach(s -> result.put(s, nodeValues.get(node).get(s))); - return result; - } - - @Override - @SuppressWarnings({"unchecked", "rawtypes"}) - public Map>> getReplicaInfo(String node, Collection keys) { - if (node.equals("node1")) { - Map m = Utils.makeMap("newColl", - Utils.makeMap("shard1", Collections.singletonList(new Replica("r1", "node1", "newColl", "shard1", "core1", - Replica.State.ACTIVE, Replica.Type.NRT, Utils.makeMap(FREEDISK.perReplicaValue, 200))))); - return m; - } else if (node.equals("node2")) { - Map m = Utils.makeMap("newColl", - Utils.makeMap("shard2", Collections.singletonList(new Replica("r1", "node2", "newColl", "shard2", "core2", - Replica.State.ACTIVE, Replica.Type.NRT, Utils.makeMap(FREEDISK.perReplicaValue, 200))))); - return m; - } - return new HashMap<>(); - } - }; - } - - @Override - public ClusterStateProvider getClusterStateProvider() { - return new DelegatingClusterStateProvider(null) { - @Override - public Set getLiveNodes() { - return new HashSet<>(Arrays.asList("node1", "node2", "node3", "node4")); - } - - @Override - public DocCollection getCollection(String name) { - return new DocCollection(name, Collections.emptyMap(), Collections.emptyMap(), DocRouter.DEFAULT) { - @Override - public Replica getLeader(String sliceName) { - if (sliceName.equals("shard1")) - return new Replica("r1", Utils.makeMap(ZkStateReader.NODE_NAME_PROP, "node1", ZkStateReader.CORE_NAME_PROP, "core1"), name, "shard1"); - if (sliceName.equals("shard2")) - return new Replica("r2", Utils.makeMap(ZkStateReader.NODE_NAME_PROP, "node2", ZkStateReader.CORE_NAME_PROP, "core2"),name, "shard2"); - return null; - } - }; - } - }; - } - }; - @SuppressWarnings({"unchecked"}) - List locations = PolicyHelper.getReplicaLocations( - "newColl", new AutoScalingConfig((Map) Utils.fromJSONString(autoScaleJson)), - cloudManager, null, Arrays.asList("shard1", "shard2"), 1, 0, 0, null); - assertTrue(locations.stream().allMatch(it -> "node3".equals(it.node))); - } - - public void testMoveReplicaLeaderlast() { - - List> validReplicas = new ArrayList<>(); - Map propMap = Utils.makeMap( - "leader", "true", - ZkStateReader.NODE_NAME_PROP, "node1", - ZkStateReader.REPLICA_TYPE, Replica.Type.NRT.toString(), - ZkStateReader.CORE_NAME_PROP, "core1"); - Replica replica = new Replica("r1", propMap, "c1", "s1"); - Replica replicaInfo = new Replica(replica.name, replica.node, replica.collection, replica.shard, replica.core, - replica.getState(), replica.type, replica.getProperties()); - validReplicas.add(new Pair<>(replicaInfo, null)); - - replicaInfo = new Replica("r4", "n1", "c1_s2_r1", "c1", "s2", Replica.State.ACTIVE, Replica.Type.NRT, - Utils.makeMap(ZkStateReader.LEADER_PROP, "true")); - validReplicas.add(new Pair<>(replicaInfo, null)); - - - propMap.put("leader", false); - propMap.put("core", "r2"); - propMap.put("node_name", "n1"); - replica = new Replica("r2", propMap,"c1","s1"); - replicaInfo = new Replica(replica.name, replica.node, replica.collection, replica.shard, replica.core, - replica.getState(), replica.type, new HashMap<>()); - validReplicas.add(new Pair<>(replicaInfo, null)); - - propMap.put("core", "r2"); - replica = new Replica("r3", propMap,"c1","s1"); - replicaInfo = new Replica(replica.name, replica.node, replica.collection, replica.shard, replica.core, - replica.getState(), replica.type, new HashMap<>()); - validReplicas.add(new Pair<>(replicaInfo, null)); - - - validReplicas.sort(MoveReplicaSuggester.leaderLast); - assertEquals("r2", validReplicas.get(0).first().getName()); - assertEquals("r3", validReplicas.get(1).first().getName()); - assertEquals("r1", validReplicas.get(2).first().getName()); - assertEquals("r4", validReplicas.get(3).first().getName()); - - } - - public void testScheduledTriggerFailure() throws Exception { - @SuppressWarnings({"rawtypes"}) - Map jsonObj = (Map) loadFromResource("testScheduledTriggerFailure.json"); - SolrCloudManager cloudManager = createCloudManager(jsonObj); - Suggester suggester = createSuggester(cloudManager, jsonObj, null); - int count = 0; - while (count < 10) { - CollectionAdminRequest.MoveReplica op = (CollectionAdminRequest.MoveReplica) suggester.getSuggestion(); - if (op == null) break; - count++; - if (log.isInfoEnabled()) { - log.info("OP:{}", op.getParams()); - } - suggester = createSuggester(cloudManager, jsonObj, suggester); - } - - assertEquals(0, count); - } - - public void testUtilizeNodeFailure() throws Exception { - @SuppressWarnings({"rawtypes"}) - Map jsonObj = (Map) loadFromResource("testUtilizeNodeFailure.json"); //(Map) Utils.fromJSONString(state); - SolrCloudManager cloudManager = createCloudManager(jsonObj); - Suggester suggester = createSuggester(cloudManager, jsonObj, null); - int count = 0; - while (count < 100) { - CollectionAdminRequest.MoveReplica op = (CollectionAdminRequest.MoveReplica) suggester.getSuggestion(); - if (op == null) break; - count++; - if (log.isInfoEnabled()) { - log.info("OP:{}", op.getParams()); - } - suggester = createSuggester(cloudManager, jsonObj, suggester); - } - - assertEquals("count = " + count, 0, count); - } - - public void testUtilizeNodeFailure2() throws Exception { - @SuppressWarnings({"rawtypes"}) - Map jsonObj = (Map) loadFromResource("testUtilizeNodeFailure2.json"); - SolrCloudManager cloudManager = createCloudManager(jsonObj); - Suggester suggester = createSuggester(cloudManager, jsonObj, null); - int count = 0; - while (count < 100) { - CollectionAdminRequest.MoveReplica op = (CollectionAdminRequest.MoveReplica) suggester.getSuggestion(); - if (op == null) break; - count++; - if (log.isInfoEnabled()) { - log.info("OP:{}", op.getParams()); - } - suggester = createSuggester(cloudManager, jsonObj, suggester); - } - - assertEquals("count = " + count, 1, count); - } - - //SOLR-12358 - @SuppressWarnings({"unchecked", "rawtypes"}) - public void testSortError() { - Policy policy = new Policy((Map) Utils.fromJSONString("{cluster-preferences: [{minimize : cores, precision:1}, " + - "{maximize : freedisk, precision: 50}, " + - "{minimize: sysLoadAvg}]}")); - - List l = (List) loadFromResource("testSortError.json"); - List params = new ArrayList<>(); - params.add(CORES); - params.add(Variable.Type.FREEDISK); - params.add(Variable.Type.SYSLOADAVG); - params.add(Variable.Type.NODE); - List rows = new ArrayList<>(); - for (Object o : l) { - Map m = (Map) o; - Cell[] c = new Cell[params.size()]; - List attrs = (List) m.get("attributes"); - for (int i = 0; i < params.size(); i++) { - Variable.Type param = params.get(i); - for (Object attr : attrs) { - Object o1 = ((Map) attr).get(param.tagName); - if (o1 != null) { - o1 = param.validate(param.tagName, o1, false); - c[i] = new Cell(i, param.tagName, o1, o1, param, null); - } - } - } - rows.add(new Row((String) m.get("node"), c, false, - new HashMap<>(), - (Boolean) m.get("isLive"), null, new HashMap(), new HashMap())); - } - int deadNodes = 0; - for (Row row : rows) { - if (!row.isLive) deadNodes++; - } - - Policy.setApproxValuesAndSortNodes(policy.getClusterPreferences(), rows); - - for (int i = 0; i < deadNodes; i++) { - assertFalse(rows.get(i).isLive); - } - - for (int i = deadNodes; i < rows.size(); i++) { - assertTrue(rows.get(i).isLive); - } - - - } - - public void testViolationOutput() throws IOException { - String autoScalingjson = "{" + - " 'cluster-preferences': [" + - " { 'maximize': 'freedisk', 'precision': 50}," + - " { 'minimize': 'cores', 'precision': 3}" + - " ]," + - " 'cluster-policy': [" + - " { 'replica': 0, shard:'#EACH', port : '8983'}" + - " ]" + - "}"; - - @SuppressWarnings({"unchecked"}) - AutoScalingConfig cfg = new AutoScalingConfig((Map) Utils.fromJSONString(autoScalingjson)); - List violations = cfg.getPolicy().createSession(cloudManagerWithData((Map) loadFromResource("testViolationOutput.json"))).getViolations(); - StringWriter writer = new StringWriter(); - NamedList val = new NamedList<>(); - val.add("violations", violations); - - if (random().nextBoolean()) { - new SolrJSONWriter(writer) - .writeObj(val) - .close(); - } else { - JSONWriter.write(writer, true, JsonTextWriter.JSON_NL_MAP, val); - } - - Object root = Utils.fromJSONString(writer.toString()); - assertEquals(2l, - Utils.getObjectByPath(root, true, "violations[0]/violation/replica/NRT")); - } - - - @SuppressWarnings({"unchecked"}) - public void testFreediskPercentage() { - - String autoScalingjson = "{" + - " 'cluster-preferences': [" + - " { 'maximize': 'freedisk', 'precision': 50}," + - " { 'minimize': 'cores', 'precision': 3}" + - " ]," + - " 'cluster-policy': [" + - " { 'replica': 0, freedisk : '<30%'}" + - " ]" + - "}"; - AutoScalingConfig cfg = new AutoScalingConfig((Map) Utils.fromJSONString(autoScalingjson)); - List violations = cfg.getPolicy().createSession(cloudManagerWithData((Map) loadFromResource("testFreediskPercentage.json"))).getViolations(); - assertEquals(1, violations.size()); - assertEquals(4, violations.get(0).getViolatingReplicas().size()); - for (Violation.ReplicaInfoAndErr r : violations.get(0).getViolatingReplicas()) { - assertEquals(10.0d, r.delta.doubleValue(), 0.1); - } - autoScalingjson = "{" + - " 'cluster-preferences': [" + - " { 'maximize': 'freedisk', 'precision': 50}," + - " { 'minimize': 'cores', 'precision': 3}" + - " ]," + - " 'cluster-policy': [" + - " { 'replica':'#ALL' , freedisk : '>30%'}" + - " ]" + - "}"; - cfg = new AutoScalingConfig((Map) Utils.fromJSONString(autoScalingjson)); - violations = cfg.getPolicy().createSession(cloudManagerWithData((Map) loadFromResource("testFreediskPercentage.json"))).getViolations(); - assertEquals(1, violations.size()); - assertEquals(-4d, violations.get(0).replicaCountDelta, 0.01); - for (Violation.ReplicaInfoAndErr r : violations.get(0).getViolatingReplicas()) { - assertEquals(10.0d, r.delta.doubleValue(), 0.1); - } - - } - - @SuppressWarnings({"unchecked", "rawtypes"}) - public static void fixRequiredProps(Map testData) { - Map clusterState = (Map) testData.get("clusterstate"); - clusterState.forEach((collection, val) -> { - Map docColl = (Map) val; - Map shards = (Map) docColl.get("shards"); - shards.forEach((shardName, val2) -> { - Map shard = (Map) val2; - Map replicas = (Map) shard.get("replicas"); - replicas.forEach((coreNode, val3) -> { - Map replica = (Map) val3; - if (!replica.containsKey("node_name")) { - replica.put("node_name", "node1"); - } - if (!replica.containsKey("core")) { - replica.put("core", "core_" + coreNode); - } - }); - }); - }); - Map replicaInfo = (Map) testData.get("replicaInfo"); - replicaInfo.forEach((node, val) -> { - Map m1 = (Map) val; - m1.forEach((coll, val2) -> { - Map m2 = (Map) val2; - m2.forEach((shard, val3) -> { - List l3 = (List) val3; - l3.forEach(o -> { - Map replica = (Map) o; - String coreNode = replica.keySet().iterator().next().toString(); - replica = (Map) replica.get(coreNode); - if (!replica.containsKey("node_name")) { - replica.put("node_name", "node1"); - } - if (!replica.containsKey("core")) { - replica.put("core", "core_" + coreNode); - } - }); - }); - }); - }); - } - - @SuppressWarnings({"unchecked", "rawtypes"}) - public void testAutoscalingPreferencesUsedWithNoPolicy() throws IOException, InterruptedException { - Map m = (Map) loadFromResource("testAutoscalingPreferencesUsedWithNoPolicy.json"); - fixRequiredProps(m); - Map clusterState = (Map) m.remove("clusterstate"); - - Map replicaInfo = (Map) m.get("replicaInfo"); - replicaInfo.forEach((node, val) -> { - Map m1 = (Map) val; - m1.forEach((coll, val2) -> { - Map m2 = (Map) val2; - m2.forEach((shard, val3) -> { - List l3 = (List) val3; - for (int i = 0; i < l3.size(); i++) { - Object o = l3.get(i); - Map m3 = (Map) o; - String name = m3.keySet().iterator().next().toString(); - m3 = (Map) m3.get(name); - Replica.Type type = Replica.Type.get((String) m3.get("type")); - l3.set(i, new Replica(name, (String) node, coll.toString(), shard.toString(), - name, Replica.State.ACTIVE, type, m3)); - } - }); - - }); - }); - @SuppressWarnings({"unchecked"}) - AutoScalingConfig asc = m.containsKey("autoscalingJson") ? new AutoScalingConfig((Map) m.get("autoscalingJson")) : new AutoScalingConfig(Collections.emptyMap()); - DelegatingCloudManager cloudManager = new DelegatingCloudManager(null) { - - @Override - public DistribStateManager getDistribStateManager() { - return new DelegatingDistribStateManager(null) { - @Override - public AutoScalingConfig getAutoScalingConfig() { - return asc; - } - }; - } - - @Override - public ClusterStateProvider getClusterStateProvider() { - return new DelegatingClusterStateProvider(null) { - @Override - @SuppressWarnings({"unchecked"}) - public Set getLiveNodes() { - return new HashSet<>((Collection) m.get("liveNodes")); - } - - @Override - public ClusterState getClusterState() { - return ClusterState.createFromCollectionMap(0, clusterState, getLiveNodes()); - } - - @Override - public Map getClusterProperties() { - return Collections.singletonMap("defaults", Collections.singletonMap("cluster", Collections.singletonMap(CollectionAdminParams.USE_LEGACY_REPLICA_ASSIGNMENT, false))); - } - }; - } - - @Override - public NodeStateProvider getNodeStateProvider() { - return new DelegatingNodeStateProvider(null) { - @Override - public Map getNodeValues(String node, Collection tags) { - @SuppressWarnings({"unchecked"}) - Map result = (Map) Utils.getObjectByPath(m, false, Arrays.asList("nodeValues", node)); - return result == null ? new HashMap<>() : result; - } - - @Override - public Map>> getReplicaInfo(String node, Collection keys) { - @SuppressWarnings({"unchecked"}) - Map>> result = (Map>>) Utils.getObjectByPath(m, false, Arrays.asList("replicaInfo", node)); - return result == null ? new HashMap<>() : result; - } - }; - } - }; - - Assign.AssignRequest assignRequest = new Assign.AssignRequestBuilder() - .forCollection("c1") - .forShard(Collections.singletonList("s1")) - .assignNrtReplicas(1) - .build(); - Assign.AssignStrategyFactory assignStrategyFactory = new Assign.AssignStrategyFactory(cloudManager); - ClusterState state = cloudManager.getClusterStateProvider().getClusterState(); - DocCollection collection = state.getCollection("c1"); - Assign.AssignStrategy assignStrategy = assignStrategyFactory.create(state, collection); - List replicaPositions = assignStrategy.assign(cloudManager, assignRequest); - - assertEquals(1, replicaPositions.size()); - ReplicaPosition replicaPosition = replicaPositions.get(0); - assertEquals("node3:8985", replicaPosition.node); // only node3:8985 has enough space to handle the new replica - assertEquals("s1", replicaPosition.shard); // sanity check - } - - public void testPolicyForEmptyCollection() throws IOException, InterruptedException { - @SuppressWarnings({"rawtypes"}) - Map m = (Map) loadFromResource("testEmptyCollection.json"); - @SuppressWarnings({"unchecked", "rawtypes"}) - Map clusterStateMap = (Map) m.remove("clusterstate"); - @SuppressWarnings({"unchecked", "rawtypes"}) - Map replicaInfoMap = (Map) m.remove("replicaInfo"); - - @SuppressWarnings({"unchecked"}) - ClusterState clusterState = ClusterState.createFromCollectionMap(1, clusterStateMap, ImmutableSet.of("node1", "node2")); - - List shards = Arrays.asList("shard1", "shard2", "shard3"); - - Assign.AssignRequest assignRequest = new Assign.AssignRequestBuilder() - .forCollection("test_empty_collection") - .forShard(shards) - .assignNrtReplicas(1) - .build(); - - DelegatingCloudManager cloudManager = new DelegatingCloudManager(null) { - @Override - public ClusterStateProvider getClusterStateProvider() { - return new DelegatingClusterStateProvider(null) { - @Override - public ClusterState getClusterState() { - return clusterState; - } - - @Override - public Set getLiveNodes() { - return clusterState.getLiveNodes(); - } - }; - } - - @Override - public DistribStateManager getDistribStateManager() { - return new DelegatingDistribStateManager(null) { - @Override - public AutoScalingConfig getAutoScalingConfig() { - return new AutoScalingConfig(new HashMap<>()); - } - }; - } - - public NodeStateProvider getNodeStateProvider() { - return new DelegatingNodeStateProvider(null) { - @Override - @SuppressWarnings({"unchecked"}) - public Map getNodeValues(String node, Collection keys) { - return Collections.EMPTY_MAP; - } - - @Override - @SuppressWarnings({"unchecked"}) - public Map>> getReplicaInfo(String node, Collection keys) { - //return Collections.EMPTY_MAP; - return replicaInfoMap; - } - }; - } - - }; - - Assign.AssignStrategyFactory assignStrategyFactory = new Assign.AssignStrategyFactory(cloudManager); - ClusterState state = cloudManager.getClusterStateProvider().getClusterState(); - DocCollection collection = state.getCollection("test_empty_collection"); - - Assign.AssignStrategy assignStrategy = assignStrategyFactory.create(state, collection); - List replicaPositions = assignStrategy.assign(cloudManager, assignRequest); - assertEquals(2,replicaPositions.stream().map((rp)-> rp.node).distinct().count()); - assertEquals(3,replicaPositions.stream().map((rp)-> rp.shard).distinct().count()); - } - - /** - * Tests that an empty policy should not persist implicitly added keys to MapWriter - *

- * The reason behind doing this is to ensure that implicitly added cluster preferences do not ever - * go to ZooKeeper so that we can decide whether to enable autoscaling policy framework or not. - * - * @see Assign#usePolicyFramework(DocCollection, SolrCloudManager) - */ - public void testPolicyMapWriterWithEmptyPreferences() throws IOException { - @SuppressWarnings({"rawtypes"}) - List defaultPreferences = Policy.DEFAULT_PREFERENCES - .stream().map(preference -> preference.getOriginal()).collect(Collectors.toList()); - - // first we create a completely empty policy - Policy policy = new Policy(); - // sanity check that the default cluster preferences were added implicitly - assertNotNull(policy.getClusterPreferences()); - // and they were the same as the default preferences - assertEquals(policy.getClusterPreferences().size(), defaultPreferences.size()); - Set writtenKeys = new HashSet<>(); - policy.writeMap(new MapWriter.EntryWriter() { - @Override - public MapWriter.EntryWriter put(CharSequence k, Object v) throws IOException { - writtenKeys.add(k.toString()); - return this; - } - }); - // but those implicitly added cluster preferences are never written by MapWriter - assertEquals(0, writtenKeys.size()); - - // reset - writtenKeys.clear(); - // now we create a policy that only has cluster preferences which happen to be the same as the default - // preferences - policy = new Policy(Utils.makeMap(CLUSTER_PREFERENCES, defaultPreferences)); - // sanity checks - assertNotNull(policy.getClusterPreferences()); - assertEquals(policy.getClusterPreferences().size(), defaultPreferences.size()); - policy.writeMap(new MapWriter.EntryWriter() { - @Override - public MapWriter.EntryWriter put(CharSequence k, Object v) throws IOException { - writtenKeys.add(k.toString()); - return this; - } - }); - // since the user explicitly added those preferences, they should be written by MapWriter - assertEquals(1, writtenKeys.size()); - assertTrue(writtenKeys.contains(CLUSTER_PREFERENCES)); - - // reset - writtenKeys.clear(); - // now we create a cluster policy that is intentionally empty which should prevent the implicit - // cluster policy from being written but should emit an empty key/val pair for cluster policy - policy = new Policy(Utils.makeMap(CLUSTER_POLICY, Collections.emptyList())); - // sanity checks - assertFalse(policy.isEmpty()); - assertTrue(policy.hasEmptyPreferences()); - assertFalse(policy.hasEmptyClusterPolicy()); - policy.writeMap(new MapWriter.EntryWriter() { - @Override - public MapWriter.EntryWriter put(CharSequence k, Object v) throws IOException { - writtenKeys.add(k.toString()); - return this; - } - }); - assertEquals(1, writtenKeys.size()); - assertTrue(writtenKeys.contains(CLUSTER_POLICY)); - } -} diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy2.java b/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy2.java deleted file mode 100644 index e20b28f8821..00000000000 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy2.java +++ /dev/null @@ -1,540 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.client.solrj.cloud.autoscaling; - -import java.io.IOException; -import java.io.InputStream; -import java.lang.invoke.MethodHandles; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.stream.Collectors; - -import com.google.common.collect.ImmutableSet; -import org.apache.solr.SolrTestCaseJ4; -import org.apache.solr.client.solrj.cloud.NodeStateProvider; -import org.apache.solr.client.solrj.cloud.SolrCloudManager; -import org.apache.solr.client.solrj.impl.ClusterStateProvider; -import org.apache.solr.client.solrj.impl.SolrClientNodeStateProvider; -import org.apache.solr.common.SolrException; -import org.apache.solr.common.cloud.ClusterState; -import org.apache.solr.common.cloud.Replica; -import org.apache.solr.common.cloud.ReplicaPosition; -import org.apache.solr.common.util.Utils; -import org.junit.Ignore; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import static java.util.Collections.EMPTY_MAP; -import static org.apache.solr.client.solrj.cloud.autoscaling.Variable.Type.CORES; -import static org.apache.solr.common.util.Utils.MAPOBJBUILDER; -import static org.apache.solr.common.util.Utils.getObjectByPath; - -public class TestPolicy2 extends SolrTestCaseJ4 { - boolean useNodeset ; - public TestPolicy2(){ - useNodeset = true; - } - - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - @SuppressWarnings({"unchecked", "rawtypes"}) - public void testEqualOnNonNode() { - List l = (List) loadFromResource("testEqualOnNonNode.json"); - String autoScalingjson = "{cluster-policy:[" + - " { replica : '<3' , shard : '#EACH', sysprop.zone: east}," + - "{ replica : '<3' , shard : '#EACH', sysprop.zone: west} ]," + - " 'cluster-preferences':[{ minimize : cores},{maximize : freedisk, precision : 50}]}"; - if(useNodeset){ - autoScalingjson = "{cluster-policy:[" + - "{ replica : '<3' , shard : '#EACH', nodeset:{ sysprop.zone: east}}," + - "{ replica : '<3' , shard : '#EACH', nodeset:{ sysprop.zone: west}} ]," + - " 'cluster-preferences':[{ minimize : cores},{maximize : freedisk, precision : 50}]}"; - - } - Policy policy = new Policy((Map) Utils.fromJSONString(autoScalingjson)); - Policy.Session session = policy.createSession(createCloudManager(l.get(0), l.get(1))); - List violations = session.getViolations(); - assertEquals(1, violations.size()); - assertEquals(3, violations.get(0).getViolatingReplicas().size()); - assertEquals(1.0, violations.get(0).replicaCountDelta, 0.01); - for (Violation.ReplicaInfoAndErr r : violations.get(0).getViolatingReplicas()) { - assertEquals("shard2", r.replicaInfo.getShard()); - } - - l = (List) loadFromResource("testEqualOnNonNode.json"); - autoScalingjson = "{cluster-policy:[" + - " { replica : '<3' , shard : '#EACH', sysprop.zone: '#EACH' } ]," + - " 'cluster-preferences':[{ minimize : cores},{maximize : freedisk, precision : 50}]}"; - if(useNodeset){ - autoScalingjson = "{cluster-policy:[" + - "{ replica : '<3' , shard : '#EACH', nodeset:{sysprop.zone: east}} , " + - "{ replica : '<3' , shard : '#EACH', nodeset:{sysprop.zone: west}} ]," + - " 'cluster-preferences':[{ minimize : cores},{maximize : freedisk, precision : 50}]}"; - } - policy = new Policy((Map) Utils.fromJSONString(autoScalingjson)); - session = policy.createSession(createCloudManager(l.get(0), l.get(1))); - violations = session.getViolations(); - assertEquals(1, violations.size()); - assertEquals(3, violations.get(0).getViolatingReplicas().size()); - assertEquals(1.0, violations.get(0).replicaCountDelta, 0.01); - for (Violation.ReplicaInfoAndErr r : violations.get(0).getViolatingReplicas()) { - assertEquals("shard2", r.replicaInfo.getShard()); - } - l = (List) loadFromResource("testEqualOnNonNode.json"); - autoScalingjson = "{cluster-policy:[" + - " { replica : '#EQUAL' , node: '#ANY' } ]," + - " 'cluster-preferences':[{ minimize : cores},{maximize : freedisk, precision : 50}]}"; - policy = new Policy((Map) Utils.fromJSONString(autoScalingjson)); - session = policy.createSession(createCloudManager(l.get(0), l.get(1))); - violations = session.getViolations(); - List suggestions = null; - assertEquals(2, violations.size()); - for (Violation violation : violations) { - if (violation.node.equals("node1")) { - assertEquals(1.0d, violation.replicaCountDelta, 0.001); - assertEquals(3, violation.getViolatingReplicas().size()); - } else if (violation.node.equals("node5")) { - assertEquals(-1.0d, violation.replicaCountDelta, 0.001); - assertEquals(0, violation.getViolatingReplicas().size()); - } else { - fail(); - } - } - l = (List) loadFromResource("testEqualOnNonNode.json"); - suggestions = PolicyHelper.getSuggestions(new AutoScalingConfig((Map) Utils.fromJSONString(autoScalingjson)) - , createCloudManager(l.get(0), l.get(1))); - assertEquals(1, suggestions.size()); - String repName = (String) suggestions.get(0)._get("operation/command/move-replica/replica", null); - - AtomicBoolean found = new AtomicBoolean(false); - session.getNode("node1").forEachReplica(replicaInfo -> { - if (replicaInfo.getName().equals(repName)) { - found.set(true); - } - }); - assertTrue(found.get()); - - l = (List) loadFromResource("testEqualOnNonNode.json"); - autoScalingjson = "{cluster-policy:[" + - " { cores : '#EQUAL' , node: '#ANY' } ]," + - " 'cluster-preferences':[{ minimize : cores},{minimize : freedisk, precision : 50}]}"; - policy = new Policy((Map) Utils.fromJSONString(autoScalingjson)); - session = policy.createSession(createCloudManager(l.get(0), l.get(1))); - violations = session.getViolations(); - assertEquals(2, violations.size()); - for (Violation violation : violations) { - if(violation.node.equals("node1")) { - assertEquals(1.0d, violation.replicaCountDelta, 0.001); - assertEquals(3, violation.getViolatingReplicas().size()); - } else if(violation.node.equals("node5")){ - assertEquals(-1.0d, violation.replicaCountDelta, 0.001); - assertEquals(0, violation.getViolatingReplicas().size()); - } else { - fail(); - } - - } - l = (List) loadFromResource("testEqualOnNonNode.json"); - suggestions = PolicyHelper.getSuggestions(new AutoScalingConfig((Map) Utils.fromJSONString(autoScalingjson)), - createCloudManager(l.get(0), l.get(1))); - assertEquals(1, suggestions.size()); - assertEquals("node5", suggestions.get(0)._get("operation/command/move-replica/targetNode", null)); - - String rName = (String) suggestions.get(0)._get("operation/command/move-replica/replica", null); - - found.set(false); - session.getNode("node1").forEachReplica(replicaInfo -> { - if (replicaInfo.getName().equals(rName)) { - found.set(true); - } - }); - assertTrue(found.get()); - - } - - @SuppressWarnings({"unchecked", "rawtypes"}) - static SolrCloudManager createCloudManager(Map m, Map meta) { - Map nodeVals = (Map) meta.get("nodeValues"); - List replicaVals = (List) meta.get("replicaValues"); - ClusterState clusterState = ClusterState.createFromCollectionMap(0, m, Collections.emptySet()); - Map coreCount = new LinkedHashMap<>(); - Set nodes = new HashSet<>(nodeVals.keySet()); - clusterState.getCollectionStates().forEach((s, collectionRef) -> collectionRef.get() - .forEachReplica((s12, replica) -> { - nodes.add(replica.getNodeName()); - coreCount.computeIfAbsent(replica.getNodeName(), s1 -> new AtomicInteger(0)) - .incrementAndGet(); - })); - - DelegatingClusterStateProvider delegatingClusterStateProvider = new DelegatingClusterStateProvider(null) { - @Override - public ClusterState getClusterState() throws IOException { - return clusterState; - } - - @Override - public Set getLiveNodes() { - return nodes; - } - }; - - return new DelegatingCloudManager(null) { - - @Override - public ClusterStateProvider getClusterStateProvider() { - return delegatingClusterStateProvider; - } - - @Override - public NodeStateProvider getNodeStateProvider() { - - return new SolrClientNodeStateProvider(null) { - @Override - protected ClusterStateProvider getClusterStateProvider() { - return delegatingClusterStateProvider; - } - - @Override - protected Map fetchTagValues(String node, Collection tags) { - Map result = new LinkedHashMap<>(); - for (String tag : tags) { - if (tag.equals(CORES.tagName)) - result.put(CORES.tagName, coreCount.getOrDefault(node, new AtomicInteger(0)).get()); - result.put(tag, getObjectByPath(nodeVals, true, Arrays.asList(node, tag))); - } - return result; - } - - @Override - public Map>> getReplicaInfo(String node, Collection keys) { - @SuppressWarnings({"unchecked"}) - Map>> result = nodeVsCollectionVsShardVsReplicaInfo.computeIfAbsent(node, Utils.NEW_HASHMAP_FUN); - if (!keys.isEmpty()) { - Row.forEachReplica(result, replicaInfo -> { - for (String key : keys) { - if (!replicaInfo.getProperties().containsKey(key)) { - replicaVals.stream() - .filter(it -> replicaInfo.getCoreName().equals(it.get("core"))) - .findFirst() - .ifPresent(map -> replicaInfo.getProperties().put(key, map.get(key))); - } - } - }); - } - return result; - } - }; - } - }; - } - - @SuppressWarnings({"unchecked"}) - public void testAutoScalingHandlerFailure() { - Map m = (Map) loadFromResource("testAutoScalingHandlerFailure.json"); - - Policy policy = new Policy((Map) getObjectByPath(m, false, "diagnostics/config")); - SolrCloudManager cloudManagerFromDiagnostics = createCloudManagerFromDiagnostics(m); - Policy.Session session = policy.createSession(cloudManagerFromDiagnostics); - List suggestions = PolicyHelper.getSuggestions(new AutoScalingConfig((Map) getObjectByPath(m, false, "diagnostics/config")), cloudManagerFromDiagnostics); -// System.out.println(Utils.writeJson(suggestions, new StringWriter(),true).toString()); - assertEquals(4, suggestions.size()); - - } - - @SuppressWarnings({"unchecked", "rawtypes"}) - static SolrCloudManager createCloudManagerFromDiagnostics(Map m) { - List sortedNodes = (List) getObjectByPath(m, false, "diagnostics/sortedNodes"); - Set liveNodes = new HashSet<>(); - SolrClientNodeStateProvider nodeStateProvider = new SolrClientNodeStateProvider(null) { - @Override - protected void readReplicaDetails() { - for (Object o : sortedNodes) { - String node = (String) ((Map) o).get("node"); - liveNodes.add(node); - Map nodeDetails = nodeVsCollectionVsShardVsReplicaInfo.computeIfAbsent(node, s -> new LinkedHashMap<>()); - Map>> replicas = (Map>>) ((Map) o).get("replicas"); - replicas.forEach((coll, shardVsReplicas) -> shardVsReplicas - .forEach((shard, repDetails) -> { - List reps = (List) ((Map) nodeDetails - .computeIfAbsent(coll, o1 -> new LinkedHashMap<>())) - .computeIfAbsent(shard, o12 -> new ArrayList()); - for (Map map : repDetails) reps.add(new Replica(map)); - })); - } - - } - - @Override - public Map>> getReplicaInfo(String node, Collection keys) { - return nodeVsCollectionVsShardVsReplicaInfo.get(node) == null ? - Collections.emptyMap() : - nodeVsCollectionVsShardVsReplicaInfo.get(node); - } - - @Override - @SuppressWarnings({"unchecked", "rawtypes"}) - public Map getNodeValues(String node, Collection tags) { - for (Map n : sortedNodes) if (n.get("node").equals(node)) return n; - return Collections.emptyMap(); - } - }; - return new DelegatingCloudManager(null) { - ClusterState clusterState = null; - @Override - public NodeStateProvider getNodeStateProvider() { - return nodeStateProvider; - } - - @Override - public ClusterStateProvider getClusterStateProvider() { - if (clusterState == null) { - @SuppressWarnings({"rawtypes"}) - Map map = (Map) getObjectByPath(m, false, "cluster/collections"); - if (map == null) map = new HashMap<>(); - clusterState = ClusterState.createFromCollectionMap(0, map, liveNodes); - } - - return new DelegatingClusterStateProvider(null) { - - @Override - public ClusterState getClusterState() { - return clusterState; - } - - @Override - public ClusterState.CollectionRef getState(String collection) { - return clusterState.getCollectionRef(collection); - } - - @Override - public Set getLiveNodes() { - return liveNodes; - } - }; - } - }; - } - - public void testHostAttribute() { - @SuppressWarnings({"unchecked"}) - Map m = (Map) loadFromResource("testHostAttribute.json"); - @SuppressWarnings({"unchecked"}) - Map conf = (Map) getObjectByPath(m, false, "diagnostics/config"); - Policy policy = new Policy(conf); - SolrCloudManager cloudManagerFromDiagnostics = createCloudManagerFromDiagnostics(m); - Policy.Session session = policy.createSession(cloudManagerFromDiagnostics); - List violations = session.getViolations(); - for (Violation violation : violations) { - assertEquals(1.0d, violation.replicaCountDelta.doubleValue(), 0.0001); - } - assertEquals(2, violations.size()); - List suggestions = PolicyHelper.getSuggestions(new AutoScalingConfig(conf), cloudManagerFromDiagnostics); - assertEquals(2, suggestions.size()); - for (Suggester.SuggestionInfo suggestion : suggestions) { - assertTrue(ImmutableSet.of("127.0.0.219:63219_solr", "127.0.0.219:63229_solr").contains( - suggestion._get("operation/command/move-replica/targetNode", null))); - } - } - @SuppressWarnings({"unchecked"}) - public void testSysPropSuggestions() { - - Map m = (Map) loadFromResource("testSysPropSuggestions.json"); - - Map conf = (Map) getObjectByPath(m, false, "diagnostics/config"); - if(useNodeset){ - conf = (Map) Utils.fromJSONString("{" + - " 'cluster-preferences':[{" + - " 'minimize':'cores'," + - " 'precision':1}," + - " {" + - " 'maximize':'freedisk'," + - " 'precision':100}," + - " {" + - " 'minimize':'sysLoadAvg'," + - " 'precision':10}]," + - " 'cluster-policy':[" + - "{'replica':'<3'," + - " 'shard':'#EACH'," + - " nodeset: {'sysprop.zone':'east'}}, " + - "{'replica':'<3'," + - " 'shard':'#EACH'," + - " nodeset: {'sysprop.zone':'west'}} " + - " ]}"); - } - Policy policy = new Policy(conf); - SolrCloudManager cloudManagerFromDiagnostics = createCloudManagerFromDiagnostics(m); - Policy.Session session = policy.createSession(cloudManagerFromDiagnostics); - List violations = session.getViolations(); - for (Violation violation : violations) { - assertEquals(1.0d, violation.replicaCountDelta.doubleValue(), 0.0001); - } - assertEquals(2, violations.size()); - List suggestions = PolicyHelper.getSuggestions(new AutoScalingConfig(conf), cloudManagerFromDiagnostics); - assertEquals(2, suggestions.size()); - for (Suggester.SuggestionInfo suggestion : suggestions) { - assertTrue(ImmutableSet.of("127.0.0.1:63219_solr", "127.0.0.1:63229_solr").contains( - suggestion._get("operation/command/move-replica/targetNode", null))); - - } - } - - public void testSuggestionsRebalanceOnly() { - String conf = " {" + - " 'cluster-preferences':[{" + - " 'minimize':'cores'," + - " 'precision':1}," + - " {'maximize':'freedisk','precision':100}," + - " {'minimize':'sysLoadAvg','precision':10}]," + - " 'cluster-policy':[" + - "{'replica':'<5','shard':'#EACH','sysprop.zone':['east','west']}]}"; - if(useNodeset){ - conf = " {" + - " 'cluster-preferences':[{" + - " 'minimize':'cores'," + - " 'precision':1}," + - " {'maximize':'freedisk','precision':100}," + - " {'minimize':'sysLoadAvg','precision':10}]," + - " 'cluster-policy':[" + - "{'replica':'<5','shard':'#EACH', nodeset:{'sysprop.zone':['east','west']}}]}"; - - } - @SuppressWarnings({"unchecked"}) - Map m = (Map) loadFromResource("testSuggestionsRebalanceOnly.json"); - SolrCloudManager cloudManagerFromDiagnostics = createCloudManagerFromDiagnostics(m); - @SuppressWarnings({"unchecked"}) - AutoScalingConfig autoScalingConfig = new AutoScalingConfig((Map) Utils.fromJSONString(conf)); - List suggestions = PolicyHelper.getSuggestions(autoScalingConfig, cloudManagerFromDiagnostics); - - assertEquals(2, suggestions.size()); - assertEquals("improvement", suggestions.get(0)._get("type",null)); - assertEquals("127.0.0.1:63229_solr", suggestions.get(0)._get("operation/command/move-replica/targetNode", null)); - assertEquals("improvement", suggestions.get(1)._get( "type",null)); - assertEquals("127.0.0.1:63219_solr", suggestions.get(1)._get("operation/command/move-replica/targetNode", null)); - } - - public void testSuggestionsRebalance2() { - @SuppressWarnings({"unchecked"}) - Map m = (Map) loadFromResource("testSuggestionsRebalance2.json"); - SolrCloudManager cloudManagerFromDiagnostics = createCloudManagerFromDiagnostics(m); - - @SuppressWarnings({"unchecked"}) - AutoScalingConfig autoScalingConfig = new AutoScalingConfig((Map) getObjectByPath(m, false, "diagnostics/config")); - List suggestions = PolicyHelper.getSuggestions(autoScalingConfig, cloudManagerFromDiagnostics); - - assertEquals(3, suggestions.size()); - - for (Suggester.SuggestionInfo suggestion : suggestions) { - assertEquals("improvement", suggestion._get("type", null)); - assertEquals("10.0.0.79:8983_solr", suggestion._get("operation/command/move-replica/targetNode",null)); - } - - } - - public void testAddMissingReplica() { - @SuppressWarnings({"unchecked"}) - Map m = (Map) loadFromResource("testAddMissingReplica.json"); - SolrCloudManager cloudManagerFromDiagnostics = createCloudManagerFromDiagnostics(m); - @SuppressWarnings({"unchecked"}) - AutoScalingConfig autoScalingConfig = new AutoScalingConfig((Map) getObjectByPath(m, false, "diagnostics/config")); - - List suggestions = PolicyHelper.getSuggestions(autoScalingConfig, cloudManagerFromDiagnostics); - - assertEquals(1, suggestions.size()); - assertEquals("repair", suggestions.get(0)._get("type",null)); - assertEquals("add-replica", suggestions.get(0)._get("operation/command[0]/key",null)); - assertEquals("shard2", suggestions.get(0)._get("operation/command/add-replica/shard",null)); - assertEquals("NRT", suggestions.get(0)._get("operation/command/add-replica/type",null)); - - } - - public void testCreateCollectionWithEmptyPolicy() { - @SuppressWarnings({"rawtypes"}) - Map m = (Map) loadFromResource("testCreateCollectionWithEmptyPolicy.json"); - @SuppressWarnings({"unchecked"}) - SolrCloudManager cloudManagerFromDiagnostics = createCloudManagerFromDiagnostics(m); - AutoScalingConfig autoScalingConfig = new AutoScalingConfig(new HashMap<>()); - //POSITIONS : [shard1:1[NRT] @127.0.0.1:49469_solr, shard1:2[NRT] @127.0.0.1:49469_solr] - @SuppressWarnings({"unchecked"}) - List positions = PolicyHelper.getReplicaLocations("coll_new", autoScalingConfig, cloudManagerFromDiagnostics, - EMPTY_MAP, Collections.singletonList("shard1"), 2, 0, 0, null); - - List nodes = positions.stream().map(count -> count.node).collect(Collectors.toList()); - assertTrue(nodes.contains("127.0.0.1:49469_solr")); - assertTrue(nodes.contains("127.0.0.1:49470_solr")); - - - } - - public void testUnresolvedSuggestion() { - @SuppressWarnings({"unchecked"}) - Map m = (Map) loadFromResource("testUnresolvedSuggestion.json"); - - SolrCloudManager cloudManagerFromDiagnostics = createCloudManagerFromDiagnostics(m); - @SuppressWarnings({"unchecked"}) - List suggestions = PolicyHelper.getSuggestions(new AutoScalingConfig((Map) getObjectByPath(m, false, "diagnostics/config")) - , cloudManagerFromDiagnostics); - for (Suggester.SuggestionInfo suggestion : suggestions) { - assertEquals("unresolved_violation", suggestion._get("type", null)); - assertEquals("1.0", suggestion._getStr("violation/violation/delta", null)); - } - } - - - @Ignore("This takes too long to run. enable it for perf testing") - @SuppressWarnings({"unchecked"}) - public void testInfiniteLoop() { - Row.cacheStats.clear(); - Map m = (Map) loadFromResource("testInfiniteLoop.json"); - SolrCloudManager cloudManagerFromDiagnostics = createCloudManagerFromDiagnostics(m); - List suggestions = PolicyHelper.getSuggestions( - new AutoScalingConfig((Map) getObjectByPath(m, false, "diagnostics/config")) - , cloudManagerFromDiagnostics, 200, 1200, null); - - System.out.println(suggestions); - } - - @SuppressWarnings({"unchecked"}) - public void testAddTooManyPerPolicy() { - Map m = (Map) loadFromResource("testAddTooManyPerPolicy.json"); - SolrCloudManager cloudManagerFromDiagnostics = createCloudManagerFromDiagnostics(m); - AutoScalingConfig autoScalingConfig = new AutoScalingConfig((Map) getObjectByPath(m, false, "diagnostics/config")); - SolrException exp = expectThrows(SolrException.class, () -> PolicyHelper.getReplicaLocations("TooManyPerPolicy", autoScalingConfig, cloudManagerFromDiagnostics, - EMPTY_MAP, Collections.singletonList("shard1"), 1, 0, 0, null)); - assertTrue(exp.getMessage().contains("No node can satisfy the rules")); - - } - - public static Object loadFromResource(String file) { - try (InputStream is = TestPolicy2.class.getResourceAsStream("/solrj/solr/autoscaling/" + file)) { - return Utils.fromJSON(is, MAPOBJBUILDER); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - -} diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy2Old.java b/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy2Old.java deleted file mode 100644 index 8b411d65db8..00000000000 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy2Old.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.client.solrj.cloud.autoscaling; - -public class TestPolicy2Old extends TestPolicy2 { - public TestPolicy2Old(){ - super(); - useNodeset = false; - } - -} diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicyOld.java b/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicyOld.java deleted file mode 100644 index 44cdbf01dc8..00000000000 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicyOld.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.client.solrj.cloud.autoscaling; - -public class TestPolicyOld extends TestPolicy { - public TestPolicyOld(){ - super(); - useNodeset = false; - } -} diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrClientCacheTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrClientCacheTest.java index d35f4d928ed..73935256174 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrClientCacheTest.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrClientCacheTest.java @@ -31,7 +31,7 @@ import java.util.function.Function; import com.google.common.collect.ImmutableSet; import org.apache.http.NoHttpResponseException; import org.apache.solr.SolrTestCaseJ4; -import org.apache.solr.client.solrj.cloud.autoscaling.DelegatingClusterStateProvider; +import org.apache.solr.client.solrj.cloud.DelegatingClusterStateProvider; import org.apache.solr.client.solrj.request.UpdateRequest; import org.apache.solr.common.cloud.ClusterState; import org.apache.solr.common.cloud.DocCollection; @@ -157,7 +157,6 @@ public class CloudSolrClientCacheTest extends SolrTestCaseJ4 { private String coll1State = "{'gettingstarted':{\n" + " 'replicationFactor':'2',\n" + " 'router':{'name':'compositeId'},\n" + - " 'autoAddReplicas':'false',\n" + " 'shards':{\n" + " 'shard1':{\n" + " 'range':'80000000-ffffffff',\n" + diff --git a/solr/webapp/web/css/angular/menu.css b/solr/webapp/web/css/angular/menu.css index 08ab60d5b45..87b51695c7a 100644 --- a/solr/webapp/web/css/angular/menu.css +++ b/solr/webapp/web/css/angular/menu.css @@ -264,7 +264,6 @@ limitations under the License. #menu #collections.global p a { background-image: url( ../../img/ico/documents-stack.png ); } #menu #cores.global p a { background-image: url( ../../img/ico/databases.png ); } -#menu #cluster-suggestions.global p a { background-image: url( ../../img/ico/idea.png ); } #menu #cloud.global p a { background-image: url( ../../img/ico/network-cloud.png ); } #menu #cloud.global .tree a { background-image: url( ../../img/ico/folder-tree.png ); } diff --git a/solr/webapp/web/css/angular/suggestions.css b/solr/webapp/web/css/angular/suggestions.css deleted file mode 100644 index 6d9fa659970..00000000000 --- a/solr/webapp/web/css/angular/suggestions.css +++ /dev/null @@ -1,64 +0,0 @@ -/* - -Licensed to the Apache Software Foundation (ASF) under one or more -contributor license agreements. See the NOTICE file distributed with -this work for additional information regarding copyright ownership. -The ASF licenses this file to You under the Apache License, Version 2.0 -(the "License"); you may not use this file except in compliance with -the License. You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -*/ -#cluster-suggestions -.s-container{ - text-align:center; -} -#cluster-suggestions -.s-box1{ - background-image: url( ../../img/ico/run.png ); - background-color: transparent; - background-repeat: no-repeat; - border: none; - cursor: pointer; - vertical-align: middle; - display:inline-block; - width:20px; -} -#cluster-suggestions -.s-box2{ - display:inline-block; -} -#cluster-suggestions -.s-box3{ - display:inline-block; -} -#cluster-suggestions -.s-box4{ - display:inline-block; -} -#s-table { - border-collapse: collapse; - width: 60%; -} - -#s-table td, #customers th { - border: 1px solid #ddd; - padding: 8px; -} - -#cluster-suggestions #s-table tr:nth-child(even){background-color: #f2f2f2;} - -#cluster-suggestions #s-table th { - padding-top: 12px; - padding-bottom: 12px; - text-align: left; - background-color: #4CAF50; - color: white; -} diff --git a/solr/webapp/web/index.html b/solr/webapp/web/index.html index 1fde25f3c74..db50db7d355 100644 --- a/solr/webapp/web/index.html +++ b/solr/webapp/web/index.html @@ -43,7 +43,6 @@ limitations under the License. - @@ -90,7 +89,6 @@ limitations under the License. - @@ -173,7 +171,6 @@ limitations under the License.

  • Java Properties

  • Thread Dump

    • -

  • Suggestions

    • diff --git a/solr/webapp/web/js/angular/app.js b/solr/webapp/web/js/angular/app.js index 7e6150ce7cc..d7d0cef2ca8 100644 --- a/solr/webapp/web/js/angular/app.js +++ b/solr/webapp/web/js/angular/app.js @@ -114,10 +114,6 @@ solrAdminApp.config([ templateUrl: 'partials/java-properties.html', controller: 'JavaPropertiesController' }). - when('/~cluster-suggestions', { - templateUrl: 'partials/cluster_suggestions.html', - controller: 'ClusterSuggestionsController' - }). when('/:core/core-overview', { templateUrl: 'partials/core_overview.html', controller: 'CoreOverviewController' diff --git a/solr/webapp/web/js/angular/controllers/cloud.js b/solr/webapp/web/js/angular/controllers/cloud.js index 1327c6e0760..bf8fc336ee2 100644 --- a/solr/webapp/web/js/angular/controllers/cloud.js +++ b/solr/webapp/web/js/angular/controllers/cloud.js @@ -753,7 +753,6 @@ var graphSubController = function ($scope, Zookeeper) { pullReplicas: state[c].pullReplicas, replicationFactor: state[c].replicationFactor, router: state[c].router.name, - autoAddReplicas: state[c].autoAddReplicas, nrtReplicas: state[c].nrtReplicas, tlogReplicas: state[c].tlogReplicas, numShards: shards.length @@ -872,7 +871,6 @@ solrAdminApp.directive('graph', function(Constants) { tooltip = d.name + " {
    "; tooltip += "numShards: [" + d.data.numShards + "],
    "; tooltip += "router: [" + d.data.router + "],
    "; - tooltip += "autoAddReplicas: [" + d.data.autoAddReplicas + "],
    "; tooltip += "replicationFactor: [" + d.data.replicationFactor + "],
    "; tooltip += "nrtReplicas: [" + d.data.nrtReplicas + "],
    "; tooltip += "pullReplicas: [" + d.data.pullReplicas + "],
    "; diff --git a/solr/webapp/web/js/angular/controllers/cluster-suggestions.js b/solr/webapp/web/js/angular/controllers/cluster-suggestions.js deleted file mode 100644 index 01e1964e0b6..00000000000 --- a/solr/webapp/web/js/angular/controllers/cluster-suggestions.js +++ /dev/null @@ -1,62 +0,0 @@ -/* - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ -solrAdminApp.controller('ClusterSuggestionsController', -function($scope, $http, Constants) { - $scope.resetMenu("cluster-suggestion", Constants.IS_COLLECTION_PAGE); - $scope.data={}; - var dataArr =[]; - var dataJson = {}; - //function to display suggestion - $http({ - method: 'GET', - url: '/api/cluster/autoscaling/suggestions' - }).then(function successCallback(response) { - $scope.data = response.data; - $scope.parsedData = $scope.data.suggestions; - }, function errorCallback(response) { - }); - //function to perform operation - $scope.postdata = function (x) { - x.loading = true; - var path=x.operation.path; - var command=x.operation.command; - var fullPath='/api/'+path; - console.log(fullPath); - console.log(command); - $http.post(fullPath, JSON.stringify(command)).then(function (response) { - if (response.data) - console.log(response.data); - x.loading = false; - x.done = true; - x.run=true; - $scope.msg = "Command Submitted Successfully!"; - }, function (response) { - x.failed=true; - $scope.msg = "Service does not exist"; - $scope.statusval = response.status; - $scope.statustext = response.statusText; - $scope.headers = response.headers(); - }); - }; - $scope.showPopover = function() { - $scope.popup = true; - }; - - $scope.hidePopover = function () { - $scope.popup = false; - }; -}); diff --git a/solr/webapp/web/js/angular/controllers/collections.js b/solr/webapp/web/js/angular/controllers/collections.js index 733922843c4..2d1ce5c6127 100644 --- a/solr/webapp/web/js/angular/controllers/collections.js +++ b/solr/webapp/web/js/angular/controllers/collections.js @@ -101,8 +101,7 @@ solrAdminApp.controller('CollectionsController', routerName: "compositeId", numShards: 1, configName: "", - replicationFactor: 1, - autoAddReplicas: 'false' + replicationFactor: 1 }; }; @@ -151,8 +150,7 @@ solrAdminApp.controller('CollectionsController', "router.name": coll.routerName, numShards: coll.numShards, "collection.configName": coll.configName, - replicationFactor: coll.replicationFactor, - autoAddReplicas: coll.autoAddReplicas + replicationFactor: coll.replicationFactor }; if (coll.shards) params.shards = coll.shards; if (coll.routerField) params["router.field"] = coll.routerField; diff --git a/solr/webapp/web/partials/cluster_suggestions.html b/solr/webapp/web/partials/cluster_suggestions.html deleted file mode 100644 index 892b0bbc97f..00000000000 --- a/solr/webapp/web/partials/cluster_suggestions.html +++ /dev/null @@ -1,49 +0,0 @@ - - -
    -

    Cluster Suggestions

    -
    - - - - - - - - - - - - - - - - -
    TypeReasonDetailsAction
    NA
    {{ x.type }}{{ x.violation.clause }}node: {{ x.violation.node }}, collection : {{ x.violation.collection }}, shard : {{ x.violation.shard }}   -
    - -
    -
    -
    -
    -
    - -
    - diff --git a/solr/webapp/web/partials/collection_overview.html b/solr/webapp/web/partials/collection_overview.html index 75287d8320d..d6c70bb4461 100644 --- a/solr/webapp/web/partials/collection_overview.html +++ b/solr/webapp/web/partials/collection_overview.html @@ -32,9 +32,6 @@ limitations under the License.
    Replication factor:
    {{selectedCollection.replicationFactor}}
    -
    Auto-add replicas:
    -
    yes
    -
    Router name:
    {{selectedCollection.router.name}}
    diff --git a/solr/webapp/web/partials/collections.html b/solr/webapp/web/partials/collections.html index eb962f1411b..d3bbf56e926 100644 --- a/solr/webapp/web/partials/collections.html +++ b/solr/webapp/web/partials/collections.html @@ -55,13 +55,6 @@ limitations under the License.

    -

    - -

    -

    {{addMessage}} @@ -216,11 +209,6 @@ limitations under the License.

    {{collection.router.name}}
    -
  • -
    autoAddReplicas:
    -
    {{collection.autoAddReplicas}}
    -
    • -
  • aliases: