From 4d7bc9477144937335e997ad630c4b89f558ddc5 Mon Sep 17 00:00:00 2001 From: Andrzej Bialecki Date: Tue, 7 Mar 2017 22:00:38 +0100 Subject: [PATCH] SOLR-9858: Collect aggregated metrics from nodes and shard leaders in overseer. --- solr/CHANGES.txt | 4 + .../apache/solr/cloud/ElectionContext.java | 5 +- .../java/org/apache/solr/cloud/Overseer.java | 7 +- .../solr/cloud/OverseerNodePrioritizer.java | 2 +- .../solr/cloud/OverseerTaskProcessor.java | 6 +- .../org/apache/solr/cloud/ZkController.java | 2 +- .../org/apache/solr/core/CoreContainer.java | 30 +- .../org/apache/solr/core/JmxMonitoredMap.java | 9 +- .../java/org/apache/solr/core/SolrCore.java | 4 +- .../org/apache/solr/core/SolrInfoMBean.java | 4 +- .../org/apache/solr/core/SolrXmlConfig.java | 3 +- .../admin/MetricsCollectorHandler.java | 228 ++++++++++ .../solr/handler/admin/MetricsHandler.java | 2 +- .../apache/solr/metrics/AggregateMetric.java | 200 +++++++++ .../solr/metrics/SolrCoreMetricManager.java | 127 +++++- .../solr/metrics/SolrMetricManager.java | 325 ++++++++++++++- .../reporters/JmxObjectNameFactory.java | 6 +- .../reporters/solr/SolrClusterReporter.java | 277 +++++++++++++ .../metrics/reporters/solr/SolrReporter.java | 392 ++++++++++++++++++ .../reporters/solr/SolrShardReporter.java | 188 +++++++++ .../metrics/reporters/solr/package-info.java | 22 + .../java/org/apache/solr/update/PeerSync.java | 8 +- .../apache/solr/util/stats/MetricUtils.java | 261 ++++++++---- .../src/test-files/solr/solr-solrreporter.xml | 66 +++ .../apache/solr/cloud/TestCloudRecovery.java | 6 +- .../apache/solr/core/TestJmxMonitoredMap.java | 2 +- .../metrics/SolrCoreMetricManagerTest.java | 31 +- .../solr/metrics/SolrMetricManagerTest.java | 30 +- .../metrics/SolrMetricsIntegrationTest.java | 15 +- .../reporters/SolrJmxReporterTest.java | 13 +- .../solr/SolrCloudReportersTest.java | 163 ++++++++ .../reporters/solr/SolrShardReporterTest.java | 117 ++++++ .../solr/util/stats/MetricUtilsTest.java | 54 ++- .../solrj/impl/BinaryRequestWriter.java | 4 +- .../solr/client/solrj/io/SolrClientCache.java | 26 +- .../client/solrj/request/TestCoreAdmin.java | 4 +- 36 files changed, 2434 insertions(+), 209 deletions(-) create mode 100644 solr/core/src/java/org/apache/solr/handler/admin/MetricsCollectorHandler.java create mode 100644 solr/core/src/java/org/apache/solr/metrics/AggregateMetric.java create mode 100644 solr/core/src/java/org/apache/solr/metrics/reporters/solr/SolrClusterReporter.java create mode 100644 solr/core/src/java/org/apache/solr/metrics/reporters/solr/SolrReporter.java create mode 100644 solr/core/src/java/org/apache/solr/metrics/reporters/solr/SolrShardReporter.java create mode 100644 solr/core/src/java/org/apache/solr/metrics/reporters/solr/package-info.java create mode 100644 solr/core/src/test-files/solr/solr-solrreporter.xml create mode 100644 solr/core/src/test/org/apache/solr/metrics/reporters/solr/SolrCloudReportersTest.java create mode 100644 solr/core/src/test/org/apache/solr/metrics/reporters/solr/SolrShardReporterTest.java diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index dc974563c8e..0e7853503e3 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -50,6 +50,10 @@ Upgrading from Solr 6.x factors should be indexed in a separate field and combined with the query score using a function query. +New Features +---------------------- +* SOLR-9857, SOLR-9858: Collect aggregated metrics from nodes and shard leaders in overseer. (ab) + Bug Fixes ---------------------- * SOLR-9262: Connection and read timeouts are being ignored by UpdateShardHandler after SOLR-4509. diff --git a/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java b/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java index ff6fb3031c1..d3ad3224382 100644 --- a/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java +++ b/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java @@ -714,14 +714,13 @@ final class OverseerElectionContext extends ElectionContext { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); private final SolrZkClient zkClient; private Overseer overseer; - public static final String OVERSEER_ELECT = "/overseer_elect"; public OverseerElectionContext(SolrZkClient zkClient, Overseer overseer, final String zkNodeName) { - super(zkNodeName, OVERSEER_ELECT, OVERSEER_ELECT + "/leader", null, zkClient); + super(zkNodeName, Overseer.OVERSEER_ELECT, Overseer.OVERSEER_ELECT + "/leader", null, zkClient); this.overseer = overseer; this.zkClient = zkClient; try { - new ZkCmdExecutor(zkClient.getZkClientTimeout()).ensureExists(OVERSEER_ELECT, zkClient); + new ZkCmdExecutor(zkClient.getZkClientTimeout()).ensureExists(Overseer.OVERSEER_ELECT, zkClient); } catch (KeeperException e) { throw new SolrException(ErrorCode.SERVER_ERROR, e); } catch (InterruptedException e) { diff --git a/solr/core/src/java/org/apache/solr/cloud/Overseer.java b/solr/core/src/java/org/apache/solr/cloud/Overseer.java index 3a8aa3edfca..61f15fca6eb 100644 --- a/solr/core/src/java/org/apache/solr/cloud/Overseer.java +++ b/solr/core/src/java/org/apache/solr/cloud/Overseer.java @@ -65,7 +65,8 @@ public class Overseer implements Closeable { public static final int STATE_UPDATE_DELAY = 1500; // delay between cloud state updates public static final int NUM_RESPONSES_TO_STORE = 10000; - + public static final String OVERSEER_ELECT = "/overseer_elect"; + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); enum LeaderStatus {DONT_KNOW, NO, YES} @@ -281,7 +282,7 @@ public class Overseer implements Closeable { private void checkIfIamStillLeader() { if (zkController != null && zkController.getCoreContainer().isShutDown()) return;//shutting down no need to go further org.apache.zookeeper.data.Stat stat = new org.apache.zookeeper.data.Stat(); - String path = OverseerElectionContext.OVERSEER_ELECT + "/leader"; + String path = OVERSEER_ELECT + "/leader"; byte[] data; try { data = zkClient.getData(path, null, stat, true); @@ -394,7 +395,7 @@ public class Overseer implements Closeable { boolean success = true; try { ZkNodeProps props = ZkNodeProps.load(zkClient.getData( - OverseerElectionContext.OVERSEER_ELECT + "/leader", null, null, true)); + OVERSEER_ELECT + "/leader", null, null, true)); if (myId.equals(props.getStr("id"))) { return LeaderStatus.YES; } diff --git a/solr/core/src/java/org/apache/solr/cloud/OverseerNodePrioritizer.java b/solr/core/src/java/org/apache/solr/cloud/OverseerNodePrioritizer.java index 6512d260425..798eca39950 100644 --- a/solr/core/src/java/org/apache/solr/cloud/OverseerNodePrioritizer.java +++ b/solr/core/src/java/org/apache/solr/cloud/OverseerNodePrioritizer.java @@ -65,7 +65,7 @@ public class OverseerNodePrioritizer { String ldr = OverseerTaskProcessor.getLeaderNode(zk); if(overseerDesignates.contains(ldr)) return; log.info("prioritizing overseer nodes at {} overseer designates are {}", overseerId, overseerDesignates); - List electionNodes = OverseerTaskProcessor.getSortedElectionNodes(zk, OverseerElectionContext.OVERSEER_ELECT + LeaderElector.ELECTION_NODE); + List electionNodes = OverseerTaskProcessor.getSortedElectionNodes(zk, Overseer.OVERSEER_ELECT + LeaderElector.ELECTION_NODE); if(electionNodes.size()<2) return; log.info("sorted nodes {}", electionNodes); diff --git a/solr/core/src/java/org/apache/solr/cloud/OverseerTaskProcessor.java b/solr/core/src/java/org/apache/solr/cloud/OverseerTaskProcessor.java index ad533462646..bed71a63734 100644 --- a/solr/core/src/java/org/apache/solr/cloud/OverseerTaskProcessor.java +++ b/solr/core/src/java/org/apache/solr/cloud/OverseerTaskProcessor.java @@ -337,7 +337,7 @@ public class OverseerTaskProcessor implements Runnable, Closeable { public static List getSortedOverseerNodeNames(SolrZkClient zk) throws KeeperException, InterruptedException { List children = null; try { - children = zk.getChildren(OverseerElectionContext.OVERSEER_ELECT + LeaderElector.ELECTION_NODE, null, true); + children = zk.getChildren(Overseer.OVERSEER_ELECT + LeaderElector.ELECTION_NODE, null, true); } catch (Exception e) { log.warn("error ", e); return new ArrayList<>(); @@ -370,7 +370,7 @@ public class OverseerTaskProcessor implements Runnable, Closeable { public static String getLeaderId(SolrZkClient zkClient) throws KeeperException,InterruptedException{ byte[] data = null; try { - data = zkClient.getData(OverseerElectionContext.OVERSEER_ELECT + "/leader", null, new Stat(), true); + data = zkClient.getData(Overseer.OVERSEER_ELECT + "/leader", null, new Stat(), true); } catch (KeeperException.NoNodeException e) { return null; } @@ -384,7 +384,7 @@ public class OverseerTaskProcessor implements Runnable, Closeable { boolean success = true; try { ZkNodeProps props = ZkNodeProps.load(zkStateReader.getZkClient().getData( - OverseerElectionContext.OVERSEER_ELECT + "/leader", null, null, true)); + Overseer.OVERSEER_ELECT + "/leader", null, null, true)); if (myId.equals(props.getStr("id"))) { return LeaderStatus.YES; } diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkController.java b/solr/core/src/java/org/apache/solr/cloud/ZkController.java index c0837365bd5..333acd419d7 100644 --- a/solr/core/src/java/org/apache/solr/cloud/ZkController.java +++ b/solr/core/src/java/org/apache/solr/cloud/ZkController.java @@ -1715,7 +1715,7 @@ public class ZkController { //however delete it . This is possible when the last attempt at deleting the election node failed. if (electionNode.startsWith(getNodeName())) { try { - zkClient.delete(OverseerElectionContext.OVERSEER_ELECT + LeaderElector.ELECTION_NODE + "/" + electionNode, -1, true); + zkClient.delete(Overseer.OVERSEER_ELECT + LeaderElector.ELECTION_NODE + "/" + electionNode, -1, true); } catch (NoNodeException e) { //no problem } catch (InterruptedException e) { diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java index e3977d7796b..b9597ae985f 100644 --- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java +++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java @@ -69,6 +69,7 @@ import org.apache.solr.handler.admin.CollectionsHandler; import org.apache.solr.handler.admin.ConfigSetsHandler; import org.apache.solr.handler.admin.CoreAdminHandler; import org.apache.solr.handler.admin.InfoHandler; +import org.apache.solr.handler.admin.MetricsCollectorHandler; import org.apache.solr.handler.admin.MetricsHandler; import org.apache.solr.handler.admin.SecurityConfHandler; import org.apache.solr.handler.admin.SecurityConfHandlerLocal; @@ -177,6 +178,8 @@ public class CoreContainer { protected MetricsHandler metricsHandler; + protected MetricsCollectorHandler metricsCollectorHandler; + private enum CoreInitFailedAction { fromleader, none } /** @@ -511,15 +514,18 @@ public class CoreContainer { coreAdminHandler = createHandler(CORES_HANDLER_PATH, cfg.getCoreAdminHandlerClass(), CoreAdminHandler.class); configSetsHandler = createHandler(CONFIGSETS_HANDLER_PATH, cfg.getConfigSetsHandlerClass(), ConfigSetsHandler.class); metricsHandler = createHandler(METRICS_PATH, MetricsHandler.class.getName(), MetricsHandler.class); + metricsCollectorHandler = createHandler(MetricsCollectorHandler.HANDLER_PATH, MetricsCollectorHandler.class.getName(), MetricsCollectorHandler.class); + // may want to add some configuration here in the future + metricsCollectorHandler.init(null); containerHandlers.put(AUTHZ_PATH, securityConfHandler); securityConfHandler.initializeMetrics(metricManager, SolrInfoMBean.Group.node.toString(), AUTHZ_PATH); containerHandlers.put(AUTHC_PATH, securityConfHandler); if(pkiAuthenticationPlugin != null) containerHandlers.put(PKIAuthenticationPlugin.PATH, pkiAuthenticationPlugin.getRequestHandler()); - metricManager.loadReporters(cfg.getMetricReporterPlugins(), loader, SolrInfoMBean.Group.node); - metricManager.loadReporters(cfg.getMetricReporterPlugins(), loader, SolrInfoMBean.Group.jvm); - metricManager.loadReporters(cfg.getMetricReporterPlugins(), loader, SolrInfoMBean.Group.jetty); + metricManager.loadReporters(cfg.getMetricReporterPlugins(), loader, null, SolrInfoMBean.Group.node); + metricManager.loadReporters(cfg.getMetricReporterPlugins(), loader, null, SolrInfoMBean.Group.jvm); + metricManager.loadReporters(cfg.getMetricReporterPlugins(), loader, null, SolrInfoMBean.Group.jetty); coreConfigService = ConfigSetService.createConfigSetService(cfg, loader, zkSys.zkController); @@ -537,6 +543,10 @@ public class CoreContainer { metricManager.register(SolrMetricManager.getRegistryName(SolrInfoMBean.Group.node), unloadedCores, true, "unloaded",SolrInfoMBean.Category.CONTAINER.toString(), "cores"); + if (isZooKeeperAware()) { + metricManager.loadClusterReporters(cfg.getMetricReporterPlugins(), this); + } + // setup executor to load cores in parallel ExecutorService coreLoadExecutor = MetricUtils.instrumentedExecutorService( ExecutorUtil.newMDCAwareFixedThreadPool( @@ -660,10 +670,16 @@ public class CoreContainer { isShutDown = true; ExecutorUtil.shutdownAndAwaitTermination(coreContainerWorkExecutor); + if (metricManager != null) { + metricManager.closeReporters(SolrMetricManager.getRegistryName(SolrInfoMBean.Group.node)); + } if (isZooKeeperAware()) { cancelCoreRecoveries(); - zkSys.zkController.publishNodeAsDown(zkSys.zkController.getNodeName()); + zkSys.zkController.publishNodeAsDown(zkSys.zkController.getNodeName()); + if (metricManager != null) { + metricManager.closeReporters(SolrMetricManager.getRegistryName(SolrInfoMBean.Group.cluster)); + } } try { @@ -722,10 +738,6 @@ public class CoreContainer { } } - if (metricManager != null) { - metricManager.closeReporters(SolrMetricManager.getRegistryName(SolrInfoMBean.Group.node)); - } - // It should be safe to close the authorization plugin at this point. try { if(authorizationPlugin != null) { @@ -1232,7 +1244,7 @@ public class CoreContainer { try (SolrCore core = getCore(name)) { if (core != null) { String oldRegistryName = core.getCoreMetricManager().getRegistryName(); - String newRegistryName = SolrCoreMetricManager.createRegistryName(core.getCoreDescriptor().getCollectionName(), toName); + String newRegistryName = SolrCoreMetricManager.createRegistryName(core, toName); metricManager.swapRegistries(oldRegistryName, newRegistryName); registerCore(toName, core, true, false); SolrCore old = solrCores.remove(name); diff --git a/solr/core/src/java/org/apache/solr/core/JmxMonitoredMap.java b/solr/core/src/java/org/apache/solr/core/JmxMonitoredMap.java index b2a5c79e5cb..8bfa662d548 100644 --- a/solr/core/src/java/org/apache/solr/core/JmxMonitoredMap.java +++ b/solr/core/src/java/org/apache/solr/core/JmxMonitoredMap.java @@ -20,6 +20,7 @@ import javax.management.Attribute; import javax.management.AttributeList; import javax.management.AttributeNotFoundException; import javax.management.DynamicMBean; +import javax.management.InstanceNotFoundException; import javax.management.InvalidAttributeValueException; import javax.management.MBeanAttributeInfo; import javax.management.MBeanException; @@ -53,7 +54,6 @@ import org.apache.lucene.store.AlreadyClosedException; import org.apache.solr.common.SolrException; import org.apache.solr.common.util.NamedList; import org.apache.solr.core.SolrConfig.JmxConfiguration; -import org.apache.solr.metrics.SolrCoreMetricManager; import org.apache.solr.metrics.reporters.JmxObjectNameFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -93,9 +93,10 @@ public class JmxMonitoredMap extends private final String registryName; - public JmxMonitoredMap(String coreName, String coreHashCode, + public JmxMonitoredMap(String coreName, String coreHashCode, String registryName, final JmxConfiguration jmxConfig) { this.coreHashCode = coreHashCode; + this.registryName = registryName; jmxRootName = (null != jmxConfig.rootName ? jmxConfig.rootName : ("solr" + (null != coreName ? "/" + coreName : ""))); @@ -117,7 +118,6 @@ public class JmxMonitoredMap extends if (servers == null || servers.isEmpty()) { server = null; - registryName = null; nameFactory = null; log.debug("No JMX servers found, not exposing Solr information with JMX."); return; @@ -141,7 +141,6 @@ public class JmxMonitoredMap extends } server = newServer; } - registryName = SolrCoreMetricManager.createRegistryName(null, coreName); nameFactory = new JmxObjectNameFactory(REPORTER_NAME + coreHashCode, registryName); } @@ -166,6 +165,8 @@ public class JmxMonitoredMap extends for (ObjectName name : objectNames) { try { server.unregisterMBean(name); + } catch (InstanceNotFoundException ie) { + // ignore - someone else already deleted this one } catch (Exception e) { log.warn("Exception un-registering mbean {}", name, e); } diff --git a/solr/core/src/java/org/apache/solr/core/SolrCore.java b/solr/core/src/java/org/apache/solr/core/SolrCore.java index f22c4722222..13c3bdd8a1e 100644 --- a/solr/core/src/java/org/apache/solr/core/SolrCore.java +++ b/solr/core/src/java/org/apache/solr/core/SolrCore.java @@ -860,6 +860,7 @@ public final class SolrCore implements SolrInfoMBean, Closeable { this.configSetProperties = configSetProperties; // Initialize the metrics manager this.coreMetricManager = initCoreMetricManager(config); + this.coreMetricManager.loadReporters(); if (updateHandler == null) { directoryFactory = initDirectoryFactory(); @@ -1101,13 +1102,12 @@ public final class SolrCore implements SolrInfoMBean, Closeable { */ private SolrCoreMetricManager initCoreMetricManager(SolrConfig config) { SolrCoreMetricManager coreMetricManager = new SolrCoreMetricManager(this); - coreMetricManager.loadReporters(); return coreMetricManager; } private Map initInfoRegistry(String name, SolrConfig config) { if (config.jmxConfig.enabled) { - return new JmxMonitoredMap(name, String.valueOf(this.hashCode()), config.jmxConfig); + return new JmxMonitoredMap(name, coreMetricManager.getRegistryName(), String.valueOf(this.hashCode()), config.jmxConfig); } else { log.debug("JMX monitoring not detected for core: " + name); return new ConcurrentHashMap<>(); diff --git a/solr/core/src/java/org/apache/solr/core/SolrInfoMBean.java b/solr/core/src/java/org/apache/solr/core/SolrInfoMBean.java index bf77db4b7b4..63bdef0f7bc 100644 --- a/solr/core/src/java/org/apache/solr/core/SolrInfoMBean.java +++ b/solr/core/src/java/org/apache/solr/core/SolrInfoMBean.java @@ -36,9 +36,9 @@ public interface SolrInfoMBean { SEARCHER, REPLICATION, TLOG, INDEX, DIRECTORY, HTTP, OTHER } /** - * Top-level group of beans for a subsystem. + * Top-level group of beans or metrics for a subsystem. */ - enum Group { jvm, jetty, node, core } + enum Group { jvm, jetty, node, core, collection, shard, cluster, overseer } /** * Simple common usage name, e.g. BasicQueryHandler, diff --git a/solr/core/src/java/org/apache/solr/core/SolrXmlConfig.java b/solr/core/src/java/org/apache/solr/core/SolrXmlConfig.java index e41cd8d44ef..951d8d54ca7 100644 --- a/solr/core/src/java/org/apache/solr/core/SolrXmlConfig.java +++ b/solr/core/src/java/org/apache/solr/core/SolrXmlConfig.java @@ -451,7 +451,8 @@ public class SolrXmlConfig { return new PluginInfo[0]; PluginInfo[] configs = new PluginInfo[nodes.getLength()]; for (int i = 0; i < nodes.getLength(); i++) { - configs[i] = new PluginInfo(nodes.item(i), "SolrMetricReporter", true, true); + // we don't require class in order to support predefined replica and node reporter classes + configs[i] = new PluginInfo(nodes.item(i), "SolrMetricReporter", true, false); } return configs; } diff --git a/solr/core/src/java/org/apache/solr/handler/admin/MetricsCollectorHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/MetricsCollectorHandler.java new file mode 100644 index 00000000000..de39a615606 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/handler/admin/MetricsCollectorHandler.java @@ -0,0 +1,228 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.handler.admin; + +import java.io.IOException; +import java.lang.invoke.MethodHandles; +import java.util.HashMap; +import java.util.Map; + +import com.codahale.metrics.MetricRegistry; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.SolrInputDocument; +import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.common.params.SolrParams; +import org.apache.solr.common.util.ContentStream; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.core.CoreContainer; +import org.apache.solr.handler.loader.ContentStreamLoader; +import org.apache.solr.handler.RequestHandlerBase; +import org.apache.solr.handler.loader.CSVLoader; +import org.apache.solr.handler.loader.JavabinLoader; +import org.apache.solr.handler.loader.JsonLoader; +import org.apache.solr.handler.loader.XMLLoader; +import org.apache.solr.metrics.AggregateMetric; +import org.apache.solr.metrics.SolrMetricManager; +import org.apache.solr.metrics.reporters.solr.SolrReporter; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.response.SolrQueryResponse; +import org.apache.solr.update.AddUpdateCommand; +import org.apache.solr.update.CommitUpdateCommand; +import org.apache.solr.update.DeleteUpdateCommand; +import org.apache.solr.update.MergeIndexesCommand; +import org.apache.solr.update.RollbackUpdateCommand; +import org.apache.solr.update.processor.UpdateRequestProcessor; +import org.apache.solr.util.stats.MetricUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Handler to collect and aggregate metric reports. Each report indicates the target registry where + * metrics values should be collected and aggregated. Metrics with the same names are + * aggregated using {@link AggregateMetric} instances, which track the source of updates and + * their count, as well as providing simple statistics over collected values. + * + * Each report consists of {@link SolrInputDocument}-s that are expected to contain + * the following fields: + *
    + *
  • {@link SolrReporter#GROUP_ID} - (required) specifies target registry name where metrics will be grouped.
    • + *
  • {@link SolrReporter#REPORTER_ID} - (required) id of the reporter that sent this update. This can be eg. + * node name or replica name or other id that uniquely identifies the source of metrics values.
    • + *
  • {@link MetricUtils#METRIC_NAME} - (required) metric name (in the source registry)
    • + *
  • {@link SolrReporter#LABEL_ID} - (optional) label to prepend to metric names in the target registry.
    • + *
  • {@link SolrReporter#REGISTRY_ID} - (optional) name of the source registry.
    • + *
+ * Remaining fields are assumed to be single-valued, and to contain metric attributes and their values. Example: + * 
+ *   <doc>
+ *     <field name="_group_">solr.core.collection1.shard1.leader</field>
+ *     <field name="_reporter_">core_node3</field>
+ *     <field name="metric">INDEX.merge.errors</field>
+ *     <field name="value">0</field>
+ *   </doc>
+ *
+ */ +public class MetricsCollectorHandler extends RequestHandlerBase { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + public static final String HANDLER_PATH = "/admin/metrics/collector"; + + private final CoreContainer coreContainer; + private final SolrMetricManager metricManager; + private final Map loaders = new HashMap<>(); + private SolrParams params; + + public MetricsCollectorHandler(final CoreContainer coreContainer) { + this.coreContainer = coreContainer; + this.metricManager = coreContainer.getMetricManager(); + + } + + @Override + public void init(NamedList initArgs) { + super.init(initArgs); + if (initArgs != null) { + params = SolrParams.toSolrParams(initArgs); + } else { + params = new ModifiableSolrParams(); + } + loaders.put("application/xml", new XMLLoader().init(params) ); + loaders.put("application/json", new JsonLoader().init(params) ); + loaders.put("application/csv", new CSVLoader().init(params) ); + loaders.put("application/javabin", new JavabinLoader().init(params) ); + loaders.put("text/csv", loaders.get("application/csv") ); + loaders.put("text/xml", loaders.get("application/xml") ); + loaders.put("text/json", loaders.get("application/json")); + } + + @Override + public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception { + if (coreContainer == null || coreContainer.isShutDown()) { + // silently drop request + return; + } + //log.info("#### " + req.toString()); + if (req.getContentStreams() == null) { // no content + return; + } + for (ContentStream cs : req.getContentStreams()) { + if (cs.getContentType() == null) { + log.warn("Missing content type - ignoring"); + continue; + } + ContentStreamLoader loader = loaders.get(cs.getContentType()); + if (loader == null) { + throw new SolrException(SolrException.ErrorCode.UNSUPPORTED_MEDIA_TYPE, "Unsupported content type for stream: " + cs.getSourceInfo() + ", contentType=" + cs.getContentType()); + } + loader.load(req, rsp, cs, new MetricUpdateProcessor(metricManager)); + } + } + + @Override + public String getDescription() { + return "Handler for collecting and aggregating metric reports."; + } + + private static class MetricUpdateProcessor extends UpdateRequestProcessor { + private final SolrMetricManager metricManager; + + public MetricUpdateProcessor(SolrMetricManager metricManager) { + super(null); + this.metricManager = metricManager; + } + + @Override + public void processAdd(AddUpdateCommand cmd) throws IOException { + SolrInputDocument doc = cmd.solrDoc; + if (doc == null) { + return; + } + String metricName = (String)doc.getFieldValue(MetricUtils.METRIC_NAME); + if (metricName == null) { + log.warn("Missing " + MetricUtils.METRIC_NAME + " field in document, skipping: " + doc); + return; + } + doc.remove(MetricUtils.METRIC_NAME); + // XXX we could modify keys by using this original registry name + doc.remove(SolrReporter.REGISTRY_ID); + String groupId = (String)doc.getFieldValue(SolrReporter.GROUP_ID); + if (groupId == null) { + log.warn("Missing " + SolrReporter.GROUP_ID + " field in document, skipping: " + doc); + return; + } + doc.remove(SolrReporter.GROUP_ID); + String reporterId = (String)doc.getFieldValue(SolrReporter.REPORTER_ID); + if (reporterId == null) { + log.warn("Missing " + SolrReporter.REPORTER_ID + " field in document, skipping: " + doc); + return; + } + doc.remove(SolrReporter.REPORTER_ID); + String labelId = (String)doc.getFieldValue(SolrReporter.LABEL_ID); + doc.remove(SolrReporter.LABEL_ID); + doc.forEach(f -> { + String key = MetricRegistry.name(labelId, metricName, f.getName()); + MetricRegistry registry = metricManager.registry(groupId); + AggregateMetric metric = getOrRegister(registry, key, new AggregateMetric()); + Object o = f.getFirstValue(); + if (o != null) { + metric.set(reporterId, o); + } else { + // remove missing values + metric.clear(reporterId); + } + }); + } + + private AggregateMetric getOrRegister(MetricRegistry registry, String name, AggregateMetric add) { + AggregateMetric existing = (AggregateMetric)registry.getMetrics().get(name); + if (existing != null) { + return existing; + } + try { + registry.register(name, add); + return add; + } catch (IllegalArgumentException e) { + // someone added before us + existing = (AggregateMetric)registry.getMetrics().get(name); + if (existing == null) { // now, that is weird... + throw new IllegalArgumentException("Inconsistent metric status, " + name); + } + return existing; + } + } + + @Override + public void processDelete(DeleteUpdateCommand cmd) throws IOException { + throw new UnsupportedOperationException("processDelete"); + } + + @Override + public void processMergeIndexes(MergeIndexesCommand cmd) throws IOException { + throw new UnsupportedOperationException("processMergeIndexes"); + } + + @Override + public void processCommit(CommitUpdateCommand cmd) throws IOException { + throw new UnsupportedOperationException("processCommit"); + } + + @Override + public void processRollback(RollbackUpdateCommand cmd) throws IOException { + throw new UnsupportedOperationException("processRollback"); + } + } +} diff --git a/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java index 385317bb159..b53c818ec7e 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java @@ -79,7 +79,7 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionName NamedList response = new NamedList(); for (String registryName : requestedRegistries) { MetricRegistry registry = metricManager.registry(registryName); - response.add(registryName, MetricUtils.toNamedList(registry, metricFilters, mustMatchFilter)); + response.add(registryName, MetricUtils.toNamedList(registry, metricFilters, mustMatchFilter, false, false, null)); } rsp.getValues().add("metrics", response); } diff --git a/solr/core/src/java/org/apache/solr/metrics/AggregateMetric.java b/solr/core/src/java/org/apache/solr/metrics/AggregateMetric.java new file mode 100644 index 00000000000..babc99d44cb --- /dev/null +++ b/solr/core/src/java/org/apache/solr/metrics/AggregateMetric.java @@ -0,0 +1,200 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.metrics; + +import java.util.Collections; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; + +import com.codahale.metrics.Metric; + +/** + * This class is used for keeping several partial named values and providing useful statistics over them. + */ +public class AggregateMetric implements Metric { + + /** + * Simple class to represent current value and how many times it was set. + */ + public static class Update { + public Object value; + public final AtomicInteger updateCount = new AtomicInteger(); + + public Update(Object value) { + update(value); + } + + public void update(Object value) { + this.value = value; + updateCount.incrementAndGet(); + } + + @Override + public String toString() { + return "Update{" + + "value=" + value + + ", updateCount=" + updateCount + + '}'; + } + } + + private final Map values = new ConcurrentHashMap<>(); + + public void set(String name, Object value) { + final Update existing = values.get(name); + if (existing == null) { + final Update created = new Update(value); + final Update raced = values.putIfAbsent(name, created); + if (raced != null) { + raced.update(value); + } + } else { + existing.update(value); + } + } + + public void clear(String name) { + values.remove(name); + } + + public void clear() { + values.clear(); + } + + public int size() { + return values.size(); + } + + public boolean isEmpty() { + return values.isEmpty(); + } + + public Map getValues() { + return Collections.unmodifiableMap(values); + } + + // --------- stats --------- + public double getMax() { + if (values.isEmpty()) { + return 0; + } + Double res = null; + for (Update u : values.values()) { + if (!(u.value instanceof Number)) { + continue; + } + Number n = (Number)u.value; + if (res == null) { + res = n.doubleValue(); + continue; + } + if (n.doubleValue() > res) { + res = n.doubleValue(); + } + } + return res; + } + + public double getMin() { + if (values.isEmpty()) { + return 0; + } + Double res = null; + for (Update u : values.values()) { + if (!(u.value instanceof Number)) { + continue; + } + Number n = (Number)u.value; + if (res == null) { + res = n.doubleValue(); + continue; + } + if (n.doubleValue() < res) { + res = n.doubleValue(); + } + } + return res; + } + + public double getMean() { + if (values.isEmpty()) { + return 0; + } + double total = 0; + for (Update u : values.values()) { + if (!(u.value instanceof Number)) { + continue; + } + Number n = (Number)u.value; + total += n.doubleValue(); + } + return total / values.size(); + } + + public double getStdDev() { + int size = values.size(); + if (size < 2) { + return 0; + } + final double mean = getMean(); + double sum = 0; + int count = 0; + for (Update u : values.values()) { + if (!(u.value instanceof Number)) { + continue; + } + count++; + Number n = (Number)u.value; + final double diff = n.doubleValue() - mean; + sum += diff * diff; + } + if (count < 2) { + return 0; + } + final double variance = sum / (count - 1); + return Math.sqrt(variance); + } + + public double getSum() { + if (values.isEmpty()) { + return 0; + } + double res = 0; + for (Update u : values.values()) { + if (!(u.value instanceof Number)) { + continue; + } + Number n = (Number)u.value; + res += n.doubleValue(); + } + return res; + } + + @Override + public String toString() { + return "AggregateMetric{" + + "size=" + size() + + ", max=" + getMax() + + ", min=" + getMin() + + ", mean=" + getMean() + + ", stddev=" + getStdDev() + + ", sum=" + getSum() + + ", values=" + values + + '}'; + } +} diff --git a/solr/core/src/java/org/apache/solr/metrics/SolrCoreMetricManager.java b/solr/core/src/java/org/apache/solr/metrics/SolrCoreMetricManager.java index eb5b68715b3..43f35352ebf 100644 --- a/solr/core/src/java/org/apache/solr/metrics/SolrCoreMetricManager.java +++ b/solr/core/src/java/org/apache/solr/metrics/SolrCoreMetricManager.java @@ -20,6 +20,7 @@ import java.io.Closeable; import java.io.IOException; import java.lang.invoke.MethodHandles; +import org.apache.solr.cloud.CloudDescriptor; import org.apache.solr.core.NodeConfig; import org.apache.solr.core.PluginInfo; import org.apache.solr.core.SolrCore; @@ -36,8 +37,14 @@ public class SolrCoreMetricManager implements Closeable { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); private final SolrCore core; + private final String tag; private final SolrMetricManager metricManager; private String registryName; + private String collectionName; + private String shardName; + private String replicaName; + private String leaderRegistryName; + private boolean cloudMode; /** * Constructs a metric manager. @@ -46,8 +53,26 @@ public class SolrCoreMetricManager implements Closeable { */ public SolrCoreMetricManager(SolrCore core) { this.core = core; + this.tag = String.valueOf(core.hashCode()); this.metricManager = core.getCoreDescriptor().getCoreContainer().getMetricManager(); - registryName = createRegistryName(core.getCoreDescriptor().getCollectionName(), core.getName()); + initCloudMode(); + registryName = createRegistryName(cloudMode, collectionName, shardName, replicaName, core.getName()); + leaderRegistryName = createLeaderRegistryName(cloudMode, collectionName, shardName); + } + + private void initCloudMode() { + CloudDescriptor cd = core.getCoreDescriptor().getCloudDescriptor(); + if (cd != null) { + cloudMode = true; + collectionName = core.getCoreDescriptor().getCollectionName(); + shardName = cd.getShardId(); + //replicaName = cd.getCoreNodeName(); + String coreName = core.getName(); + replicaName = parseReplicaName(collectionName, coreName); + if (replicaName == null) { + replicaName = cd.getCoreNodeName(); + } + } } /** @@ -57,7 +82,11 @@ public class SolrCoreMetricManager implements Closeable { public void loadReporters() { NodeConfig nodeConfig = core.getCoreDescriptor().getCoreContainer().getConfig(); PluginInfo[] pluginInfos = nodeConfig.getMetricReporterPlugins(); - metricManager.loadReporters(pluginInfos, core.getResourceLoader(), SolrInfoMBean.Group.core, registryName); + metricManager.loadReporters(pluginInfos, core.getResourceLoader(), tag, + SolrInfoMBean.Group.core, registryName); + if (cloudMode) { + metricManager.loadShardReporters(pluginInfos, core); + } } /** @@ -67,12 +96,18 @@ public class SolrCoreMetricManager implements Closeable { */ public void afterCoreSetName() { String oldRegistryName = registryName; - registryName = createRegistryName(core.getCoreDescriptor().getCollectionName(), core.getName()); + String oldLeaderRegistryName = leaderRegistryName; + initCloudMode(); + registryName = createRegistryName(cloudMode, collectionName, shardName, replicaName, core.getName()); + leaderRegistryName = createLeaderRegistryName(cloudMode, collectionName, shardName); if (oldRegistryName.equals(registryName)) { return; } // close old reporters - metricManager.closeReporters(oldRegistryName); + metricManager.closeReporters(oldRegistryName, tag); + if (oldLeaderRegistryName != null) { + metricManager.closeReporters(oldLeaderRegistryName, tag); + } // load reporters again, using the new core name loadReporters(); } @@ -96,7 +131,7 @@ public class SolrCoreMetricManager implements Closeable { */ @Override public void close() throws IOException { - metricManager.closeReporters(getRegistryName()); + metricManager.closeReporters(getRegistryName(), tag); } public SolrCore getCore() { @@ -104,7 +139,7 @@ public class SolrCoreMetricManager implements Closeable { } /** - * Retrieves the metric registry name of the manager. + * Metric registry name of the manager. * * In order to make it easier for reporting tools to aggregate metrics from * different cores that logically belong to a single collection we convert the @@ -124,22 +159,74 @@ public class SolrCoreMetricManager implements Closeable { return registryName; } - public static String createRegistryName(String collectionName, String coreName) { - if (collectionName == null || (collectionName != null && !coreName.startsWith(collectionName + "_"))) { - // single core, or unknown naming scheme + /** + * Metric registry name for leader metrics. This is null if not in cloud mode. + * @return metric registry name for leader metrics + */ + public String getLeaderRegistryName() { + return leaderRegistryName; + } + + /** + * Return a tag specific to this instance. + */ + public String getTag() { + return tag; + } + + public static String createRegistryName(boolean cloud, String collectionName, String shardName, String replicaName, String coreName) { + if (cloud) { // build registry name from logical names + return SolrMetricManager.getRegistryName(SolrInfoMBean.Group.core, collectionName, shardName, replicaName); + } else { return SolrMetricManager.getRegistryName(SolrInfoMBean.Group.core, coreName); } - // split "collection1_shard1_1_replica1" into parts - String str = coreName.substring(collectionName.length() + 1); - String shard; - String replica = null; - int pos = str.lastIndexOf("_replica"); - if (pos == -1) { // ?? no _replicaN part ?? - shard = str; - } else { - shard = str.substring(0, pos); - replica = str.substring(pos + 1); + } + + /** + * This method is used by {@link org.apache.solr.core.CoreContainer#rename(String, String)}. + * @param aCore existing core with old name + * @param coreName new name + * @return new registry name + */ + public static String createRegistryName(SolrCore aCore, String coreName) { + CloudDescriptor cd = aCore.getCoreDescriptor().getCloudDescriptor(); + String replicaName = null; + if (cd != null) { + replicaName = parseReplicaName(cd.getCollectionName(), coreName); + } + return createRegistryName( + cd != null, + cd != null ? cd.getCollectionName() : null, + cd != null ? cd.getShardId() : null, + replicaName, + coreName + ); + } + + public static String parseReplicaName(String collectionName, String coreName) { + if (collectionName == null || !coreName.startsWith(collectionName)) { + return null; + } else { + // split "collection1_shard1_1_replica1" into parts + if (coreName.length() > collectionName.length()) { + String str = coreName.substring(collectionName.length() + 1); + int pos = str.lastIndexOf("_replica"); + if (pos == -1) { // ?? no _replicaN part ?? + return str; + } else { + return str.substring(pos + 1); + } + } else { + return null; + } + } + } + + public static String createLeaderRegistryName(boolean cloud, String collectionName, String shardName) { + if (cloud) { + return SolrMetricManager.getRegistryName(SolrInfoMBean.Group.collection, collectionName, shardName, "leader"); + } else { + return null; } - return SolrMetricManager.getRegistryName(SolrInfoMBean.Group.core, collectionName, shard, replica); } } diff --git a/solr/core/src/java/org/apache/solr/metrics/SolrMetricManager.java b/solr/core/src/java/org/apache/solr/metrics/SolrMetricManager.java index cac53891696..3a4c3fed770 100644 --- a/solr/core/src/java/org/apache/solr/metrics/SolrMetricManager.java +++ b/solr/core/src/java/org/apache/solr/metrics/SolrMetricManager.java @@ -18,9 +18,13 @@ package org.apache.solr.metrics; import java.io.IOException; import java.lang.invoke.MethodHandles; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; @@ -29,6 +33,9 @@ import java.util.concurrent.ConcurrentMap; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; +import java.util.regex.Pattern; +import java.util.regex.PatternSyntaxException; +import java.util.stream.Collectors; import com.codahale.metrics.Counter; import com.codahale.metrics.Histogram; @@ -39,9 +46,14 @@ import com.codahale.metrics.MetricRegistry; import com.codahale.metrics.MetricSet; import com.codahale.metrics.SharedMetricRegistries; import com.codahale.metrics.Timer; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.core.CoreContainer; import org.apache.solr.core.PluginInfo; +import org.apache.solr.core.SolrCore; import org.apache.solr.core.SolrInfoMBean; import org.apache.solr.core.SolrResourceLoader; +import org.apache.solr.metrics.reporters.solr.SolrClusterReporter; +import org.apache.solr.metrics.reporters.solr.SolrShardReporter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -87,27 +99,39 @@ public class SolrMetricManager { private final Lock reportersLock = new ReentrantLock(); private final Lock swapLock = new ReentrantLock(); + public static final int DEFAULT_CLOUD_REPORTER_PERIOD = 60; + public SolrMetricManager() { } /** * An implementation of {@link MetricFilter} that selects metrics - * with names that start with a prefix. + * with names that start with one of prefixes. */ public static class PrefixFilter implements MetricFilter { - private final String[] prefixes; + private final Set prefixes = new HashSet<>(); private final Set matched = new HashSet<>(); private boolean allMatch = false; /** - * Create a filter that uses the provided prefix. + * Create a filter that uses the provided prefixes. * @param prefixes prefixes to use, must not be null. If empty then any * name will match, if not empty then match on any prefix will * succeed (logical OR). */ public PrefixFilter(String... prefixes) { Objects.requireNonNull(prefixes); - this.prefixes = prefixes; - if (prefixes.length == 0) { + if (prefixes.length > 0) { + this.prefixes.addAll(Arrays.asList(prefixes)); + } + if (this.prefixes.isEmpty()) { + allMatch = true; + } + } + + public PrefixFilter(Collection prefixes) { + Objects.requireNonNull(prefixes); + this.prefixes.addAll(prefixes); + if (this.prefixes.isEmpty()) { allMatch = true; } } @@ -141,6 +165,85 @@ public class SolrMetricManager { public void reset() { matched.clear(); } + + @Override + public String toString() { + return "PrefixFilter{" + + "prefixes=" + prefixes + + '}'; + } + } + + /** + * An implementation of {@link MetricFilter} that selects metrics + * with names that match regular expression patterns. + */ + public static class RegexFilter implements MetricFilter { + private final Set compiledPatterns = new HashSet<>(); + private final Set matched = new HashSet<>(); + private boolean allMatch = false; + + /** + * Create a filter that uses the provided prefix. + * @param patterns regex patterns to use, must not be null. If empty then any + * name will match, if not empty then match on any pattern will + * succeed (logical OR). + */ + public RegexFilter(String... patterns) throws PatternSyntaxException { + this(patterns != null ? Arrays.asList(patterns) : Collections.emptyList()); + } + + public RegexFilter(Collection patterns) throws PatternSyntaxException { + Objects.requireNonNull(patterns); + if (patterns.isEmpty()) { + allMatch = true; + return; + } + patterns.forEach(p -> { + Pattern pattern = Pattern.compile(p); + compiledPatterns.add(pattern); + }); + if (patterns.isEmpty()) { + allMatch = true; + } + } + + @Override + public boolean matches(String name, Metric metric) { + if (allMatch) { + matched.add(name); + return true; + } + for (Pattern p : compiledPatterns) { + if (p.matcher(name).matches()) { + matched.add(name); + return true; + } + } + return false; + } + + /** + * Return the set of names that matched this filter. + * @return matching names + */ + public Set getMatched() { + return Collections.unmodifiableSet(matched); + } + + /** + * Clear the set of names that matched. + */ + public void reset() { + matched.clear(); + } + + @Override + public String toString() { + return "RegexFilter{" + + "compiledPatterns=" + compiledPatterns + + '}'; + } } /** @@ -150,7 +253,40 @@ public class SolrMetricManager { Set set = new HashSet<>(); set.addAll(registries.keySet()); set.addAll(SharedMetricRegistries.names()); - return Collections.unmodifiableSet(set); + return set; + } + + /** + * Return set of existing registry names that match a regex pattern + * @param patterns regex patterns. NOTE: users need to make sure that patterns that + * don't start with a wildcard use the full registry name starting with + * {@link #REGISTRY_NAME_PREFIX} + * @return set of existing registry names where at least one pattern matched. + */ + public Set registryNames(String... patterns) throws PatternSyntaxException { + if (patterns == null || patterns.length == 0) { + return registryNames(); + } + List compiled = new ArrayList<>(); + for (String pattern : patterns) { + compiled.add(Pattern.compile(pattern)); + } + return registryNames((Pattern[])compiled.toArray(new Pattern[compiled.size()])); + } + + public Set registryNames(Pattern... patterns) { + Set allNames = registryNames(); + if (patterns == null || patterns.length == 0) { + return allNames; + } + return allNames.stream().filter(s -> { + for (Pattern p : patterns) { + if (p.matcher(s).matches()) { + return true; + } + } + return false; + }).collect(Collectors.toSet()); } /** @@ -209,7 +345,7 @@ public class SolrMetricManager { */ public void removeRegistry(String registry) { // close any reporters for this registry first - closeReporters(registry); + closeReporters(registry, null); // make sure we use a name with prefix, with overrides registry = overridableRegistryName(registry); if (isSharedRegistry(registry)) { @@ -490,10 +626,12 @@ public class SolrMetricManager { * the list. If both attributes are present then only "group" attribute will be processed. * @param pluginInfos plugin configurations * @param loader resource loader + * @param tag optional tag for the reporters, to distinguish reporters logically created for different parent + * component instances. * @param group selected group, not null * @param registryNames optional child registry name elements */ - public void loadReporters(PluginInfo[] pluginInfos, SolrResourceLoader loader, SolrInfoMBean.Group group, String... registryNames) { + public void loadReporters(PluginInfo[] pluginInfos, SolrResourceLoader loader, String tag, SolrInfoMBean.Group group, String... registryNames) { if (pluginInfos == null || pluginInfos.length == 0) { return; } @@ -533,7 +671,7 @@ public class SolrMetricManager { } } try { - loadReporter(registryName, loader, info); + loadReporter(registryName, loader, info, tag); } catch (Exception e) { log.warn("Error loading metrics reporter, plugin info: " + info, e); } @@ -545,9 +683,12 @@ public class SolrMetricManager { * @param registry reporter is associated with this registry * @param loader loader to use when creating an instance of the reporter * @param pluginInfo plugin configuration. Plugin "name" and "class" attributes are required. + * @param tag optional tag for the reporter, to distinguish reporters logically created for different parent + * component instances. + * @return instance of newly created and registered reporter * @throws Exception if any argument is missing or invalid */ - public void loadReporter(String registry, SolrResourceLoader loader, PluginInfo pluginInfo) throws Exception { + public SolrMetricReporter loadReporter(String registry, SolrResourceLoader loader, PluginInfo pluginInfo, String tag) throws Exception { if (registry == null || pluginInfo == null || pluginInfo.name == null || pluginInfo.className == null) { throw new IllegalArgumentException("loadReporter called with missing arguments: " + "registry=" + registry + ", loader=" + loader + ", pluginInfo=" + pluginInfo); @@ -558,14 +699,19 @@ public class SolrMetricManager { pluginInfo.className, SolrMetricReporter.class, new String[0], - new Class[] { SolrMetricManager.class, String.class }, - new Object[] { this, registry } + new Class[]{SolrMetricManager.class, String.class}, + new Object[]{this, registry} ); try { reporter.init(pluginInfo); } catch (IllegalStateException e) { throw new IllegalArgumentException("reporter init failed: " + pluginInfo, e); } + registerReporter(registry, pluginInfo.name, tag, reporter); + return reporter; + } + + private void registerReporter(String registry, String name, String tag, SolrMetricReporter reporter) throws Exception { try { if (!reportersLock.tryLock(10, TimeUnit.SECONDS)) { throw new Exception("Could not obtain lock to modify reporters registry: " + registry); @@ -579,12 +725,15 @@ public class SolrMetricManager { perRegistry = new HashMap<>(); reporters.put(registry, perRegistry); } - SolrMetricReporter oldReporter = perRegistry.get(pluginInfo.name); + if (tag != null && !tag.isEmpty()) { + name = name + "@" + tag; + } + SolrMetricReporter oldReporter = perRegistry.get(name); if (oldReporter != null) { // close it - log.info("Replacing existing reporter '" + pluginInfo.name + "' in registry '" + registry + "': " + oldReporter.toString()); + log.info("Replacing existing reporter '" + name + "' in registry '" + registry + "': " + oldReporter.toString()); oldReporter.close(); } - perRegistry.put(pluginInfo.name, reporter); + perRegistry.put(name, reporter); } finally { reportersLock.unlock(); @@ -595,9 +744,11 @@ public class SolrMetricManager { * Close and unregister a named {@link SolrMetricReporter} for a registry. * @param registry registry name * @param name reporter name + * @param tag optional tag for the reporter, to distinguish reporters logically created for different parent + * component instances. * @return true if a named reporter existed and was closed. */ - public boolean closeReporter(String registry, String name) { + public boolean closeReporter(String registry, String name, String tag) { // make sure we use a name with prefix, with overrides registry = overridableRegistryName(registry); try { @@ -614,6 +765,9 @@ public class SolrMetricManager { if (perRegistry == null) { return false; } + if (tag != null && !tag.isEmpty()) { + name = name + "@" + tag; + } SolrMetricReporter reporter = perRegistry.remove(name); if (reporter == null) { return false; @@ -635,6 +789,17 @@ public class SolrMetricManager { * @return names of closed reporters */ public Set closeReporters(String registry) { + return closeReporters(registry, null); + } + + /** + * Close and unregister all {@link SolrMetricReporter}-s for a registry. + * @param registry registry name + * @param tag optional tag for the reporter, to distinguish reporters logically created for different parent + * component instances. + * @return names of closed reporters + */ + public Set closeReporters(String registry, String tag) { // make sure we use a name with prefix, with overrides registry = overridableRegistryName(registry); try { @@ -646,18 +811,28 @@ public class SolrMetricManager { log.warn("Interrupted while trying to obtain lock to modify reporters registry: " + registry); return Collections.emptySet(); } - log.info("Closing metric reporters for: " + registry); + log.info("Closing metric reporters for registry=" + registry + ", tag=" + tag); try { - Map perRegistry = reporters.remove(registry); + Map perRegistry = reporters.get(registry); if (perRegistry != null) { - for (SolrMetricReporter reporter : perRegistry.values()) { + Set names = new HashSet<>(perRegistry.keySet()); + Set removed = new HashSet<>(); + names.forEach(name -> { + if (tag != null && !tag.isEmpty() && !name.endsWith("@" + tag)) { + return; + } + SolrMetricReporter reporter = perRegistry.remove(name); try { reporter.close(); } catch (IOException ioe) { log.warn("Exception closing reporter " + reporter, ioe); } + removed.add(name); + }); + if (removed.size() == names.size()) { + reporters.remove(registry); } - return perRegistry.keySet(); + return removed; } else { return Collections.emptySet(); } @@ -695,4 +870,114 @@ public class SolrMetricManager { reportersLock.unlock(); } } + + private List prepareCloudPlugins(PluginInfo[] pluginInfos, String group, String className, + Map defaultAttributes, + Map defaultInitArgs, + PluginInfo defaultPlugin) { + List result = new ArrayList<>(); + if (pluginInfos == null) { + pluginInfos = new PluginInfo[0]; + } + for (PluginInfo info : pluginInfos) { + String groupAttr = info.attributes.get("group"); + if (!group.equals(groupAttr)) { + continue; + } + info = preparePlugin(info, className, defaultAttributes, defaultInitArgs); + if (info != null) { + result.add(info); + } + } + if (result.isEmpty() && defaultPlugin != null) { + defaultPlugin = preparePlugin(defaultPlugin, className, defaultAttributes, defaultInitArgs); + if (defaultPlugin != null) { + result.add(defaultPlugin); + } + } + return result; + } + + private PluginInfo preparePlugin(PluginInfo info, String className, Map defaultAttributes, + Map defaultInitArgs) { + if (info == null) { + return null; + } + String classNameAttr = info.attributes.get("class"); + if (className != null) { + if (classNameAttr != null && !className.equals(classNameAttr)) { + log.warn("Conflicting class name attributes, expected " + className + " but was " + classNameAttr + ", skipping " + info); + return null; + } + } + + Map attrs = new HashMap<>(info.attributes); + defaultAttributes.forEach((k, v) -> { + if (!attrs.containsKey(k)) { + attrs.put(k, v); + } + }); + attrs.put("class", className); + Map initArgs = new HashMap<>(); + if (info.initArgs != null) { + initArgs.putAll(info.initArgs.asMap(10)); + } + defaultInitArgs.forEach((k, v) -> { + if (!initArgs.containsKey(k)) { + initArgs.put(k, v); + } + }); + return new PluginInfo(info.type, attrs, new NamedList(initArgs), null); + } + + public void loadShardReporters(PluginInfo[] pluginInfos, SolrCore core) { + // don't load for non-cloud cores + if (core.getCoreDescriptor().getCloudDescriptor() == null) { + return; + } + // prepare default plugin if none present in the config + Map attrs = new HashMap<>(); + attrs.put("name", "shardDefault"); + attrs.put("group", SolrInfoMBean.Group.shard.toString()); + Map initArgs = new HashMap<>(); + initArgs.put("period", DEFAULT_CLOUD_REPORTER_PERIOD); + + String registryName = core.getCoreMetricManager().getRegistryName(); + // collect infos and normalize + List infos = prepareCloudPlugins(pluginInfos, SolrInfoMBean.Group.shard.toString(), SolrShardReporter.class.getName(), + attrs, initArgs, null); + for (PluginInfo info : infos) { + try { + SolrMetricReporter reporter = loadReporter(registryName, core.getResourceLoader(), info, + String.valueOf(core.hashCode())); + ((SolrShardReporter)reporter).setCore(core); + } catch (Exception e) { + log.warn("Could not load shard reporter, pluginInfo=" + info, e); + } + } + } + + public void loadClusterReporters(PluginInfo[] pluginInfos, CoreContainer cc) { + // don't load for non-cloud instances + if (!cc.isZooKeeperAware()) { + return; + } + Map attrs = new HashMap<>(); + attrs.put("name", "clusterDefault"); + attrs.put("group", SolrInfoMBean.Group.cluster.toString()); + Map initArgs = new HashMap<>(); + initArgs.put("period", DEFAULT_CLOUD_REPORTER_PERIOD); + List infos = prepareCloudPlugins(pluginInfos, SolrInfoMBean.Group.cluster.toString(), SolrClusterReporter.class.getName(), + attrs, initArgs, null); + String registryName = getRegistryName(SolrInfoMBean.Group.cluster); + for (PluginInfo info : infos) { + try { + SolrMetricReporter reporter = loadReporter(registryName, cc.getResourceLoader(), info, null); + ((SolrClusterReporter)reporter).setCoreContainer(cc); + } catch (Exception e) { + log.warn("Could not load node reporter, pluginInfo=" + info, e); + } + } + } + } diff --git a/solr/core/src/java/org/apache/solr/metrics/reporters/JmxObjectNameFactory.java b/solr/core/src/java/org/apache/solr/metrics/reporters/JmxObjectNameFactory.java index 4df5257ccba..1f5b4f01513 100644 --- a/solr/core/src/java/org/apache/solr/metrics/reporters/JmxObjectNameFactory.java +++ b/solr/core/src/java/org/apache/solr/metrics/reporters/JmxObjectNameFactory.java @@ -41,9 +41,9 @@ public class JmxObjectNameFactory implements ObjectNameFactory { * @param additionalProperties additional properties as key, value pairs. */ public JmxObjectNameFactory(String reporterName, String domain, String... additionalProperties) { - this.reporterName = reporterName; + this.reporterName = reporterName.replaceAll(":", "_"); this.domain = domain; - this.subdomains = domain.split("\\."); + this.subdomains = domain.replaceAll(":", "_").split("\\."); if (additionalProperties != null && (additionalProperties.length % 2) != 0) { throw new IllegalArgumentException("additionalProperties length must be even: " + Arrays.toString(additionalProperties)); } @@ -83,7 +83,7 @@ public class JmxObjectNameFactory implements ObjectNameFactory { } sb.append(','); // separate from other properties } else { - sb.append(currentDomain); + sb.append(currentDomain.replaceAll(":", "_")); sb.append(':'); } } else { diff --git a/solr/core/src/java/org/apache/solr/metrics/reporters/solr/SolrClusterReporter.java b/solr/core/src/java/org/apache/solr/metrics/reporters/solr/SolrClusterReporter.java new file mode 100644 index 00000000000..846e805cf68 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/metrics/reporters/solr/SolrClusterReporter.java @@ -0,0 +1,277 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.metrics.reporters.solr; + +import java.io.IOException; +import java.lang.invoke.MethodHandles; +import java.net.MalformedURLException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.function.Supplier; + +import org.apache.http.client.HttpClient; +import org.apache.solr.cloud.Overseer; +import org.apache.solr.cloud.ZkController; +import org.apache.solr.common.cloud.SolrZkClient; +import org.apache.solr.common.cloud.ZkNodeProps; +import org.apache.solr.core.CoreContainer; +import org.apache.solr.core.SolrInfoMBean; +import org.apache.solr.handler.admin.MetricsCollectorHandler; +import org.apache.solr.metrics.SolrMetricManager; +import org.apache.solr.metrics.SolrMetricReporter; +import org.apache.zookeeper.KeeperException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * This reporter sends selected metrics from local registries to {@link Overseer}. + *

The following configuration properties are supported:

+ *
    + *
  • handler - (optional str) handler path where reports are sent. Default is + * {@link MetricsCollectorHandler#HANDLER_PATH}.
    • + *
  • period - (optional int) how often reports are sent, in seconds. Default is 60. Setting this + * to 0 disables the reporter.
    • + *
  • report - (optional multiple lst) report configuration(s), see below.
    • + *
+ * Each report configuration consist of the following properties: + *
    + *
  • registry - (required str) regex pattern matching source registries (see {@link SolrMetricManager#registryNames(String...)}), + * may contain capture groups.
    • + *
  • group - (required str) target registry name where metrics will be grouped. This can be a regex pattern that + * contains back-references to capture groups collected by registry pattern
    • + *
  • label - (optional str) optional prefix to prepend to metric names, may contain back-references to + * capture groups collected by registry pattern
    • + *
  • filter - (optional multiple str) regex expression(s) matching selected metrics to be reported.
    • + *
+ * NOTE: this reporter uses predefined "overseer" group, and it's always created even if explicit configuration + * is missing. Default configuration uses report specifications from {@link #DEFAULT_REPORTS}. + *

Example configuration:

+ * 
+ *       <reporter name="test" group="overseer">
+ *         <str name="handler">/admin/metrics/collector</str>
+ *         <int name="period">11</int>
+ *         <lst name="report">
+ *           <str name="group">overseer</str>
+ *           <str name="label">jvm</str>
+ *           <str name="registry">solr\.jvm</str>
+ *           <str name="filter">memory\.total\..*</str>
+ *           <str name="filter">memory\.heap\..*</str>
+ *           <str name="filter">os\.SystemLoadAverage</str>
+ *           <str name="filter">threads\.count</str>
+ *         </lst>
+ *         <lst name="report">
+ *           <str name="group">overseer</str>
+ *           <str name="label">leader.$1</str>
+ *           <str name="registry">solr\.core\.(.*)\.leader</str>
+ *           <str name="filter">UPDATE\./update/.*</str>
+ *         </lst>
+ *       </reporter>
+ *
+ * + */ +public class SolrClusterReporter extends SolrMetricReporter { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + public static final String CLUSTER_GROUP = SolrMetricManager.overridableRegistryName(SolrInfoMBean.Group.cluster.toString()); + + public static final List DEFAULT_REPORTS = new ArrayList() {{ + add(new SolrReporter.Report(CLUSTER_GROUP, "jetty", + SolrMetricManager.overridableRegistryName(SolrInfoMBean.Group.jetty.toString()), + Collections.emptySet())); // all metrics + add(new SolrReporter.Report(CLUSTER_GROUP, "jvm", + SolrMetricManager.overridableRegistryName(SolrInfoMBean.Group.jvm.toString()), + new HashSet() {{ + add("memory\\.total\\..*"); + add("memory\\.heap\\..*"); + add("os\\.SystemLoadAverage"); + add("os\\.FreePhysicalMemorySize"); + add("os\\.FreeSwapSpaceSize"); + add("os\\.OpenFileDescriptorCount"); + add("threads\\.count"); + }})); // all metrics + // XXX anything interesting here? + //add(new SolrReporter.Specification(OVERSEER_GROUP, "node", SolrMetricManager.overridableRegistryName(SolrInfoMBean.Group.node.toString()), + // Collections.emptySet())); // all metrics + add(new SolrReporter.Report(CLUSTER_GROUP, "leader.$1", "solr\\.collection\\.(.*)\\.leader", + new HashSet(){{ + add("UPDATE\\./update/.*"); + add("QUERY\\./select.*"); + add("INDEX\\..*"); + add("TLOG\\..*"); + }})); + }}; + + private String handler = MetricsCollectorHandler.HANDLER_PATH; + private int period = SolrMetricManager.DEFAULT_CLOUD_REPORTER_PERIOD; + private List reports = new ArrayList<>(); + + private SolrReporter reporter; + + /** + * Create a reporter for metrics managed in a named registry. + * + * @param metricManager metric manager + * @param registryName this is ignored + */ + public SolrClusterReporter(SolrMetricManager metricManager, String registryName) { + super(metricManager, registryName); + } + + public void setHandler(String handler) { + this.handler = handler; + } + + public void setPeriod(int period) { + this.period = period; + } + + public void setReport(List reportConfig) { + if (reportConfig == null || reportConfig.isEmpty()) { + return; + } + reportConfig.forEach(map -> { + SolrReporter.Report r = SolrReporter.Report.fromMap(map); + if (r != null) { + reports.add(r); + } + }); + } + + // for unit tests + int getPeriod() { + return period; + } + + List getReports() { + return reports; + } + + @Override + protected void validate() throws IllegalStateException { + if (period < 1) { + log.info("Turning off node reporter, period=" + period); + } + if (reports.isEmpty()) { // set defaults + reports = DEFAULT_REPORTS; + } + } + + @Override + public void close() throws IOException { + if (reporter != null) { + reporter.close();; + } + } + + public void setCoreContainer(CoreContainer cc) { + if (reporter != null) { + reporter.close();; + } + // start reporter only in cloud mode + if (!cc.isZooKeeperAware()) { + log.warn("Not ZK-aware, not starting..."); + return; + } + if (period < 1) { // don't start it + return; + } + HttpClient httpClient = cc.getUpdateShardHandler().getHttpClient(); + ZkController zk = cc.getZkController(); + String reporterId = zk.getNodeName(); + reporter = SolrReporter.Builder.forReports(metricManager, reports) + .convertRatesTo(TimeUnit.SECONDS) + .convertDurationsTo(TimeUnit.MILLISECONDS) + .withHandler(handler) + .withReporterId(reporterId) + .cloudClient(false) // we want to send reports specifically to a selected leader instance + .skipAggregateValues(true) // we don't want to transport details of aggregates + .skipHistograms(true) // we don't want to transport histograms + .build(httpClient, new OverseerUrlSupplier(zk)); + + reporter.start(period, TimeUnit.SECONDS); + } + + // TODO: fix this when there is an elegant way to retrieve URL of a node that runs Overseer leader. + // package visibility for unit tests + static class OverseerUrlSupplier implements Supplier { + private static final long DEFAULT_INTERVAL = 30000000; // 30s + private ZkController zk; + private String lastKnownUrl = null; + private long lastCheckTime = 0; + private long interval = DEFAULT_INTERVAL; + + OverseerUrlSupplier(ZkController zk) { + this.zk = zk; + } + + @Override + public String get() { + if (zk == null) { + return null; + } + // primitive caching for lastKnownUrl + long now = System.nanoTime(); + if (lastKnownUrl != null && (now - lastCheckTime) < interval) { + return lastKnownUrl; + } + if (!zk.isConnected()) { + return lastKnownUrl; + } + lastCheckTime = now; + SolrZkClient zkClient = zk.getZkClient(); + ZkNodeProps props; + try { + props = ZkNodeProps.load(zkClient.getData( + Overseer.OVERSEER_ELECT + "/leader", null, null, true)); + } catch (KeeperException e) { + log.warn("Could not obtain overseer's address, skipping.", e); + return lastKnownUrl; + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return lastKnownUrl; + } + if (props == null) { + return lastKnownUrl; + } + String oid = props.getStr("id"); + if (oid == null) { + return lastKnownUrl; + } + String[] ids = oid.split("-"); + if (ids.length != 3) { // unknown format + log.warn("Unknown format of leader id, skipping: " + oid); + return lastKnownUrl; + } + // convert nodeName back to URL + String url = zk.getZkStateReader().getBaseUrlForNodeName(ids[1]); + // check that it's parseable + try { + new java.net.URL(url); + } catch (MalformedURLException mue) { + log.warn("Malformed Overseer's leader URL: url", mue); + return lastKnownUrl; + } + lastKnownUrl = url; + return url; + } + } + +} diff --git a/solr/core/src/java/org/apache/solr/metrics/reporters/solr/SolrReporter.java b/solr/core/src/java/org/apache/solr/metrics/reporters/solr/SolrReporter.java new file mode 100644 index 00000000000..e9b8c3dc2c7 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/metrics/reporters/solr/SolrReporter.java @@ -0,0 +1,392 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.metrics.reporters.solr; + +import java.lang.invoke.MethodHandles; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.SortedMap; +import java.util.concurrent.TimeUnit; +import java.util.function.Supplier; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.regex.PatternSyntaxException; + +import com.codahale.metrics.Counter; +import com.codahale.metrics.Gauge; +import com.codahale.metrics.Histogram; +import com.codahale.metrics.Meter; +import com.codahale.metrics.MetricFilter; +import com.codahale.metrics.ScheduledReporter; +import com.codahale.metrics.Timer; +import org.apache.http.client.HttpClient; +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.io.SolrClientCache; +import org.apache.solr.client.solrj.request.UpdateRequest; +import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.common.params.SolrParams; +import org.apache.solr.handler.admin.MetricsCollectorHandler; +import org.apache.solr.metrics.SolrMetricManager; +import org.apache.solr.util.stats.MetricUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Implementation of {@link ScheduledReporter} that reports metrics from selected registries and sends + * them periodically as update requests to a selected Solr collection and to a configured handler. + */ +public class SolrReporter extends ScheduledReporter { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + public static final String REGISTRY_ID = "_registry_"; + public static final String REPORTER_ID = "_reporter_"; + public static final String GROUP_ID = "_group_"; + public static final String LABEL_ID = "_label_"; + + + /** + * Specification of what registries and what metrics to send. + */ + public static final class Report { + public String groupPattern; + public String labelPattern; + public String registryPattern; + public Set metricFilters = new HashSet<>(); + + /** + * Create a report specification + * @param groupPattern logical group for these metrics. This is used in {@link MetricsCollectorHandler} + * to select the target registry for metrics to aggregate. Must not be null or empty. + * It may contain back-references to capture groups from {@code registryPattern} + * @param labelPattern name of this group of metrics. This is used in {@link MetricsCollectorHandler} + * to prefix metric names. May be null or empty. It may contain back-references + * to capture groups from {@code registryPattern}. + * @param registryPattern pattern for selecting matching registries, see {@link SolrMetricManager#registryNames(String...)} + * @param metricFilters patterns for selecting matching metrics, see {@link org.apache.solr.metrics.SolrMetricManager.RegexFilter} + */ + public Report(String groupPattern, String labelPattern, String registryPattern, Collection metricFilters) { + this.groupPattern = groupPattern; + this.labelPattern = labelPattern; + this.registryPattern = registryPattern; + if (metricFilters != null) { + this.metricFilters.addAll(metricFilters); + } + } + + public static Report fromMap(Map map) { + String groupPattern = (String)map.get("group"); + String labelPattern = (String)map.get("label"); + String registryPattern = (String)map.get("registry"); + Object oFilters = map.get("filter"); + Collection metricFilters = Collections.emptyList(); + if (oFilters != null) { + if (oFilters instanceof String) { + metricFilters = Collections.singletonList((String)oFilters); + } else if (oFilters instanceof Collection) { + metricFilters = (Collection)oFilters; + } else { + log.warn("Invalid report filters, ignoring: " + oFilters); + } + } + if (groupPattern == null || registryPattern == null) { + log.warn("Invalid report configuration, group and registry required!: " + map); + return null; + } + return new Report(groupPattern, labelPattern, registryPattern, metricFilters); + } + } + + public static class Builder { + private final SolrMetricManager metricManager; + private final List reports; + private String reporterId; + private TimeUnit rateUnit; + private TimeUnit durationUnit; + private String handler; + private boolean skipHistograms; + private boolean skipAggregateValues; + private boolean cloudClient; + private SolrParams params; + + /** + * Create a builder for SolrReporter. + * @param metricManager metric manager that is the source of metrics + * @param reports report definitions + * @return builder + */ + public static Builder forReports(SolrMetricManager metricManager, List reports) { + return new Builder(metricManager, reports); + } + + private Builder(SolrMetricManager metricManager, List reports) { + this.metricManager = metricManager; + this.reports = reports; + this.rateUnit = TimeUnit.SECONDS; + this.durationUnit = TimeUnit.MILLISECONDS; + this.skipHistograms = false; + this.skipAggregateValues = false; + this.cloudClient = false; + this.params = null; + } + + /** + * Additional {@link SolrParams} to add to every request. + * @param params additional params + * @return {@code this} + */ + public Builder withSolrParams(SolrParams params) { + this.params = params; + return this; + } + /** + * If true then use {@link org.apache.solr.client.solrj.impl.CloudSolrClient} for communication. + * Default is false. + * @param cloudClient use CloudSolrClient when true, {@link org.apache.solr.client.solrj.impl.HttpSolrClient} otherwise. + * @return {@code this} + */ + public Builder cloudClient(boolean cloudClient) { + this.cloudClient = cloudClient; + return this; + } + + /** + * Histograms are difficult / impossible to aggregate, so it may not be + * worth to report them. + * @param skipHistograms when true then skip histograms from reports + * @return {@code this} + */ + public Builder skipHistograms(boolean skipHistograms) { + this.skipHistograms = skipHistograms; + return this; + } + + /** + * Individual values from {@link org.apache.solr.metrics.AggregateMetric} may not be worth to report. + * @param skipAggregateValues when tru then skip reporting individual values from the metric + * @return {@code this} + */ + public Builder skipAggregateValues(boolean skipAggregateValues) { + this.skipAggregateValues = skipAggregateValues; + return this; + } + + /** + * Handler name to use at the remote end. + * + * @param handler handler name, eg. "/admin/metricsCollector" + * @return {@code this} + */ + public Builder withHandler(String handler) { + this.handler = handler; + return this; + } + + /** + * Use this id to identify metrics from this instance. + * + * @param reporterId reporter id + * @return {@code this} + */ + public Builder withReporterId(String reporterId) { + this.reporterId = reporterId; + return this; + } + + /** + * Convert rates to the given time unit. + * + * @param rateUnit a unit of time + * @return {@code this} + */ + public Builder convertRatesTo(TimeUnit rateUnit) { + this.rateUnit = rateUnit; + return this; + } + + /** + * Convert durations to the given time unit. + * + * @param durationUnit a unit of time + * @return {@code this} + */ + public Builder convertDurationsTo(TimeUnit durationUnit) { + this.durationUnit = durationUnit; + return this; + } + + /** + * Build it. + * @param client an instance of {@link HttpClient} to be used for making calls. + * @param urlProvider function that returns the base URL of Solr instance to target. May return + * null to indicate that reporting should be skipped. Note: this + * function will be called every time just before report is sent. + * @return configured instance of reporter + */ + public SolrReporter build(HttpClient client, Supplier urlProvider) { + return new SolrReporter(client, urlProvider, metricManager, reports, handler, reporterId, rateUnit, durationUnit, + params, skipHistograms, skipAggregateValues, cloudClient); + } + + } + + private String reporterId; + private String handler; + private Supplier urlProvider; + private SolrClientCache clientCache; + private List compiledReports; + private SolrMetricManager metricManager; + private boolean skipHistograms; + private boolean skipAggregateValues; + private boolean cloudClient; + private ModifiableSolrParams params; + private Map metadata; + + private static final class CompiledReport { + String group; + String label; + Pattern registryPattern; + MetricFilter filter; + + CompiledReport(Report report) throws PatternSyntaxException { + this.group = report.groupPattern; + this.label = report.labelPattern; + this.registryPattern = Pattern.compile(report.registryPattern); + this.filter = new SolrMetricManager.RegexFilter(report.metricFilters); + } + + @Override + public String toString() { + return "CompiledReport{" + + "group='" + group + '\'' + + ", label='" + label + '\'' + + ", registryPattern=" + registryPattern + + ", filter=" + filter + + '}'; + } + } + + public SolrReporter(HttpClient httpClient, Supplier urlProvider, SolrMetricManager metricManager, + List metrics, String handler, + String reporterId, TimeUnit rateUnit, TimeUnit durationUnit, + SolrParams params, boolean skipHistograms, boolean skipAggregateValues, boolean cloudClient) { + super(null, "solr-reporter", MetricFilter.ALL, rateUnit, durationUnit); + this.metricManager = metricManager; + this.urlProvider = urlProvider; + this.reporterId = reporterId; + if (handler == null) { + handler = MetricsCollectorHandler.HANDLER_PATH; + } + this.handler = handler; + this.clientCache = new SolrClientCache(httpClient); + this.compiledReports = new ArrayList<>(); + metrics.forEach(report -> { + MetricFilter filter = new SolrMetricManager.RegexFilter(report.metricFilters); + try { + CompiledReport cs = new CompiledReport(report); + compiledReports.add(cs); + } catch (PatternSyntaxException e) { + log.warn("Skipping report with invalid registryPattern: " + report.registryPattern, e); + } + }); + this.skipHistograms = skipHistograms; + this.skipAggregateValues = skipAggregateValues; + this.cloudClient = cloudClient; + this.params = new ModifiableSolrParams(); + this.params.set(REPORTER_ID, reporterId); + // allow overrides to take precedence + if (params != null) { + this.params.add(params); + } + metadata = new HashMap<>(); + metadata.put(REPORTER_ID, reporterId); + } + + @Override + public void close() { + clientCache.close(); + super.close(); + } + + @Override + public void report() { + String url = urlProvider.get(); + // if null then suppress reporting + if (url == null) { + return; + } + + SolrClient solr; + if (cloudClient) { + solr = clientCache.getCloudSolrClient(url); + } else { + solr = clientCache.getHttpSolrClient(url); + } + UpdateRequest req = new UpdateRequest(handler); + req.setParams(params); + compiledReports.forEach(report -> { + Set registryNames = metricManager.registryNames(report.registryPattern); + registryNames.forEach(registryName -> { + String label = report.label; + if (label != null && label.indexOf('$') != -1) { + // label with back-references + Matcher m = report.registryPattern.matcher(registryName); + label = m.replaceFirst(label); + } + final String effectiveLabel = label; + String group = report.group; + if (group.indexOf('$') != -1) { + // group with back-references + Matcher m = report.registryPattern.matcher(registryName); + group = m.replaceFirst(group); + } + final String effectiveGroup = group; + MetricUtils.toSolrInputDocuments(metricManager.registry(registryName), Collections.singletonList(report.filter), MetricFilter.ALL, + skipHistograms, skipAggregateValues, metadata, doc -> { + doc.setField(REGISTRY_ID, registryName); + doc.setField(GROUP_ID, effectiveGroup); + if (effectiveLabel != null) { + doc.setField(LABEL_ID, effectiveLabel); + } + req.add(doc); + }); + }); + }); + + // if no docs added then don't send a report + if (req.getDocuments() == null || req.getDocuments().isEmpty()) { + return; + } + try { + //log.info("%%% sending to " + url + ": " + req.getParams()); + solr.request(req); + } catch (Exception e) { + log.debug("Error sending metric report", e.toString()); + } + + } + + @Override + public void report(SortedMap gauges, SortedMap counters, SortedMap histograms, SortedMap meters, SortedMap timers) { + // no-op - we do all the work in report() + } +} \ No newline at end of file diff --git a/solr/core/src/java/org/apache/solr/metrics/reporters/solr/SolrShardReporter.java b/solr/core/src/java/org/apache/solr/metrics/reporters/solr/SolrShardReporter.java new file mode 100644 index 00000000000..2b202749504 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/metrics/reporters/solr/SolrShardReporter.java @@ -0,0 +1,188 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.metrics.reporters.solr; + +import java.io.IOException; +import java.lang.invoke.MethodHandles; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.function.Supplier; + +import org.apache.solr.cloud.CloudDescriptor; +import org.apache.solr.common.cloud.ClusterState; +import org.apache.solr.common.cloud.DocCollection; +import org.apache.solr.common.cloud.Replica; +import org.apache.solr.core.SolrCore; +import org.apache.solr.handler.admin.MetricsCollectorHandler; +import org.apache.solr.metrics.SolrMetricManager; +import org.apache.solr.metrics.SolrMetricReporter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * This class reports selected metrics from replicas to shard leader. + *

The following configuration properties are supported:

+ *
    + *
  • handler - (optional str) handler path where reports are sent. Default is + * {@link MetricsCollectorHandler#HANDLER_PATH}.
    • + *
  • period - (optional int) how often reports are sent, in seconds. Default is 60. Setting this + * to 0 disables the reporter.
    • + *
  • filter - (optional multiple str) regex expression(s) matching selected metrics to be reported.
    • + *
+ * NOTE: this reporter uses predefined "replica" group, and it's always created even if explicit configuration + * is missing. Default configuration uses filters defined in {@link #DEFAULT_FILTERS}. + *

Example configuration:

+ * 
+ *    <reporter name="test" group="replica">
+ *      <int name="period">11</int>
+ *      <str name="filter">UPDATE\./update/.*requests</str>
+ *      <str name="filter">QUERY\./select.*requests</str>
+ *    </reporter>
+ *
+ */ +public class SolrShardReporter extends SolrMetricReporter { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + public static final List DEFAULT_FILTERS = new ArrayList(){{ + add("TLOG.*"); + add("REPLICATION.*"); + add("INDEX.flush.*"); + add("INDEX.merge.major.*"); + add("UPDATE\\./update/.*requests"); + add("QUERY\\./select.*requests"); + }}; + + private String handler = MetricsCollectorHandler.HANDLER_PATH; + private int period = SolrMetricManager.DEFAULT_CLOUD_REPORTER_PERIOD; + private List filters = new ArrayList<>(); + + private SolrReporter reporter; + + /** + * Create a reporter for metrics managed in a named registry. + * + * @param metricManager metric manager + * @param registryName registry to use, one of registries managed by + * {@link SolrMetricManager} + */ + public SolrShardReporter(SolrMetricManager metricManager, String registryName) { + super(metricManager, registryName); + } + + public void setHandler(String handler) { + this.handler = handler; + } + + public void setPeriod(int period) { + this.period = period; + } + + public void setFilter(List filterConfig) { + if (filterConfig == null || filterConfig.isEmpty()) { + return; + } + filters = filterConfig; + } + + // for unit tests + int getPeriod() { + return period; + } + + @Override + protected void validate() throws IllegalStateException { + if (period < 1) { + log.info("Turning off shard reporter, period=" + period); + } + if (filters.isEmpty()) { + filters = DEFAULT_FILTERS; + } + // start in inform(...) only when core is available + } + + @Override + public void close() throws IOException { + if (reporter != null) { + reporter.close(); + } + } + + public void setCore(SolrCore core) { + if (reporter != null) { + reporter.close(); + } + if (core.getCoreDescriptor().getCloudDescriptor() == null) { + // not a cloud core + log.warn("Not initializing shard reporter for non-cloud core " + core.getName()); + return; + } + if (period < 1) { // don't start it + log.warn("Not starting shard reporter "); + return; + } + // our id is coreNodeName + String id = core.getCoreDescriptor().getCloudDescriptor().getCoreNodeName(); + // target registry is the leaderRegistryName + String groupId = core.getCoreMetricManager().getLeaderRegistryName(); + if (groupId == null) { + log.warn("No leaderRegistryName for core " + core + ", not starting the reporter..."); + return; + } + SolrReporter.Report spec = new SolrReporter.Report(groupId, null, registryName, filters); + reporter = SolrReporter.Builder.forReports(metricManager, Collections.singletonList(spec)) + .convertRatesTo(TimeUnit.SECONDS) + .convertDurationsTo(TimeUnit.MILLISECONDS) + .withHandler(handler) + .withReporterId(id) + .cloudClient(false) // we want to send reports specifically to a selected leader instance + .skipAggregateValues(true) // we don't want to transport details of aggregates + .skipHistograms(true) // we don't want to transport histograms + .build(core.getCoreDescriptor().getCoreContainer().getUpdateShardHandler().getHttpClient(), new LeaderUrlSupplier(core)); + + reporter.start(period, TimeUnit.SECONDS); + } + + private static class LeaderUrlSupplier implements Supplier { + private SolrCore core; + + LeaderUrlSupplier(SolrCore core) { + this.core = core; + } + + @Override + public String get() { + CloudDescriptor cd = core.getCoreDescriptor().getCloudDescriptor(); + if (cd == null) { + return null; + } + ClusterState state = core.getCoreDescriptor().getCoreContainer().getZkController().getClusterState(); + DocCollection collection = state.getCollection(core.getCoreDescriptor().getCollectionName()); + Replica replica = collection.getLeader(core.getCoreDescriptor().getCloudDescriptor().getShardId()); + if (replica == null) { + log.warn("No leader for " + collection.getName() + "/" + core.getCoreDescriptor().getCloudDescriptor().getShardId()); + return null; + } + String baseUrl = replica.getStr("base_url"); + if (baseUrl == null) { + log.warn("No base_url for replica " + replica); + } + return baseUrl; + } + } +} diff --git a/solr/core/src/java/org/apache/solr/metrics/reporters/solr/package-info.java b/solr/core/src/java/org/apache/solr/metrics/reporters/solr/package-info.java new file mode 100644 index 00000000000..740bccebf3a --- /dev/null +++ b/solr/core/src/java/org/apache/solr/metrics/reporters/solr/package-info.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * This package contains {@link org.apache.solr.metrics.SolrMetricReporter} implementations + * specific to SolrCloud reporting. + */ +package org.apache.solr.metrics.reporters.solr; diff --git a/solr/core/src/java/org/apache/solr/update/PeerSync.java b/solr/core/src/java/org/apache/solr/update/PeerSync.java index ac0741364ea..874e39c33c0 100644 --- a/solr/core/src/java/org/apache/solr/update/PeerSync.java +++ b/solr/core/src/java/org/apache/solr/update/PeerSync.java @@ -161,11 +161,13 @@ public class PeerSync implements SolrMetricProducer { core.getCoreMetricManager().registerMetricProducer(SolrInfoMBean.Category.REPLICATION.toString(), this); } + public static final String METRIC_SCOPE = "peerSync"; + @Override public void initializeMetrics(SolrMetricManager manager, String registry, String scope) { - syncTime = manager.timer(registry, "time", scope); - syncErrors = manager.counter(registry, "errors", scope); - syncSkipped = manager.counter(registry, "skipped", scope); + syncTime = manager.timer(registry, "time", scope, METRIC_SCOPE); + syncErrors = manager.counter(registry, "errors", scope, METRIC_SCOPE); + syncSkipped = manager.counter(registry, "skipped", scope, METRIC_SCOPE); } /** optional list of updates we had before possibly receiving new updates */ diff --git a/solr/core/src/java/org/apache/solr/util/stats/MetricUtils.java b/solr/core/src/java/org/apache/solr/util/stats/MetricUtils.java index 80f035bda7d..5a7c680494a 100644 --- a/solr/core/src/java/org/apache/solr/util/stats/MetricUtils.java +++ b/solr/core/src/java/org/apache/solr/util/stats/MetricUtils.java @@ -16,11 +16,15 @@ */ package org.apache.solr.util.stats; +import java.util.LinkedHashMap; +import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.SortedSet; import java.util.concurrent.ExecutorService; import java.util.concurrent.TimeUnit; +import java.util.function.BiConsumer; +import java.util.function.Consumer; import com.codahale.metrics.Counter; import com.codahale.metrics.Gauge; @@ -32,13 +36,40 @@ import com.codahale.metrics.MetricFilter; import com.codahale.metrics.MetricRegistry; import com.codahale.metrics.Snapshot; import com.codahale.metrics.Timer; +import org.apache.solr.common.SolrInputDocument; import org.apache.solr.common.util.NamedList; +import org.apache.solr.metrics.AggregateMetric; /** * Metrics specific utility functions. */ public class MetricUtils { + public static final String METRIC_NAME = "metric"; + public static final String VALUES = "values"; + + static final String MS = "_ms"; + + static final String MIN = "min"; + static final String MIN_MS = MIN + MS; + static final String MAX = "max"; + static final String MAX_MS = MAX + MS; + static final String MEAN = "mean"; + static final String MEAN_MS = MEAN + MS; + static final String MEDIAN = "median"; + static final String MEDIAN_MS = MEDIAN + MS; + static final String STDDEV = "stddev"; + static final String STDDEV_MS = STDDEV + MS; + static final String SUM = "sum"; + static final String P75 = "p75"; + static final String P75_MS = P75 + MS; + static final String P95 = "p95"; + static final String P95_MS = P95 + MS; + static final String P99 = "p99"; + static final String P99_MS = P99 + MS; + static final String P999 = "p999"; + static final String P999_MS = P999 + MS; + /** * Adds metrics from a Timer to a NamedList, using well-known back-compat names. * @param lst The NamedList to add the metrics data to @@ -77,41 +108,138 @@ public class MetricUtils { * included in the output * @param mustMatchFilter a {@link MetricFilter}. * A metric must match this filter to be included in the output. + * @param skipHistograms discard any {@link Histogram}-s and histogram parts of {@link Timer}-s. + * @param metadata optional metadata. If not null and not empty then this map will be added under a + * {@code _metadata_} key. * @return a {@link NamedList} */ - public static NamedList toNamedList(MetricRegistry registry, List shouldMatchFilters, MetricFilter mustMatchFilter) { - NamedList response = new NamedList(); + public static NamedList toNamedList(MetricRegistry registry, List shouldMatchFilters, + MetricFilter mustMatchFilter, boolean skipHistograms, + boolean skipAggregateValues, + Map metadata) { + NamedList result = new NamedList(); + toNamedMaps(registry, shouldMatchFilters, mustMatchFilter, skipHistograms, skipAggregateValues, (k, v) -> { + result.add(k, new NamedList(v)); + }); + if (metadata != null && !metadata.isEmpty()) { + result.add("_metadata_", new NamedList(metadata)); + } + return result; + } + + /** + * Returns a representation of the given metric registry as a list of {@link SolrInputDocument}-s. + Only those metrics + * are converted to NamedList which match at least one of the given MetricFilter instances. + * + * @param registry the {@link MetricRegistry} to be converted to NamedList + * @param shouldMatchFilters a list of {@link MetricFilter} instances. + * A metric must match any one of the filters from this list to be + * included in the output + * @param mustMatchFilter a {@link MetricFilter}. + * A metric must match this filter to be included in the output. + * @param skipHistograms discard any {@link Histogram}-s and histogram parts of {@link Timer}-s. + * @param metadata optional metadata. If not null and not empty then this map will be added under a + * {@code _metadata_} key. + * @return a list of {@link SolrInputDocument}-s + */ + public static List toSolrInputDocuments(MetricRegistry registry, List shouldMatchFilters, + MetricFilter mustMatchFilter, boolean skipHistograms, + boolean skipAggregateValues, + Map metadata) { + List result = new LinkedList<>(); + toSolrInputDocuments(registry, shouldMatchFilters, mustMatchFilter, skipHistograms, + skipAggregateValues, metadata, doc -> { + result.add(doc); + }); + return result; + } + + public static void toSolrInputDocuments(MetricRegistry registry, List shouldMatchFilters, + MetricFilter mustMatchFilter, boolean skipHistograms, + boolean skipAggregateValues, + Map metadata, Consumer consumer) { + boolean addMetadata = metadata != null && !metadata.isEmpty(); + toNamedMaps(registry, shouldMatchFilters, mustMatchFilter, skipHistograms, skipAggregateValues, (k, v) -> { + SolrInputDocument doc = new SolrInputDocument(); + doc.setField(METRIC_NAME, k); + toSolrInputDocument(null, doc, v); + if (addMetadata) { + toSolrInputDocument(null, doc, metadata); + } + consumer.accept(doc); + }); + } + + public static void toSolrInputDocument(String prefix, SolrInputDocument doc, Map map) { + for (Map.Entry entry : map.entrySet()) { + if (entry.getValue() instanceof Map) { // flatten recursively + toSolrInputDocument(entry.getKey(), doc, (Map)entry.getValue()); + } else { + String key = prefix != null ? prefix + "." + entry.getKey() : entry.getKey(); + doc.addField(key, entry.getValue()); + } + } + } + + public static void toNamedMaps(MetricRegistry registry, List shouldMatchFilters, + MetricFilter mustMatchFilter, boolean skipHistograms, boolean skipAggregateValues, + BiConsumer> consumer) { Map metrics = registry.getMetrics(); SortedSet names = registry.getNames(); names.stream() .filter(s -> shouldMatchFilters.stream().anyMatch(metricFilter -> metricFilter.matches(s, metrics.get(s)))) .filter(s -> mustMatchFilter.matches(s, metrics.get(s))) .forEach(n -> { - Metric metric = metrics.get(n); - if (metric instanceof Counter) { - Counter counter = (Counter) metric; - response.add(n, counterToNamedList(counter)); - } else if (metric instanceof Gauge) { - Gauge gauge = (Gauge) metric; - response.add(n, gaugeToNamedList(gauge)); - } else if (metric instanceof Meter) { - Meter meter = (Meter) metric; - response.add(n, meterToNamedList(meter)); - } else if (metric instanceof Timer) { - Timer timer = (Timer) metric; - response.add(n, timerToNamedList(timer)); - } else if (metric instanceof Histogram) { - Histogram histogram = (Histogram) metric; - response.add(n, histogramToNamedList(histogram)); - } - }); + Metric metric = metrics.get(n); + if (metric instanceof Counter) { + Counter counter = (Counter) metric; + consumer.accept(n, counterToMap(counter)); + } else if (metric instanceof Gauge) { + Gauge gauge = (Gauge) metric; + consumer.accept(n, gaugeToMap(gauge)); + } else if (metric instanceof Meter) { + Meter meter = (Meter) metric; + consumer.accept(n, meterToMap(meter)); + } else if (metric instanceof Timer) { + Timer timer = (Timer) metric; + consumer.accept(n, timerToMap(timer, skipHistograms)); + } else if (metric instanceof Histogram) { + if (!skipHistograms) { + Histogram histogram = (Histogram) metric; + consumer.accept(n, histogramToMap(histogram)); + } + } else if (metric instanceof AggregateMetric) { + consumer.accept(n, aggregateMetricToMap((AggregateMetric)metric, skipAggregateValues)); + } + }); + } + + static Map aggregateMetricToMap(AggregateMetric metric, boolean skipAggregateValues) { + Map response = new LinkedHashMap<>(); + response.put("count", metric.size()); + response.put(MAX, metric.getMax()); + response.put(MIN, metric.getMin()); + response.put(MEAN, metric.getMean()); + response.put(STDDEV, metric.getStdDev()); + response.put(SUM, metric.getSum()); + if (!(metric.isEmpty() || skipAggregateValues)) { + Map values = new LinkedHashMap<>(); + response.put(VALUES, values); + metric.getValues().forEach((k, v) -> { + Map map = new LinkedHashMap<>(); + map.put("value", v.value); + map.put("updateCount", v.updateCount.get()); + values.put(k, map); + }); + } return response; } - static NamedList histogramToNamedList(Histogram histogram) { - NamedList response = new NamedList(); + static Map histogramToMap(Histogram histogram) { + Map response = new LinkedHashMap<>(); Snapshot snapshot = histogram.getSnapshot(); - response.add("count", histogram.getCount()); + response.put("count", histogram.getCount()); // non-time based values addSnapshot(response, snapshot, false); return response; @@ -126,71 +254,52 @@ public class MetricUtils { } } - static final String MS = "_ms"; - - static final String MIN = "min"; - static final String MIN_MS = MIN + MS; - static final String MAX = "max"; - static final String MAX_MS = MAX + MS; - static final String MEAN = "mean"; - static final String MEAN_MS = MEAN + MS; - static final String MEDIAN = "median"; - static final String MEDIAN_MS = MEDIAN + MS; - static final String STDDEV = "stddev"; - static final String STDDEV_MS = STDDEV + MS; - static final String P75 = "p75"; - static final String P75_MS = P75 + MS; - static final String P95 = "p95"; - static final String P95_MS = P95 + MS; - static final String P99 = "p99"; - static final String P99_MS = P99 + MS; - static final String P999 = "p999"; - static final String P999_MS = P999 + MS; - // some snapshots represent time in ns, other snapshots represent raw values (eg. chunk size) - static void addSnapshot(NamedList response, Snapshot snapshot, boolean ms) { - response.add((ms ? MIN_MS: MIN), nsToMs(ms, snapshot.getMin())); - response.add((ms ? MAX_MS: MAX), nsToMs(ms, snapshot.getMax())); - response.add((ms ? MEAN_MS : MEAN), nsToMs(ms, snapshot.getMean())); - response.add((ms ? MEDIAN_MS: MEDIAN), nsToMs(ms, snapshot.getMedian())); - response.add((ms ? STDDEV_MS: STDDEV), nsToMs(ms, snapshot.getStdDev())); - response.add((ms ? P75_MS: P75), nsToMs(ms, snapshot.get75thPercentile())); - response.add((ms ? P95_MS: P95), nsToMs(ms, snapshot.get95thPercentile())); - response.add((ms ? P99_MS: P99), nsToMs(ms, snapshot.get99thPercentile())); - response.add((ms ? P999_MS: P999), nsToMs(ms, snapshot.get999thPercentile())); + static void addSnapshot(Map response, Snapshot snapshot, boolean ms) { + response.put((ms ? MIN_MS: MIN), nsToMs(ms, snapshot.getMin())); + response.put((ms ? MAX_MS: MAX), nsToMs(ms, snapshot.getMax())); + response.put((ms ? MEAN_MS : MEAN), nsToMs(ms, snapshot.getMean())); + response.put((ms ? MEDIAN_MS: MEDIAN), nsToMs(ms, snapshot.getMedian())); + response.put((ms ? STDDEV_MS: STDDEV), nsToMs(ms, snapshot.getStdDev())); + response.put((ms ? P75_MS: P75), nsToMs(ms, snapshot.get75thPercentile())); + response.put((ms ? P95_MS: P95), nsToMs(ms, snapshot.get95thPercentile())); + response.put((ms ? P99_MS: P99), nsToMs(ms, snapshot.get99thPercentile())); + response.put((ms ? P999_MS: P999), nsToMs(ms, snapshot.get999thPercentile())); } - static NamedList timerToNamedList(Timer timer) { - NamedList response = new NamedList(); - response.add("count", timer.getCount()); - response.add("meanRate", timer.getMeanRate()); - response.add("1minRate", timer.getOneMinuteRate()); - response.add("5minRate", timer.getFiveMinuteRate()); - response.add("15minRate", timer.getFifteenMinuteRate()); - // time-based values in nanoseconds - addSnapshot(response, timer.getSnapshot(), true); + static Map timerToMap(Timer timer, boolean skipHistograms) { + Map response = new LinkedHashMap<>(); + response.put("count", timer.getCount()); + response.put("meanRate", timer.getMeanRate()); + response.put("1minRate", timer.getOneMinuteRate()); + response.put("5minRate", timer.getFiveMinuteRate()); + response.put("15minRate", timer.getFifteenMinuteRate()); + if (!skipHistograms) { + // time-based values in nanoseconds + addSnapshot(response, timer.getSnapshot(), true); + } return response; } - static NamedList meterToNamedList(Meter meter) { - NamedList response = new NamedList(); - response.add("count", meter.getCount()); - response.add("meanRate", meter.getMeanRate()); - response.add("1minRate", meter.getOneMinuteRate()); - response.add("5minRate", meter.getFiveMinuteRate()); - response.add("15minRate", meter.getFifteenMinuteRate()); + static Map meterToMap(Meter meter) { + Map response = new LinkedHashMap<>(); + response.put("count", meter.getCount()); + response.put("meanRate", meter.getMeanRate()); + response.put("1minRate", meter.getOneMinuteRate()); + response.put("5minRate", meter.getFiveMinuteRate()); + response.put("15minRate", meter.getFifteenMinuteRate()); return response; } - static NamedList gaugeToNamedList(Gauge gauge) { - NamedList response = new NamedList(); - response.add("value", gauge.getValue()); + static Map gaugeToMap(Gauge gauge) { + Map response = new LinkedHashMap<>(); + response.put("value", gauge.getValue()); return response; } - static NamedList counterToNamedList(Counter counter) { - NamedList response = new NamedList(); - response.add("count", counter.getCount()); + static Map counterToMap(Counter counter) { + Map response = new LinkedHashMap<>(); + response.put("count", counter.getCount()); return response; } diff --git a/solr/core/src/test-files/solr/solr-solrreporter.xml b/solr/core/src/test-files/solr/solr-solrreporter.xml new file mode 100644 index 00000000000..db03e421887 --- /dev/null +++ b/solr/core/src/test-files/solr/solr-solrreporter.xml @@ -0,0 +1,66 @@ + + + + + + ${urlScheme:} + ${socketTimeout:90000} + ${connTimeout:15000} + + + + 127.0.0.1 + ${hostPort:8983} + ${hostContext:solr} + ${solr.zkclienttimeout:30000} + ${genericCoreNodeNames:true} + ${leaderVoteWait:10000} + ${distribUpdateConnTimeout:45000} + ${distribUpdateSoTimeout:340000} + ${autoReplicaFailoverWaitAfterExpiration:10000} + ${autoReplicaFailoverWorkLoopDelay:10000} + ${autoReplicaFailoverBadNodeExpiration:60000} + + + + + 5 + UPDATE\./update/.*requests + QUERY\./select.*requests + + + /admin/metrics/collector + 5 + + cluster + jvm + solr\.jvm + memory\.total\..* + memory\.heap\..* + os\.SystemLoadAverage + threads\.count + + + cluster + leader.$1 + solr\.collection\.(.*)\.leader + UPDATE\./update/.* + + + + diff --git a/solr/core/src/test/org/apache/solr/cloud/TestCloudRecovery.java b/solr/core/src/test/org/apache/solr/cloud/TestCloudRecovery.java index 164eeabcc93..1af09f4e817 100644 --- a/solr/core/src/test/org/apache/solr/cloud/TestCloudRecovery.java +++ b/solr/core/src/test/org/apache/solr/cloud/TestCloudRecovery.java @@ -119,9 +119,9 @@ public class TestCloudRecovery extends SolrCloudTestCase { .filter(s -> s.startsWith("solr.core.")).collect(Collectors.toList()); for (String registry : registryNames) { Map metrics = manager.registry(registry).getMetrics(); - Timer timer = (Timer)metrics.get("REPLICATION.time"); - Counter counter = (Counter)metrics.get("REPLICATION.errors"); - Counter skipped = (Counter)metrics.get("REPLICATION.skipped"); + Timer timer = (Timer)metrics.get("REPLICATION.peerSync.time"); + Counter counter = (Counter)metrics.get("REPLICATION.peerSync.errors"); + Counter skipped = (Counter)metrics.get("REPLICATION.peerSync.skipped"); replicationCount += timer.getCount(); errorsCount += counter.getCount(); skippedCount += skipped.getCount(); diff --git a/solr/core/src/test/org/apache/solr/core/TestJmxMonitoredMap.java b/solr/core/src/test/org/apache/solr/core/TestJmxMonitoredMap.java index 2cad6e87ffc..aa107bce0bf 100644 --- a/solr/core/src/test/org/apache/solr/core/TestJmxMonitoredMap.java +++ b/solr/core/src/test/org/apache/solr/core/TestJmxMonitoredMap.java @@ -85,7 +85,7 @@ public class TestJmxMonitoredMap extends LuceneTestCase { log.info("Using port: " + port); String url = "service:jmx:rmi:///jndi/rmi://127.0.0.1:"+port+"/solrjmx"; JmxConfiguration config = new JmxConfiguration(true, null, url, null); - monitoredMap = new JmxMonitoredMap<>("", "", config); + monitoredMap = new JmxMonitoredMap<>("", "", "", config); JMXServiceURL u = new JMXServiceURL(url); connector = JMXConnectorFactory.connect(u); mbeanServer = connector.getMBeanServerConnection(); diff --git a/solr/core/src/test/org/apache/solr/metrics/SolrCoreMetricManagerTest.java b/solr/core/src/test/org/apache/solr/metrics/SolrCoreMetricManagerTest.java index 1df60212d6f..6e8e1e58e92 100644 --- a/solr/core/src/test/org/apache/solr/metrics/SolrCoreMetricManagerTest.java +++ b/solr/core/src/test/org/apache/solr/metrics/SolrCoreMetricManagerTest.java @@ -103,6 +103,7 @@ public class SolrCoreMetricManagerTest extends SolrTestCaseJ4 { String className = MockMetricReporter.class.getName(); String reporterName = TestUtil.randomUnicodeString(random); + String taggedName = reporterName + "@" + coreMetricManager.getTag(); Map attrs = new HashMap<>(); attrs.put(FieldType.CLASS_NAME, className); @@ -116,15 +117,16 @@ public class SolrCoreMetricManagerTest extends SolrTestCaseJ4 { PluginInfo pluginInfo = shouldDefinePlugin ? new PluginInfo(TestUtil.randomUnicodeString(random), attrs) : null; try { - metricManager.loadReporter(coreMetricManager.getRegistryName(), coreMetricManager.getCore().getResourceLoader(), pluginInfo); + metricManager.loadReporter(coreMetricManager.getRegistryName(), coreMetricManager.getCore().getResourceLoader(), + pluginInfo, String.valueOf(coreMetricManager.getCore().hashCode())); assertNotNull(pluginInfo); Map reporters = metricManager.getReporters(coreMetricManager.getRegistryName()); assertTrue("reporters.size should be > 0, but was + " + reporters.size(), reporters.size() > 0); - assertNotNull("reporter " + reporterName + " not present among " + reporters, reporters.get(reporterName)); - assertTrue("wrong reporter class: " + reporters.get(reporterName), reporters.get(reporterName) instanceof MockMetricReporter); + assertNotNull("reporter " + reporterName + " not present among " + reporters, reporters.get(taggedName)); + assertTrue("wrong reporter class: " + reporters.get(taggedName), reporters.get(taggedName) instanceof MockMetricReporter); } catch (IllegalArgumentException e) { assertTrue(pluginInfo == null || attrs.get("configurable") == null); - assertNull(metricManager.getReporters(coreMetricManager.getRegistryName()).get(reporterName)); + assertNull(metricManager.getReporters(coreMetricManager.getRegistryName()).get(taggedName)); } } @@ -152,20 +154,11 @@ public class SolrCoreMetricManagerTest extends SolrTestCaseJ4 { } @Test - public void testRegistryName() throws Exception { - String collectionName = "my_collection_"; - String cloudCoreName = "my_collection__shard1_0_replica0"; - String simpleCoreName = "collection_1_replica0"; - String simpleRegistryName = "solr.core." + simpleCoreName; - String cloudRegistryName = "solr.core." + cloudCoreName; - String nestedRegistryName = "solr.core.my_collection_.shard1_0.replica0"; - // pass through - assertEquals(cloudRegistryName, coreMetricManager.createRegistryName(null, cloudCoreName)); - assertEquals(simpleRegistryName, coreMetricManager.createRegistryName(null, simpleCoreName)); - // unknown naming scheme -> pass through - assertEquals(simpleRegistryName, coreMetricManager.createRegistryName(collectionName, simpleCoreName)); - // cloud collection - assertEquals(nestedRegistryName, coreMetricManager.createRegistryName(collectionName, cloudCoreName)); - + public void testNonCloudRegistryName() throws Exception { + String registryName = h.getCore().getCoreMetricManager().getRegistryName(); + String leaderRegistryName = h.getCore().getCoreMetricManager().getLeaderRegistryName(); + assertNotNull(registryName); + assertEquals("solr.core.collection1", registryName); + assertNull(leaderRegistryName); } } diff --git a/solr/core/src/test/org/apache/solr/metrics/SolrMetricManagerTest.java b/solr/core/src/test/org/apache/solr/metrics/SolrMetricManagerTest.java index ee2acd3d290..1c29c5e9a73 100644 --- a/solr/core/src/test/org/apache/solr/metrics/SolrMetricManagerTest.java +++ b/solr/core/src/test/org/apache/solr/metrics/SolrMetricManagerTest.java @@ -205,32 +205,32 @@ public class SolrMetricManagerTest extends SolrTestCaseJ4 { createPluginInfo("node_foo", "node", null), createPluginInfo("core_foo", "core", null) }; - - metricManager.loadReporters(plugins, loader, SolrInfoMBean.Group.node); + String tag = "xyz"; + metricManager.loadReporters(plugins, loader, tag, SolrInfoMBean.Group.node); Map reporters = metricManager.getReporters( SolrMetricManager.getRegistryName(SolrInfoMBean.Group.node)); assertEquals(4, reporters.size()); - assertTrue(reporters.containsKey("universal_foo")); - assertTrue(reporters.containsKey("multigroup_foo")); - assertTrue(reporters.containsKey("node_foo")); - assertTrue(reporters.containsKey("multiregistry_foo")); + assertTrue(reporters.containsKey("universal_foo@" + tag)); + assertTrue(reporters.containsKey("multigroup_foo@" + tag)); + assertTrue(reporters.containsKey("node_foo@" + tag)); + assertTrue(reporters.containsKey("multiregistry_foo@" + tag)); - metricManager.loadReporters(plugins, loader, SolrInfoMBean.Group.core, "collection1"); + metricManager.loadReporters(plugins, loader, tag, SolrInfoMBean.Group.core, "collection1"); reporters = metricManager.getReporters( SolrMetricManager.getRegistryName(SolrInfoMBean.Group.core, "collection1")); assertEquals(5, reporters.size()); - assertTrue(reporters.containsKey("universal_foo")); - assertTrue(reporters.containsKey("multigroup_foo")); - assertTrue(reporters.containsKey("specific_foo")); - assertTrue(reporters.containsKey("core_foo")); - assertTrue(reporters.containsKey("multiregistry_foo")); + assertTrue(reporters.containsKey("universal_foo@" + tag)); + assertTrue(reporters.containsKey("multigroup_foo@" + tag)); + assertTrue(reporters.containsKey("specific_foo@" + tag)); + assertTrue(reporters.containsKey("core_foo@" + tag)); + assertTrue(reporters.containsKey("multiregistry_foo@" + tag)); - metricManager.loadReporters(plugins, loader, SolrInfoMBean.Group.jvm); + metricManager.loadReporters(plugins, loader, tag, SolrInfoMBean.Group.jvm); reporters = metricManager.getReporters( SolrMetricManager.getRegistryName(SolrInfoMBean.Group.jvm)); assertEquals(2, reporters.size()); - assertTrue(reporters.containsKey("universal_foo")); - assertTrue(reporters.containsKey("multigroup_foo")); + assertTrue(reporters.containsKey("universal_foo@" + tag)); + assertTrue(reporters.containsKey("multigroup_foo@" + tag)); metricManager.removeRegistry("solr.jvm"); reporters = metricManager.getReporters( diff --git a/solr/core/src/test/org/apache/solr/metrics/SolrMetricsIntegrationTest.java b/solr/core/src/test/org/apache/solr/metrics/SolrMetricsIntegrationTest.java index 27c038bf39c..dfb5a0fa2da 100644 --- a/solr/core/src/test/org/apache/solr/metrics/SolrMetricsIntegrationTest.java +++ b/solr/core/src/test/org/apache/solr/metrics/SolrMetricsIntegrationTest.java @@ -19,7 +19,6 @@ package org.apache.solr.metrics; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.Arrays; import java.util.Map; import java.util.Random; @@ -55,6 +54,11 @@ public class SolrMetricsIntegrationTest extends SolrTestCaseJ4 { private CoreContainer cc; private SolrMetricManager metricManager; + private String tag; + + private void assertTagged(Map reporters, String name) { + assertTrue("Reporter '" + name + "' missing in " + reporters, reporters.containsKey(name + "@" + tag)); + } @Before public void beforeTest() throws Exception { @@ -68,10 +72,13 @@ public class SolrMetricsIntegrationTest extends SolrTestCaseJ4 { new TestHarness.TestCoresLocator(DEFAULT_TEST_CORENAME, initCoreDataDir.getAbsolutePath(), "solrconfig.xml", "schema.xml")); h.coreName = DEFAULT_TEST_CORENAME; metricManager = cc.getMetricManager(); + tag = h.getCore().getCoreMetricManager().getTag(); // initially there are more reporters, because two of them are added via a matching collection name Map reporters = metricManager.getReporters("solr.core." + DEFAULT_TEST_CORENAME); assertEquals(INITIAL_REPORTERS.length, reporters.size()); - assertTrue(reporters.keySet().containsAll(Arrays.asList(INITIAL_REPORTERS))); + for (String r : INITIAL_REPORTERS) { + assertTagged(reporters, r); + } // test rename operation cc.rename(DEFAULT_TEST_CORENAME, CORE_NAME); h.coreName = CORE_NAME; @@ -101,7 +108,7 @@ public class SolrMetricsIntegrationTest extends SolrTestCaseJ4 { deleteCore(); for (String reporterName : RENAMED_REPORTERS) { - SolrMetricReporter reporter = reporters.get(reporterName); + SolrMetricReporter reporter = reporters.get(reporterName + "@" + tag); MockMetricReporter mockReporter = (MockMetricReporter) reporter; assertTrue("Reporter " + reporterName + " was not closed: " + mockReporter, mockReporter.didClose); } @@ -130,7 +137,7 @@ public class SolrMetricsIntegrationTest extends SolrTestCaseJ4 { // SPECIFIC and MULTIREGISTRY were skipped because they were // specific to collection1 for (String reporterName : RENAMED_REPORTERS) { - SolrMetricReporter reporter = reporters.get(reporterName); + SolrMetricReporter reporter = reporters.get(reporterName + "@" + tag); assertNotNull("Reporter " + reporterName + " was not found.", reporter); assertTrue(reporter instanceof MockMetricReporter); diff --git a/solr/core/src/test/org/apache/solr/metrics/reporters/SolrJmxReporterTest.java b/solr/core/src/test/org/apache/solr/metrics/reporters/SolrJmxReporterTest.java index ea452b2ff13..82b9d58cc85 100644 --- a/solr/core/src/test/org/apache/solr/metrics/reporters/SolrJmxReporterTest.java +++ b/solr/core/src/test/org/apache/solr/metrics/reporters/SolrJmxReporterTest.java @@ -64,15 +64,17 @@ public class SolrJmxReporterTest extends SolrTestCaseJ4 { coreMetricManager = core.getCoreMetricManager(); metricManager = core.getCoreDescriptor().getCoreContainer().getMetricManager(); PluginInfo pluginInfo = createReporterPluginInfo(); - metricManager.loadReporter(coreMetricManager.getRegistryName(), coreMetricManager.getCore().getResourceLoader(), pluginInfo); + metricManager.loadReporter(coreMetricManager.getRegistryName(), coreMetricManager.getCore().getResourceLoader(), + pluginInfo, coreMetricManager.getTag()); Map reporters = metricManager.getReporters(coreMetricManager.getRegistryName()); assertTrue("reporters.size should be > 0, but was + " + reporters.size(), reporters.size() > 0); reporterName = pluginInfo.name; - assertNotNull("reporter " + reporterName + " not present among " + reporters, reporters.get(reporterName)); - assertTrue("wrong reporter class: " + reporters.get(reporterName), reporters.get(reporterName) instanceof SolrJmxReporter); + String taggedName = reporterName + "@" + coreMetricManager.getTag(); + assertNotNull("reporter " + taggedName + " not present among " + reporters, reporters.get(taggedName)); + assertTrue("wrong reporter class: " + reporters.get(taggedName), reporters.get(taggedName) instanceof SolrJmxReporter); - reporter = (SolrJmxReporter) reporters.get(reporterName); + reporter = (SolrJmxReporter) reporters.get(taggedName); mBeanServer = reporter.getMBeanServer(); assertNotNull("MBean server not found.", mBeanServer); } @@ -144,7 +146,8 @@ public class SolrJmxReporterTest extends SolrTestCaseJ4 { h.getCoreContainer().reload(h.getCore().getName()); PluginInfo pluginInfo = createReporterPluginInfo(); - metricManager.loadReporter(coreMetricManager.getRegistryName(), coreMetricManager.getCore().getResourceLoader(), pluginInfo); + metricManager.loadReporter(coreMetricManager.getRegistryName(), coreMetricManager.getCore().getResourceLoader(), + pluginInfo, String.valueOf(coreMetricManager.getCore().hashCode())); coreMetricManager.registerMetricProducer(scope, producer); objects = mBeanServer.queryMBeans(null, null); diff --git a/solr/core/src/test/org/apache/solr/metrics/reporters/solr/SolrCloudReportersTest.java b/solr/core/src/test/org/apache/solr/metrics/reporters/solr/SolrCloudReportersTest.java new file mode 100644 index 00000000000..91952b889d8 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/metrics/reporters/solr/SolrCloudReportersTest.java @@ -0,0 +1,163 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.metrics.reporters.solr; + +import java.nio.file.Paths; +import java.util.Map; + +import com.codahale.metrics.Metric; +import org.apache.commons.io.IOUtils; +import org.apache.solr.client.solrj.request.CollectionAdminRequest; +import org.apache.solr.cloud.SolrCloudTestCase; +import org.apache.solr.core.CoreContainer; +import org.apache.solr.core.SolrCore; +import org.apache.solr.metrics.AggregateMetric; +import org.apache.solr.metrics.SolrMetricManager; +import org.apache.solr.metrics.SolrMetricReporter; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +/** + * + */ +public class SolrCloudReportersTest extends SolrCloudTestCase { + int leaderRegistries; + int clusterRegistries; + + + @BeforeClass + public static void configureDummyCluster() throws Exception { + configureCluster(0).configure(); + } + + @Before + public void closePreviousCluster() throws Exception { + shutdownCluster(); + leaderRegistries = 0; + clusterRegistries = 0; + } + + @Test + public void testExplicitConfiguration() throws Exception { + String solrXml = IOUtils.toString(SolrCloudReportersTest.class.getResourceAsStream("/solr/solr-solrreporter.xml"), "UTF-8"); + configureCluster(2) + .withSolrXml(solrXml).configure(); + cluster.uploadConfigSet(Paths.get(TEST_PATH().toString(), "configsets", "minimal", "conf"), "test"); + System.out.println("ZK: " + cluster.getZkServer().getZkAddress()); + CollectionAdminRequest.createCollection("test_collection", "test", 2, 2) + .setMaxShardsPerNode(4) + .process(cluster.getSolrClient()); + waitForState("Expected test_collection with 2 shards and 2 replicas", "test_collection", clusterShape(2, 2)); + Thread.sleep(15000); + cluster.getJettySolrRunners().forEach(jetty -> { + CoreContainer cc = jetty.getCoreContainer(); + // verify registry names + for (String name : cc.getCoreNames()) { + SolrCore core = cc.getCore(name); + try { + String registryName = core.getCoreMetricManager().getRegistryName(); + String leaderRegistryName = core.getCoreMetricManager().getLeaderRegistryName(); + String coreName = core.getName(); + String collectionName = core.getCoreDescriptor().getCollectionName(); + String coreNodeName = core.getCoreDescriptor().getCloudDescriptor().getCoreNodeName(); + String replicaName = coreName.split("_")[3]; + String shardId = core.getCoreDescriptor().getCloudDescriptor().getShardId(); + + assertEquals("solr.core." + collectionName + "." + shardId + "." + replicaName, registryName); + assertEquals("solr.collection." + collectionName + "." + shardId + ".leader", leaderRegistryName); + + } finally { + if (core != null) { + core.close(); + } + } + } + SolrMetricManager metricManager = cc.getMetricManager(); + Map reporters = metricManager.getReporters("solr.cluster"); + assertEquals(reporters.toString(), 1, reporters.size()); + SolrMetricReporter reporter = reporters.get("test"); + assertNotNull(reporter); + assertTrue(reporter.toString(), reporter instanceof SolrClusterReporter); + SolrClusterReporter sor = (SolrClusterReporter)reporter; + assertEquals(5, sor.getPeriod()); + for (String registryName : metricManager.registryNames(".*\\.shard[0-9]\\.replica.*")) { + reporters = metricManager.getReporters(registryName); + assertEquals(reporters.toString(), 1, reporters.size()); + reporter = null; + for (String name : reporters.keySet()) { + if (name.startsWith("test")) { + reporter = reporters.get(name); + } + } + assertNotNull(reporter); + assertTrue(reporter.toString(), reporter instanceof SolrShardReporter); + SolrShardReporter srr = (SolrShardReporter)reporter; + assertEquals(5, srr.getPeriod()); + } + for (String registryName : metricManager.registryNames(".*\\.leader")) { + leaderRegistries++; + reporters = metricManager.getReporters(registryName); + // no reporters registered for leader registry + assertEquals(reporters.toString(), 0, reporters.size()); + // verify specific metrics + Map metrics = metricManager.registry(registryName).getMetrics(); + String key = "QUERY./select.requests.count"; + assertTrue(key, metrics.containsKey(key)); + assertTrue(key, metrics.get(key) instanceof AggregateMetric); + key = "UPDATE./update/json.requests.count"; + assertTrue(key, metrics.containsKey(key)); + assertTrue(key, metrics.get(key) instanceof AggregateMetric); + } + if (metricManager.registryNames().contains("solr.cluster")) { + clusterRegistries++; + Map metrics = metricManager.registry("solr.cluster").getMetrics(); + String key = "jvm.memory.heap.init.value"; + assertTrue(key, metrics.containsKey(key)); + assertTrue(key, metrics.get(key) instanceof AggregateMetric); + key = "leader.test_collection.shard1.UPDATE./update/json.requests.count.max"; + assertTrue(key, metrics.containsKey(key)); + assertTrue(key, metrics.get(key) instanceof AggregateMetric); + } + }); + assertEquals("leaderRegistries", 2, leaderRegistries); + assertEquals("clusterRegistries", 1, clusterRegistries); + } + + @Test + public void testDefaultPlugins() throws Exception { + String solrXml = IOUtils.toString(SolrCloudReportersTest.class.getResourceAsStream("/solr/solr.xml"), "UTF-8"); + configureCluster(2) + .withSolrXml(solrXml).configure(); + cluster.uploadConfigSet(Paths.get(TEST_PATH().toString(), "configsets", "minimal", "conf"), "test"); + System.out.println("ZK: " + cluster.getZkServer().getZkAddress()); + CollectionAdminRequest.createCollection("test_collection", "test", 2, 2) + .setMaxShardsPerNode(4) + .process(cluster.getSolrClient()); + waitForState("Expected test_collection with 2 shards and 2 replicas", "test_collection", clusterShape(2, 2)); + cluster.getJettySolrRunners().forEach(jetty -> { + CoreContainer cc = jetty.getCoreContainer(); + SolrMetricManager metricManager = cc.getMetricManager(); + Map reporters = metricManager.getReporters("solr.cluster"); + assertEquals(reporters.toString(), 0, reporters.size()); + for (String registryName : metricManager.registryNames(".*\\.shard[0-9]\\.replica.*")) { + reporters = metricManager.getReporters(registryName); + assertEquals(reporters.toString(), 0, reporters.size()); + } + }); + } +} diff --git a/solr/core/src/test/org/apache/solr/metrics/reporters/solr/SolrShardReporterTest.java b/solr/core/src/test/org/apache/solr/metrics/reporters/solr/SolrShardReporterTest.java new file mode 100644 index 00000000000..9ce37627cde --- /dev/null +++ b/solr/core/src/test/org/apache/solr/metrics/reporters/solr/SolrShardReporterTest.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.metrics.reporters.solr; + +import java.lang.invoke.MethodHandles; +import java.util.Map; + +import com.codahale.metrics.Metric; +import org.apache.solr.client.solrj.embedded.JettySolrRunner; +import org.apache.solr.cloud.AbstractFullDistribZkTestBase; +import org.apache.solr.cloud.CloudDescriptor; +import org.apache.solr.common.cloud.ClusterState; +import org.apache.solr.common.cloud.DocCollection; +import org.apache.solr.common.cloud.Slice; +import org.apache.solr.core.CoreContainer; +import org.apache.solr.core.CoreDescriptor; +import org.apache.solr.metrics.AggregateMetric; +import org.apache.solr.metrics.SolrCoreMetricManager; +import org.apache.solr.metrics.SolrMetricManager; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * + */ +public class SolrShardReporterTest extends AbstractFullDistribZkTestBase { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + public SolrShardReporterTest() { + schemaString = "schema15.xml"; // we need a string id + } + + @Override + public String getSolrXml() { + return "solr-solrreporter.xml"; + } + + @Test + public void test() throws Exception { + waitForRecoveriesToFinish("control_collection", + jettys.get(0).getCoreContainer().getZkController().getZkStateReader(), false); + waitForRecoveriesToFinish("collection1", + jettys.get(0).getCoreContainer().getZkController().getZkStateReader(), false); + printLayout(); + // wait for at least two reports + Thread.sleep(10000); + ClusterState state = jettys.get(0).getCoreContainer().getZkController().getClusterState(); + for (JettySolrRunner jetty : jettys) { + CoreContainer cc = jetty.getCoreContainer(); + SolrMetricManager metricManager = cc.getMetricManager(); + for (final String coreName : cc.getCoreNames()) { + CoreDescriptor cd = cc.getCoreDescriptor(coreName); + if (cd.getCloudDescriptor() == null) { // not a cloud collection + continue; + } + CloudDescriptor cloudDesc = cd.getCloudDescriptor(); + DocCollection docCollection = state.getCollection(cloudDesc.getCollectionName()); + String replicaName = SolrCoreMetricManager.parseReplicaName(cloudDesc.getCollectionName(), coreName); + if (replicaName == null) { + replicaName = cloudDesc.getCoreNodeName(); + } + String registryName = SolrCoreMetricManager.createRegistryName(true, + cloudDesc.getCollectionName(), cloudDesc.getShardId(), replicaName, null); + String leaderRegistryName = SolrCoreMetricManager.createLeaderRegistryName(true, + cloudDesc.getCollectionName(), cloudDesc.getShardId()); + boolean leader = cloudDesc.isLeader(); + Slice slice = docCollection.getSlice(cloudDesc.getShardId()); + int numReplicas = slice.getReplicas().size(); + if (leader) { + assertTrue(metricManager.registryNames() + " doesn't contain " + leaderRegistryName, + metricManager.registryNames().contains(leaderRegistryName)); + Map metrics = metricManager.registry(leaderRegistryName).getMetrics(); + metrics.forEach((k, v) -> { + assertTrue("Unexpected type of " + k + ": " + v.getClass().getName() + ", " + v, + v instanceof AggregateMetric); + AggregateMetric am = (AggregateMetric)v; + if (!k.startsWith("REPLICATION.peerSync")) { + assertEquals(coreName + "::" + registryName + "::" + k + ": " + am.toString(), numReplicas, am.size()); + } + }); + } else { + assertFalse(metricManager.registryNames() + " contains " + leaderRegistryName + + " but it's not a leader!", + metricManager.registryNames().contains(leaderRegistryName)); + Map metrics = metricManager.registry(leaderRegistryName).getMetrics(); + metrics.forEach((k, v) -> { + assertTrue("Unexpected type of " + k + ": " + v.getClass().getName() + ", " + v, + v instanceof AggregateMetric); + AggregateMetric am = (AggregateMetric)v; + if (!k.startsWith("REPLICATION.peerSync")) { + assertEquals(coreName + "::" + registryName + "::" + k + ": " + am.toString(), 1, am.size()); + } + }); + } + assertTrue(metricManager.registryNames() + " doesn't contain " + registryName, + metricManager.registryNames().contains(registryName)); + } + } + SolrMetricManager metricManager = controlJetty.getCoreContainer().getMetricManager(); + assertTrue(metricManager.registryNames().contains("solr.cluster")); + } +} diff --git a/solr/core/src/test/org/apache/solr/util/stats/MetricUtilsTest.java b/solr/core/src/test/org/apache/solr/util/stats/MetricUtilsTest.java index e39ad6e6040..8717ad641b3 100644 --- a/solr/core/src/test/org/apache/solr/util/stats/MetricUtilsTest.java +++ b/solr/core/src/test/org/apache/solr/util/stats/MetricUtilsTest.java @@ -17,12 +17,20 @@ package org.apache.solr.util.stats; +import java.util.Collections; +import java.util.Map; import java.util.concurrent.TimeUnit; +import com.codahale.metrics.Counter; +import com.codahale.metrics.Histogram; +import com.codahale.metrics.Meter; +import com.codahale.metrics.MetricFilter; +import com.codahale.metrics.MetricRegistry; import com.codahale.metrics.Snapshot; import com.codahale.metrics.Timer; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.common.util.NamedList; +import org.apache.solr.metrics.AggregateMetric; import org.junit.Test; public class MetricUtilsTest extends SolrTestCaseJ4 { @@ -36,7 +44,7 @@ public class MetricUtilsTest extends SolrTestCaseJ4 { timer.update(Math.abs(random().nextInt()) + 1, TimeUnit.NANOSECONDS); } // obtain timer metrics - NamedList lst = MetricUtils.timerToNamedList(timer); + NamedList lst = new NamedList(MetricUtils.timerToMap(timer, false)); // check that expected metrics were obtained assertEquals(14, lst.size()); final Snapshot snapshot = timer.getSnapshot(); @@ -52,5 +60,49 @@ public class MetricUtilsTest extends SolrTestCaseJ4 { assertEquals(MetricUtils.nsToMs(snapshot.get999thPercentile()), lst.get("p999_ms")); } + @Test + public void testMetrics() throws Exception { + MetricRegistry registry = new MetricRegistry(); + Counter counter = registry.counter("counter"); + counter.inc(); + Timer timer = registry.timer("timer"); + Timer.Context ctx = timer.time(); + Thread.sleep(150); + ctx.stop(); + Meter meter = registry.meter("meter"); + meter.mark(); + Histogram histogram = registry.histogram("histogram"); + histogram.update(10); + AggregateMetric am = new AggregateMetric(); + registry.register("aggregate", am); + am.set("foo", 10); + am.set("bar", 1); + am.set("bar", 2); + MetricUtils.toNamedMaps(registry, Collections.singletonList(MetricFilter.ALL), MetricFilter.ALL, + false, false, (k, v) -> { + if (k.startsWith("counter")) { + assertEquals(1L, v.get("count")); + } else if (k.startsWith("timer")) { + assertEquals(1L, v.get("count")); + assertTrue(((Number)v.get("min_ms")).intValue() > 100); + } else if (k.startsWith("meter")) { + assertEquals(1L, v.get("count")); + } else if (k.startsWith("histogram")) { + assertEquals(1L, v.get("count")); + } else if (k.startsWith("aggregate")) { + assertEquals(2, v.get("count")); + Map values = (Map)v.get("values"); + assertNotNull(values); + assertEquals(2, values.size()); + Map update = (Map)values.get("foo"); + assertEquals(10, update.get("value")); + assertEquals(1, update.get("updateCount")); + update = (Map)values.get("bar"); + assertEquals(2, update.get("value")); + assertEquals(2, update.get("updateCount")); + } + }); + } + } diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/BinaryRequestWriter.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/BinaryRequestWriter.java index 67274c2cd27..310c282d32f 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/BinaryRequestWriter.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/BinaryRequestWriter.java @@ -112,8 +112,8 @@ public class BinaryRequestWriter extends RequestWriter { /* * A hack to get access to the protected internal buffer and avoid an additional copy */ - class BAOS extends ByteArrayOutputStream { - byte[] getbuf() { + public static class BAOS extends ByteArrayOutputStream { + public byte[] getbuf() { return super.buf; } } diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/SolrClientCache.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/SolrClientCache.java index da9416211d1..132a1a89576 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/SolrClientCache.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/SolrClientCache.java @@ -22,6 +22,7 @@ import java.lang.invoke.MethodHandles; import java.util.Map; import java.util.HashMap; +import org.apache.http.client.HttpClient; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.impl.CloudSolrClient; import org.apache.solr.client.solrj.impl.HttpSolrClient; @@ -38,15 +39,27 @@ public class SolrClientCache implements Serializable { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); private final Map solrClients = new HashMap<>(); + private final HttpClient httpClient; + + public SolrClientCache() { + httpClient = null; + } + + public SolrClientCache(HttpClient httpClient) { + this.httpClient = httpClient; + } public synchronized CloudSolrClient getCloudSolrClient(String zkHost) { CloudSolrClient client; if (solrClients.containsKey(zkHost)) { client = (CloudSolrClient) solrClients.get(zkHost); } else { - client = new CloudSolrClient.Builder() - .withZkHost(zkHost) - .build(); + CloudSolrClient.Builder builder = new CloudSolrClient.Builder() + .withZkHost(zkHost); + if (httpClient != null) { + builder = builder.withHttpClient(httpClient); + } + client = builder.build(); client.connect(); solrClients.put(zkHost, client); } @@ -59,8 +72,11 @@ public class SolrClientCache implements Serializable { if (solrClients.containsKey(host)) { client = (HttpSolrClient) solrClients.get(host); } else { - client = new HttpSolrClient.Builder(host) - .build(); + HttpSolrClient.Builder builder = new HttpSolrClient.Builder(host); + if (httpClient != null) { + builder = builder.withHttpClient(httpClient); + } + client = builder.build(); solrClients.put(host, client); } return client; diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/request/TestCoreAdmin.java b/solr/solrj/src/test/org/apache/solr/client/solrj/request/TestCoreAdmin.java index b2174cd369b..de7c620422b 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/request/TestCoreAdmin.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/request/TestCoreAdmin.java @@ -251,8 +251,8 @@ public class TestCoreAdmin extends AbstractEmbeddedSolrServerTestCase { // assert initial metrics SolrMetricManager metricManager = cores.getMetricManager(); - String core0RegistryName = SolrCoreMetricManager.createRegistryName(null, "core0"); - String core1RegistryName = SolrCoreMetricManager.createRegistryName(null, "core1"); + String core0RegistryName = SolrCoreMetricManager.createRegistryName(false, null, null, null, "core0"); + String core1RegistryName = SolrCoreMetricManager.createRegistryName(false, null, null,null, "core1"); MetricRegistry core0Registry = metricManager.registry(core0RegistryName); MetricRegistry core1Registry = metricManager.registry(core1RegistryName);