Apache/lucene
1
0
Fork 0
mirror of https://github.com/apache/lucene.git synced 2025-02-21 17:46:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

SOLR-14914: Add option to disable metrics collection.

Browse Source
This commit is contained in:
Andrzej Bialecki 2020-10-15 10:13:47 +02:00
parent 9805b125dc
commit 2a3da99e2d
35 changed files with 517 additions and 94 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
solr/CHANGES.txt
View File

@ -162,6 +162,8 @@ New Features
* SOLR-14907: Support single file upload/overwrite in configSet API. (Houston Putman)
* SOLR-14914: Add option to disable metrics collection. (ab)
Improvements
---------------------

1
solr/contrib/prometheus-exporter/src/test/org/apache/solr/prometheus/PrometheusExporterTestBase.java
View File

@ -59,6 +59,7 @@ public class PrometheusExporterTestBase extends SolrCloudTestCase {
@BeforeClass
public static void setupCluster() throws Exception {
System.setProperty("metricsEnabled", "true");
configureCluster(NUM_NODES)
.addConfig(CONF_NAME, getFile(CONF_DIR).toPath())
.configure();

9
solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java
View File

@ -129,9 +129,12 @@ public class SplitShardCmd implements OverseerCollectionMessageHandler.Cmd {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Interrupted.");
}
RTimerTree t = timings.sub("checkDiskSpace");
checkDiskSpace(collectionName, slice.get(), parentShardLeader, splitMethod, ocmh.cloudManager);
t.stop();
RTimerTree t;
if (ocmh.overseer.getCoreContainer().getNodeConfig().getMetricsConfig().isEnabled()) {
t = timings.sub("checkDiskSpace");
checkDiskSpace(collectionName, slice.get(), parentShardLeader, splitMethod, ocmh.cloudManager);
t.stop();
}
// let's record the ephemeralOwner of the parent leader node
Stat leaderZnodeStat = zkStateReader.getZkClient().exists(ZkStateReader.LIVE_NODES_ZKNODE + "/" + parentShardLeader.getNodeName(), null, true);

12
solr/core/src/java/org/apache/solr/core/CoreContainer.java
View File

@ -742,9 +742,11 @@ public class CoreContainer {
createMetricsHistoryHandler();
metricsCollectorHandler = createHandler(MetricsCollectorHandler.HANDLER_PATH, MetricsCollectorHandler.class.getName(), MetricsCollectorHandler.class);
// may want to add some configuration here in the future
metricsCollectorHandler.init(null);
if (cfg.getMetricsConfig().isEnabled()) {
metricsCollectorHandler = createHandler(MetricsCollectorHandler.HANDLER_PATH, MetricsCollectorHandler.class.getName(), MetricsCollectorHandler.class);
// may want to add some configuration here in the future
metricsCollectorHandler.init(null);
}
containerHandlers.put(AUTHZ_PATH, securityConfHandler);
securityConfHandler.initializeMetrics(solrMetricsContext, AUTHZ_PATH);
@ -902,6 +904,10 @@ public class CoreContainer {
@SuppressWarnings({"unchecked"})
private void createMetricsHistoryHandler() {
PluginInfo plugin = cfg.getMetricsConfig().getHistoryHandler();
if (plugin != null && MetricsConfig.NOOP_IMPL_CLASS.equals(plugin.className)) {
// still create the handler but it will be disabled
plugin = null;
}
Map<String, Object> initArgs;
if (plugin != null && plugin.initArgs != null) {
initArgs = plugin.initArgs.asMap(5);

71
solr/core/src/java/org/apache/solr/core/MetricsConfig.java
View File

@ -16,6 +16,7 @@
*/
package org.apache.solr.core;
import java.util.Collections;
import java.util.HashSet;
import java.util.Set;
@ -31,11 +32,14 @@ public class MetricsConfig {
private final PluginInfo timerSupplier;
private final PluginInfo histogramSupplier;
private final PluginInfo historyHandler;
private final boolean enabled;
private MetricsConfig(PluginInfo[] metricReporters, Set<String> hiddenSysProps,
private MetricsConfig(boolean enabled,
PluginInfo[] metricReporters, Set<String> hiddenSysProps,
PluginInfo counterSupplier, PluginInfo meterSupplier,
PluginInfo timerSupplier, PluginInfo histogramSupplier,
PluginInfo historyHandler) {
this.enabled = enabled;
this.metricReporters = metricReporters;
this.hiddenSysProps = hiddenSysProps;
this.counterSupplier = counterSupplier;
@ -45,32 +49,74 @@ public class MetricsConfig {
this.historyHandler = historyHandler;
}
public boolean isEnabled() {
return enabled;
}
private static final PluginInfo[] NO_OP_REPORTERS = new PluginInfo[0];
public PluginInfo[] getMetricReporters() {
return metricReporters;
if (enabled) {
return metricReporters;
} else {
return NO_OP_REPORTERS;
}
}
public Set<String> getHiddenSysProps() {
return hiddenSysProps;
if (enabled) {
return hiddenSysProps;
} else {
return Collections.emptySet();
}
}
/** Symbolic name to use as plugin class name for no-op implementations. */
public static final String NOOP_IMPL_CLASS = "__noop__";
private static final PluginInfo NO_OP_PLUGIN =
new PluginInfo("typeUnused",
Collections.singletonMap("class", NOOP_IMPL_CLASS),
null, null);
public PluginInfo getCounterSupplier() {
return counterSupplier;
if (enabled) {
return counterSupplier;
} else {
return NO_OP_PLUGIN;
}
}
public PluginInfo getMeterSupplier() {
return meterSupplier;
if (enabled) {
return meterSupplier;
} else {
return NO_OP_PLUGIN;
}
}
public PluginInfo getTimerSupplier() {
return timerSupplier;
if (enabled) {
return timerSupplier;
} else {
return NO_OP_PLUGIN;
}
}
public PluginInfo getHistogramSupplier() {
return histogramSupplier;
if (enabled) {
return histogramSupplier;
} else {
return NO_OP_PLUGIN;
}
}
public PluginInfo getHistoryHandler() {
return historyHandler;
if (enabled) {
return historyHandler;
} else {
return NO_OP_PLUGIN;
}
}
public static class MetricsConfigBuilder {
@ -81,11 +127,18 @@ public class MetricsConfig {
private PluginInfo timerSupplier;
private PluginInfo histogramSupplier;
private PluginInfo historyHandler;
// default to metrics enabled
private boolean enabled = true;
public MetricsConfigBuilder() {
}
public MetricsConfigBuilder setEnabled(boolean enabled) {
this.enabled = enabled;
return this;
}
public MetricsConfigBuilder setHiddenSysProps(Set<String> hiddenSysProps) {
if (hiddenSysProps != null && !hiddenSysProps.isEmpty()) {
this.hiddenSysProps.clear();
@ -125,7 +178,7 @@ public class MetricsConfig {
}
public MetricsConfig build() {
return new MetricsConfig(metricReporterPlugins, hiddenSysProps, counterSupplier, meterSupplier,
return new MetricsConfig(enabled, metricReporterPlugins, hiddenSysProps, counterSupplier, meterSupplier,
timerSupplier, histogramSupplier, historyHandler);
}

13
solr/core/src/java/org/apache/solr/core/SolrXmlConfig.java
View File

@ -500,7 +500,18 @@ public class SolrXmlConfig {
private static MetricsConfig getMetricsConfig(XmlConfigFile config) {
MetricsConfig.MetricsConfigBuilder builder = new MetricsConfig.MetricsConfigBuilder();
Node node = config.getNode("solr/metrics/suppliers/counter", false);
Node node = config.getNode("solr/metrics", false);
// enabled by default
boolean enabled = true;
if (node != null) {
enabled = Boolean.parseBoolean(DOMUtil.getAttrOrDefault(node, "enabled", "true"));
}
builder.setEnabled(enabled);
if (!enabled) {
log.info("Metrics collection is disabled.");
return builder.build();
}
node = config.getNode("solr/metrics/suppliers/counter", false);
if (node != null) {
builder = builder.setCounterSupplier(new PluginInfo(node, "counterSupplier", false, false));
}

18
solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java
View File

@ -71,20 +71,24 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionName
public static final String ALL = "all";
private static final Pattern KEY_REGEX = Pattern.compile("(?<!" + Pattern.quote("\\") + ")" + Pattern.quote(":"));
private CoreContainer cc;
private final CoreContainer cc;
private final Map<String, String> injectedSysProps = CommonTestInjection.injectAdditionalProps();
public MetricsHandler() {
this.metricManager = null;
}
private final boolean enabled;
public MetricsHandler(CoreContainer coreContainer) {
this.metricManager = coreContainer.getMetricManager();
this.cc = coreContainer;
this.enabled = coreContainer.getConfig().getMetricsConfig().isEnabled();
}
public MetricsHandler(SolrMetricManager metricManager) {
this.metricManager = metricManager;
this.cc = null;
this.enabled = true;
}
public boolean isEnabled() {
return enabled;
}
@Override
@ -107,6 +111,10 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionName
@SuppressWarnings({"unchecked"})
public void handleRequest(SolrParams params, BiConsumer<String, Object> consumer) throws Exception {
if (!enabled) {
consumer.accept("error", "metrics collection is disabled");
return;
}
boolean compact = params.getBool(COMPACT_PARAM, true);
String[] keys = params.getParams(KEY_PARAM);
if (keys != null && keys.length > 0) {

15
solr/core/src/java/org/apache/solr/handler/admin/MetricsHistoryHandler.java
View File

@ -196,7 +196,9 @@ public class MetricsHistoryHandler extends RequestHandlerBase implements Permiss
}
this.nodeName = nodeName;
this.enable = Boolean.parseBoolean(String.valueOf(args.getOrDefault(ENABLE_PROP, "true")));
// disable when metrics reporting is disabled
this.enable = Boolean.parseBoolean(String.valueOf(args.getOrDefault(ENABLE_PROP, "true")))
&& metricsHandler.isEnabled();
// default to false - don't collect local per-replica metrics
this.enableReplicas = Boolean.parseBoolean(String.valueOf(args.getOrDefault(ENABLE_REPLICAS_PROP, "false")));
this.enableNodes = Boolean.parseBoolean(String.valueOf(args.getOrDefault(ENABLE_NODES_PROP, "false")));
@ -245,6 +247,9 @@ public class MetricsHistoryHandler extends RequestHandlerBase implements Permiss
// check that .system exists
public void checkSystemCollection() {
if (!enable) {
return;
}
if (cloudManager != null) {
try {
if (cloudManager.isClosed() || Thread.interrupted()) {
@ -306,7 +311,9 @@ public class MetricsHistoryHandler extends RequestHandlerBase implements Permiss
public void removeHistory(String registry) throws IOException {
registry = SolrMetricManager.enforcePrefix(registry);
knownDbs.remove(registry);
factory.remove(registry);
if (factory != null) {
factory.remove(registry);
}
}
@VisibleForTesting
@ -699,6 +706,10 @@ public class MetricsHistoryHandler extends RequestHandlerBase implements Permiss
@Override
public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception {
if (!enable) {
rsp.add("error", "metrics history collection is disabled");
return;
}
String actionStr = req.getParams().get(CommonParams.ACTION);
if (actionStr == null) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "'action' is a required param");

267
solr/core/src/java/org/apache/solr/metrics/MetricSuppliers.java
View File

@ -19,19 +19,13 @@ package org.apache.solr.metrics;
import java.lang.invoke.MethodHandles;
import java.lang.management.ManagementFactory;
import java.lang.management.ThreadMXBean;
import java.time.Duration;
import java.util.concurrent.Callable;
import java.util.concurrent.TimeUnit;
import java.util.function.Supplier;
import com.codahale.metrics.Clock;
import com.codahale.metrics.Counter;
import com.codahale.metrics.ExponentiallyDecayingReservoir;
import com.codahale.metrics.Histogram;
import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.Reservoir;
import com.codahale.metrics.SlidingTimeWindowReservoir;
import com.codahale.metrics.SlidingWindowReservoir;
import com.codahale.metrics.Timer;
import com.codahale.metrics.UniformReservoir;
import com.codahale.metrics.*;
import org.apache.solr.core.MetricsConfig;
import org.apache.solr.core.PluginInfo;
import org.apache.solr.core.SolrResourceLoader;
import org.apache.solr.util.SolrPluginUtils;
@ -197,17 +191,21 @@ public class MetricSuppliers {
return new SlidingWindowReservoir(size);
} else { // custom reservoir
Reservoir reservoir;
try {
reservoir = loader.newInstance(clazz, Reservoir.class);
if (reservoir instanceof PluginInfoInitialized) {
((PluginInfoInitialized)reservoir).init(info);
} else {
SolrPluginUtils.invokeSetters(reservoir, info.initArgs, true);
}
return reservoir;
} catch (Exception e) {
log.warn("Error initializing custom Reservoir implementation (will use default): {}", info, e);
if (loader == null) {
return new ExponentiallyDecayingReservoir(size, alpha, clk);
} else {
try {
reservoir = loader.newInstance(clazz, Reservoir.class);
if (reservoir instanceof PluginInfoInitialized) {
((PluginInfoInitialized)reservoir).init(info);
} else {
SolrPluginUtils.invokeSetters(reservoir, info.initArgs, true);
}
return reservoir;
} catch (Exception e) {
log.warn("Error initializing custom Reservoir implementation (will use default): {}", info, e);
return new ExponentiallyDecayingReservoir(size, alpha, clk);
}
}
}
}
@ -282,13 +280,19 @@ public class MetricSuppliers {
if (info == null || info.className == null || info.className.trim().isEmpty()) {
return new DefaultCounterSupplier();
}
if (MetricsConfig.NOOP_IMPL_CLASS.equals(info.className)) {
return NoOpCounterSupplier.INSTANCE;
}
MetricRegistry.MetricSupplier<Counter> supplier;
try {
supplier = loader.newInstance(info.className, MetricRegistry.MetricSupplier.class);
} catch (Exception e) {
log.warn("Error creating custom Counter supplier (will use default): {}", info, e);
if (loader == null) {
supplier = new DefaultCounterSupplier();
} else {
try {
supplier = loader.newInstance(info.className, MetricRegistry.MetricSupplier.class);
} catch (Exception e) {
log.warn("Error creating custom Counter supplier (will use default): {}", info, e);
supplier = new DefaultCounterSupplier();
}
}
if (supplier instanceof PluginInfoInitialized) {
((PluginInfoInitialized)supplier).init(info);
@ -310,11 +314,18 @@ public class MetricSuppliers {
if (info == null || info.className == null || info.className.isEmpty()) {
supplier = new DefaultMeterSupplier();
} else {
try {
supplier = loader.newInstance(info.className, MetricRegistry.MetricSupplier.class);
} catch (Exception e) {
log.warn("Error creating custom Meter supplier (will use default): {}",info, e);
if (MetricsConfig.NOOP_IMPL_CLASS.equals(info.className)) {
return NoOpMeterSupplier.INSTANCE;
}
if (loader == null) {
supplier = new DefaultMeterSupplier();
} else {
try {
supplier = loader.newInstance(info.className, MetricRegistry.MetricSupplier.class);
} catch (Exception e) {
log.warn("Error creating custom Meter supplier (will use default): {}",info, e);
supplier = new DefaultMeterSupplier();
}
}
}
if (supplier instanceof PluginInfoInitialized) {
@ -337,11 +348,18 @@ public class MetricSuppliers {
if (info == null || info.className == null || info.className.isEmpty()) {
supplier = new DefaultTimerSupplier(loader);
} else {
try {
supplier = loader.newInstance(info.className, MetricRegistry.MetricSupplier.class);
} catch (Exception e) {
log.warn("Error creating custom Timer supplier (will use default): {}", info, e);
supplier = new DefaultTimerSupplier(loader);
if (MetricsConfig.NOOP_IMPL_CLASS.equals(info.className)) {
return NoOpTimerSupplier.INSTANCE;
}
if (loader == null) {
supplier = new DefaultTimerSupplier(null);
} else {
try {
supplier = loader.newInstance(info.className, MetricRegistry.MetricSupplier.class);
} catch (Exception e) {
log.warn("Error creating custom Timer supplier (will use default): {}", info, e);
supplier = new DefaultTimerSupplier(loader);
}
}
}
if (supplier instanceof PluginInfoInitialized) {
@ -363,11 +381,18 @@ public class MetricSuppliers {
if (info == null || info.className == null || info.className.isEmpty()) {
supplier = new DefaultHistogramSupplier(loader);
} else {
try {
supplier = loader.newInstance(info.className, MetricRegistry.MetricSupplier.class);
} catch (Exception e) {
log.warn("Error creating custom Histogram supplier (will use default): {}", info, e);
supplier = new DefaultHistogramSupplier(loader);
if (MetricsConfig.NOOP_IMPL_CLASS.equals(info.className)) {
return NoOpHistogramSupplier.INSTANCE;
}
if (loader == null) {
supplier = new DefaultHistogramSupplier(null);
} else {
try {
supplier = loader.newInstance(info.className, MetricRegistry.MetricSupplier.class);
} catch (Exception e) {
log.warn("Error creating custom Histogram supplier (will use default): {}", info, e);
supplier = new DefaultHistogramSupplier(loader);
}
}
}
if (supplier instanceof PluginInfoInitialized) {
@ -377,4 +402,166 @@ public class MetricSuppliers {
}
return supplier;
}
// NO-OP implementations. These can be used to effectively "turn off" the metrics
// collection, or rather eliminate the overhead of the metrics accounting and their
// memory footprint.
/**
* Marker interface for all no-op metrics.
*/
public interface NoOpMetric {
}
/**
* No-op implementation of {@link Counter} supplier.
*/
public static final class NoOpCounterSupplier implements MetricRegistry.MetricSupplier<Counter> {
public static final NoOpCounterSupplier INSTANCE = new NoOpCounterSupplier();
private static final class NoOpCounter extends Counter implements NoOpMetric {
@Override
public void inc() {
// no-op
}
@Override
public void inc(long n) {
// no-op
}
@Override
public void dec() {
// no-op
}
@Override
public void dec(long n) {
// no-op
}
}
private static final Counter METRIC = new NoOpCounter();
@Override
public Counter newMetric() {
return METRIC;
}
}
/**
* No-op implementation of {@link Histogram} supplier.
*/
public static final class NoOpHistogramSupplier implements MetricRegistry.MetricSupplier<Histogram> {
public static final NoOpHistogramSupplier INSTANCE = new NoOpHistogramSupplier();
private static final class NoOpHistogram extends Histogram implements NoOpMetric {
public NoOpHistogram() {
super(new UniformReservoir(1));
}
@Override
public void update(int value) {
// no-op
}
@Override
public void update(long value) {
// no-op
}
}
private static final NoOpHistogram METRIC = new NoOpHistogram();
@Override
public Histogram newMetric() {
return METRIC;
}
}
/**
* No-op implementation of {@link Meter} supplier.
*/
public static final class NoOpMeterSupplier implements MetricRegistry.MetricSupplier<Meter> {
public static final NoOpMeterSupplier INSTANCE = new NoOpMeterSupplier();
private static final class NoOpMeter extends Meter implements NoOpMetric {
@Override
public void mark() {
// no-op
}
@Override
public void mark(long n) {
// no-op
}
}
private static final NoOpMeter METRIC = new NoOpMeter();
@Override
public Meter newMetric() {
return METRIC;
}
}
/**
* No-op implementation of {@link Timer} supplier.
*/
public static final class NoOpTimerSupplier implements MetricRegistry.MetricSupplier<Timer> {
public static final NoOpTimerSupplier INSTANCE = new NoOpTimerSupplier();
private static final class NoOpTimer extends Timer implements NoOpMetric {
@Override
public void update(long duration, TimeUnit unit) {
// no-op
}
@Override
public void update(Duration duration) {
// no-op
}
@Override
public <T> T time(Callable<T> event) throws Exception {
return event.call();
}
@Override
public <T> T timeSupplier(Supplier<T> event) {
return event.get();
}
@Override
public void time(Runnable event) {
event.run();
}
}
private static final NoOpTimer METRIC = new NoOpTimer();
@Override
public Timer newMetric() {
return METRIC;
}
}
/**
* No-op implementation of {@link Gauge}.
*/
public static final class NoOpGauge implements Gauge<Object>, NoOpMetric {
public static final NoOpGauge INSTANCE = new NoOpGauge();
@Override
public Object getValue() {
return null;
}
}
@SuppressWarnings("unchecked")
public static <T> Gauge<T> getNoOpGauge(Gauge<T> actual) {
return (Gauge<T>) NoOpGauge.INSTANCE;
}
}

14
solr/core/src/java/org/apache/solr/metrics/SolrMetricManager.java
View File

@ -110,12 +110,14 @@ public class SolrMetricManager {
public static final int DEFAULT_CLOUD_REPORTER_PERIOD = 60;
private MetricRegistry.MetricSupplier<Counter> counterSupplier;
private MetricRegistry.MetricSupplier<Meter> meterSupplier;
private MetricRegistry.MetricSupplier<Timer> timerSupplier;
private MetricRegistry.MetricSupplier<Histogram> histogramSupplier;
private final MetricsConfig metricsConfig;
private final MetricRegistry.MetricSupplier<Counter> counterSupplier;
private final MetricRegistry.MetricSupplier<Meter> meterSupplier;
private final MetricRegistry.MetricSupplier<Timer> timerSupplier;
private final MetricRegistry.MetricSupplier<Histogram> histogramSupplier;
public SolrMetricManager() {
metricsConfig = new MetricsConfig.MetricsConfigBuilder().build();
counterSupplier = MetricSuppliers.counterSupplier(null, null);
meterSupplier = MetricSuppliers.meterSupplier(null, null);
timerSupplier = MetricSuppliers.timerSupplier(null, null);
@ -123,6 +125,7 @@ public class SolrMetricManager {
}
public SolrMetricManager(SolrResourceLoader loader, MetricsConfig metricsConfig) {
this.metricsConfig = metricsConfig;
counterSupplier = MetricSuppliers.counterSupplier(loader, metricsConfig.getCounterSupplier());
meterSupplier = MetricSuppliers.meterSupplier(loader, metricsConfig.getMeterSupplier());
timerSupplier = MetricSuppliers.timerSupplier(loader, metricsConfig.getTimerSupplier());
@ -756,6 +759,9 @@ public class SolrMetricManager {
}
public <T> void registerGauge(SolrMetricsContext context, String registry, Gauge<T> gauge, String tag, ResolutionStrategy strategy, String metricName, String... metricPath) {
if (!metricsConfig.isEnabled()) {
gauge = MetricSuppliers.getNoOpGauge(gauge);
}
registerMetric(context, registry, new GaugeWrapper<>(gauge, tag), strategy, metricName, metricPath);
}

2
solr/core/src/test-files/solr/solr-metricsconfig.xml
View File

@ -17,7 +17,7 @@
-->
<solr>
<metrics>
<metrics enabled="${metricsEnabled:true}">
<suppliers>
<counter class="${counter.class:}">
<str name="foo">bar</str>

2
solr/core/src/test-files/solr/solr.xml
View File

@ -21,6 +21,8 @@
-->
<solr>
<metrics enabled="${metricsEnabled:true}"/>
<str name="shareSchema">${shareSchema:false}</str>
<str name="configSetBaseDir">${configSetBaseDir:configsets}</str>
<str name="coreRootDirectory">${coreRootDirectory:.}</str>

1
solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java
View File

@ -83,6 +83,7 @@ public class CollectionsAPISolrJTest extends SolrCloudTestCase {
@Before
public void beforeTest() throws Exception {
//System.setProperty("metricsEnabled", "true");
configureCluster(4)
.addConfig("conf", configset("cloud-minimal"))
.addConfig("conf2", configset("cloud-dynamic"))

1
solr/core/src/test/org/apache/solr/cloud/MetricsHistoryIntegrationTest.java
View File

@ -59,6 +59,7 @@ public class MetricsHistoryIntegrationTest extends SolrCloudTestCase {
@BeforeClass
public static void setupCluster() throws Exception {
System.setProperty("metricsEnabled", "true");
configureCluster(1)
.addConfig("conf", configset("cloud-minimal"))
.configure();

2
solr/core/src/test/org/apache/solr/cloud/MetricsHistoryWithAuthIntegrationTest.java
View File

@ -55,7 +55,7 @@ public class MetricsHistoryWithAuthIntegrationTest extends SolrCloudTestCase {
@BeforeClass
public static void setupCluster() throws Exception {
String solrXml = MiniSolrCloudCluster.DEFAULT_CLOUD_SOLR_XML.replace("<metrics>\n",
String solrXml = MiniSolrCloudCluster.DEFAULT_CLOUD_SOLR_XML.replace("<metrics enabled=\"${metricsEnabled:false}\">\n",
"<metrics>\n" + SOLR_XML_HISTORY_CONFIG);
// Spin up a cluster with a protected /admin/metrics handler, and a 2 seconds metrics collectPeriod
configureCluster(1)

1
solr/core/src/test/org/apache/solr/cloud/SplitShardTest.java
View File

@ -54,6 +54,7 @@ public class SplitShardTest extends SolrCloudTestCase {
@BeforeClass
public static void setupCluster() throws Exception {
System.setProperty("metricsEnabled", "true");
configureCluster(1)
.addConfig("conf", configset("cloud-minimal"))
.configure();

1
solr/core/src/test/org/apache/solr/cloud/TestBaseStatsCacheCloud.java
View File

@ -80,6 +80,7 @@ public abstract class TestBaseStatsCacheCloud extends SolrCloudTestCase {
@Before
public void setupCluster() throws Exception {
System.setProperty("metricsEnabled", "true");
// create control core & client
System.setProperty("solr.statsCache", getImplementationName());
System.setProperty("solr.similarity", CustomSimilarityFactory.class.getName());

1
solr/core/src/test/org/apache/solr/cloud/TestCloudRecovery.java
View File

@ -58,6 +58,7 @@ public class TestCloudRecovery extends SolrCloudTestCase {
@BeforeClass
public static void setupCluster() throws Exception {
System.setProperty("metricsEnabled", "true");
System.setProperty("solr.directoryFactory", "solr.StandardDirectoryFactory");
System.setProperty("solr.ulog.numRecordsToKeep", "1000");
}

1
solr/core/src/test/org/apache/solr/cloud/TestTlogReplica.java
View File

@ -91,6 +91,7 @@ public class TestTlogReplica extends SolrCloudTestCase {
@BeforeClass
public static void setupCluster() throws Exception {
System.setProperty("metricsEnabled", "true");
System.setProperty("solr.waitToSeeReplicasInStateTimeoutSeconds", "30");
configureCluster(2) // 2 + random().nextInt(3)
.addConfig("conf", configset("cloud-minimal-inplace-updates"))

1
solr/core/src/test/org/apache/solr/cloud/rule/RulesTest.java
View File

@ -56,6 +56,7 @@ public class RulesTest extends SolrCloudTestCase {
@BeforeClass
public static void setupCluster() throws Exception {
System.setProperty("metricsEnabled", "true");
configureCluster(5)
.addConfig("conf", configset("cloud-minimal"))
.configure();

1
solr/core/src/test/org/apache/solr/handler/admin/AdminHandlersProxyTest.java
View File

@ -44,6 +44,7 @@ public class AdminHandlersProxyTest extends SolrCloudTestCase {
@BeforeClass
public static void setupCluster() throws Exception {
System.setProperty("metricsEnabled", "true");
configureCluster(2)
.addConfig("conf", configset("cloud-minimal"))
.configure();

1
solr/core/src/test/org/apache/solr/handler/admin/MetricsHistoryHandlerTest.java
View File

@ -58,6 +58,7 @@ public class MetricsHistoryHandlerTest extends SolrCloudTestCase {
@BeforeClass
public static void beforeClass() throws Exception {
System.setProperty("metricsEnabled", "true");
Map<String, Object> args = new HashMap<>();
args.put(MetricsHistoryHandler.SYNC_PERIOD_PROP, 1);
args.put(MetricsHistoryHandler.COLLECT_PERIOD_PROP, 1);

12
solr/core/src/test/org/apache/solr/metrics/MetricsConfigTest.java
View File

@ -116,6 +116,18 @@ public class MetricsConfigTest extends SolrTestCaseJ4 {
assertNotNull(mockHistogramSupplier.info);
}
@Test
public void testDisabledMetrics() throws Exception {
System.setProperty("metricsEnabled", "false");
NodeConfig cfg = loadNodeConfig();
SolrMetricManager mgr = new SolrMetricManager(cfg.getSolrResourceLoader(), cfg.getMetricsConfig());
assertTrue(mgr.getCounterSupplier() instanceof MetricSuppliers.NoOpCounterSupplier);
assertTrue(mgr.getMeterSupplier() instanceof MetricSuppliers.NoOpMeterSupplier);
assertTrue(mgr.getTimerSupplier() instanceof MetricSuppliers.NoOpTimerSupplier);
assertTrue(mgr.getHistogramSupplier() instanceof MetricSuppliers.NoOpHistogramSupplier);
}
private NodeConfig loadNodeConfig() throws Exception {
InputStream is = MetricsConfigTest.class.getResourceAsStream("/solr/solr-metricsconfig.xml");
return SolrXmlConfig.fromInputStream(TEST_PATH(), is, new Properties()); //TODO pass in props

72
solr/core/src/test/org/apache/solr/metrics/MetricsDisabledCloudTest.java Normal file
View File

@ -0,0 +1,72 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.metrics;
import com.codahale.metrics.MetricRegistry;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.cloud.SolrCloudTestCase;
import org.apache.solr.core.MetricsConfig;
import org.apache.solr.core.NodeConfig;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
/**
*
*/
public class MetricsDisabledCloudTest extends SolrCloudTestCase {
@BeforeClass
public static void startCluster() throws Exception {
System.setProperty("metricsEnabled", "false");
configureCluster(2).configure();
CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection("test",
"config", 1, 2);
}
@Test
public void testBasic() throws Exception {
NodeConfig cfg = cluster.getRandomJetty(random()).getCoreContainer().getNodeConfig();
MetricsConfig metricsConfig = cfg.getMetricsConfig();
assertFalse("metrics should be disabled", metricsConfig.isEnabled());
SolrMetricManager metricManager = cluster.getRandomJetty(random()).getCoreContainer().getMetricManager();
assertTrue("wrong type of supplier: " + metricManager.getCounterSupplier(),
metricManager.getCounterSupplier() instanceof MetricSuppliers.NoOpCounterSupplier);
assertTrue("wrong type of supplier: " + metricManager.getHistogramSupplier(),
metricManager.getHistogramSupplier() instanceof MetricSuppliers.NoOpHistogramSupplier);
assertTrue("wrong type of supplier: " + metricManager.getTimerSupplier(),
metricManager.getTimerSupplier() instanceof MetricSuppliers.NoOpTimerSupplier);
assertTrue("wrong type of supplier: " + metricManager.getMeterSupplier(),
metricManager.getMeterSupplier() instanceof MetricSuppliers.NoOpMeterSupplier);
for (String registryName : metricManager.registryNames()) {
if (!registryName.startsWith("solr.core.")) {
continue;
}
MetricRegistry registry = metricManager.registry(registryName);
registry.getMetrics().forEach((name, metric) -> {
assertTrue("should be NoOpMetric but was: " + name + "=" +
metric + "(" + metric.getClass() + ")",
metric instanceof MetricSuppliers.NoOpMetric);
});
}
}
@AfterClass
public static void stopCluster() throws Exception {
shutdownCluster();
}
}

1
solr/core/src/test/org/apache/solr/search/stats/TestDefaultStatsCache.java
View File

@ -29,6 +29,7 @@ public class TestDefaultStatsCache extends BaseDistributedSearchTestCase {
@Override
public void distribSetUp() throws Exception {
System.setProperty("metricsEnabled", "true");
super.distribSetUp();
System.setProperty("solr.statsCache", LocalStatsCache.class.getName());
}

2
solr/server/solr/solr.xml
View File

@ -55,4 +55,6 @@
<str name="shardsWhitelist">${solr.shardsWhitelist:}</str>
</shardHandlerFactory>
<metrics enabled="${metricsEnabled:true}"/>
</solr>

3
solr/solr-ref-guide/src/metrics-history.adoc
View File

@ -121,6 +121,9 @@ The handler assumes that a simple aggregation (sum of partial metric values from
sufficient. This happens to make sense for the default built-in sets of metrics. Future extensions will
provide other aggregation strategies (such as, average, max, min, etc.).
This handler is automatically disabled when metrics collection is disabled using the
`<metrics enabled="false">` element in `solr.xml`.
== Metrics History Configuration
There are two ways to configure this subsystem:

15
solr/solr-ref-guide/src/metrics-reporting.adoc
View File

@ -87,6 +87,21 @@ The metrics available in your system can be customized by modifying the `<metric
TIP: See also the section <<format-of-solr-xml.adoc#format-of-solr-xml,Format of Solr.xml>> for more information about the `solr.xml` file, where to find it, and how to edit it.
=== Disabling the metrics collection ===
The `<metrics>` element in `solr.xml` supports one attribute `enabled`, which takes a boolean value,
for example `<metrics enabled="true">`.
The default value of this attribute is `true`, meaning that metrics are being collected, processed and
reported by Solr according to the configured metric reporters. They are also available from the
metrics and metrics history APIs.
The `false` value of this attribute (`<metrics enabled="false">`) turns off metrics collection, processing,
and the collection of metrics history. Internally, all metrics suppliers are replaced by singleton no-op
implementations, which effectively removes nearly all overheads related to metrics collection.
All reporter configurations are skipped, and the metrics
and metrics history APIs stop reporting any metrics and only return an `<error>`
element in their responses.
=== The <metrics> <hiddenSysProps> Element
This section of `solr.xml` allows you to define the system properties which are considered system-sensitive and should not be exposed via the Metrics API.

50
solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java
View File

@ -267,31 +267,33 @@ public class SolrClientNodeStateProvider implements NodeStateProvider, MapWriter
try {
SimpleSolrResponse rsp = snitchContext.invokeWithRetry(solrNode, CommonParams.METRICS_PATH, params);
NamedList<?> metrics = (NamedList<?>) rsp.nl.get("metrics");
if (requestedTags.contains(Variable.FREEDISK.tagName)) {
Object n = Utils.getObjectByPath(metrics, true, "solr.node/CONTAINER.fs.usableSpace");
if (n != null) ctx.getTags().put(Variable.FREEDISK.tagName, Variable.FREEDISK.convertVal(n));
}
if (requestedTags.contains(Variable.TOTALDISK.tagName)) {
Object n = Utils.getObjectByPath(metrics, true, "solr.node/CONTAINER.fs.totalSpace");
if (n != null) ctx.getTags().put(Variable.TOTALDISK.tagName, Variable.TOTALDISK.convertVal(n));
}
if (requestedTags.contains(CORES)) {
NamedList<?> node = (NamedList<?>) metrics.get("solr.node");
int count = 0;
for (String leafCoreMetricName : new String[]{"lazy", "loaded", "unloaded"}) {
Number n = (Number) node.get("CONTAINER.cores." + leafCoreMetricName);
if (n != null) count += n.intValue();
if (metrics != null) {
// metrics enabled
if (requestedTags.contains(Variable.FREEDISK.tagName)) {
Object n = Utils.getObjectByPath(metrics, true, "solr.node/CONTAINER.fs.usableSpace");
if (n != null) ctx.getTags().put(Variable.FREEDISK.tagName, Variable.FREEDISK.convertVal(n));
}
if (requestedTags.contains(Variable.TOTALDISK.tagName)) {
Object n = Utils.getObjectByPath(metrics, true, "solr.node/CONTAINER.fs.totalSpace");
if (n != null) ctx.getTags().put(Variable.TOTALDISK.tagName, Variable.TOTALDISK.convertVal(n));
}
if (requestedTags.contains(CORES)) {
NamedList<?> node = (NamedList<?>) metrics.get("solr.node");
int count = 0;
for (String leafCoreMetricName : new String[]{"lazy", "loaded", "unloaded"}) {
Number n = (Number) node.get("CONTAINER.cores." + leafCoreMetricName);
if (n != null) count += n.intValue();
}
ctx.getTags().put(CORES, count);
}
if (requestedTags.contains(SYSLOADAVG)) {
Number n = (Number) Utils.getObjectByPath(metrics, true, "solr.jvm/os.systemLoadAverage");
if (n != null) ctx.getTags().put(SYSLOADAVG, n.doubleValue() * 100.0d);
}
if (requestedTags.contains(HEAPUSAGE)) {
Number n = (Number) Utils.getObjectByPath(metrics, true, "solr.jvm/memory.heap.usage");
if (n != null) ctx.getTags().put(HEAPUSAGE, n.doubleValue() * 100.0d);
}
ctx.getTags().put(CORES, count);
}
if (requestedTags.contains(SYSLOADAVG)) {
Number n = (Number) Utils.getObjectByPath(metrics, true, "solr.jvm/os.systemLoadAverage");
if (n != null) ctx.getTags().put(SYSLOADAVG, n.doubleValue() * 100.0d);
}
if (requestedTags.contains(HEAPUSAGE)) {
Number n = (Number) Utils.getObjectByPath(metrics, true, "solr.jvm/memory.heap.usage");
if (n != null) ctx.getTags().put(HEAPUSAGE, n.doubleValue() * 100.0d);
}
} catch (Exception e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Error getting remote info", e);

1
solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudHttp2SolrClientRetryTest.java
View File

@ -37,6 +37,7 @@ public class CloudHttp2SolrClientRetryTest extends SolrCloudTestCase {
@BeforeClass
public static void setupCluster() throws Exception {
System.setProperty("metricsEnabled", "true");
configureCluster(NODE_COUNT)
.addConfig("conf", getFile("solrj").toPath().resolve("solr").resolve("configsets").resolve("streaming").resolve("conf"))
.configure();

1
solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudHttp2SolrClientTest.java
View File

@ -104,6 +104,7 @@ public class CloudHttp2SolrClientTest extends SolrCloudTestCase {
@Before
public void setupCluster() throws Exception {
System.setProperty("metricsEnabled", "true");
configureCluster(NODE_COUNT)
.addConfig("conf", getFile("solrj").toPath().resolve("solr").resolve("configsets").resolve("streaming").resolve("conf"))
.configure();

1
solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrClientRetryTest.java
View File

@ -34,6 +34,7 @@ public class CloudSolrClientRetryTest extends SolrCloudTestCase {
@BeforeClass
public static void setupCluster() throws Exception {
System.setProperty("metricsEnabled", "true");
configureCluster(NODE_COUNT)
.addConfig("conf", getFile("solrj").toPath().resolve("solr").resolve("configsets").resolve("streaming").resolve("conf"))
.configure();

1
solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrClientTest.java
View File

@ -102,6 +102,7 @@ public class CloudSolrClientTest extends SolrCloudTestCase {
@Before
public void setupCluster() throws Exception {
System.setProperty("metricsEnabled", "true");
configureCluster(NODE_COUNT)
.addConfig("conf", getFile("solrj").toPath().resolve("solr").resolve("configsets").resolve("streaming").resolve("conf"))
.configure();

3
solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java
View File

@ -118,7 +118,8 @@ public class MiniSolrCloudCluster {
" <str name=\"pkiHandlerPrivateKeyPath\">${pkiHandlerPrivateKeyPath:cryptokeys/priv_key512_pkcs8.pem}</str> \n" +
" <str name=\"pkiHandlerPublicKeyPath\">${pkiHandlerPublicKeyPath:cryptokeys/pub_key512.der}</str> \n" +
" </solrcloud>\n" +
" <metrics>\n" +
// NOTE: this turns off the metrics collection unless overriden by a sysprop
" <metrics enabled=\"${metricsEnabled:false}\">\n" +
" <reporter name=\"default\" class=\"org.apache.solr.metrics.reporters.SolrJmxReporter\">\n" +
" <str name=\"rootName\">solr_${hostPort:8983}</str>\n" +
" </reporter>\n" +

12
solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudAuthTestCase.java
View File

@ -46,6 +46,8 @@ import org.apache.solr.common.util.Utils;
import org.apache.solr.util.TimeOut;
import org.jose4j.jws.JsonWebSignature;
import org.jose4j.lang.JoseException;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -70,6 +72,16 @@ public class SolrCloudAuthTestCase extends SolrCloudTestCase {
private static final List<String> AUTH_METRICS_TO_COMPARE = Arrays.asList("requests", "authenticated", "passThrough", "failWrongCredentials", "failMissingCredentials", "errors");
private static final List<String> AUDIT_METRICS_TO_COMPARE = Arrays.asList("count");
@BeforeClass
public static void enableMetrics() {
System.setProperty("metricsEnabled", "true");
}
@AfterClass
public static void disableMetrics() {
System.clearProperty("metricsEnabled");
}
/**
* Used to check metric counts for PKI auth
*/