Apache/lucene
1
0
Fork 0
mirror of https://github.com/apache/lucene.git synced 2025-02-09 11:35:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

SOLR-13677: All Metrics Gauges should be unregistered by components that registered them.

Browse Source
This commit is contained in:
Andrzej Bialecki 2019-10-18 17:13:30 +02:00
parent b8213cef1a
commit 441af3e7aa
41 changed files with 834 additions and 501 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
solr/CHANGES.txt
View File

@ -273,6 +273,8 @@ Bug Fixes
* SOLR-13843: The MOVEREPLICA API ignores replica type and always adds 'nrt' replicas (Amrit Sarkar via shalin) * SOLR-13843: The MOVEREPLICA API ignores replica type and always adds 'nrt' replicas (Amrit Sarkar via shalin)
* SOLR-13677: All Metrics Gauges should be unregistered by components that registered them. (noble, ab)
Other Changes Other Changes
---------------------- ----------------------

8
solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataImportHandler.java
View File

@ -36,7 +36,7 @@ import org.apache.solr.core.SolrCore;
import org.apache.solr.core.SolrResourceLoader; import org.apache.solr.core.SolrResourceLoader;
import org.apache.solr.handler.RequestHandlerBase; import org.apache.solr.handler.RequestHandlerBase;
import org.apache.solr.metrics.MetricsMap; import org.apache.solr.metrics.MetricsMap;
import org.apache.solr.metrics.SolrMetricManager; import org.apache.solr.metrics.SolrMetricsContext;
import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.RawResponseWriter; import org.apache.solr.response.RawResponseWriter;
import org.apache.solr.response.SolrQueryResponse; import org.apache.solr.response.SolrQueryResponse;
@ -275,8 +275,8 @@ public class DataImportHandler extends RequestHandlerBase implements
} }
@Override @Override
public void initializeMetrics(SolrMetricManager manager, String registryName, String tag, String scope) { public void initializeMetrics(SolrMetricsContext parentContext, String scope) {
super.initializeMetrics(manager, registryName, tag, scope); super.initializeMetrics(parentContext, scope);
metrics = new MetricsMap((detailed, map) -> { metrics = new MetricsMap((detailed, map) -> {
if (importer != null) { if (importer != null) {
DocBuilder.Statistics cumulative = importer.cumulativeStatistics; DocBuilder.Statistics cumulative = importer.cumulativeStatistics;
@ -299,7 +299,7 @@ public class DataImportHandler extends RequestHandlerBase implements
map.put(DataImporter.MSG.TOTAL_DOCS_SKIPPED, cumulative.skipDocCount); map.put(DataImporter.MSG.TOTAL_DOCS_SKIPPED, cumulative.skipDocCount);
} }
}); });
manager.registerGauge(this, registryName, metrics, tag, true, "importer", getCategory().toString(), scope); solrMetricsContext.gauge(this, metrics, true, "importer", getCategory().toString(), scope);
} }
// //////////////////////SolrInfoMBeans methods ////////////////////// // //////////////////////SolrInfoMBeans methods //////////////////////

2
solr/core/src/java/org/apache/solr/cloud/ReplicateFromLeader.java
View File

@ -133,7 +133,7 @@ public class ReplicateFromLeader {
public void stopReplication() { public void stopReplication() {
if (replicationProcess != null) { if (replicationProcess != null) {
replicationProcess.close(); replicationProcess.shutdown();
} }
} }
} }

79
solr/core/src/java/org/apache/solr/core/CoreContainer.java
View File

@ -100,6 +100,7 @@ import org.apache.solr.logging.MDCLoggingContext;
import org.apache.solr.metrics.SolrCoreMetricManager; import org.apache.solr.metrics.SolrCoreMetricManager;
import org.apache.solr.metrics.SolrMetricManager; import org.apache.solr.metrics.SolrMetricManager;
import org.apache.solr.metrics.SolrMetricProducer; import org.apache.solr.metrics.SolrMetricProducer;
import org.apache.solr.metrics.SolrMetricsContext;
import org.apache.solr.request.SolrRequestHandler; import org.apache.solr.request.SolrRequestHandler;
import org.apache.solr.request.SolrRequestInfo; import org.apache.solr.request.SolrRequestInfo;
import org.apache.solr.search.SolrFieldCacheBean; import org.apache.solr.search.SolrFieldCacheBean;
@ -210,7 +211,9 @@ public class CoreContainer {
protected volatile SolrMetricManager metricManager; protected volatile SolrMetricManager metricManager;
protected volatile String metricTag = Integer.toHexString(hashCode()); protected volatile String metricTag = SolrMetricProducer.getUniqueMetricTag(this, null);
protected volatile SolrMetricsContext solrMetricsContext;
protected MetricsHandler metricsHandler; protected MetricsHandler metricsHandler;
@ -609,6 +612,8 @@ public class CoreContainer {
containerHandlers.getApiBag().register(new AnnotatedApi(packageStoreAPI.writeAPI), Collections.EMPTY_MAP); containerHandlers.getApiBag().register(new AnnotatedApi(packageStoreAPI.writeAPI), Collections.EMPTY_MAP);
metricManager = new SolrMetricManager(loader, cfg.getMetricsConfig()); metricManager = new SolrMetricManager(loader, cfg.getMetricsConfig());
String registryName = SolrMetricManager.getRegistryName(SolrInfoBean.Group.node);
solrMetricsContext = new SolrMetricsContext(metricManager, registryName, metricTag);
coreContainerWorkExecutor = MetricUtils.instrumentedExecutorService( coreContainerWorkExecutor = MetricUtils.instrumentedExecutorService(
coreContainerWorkExecutor, null, coreContainerWorkExecutor, null,
@ -622,7 +627,7 @@ public class CoreContainer {
} }
updateShardHandler = new UpdateShardHandler(cfg.getUpdateShardHandlerConfig()); updateShardHandler = new UpdateShardHandler(cfg.getUpdateShardHandlerConfig());
updateShardHandler.initializeMetrics(metricManager, SolrInfoBean.Group.node.toString(), metricTag, "updateShardHandler"); updateShardHandler.initializeMetrics(solrMetricsContext, "updateShardHandler");
solrCores.load(loader); solrCores.load(loader);
@ -635,7 +640,9 @@ public class CoreContainer {
if (isZooKeeperAware()) { if (isZooKeeperAware()) {
pkiAuthenticationPlugin = new PKIAuthenticationPlugin(this, zkSys.getZkController().getNodeName(), pkiAuthenticationPlugin = new PKIAuthenticationPlugin(this, zkSys.getZkController().getNodeName(),
(PublicKeyHandler) containerHandlers.get(PublicKeyHandler.PATH)); (PublicKeyHandler) containerHandlers.get(PublicKeyHandler.PATH));
pkiAuthenticationPlugin.initializeMetrics(metricManager, SolrInfoBean.Group.node.toString(), metricTag, "/authentication/pki"); // use deprecated API for back-compat, remove in 9.0
pkiAuthenticationPlugin.initializeMetrics(
solrMetricsContext.metricManager, solrMetricsContext.registry, solrMetricsContext.tag, "/authentication/pki");
TracerConfigurator.loadTracer(loader, cfg.getTracerConfiguratorPluginInfo(), getZkController().getZkStateReader()); TracerConfigurator.loadTracer(loader, cfg.getTracerConfiguratorPluginInfo(), getZkController().getZkStateReader());
} }
@ -665,7 +672,7 @@ public class CoreContainer {
metricsCollectorHandler.init(null); metricsCollectorHandler.init(null);
containerHandlers.put(AUTHZ_PATH, securityConfHandler); containerHandlers.put(AUTHZ_PATH, securityConfHandler);
securityConfHandler.initializeMetrics(metricManager, SolrInfoBean.Group.node.toString(), metricTag, AUTHZ_PATH); securityConfHandler.initializeMetrics(solrMetricsContext, AUTHZ_PATH);
containerHandlers.put(AUTHC_PATH, securityConfHandler); containerHandlers.put(AUTHC_PATH, securityConfHandler);
@ -680,22 +687,20 @@ public class CoreContainer {
// initialize gauges for reporting the number of cores and disk total/free // initialize gauges for reporting the number of cores and disk total/free
String registryName = SolrMetricManager.getRegistryName(SolrInfoBean.Group.node); solrMetricsContext.gauge(null, () -> solrCores.getCores().size(),
String metricTag = Integer.toHexString(hashCode()); true, "loaded", SolrInfoBean.Category.CONTAINER.toString(), "cores");
metricManager.registerGauge(null, registryName, () -> solrCores.getCores().size(), solrMetricsContext.gauge(null, () -> solrCores.getLoadedCoreNames().size() - solrCores.getCores().size(),
metricTag, true, "loaded", SolrInfoBean.Category.CONTAINER.toString(), "cores"); true, "lazy", SolrInfoBean.Category.CONTAINER.toString(), "cores");
metricManager.registerGauge(null, registryName, () -> solrCores.getLoadedCoreNames().size() - solrCores.getCores().size(), solrMetricsContext.gauge(null, () -> solrCores.getAllCoreNames().size() - solrCores.getLoadedCoreNames().size(),
metricTag, true, "lazy", SolrInfoBean.Category.CONTAINER.toString(), "cores"); true, "unloaded", SolrInfoBean.Category.CONTAINER.toString(), "cores");
metricManager.registerGauge(null, registryName, () -> solrCores.getAllCoreNames().size() - solrCores.getLoadedCoreNames().size(),
metricTag, true, "unloaded", SolrInfoBean.Category.CONTAINER.toString(), "cores");
Path dataHome = cfg.getSolrDataHome() != null ? cfg.getSolrDataHome() : cfg.getCoreRootDirectory(); Path dataHome = cfg.getSolrDataHome() != null ? cfg.getSolrDataHome() : cfg.getCoreRootDirectory();
metricManager.registerGauge(null, registryName, () -> dataHome.toFile().getTotalSpace(), solrMetricsContext.gauge(null, () -> dataHome.toFile().getTotalSpace(),
metricTag, true, "totalSpace", SolrInfoBean.Category.CONTAINER.toString(), "fs"); true, "totalSpace", SolrInfoBean.Category.CONTAINER.toString(), "fs");
metricManager.registerGauge(null, registryName, () -> dataHome.toFile().getUsableSpace(), solrMetricsContext.gauge(null, () -> dataHome.toFile().getUsableSpace(),
metricTag, true, "usableSpace", SolrInfoBean.Category.CONTAINER.toString(), "fs"); true, "usableSpace", SolrInfoBean.Category.CONTAINER.toString(), "fs");
metricManager.registerGauge(null, registryName, () -> dataHome.toAbsolutePath().toString(), solrMetricsContext.gauge(null, () -> dataHome.toAbsolutePath().toString(),
metricTag, true, "path", SolrInfoBean.Category.CONTAINER.toString(), "fs"); true, "path", SolrInfoBean.Category.CONTAINER.toString(), "fs");
metricManager.registerGauge(null, registryName, () -> { solrMetricsContext.gauge(null, () -> {
try { try {
return org.apache.lucene.util.IOUtils.spins(dataHome.toAbsolutePath()); return org.apache.lucene.util.IOUtils.spins(dataHome.toAbsolutePath());
} catch (IOException e) { } catch (IOException e) {
@ -703,14 +708,14 @@ public class CoreContainer {
return true; return true;
} }
}, },
metricTag, true, "spins", SolrInfoBean.Category.CONTAINER.toString(), "fs"); true, "spins", SolrInfoBean.Category.CONTAINER.toString(), "fs");
metricManager.registerGauge(null, registryName, () -> cfg.getCoreRootDirectory().toFile().getTotalSpace(), solrMetricsContext.gauge(null, () -> cfg.getCoreRootDirectory().toFile().getTotalSpace(),
metricTag, true, "totalSpace", SolrInfoBean.Category.CONTAINER.toString(), "fs", "coreRoot"); true, "totalSpace", SolrInfoBean.Category.CONTAINER.toString(), "fs", "coreRoot");
metricManager.registerGauge(null, registryName, () -> cfg.getCoreRootDirectory().toFile().getUsableSpace(), solrMetricsContext.gauge(null, () -> cfg.getCoreRootDirectory().toFile().getUsableSpace(),
metricTag, true, "usableSpace", SolrInfoBean.Category.CONTAINER.toString(), "fs", "coreRoot"); true, "usableSpace", SolrInfoBean.Category.CONTAINER.toString(), "fs", "coreRoot");
metricManager.registerGauge(null, registryName, () -> cfg.getCoreRootDirectory().toAbsolutePath().toString(), solrMetricsContext.gauge(null, () -> cfg.getCoreRootDirectory().toAbsolutePath().toString(),
metricTag, true, "path", SolrInfoBean.Category.CONTAINER.toString(), "fs", "coreRoot"); true, "path", SolrInfoBean.Category.CONTAINER.toString(), "fs", "coreRoot");
metricManager.registerGauge(null, registryName, () -> { solrMetricsContext.gauge(null, () -> {
try { try {
return org.apache.lucene.util.IOUtils.spins(cfg.getCoreRootDirectory().toAbsolutePath()); return org.apache.lucene.util.IOUtils.spins(cfg.getCoreRootDirectory().toAbsolutePath());
} catch (IOException e) { } catch (IOException e) {
@ -718,15 +723,15 @@ public class CoreContainer {
return true; return true;
} }
}, },
metricTag, true, "spins", SolrInfoBean.Category.CONTAINER.toString(), "fs", "coreRoot"); true, "spins", SolrInfoBean.Category.CONTAINER.toString(), "fs", "coreRoot");
// add version information // add version information
metricManager.registerGauge(null, registryName, () -> this.getClass().getPackage().getSpecificationVersion(), solrMetricsContext.gauge(null, () -> this.getClass().getPackage().getSpecificationVersion(),
metricTag, true, "specification", SolrInfoBean.Category.CONTAINER.toString(), "version"); true, "specification", SolrInfoBean.Category.CONTAINER.toString(), "version");
metricManager.registerGauge(null, registryName, () -> this.getClass().getPackage().getImplementationVersion(), solrMetricsContext.gauge(null, () -> this.getClass().getPackage().getImplementationVersion(),
metricTag, true, "implementation", SolrInfoBean.Category.CONTAINER.toString(), "version"); true, "implementation", SolrInfoBean.Category.CONTAINER.toString(), "version");
SolrFieldCacheBean fieldCacheBean = new SolrFieldCacheBean(); SolrFieldCacheBean fieldCacheBean = new SolrFieldCacheBean();
fieldCacheBean.initializeMetrics(metricManager, registryName, metricTag, null); fieldCacheBean.initializeMetrics(solrMetricsContext, null);
if (isZooKeeperAware()) { if (isZooKeeperAware()) {
metricManager.loadClusterReporters(metricReporters, this); metricManager.loadClusterReporters(metricReporters, this);
@ -815,7 +820,7 @@ public class CoreContainer {
// initialize this handler here when SolrCloudManager is ready // initialize this handler here when SolrCloudManager is ready
autoScalingHandler = new AutoScalingHandler(getZkController().getSolrCloudManager(), loader); autoScalingHandler = new AutoScalingHandler(getZkController().getSolrCloudManager(), loader);
containerHandlers.put(AutoScalingHandler.HANDLER_PATH, autoScalingHandler); containerHandlers.put(AutoScalingHandler.HANDLER_PATH, autoScalingHandler);
autoScalingHandler.initializeMetrics(metricManager, SolrInfoBean.Group.node.toString(), metricTag, AutoScalingHandler.HANDLER_PATH); autoScalingHandler.initializeMetrics(solrMetricsContext, AutoScalingHandler.HANDLER_PATH);
} }
// This is a bit redundant but these are two distinct concepts for all they're accomplished at the same time. // This is a bit redundant but these are two distinct concepts for all they're accomplished at the same time.
status |= LOAD_COMPLETE | INITIAL_CORE_LOAD_COMPLETE; status |= LOAD_COMPLETE | INITIAL_CORE_LOAD_COMPLETE;
@ -863,7 +868,7 @@ public class CoreContainer {
metricsHistoryHandler = new MetricsHistoryHandler(name, metricsHandler, metricsHistoryHandler = new MetricsHistoryHandler(name, metricsHandler,
client, cloudManager, initArgs); client, cloudManager, initArgs);
containerHandlers.put(METRICS_HISTORY_PATH, metricsHistoryHandler); containerHandlers.put(METRICS_HISTORY_PATH, metricsHistoryHandler);
metricsHistoryHandler.initializeMetrics(metricManager, SolrInfoBean.Group.node.toString(), metricTag, METRICS_HISTORY_PATH); metricsHistoryHandler.initializeMetrics(solrMetricsContext, METRICS_HISTORY_PATH);
} }
public void securityNodeChanged() { public void securityNodeChanged() {
@ -1796,7 +1801,9 @@ public class CoreContainer {
containerHandlers.put(path, (SolrRequestHandler) handler); containerHandlers.put(path, (SolrRequestHandler) handler);
} }
if (handler instanceof SolrMetricProducer) { if (handler instanceof SolrMetricProducer) {
((SolrMetricProducer) handler).initializeMetrics(metricManager, SolrInfoBean.Group.node.toString(), metricTag, path); // use deprecated method for back-compat, remove in 9.0
((SolrMetricProducer) handler).initializeMetrics(solrMetricsContext.metricManager,
solrMetricsContext.registry, solrMetricsContext.tag, path);
} }
return handler; return handler;
} }

15
solr/core/src/java/org/apache/solr/core/HdfsDirectoryFactory.java
View File

@ -53,8 +53,8 @@ import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.params.SolrParams; import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.IOUtils; import org.apache.solr.common.util.IOUtils;
import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.NamedList;
import org.apache.solr.metrics.SolrMetricManager;
import org.apache.solr.metrics.SolrMetricProducer; import org.apache.solr.metrics.SolrMetricProducer;
import org.apache.solr.metrics.SolrMetricsContext;
import org.apache.solr.store.blockcache.BlockCache; import org.apache.solr.store.blockcache.BlockCache;
import org.apache.solr.store.blockcache.BlockDirectory; import org.apache.solr.store.blockcache.BlockDirectory;
import org.apache.solr.store.blockcache.BlockDirectoryCache; import org.apache.solr.store.blockcache.BlockDirectoryCache;
@ -141,6 +141,13 @@ public class HdfsDirectoryFactory extends CachingDirectoryFactory implements Sol
} }
tmpFsCache.invalidateAll(); tmpFsCache.invalidateAll();
tmpFsCache.cleanUp(); tmpFsCache.cleanUp();
try {
SolrMetricProducer.super.close();
MetricsHolder.metrics.close();
LocalityHolder.reporter.close();
} catch (Exception e) {
throw new IOException(e);
}
} }
private final static class LocalityHolder { private final static class LocalityHolder {
@ -497,9 +504,9 @@ public class HdfsDirectoryFactory extends CachingDirectoryFactory implements Sol
} }
@Override @Override
public void initializeMetrics(SolrMetricManager manager, String registry, String tag, String scope) { public void initializeMetrics(SolrMetricsContext parentContext, String scope) {
MetricsHolder.metrics.initializeMetrics(manager, registry, tag, scope); MetricsHolder.metrics.initializeMetrics(parentContext, scope);
LocalityHolder.reporter.initializeMetrics(manager, registry, tag, scope); LocalityHolder.reporter.initializeMetrics(parentContext, scope);
} }
@Override @Override

18
solr/core/src/java/org/apache/solr/core/PluginBag.java
View File

@ -195,7 +195,7 @@ public class PluginBag<T> implements AutoCloseable {
return old == null ? null : old.get(); return old == null ? null : old.get();
} }
PluginHolder<T> put(String name, PluginHolder<T> plugin) { public PluginHolder<T> put(String name, PluginHolder<T> plugin) {
Boolean registerApi = null; Boolean registerApi = null;
Boolean disableHandler = null; Boolean disableHandler = null;
if (plugin.pluginInfo != null) { if (plugin.pluginInfo != null) {
@ -231,11 +231,15 @@ public class PluginBag<T> implements AutoCloseable {
apiBag.registerLazy((PluginHolder<SolrRequestHandler>) plugin, plugin.pluginInfo); apiBag.registerLazy((PluginHolder<SolrRequestHandler>) plugin, plugin.pluginInfo);
} }
} }
if(disableHandler == null) disableHandler = Boolean.FALSE; if (disableHandler == null) disableHandler = Boolean.FALSE;
PluginHolder<T> old = null; PluginHolder<T> old = null;
if(!disableHandler) old = registry.put(name, plugin); if (!disableHandler) old = registry.put(name, plugin);
if (plugin.pluginInfo != null && plugin.pluginInfo.isDefault()) setDefault(name); if (plugin.pluginInfo != null && plugin.pluginInfo.isDefault()) setDefault(name);
if (plugin.isLoaded()) registerMBean(plugin.get(), core, name); if (plugin.isLoaded()) registerMBean(plugin.get(), core, name);
// old instance has been replaced - close it to prevent mem leaks
if (old != null && old != plugin) {
closeQuietly(old);
}
return old; return old;
} }
@ -324,6 +328,14 @@ public class PluginBag<T> implements AutoCloseable {
} }
} }
public static void closeQuietly(Object inst) {
try {
if (inst != null && inst instanceof AutoCloseable) ((AutoCloseable) inst).close();
} catch (Exception e) {
log.error("Error closing "+ inst , e);
}
}
/** /**
* An indirect reference to a plugin. It just wraps a plugin instance. * An indirect reference to a plugin. It just wraps a plugin instance.
* subclasses may choose to lazily load the plugin * subclasses may choose to lazily load the plugin

68
solr/core/src/java/org/apache/solr/core/SolrCore.java
View File

@ -108,8 +108,8 @@ import org.apache.solr.handler.component.HighlightComponent;
import org.apache.solr.handler.component.SearchComponent; import org.apache.solr.handler.component.SearchComponent;
import org.apache.solr.logging.MDCLoggingContext; import org.apache.solr.logging.MDCLoggingContext;
import org.apache.solr.metrics.SolrCoreMetricManager; import org.apache.solr.metrics.SolrCoreMetricManager;
import org.apache.solr.metrics.SolrMetricManager;
import org.apache.solr.metrics.SolrMetricProducer; import org.apache.solr.metrics.SolrMetricProducer;
import org.apache.solr.metrics.SolrMetricsContext;
import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.request.SolrRequestHandler; import org.apache.solr.request.SolrRequestHandler;
import org.apache.solr.response.BinaryResponseWriter; import org.apache.solr.response.BinaryResponseWriter;
@ -231,7 +231,8 @@ public final class SolrCore implements SolrInfoBean, SolrMetricProducer, Closeab
private final CoreContainer coreContainer; private final CoreContainer coreContainer;
private Set<String> metricNames = ConcurrentHashMap.newKeySet(); private Set<String> metricNames = ConcurrentHashMap.newKeySet();
private String metricTag = Integer.toHexString(hashCode()); private final String metricTag = SolrMetricProducer.getUniqueMetricTag(this, null);
private final SolrMetricsContext solrMetricsContext;
public volatile boolean searchEnabled = true; public volatile boolean searchEnabled = true;
public volatile boolean indexEnabled = true; public volatile boolean indexEnabled = true;
@ -919,6 +920,7 @@ public final class SolrCore implements SolrInfoBean, SolrMetricProducer, Closeab
this.configSetProperties = configSetProperties; this.configSetProperties = configSetProperties;
// Initialize the metrics manager // Initialize the metrics manager
this.coreMetricManager = initCoreMetricManager(config); this.coreMetricManager = initCoreMetricManager(config);
solrMetricsContext = coreMetricManager.getSolrMetricsContext();
this.coreMetricManager.loadReporters(); this.coreMetricManager.loadReporters();
if (updateHandler == null) { if (updateHandler == null) {
@ -940,15 +942,13 @@ public final class SolrCore implements SolrInfoBean, SolrMetricProducer, Closeab
checkVersionFieldExistsInSchema(schema, coreDescriptor); checkVersionFieldExistsInSchema(schema, coreDescriptor);
SolrMetricManager metricManager = coreContainer.getMetricManager();
// initialize searcher-related metrics // initialize searcher-related metrics
initializeMetrics(metricManager, coreMetricManager.getRegistryName(), metricTag, null); initializeMetrics(solrMetricsContext, null);
SolrFieldCacheBean solrFieldCacheBean = new SolrFieldCacheBean(); SolrFieldCacheBean solrFieldCacheBean = new SolrFieldCacheBean();
// this is registered at the CONTAINER level because it's not core-specific - for now we // this is registered at the CONTAINER level because it's not core-specific - for now we
// also register it here for back-compat // also register it here for back-compat
solrFieldCacheBean.initializeMetrics(metricManager, coreMetricManager.getRegistryName(), metricTag, "core"); solrFieldCacheBean.initializeMetrics(solrMetricsContext, "core");
infoRegistry.put("fieldCache", solrFieldCacheBean); infoRegistry.put("fieldCache", solrFieldCacheBean);
initSchema(config, schema); initSchema(config, schema);
@ -1015,8 +1015,9 @@ public final class SolrCore implements SolrInfoBean, SolrMetricProducer, Closeab
// Allow the directory factory to report metrics // Allow the directory factory to report metrics
if (directoryFactory instanceof SolrMetricProducer) { if (directoryFactory instanceof SolrMetricProducer) {
((SolrMetricProducer) directoryFactory).initializeMetrics(metricManager, coreMetricManager.getRegistryName(), // XXX use deprecated method for back-compat, remove in 9.0
metricTag, "directoryFactory"); ((SolrMetricProducer) directoryFactory).initializeMetrics(
solrMetricsContext.metricManager, solrMetricsContext.registry, solrMetricsContext.tag, "directoryFactory");
} }
// seed version buckets with max from index during core initialization ... requires a searcher! // seed version buckets with max from index during core initialization ... requires a searcher!
@ -1163,61 +1164,66 @@ public final class SolrCore implements SolrInfoBean, SolrMetricProducer, Closeab
} }
@Override @Override
public void initializeMetrics(SolrMetricManager manager, String registry, String tag, String scope) { public void initializeMetrics(SolrMetricsContext parentContext, String scope) {
newSearcherCounter = manager.counter(this, registry, "new", Category.SEARCHER.toString()); newSearcherCounter = parentContext.counter(this, "new", Category.SEARCHER.toString());
newSearcherTimer = manager.timer(this, registry, "time", Category.SEARCHER.toString(), "new"); newSearcherTimer = parentContext.timer(this, "time", Category.SEARCHER.toString(), "new");
newSearcherWarmupTimer = manager.timer(this, registry, "warmup", Category.SEARCHER.toString(), "new"); newSearcherWarmupTimer = parentContext.timer(this, "warmup", Category.SEARCHER.toString(), "new");
newSearcherMaxReachedCounter = manager.counter(this, registry, "maxReached", Category.SEARCHER.toString(), "new"); newSearcherMaxReachedCounter = parentContext.counter(this, "maxReached", Category.SEARCHER.toString(), "new");
newSearcherOtherErrorsCounter = manager.counter(this, registry, "errors", Category.SEARCHER.toString(), "new"); newSearcherOtherErrorsCounter = parentContext.counter(this, "errors", Category.SEARCHER.toString(), "new");
manager.registerGauge(this, registry, () -> name == null ? "(null)" : name, getMetricTag(), true, "coreName", Category.CORE.toString()); parentContext.gauge(this, () -> name == null ? "(null)" : name, true, "coreName", Category.CORE.toString());
manager.registerGauge(this, registry, () -> startTime, getMetricTag(), true, "startTime", Category.CORE.toString()); parentContext.gauge(this, () -> startTime, true, "startTime", Category.CORE.toString());
manager.registerGauge(this, registry, () -> getOpenCount(), getMetricTag(), true, "refCount", Category.CORE.toString()); parentContext.gauge(this, () -> getOpenCount(), true, "refCount", Category.CORE.toString());
manager.registerGauge(this, registry, () -> resourceLoader.getInstancePath().toString(), getMetricTag(), true, "instanceDir", Category.CORE.toString()); parentContext.gauge(this, () -> resourceLoader.getInstancePath().toString(), true, "instanceDir", Category.CORE.toString());
manager.registerGauge(this, registry, () -> isClosed() ? "(closed)" : getIndexDir(), getMetricTag(), true, "indexDir", Category.CORE.toString()); parentContext.gauge(this, () -> isClosed() ? "(closed)" : getIndexDir(), true, "indexDir", Category.CORE.toString());
manager.registerGauge(this, registry, () -> isClosed() ? 0 : getIndexSize(), getMetricTag(), true, "sizeInBytes", Category.INDEX.toString()); parentContext.gauge(this, () -> isClosed() ? 0 : getIndexSize(), true, "sizeInBytes", Category.INDEX.toString());
manager.registerGauge(this, registry, () -> isClosed() ? "(closed)" : NumberUtils.readableSize(getIndexSize()), getMetricTag(), true, "size", Category.INDEX.toString()); parentContext.gauge(this, () -> isClosed() ? "(closed)" : NumberUtils.readableSize(getIndexSize()), true, "size", Category.INDEX.toString());
if (coreContainer != null) { if (coreContainer != null) {
manager.registerGauge(this, registry, () -> coreContainer.getNamesForCore(this), getMetricTag(), true, "aliases", Category.CORE.toString()); parentContext.gauge(this, () -> coreContainer.getNamesForCore(this), true, "aliases", Category.CORE.toString());
final CloudDescriptor cd = getCoreDescriptor().getCloudDescriptor(); final CloudDescriptor cd = getCoreDescriptor().getCloudDescriptor();
if (cd != null) { if (cd != null) {
manager.registerGauge(this, registry, () -> { parentContext.gauge(this, () -> {
if (cd.getCollectionName() != null) { if (cd.getCollectionName() != null) {
return cd.getCollectionName(); return cd.getCollectionName();
} else { } else {
return "_notset_"; return "_notset_";
} }
}, getMetricTag(), true, "collection", Category.CORE.toString()); }, true, "collection", Category.CORE.toString());
manager.registerGauge(this, registry, () -> { parentContext.gauge(this, () -> {
if (cd.getShardId() != null) { if (cd.getShardId() != null) {
return cd.getShardId(); return cd.getShardId();
} else { } else {
return "_auto_"; return "_auto_";
} }
}, getMetricTag(), true, "shard", Category.CORE.toString()); }, true, "shard", Category.CORE.toString());
} }
} }
// initialize disk total / free metrics // initialize disk total / free metrics
Path dataDirPath = Paths.get(dataDir); Path dataDirPath = Paths.get(dataDir);
File dataDirFile = dataDirPath.toFile(); File dataDirFile = dataDirPath.toFile();
manager.registerGauge(this, registry, () -> dataDirFile.getTotalSpace(), getMetricTag(), true, "totalSpace", Category.CORE.toString(), "fs"); parentContext.gauge(this, () -> dataDirFile.getTotalSpace(), true, "totalSpace", Category.CORE.toString(), "fs");
manager.registerGauge(this, registry, () -> dataDirFile.getUsableSpace(), getMetricTag(), true, "usableSpace", Category.CORE.toString(), "fs"); parentContext.gauge(this, () -> dataDirFile.getUsableSpace(), true, "usableSpace", Category.CORE.toString(), "fs");
manager.registerGauge(this, registry, () -> dataDirPath.toAbsolutePath().toString(), getMetricTag(), true, "path", Category.CORE.toString(), "fs"); parentContext.gauge(this, () -> dataDirPath.toAbsolutePath().toString(), true, "path", Category.CORE.toString(), "fs");
manager.registerGauge(this, registry, () -> { parentContext.gauge(this, () -> {
try { try {
return org.apache.lucene.util.IOUtils.spins(dataDirPath.toAbsolutePath()); return org.apache.lucene.util.IOUtils.spins(dataDirPath.toAbsolutePath());
} catch (IOException e) { } catch (IOException e) {
// default to spinning // default to spinning
return true; return true;
} }
}, getMetricTag(), true, "spins", Category.CORE.toString(), "fs"); }, true, "spins", Category.CORE.toString(), "fs");
} }
public String getMetricTag() { public String getMetricTag() {
return metricTag; return metricTag;
} }
@Override
public SolrMetricsContext getSolrMetricsContext() {
return solrMetricsContext;
}
private void checkVersionFieldExistsInSchema(IndexSchema schema, CoreDescriptor coreDescriptor) { private void checkVersionFieldExistsInSchema(IndexSchema schema, CoreDescriptor coreDescriptor) {
if (null != coreDescriptor.getCloudDescriptor()) { if (null != coreDescriptor.getCloudDescriptor()) {
// we are evidently running in cloud mode. // we are evidently running in cloud mode.

6
solr/core/src/java/org/apache/solr/core/SolrInfoBean.java
View File

@ -21,6 +21,8 @@ import java.util.Set;
import com.codahale.metrics.MetricRegistry; import com.codahale.metrics.MetricRegistry;
import org.apache.solr.metrics.SolrMetricManager; import org.apache.solr.metrics.SolrMetricManager;
import org.apache.solr.metrics.SolrMetricProducer;
import org.apache.solr.metrics.SolrMetricsContext;
import org.apache.solr.util.stats.MetricUtils; import org.apache.solr.util.stats.MetricUtils;
/** /**
@ -77,6 +79,10 @@ public interface SolrInfoBean {
* (default is null, which means no registry). * (default is null, which means no registry).
*/ */
default MetricRegistry getMetricRegistry() { default MetricRegistry getMetricRegistry() {
if (this instanceof SolrMetricProducer) {
SolrMetricsContext context = ((SolrMetricProducer)this).getSolrMetricsContext();
return context != null ? context.getMetricRegistry() : null;
}
return null; return null;
} }

47
solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java
View File

@ -93,7 +93,7 @@ import org.apache.solr.core.backup.repository.BackupRepository;
import org.apache.solr.core.backup.repository.LocalFileSystemRepository; import org.apache.solr.core.backup.repository.LocalFileSystemRepository;
import org.apache.solr.handler.IndexFetcher.IndexFetchResult; import org.apache.solr.handler.IndexFetcher.IndexFetchResult;
import org.apache.solr.metrics.MetricsMap; import org.apache.solr.metrics.MetricsMap;
import org.apache.solr.metrics.SolrMetricManager; import org.apache.solr.metrics.SolrMetricsContext;
import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.SolrQueryResponse; import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.search.SolrIndexSearcher; import org.apache.solr.search.SolrIndexSearcher;
@ -865,21 +865,20 @@ public class ReplicationHandler extends RequestHandlerBase implements SolrCoreAw
} }
@Override @Override
public void initializeMetrics(SolrMetricManager manager, String registry, String tag, String scope) { public void initializeMetrics(SolrMetricsContext parentContext, String scope) {
super.initializeMetrics(manager, registry, tag, scope); super.initializeMetrics(parentContext, scope);
solrMetricsContext.gauge(this, () -> (core != null && !core.isClosed() ? NumberUtils.readableSize(core.getIndexSize()) : ""),
manager.registerGauge(this, registry, () -> (core != null && !core.isClosed() ? NumberUtils.readableSize(core.getIndexSize()) : ""), true, "indexSize", getCategory().toString(), scope);
tag, true, "indexSize", getCategory().toString(), scope); solrMetricsContext.gauge(this, () -> (core != null && !core.isClosed() ? getIndexVersion().toString() : ""),
manager.registerGauge(this, registry, () -> (core != null && !core.isClosed() ? getIndexVersion().toString() : ""), true, "indexVersion", getCategory().toString(), scope);
tag, true, "indexVersion", getCategory().toString(), scope); solrMetricsContext.gauge(this, () -> (core != null && !core.isClosed() ? getIndexVersion().generation : 0),
manager.registerGauge(this, registry, () -> (core != null && !core.isClosed() ? getIndexVersion().generation : 0), true, GENERATION, getCategory().toString(), scope);
tag, true, GENERATION, getCategory().toString(), scope); solrMetricsContext.gauge(this, () -> (core != null && !core.isClosed() ? core.getIndexDir() : ""),
manager.registerGauge(this, registry, () -> (core != null && !core.isClosed() ? core.getIndexDir() : ""), true, "indexPath", getCategory().toString(), scope);
tag, true, "indexPath", getCategory().toString(), scope); solrMetricsContext.gauge(this, () -> isMaster,
manager.registerGauge(this, registry, () -> isMaster, true, "isMaster", getCategory().toString(), scope);
tag, true, "isMaster", getCategory().toString(), scope); solrMetricsContext.gauge(this, () -> isSlave,
manager.registerGauge(this, registry, () -> isSlave, true, "isSlave", getCategory().toString(), scope);
tag, true, "isSlave", getCategory().toString(), scope);
final MetricsMap fetcherMap = new MetricsMap((detailed, map) -> { final MetricsMap fetcherMap = new MetricsMap((detailed, map) -> {
IndexFetcher fetcher = currentIndexFetcher; IndexFetcher fetcher = currentIndexFetcher;
if (fetcher != null) { if (fetcher != null) {
@ -908,13 +907,13 @@ public class ReplicationHandler extends RequestHandlerBase implements SolrCoreAw
addVal(map, IndexFetcher.CONF_FILES_REPLICATED, props, String.class); addVal(map, IndexFetcher.CONF_FILES_REPLICATED, props, String.class);
} }
}); });
manager.registerGauge(this, registry, fetcherMap, tag, true, "fetcher", getCategory().toString(), scope); solrMetricsContext.gauge(this , fetcherMap, true, "fetcher", getCategory().toString(), scope);
manager.registerGauge(this, registry, () -> isMaster && includeConfFiles != null ? includeConfFiles : "", solrMetricsContext.gauge(this, () -> isMaster && includeConfFiles != null ? includeConfFiles : "",
tag, true, "confFilesToReplicate", getCategory().toString(), scope); true, "confFilesToReplicate", getCategory().toString(), scope);
manager.registerGauge(this, registry, () -> isMaster ? getReplicateAfterStrings() : Collections.<String>emptyList(), solrMetricsContext.gauge(this, () -> isMaster ? getReplicateAfterStrings() : Collections.<String>emptyList(),
tag, true, REPLICATE_AFTER, getCategory().toString(), scope); true, REPLICATE_AFTER, getCategory().toString(), scope);
manager.registerGauge(this, registry, () -> isMaster && replicationEnabled.get(), solrMetricsContext.gauge(this, () -> isMaster && replicationEnabled.get(),
tag, true, "replicationEnabled", getCategory().toString(), scope); true, "replicationEnabled", getCategory().toString(), scope);
} }
//TODO Should a failure retrieving any piece of info mark the overall request as a failure? Is there a core set of values that are required to make a response here useful? //TODO Should a failure retrieving any piece of info mark the overall request as a failure? Is there a core set of values that are required to make a response here useful?
@ -1394,7 +1393,7 @@ public class ReplicationHandler extends RequestHandlerBase implements SolrCoreAw
}); });
} }
public void close() { public void shutdown() {
if (executorService != null) executorService.shutdown(); if (executorService != null) executorService.shutdown();
if (pollingIndexFetcher != null) { if (pollingIndexFetcher != null) {
pollingIndexFetcher.destroy(); pollingIndexFetcher.destroy();

49
solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java
View File

@ -22,11 +22,13 @@ import java.util.Map;
import java.util.Set; import java.util.Set;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
import com.codahale.metrics.MetricRegistry;
import com.google.common.collect.ImmutableList;
import com.codahale.metrics.Counter; import com.codahale.metrics.Counter;
import com.codahale.metrics.Meter; import com.codahale.metrics.Meter;
import com.codahale.metrics.Timer; import com.codahale.metrics.Timer;
import com.google.common.collect.ImmutableList;
import org.apache.solr.api.Api;
import org.apache.solr.api.ApiBag;
import org.apache.solr.api.ApiSupport;
import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.ShardParams; import org.apache.solr.common.params.ShardParams;
import org.apache.solr.common.params.SolrParams; import org.apache.solr.common.params.SolrParams;
@ -36,16 +38,13 @@ import org.apache.solr.core.PluginBag;
import org.apache.solr.core.PluginInfo; import org.apache.solr.core.PluginInfo;
import org.apache.solr.core.SolrInfoBean; import org.apache.solr.core.SolrInfoBean;
import org.apache.solr.metrics.MetricsMap; import org.apache.solr.metrics.MetricsMap;
import org.apache.solr.metrics.SolrMetricManager;
import org.apache.solr.metrics.SolrMetricProducer; import org.apache.solr.metrics.SolrMetricProducer;
import org.apache.solr.metrics.SolrMetricsContext;
import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.request.SolrRequestHandler; import org.apache.solr.request.SolrRequestHandler;
import org.apache.solr.response.SolrQueryResponse; import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.search.SyntaxError; import org.apache.solr.search.SyntaxError;
import org.apache.solr.util.SolrPluginUtils; import org.apache.solr.util.SolrPluginUtils;
import org.apache.solr.api.Api;
import org.apache.solr.api.ApiBag;
import org.apache.solr.api.ApiSupport;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@ -79,9 +78,7 @@ public abstract class RequestHandlerBase implements SolrRequestHandler, SolrInfo
private PluginInfo pluginInfo; private PluginInfo pluginInfo;
private Set<String> metricNames = ConcurrentHashMap.newKeySet(); private Set<String> metricNames = ConcurrentHashMap.newKeySet();
private MetricRegistry registry; protected SolrMetricsContext solrMetricsContext;
protected String registryName;
protected SolrMetricManager metricManager;
@SuppressForbidden(reason = "Need currentTimeMillis, used only for stats output") @SuppressForbidden(reason = "Need currentTimeMillis, used only for stats output")
@ -144,21 +141,24 @@ public abstract class RequestHandlerBase implements SolrRequestHandler, SolrInfo
} }
@Override @Override
public void initializeMetrics(SolrMetricManager manager, String registryName, String tag, final String scope) { public SolrMetricsContext getSolrMetricsContext() {
this.metricManager = manager; return solrMetricsContext;
this.registryName = registryName; }
this.registry = manager.registry(registryName);
numErrors = manager.meter(this, registryName, "errors", getCategory().toString(), scope); @Override
numServerErrors = manager.meter(this, registryName, "serverErrors", getCategory().toString(), scope); public void initializeMetrics(SolrMetricsContext parentContext, String scope) {
numClientErrors = manager.meter(this, registryName, "clientErrors", getCategory().toString(), scope); this.solrMetricsContext = parentContext.getChildContext(this);
numTimeouts = manager.meter(this, registryName, "timeouts", getCategory().toString(), scope); numErrors = solrMetricsContext.meter(this, "errors", getCategory().toString(), scope);
requests = manager.counter(this, registryName, "requests", getCategory().toString(), scope); numServerErrors = solrMetricsContext.meter(this, "serverErrors", getCategory().toString(), scope);
numClientErrors = solrMetricsContext.meter(this, "clientErrors", getCategory().toString(), scope);
numTimeouts = solrMetricsContext.meter(this, "timeouts", getCategory().toString(), scope);
requests = solrMetricsContext.counter(this, "requests", getCategory().toString(), scope);
MetricsMap metricsMap = new MetricsMap((detail, map) -> MetricsMap metricsMap = new MetricsMap((detail, map) ->
shardPurposes.forEach((k, v) -> map.put(k, v.getCount()))); shardPurposes.forEach((k, v) -> map.put(k, v.getCount())));
manager.registerGauge(this, registryName, metricsMap, tag, true, "shardRequests", getCategory().toString(), scope); solrMetricsContext.gauge(this, metricsMap, true, "shardRequests", getCategory().toString(), scope);
requestTimes = manager.timer(this, registryName, "requestTimes", getCategory().toString(), scope); requestTimes = solrMetricsContext.timer(this,"requestTimes", getCategory().toString(), scope);
totalTime = manager.counter(this, registryName, "totalTime", getCategory().toString(), scope); totalTime = solrMetricsContext.counter(this, "totalTime", getCategory().toString(), scope);
manager.registerGauge(this, registryName, () -> handlerStart, tag, true, "handlerStart", getCategory().toString(), scope); solrMetricsContext.gauge(this, () -> handlerStart, true, "handlerStart", getCategory().toString(), scope);
} }
public static SolrParams getSolrParamsFromNamedList(NamedList args, String key) { public static SolrParams getSolrParamsFromNamedList(NamedList args, String key) {
@ -272,11 +272,6 @@ public abstract class RequestHandlerBase implements SolrRequestHandler, SolrInfo
return metricNames; return metricNames;
} }
@Override
public MetricRegistry getMetricRegistry() {
return registry;
}
@Override @Override
public SolrRequestHandler getSubHandler(String subPath) { public SolrRequestHandler getSubHandler(String subPath) {
return null; return null;

9
solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java
View File

@ -46,6 +46,7 @@ import org.apache.solr.core.CoreDescriptor;
import org.apache.solr.handler.RequestHandlerBase; import org.apache.solr.handler.RequestHandlerBase;
import org.apache.solr.logging.MDCLoggingContext; import org.apache.solr.logging.MDCLoggingContext;
import org.apache.solr.metrics.SolrMetricManager; import org.apache.solr.metrics.SolrMetricManager;
import org.apache.solr.metrics.SolrMetricsContext;
import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.SolrQueryResponse; import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.security.AuthorizationContext; import org.apache.solr.security.AuthorizationContext;
@ -120,10 +121,10 @@ public class CoreAdminHandler extends RequestHandlerBase implements PermissionNa
} }
@Override @Override
public void initializeMetrics(SolrMetricManager manager, String registryName, String tag, String scope) { public void initializeMetrics(SolrMetricsContext parentContext, String scope) {
super.initializeMetrics(manager, registryName, tag, scope); super.initializeMetrics(parentContext, scope);
parallelExecutor = MetricUtils.instrumentedExecutorService(parallelExecutor, this, manager.registry(registryName), parallelExecutor = MetricUtils.instrumentedExecutorService(parallelExecutor, this, solrMetricsContext.getMetricRegistry(),
SolrMetricManager.mkName("parallelCoreAdminExecutor", getCategory().name(),scope, "threadPool")); SolrMetricManager.mkName("parallelCoreAdminExecutor", getCategory().name(), scope, "threadPool"));
} }
@Override @Override
public Boolean registerV2() { public Boolean registerV2() {

21
solr/core/src/java/org/apache/solr/handler/component/SuggestComponent.java
View File

@ -48,8 +48,8 @@ import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.core.SolrCore; import org.apache.solr.core.SolrCore;
import org.apache.solr.core.SolrEventListener; import org.apache.solr.core.SolrEventListener;
import org.apache.solr.metrics.MetricsMap; import org.apache.solr.metrics.MetricsMap;
import org.apache.solr.metrics.SolrMetricManager;
import org.apache.solr.metrics.SolrMetricProducer; import org.apache.solr.metrics.SolrMetricProducer;
import org.apache.solr.metrics.SolrMetricsContext;
import org.apache.solr.search.SolrIndexSearcher; import org.apache.solr.search.SolrIndexSearcher;
import org.apache.solr.spelling.suggest.SolrSuggester; import org.apache.solr.spelling.suggest.SolrSuggester;
import org.apache.solr.spelling.suggest.SuggesterOptions; import org.apache.solr.spelling.suggest.SuggesterOptions;
@ -88,8 +88,7 @@ public class SuggestComponent extends SearchComponent implements SolrCoreAware,
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
protected NamedList initParams; protected NamedList initParams;
protected SolrMetricManager metricManager; protected SolrMetricsContext metricsContext;
protected String registryName;
/** /**
* Key is the dictionary name used in SolrConfig, value is the corresponding {@link SolrSuggester} * Key is the dictionary name used in SolrConfig, value is the corresponding {@link SolrSuggester}
@ -351,18 +350,22 @@ public class SuggestComponent extends SearchComponent implements SolrCoreAware,
} }
@Override @Override
public void initializeMetrics(SolrMetricManager manager, String registryName, String tag, String scope) { public SolrMetricsContext getSolrMetricsContext() {
this.registryName = registryName; return metricsContext;
this.metricManager = manager; }
registry = manager.registry(registryName);
manager.registerGauge(this, registryName, () -> ramBytesUsed(), tag, true, "totalSizeInBytes", getCategory().toString(), scope); @Override
public void initializeMetrics(SolrMetricsContext parentContext, String scope) {
this.metricsContext = parentContext.getChildContext(this);
this.metricsContext.gauge(this, () -> ramBytesUsed(), true, "totalSizeInBytes", getCategory().toString());
MetricsMap suggestersMap = new MetricsMap((detailed, map) -> { MetricsMap suggestersMap = new MetricsMap((detailed, map) -> {
for (Map.Entry<String, SolrSuggester> entry : suggesters.entrySet()) { for (Map.Entry<String, SolrSuggester> entry : suggesters.entrySet()) {
SolrSuggester suggester = entry.getValue(); SolrSuggester suggester = entry.getValue();
map.put(entry.getKey(), suggester.toString()); map.put(entry.getKey(), suggester.toString());
} }
}); });
manager.registerGauge(this, registryName, suggestersMap, tag, true, "suggesters", getCategory().toString(), scope); this.metricsContext.gauge(this, suggestersMap, true, "suggesters", getCategory().toString(), scope);
} }
@Override @Override

47
solr/core/src/java/org/apache/solr/metrics/SolrCoreMetricManager.java
View File

@ -40,9 +40,8 @@ public class SolrCoreMetricManager implements Closeable {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private final SolrCore core; private final SolrCore core;
private final String tag; private SolrMetricsContext solrMetricsContext;
private final SolrMetricManager metricManager; private SolrMetricManager metricManager;
private String registryName;
private String collectionName; private String collectionName;
private String shardName; private String shardName;
private String replicaName; private String replicaName;
@ -56,10 +55,10 @@ public class SolrCoreMetricManager implements Closeable {
*/ */
public SolrCoreMetricManager(SolrCore core) { public SolrCoreMetricManager(SolrCore core) {
this.core = core; this.core = core;
this.tag = core.getMetricTag();
this.metricManager = core.getCoreContainer().getMetricManager();
initCloudMode(); initCloudMode();
registryName = createRegistryName(cloudMode, collectionName, shardName, replicaName, core.getName()); metricManager = core.getCoreContainer().getMetricManager();
String registryName = createRegistryName(cloudMode, collectionName, shardName, replicaName, core.getName());
solrMetricsContext = new SolrMetricsContext(metricManager, registryName, core.getMetricTag());
leaderRegistryName = createLeaderRegistryName(cloudMode, collectionName, shardName); leaderRegistryName = createLeaderRegistryName(cloudMode, collectionName, shardName);
} }
@ -86,8 +85,8 @@ public class SolrCoreMetricManager implements Closeable {
CoreContainer coreContainer = core.getCoreContainer(); CoreContainer coreContainer = core.getCoreContainer();
NodeConfig nodeConfig = coreContainer.getConfig(); NodeConfig nodeConfig = coreContainer.getConfig();
PluginInfo[] pluginInfos = nodeConfig.getMetricsConfig().getMetricReporters(); PluginInfo[] pluginInfos = nodeConfig.getMetricsConfig().getMetricReporters();
metricManager.loadReporters(pluginInfos, core.getResourceLoader(), coreContainer, core, tag, metricManager.loadReporters(pluginInfos, core.getResourceLoader(), coreContainer, core, solrMetricsContext.tag,
SolrInfoBean.Group.core, registryName); SolrInfoBean.Group.core, solrMetricsContext.registry);
if (cloudMode) { if (cloudMode) {
metricManager.loadShardReporters(pluginInfos, core); metricManager.loadShardReporters(pluginInfos, core);
} }
@ -99,19 +98,20 @@ public class SolrCoreMetricManager implements Closeable {
* This method also reloads reporters so that they use the new core name. * This method also reloads reporters so that they use the new core name.
*/ */
public void afterCoreSetName() { public void afterCoreSetName() {
String oldRegistryName = registryName; String oldRegistryName = solrMetricsContext.registry;
String oldLeaderRegistryName = leaderRegistryName; String oldLeaderRegistryName = leaderRegistryName;
initCloudMode(); initCloudMode();
registryName = createRegistryName(cloudMode, collectionName, shardName, replicaName, core.getName()); String newRegistryName = createRegistryName(cloudMode, collectionName, shardName, replicaName, core.getName());
leaderRegistryName = createLeaderRegistryName(cloudMode, collectionName, shardName); leaderRegistryName = createLeaderRegistryName(cloudMode, collectionName, shardName);
if (oldRegistryName.equals(registryName)) { if (oldRegistryName.equals(newRegistryName)) {
return; return;
} }
// close old reporters // close old reporters
metricManager.closeReporters(oldRegistryName, tag); metricManager.closeReporters(oldRegistryName, solrMetricsContext.tag);
if (oldLeaderRegistryName != null) { if (oldLeaderRegistryName != null) {
metricManager.closeReporters(oldLeaderRegistryName, tag); metricManager.closeReporters(oldLeaderRegistryName, solrMetricsContext.tag);
} }
solrMetricsContext = new SolrMetricsContext(metricManager, newRegistryName, solrMetricsContext.tag);
// load reporters again, using the new core name // load reporters again, using the new core name
loadReporters(); loadReporters();
} }
@ -127,15 +127,16 @@ public class SolrCoreMetricManager implements Closeable {
throw new IllegalArgumentException("registerMetricProducer() called with illegal arguments: " + throw new IllegalArgumentException("registerMetricProducer() called with illegal arguments: " +
"scope = " + scope + ", producer = " + producer); "scope = " + scope + ", producer = " + producer);
} }
producer.initializeMetrics(metricManager, getRegistryName(), tag, scope); // use deprecated method for back-compat, remove in 9.0
producer.initializeMetrics(solrMetricsContext.metricManager, solrMetricsContext.registry, solrMetricsContext.tag, scope);
} }
/** /**
* Return the registry used by this SolrCore. * Return the registry used by this SolrCore.
*/ */
public MetricRegistry getRegistry() { public MetricRegistry getRegistry() {
if (registryName != null) { if (solrMetricsContext != null) {
return metricManager.registry(registryName); return solrMetricsContext.getMetricRegistry();
} else { } else {
return null; return null;
} }
@ -146,11 +147,15 @@ public class SolrCoreMetricManager implements Closeable {
*/ */
@Override @Override
public void close() throws IOException { public void close() throws IOException {
metricManager.closeReporters(getRegistryName(), tag); metricManager.closeReporters(solrMetricsContext.registry, solrMetricsContext.tag);
if (getLeaderRegistryName() != null) { if (getLeaderRegistryName() != null) {
metricManager.closeReporters(getLeaderRegistryName(), tag); metricManager.closeReporters(getLeaderRegistryName(), solrMetricsContext.tag);
} }
metricManager.unregisterGauges(getRegistryName(), tag); metricManager.unregisterGauges(solrMetricsContext.registry, solrMetricsContext.tag);
}
public SolrMetricsContext getSolrMetricsContext() {
return solrMetricsContext;
} }
public SolrCore getCore() { public SolrCore getCore() {
@ -175,7 +180,7 @@ public class SolrCoreMetricManager implements Closeable {
* @return the metric registry name of the manager. * @return the metric registry name of the manager.
*/ */
public String getRegistryName() { public String getRegistryName() {
return registryName; return solrMetricsContext != null ? solrMetricsContext.registry : null;
} }
/** /**
@ -190,7 +195,7 @@ public class SolrCoreMetricManager implements Closeable {
* Return a tag specific to this instance. * Return a tag specific to this instance.
*/ */
public String getTag() { public String getTag() {
return tag; return solrMetricsContext.tag;
} }
public static String createRegistryName(boolean cloud, String collectionName, String shardName, String replicaName, String coreName) { public static String createRegistryName(boolean cloud, String collectionName, String shardName, String replicaName, String coreName) {

26
solr/core/src/java/org/apache/solr/metrics/SolrMetricManager.java
View File

@ -724,20 +724,24 @@ public class SolrMetricManager {
registerMetric(info, registry, new GaugeWrapper(gauge, tag), force, metricName, metricPath); registerMetric(info, registry, new GaugeWrapper(gauge, tag), force, metricName, metricPath);
} }
public int unregisterGauges(String registryName, String tag) { public int unregisterGauges(String registryName, String tagSegment) {
if (tag == null) { if (tagSegment == null) {
return 0; return 0;
} }
MetricRegistry registry = registry(registryName); MetricRegistry registry = registry(registryName);
if (registry == null) return 0;
AtomicInteger removed = new AtomicInteger(); AtomicInteger removed = new AtomicInteger();
registry.removeMatching((name, metric) -> { registry.removeMatching((name, metric) -> {
if (metric instanceof GaugeWrapper && if (metric instanceof GaugeWrapper) {
tag.equals(((GaugeWrapper) metric).getTag())) { GaugeWrapper wrapper = (GaugeWrapper) metric;
boolean toRemove = wrapper.getTag().contains(tagSegment);
if (toRemove) {
removed.incrementAndGet(); removed.incrementAndGet();
return true;
} else {
return false;
} }
return toRemove;
}
return false;
}); });
return removed.get(); return removed.get();
} }
@ -752,10 +756,16 @@ public class SolrMetricManager {
* segments prepended to the name. * segments prepended to the name.
*/ */
public static String mkName(String name, String... path) { public static String mkName(String name, String... path) {
return makeName(path == null || path.length == 0 ? Collections.emptyList() : Arrays.asList(path),
name);
}
public static String makeName(List<String> path, String name) {
if (name == null || name.isEmpty()) { if (name == null || name.isEmpty()) {
throw new IllegalArgumentException("name must not be empty"); throw new IllegalArgumentException("name must not be empty");
} }
if (path == null || path.length == 0) { if (path == null || path.size() == 0) {
return name; return name;
} else { } else {
StringBuilder sb = new StringBuilder(); StringBuilder sb = new StringBuilder();

76
solr/core/src/java/org/apache/solr/metrics/SolrMetricProducer.java
View File

@ -19,17 +19,83 @@ package org.apache.solr.metrics;
/** /**
* Used by objects that expose metrics through {@link SolrMetricManager}. * Used by objects that expose metrics through {@link SolrMetricManager}.
*/ */
public interface SolrMetricProducer { public interface SolrMetricProducer extends AutoCloseable {
/**
* Unique metric tag identifies components with the same life-cycle, which should
* be registered / unregistered together. It is in the format of A:B:C, where
* A is the parent of B is the parent of C and so on.
* If object "B" is unregistered C also must get unregistered.
* If object "A" is unregistered B and C also must get unregistered.
* @param o object to create a tag for
* @param parentName parent object name, or null if no parent exists
*/
static String getUniqueMetricTag(Object o, String parentName) {
String name = o.getClass().getSimpleName() + "@" + Integer.toHexString(o.hashCode());
if (parentName != null && parentName.contains(name)) {
throw new RuntimeException("Parent already includes this component! parent=" + parentName + ", this=" + name);
}
return parentName == null ?
name :
parentName + ":" + name;
}
/** /**
* Initializes metrics specific to this producer * Initializes metrics specific to this producer
*
* @param manager an instance of {@link SolrMetricManager} * @param manager an instance of {@link SolrMetricManager}
* @param registry registry name where metrics are registered * @param registry registry name where metrics are registered
* @param tag a symbolic tag that represents this instance of the producer, * @param tag a symbolic tag that represents this instance of the producer,
* or a group of related instances that have the same life-cycle. This tag is * or a group of related instances that have the same life-cycle. This tag is
* used when managing life-cycle of some metrics and is set when * used when managing life-cycle of some metrics.
* {@link #initializeMetrics(SolrMetricManager, String, String, String)} is called. * @param scope scope of the metrics (eg. handler name) to separate metrics of components with
* @param scope scope of the metrics (eg. handler name) to separate metrics of * the same implementation but different scope.
* @deprecated use {@link #initializeMetrics(SolrMetricsContext, String)} instead
*/ */
void initializeMetrics(SolrMetricManager manager, String registry, String tag, String scope); @Deprecated
default void initializeMetrics(SolrMetricManager manager, String registry, String tag, String scope) {
initializeMetrics(new SolrMetricsContext(manager, registry, tag), scope);
}
/**
* Initialize metrics specific to this producer.
* @param parentContext parent metrics context. If this component has the same life-cycle as the parent
* it can simply use the parent context, otherwise it should obtain a child context
* using {@link SolrMetricsContext#getChildContext(Object)} passing <code>this</code>
* as the child.
* @param scope component scope
*/
default void initializeMetrics(SolrMetricsContext parentContext, String scope) {
throw new RuntimeException("In class " + getClass().getName() +
" you must implement either initializeMetrics(SolrMetricsContext, String) or " +
"initializeMetrics(SolrMetricManager, String, String, String)");
}
/**
* Implementing classes should override this method to provide the context obtained in
* {@link #initializeMetrics(SolrMetricsContext, String)} to ensure proper cleanup of metrics
* at the end of the life-cycle of this component.
*/
default SolrMetricsContext getSolrMetricsContext() {
return null;
}
/**
* Implementations should always call <code>SolrMetricProducer.super.close()</code> to ensure that
* metrics with the same life-cycle as this component are properly unregistered. This prevents
* obscure memory leaks.
*/
@Override
default void close() throws Exception {
SolrMetricsContext context = getSolrMetricsContext();
if (context == null) {
return;
} else {
context.unregister();
}
// ??? (ab) no idea what this was supposed to avoid
//if (info == null || info.tag.indexOf(':') == -1) return;//this will end up unregistering the root itself
}
} }

114
solr/core/src/java/org/apache/solr/metrics/SolrMetricsContext.java Normal file
View File

@ -0,0 +1,114 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.metrics;
import com.codahale.metrics.Counter;
import com.codahale.metrics.Gauge;
import com.codahale.metrics.Histogram;
import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.Timer;
import org.apache.solr.core.SolrInfoBean;
/**
* This class represents a metrics context that ties together components with the same life-cycle
* and provides convenient access to the metric registry.
*/
public class SolrMetricsContext {
public final String registry;
public final SolrMetricManager metricManager;
public final String tag;
public SolrMetricsContext(SolrMetricManager metricManager, String registry, String tag) {
this.registry = registry;
this.metricManager = metricManager;
this.tag = tag;
}
/**
* Metrics tag that represents objects with the same life-cycle.
*/
public String getTag() {
return tag;
}
/**
* Unregister all {@link Gauge} metrics that use this context's tag.
*
* <p><b>NOTE: This method MUST be called at the end of a life-cycle (typically in <code>close()</code>)
* of components that register gauge metrics with references to the current object's instance. Failure to
* do so may result in hard-to-debug memory leaks.</b></p>
*/
public void unregister() {
metricManager.unregisterGauges(registry, tag);
}
/**
* Get a context with the same registry name but a tag that represents a parent-child relationship.
* Since it's a different tag than the parent's context it is assumed that the life-cycle of the parent
* and child are different.
* @param child child object that produces metrics with a different life-cycle than the parent.
*/
public SolrMetricsContext getChildContext(Object child) {
SolrMetricsContext childContext = new SolrMetricsContext(metricManager, registry, SolrMetricProducer.getUniqueMetricTag(child, tag));
return childContext;
}
/**
* Convenience method for {@link SolrMetricManager#meter(SolrInfoBean, String, String, String...)}.
*/
public Meter meter(SolrInfoBean info, String metricName, String... metricPath) {
return metricManager.meter(info, registry, metricName, metricPath);
}
/**
* Convenience method for {@link SolrMetricManager#counter(SolrInfoBean, String, String, String...)}.
*/
public Counter counter(SolrInfoBean info, String metricName, String... metricPath) {
return metricManager.counter(info, registry, metricName, metricPath);
}
/**
* Convenience method for {@link SolrMetricManager#registerGauge(SolrInfoBean, String, Gauge, String, boolean, String, String...)}.
*/
public void gauge(SolrInfoBean info, Gauge<?> gauge, boolean force, String metricName, String... metricPath) {
metricManager.registerGauge(info, registry, gauge, tag, force, metricName, metricPath);
}
/**
* Convenience method for {@link SolrMetricManager#meter(SolrInfoBean, String, String, String...)}.
*/
public Timer timer(SolrInfoBean info, String metricName, String... metricPath) {
return metricManager.timer(info, registry, metricName, metricPath);
}
/**
* Convenience method for {@link SolrMetricManager#histogram(SolrInfoBean, String, String, String...)}.
*/
public Histogram histogram(SolrInfoBean info, String metricName, String... metricPath) {
return metricManager.histogram(info, registry, metricName, metricPath);
}
/**
* Get the MetricRegistry instance that is used for registering metrics in this context.
*/
public MetricRegistry getMetricRegistry() {
return metricManager.registry(registry);
}
}

18
solr/core/src/java/org/apache/solr/search/CaffeineCache.java
View File

@ -31,14 +31,13 @@ import java.util.concurrent.ForkJoinPool;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.LongAdder; import java.util.concurrent.atomic.LongAdder;
import com.codahale.metrics.MetricRegistry;
import com.github.benmanes.caffeine.cache.RemovalCause; import com.github.benmanes.caffeine.cache.RemovalCause;
import com.github.benmanes.caffeine.cache.RemovalListener; import com.github.benmanes.caffeine.cache.RemovalListener;
import org.apache.lucene.util.Accountable; import org.apache.lucene.util.Accountable;
import org.apache.lucene.util.RamUsageEstimator; import org.apache.lucene.util.RamUsageEstimator;
import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException;
import org.apache.solr.metrics.MetricsMap; import org.apache.solr.metrics.MetricsMap;
import org.apache.solr.metrics.SolrMetricManager; import org.apache.solr.metrics.SolrMetricsContext;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@ -88,7 +87,7 @@ public class CaffeineCache<K, V> extends SolrCacheBase implements SolrCache<K, V
private Set<String> metricNames = ConcurrentHashMap.newKeySet(); private Set<String> metricNames = ConcurrentHashMap.newKeySet();
private MetricsMap cacheMap; private MetricsMap cacheMap;
private MetricRegistry registry; private SolrMetricsContext solrMetricsContext;
private long initialRamBytes = 0; private long initialRamBytes = 0;
private final LongAdder ramBytes = new LongAdder(); private final LongAdder ramBytes = new LongAdder();
@ -202,7 +201,8 @@ public class CaffeineCache<K, V> extends SolrCacheBase implements SolrCache<K, V
} }
@Override @Override
public void close() { public void close() throws Exception {
SolrCache.super.close();
cache.invalidateAll(); cache.invalidateAll();
cache.cleanUp(); cache.cleanUp();
if (executor instanceof ExecutorService) { if (executor instanceof ExecutorService) {
@ -322,8 +322,8 @@ public class CaffeineCache<K, V> extends SolrCacheBase implements SolrCache<K, V
} }
@Override @Override
public MetricRegistry getMetricRegistry() { public SolrMetricsContext getSolrMetricsContext() {
return registry; return solrMetricsContext;
} }
@Override @Override
@ -337,8 +337,8 @@ public class CaffeineCache<K, V> extends SolrCacheBase implements SolrCache<K, V
} }
@Override @Override
public void initializeMetrics(SolrMetricManager manager, String registryName, String tag, String scope) { public void initializeMetrics(SolrMetricsContext parentContext, String scope) {
registry = manager.registry(registryName); solrMetricsContext = parentContext.getChildContext(this);
cacheMap = new MetricsMap((detailed, map) -> { cacheMap = new MetricsMap((detailed, map) -> {
if (cache != null) { if (cache != null) {
CacheStats stats = cache.stats(); CacheStats stats = cache.stats();
@ -362,6 +362,6 @@ public class CaffeineCache<K, V> extends SolrCacheBase implements SolrCache<K, V
map.put("cumulative_evictions", cumulativeStats.evictionCount()); map.put("cumulative_evictions", cumulativeStats.evictionCount());
} }
}); });
manager.registerGauge(this, registryName, cacheMap, tag, true, scope, getCategory().toString()); solrMetricsContext.gauge(this, cacheMap, true, scope, getCategory().toString());
} }
} }

51
solr/core/src/java/org/apache/solr/search/FastLRUCache.java
View File

@ -16,24 +16,23 @@
*/ */
package org.apache.solr.search; package org.apache.solr.search;
import com.codahale.metrics.MetricRegistry; import java.lang.invoke.MethodHandles;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.TimeUnit;
import org.apache.lucene.util.Accountable; import org.apache.lucene.util.Accountable;
import org.apache.lucene.util.RamUsageEstimator; import org.apache.lucene.util.RamUsageEstimator;
import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException;
import org.apache.solr.metrics.MetricsMap; import org.apache.solr.metrics.MetricsMap;
import org.apache.solr.metrics.SolrMetricManager; import org.apache.solr.metrics.SolrMetricsContext;
import org.apache.solr.util.ConcurrentLRUCache; import org.apache.solr.util.ConcurrentLRUCache;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import java.lang.invoke.MethodHandles;
import java.util.concurrent.ConcurrentHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.TimeUnit;
/** /**
* SolrCache based on ConcurrentLRUCache implementation. * SolrCache based on ConcurrentLRUCache implementation.
* <p> * <p>
@ -42,12 +41,11 @@ import java.util.concurrent.TimeUnit;
* <p> * <p>
* Also see <a href="http://wiki.apache.org/solr/SolrCaching">SolrCaching</a> * Also see <a href="http://wiki.apache.org/solr/SolrCaching">SolrCaching</a>
* *
*
* @see org.apache.solr.util.ConcurrentLRUCache * @see org.apache.solr.util.ConcurrentLRUCache
* @see org.apache.solr.search.SolrCache * @see org.apache.solr.search.SolrCache
* @since solr 1.4 * @since solr 1.4
*/ */
public class FastLRUCache<K, V> extends SolrCacheBase implements SolrCache<K,V>, Accountable { public class FastLRUCache<K, V> extends SolrCacheBase implements SolrCache<K, V>, Accountable {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(FastLRUCache.class); private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(FastLRUCache.class);
@ -61,7 +59,7 @@ public class FastLRUCache<K, V> extends SolrCacheBase implements SolrCache<K,V>,
private long warmupTime = 0; private long warmupTime = 0;
private String description = "Concurrent LRU Cache"; private String description = "Concurrent LRU Cache";
private ConcurrentLRUCache<K,V> cache; private ConcurrentLRUCache<K, V> cache;
private int showItems = 0; private int showItems = 0;
private long maxRamBytes; private long maxRamBytes;
@ -75,7 +73,7 @@ public class FastLRUCache<K, V> extends SolrCacheBase implements SolrCache<K,V>,
private MetricsMap cacheMap; private MetricsMap cacheMap;
private Set<String> metricNames = ConcurrentHashMap.newKeySet(); private Set<String> metricNames = ConcurrentHashMap.newKeySet();
private MetricRegistry registry; private SolrMetricsContext solrMetricsContext;
@Override @Override
public Object init(Map args, Object persistence, CacheRegenerator regenerator) { public Object init(Map args, Object persistence, CacheRegenerator regenerator) {
@ -213,7 +211,7 @@ public class FastLRUCache<K, V> extends SolrCacheBase implements SolrCache<K,V>,
*/ */
protected String generateDescription(int limit, int initialSize, int minLimit, int acceptableLimit, boolean newThread) { protected String generateDescription(int limit, int initialSize, int minLimit, int acceptableLimit, boolean newThread) {
String description = "Concurrent LRU Cache(maxSize=" + limit + ", initialSize=" + initialSize + String description = "Concurrent LRU Cache(maxSize=" + limit + ", initialSize=" + initialSize +
", minSize="+minLimit + ", acceptableSize="+acceptableLimit+", cleanupThread="+newThread; ", minSize=" + minLimit + ", acceptableSize=" + acceptableLimit + ", cleanupThread=" + newThread;
if (isAutowarmingOn()) { if (isAutowarmingOn()) {
description += ", " + getAutowarmDescription(); description += ", " + getAutowarmDescription();
} }
@ -276,8 +274,7 @@ public class FastLRUCache<K, V> extends SolrCacheBase implements SolrCache<K,V>,
boolean continueRegen = regenerator.regenerateItem(searcher, boolean continueRegen = regenerator.regenerateItem(searcher,
this, old, itemsArr[i].getKey(), itemsArr[i].getValue()); this, old, itemsArr[i].getKey(), itemsArr[i].getValue());
if (!continueRegen) break; if (!continueRegen) break;
} } catch (Exception e) {
catch (Exception e) {
SolrException.log(log, "Error during auto-warming of key:" + itemsArr[i].getKey(), e); SolrException.log(log, "Error during auto-warming of key:" + itemsArr[i].getKey(), e);
} }
} }
@ -287,7 +284,8 @@ public class FastLRUCache<K, V> extends SolrCacheBase implements SolrCache<K,V>,
@Override @Override
public void close() { public void close() throws Exception {
SolrCache.super.close();
// add the stats to the cumulative stats object (the first in the statsList) // add the stats to the cumulative stats object (the first in the statsList)
statsList.get(0).add(cache.getStats()); statsList.get(0).add(cache.getStats());
statsList.remove(cache.getStats()); statsList.remove(cache.getStats());
@ -310,10 +308,16 @@ public class FastLRUCache<K, V> extends SolrCacheBase implements SolrCache<K,V>,
return metricNames; return metricNames;
} }
@Override @Override
public void initializeMetrics(SolrMetricManager manager, String registryName, String tag, String scope) { public SolrMetricsContext getSolrMetricsContext() {
registry = manager.registry(registryName); return solrMetricsContext;
manager.registerGauge(this, registryName, cacheMap, tag, true, scope, getCategory().toString()); }
@Override
public void initializeMetrics(SolrMetricsContext parentContext, String scope) {
this.solrMetricsContext = parentContext.getChildContext(this);
this.solrMetricsContext.gauge(this, cacheMap, true, scope, getCategory().toString());
} }
// for unit tests only // for unit tests only
@ -321,11 +325,6 @@ public class FastLRUCache<K, V> extends SolrCacheBase implements SolrCache<K,V>,
return cacheMap; return cacheMap;
} }
@Override
public MetricRegistry getMetricRegistry() {
return registry;
}
@Override @Override
public String toString() { public String toString() {
return name() + (cacheMap != null ? cacheMap.getValue().toString() : ""); return name() + (cacheMap != null ? cacheMap.getValue().toString() : "");

27
solr/core/src/java/org/apache/solr/search/LFUCache.java
View File

@ -17,19 +17,18 @@
package org.apache.solr.search; package org.apache.solr.search;
import java.lang.invoke.MethodHandles; import java.lang.invoke.MethodHandles;
import java.util.concurrent.ConcurrentHashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Set; import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import com.codahale.metrics.MetricRegistry;
import org.apache.lucene.util.Accountable; import org.apache.lucene.util.Accountable;
import org.apache.lucene.util.RamUsageEstimator; import org.apache.lucene.util.RamUsageEstimator;
import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException;
import org.apache.solr.metrics.MetricsMap; import org.apache.solr.metrics.MetricsMap;
import org.apache.solr.metrics.SolrMetricManager; import org.apache.solr.metrics.SolrMetricsContext;
import org.apache.solr.util.ConcurrentLFUCache; import org.apache.solr.util.ConcurrentLFUCache;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@ -79,7 +78,8 @@ public class LFUCache<K, V> implements SolrCache<K, V>, Accountable {
private int maxIdleTimeSec; private int maxIdleTimeSec;
private MetricsMap cacheMap; private MetricsMap cacheMap;
private Set<String> metricNames = ConcurrentHashMap.newKeySet(); private Set<String> metricNames = ConcurrentHashMap.newKeySet();
private MetricRegistry registry; private SolrMetricsContext solrMetricsContext;
private int maxSize; private int maxSize;
private int minSizeLimit; private int minSizeLimit;
@ -230,7 +230,8 @@ public class LFUCache<K, V> implements SolrCache<K, V>, Accountable {
@Override @Override
public void close() { public void close() throws Exception {
SolrCache.super.close();
// add the stats to the cumulative stats object (the first in the statsList) // add the stats to the cumulative stats object (the first in the statsList)
statsList.get(0).add(cache.getStats()); statsList.get(0).add(cache.getStats());
statsList.remove(cache.getStats()); statsList.remove(cache.getStats());
@ -263,8 +264,13 @@ public class LFUCache<K, V> implements SolrCache<K, V>, Accountable {
} }
@Override @Override
public void initializeMetrics(SolrMetricManager manager, String registryName, String tag, String scope) { public SolrMetricsContext getSolrMetricsContext() {
registry = manager.registry(registryName); return solrMetricsContext;
}
@Override
public void initializeMetrics(SolrMetricsContext parentContext, String scope) {
solrMetricsContext = parentContext.getChildContext(this);
cacheMap = new MetricsMap((detailed, map) -> { cacheMap = new MetricsMap((detailed, map) -> {
if (cache != null) { if (cache != null) {
ConcurrentLFUCache.Stats stats = cache.getStats(); ConcurrentLFUCache.Stats stats = cache.getStats();
@ -330,7 +336,7 @@ public class LFUCache<K, V> implements SolrCache<K, V>, Accountable {
} }
}); });
manager.registerGauge(this, registryName, cacheMap, tag, true, scope, getCategory().toString()); solrMetricsContext.gauge(this, cacheMap, true, scope, getCategory().toString());
} }
// for unit tests only // for unit tests only
@ -343,11 +349,6 @@ public class LFUCache<K, V> implements SolrCache<K, V>, Accountable {
return metricNames; return metricNames;
} }
@Override
public MetricRegistry getMetricRegistry() {
return registry;
}
@Override @Override
public String toString() { public String toString() {
return name + (cacheMap != null ? cacheMap.getValue().toString() : ""); return name + (cacheMap != null ? cacheMap.getValue().toString() : "");

29
solr/core/src/java/org/apache/solr/search/LRUCache.java
View File

@ -18,22 +18,21 @@ package org.apache.solr.search;
import java.lang.invoke.MethodHandles; import java.lang.invoke.MethodHandles;
import java.util.Collection; import java.util.Collection;
import java.util.concurrent.ConcurrentHashMap;
import java.util.Iterator; import java.util.Iterator;
import java.util.LinkedHashMap; import java.util.LinkedHashMap;
import java.util.Map; import java.util.Map;
import java.util.Set; import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.LongAdder; import java.util.concurrent.atomic.LongAdder;
import com.codahale.metrics.MetricRegistry;
import org.apache.lucene.util.Accountable; import org.apache.lucene.util.Accountable;
import org.apache.lucene.util.Accountables; import org.apache.lucene.util.Accountables;
import org.apache.lucene.util.RamUsageEstimator; import org.apache.lucene.util.RamUsageEstimator;
import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException;
import org.apache.solr.common.util.TimeSource; import org.apache.solr.common.util.TimeSource;
import org.apache.solr.metrics.MetricsMap; import org.apache.solr.metrics.MetricsMap;
import org.apache.solr.metrics.SolrMetricManager; import org.apache.solr.metrics.SolrMetricsContext;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@ -77,7 +76,7 @@ public class LRUCache<K,V> extends SolrCacheBase implements SolrCache<K,V>, Acco
private String description="LRU Cache"; private String description="LRU Cache";
private MetricsMap cacheMap; private MetricsMap cacheMap;
private Set<String> metricNames = ConcurrentHashMap.newKeySet(); private Set<String> metricNames = ConcurrentHashMap.newKeySet();
private MetricRegistry registry; private SolrMetricsContext solrMetricsContext;
private int maxSize; private int maxSize;
private int initialSize; private int initialSize;
@ -378,12 +377,6 @@ public class LRUCache<K,V> extends SolrCacheBase implements SolrCache<K,V>, Acco
warmupTime = TimeUnit.MILLISECONDS.convert(System.nanoTime() - warmingStartTime, TimeUnit.NANOSECONDS); warmupTime = TimeUnit.MILLISECONDS.convert(System.nanoTime() - warmingStartTime, TimeUnit.NANOSECONDS);
} }
@Override
public void close() {
}
//////////////////////// SolrInfoMBeans methods ////////////////////// //////////////////////// SolrInfoMBeans methods //////////////////////
@ -403,8 +396,13 @@ public class LRUCache<K,V> extends SolrCacheBase implements SolrCache<K,V>, Acco
} }
@Override @Override
public void initializeMetrics(SolrMetricManager manager, String registryName, String tag, String scope) { public SolrMetricsContext getSolrMetricsContext() {
registry = manager.registry(registryName); return solrMetricsContext;
}
@Override
public void initializeMetrics(SolrMetricsContext parentContext, String scope) {
solrMetricsContext = parentContext.getChildContext(this);
cacheMap = new MetricsMap((detailed, res) -> { cacheMap = new MetricsMap((detailed, res) -> {
synchronized (map) { synchronized (map) {
res.put(LOOKUPS_PARAM, lookups); res.put(LOOKUPS_PARAM, lookups);
@ -433,7 +431,7 @@ public class LRUCache<K,V> extends SolrCacheBase implements SolrCache<K,V>, Acco
res.put("cumulative_evictionsRamUsage", stats.evictionsRamUsage.longValue()); res.put("cumulative_evictionsRamUsage", stats.evictionsRamUsage.longValue());
res.put("cumulative_evictionsIdleTime", stats.evictionsIdleTime.longValue()); res.put("cumulative_evictionsIdleTime", stats.evictionsIdleTime.longValue());
}); });
manager.registerGauge(this, registryName, cacheMap, tag, true, scope, getCategory().toString()); solrMetricsContext.gauge(this, cacheMap, true, scope, getCategory().toString());
} }
// for unit tests only // for unit tests only
@ -441,11 +439,6 @@ public class LRUCache<K,V> extends SolrCacheBase implements SolrCache<K,V>, Acco
return cacheMap; return cacheMap;
} }
@Override
public MetricRegistry getMetricRegistry() {
return registry;
}
@Override @Override
public String toString() { public String toString() {
return name() + (cacheMap != null ? cacheMap.getValue().toString() : ""); return name() + (cacheMap != null ? cacheMap.getValue().toString() : "");

4
solr/core/src/java/org/apache/solr/search/SolrCache.java
View File

@ -137,7 +137,9 @@ public interface SolrCache<K,V> extends SolrInfoBean, SolrMetricProducer {
/** Frees any non-memory resources */ /** Frees any non-memory resources */
public void close(); default void close() throws Exception {
SolrMetricProducer.super.close();
}
/** Returns maximum size limit (number of items) if set and supported, -1 otherwise. */ /** Returns maximum size limit (number of items) if set and supported, -1 otherwise. */
int getMaxSize(); int getMaxSize();

12
solr/core/src/java/org/apache/solr/search/SolrCacheHolder.java
View File

@ -22,8 +22,7 @@ import java.util.Map;
import java.util.Set; import java.util.Set;
import com.codahale.metrics.MetricRegistry; import com.codahale.metrics.MetricRegistry;
import org.apache.solr.common.util.Utils; import org.apache.solr.metrics.SolrMetricsContext;
import org.apache.solr.metrics.SolrMetricManager;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@ -77,7 +76,7 @@ public class SolrCacheHolder<K, V> implements SolrCache<K,V> {
return delegate; return delegate;
} }
public void close() { public void close() throws Exception {
delegate.close(); delegate.close();
} }
@ -142,11 +141,8 @@ public class SolrCacheHolder<K, V> implements SolrCache<K,V> {
} }
@Override @Override
public void initializeMetrics(SolrMetricManager manager, String registry, String tag, String scope) { public void initializeMetrics(SolrMetricsContext parentContext, String scope) {
log.debug("Going to register cachemetrics " + Utils.toJSONString(factory)); delegate.initializeMetrics(parentContext, scope);
delegate.initializeMetrics(manager, registry, tag,scope);
} }
} }

16
solr/core/src/java/org/apache/solr/search/SolrFieldCacheBean.java
View File

@ -19,11 +19,10 @@ package org.apache.solr.search;
import java.util.Set; import java.util.Set;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
import com.codahale.metrics.MetricRegistry;
import org.apache.solr.core.SolrInfoBean; import org.apache.solr.core.SolrInfoBean;
import org.apache.solr.metrics.MetricsMap; import org.apache.solr.metrics.MetricsMap;
import org.apache.solr.metrics.SolrMetricManager;
import org.apache.solr.metrics.SolrMetricProducer; import org.apache.solr.metrics.SolrMetricProducer;
import org.apache.solr.metrics.SolrMetricsContext;
import org.apache.solr.uninverting.UninvertingReader; import org.apache.solr.uninverting.UninvertingReader;
/** /**
@ -35,7 +34,7 @@ public class SolrFieldCacheBean implements SolrInfoBean, SolrMetricProducer {
private boolean disableEntryList = Boolean.getBoolean("disableSolrFieldCacheMBeanEntryList"); private boolean disableEntryList = Boolean.getBoolean("disableSolrFieldCacheMBeanEntryList");
private boolean disableJmxEntryList = Boolean.getBoolean("disableSolrFieldCacheMBeanEntryListJmx"); private boolean disableJmxEntryList = Boolean.getBoolean("disableSolrFieldCacheMBeanEntryListJmx");
private MetricRegistry registry; private SolrMetricsContext solrMetricsContext;
private Set<String> metricNames = ConcurrentHashMap.newKeySet(); private Set<String> metricNames = ConcurrentHashMap.newKeySet();
@Override @Override
@ -50,14 +49,15 @@ public class SolrFieldCacheBean implements SolrInfoBean, SolrMetricProducer {
public Set<String> getMetricNames() { public Set<String> getMetricNames() {
return metricNames; return metricNames;
} }
@Override @Override
public MetricRegistry getMetricRegistry() { public SolrMetricsContext getSolrMetricsContext() {
return registry; return solrMetricsContext;
} }
@Override @Override
public void initializeMetrics(SolrMetricManager manager, String registryName, String tag, String scope) { public void initializeMetrics(SolrMetricsContext parentContext, String scope) {
registry = manager.registry(registryName); this.solrMetricsContext = parentContext;
MetricsMap metricsMap = new MetricsMap((detailed, map) -> { MetricsMap metricsMap = new MetricsMap((detailed, map) -> {
if (detailed && !disableEntryList && !disableJmxEntryList) { if (detailed && !disableEntryList && !disableJmxEntryList) {
UninvertingReader.FieldCacheStats fieldCacheStats = UninvertingReader.getUninvertedStats(); UninvertingReader.FieldCacheStats fieldCacheStats = UninvertingReader.getUninvertedStats();
@ -72,6 +72,6 @@ public class SolrFieldCacheBean implements SolrInfoBean, SolrMetricProducer {
map.put("entries_count", UninvertingReader.getUninvertedStatsSize()); map.put("entries_count", UninvertingReader.getUninvertedStatsSize());
} }
}); });
manager.registerGauge(this, registryName, metricsMap, tag, true, "fieldCache", Category.CACHE.toString(), scope); solrMetricsContext.gauge(this, metricsMap, true, "fieldCache", Category.CACHE.toString(), scope);
} }
} }

63
solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
View File

@ -35,7 +35,6 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.atomic.AtomicReference;
import com.codahale.metrics.MetricRegistry;
import com.google.common.collect.Iterables; import com.google.common.collect.Iterables;
import org.apache.lucene.document.Document; import org.apache.lucene.document.Document;
import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.DirectoryReader;
@ -69,6 +68,7 @@ import org.apache.solr.index.SlowCompositeReaderWrapper;
import org.apache.solr.metrics.MetricsMap; import org.apache.solr.metrics.MetricsMap;
import org.apache.solr.metrics.SolrMetricManager; import org.apache.solr.metrics.SolrMetricManager;
import org.apache.solr.metrics.SolrMetricProducer; import org.apache.solr.metrics.SolrMetricProducer;
import org.apache.solr.metrics.SolrMetricsContext;
import org.apache.solr.request.LocalSolrQueryRequest; import org.apache.solr.request.LocalSolrQueryRequest;
import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.request.SolrRequestInfo; import org.apache.solr.request.SolrRequestInfo;
@ -140,8 +140,7 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrI
private final StatsCache statsCache; private final StatsCache statsCache;
private Set<String> metricNames = ConcurrentHashMap.newKeySet(); private Set<String> metricNames = ConcurrentHashMap.newKeySet();
private SolrMetricManager metricManager; private SolrMetricsContext solrMetricsContext;
private String registryName;
private static DirectoryReader getReader(SolrCore core, SolrIndexConfig config, DirectoryFactory directoryFactory, private static DirectoryReader getReader(SolrCore core, SolrIndexConfig config, DirectoryFactory directoryFactory,
String path) throws IOException { String path) throws IOException {
@ -431,12 +430,13 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrI
cache.setState(SolrCache.State.LIVE); cache.setState(SolrCache.State.LIVE);
infoRegistry.put(cache.name(), cache); infoRegistry.put(cache.name(), cache);
} }
metricManager = core.getCoreContainer().getMetricManager(); this.solrMetricsContext = core.getSolrMetricsContext().getChildContext(this);
registryName = core.getCoreMetricManager().getRegistryName();
for (SolrCache cache : cacheList) { for (SolrCache cache : cacheList) {
cache.initializeMetrics(metricManager, registryName, core.getMetricTag(), SolrMetricManager.mkName(cache.name(), STATISTICS_KEY)); // XXX use the deprecated method for back-compat. remove in 9.0
cache.initializeMetrics(solrMetricsContext.metricManager,
solrMetricsContext.registry, solrMetricsContext.tag, SolrMetricManager.mkName(cache.name(), STATISTICS_KEY));
} }
initializeMetrics(metricManager, registryName, core.getMetricTag(), STATISTICS_KEY); initializeMetrics(solrMetricsContext, STATISTICS_KEY);
registerTime = new Date(); registerTime = new Date();
} }
@ -479,7 +479,11 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrI
} }
for (SolrCache cache : cacheList) { for (SolrCache cache : cacheList) {
try {
cache.close(); cache.close();
} catch (Exception e) {
SolrException.log(log, "Exception closing cache " + cache.name(), e);
}
} }
if (releaseDirectory) { if (releaseDirectory) {
@ -2275,23 +2279,26 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrI
} }
@Override @Override
public void initializeMetrics(SolrMetricManager manager, String registry, String tag, String scope) { public SolrMetricsContext getSolrMetricsContext() {
this.registryName = registry; return solrMetricsContext;
this.metricManager = manager; }
manager.registerGauge(this, registry, () -> name, tag, true, "searcherName", Category.SEARCHER.toString(), scope);
manager.registerGauge(this, registry, () -> cachingEnabled, tag, true, "caching", Category.SEARCHER.toString(), scope); @Override
manager.registerGauge(this, registry, () -> openTime, tag, true, "openedAt", Category.SEARCHER.toString(), scope); public void initializeMetrics(SolrMetricsContext parentContext, String scope) {
manager.registerGauge(this, registry, () -> warmupTime, tag, true, "warmupTime", Category.SEARCHER.toString(), scope); parentContext.gauge(this, () -> name, true, "searcherName", Category.SEARCHER.toString(), scope);
manager.registerGauge(this, registry, () -> registerTime, tag, true, "registeredAt", Category.SEARCHER.toString(), scope); parentContext.gauge(this, () -> cachingEnabled, true, "caching", Category.SEARCHER.toString(), scope);
parentContext.gauge(this, () -> openTime, true, "openedAt", Category.SEARCHER.toString(), scope);
parentContext.gauge(this, () -> warmupTime, true, "warmupTime", Category.SEARCHER.toString(), scope);
parentContext.gauge(this, () -> registerTime, true, "registeredAt", Category.SEARCHER.toString(), scope);
// reader stats // reader stats
manager.registerGauge(this, registry, () -> reader.numDocs(), tag, true, "numDocs", Category.SEARCHER.toString(), scope); parentContext.gauge(this, () -> reader.numDocs(), true, "numDocs", Category.SEARCHER.toString(), scope);
manager.registerGauge(this, registry, () -> reader.maxDoc(), tag, true, "maxDoc", Category.SEARCHER.toString(), scope); parentContext.gauge(this, () -> reader.maxDoc(), true, "maxDoc", Category.SEARCHER.toString(), scope);
manager.registerGauge(this, registry, () -> reader.maxDoc() - reader.numDocs(), tag, true, "deletedDocs", Category.SEARCHER.toString(), scope); parentContext.gauge(this, () -> reader.maxDoc() - reader.numDocs(), true, "deletedDocs", Category.SEARCHER.toString(), scope);
manager.registerGauge(this, registry, () -> reader.toString(), tag, true, "reader", Category.SEARCHER.toString(), scope); parentContext.gauge(this, () -> reader.toString(), true, "reader", Category.SEARCHER.toString(), scope);
manager.registerGauge(this, registry, () -> reader.directory().toString(), tag, true, "readerDir", Category.SEARCHER.toString(), scope); parentContext.gauge(this, () -> reader.directory().toString(), true, "readerDir", Category.SEARCHER.toString(), scope);
manager.registerGauge(this, registry, () -> reader.getVersion(), tag, true, "indexVersion", Category.SEARCHER.toString(), scope); parentContext.gauge(this, () -> reader.getVersion(), true, "indexVersion", Category.SEARCHER.toString(), scope);
// size of the currently opened commit // size of the currently opened commit
manager.registerGauge(this, registry, () -> { parentContext.gauge(this, () -> {
try { try {
Collection<String> files = reader.getIndexCommit().getFileNames(); Collection<String> files = reader.getIndexCommit().getFileNames();
long total = 0; long total = 0;
@ -2302,19 +2309,13 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrI
} catch (Exception e) { } catch (Exception e) {
return -1; return -1;
} }
}, tag, true, "indexCommitSize", Category.SEARCHER.toString(), scope); }, true, "indexCommitSize", Category.SEARCHER.toString(), scope);
// statsCache metrics // statsCache metrics
manager.registerGauge(this, registry, parentContext.gauge(this,
new MetricsMap((detailed, map) -> { new MetricsMap((detailed, map) -> {
statsCache.getCacheMetrics().getSnapshot(map::put); statsCache.getCacheMetrics().getSnapshot(map::put);
map.put("statsCacheImpl", statsCache.getClass().getSimpleName()); map.put("statsCacheImpl", statsCache.getClass().getSimpleName());
}), }), true, "statsCache", Category.CACHE.toString(), scope);
tag, true, "statsCache", Category.CACHE.toString(), scope);
}
@Override
public MetricRegistry getMetricRegistry() {
return core.getMetricRegistry();
} }
private static class FilterImpl extends Filter { private static class FilterImpl extends Filter {

41
solr/core/src/java/org/apache/solr/security/AuditLoggerPlugin.java
View File

@ -36,7 +36,6 @@ import java.util.concurrent.atomic.AtomicInteger;
import com.codahale.metrics.Counter; import com.codahale.metrics.Counter;
import com.codahale.metrics.Meter; import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.Timer; import com.codahale.metrics.Timer;
import com.fasterxml.jackson.annotation.JsonInclude.Include; import com.fasterxml.jackson.annotation.JsonInclude.Include;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
@ -45,8 +44,8 @@ import org.apache.solr.common.SolrException;
import org.apache.solr.common.util.ExecutorUtil; import org.apache.solr.common.util.ExecutorUtil;
import org.apache.solr.common.util.SolrjNamedThreadFactory; import org.apache.solr.common.util.SolrjNamedThreadFactory;
import org.apache.solr.core.SolrInfoBean; import org.apache.solr.core.SolrInfoBean;
import org.apache.solr.metrics.SolrMetricManager;
import org.apache.solr.metrics.SolrMetricProducer; import org.apache.solr.metrics.SolrMetricProducer;
import org.apache.solr.metrics.SolrMetricsContext;
import org.apache.solr.security.AuditEvent.EventType; import org.apache.solr.security.AuditEvent.EventType;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@ -75,14 +74,12 @@ public abstract class AuditLoggerPlugin implements Closeable, Runnable, SolrInfo
int blockingQueueSize; int blockingQueueSize;
protected AuditEventFormatter formatter; protected AuditEventFormatter formatter;
private MetricRegistry registry;
private Set<String> metricNames = ConcurrentHashMap.newKeySet(); private Set<String> metricNames = ConcurrentHashMap.newKeySet();
private ExecutorService executorService; private ExecutorService executorService;
private boolean closed; private boolean closed;
private MuteRules muteRules; private MuteRules muteRules;
protected String registryName; protected SolrMetricsContext solrMetricsContext;
protected SolrMetricManager metricManager;
protected Meter numErrors = new Meter(); protected Meter numErrors = new Meter();
protected Meter numLost = new Meter(); protected Meter numLost = new Meter();
protected Meter numLogged = new Meter(); protected Meter numLogged = new Meter();
@ -239,24 +236,21 @@ public abstract class AuditLoggerPlugin implements Closeable, Runnable, SolrInfo
} }
@Override @Override
public void initializeMetrics(SolrMetricManager manager, String registryName, String tag, final String scope) { public void initializeMetrics(SolrMetricsContext parentContext, final String scope) {
solrMetricsContext = parentContext.getChildContext(this);
String className = this.getClass().getSimpleName(); String className = this.getClass().getSimpleName();
log.debug("Initializing metrics for {}", className); log.debug("Initializing metrics for {}", className);
this.metricManager = manager; numErrors = solrMetricsContext.meter(this, "errors", getCategory().toString(), scope, className);
this.registryName = registryName; numLost = solrMetricsContext.meter(this, "lost", getCategory().toString(), scope, className);
// Metrics numLogged = solrMetricsContext.meter(this, "count", getCategory().toString(), scope, className);
registry = manager.registry(registryName); requestTimes = solrMetricsContext.timer(this, "requestTimes", getCategory().toString(), scope, className);
numErrors = manager.meter(this, registryName, "errors", getCategory().toString(), scope, className); totalTime = solrMetricsContext.counter(this, "totalTime", getCategory().toString(), scope, className);
numLost = manager.meter(this, registryName, "lost", getCategory().toString(), scope, className);
numLogged = manager.meter(this, registryName, "count", getCategory().toString(), scope, className);
requestTimes = manager.timer(this, registryName, "requestTimes", getCategory().toString(), scope, className);
totalTime = manager.counter(this, registryName, "totalTime", getCategory().toString(), scope, className);
if (async) { if (async) {
manager.registerGauge(this, registryName, () -> blockingQueueSize, "queueCapacity", true, "queueCapacity", getCategory().toString(), scope, className); solrMetricsContext.gauge(this, () -> blockingQueueSize, true, "queueCapacity", getCategory().toString(), scope, className);
manager.registerGauge(this, registryName, () -> blockingQueueSize - queue.remainingCapacity(), "queueSize", true, "queueSize", getCategory().toString(), scope, className); solrMetricsContext.gauge(this, () -> blockingQueueSize - queue.remainingCapacity(), true, "queueSize", getCategory().toString(), scope, className);
queuedTime = manager.timer(this, registryName, "queuedTime", getCategory().toString(), scope, className); queuedTime = solrMetricsContext.timer(this, "queuedTime", getCategory().toString(), scope, className);
} }
manager.registerGauge(this, registryName, () -> async, "async", true, "async", getCategory().toString(), scope, className); solrMetricsContext.gauge(this, () -> async, true, "async", getCategory().toString(), scope, className);
} }
@Override @Override
@ -280,8 +274,8 @@ public abstract class AuditLoggerPlugin implements Closeable, Runnable, SolrInfo
} }
@Override @Override
public MetricRegistry getMetricRegistry() { public SolrMetricsContext getSolrMetricsContext() {
return registry; return solrMetricsContext;
} }
/** /**
@ -325,6 +319,11 @@ public abstract class AuditLoggerPlugin implements Closeable, Runnable, SolrInfo
closed = true; closed = true;
log.info("Shutting down async Auditlogger background thread(s)"); log.info("Shutting down async Auditlogger background thread(s)");
executorService.shutdownNow(); executorService.shutdownNow();
try {
SolrMetricProducer.super.close();
} catch (Exception e) {
throw new IOException("Exception closing", e);
}
} }
} }

51
solr/core/src/java/org/apache/solr/security/AuthenticationPlugin.java
View File

@ -19,39 +19,33 @@ package org.apache.solr.security;
import javax.servlet.FilterChain; import javax.servlet.FilterChain;
import javax.servlet.ServletRequest; import javax.servlet.ServletRequest;
import javax.servlet.ServletResponse; import javax.servlet.ServletResponse;
import java.io.Closeable;
import java.util.Arrays;
import java.util.Map; import java.util.Map;
import java.util.Set; import java.util.Set;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
import com.codahale.metrics.Counter; import com.codahale.metrics.Counter;
import com.codahale.metrics.Meter; import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.Timer; import com.codahale.metrics.Timer;
import org.apache.solr.core.SolrInfoBean;
import org.apache.solr.metrics.SolrMetricManager;
import org.apache.solr.metrics.SolrMetricProducer;
import org.apache.http.HttpRequest; import org.apache.http.HttpRequest;
import org.apache.http.protocol.HttpContext; import org.apache.http.protocol.HttpContext;
import org.apache.solr.core.SolrInfoBean;
import org.apache.solr.metrics.SolrMetricProducer;
import org.apache.solr.metrics.SolrMetricsContext;
import org.eclipse.jetty.client.api.Request; import org.eclipse.jetty.client.api.Request;
/** /**
* *
* @lucene.experimental * @lucene.experimental
*/ */
public abstract class AuthenticationPlugin implements Closeable, SolrInfoBean, SolrMetricProducer { public abstract class AuthenticationPlugin implements SolrInfoBean, SolrMetricProducer {
final public static String AUTHENTICATION_PLUGIN_PROP = "authenticationPlugin"; final public static String AUTHENTICATION_PLUGIN_PROP = "authenticationPlugin";
final public static String HTTP_HEADER_X_SOLR_AUTHDATA = "X-Solr-AuthData"; final public static String HTTP_HEADER_X_SOLR_AUTHDATA = "X-Solr-AuthData";
// Metrics // Metrics
private Set<String> metricNames = ConcurrentHashMap.newKeySet(); private Set<String> metricNames = ConcurrentHashMap.newKeySet();
private MetricRegistry registry; protected SolrMetricsContext solrMetricsContext;
protected String registryName;
protected SolrMetricManager metricManager;
protected Meter numErrors = new Meter(); protected Meter numErrors = new Meter();
protected Counter requests = new Counter(); protected Counter requests = new Counter();
protected Timer requestTimes = new Timer(); protected Timer requestTimes = new Timer();
@ -145,21 +139,22 @@ public abstract class AuthenticationPlugin implements Closeable, SolrInfoBean, S
} }
@Override @Override
public void initializeMetrics(SolrMetricManager manager, String registryName, String tag, final String scope) { public SolrMetricsContext getSolrMetricsContext() {
this.metricManager = manager; return solrMetricsContext;
this.registryName = registryName; }
@Override
public void initializeMetrics(SolrMetricsContext parentContext, String scope) {
this.solrMetricsContext = parentContext.getChildContext(this);
// Metrics // Metrics
registry = manager.registry(registryName); numErrors = this.solrMetricsContext.meter(this, "errors", getCategory().toString(), scope);
numErrors = manager.meter(this, registryName, "errors", getCategory().toString(), scope); requests = this.solrMetricsContext.counter(this, "requests", getCategory().toString(), scope);
requests = manager.counter(this, registryName, "requests", getCategory().toString(), scope); numAuthenticated = this.solrMetricsContext.counter(this, "authenticated",getCategory().toString(), scope);
numAuthenticated = manager.counter(this, registryName, "authenticated", getCategory().toString(), scope); numPassThrough = this.solrMetricsContext.counter(this, "passThrough", getCategory().toString(), scope);
numPassThrough = manager.counter(this, registryName, "passThrough", getCategory().toString(), scope); numWrongCredentials = this.solrMetricsContext.counter(this, "failWrongCredentials",getCategory().toString(), scope);
numWrongCredentials = manager.counter(this, registryName, "failWrongCredentials", getCategory().toString(), scope); numMissingCredentials = this.solrMetricsContext.counter(this, "failMissingCredentials",getCategory().toString(), scope);
numMissingCredentials = manager.counter(this, registryName, "failMissingCredentials", getCategory().toString(), scope); requestTimes = this.solrMetricsContext.timer(this,"requestTimes", getCategory().toString(), scope);
requestTimes = manager.timer(this, registryName, "requestTimes", getCategory().toString(), scope); totalTime = this.solrMetricsContext.counter(this,"totalTime", getCategory().toString(), scope);
totalTime = manager.counter(this, registryName, "totalTime", getCategory().toString(), scope);
metricNames.addAll(Arrays.asList("errors", "requests", "authenticated", "passThrough",
"failWrongCredentials", "failMissingCredentials", "requestTimes", "totalTime"));
} }
@Override @Override
@ -181,10 +176,4 @@ public abstract class AuthenticationPlugin implements Closeable, SolrInfoBean, S
public Set<String> getMetricNames() { public Set<String> getMetricNames() {
return metricNames; return metricNames;
} }
@Override
public MetricRegistry getMetricRegistry() {
return registry;
}
} }

3
solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java
View File

@ -77,6 +77,7 @@ import org.apache.solr.metrics.AltBufferPoolMetricSet;
import org.apache.solr.metrics.MetricsMap; import org.apache.solr.metrics.MetricsMap;
import org.apache.solr.metrics.OperatingSystemMetricSet; import org.apache.solr.metrics.OperatingSystemMetricSet;
import org.apache.solr.metrics.SolrMetricManager; import org.apache.solr.metrics.SolrMetricManager;
import org.apache.solr.metrics.SolrMetricProducer;
import org.apache.solr.security.AuditEvent; import org.apache.solr.security.AuditEvent;
import org.apache.solr.security.AuthenticationPlugin; import org.apache.solr.security.AuthenticationPlugin;
import org.apache.solr.security.PKIAuthenticationPlugin; import org.apache.solr.security.PKIAuthenticationPlugin;
@ -108,7 +109,7 @@ public class SolrDispatchFilter extends BaseSolrFilter {
private boolean isV2Enabled = !"true".equals(System.getProperty("disable.v2.api", "false")); private boolean isV2Enabled = !"true".equals(System.getProperty("disable.v2.api", "false"));
private final String metricTag = Integer.toHexString(hashCode()); private final String metricTag = SolrMetricProducer.getUniqueMetricTag(this, null);
private SolrMetricManager metricManager; private SolrMetricManager metricManager;
private String registryName; private String registryName;
private volatile boolean closeOnDestroy = true; private volatile boolean closeOnDestroy = true;

20
solr/core/src/java/org/apache/solr/store/blockcache/Metrics.java
View File

@ -20,11 +20,10 @@ import java.util.Set;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicLong;
import com.codahale.metrics.MetricRegistry;
import org.apache.solr.core.SolrInfoBean; import org.apache.solr.core.SolrInfoBean;
import org.apache.solr.metrics.MetricsMap; import org.apache.solr.metrics.MetricsMap;
import org.apache.solr.metrics.SolrMetricManager;
import org.apache.solr.metrics.SolrMetricProducer; import org.apache.solr.metrics.SolrMetricProducer;
import org.apache.solr.metrics.SolrMetricsContext;
import org.apache.solr.search.SolrCacheBase; import org.apache.solr.search.SolrCacheBase;
/** /**
@ -54,17 +53,13 @@ public class Metrics extends SolrCacheBase implements SolrInfoBean, SolrMetricPr
public AtomicLong shardBuffercacheLost = new AtomicLong(0); public AtomicLong shardBuffercacheLost = new AtomicLong(0);
private MetricsMap metricsMap; private MetricsMap metricsMap;
private MetricRegistry registry;
private Set<String> metricNames = ConcurrentHashMap.newKeySet(); private Set<String> metricNames = ConcurrentHashMap.newKeySet();
private SolrMetricManager metricManager; private SolrMetricsContext solrMetricsContext;
private String registryName;
private long previous = System.nanoTime(); private long previous = System.nanoTime();
@Override @Override
public void initializeMetrics(SolrMetricManager manager, String registryName, String tag, String scope) { public void initializeMetrics(SolrMetricsContext parentContext, String scope) {
this.metricManager = manager; solrMetricsContext = parentContext.getChildContext(this);
this.registryName = registryName;
registry = manager.registry(registryName);
metricsMap = new MetricsMap((detailed, map) -> { metricsMap = new MetricsMap((detailed, map) -> {
long now = System.nanoTime(); long now = System.nanoTime();
long delta = Math.max(now - previous, 1); long delta = Math.max(now - previous, 1);
@ -108,7 +103,7 @@ public class Metrics extends SolrCacheBase implements SolrInfoBean, SolrMetricPr
previous = now; previous = now;
}); });
manager.registerGauge(this, registryName, metricsMap, tag, true, getName(), getCategory().toString(), scope); solrMetricsContext.gauge(this, metricsMap, true, getName(), getCategory().toString(), scope);
} }
private float getPerSecond(long value, double seconds) { private float getPerSecond(long value, double seconds) {
@ -133,8 +128,7 @@ public class Metrics extends SolrCacheBase implements SolrInfoBean, SolrMetricPr
} }
@Override @Override
public MetricRegistry getMetricRegistry() { public SolrMetricsContext getSolrMetricsContext() {
return registry; return solrMetricsContext;
} }
} }

21
solr/core/src/java/org/apache/solr/store/hdfs/HdfsLocalityReporter.java
View File

@ -32,8 +32,8 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileSystem;
import org.apache.solr.core.SolrInfoBean; import org.apache.solr.core.SolrInfoBean;
import org.apache.solr.metrics.MetricsMap; import org.apache.solr.metrics.MetricsMap;
import org.apache.solr.metrics.SolrMetricManager;
import org.apache.solr.metrics.SolrMetricProducer; import org.apache.solr.metrics.SolrMetricProducer;
import org.apache.solr.metrics.SolrMetricsContext;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@ -51,9 +51,7 @@ public class HdfsLocalityReporter implements SolrInfoBean, SolrMetricProducer {
private final ConcurrentMap<HdfsDirectory,ConcurrentMap<FileStatus,BlockLocation[]>> cache; private final ConcurrentMap<HdfsDirectory,ConcurrentMap<FileStatus,BlockLocation[]>> cache;
private final Set<String> metricNames = ConcurrentHashMap.newKeySet(); private final Set<String> metricNames = ConcurrentHashMap.newKeySet();
private MetricRegistry registry; private SolrMetricsContext solrMetricsContext;
private SolrMetricManager metricManager;
private String registryName;
public HdfsLocalityReporter() { public HdfsLocalityReporter() {
cache = new ConcurrentHashMap<>(); cache = new ConcurrentHashMap<>();
@ -89,17 +87,20 @@ public class HdfsLocalityReporter implements SolrInfoBean, SolrMetricProducer {
@Override @Override
public MetricRegistry getMetricRegistry() { public MetricRegistry getMetricRegistry() {
return registry; return solrMetricsContext != null ? solrMetricsContext.getMetricRegistry() : null;
}
@Override
public SolrMetricsContext getSolrMetricsContext() {
return solrMetricsContext;
} }
/** /**
* Provide statistics on HDFS block locality, both in terms of bytes and block counts. * Provide statistics on HDFS block locality, both in terms of bytes and block counts.
*/ */
@Override @Override
public void initializeMetrics(SolrMetricManager manager, String registryName, String tag, String scope) { public void initializeMetrics(SolrMetricsContext parentContext, String scope) {
this.metricManager = manager; solrMetricsContext = parentContext.getChildContext(this);
this.registryName = registryName;
registry = manager.registry(registryName);
MetricsMap metricsMap = new MetricsMap((detailed, map) -> { MetricsMap metricsMap = new MetricsMap((detailed, map) -> {
long totalBytes = 0; long totalBytes = 0;
long localBytes = 0; long localBytes = 0;
@ -149,7 +150,7 @@ public class HdfsLocalityReporter implements SolrInfoBean, SolrMetricProducer {
map.put(LOCALITY_BLOCKS_RATIO, localCount / (double) totalCount); map.put(LOCALITY_BLOCKS_RATIO, localCount / (double) totalCount);
} }
}); });
manager.registerGauge(this, registryName, metricsMap, tag, true, "hdfsLocality", getCategory().toString(), scope); solrMetricsContext.gauge(this, metricsMap, true, "hdfsLocality", getCategory().toString(), scope);
} }
/** /**

62
solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java
View File

@ -51,8 +51,8 @@ import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.NamedList;
import org.apache.solr.core.SolrConfig.UpdateHandlerInfo; import org.apache.solr.core.SolrConfig.UpdateHandlerInfo;
import org.apache.solr.core.SolrCore; import org.apache.solr.core.SolrCore;
import org.apache.solr.metrics.SolrMetricManager;
import org.apache.solr.metrics.SolrMetricProducer; import org.apache.solr.metrics.SolrMetricProducer;
import org.apache.solr.metrics.SolrMetricsContext;
import org.apache.solr.request.LocalSolrQueryRequest; import org.apache.solr.request.LocalSolrQueryRequest;
import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.request.SolrRequestInfo; import org.apache.solr.request.SolrRequestInfo;
@ -96,8 +96,7 @@ public class DirectUpdateHandler2 extends UpdateHandler implements SolrCoreState
LongAdder numDocsPending = new LongAdder(); LongAdder numDocsPending = new LongAdder();
LongAdder numErrors = new LongAdder(); LongAdder numErrors = new LongAdder();
Meter numErrorsCumulative; Meter numErrorsCumulative;
SolrMetricManager metricManager; SolrMetricsContext solrMetricsContext;
String registryName;
// tracks when auto-commit should occur // tracks when auto-commit should occur
protected final CommitTracker commitTracker; protected final CommitTracker commitTracker;
@ -170,48 +169,46 @@ public class DirectUpdateHandler2 extends UpdateHandler implements SolrCoreState
} }
@Override @Override
public void initializeMetrics(SolrMetricManager manager, String registryName, String tag, String scope) { public void initializeMetrics(SolrMetricsContext parentContext, String scope) {
this.metricManager = manager; solrMetricsContext = parentContext.getChildContext(this);
this.registryName = registryName; commitCommands = solrMetricsContext.meter(this, "commits", getCategory().toString(), scope);
this.registry = manager.registry(registryName); solrMetricsContext.gauge(this, () -> commitTracker.getCommitCount(), true, "autoCommits", getCategory().toString(), scope);
commitCommands = manager.meter(this, registryName, "commits", getCategory().toString(), scope); solrMetricsContext.gauge(this, () -> softCommitTracker.getCommitCount(), true, "softAutoCommits", getCategory().toString(), scope);
manager.registerGauge(this, registryName, () -> commitTracker.getCommitCount(), tag, true, "autoCommits", getCategory().toString(), scope);
manager.registerGauge(this, registryName, () -> softCommitTracker.getCommitCount(), tag, true, "softAutoCommits", getCategory().toString(), scope);
if (commitTracker.getDocsUpperBound() > 0) { if (commitTracker.getDocsUpperBound() > 0) {
manager.registerGauge(this, registryName, () -> commitTracker.getDocsUpperBound(), tag, true, "autoCommitMaxDocs", solrMetricsContext.gauge(this, () -> commitTracker.getDocsUpperBound(), true, "autoCommitMaxDocs",
getCategory().toString(), scope); getCategory().toString(), scope);
} }
if (commitTracker.getTimeUpperBound() > 0) { if (commitTracker.getTimeUpperBound() > 0) {
manager.registerGauge(this, registryName, () -> "" + commitTracker.getTimeUpperBound() + "ms", tag, true, "autoCommitMaxTime", solrMetricsContext.gauge(this, () -> "" + commitTracker.getTimeUpperBound() + "ms", true, "autoCommitMaxTime",
getCategory().toString(), scope); getCategory().toString(), scope);
} }
if (commitTracker.getTLogFileSizeUpperBound() > 0) { if (commitTracker.getTLogFileSizeUpperBound() > 0) {
manager.registerGauge(this, registryName, () -> commitTracker.getTLogFileSizeUpperBound(), tag, true, "autoCommitMaxSize", solrMetricsContext.gauge(this, () -> commitTracker.getTLogFileSizeUpperBound(), true, "autoCommitMaxSize",
getCategory().toString(), scope); getCategory().toString(), scope);
} }
if (softCommitTracker.getDocsUpperBound() > 0) { if (softCommitTracker.getDocsUpperBound() > 0) {
manager.registerGauge(this, registryName, () -> softCommitTracker.getDocsUpperBound(), tag, true, "softAutoCommitMaxDocs", solrMetricsContext.gauge(this, () -> softCommitTracker.getDocsUpperBound(), true, "softAutoCommitMaxDocs",
getCategory().toString(), scope); getCategory().toString(), scope);
} }
if (softCommitTracker.getTimeUpperBound() > 0) { if (softCommitTracker.getTimeUpperBound() > 0) {
manager.registerGauge(this, registryName, () -> "" + softCommitTracker.getTimeUpperBound() + "ms", tag, true, "softAutoCommitMaxTime", solrMetricsContext.gauge(this, () -> "" + softCommitTracker.getTimeUpperBound() + "ms", true, "softAutoCommitMaxTime",
getCategory().toString(), scope); getCategory().toString(), scope);
} }
optimizeCommands = manager.meter(this, registryName, "optimizes", getCategory().toString(), scope); optimizeCommands = solrMetricsContext.meter(this, "optimizes", getCategory().toString(), scope);
rollbackCommands = manager.meter(this, registryName, "rollbacks", getCategory().toString(), scope); rollbackCommands = solrMetricsContext.meter(this, "rollbacks", getCategory().toString(), scope);
splitCommands = manager.meter(this, registryName, "splits", getCategory().toString(), scope); splitCommands = solrMetricsContext.meter(this, "splits", getCategory().toString(), scope);
mergeIndexesCommands = manager.meter(this, registryName, "merges", getCategory().toString(), scope); mergeIndexesCommands = solrMetricsContext.meter(this, "merges", getCategory().toString(), scope);
expungeDeleteCommands = manager.meter(this, registryName, "expungeDeletes", getCategory().toString(), scope); expungeDeleteCommands = solrMetricsContext.meter(this, "expungeDeletes", getCategory().toString(), scope);
manager.registerGauge(this, registryName, () -> numDocsPending.longValue(), tag, true, "docsPending", getCategory().toString(), scope); solrMetricsContext.gauge(this, () -> numDocsPending.longValue(), true, "docsPending", getCategory().toString(), scope);
manager.registerGauge(this, registryName, () -> addCommands.longValue(), tag, true, "adds", getCategory().toString(), scope); solrMetricsContext.gauge(this, () -> addCommands.longValue(), true, "adds", getCategory().toString(), scope);
manager.registerGauge(this, registryName, () -> deleteByIdCommands.longValue(), tag, true, "deletesById", getCategory().toString(), scope); solrMetricsContext.gauge(this, () -> deleteByIdCommands.longValue(), true, "deletesById", getCategory().toString(), scope);
manager.registerGauge(this, registryName, () -> deleteByQueryCommands.longValue(), tag, true, "deletesByQuery", getCategory().toString(), scope); solrMetricsContext.gauge(this, () -> deleteByQueryCommands.longValue(), true, "deletesByQuery", getCategory().toString(), scope);
manager.registerGauge(this, registryName, () -> numErrors.longValue(), tag, true, "errors", getCategory().toString(), scope); solrMetricsContext.gauge(this, () -> numErrors.longValue(), true, "errors", getCategory().toString(), scope);
addCommandsCumulative = manager.meter(this, registryName, "cumulativeAdds", getCategory().toString(), scope); addCommandsCumulative = solrMetricsContext.meter(this, "cumulativeAdds", getCategory().toString(), scope);
deleteByIdCommandsCumulative = manager.meter(this, registryName, "cumulativeDeletesById", getCategory().toString(), scope); deleteByIdCommandsCumulative = solrMetricsContext.meter(this, "cumulativeDeletesById", getCategory().toString(), scope);
deleteByQueryCommandsCumulative = manager.meter(this, registryName, "cumulativeDeletesByQuery", getCategory().toString(), scope); deleteByQueryCommandsCumulative = solrMetricsContext.meter(this, "cumulativeDeletesByQuery", getCategory().toString(), scope);
numErrorsCumulative = manager.meter(this, registryName, "cumulativeErrors", getCategory().toString(), scope); numErrorsCumulative = solrMetricsContext.meter(this, "cumulativeErrors", getCategory().toString(), scope);
} }
private void deleteAll() throws IOException { private void deleteAll() throws IOException {
@ -805,6 +802,11 @@ public class DirectUpdateHandler2 extends UpdateHandler implements SolrCoreState
softCommitTracker.close(); softCommitTracker.close();
numDocsPending.reset(); numDocsPending.reset();
try {
SolrMetricProducer.super.close();
} catch (Exception e) {
throw new IOException("Error closing", e);
}
} }
@ -915,7 +917,7 @@ public class DirectUpdateHandler2 extends UpdateHandler implements SolrCoreState
} }
/** /**
* Calls either {@link IndexWriter#updateDocValues} or {@link IndexWriter#updateDocument}(s) as * Calls either {@link IndexWriter#updateDocValues} or <code>IndexWriter#updateDocument</code>(s) as
* needed based on {@link AddUpdateCommand#isInPlaceUpdate}. * needed based on {@link AddUpdateCommand#isInPlaceUpdate}.
* <p> * <p>
* If the this is an UPDATE_INPLACE cmd, then all fields included in * If the this is an UPDATE_INPLACE cmd, then all fields included in

38
solr/core/src/java/org/apache/solr/update/SolrIndexWriter.java
View File

@ -42,7 +42,7 @@ import org.apache.solr.core.DirectoryFactory;
import org.apache.solr.core.DirectoryFactory.DirContext; import org.apache.solr.core.DirectoryFactory.DirContext;
import org.apache.solr.core.SolrCore; import org.apache.solr.core.SolrCore;
import org.apache.solr.core.SolrInfoBean; import org.apache.solr.core.SolrInfoBean;
import org.apache.solr.metrics.SolrMetricManager; import org.apache.solr.metrics.SolrMetricsContext;
import org.apache.solr.schema.IndexSchema; import org.apache.solr.schema.IndexSchema;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@ -88,8 +88,7 @@ public class SolrIndexWriter extends IndexWriter {
private final AtomicLong runningMajorMergesDocs = new AtomicLong(); private final AtomicLong runningMajorMergesDocs = new AtomicLong();
private final AtomicLong runningMinorMergesDocs = new AtomicLong(); private final AtomicLong runningMinorMergesDocs = new AtomicLong();
private final SolrMetricManager metricManager; private final SolrMetricsContext solrMetricsContext;
private final String registryName;
// merge diagnostics. // merge diagnostics.
private final Map<String, Long> runningMerges = new ConcurrentHashMap<>(); private final Map<String, Long> runningMerges = new ConcurrentHashMap<>();
@ -120,8 +119,7 @@ public class SolrIndexWriter extends IndexWriter {
// no metrics // no metrics
mergeTotals = false; mergeTotals = false;
mergeDetails = false; mergeDetails = false;
metricManager = null; solrMetricsContext = null;
registryName = null;
} }
private SolrIndexWriter(SolrCore core, String name, String path, Directory directory, boolean create, IndexSchema schema, SolrIndexConfig config, IndexDeletionPolicy delPolicy, Codec codec) throws IOException { private SolrIndexWriter(SolrCore core, String name, String path, Directory directory, boolean create, IndexSchema schema, SolrIndexConfig config, IndexDeletionPolicy delPolicy, Codec codec) throws IOException {
@ -135,8 +133,7 @@ public class SolrIndexWriter extends IndexWriter {
infoStream = getConfig().getInfoStream(); infoStream = getConfig().getInfoStream();
this.directory = directory; this.directory = directory;
numOpens.incrementAndGet(); numOpens.incrementAndGet();
metricManager = core.getCoreContainer().getMetricManager(); solrMetricsContext = core.getSolrMetricsContext().getChildContext(this);
registryName = core.getCoreMetricManager().getRegistryName();
if (config.metricsInfo != null && config.metricsInfo.initArgs != null) { if (config.metricsInfo != null && config.metricsInfo.initArgs != null) {
Object v = config.metricsInfo.initArgs.get("majorMergeDocs"); Object v = config.metricsInfo.initArgs.get("majorMergeDocs");
if (v != null) { if (v != null) {
@ -160,21 +157,21 @@ public class SolrIndexWriter extends IndexWriter {
} }
if (mergeDetails) { if (mergeDetails) {
mergeTotals = true; // override mergeTotals = true; // override
majorMergedDocs = metricManager.meter(null, registryName, "docs", SolrInfoBean.Category.INDEX.toString(), "merge", "major"); majorMergedDocs = solrMetricsContext.meter(null, "docs", SolrInfoBean.Category.INDEX.toString(), "merge", "major");
majorDeletedDocs = metricManager.meter(null, registryName, "deletedDocs", SolrInfoBean.Category.INDEX.toString(), "merge", "major"); majorDeletedDocs = solrMetricsContext.meter(null, "deletedDocs", SolrInfoBean.Category.INDEX.toString(), "merge", "major");
} }
if (mergeTotals) { if (mergeTotals) {
minorMerge = metricManager.timer(null, registryName, "minor", SolrInfoBean.Category.INDEX.toString(), "merge"); minorMerge = solrMetricsContext.timer(null, "minor", SolrInfoBean.Category.INDEX.toString(), "merge");
majorMerge = metricManager.timer(null, registryName, "major", SolrInfoBean.Category.INDEX.toString(), "merge"); majorMerge = solrMetricsContext.timer(null, "major", SolrInfoBean.Category.INDEX.toString(), "merge");
mergeErrors = metricManager.counter(null, registryName, "errors", SolrInfoBean.Category.INDEX.toString(), "merge"); mergeErrors = solrMetricsContext.counter(null, "errors", SolrInfoBean.Category.INDEX.toString(), "merge");
String tag = core.getMetricTag(); String tag = core.getMetricTag();
metricManager.registerGauge(null, registryName, () -> runningMajorMerges.get(), tag, true, "running", SolrInfoBean.Category.INDEX.toString(), "merge", "major"); solrMetricsContext.gauge(null, () -> runningMajorMerges.get(), true, "running", SolrInfoBean.Category.INDEX.toString(), "merge", "major");
metricManager.registerGauge(null, registryName, () -> runningMinorMerges.get(), tag, true, "running", SolrInfoBean.Category.INDEX.toString(), "merge", "minor"); solrMetricsContext.gauge(null, () -> runningMinorMerges.get(), true, "running", SolrInfoBean.Category.INDEX.toString(), "merge", "minor");
metricManager.registerGauge(null, registryName, () -> runningMajorMergesDocs.get(), tag, true, "running.docs", SolrInfoBean.Category.INDEX.toString(), "merge", "major"); solrMetricsContext.gauge(null, () -> runningMajorMergesDocs.get(), true, "running.docs", SolrInfoBean.Category.INDEX.toString(), "merge", "major");
metricManager.registerGauge(null, registryName, () -> runningMinorMergesDocs.get(), tag, true, "running.docs", SolrInfoBean.Category.INDEX.toString(), "merge", "minor"); solrMetricsContext.gauge(null, () -> runningMinorMergesDocs.get(), true, "running.docs", SolrInfoBean.Category.INDEX.toString(), "merge", "minor");
metricManager.registerGauge(null, registryName, () -> runningMajorMergesSegments.get(), tag, true, "running.segments", SolrInfoBean.Category.INDEX.toString(), "merge", "major"); solrMetricsContext.gauge(null, () -> runningMajorMergesSegments.get(), true, "running.segments", SolrInfoBean.Category.INDEX.toString(), "merge", "major");
metricManager.registerGauge(null, registryName, () -> runningMinorMergesSegments.get(), tag, true, "running.segments", SolrInfoBean.Category.INDEX.toString(), "merge", "minor"); solrMetricsContext.gauge(null, () -> runningMinorMergesSegments.get(), true, "running.segments", SolrInfoBean.Category.INDEX.toString(), "merge", "minor");
flushMeter = metricManager.meter(null, registryName, "flush", SolrInfoBean.Category.INDEX.toString()); flushMeter = solrMetricsContext.meter(null, "flush", SolrInfoBean.Category.INDEX.toString());
} }
} }
} }
@ -345,6 +342,9 @@ public class SolrIndexWriter extends IndexWriter {
if (directoryFactory != null) { if (directoryFactory != null) {
directoryFactory.release(directory); directoryFactory.release(directory);
} }
if (solrMetricsContext != null) {
solrMetricsContext.unregister();
}
} }
} }

6
solr/core/src/java/org/apache/solr/update/UpdateHandler.java
View File

@ -22,7 +22,6 @@ import java.util.Set;
import java.util.Vector; import java.util.Vector;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
import com.codahale.metrics.MetricRegistry;
import org.apache.solr.core.DirectoryFactory; import org.apache.solr.core.DirectoryFactory;
import org.apache.solr.core.HdfsDirectoryFactory; import org.apache.solr.core.HdfsDirectoryFactory;
import org.apache.solr.core.PluginInfo; import org.apache.solr.core.PluginInfo;
@ -57,7 +56,6 @@ public abstract class UpdateHandler implements SolrInfoBean {
protected final UpdateLog ulog; protected final UpdateLog ulog;
protected Set<String> metricNames = ConcurrentHashMap.newKeySet(); protected Set<String> metricNames = ConcurrentHashMap.newKeySet();
protected MetricRegistry registry;
private void parseEventListeners() { private void parseEventListeners() {
final Class<SolrEventListener> clazz = SolrEventListener.class; final Class<SolrEventListener> clazz = SolrEventListener.class;
@ -211,8 +209,4 @@ public abstract class UpdateHandler implements SolrInfoBean {
public Set<String> getMetricNames() { public Set<String> getMetricNames() {
return metricNames; return metricNames;
} }
@Override
public MetricRegistry getMetricRegistry() {
return registry;
}
} }

25
solr/core/src/java/org/apache/solr/update/UpdateShardHandler.java
View File

@ -25,7 +25,6 @@ import java.util.concurrent.SynchronousQueue;
import java.util.concurrent.ThreadFactory; import java.util.concurrent.ThreadFactory;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import com.codahale.metrics.MetricRegistry;
import com.google.common.annotations.VisibleForTesting; import com.google.common.annotations.VisibleForTesting;
import org.apache.http.client.HttpClient; import org.apache.http.client.HttpClient;
import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.CloseableHttpClient;
@ -40,6 +39,7 @@ import org.apache.solr.common.util.SolrjNamedThreadFactory;
import org.apache.solr.core.SolrInfoBean; import org.apache.solr.core.SolrInfoBean;
import org.apache.solr.metrics.SolrMetricManager; import org.apache.solr.metrics.SolrMetricManager;
import org.apache.solr.metrics.SolrMetricProducer; import org.apache.solr.metrics.SolrMetricProducer;
import org.apache.solr.metrics.SolrMetricsContext;
import org.apache.solr.security.HttpClientBuilderPlugin; import org.apache.solr.security.HttpClientBuilderPlugin;
import org.apache.solr.update.processor.DistributedUpdateProcessor; import org.apache.solr.update.processor.DistributedUpdateProcessor;
import org.apache.solr.update.processor.DistributingUpdateProcessorFactory; import org.apache.solr.update.processor.DistributingUpdateProcessorFactory;
@ -90,7 +90,7 @@ public class UpdateShardHandler implements SolrMetricProducer, SolrInfoBean {
private final Set<String> metricNames = ConcurrentHashMap.newKeySet(); private final Set<String> metricNames = ConcurrentHashMap.newKeySet();
private MetricRegistry registry; private SolrMetricsContext solrMetricsContext;
private int socketTimeout = HttpClientUtil.DEFAULT_SO_TIMEOUT; private int socketTimeout = HttpClientUtil.DEFAULT_SO_TIMEOUT;
private int connectionTimeout = HttpClientUtil.DEFAULT_CONNECT_TIMEOUT; private int connectionTimeout = HttpClientUtil.DEFAULT_CONNECT_TIMEOUT;
@ -179,14 +179,14 @@ public class UpdateShardHandler implements SolrMetricProducer, SolrInfoBean {
} }
@Override @Override
public void initializeMetrics(SolrMetricManager manager, String registryName, String tag, String scope) { public void initializeMetrics(SolrMetricsContext parentContext, String scope) {
registry = manager.registry(registryName); solrMetricsContext = parentContext.getChildContext(this);
String expandedScope = SolrMetricManager.mkName(scope, getCategory().name()); String expandedScope = SolrMetricManager.mkName(scope, getCategory().name());
updateHttpListenerFactory.initializeMetrics(manager, registryName, tag, expandedScope); updateHttpListenerFactory.initializeMetrics(solrMetricsContext, expandedScope);
defaultConnectionManager.initializeMetrics(manager, registryName, tag, expandedScope); defaultConnectionManager.initializeMetrics(solrMetricsContext, expandedScope);
updateExecutor = MetricUtils.instrumentedExecutorService(updateExecutor, this, registry, updateExecutor = MetricUtils.instrumentedExecutorService(updateExecutor, this, solrMetricsContext.getMetricRegistry(),
SolrMetricManager.mkName("updateOnlyExecutor", expandedScope, "threadPool")); SolrMetricManager.mkName("updateOnlyExecutor", expandedScope, "threadPool"));
recoveryExecutor = MetricUtils.instrumentedExecutorService(recoveryExecutor, this, registry, recoveryExecutor = MetricUtils.instrumentedExecutorService(recoveryExecutor, this, solrMetricsContext.getMetricRegistry(),
SolrMetricManager.mkName("recoveryExecutor", expandedScope, "threadPool")); SolrMetricManager.mkName("recoveryExecutor", expandedScope, "threadPool"));
} }
@ -206,8 +206,8 @@ public class UpdateShardHandler implements SolrMetricProducer, SolrInfoBean {
} }
@Override @Override
public MetricRegistry getMetricRegistry() { public SolrMetricsContext getSolrMetricsContext() {
return registry; return solrMetricsContext;
} }
// if you are looking for a client to use, it's probably this one. // if you are looking for a client to use, it's probably this one.
@ -259,6 +259,11 @@ public class UpdateShardHandler implements SolrMetricProducer, SolrInfoBean {
} catch (Exception e) { } catch (Exception e) {
throw new RuntimeException(e); throw new RuntimeException(e);
} finally { } finally {
try {
SolrMetricProducer.super.close();
} catch (Exception e) {
// do nothing
}
IOUtils.closeQuietly(updateOnlyClient); IOUtils.closeQuietly(updateOnlyClient);
HttpClientUtil.close(recoveryOnlyClient); HttpClientUtil.close(recoveryOnlyClient);
HttpClientUtil.close(defaultClient); HttpClientUtil.close(defaultClient);

17
solr/core/src/java/org/apache/solr/util/stats/InstrumentedHttpListenerFactory.java
View File

@ -21,11 +21,10 @@ import java.util.HashMap;
import java.util.Locale; import java.util.Locale;
import java.util.Map; import java.util.Map;
import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.Timer; import com.codahale.metrics.Timer;
import org.apache.solr.client.solrj.impl.HttpListenerFactory; import org.apache.solr.client.solrj.impl.HttpListenerFactory;
import org.apache.solr.metrics.SolrMetricManager;
import org.apache.solr.metrics.SolrMetricProducer; import org.apache.solr.metrics.SolrMetricProducer;
import org.apache.solr.metrics.SolrMetricsContext;
import org.eclipse.jetty.client.api.Request; import org.eclipse.jetty.client.api.Request;
import org.eclipse.jetty.client.api.Result; import org.eclipse.jetty.client.api.Result;
@ -64,9 +63,7 @@ public class InstrumentedHttpListenerFactory implements SolrMetricProducer, Http
KNOWN_METRIC_NAME_STRATEGIES.put("methodOnly", METHOD_ONLY); KNOWN_METRIC_NAME_STRATEGIES.put("methodOnly", METHOD_ONLY);
} }
protected MetricRegistry metricsRegistry; protected SolrMetricsContext solrMetricsContext;
protected SolrMetricManager metricManager;
protected String registryName;
protected String scope; protected String scope;
protected NameStrategy nameStrategy; protected NameStrategy nameStrategy;
@ -85,7 +82,7 @@ public class InstrumentedHttpListenerFactory implements SolrMetricProducer, Http
@Override @Override
public void onBegin(Request request) { public void onBegin(Request request) {
if (metricsRegistry != null) { if (solrMetricsContext != null) {
timerContext = timer(request).time(); timerContext = timer(request).time();
} }
} }
@ -100,14 +97,12 @@ public class InstrumentedHttpListenerFactory implements SolrMetricProducer, Http
} }
private Timer timer(Request request) { private Timer timer(Request request) {
return metricsRegistry.timer(nameStrategy.getNameFor(scope, request)); return solrMetricsContext.timer(null, nameStrategy.getNameFor(scope, request));
} }
@Override @Override
public void initializeMetrics(SolrMetricManager manager, String registry, String tag, String scope) { public void initializeMetrics(SolrMetricsContext parentContext, String scope) {
this.metricManager = manager; this.solrMetricsContext = parentContext;
this.registryName = registry;
this.metricsRegistry = manager.registry(registry);
this.scope = scope; this.scope = scope;
} }
} }

30
solr/core/src/java/org/apache/solr/util/stats/InstrumentedPoolingHttpClientConnectionManager.java
View File

@ -22,6 +22,7 @@ import org.apache.http.conn.socket.ConnectionSocketFactory;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.solr.metrics.SolrMetricManager; import org.apache.solr.metrics.SolrMetricManager;
import org.apache.solr.metrics.SolrMetricProducer; import org.apache.solr.metrics.SolrMetricProducer;
import org.apache.solr.metrics.SolrMetricsContext;
/** /**
* Sub-class of PoolingHttpClientConnectionManager which tracks metrics interesting to Solr. * Sub-class of PoolingHttpClientConnectionManager which tracks metrics interesting to Solr.
@ -29,25 +30,28 @@ import org.apache.solr.metrics.SolrMetricProducer;
*/ */
public class InstrumentedPoolingHttpClientConnectionManager extends PoolingHttpClientConnectionManager implements SolrMetricProducer { public class InstrumentedPoolingHttpClientConnectionManager extends PoolingHttpClientConnectionManager implements SolrMetricProducer {
private SolrMetricManager metricManager; private SolrMetricsContext solrMetricsContext;
private String registryName;
public InstrumentedPoolingHttpClientConnectionManager(Registry<ConnectionSocketFactory> socketFactoryRegistry) { public InstrumentedPoolingHttpClientConnectionManager(Registry<ConnectionSocketFactory> socketFactoryRegistry) {
super(socketFactoryRegistry); super(socketFactoryRegistry);
} }
@Override @Override
public void initializeMetrics(SolrMetricManager manager, String registry, String tag, String scope) { public SolrMetricsContext getSolrMetricsContext() {
this.metricManager = manager; return solrMetricsContext;
this.registryName = registry; }
manager.registerGauge(null, registry, () -> getTotalStats().getAvailable(),
tag, true, SolrMetricManager.mkName("availableConnections", scope)); @Override
public void initializeMetrics(SolrMetricsContext parentContext, String scope) {
this.solrMetricsContext = parentContext.getChildContext(this);
parentContext.gauge(null, () -> getTotalStats().getAvailable(),
true, SolrMetricManager.mkName("availableConnections", scope));
// this acquires a lock on the connection pool; remove if contention sucks // this acquires a lock on the connection pool; remove if contention sucks
manager.registerGauge(null, registry, () -> getTotalStats().getLeased(), parentContext.gauge(null, () -> getTotalStats().getLeased(),
tag, true, SolrMetricManager.mkName("leasedConnections", scope)); true, SolrMetricManager.mkName("leasedConnections", scope));
manager.registerGauge(null, registry, () -> getTotalStats().getMax(), parentContext.gauge(null, () -> getTotalStats().getMax(),
tag, true, SolrMetricManager.mkName("maxConnections", scope)); true, SolrMetricManager.mkName("maxConnections", scope));
manager.registerGauge(null, registry, () -> getTotalStats().getPending(), parentContext.gauge(null, () -> getTotalStats().getPending(),
tag, true, SolrMetricManager.mkName("pendingConnections", scope)); true, SolrMetricManager.mkName("pendingConnections", scope));
} }
} }

144
solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java
View File

@ -17,6 +17,7 @@
package org.apache.solr.handler.admin; package org.apache.solr.handler.admin;
import java.util.Arrays;
import java.util.Map; import java.util.Map;
import com.codahale.metrics.Counter; import com.codahale.metrics.Counter;
@ -24,6 +25,15 @@ import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap; import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.common.util.Utils;
import org.apache.solr.core.PluginBag;
import org.apache.solr.core.PluginInfo;
import org.apache.solr.core.SolrCore;
import org.apache.solr.handler.RequestHandlerBase;
import org.apache.solr.metrics.MetricsMap;
import org.apache.solr.metrics.SolrMetricsContext;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.request.SolrRequestHandler;
import org.apache.solr.response.SolrQueryResponse; import org.apache.solr.response.SolrQueryResponse;
import org.junit.AfterClass; import org.junit.AfterClass;
import org.junit.BeforeClass; import org.junit.BeforeClass;
@ -53,7 +63,7 @@ public class MetricsHandlerTest extends SolrTestCaseJ4 {
@AfterClass @AfterClass
public static void cleanupMetrics() throws Exception { public static void cleanupMetrics() throws Exception {
if (null != h) { if (null != h) {
h.getCoreContainer().getMetricManager().registry("solr.jvm" ).remove("solrtest_foo"); h.getCoreContainer().getMetricManager().registry("solr.jvm").remove("solrtest_foo");
h.getCoreContainer().getMetricManager().registry("solr.jetty").remove("solrtest_foo"); h.getCoreContainer().getMetricManager().registry("solr.jetty").remove("solrtest_foo");
h.getCoreContainer().getMetricManager().registry("solr.jetty").remove("solrtest_foo:bar"); h.getCoreContainer().getMetricManager().registry("solr.jetty").remove("solrtest_foo:bar");
} }
@ -145,7 +155,7 @@ public class MetricsHandlerTest extends SolrTestCaseJ4 {
assertNotNull(values.get("metrics")); assertNotNull(values.get("metrics"));
values = (NamedList) values.get("metrics"); values = (NamedList) values.get("metrics");
assertEquals(1, values.size()); assertEquals(1, values.size());
assertEquals(13, ((NamedList)values.get("solr.node")).size()); assertEquals(13, ((NamedList) values.get("solr.node")).size());
assertNotNull(values.get("solr.node")); assertNotNull(values.get("solr.node"));
values = (NamedList) values.get("solr.node"); values = (NamedList) values.get("solr.node");
assertNotNull(values.get("CONTAINER.cores.lazy")); // this is a gauge node assertNotNull(values.get("CONTAINER.cores.lazy")); // this is a gauge node
@ -171,7 +181,7 @@ public class MetricsHandlerTest extends SolrTestCaseJ4 {
assertNotNull(values.get("solr.core.collection1")); assertNotNull(values.get("solr.core.collection1"));
values = (NamedList) values.get("solr.core.collection1"); values = (NamedList) values.get("solr.core.collection1");
assertEquals(1, values.size()); assertEquals(1, values.size());
Map m = (Map)values.get("CACHE.core.fieldCache"); Map m = (Map) values.get("CACHE.core.fieldCache");
assertNotNull(m); assertNotNull(m);
assertNotNull(m.get("entries_count")); assertNotNull(m.get("entries_count"));
@ -223,7 +233,7 @@ public class MetricsHandlerTest extends SolrTestCaseJ4 {
assertTrue(nl.size() > 0); assertTrue(nl.size() > 0);
nl.forEach((k, v) -> { nl.forEach((k, v) -> {
assertTrue(v instanceof Map); assertTrue(v instanceof Map);
Map map = (Map)v; Map map = (Map) v;
assertTrue(map.size() > 2); assertTrue(map.size() > 2);
}); });
@ -238,7 +248,7 @@ public class MetricsHandlerTest extends SolrTestCaseJ4 {
assertTrue(nl.size() > 0); assertTrue(nl.size() > 0);
nl.forEach((k, v) -> { nl.forEach((k, v) -> {
assertTrue(v instanceof Map); assertTrue(v instanceof Map);
Map map = (Map)v; Map map = (Map) v;
assertEquals(2, map.size()); assertEquals(2, map.size());
assertNotNull(map.get("inserts")); assertNotNull(map.get("inserts"));
assertNotNull(map.get("size")); assertNotNull(map.get("size"));
@ -257,7 +267,7 @@ public class MetricsHandlerTest extends SolrTestCaseJ4 {
Object val = values.findRecursive("metrics", key1); Object val = values.findRecursive("metrics", key1);
assertNotNull(val); assertNotNull(val);
assertTrue(val instanceof Map); assertTrue(val instanceof Map);
assertTrue(((Map)val).size() >= 2); assertTrue(((Map) val).size() >= 2);
String key2 = "solr.core.collection1:CACHE.core.fieldCache:entries_count"; String key2 = "solr.core.collection1:CACHE.core.fieldCache:entries_count";
resp = new SolrQueryResponse(); resp = new SolrQueryResponse();
@ -276,7 +286,7 @@ public class MetricsHandlerTest extends SolrTestCaseJ4 {
val = values.findRecursive("metrics", key3); val = values.findRecursive("metrics", key3);
assertNotNull(val); assertNotNull(val);
assertTrue(val instanceof Number); assertTrue(val instanceof Number);
assertEquals(3, ((Number)val).intValue()); assertEquals(3, ((Number) val).intValue());
// test multiple keys // test multiple keys
resp = new SolrQueryResponse(); resp = new SolrQueryResponse();
@ -306,7 +316,7 @@ public class MetricsHandlerTest extends SolrTestCaseJ4 {
handler.handleRequestBody(req(CommonParams.QT, "/admin/metrics", CommonParams.WT, "json", handler.handleRequestBody(req(CommonParams.QT, "/admin/metrics", CommonParams.WT, "json",
MetricsHandler.KEY_PARAM, "foo", MetricsHandler.KEY_PARAM, "foo:bar:baz:xyz"), resp); MetricsHandler.KEY_PARAM, "foo", MetricsHandler.KEY_PARAM, "foo:bar:baz:xyz"), resp);
values = resp.getValues(); values = resp.getValues();
NamedList metrics = (NamedList)values.get("metrics"); NamedList metrics = (NamedList) values.get("metrics");
assertEquals(0, metrics.size()); assertEquals(0, metrics.size());
assertNotNull(values.findRecursive("errors", "foo")); assertNotNull(values.findRecursive("errors", "foo"));
assertNotNull(values.findRecursive("errors", "foo:bar:baz:xyz")); assertNotNull(values.findRecursive("errors", "foo:bar:baz:xyz"));
@ -316,7 +326,7 @@ public class MetricsHandlerTest extends SolrTestCaseJ4 {
handler.handleRequestBody(req(CommonParams.QT, "/admin/metrics", CommonParams.WT, "json", handler.handleRequestBody(req(CommonParams.QT, "/admin/metrics", CommonParams.WT, "json",
MetricsHandler.KEY_PARAM, "foo:bar:baz"), resp); MetricsHandler.KEY_PARAM, "foo:bar:baz"), resp);
values = resp.getValues(); values = resp.getValues();
metrics = (NamedList)values.get("metrics"); metrics = (NamedList) values.get("metrics");
assertEquals(0, metrics.size()); assertEquals(0, metrics.size());
assertNotNull(values.findRecursive("errors", "foo:bar:baz")); assertNotNull(values.findRecursive("errors", "foo:bar:baz"));
@ -325,8 +335,122 @@ public class MetricsHandlerTest extends SolrTestCaseJ4 {
handler.handleRequestBody(req(CommonParams.QT, "/admin/metrics", CommonParams.WT, "json", handler.handleRequestBody(req(CommonParams.QT, "/admin/metrics", CommonParams.WT, "json",
MetricsHandler.KEY_PARAM, "solr.jetty:unknown:baz"), resp); MetricsHandler.KEY_PARAM, "solr.jetty:unknown:baz"), resp);
values = resp.getValues(); values = resp.getValues();
metrics = (NamedList)values.get("metrics"); metrics = (NamedList) values.get("metrics");
assertEquals(0, metrics.size()); assertEquals(0, metrics.size());
assertNotNull(values.findRecursive("errors", "solr.jetty:unknown:baz")); assertNotNull(values.findRecursive("errors", "solr.jetty:unknown:baz"));
} }
@Test
public void testMetricsUnload() throws Exception {
SolrCore core = h.getCoreContainer().getCore("collection1");//;.getRequestHandlers().put("/dumphandler", new DumpRequestHandler());
RefreshablePluginHolder pluginHolder =null;
try {
PluginInfo info = new PluginInfo(SolrRequestHandler.TYPE, Utils.makeMap("name", "/dumphandler", "class", DumpRequestHandler.class.getName()));
DumpRequestHandler requestHandler = new DumpRequestHandler();
requestHandler.gaugevals = Utils.makeMap("d_k1","v1", "d_k2","v2");
pluginHolder = new RefreshablePluginHolder(info, requestHandler);
core.getRequestHandlers().put("/dumphandler",
pluginHolder);
} finally {
core.close();
}
MetricsHandler handler = new MetricsHandler(h.getCoreContainer());
SolrQueryResponse resp = new SolrQueryResponse();
handler.handleRequestBody(req(CommonParams.QT, "/admin/metrics", CommonParams.WT, "json", MetricsHandler.COMPACT_PARAM, "true", "key", "solr.core.collection1:QUERY./dumphandler.dumphandlergauge"),
resp);
assertEquals("v1", resp.getValues()._getStr(Arrays.asList("metrics", "solr.core.collection1:QUERY./dumphandler.dumphandlergauge","d_k1"), null));
assertEquals("v2", resp.getValues()._getStr(Arrays.asList("metrics","solr.core.collection1:QUERY./dumphandler.dumphandlergauge","d_k2"), null));
pluginHolder.closeHandler();
resp = new SolrQueryResponse();
handler.handleRequestBody(req(CommonParams.QT, "/admin/metrics", CommonParams.WT, "json", MetricsHandler.COMPACT_PARAM, "true", "key", "solr.core.collection1:QUERY./dumphandler.dumphandlergauge"),
resp);
assertEquals(null, resp.getValues()._getStr(Arrays.asList("metrics", "solr.core.collection1:QUERY./dumphandler.dumphandlergauge","d_k1"), null));
assertEquals(null, resp.getValues()._getStr(Arrays.asList("metrics","solr.core.collection1:QUERY./dumphandler.dumphandlergauge","d_k2"), null));
DumpRequestHandler requestHandler = new DumpRequestHandler();
requestHandler.gaugevals = Utils.makeMap("d_k1","v1.1", "d_k2","v2.1");
pluginHolder.reset(requestHandler);
resp = new SolrQueryResponse();
handler.handleRequestBody(req(CommonParams.QT, "/admin/metrics", CommonParams.WT, "json", MetricsHandler.COMPACT_PARAM, "true", "key", "solr.core.collection1:QUERY./dumphandler.dumphandlergauge"),
resp);
assertEquals("v1.1", resp.getValues()._getStr(Arrays.asList("metrics", "solr.core.collection1:QUERY./dumphandler.dumphandlergauge","d_k1"), null));
assertEquals("v2.1", resp.getValues()._getStr(Arrays.asList("metrics","solr.core.collection1:QUERY./dumphandler.dumphandlergauge","d_k2"), null));
}
static class RefreshablePluginHolder extends PluginBag.PluginHolder<SolrRequestHandler> {
private DumpRequestHandler rh;
private SolrMetricsContext metricsInfo;
public RefreshablePluginHolder(PluginInfo info, DumpRequestHandler rh) {
super(info);
this.rh = rh;
}
@Override
public boolean isLoaded() {
return true;
}
void closeHandler() throws Exception {
this.metricsInfo = rh.getSolrMetricsContext();
// if(metricsInfo.tag.contains(String.valueOf(rh.hashCode()))){
// //this created a new child metrics
// metricsInfo = metricsInfo.getParent();
// }
this.rh.close();
}
void reset(DumpRequestHandler rh) throws Exception {
this.rh = rh;
if(metricsInfo != null)
this.rh.initializeMetrics(metricsInfo, "/dumphandler");
}
@Override
public SolrRequestHandler get() {
return rh;
}
}
public static class DumpRequestHandler extends RequestHandlerBase {
static String key = DumpRequestHandler.class.getName();
Map<String, Object> gaugevals ;
@Override
public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception {
rsp.add("key", key);
}
@Override
public String getDescription() {
return "DO nothing";
}
@Override
public void initializeMetrics(SolrMetricsContext parentContext, String scope) {
super.initializeMetrics(parentContext, scope);
MetricsMap metrics = new MetricsMap((detailed, map) -> map.putAll(gaugevals));
solrMetricsContext.gauge(this,
metrics, true, "dumphandlergauge", getCategory().toString(), scope);
}
@Override
public Boolean registerV2() {
return Boolean.FALSE;
}
}
} }

1
solr/core/src/test/org/apache/solr/search/TestCaffeineCache.java
View File

@ -281,5 +281,6 @@ public class TestCaffeineCache extends SolrTestCase {
} }
assertTrue("total ram bytes should be greater than 0", total > 0); assertTrue("total ram bytes should be greater than 0", total > 0);
assertTrue("total ram bytes exceeded limit", total < 1024 * 1024); assertTrue("total ram bytes exceeded limit", total < 1024 * 1024);
cache.close();
} }
} }

15
solr/core/src/test/org/apache/solr/search/TestFastLRUCache.java
View File

@ -30,7 +30,6 @@ import org.apache.solr.metrics.SolrMetricManager;
import org.apache.solr.util.ConcurrentLRUCache; import org.apache.solr.util.ConcurrentLRUCache;
import org.apache.solr.util.RTimer; import org.apache.solr.util.RTimer;
import java.io.IOException;
import java.util.HashMap; import java.util.HashMap;
import java.util.Map; import java.util.Map;
import java.util.Random; import java.util.Random;
@ -52,7 +51,7 @@ public class TestFastLRUCache extends SolrTestCase {
String registry = TestUtil.randomSimpleString(random(), 2, 10); String registry = TestUtil.randomSimpleString(random(), 2, 10);
String scope = TestUtil.randomSimpleString(random(), 2, 10); String scope = TestUtil.randomSimpleString(random(), 2, 10);
public void testPercentageAutowarm() throws IOException { public void testPercentageAutowarm() throws Exception {
FastLRUCache<Object, Object> fastCache = new FastLRUCache<>(); FastLRUCache<Object, Object> fastCache = new FastLRUCache<>();
Map<String, String> params = new HashMap<>(); Map<String, String> params = new HashMap<>();
params.put("size", "100"); params.put("size", "100");
@ -94,7 +93,7 @@ public class TestFastLRUCache extends SolrTestCase {
fastCacheNew.close(); fastCacheNew.close();
} }
public void testPercentageAutowarmMultiple() throws IOException { public void testPercentageAutowarmMultiple() throws Exception {
doTestPercentageAutowarm(100, 50, new int[]{51, 55, 60, 70, 80, 99, 100}, new int[]{1, 2, 3, 5, 10, 20, 30, 40, 50}); doTestPercentageAutowarm(100, 50, new int[]{51, 55, 60, 70, 80, 99, 100}, new int[]{1, 2, 3, 5, 10, 20, 30, 40, 50});
doTestPercentageAutowarm(100, 25, new int[]{76, 80, 99, 100}, new int[]{1, 2, 3, 5, 10, 20, 30, 40, 50, 51, 55, 60, 70}); doTestPercentageAutowarm(100, 25, new int[]{76, 80, 99, 100}, new int[]{1, 2, 3, 5, 10, 20, 30, 40, 50, 51, 55, 60, 70});
doTestPercentageAutowarm(1000, 10, new int[]{901, 930, 950, 999, 1000}, new int[]{1, 5, 100, 200, 300, 400, 800, 899, 900}); doTestPercentageAutowarm(1000, 10, new int[]{901, 930, 950, 999, 1000}, new int[]{1, 5, 100, 200, 300, 400, 800, 899, 900});
@ -102,7 +101,7 @@ public class TestFastLRUCache extends SolrTestCase {
doTestPercentageAutowarm(100, 0, new int[]{}, new int[]{1, 10, 25, 51, 55, 60, 70, 80, 99, 100, 200, 300}); doTestPercentageAutowarm(100, 0, new int[]{}, new int[]{1, 10, 25, 51, 55, 60, 70, 80, 99, 100, 200, 300});
} }
private void doTestPercentageAutowarm(int limit, int percentage, int[] hits, int[]misses) { private void doTestPercentageAutowarm(int limit, int percentage, int[] hits, int[]misses) throws Exception {
FastLRUCache<Object, Object> fastCache = new FastLRUCache<>(); FastLRUCache<Object, Object> fastCache = new FastLRUCache<>();
Map<String, String> params = new HashMap<>(); Map<String, String> params = new HashMap<>();
params.put("size", String.valueOf(limit)); params.put("size", String.valueOf(limit));
@ -136,7 +135,7 @@ public class TestFastLRUCache extends SolrTestCase {
fastCacheNew.close(); fastCacheNew.close();
} }
public void testNoAutowarm() throws IOException { public void testNoAutowarm() throws Exception {
FastLRUCache<Object, Object> fastCache = new FastLRUCache<>(); FastLRUCache<Object, Object> fastCache = new FastLRUCache<>();
Map<String, String> params = new HashMap<>(); Map<String, String> params = new HashMap<>();
params.put("size", "100"); params.put("size", "100");
@ -166,7 +165,7 @@ public class TestFastLRUCache extends SolrTestCase {
fastCacheNew.close(); fastCacheNew.close();
} }
public void testFullAutowarm() throws IOException { public void testFullAutowarm() throws Exception {
FastLRUCache<Object, Object> cache = new FastLRUCache<>(); FastLRUCache<Object, Object> cache = new FastLRUCache<>();
Map<Object, Object> params = new HashMap<>(); Map<Object, Object> params = new HashMap<>();
params.put("size", "100"); params.put("size", "100");
@ -196,7 +195,7 @@ public class TestFastLRUCache extends SolrTestCase {
cacheNew.close(); cacheNew.close();
} }
public void testSimple() throws IOException { public void testSimple() throws Exception {
FastLRUCache sc = new FastLRUCache(); FastLRUCache sc = new FastLRUCache();
Map l = new HashMap(); Map l = new HashMap();
l.put("size", "100"); l.put("size", "100");
@ -304,7 +303,7 @@ public class TestFastLRUCache extends SolrTestCase {
System.out.println("time=" + timer.getTime() + ", minSize="+minSize+",maxSize="+maxSize); System.out.println("time=" + timer.getTime() + ", minSize="+minSize+",maxSize="+maxSize);
} }
public void testAccountable() { public void testAccountable() throws Exception {
FastLRUCache<Query, DocSet> sc = new FastLRUCache<>(); FastLRUCache<Query, DocSet> sc = new FastLRUCache<>();
try { try {
Map l = new HashMap(); Map l = new HashMap();

2
solr/core/src/test/org/apache/solr/search/TestLFUCache.java
View File

@ -134,7 +134,7 @@ public class TestLFUCache extends SolrTestCaseJ4 {
@Test @Test
public void testSimple() throws IOException { public void testSimple() throws Exception {
SolrMetricManager metricManager = new SolrMetricManager(); SolrMetricManager metricManager = new SolrMetricManager();
Random r = random(); Random r = random();
String registry = TestUtil.randomSimpleString(r, 2, 10); String registry = TestUtil.randomSimpleString(r, 2, 10);

9
solr/core/src/test/org/apache/solr/search/TestLRUCache.java
View File

@ -16,7 +16,6 @@
*/ */
package org.apache.solr.search; package org.apache.solr.search;
import java.io.IOException;
import java.util.HashMap; import java.util.HashMap;
import java.util.Map; import java.util.Map;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
@ -38,7 +37,7 @@ public class TestLRUCache extends SolrTestCase {
String registry = TestUtil.randomSimpleString(random(), 2, 10); String registry = TestUtil.randomSimpleString(random(), 2, 10);
String scope = TestUtil.randomSimpleString(random(), 2, 10); String scope = TestUtil.randomSimpleString(random(), 2, 10);
public void testFullAutowarm() throws IOException { public void testFullAutowarm() throws Exception {
LRUCache<Object, Object> lruCache = new LRUCache<>(); LRUCache<Object, Object> lruCache = new LRUCache<>();
Map<String, String> params = new HashMap<>(); Map<String, String> params = new HashMap<>();
params.put("size", "100"); params.put("size", "100");
@ -64,14 +63,14 @@ public class TestLRUCache extends SolrTestCase {
lruCacheNew.close(); lruCacheNew.close();
} }
public void testPercentageAutowarm() throws IOException { public void testPercentageAutowarm() throws Exception {
doTestPercentageAutowarm(100, 50, new int[]{51, 55, 60, 70, 80, 99, 100}, new int[]{1, 2, 3, 5, 10, 20, 30, 40, 50}); doTestPercentageAutowarm(100, 50, new int[]{51, 55, 60, 70, 80, 99, 100}, new int[]{1, 2, 3, 5, 10, 20, 30, 40, 50});
doTestPercentageAutowarm(100, 25, new int[]{76, 80, 99, 100}, new int[]{1, 2, 3, 5, 10, 20, 30, 40, 50, 51, 55, 60, 70}); doTestPercentageAutowarm(100, 25, new int[]{76, 80, 99, 100}, new int[]{1, 2, 3, 5, 10, 20, 30, 40, 50, 51, 55, 60, 70});
doTestPercentageAutowarm(1000, 10, new int[]{901, 930, 950, 999, 1000}, new int[]{1, 5, 100, 200, 300, 400, 800, 899, 900}); doTestPercentageAutowarm(1000, 10, new int[]{901, 930, 950, 999, 1000}, new int[]{1, 5, 100, 200, 300, 400, 800, 899, 900});
doTestPercentageAutowarm(10, 10, new int[]{10}, new int[]{1, 5, 9, 100, 200, 300, 400, 800, 899, 900}); doTestPercentageAutowarm(10, 10, new int[]{10}, new int[]{1, 5, 9, 100, 200, 300, 400, 800, 899, 900});
} }
private void doTestPercentageAutowarm(int limit, int percentage, int[] hits, int[]misses) { private void doTestPercentageAutowarm(int limit, int percentage, int[] hits, int[]misses) throws Exception {
LRUCache<Object, Object> lruCache = new LRUCache<>(); LRUCache<Object, Object> lruCache = new LRUCache<>();
Map<String, String> params = new HashMap<>(); Map<String, String> params = new HashMap<>();
params.put("size", String.valueOf(limit)); params.put("size", String.valueOf(limit));
@ -101,7 +100,7 @@ public class TestLRUCache extends SolrTestCase {
} }
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
public void testNoAutowarm() throws IOException { public void testNoAutowarm() throws Exception {
LRUCache<Object, Object> lruCache = new LRUCache<>(); LRUCache<Object, Object> lruCache = new LRUCache<>();
lruCache.initializeMetrics(metricManager, registry, "foo", scope); lruCache.initializeMetrics(metricManager, registry, "foo", scope);
Map<String, String> params = new HashMap<>(); Map<String, String> params = new HashMap<>();

1
solr/solrj/src/java/org/apache/solr/common/util/ExecutorUtil.java
View File

@ -72,6 +72,7 @@ public class ExecutorUtil {
} }
public static void shutdownAndAwaitTermination(ExecutorService pool) { public static void shutdownAndAwaitTermination(ExecutorService pool) {
if(pool == null) return;
pool.shutdown(); // Disable new tasks from being submitted pool.shutdown(); // Disable new tasks from being submitted
awaitTermination(pool); awaitTermination(pool);
} }