SOLR-10247: Support non-numeric metrics and a "compact" format of /admin/metrics.

This commit is contained in:
Andrzej Bialecki 2017-03-14 11:17:56 +01:00 committed by Shalin Shekhar Mangar
parent 6083f370a1
commit 52b3bc2f6d
15 changed files with 195 additions and 147 deletions

View File

@ -339,6 +339,8 @@ Other Changes
* SOLR-8876: change morphline test config files to work around 'importCommands' bug when using java9 (hossman)
* SOLR-10247: Support non-numeric metrics and a "compact" format of /admin/metrics output. (ab)
================== 6.4.2 ==================
Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release.

View File

@ -36,7 +36,6 @@ import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
import com.codahale.metrics.Gauge;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Maps;
import org.apache.http.auth.AuthSchemeProvider;
@ -532,16 +531,13 @@ public class CoreContainer {
containerProperties.putAll(cfg.getSolrProperties());
// initialize gauges for reporting the number of cores
Gauge<Integer> loadedCores = () -> solrCores.getCores().size();
Gauge<Integer> lazyCores = () -> solrCores.getCoreNames().size() - solrCores.getCores().size();
Gauge<Integer> unloadedCores = () -> solrCores.getAllCoreNames().size() - solrCores.getCoreNames().size();
metricManager.register(SolrMetricManager.getRegistryName(SolrInfoMBean.Group.node),
loadedCores, true, "loaded", SolrInfoMBean.Category.CONTAINER.toString(), "cores");
metricManager.register(SolrMetricManager.getRegistryName(SolrInfoMBean.Group.node),
lazyCores, true, "lazy",SolrInfoMBean.Category.CONTAINER.toString(), "cores");
metricManager.register(SolrMetricManager.getRegistryName(SolrInfoMBean.Group.node),
unloadedCores, true, "unloaded",SolrInfoMBean.Category.CONTAINER.toString(), "cores");
String registryName = SolrMetricManager.getRegistryName(SolrInfoMBean.Group.node);
metricManager.registerGauge(registryName, () -> solrCores.getCores().size(),
true, "loaded", SolrInfoMBean.Category.CONTAINER.toString(), "cores");
metricManager.registerGauge(registryName, () -> solrCores.getCoreNames().size() - solrCores.getCores().size(),
true, "lazy",SolrInfoMBean.Category.CONTAINER.toString(), "cores");
metricManager.registerGauge(registryName, () -> solrCores.getAllCoreNames().size() - solrCores.getCoreNames().size(),
true, "unloaded",SolrInfoMBean.Category.CONTAINER.toString(), "cores");
if (isZooKeeperAware()) {
metricManager.loadClusterReporters(cfg.getMetricReporterPlugins(), this);

View File

@ -169,7 +169,7 @@ import static org.apache.solr.common.params.CommonParams.PATH;
/**
*
*/
public final class SolrCore implements SolrInfoMBean, Closeable {
public final class SolrCore implements SolrInfoMBean, SolrMetricProducer, Closeable {
public static final String version="1.0";
@ -214,11 +214,11 @@ public final class SolrCore implements SolrInfoMBean, Closeable {
private final ReentrantLock ruleExpiryLock;
private final ReentrantLock snapshotDelLock; // A lock instance to guard against concurrent deletions.
private final Timer newSearcherTimer;
private final Timer newSearcherWarmupTimer;
private final Counter newSearcherCounter;
private final Counter newSearcherMaxReachedCounter;
private final Counter newSearcherOtherErrorsCounter;
private Timer newSearcherTimer;
private Timer newSearcherWarmupTimer;
private Counter newSearcherCounter;
private Counter newSearcherMaxReachedCounter;
private Counter newSearcherOtherErrorsCounter;
public Date getStartTimeStamp() { return startTime; }
@ -901,11 +901,7 @@ public final class SolrCore implements SolrInfoMBean, Closeable {
SolrMetricManager metricManager = this.coreDescriptor.getCoreContainer().getMetricManager();
// initialize searcher-related metrics
newSearcherCounter = metricManager.counter(coreMetricManager.getRegistryName(), "new", Category.SEARCHER.toString());
newSearcherTimer = metricManager.timer(coreMetricManager.getRegistryName(), "time", Category.SEARCHER.toString(), "new");
newSearcherWarmupTimer = metricManager.timer(coreMetricManager.getRegistryName(), "warmup", Category.SEARCHER.toString(), "new");
newSearcherMaxReachedCounter = metricManager.counter(coreMetricManager.getRegistryName(), "maxReached", Category.SEARCHER.toString(), "new");
newSearcherOtherErrorsCounter = metricManager.counter(coreMetricManager.getRegistryName(), "errors", Category.SEARCHER.toString(), "new");
initializeMetrics(metricManager, coreMetricManager.getRegistryName(), null);
// Initialize JMX
this.infoRegistry = initInfoRegistry(name, config);
@ -1125,6 +1121,24 @@ public final class SolrCore implements SolrInfoMBean, Closeable {
return coreMetricManager;
}
@Override
public void initializeMetrics(SolrMetricManager manager, String registry, String scope) {
newSearcherCounter = manager.counter(registry, "new", Category.SEARCHER.toString());
newSearcherTimer = manager.timer(registry, "time", Category.SEARCHER.toString(), "new");
newSearcherWarmupTimer = manager.timer(registry, "warmup", Category.SEARCHER.toString(), "new");
newSearcherMaxReachedCounter = manager.counter(registry, "maxReached", Category.SEARCHER.toString(), "new");
newSearcherOtherErrorsCounter = manager.counter(registry, "errors", Category.SEARCHER.toString(), "new");
manager.registerGauge(registry, () -> name == null ? "(null)" : name, true, "coreName", Category.CORE.toString());
manager.registerGauge(registry, () -> startTime, true, "startTime", Category.CORE.toString());
manager.registerGauge(registry, () -> getOpenCount(), true, "refCount", Category.CORE.toString());
manager.registerGauge(registry, () -> resourceLoader.getInstancePath(), true, "instanceDir", Category.CORE.toString());
manager.registerGauge(registry, () -> getIndexDir(), true, "indexDir", Category.CORE.toString());
manager.registerGauge(registry, () -> getIndexSize(), true, "sizeInBytes", Category.INDEX.toString());
manager.registerGauge(registry, () -> NumberUtils.readableSize(getIndexSize()), true, "size", Category.INDEX.toString());
manager.registerGauge(registry, () -> coreDescriptor.getCoreContainer().getCoreNames(this), true, "aliases", Category.CORE.toString());
}
private Map<String,SolrInfoMBean> initInfoRegistry(String name, SolrConfig config) {
if (config.jmxConfig.enabled) {
return new JmxMonitoredMap<String, SolrInfoMBean>(name, coreMetricManager.getRegistryName(), String.valueOf(this.hashCode()), config.jmxConfig);

View File

@ -50,6 +50,8 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionName
final CoreContainer container;
final SolrMetricManager metricManager;
public static final String COMPACT_PARAM = "compact";
public MetricsHandler() {
this.container = null;
this.metricManager = null;
@ -71,6 +73,7 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionName
throw new SolrException(SolrException.ErrorCode.INVALID_STATE, "Core container instance not initialized");
}
boolean compact = req.getParams().getBool(COMPACT_PARAM, false);
MetricFilter mustMatchFilter = parseMustMatchFilter(req);
List<MetricType> metricTypes = parseMetricTypes(req);
List<MetricFilter> metricFilters = metricTypes.stream().map(MetricType::asMetricFilter).collect(Collectors.toList());
@ -79,7 +82,8 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionName
NamedList response = new NamedList();
for (String registryName : requestedRegistries) {
MetricRegistry registry = metricManager.registry(registryName);
response.add(registryName, MetricUtils.toNamedList(registry, metricFilters, mustMatchFilter, false, false, null));
response.add(registryName, MetricUtils.toNamedList(registry, metricFilters, mustMatchFilter, false,
false, compact, null));
}
rsp.getValues().add("metrics", response);
}

View File

@ -38,6 +38,7 @@ import java.util.regex.PatternSyntaxException;
import java.util.stream.Collectors;
import com.codahale.metrics.Counter;
import com.codahale.metrics.Gauge;
import com.codahale.metrics.Histogram;
import com.codahale.metrics.Meter;
import com.codahale.metrics.Metric;
@ -520,7 +521,9 @@ public class SolrMetricManager {
}
}
public void registerGauge(String registry, Gauge<?> gauge, boolean force, String metricName, String... metricPath) {
register(registry, gauge, force, metricName, metricPath);
}
/**
* This method creates a hierarchical name with arbitrary levels of hierarchy

View File

@ -361,7 +361,7 @@ public class SolrReporter extends ScheduledReporter {
}
final String effectiveGroup = group;
MetricUtils.toSolrInputDocuments(metricManager.registry(registryName), Collections.singletonList(report.filter), MetricFilter.ALL,
skipHistograms, skipAggregateValues, metadata, doc -> {
skipHistograms, skipAggregateValues, false, metadata, doc -> {
doc.setField(REGISTRY_ID, registryName);
doc.setField(GROUP_ID, effectiveGroup);
if (effectiveLabel != null) {

View File

@ -26,7 +26,6 @@ import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.concurrent.atomic.LongAdder;
import com.codahale.metrics.Gauge;
import com.codahale.metrics.Meter;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
@ -164,25 +163,18 @@ public class DirectUpdateHandler2 extends UpdateHandler implements SolrCoreState
@Override
public void initializeMetrics(SolrMetricManager manager, String registry, String scope) {
commitCommands = manager.meter(registry, "commits", getCategory().toString(), scope);
Gauge<Integer> autoCommits = () -> commitTracker.getCommitCount();
manager.register(registry, autoCommits, true, "autoCommits", getCategory().toString(), scope);
Gauge<Integer> softAutoCommits = () -> softCommitTracker.getCommitCount();
manager.register(registry, softAutoCommits, true, "softAutoCommits", getCategory().toString(), scope);
manager.registerGauge(registry, () -> commitTracker.getCommitCount(), true, "autoCommits", getCategory().toString(), scope);
manager.registerGauge(registry, () -> softCommitTracker.getCommitCount(), true, "softAutoCommits", getCategory().toString(), scope);
optimizeCommands = manager.meter(registry, "optimizes", getCategory().toString(), scope);
rollbackCommands = manager.meter(registry, "rollbacks", getCategory().toString(), scope);
splitCommands = manager.meter(registry, "splits", getCategory().toString(), scope);
mergeIndexesCommands = manager.meter(registry, "merges", getCategory().toString(), scope);
expungeDeleteCommands = manager.meter(registry, "expungeDeletes", getCategory().toString(), scope);
Gauge<Long> docsPending = () -> numDocsPending.longValue();
manager.register(registry, docsPending, true, "docsPending", getCategory().toString(), scope);
Gauge<Long> adds = () -> addCommands.longValue();
manager.register(registry, adds, true, "adds", getCategory().toString(), scope);
Gauge<Long> deletesById = () -> deleteByIdCommands.longValue();
manager.register(registry, deletesById, true, "deletesById", getCategory().toString(), scope);
Gauge<Long> deletesByQuery = () -> deleteByQueryCommands.longValue();
manager.register(registry, deletesByQuery, true, "deletesByQuery", getCategory().toString(), scope);
Gauge<Long> errors = () -> numErrors.longValue();
manager.register(registry, errors, true, "errors", getCategory().toString(), scope);
manager.registerGauge(registry, () -> numDocsPending.longValue(), true, "docsPending", getCategory().toString(), scope);
manager.registerGauge(registry, () -> addCommands.longValue(), true, "adds", getCategory().toString(), scope);
manager.registerGauge(registry, () -> deleteByIdCommands.longValue(), true, "deletesById", getCategory().toString(), scope);
manager.registerGauge(registry, () -> deleteByQueryCommands.longValue(), true, "deletesByQuery", getCategory().toString(), scope);
manager.registerGauge(registry, () -> numErrors.longValue(), true, "errors", getCategory().toString(), scope);
addCommandsCumulative = manager.meter(registry, "cumulativeAdds", getCategory().toString(), scope);
deleteByIdCommandsCumulative = manager.meter(registry, "cumulativeDeletesById", getCategory().toString(), scope);

View File

@ -24,7 +24,6 @@ import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import com.codahale.metrics.Counter;
import com.codahale.metrics.Gauge;
import com.codahale.metrics.Meter;
import com.codahale.metrics.Timer;
import org.apache.lucene.codecs.Codec;
@ -81,17 +80,11 @@ public class SolrIndexWriter extends IndexWriter {
private boolean mergeTotals = false;
private boolean mergeDetails = false;
private final AtomicInteger runningMajorMerges = new AtomicInteger();
private Gauge<Integer> runningMajorMergesGauge;
private final AtomicInteger runningMinorMerges = new AtomicInteger();
private Gauge<Integer> runningMinorMergesGauge;
private final AtomicInteger runningMajorMergesSegments = new AtomicInteger();
private Gauge<Integer> runningMajorMergesSegmentsGauge;
private final AtomicInteger runningMinorMergesSegments = new AtomicInteger();
private Gauge<Integer> runningMinorMergesSegmentsGauge;
private final AtomicLong runningMajorMergesDocs = new AtomicLong();
private Gauge<Long> runningMajorMergesDocsGauge;
private final AtomicLong runningMinorMergesDocs = new AtomicLong();
private Gauge<Long> runningMinorMergesDocsGauge;
public static SolrIndexWriter create(SolrCore core, String name, String path, DirectoryFactory directoryFactory, boolean create, IndexSchema schema, SolrIndexConfig config, IndexDeletionPolicy delPolicy, Codec codec) throws IOException {
@ -165,18 +158,12 @@ public class SolrIndexWriter extends IndexWriter {
minorMerge = metricManager.timer(registry, "minor", SolrInfoMBean.Category.INDEX.toString(), "merge");
majorMerge = metricManager.timer(registry, "major", SolrInfoMBean.Category.INDEX.toString(), "merge");
mergeErrors = metricManager.counter(registry, "errors", SolrInfoMBean.Category.INDEX.toString(), "merge");
runningMajorMergesGauge = () -> runningMajorMerges.get();
runningMinorMergesGauge = () -> runningMinorMerges.get();
runningMajorMergesDocsGauge = () -> runningMajorMergesDocs.get();
runningMinorMergesDocsGauge = () -> runningMinorMergesDocs.get();
runningMajorMergesSegmentsGauge = () -> runningMajorMergesSegments.get();
runningMinorMergesSegmentsGauge = () -> runningMinorMergesSegments.get();
metricManager.register(registry, runningMajorMergesGauge, true, "running", SolrInfoMBean.Category.INDEX.toString(), "merge", "major");
metricManager.register(registry, runningMinorMergesGauge, true, "running", SolrInfoMBean.Category.INDEX.toString(), "merge", "minor");
metricManager.register(registry, runningMajorMergesDocsGauge, true, "running.docs", SolrInfoMBean.Category.INDEX.toString(), "merge", "major");
metricManager.register(registry, runningMinorMergesDocsGauge, true, "running.docs", SolrInfoMBean.Category.INDEX.toString(), "merge", "minor");
metricManager.register(registry, runningMajorMergesSegmentsGauge, true, "running.segments", SolrInfoMBean.Category.INDEX.toString(), "merge", "major");
metricManager.register(registry, runningMinorMergesSegmentsGauge, true, "running.segments", SolrInfoMBean.Category.INDEX.toString(), "merge", "minor");
metricManager.registerGauge(registry, () -> runningMajorMerges.get(), true, "running", SolrInfoMBean.Category.INDEX.toString(), "merge", "major");
metricManager.registerGauge(registry, () -> runningMinorMerges.get(), true, "running", SolrInfoMBean.Category.INDEX.toString(), "merge", "minor");
metricManager.registerGauge(registry, () -> runningMajorMergesDocs.get(), true, "running.docs", SolrInfoMBean.Category.INDEX.toString(), "merge", "major");
metricManager.registerGauge(registry, () -> runningMinorMergesDocs.get(), true, "running.docs", SolrInfoMBean.Category.INDEX.toString(), "merge", "minor");
metricManager.registerGauge(registry, () -> runningMajorMergesSegments.get(), true, "running.segments", SolrInfoMBean.Category.INDEX.toString(), "merge", "major");
metricManager.registerGauge(registry, () -> runningMinorMergesSegments.get(), true, "running.segments", SolrInfoMBean.Category.INDEX.toString(), "merge", "minor");
flushMeter = metricManager.meter(registry, "flush", SolrInfoMBean.Category.INDEX.toString());
}
}

View File

@ -234,9 +234,6 @@ public static final int VERSION_IDX = 1;
// metrics
protected Gauge<Integer> bufferedOpsGauge;
protected Gauge<Integer> replayLogsCountGauge;
protected Gauge<Long> replayBytesGauge;
protected Gauge<Integer> stateGauge;
protected Meter applyingBufferedOpsMeter;
protected Meter replayOpsMeter;
@ -424,16 +421,13 @@ public static final int VERSION_IDX = 1;
return 0;
}
};
replayLogsCountGauge = () -> logs.size();
replayBytesGauge = () -> getTotalLogsSize();
manager.register(registry, bufferedOpsGauge, true, "ops", scope, "buffered");
manager.register(registry, replayLogsCountGauge, true, "logs", scope, "replay", "remaining");
manager.register(registry, replayBytesGauge, true, "bytes", scope, "replay", "remaining");
manager.registerGauge(registry, bufferedOpsGauge, true, "ops", scope, "buffered");
manager.registerGauge(registry, () -> logs.size(), true, "logs", scope, "replay", "remaining");
manager.registerGauge(registry, () -> getTotalLogsSize(), true, "bytes", scope, "replay", "remaining");
applyingBufferedOpsMeter = manager.meter(registry, "ops", scope, "applyingBuffered");
replayOpsMeter = manager.meter(registry, "ops", scope, "replay");
stateGauge = () -> state.getValue();
manager.register(registry, stateGauge, true, "state", scope);
manager.registerGauge(registry, () -> state.getValue(), true, "state", scope);
}
/**

View File

@ -17,8 +17,6 @@
package org.apache.solr.util.stats;
import com.codahale.metrics.Gauge;
import com.codahale.metrics.MetricRegistry;
import org.apache.http.config.Registry;
import org.apache.http.conn.socket.ConnectionSocketFactory;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
@ -31,42 +29,16 @@ import org.apache.solr.metrics.SolrMetricProducer;
*/
public class InstrumentedPoolingHttpClientConnectionManager extends PoolingHttpClientConnectionManager implements SolrMetricProducer {
protected MetricRegistry metricsRegistry;
public InstrumentedPoolingHttpClientConnectionManager(Registry<ConnectionSocketFactory> socketFactoryRegistry) {
super(socketFactoryRegistry);
}
public MetricRegistry getMetricsRegistry() {
return metricsRegistry;
}
public void setMetricsRegistry(MetricRegistry metricRegistry) {
this.metricsRegistry = metricRegistry;
}
@Override
public void initializeMetrics(SolrMetricManager manager, String registry, String scope) {
this.metricsRegistry = manager.registry(registry);
metricsRegistry.register(SolrMetricManager.mkName("availableConnections", scope),
(Gauge<Integer>) () -> {
// this acquires a lock on the connection pool; remove if contention sucks
return getTotalStats().getAvailable();
});
metricsRegistry.register(SolrMetricManager.mkName("leasedConnections", scope),
(Gauge<Integer>) () -> {
// this acquires a lock on the connection pool; remove if contention sucks
return getTotalStats().getLeased();
});
metricsRegistry.register(SolrMetricManager.mkName("maxConnections", scope),
(Gauge<Integer>) () -> {
// this acquires a lock on the connection pool; remove if contention sucks
return getTotalStats().getMax();
});
metricsRegistry.register(SolrMetricManager.mkName("pendingConnections", scope),
(Gauge<Integer>) () -> {
// this acquires a lock on the connection pool; remove if contention sucks
return getTotalStats().getPending();
});
manager.registerGauge(registry, () -> getTotalStats().getAvailable(), true, SolrMetricManager.mkName("availableConnections", scope));
// this acquires a lock on the connection pool; remove if contention sucks
manager.registerGauge(registry, () -> getTotalStats().getLeased(), true, SolrMetricManager.mkName("leasedConnections", scope));
manager.registerGauge(registry, () -> getTotalStats().getMax(), true, SolrMetricManager.mkName("maxConnections", scope));
manager.registerGauge(registry, () -> getTotalStats().getPending(), true, SolrMetricManager.mkName("pendingConnections", scope));
}
}

View File

@ -47,6 +47,7 @@ import org.apache.solr.metrics.AggregateMetric;
public class MetricUtils {
public static final String METRIC_NAME = "metric";
public static final String VALUE = "value";
public static final String VALUES = "values";
static final String MS = "_ms";
@ -110,16 +111,17 @@ public class MetricUtils {
* @param mustMatchFilter a {@link MetricFilter}.
* A metric <em>must</em> match this filter to be included in the output.
* @param skipHistograms discard any {@link Histogram}-s and histogram parts of {@link Timer}-s.
* @param compact use compact representation for counters and gauges.
* @param metadata optional metadata. If not null and not empty then this map will be added under a
* {@code _metadata_} key.
* @return a {@link NamedList}
*/
public static NamedList toNamedList(MetricRegistry registry, List<MetricFilter> shouldMatchFilters,
MetricFilter mustMatchFilter, boolean skipHistograms,
boolean skipAggregateValues,
boolean skipAggregateValues, boolean compact,
Map<String, Object> metadata) {
NamedList result = new SimpleOrderedMap();
toNamedMaps(registry, shouldMatchFilters, mustMatchFilter, skipHistograms, skipAggregateValues, (k, v) -> {
toMaps(registry, shouldMatchFilters, mustMatchFilter, skipHistograms, skipAggregateValues, compact, (k, v) -> {
result.add(k, v);
});
if (metadata != null && !metadata.isEmpty()) {
@ -140,17 +142,18 @@ public class MetricUtils {
* @param mustMatchFilter a {@link MetricFilter}.
* A metric <em>must</em> match this filter to be included in the output.
* @param skipHistograms discard any {@link Histogram}-s and histogram parts of {@link Timer}-s.
* @param compact use compact representation for counters and gauges.
* @param metadata optional metadata. If not null and not empty then this map will be added under a
* {@code _metadata_} key.
* @return a list of {@link SolrInputDocument}-s
*/
public static List<SolrInputDocument> toSolrInputDocuments(MetricRegistry registry, List<MetricFilter> shouldMatchFilters,
MetricFilter mustMatchFilter, boolean skipHistograms,
boolean skipAggregateValues,
boolean skipAggregateValues, boolean compact,
Map<String, Object> metadata) {
List<SolrInputDocument> result = new LinkedList<>();
toSolrInputDocuments(registry, shouldMatchFilters, mustMatchFilter, skipHistograms,
skipAggregateValues, metadata, doc -> {
skipAggregateValues, compact, metadata, doc -> {
result.add(doc);
});
return result;
@ -158,10 +161,10 @@ public class MetricUtils {
public static void toSolrInputDocuments(MetricRegistry registry, List<MetricFilter> shouldMatchFilters,
MetricFilter mustMatchFilter, boolean skipHistograms,
boolean skipAggregateValues,
boolean skipAggregateValues, boolean compact,
Map<String, Object> metadata, Consumer<SolrInputDocument> consumer) {
boolean addMetadata = metadata != null && !metadata.isEmpty();
toNamedMaps(registry, shouldMatchFilters, mustMatchFilter, skipHistograms, skipAggregateValues, (k, v) -> {
toMaps(registry, shouldMatchFilters, mustMatchFilter, skipHistograms, skipAggregateValues, compact, (k, v) -> {
SolrInputDocument doc = new SolrInputDocument();
doc.setField(METRIC_NAME, k);
toSolrInputDocument(null, doc, v);
@ -172,10 +175,16 @@ public class MetricUtils {
});
}
public static void toSolrInputDocument(String prefix, SolrInputDocument doc, Map<String, Object> map) {
public static void toSolrInputDocument(String prefix, SolrInputDocument doc, Object o) {
if (!(o instanceof Map)) {
String key = prefix != null ? prefix : VALUE;
doc.addField(key, o);
return;
}
Map<String, Object> map = (Map<String, Object>)o;
for (Map.Entry<String, Object> entry : map.entrySet()) {
if (entry.getValue() instanceof Map) { // flatten recursively
toSolrInputDocument(entry.getKey(), doc, (Map<String, Object>)entry.getValue());
toSolrInputDocument(entry.getKey(), doc, entry.getValue());
} else {
String key = prefix != null ? prefix + "." + entry.getKey() : entry.getKey();
doc.addField(key, entry.getValue());
@ -183,9 +192,10 @@ public class MetricUtils {
}
}
public static void toNamedMaps(MetricRegistry registry, List<MetricFilter> shouldMatchFilters,
MetricFilter mustMatchFilter, boolean skipHistograms, boolean skipAggregateValues,
BiConsumer<String, Map<String, Object>> consumer) {
public static void toMaps(MetricRegistry registry, List<MetricFilter> shouldMatchFilters,
MetricFilter mustMatchFilter, boolean skipHistograms, boolean skipAggregateValues,
boolean compact,
BiConsumer<String, Object> consumer) {
Map<String, Metric> metrics = registry.getMetrics();
SortedSet<String> names = registry.getNames();
names.stream()
@ -195,28 +205,28 @@ public class MetricUtils {
Metric metric = metrics.get(n);
if (metric instanceof Counter) {
Counter counter = (Counter) metric;
consumer.accept(n, counterToMap(counter));
consumer.accept(n, convertCounter(counter, compact));
} else if (metric instanceof Gauge) {
Gauge gauge = (Gauge) metric;
consumer.accept(n, gaugeToMap(gauge));
consumer.accept(n, convertGauge(gauge, compact));
} else if (metric instanceof Meter) {
Meter meter = (Meter) metric;
consumer.accept(n, meterToMap(meter));
consumer.accept(n, convertMeter(meter));
} else if (metric instanceof Timer) {
Timer timer = (Timer) metric;
consumer.accept(n, timerToMap(timer, skipHistograms));
consumer.accept(n, convertTimer(timer, skipHistograms));
} else if (metric instanceof Histogram) {
if (!skipHistograms) {
Histogram histogram = (Histogram) metric;
consumer.accept(n, histogramToMap(histogram));
consumer.accept(n, convertHistogram(histogram));
}
} else if (metric instanceof AggregateMetric) {
consumer.accept(n, aggregateMetricToMap((AggregateMetric)metric, skipAggregateValues));
consumer.accept(n, convertAggregateMetric((AggregateMetric)metric, skipAggregateValues));
}
});
}
static Map<String, Object> aggregateMetricToMap(AggregateMetric metric, boolean skipAggregateValues) {
static Map<String, Object> convertAggregateMetric(AggregateMetric metric, boolean skipAggregateValues) {
Map<String, Object> response = new LinkedHashMap<>();
response.put("count", metric.size());
response.put(MAX, metric.getMax());
@ -237,7 +247,7 @@ public class MetricUtils {
return response;
}
static Map<String, Object> histogramToMap(Histogram histogram) {
static Map<String, Object> convertHistogram(Histogram histogram) {
Map<String, Object> response = new LinkedHashMap<>();
Snapshot snapshot = histogram.getSnapshot();
response.put("count", histogram.getCount());
@ -268,7 +278,7 @@ public class MetricUtils {
response.put((ms ? P999_MS: P999), nsToMs(ms, snapshot.get999thPercentile()));
}
static Map<String,Object> timerToMap(Timer timer, boolean skipHistograms) {
static Map<String,Object> convertTimer(Timer timer, boolean skipHistograms) {
Map<String, Object> response = new LinkedHashMap<>();
response.put("count", timer.getCount());
response.put("meanRate", timer.getMeanRate());
@ -282,7 +292,7 @@ public class MetricUtils {
return response;
}
static Map<String, Object> meterToMap(Meter meter) {
static Map<String, Object> convertMeter(Meter meter) {
Map<String, Object> response = new LinkedHashMap<>();
response.put("count", meter.getCount());
response.put("meanRate", meter.getMeanRate());
@ -292,16 +302,24 @@ public class MetricUtils {
return response;
}
static Map<String, Object> gaugeToMap(Gauge gauge) {
Map<String, Object> response = new LinkedHashMap<>();
response.put("value", gauge.getValue());
return response;
static Object convertGauge(Gauge gauge, boolean compact) {
if (compact) {
return gauge.getValue();
} else {
Map<String, Object> response = new LinkedHashMap<>();
response.put("value", gauge.getValue());
return response;
}
}
static Map<String, Object> counterToMap(Counter counter) {
Map<String, Object> response = new LinkedHashMap<>();
response.put("count", counter.getCount());
return response;
static Object convertCounter(Counter counter, boolean compact) {
if (compact) {
return counter.getCount();
} else {
Map<String, Object> response = new LinkedHashMap<>();
response.put("count", counter.getCount());
return response;
}
}
/**

View File

@ -49,9 +49,11 @@ public class MetricsHandlerTest extends SolrTestCaseJ4 {
assertNotNull(values.get("solr.node"));
NamedList nl = (NamedList) values.get("solr.core.collection1");
assertNotNull(nl);
assertNotNull(nl.get("SEARCHER.new.errors")); // counter type
assertNotNull(((Map) nl.get("SEARCHER.new.errors")).get("count"));
Object o = nl.get("SEARCHER.new.errors");
assertNotNull(o); // counter type
assertTrue(o instanceof Map);
// response wasn't serialized so we get here whatever MetricUtils produced instead of NamedList
assertNotNull(((Map) o).get("count"));
assertEquals(0L, ((Map) nl.get("SEARCHER.new.errors")).get("count"));
nl = (NamedList) values.get("solr.node");
assertNotNull(nl.get("CONTAINER.cores.loaded")); // int gauge
@ -149,4 +151,20 @@ public class MetricsHandlerTest extends SolrTestCaseJ4 {
assertEquals(1, values.size());
assertEquals(0, ((NamedList)values.get("solr.node")).size());
}
@Test
public void testCompact() throws Exception {
MetricsHandler handler = new MetricsHandler(h.getCoreContainer());
SolrQueryResponse resp = new SolrQueryResponse();
handler.handleRequestBody(req(CommonParams.QT, "/admin/metrics", CommonParams.WT, "json", MetricsHandler.COMPACT_PARAM, "true"), resp);
NamedList values = resp.getValues();
assertNotNull(values.get("metrics"));
values = (NamedList) values.get("metrics");
NamedList nl = (NamedList) values.get("solr.core.collection1");
assertNotNull(nl);
Object o = nl.get("SEARCHER.new.errors");
assertNotNull(o); // counter type
assertTrue(o instanceof Number);
}
}

View File

@ -119,7 +119,7 @@ public class DirectUpdateHandlerTest extends SolrTestCaseJ4 {
String delsQName = PREFIX + "deletesByQuery";
String cumulativeDelsQName = PREFIX + "cumulativeDeletesByQuery";
long commits = ((Meter) metrics.get(commitsName)).getCount();
long adds = ((Gauge<Long>) metrics.get(addsName)).getValue();
long adds = ((Gauge<Number>) metrics.get(addsName)).getValue().longValue();
long cumulativeAdds = ((Meter) metrics.get(cumulativeAddsName)).getCount();
long cumulativeDelsI = ((Meter) metrics.get(cumulativeDelsIName)).getCount();
long cumulativeDelsQ = ((Meter) metrics.get(cumulativeDelsQName)).getCount();
@ -137,7 +137,7 @@ public class DirectUpdateHandlerTest extends SolrTestCaseJ4 {
assertQ(req("q","id:5"), "//*[@numFound='0']");
assertQ(req("q","id:6"), "//*[@numFound='0']");
long newAdds = ((Gauge<Long>) metrics.get(addsName)).getValue();
long newAdds = ((Gauge<Number>) metrics.get(addsName)).getValue().longValue();
long newCumulativeAdds = ((Meter) metrics.get(cumulativeAddsName)).getCount();
assertEquals("new adds", 2, newAdds - adds);
assertEquals("new cumulative adds", 2, newCumulativeAdds - cumulativeAdds);
@ -147,7 +147,7 @@ public class DirectUpdateHandlerTest extends SolrTestCaseJ4 {
long newCommits = ((Meter) metrics.get(commitsName)).getCount();
assertEquals("new commits", 1, newCommits - commits);
newAdds = ((Gauge<Long>) metrics.get(addsName)).getValue();
newAdds = ((Gauge<Number>) metrics.get(addsName)).getValue().longValue();
newCumulativeAdds = ((Meter) metrics.get(cumulativeAddsName)).getCount();
// adds should be reset to 0 after commit
assertEquals("new adds after commit", 0, newAdds);
@ -161,7 +161,7 @@ public class DirectUpdateHandlerTest extends SolrTestCaseJ4 {
// now delete one
assertU(delI("5"));
long newDelsI = ((Gauge<Long>) metrics.get(delsIName)).getValue();
long newDelsI = ((Gauge<Number>) metrics.get(delsIName)).getValue().longValue();
long newCumulativeDelsI = ((Meter) metrics.get(cumulativeDelsIName)).getCount();
assertEquals("new delsI", 1, newDelsI);
assertEquals("new cumulative delsI", 1, newCumulativeDelsI - cumulativeDelsI);
@ -171,7 +171,7 @@ public class DirectUpdateHandlerTest extends SolrTestCaseJ4 {
assertU(commit());
// delsI should be reset to 0 after commit
newDelsI = ((Gauge<Long>) metrics.get(delsIName)).getValue();
newDelsI = ((Gauge<Number>) metrics.get(delsIName)).getValue().longValue();
newCumulativeDelsI = ((Meter) metrics.get(cumulativeDelsIName)).getCount();
assertEquals("new delsI after commit", 0, newDelsI);
assertEquals("new cumulative delsI after commit", 1, newCumulativeDelsI - cumulativeDelsI);
@ -183,7 +183,7 @@ public class DirectUpdateHandlerTest extends SolrTestCaseJ4 {
// now delete all
assertU(delQ("*:*"));
long newDelsQ = ((Gauge<Long>) metrics.get(delsQName)).getValue();
long newDelsQ = ((Gauge<Number>) metrics.get(delsQName)).getValue().longValue();
long newCumulativeDelsQ = ((Meter) metrics.get(cumulativeDelsQName)).getCount();
assertEquals("new delsQ", 1, newDelsQ);
assertEquals("new cumulative delsQ", 1, newCumulativeDelsQ - cumulativeDelsQ);
@ -193,7 +193,7 @@ public class DirectUpdateHandlerTest extends SolrTestCaseJ4 {
assertU(commit());
newDelsQ = ((Gauge<Long>) metrics.get(delsQName)).getValue();
newDelsQ = ((Gauge<Number>) metrics.get(delsQName)).getValue().longValue();
newCumulativeDelsQ = ((Meter) metrics.get(cumulativeDelsQName)).getCount();
assertEquals("new delsQ after commit", 0, newDelsQ);
assertEquals("new cumulative delsQ after commit", 1, newCumulativeDelsQ - cumulativeDelsQ);
@ -204,11 +204,11 @@ public class DirectUpdateHandlerTest extends SolrTestCaseJ4 {
// verify final metrics
newCommits = ((Meter) metrics.get(commitsName)).getCount();
assertEquals("new commits", 3, newCommits - commits);
newAdds = ((Gauge<Long>) metrics.get(addsName)).getValue();
newAdds = ((Gauge<Number>) metrics.get(addsName)).getValue().longValue();
assertEquals("new adds", 0, newAdds);
newCumulativeAdds = ((Meter) metrics.get(cumulativeAddsName)).getCount();
assertEquals("new cumulative adds", 2, newCumulativeAdds - cumulativeAdds);
newDelsI = ((Gauge<Long>) metrics.get(delsIName)).getValue();
newDelsI = ((Gauge<Number>) metrics.get(delsIName)).getValue().longValue();
assertEquals("new delsI", 0, newDelsI);
newCumulativeDelsI = ((Meter) metrics.get(cumulativeDelsIName)).getCount();
assertEquals("new cumulative delsI", 1, newCumulativeDelsI - cumulativeDelsI);

View File

@ -67,7 +67,7 @@ public class SolrIndexMetricsTest extends SolrTestCaseJ4 {
Map<String, Metric> metrics = registry.getMetrics();
assertEquals(10, metrics.entrySet().stream().filter(e -> e.getKey().startsWith("INDEX")).count());
assertEquals(12, metrics.entrySet().stream().filter(e -> e.getKey().startsWith("INDEX")).count());
// check basic index meters
Timer timer = (Timer)metrics.get("INDEX.merge.minor");
@ -92,7 +92,8 @@ public class SolrIndexMetricsTest extends SolrTestCaseJ4 {
assertNotNull(registry);
Map<String, Metric> metrics = registry.getMetrics();
assertEquals(0, metrics.entrySet().stream().filter(e -> e.getKey().startsWith("INDEX")).count());
// INDEX.size, INDEX.sizeInBytes
assertEquals(2, metrics.entrySet().stream().filter(e -> e.getKey().startsWith("INDEX")).count());
}
@Test

View File

@ -22,6 +22,7 @@ import java.util.Map;
import java.util.concurrent.TimeUnit;
import com.codahale.metrics.Counter;
import com.codahale.metrics.Gauge;
import com.codahale.metrics.Histogram;
import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricFilter;
@ -44,7 +45,7 @@ public class MetricUtilsTest extends SolrTestCaseJ4 {
timer.update(Math.abs(random().nextInt()) + 1, TimeUnit.NANOSECONDS);
}
// obtain timer metrics
NamedList lst = new NamedList(MetricUtils.timerToMap(timer, false));
NamedList lst = new NamedList(MetricUtils.convertTimer(timer, false));
// check that expected metrics were obtained
assertEquals(14, lst.size());
final Snapshot snapshot = timer.getSnapshot();
@ -78,10 +79,15 @@ public class MetricUtilsTest extends SolrTestCaseJ4 {
am.set("foo", 10);
am.set("bar", 1);
am.set("bar", 2);
MetricUtils.toNamedMaps(registry, Collections.singletonList(MetricFilter.ALL), MetricFilter.ALL,
false, false, (k, v) -> {
Gauge<String> gauge = () -> "foobar";
registry.register("gauge", gauge);
MetricUtils.toMaps(registry, Collections.singletonList(MetricFilter.ALL), MetricFilter.ALL,
false, false, false, (k, o) -> {
Map v = (Map)o;
if (k.startsWith("counter")) {
assertEquals(1L, v.get("count"));
} else if (k.startsWith("gauge")) {
assertEquals("foobar", v.get("value"));
} else if (k.startsWith("timer")) {
assertEquals(1L, v.get("count"));
assertTrue(((Number)v.get("min_ms")).intValue() > 100);
@ -102,6 +108,47 @@ public class MetricUtilsTest extends SolrTestCaseJ4 {
assertEquals(2, update.get("updateCount"));
}
});
// test compact format
MetricUtils.toMaps(registry, Collections.singletonList(MetricFilter.ALL), MetricFilter.ALL,
false, false, true, (k, o) -> {
if (k.startsWith("counter")) {
assertTrue(o instanceof Long);
assertEquals(1L, o);
} else if (k.startsWith("gauge")) {
assertTrue(o instanceof String);
assertEquals("foobar", o);
} else if (k.startsWith("timer")) {
assertTrue(o instanceof Map);
Map v = (Map)o;
assertEquals(1L, v.get("count"));
assertTrue(((Number)v.get("min_ms")).intValue() > 100);
} else if (k.startsWith("meter")) {
assertTrue(o instanceof Map);
Map v = (Map)o;
assertEquals(1L, v.get("count"));
} else if (k.startsWith("histogram")) {
assertTrue(o instanceof Map);
Map v = (Map)o;
assertEquals(1L, v.get("count"));
} else if (k.startsWith("aggregate")) {
assertTrue(o instanceof Map);
Map v = (Map)o;
assertEquals(2, v.get("count"));
Map<String, Object> values = (Map<String, Object>)v.get("values");
assertNotNull(values);
assertEquals(2, values.size());
Map<String, Object> update = (Map<String, Object>)values.get("foo");
assertEquals(10, update.get("value"));
assertEquals(1, update.get("updateCount"));
update = (Map<String, Object>)values.get("bar");
assertEquals(2, update.get("value"));
assertEquals(2, update.get("updateCount"));
} else {
Map v = (Map)o;
assertEquals(1L, v.get("count"));
}
});
}
}