Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-11215: Make a metric accessible through a single param.

This commit is contained in:
Andrzej Bialecki 2017-08-17 12:32:22 +02:00
parent ac97931c7e
commit 1c36569176
7 changed files with 183 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
solr/CHANGES.txt
View File

@ -32,7 +32,6 @@ Jetty 9.3.14.v20161028
(No Changes)
================== 7.1.0 ==================
Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release.
@ -73,6 +72,7 @@ New Features
A new "sum" function is also added. Example :
{!payload_score f=payload_field func=sum operator=or}A B C" (Varun Thacker)
* SOLR-11215: Make a metric accessible through a single param. (ab)
Bug Fixes
----------------------

2
solr/contrib/analytics/src/java/org/apache/solr/analytics/plugin/AnalyticsStatisticsCollector.java
View File

@ -88,7 +88,7 @@ public class AnalyticsStatisticsCollector {
public Map<String, Object> getStatistics() {
Map<String, Object> map = new HashMap<>();
MetricUtils.convertTimer("", requestTimes, MetricUtils.PropertyFilter.ALL, false, false, (k, v) -> {
MetricUtils.convertTimer("", requestTimes, MetricUtils.PropertyFilter.ALL, false, false, ".", (k, v) -> {
map.putAll((Map<String, Object>)v);
});
map.put("requests", numRequests.longValue());

66
solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java
View File

@ -23,12 +23,14 @@ import java.util.EnumSet;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import com.codahale.metrics.Counter;
import com.codahale.metrics.Gauge;
import com.codahale.metrics.Histogram;
import com.codahale.metrics.Meter;
import com.codahale.metrics.Metric;
import com.codahale.metrics.MetricFilter;
import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.Timer;
@ -58,10 +60,13 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionName
public static final String PROPERTY_PARAM = "property";
public static final String REGISTRY_PARAM = "registry";
public static final String GROUP_PARAM = "group";
public static final String KEY_PARAM = "key";
public static final String TYPE_PARAM = "type";
public static final String ALL = "all";
private static final Pattern KEY_REGEX = Pattern.compile("(?<!" + Pattern.quote("\\") + ")" + Pattern.quote(":"));
public MetricsHandler() {
this.container = null;
this.metricManager = null;
@ -84,6 +89,11 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionName
}
boolean compact = req.getParams().getBool(COMPACT_PARAM, true);
String[] keys = req.getParams().getParams(KEY_PARAM);
if (keys != null && keys.length > 0) {
handleKeyRequest(keys, req, rsp);
return;
}
MetricFilter mustMatchFilter = parseMustMatchFilter(req);
MetricUtils.PropertyFilter propertyFilter = parsePropertyFilter(req);
List<MetricType> metricTypes = parseMetricTypes(req);
@ -103,6 +113,62 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionName
rsp.getValues().add("metrics", response);
}
private void handleKeyRequest(String[] keys, SolrQueryRequest req, SolrQueryResponse rsp) throws Exception {
SimpleOrderedMap result = new SimpleOrderedMap();
SimpleOrderedMap errors = new SimpleOrderedMap();
for (String key : keys) {
if (key == null || key.isEmpty()) {
continue;
}
String[] parts = KEY_REGEX.split(key);
if (parts.length < 2 || parts.length > 3) {
errors.add(key, "at least two and at most three colon-separated parts must be provided");
continue;
}
final String registryName = unescape(parts[0]);
final String metricName = unescape(parts[1]);
final String propertyName = parts.length > 2 ? unescape(parts[2]) : null;
if (!metricManager.hasRegistry(registryName)) {
errors.add(key, "registry '" + registryName + "' not found");
continue;
}
MetricRegistry registry = metricManager.registry(registryName);
Metric m = registry.getMetrics().get(metricName);
if (m == null) {
errors.add(key, "metric '" + metricName + "' not found");
continue;
}
MetricUtils.PropertyFilter propertyFilter = MetricUtils.PropertyFilter.ALL;
boolean simple = false;
if (propertyName != null) {
propertyFilter = (name) -> name.equals(propertyName);
simple = true;
// use escaped versions
key = parts[0] + ":" + parts[1];
}
MetricUtils.convertMetric(key, m, propertyFilter, false, true, true, simple, ":", (k, v) -> result.add(k, v));
}
rsp.getValues().add("metrics", result);
if (errors.size() > 0) {
rsp.getValues().add("errors", errors);
}
}
private static String unescape(String s) {
if (s.indexOf('\\') == -1) {
return s;
}
StringBuilder sb = new StringBuilder(s.length());
for (int i = 0; i < s.length(); i++) {
char c = s.charAt(i);
if (c == '\\') {
continue;
}
sb.append(c);
}
return sb.toString();
}
private MetricFilter parseMustMatchFilter(SolrQueryRequest req) {
String[] prefixes = req.getParams().getParams(PREFIX_PARAM);
MetricFilter prefixFilter = null;

11
solr/core/src/java/org/apache/solr/metrics/SolrMetricManager.java
View File

@ -352,6 +352,17 @@ public class SolrMetricManager {
return set;
}
/**
* Check whether a registry with a given (overridable) name already exists.
* @param name registry name
* @return true if this name points to a registry that already exists, false otherwise
*/
public boolean hasRegistry(String name) {
Set<String> names = registryNames();
name = overridableRegistryName(name);
return names.contains(name);
}
/**
* Return set of existing registry names that match a regex pattern
* @param patterns regex patterns. NOTE: users need to make sure that patterns that

38
solr/core/src/java/org/apache/solr/util/stats/MetricUtils.java
View File

@ -210,7 +210,7 @@ public class MetricUtils {
.filter(s -> mustMatchFilter.matches(s, metrics.get(s)))
.forEach(n -> {
Metric metric = metrics.get(n);
convertMetric(n, metric, propertyFilter, skipHistograms, skipAggregateValues, compact, simple, consumer);
convertMetric(n, metric, propertyFilter, skipHistograms, skipAggregateValues, compact, simple, ".", consumer);
});
}
@ -247,7 +247,7 @@ public class MetricUtils {
names.stream()
.forEach(n -> {
Metric metric = metrics.get(n);
convertMetric(n, metric, PropertyFilter.ALL, skipHistograms, skipAggregateValues, compact, simple, consumer);
convertMetric(n, metric, PropertyFilter.ALL, skipHistograms, skipAggregateValues, compact, simple, ".", consumer);
});
}
@ -263,15 +263,15 @@ public class MetricUtils {
* only the selected (name "." key, value) pairs will be produced.
* @param consumer consumer that accepts produced objects
*/
static void convertMetric(String n, Metric metric, PropertyFilter propertyFilter, boolean skipHistograms, boolean skipAggregateValues,
boolean compact, boolean simple, BiConsumer<String, Object> consumer) {
public static void convertMetric(String n, Metric metric, PropertyFilter propertyFilter, boolean skipHistograms, boolean skipAggregateValues,
boolean compact, boolean simple, String separator, BiConsumer<String, Object> consumer) {
if (metric instanceof Counter) {
Counter counter = (Counter) metric;
convertCounter(n, counter, propertyFilter, compact, consumer);
} else if (metric instanceof Gauge) {
Gauge gauge = (Gauge) metric;
try {
convertGauge(n, gauge, propertyFilter, simple, compact, consumer);
convertGauge(n, gauge, propertyFilter, simple, compact, separator, consumer);
} catch (InternalError ie) {
if (n.startsWith("memory.") && ie.getMessage().contains("Memory Pool not found")) {
LOG.warn("Error converting gauge '" + n + "', possible JDK bug: SOLR-10362", ie);
@ -282,17 +282,17 @@ public class MetricUtils {
}
} else if (metric instanceof Meter) {
Meter meter = (Meter) metric;
convertMeter(n, meter, propertyFilter, simple, consumer);
convertMeter(n, meter, propertyFilter, simple, separator, consumer);
} else if (metric instanceof Timer) {
Timer timer = (Timer) metric;
convertTimer(n, timer, propertyFilter, skipHistograms, simple, consumer);
convertTimer(n, timer, propertyFilter, skipHistograms, simple, separator, consumer);
} else if (metric instanceof Histogram) {
if (!skipHistograms) {
Histogram histogram = (Histogram) metric;
convertHistogram(n, histogram, propertyFilter, simple, consumer);
convertHistogram(n, histogram, propertyFilter, simple, separator, consumer);
}
} else if (metric instanceof AggregateMetric) {
convertAggregateMetric(n, (AggregateMetric)metric, propertyFilter, skipAggregateValues, simple, consumer);
convertAggregateMetric(n, (AggregateMetric)metric, propertyFilter, skipAggregateValues, simple, separator, consumer);
}
}
@ -308,10 +308,10 @@ public class MetricUtils {
*/
static void convertAggregateMetric(String name, AggregateMetric metric,
PropertyFilter propertyFilter,
boolean skipAggregateValues, boolean simple, BiConsumer<String, Object> consumer) {
boolean skipAggregateValues, boolean simple, String separator, BiConsumer<String, Object> consumer) {
if (simple) {
if (propertyFilter.accept(MEAN)) {
consumer.accept(name + "." + MEAN, metric.getMean());
consumer.accept(name + separator + MEAN, metric.getMean());
}
} else {
Map<String, Object> response = new LinkedHashMap<>();
@ -353,11 +353,11 @@ public class MetricUtils {
* @param consumer consumer that accepts produced objects
*/
static void convertHistogram(String name, Histogram histogram, PropertyFilter propertyFilter,
boolean simple, BiConsumer<String, Object> consumer) {
boolean simple, String separator, BiConsumer<String, Object> consumer) {
Snapshot snapshot = histogram.getSnapshot();
if (simple) {
if (propertyFilter.accept(MEAN)) {
consumer.accept(name + "." + MEAN, snapshot.getMean());
consumer.accept(name + separator + MEAN, snapshot.getMean());
}
} else {
Map<String, Object> response = new LinkedHashMap<>();
@ -411,11 +411,11 @@ public class MetricUtils {
* @param consumer consumer that accepts produced objects
*/
public static void convertTimer(String name, Timer timer, PropertyFilter propertyFilter, boolean skipHistograms,
boolean simple, BiConsumer<String, Object> consumer) {
boolean simple, String separator, BiConsumer<String, Object> consumer) {
if (simple) {
String prop = "meanRate";
if (propertyFilter.accept(prop)) {
consumer.accept(name + "." + prop, timer.getMeanRate());
consumer.accept(name + separator + prop, timer.getMeanRate());
}
} else {
Map<String, Object> response = new LinkedHashMap<>();
@ -448,10 +448,10 @@ public class MetricUtils {
* only the selected (name "." key, value) pairs will be produced.
* @param consumer consumer that accepts produced objects
*/
static void convertMeter(String name, Meter meter, PropertyFilter propertyFilter, boolean simple, BiConsumer<String, Object> consumer) {
static void convertMeter(String name, Meter meter, PropertyFilter propertyFilter, boolean simple, String separator, BiConsumer<String, Object> consumer) {
if (simple) {
if (propertyFilter.accept("count")) {
consumer.accept(name + ".count", meter.getCount());
consumer.accept(name + separator + "count", meter.getCount());
}
} else {
Map<String, Object> response = new LinkedHashMap<>();
@ -483,7 +483,7 @@ public class MetricUtils {
* @param consumer consumer that accepts produced objects
*/
static void convertGauge(String name, Gauge gauge, PropertyFilter propertyFilter, boolean simple, boolean compact,
BiConsumer<String, Object> consumer) {
String separator, BiConsumer<String, Object> consumer) {
if (compact || simple) {
Object o = gauge.getValue();
if (o instanceof Map) {
@ -491,7 +491,7 @@ public class MetricUtils {
for (Map.Entry<?, ?> entry : ((Map<?, ?>)o).entrySet()) {
String prop = entry.getKey().toString();
if (propertyFilter.accept(prop)) {
consumer.accept(name + "." + prop, entry.getValue());
consumer.accept(name + separator + prop, entry.getValue());
}
}
} else {

86
solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java
View File

@ -19,6 +19,7 @@ package org.apache.solr.handler.admin;
import java.util.Map;
import com.codahale.metrics.Counter;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.util.NamedList;
@ -36,8 +37,13 @@ public class MetricsHandlerTest extends SolrTestCaseJ4 {
initCore("solrconfig.xml", "schema.xml");
// manually register some metrics in solr.jvm and solr.jetty - TestHarness doesn't init them
h.getCoreContainer().getMetricManager().counter(null, "solr.jvm", "foo");
h.getCoreContainer().getMetricManager().counter(null, "solr.jetty", "foo");
Counter c = h.getCoreContainer().getMetricManager().counter(null, "solr.jvm", "foo");
c.inc();
c = h.getCoreContainer().getMetricManager().counter(null, "solr.jetty", "foo");
c.inc(2);
// test escapes
c = h.getCoreContainer().getMetricManager().counter(null, "solr.jetty", "foo:bar");
c.inc(3);
}
@Test
@ -225,4 +231,80 @@ public class MetricsHandlerTest extends SolrTestCaseJ4 {
assertNotNull(map.get("size"));
});
}
@Test
public void testKeyMetrics() throws Exception {
MetricsHandler handler = new MetricsHandler(h.getCoreContainer());
String key1 = "solr.core.collection1:CACHE.core.fieldCache";
SolrQueryResponse resp = new SolrQueryResponse();
handler.handleRequestBody(req(CommonParams.QT, "/admin/metrics", CommonParams.WT, "json",
MetricsHandler.KEY_PARAM, key1), resp);
NamedList values = resp.getValues();
Object val = values.findRecursive("metrics", key1);
assertNotNull(val);
assertTrue(val instanceof Map);
assertTrue(((Map)val).size() >= 2);
String key2 = "solr.core.collection1:CACHE.core.fieldCache:entries_count";
resp = new SolrQueryResponse();
handler.handleRequestBody(req(CommonParams.QT, "/admin/metrics", CommonParams.WT, "json",
MetricsHandler.KEY_PARAM, key2), resp);
values = resp.getValues();
val = values.findRecursive("metrics", key2);
assertNotNull(val);
assertTrue(val instanceof Number);
String key3 = "solr.jetty:foo\\:bar";
resp = new SolrQueryResponse();
handler.handleRequestBody(req(CommonParams.QT, "/admin/metrics", CommonParams.WT, "json",
MetricsHandler.KEY_PARAM, key3), resp);
values = resp.getValues();
val = values.findRecursive("metrics", key3);
assertNotNull(val);
assertTrue(val instanceof Number);
assertEquals(3, ((Number)val).intValue());
// test multiple keys
resp = new SolrQueryResponse();
handler.handleRequestBody(req(CommonParams.QT, "/admin/metrics", CommonParams.WT, "json",
MetricsHandler.KEY_PARAM, key1, MetricsHandler.KEY_PARAM, key2, MetricsHandler.KEY_PARAM, key3), resp);
values = resp.getValues();
val = values.findRecursive("metrics", key1);
assertNotNull(val);
val = values.findRecursive("metrics", key2);
assertNotNull(val);
val = values.findRecursive("metrics", key3);
assertNotNull(val);
// test errors
// invalid keys
resp = new SolrQueryResponse();
handler.handleRequestBody(req(CommonParams.QT, "/admin/metrics", CommonParams.WT, "json",
MetricsHandler.KEY_PARAM, "foo", MetricsHandler.KEY_PARAM, "foo:bar:baz:xyz"), resp);
values = resp.getValues();
NamedList metrics = (NamedList)values.get("metrics");
assertEquals(0, metrics.size());
assertNotNull(values.findRecursive("errors", "foo"));
assertNotNull(values.findRecursive("errors", "foo:bar:baz:xyz"));
// unknown registry
resp = new SolrQueryResponse();
handler.handleRequestBody(req(CommonParams.QT, "/admin/metrics", CommonParams.WT, "json",
MetricsHandler.KEY_PARAM, "foo:bar:baz"), resp);
values = resp.getValues();
metrics = (NamedList)values.get("metrics");
assertEquals(0, metrics.size());
assertNotNull(values.findRecursive("errors", "foo:bar:baz"));
// unknown metric
resp = new SolrQueryResponse();
handler.handleRequestBody(req(CommonParams.QT, "/admin/metrics", CommonParams.WT, "json",
MetricsHandler.KEY_PARAM, "solr.jetty:unknown:baz"), resp);
values = resp.getValues();
metrics = (NamedList)values.get("metrics");
assertEquals(0, metrics.size());
assertNotNull(values.findRecursive("errors", "solr.jetty:unknown:baz"));
}
}

2
solr/core/src/test/org/apache/solr/util/stats/MetricUtilsTest.java
View File

@ -47,7 +47,7 @@ public class MetricUtilsTest extends SolrTestCaseJ4 {
}
// obtain timer metrics
Map<String,Object> map = new HashMap<>();
MetricUtils.convertTimer("", timer, MetricUtils.PropertyFilter.ALL, false, false, (k, v) -> {
MetricUtils.convertTimer("", timer, MetricUtils.PropertyFilter.ALL, false, false, ".", (k, v) -> {
map.putAll((Map<String,Object>)v);
});
NamedList lst = new NamedList(map);