honeymoose/OpenSearch
1
0
Fork 0
mirror of https://github.com/honeymoose/OpenSearch.git synced 2025-03-25 09:28:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

[7.x] Add analytics plugin usage stats to _xpack/usage (#54911) (#55162)

Browse Source 
Adds analytics plugin usage stats to _xpack/usage.

Closes #54847
This commit is contained in:
Igor Motov 2020-04-14 17:03:14 -04:00 committed by GitHub
parent ce85063653
commit 1754e50cbd
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 619 additions and 126 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
docs/reference/rest-api/usage.asciidoc
View File

@ -16,7 +16,7 @@ Provides usage information about the installed {xpack} features.
=== {api-description-title}
This API provides information about which features are currently enabled and
available under the current license and some usage statistics.
available under the current license and some usage statistics.
[discrete]
[[usage-api-query-parms]]
@ -291,9 +291,9 @@ GET /_xpack/usage
// 1. Handling eql, which is disabled by default on release builds and enabled
// everywhere else during the initial implementation phase until its release
// 2. Ignore the contents of the `ilm` and `slm` objects because they don't know
// all of the policies that will be in them.
// all of the policies that will be in them.
// 3. Ignore the contents of the `analytics` object because it might contain
// additional stats
// 4. All of the numbers and strings on the right hand side of *every* field in
// the response are ignored. So we're really only asserting things about the
// the shape of this response, not the values in it.
// the shape of this response, not the values in it.

12
x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/AnalyticsPlugin.java
View File

@ -74,7 +74,7 @@ public class AnalyticsPlugin extends Plugin implements SearchPlugin, ActionPlugi
CumulativeCardinalityPipelineAggregationBuilder.NAME,
CumulativeCardinalityPipelineAggregationBuilder::new,
CumulativeCardinalityPipelineAggregator::new,
usage.track(AnalyticsUsage.Item.CUMULATIVE_CARDINALITY,
usage.track(AnalyticsStatsAction.Item.CUMULATIVE_CARDINALITY,
checkLicense(CumulativeCardinalityPipelineAggregationBuilder.PARSER)))
);
}
@ -85,22 +85,22 @@ public class AnalyticsPlugin extends Plugin implements SearchPlugin, ActionPlugi
new AggregationSpec(
StringStatsAggregationBuilder.NAME,
StringStatsAggregationBuilder::new,
usage.track(AnalyticsUsage.Item.STRING_STATS, checkLicense(StringStatsAggregationBuilder.PARSER)))
usage.track(AnalyticsStatsAction.Item.STRING_STATS, checkLicense(StringStatsAggregationBuilder.PARSER)))
.addResultReader(InternalStringStats::new),
new AggregationSpec(
BoxplotAggregationBuilder.NAME,
BoxplotAggregationBuilder::new,
usage.track(AnalyticsUsage.Item.BOXPLOT, checkLicense(BoxplotAggregationBuilder.PARSER)))
usage.track(AnalyticsStatsAction.Item.BOXPLOT, checkLicense(BoxplotAggregationBuilder.PARSER)))
.addResultReader(InternalBoxplot::new),
new AggregationSpec(
TopMetricsAggregationBuilder.NAME,
TopMetricsAggregationBuilder::new,
usage.track(AnalyticsUsage.Item.TOP_METRICS, checkLicense(TopMetricsAggregationBuilder.PARSER)))
usage.track(AnalyticsStatsAction.Item.TOP_METRICS, checkLicense(TopMetricsAggregationBuilder.PARSER)))
.addResultReader(InternalTopMetrics::new),
new AggregationSpec(
TTestAggregationBuilder.NAME,
TTestAggregationBuilder::new,
usage.track(AnalyticsUsage.Item.T_TEST, checkLicense(TTestAggregationBuilder.PARSER)))
usage.track(AnalyticsStatsAction.Item.T_TEST, checkLicense(TTestAggregationBuilder.PARSER)))
.addResultReader(InternalTTest::new)
);
}
@ -138,7 +138,7 @@ public class AnalyticsPlugin extends Plugin implements SearchPlugin, ActionPlugi
ResourceWatcherService resourceWatcherService, ScriptService scriptService, NamedXContentRegistry xContentRegistry,
Environment environment, NodeEnvironment nodeEnvironment, NamedWriteableRegistry namedWriteableRegistry,
IndexNameExpressionResolver indexNameExpressionResolver) {
return singletonList(new AnalyticsUsage());
return singletonList(usage);
}
@Override

32
x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/AnalyticsUsage.java
View File

@ -8,54 +8,32 @@ package org.elasticsearch.xpack.analytics;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.common.xcontent.ContextParser;
import org.elasticsearch.xpack.core.analytics.EnumCounters;
import org.elasticsearch.xpack.core.analytics.action.AnalyticsStatsAction;
import java.util.EnumMap;
import java.util.Map;
import java.util.concurrent.atomic.AtomicLong;
/**
* Tracks usage of the Analytics aggregations.
*/
public class AnalyticsUsage {
/**
* Items to track.
*/
public enum Item {
BOXPLOT,
CUMULATIVE_CARDINALITY,
STRING_STATS,
TOP_METRICS,
T_TEST;
}
private final Map<Item, AtomicLong> trackers = new EnumMap<>(Item.class);
private final EnumCounters<AnalyticsStatsAction.Item> counters = new EnumCounters<>(AnalyticsStatsAction.Item.class);
public AnalyticsUsage() {
for (Item item: Item.values()) {
trackers.put(item, new AtomicLong(0));
}
}
/**
* Track successful parsing.
*/
public <C, T> ContextParser<C, T> track(Item item, ContextParser<C, T> realParser) {
AtomicLong usage = trackers.get(item);
public <C, T> ContextParser<C, T> track(AnalyticsStatsAction.Item item, ContextParser<C, T> realParser) {
return (parser, context) -> {
T value = realParser.parse(parser, context);
// Intentionally doesn't count unless the parser returns cleanly.
usage.incrementAndGet();
counters.inc(item);
return value;
};
}
public AnalyticsStatsAction.NodeResponse stats(DiscoveryNode node) {
return new AnalyticsStatsAction.NodeResponse(node,
trackers.get(Item.BOXPLOT).get(),
trackers.get(Item.CUMULATIVE_CARDINALITY).get(),
trackers.get(Item.STRING_STATS).get(),
trackers.get(Item.TOP_METRICS).get(),
trackers.get(Item.T_TEST).get());
return new AnalyticsStatsAction.NodeResponse(node, counters);
}
}

49
x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/action/AnalyticsStatsActionNodeResponseTests.java Normal file
View File

@ -0,0 +1,49 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.analytics.action;
import org.elasticsearch.Version;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.common.io.stream.Writeable;
import org.elasticsearch.test.AbstractWireSerializingTestCase;
import org.elasticsearch.xpack.core.analytics.EnumCounters;
import org.elasticsearch.xpack.core.analytics.action.AnalyticsStatsAction;
import static org.hamcrest.Matchers.equalTo;
public class AnalyticsStatsActionNodeResponseTests extends AbstractWireSerializingTestCase<AnalyticsStatsAction.NodeResponse> {
@Override
protected Writeable.Reader<AnalyticsStatsAction.NodeResponse> instanceReader() {
return AnalyticsStatsAction.NodeResponse::new;
}
@Override
protected AnalyticsStatsAction.NodeResponse createTestInstance() {
String nodeName = randomAlphaOfLength(10);
DiscoveryNode node = new DiscoveryNode(nodeName, buildNewFakeTransportAddress(), Version.CURRENT);
EnumCounters<AnalyticsStatsAction.Item> counters = new EnumCounters<>(AnalyticsStatsAction.Item.class);
for (AnalyticsStatsAction.Item item : AnalyticsStatsAction.Item.values()) {
if (randomBoolean()) {
counters.inc(item, randomLongBetween(0, 1000));
}
}
return new AnalyticsStatsAction.NodeResponse(node, counters);
}
public void testItemEnum() {
int i = 0;
// We rely on the ordinals for serialization, so they shouldn't change between version
assertThat(AnalyticsStatsAction.Item.BOXPLOT.ordinal(), equalTo(i++));
assertThat(AnalyticsStatsAction.Item.CUMULATIVE_CARDINALITY.ordinal(), equalTo(i++));
assertThat(AnalyticsStatsAction.Item.STRING_STATS.ordinal(), equalTo(i++));
assertThat(AnalyticsStatsAction.Item.TOP_METRICS.ordinal(), equalTo(i++));
assertThat(AnalyticsStatsAction.Item.T_TEST.ordinal(), equalTo(i++));
// Please add tests for newly added items here
assertThat(AnalyticsStatsAction.Item.values().length, equalTo(i));
}
}

23
x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/action/TransportAnalyticsStatsActionTests.java
View File

@ -22,6 +22,7 @@ import org.elasticsearch.test.rest.yaml.ObjectPath;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.transport.TransportService;
import org.elasticsearch.xpack.analytics.AnalyticsUsage;
import org.elasticsearch.xpack.core.analytics.AnalyticsFeatureSetUsage;
import org.elasticsearch.xpack.core.analytics.action.AnalyticsStatsAction;
import java.io.IOException;
@ -54,40 +55,36 @@ public class TransportAnalyticsStatsActionTests extends ESTestCase {
when(clusterService.state()).thenReturn(clusterState);
return new TransportAnalyticsStatsAction(transportService, clusterService, threadPool,
new ActionFilters(Collections.emptySet()), usage);
new ActionFilters(Collections.emptySet()), usage);
}
public void test() throws IOException {
for (AnalyticsUsage.Item item : AnalyticsUsage.Item.values()) {
for (AnalyticsStatsAction.Item item : AnalyticsStatsAction.Item.values()) {
AnalyticsUsage realUsage = new AnalyticsUsage();
AnalyticsUsage emptyUsage = new AnalyticsUsage();
ContextParser<Void, Void> parser = realUsage.track(item, (p, c) -> c);
ObjectPath unused = run(realUsage, emptyUsage);
assertThat(unused.evaluate("stats.0." + item.name().toLowerCase(Locale.ROOT) + "_usage"), equalTo(0));
assertThat(unused.evaluate("stats.1." + item.name().toLowerCase(Locale.ROOT) + "_usage"), equalTo(0));
assertThat(unused.evaluate("stats." + item.name().toLowerCase(Locale.ROOT) + "_usage"), equalTo(0));
int count = between(1, 10000);
for (int i = 0; i < count; i++) {
assertNull(parser.parse(null, null));
}
ObjectPath used = run(realUsage, emptyUsage);
assertThat(item.name(), used.evaluate("stats.0." + item.name().toLowerCase(Locale.ROOT) + "_usage"), equalTo(count));
assertThat(item.name(), used.evaluate("stats.1." + item.name().toLowerCase(Locale.ROOT) + "_usage"), equalTo(0));
assertThat(item.name(), used.evaluate("stats." + item.name().toLowerCase(Locale.ROOT) + "_usage"), equalTo(count));
}
}
private ObjectPath run(AnalyticsUsage... nodeUsages) throws IOException {
AnalyticsStatsAction.Request request = new AnalyticsStatsAction.Request();
List<AnalyticsStatsAction.NodeResponse> nodeResponses = Arrays.stream(nodeUsages)
.map(usage -> action(usage).nodeOperation(new AnalyticsStatsAction.NodeRequest(request)))
.collect(toList());
.map(usage -> action(usage).nodeOperation(new AnalyticsStatsAction.NodeRequest(request)))
.collect(toList());
AnalyticsStatsAction.Response response = new AnalyticsStatsAction.Response(
new ClusterName("cluster_name"), nodeResponses, emptyList());
new ClusterName("cluster_name"), nodeResponses, emptyList());
AnalyticsFeatureSetUsage usage = new AnalyticsFeatureSetUsage(true, true, response);
try (XContentBuilder builder = jsonBuilder()) {
builder.startObject();
response.toXContent(builder, ToXContent.EMPTY_PARAMS);
builder.endObject();
usage.toXContent(builder, ToXContent.EMPTY_PARAMS);
return ObjectPath.createFromXContent(JsonXContent.jsonXContent, BytesReference.bytes(builder));
}
}

2
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageResponse.java
View File

@ -43,4 +43,4 @@ public class XPackUsageResponse extends ActionResponse {
}
}
}
}

138
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/analytics/EnumCounters.java Normal file
View File

@ -0,0 +1,138 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.core.analytics;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.io.stream.Writeable;
import java.io.IOException;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.concurrent.atomic.AtomicLongArray;
/**
* Utility class similar to org.elasticsearch.xpack.core.watcher.common.stats.Counters, but it is using Enum instead
* of string to identify the counter. The serialization happens using enum ordinals similar to
* {@link StreamOutput#writeEnum(Enum)}, which means that ordinal for existing enums should remain the same for backward
* and forward compatibility of the serialization protocol.
*/
public class EnumCounters<E extends Enum<E>> implements Writeable {
private final AtomicLongArray counters;
private final E[] enums;
public EnumCounters(Class<E> enumClass) {
counters = new AtomicLongArray(enumClass.getEnumConstants().length);
enums = enumClass.getEnumConstants();
}
public EnumCounters(StreamInput in, Class<E> enumClass) throws IOException {
int size = in.readVInt();
enums = enumClass.getEnumConstants();
long[] vals = new long[enums.length];
for (int i = 0; i < size; i++) {
long val = in.readVLong();
if (i < vals.length) {
vals[i] = val;
}
}
counters = new AtomicLongArray(vals);
}
@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeVInt(counters.length());
for (int i = 0; i < counters.length(); i++) {
out.writeVLong(counters.get(i));
}
}
public void set(E name) {
counters.set(name.ordinal(), 0);
}
public void inc(E name) {
counters.incrementAndGet(name.ordinal());
}
public void inc(E name, long count) {
counters.addAndGet(name.ordinal(), count);
}
public long get(E name) {
return counters.get(name.ordinal());
}
public long size() {
return counters.length();
}
public boolean hasCounters() {
return size() > 0;
}
public Map<String, Object> toMap() {
Map<String, Object> map = new HashMap<>();
for (E e : enums) {
map.put(e.name().toLowerCase(Locale.ROOT), counters.get(e.ordinal()));
}
return map;
}
public static <E extends Enum<E>> EnumCounters<E> merge(Class<E> enumClass, List<EnumCounters<E>> counters) {
EnumCounters<E> result = new EnumCounters<>(enumClass);
E[] enums = enumClass.getEnumConstants();
for (EnumCounters<E> c : counters) {
for (E e : enums) {
result.inc(e, c.get(e));
}
}
return result;
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
EnumCounters<?> that = (EnumCounters<?>) o;
return Arrays.equals(toArray(), that.toArray()) &&
Arrays.equals(enums, that.enums);
}
@Override
public int hashCode() {
int result = Arrays.hashCode(toArray());
result = 31 * result + Arrays.hashCode(enums);
return result;
}
@Override
public String toString() {
StringBuilder buf = new StringBuilder("[");
boolean first = true;
for (E e : enums) {
buf.append(e.name().toLowerCase(Locale.ROOT)).append(": ").append(get(e));
if (first) {
buf.append(", ");
first = false;
}
}
buf.append("]");
return buf.toString();
}
private long[] toArray() {
long[] res = new long[enums.length];
for (int i = 0; i < res.length; i++) {
res[i] = counters.get(i);
}
return res;
}
}

140
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/analytics/action/AnalyticsStatsAction.java
View File

@ -14,16 +14,18 @@ import org.elasticsearch.action.support.nodes.BaseNodesRequest;
import org.elasticsearch.action.support.nodes.BaseNodesResponse;
import org.elasticsearch.cluster.ClusterName;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.io.stream.Writeable;
import org.elasticsearch.common.xcontent.ToXContentObject;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.xpack.core.analytics.EnumCounters;
import java.io.IOException;
import java.util.List;
import java.util.Locale;
import java.util.Objects;
import java.util.stream.Collectors;
public class AnalyticsStatsAction extends ActionType<AnalyticsStatsAction.Response> {
public static final AnalyticsStatsAction INSTANCE = new AnalyticsStatsAction();
@ -33,6 +35,17 @@ public class AnalyticsStatsAction extends ActionType<AnalyticsStatsAction.Respon
super(NAME, Response::new);
}
/**
* Items to track. Serialized by ordinals. Append only, don't remove or change order of items in this list.
*/
public enum Item {
BOXPLOT,
CUMULATIVE_CARDINALITY,
STRING_STATS,
TOP_METRICS,
T_TEST;
}
public static class Request extends BaseNodesRequest<Request> implements ToXContentObject {
public Request() {
@ -97,109 +110,84 @@ public class AnalyticsStatsAction extends ActionType<AnalyticsStatsAction.Respon
out.writeList(nodes);
}
public EnumCounters<Item> getStats() {
List<EnumCounters<Item>> countersPerNode = getNodes()
.stream()
.map(AnalyticsStatsAction.NodeResponse::getStats)
.collect(Collectors.toList());
return EnumCounters.merge(Item.class, countersPerNode);
}
@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startArray("stats");
for (NodeResponse node : getNodes()) {
node.toXContent(builder, params);
EnumCounters<Item> stats = getStats();
builder.startObject("stats");
for (Item item : Item.values()) {
builder.field(item.name().toLowerCase(Locale.ROOT) + "_usage", stats.get(item));
}
builder.endArray();
builder.endObject();
return builder;
}
}
public static class NodeResponse extends BaseNodeResponse implements ToXContentObject {
static final ParseField BOXPLOT_USAGE = new ParseField("boxplot_usage");
static final ParseField CUMULATIVE_CARDINALITY_USAGE = new ParseField("cumulative_cardinality_usage");
static final ParseField STRING_STATS_USAGE = new ParseField("string_stats_usage");
static final ParseField TOP_METRICS_USAGE = new ParseField("top_metrics_usage");
static final ParseField T_TEST_USAGE = new ParseField("t_test_usage");
public static class NodeResponse extends BaseNodeResponse {
private final EnumCounters<Item> counters;
private final long boxplotUsage;
private final long cumulativeCardinalityUsage;
private final long stringStatsUsage;
private final long topMetricsUsage;
private final long ttestUsage;
public NodeResponse(DiscoveryNode node, long boxplotUsage, long cumulativeCardinalityUsage, long stringStatsUsage,
long topMetricsUsage, long ttestUsage) {
public NodeResponse(DiscoveryNode node, EnumCounters<Item> counters) {
super(node);
this.boxplotUsage = boxplotUsage;
this.cumulativeCardinalityUsage = cumulativeCardinalityUsage;
this.stringStatsUsage = stringStatsUsage;
this.topMetricsUsage = topMetricsUsage;
this.ttestUsage = ttestUsage;
this.counters = counters;
}
public NodeResponse(StreamInput in) throws IOException {
super(in);
if (in.getVersion().onOrAfter(Version.V_7_7_0)) {
boxplotUsage = in.readVLong();
} else {
boxplotUsage = 0;
}
cumulativeCardinalityUsage = in.readZLong();
if (in.getVersion().onOrAfter(Version.V_7_7_0)) {
stringStatsUsage = in.readVLong();
topMetricsUsage = in.readVLong();
} else {
topMetricsUsage = 0;
stringStatsUsage = 0;
}
if (in.getVersion().onOrAfter(Version.V_7_8_0)) {
ttestUsage = in.readVLong();
counters = new EnumCounters<>(in, Item.class);
} else {
ttestUsage = 0;
counters = new EnumCounters<>(Item.class);
if (in.getVersion().onOrAfter(Version.V_7_7_0)) {
counters.inc(Item.BOXPLOT, in.readVLong());
}
counters.inc(Item.CUMULATIVE_CARDINALITY, in.readZLong());
if (in.getVersion().onOrAfter(Version.V_7_7_0)) {
counters.inc(Item.STRING_STATS, in.readVLong());
counters.inc(Item.TOP_METRICS, in.readVLong());
}
}
}
@Override
public void writeTo(StreamOutput out) throws IOException {
super.writeTo(out);
if (out.getVersion().onOrAfter(Version.V_7_7_0)) {
out.writeVLong(boxplotUsage);
}
out.writeVLong(cumulativeCardinalityUsage);
if (out.getVersion().onOrAfter(Version.V_7_7_0)) {
out.writeVLong(stringStatsUsage);
out.writeVLong(topMetricsUsage);
}
if (out.getVersion().onOrAfter(Version.V_7_8_0)) {
out.writeVLong(ttestUsage);
counters.writeTo(out);
} else {
if (out.getVersion().onOrAfter(Version.V_7_7_0)) {
out.writeVLong(counters.get(Item.BOXPLOT));
}
out.writeZLong(counters.get(Item.CUMULATIVE_CARDINALITY));
if (out.getVersion().onOrAfter(Version.V_7_7_0)) {
out.writeVLong(counters.get(Item.STRING_STATS));
out.writeVLong(counters.get(Item.TOP_METRICS));
}
}
}
public EnumCounters<Item> getStats() {
return counters;
}
@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject();
builder.field(BOXPLOT_USAGE.getPreferredName(), boxplotUsage);
builder.field(CUMULATIVE_CARDINALITY_USAGE.getPreferredName(), cumulativeCardinalityUsage);
builder.field(STRING_STATS_USAGE.getPreferredName(), stringStatsUsage);
builder.field(TOP_METRICS_USAGE.getPreferredName(), topMetricsUsage);
builder.field(T_TEST_USAGE.getPreferredName(), ttestUsage);
builder.endObject();
return builder;
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
NodeResponse that = (NodeResponse) o;
return counters.equals(that.counters) &&
getNode().equals(that.getNode());
}
public long getBoxplotUsage() {
return boxplotUsage;
}
public long getCumulativeCardinalityUsage() {
return cumulativeCardinalityUsage;
}
public long getStringStatsUsage() {
return stringStatsUsage;
}
public long getTopMetricsUsage() {
return topMetricsUsage;
}
public long getTTestUsage() {
return topMetricsUsage;
@Override
public int hashCode() {
return Objects.hash(counters, getNode());
}
}
}

112
x-pack/plugin/core/src/test/java/org/elasticsearch/analytics/EnumCountersTests.java Normal file
View File

@ -0,0 +1,112 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.analytics;
import org.elasticsearch.Version;
import org.elasticsearch.common.io.stream.BytesStreamOutput;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.Writeable;
import org.elasticsearch.test.AbstractWireTestCase;
import org.elasticsearch.xpack.core.analytics.EnumCounters;
import java.io.IOException;
import java.util.Map;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.hasSize;
public class EnumCountersTests extends AbstractWireTestCase<EnumCounters<EnumCountersTests.TestV2>> {
enum TestV1 {A, B, C}
enum TestV2 {A, B, C, D}
@Override
protected EnumCounters<TestV2> createTestInstance() {
EnumCounters<TestV2> inst = new EnumCounters<>(TestV2.class);
inst.inc(TestV2.A, randomNonNegativeLong());
inst.inc(TestV2.B, randomNonNegativeLong());
inst.inc(TestV2.C, randomNonNegativeLong());
inst.inc(TestV2.D, randomNonNegativeLong());
return inst;
}
@Override
protected EnumCounters<TestV2> copyInstance(EnumCounters<TestV2> instance, Version version) throws IOException {
return serialize(instance, in -> new EnumCounters<>(in, TestV2.class));
}
public void testIncrements() {
EnumCounters<TestV1> counters = new EnumCounters<>(TestV1.class);
int a = randomIntBetween(0, 100);
int b = randomIntBetween(0, 100);
int c = randomIntBetween(0, 100);
incrementRandomly(counters, TestV1.A, a);
incrementRandomly(counters, TestV1.B, b);
incrementRandomly(counters, TestV1.C, c);
assertEquals(a, counters.get(TestV1.A));
assertEquals(b, counters.get(TestV1.B));
assertEquals(c, counters.get(TestV1.C));
Map<String, Object> map = counters.toMap();
assertThat(map.keySet(), hasSize(3));
assertThat(map.get("a"), equalTo((long) a));
assertThat(map.get("b"), equalTo((long) b));
assertThat(map.get("c"), equalTo((long) c));
}
public void testBackwardCompatibility() throws Exception {
EnumCounters<TestV2> counters = new EnumCounters<>(TestV2.class);
counters.inc(TestV2.A, 1);
counters.inc(TestV2.B, 2);
counters.inc(TestV2.C, 3);
counters.inc(TestV2.D, 4);
EnumCounters<TestV1> oldCounters = serialize(counters, in -> new EnumCounters<>(in, TestV1.class));
assertEquals(counters.get(TestV2.A), oldCounters.get(TestV1.A));
assertEquals(counters.get(TestV2.B), oldCounters.get(TestV1.B));
assertEquals(counters.get(TestV2.C), oldCounters.get(TestV1.C));
}
public void testForwardCompatibility() throws Exception {
EnumCounters<TestV1> counters = new EnumCounters<>(TestV1.class);
counters.inc(TestV1.A, 1);
counters.inc(TestV1.B, 2);
counters.inc(TestV1.C, 3);
EnumCounters<TestV2> newCounters = serialize(counters, in -> new EnumCounters<>(in, TestV2.class));
assertEquals(counters.get(TestV1.A), newCounters.get(TestV2.A));
assertEquals(counters.get(TestV1.B), newCounters.get(TestV2.B));
assertEquals(counters.get(TestV1.C), newCounters.get(TestV2.C));
assertEquals(0, newCounters.get(TestV2.D));
}
private <E1 extends Enum<E1>, E2 extends Enum<E2>> EnumCounters<E2> serialize(
EnumCounters<E1> source, Writeable.Reader<EnumCounters<E2>> targetReader) throws IOException {
try (BytesStreamOutput output = new BytesStreamOutput()) {
source.writeTo(output);
try (StreamInput in = output.bytes().streamInput()) {
return targetReader.read(in);
}
}
}
private <E extends Enum<E>> void incrementRandomly(EnumCounters<E> counters, E e, int inc) {
int single = randomIntBetween(0, inc);
if (randomBoolean()) {
for (int i = 0; i < single; i++) {
counters.inc(e);
}
counters.inc(e, inc - single);
} else {
counters.inc(e, inc - single);
for (int i = 0; i < single; i++) {
counters.inc(e);
}
}
}
}

158
x-pack/plugin/src/test/resources/rest-api-spec/test/analytics/usage.yml Normal file
View File

@ -0,0 +1,158 @@
---
setup:
- skip:
version: " - 7.7.99"
reason: "stats are not working in earlier versions at the moment"
- do:
bulk:
index: test
refresh: true
body:
- '{"index": {}}'
- '{"timestamp": "2017-01-01T05:00:00Z", "s": 1, "v1": 3.1415, "v2": 2.1415, "str": "a"}'
- '{"index": {}}'
- '{"timestamp": "2017-01-01T05:00:00Z", "s": 2, "v1": 1.0, "v2": 2.0, "str": "a"}'
- '{"index": {}}'
- '{"timestamp": "2017-01-01T05:00:00Z", "s": 3, "v1": 2.71828, "v2": 3.71828, "str": "b"}'
---
"Usage stats on analytics indices":
- do: {xpack.usage: {}}
- match: { analytics.available: true }
- match: { analytics.enabled: true }
- set: {analytics.stats.boxplot_usage: boxplot_usage}
- set: {analytics.stats.top_metrics_usage: top_metrics_usage}
- set: {analytics.stats.cumulative_cardinality_usage: cumulative_cardinality_usage}
- set: {analytics.stats.t_test_usage: t_test_usage}
- set: {analytics.stats.string_stats_usage: string_stats_usage}
# use boxplot agg
- do:
search:
index: "test"
body:
size: 0
aggs:
plot:
boxplot:
field: "s"
- match: { aggregations.plot.q2: 2.0 }
- do: {xpack.usage: {}}
- match: { analytics.available: true }
- match: { analytics.enabled: true }
- gt: { analytics.stats.boxplot_usage: $boxplot_usage }
- set: {analytics.stats.boxplot_usage: boxplot_usage}
- match: {analytics.stats.top_metrics_usage: $top_metrics_usage}
- match: {analytics.stats.cumulative_cardinality_usage: $cumulative_cardinality_usage}
- match: {analytics.stats.t_test_usage: $t_test_usage}
- match: {analytics.stats.string_stats_usage: $string_stats_usage}
# use top_metrics agg
- do:
search:
index: "test"
size: 0
body:
aggs:
tm:
top_metrics:
metrics:
field: v1
sort:
s: desc
- match: { aggregations.tm.top.0.metrics.v1: 2.718280076980591 }
- match: { aggregations.tm.top.0.sort: [3] }
- do: {xpack.usage: {}}
- match: { analytics.available: true }
- match: { analytics.enabled: true }
- match: {analytics.stats.boxplot_usage: $boxplot_usage}
- gt: { analytics.stats.top_metrics_usage: $top_metrics_usage }
- set: {analytics.stats.top_metrics_usage: top_metrics_usage}
- match: {analytics.stats.cumulative_cardinality_usage: $cumulative_cardinality_usage}
- match: {analytics.stats.t_test_usage: $t_test_usage}
- match: {analytics.stats.string_stats_usage: $string_stats_usage}
# use cumulative_cardinality agg
- do:
search:
index: "test"
body:
size: 0
aggs:
histo:
date_histogram:
field: "timestamp"
calendar_interval: "day"
aggs:
distinct_s:
cardinality:
field: "s"
total_users:
cumulative_cardinality:
buckets_path: "distinct_s"
- length: { aggregations.histo.buckets: 1 }
- do: {xpack.usage: {}}
- match: { analytics.available: true }
- match: { analytics.enabled: true }
- match: {analytics.stats.boxplot_usage: $boxplot_usage}
- match: {analytics.stats.top_metrics_usage: $top_metrics_usage}
- gt: { analytics.stats.cumulative_cardinality_usage: $cumulative_cardinality_usage }
- set: {analytics.stats.cumulative_cardinality_usage: cumulative_cardinality_usage}
- match: {analytics.stats.t_test_usage: $t_test_usage}
- match: {analytics.stats.string_stats_usage: $string_stats_usage}
# use t-test agg
- do:
search:
size: 0
index: "test"
body:
aggs:
ttest:
t_test:
a:
field: v1
b:
field: v2
- match: { aggregations.ttest.value: 0.7172402682151968 }
- do: {xpack.usage: {}}
- match: { analytics.available: true }
- match: { analytics.enabled: true }
- match: {analytics.stats.boxplot_usage: $boxplot_usage}
- match: {analytics.stats.top_metrics_usage: $top_metrics_usage}
- match: {analytics.stats.cumulative_cardinality_usage: $cumulative_cardinality_usage}
- gt: { analytics.stats.t_test_usage: $t_test_usage }
- set: {analytics.stats.t_test_usage: t_test_usage}
- match: {analytics.stats.string_stats_usage: $string_stats_usage}
- do:
search:
size: 0
index: "test"
body:
aggs:
my_agg:
string_stats:
field: str.keyword
- match: { aggregations.my_agg.count: 3 }
- do: {xpack.usage: {}}
- match: { analytics.available: true }
- match: { analytics.enabled: true }
- match: {analytics.stats.boxplot_usage: $boxplot_usage}
- match: {analytics.stats.top_metrics_usage: $top_metrics_usage}
- match: {analytics.stats.cumulative_cardinality_usage: $cumulative_cardinality_usage}
- match: {analytics.stats.t_test_usage: $t_test_usage}
- gt: { analytics.stats.string_stats_usage: $string_stats_usage }
- set: {analytics.stats.string_stats_usage: string_stats_usage}

73
x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/100_analytics_usage.yml Normal file
View File

@ -0,0 +1,73 @@
---
setup:
- skip:
version: " - 7.4.99"
reason: "analytics wasn't available before 7.5"
features: allowed_warnings
- do:
bulk:
index: "analytics_usage"
refresh: true
body:
- '{"index": {}}'
- '{"timestamp": "2017-01-01T05:00:00Z", "s": 1, "v1": 3.1415, "v2": 2.1415, "str": "a"}'
- '{"index": {}}'
- '{"timestamp": "2017-01-01T05:00:00Z", "s": 2, "v1": 1.0, "v2": 2.0, "str": "a"}'
- '{"index": {}}'
- '{"timestamp": "2017-01-01T05:00:00Z", "s": 3, "v1": 2.71828, "v2": 3.71828, "str": "b"}'
---
"Smoke test for usage stats on analytics indices":
- skip:
version: "7.8.0 - "
reason: "stats are not working in earlier versions"
- do:
allowed_warnings:
- '[interval] on [date_histogram] is deprecated, use [fixed_interval] or [calendar_interval] in the future.'
xpack.usage: {}
- match: { analytics.available: true }
- match: { analytics.enabled: true }
- is_true: analytics.stats
---
"Basic test for usage stats on analytics indices":
- skip:
version: " - 7.7.99"
reason: "stats are not working in earlier versions"
- do:
allowed_warnings:
- '[interval] on [date_histogram] is deprecated, use [fixed_interval] or [calendar_interval] in the future.'
xpack.usage: {}
- match: { analytics.available: true }
- match: { analytics.enabled: true }
- is_true: analytics.stats
- set: {analytics.stats.cumulative_cardinality_usage: cumulative_cardinality_usage}
# use cumulative_cardinality agg
- do:
search:
index: "analytics_usage"
body:
size: 0
aggs:
histo:
date_histogram:
field: "timestamp"
calendar_interval: "day"
aggs:
distinct_s:
cardinality:
field: "s"
total_users:
cumulative_cardinality:
buckets_path: "distinct_s"
- length: { aggregations.histo.buckets: 1 }
- do:
allowed_warnings:
- '[interval] on [date_histogram] is deprecated, use [fixed_interval] or [calendar_interval] in the future.'
xpack.usage: {}
- match: { analytics.available: true }
- match: { analytics.enabled: true }
- gt: { analytics.stats.cumulative_cardinality_usage: $cumulative_cardinality_usage }