From 5c9d18fc3473e9523691952b8f0b2d95eab091cd Mon Sep 17 00:00:00 2001 From: Chris Earle Date: Mon, 9 May 2016 19:10:14 -0400 Subject: [PATCH] Adding {index} option for _xpack/monitoring/_bulk This adds it so that a system can specify "_data" as the index to index into the data index (without having to know its name). _Not_ supplying an index will use the timestamped index. Any other index name (including wrong case) is invalid. Original commit: elastic/x-pack-elasticsearch@6eeadfb3c85c47d8bdd1284478693ecaee363578 --- .../marvel/action/MonitoringBulkDoc.java | 22 ++++-- .../marvel/action/MonitoringBulkRequest.java | 13 ++-- .../marvel/action/MonitoringIndex.java | 76 +++++++++++++++++++ .../resolver/MonitoringIndexNameResolver.java | 2 +- .../agent/resolver/ResolversRegistry.java | 16 ++-- .../bulk/MonitoringBulkDataResolver.java | 35 +++++++++ ...=> MonitoringBulkTimestampedResolver.java} | 4 +- .../rest/action/RestMonitoringBulkAction.java | 15 ++-- .../marvel/action/MonitoringBulkDocTests.java | 2 +- .../action/MonitoringBulkRequestTests.java | 9 ++- .../marvel/action/MonitoringBulkTests.java | 8 +- .../marvel/action/MonitoringIndexTests.java | 69 +++++++++++++++++ .../exporter/http/HttpExporterTests.java | 6 +- .../MonitoringIndexNameResolverTestCase.java | 11 ++- .../resolver/TimestampedResolverTests.java | 2 +- .../bulk/MonitoringBulkDataResolverTests.java | 73 ++++++++++++++++++ ...nitoringBulkTimestampedResolverTests.java} | 35 +++++---- .../test/monitoring/bulk/10_basic.yaml | 11 ++- .../shield/authz/permission/KibanaRole.java | 11 +-- .../integration/KibanaRoleTests.java | 3 + .../authz/permission/KibanaRoleTests.java | 2 + 21 files changed, 362 insertions(+), 63 deletions(-) create mode 100644 elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/action/MonitoringIndex.java create mode 100644 elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/resolver/bulk/MonitoringBulkDataResolver.java rename elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/resolver/bulk/{MonitoringBulkResolver.java => MonitoringBulkTimestampedResolver.java} (85%) create mode 100644 elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/action/MonitoringIndexTests.java create mode 100644 elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/resolver/bulk/MonitoringBulkDataResolverTests.java rename elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/resolver/bulk/{MonitoringBulkResolverTests.java => MonitoringBulkTimestampedResolverTests.java} (78%) diff --git a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/action/MonitoringBulkDoc.java b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/action/MonitoringBulkDoc.java index 25bc836d555..3a27ac8663d 100644 --- a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/action/MonitoringBulkDoc.java +++ b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/action/MonitoringBulkDoc.java @@ -14,22 +14,31 @@ import java.io.IOException; public class MonitoringBulkDoc extends MonitoringDoc { - private String index; + private MonitoringIndex index = MonitoringIndex.TIMESTAMPED; private String type; private String id; - private BytesReference source; public MonitoringBulkDoc(String monitoringId, String monitoringVersion) { super(monitoringId, monitoringVersion); } + public MonitoringBulkDoc(String monitoringId, String monitoringVersion, + MonitoringIndex index, String type, String id, + BytesReference source) { + super(monitoringId, monitoringVersion); + this.index = index; + this.type = type; + this.id = id; + this.source = source; + } + /** * Read from a stream. */ public MonitoringBulkDoc(StreamInput in) throws IOException { super(in); - index = in.readOptionalString(); + index = MonitoringIndex.readFrom(in); type = in.readOptionalString(); id = in.readOptionalString(); source = in.readBytesReference(); @@ -38,17 +47,17 @@ public class MonitoringBulkDoc extends MonitoringDoc { @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); - out.writeOptionalString(index); + index.writeTo(out); out.writeOptionalString(type); out.writeOptionalString(id); out.writeBytesReference(source); } - public String getIndex() { + public MonitoringIndex getIndex() { return index; } - public void setIndex(String index) { + public void setIndex(MonitoringIndex index) { this.index = index; } @@ -75,4 +84,5 @@ public class MonitoringBulkDoc extends MonitoringDoc { public void setSource(BytesReference source) { this.source = source; } + } diff --git a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/action/MonitoringBulkRequest.java b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/action/MonitoringBulkRequest.java index 27a0626fd6e..7ada7f86a25 100644 --- a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/action/MonitoringBulkRequest.java +++ b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/action/MonitoringBulkRequest.java @@ -95,11 +95,14 @@ public class MonitoringBulkRequest extends ActionRequest IndexRequest indexRequest = (IndexRequest) request; // builds a new monitoring document based on the index request - MonitoringBulkDoc doc = new MonitoringBulkDoc(defaultMonitoringId, defaultMonitoringVersion); - doc.setIndex(indexRequest.index()); - doc.setType(indexRequest.type()); - doc.setId(indexRequest.id()); - doc.setSource(indexRequest.source()); + MonitoringBulkDoc doc = + new MonitoringBulkDoc(defaultMonitoringId, + defaultMonitoringVersion, + MonitoringIndex.from(indexRequest.index()), + indexRequest.type(), + indexRequest.id(), + indexRequest.source()); + add(doc); } else { throw new IllegalArgumentException("monitoring bulk requests should only contain index requests"); diff --git a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/action/MonitoringIndex.java b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/action/MonitoringIndex.java new file mode 100644 index 00000000000..ce156bc2139 --- /dev/null +++ b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/action/MonitoringIndex.java @@ -0,0 +1,76 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.marvel.action; + +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; + +import java.io.IOException; + +/** + * {@code MonitoringIndex} represents the receivable index from any request. + *

+ * This allows external systems to provide details for an index without having to know its exact name. + */ +public enum MonitoringIndex implements Writeable { + + /** + * Data that drives information about the "cluster" (e.g., a node or instance). + */ + DATA { + @Override + public boolean matchesIndexName(String indexName) { + return "_data".equals(indexName); + } + }, + + /** + * Timestamped data that drives the charts (e.g., memory statistics). + */ + TIMESTAMPED { + @Override + public boolean matchesIndexName(String indexName) { + return Strings.isEmpty(indexName); + } + }; + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeByte((byte)ordinal()); + } + + public static MonitoringIndex readFrom(StreamInput in) throws IOException { + return values()[in.readByte()]; + } + + /** + * Determine if the {@code indexName} matches {@code this} monitoring index. + * + * @param indexName The name of the index. + * @return {@code true} if {@code this} matches the {@code indexName} + */ + public abstract boolean matchesIndexName(String indexName); + + /** + * Find the {@link MonitoringIndex} to use for the request. + * + * @param indexName The name of the index. + * @return Never {@code null}. + * @throws IllegalArgumentException if {@code indexName} is unrecognized + */ + public static MonitoringIndex from(String indexName) { + for (MonitoringIndex index : values()) { + if (index.matchesIndexName(indexName)) { + return index; + } + } + + throw new IllegalArgumentException("unrecognized index name [" + indexName + "]"); + } + +} diff --git a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/resolver/MonitoringIndexNameResolver.java b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/resolver/MonitoringIndexNameResolver.java index 6b89eec592b..6bfaf6e8dfd 100644 --- a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/resolver/MonitoringIndexNameResolver.java +++ b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/resolver/MonitoringIndexNameResolver.java @@ -163,7 +163,7 @@ public abstract class MonitoringIndexNameResolver { */ public static abstract class Timestamped extends MonitoringIndexNameResolver { - public static final Setting INDEX_NAME_TIME_FORMAT_SETTING = new Setting<>("index.name.time_format","YYYY.MM.dd", + public static final Setting INDEX_NAME_TIME_FORMAT_SETTING = new Setting<>("index.name.time_format", "YYYY.MM.dd", Function.identity(), Setting.Property.NodeScope); private final MonitoredSystem system; diff --git a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/resolver/ResolversRegistry.java b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/resolver/ResolversRegistry.java index 86d5d9c3913..5aadf12ecfc 100644 --- a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/resolver/ResolversRegistry.java +++ b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/resolver/ResolversRegistry.java @@ -9,6 +9,7 @@ import org.elasticsearch.Version; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.marvel.MonitoredSystem; import org.elasticsearch.marvel.action.MonitoringBulkDoc; +import org.elasticsearch.marvel.action.MonitoringIndex; import org.elasticsearch.marvel.agent.collector.cluster.ClusterInfoMonitoringDoc; import org.elasticsearch.marvel.agent.collector.cluster.ClusterStateMonitoringDoc; import org.elasticsearch.marvel.agent.collector.cluster.ClusterStateNodeMonitoringDoc; @@ -20,7 +21,8 @@ import org.elasticsearch.marvel.agent.collector.indices.IndicesStatsMonitoringDo import org.elasticsearch.marvel.agent.collector.node.NodeStatsMonitoringDoc; import org.elasticsearch.marvel.agent.collector.shards.ShardMonitoringDoc; import org.elasticsearch.marvel.agent.exporter.MonitoringDoc; -import org.elasticsearch.marvel.agent.resolver.bulk.MonitoringBulkResolver; +import org.elasticsearch.marvel.agent.resolver.bulk.MonitoringBulkDataResolver; +import org.elasticsearch.marvel.agent.resolver.bulk.MonitoringBulkTimestampedResolver; import org.elasticsearch.marvel.agent.resolver.cluster.ClusterInfoResolver; import org.elasticsearch.marvel.agent.resolver.cluster.ClusterStateNodeResolver; import org.elasticsearch.marvel.agent.resolver.cluster.ClusterStateResolver; @@ -72,8 +74,10 @@ public class ResolversRegistry implements Iterable * Registers resolvers for monitored systems */ private void registerMonitoredSystem(MonitoredSystem id, Settings settings) { - final MonitoringBulkResolver resolver = new MonitoringBulkResolver(id, settings); - registrations.add(resolveByClassSystemVersion(MonitoringBulkDoc.class, id, Version.CURRENT, resolver)); + final MonitoringBulkDataResolver dataResolver = new MonitoringBulkDataResolver(); + final MonitoringBulkTimestampedResolver timestampedResolver = new MonitoringBulkTimestampedResolver(id, settings); + registrations.add(resolveByClassSystemVersion(id, dataResolver, MonitoringIndex.DATA, Version.CURRENT)); + registrations.add(resolveByClassSystemVersion(id, timestampedResolver, MonitoringIndex.TIMESTAMPED, Version.CURRENT)); } /** @@ -97,11 +101,11 @@ public class ResolversRegistry implements Iterable return new Registration(resolver, type::isInstance); } - static Registration resolveByClassSystemVersion(Class type, MonitoredSystem system, Version version, - MonitoringIndexNameResolver resolver) { + static Registration resolveByClassSystemVersion(MonitoredSystem system, MonitoringIndexNameResolver resolver, MonitoringIndex index, + Version version) { return new Registration(resolver, doc -> { try { - if (type.isInstance(doc) == false) { + if (doc instanceof MonitoringBulkDoc == false || index != ((MonitoringBulkDoc)doc).getIndex()) { return false; } if (system != MonitoredSystem.fromSystem(doc.getMonitoringId())) { diff --git a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/resolver/bulk/MonitoringBulkDataResolver.java b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/resolver/bulk/MonitoringBulkDataResolver.java new file mode 100644 index 00000000000..d93cdfa1a4f --- /dev/null +++ b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/resolver/bulk/MonitoringBulkDataResolver.java @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.marvel.agent.resolver.bulk; + +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.marvel.action.MonitoringBulkDoc; +import org.elasticsearch.marvel.agent.resolver.MonitoringIndexNameResolver; + +import java.io.IOException; + +public class MonitoringBulkDataResolver extends MonitoringIndexNameResolver.Data { + + @Override + public String type(MonitoringBulkDoc document) { + return document.getType(); + } + + @Override + public String id(MonitoringBulkDoc document) { + return document.getId(); + } + + @Override + protected void buildXContent(MonitoringBulkDoc document, XContentBuilder builder, ToXContent.Params params) throws IOException { + BytesReference source = document.getSource(); + if (source != null && source.length() > 0) { + builder.rawField(type(document), source); + } + } +} \ No newline at end of file diff --git a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/resolver/bulk/MonitoringBulkResolver.java b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/resolver/bulk/MonitoringBulkTimestampedResolver.java similarity index 85% rename from elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/resolver/bulk/MonitoringBulkResolver.java rename to elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/resolver/bulk/MonitoringBulkTimestampedResolver.java index e4fa402075d..7af264fc2fb 100644 --- a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/resolver/bulk/MonitoringBulkResolver.java +++ b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/resolver/bulk/MonitoringBulkTimestampedResolver.java @@ -15,9 +15,9 @@ import org.elasticsearch.marvel.agent.resolver.MonitoringIndexNameResolver; import java.io.IOException; -public class MonitoringBulkResolver extends MonitoringIndexNameResolver.Timestamped { +public class MonitoringBulkTimestampedResolver extends MonitoringIndexNameResolver.Timestamped { - public MonitoringBulkResolver(MonitoredSystem id, Settings settings) { + public MonitoringBulkTimestampedResolver(MonitoredSystem id, Settings settings) { super(id, settings); } diff --git a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/rest/action/RestMonitoringBulkAction.java b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/rest/action/RestMonitoringBulkAction.java index 0c3ee6b4878..59226d3cd9f 100644 --- a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/rest/action/RestMonitoringBulkAction.java +++ b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/rest/action/RestMonitoringBulkAction.java @@ -23,6 +23,9 @@ import org.elasticsearch.rest.RestResponse; import org.elasticsearch.rest.action.support.RestActions; import org.elasticsearch.rest.action.support.RestBuilderListener; +import static org.elasticsearch.rest.RestRequest.Method.POST; +import static org.elasticsearch.rest.RestRequest.Method.PUT; + public class RestMonitoringBulkAction extends MonitoringRestHandler { public static final String MONITORING_ID = "system_id"; @@ -31,12 +34,12 @@ public class RestMonitoringBulkAction extends MonitoringRestHandler { @Inject public RestMonitoringBulkAction(Settings settings, RestController controller, Client client) { super(settings, client); - controller.registerHandler(RestRequest.Method.POST, URI_BASE + "/_bulk", this); - controller.registerHandler(RestRequest.Method.PUT, URI_BASE + "/_bulk", this); - controller.registerHandler(RestRequest.Method.POST, URI_BASE + "/{index}/_bulk", this); - controller.registerHandler(RestRequest.Method.PUT, URI_BASE + "/{index}/_bulk", this); - controller.registerHandler(RestRequest.Method.POST, URI_BASE + "/{index}/{type}/_bulk", this); - controller.registerHandler(RestRequest.Method.PUT, URI_BASE + "/{index}/{type}/_bulk", this); + controller.registerHandler(POST, URI_BASE + "/_bulk", this); + controller.registerHandler(PUT, URI_BASE + "/_bulk", this); + controller.registerHandler(POST, URI_BASE + "/{type}/_bulk", this); + controller.registerHandler(PUT, URI_BASE + "/{type}/_bulk", this); + controller.registerHandler(POST, URI_BASE + "/{index}/{type}/_bulk", this); + controller.registerHandler(PUT, URI_BASE + "/{index}/{type}/_bulk", this); } @Override diff --git a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/action/MonitoringBulkDocTests.java b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/action/MonitoringBulkDocTests.java index ca6914f3e95..876d6fd0891 100644 --- a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/action/MonitoringBulkDocTests.java +++ b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/action/MonitoringBulkDocTests.java @@ -67,7 +67,7 @@ public class MonitoringBulkDocTests extends ESTestCase { doc.setSource(new BytesArray("{\"key\" : \"value\"}")); } if (rarely()) { - doc.setIndex(randomAsciiOfLength(5)); + doc.setIndex(MonitoringIndex.DATA); doc.setId(randomAsciiOfLength(2)); } return doc; diff --git a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/action/MonitoringBulkRequestTests.java b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/action/MonitoringBulkRequestTests.java index 4e9f2ef9e79..3f67e00c0c3 100644 --- a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/action/MonitoringBulkRequestTests.java +++ b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/action/MonitoringBulkRequestTests.java @@ -23,6 +23,7 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasItems; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.nullValue; +import static org.hamcrest.Matchers.sameInstance; public class MonitoringBulkRequestTests extends ESTestCase { @@ -122,9 +123,11 @@ public class MonitoringBulkRequestTests extends ESTestCase { String defaultMonitoringId = randomBoolean() ? randomAsciiOfLength(2) : null; String defaultMonitoringVersion = randomBoolean() ? randomAsciiOfLength(3) : null; - String defaultIndex = randomBoolean() ? randomAsciiOfLength(5) : null; + String defaultIndex = randomFrom("_data", null); String defaultType = randomBoolean() ? randomAsciiOfLength(4) : null; + MonitoringIndex index = MonitoringIndex.from(defaultIndex); + MonitoringBulkRequest request = new MonitoringBulkRequest(); request.add(content.bytes(), defaultMonitoringId, defaultMonitoringVersion, defaultIndex, defaultType); assertThat(request.getDocs(), hasSize(nbDocs)); @@ -132,7 +135,7 @@ public class MonitoringBulkRequestTests extends ESTestCase { for (MonitoringBulkDoc doc : request.getDocs()) { assertThat(doc.getMonitoringId(), equalTo(defaultMonitoringId)); assertThat(doc.getMonitoringVersion(), equalTo(defaultMonitoringVersion)); - assertThat(doc.getIndex(), equalTo(defaultIndex)); + assertThat(doc.getIndex(), sameInstance(index)); assertThat(doc.getType(), equalTo(defaultType)); } } @@ -147,7 +150,7 @@ public class MonitoringBulkRequestTests extends ESTestCase { doc.setType(randomFrom("type1", "type2", "type3")); doc.setSource(SOURCE); if (randomBoolean()) { - doc.setIndex("index"); + doc.setIndex(MonitoringIndex.DATA); } if (randomBoolean()) { doc.setId(randomAsciiOfLength(3)); diff --git a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/action/MonitoringBulkTests.java b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/action/MonitoringBulkTests.java index 55ea6e7928a..374077beff9 100644 --- a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/action/MonitoringBulkTests.java +++ b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/action/MonitoringBulkTests.java @@ -10,7 +10,7 @@ import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.marvel.MonitoredSystem; -import org.elasticsearch.marvel.agent.resolver.bulk.MonitoringBulkResolver; +import org.elasticsearch.marvel.agent.resolver.bulk.MonitoringBulkTimestampedResolver; import org.elasticsearch.marvel.test.MarvelIntegTestCase; import org.elasticsearch.search.SearchHit; import org.elasticsearch.test.junit.annotations.TestLogging; @@ -58,9 +58,9 @@ public class MonitoringBulkTests extends MarvelIntegTestCase { for (SearchHit searchHit : searchResponse.getHits()) { Map source = searchHit.sourceAsMap(); - assertNotNull(source.get(MonitoringBulkResolver.Fields.CLUSTER_UUID)); - assertNotNull(source.get(MonitoringBulkResolver.Fields.TIMESTAMP)); - assertNotNull(source.get(MonitoringBulkResolver.Fields.SOURCE_NODE)); + assertNotNull(source.get(MonitoringBulkTimestampedResolver.Fields.CLUSTER_UUID)); + assertNotNull(source.get(MonitoringBulkTimestampedResolver.Fields.TIMESTAMP)); + assertNotNull(source.get(MonitoringBulkTimestampedResolver.Fields.SOURCE_NODE)); } } diff --git a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/action/MonitoringIndexTests.java b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/action/MonitoringIndexTests.java new file mode 100644 index 00000000000..9569074fd1f --- /dev/null +++ b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/action/MonitoringIndexTests.java @@ -0,0 +1,69 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.marvel.action; + +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; + +/** + * Tests {@link MonitoringIndex} + */ +public class MonitoringIndexTests extends ESTestCase { + + public void testDataMatchesIndexName() { + assertTrue(MonitoringIndex.DATA.matchesIndexName("_data")); + assertFalse(MonitoringIndex.DATA.matchesIndexName("_DATA")); + assertFalse(MonitoringIndex.DATA.matchesIndexName("_dAtA")); + assertFalse(MonitoringIndex.DATA.matchesIndexName("_data ")); + assertFalse(MonitoringIndex.DATA.matchesIndexName(" _data ")); + assertFalse(MonitoringIndex.DATA.matchesIndexName("")); + assertFalse(MonitoringIndex.DATA.matchesIndexName(null)); + } + + public void testTimestampMatchesIndexName() { + assertTrue(MonitoringIndex.TIMESTAMPED.matchesIndexName("")); + assertTrue(MonitoringIndex.TIMESTAMPED.matchesIndexName(null)); + assertFalse(MonitoringIndex.TIMESTAMPED.matchesIndexName(" ")); + assertFalse(MonitoringIndex.TIMESTAMPED.matchesIndexName("_data")); + } + + public void testFrom() { + assertSame(MonitoringIndex.DATA, MonitoringIndex.from("_data")); + assertSame(MonitoringIndex.TIMESTAMPED, MonitoringIndex.from("")); + assertSame(MonitoringIndex.TIMESTAMPED, MonitoringIndex.from(null)); + } + + public void testFromFails() { + String[] invalidNames = { "_DATA", "other", " " }; + + for (String name : invalidNames) { + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> MonitoringIndex.from(name)); + + assertEquals("unrecognized index name [" + name + "]", e.getMessage()); + } + } + + public void testStreaming() throws IOException { + MonitoringIndex index = randomFrom(MonitoringIndex.values()); + + final BytesStreamOutput out = new BytesStreamOutput(); + + index.writeTo(out); + + final StreamInput in = StreamInput.wrap(out.bytes().toBytes()); + + assertSame(index, MonitoringIndex.readFrom(in)); + + assertEquals(0, in.available()); + + in.close(); + out.close(); + } + +} diff --git a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/http/HttpExporterTests.java b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/http/HttpExporterTests.java index 60338a77393..4953c196b33 100644 --- a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/http/HttpExporterTests.java +++ b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/http/HttpExporterTests.java @@ -31,7 +31,7 @@ import org.elasticsearch.marvel.agent.collector.indices.IndexRecoveryMonitoringD import org.elasticsearch.marvel.agent.exporter.Exporters; import org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils; import org.elasticsearch.marvel.agent.exporter.MonitoringDoc; -import org.elasticsearch.marvel.agent.resolver.bulk.MonitoringBulkResolver; +import org.elasticsearch.marvel.agent.resolver.bulk.MonitoringBulkTimestampedResolver; import org.elasticsearch.marvel.test.MarvelIntegTestCase; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.Scope; @@ -229,7 +229,7 @@ public class HttpExporterTests extends MarvelIntegTestCase { enqueueGetClusterVersionResponse(secondWebServer, Version.CURRENT); for (String template : monitoringTemplates().keySet()) { - if (template.contains(MonitoringBulkResolver.Data.DATA)) { + if (template.contains(MonitoringBulkTimestampedResolver.Data.DATA)) { enqueueResponse(secondWebServer, 200, "template [" + template + "] exist"); } else { enqueueResponse(secondWebServer, 404, "template [" + template + "] does not exist"); @@ -252,7 +252,7 @@ public class HttpExporterTests extends MarvelIntegTestCase { assertThat(recordedRequest.getMethod(), equalTo("GET")); assertThat(recordedRequest.getPath(), equalTo("/_template/" + template.getKey())); - if (template.getKey().contains(MonitoringBulkResolver.Data.DATA) == false) { + if (template.getKey().contains(MonitoringBulkTimestampedResolver.Data.DATA) == false) { recordedRequest = secondWebServer.takeRequest(); assertThat(recordedRequest.getMethod(), equalTo("PUT")); assertThat(recordedRequest.getPath(), equalTo("/_template/" + template.getKey())); diff --git a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/resolver/MonitoringIndexNameResolverTestCase.java b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/resolver/MonitoringIndexNameResolverTestCase.java index 8443e960297..00e8a8d818a 100644 --- a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/resolver/MonitoringIndexNameResolverTestCase.java +++ b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/resolver/MonitoringIndexNameResolverTestCase.java @@ -30,7 +30,6 @@ import static org.hamcrest.Matchers.anyOf; import static org.hamcrest.Matchers.emptyArray; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; -import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.hasItems; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.isEmptyOrNullString; @@ -48,7 +47,15 @@ public abstract class MonitoringIndexNameResolverTestCase { + + private final String id = randomBoolean() ? randomAsciiOfLength(35) : null; + + @Override + protected MonitoringBulkDoc newMarvelDoc() { + MonitoringBulkDoc doc = new MonitoringBulkDoc(MonitoredSystem.KIBANA.getSystem(), Version.CURRENT.toString(), + MonitoringIndex.DATA, "kibana", id, + new BytesArray("{\"field1\" : \"value1\"}")); + + if (randomBoolean()) { + doc.setClusterUUID(randomAsciiOfLength(5)); + } + + doc.setClusterUUID(randomAsciiOfLength(5)); + doc.setTimestamp(Math.abs(randomLong())); + doc.setSourceNode(new DiscoveryNode("id", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT)); + return doc; + } + + @Override + public void testId() { + MonitoringBulkDoc doc = newMarvelDoc(); + + assertThat(newResolver(doc).id(doc), sameInstance(id)); + } + + @Override + protected boolean checkFilters() { + return false; + } + + public void testMonitoringBulkResolver() throws Exception { + MonitoringBulkDoc doc = newMarvelDoc(); + + MonitoringBulkDataResolver resolver = newResolver(doc); + assertThat(resolver.index(doc), equalTo(".monitoring-data-2")); + assertThat(resolver.type(doc), equalTo(doc.getType())); + assertThat(resolver.id(doc), sameInstance(id)); + + assertSource(resolver.source(doc, XContentType.JSON), + "cluster_uuid", + "timestamp", + "source_node", + "kibana", + "kibana.field1"); + } +} diff --git a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/resolver/bulk/MonitoringBulkResolverTests.java b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/resolver/bulk/MonitoringBulkTimestampedResolverTests.java similarity index 78% rename from elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/resolver/bulk/MonitoringBulkResolverTests.java rename to elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/resolver/bulk/MonitoringBulkTimestampedResolverTests.java index 0e7c0825553..636f7bede27 100644 --- a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/resolver/bulk/MonitoringBulkResolverTests.java +++ b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/resolver/bulk/MonitoringBulkTimestampedResolverTests.java @@ -12,6 +12,7 @@ import org.elasticsearch.common.transport.DummyTransportAddress; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.marvel.MonitoredSystem; import org.elasticsearch.marvel.action.MonitoringBulkDoc; +import org.elasticsearch.marvel.action.MonitoringIndex; import org.elasticsearch.marvel.agent.resolver.MonitoringIndexNameResolverTestCase; import static java.util.Collections.emptyMap; @@ -19,16 +20,28 @@ import static java.util.Collections.emptySet; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.nullValue; -public class MonitoringBulkResolverTests extends MonitoringIndexNameResolverTestCase { +/** + * Tests {@link MonitoringBulkTimestampedResolver}. + */ +public class MonitoringBulkTimestampedResolverTests + extends MonitoringIndexNameResolverTestCase { @Override protected MonitoringBulkDoc newMarvelDoc() { - MonitoringBulkDoc doc = new MonitoringBulkDoc(MonitoredSystem.KIBANA.getSystem(), Version.CURRENT.toString()); + MonitoringBulkDoc doc = new MonitoringBulkDoc(MonitoredSystem.KIBANA.getSystem(), Version.CURRENT.toString(), + MonitoringIndex.TIMESTAMPED, "kibana_stats", null, + new BytesArray("{\"field1\" : \"value1\"}")); + + doc.setTimestamp(1437580442979L); + if (randomBoolean()) { + doc.setId(randomAsciiOfLength(35)); + } + if (randomBoolean()) { + doc.setClusterUUID(randomAsciiOfLength(5)); + } + doc.setClusterUUID(randomAsciiOfLength(5)); - doc.setTimestamp(Math.abs(randomLong())); doc.setSourceNode(new DiscoveryNode("id", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT)); - doc.setType("kibana_stats"); - doc.setSource(new BytesArray("{\"field1\" : \"value1\"}")); return doc; } @@ -44,18 +57,8 @@ public class MonitoringBulkResolverTests extends MonitoringIndexNameResolverTest public void testMonitoringBulkResolver() throws Exception { MonitoringBulkDoc doc = newMarvelDoc(); - doc.setTimestamp(1437580442979L); - if (randomBoolean()) { - doc.setIndex(randomAsciiOfLength(5)); - } - if (randomBoolean()) { - doc.setId(randomAsciiOfLength(35)); - } - if (randomBoolean()) { - doc.setClusterUUID(randomAsciiOfLength(5)); - } - MonitoringBulkResolver resolver = newResolver(); + MonitoringBulkTimestampedResolver resolver = newResolver(doc); assertThat(resolver.index(doc), equalTo(".monitoring-kibana-2-2015.07.22")); assertThat(resolver.type(doc), equalTo(doc.getType())); assertThat(resolver.id(doc), nullValue()); diff --git a/elasticsearch/x-pack/marvel/src/test/resources/rest-api-spec/test/monitoring/bulk/10_basic.yaml b/elasticsearch/x-pack/marvel/src/test/resources/rest-api-spec/test/monitoring/bulk/10_basic.yaml index 7529c85be13..58146378c55 100644 --- a/elasticsearch/x-pack/marvel/src/test/resources/rest-api-spec/test/monitoring/bulk/10_basic.yaml +++ b/elasticsearch/x-pack/marvel/src/test/resources/rest-api-spec/test/monitoring/bulk/10_basic.yaml @@ -26,7 +26,6 @@ setup: system: 1.51 iowait: 0.85 idle: 84.20 - - index: _type: test_type - avg-cpu: @@ -52,7 +51,6 @@ setup: monitoring.bulk: system_id: "kibana" system_version: $version - index: "default_index" type: "default_type" body: - '{"index": {}}' @@ -61,6 +59,8 @@ setup: - '{"field_1": "value_2"}' - '{"index": {}}' - '{"field_1": "value_3"}' + - '{"index": {"_index": "_data", "_type": "kibana"}}' + - '{"field_1": "value_4"}' - is_false: errors @@ -80,3 +80,10 @@ setup: type: custom_type - match: { hits.total: 1 } + + - do: + search: + index: .monitoring-data-* + type: kibana + + - match: { hits.total: 1 } diff --git a/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authz/permission/KibanaRole.java b/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authz/permission/KibanaRole.java index b35d1cebfad..83dede30eb6 100644 --- a/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authz/permission/KibanaRole.java +++ b/elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authz/permission/KibanaRole.java @@ -5,6 +5,7 @@ */ package org.elasticsearch.shield.authz.permission; +import org.elasticsearch.marvel.action.MonitoringBulkAction; import org.elasticsearch.shield.authz.RoleDescriptor; import org.elasticsearch.shield.authz.privilege.ClusterPrivilege; import org.elasticsearch.shield.authz.privilege.Privilege.Name; @@ -14,9 +15,9 @@ import org.elasticsearch.shield.authz.privilege.Privilege.Name; */ public class KibanaRole extends Role { - private static final String[] CLUSTER_PRIVILEGES = new String[] { "monitor" }; + private static final String[] CLUSTER_PRIVILEGES = new String[] { "monitor", MonitoringBulkAction.NAME}; private static final RoleDescriptor.IndicesPrivileges[] INDICES_PRIVILEGES = new RoleDescriptor.IndicesPrivileges[] { - RoleDescriptor.IndicesPrivileges.builder().indices(".kibana" ).privileges("all").build() }; + RoleDescriptor.IndicesPrivileges.builder().indices(".kibana").privileges("all").build() }; public static final String NAME = "kibana"; public static final RoleDescriptor DESCRIPTOR = new RoleDescriptor(NAME, CLUSTER_PRIVILEGES, INDICES_PRIVILEGES, null); @@ -24,8 +25,8 @@ public class KibanaRole extends Role { private KibanaRole() { super(DESCRIPTOR.getName(), - new ClusterPermission.Core(ClusterPrivilege.get(new Name(DESCRIPTOR.getClusterPrivileges()))), - new IndicesPermission.Core(Role.Builder.convertFromIndicesPrivileges(DESCRIPTOR.getIndicesPrivileges())), - RunAsPermission.Core.NONE); + new ClusterPermission.Core(ClusterPrivilege.get(new Name(DESCRIPTOR.getClusterPrivileges()))), + new IndicesPermission.Core(Role.Builder.convertFromIndicesPrivileges(DESCRIPTOR.getIndicesPrivileges())), + RunAsPermission.Core.NONE); } } diff --git a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/KibanaRoleTests.java b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/KibanaRoleTests.java index 9cf8a5efe32..4a7bb5dfdb1 100644 --- a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/KibanaRoleTests.java +++ b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/KibanaRoleTests.java @@ -167,4 +167,7 @@ public class KibanaRoleTests extends ShieldIntegTestCase { .setIndices(index).get(); assertThat(response.getIndices(), arrayContaining(index)); } + + // TODO: When we have an XPackIntegTestCase, this should test that we can send MonitoringBulkActions + } diff --git a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/authz/permission/KibanaRoleTests.java b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/authz/permission/KibanaRoleTests.java index 7ce018a51e3..b4452578074 100644 --- a/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/authz/permission/KibanaRoleTests.java +++ b/elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/authz/permission/KibanaRoleTests.java @@ -16,6 +16,7 @@ import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsAction; import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateAction; import org.elasticsearch.action.delete.DeleteAction; import org.elasticsearch.action.index.IndexAction; +import org.elasticsearch.marvel.action.MonitoringBulkAction; import org.elasticsearch.shield.user.KibanaUser; import org.elasticsearch.shield.user.User; import org.elasticsearch.test.ESTestCase; @@ -34,6 +35,7 @@ public class KibanaRoleTests extends ESTestCase { assertThat(KibanaRole.INSTANCE.cluster().check(ClusterHealthAction.NAME, request, user), is(true)); assertThat(KibanaRole.INSTANCE.cluster().check(ClusterStateAction.NAME, request, user), is(true)); assertThat(KibanaRole.INSTANCE.cluster().check(ClusterStatsAction.NAME, request, user), is(true)); + assertThat(KibanaRole.INSTANCE.cluster().check(MonitoringBulkAction.NAME, request, user), is(true)); assertThat(KibanaRole.INSTANCE.cluster().check(PutIndexTemplateAction.NAME, request, user), is(false)); assertThat(KibanaRole.INSTANCE.cluster().check(ClusterRerouteAction.NAME, request, user), is(false)); assertThat(KibanaRole.INSTANCE.cluster().check(ClusterUpdateSettingsAction.NAME, request, user), is(false));