honeymoose/OpenSearch
1
0
Fork 0
mirror of https://github.com/honeymoose/OpenSearch.git synced 2025-02-25 22:36:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

Monitoring: Manage multiple index templates

Browse Source 
Original commit: elastic/x-pack-elasticsearch@fef9dcc5d1
This commit is contained in:
Tanguy Leroux 2016-03-23 13:36:25 +01:00
parent 29263637c0
commit 98fc26c614
18 changed files with 400 additions and 313 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
elasticsearch/x-pack/build.gradle
View File

@ -71,8 +71,6 @@ compileTestJava.options.compilerArgs << "-Xlint:-deprecation,-rawtypes,-serial,-
ext.expansions = [
'project.version': version,
// Used in marvel index templates
'monitoring.template.version': '1',
// Used in watcher index templates
'watcher.template.version': '1',
]

10
elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/MarvelSettings.java
View File

@ -13,7 +13,6 @@ import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.settings.SettingsModule;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils;
import org.elasticsearch.xpack.XPackPlugin;
import java.util.Collections;
@ -23,7 +22,6 @@ import java.util.function.Function;
import static org.elasticsearch.common.settings.Setting.Property;
import static org.elasticsearch.common.settings.Setting.boolSetting;
import static org.elasticsearch.common.settings.Setting.groupSetting;
import static org.elasticsearch.common.settings.Setting.intSetting;
import static org.elasticsearch.common.settings.Setting.listSetting;
import static org.elasticsearch.common.settings.Setting.timeSetting;
@ -121,13 +119,6 @@ public class MarvelSettings extends AbstractComponent {
HISTORY_DURATION_MINIMUM, // minimum value
Property.Dynamic, Property.NodeScope);
/**
* The index setting that holds the template version
*/
public static final Setting<Integer> INDEX_TEMPLATE_VERSION =
intSetting("index.xpack.monitoring.template.version", MarvelTemplateUtils.TEMPLATE_VERSION,
Property.Dynamic, Property.IndexScope);
/**
* Settings/Options per configured exporter
*/
@ -147,7 +138,6 @@ public class MarvelSettings extends AbstractComponent {
module.registerSetting(HISTORY_DURATION);
module.registerSetting(EXPORTERS_SETTINGS);
module.registerSetting(ENABLED);
module.registerSetting(INDEX_TEMPLATE_VERSION);
module.registerSettingsFilter("xpack.monitoring.agent.exporters.*.auth.*");
module.registerSettingsFilter("xpack.monitoring.agent.exporters.*.ssl.*");

113
elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/MarvelTemplateUtils.java
View File

@ -5,103 +5,80 @@
*/
package org.elasticsearch.marvel.agent.exporter;
import org.elasticsearch.common.Strings;
import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.compress.NotXContentException;
import org.elasticsearch.common.io.Streams;
import org.elasticsearch.common.xcontent.XContentHelper;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.Properties;
import java.util.Locale;
import java.util.regex.Pattern;
public final class MarvelTemplateUtils {
static final String INDEX_TEMPLATE_FILE = "/monitoring-es.json";
static final String INDEX_TEMPLATE_NAME_PREFIX = ".monitoring-es-";
private static final String TEMPLATE_FILE = "/monitoring-%s.json";
private static final String TEMPLATE_VERSION_PROPERTY = Pattern.quote("${monitoring.template.version}");
static final String DATA_TEMPLATE_FILE = "/monitoring-data.json";
static final String DATA_TEMPLATE_NAME_PREFIX = ".monitoring-data-";
static final String PROPERTIES_FILE = "/monitoring.properties";
static final String TEMPLATE_VERSION_PROPERTY = "template.version";
public static final Integer TEMPLATE_VERSION = loadTemplateVersion();
/** Current version of es and data templates **/
public static final Integer TEMPLATE_VERSION = 2;
private MarvelTemplateUtils() {
}
/**
* Loads the default template for the timestamped indices
* Loads a built-in template and returns its source.
*/
public static byte[] loadTimestampedIndexTemplate() {
public static String loadTemplate(String id, Integer version) {
String resource = String.format(Locale.ROOT, TEMPLATE_FILE, id);
try {
return load(INDEX_TEMPLATE_FILE);
} catch (IOException e) {
throw new IllegalStateException("unable to load monitoring template", e);
BytesReference source = load(resource);
validate(source);
return filter(source, version);
} catch (Exception e) {
throw new IllegalArgumentException("Unable to load monitoring template [" + resource + "]", e);
}
}
/**
* Loads the default template for the data index
* Loads a resource from the classpath and returns it as a {@link BytesReference}
*/
public static byte[] loadDataIndexTemplate() {
try {
return load(DATA_TEMPLATE_FILE);
} catch (IOException e) {
throw new IllegalStateException("unable to load monitoring data template", e);
}
}
/**
* Loads a resource with a given name and returns it as a byte array.
*/
static byte[] load(String name) throws IOException {
static BytesReference load(String name) throws IOException {
try (InputStream is = MarvelTemplateUtils.class.getResourceAsStream(name)) {
ByteArrayOutputStream out = new ByteArrayOutputStream();
Streams.copy(is, out);
return out.toByteArray();
try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
Streams.copy(is, out);
return new BytesArray(out.toByteArray());
}
}
}
/**
* Loads the current version of templates
*
* When executing tests in Intellij, the properties file might not be
* resolved: try running 'gradle processResources' first.
* Parses and validates that the source is not empty.
*/
static Integer loadTemplateVersion() {
try (InputStream is = MarvelTemplateUtils.class.getResourceAsStream(PROPERTIES_FILE)) {
Properties properties = new Properties();
properties.load(is);
String version = properties.getProperty(TEMPLATE_VERSION_PROPERTY);
if (Strings.hasLength(version)) {
return Integer.parseInt(version);
}
throw new IllegalArgumentException("no monitoring template version found");
} catch (NumberFormatException e) {
throw new IllegalArgumentException("failed to parse monitoring template version");
} catch (IOException e) {
throw new IllegalArgumentException("failed to load monitoring template version");
static void validate(BytesReference source) {
if (source == null) {
throw new ElasticsearchParseException("Monitoring template must not be null");
}
try {
XContentHelper.convertToMap(source, false).v2();
} catch (NotXContentException e) {
throw new ElasticsearchParseException("Monitoring template must not be empty");
} catch (Exception e) {
throw new ElasticsearchParseException("Invalid monitoring template", e);
}
}
public static String indexTemplateName() {
return indexTemplateName(TEMPLATE_VERSION);
}
public static String indexTemplateName(Integer version) {
return templateName(INDEX_TEMPLATE_NAME_PREFIX, version);
}
public static String dataTemplateName() {
return dataTemplateName(TEMPLATE_VERSION);
}
public static String dataTemplateName(Integer version) {
return templateName(DATA_TEMPLATE_NAME_PREFIX, version);
}
static String templateName(String prefix, Integer version) {
assert version != null && version >= 0 : "version must be not null and greater or equal to zero";
return prefix + String.valueOf(version);
/**
* Filters the source: replaces any template version property with the version number
*/
static String filter(BytesReference source, Integer version) {
return Pattern.compile(TEMPLATE_VERSION_PROPERTY)
.matcher(source.toUtf8())
.replaceAll(String.valueOf(version));
}
}

59
elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/http/HttpExporter.java
View File

@ -25,7 +25,6 @@ import org.elasticsearch.env.Environment;
import org.elasticsearch.marvel.agent.exporter.ExportBulk;
import org.elasticsearch.marvel.agent.exporter.ExportException;
import org.elasticsearch.marvel.agent.exporter.Exporter;
import org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils;
import org.elasticsearch.marvel.agent.exporter.MonitoringDoc;
import org.elasticsearch.marvel.agent.resolver.MonitoringIndexNameResolver;
import org.elasticsearch.marvel.agent.resolver.ResolversRegistry;
@ -47,6 +46,7 @@ import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URISyntaxException;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.security.AccessController;
@ -55,6 +55,8 @@ import java.security.PrivilegedAction;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Map;
import java.util.stream.Collectors;
import java.util.stream.StreamSupport;
/**
*
@ -107,11 +109,6 @@ public class HttpExporter extends Exporter {
volatile boolean checkedAndUploadedIndexTemplate = false;
volatile boolean supportedClusterVersion = false;
/**
* Version number of built-in templates
**/
private final Integer templateVersion;
boolean keepAlive;
final ConnectionKeepAliveWorker keepAliveWorker;
Thread keepAliveThread;
@ -145,14 +142,16 @@ public class HttpExporter extends Exporter {
sslSocketFactory = createSSLSocketFactory(config.settings().getAsSettings(SSL_SETTING));
hostnameVerification = config.settings().getAsBoolean(SSL_HOSTNAME_VERIFICATION_SETTING, true);
// Loads the current version number of built-in templates
templateVersion = MarvelTemplateUtils.TEMPLATE_VERSION;
if (templateVersion == null) {
throw new IllegalStateException("unable to find built-in template version");
}
resolvers = new ResolversRegistry(config.settings());
logger.debug("initialized with hosts [{}], index prefix [{}], template version [{}]",
Strings.arrayToCommaDelimitedString(hosts), MonitoringIndexNameResolver.PREFIX, templateVersion);
// Checks that required templates are loaded
for (MonitoringIndexNameResolver resolver : resolvers) {
if (resolver.template() == null) {
throw new IllegalStateException("unable to find built-in template " + resolver.templateName());
}
}
logger.debug("initialized with hosts [{}], index prefix [{}]",
Strings.arrayToCommaDelimitedString(hosts), MonitoringIndexNameResolver.PREFIX);
}
ResolversRegistry getResolvers() {
@ -424,24 +423,18 @@ public class HttpExporter extends Exporter {
* @return true if template exists or was uploaded successfully.
*/
private boolean checkAndUploadIndexTemplate(final String host) {
String templateName = MarvelTemplateUtils.indexTemplateName(templateVersion);
boolean templateInstalled = hasTemplate(templateName, host);
// List of distinct templates
Map<String, String> templates = StreamSupport.stream(new ResolversRegistry(Settings.EMPTY).spliterator(), false)
.collect(Collectors.toMap(MonitoringIndexNameResolver::templateName, MonitoringIndexNameResolver::template, (a, b) -> a));
// Works like LocalExporter on master:
// Install the index template for timestamped indices first, so that other nodes can ship data
if (!templateInstalled) {
logger.debug("could not find existing monitoring template, installing a new one");
if (!putTemplate(host, templateName, MarvelTemplateUtils.loadTimestampedIndexTemplate())) {
return false;
}
}
// Install the index template for data index
templateName = MarvelTemplateUtils.dataTemplateName(templateVersion);
if (!hasTemplate(templateName, host)) {
logger.debug("could not find existing monitoring template for data index, installing a new one");
if (!putTemplate(host, templateName, MarvelTemplateUtils.loadDataIndexTemplate())) {
return false;
for (Map.Entry<String, String> template : templates.entrySet()) {
if (hasTemplate(template.getKey(), host) == false) {
logger.debug("template [{}] not found", template.getKey());
if (!putTemplate(host, template.getKey(), template.getValue())) {
return false;
}
} else {
logger.debug("template [{}] found", template.getKey());
}
}
return true;
@ -481,7 +474,7 @@ public class HttpExporter extends Exporter {
return false;
}
boolean putTemplate(String host, String template, byte[] source) {
boolean putTemplate(String host, String template, String source) {
logger.debug("installing template [{}]", template);
HttpURLConnection connection = null;
try {
@ -492,13 +485,13 @@ public class HttpExporter extends Exporter {
}
// Uploads the template and closes the outputstream
Streams.copy(source, connection.getOutputStream());
Streams.copy(source.getBytes(StandardCharsets.UTF_8), connection.getOutputStream());
if (connection.getResponseCode() != 200 && connection.getResponseCode() != 201) {
logConnectionError("error adding the monitoring template [" + template + "] to [" + host + "]", connection);
return false;
}
logger.info("monitoring template [{}] updated to version [{}]", template, templateVersion);
logger.info("monitoring template [{}] updated ", template);
return true;
} catch (IOException e) {
logger.error("failed to update monitoring template [{}] on host [{}]:\n{}", template, host, e.getMessage());

69
elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/local/LocalExporter.java
View File

@ -21,11 +21,11 @@ import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.collect.ImmutableOpenMap;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.regex.Regex;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.gateway.GatewayService;
import org.elasticsearch.marvel.agent.exporter.ExportBulk;
import org.elasticsearch.marvel.agent.exporter.Exporter;
import org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils;
import org.elasticsearch.marvel.agent.exporter.MonitoringDoc;
import org.elasticsearch.marvel.agent.resolver.MonitoringIndexNameResolver;
import org.elasticsearch.marvel.agent.resolver.ResolversRegistry;
@ -35,6 +35,7 @@ import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.atomic.AtomicReference;
import java.util.stream.Collectors;
@ -56,23 +57,21 @@ public class LocalExporter extends Exporter implements ClusterStateListener, Cle
private final AtomicReference<State> state = new AtomicReference<>(State.INITIALIZED);
/** Version number of built-in templates **/
private final Integer templateVersion;
public LocalExporter(Exporter.Config config, MonitoringClientProxy client,
ClusterService clusterService, CleanerService cleanerService) {
super(TYPE, config);
this.client = client;
this.clusterService = clusterService;
this.cleanerService = cleanerService;
this.resolvers = new ResolversRegistry(config.settings());
// Loads the current version number of built-in templates
templateVersion = MarvelTemplateUtils.TEMPLATE_VERSION;
if (templateVersion == null) {
throw new IllegalStateException("unable to find built-in template version");
// Checks that required templates are loaded
for (MonitoringIndexNameResolver resolver : resolvers) {
if (resolver.template() == null) {
throw new IllegalStateException("unable to find built-in template " + resolver.templateName());
}
}
resolvers = new ResolversRegistry(config.settings());
clusterService.add(this);
cleanerService.add(this);
}
@ -117,21 +116,23 @@ public class LocalExporter extends Exporter implements ClusterStateListener, Cle
return null;
}
String templateName = MarvelTemplateUtils.indexTemplateName(templateVersion);
boolean templateInstalled = hasTemplate(templateName, clusterState);
// List of distinct templates
Map<String, String> templates = StreamSupport.stream(new ResolversRegistry(Settings.EMPTY).spliterator(), false)
.collect(Collectors.toMap(MonitoringIndexNameResolver::templateName, MonitoringIndexNameResolver::template, (a, b) -> a));
// if this is not the master, we'll just look to see if the monitoring timestamped template is already
// installed and if so, if it has a compatible version. If it is (installed and compatible)
// we'll be able to start this exporter. Otherwise, we'll just wait for a new cluster state.
// if this is not the master, we'll just look to see if the monitoring templates are installed.
// If they all are, we'll be able to start this exporter. Otherwise, we'll just wait for a new cluster state.
if (clusterService.state().nodes().isLocalNodeElectedMaster() == false) {
// We only need to check the index template for timestamped indices
if (templateInstalled == false) {
// the template for timestamped indices is not yet installed in the given cluster state, we'll wait.
logger.debug("monitoring index template does not exist, so service cannot start");
return null;
for (String template : templates.keySet()) {
if (hasTemplate(template, clusterState) == false) {
// the required template is not yet installed in the given cluster state, we'll wait.
logger.debug("monitoring index template [{}] does not exist, so service cannot start", template);
return null;
}
}
logger.debug("monitoring index template found, service can start");
logger.debug("monitoring index templates are installed, service can start");
} else {
@ -143,24 +144,18 @@ public class LocalExporter extends Exporter implements ClusterStateListener, Cle
return null;
}
// Install the index template for timestamped indices first, so that other nodes can ship data
if (templateInstalled == false) {
logger.debug("could not find existing monitoring template for timestamped indices, installing a new one");
putTemplate(templateName, MarvelTemplateUtils.loadTimestampedIndexTemplate());
// we'll get that template on the next cluster state update
return null;
// Check that each required template exist, installing it if needed
for (Map.Entry<String, String> template : templates.entrySet()) {
if (hasTemplate(template.getKey(), clusterState) == false) {
logger.debug("template [{}] not found", template.getKey());
putTemplate(template.getKey(), template.getValue());
return null;
} else {
logger.debug("template [{}] found", template.getKey());
}
}
// Install the index template for data index
templateName = MarvelTemplateUtils.dataTemplateName(templateVersion);
if (hasTemplate(templateName, clusterState) == false) {
logger.debug("could not find existing monitoring template for data index, installing a new one");
putTemplate(templateName, MarvelTemplateUtils.loadDataIndexTemplate());
// we'll get that template on the next cluster state update
return null;
}
logger.debug("monitoring index template found on master node, service can start");
logger.debug("monitoring index templates are installed on master node, service can start");
}
if (state.compareAndSet(State.INITIALIZED, State.RUNNING)) {
@ -197,7 +192,7 @@ public class LocalExporter extends Exporter implements ClusterStateListener, Cle
return templates.size() > 0;
}
void putTemplate(String template, byte[] source) {
void putTemplate(String template, String source) {
logger.debug("installing template [{}]",template);
PutIndexTemplateRequest request = new PutIndexTemplateRequest(template).source(source);

32
elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/resolver/MonitoringIndexNameResolver.java
View File

@ -16,6 +16,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentBuilderString;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.marvel.MonitoredSystem;
import org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils;
import org.elasticsearch.marvel.agent.exporter.MonitoringDoc;
import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;
@ -23,6 +24,7 @@ import org.joda.time.format.DateTimeFormat;
import org.joda.time.format.DateTimeFormatter;
import java.io.IOException;
import java.util.Locale;
import java.util.function.Function;
/**
@ -100,6 +102,16 @@ public abstract class MonitoringIndexNameResolver<T extends MonitoringDoc> {
}
}
/**
* @return the template name that is required by the resolver.
*/
public abstract String templateName();
/**
* @return the template source required by the resolver
*/
public abstract String template();
int getVersion() {
return version;
}
@ -145,6 +157,16 @@ public abstract class MonitoringIndexNameResolver<T extends MonitoringDoc> {
public String indexPattern() {
return index;
}
@Override
public String templateName() {
return String.format(Locale.ROOT, "%s-%s-%d", PREFIX, DATA, getVersion());
}
@Override
public String template() {
return MarvelTemplateUtils.loadTemplate(DATA, getVersion());
}
}
/**
@ -188,6 +210,16 @@ public abstract class MonitoringIndexNameResolver<T extends MonitoringDoc> {
return null;
}
@Override
public String templateName() {
return String.format(Locale.ROOT, "%s-%s-%d", PREFIX, getId(), getVersion());
}
@Override
public String template() {
return MarvelTemplateUtils.loadTemplate(getId(), getVersion());
}
String getId() {
return system.getSystem();
}

17
elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/resolver/ResolversRegistry.java
View File

@ -43,14 +43,17 @@ import static org.elasticsearch.marvel.MonitoredSystem.ES;
public class ResolversRegistry implements Iterable<MonitoringIndexNameResolver> {
private static final int ES_TEMPLATE_VERSION = MarvelTemplateUtils.TEMPLATE_VERSION;
private static final int KIBANA_TEMPLATE_VERSION = MarvelTemplateUtils.TEMPLATE_VERSION;
private final List<Registration> registrations = new ArrayList<>();
public ResolversRegistry(Settings settings) {
// register built-in defaults resolvers
registerBuiltIn(ES, MarvelTemplateUtils.TEMPLATE_VERSION, settings);
registerBuiltIn(ES, ES_TEMPLATE_VERSION, settings);
// register resolvers for external applications
registerKibana(settings);
// register resolvers for monitored systems
registerMonitoredSystem(MonitoredSystem.KIBANA, KIBANA_TEMPLATE_VERSION, settings);
}
/**
@ -70,11 +73,11 @@ public class ResolversRegistry implements Iterable<MonitoringIndexNameResolver>
}
/**
* Registers resolvers for Kibana
* Registers resolvers for monitored systems
*/
private void registerKibana(Settings settings) {
final MonitoringBulkResolver kibana = new MonitoringBulkResolver(MonitoredSystem.KIBANA, 0, settings);
registrations.add(resolveByClassSystemVersion(MonitoringBulkDoc.class, MonitoredSystem.KIBANA, Version.CURRENT, kibana));
private void registerMonitoredSystem(MonitoredSystem id, int version, Settings settings) {
final MonitoringBulkResolver resolver = new MonitoringBulkResolver(id, version, settings);
registrations.add(resolveByClassSystemVersion(MonitoringBulkDoc.class, id, Version.CURRENT, resolver));
}
/**

57
elasticsearch/x-pack/marvel/src/main/resources/monitoring-kibana.json Normal file
View File

@ -0,0 +1,57 @@
{
"template": ".monitoring-kibana-${monitoring.template.version}-*",
"settings": {
"index.number_of_shards": 1,
"index.number_of_replicas": 1,
"index.codec": "best_compression"
},
"mappings": {
"_default_": {
"_all": {
"enabled": false
},
"properties": {
"cluster_uuid": {
"type": "keyword"
},
"timestamp": {
"type": "date",
"format": "date_time"
},
"source_node": {
"properties": {
"uuid": {
"type": "keyword"
},
"host": {
"type": "keyword"
},
"transport_address": {
"type": "keyword"
},
"ip": {
"type": "keyword"
},
"name": {
"type": "keyword"
},
"attributes": {
"dynamic": true,
"properties": {
"data": {
"type": "boolean"
},
"master": {
"type": "boolean"
},
"client": {
"type": "boolean"
}
}
}
}
}
}
}
}
}

1
elasticsearch/x-pack/marvel/src/main/resources/monitoring.properties
View File

@ -1 +0,0 @@
template.version=${monitoring.template.version}

37
elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/AbstractExporterTemplateTestCase.java
View File

@ -19,8 +19,6 @@ import java.io.IOException;
import java.util.Map;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils.dataTemplateName;
import static org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils.indexTemplateName;
import static org.elasticsearch.test.ESIntegTestCase.Scope.TEST;
import static org.hamcrest.Matchers.notNullValue;
@ -58,8 +56,9 @@ public abstract class AbstractExporterTemplateTestCase extends MarvelIntegTestCa
doExporting();
logger.debug("--> templates does not exist: it should have been created in the current version");
assertTemplateExist(indexTemplateName());
assertTemplateExist(dataTemplateName());
for (String template : monitoringTemplates().keySet()) {
assertTemplateExist(template);
}
doExporting();
@ -82,8 +81,9 @@ public abstract class AbstractExporterTemplateTestCase extends MarvelIntegTestCa
assertTemplateExist(indexTemplateName(version));
logger.debug("--> existing templates are old: new templates should be created");
assertTemplateExist(indexTemplateName());
assertTemplateExist(dataTemplateName());
for (String template : monitoringTemplates().keySet()) {
assertTemplateExist(template);
}
doExporting();
@ -101,12 +101,13 @@ public abstract class AbstractExporterTemplateTestCase extends MarvelIntegTestCa
doExporting();
logger.debug("--> existing templates are up to date");
assertTemplateExist(indexTemplateName());
assertTemplateExist(dataTemplateName());
for (String template : monitoringTemplates().keySet()) {
assertTemplateExist(template);
}
logger.debug("--> existing templates has the same version: they should not be changed");
assertTemplateNotUpdated(indexTemplateName());
assertTemplateNotUpdated(dataTemplateName());
assertTemplateNotUpdated(indexTemplateName(currentVersion));
assertTemplateNotUpdated(dataTemplateName(currentVersion));
doExporting();
@ -135,8 +136,9 @@ public abstract class AbstractExporterTemplateTestCase extends MarvelIntegTestCa
doExporting();
logger.debug("--> templates should exist in current version");
assertTemplateExist(indexTemplateName());
assertTemplateExist(dataTemplateName());
for (String template : monitoringTemplates().keySet()) {
assertTemplateExist(template);
}
if (previousIndexTemplateExist) {
logger.debug("--> index template should exist in version [{}]", previousIndexTemplateVersion);
@ -167,6 +169,16 @@ public abstract class AbstractExporterTemplateTestCase extends MarvelIntegTestCa
exporters.export(collector.collect());
}
private String dataTemplateName(Integer version) {
MockDataIndexNameResolver resolver = new MockDataIndexNameResolver(version);
return resolver.templateName();
}
private String indexTemplateName(Integer version) {
MockTimestampedIndexNameResolver resolver = new MockTimestampedIndexNameResolver(MonitoredSystem.ES, version, exporterSettings());
return resolver.templateName();
}
private String currentDataIndexName() {
MockDataIndexNameResolver resolver = new MockDataIndexNameResolver(currentVersion);
return resolver.index(null);
@ -188,7 +200,6 @@ public abstract class AbstractExporterTemplateTestCase extends MarvelIntegTestCa
.startObject("settings")
.field("index.number_of_shards", 1)
.field("index.number_of_replicas", 1)
.field(MarvelSettings.INDEX_TEMPLATE_VERSION.getKey(), String.valueOf(version))
.endObject()
.startObject("mappings")
.startObject("_default_")

92
elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/MarvelTemplateUtilsTests.java
View File

@ -5,52 +5,88 @@
*/
package org.elasticsearch.marvel.agent.exporter;
import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.test.ESTestCase;
import org.junit.Rule;
import org.junit.rules.ExpectedException;
import java.io.IOException;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.greaterThan;
import static org.hamcrest.Matchers.notNullValue;
public class MarvelTemplateUtilsTests extends ESTestCase {
public void testLoadTimestampedIndexTemplate() {
byte[] template = MarvelTemplateUtils.loadTimestampedIndexTemplate();
assertNotNull(template);
assertThat(template.length, greaterThan(0));
}
private static final String TEST_TEMPLATE = "/monitoring-test.json";
public void testLoadDataIndexTemplate() {
byte[] template = MarvelTemplateUtils.loadDataIndexTemplate();
assertNotNull(template);
assertThat(template.length, greaterThan(0));
@Rule
public ExpectedException expectedException = ExpectedException.none();
public void testLoadTemplate() throws IOException {
final int version = randomIntBetween(0, 10_000);
String source = MarvelTemplateUtils.loadTemplate("test", version);
assertThat(source, notNullValue());
assertThat(source.length(), greaterThan(0));
assertThat(source, equalTo("{\n" +
" \"template\": \".monitoring-data-" + version + "\",\n" +
" \"mappings\": {\n" +
" \"type_1\": {\n" +
" \"_meta\": {\n" +
" \"template.version\": \"" + version + "\"\n" +
" }\n" +
" }\n" +
" }\n" +
"}"));
}
public void testLoad() throws IOException {
String resource = randomFrom(MarvelTemplateUtils.INDEX_TEMPLATE_FILE, MarvelTemplateUtils.DATA_TEMPLATE_FILE);
byte[] template = MarvelTemplateUtils.load(resource);
assertNotNull(template);
assertThat(template.length, greaterThan(0));
BytesReference source = MarvelTemplateUtils.load(TEST_TEMPLATE);
assertThat(source, notNullValue());
assertThat(source.length(), greaterThan(0));
}
public void testLoadTemplateVersion() {
Integer version = MarvelTemplateUtils.loadTemplateVersion();
assertNotNull(version);
assertThat(version, greaterThan(0));
assertThat(version, equalTo(MarvelTemplateUtils.TEMPLATE_VERSION));
public void testValidateNullSource() {
expectedException.expect(ElasticsearchParseException.class);
expectedException.expectMessage("Monitoring template must not be null");
MarvelTemplateUtils.validate(null);
}
public void testIndexTemplateName() {
assertThat(MarvelTemplateUtils.indexTemplateName(),
equalTo(MarvelTemplateUtils.INDEX_TEMPLATE_NAME_PREFIX + MarvelTemplateUtils.TEMPLATE_VERSION));
int version = randomIntBetween(1, 100);
assertThat(MarvelTemplateUtils.indexTemplateName(version), equalTo(".monitoring-es-" + version));
public void testValidateEmptySource() {
expectedException.expect(ElasticsearchParseException.class);
expectedException.expectMessage("Monitoring template must not be empty");
MarvelTemplateUtils.validate(new BytesArray(""));
}
public void testDataTemplateName() {
assertThat(MarvelTemplateUtils.dataTemplateName(),
equalTo(MarvelTemplateUtils.DATA_TEMPLATE_NAME_PREFIX + MarvelTemplateUtils.TEMPLATE_VERSION));
int version = randomIntBetween(1, 100);
assertThat(MarvelTemplateUtils.dataTemplateName(version), equalTo(".monitoring-data-" + version));
public void testValidateInvalidSource() {
expectedException.expect(ElasticsearchParseException.class);
expectedException.expectMessage("Invalid monitoring template");
MarvelTemplateUtils.validate(new BytesArray("{\"foo\": \"bar"));
}
public void testValidate() {
try {
MarvelTemplateUtils.validate(MarvelTemplateUtils.load(TEST_TEMPLATE));
} catch (Exception e) {
fail("failed to validate test template: " + e.getMessage());
}
}
public void testFilter() {
assertThat(MarvelTemplateUtils.filter(new BytesArray("${monitoring.template.version}"), 0), equalTo("0"));
assertThat(MarvelTemplateUtils.filter(new BytesArray("{\"template\": \"test-${monitoring.template.version}\"}"), 1),
equalTo("{\"template\": \"test-1\"}"));
assertThat(MarvelTemplateUtils.filter(new BytesArray("{\"template\": \"${monitoring.template.version}-test\"}"), 2),
equalTo("{\"template\": \"2-test\"}"));
assertThat(MarvelTemplateUtils.filter(new BytesArray("{\"template\": \"test-${monitoring.template.version}-test\"}"), 3),
equalTo("{\"template\": \"test-3-test\"}"));
final int version = randomIntBetween(0, 100);
assertThat(MarvelTemplateUtils.filter(new BytesArray("{\"foo-${monitoring.template.version}\": " +
"\"bar-${monitoring.template.version}\"}"), version),
equalTo("{\"foo-" + version + "\": \"bar-" + version + "\"}"));
}
}

166
elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/http/HttpExporterTests.java
View File

@ -31,6 +31,7 @@ import org.elasticsearch.marvel.agent.collector.indices.IndexRecoveryMonitoringD
import org.elasticsearch.marvel.agent.exporter.Exporters;
import org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils;
import org.elasticsearch.marvel.agent.exporter.MonitoringDoc;
import org.elasticsearch.marvel.agent.resolver.bulk.MonitoringBulkResolver;
import org.elasticsearch.marvel.test.MarvelIntegTestCase;
import org.elasticsearch.test.ESIntegTestCase;
import org.elasticsearch.test.ESIntegTestCase.Scope;
@ -50,8 +51,6 @@ import java.util.Map;
import static java.util.Collections.emptyMap;
import static java.util.Collections.emptySet;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils.dataTemplateName;
import static org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils.indexTemplateName;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.instanceOf;
@ -64,9 +63,6 @@ public class HttpExporterTests extends MarvelIntegTestCase {
private int webPort;
private MockWebServer webServer;
private static final byte[] TIMESTAMPED_TEMPLATE = MarvelTemplateUtils.loadTimestampedIndexTemplate();
private static final byte[] DATA_TEMPLATE = MarvelTemplateUtils.loadDataIndexTemplate();
@Before
public void startWebservice() throws Exception {
for (webPort = 9250; webPort < 9300; webPort++) {
@ -91,10 +87,10 @@ public class HttpExporterTests extends MarvelIntegTestCase {
public void testExport() throws Exception {
enqueueGetClusterVersionResponse(Version.CURRENT);
enqueueResponse(404, "template for timestamped indices does not exist");
enqueueResponse(201, "template for timestamped indices created");
enqueueResponse(404, "template for data index does not exist");
enqueueResponse(201, "template for data index created");
for (String template : monitoringTemplates().keySet()) {
enqueueResponse(404, "template [" + template + "] does not exist");
enqueueResponse(201, "template [" + template + "] created");
}
enqueueResponse(200, "{\"errors\": false, \"msg\": \"successful bulk request\"}");
Settings.Builder builder = Settings.builder()
@ -109,29 +105,22 @@ public class HttpExporterTests extends MarvelIntegTestCase {
final int nbDocs = randomIntBetween(1, 25);
export(newRandomMarvelDocs(nbDocs));
assertThat(webServer.getRequestCount(), equalTo(6));
assertThat(webServer.getRequestCount(), equalTo(2 + monitoringTemplates().size() * 2));
RecordedRequest recordedRequest = webServer.takeRequest();
assertThat(recordedRequest.getMethod(), equalTo("GET"));
assertThat(recordedRequest.getPath(), equalTo("/"));
recordedRequest = webServer.takeRequest();
assertThat(recordedRequest.getMethod(), equalTo("GET"));
assertThat(recordedRequest.getPath(), equalTo("/_template/" + indexTemplateName()));
for (Map.Entry<String, String> template : monitoringTemplates().entrySet()) {
recordedRequest = webServer.takeRequest();
assertThat(recordedRequest.getMethod(), equalTo("GET"));
assertThat(recordedRequest.getPath(), equalTo("/_template/" + template.getKey()));
recordedRequest = webServer.takeRequest();
assertThat(recordedRequest.getMethod(), equalTo("PUT"));
assertThat(recordedRequest.getPath(), equalTo("/_template/" + indexTemplateName()));
assertThat(recordedRequest.getBody().readByteArray(), equalTo(TIMESTAMPED_TEMPLATE));
recordedRequest = webServer.takeRequest();
assertThat(recordedRequest.getMethod(), equalTo("GET"));
assertThat(recordedRequest.getPath(), equalTo("/_template/" + dataTemplateName()));
recordedRequest = webServer.takeRequest();
assertThat(recordedRequest.getMethod(), equalTo("PUT"));
assertThat(recordedRequest.getPath(), equalTo("/_template/" + dataTemplateName()));
assertThat(recordedRequest.getBody().readByteArray(), equalTo(DATA_TEMPLATE));
recordedRequest = webServer.takeRequest();
assertThat(recordedRequest.getMethod(), equalTo("PUT"));
assertThat(recordedRequest.getPath(), equalTo("/_template/" + template.getKey()));
assertThat(recordedRequest.getBody().readUtf8(), equalTo(template.getValue()));
}
recordedRequest = webServer.takeRequest();
assertThat(recordedRequest.getMethod(), equalTo("POST"));
@ -176,10 +165,10 @@ public class HttpExporterTests extends MarvelIntegTestCase {
logger.info("--> starting node");
enqueueGetClusterVersionResponse(Version.CURRENT);
enqueueResponse(404, "template for timestamped indices does not exist");
enqueueResponse(201, "template for timestamped indices created");
enqueueResponse(404, "template for data index does not exist");
enqueueResponse(201, "template for data index created");
for (String template : monitoringTemplates().keySet()) {
enqueueResponse(404, "template [" + template + "] does not exist");
enqueueResponse(201, "template [" + template + "] created");
}
enqueueResponse(200, "{\"errors\": false, \"msg\": \"successful bulk request\"}");
String agentNode = internalCluster().startNode(builder);
@ -190,29 +179,22 @@ public class HttpExporterTests extends MarvelIntegTestCase {
export(Collections.singletonList(newRandomMarvelDoc()));
assertThat(exporter.supportedClusterVersion, is(true));
assertThat(webServer.getRequestCount(), equalTo(6));
assertThat(webServer.getRequestCount(), equalTo(2 + monitoringTemplates().size() * 2));
RecordedRequest recordedRequest = webServer.takeRequest();
assertThat(recordedRequest.getMethod(), equalTo("GET"));
assertThat(recordedRequest.getPath(), equalTo("/"));
recordedRequest = webServer.takeRequest();
assertThat(recordedRequest.getMethod(), equalTo("GET"));
assertThat(recordedRequest.getPath(), equalTo("/_template/" + indexTemplateName()));
for (Map.Entry<String, String> template : monitoringTemplates().entrySet()) {
recordedRequest = webServer.takeRequest();
assertThat(recordedRequest.getMethod(), equalTo("GET"));
assertThat(recordedRequest.getPath(), equalTo("/_template/" + template.getKey()));
recordedRequest = webServer.takeRequest();
assertThat(recordedRequest.getMethod(), equalTo("PUT"));
assertThat(recordedRequest.getPath(), equalTo("/_template/" + indexTemplateName()));
assertThat(recordedRequest.getBody().readByteArray(), equalTo(TIMESTAMPED_TEMPLATE));
recordedRequest = webServer.takeRequest();
assertThat(recordedRequest.getMethod(), equalTo("GET"));
assertThat(recordedRequest.getPath(), equalTo("/_template/" + dataTemplateName()));
recordedRequest = webServer.takeRequest();
assertThat(recordedRequest.getMethod(), equalTo("PUT"));
assertThat(recordedRequest.getPath(), equalTo("/_template/" + dataTemplateName()));
assertThat(recordedRequest.getBody().readByteArray(), equalTo(DATA_TEMPLATE));
recordedRequest = webServer.takeRequest();
assertThat(recordedRequest.getMethod(), equalTo("PUT"));
assertThat(recordedRequest.getPath(), equalTo("/_template/" + template.getKey()));
assertThat(recordedRequest.getBody().readUtf8(), equalTo(template.getValue()));
}
recordedRequest = webServer.takeRequest();
assertThat(recordedRequest.getMethod(), equalTo("POST"));
@ -246,32 +228,37 @@ public class HttpExporterTests extends MarvelIntegTestCase {
exporter = getExporter(agentNode);
enqueueGetClusterVersionResponse(secondWebServer, Version.CURRENT);
enqueueResponse(secondWebServer, 404, "template for timestamped indices does not exist");
enqueueResponse(secondWebServer, 201, "template for timestamped indices created");
enqueueResponse(secondWebServer, 200, "template for data index exist");
for (String template : monitoringTemplates().keySet()) {
if (template.contains(MonitoringBulkResolver.Data.DATA)) {
enqueueResponse(secondWebServer, 200, "template [" + template + "] exist");
} else {
enqueueResponse(secondWebServer, 404, "template [" + template + "] does not exist");
enqueueResponse(secondWebServer, 201, "template [" + template + "] created");
}
}
enqueueResponse(secondWebServer, 200, "{\"errors\": false, \"msg\": \"successful bulk request\"}");
logger.info("--> exporting a second event");
export(Collections.singletonList(newRandomMarvelDoc()));
assertThat(secondWebServer.getRequestCount(), equalTo(5));
assertThat(secondWebServer.getRequestCount(), equalTo(2 + monitoringTemplates().size() * 2 - 1));
recordedRequest = secondWebServer.takeRequest();
assertThat(recordedRequest.getMethod(), equalTo("GET"));
assertThat(recordedRequest.getPath(), equalTo("/"));
recordedRequest = secondWebServer.takeRequest();
assertThat(recordedRequest.getMethod(), equalTo("GET"));
assertThat(recordedRequest.getPath(), equalTo("/_template/" + indexTemplateName()));
for (Map.Entry<String, String> template : monitoringTemplates().entrySet()) {
recordedRequest = secondWebServer.takeRequest();
assertThat(recordedRequest.getMethod(), equalTo("GET"));
assertThat(recordedRequest.getPath(), equalTo("/_template/" + template.getKey()));
recordedRequest = secondWebServer.takeRequest();
assertThat(recordedRequest.getMethod(), equalTo("PUT"));
assertThat(recordedRequest.getPath(), equalTo("/_template/" + indexTemplateName()));
assertThat(recordedRequest.getBody().readByteArray(), equalTo(TIMESTAMPED_TEMPLATE));
recordedRequest = secondWebServer.takeRequest();
assertThat(recordedRequest.getMethod(), equalTo("GET"));
assertThat(recordedRequest.getPath(), equalTo("/_template/" + dataTemplateName()));;
if (template.getKey().contains(MonitoringBulkResolver.Data.DATA) == false) {
recordedRequest = secondWebServer.takeRequest();
assertThat(recordedRequest.getMethod(), equalTo("PUT"));
assertThat(recordedRequest.getPath(), equalTo("/_template/" + template.getKey()));
assertThat(recordedRequest.getBody().readUtf8(), equalTo(template.getValue()));
}
}
recordedRequest = secondWebServer.takeRequest();
assertThat(recordedRequest.getMethod(), equalTo("POST"));
@ -325,10 +312,10 @@ public class HttpExporterTests extends MarvelIntegTestCase {
logger.info("--> exporting a first event");
enqueueGetClusterVersionResponse(Version.CURRENT);
enqueueResponse(404, "template for timestamped indices does not exist");
enqueueResponse(201, "template for timestamped indices created");
enqueueResponse(404, "template for data index does not exist");
enqueueResponse(201, "template for data index created");
for (String template : monitoringTemplates().keySet()) {
enqueueResponse(404, "template [" + template + "] does not exist");
enqueueResponse(201, "template [" + template + "] created");
}
enqueueResponse(200, "{\"errors\": false, \"msg\": \"successful bulk request\"}");
HttpExporter exporter = getExporter(agentNode);
@ -336,29 +323,23 @@ public class HttpExporterTests extends MarvelIntegTestCase {
MonitoringDoc doc = newRandomMarvelDoc();
export(Collections.singletonList(doc));
assertThat(webServer.getRequestCount(), equalTo(6));
final int expectedRequests = 2 + monitoringTemplates().size() * 2;
assertThat(webServer.getRequestCount(), equalTo(expectedRequests));
RecordedRequest recordedRequest = webServer.takeRequest();
assertThat(recordedRequest.getMethod(), equalTo("GET"));
assertThat(recordedRequest.getPath(), equalTo("/"));
recordedRequest = webServer.takeRequest();
assertThat(recordedRequest.getMethod(), equalTo("GET"));
assertThat(recordedRequest.getPath(), equalTo("/_template/" + indexTemplateName()));
for (Map.Entry<String, String> template : monitoringTemplates().entrySet()) {
recordedRequest = webServer.takeRequest();
assertThat(recordedRequest.getMethod(), equalTo("GET"));
assertThat(recordedRequest.getPath(), equalTo("/_template/" + template.getKey()));
recordedRequest = webServer.takeRequest();
assertThat(recordedRequest.getMethod(), equalTo("PUT"));
assertThat(recordedRequest.getPath(), equalTo("/_template/" + indexTemplateName()));
assertThat(recordedRequest.getBody().readByteArray(), equalTo(TIMESTAMPED_TEMPLATE));
recordedRequest = webServer.takeRequest();
assertThat(recordedRequest.getMethod(), equalTo("GET"));
assertThat(recordedRequest.getPath(), equalTo("/_template/" + dataTemplateName()));
recordedRequest = webServer.takeRequest();
assertThat(recordedRequest.getMethod(), equalTo("PUT"));
assertThat(recordedRequest.getPath(), equalTo("/_template/" + dataTemplateName()));
assertThat(recordedRequest.getBody().readByteArray(), equalTo(DATA_TEMPLATE));
recordedRequest = webServer.takeRequest();
assertThat(recordedRequest.getMethod(), equalTo("PUT"));
assertThat(recordedRequest.getPath(), equalTo("/_template/" + template.getKey()));
assertThat(recordedRequest.getBody().readUtf8(), equalTo(template.getValue()));
}
recordedRequest = webServer.takeRequest();
assertThat(recordedRequest.getMethod(), equalTo("POST"));
@ -381,8 +362,9 @@ public class HttpExporterTests extends MarvelIntegTestCase {
logger.info("--> exporting a second event");
enqueueGetClusterVersionResponse(Version.CURRENT);
enqueueResponse(200, "template for timestamped indices exist");
enqueueResponse(200, "template for data index exist");
for (String template : monitoringTemplates().keySet()) {
enqueueResponse(200, "template [" + template + "] exist");
}
enqueueResponse(200, "{\"errors\": false, \"msg\": \"successful bulk request\"}");
doc = newRandomMarvelDoc();
@ -391,19 +373,17 @@ public class HttpExporterTests extends MarvelIntegTestCase {
String expectedMarvelIndex = ".monitoring-es-" + MarvelTemplateUtils.TEMPLATE_VERSION + "-"
+ DateTimeFormat.forPattern(newTimeFormat).withZoneUTC().print(doc.getTimestamp());
assertThat(webServer.getRequestCount(), equalTo(6 + 4));
assertThat(webServer.getRequestCount(), equalTo(expectedRequests + 2 + monitoringTemplates().size()));
recordedRequest = webServer.takeRequest();
assertThat(recordedRequest.getMethod(), equalTo("GET"));
assertThat(recordedRequest.getPath(), equalTo("/"));
recordedRequest = webServer.takeRequest();
assertThat(recordedRequest.getMethod(), equalTo("GET"));
assertThat(recordedRequest.getPath(), equalTo("/_template/" + indexTemplateName()));
recordedRequest = webServer.takeRequest();
assertThat(recordedRequest.getMethod(), equalTo("GET"));
assertThat(recordedRequest.getPath(), equalTo("/_template/" + dataTemplateName()));
for (String template : monitoringTemplates().keySet()) {
recordedRequest = webServer.takeRequest();
assertThat(recordedRequest.getMethod(), equalTo("GET"));
assertThat(recordedRequest.getPath(), equalTo("/_template/" + template));
}
recordedRequest = webServer.takeRequest();
assertThat(recordedRequest.getMethod(), equalTo("POST"));

7
elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/local/LocalExporterTests.java
View File

@ -36,12 +36,9 @@ import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
import static java.util.Collections.emptyMap;
import static java.util.Collections.emptySet;
import static org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils.dataTemplateName;
import static org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils.indexTemplateName;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.equalTo;
@ -51,7 +48,6 @@ import static org.hamcrest.Matchers.notNullValue;
@ClusterScope(scope = Scope.TEST, numDataNodes = 0, numClientNodes = 0, transportClientRatio = 0.0)
public class LocalExporterTests extends MarvelIntegTestCase {
private final static AtomicLong timeStampGenerator = new AtomicLong();
@Override
protected Settings nodeSettings(int nodeOrdinal) {
@ -108,8 +104,7 @@ public class LocalExporterTests extends MarvelIntegTestCase {
updateMarvelInterval(3L, TimeUnit.SECONDS);
// lets wait until the monitoring template will be installed
waitForMarvelTemplate(indexTemplateName());
waitForMarvelTemplate(dataTemplateName());
waitForMarvelTemplates();
}
public void testIndexTimestampFormat() throws Exception {

2
elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/resolver/bulk/MonitoringBulkResolverTests.java
View File

@ -56,7 +56,7 @@ public class MonitoringBulkResolverTests extends MonitoringIndexNameResolverTest
}
MonitoringBulkResolver resolver = newResolver();
assertThat(resolver.index(doc), equalTo(".monitoring-kibana-0-2015.07.22"));
assertThat(resolver.index(doc), equalTo(".monitoring-kibana-2-2015.07.22"));
assertThat(resolver.type(doc), equalTo(doc.getType()));
assertThat(resolver.id(doc), nullValue());

5
elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/resolver/cluster/ClusterInfoTests.java
View File

@ -23,8 +23,6 @@ import org.junit.Before;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import static org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils.dataTemplateName;
import static org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils.indexTemplateName;
import static org.elasticsearch.test.ESIntegTestCase.Scope.TEST;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
import static org.hamcrest.Matchers.equalTo;
@ -119,8 +117,7 @@ public class ClusterInfoTests extends MarvelIntegTestCase {
assertThat(clusterStats, instanceOf(Map.class));
assertThat(((Map) clusterStats).size(), greaterThan(0));
waitForMarvelTemplate(indexTemplateName());
waitForMarvelTemplate(dataTemplateName());
waitForMarvelTemplates();
logger.debug("--> check that the cluster_info is not indexed");
securedFlush();

16
elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/shield/MarvelInternalClientTests.java
View File

@ -11,11 +11,12 @@ import org.elasticsearch.common.network.NetworkModule;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.IndexNotFoundException;
import org.elasticsearch.marvel.MarvelSettings;
import org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils;
import org.elasticsearch.marvel.test.MarvelIntegTestCase;
import org.elasticsearch.rest.RestStatus;
import org.elasticsearch.shield.InternalClient;
import java.util.ArrayList;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
import static org.hamcrest.Matchers.is;
@ -48,15 +49,13 @@ public class MarvelInternalClientTests extends MarvelIntegTestCase {
assertAccessIsAllowed(internalClient.admin().indices().prepareDelete(MONITORING_INDICES_PREFIX + "*"));
assertAccessIsAllowed(internalClient.admin().indices().prepareCreate(MONITORING_INDICES_PREFIX + "test"));
assertAccessIsAllowed(internalClient.admin().indices().preparePutTemplate("foo")
.setSource(MarvelTemplateUtils.loadTimestampedIndexTemplate()));
assertAccessIsAllowed(internalClient.admin().indices().preparePutTemplate("foo").setSource(randomTemplateSource()));
assertAccessIsAllowed(internalClient.admin().indices().prepareGetTemplates("foo"));
}
public void testAllowAllAccess() {
InternalClient internalClient = internalCluster().getInstance(InternalClient.class);
assertAcked(internalClient.admin().indices().preparePutTemplate("foo")
.setSource(MarvelTemplateUtils.loadDataIndexTemplate()).get());
assertAcked(internalClient.admin().indices().preparePutTemplate("foo").setSource(randomTemplateSource()).get());
assertAccessIsAllowed(internalClient.admin().indices().prepareDeleteTemplate("foo"));
assertAccessIsAllowed(internalClient.admin().cluster().prepareGetRepositories());
@ -83,5 +82,12 @@ public class MarvelInternalClientTests extends MarvelIntegTestCase {
assertThat(e.status(), is(RestStatus.FORBIDDEN));
}
}
/**
* @return the source of a random monitoring template
*/
private String randomTemplateSource() {
return randomFrom(new ArrayList<>(monitoringTemplates().values()));
}
}

18
elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/test/MarvelIntegTestCase.java
View File

@ -20,9 +20,9 @@ import org.elasticsearch.index.IndexNotFoundException;
import org.elasticsearch.marvel.MarvelSettings;
import org.elasticsearch.marvel.MonitoredSystem;
import org.elasticsearch.marvel.agent.AgentService;
import org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils;
import org.elasticsearch.marvel.agent.exporter.MonitoringDoc;
import org.elasticsearch.marvel.agent.resolver.MonitoringIndexNameResolver;
import org.elasticsearch.marvel.agent.resolver.ResolversRegistry;
import org.elasticsearch.marvel.client.MonitoringClient;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.shield.authc.file.FileRealm;
@ -57,6 +57,8 @@ import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.StreamSupport;
import static org.elasticsearch.shield.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
@ -164,10 +166,7 @@ public abstract class MarvelIntegTestCase extends ESIntegTestCase {
@Override
protected Set<String> excludeTemplates() {
Set<String> templates = new HashSet<>();
templates.add(MarvelTemplateUtils.indexTemplateName());
templates.add(MarvelTemplateUtils.dataTemplateName());
return templates;
return monitoringTemplates().keySet();
}
@Before
@ -275,6 +274,11 @@ public abstract class MarvelIntegTestCase extends ESIntegTestCase {
}
}
protected Map<String, String> monitoringTemplates() {
return StreamSupport.stream(new ResolversRegistry(Settings.EMPTY).spliterator(), false)
.collect(Collectors.toMap(MonitoringIndexNameResolver::templateName, MonitoringIndexNameResolver::template, (a, b) -> a));
}
protected void assertTemplateInstalled(String name) {
boolean found = false;
for (IndexTemplateMetaData template : client().admin().indices().prepareGetTemplates().get().getIndexTemplates()) {
@ -294,6 +298,10 @@ public abstract class MarvelIntegTestCase extends ESIntegTestCase {
}, 30, TimeUnit.SECONDS);
}
protected void waitForMarvelTemplates() throws Exception {
assertBusy(() -> monitoringTemplates().keySet().forEach(this::assertTemplateInstalled), 30, TimeUnit.SECONDS);
}
protected void waitForMarvelIndices() throws Exception {
awaitIndexExists(MONITORING_INDICES_PREFIX + "*");
assertBusy(this::ensureMarvelIndicesYellow);

10
elasticsearch/x-pack/marvel/src/test/resources/monitoring-test.json Normal file
View File

@ -0,0 +1,10 @@
{
"template": ".monitoring-data-${monitoring.template.version}",
"mappings": {
"type_1": {
"_meta": {
"template.version": "${monitoring.template.version}"
}
}
}
}