diff --git a/elasticsearch/x-pack/build.gradle b/elasticsearch/x-pack/build.gradle index 279210b131e..c8b01a903fc 100644 --- a/elasticsearch/x-pack/build.gradle +++ b/elasticsearch/x-pack/build.gradle @@ -70,6 +70,9 @@ compileTestJava.options.compilerArgs << "-Xlint:-deprecation,-rawtypes,-serial,- ext.expansions = [ 'project.version': version, + // Used in marvel index templates + 'marvel.plugin.version': version, + 'marvel.template.version': '1', ] processResources { @@ -82,8 +85,8 @@ processResources { String licenseKeyPath = "license-plugin/keys/${licenseKeyName}/public.key" if (file(licenseKeyPath).exists() == false) { throw new GradleException("no public key found for '${licenseKeyName}'") - } - from licenseKeyPath + } + from licenseKeyPath } processTestResources { diff --git a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/collector/AbstractCollector.java b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/collector/AbstractCollector.java index 76c44e28fc6..734520134ef 100644 --- a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/collector/AbstractCollector.java +++ b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/collector/AbstractCollector.java @@ -11,11 +11,14 @@ import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.common.component.AbstractLifecycleComponent; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.marvel.agent.exporter.IndexNameResolver; import org.elasticsearch.marvel.agent.exporter.MarvelDoc; +import org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils; import org.elasticsearch.marvel.agent.settings.MarvelSettings; import org.elasticsearch.marvel.license.MarvelLicensee; import java.util.Collection; +import java.util.Objects; public abstract class AbstractCollector extends AbstractLifecycleComponent implements Collector { @@ -24,6 +27,7 @@ public abstract class AbstractCollector extends AbstractLifecycleComponent protected final ClusterService clusterService; protected final MarvelSettings marvelSettings; protected final MarvelLicensee licensee; + protected final IndexNameResolver dataIndexNameResolver; @Inject public AbstractCollector(Settings settings, String name, ClusterService clusterService, MarvelSettings marvelSettings, MarvelLicensee licensee) { @@ -32,6 +36,7 @@ public abstract class AbstractCollector extends AbstractLifecycleComponent this.clusterService = clusterService; this.marvelSettings = marvelSettings; this.licensee = licensee; + this.dataIndexNameResolver = new DataIndexNameResolver(MarvelTemplateUtils.TEMPLATE_VERSION); } @Override @@ -109,4 +114,27 @@ public abstract class AbstractCollector extends AbstractLifecycleComponent protected String clusterUUID() { return clusterService.state().metaData().clusterUUID(); } + + /** + * Resolves marvel's data index name + */ + public class DataIndexNameResolver implements IndexNameResolver { + + private final String index; + + DataIndexNameResolver(Integer version) { + Objects.requireNonNull(version, "index version cannot be null"); + this.index = MarvelSettings.MARVEL_DATA_INDEX_PREFIX + String.valueOf(version); + } + + @Override + public String resolve(MarvelDoc doc) { + return index; + } + + @Override + public String resolve(long timestamp) { + return index; + } + } } \ No newline at end of file diff --git a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/collector/cluster/ClusterInfoCollector.java b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/collector/cluster/ClusterInfoCollector.java index 6e2d19d1a6e..4d8457cbab8 100644 --- a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/collector/cluster/ClusterInfoCollector.java +++ b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/collector/cluster/ClusterInfoCollector.java @@ -77,7 +77,8 @@ public class ClusterInfoCollector extends AbstractCollector= 0 : "version must be not null and greater or equal to zero"; + return prefix + String.valueOf(version); } } diff --git a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/http/HttpExporter.java b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/http/HttpExporter.java index 041a3b437cf..21a279313a3 100644 --- a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/http/HttpExporter.java +++ b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/http/HttpExporter.java @@ -8,11 +8,9 @@ package org.elasticsearch.marvel.agent.exporter.http; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.Version; -import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest; import org.elasticsearch.common.Base64; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.io.Streams; import org.elasticsearch.common.io.stream.BytesStreamOutput; @@ -20,7 +18,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsException; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.env.Environment; @@ -55,12 +52,7 @@ import java.nio.file.Path; import java.security.KeyStore; import java.util.ArrayList; import java.util.Collection; -import java.util.HashSet; import java.util.Map; -import java.util.Set; - -import static org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils.installedTemplateVersionIsSufficient; -import static org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils.installedTemplateVersionMandatesAnUpdate; /** * @@ -106,8 +98,8 @@ public class HttpExporter extends Exporter { volatile boolean checkedAndUploadedIndexTemplate = false; volatile boolean supportedClusterVersion = false; - /** Version of the built-in template **/ - final Version templateVersion; + /** Version number of built-in templates **/ + private final Integer templateVersion; boolean keepAlive; final ConnectionKeepAliveWorker keepAliveWorker; @@ -142,8 +134,8 @@ public class HttpExporter extends Exporter { sslSocketFactory = createSSLSocketFactory(config.settings().getAsSettings(SSL_SETTING)); hostnameVerification = config.settings().getAsBoolean(SSL_HOSTNAME_VERIFICATION_SETTING, true); - // Checks that the built-in template is versioned - templateVersion = MarvelTemplateUtils.loadDefaultTemplateVersion(); + // Loads the current version number of built-in templates + templateVersion = MarvelTemplateUtils.TEMPLATE_VERSION; if (templateVersion == null) { throw new IllegalStateException("unable to find built-in template version"); } @@ -202,7 +194,7 @@ public class HttpExporter extends Exporter { builder.startObject(); builder.startObject("index"); - // we need the index to be based on the document timestamp + // we need the index to be based on the document timestamp and/or template version builder.field("_index", indexNameResolver.resolve(marvelDoc)); if (marvelDoc.type() != null) { @@ -395,101 +387,51 @@ public class HttpExporter extends Exporter { * @return true if template exists or was uploaded successfully. */ private boolean checkAndUploadIndexTemplate(final String host) { - byte[] installedTemplate; - try { - installedTemplate = findMarvelTemplate(host); - } catch (Exception e) { - logger.debug("http exporter [{}] - exception when loading the existing marvel template on host[{}]", e, name(), host); - return false; + String templateName = MarvelTemplateUtils.indexTemplateName(templateVersion); + boolean templateInstalled = hasTemplate(templateName, host); + + // Works like LocalExporter on master: + // Install the index template for timestamped indices first, so that other nodes can ship data + if (!templateInstalled) { + logger.debug("http exporter [{}] - could not find existing marvel template, installing a new one", name()); + if (!putTemplate(host, templateName, MarvelTemplateUtils.loadTimestampedIndexTemplate())) { + return false; + } } - // if we cannot find a template or a compatible template, we'll install one in / update it. - if (installedTemplate == null) { - logger.debug("http exporter [{}] - could not find existing marvel template, installing a new one", name()); - return putTemplate(host); - } - Version installedTemplateVersion = MarvelTemplateUtils.parseTemplateVersion(installedTemplate); - if (installedTemplateVersionMandatesAnUpdate(templateVersion, installedTemplateVersion, logger, name())) { - logger.debug("http exporter [{}] - installing new marvel template [{}], replacing [{}]", name(), templateVersion, installedTemplateVersion); - return putTemplate(host); - } else if (!installedTemplateVersionIsSufficient(installedTemplateVersion)) { - logger.error("http exporter [{}] - marvel template version [{}] is below the minimum compatible version [{}]. " - + "please manually update the marvel template to a more recent version" - + "and delete the current active marvel index (don't forget to back up it first if needed)", - name(), installedTemplateVersion, MarvelTemplateUtils.MIN_SUPPORTED_TEMPLATE_VERSION); - // we're not going to do anything with the template.. it's too old, and the schema might - // be too different than what this version of marvel/es can work with. For this reason we're - // not going to export any data, to avoid mapping conflicts. - return false; + // Install the index template for data index + templateName = MarvelTemplateUtils.dataTemplateName(templateVersion); + if (!hasTemplate(templateName, host)) { + logger.debug("http exporter [{}] - could not find existing marvel template for data index, installing a new one", name()); + if (!putTemplate(host, templateName, MarvelTemplateUtils.loadDataIndexTemplate())) { + return false; + } } return true; } - private byte[] findMarvelTemplate(String host) throws IOException { - String url = "_template/" + MarvelTemplateUtils.INDEX_TEMPLATE_NAME; + private boolean hasTemplate(String templateName, String host) { + String url = "_template/" + templateName; if (templateCheckTimeout != null) { url += "?timeout=" + templateCheckTimeout; } HttpURLConnection connection = null; try { - logger.debug("http exporter [{}] - checking if marvel template exists on the marvel cluster", name()); + logger.debug("http exporter [{}] - checking if marvel template [{}] exists on the marvel cluster", name(), templateName); connection = openConnection(host, "GET", url, null); if (connection == null) { - throw new IOException("no available connection to check marvel template existence"); + throw new IOException("no available connection to check for marvel template [" + templateName + "] existence"); } - byte[] remoteTemplate = null; - // 200 means that the template has been found, 404 otherwise if (connection.getResponseCode() == 200) { - logger.debug("marvel template found"); - - try (InputStream is = connection.getInputStream()) { - ByteArrayOutputStream out = new ByteArrayOutputStream(); - Streams.copy(is, out); - remoteTemplate = out.toByteArray(); - } + logger.debug("marvel template [{}] found",templateName); + return true; } - return remoteTemplate; } catch (Exception e) { - logger.error("http exporter [{}] - failed to verify the marvel template to [{}]:\n{}", name(), host, e.getMessage()); - throw e; - } finally { - if (connection != null) { - try { - connection.getInputStream().close(); - } catch (IOException e) { - // Ignore - } - } - } - } - - boolean putTemplate(String host) { - HttpURLConnection connection = null; - try { - connection = openConnection(host, "PUT", "_template/" + MarvelTemplateUtils.INDEX_TEMPLATE_NAME, XContentType.JSON.restContentType()); - if (connection == null) { - logger.debug("http exporter [{}] - no available connection to update marvel template", name()); - return false; - } - - logger.debug("http exporter [{}] - loading marvel pre-configured template", name()); - byte[] template = MarvelTemplateUtils.loadDefaultTemplate(); - - // Uploads the template and closes the outputstream - Streams.copy(template, connection.getOutputStream()); - if (connection.getResponseCode() != 200 && connection.getResponseCode() != 201) { - logConnectionError("error adding the marvel template to [" + host + "]", connection); - return false; - } - - logger.info("http exporter [{}] - marvel template updated to version [{}]", name(), templateVersion); - } catch (IOException e) { - logger.error("http exporter [{}] - failed to update the marvel template to [{}]:\n{}", name(), host, e.getMessage()); + logger.error("http exporter [{}] - failed to verify the marvel template [{}] on [{}]:\n{}", name(), templateName, host, e.getMessage()); return false; - } finally { if (connection != null) { try { @@ -499,101 +441,31 @@ public class HttpExporter extends Exporter { } } } - - if (config.settings().getAsBoolean("update_mappings", true)) { - updateMappings(host, MarvelSettings.MARVEL_DATA_INDEX_NAME); - updateMappings(host, indexNameResolver().resolve(System.currentTimeMillis())); - } - return true; + return false; } - // TODO: Remove this method once marvel indices are versioned (v 2.2.0) - void updateMappings(String host, String index) { - logger.trace("http exporter [{}] - updating mappings for index [{}]", name(), index); - - // Parse the default template to get its mappings - PutIndexTemplateRequest template = new PutIndexTemplateRequest().source(MarvelTemplateUtils.loadDefaultTemplate()); - if ((template == null) || (template.mappings() == null) || (template.mappings().isEmpty())) { - return; - } - - Set indexMappings = new HashSet<>(); - + boolean putTemplate(String host, String template, byte[] source) { + logger.debug("http exporter [{}] - installing template [{}]", name(), template); HttpURLConnection connection = null; try { - connection = openConnection(host, "GET", "/" + index + "/_mapping", XContentType.JSON.restContentType()); + connection = openConnection(host, "PUT", "_template/" + template, XContentType.JSON.restContentType()); if (connection == null) { - logger.debug("http exporter [{}] - no available connection to get index mappings", name()); - return; - } - - if (connection.getResponseCode() == 404) { - logger.trace("http exporter [{}] - index [{}] does not exist", name(), index); - return; - } else if (connection.getResponseCode() == 200) { - try (InputStream is = connection.getInputStream()) { - ByteArrayOutputStream out = new ByteArrayOutputStream(); - Streams.copy(is, out); - - Map mappings = XContentHelper.convertToMap(new BytesArray(out.toByteArray()), false).v2(); - if ((mappings.get(index) != null) && (mappings.get(index) instanceof Map)) { - Map m = (Map) ((Map) mappings.get(index)).get("mappings"); - if (m != null) { - indexMappings = m.keySet(); - } - } - } - } else { - logConnectionError("http exporter [" + name() +"] - failed to get mappings for index [" + index + "] on host [" + host + "]", connection); - return; - } - } catch (Exception e) { - logger.error("http exporter [{}] - failed to update the marvel template to [{}]:\n{}", name(), host, e.getMessage()); - return; - - } finally { - if (connection != null) { - try { - connection.getInputStream().close(); - } catch (IOException e) { - // Ignore - } - } - } - - // Iterates over document types defined in the default template - for (String type : template.mappings().keySet()) { - if (indexMappings.contains(type)) { - logger.trace("http exporter [{}] - type [{} already exists in mapping of index [{}]", name(), type, index); - continue; - } - - logger.trace("http exporter [{}] - adding type [{}] to index [{}] mappings", name(), type, index); - updateMappingForType(host, index, type, template.mappings().get(type)); - } - } - - void updateMappingForType(String host, String index, String type, String mappingSource) { - logger.trace("http exporter [{}] - updating index [{}] mappings for type [{}] on host [{}]", name(), index, type, host); - HttpURLConnection connection = null; - try { - connection = openConnection(host, "PUT", "/" + index + "/_mapping/" + type, XContentType.JSON.restContentType()); - if (connection == null) { - logger.debug("http exporter [{}] - no available connection to update index mapping", name()); - return; + logger.debug("http exporter [{}] - no available connection to update marvel template [{}]", name(), template); + return false; } // Uploads the template and closes the outputstream - Streams.copy(Strings.toUTF8Bytes(mappingSource), connection.getOutputStream()); + Streams.copy(source, connection.getOutputStream()); if (connection.getResponseCode() != 200 && connection.getResponseCode() != 201) { - logConnectionError("http exporter [" + name() +"] - mapping of index [" + index + "] failed to be updated for type [" + type + "] on host [" + host + "]", connection); - return; + logConnectionError("error adding the marvel template [" + template + "] to [" + host + "]", connection); + return false; } - logger.trace("http exporter [{}] - mapping of index [{}] updated for type [{}]", name(), index, type); - } catch (Exception e) { - logger.error("http exporter [{}] - failed to update mapping of index [{}] for type [{}]", name(), index, type); - + logger.info("http exporter [{}] - marvel template [{}] updated to version [{}]", name(), template, templateVersion); + return true; + } catch (IOException e) { + logger.error("http exporter [{}] - failed to update marvel template [{}] on host [{}]:\n{}", name(), template, host, e.getMessage()); + return false; } finally { if (connection != null) { try { diff --git a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/local/LocalBulk.java b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/local/LocalBulk.java index 6482894ca04..730b0cc7afd 100644 --- a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/local/LocalBulk.java +++ b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/local/LocalBulk.java @@ -60,11 +60,10 @@ public class LocalBulk extends ExportBulk { } IndexRequestBuilder request = client.prepareIndex(); - if (marvelDoc.index() != null) { - request.setIndex(marvelDoc.index()); - } else { - request.setIndex(indexNameResolver.resolve(marvelDoc)); - } + + // we need the index to be based on the document timestamp and/or template version + request.setIndex(indexNameResolver.resolve(marvelDoc)); + if (marvelDoc.type() != null) { request.setType(marvelDoc.type()); } diff --git a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/local/LocalExporter.java b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/local/LocalExporter.java index e275f28d459..c1e843b2ee5 100644 --- a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/local/LocalExporter.java +++ b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/local/LocalExporter.java @@ -5,12 +5,8 @@ */ package org.elasticsearch.marvel.agent.exporter.local; -import org.elasticsearch.Version; +import com.carrotsearch.hppc.cursors.ObjectCursor; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest; -import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; -import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; -import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse; import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest; import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateResponse; import org.elasticsearch.client.Client; @@ -18,22 +14,17 @@ import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterStateListener; -import org.elasticsearch.cluster.metadata.IndexTemplateMetaData; -import org.elasticsearch.cluster.metadata.MappingMetaData; +import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.regex.Regex; import org.elasticsearch.gateway.GatewayService; -import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.marvel.agent.exporter.ExportBulk; import org.elasticsearch.marvel.agent.exporter.Exporter; import org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils; import org.elasticsearch.marvel.agent.renderer.RendererRegistry; -import org.elasticsearch.marvel.agent.settings.MarvelSettings; import org.elasticsearch.marvel.shield.SecuredClient; -import static org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils.installedTemplateVersionIsSufficient; -import static org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils.installedTemplateVersionMandatesAnUpdate; - /** * */ @@ -48,8 +39,8 @@ public class LocalExporter extends Exporter implements ClusterStateListener { private volatile LocalBulk bulk; private volatile boolean active = true; - /** Version of the built-in template **/ - private final Version templateVersion; + /** Version number of built-in templates **/ + private final Integer templateVersion; public LocalExporter(Exporter.Config config, Client client, ClusterService clusterService, RendererRegistry renderers) { super(TYPE, config); @@ -57,8 +48,8 @@ public class LocalExporter extends Exporter implements ClusterStateListener { this.clusterService = clusterService; this.renderers = renderers; - // Checks that the built-in template is versioned - templateVersion = MarvelTemplateUtils.loadDefaultTemplateVersion(); + // Loads the current version number of built-in templates + templateVersion = MarvelTemplateUtils.TEMPLATE_VERSION; if (templateVersion == null) { throw new IllegalStateException("unable to find built-in template version"); } @@ -126,21 +117,17 @@ public class LocalExporter extends Exporter implements ClusterStateListener { return null; } - IndexTemplateMetaData installedTemplate = MarvelTemplateUtils.findMarvelTemplate(clusterState); + String templateName = MarvelTemplateUtils.indexTemplateName(templateVersion); + boolean templateInstalled = hasTemplate(templateName, clusterState); - // if this is not the master, we'll just look to see if the marvel template is already + // if this is not the master, we'll just look to see if the marvel timestamped template is already // installed and if so, if it has a compatible version. If it is (installed and compatible) // we'll be able to start this exporter. Otherwise, we'll just wait for a new cluster state. if (!clusterService.localNode().masterNode()) { - if (installedTemplate == null) { - // the marvel template is not yet installed in the given cluster state, we'll wait. - logger.debug("local exporter [{}] - marvel index template [{}] does not exist, so service cannot start", name(), MarvelTemplateUtils.INDEX_TEMPLATE_NAME); - return null; - } - Version installedTemplateVersion = MarvelTemplateUtils.templateVersion(installedTemplate); - if (!installedTemplateVersionIsSufficient(installedTemplateVersion)) { - logger.debug("local exporter [{}] - cannot start. the currently installed marvel template (version [{}]) is incompatible with the " + - "current elasticsearch version [{}]. waiting until the template is updated", name(), installedTemplateVersion, Version.CURRENT); + // We only need to check the index template for timestamped indices + if (!templateInstalled) { + // the template for timestamped indices is not yet installed in the given cluster state, we'll wait. + logger.debug("local exporter [{}] - marvel index template does not exist, so service cannot start", name()); return null; } @@ -151,36 +138,65 @@ public class LocalExporter extends Exporter implements ClusterStateListener { // we are on master // - // if we cannot find a template or a compatible template, we'll install one in / update it. - if (installedTemplate == null) { - logger.debug("local exporter [{}] - could not find existing marvel template, installing a new one", name()); - putTemplate(); - // we'll get that template on the next cluster state update - return null; - } - Version installedTemplateVersion = MarvelTemplateUtils.templateVersion(installedTemplate); - if (installedTemplateVersionMandatesAnUpdate(templateVersion, installedTemplateVersion, logger, name())) { - logger.debug("local exporter [{}] - installing new marvel template [{}], replacing [{}]", name(), templateVersion, installedTemplateVersion); - putTemplate(); - // we'll get that template on the next cluster state update - return null; - } else if (!installedTemplateVersionIsSufficient(installedTemplateVersion)) { - logger.error("local exporter [{}] - marvel template version [{}] is below the minimum compatible version [{}]. " - + "please manually update the marvel template to a more recent version" - + "and delete the current active marvel index (don't forget to back up it first if needed)", - name(), installedTemplateVersion, MarvelTemplateUtils.MIN_SUPPORTED_TEMPLATE_VERSION); - // we're not going to do anything with the template.. it's too old, and the schema might - // be too different than what this version of marvel/es can work with. For this reason we're - // not going to export any data, to avoid mapping conflicts. + // Check that there is nothing that could block metadata updates + if (clusterState.blocks().hasGlobalBlock(ClusterBlockLevel.METADATA_WRITE)) { + logger.debug("local exporter [{}] - waiting until metadata writes are unblocked", name()); return null; } - // ok.. we have a compatible template... we can start + // Install the index template for timestamped indices first, so that other nodes can ship data + if (!templateInstalled) { + logger.debug("local exporter [{}] - could not find existing marvel template for timestamped indices, installing a new one", name()); + putTemplate(templateName, MarvelTemplateUtils.loadTimestampedIndexTemplate()); + // we'll get that template on the next cluster state update + return null; + } + + // Install the index template for data index + templateName = MarvelTemplateUtils.dataTemplateName(templateVersion); + if (!hasTemplate(templateName, clusterState)) { + logger.debug("local exporter [{}] - could not find existing marvel template for data index, installing a new one", name()); + putTemplate(templateName, MarvelTemplateUtils.loadDataIndexTemplate()); + // we'll get that template on the next cluster state update + return null; + } + + // ok.. we have a compatible templates... we can start return currentBulk != null ? currentBulk : new LocalBulk(name(), logger, client, indexNameResolver, renderers); } - void putTemplate() { - PutIndexTemplateRequest request = new PutIndexTemplateRequest(MarvelTemplateUtils.INDEX_TEMPLATE_NAME).source(MarvelTemplateUtils.loadDefaultTemplate()); + /** + * List templates that exists in cluster state metadata and that match a given template name pattern. + */ + private ImmutableOpenMap findTemplates(String templatePattern, ClusterState state) { + if (state == null || state.getMetaData() == null || state.getMetaData().getTemplates().isEmpty()) { + return ImmutableOpenMap.of(); + } + + ImmutableOpenMap.Builder templates = ImmutableOpenMap.builder(); + for (ObjectCursor template : state.metaData().templates().keys()) { + if (Regex.simpleMatch(templatePattern, template.value)) { + try { + Integer version = Integer.parseInt(template.value.substring(templatePattern.length() - 1)); + templates.put(template.value, version); + logger.debug("found index template [{}] in version [{}]", template, version); + } catch (NumberFormatException e) { + logger.warn("cannot extract version number for template [{}]", template.value); + } + } + } + return templates.build(); + } + + private boolean hasTemplate(String templateName, ClusterState state) { + ImmutableOpenMap templates = findTemplates(templateName, state); + return templates.size() > 0; + } + + void putTemplate(String template, byte[] source) { + logger.debug("local exporter [{}] - installing template [{}]", name(), template); + + PutIndexTemplateRequest request = new PutIndexTemplateRequest(template).source(source); assert !Thread.currentThread().isInterrupted() : "current thread has been interrupted before putting index template!!!"; // async call, so we won't block cluster event thread @@ -188,80 +204,15 @@ public class LocalExporter extends Exporter implements ClusterStateListener { @Override public void onResponse(PutIndexTemplateResponse response) { if (response.isAcknowledged()) { - logger.trace("local exporter [{}] - successfully installed marvel template", name()); - - if (config.settings().getAsBoolean("update_mappings", true)) { - updateMappings(MarvelSettings.MARVEL_DATA_INDEX_NAME); - updateMappings(indexNameResolver().resolve(System.currentTimeMillis())); - } + logger.trace("local exporter [{}] - successfully installed marvel template [{}]", name(), template); } else { - logger.error("local exporter [{}] - failed to update marvel index template", name()); + logger.error("local exporter [{}] - failed to update marvel index template [{}]", name(), template); } } @Override public void onFailure(Throwable throwable) { - logger.error("local exporter [{}] - failed to update marvel index template", throwable, name()); - } - }); - } - - // TODO: Remove this method once marvel indices are versioned (v 2.2.0) - void updateMappings(String index) { - logger.trace("local exporter [{}] - updating mappings for index [{}]", name(), index); - - // Parse the default template to get its mappings - PutIndexTemplateRequest template = new PutIndexTemplateRequest().source(MarvelTemplateUtils.loadDefaultTemplate()); - if ((template == null) || (template.mappings() == null) || (template.mappings().isEmpty())) { - return; - } - - // async call, so we won't block cluster event thread - client.admin().indices().getMappings(new GetMappingsRequest().indices(index), new ActionListener() { - @Override - public void onResponse(GetMappingsResponse response) { - ImmutableOpenMap indexMappings = response.getMappings().get(index); - if (indexMappings != null) { - - // Iterates over document types defined in the default template - for (String type : template.mappings().keySet()) { - if (indexMappings.get(type) != null) { - logger.trace("local exporter [{}] - type [{} already exists in mapping of index [{}]", name(), type, index); - continue; - } - - logger.trace("local exporter [{}] - adding type [{}] to index [{}] mappings", name(), type, index); - updateMappingForType(index, type, template.mappings().get(type)); - } - } - } - - @Override - public void onFailure(Throwable e) { - if (e instanceof IndexNotFoundException) { - logger.trace("local exporter [{}] - index [{}] not found, unable to update mappings", name(), index); - } else { - logger.error("local exporter [{}] - failed to get mappings for index [{}]", name(), index); - } - } - }); - } - - void updateMappingForType(String index, String type, String mappingSource) { - logger.trace("local exporter [{}] - updating index [{}] mappings for type [{}]", name(), index, type); - client.admin().indices().putMapping(new PutMappingRequest(index).type(type).source(mappingSource), new ActionListener() { - @Override - public void onResponse(PutMappingResponse response) { - if (response.isAcknowledged()) { - logger.trace("local exporter [{}] - mapping of index [{}] updated for type [{}]", name(), index, type); - } else { - logger.trace("local exporter [{}] - mapping of index [{}] failed to be updated for type [{}]", name(), index, type); - } - } - - @Override - public void onFailure(Throwable e) { - logger.error("local exporter [{}] - failed to update mapping of index [{}] for type [{}]", name(), index, type); + logger.error("local exporter [{}] - failed to update marvel index template [{}]", throwable, name(), template); } }); } diff --git a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/settings/MarvelSettings.java b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/settings/MarvelSettings.java index e142f4c246c..5ed5b23042c 100644 --- a/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/settings/MarvelSettings.java +++ b/elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/settings/MarvelSettings.java @@ -22,7 +22,7 @@ public class MarvelSettings extends AbstractComponent { private static final String PREFIX = MarvelPlugin.NAME + ".agent."; public static final String MARVEL_INDICES_PREFIX = ".marvel-es-"; - public static final String MARVEL_DATA_INDEX_NAME = MARVEL_INDICES_PREFIX + "data"; + public static final String MARVEL_DATA_INDEX_PREFIX = MARVEL_INDICES_PREFIX + "data-"; public static final TimeValue MAX_LICENSE_GRACE_PERIOD = TimeValue.timeValueHours(7 * 24); /** Sampling interval between two collections (default to 10s) */ diff --git a/elasticsearch/x-pack/marvel/src/main/resources/marvel-es-data.json b/elasticsearch/x-pack/marvel/src/main/resources/marvel-es-data.json new file mode 100644 index 00000000000..2989eb7de0a --- /dev/null +++ b/elasticsearch/x-pack/marvel/src/main/resources/marvel-es-data.json @@ -0,0 +1,25 @@ +{ + "template": ".marvel-es-data-${marvel.template.version}", + "settings": { + "marvel.plugin.version": "${marvel.plugin.version}", + "marvel.template.version": "${marvel.template.version}", + "index.number_of_shards": 1, + "index.number_of_replicas": 1, + "index.codec": "best_compression", + "index.mapper.dynamic": false + }, + "mappings": { + "_default_": { + "_all": { + "enabled": false + }, + "date_detection": false + }, + "cluster_info": { + "enabled": false + }, + "node": { + "enabled": false + } + } +} \ No newline at end of file diff --git a/elasticsearch/x-pack/marvel/src/main/resources/marvel_index_template.json b/elasticsearch/x-pack/marvel/src/main/resources/marvel-es.json similarity index 97% rename from elasticsearch/x-pack/marvel/src/main/resources/marvel_index_template.json rename to elasticsearch/x-pack/marvel/src/main/resources/marvel-es.json index 59ba3c3c915..9944c4c28ba 100644 --- a/elasticsearch/x-pack/marvel/src/main/resources/marvel_index_template.json +++ b/elasticsearch/x-pack/marvel/src/main/resources/marvel-es.json @@ -1,7 +1,8 @@ { - "template": ".marvel-es-*", + "template": ".marvel-es-${marvel.template.version}-*", "settings": { - "marvel_version": "${project.version}", + "marvel.plugin.version": "${marvel.plugin.version}", + "marvel.template.version": "${marvel.template.version}", "index.number_of_shards": 1, "index.number_of_replicas": 1, "index.codec": "best_compression", @@ -196,12 +197,6 @@ } } }, - "cluster_info": { - "enabled": false - }, - "node": { - "enabled": false - }, "nodes": { "properties": { "state_uuid": { diff --git a/elasticsearch/x-pack/marvel/src/main/resources/marvel.properties b/elasticsearch/x-pack/marvel/src/main/resources/marvel.properties new file mode 100644 index 00000000000..c37445e2a8b --- /dev/null +++ b/elasticsearch/x-pack/marvel/src/main/resources/marvel.properties @@ -0,0 +1 @@ +marvel.template.version=${marvel.template.version} diff --git a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/collector/cluster/ClusterStateCollectorTests.java b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/collector/cluster/ClusterStateCollectorTests.java index 24e897a6fd9..d21af630df8 100644 --- a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/collector/cluster/ClusterStateCollectorTests.java +++ b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/collector/cluster/ClusterStateCollectorTests.java @@ -11,6 +11,7 @@ import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.marvel.agent.collector.AbstractCollectorTestCase; import org.elasticsearch.marvel.agent.exporter.MarvelDoc; +import org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils; import org.elasticsearch.marvel.agent.settings.MarvelSettings; import org.elasticsearch.marvel.license.MarvelLicensee; @@ -182,7 +183,7 @@ public class ClusterStateCollectorTests extends AbstractCollectorTestCase { case ClusterStateCollector.NODE_TYPE: DiscoveryNodeMarvelDoc discoveryNodeMarvelDoc = (DiscoveryNodeMarvelDoc) marvelDoc; - assertThat(discoveryNodeMarvelDoc.index(), equalTo(MarvelSettings.MARVEL_DATA_INDEX_NAME)); + assertThat(discoveryNodeMarvelDoc.index(), equalTo(MarvelSettings.MARVEL_DATA_INDEX_PREFIX + MarvelTemplateUtils.TEMPLATE_VERSION)); assertThat(discoveryNodeMarvelDoc.id(), not(isEmptyOrNullString())); assertNotNull(discoveryNodeMarvelDoc.getNode()); discoveryNodes.add(discoveryNodeMarvelDoc); diff --git a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/AbstractExporterTemplateTestCase.java b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/AbstractExporterTemplateTestCase.java index da8f7be7fed..b8646bd3cf9 100644 --- a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/AbstractExporterTemplateTestCase.java +++ b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/AbstractExporterTemplateTestCase.java @@ -5,7 +5,6 @@ */ package org.elasticsearch.marvel.agent.exporter; -import org.elasticsearch.Version; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.marvel.agent.collector.Collector; @@ -14,9 +13,6 @@ import org.elasticsearch.marvel.agent.collector.node.NodeStatsCollector; import org.elasticsearch.marvel.agent.settings.MarvelSettings; import org.elasticsearch.marvel.test.MarvelIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; -import org.elasticsearch.test.VersionUtils; -import org.joda.time.DateTime; -import org.joda.time.DateTimeZone; import java.io.IOException; import java.util.ArrayList; @@ -25,11 +21,15 @@ import java.util.List; import java.util.Map; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils.dataTemplateName; +import static org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils.indexTemplateName; import static org.elasticsearch.test.ESIntegTestCase.Scope.TEST; @ClusterScope(scope = TEST, numDataNodes = 0, numClientNodes = 0, transportClientRatio = 0.0) public abstract class AbstractExporterTemplateTestCase extends MarvelIntegTestCase { + private final Integer currentVersion = MarvelTemplateUtils.TEMPLATE_VERSION; + @Override protected Settings nodeSettings(int nodeOrdinal) { Settings.Builder settings = Settings.builder() @@ -44,149 +44,116 @@ public abstract class AbstractExporterTemplateTestCase extends MarvelIntegTestCa protected abstract Settings exporterSettings(); - protected abstract void deleteTemplate() throws Exception; + protected abstract void deleteTemplates() throws Exception; - protected abstract void putTemplate(String version) throws Exception; + protected abstract void putTemplate(String name, int version) throws Exception; - protected abstract void createMarvelIndex(String index) throws Exception; + protected abstract void assertTemplateExist(String name) throws Exception; - protected abstract void assertTemplateUpdated(Version version) throws Exception; + protected abstract void assertTemplateNotUpdated(String name) throws Exception; - protected abstract void assertTemplateNotUpdated(Version version) throws Exception; - - protected abstract void assertMappingsUpdated(String... indices) throws Exception; - - protected abstract void assertMappingsNotUpdated(String... indices) throws Exception; - - protected abstract void assertIndicesNotCreated() throws Exception; - - public void testCreateWhenNoExistingTemplate() throws Exception { + public void testCreateWhenNoExistingTemplates() throws Exception { internalCluster().startNode(); - deleteTemplate(); + deleteTemplates(); doExporting(); - logger.debug("--> template does not exist: it should have been created in the current version"); - assertTemplateUpdated(currentVersion()); + logger.debug("--> templates does not exist: it should have been created in the current version"); + assertTemplateExist(indexTemplateName()); + assertTemplateExist(dataTemplateName()); doExporting(); - logger.debug("--> mappings should be up-to-date"); - assertMappingsUpdated(currentIndices()); + logger.debug("--> indices should have been created"); + awaitIndexExists(currentDataIndexName()); + awaitIndexExists(currentTimestampedIndexName()); } - public void testUpdateWhenExistingTemplateHasNoVersion() throws Exception { + public void testCreateWhenExistingTemplatesAreOld() throws Exception { internalCluster().startNode(); - putTemplate(""); - doExporting(); - - logger.debug("--> existing template does not have a version: it should be updated to the current version"); - assertTemplateUpdated(currentVersion()); + final Integer version = randomIntBetween(0, currentVersion - 1); + putTemplate(indexTemplateName(version), version); + putTemplate(dataTemplateName(version), version); doExporting(); - logger.debug("--> mappings should be up-to-date"); - assertMappingsUpdated(currentIndices()); + logger.debug("--> existing templates are old"); + assertTemplateExist(dataTemplateName(version)); + assertTemplateExist(indexTemplateName(version)); + + logger.debug("--> existing templates are old: new templates should be created"); + assertTemplateExist(indexTemplateName()); + assertTemplateExist(dataTemplateName()); + + doExporting(); + + logger.debug("--> indices should have been created"); + awaitIndexExists(currentDataIndexName()); + awaitIndexExists(currentTimestampedIndexName()); } - public void testUpdateWhenExistingTemplateHasWrongVersion() throws Exception { + public void testCreateWhenExistingTemplateAreUpToDate() throws Exception { internalCluster().startNode(); - putTemplate(randomAsciiOfLength(5)); - doExporting(); - - logger.debug("--> existing template has a wrong version: it should be updated to the current version"); - assertTemplateUpdated(currentVersion()); + putTemplate(indexTemplateName(currentVersion), currentVersion); + putTemplate(dataTemplateName(currentVersion), currentVersion); doExporting(); - logger.debug("--> mappings should be up-to-date"); - assertMappingsUpdated(currentIndices()); + logger.debug("--> existing templates are up to date"); + assertTemplateExist(indexTemplateName()); + assertTemplateExist(dataTemplateName()); + + logger.debug("--> existing templates has the same version: they should not be changed"); + assertTemplateNotUpdated(indexTemplateName()); + assertTemplateNotUpdated(dataTemplateName()); + + doExporting(); + + logger.debug("--> indices should have been created"); + awaitIndexExists(currentDataIndexName()); + awaitIndexExists(currentTimestampedIndexName()); } - public void testNoUpdateWhenExistingTemplateIsTooOld() throws Exception { + public void testRandomTemplates() throws Exception { internalCluster().startNode(); - putTemplate(VersionUtils.getFirstVersion().number()); - doExporting(); - - logger.debug("--> existing template is too old: it should not be updated"); - assertTemplateNotUpdated(VersionUtils.getFirstVersion()); - - doExporting(); - - logger.debug("--> existing template is too old: no data is exported"); - assertIndicesNotCreated(); - } - - public void testUpdateWhenExistingTemplateIsOld() throws Exception { - internalCluster().startNode(); - - putTemplate(VersionUtils.getPreviousVersion(currentVersion()).number()); - doExporting(); - - logger.debug("--> existing template is old but supported: it should be updated to the current version"); - assertTemplateUpdated(currentVersion()); - - doExporting(); - - logger.debug("--> mappings should be up-to-date"); - assertMappingsUpdated(currentIndices()); - } - - public void testUpdateWhenExistingTemplateIsUpToDate() throws Exception { - internalCluster().startNode(); - - putTemplate(currentVersion().toString()); - doExporting(); - - logger.debug("--> existing template has the same version: it should not be updated"); - assertTemplateNotUpdated(currentVersion()); - - doExporting(); - - logger.debug("--> mappings should not have been updated"); - assertMappingsNotUpdated(currentIndices()); - } - - public void testMappingsUpdate() throws Exception { - boolean updateMappings = randomBoolean(); - logger.debug("--> update_mappings is {}", updateMappings); - internalCluster().startNode(Settings.builder().put("marvel.agent.exporters._exporter.update_mappings", updateMappings)); - - logger.debug("--> putting a template with a very old version so that it will not be updated"); - putTemplate(VersionUtils.getFirstVersion().toString()); - - logger.debug("--> creating marvel data index"); - createMarvelIndex(MarvelSettings.MARVEL_DATA_INDEX_NAME); - - logger.debug("--> creating a cold marvel index"); - createMarvelIndex(coldIndex()); - - logger.debug("--> creating an active marvel index"); - createMarvelIndex(hotIndex()); - - logger.debug("--> all indices have a old mapping now"); - assertMappingsNotUpdated(coldIndex(), hotIndex(), MarvelSettings.MARVEL_DATA_INDEX_NAME); - - logger.debug("--> updating the template with a previous version, so that it will be updated when exporting documents"); - putTemplate(VersionUtils.getPreviousVersion(currentVersion()).number()); - doExporting(); - - logger.debug("--> existing template is old: it should be updated to the current version"); - assertTemplateUpdated(currentVersion()); - - logger.debug("--> cold marvel index: mappings should not have been updated"); - assertMappingsNotUpdated(coldIndex()); - - if (updateMappings) { - logger.debug("--> marvel indices: mappings should be up-to-date"); - assertMappingsUpdated(MarvelSettings.MARVEL_DATA_INDEX_NAME, hotIndex()); - } else { - logger.debug("--> marvel indices: mappings should bnot have been updated"); - assertMappingsNotUpdated(MarvelSettings.MARVEL_DATA_INDEX_NAME, hotIndex()); + int previousIndexTemplateVersion = rarely() ? currentVersion : randomIntBetween(0, currentVersion - 1); + boolean previousIndexTemplateExist = randomBoolean(); + if (previousIndexTemplateExist) { + logger.debug("--> creating index template in version [{}]", previousIndexTemplateVersion); + putTemplate(indexTemplateName(previousIndexTemplateVersion), previousIndexTemplateVersion); } + + int previousDataTemplateVersion = rarely() ? currentVersion : randomIntBetween(0, currentVersion - 1); + boolean previousDataTemplateExist = randomBoolean(); + if (previousDataTemplateExist) { + logger.debug("--> creating data template in version [{}]", previousDataTemplateVersion); + putTemplate(dataTemplateName(previousDataTemplateVersion), previousDataTemplateVersion); + } + + doExporting(); + + logger.debug("--> templates should exist in current version"); + assertTemplateExist(indexTemplateName()); + assertTemplateExist(dataTemplateName()); + + if (previousIndexTemplateExist) { + logger.debug("--> index template should exist in version [{}]", previousIndexTemplateVersion); + assertTemplateExist(indexTemplateName(previousIndexTemplateVersion)); + } + + if (previousDataTemplateExist) { + logger.debug("--> data template should exist in version [{}]", previousDataTemplateVersion); + assertTemplateExist(dataTemplateName(previousDataTemplateVersion)); + } + + doExporting(); + + logger.debug("--> indices should exist in current version"); + awaitIndexExists(currentDataIndexName()); + awaitIndexExists(currentTimestampedIndexName()); } protected void doExporting() throws Exception { @@ -203,59 +170,23 @@ public abstract class AbstractExporterTemplateTestCase extends MarvelIntegTestCa return exporters.iterator().next(); } - private Version currentVersion() { - return MarvelTemplateUtils.loadDefaultTemplateVersion(); + private String currentDataIndexName() { + return ".marvel-es-data-" + String.valueOf(currentVersion); } - private String[] currentIndices() { - return new String[]{hotIndex(), MarvelSettings.MARVEL_DATA_INDEX_NAME}; - } - - private String coldIndex() { - return exporter().indexNameResolver().resolve(new DateTime(2012, 3, 10, 0, 0, DateTimeZone.UTC).getMillis()); - } - - private String hotIndex() { + private String currentTimestampedIndexName() { return exporter().indexNameResolver().resolve(System.currentTimeMillis()); } - /** Generates a template that looks like an old one **/ - protected static BytesReference generateTemplateSource(String version) throws IOException { + /** Generates a basic template **/ + protected static BytesReference generateTemplateSource(String name, Integer version) throws IOException { return jsonBuilder().startObject() - .field("template", ".marvel-es-*") + .field("template", name) .startObject("settings") .field("index.number_of_shards", 1) .field("index.number_of_replicas", 1) - .field("index.mapper.dynamic", false) - .field(MarvelTemplateUtils.MARVEL_VERSION_FIELD, version) + .field(MarvelTemplateUtils.VERSION_FIELD, String.valueOf(version)) .endObject() - .startObject("mappings") - .startObject("_default_") - .startObject("_all") - .field("enabled", false) - .endObject() - .field("date_detection", false) - .startObject("properties") - .startObject("cluster_uuid") - .field("type", "string") - .field("index", "not_analyzed") - .endObject() - .startObject("timestamp") - .field("type", "date") - .field("format", "date_time") - .endObject() - .endObject() - .endObject() - .startObject("cluster_info") - .field("enabled", false) - .endObject() - .startObject("node_stats") - .startObject("properties") - .startObject("node_stats") - .field("type", "object") - .endObject() - .endObject() - .endObject() - .endObject().bytes(); + .endObject().bytes(); } } diff --git a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/MarvelTemplateUtilsTests.java b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/MarvelTemplateUtilsTests.java index 959baab1a9c..0b331686622 100644 --- a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/MarvelTemplateUtilsTests.java +++ b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/MarvelTemplateUtilsTests.java @@ -5,89 +5,54 @@ */ package org.elasticsearch.marvel.agent.exporter; -import org.elasticsearch.Version; -import org.elasticsearch.common.Strings; -import org.elasticsearch.marvel.support.VersionUtils; import org.elasticsearch.test.ESTestCase; -import org.hamcrest.Matchers; import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import static org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils.MARVEL_VERSION_FIELD; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; + +//import static org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils.MARVEL_VERSION_FIELD; public class MarvelTemplateUtilsTests extends ESTestCase { - public void testLoadTemplate() { - byte[] template = MarvelTemplateUtils.loadDefaultTemplate(); + public void testLoadTimestampedIndexTemplate() { + byte[] template = MarvelTemplateUtils.loadTimestampedIndexTemplate(); assertNotNull(template); - assertThat(template.length, Matchers.greaterThan(0)); + assertThat(template.length, greaterThan(0)); } - public void testParseTemplateVersionFromByteArrayTemplate() throws IOException { - byte[] template = MarvelTemplateUtils.loadDefaultTemplate(); + public void testLoadDataIndexTemplate() { + byte[] template = MarvelTemplateUtils.loadDataIndexTemplate(); assertNotNull(template); + assertThat(template.length, greaterThan(0)); + } - Version version = MarvelTemplateUtils.parseTemplateVersion(template); + public void testLoad() throws IOException { + String resource = randomFrom(MarvelTemplateUtils.INDEX_TEMPLATE_FILE, MarvelTemplateUtils.DATA_TEMPLATE_FILE); + byte[] template = MarvelTemplateUtils.load(resource); + assertNotNull(template); + assertThat(template.length, greaterThan(0)); + } + + public void testLoadTemplateVersion() { + Integer version = MarvelTemplateUtils.loadTemplateVersion(); assertNotNull(version); + assertThat(version, greaterThan(0)); + assertThat(version, equalTo(MarvelTemplateUtils.TEMPLATE_VERSION)); } - public void testParseTemplateVersionFromStringTemplate() throws IOException { - List templates = new ArrayList<>(); - templates.add("{\"marvel_version\": \"1.4.0.Beta1\"}"); - templates.add("{\"marvel_version\": \"1.6.2-SNAPSHOT\"}"); - templates.add("{\"marvel_version\": \"1.7.1\"}"); - templates.add("{\"marvel_version\": \"2.0.0-beta1\"}"); - templates.add("{\"marvel_version\": \"2.0.0\"}"); - templates.add("{ \"template\": \".marvel*\", \"settings\": { \"marvel_version\": \"2.0.0-beta1-SNAPSHOT\", \"index.number_of_shards\": 1 } }"); - - for (String template : templates) { - Version version = MarvelTemplateUtils.parseTemplateVersion(Strings.toUTF8Bytes(template)); - assertNotNull(version); - } - - Version version = MarvelTemplateUtils.parseTemplateVersion(Strings.toUTF8Bytes("{\"marvel.index_format\": \"7\"}")); - assertNull(version); + public void testIndexTemplateName() { + assertThat(MarvelTemplateUtils.indexTemplateName(), + equalTo(MarvelTemplateUtils.INDEX_TEMPLATE_NAME_PREFIX + MarvelTemplateUtils.TEMPLATE_VERSION)); + int version = randomIntBetween(1, 100); + assertThat(MarvelTemplateUtils.indexTemplateName(version), equalTo(".marvel-es-" + version)); } - public void testParseVersion() throws IOException { - assertNotNull(VersionUtils.parseVersion(MARVEL_VERSION_FIELD, "{\"marvel_version\": \"2.0.0-beta1\"}")); - assertNotNull(VersionUtils.parseVersion(MARVEL_VERSION_FIELD, "{\"marvel_version\": \"2.0.0\"}")); - assertNotNull(VersionUtils.parseVersion(MARVEL_VERSION_FIELD, "{\"marvel_version\": \"1.5.2\"}")); - assertNotNull(VersionUtils.parseVersion(MARVEL_VERSION_FIELD, "{ \"template\": \".marvel*\", \"settings\": { \"marvel_version\": \"2.0.0-beta1-SNAPSHOT\", \"index.number_of_shards\": 1 } }")); - assertNull(VersionUtils.parseVersion(MARVEL_VERSION_FIELD, "{\"marvel.index_format\": \"7\"}")); - assertNull(VersionUtils.parseVersion(MARVEL_VERSION_FIELD + "unkown", "{\"marvel_version\": \"1.5.2\"}")); - } - - public void testTemplateVersionMandatesAnUpdate() { - // Version is unknown - assertTrue(MarvelTemplateUtils.installedTemplateVersionMandatesAnUpdate(Version.CURRENT, null, logger, "unit-test")); - - // Version is too old - Version unsupported = org.elasticsearch.test.VersionUtils.getPreviousVersion(MarvelTemplateUtils.MIN_SUPPORTED_TEMPLATE_VERSION); - assertFalse(MarvelTemplateUtils.installedTemplateVersionMandatesAnUpdate(Version.CURRENT, unsupported, logger, "unit-test")); - - // Version is old but supported - assertTrue(MarvelTemplateUtils.installedTemplateVersionMandatesAnUpdate(Version.CURRENT, MarvelTemplateUtils.MIN_SUPPORTED_TEMPLATE_VERSION, logger, "unit-test")); - - // Version is up to date - assertFalse(MarvelTemplateUtils.installedTemplateVersionMandatesAnUpdate(Version.CURRENT, Version.CURRENT, logger, "unit-test")); - - // Version is up to date - Version previous = org.elasticsearch.test.VersionUtils.getPreviousVersion(Version.CURRENT); - assertFalse(MarvelTemplateUtils.installedTemplateVersionMandatesAnUpdate(previous, Version.CURRENT, logger, "unit-test")); - } - - public void testTemplateVersionIsSufficient() { - // Version is unknown - assertFalse(MarvelTemplateUtils.installedTemplateVersionIsSufficient(null)); - - // Version is too old - Version unsupported = org.elasticsearch.test.VersionUtils.getPreviousVersion(MarvelTemplateUtils.MIN_SUPPORTED_TEMPLATE_VERSION); - assertFalse(MarvelTemplateUtils.installedTemplateVersionIsSufficient(unsupported)); - - // Version is OK - assertTrue(MarvelTemplateUtils.installedTemplateVersionIsSufficient(Version.CURRENT)); + public void testDataTemplateName() { + assertThat(MarvelTemplateUtils.dataTemplateName(), + equalTo(MarvelTemplateUtils.DATA_TEMPLATE_NAME_PREFIX + MarvelTemplateUtils.TEMPLATE_VERSION)); + int version = randomIntBetween(1, 100); + assertThat(MarvelTemplateUtils.dataTemplateName(version), equalTo(".marvel-es-data-" + version)); } } diff --git a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/http/HttpExporterTemplateTests.java b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/http/HttpExporterTemplateTests.java index cc93fca3776..44bd48edd99 100644 --- a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/http/HttpExporterTemplateTests.java +++ b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/http/HttpExporterTemplateTests.java @@ -11,26 +11,25 @@ import com.squareup.okhttp.mockwebserver.MockWebServer; import com.squareup.okhttp.mockwebserver.RecordedRequest; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.Version; -import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.bulk.BulkRequest; +import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.util.concurrent.ConcurrentCollections; import org.elasticsearch.marvel.agent.exporter.AbstractExporterTemplateTestCase; import org.elasticsearch.marvel.agent.exporter.Exporter; -import org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils; import org.junit.After; import org.junit.Before; -import java.io.IOException; import java.net.BindException; -import java.util.HashMap; import java.util.HashSet; import java.util.Map; import java.util.Set; +import java.util.concurrent.TimeUnit; -import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.core.Is.is; public class HttpExporterTemplateTests extends AbstractExporterTemplateTestCase { @@ -70,68 +69,39 @@ public class HttpExporterTemplateTests extends AbstractExporterTemplateTestCase } @Override - protected void deleteTemplate() { - dispatcher.setTemplate(null); + protected void deleteTemplates() throws Exception { + dispatcher.templates.clear(); } @Override - protected void putTemplate(String version) throws Exception { - dispatcher.setTemplate(generateTemplateSource(version).toBytes()); + protected void putTemplate(String name, int version) throws Exception { + dispatcher.templates.put(name, generateTemplateSource(name, version)); } @Override - protected void createMarvelIndex(String index) throws Exception { - dispatcher.addIndex(index); + protected void assertTemplateExist(String name) throws Exception { + assertThat("failed to find a template matching [" + name + "]", dispatcher.templates.containsKey(name), is(true)); } @Override - protected void assertTemplateUpdated(Version version) { - // Checks that a PUT Template request has been made - assertThat(dispatcher.hasRequest("PUT", "/_template/" + MarvelTemplateUtils.INDEX_TEMPLATE_NAME), is(true)); - - // Checks that the current template has the expected version - assertThat(MarvelTemplateUtils.parseTemplateVersion(dispatcher.getTemplate()), equalTo(version)); - } - - @Override - protected void assertTemplateNotUpdated(Version version) throws Exception { + protected void assertTemplateNotUpdated(String name) throws Exception { // Checks that no PUT Template request has been made - assertThat(dispatcher.hasRequest("PUT", "/_template/" + MarvelTemplateUtils.INDEX_TEMPLATE_NAME), is(false)); + assertThat(dispatcher.hasRequest("PUT", "/_template/" + name), is(false)); - // Checks that the current template has the expected version - assertThat(MarvelTemplateUtils.parseTemplateVersion(dispatcher.getTemplate()), equalTo(version)); + // Checks that the current template exists + assertThat(dispatcher.templates.containsKey(name), is(true)); } @Override - protected void assertIndicesNotCreated() throws Exception { - // Checks that no Bulk request has been made - assertThat(dispatcher.hasRequest("POST", "/_bulk"), is(false)); - assertThat(dispatcher.mappings.size(), equalTo(0)); - } - - @Override - protected void assertMappingsUpdated(String... indices) throws Exception { - // Load the mappings of the old template - Set oldMappings = new PutIndexTemplateRequest().source(generateTemplateSource(null)).mappings().keySet(); - - // Load the mappings of the latest template - Set newMappings = new PutIndexTemplateRequest().source(generateTemplateSource(null)).mappings().keySet(); - newMappings.removeAll(oldMappings); - - for (String index : indices) { - for (String mapping : newMappings) { - // Checks that a PUT Mapping request has been made for every type that was not in the old template - assertThat(dispatcher.hasRequest("PUT", "/" + index + "/_mapping/" + mapping), equalTo(true)); + protected void awaitIndexExists(String... indices) throws Exception { + assertBusy(new Runnable() { + @Override + public void run() { + for (String index : indices) { + assertThat("could not find index " + index, dispatcher.hasIndex(index), is(true)); + } } - } - } - - @Override - protected void assertMappingsNotUpdated(String... indices) throws Exception { - for (String index : indices) { - // Checks that no PUT Template request has been made - assertThat(dispatcher.hasRequest("PUT", "/" + index + "/_mapping/"), is(false)); - } + }, 10, TimeUnit.SECONDS); } class MockServerDispatcher extends Dispatcher { @@ -140,100 +110,59 @@ public class HttpExporterTemplateTests extends AbstractExporterTemplateTestCase private final MockResponse NOT_FOUND = newResponse(404, ""); private final Set requests = new HashSet<>(); - - private final Map> mappings = new HashMap<>(); - private byte[] template; + private final Map templates = ConcurrentCollections.newConcurrentMap(); + private final Set indices = ConcurrentCollections.newConcurrentSet(); @Override public MockResponse dispatch(RecordedRequest request) throws InterruptedException { - synchronized (this) { - final String requestLine = request.getRequestLine(); - requests.add(requestLine); + final String requestLine = request.getRequestLine(); + requests.add(requestLine); - switch (requestLine) { - // Cluster version - case "GET / HTTP/1.1": - return newResponse(200, "{\"version\": {\"number\": \"" + Version.CURRENT.number() + "\"}}"); - - // Template - case "GET /_template/.marvel-es HTTP/1.1": - return (template == null) ? NOT_FOUND : newResponse(200, new BytesArray(template).toUtf8()); - - case "PUT /_template/.marvel-es HTTP/1.1": - this.template = request.getBody().readByteArray(); - return OK; - - // Bulk - case "POST /_bulk HTTP/1.1": - - return OK; - default: - String[] paths = Strings.splitStringToArray(request.getPath(), '/'); - - // Index Mappings - if ((paths != null) && (paths.length > 0) && ("_mapping".equals(paths[1]))) { - - if (!mappings.containsKey(paths[0])) { - // Index does not exist - return NOT_FOUND; - } - - // Get index mappings - if ("GET".equals(request.getMethod())) { - try { - // Builds a fake mapping response - XContentBuilder builder = jsonBuilder().startObject().startObject(paths[0]).startObject("mappings"); - for (String type : mappings.get(paths[0])) { - builder.startObject(type).endObject(); - } - builder.endObject().endObject().endObject(); - return newResponse(200, builder.bytes().toUtf8()); - } catch (IOException e) { - return newResponse(500, e.getMessage()); - } - - // Put index mapping - } else if ("PUT".equals(request.getMethod()) && paths.length > 2) { - Set types = mappings.get(paths[0]); - if (types == null) { - types = new HashSet<>(); - } - types.add(paths[2]); - return OK; + switch (requestLine) { + // Cluster version + case "GET / HTTP/1.1": + return newResponse(200, "{\"version\": {\"number\": \"" + Version.CURRENT.number() + "\"}}"); + // Bulk + case "POST /_bulk HTTP/1.1": + // Parse the bulk request and extract all index names + try { + BulkRequest bulk = new BulkRequest(); + byte[] source = request.getBody().readByteArray(); + bulk.add(source, 0, source.length); + for (ActionRequest docRequest : bulk.requests()) { + if (docRequest instanceof IndexRequest) { + indices.add(((IndexRequest) docRequest).index()); } } - break; - } + } catch (Exception e) { + return newResponse(500, e.getMessage()); + } + return OK; + default: + String[] paths = Strings.splitStringToArray(request.getPath(), '/'); - return newResponse(500, "MockServerDispatcher does not support: " + request.getRequestLine()); + // Templates + if ((paths != null) && (paths.length > 0) && ("_template".equals(paths[0]))) { + String templateName = paths[1]; + boolean templateExist = templates.containsKey(templateName); + + if ("GET".equals(request.getMethod())) { + return templateExist ? newResponse(200, templates.get(templateName).toUtf8()) : NOT_FOUND; + } + if ("PUT".equals(request.getMethod())) { + templates.put(templateName, new BytesArray(request.getBody().readByteArray())); + return templateExist ? newResponse(200, "updated") : newResponse(201, "created"); + } + } + break; } + return newResponse(500, "MockServerDispatcher does not support: " + request.getRequestLine()); } MockResponse newResponse(int code, String body) { return new MockResponse().setResponseCode(code).setBody(body); } - void setTemplate(byte[] template) { - synchronized (this) { - this.template = template; - } - } - - byte[] getTemplate() { - return template; - } - - void addIndex(String index) { - synchronized (this) { - if (template != null) { - // Simulate the use of the index template when creating an index - mappings.put(index, new HashSet<>(new PutIndexTemplateRequest().source(template).mappings().keySet())); - } else { - mappings.put(index, null); - } - } - } - int countRequests(String method, String path) { int count = 0; for (String request : requests) { @@ -247,5 +176,9 @@ public class HttpExporterTemplateTests extends AbstractExporterTemplateTestCase boolean hasRequest(String method, String path) { return countRequests(method, path) > 0; } + + boolean hasIndex(String index) { + return indices.contains(index); + } } } diff --git a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/http/HttpExporterTests.java b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/http/HttpExporterTests.java index 17ba6e481fe..4b43531f73a 100644 --- a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/http/HttpExporterTests.java +++ b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/http/HttpExporterTests.java @@ -46,6 +46,8 @@ import java.util.List; import java.util.Map; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils.dataTemplateName; +import static org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils.indexTemplateName; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; @@ -53,9 +55,13 @@ import static org.hamcrest.Matchers.is; @ESIntegTestCase.ClusterScope(scope = Scope.TEST, numDataNodes = 0, numClientNodes = 0, transportClientRatio = 0.0) public class HttpExporterTests extends MarvelIntegTestCase { + private int webPort; private MockWebServer webServer; + private static final byte[] TIMESTAMPED_TEMPLATE = MarvelTemplateUtils.loadTimestampedIndexTemplate(); + private static final byte[] DATA_TEMPLATE = MarvelTemplateUtils.loadDataIndexTemplate(); + @Before public void startWebservice() throws Exception { for (webPort = 9250; webPort < 9300; webPort++) { @@ -81,8 +87,10 @@ public class HttpExporterTests extends MarvelIntegTestCase { public void testExport() throws Exception { enqueueGetClusterVersionResponse(Version.CURRENT); - enqueueResponse(404, "marvel template does not exist"); - enqueueResponse(201, "marvel template created"); + enqueueResponse(404, "template for timestamped indices does not exist"); + enqueueResponse(201, "template for timestamped indices created"); + enqueueResponse(404, "template for data index does not exist"); + enqueueResponse(201, "template for data index created"); enqueueResponse(200, "successful bulk request "); Settings.Builder builder = Settings.builder() @@ -98,7 +106,7 @@ public class HttpExporterTests extends MarvelIntegTestCase { final int nbDocs = randomIntBetween(1, 25); exporter.export(newRandomMarvelDocs(nbDocs)); - assertThat(webServer.getRequestCount(), equalTo(4)); + assertThat(webServer.getRequestCount(), equalTo(6)); RecordedRequest recordedRequest = webServer.takeRequest(); assertThat(recordedRequest.getMethod(), equalTo("GET")); @@ -106,12 +114,21 @@ public class HttpExporterTests extends MarvelIntegTestCase { recordedRequest = webServer.takeRequest(); assertThat(recordedRequest.getMethod(), equalTo("GET")); - assertThat(recordedRequest.getPath(), equalTo("/_template/.marvel-es")); + assertThat(recordedRequest.getPath(), equalTo("/_template/" + indexTemplateName())); recordedRequest = webServer.takeRequest(); assertThat(recordedRequest.getMethod(), equalTo("PUT")); - assertThat(recordedRequest.getPath(), equalTo("/_template/.marvel-es")); - assertThat(recordedRequest.getBody().readByteArray(), equalTo(MarvelTemplateUtils.loadDefaultTemplate())); + assertThat(recordedRequest.getPath(), equalTo("/_template/" + indexTemplateName())); + assertThat(recordedRequest.getBody().readByteArray(), equalTo(TIMESTAMPED_TEMPLATE)); + + recordedRequest = webServer.takeRequest(); + assertThat(recordedRequest.getMethod(), equalTo("GET")); + assertThat(recordedRequest.getPath(), equalTo("/_template/" + dataTemplateName())); + + recordedRequest = webServer.takeRequest(); + assertThat(recordedRequest.getMethod(), equalTo("PUT")); + assertThat(recordedRequest.getPath(), equalTo("/_template/" + dataTemplateName())); + assertThat(recordedRequest.getBody().readByteArray(), equalTo(DATA_TEMPLATE)); recordedRequest = webServer.takeRequest(); assertThat(recordedRequest.getMethod(), equalTo("POST")); @@ -156,8 +173,10 @@ public class HttpExporterTests extends MarvelIntegTestCase { logger.info("--> starting node"); enqueueGetClusterVersionResponse(Version.CURRENT); - enqueueResponse(404, "marvel template does not exist"); - enqueueResponse(201, "marvel template created"); + enqueueResponse(404, "template for timestamped indices does not exist"); + enqueueResponse(201, "template for timestamped indices created"); + enqueueResponse(404, "template for data index does not exist"); + enqueueResponse(201, "template for data index created"); enqueueResponse(200, "successful bulk request "); String agentNode = internalCluster().startNode(builder); @@ -168,7 +187,7 @@ public class HttpExporterTests extends MarvelIntegTestCase { exporter.export(Collections.singletonList(newRandomMarvelDoc())); assertThat(exporter.supportedClusterVersion, is(true)); - assertThat(webServer.getRequestCount(), equalTo(4)); + assertThat(webServer.getRequestCount(), equalTo(6)); RecordedRequest recordedRequest = webServer.takeRequest(); assertThat(recordedRequest.getMethod(), equalTo("GET")); @@ -176,12 +195,21 @@ public class HttpExporterTests extends MarvelIntegTestCase { recordedRequest = webServer.takeRequest(); assertThat(recordedRequest.getMethod(), equalTo("GET")); - assertThat(recordedRequest.getPath(), equalTo("/_template/.marvel-es")); + assertThat(recordedRequest.getPath(), equalTo("/_template/" + indexTemplateName())); recordedRequest = webServer.takeRequest(); assertThat(recordedRequest.getMethod(), equalTo("PUT")); - assertThat(recordedRequest.getPath(), equalTo("/_template/.marvel-es")); - assertThat(recordedRequest.getBody().readByteArray(), equalTo(MarvelTemplateUtils.loadDefaultTemplate())); + assertThat(recordedRequest.getPath(), equalTo("/_template/" + indexTemplateName())); + assertThat(recordedRequest.getBody().readByteArray(), equalTo(TIMESTAMPED_TEMPLATE)); + + recordedRequest = webServer.takeRequest(); + assertThat(recordedRequest.getMethod(), equalTo("GET")); + assertThat(recordedRequest.getPath(), equalTo("/_template/" + dataTemplateName())); + + recordedRequest = webServer.takeRequest(); + assertThat(recordedRequest.getMethod(), equalTo("PUT")); + assertThat(recordedRequest.getPath(), equalTo("/_template/" + dataTemplateName())); + assertThat(recordedRequest.getBody().readByteArray(), equalTo(DATA_TEMPLATE)); recordedRequest = webServer.takeRequest(); assertThat(recordedRequest.getMethod(), equalTo("POST")); @@ -214,14 +242,15 @@ public class HttpExporterTests extends MarvelIntegTestCase { exporter = getExporter(agentNode); enqueueGetClusterVersionResponse(secondWebServer, Version.CURRENT); - enqueueResponse(secondWebServer, 404, "marvel template does not exist"); - enqueueResponse(secondWebServer, 201, "marvel template created"); + enqueueResponse(secondWebServer, 404, "template for timestamped indices does not exist"); + enqueueResponse(secondWebServer, 201, "template for timestamped indices created"); + enqueueResponse(secondWebServer, 200, "template for data index exist"); enqueueResponse(secondWebServer, 200, "successful bulk request "); logger.info("--> exporting a second event"); exporter.export(Collections.singletonList(newRandomMarvelDoc())); - assertThat(secondWebServer.getRequestCount(), equalTo(4)); + assertThat(secondWebServer.getRequestCount(), equalTo(5)); recordedRequest = secondWebServer.takeRequest(); assertThat(recordedRequest.getMethod(), equalTo("GET")); @@ -229,12 +258,16 @@ public class HttpExporterTests extends MarvelIntegTestCase { recordedRequest = secondWebServer.takeRequest(); assertThat(recordedRequest.getMethod(), equalTo("GET")); - assertThat(recordedRequest.getPath(), equalTo("/_template/.marvel-es")); + assertThat(recordedRequest.getPath(), equalTo("/_template/" + indexTemplateName())); recordedRequest = secondWebServer.takeRequest(); assertThat(recordedRequest.getMethod(), equalTo("PUT")); - assertThat(recordedRequest.getPath(), equalTo("/_template/.marvel-es")); - assertThat(recordedRequest.getBody().readByteArray(), equalTo(MarvelTemplateUtils.loadDefaultTemplate())); + assertThat(recordedRequest.getPath(), equalTo("/_template/" + indexTemplateName())); + assertThat(recordedRequest.getBody().readByteArray(), equalTo(TIMESTAMPED_TEMPLATE)); + + recordedRequest = secondWebServer.takeRequest(); + assertThat(recordedRequest.getMethod(), equalTo("GET")); + assertThat(recordedRequest.getPath(), equalTo("/_template/" + dataTemplateName()));; recordedRequest = secondWebServer.takeRequest(); assertThat(recordedRequest.getMethod(), equalTo("POST")); @@ -287,8 +320,10 @@ public class HttpExporterTests extends MarvelIntegTestCase { logger.info("--> exporting a first event"); enqueueGetClusterVersionResponse(Version.CURRENT); - enqueueResponse(404, "marvel template does not exist"); - enqueueResponse(201, "marvel template created"); + enqueueResponse(404, "template for timestamped indices does not exist"); + enqueueResponse(201, "template for timestamped indices created"); + enqueueResponse(404, "template for data index does not exist"); + enqueueResponse(201, "template for data index created"); enqueueResponse(200, "successful bulk request "); HttpExporter exporter = getExporter(agentNode); @@ -296,7 +331,7 @@ public class HttpExporterTests extends MarvelIntegTestCase { MarvelDoc doc = newRandomMarvelDoc(); exporter.export(Collections.singletonList(doc)); - assertThat(webServer.getRequestCount(), equalTo(4)); + assertThat(webServer.getRequestCount(), equalTo(6)); RecordedRequest recordedRequest = webServer.takeRequest(); assertThat(recordedRequest.getMethod(), equalTo("GET")); @@ -304,12 +339,21 @@ public class HttpExporterTests extends MarvelIntegTestCase { recordedRequest = webServer.takeRequest(); assertThat(recordedRequest.getMethod(), equalTo("GET")); - assertThat(recordedRequest.getPath(), equalTo("/_template/.marvel-es")); + assertThat(recordedRequest.getPath(), equalTo("/_template/" + indexTemplateName())); recordedRequest = webServer.takeRequest(); assertThat(recordedRequest.getMethod(), equalTo("PUT")); - assertThat(recordedRequest.getPath(), equalTo("/_template/.marvel-es")); - assertThat(recordedRequest.getBody().readByteArray(), equalTo(MarvelTemplateUtils.loadDefaultTemplate())); + assertThat(recordedRequest.getPath(), equalTo("/_template/" + indexTemplateName())); + assertThat(recordedRequest.getBody().readByteArray(), equalTo(TIMESTAMPED_TEMPLATE)); + + recordedRequest = webServer.takeRequest(); + assertThat(recordedRequest.getMethod(), equalTo("GET")); + assertThat(recordedRequest.getPath(), equalTo("/_template/" + dataTemplateName())); + + recordedRequest = webServer.takeRequest(); + assertThat(recordedRequest.getMethod(), equalTo("PUT")); + assertThat(recordedRequest.getPath(), equalTo("/_template/" + dataTemplateName())); + assertThat(recordedRequest.getBody().readByteArray(), equalTo(DATA_TEMPLATE)); recordedRequest = webServer.takeRequest(); assertThat(recordedRequest.getMethod(), equalTo("POST")); @@ -332,18 +376,18 @@ public class HttpExporterTests extends MarvelIntegTestCase { logger.info("--> exporting a second event"); enqueueGetClusterVersionResponse(Version.CURRENT); - enqueueResponse(404, "marvel template does not exist"); - enqueueResponse(201, "marvel template created"); + enqueueResponse(200, "template for timestamped indices exist"); + enqueueResponse(200, "template for data index exist"); enqueueResponse(200, "successful bulk request "); doc = newRandomMarvelDoc(); exporter = getExporter(agentNode); exporter.export(Collections.singletonList(doc)); - String expectedMarvelIndex = MarvelSettings.MARVEL_INDICES_PREFIX + String expectedMarvelIndex = MarvelSettings.MARVEL_INDICES_PREFIX + MarvelTemplateUtils.TEMPLATE_VERSION + "-" + DateTimeFormat.forPattern(newTimeFormat).withZoneUTC().print(doc.timestamp()); - assertThat(webServer.getRequestCount(), equalTo(4 + 4)); + assertThat(webServer.getRequestCount(), equalTo(6 + 4)); recordedRequest = webServer.takeRequest(); assertThat(recordedRequest.getMethod(), equalTo("GET")); @@ -351,12 +395,11 @@ public class HttpExporterTests extends MarvelIntegTestCase { recordedRequest = webServer.takeRequest(); assertThat(recordedRequest.getMethod(), equalTo("GET")); - assertThat(recordedRequest.getPath(), equalTo("/_template/.marvel-es")); + assertThat(recordedRequest.getPath(), equalTo("/_template/" + indexTemplateName())); recordedRequest = webServer.takeRequest(); - assertThat(recordedRequest.getMethod(), equalTo("PUT")); - assertThat(recordedRequest.getPath(), equalTo("/_template/.marvel-es")); - assertThat(recordedRequest.getBody().readByteArray(), equalTo(MarvelTemplateUtils.loadDefaultTemplate())); + assertThat(recordedRequest.getMethod(), equalTo("GET")); + assertThat(recordedRequest.getPath(), equalTo("/_template/" + dataTemplateName())); recordedRequest = webServer.takeRequest(); assertThat(recordedRequest.getMethod(), equalTo("POST")); diff --git a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/local/LocalExporterTemplateTests.java b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/local/LocalExporterTemplateTests.java index 9949cd2e3b9..035254747c6 100644 --- a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/local/LocalExporterTemplateTests.java +++ b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/local/LocalExporterTemplateTests.java @@ -5,26 +5,12 @@ */ package org.elasticsearch.marvel.agent.exporter.local; -import org.elasticsearch.Version; -import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; -import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest; -import org.elasticsearch.action.support.IndicesOptions; -import org.elasticsearch.cluster.metadata.MappingMetaData; -import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.marvel.agent.exporter.AbstractExporterTemplateTestCase; -import org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils; -import org.elasticsearch.marvel.agent.settings.MarvelSettings; import java.util.Collections; -import java.util.Map; -import java.util.Set; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.notNullValue; -import static org.hamcrest.core.Is.is; public class LocalExporterTemplateTests extends AbstractExporterTemplateTestCase { @@ -34,83 +20,26 @@ public class LocalExporterTemplateTests extends AbstractExporterTemplateTestCase } @Override - protected Set excludeTemplates() { - // Always delete the template between tests - return Collections.emptySet(); - } - - @Override - protected void deleteTemplate() throws Exception { + protected void deleteTemplates() throws Exception { waitNoPendingTasksOnAll(); - assertAcked(client().admin().indices().prepareDeleteTemplate(MarvelTemplateUtils.INDEX_TEMPLATE_NAME).get()); + cluster().wipeAllTemplates(Collections.emptySet()); } @Override - protected void putTemplate(String version) throws Exception { + protected void putTemplate(String name, int version) throws Exception { waitNoPendingTasksOnAll(); - assertAcked(client().admin().indices().preparePutTemplate(MarvelTemplateUtils.INDEX_TEMPLATE_NAME).setSource(generateTemplateSource(version)).get()); + assertAcked(client().admin().indices().preparePutTemplate(name).setSource(generateTemplateSource(name, version)).get()); } @Override - protected void createMarvelIndex(String index) throws Exception { + protected void assertTemplateExist(String name) throws Exception { waitNoPendingTasksOnAll(); - createIndex(index); + waitForMarvelTemplate(name); } @Override - protected void assertTemplateUpdated(Version version) throws Exception { + protected void assertTemplateNotUpdated(String name) throws Exception { waitNoPendingTasksOnAll(); - awaitMarvelTemplateInstalled(version); - } - - @Override - protected void assertTemplateNotUpdated(Version version) throws Exception { - waitNoPendingTasksOnAll(); - awaitMarvelTemplateInstalled(version); - } - - private void assertMappings(byte[] reference, String... indices) throws Exception { - waitNoPendingTasksOnAll(); - - Map mappings = new PutIndexTemplateRequest().source(reference).mappings(); - assertBusy(new Runnable() { - @Override - public void run() { - for (String index : indices) { - GetMappingsResponse response = client().admin().indices().prepareGetMappings(index).setIndicesOptions(IndicesOptions.lenientExpandOpen()).get(); - ImmutableOpenMap indexMappings = response.getMappings().get(index); - assertNotNull(indexMappings); - assertThat(indexMappings.size(), equalTo(mappings.size())); - - for (String mapping : mappings.keySet()) { - // We just check that mapping type exists, we don't verify its content - assertThat("mapping type " + mapping + " should exist in index " + index, indexMappings.get(mapping), notNullValue()); - } - } - } - }); - } - - @Override - protected void assertMappingsUpdated(String... indices) throws Exception { - assertMappings(MarvelTemplateUtils.loadDefaultTemplate(), indices); - } - - @Override - protected void assertMappingsNotUpdated(String... indices) throws Exception { - assertMappings(generateTemplateSource(null).toBytes(), indices); - } - - @Override - protected void assertIndicesNotCreated() throws Exception { - waitNoPendingTasksOnAll(); - try { - assertThat(client().admin().indices().prepareExists(MarvelSettings.MARVEL_INDICES_PREFIX + "*").get().isExists(), is(false)); - } catch (IndexNotFoundException e) { - // with shield we might get that if wildcards were resolved to no indices - if (!shieldEnabled) { - throw e; - } - } + assertTemplateExist(name); } } diff --git a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/local/LocalExporterTests.java b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/local/LocalExporterTests.java index f816d3cf721..9e581e8e2d4 100644 --- a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/local/LocalExporterTests.java +++ b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/local/LocalExporterTests.java @@ -6,9 +6,7 @@ package org.elasticsearch.marvel.agent.exporter.local; import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.Version; import org.elasticsearch.action.admin.indices.recovery.RecoveryResponse; -import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.health.ClusterHealthStatus; @@ -36,11 +34,12 @@ import java.util.Map; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; +import static org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils.dataTemplateName; +import static org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils.indexTemplateName; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.allOf; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; @@ -105,8 +104,8 @@ public class LocalExporterTests extends MarvelIntegTestCase { updateMarvelInterval(3L, TimeUnit.SECONDS); // lets wait until the marvel template will be installed - awaitMarvelTemplateInstalled(); - assertThat(getCurrentlyInstalledTemplateVersion(), is(Version.CURRENT)); + waitForMarvelTemplate(indexTemplateName()); + waitForMarvelTemplate(dataTemplateName()); } public void testIndexTimestampFormat() throws Exception { @@ -122,12 +121,12 @@ public class LocalExporterTests extends MarvelIntegTestCase { LocalExporter exporter = getLocalExporter("_local"); // first lets test that the index resolver works with time - String indexName = MarvelSettings.MARVEL_INDICES_PREFIX + DateTimeFormat.forPattern(timeFormat).withZoneUTC().print(time); + String indexName = MarvelSettings.MARVEL_INDICES_PREFIX + MarvelTemplateUtils.TEMPLATE_VERSION + "-" + DateTimeFormat.forPattern(timeFormat).withZoneUTC().print(time); assertThat(exporter.indexNameResolver().resolve(time), equalTo(indexName)); // now lets test that the index name resolver works with a doc MarvelDoc doc = newRandomMarvelDoc(); - indexName = MarvelSettings.MARVEL_INDICES_PREFIX + DateTimeFormat.forPattern(timeFormat).withZoneUTC().print(doc.timestamp()); + indexName = MarvelSettings.MARVEL_INDICES_PREFIX + MarvelTemplateUtils.TEMPLATE_VERSION + "-" + DateTimeFormat.forPattern(timeFormat).withZoneUTC().print(doc.timestamp()); assertThat(exporter.indexNameResolver().resolve(doc), equalTo(indexName)); logger.debug("--> exporting a random marvel document"); @@ -138,7 +137,7 @@ public class LocalExporterTests extends MarvelIntegTestCase { timeFormat = randomFrom("dd", "dd.MM.YYYY", "dd.MM"); updateClusterSettings(Settings.builder().put("marvel.agent.exporters._local.index.name.time_format", timeFormat)); exporter = getLocalExporter("_local"); // we need to get it again.. as it was rebuilt - indexName = MarvelSettings.MARVEL_INDICES_PREFIX + DateTimeFormat.forPattern(timeFormat).withZoneUTC().print(doc.timestamp()); + indexName = MarvelSettings.MARVEL_INDICES_PREFIX + MarvelTemplateUtils.TEMPLATE_VERSION + "-" + DateTimeFormat.forPattern(timeFormat).withZoneUTC().print(doc.timestamp()); assertThat(exporter.indexNameResolver().resolve(doc), equalTo(indexName)); logger.debug("--> exporting the document again (this time with the the new index name time format [{}], expecting index name [{}]", timeFormat, indexName); @@ -194,13 +193,4 @@ public class LocalExporterTests extends MarvelIntegTestCase { ClusterStateCollector.TYPE, timeStampGenerator.incrementAndGet(), ClusterState.PROTO, ClusterHealthStatus.GREEN); } } - - private Version getCurrentlyInstalledTemplateVersion() { - GetIndexTemplatesResponse response = client().admin().indices().prepareGetTemplates(MarvelTemplateUtils.INDEX_TEMPLATE_NAME).get(); - assertThat(response, notNullValue()); - assertThat(response.getIndexTemplates(), notNullValue()); - assertThat(response.getIndexTemplates(), hasSize(1)); - assertThat(response.getIndexTemplates().get(0), notNullValue()); - return MarvelTemplateUtils.templateVersion(response.getIndexTemplates().get(0)); - } } diff --git a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/cluster/ClusterInfoTests.java b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/cluster/ClusterInfoTests.java index 1651a30baa3..f0336d3554e 100644 --- a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/cluster/ClusterInfoTests.java +++ b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/cluster/ClusterInfoTests.java @@ -12,6 +12,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.license.core.License; import org.elasticsearch.marvel.agent.collector.cluster.ClusterInfoCollector; +import org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils; import org.elasticsearch.marvel.agent.settings.MarvelSettings; import org.elasticsearch.marvel.test.MarvelIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; @@ -21,6 +22,8 @@ import org.junit.Before; import java.util.Map; import java.util.concurrent.TimeUnit; +import static org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils.dataTemplateName; +import static org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils.indexTemplateName; import static org.elasticsearch.test.ESIntegTestCase.Scope.TEST; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; import static org.hamcrest.Matchers.equalTo; @@ -57,20 +60,21 @@ public class ClusterInfoTests extends MarvelIntegTestCase { final String clusterUUID = client().admin().cluster().prepareState().setMetaData(true).get().getState().metaData().clusterUUID(); assertTrue(Strings.hasText(clusterUUID)); - logger.debug("--> waiting for the marvel data index to be created (it should have been created by the LicenseCollector)"); - awaitIndexExists(MarvelSettings.MARVEL_DATA_INDEX_NAME); + logger.debug("--> waiting for the marvel data index to be created (it should have been created by the ClusterInfoCollector)"); + String dataIndex = ".marvel-es-data-" + MarvelTemplateUtils.TEMPLATE_VERSION; + awaitIndexExists(dataIndex); logger.debug("--> waiting for cluster info collector to collect data"); awaitMarvelDocsCount(equalTo(1L), ClusterInfoCollector.TYPE); logger.debug("--> retrieving cluster info document"); - GetResponse response = client().prepareGet(MarvelSettings.MARVEL_DATA_INDEX_NAME, ClusterInfoCollector.TYPE, clusterUUID).get(); - assertTrue(MarvelSettings.MARVEL_DATA_INDEX_NAME + " document does not exist", response.isExists()); + GetResponse response = client().prepareGet(dataIndex, ClusterInfoCollector.TYPE, clusterUUID).get(); + assertTrue("cluster_info document does not exist in data index", response.isExists()); logger.debug("--> checking that the document contains all required information"); logger.debug("--> checking that the document contains license information"); - assertThat(response.getIndex(), equalTo(MarvelSettings.MARVEL_DATA_INDEX_NAME)); + assertThat(response.getIndex(), equalTo(dataIndex)); assertThat(response.getType(), equalTo(ClusterInfoCollector.TYPE)); assertThat(response.getId(), equalTo(clusterUUID)); @@ -111,14 +115,15 @@ public class ClusterInfoTests extends MarvelIntegTestCase { assertThat(clusterStats, instanceOf(Map.class)); assertThat(((Map) clusterStats).size(), greaterThan(0)); - assertMarvelTemplateInstalled(); + waitForMarvelTemplate(indexTemplateName()); + waitForMarvelTemplate(dataTemplateName()); logger.debug("--> check that the cluster_info is not indexed"); securedFlush(); securedRefresh(); assertHitCount(client().prepareSearch().setSize(0) - .setIndices(MarvelSettings.MARVEL_DATA_INDEX_NAME) + .setIndices(dataIndex) .setTypes(ClusterInfoCollector.TYPE) .setQuery(QueryBuilders.boolQuery() .should(QueryBuilders.matchQuery(License.XFields.STATUS.underscore().toString(), License.Status.ACTIVE.label())) diff --git a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/cluster/ClusterStateTests.java b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/cluster/ClusterStateTests.java index 47ed2b5f569..cb2d9ef1256 100644 --- a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/cluster/ClusterStateTests.java +++ b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/cluster/ClusterStateTests.java @@ -11,6 +11,7 @@ import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.marvel.agent.collector.cluster.ClusterStateCollector; +import org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils; import org.elasticsearch.marvel.agent.renderer.AbstractRenderer; import org.elasticsearch.marvel.agent.settings.MarvelSettings; import org.elasticsearch.marvel.test.MarvelIntegTestCase; @@ -156,9 +157,10 @@ public class ClusterStateTests extends MarvelIntegTestCase { for (final String nodeName : internalCluster().getNodeNames()) { final String nodeId = internalCluster().clusterService(nodeName).localNode().getId(); + final String dataIndex = ".marvel-es-data-" + MarvelTemplateUtils.TEMPLATE_VERSION; logger.debug("--> getting marvel document for node id [{}]", nodeId); - assertThat(client().prepareGet(MarvelSettings.MARVEL_DATA_INDEX_NAME, ClusterStateCollector.NODE_TYPE, nodeId).get().isExists(), is(true)); + assertThat(client().prepareGet(dataIndex, ClusterStateCollector.NODE_TYPE, nodeId).get().isExists(), is(true)); // checks that document is not indexed assertHitCount(client().prepareSearch().setSize(0) diff --git a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/shield/SecuredClientTests.java b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/shield/SecuredClientTests.java index c4afeece1b7..8a870a82804 100644 --- a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/shield/SecuredClientTests.java +++ b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/shield/SecuredClientTests.java @@ -36,13 +36,13 @@ public class SecuredClientTests extends MarvelIntegTestCase { assertAccessIsAllowed(securedClient.admin().indices().prepareDelete(MarvelSettings.MARVEL_INDICES_PREFIX)); assertAccessIsAllowed(securedClient.admin().indices().prepareCreate(MarvelSettings.MARVEL_INDICES_PREFIX + "test")); - assertAccessIsAllowed(securedClient.admin().indices().preparePutTemplate("foo").setSource(MarvelTemplateUtils.loadDefaultTemplate())); + assertAccessIsAllowed(securedClient.admin().indices().preparePutTemplate("foo").setSource(MarvelTemplateUtils.loadTimestampedIndexTemplate())); assertAccessIsAllowed(securedClient.admin().indices().prepareGetTemplates("foo")); } public void testDeniedAccess() { SecuredClient securedClient = internalCluster().getInstance(SecuredClient.class); - assertAcked(securedClient.admin().indices().preparePutTemplate("foo").setSource(MarvelTemplateUtils.loadDefaultTemplate()).get()); + assertAcked(securedClient.admin().indices().preparePutTemplate("foo").setSource(MarvelTemplateUtils.loadDataIndexTemplate()).get()); if (shieldEnabled) { assertAccessIsDenied(securedClient.admin().indices().prepareDeleteTemplate("foo")); diff --git a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/test/MarvelIntegTestCase.java b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/test/MarvelIntegTestCase.java index bc5237f3eeb..33fc17350f7 100644 --- a/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/test/MarvelIntegTestCase.java +++ b/elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/test/MarvelIntegTestCase.java @@ -6,14 +6,10 @@ package org.elasticsearch.marvel.test; import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.Version; -import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; -import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse; -import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexTemplateMetaData; -import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.Streams; +import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.CountDown; import org.elasticsearch.index.IndexModule; @@ -70,11 +66,6 @@ public abstract class MarvelIntegTestCase extends ESIntegTestCase { return super.buildTestCluster(scope, seed); } - @Override - protected Set excludeTemplates() { - return Collections.singleton(MarvelTemplateUtils.INDEX_TEMPLATE_NAME); - } - @Override protected Settings nodeSettings(int nodeOrdinal) { Settings.Builder builder = Settings.builder() @@ -212,63 +203,29 @@ public abstract class MarvelIntegTestCase extends ESIntegTestCase { } } - protected void assertMarvelTemplateInstalled() { - ClusterStateResponse clusterStateResponse = client().admin().cluster().prepareState().get(); - if (clusterStateResponse != null) { - ClusterState state = clusterStateResponse.getState(); - MetaData md = state.getMetaData(); - } - GetIndexTemplatesResponse response = client().admin().indices().prepareGetTemplates().get(); - for (IndexTemplateMetaData template : response.getIndexTemplates()) { - if (template.getName().equals(MarvelTemplateUtils.INDEX_TEMPLATE_NAME)) { - return; + protected void assertTemplateInstalled(String name) { + boolean found = false; + for (IndexTemplateMetaData template : client().admin().indices().prepareGetTemplates().get().getIndexTemplates()) { + if (Regex.simpleMatch(name, template.getName())) { + found = true; } } - fail("marvel template should exist"); + assertTrue("failed to find a template matching [" + name + "]", found); } - protected void assertMarvelTemplateMissing() { - for (IndexTemplateMetaData template : client().admin().indices().prepareGetTemplates(MarvelTemplateUtils.INDEX_TEMPLATE_NAME).get().getIndexTemplates()) { - if (template.getName().equals(MarvelTemplateUtils.INDEX_TEMPLATE_NAME)) { - fail("marvel template shouldn't exist"); - } - } - } - - protected void awaitMarvelTemplateInstalled() throws Exception { + protected void waitForMarvelTemplate(String name) throws Exception { assertBusy(new Runnable() { @Override public void run() { - assertMarvelTemplateInstalled(); + assertTemplateInstalled(name); } }, 30, TimeUnit.SECONDS); } - protected void awaitMarvelTemplateInstalled(Version version) throws Exception { - assertBusy(new Runnable() { - @Override - public void run() { - assertMarvelTemplateInstalled(version); - } - }, 30, TimeUnit.SECONDS); - } - - protected void assertMarvelTemplateInstalled(Version version) { - for (IndexTemplateMetaData template : client().admin().indices().prepareGetTemplates(MarvelTemplateUtils.INDEX_TEMPLATE_NAME).get().getIndexTemplates()) { - if (template.getName().equals(MarvelTemplateUtils.INDEX_TEMPLATE_NAME)) { - Version templateVersion = MarvelTemplateUtils.templateVersion(template); - if (templateVersion != null && templateVersion.id == version.id) { - return; - } - fail("did not find marvel template with expected version [" + version + "]. found version [" + templateVersion + "]"); - } - } - fail("marvel template could not be found"); - } - protected void waitForMarvelIndices() throws Exception { - awaitIndexExists(MarvelSettings.MARVEL_DATA_INDEX_NAME); - awaitIndexExists(MarvelSettings.MARVEL_INDICES_PREFIX + "*"); + String currentVersion = String.valueOf(MarvelTemplateUtils.TEMPLATE_VERSION); + awaitIndexExists(MarvelSettings.MARVEL_DATA_INDEX_PREFIX + currentVersion); + awaitIndexExists(MarvelSettings.MARVEL_INDICES_PREFIX + currentVersion + "-*"); assertBusy(new Runnable() { @Override public void run() {