honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Marvel: Use versioned index templates 

This commit changes the templates so that they are now versionned using a number (starting from 1). This number is used in index templates names (ex: .marvel-es-data-1, .marvel-es-1) as well as in indices names (ex: .marvel-es-1-2015-12-30, .marvel-es-data-1).

If the template does not exist, it is created. Otherwise nothing (no update) is done.

Original commit: elastic/x-pack-elasticsearch@66c1a8bed0
This commit is contained in:
Tanguy Leroux 2015-12-17 23:01:32 +01:00
parent a931e4e0b8
commit eeb5842730
24 changed files with 567 additions and 946 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
elasticsearch/x-pack/build.gradle
View File

@ -70,6 +70,9 @@ compileTestJava.options.compilerArgs << "-Xlint:-deprecation,-rawtypes,-serial,-
ext.expansions = [
'project.version': version,
// Used in marvel index templates
'marvel.plugin.version': version,
'marvel.template.version': '1',
]
processResources {
@ -82,8 +85,8 @@ processResources {
String licenseKeyPath = "license-plugin/keys/${licenseKeyName}/public.key"
if (file(licenseKeyPath).exists() == false) {
throw new GradleException("no public key found for '${licenseKeyName}'")
}
from licenseKeyPath
}
from licenseKeyPath
}
processTestResources {

28
elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/collector/AbstractCollector.java
View File

@ -11,11 +11,14 @@ import org.elasticsearch.cluster.ClusterService;
import org.elasticsearch.common.component.AbstractLifecycleComponent;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.marvel.agent.exporter.IndexNameResolver;
import org.elasticsearch.marvel.agent.exporter.MarvelDoc;
import org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils;
import org.elasticsearch.marvel.agent.settings.MarvelSettings;
import org.elasticsearch.marvel.license.MarvelLicensee;
import java.util.Collection;
import java.util.Objects;
public abstract class AbstractCollector<T> extends AbstractLifecycleComponent<T> implements Collector<T> {
@ -24,6 +27,7 @@ public abstract class AbstractCollector<T> extends AbstractLifecycleComponent<T>
protected final ClusterService clusterService;
protected final MarvelSettings marvelSettings;
protected final MarvelLicensee licensee;
protected final IndexNameResolver dataIndexNameResolver;
@Inject
public AbstractCollector(Settings settings, String name, ClusterService clusterService, MarvelSettings marvelSettings, MarvelLicensee licensee) {
@ -32,6 +36,7 @@ public abstract class AbstractCollector<T> extends AbstractLifecycleComponent<T>
this.clusterService = clusterService;
this.marvelSettings = marvelSettings;
this.licensee = licensee;
this.dataIndexNameResolver = new DataIndexNameResolver(MarvelTemplateUtils.TEMPLATE_VERSION);
}
@Override
@ -109,4 +114,27 @@ public abstract class AbstractCollector<T> extends AbstractLifecycleComponent<T>
protected String clusterUUID() {
return clusterService.state().metaData().clusterUUID();
}
/**
* Resolves marvel's data index name
*/
public class DataIndexNameResolver implements IndexNameResolver {
private final String index;
DataIndexNameResolver(Integer version) {
Objects.requireNonNull(version, "index version cannot be null");
this.index = MarvelSettings.MARVEL_DATA_INDEX_PREFIX + String.valueOf(version);
}
@Override
public String resolve(MarvelDoc doc) {
return index;
}
@Override
public String resolve(long timestamp) {
return index;
}
}
}

3
elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/collector/cluster/ClusterInfoCollector.java
View File

@ -77,7 +77,8 @@ public class ClusterInfoCollector extends AbstractCollector<ClusterInfoMarvelDoc
}
String clusterUUID = clusterUUID();
results.add(new ClusterInfoMarvelDoc(MarvelSettings.MARVEL_DATA_INDEX_NAME, TYPE, clusterUUID, clusterUUID, System.currentTimeMillis(),
long timestamp = System.currentTimeMillis();
results.add(new ClusterInfoMarvelDoc(dataIndexNameResolver.resolve(timestamp), TYPE, clusterUUID, clusterUUID, timestamp,
clusterName.value(), Version.CURRENT.toString(), license, clusterStats));
return Collections.unmodifiableCollection(results);
}

2
elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/collector/cluster/ClusterStateCollector.java
View File

@ -71,7 +71,7 @@ public class ClusterStateCollector extends AbstractCollector<ClusterStateCollect
results.add(new ClusterStateNodeMarvelDoc(clusterUUID, NODES_TYPE, timestamp, stateUUID, node.getId()));
// Adds a document for every node in the marvel data index (type "node")
results.add(new DiscoveryNodeMarvelDoc(MarvelSettings.MARVEL_DATA_INDEX_NAME, NODE_TYPE, node.getId(), clusterUUID, timestamp, node));
results.add(new DiscoveryNodeMarvelDoc(dataIndexNameResolver.resolve(timestamp), NODE_TYPE, node.getId(), clusterUUID, timestamp, node));
}
}

2
elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/Exporter.java
View File

@ -155,7 +155,7 @@ public abstract class Exporter {
@Override
public String resolve(long timestamp) {
return MarvelSettings.MARVEL_INDICES_PREFIX + indexTimeFormatter.print(timestamp);
return MarvelSettings.MARVEL_INDICES_PREFIX + String.valueOf(MarvelTemplateUtils.TEMPLATE_VERSION) + "-" + indexTimeFormatter.print(timestamp);
}
@Override

152
elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/MarvelTemplateUtils.java
View File

@ -5,113 +5,103 @@
*/
package org.elasticsearch.marvel.agent.exporter;
import org.elasticsearch.Version;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
import org.elasticsearch.cluster.metadata.MetaData;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.io.Streams;
import org.elasticsearch.common.logging.ESLogger;
import org.elasticsearch.marvel.support.VersionUtils;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.Properties;
/**
*
*/
public final class MarvelTemplateUtils {
public static final String MARVEL_TEMPLATE_FILE = "/marvel_index_template.json";
public static final String INDEX_TEMPLATE_NAME = ".marvel-es";
public static final String MARVEL_VERSION_FIELD = "marvel_version";
public static final Version MIN_SUPPORTED_TEMPLATE_VERSION = Version.V_2_0_0_beta2;
static final String INDEX_TEMPLATE_FILE = "/marvel-es.json";
static final String INDEX_TEMPLATE_NAME_PREFIX = ".marvel-es-";
static final String DATA_TEMPLATE_FILE = "/marvel-es-data.json";
static final String DATA_TEMPLATE_NAME_PREFIX = ".marvel-es-data-";
static final String PROPERTIES_FILE = "/marvel.properties";
static final String VERSION_FIELD = "marvel.template.version";
public static final Integer TEMPLATE_VERSION = loadTemplateVersion();
private MarvelTemplateUtils() {
}
/**
* Loads the default Marvel template
* Loads the default template for the timestamped indices
*/
public static byte[] loadDefaultTemplate() {
try (InputStream is = MarvelTemplateUtils.class.getResourceAsStream(MARVEL_TEMPLATE_FILE)) {
ByteArrayOutputStream out = new ByteArrayOutputStream();
Streams.copy(is, out);
return out.toByteArray();
public static byte[] loadTimestampedIndexTemplate() {
try {
return load(INDEX_TEMPLATE_FILE);
} catch (IOException e) {
throw new IllegalStateException("unable to load marvel template", e);
}
}
public static Version loadDefaultTemplateVersion() {
return parseTemplateVersion(loadDefaultTemplate());
}
public static Version templateVersion(IndexTemplateMetaData templateMetaData) {
String version = templateMetaData.settings().get("index." + MARVEL_VERSION_FIELD);
if (Strings.hasLength(version)) {
try {
return Version.fromString(version);
} catch (IllegalArgumentException e) {
return null;
}
}
return null;
}
public static IndexTemplateMetaData findMarvelTemplate(ClusterState state) {
MetaData metaData = state.getMetaData();
return metaData != null ? metaData.getTemplates().get(INDEX_TEMPLATE_NAME) : null;
}
public static Version parseTemplateVersion(byte[] template) {
/**
* Loads the default template for the data index
*/
public static byte[] loadDataIndexTemplate() {
try {
return VersionUtils.parseVersion(MARVEL_VERSION_FIELD, template);
} catch (IllegalArgumentException e) {
return load(DATA_TEMPLATE_FILE);
} catch (IOException e) {
throw new IllegalStateException("unable to load marvel data template", e);
}
}
/**
* Loads a resource with a given name and returns it as a byte array.
*/
static byte[] load(String name) throws IOException {
try (InputStream is = MarvelTemplateUtils.class.getResourceAsStream(name)) {
ByteArrayOutputStream out = new ByteArrayOutputStream();
Streams.copy(is, out);
return out.toByteArray();
}
}
/**
* Loads the current version of templates
*
* When executing tests in Intellij, the properties file might not be
* resolved: try running 'gradle processResources' first.
*/
static Integer loadTemplateVersion() {
try (InputStream is = MarvelTemplateUtils.class.getResourceAsStream(PROPERTIES_FILE)) {
Properties properties = new Properties();
properties.load(is);
String version = properties.getProperty(VERSION_FIELD);
if (Strings.hasLength(version)) {
return Integer.parseInt(version);
}
return null;
} catch (NumberFormatException e) {
throw new IllegalArgumentException("failed to parse marvel template version");
} catch (IOException e) {
throw new IllegalArgumentException("failed to load marvel template version");
}
}
public static boolean installedTemplateVersionIsSufficient(Version installed) {
// null indicates couldn't parse the version from the installed template, this means it is probably too old or invalid...
if (installed == null) {
return false;
}
// ensure the template is not too old
if (installed.before(MIN_SUPPORTED_TEMPLATE_VERSION)) {
return false;
}
// We do not enforce that versions are equivalent to the current version as we may be in a rolling upgrade scenario
// and until a master is elected with the new version, data nodes that have been upgraded will not be able to ship
// data. This means that there is an implication that the new shippers will ship data correctly even with an old template.
// There is also no upper bound and we rely on elasticsearch nodes not being able to connect to each other across major
// versions
return true;
public static String indexTemplateName() {
return indexTemplateName(TEMPLATE_VERSION);
}
public static boolean installedTemplateVersionMandatesAnUpdate(Version current, Version installed, ESLogger logger, String exporterName) {
if (installed == null) {
logger.debug("exporter [{}] - currently installed marvel template is missing a version - installing a new one [{}]", exporterName, current);
return true;
}
// Never update a very old template
if (installed.before(MIN_SUPPORTED_TEMPLATE_VERSION)) {
return false;
}
// Always update a template to the last up-to-date version
if (current.after(installed)) {
logger.debug("exporter [{}] - currently installed marvel template version [{}] will be updated to a newer version [{}]", exporterName, installed, current);
return true;
// When the template is up-to-date, do not update
} else if (current.equals(installed)) {
logger.debug("exporter [{}] - currently installed marvel template version [{}] is up-to-date", exporterName, installed);
return false;
// Never update a template that is newer than the expected one
} else {
logger.debug("exporter [{}] - currently installed marvel template version [{}] is newer than the one required [{}]... keeping it.", exporterName, installed, current);
return false;
}
public static String indexTemplateName(Integer version) {
return templateName(INDEX_TEMPLATE_NAME_PREFIX, version);
}
public static String dataTemplateName() {
return dataTemplateName(TEMPLATE_VERSION);
}
public static String dataTemplateName(Integer version) {
return templateName(DATA_TEMPLATE_NAME_PREFIX, version);
}
static String templateName(String prefix, Integer version) {
assert version != null && version >= 0 : "version must be not null and greater or equal to zero";
return prefix + String.valueOf(version);
}
}

214
elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/http/HttpExporter.java
View File

@ -8,11 +8,9 @@ package org.elasticsearch.marvel.agent.exporter.http;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.ExceptionsHelper;
import org.elasticsearch.Version;
import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest;
import org.elasticsearch.common.Base64;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.io.Streams;
import org.elasticsearch.common.io.stream.BytesStreamOutput;
@ -20,7 +18,6 @@ import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.settings.SettingsException;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentHelper;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.env.Environment;
@ -55,12 +52,7 @@ import java.nio.file.Path;
import java.security.KeyStore;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import static org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils.installedTemplateVersionIsSufficient;
import static org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils.installedTemplateVersionMandatesAnUpdate;
/**
*
@ -106,8 +98,8 @@ public class HttpExporter extends Exporter {
volatile boolean checkedAndUploadedIndexTemplate = false;
volatile boolean supportedClusterVersion = false;
/** Version of the built-in template **/
final Version templateVersion;
/** Version number of built-in templates **/
private final Integer templateVersion;
boolean keepAlive;
final ConnectionKeepAliveWorker keepAliveWorker;
@ -142,8 +134,8 @@ public class HttpExporter extends Exporter {
sslSocketFactory = createSSLSocketFactory(config.settings().getAsSettings(SSL_SETTING));
hostnameVerification = config.settings().getAsBoolean(SSL_HOSTNAME_VERIFICATION_SETTING, true);
// Checks that the built-in template is versioned
templateVersion = MarvelTemplateUtils.loadDefaultTemplateVersion();
// Loads the current version number of built-in templates
templateVersion = MarvelTemplateUtils.TEMPLATE_VERSION;
if (templateVersion == null) {
throw new IllegalStateException("unable to find built-in template version");
}
@ -202,7 +194,7 @@ public class HttpExporter extends Exporter {
builder.startObject();
builder.startObject("index");
// we need the index to be based on the document timestamp
// we need the index to be based on the document timestamp and/or template version
builder.field("_index", indexNameResolver.resolve(marvelDoc));
if (marvelDoc.type() != null) {
@ -395,101 +387,51 @@ public class HttpExporter extends Exporter {
* @return true if template exists or was uploaded successfully.
*/
private boolean checkAndUploadIndexTemplate(final String host) {
byte[] installedTemplate;
try {
installedTemplate = findMarvelTemplate(host);
} catch (Exception e) {
logger.debug("http exporter [{}] - exception when loading the existing marvel template on host[{}]", e, name(), host);
return false;
String templateName = MarvelTemplateUtils.indexTemplateName(templateVersion);
boolean templateInstalled = hasTemplate(templateName, host);
// Works like LocalExporter on master:
// Install the index template for timestamped indices first, so that other nodes can ship data
if (!templateInstalled) {
logger.debug("http exporter [{}] - could not find existing marvel template, installing a new one", name());
if (!putTemplate(host, templateName, MarvelTemplateUtils.loadTimestampedIndexTemplate())) {
return false;
}
}
// if we cannot find a template or a compatible template, we'll install one in / update it.
if (installedTemplate == null) {
logger.debug("http exporter [{}] - could not find existing marvel template, installing a new one", name());
return putTemplate(host);
}
Version installedTemplateVersion = MarvelTemplateUtils.parseTemplateVersion(installedTemplate);
if (installedTemplateVersionMandatesAnUpdate(templateVersion, installedTemplateVersion, logger, name())) {
logger.debug("http exporter [{}] - installing new marvel template [{}], replacing [{}]", name(), templateVersion, installedTemplateVersion);
return putTemplate(host);
} else if (!installedTemplateVersionIsSufficient(installedTemplateVersion)) {
logger.error("http exporter [{}] - marvel template version [{}] is below the minimum compatible version [{}]. "
+ "please manually update the marvel template to a more recent version"
+ "and delete the current active marvel index (don't forget to back up it first if needed)",
name(), installedTemplateVersion, MarvelTemplateUtils.MIN_SUPPORTED_TEMPLATE_VERSION);
// we're not going to do anything with the template.. it's too old, and the schema might
// be too different than what this version of marvel/es can work with. For this reason we're
// not going to export any data, to avoid mapping conflicts.
return false;
// Install the index template for data index
templateName = MarvelTemplateUtils.dataTemplateName(templateVersion);
if (!hasTemplate(templateName, host)) {
logger.debug("http exporter [{}] - could not find existing marvel template for data index, installing a new one", name());
if (!putTemplate(host, templateName, MarvelTemplateUtils.loadDataIndexTemplate())) {
return false;
}
}
return true;
}
private byte[] findMarvelTemplate(String host) throws IOException {
String url = "_template/" + MarvelTemplateUtils.INDEX_TEMPLATE_NAME;
private boolean hasTemplate(String templateName, String host) {
String url = "_template/" + templateName;
if (templateCheckTimeout != null) {
url += "?timeout=" + templateCheckTimeout;
}
HttpURLConnection connection = null;
try {
logger.debug("http exporter [{}] - checking if marvel template exists on the marvel cluster", name());
logger.debug("http exporter [{}] - checking if marvel template [{}] exists on the marvel cluster", name(), templateName);
connection = openConnection(host, "GET", url, null);
if (connection == null) {
throw new IOException("no available connection to check marvel template existence");
throw new IOException("no available connection to check for marvel template [" + templateName + "] existence");
}
byte[] remoteTemplate = null;
// 200 means that the template has been found, 404 otherwise
if (connection.getResponseCode() == 200) {
logger.debug("marvel template found");
try (InputStream is = connection.getInputStream()) {
ByteArrayOutputStream out = new ByteArrayOutputStream();
Streams.copy(is, out);
remoteTemplate = out.toByteArray();
}
logger.debug("marvel template [{}] found",templateName);
return true;
}
return remoteTemplate;
} catch (Exception e) {
logger.error("http exporter [{}] - failed to verify the marvel template to [{}]:\n{}", name(), host, e.getMessage());
throw e;
} finally {
if (connection != null) {
try {
connection.getInputStream().close();
} catch (IOException e) {
// Ignore
}
}
}
}
boolean putTemplate(String host) {
HttpURLConnection connection = null;
try {
connection = openConnection(host, "PUT", "_template/" + MarvelTemplateUtils.INDEX_TEMPLATE_NAME, XContentType.JSON.restContentType());
if (connection == null) {
logger.debug("http exporter [{}] - no available connection to update marvel template", name());
return false;
}
logger.debug("http exporter [{}] - loading marvel pre-configured template", name());
byte[] template = MarvelTemplateUtils.loadDefaultTemplate();
// Uploads the template and closes the outputstream
Streams.copy(template, connection.getOutputStream());
if (connection.getResponseCode() != 200 && connection.getResponseCode() != 201) {
logConnectionError("error adding the marvel template to [" + host + "]", connection);
return false;
}
logger.info("http exporter [{}] - marvel template updated to version [{}]", name(), templateVersion);
} catch (IOException e) {
logger.error("http exporter [{}] - failed to update the marvel template to [{}]:\n{}", name(), host, e.getMessage());
logger.error("http exporter [{}] - failed to verify the marvel template [{}] on [{}]:\n{}", name(), templateName, host, e.getMessage());
return false;
} finally {
if (connection != null) {
try {
@ -499,101 +441,31 @@ public class HttpExporter extends Exporter {
}
}
}
if (config.settings().getAsBoolean("update_mappings", true)) {
updateMappings(host, MarvelSettings.MARVEL_DATA_INDEX_NAME);
updateMappings(host, indexNameResolver().resolve(System.currentTimeMillis()));
}
return true;
return false;
}
// TODO: Remove this method once marvel indices are versioned (v 2.2.0)
void updateMappings(String host, String index) {
logger.trace("http exporter [{}] - updating mappings for index [{}]", name(), index);
// Parse the default template to get its mappings
PutIndexTemplateRequest template = new PutIndexTemplateRequest().source(MarvelTemplateUtils.loadDefaultTemplate());
if ((template == null) || (template.mappings() == null) || (template.mappings().isEmpty())) {
return;
}
Set<String> indexMappings = new HashSet<>();
boolean putTemplate(String host, String template, byte[] source) {
logger.debug("http exporter [{}] - installing template [{}]", name(), template);
HttpURLConnection connection = null;
try {
connection = openConnection(host, "GET", "/" + index + "/_mapping", XContentType.JSON.restContentType());
connection = openConnection(host, "PUT", "_template/" + template, XContentType.JSON.restContentType());
if (connection == null) {
logger.debug("http exporter [{}] - no available connection to get index mappings", name());
return;
}
if (connection.getResponseCode() == 404) {
logger.trace("http exporter [{}] - index [{}] does not exist", name(), index);
return;
} else if (connection.getResponseCode() == 200) {
try (InputStream is = connection.getInputStream()) {
ByteArrayOutputStream out = new ByteArrayOutputStream();
Streams.copy(is, out);
Map<String, Object> mappings = XContentHelper.convertToMap(new BytesArray(out.toByteArray()), false).v2();
if ((mappings.get(index) != null) && (mappings.get(index) instanceof Map)) {
Map m = (Map) ((Map) mappings.get(index)).get("mappings");
if (m != null) {
indexMappings = m.keySet();
}
}
}
} else {
logConnectionError("http exporter [" + name() +"] - failed to get mappings for index [" + index + "] on host [" + host + "]", connection);
return;
}
} catch (Exception e) {
logger.error("http exporter [{}] - failed to update the marvel template to [{}]:\n{}", name(), host, e.getMessage());
return;
} finally {
if (connection != null) {
try {
connection.getInputStream().close();
} catch (IOException e) {
// Ignore
}
}
}
// Iterates over document types defined in the default template
for (String type : template.mappings().keySet()) {
if (indexMappings.contains(type)) {
logger.trace("http exporter [{}] - type [{} already exists in mapping of index [{}]", name(), type, index);
continue;
}
logger.trace("http exporter [{}] - adding type [{}] to index [{}] mappings", name(), type, index);
updateMappingForType(host, index, type, template.mappings().get(type));
}
}
void updateMappingForType(String host, String index, String type, String mappingSource) {
logger.trace("http exporter [{}] - updating index [{}] mappings for type [{}] on host [{}]", name(), index, type, host);
HttpURLConnection connection = null;
try {
connection = openConnection(host, "PUT", "/" + index + "/_mapping/" + type, XContentType.JSON.restContentType());
if (connection == null) {
logger.debug("http exporter [{}] - no available connection to update index mapping", name());
return;
logger.debug("http exporter [{}] - no available connection to update marvel template [{}]", name(), template);
return false;
}
// Uploads the template and closes the outputstream
Streams.copy(Strings.toUTF8Bytes(mappingSource), connection.getOutputStream());
Streams.copy(source, connection.getOutputStream());
if (connection.getResponseCode() != 200 && connection.getResponseCode() != 201) {
logConnectionError("http exporter [" + name() +"] - mapping of index [" + index + "] failed to be updated for type [" + type + "] on host [" + host + "]", connection);
return;
logConnectionError("error adding the marvel template [" + template + "] to [" + host + "]", connection);
return false;
}
logger.trace("http exporter [{}] - mapping of index [{}] updated for type [{}]", name(), index, type);
} catch (Exception e) {
logger.error("http exporter [{}] - failed to update mapping of index [{}] for type [{}]", name(), index, type);
logger.info("http exporter [{}] - marvel template [{}] updated to version [{}]", name(), template, templateVersion);
return true;
} catch (IOException e) {
logger.error("http exporter [{}] - failed to update marvel template [{}] on host [{}]:\n{}", name(), template, host, e.getMessage());
return false;
} finally {
if (connection != null) {
try {

9
elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/local/LocalBulk.java
View File

@ -60,11 +60,10 @@ public class LocalBulk extends ExportBulk {
}
IndexRequestBuilder request = client.prepareIndex();
if (marvelDoc.index() != null) {
request.setIndex(marvelDoc.index());
} else {
request.setIndex(indexNameResolver.resolve(marvelDoc));
}
// we need the index to be based on the document timestamp and/or template version
request.setIndex(indexNameResolver.resolve(marvelDoc));
if (marvelDoc.type() != null) {
request.setType(marvelDoc.type());
}

189
elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/local/LocalExporter.java
View File

@ -5,12 +5,8 @@
*/
package org.elasticsearch.marvel.agent.exporter.local;
import org.elasticsearch.Version;
import com.carrotsearch.hppc.cursors.ObjectCursor;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest;
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse;
import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest;
import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateResponse;
import org.elasticsearch.client.Client;
@ -18,22 +14,17 @@ import org.elasticsearch.cluster.ClusterChangedEvent;
import org.elasticsearch.cluster.ClusterService;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.ClusterStateListener;
import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
import org.elasticsearch.cluster.metadata.MappingMetaData;
import org.elasticsearch.cluster.block.ClusterBlockLevel;
import org.elasticsearch.common.collect.ImmutableOpenMap;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.regex.Regex;
import org.elasticsearch.gateway.GatewayService;
import org.elasticsearch.index.IndexNotFoundException;
import org.elasticsearch.marvel.agent.exporter.ExportBulk;
import org.elasticsearch.marvel.agent.exporter.Exporter;
import org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils;
import org.elasticsearch.marvel.agent.renderer.RendererRegistry;
import org.elasticsearch.marvel.agent.settings.MarvelSettings;
import org.elasticsearch.marvel.shield.SecuredClient;
import static org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils.installedTemplateVersionIsSufficient;
import static org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils.installedTemplateVersionMandatesAnUpdate;
/**
*
*/
@ -48,8 +39,8 @@ public class LocalExporter extends Exporter implements ClusterStateListener {
private volatile LocalBulk bulk;
private volatile boolean active = true;
/** Version of the built-in template **/
private final Version templateVersion;
/** Version number of built-in templates **/
private final Integer templateVersion;
public LocalExporter(Exporter.Config config, Client client, ClusterService clusterService, RendererRegistry renderers) {
super(TYPE, config);
@ -57,8 +48,8 @@ public class LocalExporter extends Exporter implements ClusterStateListener {
this.clusterService = clusterService;
this.renderers = renderers;
// Checks that the built-in template is versioned
templateVersion = MarvelTemplateUtils.loadDefaultTemplateVersion();
// Loads the current version number of built-in templates
templateVersion = MarvelTemplateUtils.TEMPLATE_VERSION;
if (templateVersion == null) {
throw new IllegalStateException("unable to find built-in template version");
}
@ -126,21 +117,17 @@ public class LocalExporter extends Exporter implements ClusterStateListener {
return null;
}
IndexTemplateMetaData installedTemplate = MarvelTemplateUtils.findMarvelTemplate(clusterState);
String templateName = MarvelTemplateUtils.indexTemplateName(templateVersion);
boolean templateInstalled = hasTemplate(templateName, clusterState);
// if this is not the master, we'll just look to see if the marvel template is already
// if this is not the master, we'll just look to see if the marvel timestamped template is already
// installed and if so, if it has a compatible version. If it is (installed and compatible)
// we'll be able to start this exporter. Otherwise, we'll just wait for a new cluster state.
if (!clusterService.localNode().masterNode()) {
if (installedTemplate == null) {
// the marvel template is not yet installed in the given cluster state, we'll wait.
logger.debug("local exporter [{}] - marvel index template [{}] does not exist, so service cannot start", name(), MarvelTemplateUtils.INDEX_TEMPLATE_NAME);
return null;
}
Version installedTemplateVersion = MarvelTemplateUtils.templateVersion(installedTemplate);
if (!installedTemplateVersionIsSufficient(installedTemplateVersion)) {
logger.debug("local exporter [{}] - cannot start. the currently installed marvel template (version [{}]) is incompatible with the " +
"current elasticsearch version [{}]. waiting until the template is updated", name(), installedTemplateVersion, Version.CURRENT);
// We only need to check the index template for timestamped indices
if (!templateInstalled) {
// the template for timestamped indices is not yet installed in the given cluster state, we'll wait.
logger.debug("local exporter [{}] - marvel index template does not exist, so service cannot start", name());
return null;
}
@ -151,36 +138,65 @@ public class LocalExporter extends Exporter implements ClusterStateListener {
// we are on master
//
// if we cannot find a template or a compatible template, we'll install one in / update it.
if (installedTemplate == null) {
logger.debug("local exporter [{}] - could not find existing marvel template, installing a new one", name());
putTemplate();
// we'll get that template on the next cluster state update
return null;
}
Version installedTemplateVersion = MarvelTemplateUtils.templateVersion(installedTemplate);
if (installedTemplateVersionMandatesAnUpdate(templateVersion, installedTemplateVersion, logger, name())) {
logger.debug("local exporter [{}] - installing new marvel template [{}], replacing [{}]", name(), templateVersion, installedTemplateVersion);
putTemplate();
// we'll get that template on the next cluster state update
return null;
} else if (!installedTemplateVersionIsSufficient(installedTemplateVersion)) {
logger.error("local exporter [{}] - marvel template version [{}] is below the minimum compatible version [{}]. "
+ "please manually update the marvel template to a more recent version"
+ "and delete the current active marvel index (don't forget to back up it first if needed)",
name(), installedTemplateVersion, MarvelTemplateUtils.MIN_SUPPORTED_TEMPLATE_VERSION);
// we're not going to do anything with the template.. it's too old, and the schema might
// be too different than what this version of marvel/es can work with. For this reason we're
// not going to export any data, to avoid mapping conflicts.
// Check that there is nothing that could block metadata updates
if (clusterState.blocks().hasGlobalBlock(ClusterBlockLevel.METADATA_WRITE)) {
logger.debug("local exporter [{}] - waiting until metadata writes are unblocked", name());
return null;
}
// ok.. we have a compatible template... we can start
// Install the index template for timestamped indices first, so that other nodes can ship data
if (!templateInstalled) {
logger.debug("local exporter [{}] - could not find existing marvel template for timestamped indices, installing a new one", name());
putTemplate(templateName, MarvelTemplateUtils.loadTimestampedIndexTemplate());
// we'll get that template on the next cluster state update
return null;
}
// Install the index template for data index
templateName = MarvelTemplateUtils.dataTemplateName(templateVersion);
if (!hasTemplate(templateName, clusterState)) {
logger.debug("local exporter [{}] - could not find existing marvel template for data index, installing a new one", name());
putTemplate(templateName, MarvelTemplateUtils.loadDataIndexTemplate());
// we'll get that template on the next cluster state update
return null;
}
// ok.. we have a compatible templates... we can start
return currentBulk != null ? currentBulk : new LocalBulk(name(), logger, client, indexNameResolver, renderers);
}
void putTemplate() {
PutIndexTemplateRequest request = new PutIndexTemplateRequest(MarvelTemplateUtils.INDEX_TEMPLATE_NAME).source(MarvelTemplateUtils.loadDefaultTemplate());
/**
* List templates that exists in cluster state metadata and that match a given template name pattern.
*/
private ImmutableOpenMap<String, Integer> findTemplates(String templatePattern, ClusterState state) {
if (state == null || state.getMetaData() == null || state.getMetaData().getTemplates().isEmpty()) {
return ImmutableOpenMap.of();
}
ImmutableOpenMap.Builder<String, Integer> templates = ImmutableOpenMap.builder();
for (ObjectCursor<String> template : state.metaData().templates().keys()) {
if (Regex.simpleMatch(templatePattern, template.value)) {
try {
Integer version = Integer.parseInt(template.value.substring(templatePattern.length() - 1));
templates.put(template.value, version);
logger.debug("found index template [{}] in version [{}]", template, version);
} catch (NumberFormatException e) {
logger.warn("cannot extract version number for template [{}]", template.value);
}
}
}
return templates.build();
}
private boolean hasTemplate(String templateName, ClusterState state) {
ImmutableOpenMap<String, Integer> templates = findTemplates(templateName, state);
return templates.size() > 0;
}
void putTemplate(String template, byte[] source) {
logger.debug("local exporter [{}] - installing template [{}]", name(), template);
PutIndexTemplateRequest request = new PutIndexTemplateRequest(template).source(source);
assert !Thread.currentThread().isInterrupted() : "current thread has been interrupted before putting index template!!!";
// async call, so we won't block cluster event thread
@ -188,80 +204,15 @@ public class LocalExporter extends Exporter implements ClusterStateListener {
@Override
public void onResponse(PutIndexTemplateResponse response) {
if (response.isAcknowledged()) {
logger.trace("local exporter [{}] - successfully installed marvel template", name());
if (config.settings().getAsBoolean("update_mappings", true)) {
updateMappings(MarvelSettings.MARVEL_DATA_INDEX_NAME);
updateMappings(indexNameResolver().resolve(System.currentTimeMillis()));
}
logger.trace("local exporter [{}] - successfully installed marvel template [{}]", name(), template);
} else {
logger.error("local exporter [{}] - failed to update marvel index template", name());
logger.error("local exporter [{}] - failed to update marvel index template [{}]", name(), template);
}
}
@Override
public void onFailure(Throwable throwable) {
logger.error("local exporter [{}] - failed to update marvel index template", throwable, name());
}
});
}
// TODO: Remove this method once marvel indices are versioned (v 2.2.0)
void updateMappings(String index) {
logger.trace("local exporter [{}] - updating mappings for index [{}]", name(), index);
// Parse the default template to get its mappings
PutIndexTemplateRequest template = new PutIndexTemplateRequest().source(MarvelTemplateUtils.loadDefaultTemplate());
if ((template == null) || (template.mappings() == null) || (template.mappings().isEmpty())) {
return;
}
// async call, so we won't block cluster event thread
client.admin().indices().getMappings(new GetMappingsRequest().indices(index), new ActionListener<GetMappingsResponse>() {
@Override
public void onResponse(GetMappingsResponse response) {
ImmutableOpenMap<String, MappingMetaData> indexMappings = response.getMappings().get(index);
if (indexMappings != null) {
// Iterates over document types defined in the default template
for (String type : template.mappings().keySet()) {
if (indexMappings.get(type) != null) {
logger.trace("local exporter [{}] - type [{} already exists in mapping of index [{}]", name(), type, index);
continue;
}
logger.trace("local exporter [{}] - adding type [{}] to index [{}] mappings", name(), type, index);
updateMappingForType(index, type, template.mappings().get(type));
}
}
}
@Override
public void onFailure(Throwable e) {
if (e instanceof IndexNotFoundException) {
logger.trace("local exporter [{}] - index [{}] not found, unable to update mappings", name(), index);
} else {
logger.error("local exporter [{}] - failed to get mappings for index [{}]", name(), index);
}
}
});
}
void updateMappingForType(String index, String type, String mappingSource) {
logger.trace("local exporter [{}] - updating index [{}] mappings for type [{}]", name(), index, type);
client.admin().indices().putMapping(new PutMappingRequest(index).type(type).source(mappingSource), new ActionListener<PutMappingResponse>() {
@Override
public void onResponse(PutMappingResponse response) {
if (response.isAcknowledged()) {
logger.trace("local exporter [{}] - mapping of index [{}] updated for type [{}]", name(), index, type);
} else {
logger.trace("local exporter [{}] - mapping of index [{}] failed to be updated for type [{}]", name(), index, type);
}
}
@Override
public void onFailure(Throwable e) {
logger.error("local exporter [{}] - failed to update mapping of index [{}] for type [{}]", name(), index, type);
logger.error("local exporter [{}] - failed to update marvel index template [{}]", throwable, name(), template);
}
});
}

2
elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/settings/MarvelSettings.java
View File

@ -22,7 +22,7 @@ public class MarvelSettings extends AbstractComponent {
private static final String PREFIX = MarvelPlugin.NAME + ".agent.";
public static final String MARVEL_INDICES_PREFIX = ".marvel-es-";
public static final String MARVEL_DATA_INDEX_NAME = MARVEL_INDICES_PREFIX + "data";
public static final String MARVEL_DATA_INDEX_PREFIX = MARVEL_INDICES_PREFIX + "data-";
public static final TimeValue MAX_LICENSE_GRACE_PERIOD = TimeValue.timeValueHours(7 * 24);
/** Sampling interval between two collections (default to 10s) */

25
elasticsearch/x-pack/marvel/src/main/resources/marvel-es-data.json Normal file
View File

@ -0,0 +1,25 @@
{
"template": ".marvel-es-data-${marvel.template.version}",
"settings": {
"marvel.plugin.version": "${marvel.plugin.version}",
"marvel.template.version": "${marvel.template.version}",
"index.number_of_shards": 1,
"index.number_of_replicas": 1,
"index.codec": "best_compression",
"index.mapper.dynamic": false
},
"mappings": {
"_default_": {
"_all": {
"enabled": false
},
"date_detection": false
},
"cluster_info": {
"enabled": false
},
"node": {
"enabled": false
}
}
}

11
elasticsearch/x-pack/marvel/src/main/resources/marvel_index_template.json → elasticsearch/x-pack/marvel/src/main/resources/marvel-es.json
View File

@ -1,7 +1,8 @@
{
"template": ".marvel-es-*",
"template": ".marvel-es-${marvel.template.version}-*",
"settings": {
"marvel_version": "${project.version}",
"marvel.plugin.version": "${marvel.plugin.version}",
"marvel.template.version": "${marvel.template.version}",
"index.number_of_shards": 1,
"index.number_of_replicas": 1,
"index.codec": "best_compression",
@ -196,12 +197,6 @@
}
}
},
"cluster_info": {
"enabled": false
},
"node": {
"enabled": false
},
"nodes": {
"properties": {
"state_uuid": {

1
elasticsearch/x-pack/marvel/src/main/resources/marvel.properties Normal file
View File

@ -0,0 +1 @@
marvel.template.version=${marvel.template.version}

3
elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/collector/cluster/ClusterStateCollectorTests.java
View File

@ -11,6 +11,7 @@ import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.marvel.agent.collector.AbstractCollectorTestCase;
import org.elasticsearch.marvel.agent.exporter.MarvelDoc;
import org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils;
import org.elasticsearch.marvel.agent.settings.MarvelSettings;
import org.elasticsearch.marvel.license.MarvelLicensee;
@ -182,7 +183,7 @@ public class ClusterStateCollectorTests extends AbstractCollectorTestCase {
case ClusterStateCollector.NODE_TYPE:
DiscoveryNodeMarvelDoc discoveryNodeMarvelDoc = (DiscoveryNodeMarvelDoc) marvelDoc;
assertThat(discoveryNodeMarvelDoc.index(), equalTo(MarvelSettings.MARVEL_DATA_INDEX_NAME));
assertThat(discoveryNodeMarvelDoc.index(), equalTo(MarvelSettings.MARVEL_DATA_INDEX_PREFIX + MarvelTemplateUtils.TEMPLATE_VERSION));
assertThat(discoveryNodeMarvelDoc.id(), not(isEmptyOrNullString()));
assertNotNull(discoveryNodeMarvelDoc.getNode());
discoveryNodes.add(discoveryNodeMarvelDoc);

253
elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/AbstractExporterTemplateTestCase.java
View File

@ -5,7 +5,6 @@
*/
package org.elasticsearch.marvel.agent.exporter;
import org.elasticsearch.Version;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.marvel.agent.collector.Collector;
@ -14,9 +13,6 @@ import org.elasticsearch.marvel.agent.collector.node.NodeStatsCollector;
import org.elasticsearch.marvel.agent.settings.MarvelSettings;
import org.elasticsearch.marvel.test.MarvelIntegTestCase;
import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
import org.elasticsearch.test.VersionUtils;
import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;
import java.io.IOException;
import java.util.ArrayList;
@ -25,11 +21,15 @@ import java.util.List;
import java.util.Map;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils.dataTemplateName;
import static org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils.indexTemplateName;
import static org.elasticsearch.test.ESIntegTestCase.Scope.TEST;
@ClusterScope(scope = TEST, numDataNodes = 0, numClientNodes = 0, transportClientRatio = 0.0)
public abstract class AbstractExporterTemplateTestCase extends MarvelIntegTestCase {
private final Integer currentVersion = MarvelTemplateUtils.TEMPLATE_VERSION;
@Override
protected Settings nodeSettings(int nodeOrdinal) {
Settings.Builder settings = Settings.builder()
@ -44,149 +44,116 @@ public abstract class AbstractExporterTemplateTestCase extends MarvelIntegTestCa
protected abstract Settings exporterSettings();
protected abstract void deleteTemplate() throws Exception;
protected abstract void deleteTemplates() throws Exception;
protected abstract void putTemplate(String version) throws Exception;
protected abstract void putTemplate(String name, int version) throws Exception;
protected abstract void createMarvelIndex(String index) throws Exception;
protected abstract void assertTemplateExist(String name) throws Exception;
protected abstract void assertTemplateUpdated(Version version) throws Exception;
protected abstract void assertTemplateNotUpdated(String name) throws Exception;
protected abstract void assertTemplateNotUpdated(Version version) throws Exception;
protected abstract void assertMappingsUpdated(String... indices) throws Exception;
protected abstract void assertMappingsNotUpdated(String... indices) throws Exception;
protected abstract void assertIndicesNotCreated() throws Exception;
public void testCreateWhenNoExistingTemplate() throws Exception {
public void testCreateWhenNoExistingTemplates() throws Exception {
internalCluster().startNode();
deleteTemplate();
deleteTemplates();
doExporting();
logger.debug("--> template does not exist: it should have been created in the current version");
assertTemplateUpdated(currentVersion());
logger.debug("--> templates does not exist: it should have been created in the current version");
assertTemplateExist(indexTemplateName());
assertTemplateExist(dataTemplateName());
doExporting();
logger.debug("--> mappings should be up-to-date");
assertMappingsUpdated(currentIndices());
logger.debug("--> indices should have been created");
awaitIndexExists(currentDataIndexName());
awaitIndexExists(currentTimestampedIndexName());
}
public void testUpdateWhenExistingTemplateHasNoVersion() throws Exception {
public void testCreateWhenExistingTemplatesAreOld() throws Exception {
internalCluster().startNode();
putTemplate("");
doExporting();
logger.debug("--> existing template does not have a version: it should be updated to the current version");
assertTemplateUpdated(currentVersion());
final Integer version = randomIntBetween(0, currentVersion - 1);
putTemplate(indexTemplateName(version), version);
putTemplate(dataTemplateName(version), version);
doExporting();
logger.debug("--> mappings should be up-to-date");
assertMappingsUpdated(currentIndices());
logger.debug("--> existing templates are old");
assertTemplateExist(dataTemplateName(version));
assertTemplateExist(indexTemplateName(version));
logger.debug("--> existing templates are old: new templates should be created");
assertTemplateExist(indexTemplateName());
assertTemplateExist(dataTemplateName());
doExporting();
logger.debug("--> indices should have been created");
awaitIndexExists(currentDataIndexName());
awaitIndexExists(currentTimestampedIndexName());
}
public void testUpdateWhenExistingTemplateHasWrongVersion() throws Exception {
public void testCreateWhenExistingTemplateAreUpToDate() throws Exception {
internalCluster().startNode();
putTemplate(randomAsciiOfLength(5));
doExporting();
logger.debug("--> existing template has a wrong version: it should be updated to the current version");
assertTemplateUpdated(currentVersion());
putTemplate(indexTemplateName(currentVersion), currentVersion);
putTemplate(dataTemplateName(currentVersion), currentVersion);
doExporting();
logger.debug("--> mappings should be up-to-date");
assertMappingsUpdated(currentIndices());
logger.debug("--> existing templates are up to date");
assertTemplateExist(indexTemplateName());
assertTemplateExist(dataTemplateName());
logger.debug("--> existing templates has the same version: they should not be changed");
assertTemplateNotUpdated(indexTemplateName());
assertTemplateNotUpdated(dataTemplateName());
doExporting();
logger.debug("--> indices should have been created");
awaitIndexExists(currentDataIndexName());
awaitIndexExists(currentTimestampedIndexName());
}
public void testNoUpdateWhenExistingTemplateIsTooOld() throws Exception {
public void testRandomTemplates() throws Exception {
internalCluster().startNode();
putTemplate(VersionUtils.getFirstVersion().number());
doExporting();
logger.debug("--> existing template is too old: it should not be updated");
assertTemplateNotUpdated(VersionUtils.getFirstVersion());
doExporting();
logger.debug("--> existing template is too old: no data is exported");
assertIndicesNotCreated();
}
public void testUpdateWhenExistingTemplateIsOld() throws Exception {
internalCluster().startNode();
putTemplate(VersionUtils.getPreviousVersion(currentVersion()).number());
doExporting();
logger.debug("--> existing template is old but supported: it should be updated to the current version");
assertTemplateUpdated(currentVersion());
doExporting();
logger.debug("--> mappings should be up-to-date");
assertMappingsUpdated(currentIndices());
}
public void testUpdateWhenExistingTemplateIsUpToDate() throws Exception {
internalCluster().startNode();
putTemplate(currentVersion().toString());
doExporting();
logger.debug("--> existing template has the same version: it should not be updated");
assertTemplateNotUpdated(currentVersion());
doExporting();
logger.debug("--> mappings should not have been updated");
assertMappingsNotUpdated(currentIndices());
}
public void testMappingsUpdate() throws Exception {
boolean updateMappings = randomBoolean();
logger.debug("--> update_mappings is {}", updateMappings);
internalCluster().startNode(Settings.builder().put("marvel.agent.exporters._exporter.update_mappings", updateMappings));
logger.debug("--> putting a template with a very old version so that it will not be updated");
putTemplate(VersionUtils.getFirstVersion().toString());
logger.debug("--> creating marvel data index");
createMarvelIndex(MarvelSettings.MARVEL_DATA_INDEX_NAME);
logger.debug("--> creating a cold marvel index");
createMarvelIndex(coldIndex());
logger.debug("--> creating an active marvel index");
createMarvelIndex(hotIndex());
logger.debug("--> all indices have a old mapping now");
assertMappingsNotUpdated(coldIndex(), hotIndex(), MarvelSettings.MARVEL_DATA_INDEX_NAME);
logger.debug("--> updating the template with a previous version, so that it will be updated when exporting documents");
putTemplate(VersionUtils.getPreviousVersion(currentVersion()).number());
doExporting();
logger.debug("--> existing template is old: it should be updated to the current version");
assertTemplateUpdated(currentVersion());
logger.debug("--> cold marvel index: mappings should not have been updated");
assertMappingsNotUpdated(coldIndex());
if (updateMappings) {
logger.debug("--> marvel indices: mappings should be up-to-date");
assertMappingsUpdated(MarvelSettings.MARVEL_DATA_INDEX_NAME, hotIndex());
} else {
logger.debug("--> marvel indices: mappings should bnot have been updated");
assertMappingsNotUpdated(MarvelSettings.MARVEL_DATA_INDEX_NAME, hotIndex());
int previousIndexTemplateVersion = rarely() ? currentVersion : randomIntBetween(0, currentVersion - 1);
boolean previousIndexTemplateExist = randomBoolean();
if (previousIndexTemplateExist) {
logger.debug("--> creating index template in version [{}]", previousIndexTemplateVersion);
putTemplate(indexTemplateName(previousIndexTemplateVersion), previousIndexTemplateVersion);
}
int previousDataTemplateVersion = rarely() ? currentVersion : randomIntBetween(0, currentVersion - 1);
boolean previousDataTemplateExist = randomBoolean();
if (previousDataTemplateExist) {
logger.debug("--> creating data template in version [{}]", previousDataTemplateVersion);
putTemplate(dataTemplateName(previousDataTemplateVersion), previousDataTemplateVersion);
}
doExporting();
logger.debug("--> templates should exist in current version");
assertTemplateExist(indexTemplateName());
assertTemplateExist(dataTemplateName());
if (previousIndexTemplateExist) {
logger.debug("--> index template should exist in version [{}]", previousIndexTemplateVersion);
assertTemplateExist(indexTemplateName(previousIndexTemplateVersion));
}
if (previousDataTemplateExist) {
logger.debug("--> data template should exist in version [{}]", previousDataTemplateVersion);
assertTemplateExist(dataTemplateName(previousDataTemplateVersion));
}
doExporting();
logger.debug("--> indices should exist in current version");
awaitIndexExists(currentDataIndexName());
awaitIndexExists(currentTimestampedIndexName());
}
protected void doExporting() throws Exception {
@ -203,59 +170,23 @@ public abstract class AbstractExporterTemplateTestCase extends MarvelIntegTestCa
return exporters.iterator().next();
}
private Version currentVersion() {
return MarvelTemplateUtils.loadDefaultTemplateVersion();
private String currentDataIndexName() {
return ".marvel-es-data-" + String.valueOf(currentVersion);
}
private String[] currentIndices() {
return new String[]{hotIndex(), MarvelSettings.MARVEL_DATA_INDEX_NAME};
}
private String coldIndex() {
return exporter().indexNameResolver().resolve(new DateTime(2012, 3, 10, 0, 0, DateTimeZone.UTC).getMillis());
}
private String hotIndex() {
private String currentTimestampedIndexName() {
return exporter().indexNameResolver().resolve(System.currentTimeMillis());
}
/** Generates a template that looks like an old one **/
protected static BytesReference generateTemplateSource(String version) throws IOException {
/** Generates a basic template **/
protected static BytesReference generateTemplateSource(String name, Integer version) throws IOException {
return jsonBuilder().startObject()
.field("template", ".marvel-es-*")
.field("template", name)
.startObject("settings")
.field("index.number_of_shards", 1)
.field("index.number_of_replicas", 1)
.field("index.mapper.dynamic", false)
.field(MarvelTemplateUtils.MARVEL_VERSION_FIELD, version)
.field(MarvelTemplateUtils.VERSION_FIELD, String.valueOf(version))
.endObject()
.startObject("mappings")
.startObject("_default_")
.startObject("_all")
.field("enabled", false)
.endObject()
.field("date_detection", false)
.startObject("properties")
.startObject("cluster_uuid")
.field("type", "string")
.field("index", "not_analyzed")
.endObject()
.startObject("timestamp")
.field("type", "date")
.field("format", "date_time")
.endObject()
.endObject()
.endObject()
.startObject("cluster_info")
.field("enabled", false)
.endObject()
.startObject("node_stats")
.startObject("properties")
.startObject("node_stats")
.field("type", "object")
.endObject()
.endObject()
.endObject()
.endObject().bytes();
.endObject().bytes();
}
}

99
elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/MarvelTemplateUtilsTests.java
View File

@ -5,89 +5,54 @@
*/
package org.elasticsearch.marvel.agent.exporter;
import org.elasticsearch.Version;
import org.elasticsearch.common.Strings;
import org.elasticsearch.marvel.support.VersionUtils;
import org.elasticsearch.test.ESTestCase;
import org.hamcrest.Matchers;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import static org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils.MARVEL_VERSION_FIELD;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.greaterThan;
//import static org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils.MARVEL_VERSION_FIELD;
public class MarvelTemplateUtilsTests extends ESTestCase {
public void testLoadTemplate() {
byte[] template = MarvelTemplateUtils.loadDefaultTemplate();
public void testLoadTimestampedIndexTemplate() {
byte[] template = MarvelTemplateUtils.loadTimestampedIndexTemplate();
assertNotNull(template);
assertThat(template.length, Matchers.greaterThan(0));
assertThat(template.length, greaterThan(0));
}
public void testParseTemplateVersionFromByteArrayTemplate() throws IOException {
byte[] template = MarvelTemplateUtils.loadDefaultTemplate();
public void testLoadDataIndexTemplate() {
byte[] template = MarvelTemplateUtils.loadDataIndexTemplate();
assertNotNull(template);
assertThat(template.length, greaterThan(0));
}
Version version = MarvelTemplateUtils.parseTemplateVersion(template);
public void testLoad() throws IOException {
String resource = randomFrom(MarvelTemplateUtils.INDEX_TEMPLATE_FILE, MarvelTemplateUtils.DATA_TEMPLATE_FILE);
byte[] template = MarvelTemplateUtils.load(resource);
assertNotNull(template);
assertThat(template.length, greaterThan(0));
}
public void testLoadTemplateVersion() {
Integer version = MarvelTemplateUtils.loadTemplateVersion();
assertNotNull(version);
assertThat(version, greaterThan(0));
assertThat(version, equalTo(MarvelTemplateUtils.TEMPLATE_VERSION));
}
public void testParseTemplateVersionFromStringTemplate() throws IOException {
List<String> templates = new ArrayList<>();
templates.add("{\"marvel_version\": \"1.4.0.Beta1\"}");
templates.add("{\"marvel_version\": \"1.6.2-SNAPSHOT\"}");
templates.add("{\"marvel_version\": \"1.7.1\"}");
templates.add("{\"marvel_version\": \"2.0.0-beta1\"}");
templates.add("{\"marvel_version\": \"2.0.0\"}");
templates.add("{ \"template\": \".marvel*\", \"settings\": { \"marvel_version\": \"2.0.0-beta1-SNAPSHOT\", \"index.number_of_shards\": 1 } }");
for (String template : templates) {
Version version = MarvelTemplateUtils.parseTemplateVersion(Strings.toUTF8Bytes(template));
assertNotNull(version);
}
Version version = MarvelTemplateUtils.parseTemplateVersion(Strings.toUTF8Bytes("{\"marvel.index_format\": \"7\"}"));
assertNull(version);
public void testIndexTemplateName() {
assertThat(MarvelTemplateUtils.indexTemplateName(),
equalTo(MarvelTemplateUtils.INDEX_TEMPLATE_NAME_PREFIX + MarvelTemplateUtils.TEMPLATE_VERSION));
int version = randomIntBetween(1, 100);
assertThat(MarvelTemplateUtils.indexTemplateName(version), equalTo(".marvel-es-" + version));
}
public void testParseVersion() throws IOException {
assertNotNull(VersionUtils.parseVersion(MARVEL_VERSION_FIELD, "{\"marvel_version\": \"2.0.0-beta1\"}"));
assertNotNull(VersionUtils.parseVersion(MARVEL_VERSION_FIELD, "{\"marvel_version\": \"2.0.0\"}"));
assertNotNull(VersionUtils.parseVersion(MARVEL_VERSION_FIELD, "{\"marvel_version\": \"1.5.2\"}"));
assertNotNull(VersionUtils.parseVersion(MARVEL_VERSION_FIELD, "{ \"template\": \".marvel*\", \"settings\": { \"marvel_version\": \"2.0.0-beta1-SNAPSHOT\", \"index.number_of_shards\": 1 } }"));
assertNull(VersionUtils.parseVersion(MARVEL_VERSION_FIELD, "{\"marvel.index_format\": \"7\"}"));
assertNull(VersionUtils.parseVersion(MARVEL_VERSION_FIELD + "unkown", "{\"marvel_version\": \"1.5.2\"}"));
}
public void testTemplateVersionMandatesAnUpdate() {
// Version is unknown
assertTrue(MarvelTemplateUtils.installedTemplateVersionMandatesAnUpdate(Version.CURRENT, null, logger, "unit-test"));
// Version is too old
Version unsupported = org.elasticsearch.test.VersionUtils.getPreviousVersion(MarvelTemplateUtils.MIN_SUPPORTED_TEMPLATE_VERSION);
assertFalse(MarvelTemplateUtils.installedTemplateVersionMandatesAnUpdate(Version.CURRENT, unsupported, logger, "unit-test"));
// Version is old but supported
assertTrue(MarvelTemplateUtils.installedTemplateVersionMandatesAnUpdate(Version.CURRENT, MarvelTemplateUtils.MIN_SUPPORTED_TEMPLATE_VERSION, logger, "unit-test"));
// Version is up to date
assertFalse(MarvelTemplateUtils.installedTemplateVersionMandatesAnUpdate(Version.CURRENT, Version.CURRENT, logger, "unit-test"));
// Version is up to date
Version previous = org.elasticsearch.test.VersionUtils.getPreviousVersion(Version.CURRENT);
assertFalse(MarvelTemplateUtils.installedTemplateVersionMandatesAnUpdate(previous, Version.CURRENT, logger, "unit-test"));
}
public void testTemplateVersionIsSufficient() {
// Version is unknown
assertFalse(MarvelTemplateUtils.installedTemplateVersionIsSufficient(null));
// Version is too old
Version unsupported = org.elasticsearch.test.VersionUtils.getPreviousVersion(MarvelTemplateUtils.MIN_SUPPORTED_TEMPLATE_VERSION);
assertFalse(MarvelTemplateUtils.installedTemplateVersionIsSufficient(unsupported));
// Version is OK
assertTrue(MarvelTemplateUtils.installedTemplateVersionIsSufficient(Version.CURRENT));
public void testDataTemplateName() {
assertThat(MarvelTemplateUtils.dataTemplateName(),
equalTo(MarvelTemplateUtils.DATA_TEMPLATE_NAME_PREFIX + MarvelTemplateUtils.TEMPLATE_VERSION));
int version = randomIntBetween(1, 100);
assertThat(MarvelTemplateUtils.dataTemplateName(version), equalTo(".marvel-es-data-" + version));
}
}

201
elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/http/HttpExporterTemplateTests.java
View File

@ -11,26 +11,25 @@ import com.squareup.okhttp.mockwebserver.MockWebServer;
import com.squareup.okhttp.mockwebserver.RecordedRequest;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.Version;
import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest;
import org.elasticsearch.action.ActionRequest;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
import org.elasticsearch.marvel.agent.exporter.AbstractExporterTemplateTestCase;
import org.elasticsearch.marvel.agent.exporter.Exporter;
import org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils;
import org.junit.After;
import org.junit.Before;
import java.io.IOException;
import java.net.BindException;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.core.Is.is;
public class HttpExporterTemplateTests extends AbstractExporterTemplateTestCase {
@ -70,68 +69,39 @@ public class HttpExporterTemplateTests extends AbstractExporterTemplateTestCase
}
@Override
protected void deleteTemplate() {
dispatcher.setTemplate(null);
protected void deleteTemplates() throws Exception {
dispatcher.templates.clear();
}
@Override
protected void putTemplate(String version) throws Exception {
dispatcher.setTemplate(generateTemplateSource(version).toBytes());
protected void putTemplate(String name, int version) throws Exception {
dispatcher.templates.put(name, generateTemplateSource(name, version));
}
@Override
protected void createMarvelIndex(String index) throws Exception {
dispatcher.addIndex(index);
protected void assertTemplateExist(String name) throws Exception {
assertThat("failed to find a template matching [" + name + "]", dispatcher.templates.containsKey(name), is(true));
}
@Override
protected void assertTemplateUpdated(Version version) {
// Checks that a PUT Template request has been made
assertThat(dispatcher.hasRequest("PUT", "/_template/" + MarvelTemplateUtils.INDEX_TEMPLATE_NAME), is(true));
// Checks that the current template has the expected version
assertThat(MarvelTemplateUtils.parseTemplateVersion(dispatcher.getTemplate()), equalTo(version));
}
@Override
protected void assertTemplateNotUpdated(Version version) throws Exception {
protected void assertTemplateNotUpdated(String name) throws Exception {
// Checks that no PUT Template request has been made
assertThat(dispatcher.hasRequest("PUT", "/_template/" + MarvelTemplateUtils.INDEX_TEMPLATE_NAME), is(false));
assertThat(dispatcher.hasRequest("PUT", "/_template/" + name), is(false));
// Checks that the current template has the expected version
assertThat(MarvelTemplateUtils.parseTemplateVersion(dispatcher.getTemplate()), equalTo(version));
// Checks that the current template exists
assertThat(dispatcher.templates.containsKey(name), is(true));
}
@Override
protected void assertIndicesNotCreated() throws Exception {
// Checks that no Bulk request has been made
assertThat(dispatcher.hasRequest("POST", "/_bulk"), is(false));
assertThat(dispatcher.mappings.size(), equalTo(0));
}
@Override
protected void assertMappingsUpdated(String... indices) throws Exception {
// Load the mappings of the old template
Set<String> oldMappings = new PutIndexTemplateRequest().source(generateTemplateSource(null)).mappings().keySet();
// Load the mappings of the latest template
Set<String> newMappings = new PutIndexTemplateRequest().source(generateTemplateSource(null)).mappings().keySet();
newMappings.removeAll(oldMappings);
for (String index : indices) {
for (String mapping : newMappings) {
// Checks that a PUT Mapping request has been made for every type that was not in the old template
assertThat(dispatcher.hasRequest("PUT", "/" + index + "/_mapping/" + mapping), equalTo(true));
protected void awaitIndexExists(String... indices) throws Exception {
assertBusy(new Runnable() {
@Override
public void run() {
for (String index : indices) {
assertThat("could not find index " + index, dispatcher.hasIndex(index), is(true));
}
}
}
}
@Override
protected void assertMappingsNotUpdated(String... indices) throws Exception {
for (String index : indices) {
// Checks that no PUT Template request has been made
assertThat(dispatcher.hasRequest("PUT", "/" + index + "/_mapping/"), is(false));
}
}, 10, TimeUnit.SECONDS);
}
class MockServerDispatcher extends Dispatcher {
@ -140,100 +110,59 @@ public class HttpExporterTemplateTests extends AbstractExporterTemplateTestCase
private final MockResponse NOT_FOUND = newResponse(404, "");
private final Set<String> requests = new HashSet<>();
private final Map<String, Set<String>> mappings = new HashMap<>();
private byte[] template;
private final Map<String, BytesReference> templates = ConcurrentCollections.newConcurrentMap();
private final Set<String> indices = ConcurrentCollections.newConcurrentSet();
@Override
public MockResponse dispatch(RecordedRequest request) throws InterruptedException {
synchronized (this) {
final String requestLine = request.getRequestLine();
requests.add(requestLine);
final String requestLine = request.getRequestLine();
requests.add(requestLine);
switch (requestLine) {
// Cluster version
case "GET / HTTP/1.1":
return newResponse(200, "{\"version\": {\"number\": \"" + Version.CURRENT.number() + "\"}}");
// Template
case "GET /_template/.marvel-es HTTP/1.1":
return (template == null) ? NOT_FOUND : newResponse(200, new BytesArray(template).toUtf8());
case "PUT /_template/.marvel-es HTTP/1.1":
this.template = request.getBody().readByteArray();
return OK;
// Bulk
case "POST /_bulk HTTP/1.1":
return OK;
default:
String[] paths = Strings.splitStringToArray(request.getPath(), '/');
// Index Mappings
if ((paths != null) && (paths.length > 0) && ("_mapping".equals(paths[1]))) {
if (!mappings.containsKey(paths[0])) {
// Index does not exist
return NOT_FOUND;
}
// Get index mappings
if ("GET".equals(request.getMethod())) {
try {
// Builds a fake mapping response
XContentBuilder builder = jsonBuilder().startObject().startObject(paths[0]).startObject("mappings");
for (String type : mappings.get(paths[0])) {
builder.startObject(type).endObject();
}
builder.endObject().endObject().endObject();
return newResponse(200, builder.bytes().toUtf8());
} catch (IOException e) {
return newResponse(500, e.getMessage());
}
// Put index mapping
} else if ("PUT".equals(request.getMethod()) && paths.length > 2) {
Set<String> types = mappings.get(paths[0]);
if (types == null) {
types = new HashSet<>();
}
types.add(paths[2]);
return OK;
switch (requestLine) {
// Cluster version
case "GET / HTTP/1.1":
return newResponse(200, "{\"version\": {\"number\": \"" + Version.CURRENT.number() + "\"}}");
// Bulk
case "POST /_bulk HTTP/1.1":
// Parse the bulk request and extract all index names
try {
BulkRequest bulk = new BulkRequest();
byte[] source = request.getBody().readByteArray();
bulk.add(source, 0, source.length);
for (ActionRequest docRequest : bulk.requests()) {
if (docRequest instanceof IndexRequest) {
indices.add(((IndexRequest) docRequest).index());
}
}
break;
}
} catch (Exception e) {
return newResponse(500, e.getMessage());
}
return OK;
default:
String[] paths = Strings.splitStringToArray(request.getPath(), '/');
return newResponse(500, "MockServerDispatcher does not support: " + request.getRequestLine());
// Templates
if ((paths != null) && (paths.length > 0) && ("_template".equals(paths[0]))) {
String templateName = paths[1];
boolean templateExist = templates.containsKey(templateName);
if ("GET".equals(request.getMethod())) {
return templateExist ? newResponse(200, templates.get(templateName).toUtf8()) : NOT_FOUND;
}
if ("PUT".equals(request.getMethod())) {
templates.put(templateName, new BytesArray(request.getBody().readByteArray()));
return templateExist ? newResponse(200, "updated") : newResponse(201, "created");
}
}
break;
}
return newResponse(500, "MockServerDispatcher does not support: " + request.getRequestLine());
}
MockResponse newResponse(int code, String body) {
return new MockResponse().setResponseCode(code).setBody(body);
}
void setTemplate(byte[] template) {
synchronized (this) {
this.template = template;
}
}
byte[] getTemplate() {
return template;
}
void addIndex(String index) {
synchronized (this) {
if (template != null) {
// Simulate the use of the index template when creating an index
mappings.put(index, new HashSet<>(new PutIndexTemplateRequest().source(template).mappings().keySet()));
} else {
mappings.put(index, null);
}
}
}
int countRequests(String method, String path) {
int count = 0;
for (String request : requests) {
@ -247,5 +176,9 @@ public class HttpExporterTemplateTests extends AbstractExporterTemplateTestCase
boolean hasRequest(String method, String path) {
return countRequests(method, path) > 0;
}
boolean hasIndex(String index) {
return indices.contains(index);
}
}
}

107
elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/http/HttpExporterTests.java
View File

@ -46,6 +46,8 @@ import java.util.List;
import java.util.Map;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils.dataTemplateName;
import static org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils.indexTemplateName;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.instanceOf;
@ -53,9 +55,13 @@ import static org.hamcrest.Matchers.is;
@ESIntegTestCase.ClusterScope(scope = Scope.TEST, numDataNodes = 0, numClientNodes = 0, transportClientRatio = 0.0)
public class HttpExporterTests extends MarvelIntegTestCase {
private int webPort;
private MockWebServer webServer;
private static final byte[] TIMESTAMPED_TEMPLATE = MarvelTemplateUtils.loadTimestampedIndexTemplate();
private static final byte[] DATA_TEMPLATE = MarvelTemplateUtils.loadDataIndexTemplate();
@Before
public void startWebservice() throws Exception {
for (webPort = 9250; webPort < 9300; webPort++) {
@ -81,8 +87,10 @@ public class HttpExporterTests extends MarvelIntegTestCase {
public void testExport() throws Exception {
enqueueGetClusterVersionResponse(Version.CURRENT);
enqueueResponse(404, "marvel template does not exist");
enqueueResponse(201, "marvel template created");
enqueueResponse(404, "template for timestamped indices does not exist");
enqueueResponse(201, "template for timestamped indices created");
enqueueResponse(404, "template for data index does not exist");
enqueueResponse(201, "template for data index created");
enqueueResponse(200, "successful bulk request ");
Settings.Builder builder = Settings.builder()
@ -98,7 +106,7 @@ public class HttpExporterTests extends MarvelIntegTestCase {
final int nbDocs = randomIntBetween(1, 25);
exporter.export(newRandomMarvelDocs(nbDocs));
assertThat(webServer.getRequestCount(), equalTo(4));
assertThat(webServer.getRequestCount(), equalTo(6));
RecordedRequest recordedRequest = webServer.takeRequest();
assertThat(recordedRequest.getMethod(), equalTo("GET"));
@ -106,12 +114,21 @@ public class HttpExporterTests extends MarvelIntegTestCase {
recordedRequest = webServer.takeRequest();
assertThat(recordedRequest.getMethod(), equalTo("GET"));
assertThat(recordedRequest.getPath(), equalTo("/_template/.marvel-es"));
assertThat(recordedRequest.getPath(), equalTo("/_template/" + indexTemplateName()));
recordedRequest = webServer.takeRequest();
assertThat(recordedRequest.getMethod(), equalTo("PUT"));
assertThat(recordedRequest.getPath(), equalTo("/_template/.marvel-es"));
assertThat(recordedRequest.getBody().readByteArray(), equalTo(MarvelTemplateUtils.loadDefaultTemplate()));
assertThat(recordedRequest.getPath(), equalTo("/_template/" + indexTemplateName()));
assertThat(recordedRequest.getBody().readByteArray(), equalTo(TIMESTAMPED_TEMPLATE));
recordedRequest = webServer.takeRequest();
assertThat(recordedRequest.getMethod(), equalTo("GET"));
assertThat(recordedRequest.getPath(), equalTo("/_template/" + dataTemplateName()));
recordedRequest = webServer.takeRequest();
assertThat(recordedRequest.getMethod(), equalTo("PUT"));
assertThat(recordedRequest.getPath(), equalTo("/_template/" + dataTemplateName()));
assertThat(recordedRequest.getBody().readByteArray(), equalTo(DATA_TEMPLATE));
recordedRequest = webServer.takeRequest();
assertThat(recordedRequest.getMethod(), equalTo("POST"));
@ -156,8 +173,10 @@ public class HttpExporterTests extends MarvelIntegTestCase {
logger.info("--> starting node");
enqueueGetClusterVersionResponse(Version.CURRENT);
enqueueResponse(404, "marvel template does not exist");
enqueueResponse(201, "marvel template created");
enqueueResponse(404, "template for timestamped indices does not exist");
enqueueResponse(201, "template for timestamped indices created");
enqueueResponse(404, "template for data index does not exist");
enqueueResponse(201, "template for data index created");
enqueueResponse(200, "successful bulk request ");
String agentNode = internalCluster().startNode(builder);
@ -168,7 +187,7 @@ public class HttpExporterTests extends MarvelIntegTestCase {
exporter.export(Collections.singletonList(newRandomMarvelDoc()));
assertThat(exporter.supportedClusterVersion, is(true));
assertThat(webServer.getRequestCount(), equalTo(4));
assertThat(webServer.getRequestCount(), equalTo(6));
RecordedRequest recordedRequest = webServer.takeRequest();
assertThat(recordedRequest.getMethod(), equalTo("GET"));
@ -176,12 +195,21 @@ public class HttpExporterTests extends MarvelIntegTestCase {
recordedRequest = webServer.takeRequest();
assertThat(recordedRequest.getMethod(), equalTo("GET"));
assertThat(recordedRequest.getPath(), equalTo("/_template/.marvel-es"));
assertThat(recordedRequest.getPath(), equalTo("/_template/" + indexTemplateName()));
recordedRequest = webServer.takeRequest();
assertThat(recordedRequest.getMethod(), equalTo("PUT"));
assertThat(recordedRequest.getPath(), equalTo("/_template/.marvel-es"));
assertThat(recordedRequest.getBody().readByteArray(), equalTo(MarvelTemplateUtils.loadDefaultTemplate()));
assertThat(recordedRequest.getPath(), equalTo("/_template/" + indexTemplateName()));
assertThat(recordedRequest.getBody().readByteArray(), equalTo(TIMESTAMPED_TEMPLATE));
recordedRequest = webServer.takeRequest();
assertThat(recordedRequest.getMethod(), equalTo("GET"));
assertThat(recordedRequest.getPath(), equalTo("/_template/" + dataTemplateName()));
recordedRequest = webServer.takeRequest();
assertThat(recordedRequest.getMethod(), equalTo("PUT"));
assertThat(recordedRequest.getPath(), equalTo("/_template/" + dataTemplateName()));
assertThat(recordedRequest.getBody().readByteArray(), equalTo(DATA_TEMPLATE));
recordedRequest = webServer.takeRequest();
assertThat(recordedRequest.getMethod(), equalTo("POST"));
@ -214,14 +242,15 @@ public class HttpExporterTests extends MarvelIntegTestCase {
exporter = getExporter(agentNode);
enqueueGetClusterVersionResponse(secondWebServer, Version.CURRENT);
enqueueResponse(secondWebServer, 404, "marvel template does not exist");
enqueueResponse(secondWebServer, 201, "marvel template created");
enqueueResponse(secondWebServer, 404, "template for timestamped indices does not exist");
enqueueResponse(secondWebServer, 201, "template for timestamped indices created");
enqueueResponse(secondWebServer, 200, "template for data index exist");
enqueueResponse(secondWebServer, 200, "successful bulk request ");
logger.info("--> exporting a second event");
exporter.export(Collections.singletonList(newRandomMarvelDoc()));
assertThat(secondWebServer.getRequestCount(), equalTo(4));
assertThat(secondWebServer.getRequestCount(), equalTo(5));
recordedRequest = secondWebServer.takeRequest();
assertThat(recordedRequest.getMethod(), equalTo("GET"));
@ -229,12 +258,16 @@ public class HttpExporterTests extends MarvelIntegTestCase {
recordedRequest = secondWebServer.takeRequest();
assertThat(recordedRequest.getMethod(), equalTo("GET"));
assertThat(recordedRequest.getPath(), equalTo("/_template/.marvel-es"));
assertThat(recordedRequest.getPath(), equalTo("/_template/" + indexTemplateName()));
recordedRequest = secondWebServer.takeRequest();
assertThat(recordedRequest.getMethod(), equalTo("PUT"));
assertThat(recordedRequest.getPath(), equalTo("/_template/.marvel-es"));
assertThat(recordedRequest.getBody().readByteArray(), equalTo(MarvelTemplateUtils.loadDefaultTemplate()));
assertThat(recordedRequest.getPath(), equalTo("/_template/" + indexTemplateName()));
assertThat(recordedRequest.getBody().readByteArray(), equalTo(TIMESTAMPED_TEMPLATE));
recordedRequest = secondWebServer.takeRequest();
assertThat(recordedRequest.getMethod(), equalTo("GET"));
assertThat(recordedRequest.getPath(), equalTo("/_template/" + dataTemplateName()));;
recordedRequest = secondWebServer.takeRequest();
assertThat(recordedRequest.getMethod(), equalTo("POST"));
@ -287,8 +320,10 @@ public class HttpExporterTests extends MarvelIntegTestCase {
logger.info("--> exporting a first event");
enqueueGetClusterVersionResponse(Version.CURRENT);
enqueueResponse(404, "marvel template does not exist");
enqueueResponse(201, "marvel template created");
enqueueResponse(404, "template for timestamped indices does not exist");
enqueueResponse(201, "template for timestamped indices created");
enqueueResponse(404, "template for data index does not exist");
enqueueResponse(201, "template for data index created");
enqueueResponse(200, "successful bulk request ");
HttpExporter exporter = getExporter(agentNode);
@ -296,7 +331,7 @@ public class HttpExporterTests extends MarvelIntegTestCase {
MarvelDoc doc = newRandomMarvelDoc();
exporter.export(Collections.singletonList(doc));
assertThat(webServer.getRequestCount(), equalTo(4));
assertThat(webServer.getRequestCount(), equalTo(6));
RecordedRequest recordedRequest = webServer.takeRequest();
assertThat(recordedRequest.getMethod(), equalTo("GET"));
@ -304,12 +339,21 @@ public class HttpExporterTests extends MarvelIntegTestCase {
recordedRequest = webServer.takeRequest();
assertThat(recordedRequest.getMethod(), equalTo("GET"));
assertThat(recordedRequest.getPath(), equalTo("/_template/.marvel-es"));
assertThat(recordedRequest.getPath(), equalTo("/_template/" + indexTemplateName()));
recordedRequest = webServer.takeRequest();
assertThat(recordedRequest.getMethod(), equalTo("PUT"));
assertThat(recordedRequest.getPath(), equalTo("/_template/.marvel-es"));
assertThat(recordedRequest.getBody().readByteArray(), equalTo(MarvelTemplateUtils.loadDefaultTemplate()));
assertThat(recordedRequest.getPath(), equalTo("/_template/" + indexTemplateName()));
assertThat(recordedRequest.getBody().readByteArray(), equalTo(TIMESTAMPED_TEMPLATE));
recordedRequest = webServer.takeRequest();
assertThat(recordedRequest.getMethod(), equalTo("GET"));
assertThat(recordedRequest.getPath(), equalTo("/_template/" + dataTemplateName()));
recordedRequest = webServer.takeRequest();
assertThat(recordedRequest.getMethod(), equalTo("PUT"));
assertThat(recordedRequest.getPath(), equalTo("/_template/" + dataTemplateName()));
assertThat(recordedRequest.getBody().readByteArray(), equalTo(DATA_TEMPLATE));
recordedRequest = webServer.takeRequest();
assertThat(recordedRequest.getMethod(), equalTo("POST"));
@ -332,18 +376,18 @@ public class HttpExporterTests extends MarvelIntegTestCase {
logger.info("--> exporting a second event");
enqueueGetClusterVersionResponse(Version.CURRENT);
enqueueResponse(404, "marvel template does not exist");
enqueueResponse(201, "marvel template created");
enqueueResponse(200, "template for timestamped indices exist");
enqueueResponse(200, "template for data index exist");
enqueueResponse(200, "successful bulk request ");
doc = newRandomMarvelDoc();
exporter = getExporter(agentNode);
exporter.export(Collections.singletonList(doc));
String expectedMarvelIndex = MarvelSettings.MARVEL_INDICES_PREFIX
String expectedMarvelIndex = MarvelSettings.MARVEL_INDICES_PREFIX + MarvelTemplateUtils.TEMPLATE_VERSION + "-"
+ DateTimeFormat.forPattern(newTimeFormat).withZoneUTC().print(doc.timestamp());
assertThat(webServer.getRequestCount(), equalTo(4 + 4));
assertThat(webServer.getRequestCount(), equalTo(6 + 4));
recordedRequest = webServer.takeRequest();
assertThat(recordedRequest.getMethod(), equalTo("GET"));
@ -351,12 +395,11 @@ public class HttpExporterTests extends MarvelIntegTestCase {
recordedRequest = webServer.takeRequest();
assertThat(recordedRequest.getMethod(), equalTo("GET"));
assertThat(recordedRequest.getPath(), equalTo("/_template/.marvel-es"));
assertThat(recordedRequest.getPath(), equalTo("/_template/" + indexTemplateName()));
recordedRequest = webServer.takeRequest();
assertThat(recordedRequest.getMethod(), equalTo("PUT"));
assertThat(recordedRequest.getPath(), equalTo("/_template/.marvel-es"));
assertThat(recordedRequest.getBody().readByteArray(), equalTo(MarvelTemplateUtils.loadDefaultTemplate()));
assertThat(recordedRequest.getMethod(), equalTo("GET"));
assertThat(recordedRequest.getPath(), equalTo("/_template/" + dataTemplateName()));
recordedRequest = webServer.takeRequest();
assertThat(recordedRequest.getMethod(), equalTo("POST"));

87
elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/local/LocalExporterTemplateTests.java
View File

@ -5,26 +5,12 @@
*/
package org.elasticsearch.marvel.agent.exporter.local;
import org.elasticsearch.Version;
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest;
import org.elasticsearch.action.support.IndicesOptions;
import org.elasticsearch.cluster.metadata.MappingMetaData;
import org.elasticsearch.common.collect.ImmutableOpenMap;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.IndexNotFoundException;
import org.elasticsearch.marvel.agent.exporter.AbstractExporterTemplateTestCase;
import org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils;
import org.elasticsearch.marvel.agent.settings.MarvelSettings;
import java.util.Collections;
import java.util.Map;
import java.util.Set;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.notNullValue;
import static org.hamcrest.core.Is.is;
public class LocalExporterTemplateTests extends AbstractExporterTemplateTestCase {
@ -34,83 +20,26 @@ public class LocalExporterTemplateTests extends AbstractExporterTemplateTestCase
}
@Override
protected Set<String> excludeTemplates() {
// Always delete the template between tests
return Collections.emptySet();
}
@Override
protected void deleteTemplate() throws Exception {
protected void deleteTemplates() throws Exception {
waitNoPendingTasksOnAll();
assertAcked(client().admin().indices().prepareDeleteTemplate(MarvelTemplateUtils.INDEX_TEMPLATE_NAME).get());
cluster().wipeAllTemplates(Collections.emptySet());
}
@Override
protected void putTemplate(String version) throws Exception {
protected void putTemplate(String name, int version) throws Exception {
waitNoPendingTasksOnAll();
assertAcked(client().admin().indices().preparePutTemplate(MarvelTemplateUtils.INDEX_TEMPLATE_NAME).setSource(generateTemplateSource(version)).get());
assertAcked(client().admin().indices().preparePutTemplate(name).setSource(generateTemplateSource(name, version)).get());
}
@Override
protected void createMarvelIndex(String index) throws Exception {
protected void assertTemplateExist(String name) throws Exception {
waitNoPendingTasksOnAll();
createIndex(index);
waitForMarvelTemplate(name);
}
@Override
protected void assertTemplateUpdated(Version version) throws Exception {
protected void assertTemplateNotUpdated(String name) throws Exception {
waitNoPendingTasksOnAll();
awaitMarvelTemplateInstalled(version);
}
@Override
protected void assertTemplateNotUpdated(Version version) throws Exception {
waitNoPendingTasksOnAll();
awaitMarvelTemplateInstalled(version);
}
private void assertMappings(byte[] reference, String... indices) throws Exception {
waitNoPendingTasksOnAll();
Map<String, String> mappings = new PutIndexTemplateRequest().source(reference).mappings();
assertBusy(new Runnable() {
@Override
public void run() {
for (String index : indices) {
GetMappingsResponse response = client().admin().indices().prepareGetMappings(index).setIndicesOptions(IndicesOptions.lenientExpandOpen()).get();
ImmutableOpenMap<String, MappingMetaData> indexMappings = response.getMappings().get(index);
assertNotNull(indexMappings);
assertThat(indexMappings.size(), equalTo(mappings.size()));
for (String mapping : mappings.keySet()) {
// We just check that mapping type exists, we don't verify its content
assertThat("mapping type " + mapping + " should exist in index " + index, indexMappings.get(mapping), notNullValue());
}
}
}
});
}
@Override
protected void assertMappingsUpdated(String... indices) throws Exception {
assertMappings(MarvelTemplateUtils.loadDefaultTemplate(), indices);
}
@Override
protected void assertMappingsNotUpdated(String... indices) throws Exception {
assertMappings(generateTemplateSource(null).toBytes(), indices);
}
@Override
protected void assertIndicesNotCreated() throws Exception {
waitNoPendingTasksOnAll();
try {
assertThat(client().admin().indices().prepareExists(MarvelSettings.MARVEL_INDICES_PREFIX + "*").get().isExists(), is(false));
} catch (IndexNotFoundException e) {
// with shield we might get that if wildcards were resolved to no indices
if (!shieldEnabled) {
throw e;
}
}
assertTemplateExist(name);
}
}

24
elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/local/LocalExporterTests.java
View File

@ -6,9 +6,7 @@
package org.elasticsearch.marvel.agent.exporter.local;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.Version;
import org.elasticsearch.action.admin.indices.recovery.RecoveryResponse;
import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.health.ClusterHealthStatus;
@ -36,11 +34,12 @@ import java.util.Map;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
import static org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils.dataTemplateName;
import static org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils.indexTemplateName;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
import static org.hamcrest.Matchers.allOf;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.hasSize;
import static org.hamcrest.Matchers.instanceOf;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.notNullValue;
@ -105,8 +104,8 @@ public class LocalExporterTests extends MarvelIntegTestCase {
updateMarvelInterval(3L, TimeUnit.SECONDS);
// lets wait until the marvel template will be installed
awaitMarvelTemplateInstalled();
assertThat(getCurrentlyInstalledTemplateVersion(), is(Version.CURRENT));
waitForMarvelTemplate(indexTemplateName());
waitForMarvelTemplate(dataTemplateName());
}
public void testIndexTimestampFormat() throws Exception {
@ -122,12 +121,12 @@ public class LocalExporterTests extends MarvelIntegTestCase {
LocalExporter exporter = getLocalExporter("_local");
// first lets test that the index resolver works with time
String indexName = MarvelSettings.MARVEL_INDICES_PREFIX + DateTimeFormat.forPattern(timeFormat).withZoneUTC().print(time);
String indexName = MarvelSettings.MARVEL_INDICES_PREFIX + MarvelTemplateUtils.TEMPLATE_VERSION + "-" + DateTimeFormat.forPattern(timeFormat).withZoneUTC().print(time);
assertThat(exporter.indexNameResolver().resolve(time), equalTo(indexName));
// now lets test that the index name resolver works with a doc
MarvelDoc doc = newRandomMarvelDoc();
indexName = MarvelSettings.MARVEL_INDICES_PREFIX + DateTimeFormat.forPattern(timeFormat).withZoneUTC().print(doc.timestamp());
indexName = MarvelSettings.MARVEL_INDICES_PREFIX + MarvelTemplateUtils.TEMPLATE_VERSION + "-" + DateTimeFormat.forPattern(timeFormat).withZoneUTC().print(doc.timestamp());
assertThat(exporter.indexNameResolver().resolve(doc), equalTo(indexName));
logger.debug("--> exporting a random marvel document");
@ -138,7 +137,7 @@ public class LocalExporterTests extends MarvelIntegTestCase {
timeFormat = randomFrom("dd", "dd.MM.YYYY", "dd.MM");
updateClusterSettings(Settings.builder().put("marvel.agent.exporters._local.index.name.time_format", timeFormat));
exporter = getLocalExporter("_local"); // we need to get it again.. as it was rebuilt
indexName = MarvelSettings.MARVEL_INDICES_PREFIX + DateTimeFormat.forPattern(timeFormat).withZoneUTC().print(doc.timestamp());
indexName = MarvelSettings.MARVEL_INDICES_PREFIX + MarvelTemplateUtils.TEMPLATE_VERSION + "-" + DateTimeFormat.forPattern(timeFormat).withZoneUTC().print(doc.timestamp());
assertThat(exporter.indexNameResolver().resolve(doc), equalTo(indexName));
logger.debug("--> exporting the document again (this time with the the new index name time format [{}], expecting index name [{}]", timeFormat, indexName);
@ -194,13 +193,4 @@ public class LocalExporterTests extends MarvelIntegTestCase {
ClusterStateCollector.TYPE, timeStampGenerator.incrementAndGet(), ClusterState.PROTO, ClusterHealthStatus.GREEN);
}
}
private Version getCurrentlyInstalledTemplateVersion() {
GetIndexTemplatesResponse response = client().admin().indices().prepareGetTemplates(MarvelTemplateUtils.INDEX_TEMPLATE_NAME).get();
assertThat(response, notNullValue());
assertThat(response.getIndexTemplates(), notNullValue());
assertThat(response.getIndexTemplates(), hasSize(1));
assertThat(response.getIndexTemplates().get(0), notNullValue());
return MarvelTemplateUtils.templateVersion(response.getIndexTemplates().get(0));
}
}

19
elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/cluster/ClusterInfoTests.java
View File

@ -12,6 +12,7 @@ import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.license.core.License;
import org.elasticsearch.marvel.agent.collector.cluster.ClusterInfoCollector;
import org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils;
import org.elasticsearch.marvel.agent.settings.MarvelSettings;
import org.elasticsearch.marvel.test.MarvelIntegTestCase;
import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
@ -21,6 +22,8 @@ import org.junit.Before;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import static org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils.dataTemplateName;
import static org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils.indexTemplateName;
import static org.elasticsearch.test.ESIntegTestCase.Scope.TEST;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
import static org.hamcrest.Matchers.equalTo;
@ -57,20 +60,21 @@ public class ClusterInfoTests extends MarvelIntegTestCase {
final String clusterUUID = client().admin().cluster().prepareState().setMetaData(true).get().getState().metaData().clusterUUID();
assertTrue(Strings.hasText(clusterUUID));
logger.debug("--> waiting for the marvel data index to be created (it should have been created by the LicenseCollector)");
awaitIndexExists(MarvelSettings.MARVEL_DATA_INDEX_NAME);
logger.debug("--> waiting for the marvel data index to be created (it should have been created by the ClusterInfoCollector)");
String dataIndex = ".marvel-es-data-" + MarvelTemplateUtils.TEMPLATE_VERSION;
awaitIndexExists(dataIndex);
logger.debug("--> waiting for cluster info collector to collect data");
awaitMarvelDocsCount(equalTo(1L), ClusterInfoCollector.TYPE);
logger.debug("--> retrieving cluster info document");
GetResponse response = client().prepareGet(MarvelSettings.MARVEL_DATA_INDEX_NAME, ClusterInfoCollector.TYPE, clusterUUID).get();
assertTrue(MarvelSettings.MARVEL_DATA_INDEX_NAME + " document does not exist", response.isExists());
GetResponse response = client().prepareGet(dataIndex, ClusterInfoCollector.TYPE, clusterUUID).get();
assertTrue("cluster_info document does not exist in data index", response.isExists());
logger.debug("--> checking that the document contains all required information");
logger.debug("--> checking that the document contains license information");
assertThat(response.getIndex(), equalTo(MarvelSettings.MARVEL_DATA_INDEX_NAME));
assertThat(response.getIndex(), equalTo(dataIndex));
assertThat(response.getType(), equalTo(ClusterInfoCollector.TYPE));
assertThat(response.getId(), equalTo(clusterUUID));
@ -111,14 +115,15 @@ public class ClusterInfoTests extends MarvelIntegTestCase {
assertThat(clusterStats, instanceOf(Map.class));
assertThat(((Map) clusterStats).size(), greaterThan(0));
assertMarvelTemplateInstalled();
waitForMarvelTemplate(indexTemplateName());
waitForMarvelTemplate(dataTemplateName());
logger.debug("--> check that the cluster_info is not indexed");
securedFlush();
securedRefresh();
assertHitCount(client().prepareSearch().setSize(0)
.setIndices(MarvelSettings.MARVEL_DATA_INDEX_NAME)
.setIndices(dataIndex)
.setTypes(ClusterInfoCollector.TYPE)
.setQuery(QueryBuilders.boolQuery()
.should(QueryBuilders.matchQuery(License.XFields.STATUS.underscore().toString(), License.Status.ACTIVE.label()))

4
elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/cluster/ClusterStateTests.java
View File

@ -11,6 +11,7 @@ import org.elasticsearch.cluster.node.DiscoveryNodes;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.marvel.agent.collector.cluster.ClusterStateCollector;
import org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils;
import org.elasticsearch.marvel.agent.renderer.AbstractRenderer;
import org.elasticsearch.marvel.agent.settings.MarvelSettings;
import org.elasticsearch.marvel.test.MarvelIntegTestCase;
@ -156,9 +157,10 @@ public class ClusterStateTests extends MarvelIntegTestCase {
for (final String nodeName : internalCluster().getNodeNames()) {
final String nodeId = internalCluster().clusterService(nodeName).localNode().getId();
final String dataIndex = ".marvel-es-data-" + MarvelTemplateUtils.TEMPLATE_VERSION;
logger.debug("--> getting marvel document for node id [{}]", nodeId);
assertThat(client().prepareGet(MarvelSettings.MARVEL_DATA_INDEX_NAME, ClusterStateCollector.NODE_TYPE, nodeId).get().isExists(), is(true));
assertThat(client().prepareGet(dataIndex, ClusterStateCollector.NODE_TYPE, nodeId).get().isExists(), is(true));
// checks that document is not indexed
assertHitCount(client().prepareSearch().setSize(0)

4
elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/shield/SecuredClientTests.java
View File

@ -36,13 +36,13 @@ public class SecuredClientTests extends MarvelIntegTestCase {
assertAccessIsAllowed(securedClient.admin().indices().prepareDelete(MarvelSettings.MARVEL_INDICES_PREFIX));
assertAccessIsAllowed(securedClient.admin().indices().prepareCreate(MarvelSettings.MARVEL_INDICES_PREFIX + "test"));
assertAccessIsAllowed(securedClient.admin().indices().preparePutTemplate("foo").setSource(MarvelTemplateUtils.loadDefaultTemplate()));
assertAccessIsAllowed(securedClient.admin().indices().preparePutTemplate("foo").setSource(MarvelTemplateUtils.loadTimestampedIndexTemplate()));
assertAccessIsAllowed(securedClient.admin().indices().prepareGetTemplates("foo"));
}
public void testDeniedAccess() {
SecuredClient securedClient = internalCluster().getInstance(SecuredClient.class);
assertAcked(securedClient.admin().indices().preparePutTemplate("foo").setSource(MarvelTemplateUtils.loadDefaultTemplate()).get());
assertAcked(securedClient.admin().indices().preparePutTemplate("foo").setSource(MarvelTemplateUtils.loadDataIndexTemplate()).get());
if (shieldEnabled) {
assertAccessIsDenied(securedClient.admin().indices().prepareDeleteTemplate("foo"));

67
elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/test/MarvelIntegTestCase.java
View File

@ -6,14 +6,10 @@
package org.elasticsearch.marvel.test;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.Version;
import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
import org.elasticsearch.cluster.metadata.MetaData;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.io.Streams;
import org.elasticsearch.common.regex.Regex;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.concurrent.CountDown;
import org.elasticsearch.index.IndexModule;
@ -70,11 +66,6 @@ public abstract class MarvelIntegTestCase extends ESIntegTestCase {
return super.buildTestCluster(scope, seed);
}
@Override
protected Set<String> excludeTemplates() {
return Collections.singleton(MarvelTemplateUtils.INDEX_TEMPLATE_NAME);
}
@Override
protected Settings nodeSettings(int nodeOrdinal) {
Settings.Builder builder = Settings.builder()
@ -212,63 +203,29 @@ public abstract class MarvelIntegTestCase extends ESIntegTestCase {
}
}
protected void assertMarvelTemplateInstalled() {
ClusterStateResponse clusterStateResponse = client().admin().cluster().prepareState().get();
if (clusterStateResponse != null) {
ClusterState state = clusterStateResponse.getState();
MetaData md = state.getMetaData();
}
GetIndexTemplatesResponse response = client().admin().indices().prepareGetTemplates().get();
for (IndexTemplateMetaData template : response.getIndexTemplates()) {
if (template.getName().equals(MarvelTemplateUtils.INDEX_TEMPLATE_NAME)) {
return;
protected void assertTemplateInstalled(String name) {
boolean found = false;
for (IndexTemplateMetaData template : client().admin().indices().prepareGetTemplates().get().getIndexTemplates()) {
if (Regex.simpleMatch(name, template.getName())) {
found = true;
}
}
fail("marvel template should exist");
assertTrue("failed to find a template matching [" + name + "]", found);
}
protected void assertMarvelTemplateMissing() {
for (IndexTemplateMetaData template : client().admin().indices().prepareGetTemplates(MarvelTemplateUtils.INDEX_TEMPLATE_NAME).get().getIndexTemplates()) {
if (template.getName().equals(MarvelTemplateUtils.INDEX_TEMPLATE_NAME)) {
fail("marvel template shouldn't exist");
}
}
}
protected void awaitMarvelTemplateInstalled() throws Exception {
protected void waitForMarvelTemplate(String name) throws Exception {
assertBusy(new Runnable() {
@Override
public void run() {
assertMarvelTemplateInstalled();
assertTemplateInstalled(name);
}
}, 30, TimeUnit.SECONDS);
}
protected void awaitMarvelTemplateInstalled(Version version) throws Exception {
assertBusy(new Runnable() {
@Override
public void run() {
assertMarvelTemplateInstalled(version);
}
}, 30, TimeUnit.SECONDS);
}
protected void assertMarvelTemplateInstalled(Version version) {
for (IndexTemplateMetaData template : client().admin().indices().prepareGetTemplates(MarvelTemplateUtils.INDEX_TEMPLATE_NAME).get().getIndexTemplates()) {
if (template.getName().equals(MarvelTemplateUtils.INDEX_TEMPLATE_NAME)) {
Version templateVersion = MarvelTemplateUtils.templateVersion(template);
if (templateVersion != null && templateVersion.id == version.id) {
return;
}
fail("did not find marvel template with expected version [" + version + "]. found version [" + templateVersion + "]");
}
}
fail("marvel template could not be found");
}
protected void waitForMarvelIndices() throws Exception {
awaitIndexExists(MarvelSettings.MARVEL_DATA_INDEX_NAME);
awaitIndexExists(MarvelSettings.MARVEL_INDICES_PREFIX + "*");
String currentVersion = String.valueOf(MarvelTemplateUtils.TEMPLATE_VERSION);
awaitIndexExists(MarvelSettings.MARVEL_DATA_INDEX_PREFIX + currentVersion);
awaitIndexExists(MarvelSettings.MARVEL_INDICES_PREFIX + currentVersion + "-*");
assertBusy(new Runnable() {
@Override
public void run() {