honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Marvel: Update template management in Local/Http Exporters 

This commit update the Local & Http exporters so that they have the same behavior. Exporters now try to update the mappings of the current marvel indices if they exist (only new document types will be added)

It also adds dedicated tests to verify that the Marvel template is correctly updated depending of the current template version.

Original commit: elastic/x-pack-elasticsearch@ac650f5aba
This commit is contained in:
Tanguy Leroux 2015-12-07 14:07:55 +01:00
parent d377168a7f
commit fdcf1f44e4
20 changed files with 1052 additions and 407 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/Exporter.java
View File

@ -5,10 +5,7 @@
*/
package org.elasticsearch.marvel.agent.exporter;
import org.elasticsearch.Version;
import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.logging.ESLogger;
import org.elasticsearch.common.logging.Loggers;
import org.elasticsearch.common.settings.Settings;
@ -26,7 +23,6 @@ public abstract class Exporter {
public static final String INDEX_NAME_TIME_FORMAT_SETTING = "index.name.time_format";
public static final String BULK_TIMEOUT_SETTING = "bulk.timeout";
public static final Version MIN_SUPPORTED_TEMPLATE_VERSION = Version.V_2_0_0_beta2;
public static final String DEFAULT_INDEX_NAME_TIME_FORMAT = "YYYY.MM.dd";
protected final String type;

61
elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/MarvelTemplateUtils.java
View File

@ -11,6 +11,7 @@ import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
import org.elasticsearch.cluster.metadata.MetaData;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.io.Streams;
import org.elasticsearch.common.logging.ESLogger;
import org.elasticsearch.marvel.support.VersionUtils;
import java.io.ByteArrayOutputStream;
@ -25,7 +26,7 @@ public final class MarvelTemplateUtils {
public static final String MARVEL_TEMPLATE_FILE = "/marvel_index_template.json";
public static final String INDEX_TEMPLATE_NAME = ".marvel-es";
public static final String MARVEL_VERSION_FIELD = "marvel_version";
public static final Version MIN_SUPPORTED_TEMPLATE_VERSION = Version.V_2_0_0_beta2;
private MarvelTemplateUtils() {
}
@ -43,10 +44,18 @@ public final class MarvelTemplateUtils {
}
}
public static Version loadDefaultTemplateVersion() {
return parseTemplateVersion(loadDefaultTemplate());
}
public static Version templateVersion(IndexTemplateMetaData templateMetaData) {
String version = templateMetaData.settings().get("index." + MARVEL_VERSION_FIELD);
if (Strings.hasLength(version)) {
return Version.fromString(version);
try {
return Version.fromString(version);
} catch (IllegalArgumentException e) {
return null;
}
}
return null;
}
@ -57,10 +66,52 @@ public final class MarvelTemplateUtils {
}
public static Version parseTemplateVersion(byte[] template) {
return VersionUtils.parseVersion(MARVEL_VERSION_FIELD, template);
try {
return VersionUtils.parseVersion(MARVEL_VERSION_FIELD, template);
} catch (IllegalArgumentException e) {
return null;
}
}
public static Version parseTemplateVersion(String template) {
return VersionUtils.parseVersion(MARVEL_VERSION_FIELD, template);
public static boolean installedTemplateVersionIsSufficient(Version installed) {
// null indicates couldn't parse the version from the installed template, this means it is probably too old or invalid...
if (installed == null) {
return false;
}
// ensure the template is not too old
if (installed.before(MIN_SUPPORTED_TEMPLATE_VERSION)) {
return false;
}
// We do not enforce that versions are equivalent to the current version as we may be in a rolling upgrade scenario
// and until a master is elected with the new version, data nodes that have been upgraded will not be able to ship
// data. This means that there is an implication that the new shippers will ship data correctly even with an old template.
// There is also no upper bound and we rely on elasticsearch nodes not being able to connect to each other across major
// versions
return true;
}
public static boolean installedTemplateVersionMandatesAnUpdate(Version current, Version installed, ESLogger logger, String exporterName) {
if (installed == null) {
logger.debug("exporter [{}] - currently installed marvel template is missing a version - installing a new one [{}]", exporterName, current);
return true;
}
// Never update a very old template
if (installed.before(MIN_SUPPORTED_TEMPLATE_VERSION)) {
return false;
}
// Always update a template to the last up-to-date version
if (current.after(installed)) {
logger.debug("exporter [{}] - currently installed marvel template version [{}] will be updated to a newer version [{}]", exporterName, installed, current);
return true;
// When the template is up-to-date, do not update
} else if (current.equals(installed)) {
logger.debug("exporter [{}] - currently installed marvel template version [{}] is up-to-date", exporterName, installed);
return false;
// Never update a template that is newer than the expected one
} else {
logger.debug("exporter [{}] - currently installed marvel template version [{}] is newer than the one required [{}]... keeping it.", exporterName, installed, current);
return false;
}
}
}

190
elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/http/HttpExporter.java
View File

@ -8,9 +8,11 @@ package org.elasticsearch.marvel.agent.exporter.http;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.ExceptionsHelper;
import org.elasticsearch.Version;
import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest;
import org.elasticsearch.common.Base64;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.io.Streams;
import org.elasticsearch.common.io.stream.BytesStreamOutput;
@ -18,6 +20,7 @@ import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.settings.SettingsException;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentHelper;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.env.Environment;
@ -40,9 +43,10 @@ import java.net.URL;
import java.nio.file.Files;
import java.nio.file.Path;
import java.security.KeyStore;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Map;
import java.util.*;
import static org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils.installedTemplateVersionIsSufficient;
import static org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils.installedTemplateVersionMandatesAnUpdate;
/**
*
@ -88,7 +92,6 @@ public class HttpExporter extends Exporter {
volatile boolean checkedAndUploadedIndexTemplate = false;
volatile boolean supportedClusterVersion = false;
/** Version of the built-in template **/
final Version templateVersion;
@ -126,7 +129,7 @@ public class HttpExporter extends Exporter {
hostnameVerification = config.settings().getAsBoolean(SSL_HOSTNAME_VERIFICATION_SETTING, true);
// Checks that the built-in template is versioned
templateVersion = MarvelTemplateUtils.parseTemplateVersion(MarvelTemplateUtils.loadDefaultTemplate());
templateVersion = MarvelTemplateUtils.loadDefaultTemplateVersion();
if (templateVersion == null) {
throw new IllegalStateException("unable to find built-in template version");
}
@ -381,6 +384,37 @@ public class HttpExporter extends Exporter {
* @return true if template exists or was uploaded successfully.
*/
private boolean checkAndUploadIndexTemplate(final String host) {
byte[] installedTemplate;
try {
installedTemplate = findMarvelTemplate(host);
} catch (Exception e) {
logger.debug("http exporter [{}] - exception when loading the existing marvel template on host[{}]", e, name(), host);
return false;
}
// if we cannot find a template or a compatible template, we'll install one in / update it.
if (installedTemplate == null) {
logger.debug("http exporter [{}] - could not find existing marvel template, installing a new one", name());
return putTemplate(host);
}
Version installedTemplateVersion = MarvelTemplateUtils.parseTemplateVersion(installedTemplate);
if (installedTemplateVersionMandatesAnUpdate(templateVersion, installedTemplateVersion, logger, name())) {
logger.debug("http exporter [{}] - installing new marvel template [{}], replacing [{}]", name(), templateVersion, installedTemplateVersion);
return putTemplate(host);
} else if (!installedTemplateVersionIsSufficient(installedTemplateVersion)) {
logger.error("http exporter [{}] - marvel template version [{}] is below the minimum compatible version [{}]. "
+ "please manually update the marvel template to a more recent version"
+ "and delete the current active marvel index (don't forget to back up it first if needed)",
name(), installedTemplateVersion, MarvelTemplateUtils.MIN_SUPPORTED_TEMPLATE_VERSION);
// we're not going to do anything with the template.. it's too old, and the schema might
// be too different than what this version of marvel/es can work with. For this reason we're
// not going to export any data, to avoid mapping conflicts.
return false;
}
return true;
}
private byte[] findMarvelTemplate(String host) throws IOException {
String url = "_template/" + MarvelTemplateUtils.INDEX_TEMPLATE_NAME;
if (templateCheckTimeout != null) {
url += "?timeout=" + templateCheckTimeout;
@ -388,40 +422,28 @@ public class HttpExporter extends Exporter {
HttpURLConnection connection = null;
try {
logger.debug("checking if marvel template exists on the marvel cluster");
logger.debug("http exporter [{}] - checking if marvel template exists on the marvel cluster", name());
connection = openConnection(host, "GET", url, null);
if (connection == null) {
logger.debug("no available connection to check marvel template existence");
return false;
throw new IOException("no available connection to check marvel template existence");
}
byte[] remoteTemplate = null;
// 200 means that the template has been found, 404 otherwise
if (connection.getResponseCode() == 200) {
logger.debug("marvel template found, checking its version");
logger.debug("marvel template found");
byte[] remoteTemplate;
try (InputStream is = connection.getInputStream()) {
ByteArrayOutputStream out = new ByteArrayOutputStream();
Streams.copy(is, out);
remoteTemplate = out.toByteArray();
}
if ((remoteTemplate == null) || (remoteTemplate.length == 0)) {
logger.error("unable to load remote marvel template on host [{}]", host);
return false;
}
Version remoteVersion = MarvelTemplateUtils.parseTemplateVersion(remoteTemplate);
logger.debug("detected existing remote template in version [{}] on host [{}]", remoteVersion, host);
if ((remoteVersion != null) && (remoteVersion.onOrAfter(MIN_SUPPORTED_TEMPLATE_VERSION))) {
logger.debug("remote template in version [{}] is compatible with the min. supported version [{}]", remoteVersion, MIN_SUPPORTED_TEMPLATE_VERSION);
return true;
}
}
} catch (IOException e) {
logger.error("failed to verify the marvel template to [{}]:\n{}", host, e.getMessage());
return false;
return remoteTemplate;
} catch (Exception e) {
logger.error("http exporter [{}] - failed to verify the marvel template to [{}]:\n{}", name(), host, e.getMessage());
throw e;
} finally {
if (connection != null) {
try {
@ -431,28 +453,30 @@ public class HttpExporter extends Exporter {
}
}
}
}
boolean putTemplate(String host) {
HttpURLConnection connection = null;
try {
connection = openConnection(host, "PUT", url, XContentType.JSON.restContentType());
connection = openConnection(host, "PUT", "_template/" + MarvelTemplateUtils.INDEX_TEMPLATE_NAME, XContentType.JSON.restContentType());
if (connection == null) {
logger.debug("no available connection to update marvel template");
logger.debug("http exporter [{}] - no available connection to update marvel template", name());
return false;
}
logger.debug("loading marvel pre-configured template");
logger.debug("http exporter [{}] - loading marvel pre-configured template", name());
byte[] template = MarvelTemplateUtils.loadDefaultTemplate();
// Uploads the template and closes the outputstream
Streams.copy(template, connection.getOutputStream());
if (connection.getResponseCode() != 200 && connection.getResponseCode() != 201) {
logConnectionError("error adding the marvel template to [" + host + "]", connection);
return false;
}
logger.info("marvel template updated to version [{}]", templateVersion);
logger.info("http exporter [{}] - marvel template updated to version [{}]", name(), templateVersion);
} catch (IOException e) {
logger.error("failed to update the marvel template to [{}]:\n{}", host, e.getMessage());
logger.error("http exporter [{}] - failed to update the marvel template to [{}]:\n{}", name(), host, e.getMessage());
return false;
} finally {
@ -465,9 +489,111 @@ public class HttpExporter extends Exporter {
}
}
if (config.settings().getAsBoolean("update_mappings", true)) {
updateMappings(host, MarvelSettings.MARVEL_DATA_INDEX_NAME);
updateMappings(host, indexNameResolver().resolve(System.currentTimeMillis()));
}
return true;
}
// TODO: Remove this method once marvel indices are versioned (v 2.2.0)
void updateMappings(String host, String index) {
logger.trace("http exporter [{}] - updating mappings for index [{}]", name(), index);
// Parse the default template to get its mappings
PutIndexTemplateRequest template = new PutIndexTemplateRequest().source(MarvelTemplateUtils.loadDefaultTemplate());
if ((template == null) || (template.mappings() == null) || (template.mappings().isEmpty())) {
return;
}
Set<String> indexMappings = new HashSet<>();
HttpURLConnection connection = null;
try {
connection = openConnection(host, "GET", "/" + index + "/_mapping", XContentType.JSON.restContentType());
if (connection == null) {
logger.debug("http exporter [{}] - no available connection to get index mappings", name());
return;
}
if (connection.getResponseCode() == 404) {
logger.trace("http exporter [{}] - index [{}] does not exist", name(), index);
return;
} else if (connection.getResponseCode() == 200) {
try (InputStream is = connection.getInputStream()) {
ByteArrayOutputStream out = new ByteArrayOutputStream();
Streams.copy(is, out);
Map<String, Object> mappings = XContentHelper.convertToMap(new BytesArray(out.toByteArray()), false).v2();
if ((mappings.get(index) != null) && (mappings.get(index) instanceof Map)) {
Map m = (Map) ((Map) mappings.get(index)).get("mappings");
if (m != null) {
indexMappings = m.keySet();
}
}
}
} else {
logConnectionError("http exporter [" + name() +"] - failed to get mappings for index [" + index + "] on host [" + host + "]", connection);
return;
}
} catch (Exception e) {
logger.error("http exporter [{}] - failed to update the marvel template to [{}]:\n{}", name(), host, e.getMessage());
return;
} finally {
if (connection != null) {
try {
connection.getInputStream().close();
} catch (IOException e) {
// Ignore
}
}
}
// Iterates over document types defined in the default template
for (String type : template.mappings().keySet()) {
if (indexMappings.contains(type)) {
logger.trace("http exporter [{}] - type [{} already exists in mapping of index [{}]", name(), type, index);
continue;
}
logger.trace("http exporter [{}] - adding type [{}] to index [{}] mappings", name(), type, index);
updateMappingForType(host, index, type, template.mappings().get(type));
}
}
void updateMappingForType(String host, String index, String type, String mappingSource) {
logger.trace("http exporter [{}] - updating index [{}] mappings for type [{}] on host [{}]", name(), index, type, host);
HttpURLConnection connection = null;
try {
connection = openConnection(host, "PUT", "/" + index + "/_mapping/" + type, XContentType.JSON.restContentType());
if (connection == null) {
logger.debug("http exporter [{}] - no available connection to update index mapping", name());
return;
}
// Uploads the template and closes the outputstream
Streams.copy(Strings.toUTF8Bytes(mappingSource), connection.getOutputStream());
if (connection.getResponseCode() != 200 && connection.getResponseCode() != 201) {
logConnectionError("http exporter [" + name() +"] - mapping of index [" + index + "] failed to be updated for type [" + type + "] on host [" + host + "]", connection);
return;
}
logger.trace("http exporter [{}] - mapping of index [{}] updated for type [{}]", name(), index, type);
} catch (Exception e) {
logger.error("http exporter [{}] - failed to update mapping of index [{}] for type [{}]", name(), index, type);
} finally {
if (connection != null) {
try {
connection.getInputStream().close();
} catch (IOException e) {
// Ignore
}
}
}
}
private void logConnectionError(String msg, HttpURLConnection conn) {
InputStream inputStream = conn.getErrorStream();
String err = "";

183
elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/local/LocalExporter.java
View File

@ -7,6 +7,10 @@ package org.elasticsearch.marvel.agent.exporter.local;
import org.elasticsearch.Version;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest;
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse;
import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest;
import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateResponse;
import org.elasticsearch.client.Client;
@ -15,18 +19,20 @@ import org.elasticsearch.cluster.ClusterService;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.ClusterStateListener;
import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
import org.elasticsearch.cluster.metadata.MappingMetaData;
import org.elasticsearch.common.collect.ImmutableOpenMap;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.io.Streams;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.gateway.GatewayService;
import org.elasticsearch.index.IndexNotFoundException;
import org.elasticsearch.marvel.agent.exporter.ExportBulk;
import org.elasticsearch.marvel.agent.exporter.Exporter;
import org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils;
import org.elasticsearch.marvel.agent.renderer.RendererRegistry;
import org.elasticsearch.marvel.agent.settings.MarvelSettings;
import org.elasticsearch.marvel.shield.SecuredClient;
import java.io.ByteArrayOutputStream;
import java.io.InputStream;
import static org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils.installedTemplateVersionIsSufficient;
import static org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils.installedTemplateVersionMandatesAnUpdate;
/**
*
@ -42,11 +48,21 @@ public class LocalExporter extends Exporter implements ClusterStateListener {
private volatile LocalBulk bulk;
private volatile boolean active = true;
/** Version of the built-in template **/
private final Version templateVersion;
public LocalExporter(Exporter.Config config, Client client, ClusterService clusterService, RendererRegistry renderers) {
super(TYPE, config);
this.client = client;
this.clusterService = clusterService;
this.renderers = renderers;
// Checks that the built-in template is versioned
templateVersion = MarvelTemplateUtils.loadDefaultTemplateVersion();
if (templateVersion == null) {
throw new IllegalStateException("unable to find built-in template version");
}
bulk = resolveBulk(clusterService.state(), bulk);
clusterService.add(this);
}
@ -122,7 +138,7 @@ public class LocalExporter extends Exporter implements ClusterStateListener {
return null;
}
Version installedTemplateVersion = MarvelTemplateUtils.templateVersion(installedTemplate);
if (!installedTemplateVersionIsSufficient(Version.CURRENT, installedTemplateVersion)) {
if (!installedTemplateVersionIsSufficient(installedTemplateVersion)) {
logger.debug("local exporter [{}] - cannot start. the currently installed marvel template (version [{}]) is incompatible with the " +
"current elasticsearch version [{}]. waiting until the template is updated", name(), installedTemplateVersion, Version.CURRENT);
return null;
@ -138,21 +154,21 @@ public class LocalExporter extends Exporter implements ClusterStateListener {
// if we cannot find a template or a compatible template, we'll install one in / update it.
if (installedTemplate == null) {
logger.debug("local exporter [{}] - could not find existing marvel template, installing a new one", name());
putTemplate(config.settings().getAsSettings("template.settings"));
putTemplate();
// we'll get that template on the next cluster state update
return null;
}
Version installedTemplateVersion = MarvelTemplateUtils.templateVersion(installedTemplate);
if (installedTemplateVersionMandatesAnUpdate(Version.CURRENT, installedTemplateVersion)) {
logger.debug("local exporter [{}] - installing new marvel template [{}], replacing [{}]", name(), Version.CURRENT, installedTemplateVersion);
putTemplate(config.settings().getAsSettings("template.settings"));
if (installedTemplateVersionMandatesAnUpdate(templateVersion, installedTemplateVersion, logger, name())) {
logger.debug("local exporter [{}] - installing new marvel template [{}], replacing [{}]", name(), templateVersion, installedTemplateVersion);
putTemplate();
// we'll get that template on the next cluster state update
return null;
} else if (!installedTemplateVersionIsSufficient(Version.CURRENT, installedTemplateVersion)) {
} else if (!installedTemplateVersionIsSufficient(installedTemplateVersion)) {
logger.error("local exporter [{}] - marvel template version [{}] is below the minimum compatible version [{}]. "
+ "please manually update the marvel template to a more recent version"
+ "and delete the current active marvel index (don't forget to back up it first if needed)",
name(), installedTemplateVersion, MIN_SUPPORTED_TEMPLATE_VERSION);
name(), installedTemplateVersion, MarvelTemplateUtils.MIN_SUPPORTED_TEMPLATE_VERSION);
// we're not going to do anything with the template.. it's too old, and the schema might
// be too different than what this version of marvel/es can work with. For this reason we're
// not going to export any data, to avoid mapping conflicts.
@ -163,86 +179,91 @@ public class LocalExporter extends Exporter implements ClusterStateListener {
return currentBulk != null ? currentBulk : new LocalBulk(name(), logger, client, indexNameResolver, renderers);
}
boolean installedTemplateVersionIsSufficient(Version current, Version installed) {
// null indicates couldn't parse the version from the installed template, this means it is probably too old or invalid...
if (installed == null) {
return false;
}
// ensure the template is not too old
if (installed.before(MIN_SUPPORTED_TEMPLATE_VERSION)) {
return false;
}
void putTemplate() {
PutIndexTemplateRequest request = new PutIndexTemplateRequest(MarvelTemplateUtils.INDEX_TEMPLATE_NAME).source(MarvelTemplateUtils.loadDefaultTemplate());
assert !Thread.currentThread().isInterrupted() : "current thread has been interrupted before putting index template!!!";
// We do not enforce that versions are equivalent to the current version as we may be in a rolling upgrade scenario
// and until a master is elected with the new version, data nodes that have been upgraded will not be able to ship
// data. This means that there is an implication that the new shippers will ship data correctly even with an old template.
// There is also no upper bound and we rely on elasticsearch nodes not being able to connect to each other across major
// versions
return true;
}
// async call, so we won't block cluster event thread
client.admin().indices().putTemplate(request, new ActionListener<PutIndexTemplateResponse>() {
@Override
public void onResponse(PutIndexTemplateResponse response) {
if (response.isAcknowledged()) {
logger.trace("local exporter [{}] - successfully installed marvel template", name());
boolean installedTemplateVersionMandatesAnUpdate(Version current, Version installed) {
if (installed == null) {
logger.debug("local exporter [{}] - currently installed marvel template is missing a version - installing a new one [{}]", name(), current);
return true;
}
// Never update a very old template
if (installed.before(MIN_SUPPORTED_TEMPLATE_VERSION)) {
return false;
}
// Always update a template to the last up-to-date version
if (current.after(installed)) {
logger.debug("local exporter [{}] - currently installed marvel template version [{}] will be updated to a newer version [{}]", name(), installed, current);
return true;
// When the template is up-to-date, force an update for snapshot versions only
} else if (current.equals(installed)) {
logger.debug("local exporter [{}] - currently installed marvel template version [{}] is up-to-date", name(), installed);
return installed.snapshot() && !current.snapshot();
// Never update a template that is newer than the expected one
} else {
logger.debug("local exporter [{}] - currently installed marvel template version [{}] is newer than the one required [{}]... keeping it.", name(), installed, current);
return false;
}
}
void putTemplate(Settings customSettings) {
try (InputStream is = getClass().getResourceAsStream("/marvel_index_template.json")) {
ByteArrayOutputStream out = new ByteArrayOutputStream();
Streams.copy(is, out);
final byte[] template = out.toByteArray();
PutIndexTemplateRequest request = new PutIndexTemplateRequest(MarvelTemplateUtils.INDEX_TEMPLATE_NAME).source(template);
if (customSettings != null && customSettings.names().size() > 0) {
Settings updatedSettings = Settings.builder()
.put(request.settings())
.put(customSettings)
// making sure we override any other template that may apply
.put("order", Integer.MAX_VALUE)
.build();
request.settings(updatedSettings);
if (config.settings().getAsBoolean("update_mappings", true)) {
updateMappings(MarvelSettings.MARVEL_DATA_INDEX_NAME);
updateMappings(indexNameResolver().resolve(System.currentTimeMillis()));
}
} else {
logger.error("local exporter [{}] - failed to update marvel index template", name());
}
}
assert !Thread.currentThread().isInterrupted() : "current thread has been interrupted before putting index template!!!";
@Override
public void onFailure(Throwable throwable) {
logger.error("local exporter [{}] - failed to update marvel index template", throwable, name());
}
});
}
// async call, so we won't block cluster event thread
client.admin().indices().putTemplate(request, new ActionListener<PutIndexTemplateResponse>() {
@Override
public void onResponse(PutIndexTemplateResponse response) {
if (response.isAcknowledged()) {
logger.trace("local exporter [{}] - successfully installed marvel template", name());
} else {
logger.error("local exporter [{}] - failed to update marvel index template", name());
// TODO: Remove this method once marvel indices are versioned (v 2.2.0)
void updateMappings(String index) {
logger.trace("local exporter [{}] - updating mappings for index [{}]", name(), index);
// Parse the default template to get its mappings
PutIndexTemplateRequest template = new PutIndexTemplateRequest().source(MarvelTemplateUtils.loadDefaultTemplate());
if ((template == null) || (template.mappings() == null) || (template.mappings().isEmpty())) {
return;
}
// async call, so we won't block cluster event thread
client.admin().indices().getMappings(new GetMappingsRequest().indices(index), new ActionListener<GetMappingsResponse>() {
@Override
public void onResponse(GetMappingsResponse response) {
ImmutableOpenMap<String, MappingMetaData> indexMappings = response.getMappings().get(index);
if (indexMappings != null) {
// Iterates over document types defined in the default template
for (String type : template.mappings().keySet()) {
if (indexMappings.get(type) != null) {
logger.trace("local exporter [{}] - type [{} already exists in mapping of index [{}]", name(), type, index);
continue;
}
logger.trace("local exporter [{}] - adding type [{}] to index [{}] mappings", name(), type, index);
updateMappingForType(index, type, template.mappings().get(type));
}
}
}
@Override
public void onFailure(Throwable throwable) {
logger.error("local exporter [{}] - failed to update marvel index template", throwable, name());
@Override
public void onFailure(Throwable e) {
if (e instanceof IndexNotFoundException) {
logger.trace("local exporter [{}] - index [{}] not found, unable to update mappings", name(), index);
} else {
logger.error("local exporter [{}] - failed to get mappings for index [{}]", name(), index);
}
});
}
});
}
} catch (Exception e) {
throw new IllegalStateException("failed to update marvel index template", e);
}
void updateMappingForType(String index, String type, String mappingSource) {
logger.trace("local exporter [{}] - updating index [{}] mappings for type [{}]", name(), index, type);
client.admin().indices().putMapping(new PutMappingRequest(index).type(type).source(mappingSource), new ActionListener<PutMappingResponse>() {
@Override
public void onResponse(PutMappingResponse response) {
if (response.isAcknowledged()) {
logger.trace("local exporter [{}] - mapping of index [{}] updated for type [{}]", name(), index, type);
} else {
logger.trace("local exporter [{}] - mapping of index [{}] failed to be updated for type [{}]", name(), index, type);
}
}
@Override
public void onFailure(Throwable e) {
logger.error("local exporter [{}] - failed to update mapping of index [{}] for type [{}]", name(), index, type);
}
});
}
public static class Factory extends Exporter.Factory<LocalExporter> {

261
elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/AbstractExporterTemplateTestCase.java Normal file
View File

@ -0,0 +1,261 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.marvel.agent.exporter;
import org.elasticsearch.Version;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.marvel.agent.collector.Collector;
import org.elasticsearch.marvel.agent.collector.cluster.ClusterInfoCollector;
import org.elasticsearch.marvel.agent.collector.node.NodeStatsCollector;
import org.elasticsearch.marvel.agent.settings.MarvelSettings;
import org.elasticsearch.marvel.test.MarvelIntegTestCase;
import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
import org.elasticsearch.test.VersionUtils;
import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.elasticsearch.test.ESIntegTestCase.Scope.TEST;
@ClusterScope(scope = TEST, numDataNodes = 0, numClientNodes = 0, transportClientRatio = 0.0)
public abstract class AbstractExporterTemplateTestCase extends MarvelIntegTestCase {
@Override
protected Settings nodeSettings(int nodeOrdinal) {
Settings.Builder settings = Settings.builder()
.put(super.nodeSettings(nodeOrdinal))
.put(MarvelSettings.INTERVAL, "-1");
for (Map.Entry<String, String> setting : exporterSettings().getAsMap().entrySet()) {
settings.put("marvel.agent.exporters._exporter." + setting.getKey(), setting.getValue());
}
return settings.build();
}
protected abstract Settings exporterSettings();
protected abstract void deleteTemplate() throws Exception;
protected abstract void putTemplate(String version) throws Exception;
protected abstract void createMarvelIndex(String index) throws Exception;
protected abstract void assertTemplateUpdated(Version version) throws Exception;
protected abstract void assertTemplateNotUpdated(Version version) throws Exception;
protected abstract void assertMappingsUpdated(String... indices) throws Exception;
protected abstract void assertMappingsNotUpdated(String... indices) throws Exception;
protected abstract void assertIndicesNotCreated() throws Exception;
public void testCreateWhenNoExistingTemplate() throws Exception {
internalCluster().startNode();
deleteTemplate();
doExporting();
logger.debug("--> template does not exist: it should have been created in the current version");
assertTemplateUpdated(currentVersion());
doExporting();
logger.debug("--> mappings should be up-to-date");
assertMappingsUpdated(currentIndices());
}
public void testUpdateWhenExistingTemplateHasNoVersion() throws Exception {
internalCluster().startNode();
putTemplate("");
doExporting();
logger.debug("--> existing template does not have a version: it should be updated to the current version");
assertTemplateUpdated(currentVersion());
doExporting();
logger.debug("--> mappings should be up-to-date");
assertMappingsUpdated(currentIndices());
}
public void testUpdateWhenExistingTemplateHasWrongVersion() throws Exception {
internalCluster().startNode();
putTemplate(randomAsciiOfLength(5));
doExporting();
logger.debug("--> existing template has a wrong version: it should be updated to the current version");
assertTemplateUpdated(currentVersion());
doExporting();
logger.debug("--> mappings should be up-to-date");
assertMappingsUpdated(currentIndices());
}
public void testNoUpdateWhenExistingTemplateIsTooOld() throws Exception {
internalCluster().startNode();
putTemplate(VersionUtils.getFirstVersion().number());
doExporting();
logger.debug("--> existing template is too old: it should not be updated");
assertTemplateNotUpdated(VersionUtils.getFirstVersion());
doExporting();
logger.debug("--> existing template is too old: no data is exported");
assertIndicesNotCreated();
}
public void testUpdateWhenExistingTemplateIsOld() throws Exception {
internalCluster().startNode();
putTemplate(VersionUtils.getPreviousVersion(currentVersion()).number());
doExporting();
logger.debug("--> existing template is old but supported: it should be updated to the current version");
assertTemplateUpdated(currentVersion());
doExporting();
logger.debug("--> mappings should be up-to-date");
assertMappingsUpdated(currentIndices());
}
public void testUpdateWhenExistingTemplateIsUpToDate() throws Exception {
internalCluster().startNode();
putTemplate(currentVersion().toString());
doExporting();
logger.debug("--> existing template has the same version: it should not be updated");
assertTemplateNotUpdated(currentVersion());
doExporting();
logger.debug("--> mappings should not have been updated");
assertMappingsNotUpdated(currentIndices());
}
public void testMappingsUpdate() throws Exception {
boolean updateMappings = randomBoolean();
logger.debug("--> update_mappings is {}", updateMappings);
internalCluster().startNode(Settings.builder().put("marvel.agent.exporters._exporter.update_mappings", updateMappings));
logger.debug("--> putting a template with a very old version so that it will not be updated");
putTemplate(VersionUtils.getFirstVersion().toString());
logger.debug("--> creating marvel data index");
createMarvelIndex(MarvelSettings.MARVEL_DATA_INDEX_NAME);
logger.debug("--> creating a cold marvel index");
createMarvelIndex(coldIndex());
logger.debug("--> creating an active marvel index");
createMarvelIndex(hotIndex());
logger.debug("--> all indices have a old mapping now");
assertMappingsNotUpdated(coldIndex(), hotIndex(), MarvelSettings.MARVEL_DATA_INDEX_NAME);
logger.debug("--> updating the template with a previous version, so that it will be updated when exporting documents");
putTemplate(VersionUtils.getPreviousVersion(currentVersion()).number());
doExporting();
logger.debug("--> existing template is old: it should be updated to the current version");
assertTemplateUpdated(currentVersion());
logger.debug("--> cold marvel index: mappings should not have been updated");
assertMappingsNotUpdated(coldIndex());
if (updateMappings) {
logger.debug("--> marvel indices: mappings should be up-to-date");
assertMappingsUpdated(MarvelSettings.MARVEL_DATA_INDEX_NAME, hotIndex());
} else {
logger.debug("--> marvel indices: mappings should bnot have been updated");
assertMappingsNotUpdated(MarvelSettings.MARVEL_DATA_INDEX_NAME, hotIndex());
}
}
protected void doExporting() throws Exception {
List<MarvelDoc> docs = new ArrayList<>();
for (Class<? extends Collector> collectorClass : Arrays.asList(ClusterInfoCollector.class, NodeStatsCollector.class)) {
Collector collector = internalCluster().getInstance(collectorClass);
docs.addAll(collector.collect());
}
exporter().export(docs);
}
private Exporter exporter() {
Exporters exporters = internalCluster().getInstance(Exporters.class);
return exporters.iterator().next();
}
private Version currentVersion() {
return MarvelTemplateUtils.loadDefaultTemplateVersion();
}
private String[] currentIndices() {
return new String[]{hotIndex(), MarvelSettings.MARVEL_DATA_INDEX_NAME};
}
private String coldIndex() {
return exporter().indexNameResolver().resolve(new DateTime(2012, 3, 10, 0, 0, DateTimeZone.UTC).getMillis());
}
private String hotIndex() {
return exporter().indexNameResolver().resolve(System.currentTimeMillis());
}
/** Generates a template that looks like an old one **/
protected static BytesReference generateTemplateSource(String version) throws IOException {
return jsonBuilder().startObject()
.field("template", ".marvel-es-*")
.startObject("settings")
.field("index.number_of_shards", 1)
.field("index.number_of_replicas", 1)
.field("index.mapper.dynamic", false)
.field(MarvelTemplateUtils.MARVEL_VERSION_FIELD, version)
.endObject()
.startObject("mappings")
.startObject("_default_")
.startObject("_all")
.field("enabled", false)
.endObject()
.field("date_detection", false)
.startObject("properties")
.startObject("cluster_uuid")
.field("type", "string")
.field("index", "not_analyzed")
.endObject()
.startObject("timestamp")
.field("type", "date")
.field("format", "date_time")
.endObject()
.endObject()
.endObject()
.startObject("cluster_info")
.field("enabled", false)
.endObject()
.startObject("node_stats")
.startObject("properties")
.startObject("node_stats")
.field("type", "object")
.endObject()
.endObject()
.endObject()
.endObject().bytes();
}
}

93
elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/MarvelTemplateUtilsTests.java Normal file
View File

@ -0,0 +1,93 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.marvel.agent.exporter;
import org.elasticsearch.Version;
import org.elasticsearch.common.Strings;
import org.elasticsearch.marvel.support.VersionUtils;
import org.elasticsearch.test.ESTestCase;
import org.hamcrest.Matchers;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import static org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils.MARVEL_VERSION_FIELD;
public class MarvelTemplateUtilsTests extends ESTestCase {
public void testLoadTemplate() {
byte[] template = MarvelTemplateUtils.loadDefaultTemplate();
assertNotNull(template);
assertThat(template.length, Matchers.greaterThan(0));
}
public void testParseTemplateVersionFromByteArrayTemplate() throws IOException {
byte[] template = MarvelTemplateUtils.loadDefaultTemplate();
assertNotNull(template);
Version version = MarvelTemplateUtils.parseTemplateVersion(template);
assertNotNull(version);
}
public void testParseTemplateVersionFromStringTemplate() throws IOException {
List<String> templates = new ArrayList<>();
templates.add("{\"marvel_version\": \"1.4.0.Beta1\"}");
templates.add("{\"marvel_version\": \"1.6.2-SNAPSHOT\"}");
templates.add("{\"marvel_version\": \"1.7.1\"}");
templates.add("{\"marvel_version\": \"2.0.0-beta1\"}");
templates.add("{\"marvel_version\": \"2.0.0\"}");
templates.add("{ \"template\": \".marvel*\", \"settings\": { \"marvel_version\": \"2.0.0-beta1-SNAPSHOT\", \"index.number_of_shards\": 1 } }");
for (String template : templates) {
Version version = MarvelTemplateUtils.parseTemplateVersion(Strings.toUTF8Bytes(template));
assertNotNull(version);
}
Version version = MarvelTemplateUtils.parseTemplateVersion(Strings.toUTF8Bytes("{\"marvel.index_format\": \"7\"}"));
assertNull(version);
}
public void testParseVersion() throws IOException {
assertNotNull(VersionUtils.parseVersion(MARVEL_VERSION_FIELD, "{\"marvel_version\": \"2.0.0-beta1\"}"));
assertNotNull(VersionUtils.parseVersion(MARVEL_VERSION_FIELD, "{\"marvel_version\": \"2.0.0\"}"));
assertNotNull(VersionUtils.parseVersion(MARVEL_VERSION_FIELD, "{\"marvel_version\": \"1.5.2\"}"));
assertNotNull(VersionUtils.parseVersion(MARVEL_VERSION_FIELD, "{ \"template\": \".marvel*\", \"settings\": { \"marvel_version\": \"2.0.0-beta1-SNAPSHOT\", \"index.number_of_shards\": 1 } }"));
assertNull(VersionUtils.parseVersion(MARVEL_VERSION_FIELD, "{\"marvel.index_format\": \"7\"}"));
assertNull(VersionUtils.parseVersion(MARVEL_VERSION_FIELD + "unkown", "{\"marvel_version\": \"1.5.2\"}"));
}
public void testTemplateVersionMandatesAnUpdate() {
// Version is unknown
assertTrue(MarvelTemplateUtils.installedTemplateVersionMandatesAnUpdate(Version.CURRENT, null, logger, "unit-test"));
// Version is too old
Version unsupported = org.elasticsearch.test.VersionUtils.getPreviousVersion(MarvelTemplateUtils.MIN_SUPPORTED_TEMPLATE_VERSION);
assertFalse(MarvelTemplateUtils.installedTemplateVersionMandatesAnUpdate(Version.CURRENT, unsupported, logger, "unit-test"));
// Version is old but supported
assertTrue(MarvelTemplateUtils.installedTemplateVersionMandatesAnUpdate(Version.CURRENT, MarvelTemplateUtils.MIN_SUPPORTED_TEMPLATE_VERSION, logger, "unit-test"));
// Version is up to date
assertFalse(MarvelTemplateUtils.installedTemplateVersionMandatesAnUpdate(Version.CURRENT, Version.CURRENT, logger, "unit-test"));
// Version is up to date
Version previous = org.elasticsearch.test.VersionUtils.getPreviousVersion(Version.CURRENT);
assertFalse(MarvelTemplateUtils.installedTemplateVersionMandatesAnUpdate(previous, Version.CURRENT, logger, "unit-test"));
}
public void testTemplateVersionIsSufficient() {
// Version is unknown
assertFalse(MarvelTemplateUtils.installedTemplateVersionIsSufficient(null));
// Version is too old
Version unsupported = org.elasticsearch.test.VersionUtils.getPreviousVersion(MarvelTemplateUtils.MIN_SUPPORTED_TEMPLATE_VERSION);
assertFalse(MarvelTemplateUtils.installedTemplateVersionIsSufficient(unsupported));
// Version is OK
assertTrue(MarvelTemplateUtils.installedTemplateVersionIsSufficient(Version.CURRENT));
}
}

251
elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/http/HttpExporterTemplateTests.java Normal file
View File

@ -0,0 +1,251 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.marvel.agent.exporter.http;
import com.squareup.okhttp.mockwebserver.Dispatcher;
import com.squareup.okhttp.mockwebserver.MockResponse;
import com.squareup.okhttp.mockwebserver.MockWebServer;
import com.squareup.okhttp.mockwebserver.RecordedRequest;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.Version;
import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.marvel.agent.exporter.AbstractExporterTemplateTestCase;
import org.elasticsearch.marvel.agent.exporter.Exporter;
import org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils;
import org.junit.After;
import org.junit.Before;
import java.io.IOException;
import java.net.BindException;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.core.Is.is;
public class HttpExporterTemplateTests extends AbstractExporterTemplateTestCase {
private MockWebServer webServer;
private MockServerDispatcher dispatcher;
@Before
public void startWebServer() throws Exception {
for (int webPort = 9250; webPort < 9300; webPort++) {
try {
webServer = new MockWebServer();
dispatcher = new MockServerDispatcher();
webServer.setDispatcher(dispatcher);
webServer.start(webPort);
return;
} catch (BindException be) {
logger.warn("port [{}] was already in use trying next port", webPort);
}
}
throw new ElasticsearchException("unable to find open port between 9200 and 9300");
}
@After
public void stopWebServer() throws Exception {
webServer.shutdown();
}
@Override
protected Settings exporterSettings() {
return Settings.builder()
.put("type", "http")
.put("host", webServer.getHostName() + ":" + webServer.getPort())
.put("connection.keep_alive", false)
.put(Exporter.INDEX_NAME_TIME_FORMAT_SETTING, "YYYY")
.build();
}
@Override
protected void deleteTemplate() {
dispatcher.setTemplate(null);
}
@Override
protected void putTemplate(String version) throws Exception {
dispatcher.setTemplate(generateTemplateSource(version).toBytes());
}
@Override
protected void createMarvelIndex(String index) throws Exception {
dispatcher.addIndex(index);
}
@Override
protected void assertTemplateUpdated(Version version) {
// Checks that a PUT Template request has been made
assertThat(dispatcher.hasRequest("PUT", "/_template/" + MarvelTemplateUtils.INDEX_TEMPLATE_NAME), is(true));
// Checks that the current template has the expected version
assertThat(MarvelTemplateUtils.parseTemplateVersion(dispatcher.getTemplate()), equalTo(version));
}
@Override
protected void assertTemplateNotUpdated(Version version) throws Exception {
// Checks that no PUT Template request has been made
assertThat(dispatcher.hasRequest("PUT", "/_template/" + MarvelTemplateUtils.INDEX_TEMPLATE_NAME), is(false));
// Checks that the current template has the expected version
assertThat(MarvelTemplateUtils.parseTemplateVersion(dispatcher.getTemplate()), equalTo(version));
}
@Override
protected void assertIndicesNotCreated() throws Exception {
// Checks that no Bulk request has been made
assertThat(dispatcher.hasRequest("POST", "/_bulk"), is(false));
assertThat(dispatcher.mappings.size(), equalTo(0));
}
@Override
protected void assertMappingsUpdated(String... indices) throws Exception {
// Load the mappings of the old template
Set<String> oldMappings = new PutIndexTemplateRequest().source(generateTemplateSource(null)).mappings().keySet();
// Load the mappings of the latest template
Set<String> newMappings = new PutIndexTemplateRequest().source(generateTemplateSource(null)).mappings().keySet();
newMappings.removeAll(oldMappings);
for (String index : indices) {
for (String mapping : newMappings) {
// Checks that a PUT Mapping request has been made for every type that was not in the old template
assertThat(dispatcher.hasRequest("PUT", "/" + index + "/_mapping/" + mapping), equalTo(true));
}
}
}
@Override
protected void assertMappingsNotUpdated(String... indices) throws Exception {
for (String index : indices) {
// Checks that no PUT Template request has been made
assertThat(dispatcher.hasRequest("PUT", "/" + index + "/_mapping/"), is(false));
}
}
class MockServerDispatcher extends Dispatcher {
private final MockResponse OK = newResponse(200, "");
private final MockResponse NOT_FOUND = newResponse(404, "");
private final Set<String> requests = new HashSet<>();
private final Map<String, Set<String>> mappings = new HashMap<>();
private byte[] template;
@Override
public MockResponse dispatch(RecordedRequest request) throws InterruptedException {
synchronized (this) {
final String requestLine = request.getRequestLine();
requests.add(requestLine);
switch (requestLine) {
// Cluster version
case "GET / HTTP/1.1":
return newResponse(200, "{\"version\": {\"number\": \"" + Version.CURRENT.number() + "\"}}");
// Template
case "GET /_template/.marvel-es HTTP/1.1":
return (template == null) ? NOT_FOUND : newResponse(200, new BytesArray(template).toUtf8());
case "PUT /_template/.marvel-es HTTP/1.1":
this.template = request.getBody().readByteArray();
return OK;
// Bulk
case "POST /_bulk HTTP/1.1":
return OK;
default:
String[] paths = Strings.splitStringToArray(request.getPath(), '/');
// Index Mappings
if ((paths != null) && (paths.length > 0) && ("_mapping".equals(paths[1]))) {
if (!mappings.containsKey(paths[0])) {
// Index does not exist
return NOT_FOUND;
}
// Get index mappings
if ("GET".equals(request.getMethod())) {
try {
// Builds a fake mapping response
XContentBuilder builder = jsonBuilder().startObject().startObject(paths[0]).startObject("mappings");
for (String type : mappings.get(paths[0])) {
builder.startObject(type).endObject();
}
builder.endObject().endObject().endObject();
return newResponse(200, builder.bytes().toUtf8());
} catch (IOException e) {
return newResponse(500, e.getMessage());
}
// Put index mapping
} else if ("PUT".equals(request.getMethod()) && paths.length > 2) {
Set<String> types = mappings.get(paths[0]);
if (types == null) {
types = new HashSet<>();
}
types.add(paths[2]);
return OK;
}
}
break;
}
return newResponse(500, "MockServerDispatcher does not support: " + request.getRequestLine());
}
}
MockResponse newResponse(int code, String body) {
return new MockResponse().setResponseCode(code).setBody(body);
}
void setTemplate(byte[] template) {
synchronized (this) {
this.template = template;
}
}
byte[] getTemplate() {
return template;
}
void addIndex(String index) {
synchronized (this) {
if (template != null) {
// Simulate the use of the index template when creating an index
mappings.put(index, new HashSet<>(new PutIndexTemplateRequest().source(template).mappings().keySet()));
} else {
mappings.put(index, null);
}
}
}
int countRequests(String method, String path) {
int count = 0;
for (String request : requests) {
if (request.startsWith(method + " " + path)) {
count += 1;
}
}
return count;
}
boolean hasRequest(String method, String path) {
return countRequests(method, path) > 0;
}
}
}

91
elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/http/HttpExporterTests.java
View File

@ -13,7 +13,6 @@ import okio.Buffer;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.Version;
import org.elasticsearch.action.ActionRequest;
import org.elasticsearch.cluster.health.ClusterHealthStatus;
import org.elasticsearch.action.admin.indices.recovery.RecoveryResponse;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.index.IndexRequest;
@ -88,7 +87,8 @@ public class HttpExporterTests extends MarvelIntegTestCase {
.put(MarvelSettings.INTERVAL, "-1")
.put("marvel.agent.exporters._http.type", "http")
.put("marvel.agent.exporters._http.host", webServer.getHostName() + ":" + webServer.getPort())
.put("marvel.agent.exporters._http.connection.keep_alive", false);
.put("marvel.agent.exporters._http.connection.keep_alive", false)
.put("marvel.agent.exporters._http.update_mappings", false);
String agentNode = internalCluster().startNode(builder);
HttpExporter exporter = getExporter(agentNode);
@ -142,56 +142,14 @@ public class HttpExporterTests extends MarvelIntegTestCase {
assertThat(getExporter(nodeName).hosts, Matchers.arrayContaining("test3"));
}
public void testTemplateUpdate() throws Exception {
Settings.Builder builder = Settings.builder()
.put(MarvelSettings.INTERVAL, "-1")
.put("marvel.agent.exporters._http.type", "http")
.put("marvel.agent.exporters._http.host", webServer.getHostName() + ":" + webServer.getPort())
.put("marvel.agent.exporters._http.connection.keep_alive", false);
logger.info("--> starting node");
enqueueGetClusterVersionResponse(Version.CURRENT);
enqueueResponse(404, "marvel template does not exist");
enqueueResponse(201, "marvel template created");
enqueueResponse(200, "successful bulk request ");
String agentNode = internalCluster().startNode(builder);
logger.info("--> exporting data");
HttpExporter exporter = getExporter(agentNode);
final int nbDocs = randomIntBetween(1, 25);
exporter.export(newRandomMarvelDocs(nbDocs));
assertThat(webServer.getRequestCount(), equalTo(4));
RecordedRequest recordedRequest = webServer.takeRequest();
assertThat(recordedRequest.getMethod(), equalTo("GET"));
assertThat(recordedRequest.getPath(), equalTo("/"));
recordedRequest = webServer.takeRequest();
assertThat(recordedRequest.getMethod(), equalTo("GET"));
assertThat(recordedRequest.getPath(), equalTo("/_template/.marvel-es"));
recordedRequest = webServer.takeRequest();
assertThat(recordedRequest.getMethod(), equalTo("PUT"));
assertThat(recordedRequest.getPath(), equalTo("/_template/.marvel-es"));
assertThat(recordedRequest.getBody().readByteArray(), equalTo(MarvelTemplateUtils.loadDefaultTemplate()));
recordedRequest = webServer.takeRequest();
assertThat(recordedRequest.getMethod(), equalTo("POST"));
assertThat(recordedRequest.getPath(), equalTo("/_bulk"));
assertBulkRequest(recordedRequest.getBody(), nbDocs);
}
public void testHostChangeReChecksTemplate() throws Exception {
Settings.Builder builder = Settings.builder()
.put(MarvelSettings.INTERVAL, "-1")
.put("marvel.agent.exporters._http.type", "http")
.put("marvel.agent.exporters._http.host", webServer.getHostName() + ":" + webServer.getPort())
.put("marvel.agent.exporters._http.connection.keep_alive", false);
.put("marvel.agent.exporters._http.connection.keep_alive", false)
.put("marvel.agent.exporters._http.update_mappings", false);
logger.info("--> starting node");
@ -287,44 +245,6 @@ public class HttpExporterTests extends MarvelIntegTestCase {
}
}
public void testUnsupportedTemplateVersion() throws Exception {
Settings.Builder builder = Settings.builder()
.put(MarvelSettings.INTERVAL, "-1")
.put("marvel.agent.exporters._http.type", "http")
.put("marvel.agent.exporters._http.host", webServer.getHostName() + ":" + webServer.getPort())
.put("marvel.agent.exporters._http.connection.keep_alive", false);
logger.info("--> starting node");
enqueueGetClusterVersionResponse(Version.CURRENT);
// returning a fake template with an unsupported version
Version unsupportedVersion = randomFrom(Version.V_0_18_0, Version.V_1_0_0, Version.V_1_4_0);
enqueueResponse(200, XContentHelper.toString(Settings.builder().put("index.marvel_version", unsupportedVersion.toString()).build()));
String agentNode = internalCluster().startNode(builder);
logger.info("--> exporting data");
HttpExporter exporter = getExporter(agentNode);
assertThat(exporter.supportedClusterVersion, is(false));
exporter.export(Collections.singletonList(newRandomMarvelDoc()));
assertThat(exporter.supportedClusterVersion, is(true));
assertThat(webServer.getRequestCount(), equalTo(3));
RecordedRequest recordedRequest = webServer.takeRequest();
assertThat(recordedRequest.getMethod(), equalTo("GET"));
assertThat(recordedRequest.getPath(), equalTo("/"));
recordedRequest = webServer.takeRequest();
assertThat(recordedRequest.getMethod(), equalTo("GET"));
assertThat(recordedRequest.getPath(), equalTo("/_template/.marvel-es"));
recordedRequest = webServer.takeRequest();
assertThat(recordedRequest.getMethod(), equalTo("PUT"));
assertThat(recordedRequest.getPath(), equalTo("/_template/.marvel-es"));
assertThat(recordedRequest.getBody().readByteArray(), equalTo(MarvelTemplateUtils.loadDefaultTemplate()));
}
public void testUnsupportedClusterVersion() throws Exception {
Settings.Builder builder = Settings.builder()
.put(MarvelSettings.INTERVAL, "-1")
@ -357,7 +277,8 @@ public class HttpExporterTests extends MarvelIntegTestCase {
.put(MarvelSettings.INTERVAL, "-1")
.put("marvel.agent.exporters._http.type", "http")
.put("marvel.agent.exporters._http.host", webServer.getHostName() + ":" + webServer.getPort())
.put("marvel.agent.exporters._http.connection.keep_alive", false);
.put("marvel.agent.exporters._http.connection.keep_alive", false)
.put("marvel.agent.exporters._http.update_mappings", false);
String agentNode = internalCluster().startNode(builder);

48
elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/http/HttpExporterUtilsTests.java
View File

@ -5,64 +5,16 @@
*/
package org.elasticsearch.marvel.agent.exporter.http;
import org.elasticsearch.Version;
import org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils;
import org.elasticsearch.marvel.support.VersionUtils;
import org.elasticsearch.test.ESTestCase;
import org.hamcrest.Matchers;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URISyntaxException;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import static org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils.MARVEL_VERSION_FIELD;
import static org.hamcrest.CoreMatchers.equalTo;
public class HttpExporterUtilsTests extends ESTestCase {
public void testLoadTemplate() {
byte[] template = MarvelTemplateUtils.loadDefaultTemplate();
assertNotNull(template);
assertThat(template.length, Matchers.greaterThan(0));
}
public void testParseTemplateVersionFromByteArrayTemplate() throws IOException {
byte[] template = MarvelTemplateUtils.loadDefaultTemplate();
assertNotNull(template);
Version version = MarvelTemplateUtils.parseTemplateVersion(template);
assertNotNull(version);
}
public void testParseTemplateVersionFromStringTemplate() throws IOException {
List<String> templates = new ArrayList<>();
templates.add("{\"marvel_version\": \"1.4.0.Beta1\"}");
templates.add("{\"marvel_version\": \"1.6.2-SNAPSHOT\"}");
templates.add("{\"marvel_version\": \"1.7.1\"}");
templates.add("{\"marvel_version\": \"2.0.0-beta1\"}");
templates.add("{\"marvel_version\": \"2.0.0\"}");
templates.add("{ \"template\": \".marvel*\", \"settings\": { \"marvel_version\": \"2.0.0-beta1-SNAPSHOT\", \"index.number_of_shards\": 1 } }");
for (String template : templates) {
Version version = MarvelTemplateUtils.parseTemplateVersion(template);
assertNotNull(version);
}
Version version = MarvelTemplateUtils.parseTemplateVersion("{\"marvel.index_format\": \"7\"}");
assertNull(version);
}
public void testParseVersion() throws IOException {
assertNotNull(VersionUtils.parseVersion(MARVEL_VERSION_FIELD, "{\"marvel_version\": \"2.0.0-beta1\"}"));
assertNotNull(VersionUtils.parseVersion(MARVEL_VERSION_FIELD, "{\"marvel_version\": \"2.0.0\"}"));
assertNotNull(VersionUtils.parseVersion(MARVEL_VERSION_FIELD, "{\"marvel_version\": \"1.5.2\"}"));
assertNotNull(VersionUtils.parseVersion(MARVEL_VERSION_FIELD, "{ \"template\": \".marvel*\", \"settings\": { \"marvel_version\": \"2.0.0-beta1-SNAPSHOT\", \"index.number_of_shards\": 1 } }"));
assertNull(VersionUtils.parseVersion(MARVEL_VERSION_FIELD, "{\"marvel.index_format\": \"7\"}"));
assertNull(VersionUtils.parseVersion(MARVEL_VERSION_FIELD + "unkown", "{\"marvel_version\": \"1.5.2\"}"));
}
public void testHostParsing() throws MalformedURLException, URISyntaxException {
URL url = HttpExporterUtils.parseHostWithPath("localhost:9200", "");

116
elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/local/LocalExporterTemplateTests.java Normal file
View File

@ -0,0 +1,116 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.marvel.agent.exporter.local;
import org.elasticsearch.Version;
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest;
import org.elasticsearch.action.support.IndicesOptions;
import org.elasticsearch.cluster.metadata.MappingMetaData;
import org.elasticsearch.common.collect.ImmutableOpenMap;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.IndexNotFoundException;
import org.elasticsearch.marvel.agent.exporter.AbstractExporterTemplateTestCase;
import org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils;
import org.elasticsearch.marvel.agent.settings.MarvelSettings;
import java.util.Collections;
import java.util.Map;
import java.util.Set;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.notNullValue;
import static org.hamcrest.core.Is.is;
public class LocalExporterTemplateTests extends AbstractExporterTemplateTestCase {
@Override
protected Settings exporterSettings() {
return Settings.builder().put("type", LocalExporter.TYPE).build();
}
@Override
protected Set<String> excludeTemplates() {
// Always delete the template between tests
return Collections.emptySet();
}
@Override
protected void deleteTemplate() throws Exception {
waitNoPendingTasksOnAll();
assertAcked(client().admin().indices().prepareDeleteTemplate(MarvelTemplateUtils.INDEX_TEMPLATE_NAME).get());
}
@Override
protected void putTemplate(String version) throws Exception {
waitNoPendingTasksOnAll();
assertAcked(client().admin().indices().preparePutTemplate(MarvelTemplateUtils.INDEX_TEMPLATE_NAME).setSource(generateTemplateSource(version)).get());
}
@Override
protected void createMarvelIndex(String index) throws Exception {
waitNoPendingTasksOnAll();
createIndex(index);
}
@Override
protected void assertTemplateUpdated(Version version) throws Exception {
waitNoPendingTasksOnAll();
awaitMarvelTemplateInstalled(version);
}
@Override
protected void assertTemplateNotUpdated(Version version) throws Exception {
waitNoPendingTasksOnAll();
awaitMarvelTemplateInstalled(version);
}
private void assertMappings(byte[] reference, String... indices) throws Exception {
waitNoPendingTasksOnAll();
Map<String, String> mappings = new PutIndexTemplateRequest().source(reference).mappings();
assertBusy(new Runnable() {
@Override
public void run() {
for (String index : indices) {
GetMappingsResponse response = client().admin().indices().prepareGetMappings(index).setIndicesOptions(IndicesOptions.lenientExpandOpen()).get();
ImmutableOpenMap<String, MappingMetaData> indexMappings = response.getMappings().get(index);
assertNotNull(indexMappings);
assertThat(indexMappings.size(), equalTo(mappings.size()));
for (String mapping : mappings.keySet()) {
// We just check that mapping type exists, we don't verify its content
assertThat("mapping type " + mapping + " should exist in index " + index, indexMappings.get(mapping), notNullValue());
}
}
}
});
}
@Override
protected void assertMappingsUpdated(String... indices) throws Exception {
assertMappings(MarvelTemplateUtils.loadDefaultTemplate(), indices);
}
@Override
protected void assertMappingsNotUpdated(String... indices) throws Exception {
assertMappings(generateTemplateSource(null).toBytes(), indices);
}
@Override
protected void assertIndicesNotCreated() throws Exception {
waitNoPendingTasksOnAll();
try {
assertThat(client().admin().indices().prepareExists(MarvelSettings.MARVEL_INDICES_PREFIX + "*").get().isExists(), is(false));
} catch (IndexNotFoundException e) {
// with shield we might get that if wildcards were resolved to no indices
if (!shieldEnabled) {
throw e;
}
}
}
}

135
elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/local/LocalExporterTests.java
View File

@ -10,17 +10,9 @@ import org.elasticsearch.Version;
import org.elasticsearch.action.admin.indices.recovery.RecoveryResponse;
import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.Client;
import org.elasticsearch.cluster.ClusterService;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.block.ClusterBlocks;
import org.elasticsearch.cluster.health.ClusterHealthStatus;
import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
import org.elasticsearch.cluster.metadata.MetaData;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.common.collect.ImmutableOpenMap;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.gateway.GatewayService;
import org.elasticsearch.marvel.agent.collector.cluster.ClusterStateCollector;
import org.elasticsearch.marvel.agent.collector.cluster.ClusterStateMarvelDoc;
import org.elasticsearch.marvel.agent.collector.indices.IndexRecoveryCollector;
@ -29,13 +21,11 @@ import org.elasticsearch.marvel.agent.exporter.Exporter;
import org.elasticsearch.marvel.agent.exporter.Exporters;
import org.elasticsearch.marvel.agent.exporter.MarvelDoc;
import org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils;
import org.elasticsearch.marvel.agent.renderer.RendererRegistry;
import org.elasticsearch.marvel.agent.settings.MarvelSettings;
import org.elasticsearch.marvel.test.MarvelIntegTestCase;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
import org.elasticsearch.test.ESIntegTestCase.Scope;
import org.elasticsearch.test.InternalTestCluster;
import org.joda.time.format.DateTimeFormat;
import org.junit.After;
@ -43,14 +33,11 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
import static org.elasticsearch.marvel.agent.exporter.Exporter.MIN_SUPPORTED_TEMPLATE_VERSION;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
import static org.hamcrest.Matchers.*;
import static org.mockito.Mockito.*;
@ClusterScope(scope = Scope.TEST, numDataNodes = 0, numClientNodes = 0, transportClientRatio = 0.0)
public class LocalExporterTests extends MarvelIntegTestCase {
@ -105,105 +92,17 @@ public class LocalExporterTests extends MarvelIntegTestCase {
public void testTemplateCreation() throws Exception {
internalCluster().startNode(Settings.builder()
.put("marvel.agent.exporters._local.type", LocalExporter.TYPE)
.put("marvel.agent.exporters._local.template.settings.index.number_of_replicas", 0)
.build());
securedEnsureGreen();
LocalExporter exporter = getLocalExporter("_local");
assertTrue(exporter.installedTemplateVersionMandatesAnUpdate(Version.CURRENT, null));
// start collecting
updateMarvelInterval(3L, TimeUnit.SECONDS);
waitForMarvelIndices();
// lets wait until the marvel template will be installed
awaitMarvelTemplateInstalled();
awaitMarvelDocsCount(greaterThan(0L));
assertThat(getCurrentlyInstalledTemplateVersion(), is(Version.CURRENT));
}
public void testTemplateUpdate() throws Exception {
internalCluster().startNode(Settings.builder()
.put("marvel.agent.exporters._local.type", LocalExporter.TYPE)
.put("marvel.agent.exporters._local.template.settings.index.number_of_replicas", 0)
.build());
securedEnsureGreen();
LocalExporter exporter = getLocalExporter("_local");
Version fakeVersion = MIN_SUPPORTED_TEMPLATE_VERSION;
assertThat(exporter.installedTemplateVersionMandatesAnUpdate(Version.CURRENT, fakeVersion), is(true));
// start collecting
updateMarvelInterval(3L, TimeUnit.SECONDS);
waitForMarvelIndices();
// first, lets wait for the marvel template to be installed
awaitMarvelTemplateInstalled();
// stop collecting before cluster restart
updateMarvelInterval(-1, TimeUnit.SECONDS);
wipeMarvelIndices();
// now lets update the template with an old one and then restart the cluster
exporter.putTemplate(Settings.builder().put(MarvelTemplateUtils.MARVEL_VERSION_FIELD, fakeVersion.toString()).build());
logger.debug("full cluster restart");
final CountDownLatch latch = new CountDownLatch(1);
internalCluster().fullRestart(new InternalTestCluster.RestartCallback() {
@Override
public void doAfterNodes(int n, Client client) throws Exception {
latch.countDown();
}
});
if (!latch.await(30, TimeUnit.SECONDS)) {
fail("waited too long (at least 30 seconds) for the cluster to restart");
}
// start collecting again
updateMarvelInterval(3L, TimeUnit.SECONDS);
waitForMarvelIndices();
// now that the cluster is restarting, lets wait for the new template version to be installed
awaitMarvelTemplateInstalled(Version.CURRENT);
}
public void testUnsupportedTemplateVersion() throws Exception {
Exporter.Config config = new Exporter.Config("_name", Settings.EMPTY, Settings.builder()
.put("type", "local").build());
Client client = mock(Client.class);
ClusterService clusterService = mock(ClusterService.class);
boolean master = randomBoolean();
DiscoveryNode localNode = mock(DiscoveryNode.class);
when(localNode.masterNode()).thenReturn(master);
when(clusterService.localNode()).thenReturn(localNode);
RendererRegistry renderers = mock(RendererRegistry.class);
LocalExporter exporter = spy(new LocalExporter(config, client, clusterService, renderers));
// creating a cluster state mock that holds unsupported template version
Version unsupportedVersion = randomFrom(Version.V_0_18_0, Version.V_1_0_0, Version.V_1_4_0);
IndexTemplateMetaData template = mock(IndexTemplateMetaData.class);
when(template.settings()).thenReturn(Settings.builder().put("index.marvel_version", unsupportedVersion.toString()).build());
MetaData metaData = mock(MetaData.class);
when(metaData.getTemplates()).thenReturn(ImmutableOpenMap.<String, IndexTemplateMetaData>builder().fPut(MarvelTemplateUtils.INDEX_TEMPLATE_NAME, template).build());
ClusterBlocks blocks = mock(ClusterBlocks.class);
when(blocks.hasGlobalBlock(GatewayService.STATE_NOT_RECOVERED_BLOCK)).thenReturn(false);
ClusterState clusterState = mock(ClusterState.class);
when(clusterState.getMetaData()).thenReturn(metaData);
when(clusterState.blocks()).thenReturn(blocks);
when(clusterService.state()).thenReturn(clusterState);
assertThat(exporter.resolveBulk(clusterState, null), nullValue());
verifyZeroInteractions(client);
if (master) {
verify(exporter, times(1)).installedTemplateVersionMandatesAnUpdate(Version.CURRENT, unsupportedVersion);
}
verify(exporter, times(1)).installedTemplateVersionIsSufficient(Version.CURRENT, unsupportedVersion);
}
public void testIndexTimestampFormat() throws Exception {
long time = System.currentTimeMillis();
String timeFormat = randomFrom("YY", "YYYY", "YYYY.MM", "YYYY-MM", "MM.YYYY", "MM");
@ -241,40 +140,6 @@ public class LocalExporterTests extends MarvelIntegTestCase {
awaitIndexExists(indexName);
}
public void testInstalledTemplateVersionChecking() throws Exception {
Exporter.Config config = new Exporter.Config("_name", Settings.EMPTY, Settings.builder()
.put("type", "local").build());
Client client = mock(Client.class);
ClusterService clusterService = mock(ClusterService.class);
boolean master = randomBoolean();
DiscoveryNode localNode = mock(DiscoveryNode.class);
when(localNode.masterNode()).thenReturn(master);
when(clusterService.localNode()).thenReturn(localNode);
RendererRegistry renderers = mock(RendererRegistry.class);
LocalExporter exporter = new LocalExporter(config, client, clusterService, renderers);
assertTrue("current template version should always be sufficient", exporter.installedTemplateVersionIsSufficient(Version.CURRENT, Version.CURRENT));
Version version = Version.fromId(Version.CURRENT.id + 1000000);
assertTrue("future versions should be considered sufficient in case of a rolling upgrade scenario",
exporter.installedTemplateVersionIsSufficient(Version.CURRENT, version));
// make sure we test at least one snapshot and non-snapshot
String versionStr = "2.0.1";
if (randomBoolean()) {
versionStr += "-SNAPSHOT";
}
Version version1 = Version.fromString(versionStr);
assertTrue("snapshots should not matter", exporter.installedTemplateVersionIsSufficient(version1, version1));
// test the minimum version
assertTrue("minimum template version should always be sufficient", exporter.installedTemplateVersionIsSufficient(Version.CURRENT, Exporter.MIN_SUPPORTED_TEMPLATE_VERSION));
// test a version below the minimum version
assertFalse("version below minimum should not be sufficient", exporter.installedTemplateVersionIsSufficient(Version.CURRENT, Version.V_2_0_0_beta1));
assertFalse("null version should not be sufficient", exporter.installedTemplateVersionIsSufficient(Version.CURRENT, null));
}
public void testLocalExporterFlush() throws Exception {
internalCluster().startNode(Settings.builder()
.put("marvel.agent.exporters._local.type", LocalExporter.TYPE)

1
elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/cluster/ClusterStateTests.java
View File

@ -39,7 +39,6 @@ public class ClusterStateTests extends MarvelIntegTestCase {
.put(MarvelSettings.INTERVAL, "-1")
.put(MarvelSettings.COLLECTORS, ClusterStateCollector.NAME)
.put("marvel.agent.exporters.default_local.type", "local")
.put("marvel.agent.exporters.default_local.template.settings.index.number_of_replicas", 0)
.build();
}

2
elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/cluster/ClusterStatsTests.java
View File

@ -9,7 +9,6 @@ import org.elasticsearch.action.admin.cluster.stats.ClusterStatsNodes;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.marvel.agent.collector.cluster.ClusterStatsCollector;
import org.elasticsearch.marvel.agent.collector.indices.IndexStatsCollector;
import org.elasticsearch.marvel.agent.settings.MarvelSettings;
import org.elasticsearch.marvel.test.MarvelIntegTestCase;
import org.elasticsearch.search.SearchHit;
@ -33,7 +32,6 @@ public class ClusterStatsTests extends MarvelIntegTestCase {
.put(MarvelSettings.INTERVAL, "-1")
.put(MarvelSettings.COLLECTORS, ClusterStatsCollector.NAME)
.put("marvel.agent.exporters.default_local.type", "local")
.put("marvel.agent.exporters.default_local.template.settings.index.number_of_replicas", 0)
.build();
}

1
elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/indices/IndexRecoveryTests.java
View File

@ -38,7 +38,6 @@ public class IndexRecoveryTests extends MarvelIntegTestCase {
.put(MarvelSettings.INDICES, INDEX_PREFIX + "*")
.put(MarvelSettings.COLLECTORS, IndexRecoveryCollector.NAME)
.put("marvel.agent.exporters.default_local.type", "local")
.put("marvel.agent.exporters.default_local.template.settings.index.number_of_replicas", 0)
.build();
}

1
elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/indices/IndexStatsTests.java
View File

@ -30,7 +30,6 @@ public class IndexStatsTests extends MarvelIntegTestCase {
.put(MarvelSettings.INTERVAL, "-1")
.put(MarvelSettings.COLLECTORS, IndexStatsCollector.NAME)
.put("marvel.agent.exporters.default_local.type", "local")
.put("marvel.agent.exporters.default_local.template.settings.index.number_of_replicas", 0)
.build();
}

1
elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/indices/IndicesStatsTests.java
View File

@ -30,7 +30,6 @@ public class IndicesStatsTests extends MarvelIntegTestCase {
.put(MarvelSettings.INTERVAL, "-1")
.put(MarvelSettings.COLLECTORS, IndicesStatsCollector.NAME)
.put("marvel.agent.exporters.default_local.type", "local")
.put("marvel.agent.exporters.default_local.template.settings.index.number_of_replicas", 0)
.build();
}

1
elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/node/MultiNodesStatsTests.java
View File

@ -31,7 +31,6 @@ public class MultiNodesStatsTests extends MarvelIntegTestCase {
.put(super.nodeSettings(nodeOrdinal))
.put(MarvelSettings.INTERVAL, "-1")
.put("marvel.agent.exporters.default_local.type", "local")
.put("marvel.agent.exporters.default_local.template.settings.index.number_of_replicas", 0)
.build();
}

1
elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/node/NodeStatsTests.java
View File

@ -31,7 +31,6 @@ public class NodeStatsTests extends MarvelIntegTestCase {
.put(MarvelSettings.INTERVAL, "-1")
.put(MarvelSettings.COLLECTORS, NodeStatsCollector.NAME)
.put("marvel.agent.exporters.default_local.type", LocalExporter.TYPE)
.put("marvel.agent.exporters.default_local.template.settings.index.number_of_replicas", 0)
.build();
}

1
elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/shards/ShardsTests.java
View File

@ -45,7 +45,6 @@ public class ShardsTests extends MarvelIntegTestCase {
.put(MarvelSettings.COLLECTORS, ShardsCollector.NAME)
.put(MarvelSettings.INDICES, INDEX_PREFIX + "*")
.put("marvel.agent.exporters.default_local.type", "local")
.put("marvel.agent.exporters.default_local.template.settings.index.number_of_replicas", 0)
.build();
}

17
elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/test/MarvelIntegTestCase.java
View File

@ -39,7 +39,10 @@ import java.io.OutputStream;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.*;
import java.util.Collection;
import java.util.Collections;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
@ -110,9 +113,7 @@ public abstract class MarvelIntegTestCase extends ESIntegTestCase {
* Override and returns {@code false} to force running without shield
*/
protected boolean enableShield() {
boolean r = randomBoolean();
logger.info("--> shield is{}", r);
return r;
return randomBoolean();
}
protected void stopCollection() {
@ -170,15 +171,15 @@ public abstract class MarvelIntegTestCase extends ESIntegTestCase {
}, 30, TimeUnit.SECONDS);
}
protected void ensureMarvelIndicesGreen() {
protected void ensureMarvelIndicesYellow() {
if (shieldEnabled) {
try {
ensureGreen(".marvel-es-*");
ensureYellow(".marvel-es-*");
} catch (IndexNotFoundException e) {
// might happen with shield...
}
} else {
ensureGreen(".marvel-es-*");
ensureYellow(".marvel-es-*");
}
}
@ -257,7 +258,7 @@ public abstract class MarvelIntegTestCase extends ESIntegTestCase {
assertBusy(new Runnable() {
@Override
public void run() {
ensureMarvelIndicesGreen();
ensureMarvelIndicesYellow();
}
});
}