Marvel: Use versioned index templates
This commit changes the templates so that they are now versionned using a number (starting from 1). This number is used in index templates names (ex: .marvel-es-data-1, .marvel-es-1) as well as in indices names (ex: .marvel-es-1-2015-12-30, .marvel-es-data-1). If the template does not exist, it is created. Otherwise nothing (no update) is done. Original commit: elastic/x-pack-elasticsearch@66c1a8bed0
This commit is contained in:
parent
a931e4e0b8
commit
eeb5842730
|
@ -70,6 +70,9 @@ compileTestJava.options.compilerArgs << "-Xlint:-deprecation,-rawtypes,-serial,-
|
|||
|
||||
ext.expansions = [
|
||||
'project.version': version,
|
||||
// Used in marvel index templates
|
||||
'marvel.plugin.version': version,
|
||||
'marvel.template.version': '1',
|
||||
]
|
||||
|
||||
processResources {
|
||||
|
|
|
@ -11,11 +11,14 @@ import org.elasticsearch.cluster.ClusterService;
|
|||
import org.elasticsearch.common.component.AbstractLifecycleComponent;
|
||||
import org.elasticsearch.common.inject.Inject;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.marvel.agent.exporter.IndexNameResolver;
|
||||
import org.elasticsearch.marvel.agent.exporter.MarvelDoc;
|
||||
import org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils;
|
||||
import org.elasticsearch.marvel.agent.settings.MarvelSettings;
|
||||
import org.elasticsearch.marvel.license.MarvelLicensee;
|
||||
|
||||
import java.util.Collection;
|
||||
import java.util.Objects;
|
||||
|
||||
public abstract class AbstractCollector<T> extends AbstractLifecycleComponent<T> implements Collector<T> {
|
||||
|
||||
|
@ -24,6 +27,7 @@ public abstract class AbstractCollector<T> extends AbstractLifecycleComponent<T>
|
|||
protected final ClusterService clusterService;
|
||||
protected final MarvelSettings marvelSettings;
|
||||
protected final MarvelLicensee licensee;
|
||||
protected final IndexNameResolver dataIndexNameResolver;
|
||||
|
||||
@Inject
|
||||
public AbstractCollector(Settings settings, String name, ClusterService clusterService, MarvelSettings marvelSettings, MarvelLicensee licensee) {
|
||||
|
@ -32,6 +36,7 @@ public abstract class AbstractCollector<T> extends AbstractLifecycleComponent<T>
|
|||
this.clusterService = clusterService;
|
||||
this.marvelSettings = marvelSettings;
|
||||
this.licensee = licensee;
|
||||
this.dataIndexNameResolver = new DataIndexNameResolver(MarvelTemplateUtils.TEMPLATE_VERSION);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
@ -109,4 +114,27 @@ public abstract class AbstractCollector<T> extends AbstractLifecycleComponent<T>
|
|||
protected String clusterUUID() {
|
||||
return clusterService.state().metaData().clusterUUID();
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves marvel's data index name
|
||||
*/
|
||||
public class DataIndexNameResolver implements IndexNameResolver {
|
||||
|
||||
private final String index;
|
||||
|
||||
DataIndexNameResolver(Integer version) {
|
||||
Objects.requireNonNull(version, "index version cannot be null");
|
||||
this.index = MarvelSettings.MARVEL_DATA_INDEX_PREFIX + String.valueOf(version);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String resolve(MarvelDoc doc) {
|
||||
return index;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String resolve(long timestamp) {
|
||||
return index;
|
||||
}
|
||||
}
|
||||
}
|
|
@ -77,7 +77,8 @@ public class ClusterInfoCollector extends AbstractCollector<ClusterInfoMarvelDoc
|
|||
}
|
||||
|
||||
String clusterUUID = clusterUUID();
|
||||
results.add(new ClusterInfoMarvelDoc(MarvelSettings.MARVEL_DATA_INDEX_NAME, TYPE, clusterUUID, clusterUUID, System.currentTimeMillis(),
|
||||
long timestamp = System.currentTimeMillis();
|
||||
results.add(new ClusterInfoMarvelDoc(dataIndexNameResolver.resolve(timestamp), TYPE, clusterUUID, clusterUUID, timestamp,
|
||||
clusterName.value(), Version.CURRENT.toString(), license, clusterStats));
|
||||
return Collections.unmodifiableCollection(results);
|
||||
}
|
||||
|
|
|
@ -71,7 +71,7 @@ public class ClusterStateCollector extends AbstractCollector<ClusterStateCollect
|
|||
results.add(new ClusterStateNodeMarvelDoc(clusterUUID, NODES_TYPE, timestamp, stateUUID, node.getId()));
|
||||
|
||||
// Adds a document for every node in the marvel data index (type "node")
|
||||
results.add(new DiscoveryNodeMarvelDoc(MarvelSettings.MARVEL_DATA_INDEX_NAME, NODE_TYPE, node.getId(), clusterUUID, timestamp, node));
|
||||
results.add(new DiscoveryNodeMarvelDoc(dataIndexNameResolver.resolve(timestamp), NODE_TYPE, node.getId(), clusterUUID, timestamp, node));
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -155,7 +155,7 @@ public abstract class Exporter {
|
|||
|
||||
@Override
|
||||
public String resolve(long timestamp) {
|
||||
return MarvelSettings.MARVEL_INDICES_PREFIX + indexTimeFormatter.print(timestamp);
|
||||
return MarvelSettings.MARVEL_INDICES_PREFIX + String.valueOf(MarvelTemplateUtils.TEMPLATE_VERSION) + "-" + indexTimeFormatter.print(timestamp);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
|
@ -5,113 +5,103 @@
|
|||
*/
|
||||
package org.elasticsearch.marvel.agent.exporter;
|
||||
|
||||
import org.elasticsearch.Version;
|
||||
import org.elasticsearch.cluster.ClusterState;
|
||||
import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
|
||||
import org.elasticsearch.cluster.metadata.MetaData;
|
||||
import org.elasticsearch.common.Strings;
|
||||
import org.elasticsearch.common.io.Streams;
|
||||
import org.elasticsearch.common.logging.ESLogger;
|
||||
import org.elasticsearch.marvel.support.VersionUtils;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.util.Properties;
|
||||
|
||||
/**
|
||||
*
|
||||
*/
|
||||
public final class MarvelTemplateUtils {
|
||||
|
||||
public static final String MARVEL_TEMPLATE_FILE = "/marvel_index_template.json";
|
||||
public static final String INDEX_TEMPLATE_NAME = ".marvel-es";
|
||||
public static final String MARVEL_VERSION_FIELD = "marvel_version";
|
||||
public static final Version MIN_SUPPORTED_TEMPLATE_VERSION = Version.V_2_0_0_beta2;
|
||||
static final String INDEX_TEMPLATE_FILE = "/marvel-es.json";
|
||||
static final String INDEX_TEMPLATE_NAME_PREFIX = ".marvel-es-";
|
||||
|
||||
static final String DATA_TEMPLATE_FILE = "/marvel-es-data.json";
|
||||
static final String DATA_TEMPLATE_NAME_PREFIX = ".marvel-es-data-";
|
||||
|
||||
static final String PROPERTIES_FILE = "/marvel.properties";
|
||||
static final String VERSION_FIELD = "marvel.template.version";
|
||||
|
||||
public static final Integer TEMPLATE_VERSION = loadTemplateVersion();
|
||||
|
||||
private MarvelTemplateUtils() {
|
||||
}
|
||||
|
||||
/**
|
||||
* Loads the default Marvel template
|
||||
* Loads the default template for the timestamped indices
|
||||
*/
|
||||
public static byte[] loadDefaultTemplate() {
|
||||
try (InputStream is = MarvelTemplateUtils.class.getResourceAsStream(MARVEL_TEMPLATE_FILE)) {
|
||||
ByteArrayOutputStream out = new ByteArrayOutputStream();
|
||||
Streams.copy(is, out);
|
||||
return out.toByteArray();
|
||||
public static byte[] loadTimestampedIndexTemplate() {
|
||||
try {
|
||||
return load(INDEX_TEMPLATE_FILE);
|
||||
} catch (IOException e) {
|
||||
throw new IllegalStateException("unable to load marvel template", e);
|
||||
}
|
||||
}
|
||||
|
||||
public static Version loadDefaultTemplateVersion() {
|
||||
return parseTemplateVersion(loadDefaultTemplate());
|
||||
}
|
||||
|
||||
public static Version templateVersion(IndexTemplateMetaData templateMetaData) {
|
||||
String version = templateMetaData.settings().get("index." + MARVEL_VERSION_FIELD);
|
||||
if (Strings.hasLength(version)) {
|
||||
try {
|
||||
return Version.fromString(version);
|
||||
} catch (IllegalArgumentException e) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
public static IndexTemplateMetaData findMarvelTemplate(ClusterState state) {
|
||||
MetaData metaData = state.getMetaData();
|
||||
return metaData != null ? metaData.getTemplates().get(INDEX_TEMPLATE_NAME) : null;
|
||||
}
|
||||
|
||||
public static Version parseTemplateVersion(byte[] template) {
|
||||
/**
|
||||
* Loads the default template for the data index
|
||||
*/
|
||||
public static byte[] loadDataIndexTemplate() {
|
||||
try {
|
||||
return VersionUtils.parseVersion(MARVEL_VERSION_FIELD, template);
|
||||
} catch (IllegalArgumentException e) {
|
||||
return load(DATA_TEMPLATE_FILE);
|
||||
} catch (IOException e) {
|
||||
throw new IllegalStateException("unable to load marvel data template", e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Loads a resource with a given name and returns it as a byte array.
|
||||
*/
|
||||
static byte[] load(String name) throws IOException {
|
||||
try (InputStream is = MarvelTemplateUtils.class.getResourceAsStream(name)) {
|
||||
ByteArrayOutputStream out = new ByteArrayOutputStream();
|
||||
Streams.copy(is, out);
|
||||
return out.toByteArray();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Loads the current version of templates
|
||||
*
|
||||
* When executing tests in Intellij, the properties file might not be
|
||||
* resolved: try running 'gradle processResources' first.
|
||||
*/
|
||||
static Integer loadTemplateVersion() {
|
||||
try (InputStream is = MarvelTemplateUtils.class.getResourceAsStream(PROPERTIES_FILE)) {
|
||||
Properties properties = new Properties();
|
||||
properties.load(is);
|
||||
String version = properties.getProperty(VERSION_FIELD);
|
||||
if (Strings.hasLength(version)) {
|
||||
return Integer.parseInt(version);
|
||||
}
|
||||
return null;
|
||||
} catch (NumberFormatException e) {
|
||||
throw new IllegalArgumentException("failed to parse marvel template version");
|
||||
} catch (IOException e) {
|
||||
throw new IllegalArgumentException("failed to load marvel template version");
|
||||
}
|
||||
}
|
||||
|
||||
public static boolean installedTemplateVersionIsSufficient(Version installed) {
|
||||
// null indicates couldn't parse the version from the installed template, this means it is probably too old or invalid...
|
||||
if (installed == null) {
|
||||
return false;
|
||||
}
|
||||
// ensure the template is not too old
|
||||
if (installed.before(MIN_SUPPORTED_TEMPLATE_VERSION)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// We do not enforce that versions are equivalent to the current version as we may be in a rolling upgrade scenario
|
||||
// and until a master is elected with the new version, data nodes that have been upgraded will not be able to ship
|
||||
// data. This means that there is an implication that the new shippers will ship data correctly even with an old template.
|
||||
// There is also no upper bound and we rely on elasticsearch nodes not being able to connect to each other across major
|
||||
// versions
|
||||
return true;
|
||||
public static String indexTemplateName() {
|
||||
return indexTemplateName(TEMPLATE_VERSION);
|
||||
}
|
||||
|
||||
public static boolean installedTemplateVersionMandatesAnUpdate(Version current, Version installed, ESLogger logger, String exporterName) {
|
||||
if (installed == null) {
|
||||
logger.debug("exporter [{}] - currently installed marvel template is missing a version - installing a new one [{}]", exporterName, current);
|
||||
return true;
|
||||
}
|
||||
// Never update a very old template
|
||||
if (installed.before(MIN_SUPPORTED_TEMPLATE_VERSION)) {
|
||||
return false;
|
||||
}
|
||||
// Always update a template to the last up-to-date version
|
||||
if (current.after(installed)) {
|
||||
logger.debug("exporter [{}] - currently installed marvel template version [{}] will be updated to a newer version [{}]", exporterName, installed, current);
|
||||
return true;
|
||||
// When the template is up-to-date, do not update
|
||||
} else if (current.equals(installed)) {
|
||||
logger.debug("exporter [{}] - currently installed marvel template version [{}] is up-to-date", exporterName, installed);
|
||||
return false;
|
||||
// Never update a template that is newer than the expected one
|
||||
} else {
|
||||
logger.debug("exporter [{}] - currently installed marvel template version [{}] is newer than the one required [{}]... keeping it.", exporterName, installed, current);
|
||||
return false;
|
||||
}
|
||||
public static String indexTemplateName(Integer version) {
|
||||
return templateName(INDEX_TEMPLATE_NAME_PREFIX, version);
|
||||
}
|
||||
|
||||
public static String dataTemplateName() {
|
||||
return dataTemplateName(TEMPLATE_VERSION);
|
||||
}
|
||||
|
||||
public static String dataTemplateName(Integer version) {
|
||||
return templateName(DATA_TEMPLATE_NAME_PREFIX, version);
|
||||
}
|
||||
|
||||
static String templateName(String prefix, Integer version) {
|
||||
assert version != null && version >= 0 : "version must be not null and greater or equal to zero";
|
||||
return prefix + String.valueOf(version);
|
||||
}
|
||||
}
|
||||
|
|
|
@ -8,11 +8,9 @@ package org.elasticsearch.marvel.agent.exporter.http;
|
|||
import org.elasticsearch.ElasticsearchException;
|
||||
import org.elasticsearch.ExceptionsHelper;
|
||||
import org.elasticsearch.Version;
|
||||
import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest;
|
||||
import org.elasticsearch.common.Base64;
|
||||
import org.elasticsearch.common.Nullable;
|
||||
import org.elasticsearch.common.Strings;
|
||||
import org.elasticsearch.common.bytes.BytesArray;
|
||||
import org.elasticsearch.common.inject.Inject;
|
||||
import org.elasticsearch.common.io.Streams;
|
||||
import org.elasticsearch.common.io.stream.BytesStreamOutput;
|
||||
|
@ -20,7 +18,6 @@ import org.elasticsearch.common.settings.Settings;
|
|||
import org.elasticsearch.common.settings.SettingsException;
|
||||
import org.elasticsearch.common.unit.TimeValue;
|
||||
import org.elasticsearch.common.xcontent.XContentBuilder;
|
||||
import org.elasticsearch.common.xcontent.XContentHelper;
|
||||
import org.elasticsearch.common.xcontent.XContentParser;
|
||||
import org.elasticsearch.common.xcontent.XContentType;
|
||||
import org.elasticsearch.env.Environment;
|
||||
|
@ -55,12 +52,7 @@ import java.nio.file.Path;
|
|||
import java.security.KeyStore;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.HashSet;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
import static org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils.installedTemplateVersionIsSufficient;
|
||||
import static org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils.installedTemplateVersionMandatesAnUpdate;
|
||||
|
||||
/**
|
||||
*
|
||||
|
@ -106,8 +98,8 @@ public class HttpExporter extends Exporter {
|
|||
volatile boolean checkedAndUploadedIndexTemplate = false;
|
||||
volatile boolean supportedClusterVersion = false;
|
||||
|
||||
/** Version of the built-in template **/
|
||||
final Version templateVersion;
|
||||
/** Version number of built-in templates **/
|
||||
private final Integer templateVersion;
|
||||
|
||||
boolean keepAlive;
|
||||
final ConnectionKeepAliveWorker keepAliveWorker;
|
||||
|
@ -142,8 +134,8 @@ public class HttpExporter extends Exporter {
|
|||
sslSocketFactory = createSSLSocketFactory(config.settings().getAsSettings(SSL_SETTING));
|
||||
hostnameVerification = config.settings().getAsBoolean(SSL_HOSTNAME_VERIFICATION_SETTING, true);
|
||||
|
||||
// Checks that the built-in template is versioned
|
||||
templateVersion = MarvelTemplateUtils.loadDefaultTemplateVersion();
|
||||
// Loads the current version number of built-in templates
|
||||
templateVersion = MarvelTemplateUtils.TEMPLATE_VERSION;
|
||||
if (templateVersion == null) {
|
||||
throw new IllegalStateException("unable to find built-in template version");
|
||||
}
|
||||
|
@ -202,7 +194,7 @@ public class HttpExporter extends Exporter {
|
|||
builder.startObject();
|
||||
builder.startObject("index");
|
||||
|
||||
// we need the index to be based on the document timestamp
|
||||
// we need the index to be based on the document timestamp and/or template version
|
||||
builder.field("_index", indexNameResolver.resolve(marvelDoc));
|
||||
|
||||
if (marvelDoc.type() != null) {
|
||||
|
@ -395,101 +387,51 @@ public class HttpExporter extends Exporter {
|
|||
* @return true if template exists or was uploaded successfully.
|
||||
*/
|
||||
private boolean checkAndUploadIndexTemplate(final String host) {
|
||||
byte[] installedTemplate;
|
||||
try {
|
||||
installedTemplate = findMarvelTemplate(host);
|
||||
} catch (Exception e) {
|
||||
logger.debug("http exporter [{}] - exception when loading the existing marvel template on host[{}]", e, name(), host);
|
||||
return false;
|
||||
String templateName = MarvelTemplateUtils.indexTemplateName(templateVersion);
|
||||
boolean templateInstalled = hasTemplate(templateName, host);
|
||||
|
||||
// Works like LocalExporter on master:
|
||||
// Install the index template for timestamped indices first, so that other nodes can ship data
|
||||
if (!templateInstalled) {
|
||||
logger.debug("http exporter [{}] - could not find existing marvel template, installing a new one", name());
|
||||
if (!putTemplate(host, templateName, MarvelTemplateUtils.loadTimestampedIndexTemplate())) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// if we cannot find a template or a compatible template, we'll install one in / update it.
|
||||
if (installedTemplate == null) {
|
||||
logger.debug("http exporter [{}] - could not find existing marvel template, installing a new one", name());
|
||||
return putTemplate(host);
|
||||
}
|
||||
Version installedTemplateVersion = MarvelTemplateUtils.parseTemplateVersion(installedTemplate);
|
||||
if (installedTemplateVersionMandatesAnUpdate(templateVersion, installedTemplateVersion, logger, name())) {
|
||||
logger.debug("http exporter [{}] - installing new marvel template [{}], replacing [{}]", name(), templateVersion, installedTemplateVersion);
|
||||
return putTemplate(host);
|
||||
} else if (!installedTemplateVersionIsSufficient(installedTemplateVersion)) {
|
||||
logger.error("http exporter [{}] - marvel template version [{}] is below the minimum compatible version [{}]. "
|
||||
+ "please manually update the marvel template to a more recent version"
|
||||
+ "and delete the current active marvel index (don't forget to back up it first if needed)",
|
||||
name(), installedTemplateVersion, MarvelTemplateUtils.MIN_SUPPORTED_TEMPLATE_VERSION);
|
||||
// we're not going to do anything with the template.. it's too old, and the schema might
|
||||
// be too different than what this version of marvel/es can work with. For this reason we're
|
||||
// not going to export any data, to avoid mapping conflicts.
|
||||
return false;
|
||||
// Install the index template for data index
|
||||
templateName = MarvelTemplateUtils.dataTemplateName(templateVersion);
|
||||
if (!hasTemplate(templateName, host)) {
|
||||
logger.debug("http exporter [{}] - could not find existing marvel template for data index, installing a new one", name());
|
||||
if (!putTemplate(host, templateName, MarvelTemplateUtils.loadDataIndexTemplate())) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
private byte[] findMarvelTemplate(String host) throws IOException {
|
||||
String url = "_template/" + MarvelTemplateUtils.INDEX_TEMPLATE_NAME;
|
||||
private boolean hasTemplate(String templateName, String host) {
|
||||
String url = "_template/" + templateName;
|
||||
if (templateCheckTimeout != null) {
|
||||
url += "?timeout=" + templateCheckTimeout;
|
||||
}
|
||||
|
||||
HttpURLConnection connection = null;
|
||||
try {
|
||||
logger.debug("http exporter [{}] - checking if marvel template exists on the marvel cluster", name());
|
||||
logger.debug("http exporter [{}] - checking if marvel template [{}] exists on the marvel cluster", name(), templateName);
|
||||
connection = openConnection(host, "GET", url, null);
|
||||
if (connection == null) {
|
||||
throw new IOException("no available connection to check marvel template existence");
|
||||
throw new IOException("no available connection to check for marvel template [" + templateName + "] existence");
|
||||
}
|
||||
|
||||
byte[] remoteTemplate = null;
|
||||
|
||||
// 200 means that the template has been found, 404 otherwise
|
||||
if (connection.getResponseCode() == 200) {
|
||||
logger.debug("marvel template found");
|
||||
|
||||
try (InputStream is = connection.getInputStream()) {
|
||||
ByteArrayOutputStream out = new ByteArrayOutputStream();
|
||||
Streams.copy(is, out);
|
||||
remoteTemplate = out.toByteArray();
|
||||
}
|
||||
logger.debug("marvel template [{}] found",templateName);
|
||||
return true;
|
||||
}
|
||||
return remoteTemplate;
|
||||
} catch (Exception e) {
|
||||
logger.error("http exporter [{}] - failed to verify the marvel template to [{}]:\n{}", name(), host, e.getMessage());
|
||||
throw e;
|
||||
} finally {
|
||||
if (connection != null) {
|
||||
try {
|
||||
connection.getInputStream().close();
|
||||
} catch (IOException e) {
|
||||
// Ignore
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
boolean putTemplate(String host) {
|
||||
HttpURLConnection connection = null;
|
||||
try {
|
||||
connection = openConnection(host, "PUT", "_template/" + MarvelTemplateUtils.INDEX_TEMPLATE_NAME, XContentType.JSON.restContentType());
|
||||
if (connection == null) {
|
||||
logger.debug("http exporter [{}] - no available connection to update marvel template", name());
|
||||
return false;
|
||||
}
|
||||
|
||||
logger.debug("http exporter [{}] - loading marvel pre-configured template", name());
|
||||
byte[] template = MarvelTemplateUtils.loadDefaultTemplate();
|
||||
|
||||
// Uploads the template and closes the outputstream
|
||||
Streams.copy(template, connection.getOutputStream());
|
||||
if (connection.getResponseCode() != 200 && connection.getResponseCode() != 201) {
|
||||
logConnectionError("error adding the marvel template to [" + host + "]", connection);
|
||||
return false;
|
||||
}
|
||||
|
||||
logger.info("http exporter [{}] - marvel template updated to version [{}]", name(), templateVersion);
|
||||
} catch (IOException e) {
|
||||
logger.error("http exporter [{}] - failed to update the marvel template to [{}]:\n{}", name(), host, e.getMessage());
|
||||
logger.error("http exporter [{}] - failed to verify the marvel template [{}] on [{}]:\n{}", name(), templateName, host, e.getMessage());
|
||||
return false;
|
||||
|
||||
} finally {
|
||||
if (connection != null) {
|
||||
try {
|
||||
|
@ -499,101 +441,31 @@ public class HttpExporter extends Exporter {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (config.settings().getAsBoolean("update_mappings", true)) {
|
||||
updateMappings(host, MarvelSettings.MARVEL_DATA_INDEX_NAME);
|
||||
updateMappings(host, indexNameResolver().resolve(System.currentTimeMillis()));
|
||||
}
|
||||
return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
// TODO: Remove this method once marvel indices are versioned (v 2.2.0)
|
||||
void updateMappings(String host, String index) {
|
||||
logger.trace("http exporter [{}] - updating mappings for index [{}]", name(), index);
|
||||
|
||||
// Parse the default template to get its mappings
|
||||
PutIndexTemplateRequest template = new PutIndexTemplateRequest().source(MarvelTemplateUtils.loadDefaultTemplate());
|
||||
if ((template == null) || (template.mappings() == null) || (template.mappings().isEmpty())) {
|
||||
return;
|
||||
}
|
||||
|
||||
Set<String> indexMappings = new HashSet<>();
|
||||
|
||||
boolean putTemplate(String host, String template, byte[] source) {
|
||||
logger.debug("http exporter [{}] - installing template [{}]", name(), template);
|
||||
HttpURLConnection connection = null;
|
||||
try {
|
||||
connection = openConnection(host, "GET", "/" + index + "/_mapping", XContentType.JSON.restContentType());
|
||||
connection = openConnection(host, "PUT", "_template/" + template, XContentType.JSON.restContentType());
|
||||
if (connection == null) {
|
||||
logger.debug("http exporter [{}] - no available connection to get index mappings", name());
|
||||
return;
|
||||
}
|
||||
|
||||
if (connection.getResponseCode() == 404) {
|
||||
logger.trace("http exporter [{}] - index [{}] does not exist", name(), index);
|
||||
return;
|
||||
} else if (connection.getResponseCode() == 200) {
|
||||
try (InputStream is = connection.getInputStream()) {
|
||||
ByteArrayOutputStream out = new ByteArrayOutputStream();
|
||||
Streams.copy(is, out);
|
||||
|
||||
Map<String, Object> mappings = XContentHelper.convertToMap(new BytesArray(out.toByteArray()), false).v2();
|
||||
if ((mappings.get(index) != null) && (mappings.get(index) instanceof Map)) {
|
||||
Map m = (Map) ((Map) mappings.get(index)).get("mappings");
|
||||
if (m != null) {
|
||||
indexMappings = m.keySet();
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
logConnectionError("http exporter [" + name() +"] - failed to get mappings for index [" + index + "] on host [" + host + "]", connection);
|
||||
return;
|
||||
}
|
||||
} catch (Exception e) {
|
||||
logger.error("http exporter [{}] - failed to update the marvel template to [{}]:\n{}", name(), host, e.getMessage());
|
||||
return;
|
||||
|
||||
} finally {
|
||||
if (connection != null) {
|
||||
try {
|
||||
connection.getInputStream().close();
|
||||
} catch (IOException e) {
|
||||
// Ignore
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Iterates over document types defined in the default template
|
||||
for (String type : template.mappings().keySet()) {
|
||||
if (indexMappings.contains(type)) {
|
||||
logger.trace("http exporter [{}] - type [{} already exists in mapping of index [{}]", name(), type, index);
|
||||
continue;
|
||||
}
|
||||
|
||||
logger.trace("http exporter [{}] - adding type [{}] to index [{}] mappings", name(), type, index);
|
||||
updateMappingForType(host, index, type, template.mappings().get(type));
|
||||
}
|
||||
}
|
||||
|
||||
void updateMappingForType(String host, String index, String type, String mappingSource) {
|
||||
logger.trace("http exporter [{}] - updating index [{}] mappings for type [{}] on host [{}]", name(), index, type, host);
|
||||
HttpURLConnection connection = null;
|
||||
try {
|
||||
connection = openConnection(host, "PUT", "/" + index + "/_mapping/" + type, XContentType.JSON.restContentType());
|
||||
if (connection == null) {
|
||||
logger.debug("http exporter [{}] - no available connection to update index mapping", name());
|
||||
return;
|
||||
logger.debug("http exporter [{}] - no available connection to update marvel template [{}]", name(), template);
|
||||
return false;
|
||||
}
|
||||
|
||||
// Uploads the template and closes the outputstream
|
||||
Streams.copy(Strings.toUTF8Bytes(mappingSource), connection.getOutputStream());
|
||||
Streams.copy(source, connection.getOutputStream());
|
||||
if (connection.getResponseCode() != 200 && connection.getResponseCode() != 201) {
|
||||
logConnectionError("http exporter [" + name() +"] - mapping of index [" + index + "] failed to be updated for type [" + type + "] on host [" + host + "]", connection);
|
||||
return;
|
||||
logConnectionError("error adding the marvel template [" + template + "] to [" + host + "]", connection);
|
||||
return false;
|
||||
}
|
||||
|
||||
logger.trace("http exporter [{}] - mapping of index [{}] updated for type [{}]", name(), index, type);
|
||||
} catch (Exception e) {
|
||||
logger.error("http exporter [{}] - failed to update mapping of index [{}] for type [{}]", name(), index, type);
|
||||
|
||||
logger.info("http exporter [{}] - marvel template [{}] updated to version [{}]", name(), template, templateVersion);
|
||||
return true;
|
||||
} catch (IOException e) {
|
||||
logger.error("http exporter [{}] - failed to update marvel template [{}] on host [{}]:\n{}", name(), template, host, e.getMessage());
|
||||
return false;
|
||||
} finally {
|
||||
if (connection != null) {
|
||||
try {
|
||||
|
|
|
@ -60,11 +60,10 @@ public class LocalBulk extends ExportBulk {
|
|||
}
|
||||
|
||||
IndexRequestBuilder request = client.prepareIndex();
|
||||
if (marvelDoc.index() != null) {
|
||||
request.setIndex(marvelDoc.index());
|
||||
} else {
|
||||
request.setIndex(indexNameResolver.resolve(marvelDoc));
|
||||
}
|
||||
|
||||
// we need the index to be based on the document timestamp and/or template version
|
||||
request.setIndex(indexNameResolver.resolve(marvelDoc));
|
||||
|
||||
if (marvelDoc.type() != null) {
|
||||
request.setType(marvelDoc.type());
|
||||
}
|
||||
|
|
|
@ -5,12 +5,8 @@
|
|||
*/
|
||||
package org.elasticsearch.marvel.agent.exporter.local;
|
||||
|
||||
import org.elasticsearch.Version;
|
||||
import com.carrotsearch.hppc.cursors.ObjectCursor;
|
||||
import org.elasticsearch.action.ActionListener;
|
||||
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest;
|
||||
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
|
||||
import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
|
||||
import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse;
|
||||
import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest;
|
||||
import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateResponse;
|
||||
import org.elasticsearch.client.Client;
|
||||
|
@ -18,22 +14,17 @@ import org.elasticsearch.cluster.ClusterChangedEvent;
|
|||
import org.elasticsearch.cluster.ClusterService;
|
||||
import org.elasticsearch.cluster.ClusterState;
|
||||
import org.elasticsearch.cluster.ClusterStateListener;
|
||||
import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
|
||||
import org.elasticsearch.cluster.metadata.MappingMetaData;
|
||||
import org.elasticsearch.cluster.block.ClusterBlockLevel;
|
||||
import org.elasticsearch.common.collect.ImmutableOpenMap;
|
||||
import org.elasticsearch.common.inject.Inject;
|
||||
import org.elasticsearch.common.regex.Regex;
|
||||
import org.elasticsearch.gateway.GatewayService;
|
||||
import org.elasticsearch.index.IndexNotFoundException;
|
||||
import org.elasticsearch.marvel.agent.exporter.ExportBulk;
|
||||
import org.elasticsearch.marvel.agent.exporter.Exporter;
|
||||
import org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils;
|
||||
import org.elasticsearch.marvel.agent.renderer.RendererRegistry;
|
||||
import org.elasticsearch.marvel.agent.settings.MarvelSettings;
|
||||
import org.elasticsearch.marvel.shield.SecuredClient;
|
||||
|
||||
import static org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils.installedTemplateVersionIsSufficient;
|
||||
import static org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils.installedTemplateVersionMandatesAnUpdate;
|
||||
|
||||
/**
|
||||
*
|
||||
*/
|
||||
|
@ -48,8 +39,8 @@ public class LocalExporter extends Exporter implements ClusterStateListener {
|
|||
private volatile LocalBulk bulk;
|
||||
private volatile boolean active = true;
|
||||
|
||||
/** Version of the built-in template **/
|
||||
private final Version templateVersion;
|
||||
/** Version number of built-in templates **/
|
||||
private final Integer templateVersion;
|
||||
|
||||
public LocalExporter(Exporter.Config config, Client client, ClusterService clusterService, RendererRegistry renderers) {
|
||||
super(TYPE, config);
|
||||
|
@ -57,8 +48,8 @@ public class LocalExporter extends Exporter implements ClusterStateListener {
|
|||
this.clusterService = clusterService;
|
||||
this.renderers = renderers;
|
||||
|
||||
// Checks that the built-in template is versioned
|
||||
templateVersion = MarvelTemplateUtils.loadDefaultTemplateVersion();
|
||||
// Loads the current version number of built-in templates
|
||||
templateVersion = MarvelTemplateUtils.TEMPLATE_VERSION;
|
||||
if (templateVersion == null) {
|
||||
throw new IllegalStateException("unable to find built-in template version");
|
||||
}
|
||||
|
@ -126,21 +117,17 @@ public class LocalExporter extends Exporter implements ClusterStateListener {
|
|||
return null;
|
||||
}
|
||||
|
||||
IndexTemplateMetaData installedTemplate = MarvelTemplateUtils.findMarvelTemplate(clusterState);
|
||||
String templateName = MarvelTemplateUtils.indexTemplateName(templateVersion);
|
||||
boolean templateInstalled = hasTemplate(templateName, clusterState);
|
||||
|
||||
// if this is not the master, we'll just look to see if the marvel template is already
|
||||
// if this is not the master, we'll just look to see if the marvel timestamped template is already
|
||||
// installed and if so, if it has a compatible version. If it is (installed and compatible)
|
||||
// we'll be able to start this exporter. Otherwise, we'll just wait for a new cluster state.
|
||||
if (!clusterService.localNode().masterNode()) {
|
||||
if (installedTemplate == null) {
|
||||
// the marvel template is not yet installed in the given cluster state, we'll wait.
|
||||
logger.debug("local exporter [{}] - marvel index template [{}] does not exist, so service cannot start", name(), MarvelTemplateUtils.INDEX_TEMPLATE_NAME);
|
||||
return null;
|
||||
}
|
||||
Version installedTemplateVersion = MarvelTemplateUtils.templateVersion(installedTemplate);
|
||||
if (!installedTemplateVersionIsSufficient(installedTemplateVersion)) {
|
||||
logger.debug("local exporter [{}] - cannot start. the currently installed marvel template (version [{}]) is incompatible with the " +
|
||||
"current elasticsearch version [{}]. waiting until the template is updated", name(), installedTemplateVersion, Version.CURRENT);
|
||||
// We only need to check the index template for timestamped indices
|
||||
if (!templateInstalled) {
|
||||
// the template for timestamped indices is not yet installed in the given cluster state, we'll wait.
|
||||
logger.debug("local exporter [{}] - marvel index template does not exist, so service cannot start", name());
|
||||
return null;
|
||||
}
|
||||
|
||||
|
@ -151,36 +138,65 @@ public class LocalExporter extends Exporter implements ClusterStateListener {
|
|||
|
||||
// we are on master
|
||||
//
|
||||
// if we cannot find a template or a compatible template, we'll install one in / update it.
|
||||
if (installedTemplate == null) {
|
||||
logger.debug("local exporter [{}] - could not find existing marvel template, installing a new one", name());
|
||||
putTemplate();
|
||||
// we'll get that template on the next cluster state update
|
||||
return null;
|
||||
}
|
||||
Version installedTemplateVersion = MarvelTemplateUtils.templateVersion(installedTemplate);
|
||||
if (installedTemplateVersionMandatesAnUpdate(templateVersion, installedTemplateVersion, logger, name())) {
|
||||
logger.debug("local exporter [{}] - installing new marvel template [{}], replacing [{}]", name(), templateVersion, installedTemplateVersion);
|
||||
putTemplate();
|
||||
// we'll get that template on the next cluster state update
|
||||
return null;
|
||||
} else if (!installedTemplateVersionIsSufficient(installedTemplateVersion)) {
|
||||
logger.error("local exporter [{}] - marvel template version [{}] is below the minimum compatible version [{}]. "
|
||||
+ "please manually update the marvel template to a more recent version"
|
||||
+ "and delete the current active marvel index (don't forget to back up it first if needed)",
|
||||
name(), installedTemplateVersion, MarvelTemplateUtils.MIN_SUPPORTED_TEMPLATE_VERSION);
|
||||
// we're not going to do anything with the template.. it's too old, and the schema might
|
||||
// be too different than what this version of marvel/es can work with. For this reason we're
|
||||
// not going to export any data, to avoid mapping conflicts.
|
||||
// Check that there is nothing that could block metadata updates
|
||||
if (clusterState.blocks().hasGlobalBlock(ClusterBlockLevel.METADATA_WRITE)) {
|
||||
logger.debug("local exporter [{}] - waiting until metadata writes are unblocked", name());
|
||||
return null;
|
||||
}
|
||||
|
||||
// ok.. we have a compatible template... we can start
|
||||
// Install the index template for timestamped indices first, so that other nodes can ship data
|
||||
if (!templateInstalled) {
|
||||
logger.debug("local exporter [{}] - could not find existing marvel template for timestamped indices, installing a new one", name());
|
||||
putTemplate(templateName, MarvelTemplateUtils.loadTimestampedIndexTemplate());
|
||||
// we'll get that template on the next cluster state update
|
||||
return null;
|
||||
}
|
||||
|
||||
// Install the index template for data index
|
||||
templateName = MarvelTemplateUtils.dataTemplateName(templateVersion);
|
||||
if (!hasTemplate(templateName, clusterState)) {
|
||||
logger.debug("local exporter [{}] - could not find existing marvel template for data index, installing a new one", name());
|
||||
putTemplate(templateName, MarvelTemplateUtils.loadDataIndexTemplate());
|
||||
// we'll get that template on the next cluster state update
|
||||
return null;
|
||||
}
|
||||
|
||||
// ok.. we have a compatible templates... we can start
|
||||
return currentBulk != null ? currentBulk : new LocalBulk(name(), logger, client, indexNameResolver, renderers);
|
||||
}
|
||||
|
||||
void putTemplate() {
|
||||
PutIndexTemplateRequest request = new PutIndexTemplateRequest(MarvelTemplateUtils.INDEX_TEMPLATE_NAME).source(MarvelTemplateUtils.loadDefaultTemplate());
|
||||
/**
|
||||
* List templates that exists in cluster state metadata and that match a given template name pattern.
|
||||
*/
|
||||
private ImmutableOpenMap<String, Integer> findTemplates(String templatePattern, ClusterState state) {
|
||||
if (state == null || state.getMetaData() == null || state.getMetaData().getTemplates().isEmpty()) {
|
||||
return ImmutableOpenMap.of();
|
||||
}
|
||||
|
||||
ImmutableOpenMap.Builder<String, Integer> templates = ImmutableOpenMap.builder();
|
||||
for (ObjectCursor<String> template : state.metaData().templates().keys()) {
|
||||
if (Regex.simpleMatch(templatePattern, template.value)) {
|
||||
try {
|
||||
Integer version = Integer.parseInt(template.value.substring(templatePattern.length() - 1));
|
||||
templates.put(template.value, version);
|
||||
logger.debug("found index template [{}] in version [{}]", template, version);
|
||||
} catch (NumberFormatException e) {
|
||||
logger.warn("cannot extract version number for template [{}]", template.value);
|
||||
}
|
||||
}
|
||||
}
|
||||
return templates.build();
|
||||
}
|
||||
|
||||
private boolean hasTemplate(String templateName, ClusterState state) {
|
||||
ImmutableOpenMap<String, Integer> templates = findTemplates(templateName, state);
|
||||
return templates.size() > 0;
|
||||
}
|
||||
|
||||
void putTemplate(String template, byte[] source) {
|
||||
logger.debug("local exporter [{}] - installing template [{}]", name(), template);
|
||||
|
||||
PutIndexTemplateRequest request = new PutIndexTemplateRequest(template).source(source);
|
||||
assert !Thread.currentThread().isInterrupted() : "current thread has been interrupted before putting index template!!!";
|
||||
|
||||
// async call, so we won't block cluster event thread
|
||||
|
@ -188,80 +204,15 @@ public class LocalExporter extends Exporter implements ClusterStateListener {
|
|||
@Override
|
||||
public void onResponse(PutIndexTemplateResponse response) {
|
||||
if (response.isAcknowledged()) {
|
||||
logger.trace("local exporter [{}] - successfully installed marvel template", name());
|
||||
|
||||
if (config.settings().getAsBoolean("update_mappings", true)) {
|
||||
updateMappings(MarvelSettings.MARVEL_DATA_INDEX_NAME);
|
||||
updateMappings(indexNameResolver().resolve(System.currentTimeMillis()));
|
||||
}
|
||||
logger.trace("local exporter [{}] - successfully installed marvel template [{}]", name(), template);
|
||||
} else {
|
||||
logger.error("local exporter [{}] - failed to update marvel index template", name());
|
||||
logger.error("local exporter [{}] - failed to update marvel index template [{}]", name(), template);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onFailure(Throwable throwable) {
|
||||
logger.error("local exporter [{}] - failed to update marvel index template", throwable, name());
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// TODO: Remove this method once marvel indices are versioned (v 2.2.0)
|
||||
void updateMappings(String index) {
|
||||
logger.trace("local exporter [{}] - updating mappings for index [{}]", name(), index);
|
||||
|
||||
// Parse the default template to get its mappings
|
||||
PutIndexTemplateRequest template = new PutIndexTemplateRequest().source(MarvelTemplateUtils.loadDefaultTemplate());
|
||||
if ((template == null) || (template.mappings() == null) || (template.mappings().isEmpty())) {
|
||||
return;
|
||||
}
|
||||
|
||||
// async call, so we won't block cluster event thread
|
||||
client.admin().indices().getMappings(new GetMappingsRequest().indices(index), new ActionListener<GetMappingsResponse>() {
|
||||
@Override
|
||||
public void onResponse(GetMappingsResponse response) {
|
||||
ImmutableOpenMap<String, MappingMetaData> indexMappings = response.getMappings().get(index);
|
||||
if (indexMappings != null) {
|
||||
|
||||
// Iterates over document types defined in the default template
|
||||
for (String type : template.mappings().keySet()) {
|
||||
if (indexMappings.get(type) != null) {
|
||||
logger.trace("local exporter [{}] - type [{} already exists in mapping of index [{}]", name(), type, index);
|
||||
continue;
|
||||
}
|
||||
|
||||
logger.trace("local exporter [{}] - adding type [{}] to index [{}] mappings", name(), type, index);
|
||||
updateMappingForType(index, type, template.mappings().get(type));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onFailure(Throwable e) {
|
||||
if (e instanceof IndexNotFoundException) {
|
||||
logger.trace("local exporter [{}] - index [{}] not found, unable to update mappings", name(), index);
|
||||
} else {
|
||||
logger.error("local exporter [{}] - failed to get mappings for index [{}]", name(), index);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
void updateMappingForType(String index, String type, String mappingSource) {
|
||||
logger.trace("local exporter [{}] - updating index [{}] mappings for type [{}]", name(), index, type);
|
||||
client.admin().indices().putMapping(new PutMappingRequest(index).type(type).source(mappingSource), new ActionListener<PutMappingResponse>() {
|
||||
@Override
|
||||
public void onResponse(PutMappingResponse response) {
|
||||
if (response.isAcknowledged()) {
|
||||
logger.trace("local exporter [{}] - mapping of index [{}] updated for type [{}]", name(), index, type);
|
||||
} else {
|
||||
logger.trace("local exporter [{}] - mapping of index [{}] failed to be updated for type [{}]", name(), index, type);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onFailure(Throwable e) {
|
||||
logger.error("local exporter [{}] - failed to update mapping of index [{}] for type [{}]", name(), index, type);
|
||||
logger.error("local exporter [{}] - failed to update marvel index template [{}]", throwable, name(), template);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
|
|
@ -22,7 +22,7 @@ public class MarvelSettings extends AbstractComponent {
|
|||
private static final String PREFIX = MarvelPlugin.NAME + ".agent.";
|
||||
|
||||
public static final String MARVEL_INDICES_PREFIX = ".marvel-es-";
|
||||
public static final String MARVEL_DATA_INDEX_NAME = MARVEL_INDICES_PREFIX + "data";
|
||||
public static final String MARVEL_DATA_INDEX_PREFIX = MARVEL_INDICES_PREFIX + "data-";
|
||||
public static final TimeValue MAX_LICENSE_GRACE_PERIOD = TimeValue.timeValueHours(7 * 24);
|
||||
|
||||
/** Sampling interval between two collections (default to 10s) */
|
||||
|
|
|
@ -0,0 +1,25 @@
|
|||
{
|
||||
"template": ".marvel-es-data-${marvel.template.version}",
|
||||
"settings": {
|
||||
"marvel.plugin.version": "${marvel.plugin.version}",
|
||||
"marvel.template.version": "${marvel.template.version}",
|
||||
"index.number_of_shards": 1,
|
||||
"index.number_of_replicas": 1,
|
||||
"index.codec": "best_compression",
|
||||
"index.mapper.dynamic": false
|
||||
},
|
||||
"mappings": {
|
||||
"_default_": {
|
||||
"_all": {
|
||||
"enabled": false
|
||||
},
|
||||
"date_detection": false
|
||||
},
|
||||
"cluster_info": {
|
||||
"enabled": false
|
||||
},
|
||||
"node": {
|
||||
"enabled": false
|
||||
}
|
||||
}
|
||||
}
|
|
@ -1,7 +1,8 @@
|
|||
{
|
||||
"template": ".marvel-es-*",
|
||||
"template": ".marvel-es-${marvel.template.version}-*",
|
||||
"settings": {
|
||||
"marvel_version": "${project.version}",
|
||||
"marvel.plugin.version": "${marvel.plugin.version}",
|
||||
"marvel.template.version": "${marvel.template.version}",
|
||||
"index.number_of_shards": 1,
|
||||
"index.number_of_replicas": 1,
|
||||
"index.codec": "best_compression",
|
||||
|
@ -196,12 +197,6 @@
|
|||
}
|
||||
}
|
||||
},
|
||||
"cluster_info": {
|
||||
"enabled": false
|
||||
},
|
||||
"node": {
|
||||
"enabled": false
|
||||
},
|
||||
"nodes": {
|
||||
"properties": {
|
||||
"state_uuid": {
|
|
@ -0,0 +1 @@
|
|||
marvel.template.version=${marvel.template.version}
|
|
@ -11,6 +11,7 @@ import org.elasticsearch.cluster.metadata.IndexMetaData;
|
|||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.marvel.agent.collector.AbstractCollectorTestCase;
|
||||
import org.elasticsearch.marvel.agent.exporter.MarvelDoc;
|
||||
import org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils;
|
||||
import org.elasticsearch.marvel.agent.settings.MarvelSettings;
|
||||
import org.elasticsearch.marvel.license.MarvelLicensee;
|
||||
|
||||
|
@ -182,7 +183,7 @@ public class ClusterStateCollectorTests extends AbstractCollectorTestCase {
|
|||
|
||||
case ClusterStateCollector.NODE_TYPE:
|
||||
DiscoveryNodeMarvelDoc discoveryNodeMarvelDoc = (DiscoveryNodeMarvelDoc) marvelDoc;
|
||||
assertThat(discoveryNodeMarvelDoc.index(), equalTo(MarvelSettings.MARVEL_DATA_INDEX_NAME));
|
||||
assertThat(discoveryNodeMarvelDoc.index(), equalTo(MarvelSettings.MARVEL_DATA_INDEX_PREFIX + MarvelTemplateUtils.TEMPLATE_VERSION));
|
||||
assertThat(discoveryNodeMarvelDoc.id(), not(isEmptyOrNullString()));
|
||||
assertNotNull(discoveryNodeMarvelDoc.getNode());
|
||||
discoveryNodes.add(discoveryNodeMarvelDoc);
|
||||
|
|
|
@ -5,7 +5,6 @@
|
|||
*/
|
||||
package org.elasticsearch.marvel.agent.exporter;
|
||||
|
||||
import org.elasticsearch.Version;
|
||||
import org.elasticsearch.common.bytes.BytesReference;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.marvel.agent.collector.Collector;
|
||||
|
@ -14,9 +13,6 @@ import org.elasticsearch.marvel.agent.collector.node.NodeStatsCollector;
|
|||
import org.elasticsearch.marvel.agent.settings.MarvelSettings;
|
||||
import org.elasticsearch.marvel.test.MarvelIntegTestCase;
|
||||
import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
|
||||
import org.elasticsearch.test.VersionUtils;
|
||||
import org.joda.time.DateTime;
|
||||
import org.joda.time.DateTimeZone;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
|
@ -25,11 +21,15 @@ import java.util.List;
|
|||
import java.util.Map;
|
||||
|
||||
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
|
||||
import static org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils.dataTemplateName;
|
||||
import static org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils.indexTemplateName;
|
||||
import static org.elasticsearch.test.ESIntegTestCase.Scope.TEST;
|
||||
|
||||
@ClusterScope(scope = TEST, numDataNodes = 0, numClientNodes = 0, transportClientRatio = 0.0)
|
||||
public abstract class AbstractExporterTemplateTestCase extends MarvelIntegTestCase {
|
||||
|
||||
private final Integer currentVersion = MarvelTemplateUtils.TEMPLATE_VERSION;
|
||||
|
||||
@Override
|
||||
protected Settings nodeSettings(int nodeOrdinal) {
|
||||
Settings.Builder settings = Settings.builder()
|
||||
|
@ -44,149 +44,116 @@ public abstract class AbstractExporterTemplateTestCase extends MarvelIntegTestCa
|
|||
|
||||
protected abstract Settings exporterSettings();
|
||||
|
||||
protected abstract void deleteTemplate() throws Exception;
|
||||
protected abstract void deleteTemplates() throws Exception;
|
||||
|
||||
protected abstract void putTemplate(String version) throws Exception;
|
||||
protected abstract void putTemplate(String name, int version) throws Exception;
|
||||
|
||||
protected abstract void createMarvelIndex(String index) throws Exception;
|
||||
protected abstract void assertTemplateExist(String name) throws Exception;
|
||||
|
||||
protected abstract void assertTemplateUpdated(Version version) throws Exception;
|
||||
protected abstract void assertTemplateNotUpdated(String name) throws Exception;
|
||||
|
||||
protected abstract void assertTemplateNotUpdated(Version version) throws Exception;
|
||||
|
||||
protected abstract void assertMappingsUpdated(String... indices) throws Exception;
|
||||
|
||||
protected abstract void assertMappingsNotUpdated(String... indices) throws Exception;
|
||||
|
||||
protected abstract void assertIndicesNotCreated() throws Exception;
|
||||
|
||||
public void testCreateWhenNoExistingTemplate() throws Exception {
|
||||
public void testCreateWhenNoExistingTemplates() throws Exception {
|
||||
internalCluster().startNode();
|
||||
|
||||
deleteTemplate();
|
||||
deleteTemplates();
|
||||
doExporting();
|
||||
|
||||
logger.debug("--> template does not exist: it should have been created in the current version");
|
||||
assertTemplateUpdated(currentVersion());
|
||||
logger.debug("--> templates does not exist: it should have been created in the current version");
|
||||
assertTemplateExist(indexTemplateName());
|
||||
assertTemplateExist(dataTemplateName());
|
||||
|
||||
doExporting();
|
||||
|
||||
logger.debug("--> mappings should be up-to-date");
|
||||
assertMappingsUpdated(currentIndices());
|
||||
logger.debug("--> indices should have been created");
|
||||
awaitIndexExists(currentDataIndexName());
|
||||
awaitIndexExists(currentTimestampedIndexName());
|
||||
}
|
||||
|
||||
public void testUpdateWhenExistingTemplateHasNoVersion() throws Exception {
|
||||
public void testCreateWhenExistingTemplatesAreOld() throws Exception {
|
||||
internalCluster().startNode();
|
||||
|
||||
putTemplate("");
|
||||
doExporting();
|
||||
|
||||
logger.debug("--> existing template does not have a version: it should be updated to the current version");
|
||||
assertTemplateUpdated(currentVersion());
|
||||
final Integer version = randomIntBetween(0, currentVersion - 1);
|
||||
putTemplate(indexTemplateName(version), version);
|
||||
putTemplate(dataTemplateName(version), version);
|
||||
|
||||
doExporting();
|
||||
|
||||
logger.debug("--> mappings should be up-to-date");
|
||||
assertMappingsUpdated(currentIndices());
|
||||
logger.debug("--> existing templates are old");
|
||||
assertTemplateExist(dataTemplateName(version));
|
||||
assertTemplateExist(indexTemplateName(version));
|
||||
|
||||
logger.debug("--> existing templates are old: new templates should be created");
|
||||
assertTemplateExist(indexTemplateName());
|
||||
assertTemplateExist(dataTemplateName());
|
||||
|
||||
doExporting();
|
||||
|
||||
logger.debug("--> indices should have been created");
|
||||
awaitIndexExists(currentDataIndexName());
|
||||
awaitIndexExists(currentTimestampedIndexName());
|
||||
}
|
||||
|
||||
public void testUpdateWhenExistingTemplateHasWrongVersion() throws Exception {
|
||||
public void testCreateWhenExistingTemplateAreUpToDate() throws Exception {
|
||||
internalCluster().startNode();
|
||||
|
||||
putTemplate(randomAsciiOfLength(5));
|
||||
doExporting();
|
||||
|
||||
logger.debug("--> existing template has a wrong version: it should be updated to the current version");
|
||||
assertTemplateUpdated(currentVersion());
|
||||
putTemplate(indexTemplateName(currentVersion), currentVersion);
|
||||
putTemplate(dataTemplateName(currentVersion), currentVersion);
|
||||
|
||||
doExporting();
|
||||
|
||||
logger.debug("--> mappings should be up-to-date");
|
||||
assertMappingsUpdated(currentIndices());
|
||||
logger.debug("--> existing templates are up to date");
|
||||
assertTemplateExist(indexTemplateName());
|
||||
assertTemplateExist(dataTemplateName());
|
||||
|
||||
logger.debug("--> existing templates has the same version: they should not be changed");
|
||||
assertTemplateNotUpdated(indexTemplateName());
|
||||
assertTemplateNotUpdated(dataTemplateName());
|
||||
|
||||
doExporting();
|
||||
|
||||
logger.debug("--> indices should have been created");
|
||||
awaitIndexExists(currentDataIndexName());
|
||||
awaitIndexExists(currentTimestampedIndexName());
|
||||
}
|
||||
|
||||
public void testNoUpdateWhenExistingTemplateIsTooOld() throws Exception {
|
||||
public void testRandomTemplates() throws Exception {
|
||||
internalCluster().startNode();
|
||||
|
||||
putTemplate(VersionUtils.getFirstVersion().number());
|
||||
doExporting();
|
||||
|
||||
logger.debug("--> existing template is too old: it should not be updated");
|
||||
assertTemplateNotUpdated(VersionUtils.getFirstVersion());
|
||||
|
||||
doExporting();
|
||||
|
||||
logger.debug("--> existing template is too old: no data is exported");
|
||||
assertIndicesNotCreated();
|
||||
}
|
||||
|
||||
public void testUpdateWhenExistingTemplateIsOld() throws Exception {
|
||||
internalCluster().startNode();
|
||||
|
||||
putTemplate(VersionUtils.getPreviousVersion(currentVersion()).number());
|
||||
doExporting();
|
||||
|
||||
logger.debug("--> existing template is old but supported: it should be updated to the current version");
|
||||
assertTemplateUpdated(currentVersion());
|
||||
|
||||
doExporting();
|
||||
|
||||
logger.debug("--> mappings should be up-to-date");
|
||||
assertMappingsUpdated(currentIndices());
|
||||
}
|
||||
|
||||
public void testUpdateWhenExistingTemplateIsUpToDate() throws Exception {
|
||||
internalCluster().startNode();
|
||||
|
||||
putTemplate(currentVersion().toString());
|
||||
doExporting();
|
||||
|
||||
logger.debug("--> existing template has the same version: it should not be updated");
|
||||
assertTemplateNotUpdated(currentVersion());
|
||||
|
||||
doExporting();
|
||||
|
||||
logger.debug("--> mappings should not have been updated");
|
||||
assertMappingsNotUpdated(currentIndices());
|
||||
}
|
||||
|
||||
public void testMappingsUpdate() throws Exception {
|
||||
boolean updateMappings = randomBoolean();
|
||||
logger.debug("--> update_mappings is {}", updateMappings);
|
||||
internalCluster().startNode(Settings.builder().put("marvel.agent.exporters._exporter.update_mappings", updateMappings));
|
||||
|
||||
logger.debug("--> putting a template with a very old version so that it will not be updated");
|
||||
putTemplate(VersionUtils.getFirstVersion().toString());
|
||||
|
||||
logger.debug("--> creating marvel data index");
|
||||
createMarvelIndex(MarvelSettings.MARVEL_DATA_INDEX_NAME);
|
||||
|
||||
logger.debug("--> creating a cold marvel index");
|
||||
createMarvelIndex(coldIndex());
|
||||
|
||||
logger.debug("--> creating an active marvel index");
|
||||
createMarvelIndex(hotIndex());
|
||||
|
||||
logger.debug("--> all indices have a old mapping now");
|
||||
assertMappingsNotUpdated(coldIndex(), hotIndex(), MarvelSettings.MARVEL_DATA_INDEX_NAME);
|
||||
|
||||
logger.debug("--> updating the template with a previous version, so that it will be updated when exporting documents");
|
||||
putTemplate(VersionUtils.getPreviousVersion(currentVersion()).number());
|
||||
doExporting();
|
||||
|
||||
logger.debug("--> existing template is old: it should be updated to the current version");
|
||||
assertTemplateUpdated(currentVersion());
|
||||
|
||||
logger.debug("--> cold marvel index: mappings should not have been updated");
|
||||
assertMappingsNotUpdated(coldIndex());
|
||||
|
||||
if (updateMappings) {
|
||||
logger.debug("--> marvel indices: mappings should be up-to-date");
|
||||
assertMappingsUpdated(MarvelSettings.MARVEL_DATA_INDEX_NAME, hotIndex());
|
||||
} else {
|
||||
logger.debug("--> marvel indices: mappings should bnot have been updated");
|
||||
assertMappingsNotUpdated(MarvelSettings.MARVEL_DATA_INDEX_NAME, hotIndex());
|
||||
int previousIndexTemplateVersion = rarely() ? currentVersion : randomIntBetween(0, currentVersion - 1);
|
||||
boolean previousIndexTemplateExist = randomBoolean();
|
||||
if (previousIndexTemplateExist) {
|
||||
logger.debug("--> creating index template in version [{}]", previousIndexTemplateVersion);
|
||||
putTemplate(indexTemplateName(previousIndexTemplateVersion), previousIndexTemplateVersion);
|
||||
}
|
||||
|
||||
int previousDataTemplateVersion = rarely() ? currentVersion : randomIntBetween(0, currentVersion - 1);
|
||||
boolean previousDataTemplateExist = randomBoolean();
|
||||
if (previousDataTemplateExist) {
|
||||
logger.debug("--> creating data template in version [{}]", previousDataTemplateVersion);
|
||||
putTemplate(dataTemplateName(previousDataTemplateVersion), previousDataTemplateVersion);
|
||||
}
|
||||
|
||||
doExporting();
|
||||
|
||||
logger.debug("--> templates should exist in current version");
|
||||
assertTemplateExist(indexTemplateName());
|
||||
assertTemplateExist(dataTemplateName());
|
||||
|
||||
if (previousIndexTemplateExist) {
|
||||
logger.debug("--> index template should exist in version [{}]", previousIndexTemplateVersion);
|
||||
assertTemplateExist(indexTemplateName(previousIndexTemplateVersion));
|
||||
}
|
||||
|
||||
if (previousDataTemplateExist) {
|
||||
logger.debug("--> data template should exist in version [{}]", previousDataTemplateVersion);
|
||||
assertTemplateExist(dataTemplateName(previousDataTemplateVersion));
|
||||
}
|
||||
|
||||
doExporting();
|
||||
|
||||
logger.debug("--> indices should exist in current version");
|
||||
awaitIndexExists(currentDataIndexName());
|
||||
awaitIndexExists(currentTimestampedIndexName());
|
||||
}
|
||||
|
||||
protected void doExporting() throws Exception {
|
||||
|
@ -203,59 +170,23 @@ public abstract class AbstractExporterTemplateTestCase extends MarvelIntegTestCa
|
|||
return exporters.iterator().next();
|
||||
}
|
||||
|
||||
private Version currentVersion() {
|
||||
return MarvelTemplateUtils.loadDefaultTemplateVersion();
|
||||
private String currentDataIndexName() {
|
||||
return ".marvel-es-data-" + String.valueOf(currentVersion);
|
||||
}
|
||||
|
||||
private String[] currentIndices() {
|
||||
return new String[]{hotIndex(), MarvelSettings.MARVEL_DATA_INDEX_NAME};
|
||||
}
|
||||
|
||||
private String coldIndex() {
|
||||
return exporter().indexNameResolver().resolve(new DateTime(2012, 3, 10, 0, 0, DateTimeZone.UTC).getMillis());
|
||||
}
|
||||
|
||||
private String hotIndex() {
|
||||
private String currentTimestampedIndexName() {
|
||||
return exporter().indexNameResolver().resolve(System.currentTimeMillis());
|
||||
}
|
||||
|
||||
/** Generates a template that looks like an old one **/
|
||||
protected static BytesReference generateTemplateSource(String version) throws IOException {
|
||||
/** Generates a basic template **/
|
||||
protected static BytesReference generateTemplateSource(String name, Integer version) throws IOException {
|
||||
return jsonBuilder().startObject()
|
||||
.field("template", ".marvel-es-*")
|
||||
.field("template", name)
|
||||
.startObject("settings")
|
||||
.field("index.number_of_shards", 1)
|
||||
.field("index.number_of_replicas", 1)
|
||||
.field("index.mapper.dynamic", false)
|
||||
.field(MarvelTemplateUtils.MARVEL_VERSION_FIELD, version)
|
||||
.field(MarvelTemplateUtils.VERSION_FIELD, String.valueOf(version))
|
||||
.endObject()
|
||||
.startObject("mappings")
|
||||
.startObject("_default_")
|
||||
.startObject("_all")
|
||||
.field("enabled", false)
|
||||
.endObject()
|
||||
.field("date_detection", false)
|
||||
.startObject("properties")
|
||||
.startObject("cluster_uuid")
|
||||
.field("type", "string")
|
||||
.field("index", "not_analyzed")
|
||||
.endObject()
|
||||
.startObject("timestamp")
|
||||
.field("type", "date")
|
||||
.field("format", "date_time")
|
||||
.endObject()
|
||||
.endObject()
|
||||
.endObject()
|
||||
.startObject("cluster_info")
|
||||
.field("enabled", false)
|
||||
.endObject()
|
||||
.startObject("node_stats")
|
||||
.startObject("properties")
|
||||
.startObject("node_stats")
|
||||
.field("type", "object")
|
||||
.endObject()
|
||||
.endObject()
|
||||
.endObject()
|
||||
.endObject().bytes();
|
||||
.endObject().bytes();
|
||||
}
|
||||
}
|
||||
|
|
|
@ -5,89 +5,54 @@
|
|||
*/
|
||||
package org.elasticsearch.marvel.agent.exporter;
|
||||
|
||||
import org.elasticsearch.Version;
|
||||
import org.elasticsearch.common.Strings;
|
||||
import org.elasticsearch.marvel.support.VersionUtils;
|
||||
import org.elasticsearch.test.ESTestCase;
|
||||
import org.hamcrest.Matchers;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import static org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils.MARVEL_VERSION_FIELD;
|
||||
import static org.hamcrest.Matchers.equalTo;
|
||||
import static org.hamcrest.Matchers.greaterThan;
|
||||
|
||||
//import static org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils.MARVEL_VERSION_FIELD;
|
||||
|
||||
public class MarvelTemplateUtilsTests extends ESTestCase {
|
||||
|
||||
public void testLoadTemplate() {
|
||||
byte[] template = MarvelTemplateUtils.loadDefaultTemplate();
|
||||
public void testLoadTimestampedIndexTemplate() {
|
||||
byte[] template = MarvelTemplateUtils.loadTimestampedIndexTemplate();
|
||||
assertNotNull(template);
|
||||
assertThat(template.length, Matchers.greaterThan(0));
|
||||
assertThat(template.length, greaterThan(0));
|
||||
}
|
||||
|
||||
public void testParseTemplateVersionFromByteArrayTemplate() throws IOException {
|
||||
byte[] template = MarvelTemplateUtils.loadDefaultTemplate();
|
||||
public void testLoadDataIndexTemplate() {
|
||||
byte[] template = MarvelTemplateUtils.loadDataIndexTemplate();
|
||||
assertNotNull(template);
|
||||
assertThat(template.length, greaterThan(0));
|
||||
}
|
||||
|
||||
Version version = MarvelTemplateUtils.parseTemplateVersion(template);
|
||||
public void testLoad() throws IOException {
|
||||
String resource = randomFrom(MarvelTemplateUtils.INDEX_TEMPLATE_FILE, MarvelTemplateUtils.DATA_TEMPLATE_FILE);
|
||||
byte[] template = MarvelTemplateUtils.load(resource);
|
||||
assertNotNull(template);
|
||||
assertThat(template.length, greaterThan(0));
|
||||
}
|
||||
|
||||
public void testLoadTemplateVersion() {
|
||||
Integer version = MarvelTemplateUtils.loadTemplateVersion();
|
||||
assertNotNull(version);
|
||||
assertThat(version, greaterThan(0));
|
||||
assertThat(version, equalTo(MarvelTemplateUtils.TEMPLATE_VERSION));
|
||||
}
|
||||
|
||||
public void testParseTemplateVersionFromStringTemplate() throws IOException {
|
||||
List<String> templates = new ArrayList<>();
|
||||
templates.add("{\"marvel_version\": \"1.4.0.Beta1\"}");
|
||||
templates.add("{\"marvel_version\": \"1.6.2-SNAPSHOT\"}");
|
||||
templates.add("{\"marvel_version\": \"1.7.1\"}");
|
||||
templates.add("{\"marvel_version\": \"2.0.0-beta1\"}");
|
||||
templates.add("{\"marvel_version\": \"2.0.0\"}");
|
||||
templates.add("{ \"template\": \".marvel*\", \"settings\": { \"marvel_version\": \"2.0.0-beta1-SNAPSHOT\", \"index.number_of_shards\": 1 } }");
|
||||
|
||||
for (String template : templates) {
|
||||
Version version = MarvelTemplateUtils.parseTemplateVersion(Strings.toUTF8Bytes(template));
|
||||
assertNotNull(version);
|
||||
}
|
||||
|
||||
Version version = MarvelTemplateUtils.parseTemplateVersion(Strings.toUTF8Bytes("{\"marvel.index_format\": \"7\"}"));
|
||||
assertNull(version);
|
||||
public void testIndexTemplateName() {
|
||||
assertThat(MarvelTemplateUtils.indexTemplateName(),
|
||||
equalTo(MarvelTemplateUtils.INDEX_TEMPLATE_NAME_PREFIX + MarvelTemplateUtils.TEMPLATE_VERSION));
|
||||
int version = randomIntBetween(1, 100);
|
||||
assertThat(MarvelTemplateUtils.indexTemplateName(version), equalTo(".marvel-es-" + version));
|
||||
}
|
||||
|
||||
public void testParseVersion() throws IOException {
|
||||
assertNotNull(VersionUtils.parseVersion(MARVEL_VERSION_FIELD, "{\"marvel_version\": \"2.0.0-beta1\"}"));
|
||||
assertNotNull(VersionUtils.parseVersion(MARVEL_VERSION_FIELD, "{\"marvel_version\": \"2.0.0\"}"));
|
||||
assertNotNull(VersionUtils.parseVersion(MARVEL_VERSION_FIELD, "{\"marvel_version\": \"1.5.2\"}"));
|
||||
assertNotNull(VersionUtils.parseVersion(MARVEL_VERSION_FIELD, "{ \"template\": \".marvel*\", \"settings\": { \"marvel_version\": \"2.0.0-beta1-SNAPSHOT\", \"index.number_of_shards\": 1 } }"));
|
||||
assertNull(VersionUtils.parseVersion(MARVEL_VERSION_FIELD, "{\"marvel.index_format\": \"7\"}"));
|
||||
assertNull(VersionUtils.parseVersion(MARVEL_VERSION_FIELD + "unkown", "{\"marvel_version\": \"1.5.2\"}"));
|
||||
}
|
||||
|
||||
public void testTemplateVersionMandatesAnUpdate() {
|
||||
// Version is unknown
|
||||
assertTrue(MarvelTemplateUtils.installedTemplateVersionMandatesAnUpdate(Version.CURRENT, null, logger, "unit-test"));
|
||||
|
||||
// Version is too old
|
||||
Version unsupported = org.elasticsearch.test.VersionUtils.getPreviousVersion(MarvelTemplateUtils.MIN_SUPPORTED_TEMPLATE_VERSION);
|
||||
assertFalse(MarvelTemplateUtils.installedTemplateVersionMandatesAnUpdate(Version.CURRENT, unsupported, logger, "unit-test"));
|
||||
|
||||
// Version is old but supported
|
||||
assertTrue(MarvelTemplateUtils.installedTemplateVersionMandatesAnUpdate(Version.CURRENT, MarvelTemplateUtils.MIN_SUPPORTED_TEMPLATE_VERSION, logger, "unit-test"));
|
||||
|
||||
// Version is up to date
|
||||
assertFalse(MarvelTemplateUtils.installedTemplateVersionMandatesAnUpdate(Version.CURRENT, Version.CURRENT, logger, "unit-test"));
|
||||
|
||||
// Version is up to date
|
||||
Version previous = org.elasticsearch.test.VersionUtils.getPreviousVersion(Version.CURRENT);
|
||||
assertFalse(MarvelTemplateUtils.installedTemplateVersionMandatesAnUpdate(previous, Version.CURRENT, logger, "unit-test"));
|
||||
}
|
||||
|
||||
public void testTemplateVersionIsSufficient() {
|
||||
// Version is unknown
|
||||
assertFalse(MarvelTemplateUtils.installedTemplateVersionIsSufficient(null));
|
||||
|
||||
// Version is too old
|
||||
Version unsupported = org.elasticsearch.test.VersionUtils.getPreviousVersion(MarvelTemplateUtils.MIN_SUPPORTED_TEMPLATE_VERSION);
|
||||
assertFalse(MarvelTemplateUtils.installedTemplateVersionIsSufficient(unsupported));
|
||||
|
||||
// Version is OK
|
||||
assertTrue(MarvelTemplateUtils.installedTemplateVersionIsSufficient(Version.CURRENT));
|
||||
public void testDataTemplateName() {
|
||||
assertThat(MarvelTemplateUtils.dataTemplateName(),
|
||||
equalTo(MarvelTemplateUtils.DATA_TEMPLATE_NAME_PREFIX + MarvelTemplateUtils.TEMPLATE_VERSION));
|
||||
int version = randomIntBetween(1, 100);
|
||||
assertThat(MarvelTemplateUtils.dataTemplateName(version), equalTo(".marvel-es-data-" + version));
|
||||
}
|
||||
}
|
||||
|
|
|
@ -11,26 +11,25 @@ import com.squareup.okhttp.mockwebserver.MockWebServer;
|
|||
import com.squareup.okhttp.mockwebserver.RecordedRequest;
|
||||
import org.elasticsearch.ElasticsearchException;
|
||||
import org.elasticsearch.Version;
|
||||
import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest;
|
||||
import org.elasticsearch.action.ActionRequest;
|
||||
import org.elasticsearch.action.bulk.BulkRequest;
|
||||
import org.elasticsearch.action.index.IndexRequest;
|
||||
import org.elasticsearch.common.Strings;
|
||||
import org.elasticsearch.common.bytes.BytesArray;
|
||||
import org.elasticsearch.common.bytes.BytesReference;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.common.xcontent.XContentBuilder;
|
||||
import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
|
||||
import org.elasticsearch.marvel.agent.exporter.AbstractExporterTemplateTestCase;
|
||||
import org.elasticsearch.marvel.agent.exporter.Exporter;
|
||||
import org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils;
|
||||
import org.junit.After;
|
||||
import org.junit.Before;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.BindException;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
|
||||
import static org.hamcrest.Matchers.equalTo;
|
||||
import static org.hamcrest.core.Is.is;
|
||||
|
||||
public class HttpExporterTemplateTests extends AbstractExporterTemplateTestCase {
|
||||
|
@ -70,68 +69,39 @@ public class HttpExporterTemplateTests extends AbstractExporterTemplateTestCase
|
|||
}
|
||||
|
||||
@Override
|
||||
protected void deleteTemplate() {
|
||||
dispatcher.setTemplate(null);
|
||||
protected void deleteTemplates() throws Exception {
|
||||
dispatcher.templates.clear();
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void putTemplate(String version) throws Exception {
|
||||
dispatcher.setTemplate(generateTemplateSource(version).toBytes());
|
||||
protected void putTemplate(String name, int version) throws Exception {
|
||||
dispatcher.templates.put(name, generateTemplateSource(name, version));
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void createMarvelIndex(String index) throws Exception {
|
||||
dispatcher.addIndex(index);
|
||||
protected void assertTemplateExist(String name) throws Exception {
|
||||
assertThat("failed to find a template matching [" + name + "]", dispatcher.templates.containsKey(name), is(true));
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void assertTemplateUpdated(Version version) {
|
||||
// Checks that a PUT Template request has been made
|
||||
assertThat(dispatcher.hasRequest("PUT", "/_template/" + MarvelTemplateUtils.INDEX_TEMPLATE_NAME), is(true));
|
||||
|
||||
// Checks that the current template has the expected version
|
||||
assertThat(MarvelTemplateUtils.parseTemplateVersion(dispatcher.getTemplate()), equalTo(version));
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void assertTemplateNotUpdated(Version version) throws Exception {
|
||||
protected void assertTemplateNotUpdated(String name) throws Exception {
|
||||
// Checks that no PUT Template request has been made
|
||||
assertThat(dispatcher.hasRequest("PUT", "/_template/" + MarvelTemplateUtils.INDEX_TEMPLATE_NAME), is(false));
|
||||
assertThat(dispatcher.hasRequest("PUT", "/_template/" + name), is(false));
|
||||
|
||||
// Checks that the current template has the expected version
|
||||
assertThat(MarvelTemplateUtils.parseTemplateVersion(dispatcher.getTemplate()), equalTo(version));
|
||||
// Checks that the current template exists
|
||||
assertThat(dispatcher.templates.containsKey(name), is(true));
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void assertIndicesNotCreated() throws Exception {
|
||||
// Checks that no Bulk request has been made
|
||||
assertThat(dispatcher.hasRequest("POST", "/_bulk"), is(false));
|
||||
assertThat(dispatcher.mappings.size(), equalTo(0));
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void assertMappingsUpdated(String... indices) throws Exception {
|
||||
// Load the mappings of the old template
|
||||
Set<String> oldMappings = new PutIndexTemplateRequest().source(generateTemplateSource(null)).mappings().keySet();
|
||||
|
||||
// Load the mappings of the latest template
|
||||
Set<String> newMappings = new PutIndexTemplateRequest().source(generateTemplateSource(null)).mappings().keySet();
|
||||
newMappings.removeAll(oldMappings);
|
||||
|
||||
for (String index : indices) {
|
||||
for (String mapping : newMappings) {
|
||||
// Checks that a PUT Mapping request has been made for every type that was not in the old template
|
||||
assertThat(dispatcher.hasRequest("PUT", "/" + index + "/_mapping/" + mapping), equalTo(true));
|
||||
protected void awaitIndexExists(String... indices) throws Exception {
|
||||
assertBusy(new Runnable() {
|
||||
@Override
|
||||
public void run() {
|
||||
for (String index : indices) {
|
||||
assertThat("could not find index " + index, dispatcher.hasIndex(index), is(true));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void assertMappingsNotUpdated(String... indices) throws Exception {
|
||||
for (String index : indices) {
|
||||
// Checks that no PUT Template request has been made
|
||||
assertThat(dispatcher.hasRequest("PUT", "/" + index + "/_mapping/"), is(false));
|
||||
}
|
||||
}, 10, TimeUnit.SECONDS);
|
||||
}
|
||||
|
||||
class MockServerDispatcher extends Dispatcher {
|
||||
|
@ -140,100 +110,59 @@ public class HttpExporterTemplateTests extends AbstractExporterTemplateTestCase
|
|||
private final MockResponse NOT_FOUND = newResponse(404, "");
|
||||
|
||||
private final Set<String> requests = new HashSet<>();
|
||||
|
||||
private final Map<String, Set<String>> mappings = new HashMap<>();
|
||||
private byte[] template;
|
||||
private final Map<String, BytesReference> templates = ConcurrentCollections.newConcurrentMap();
|
||||
private final Set<String> indices = ConcurrentCollections.newConcurrentSet();
|
||||
|
||||
@Override
|
||||
public MockResponse dispatch(RecordedRequest request) throws InterruptedException {
|
||||
synchronized (this) {
|
||||
final String requestLine = request.getRequestLine();
|
||||
requests.add(requestLine);
|
||||
final String requestLine = request.getRequestLine();
|
||||
requests.add(requestLine);
|
||||
|
||||
switch (requestLine) {
|
||||
// Cluster version
|
||||
case "GET / HTTP/1.1":
|
||||
return newResponse(200, "{\"version\": {\"number\": \"" + Version.CURRENT.number() + "\"}}");
|
||||
|
||||
// Template
|
||||
case "GET /_template/.marvel-es HTTP/1.1":
|
||||
return (template == null) ? NOT_FOUND : newResponse(200, new BytesArray(template).toUtf8());
|
||||
|
||||
case "PUT /_template/.marvel-es HTTP/1.1":
|
||||
this.template = request.getBody().readByteArray();
|
||||
return OK;
|
||||
|
||||
// Bulk
|
||||
case "POST /_bulk HTTP/1.1":
|
||||
|
||||
return OK;
|
||||
default:
|
||||
String[] paths = Strings.splitStringToArray(request.getPath(), '/');
|
||||
|
||||
// Index Mappings
|
||||
if ((paths != null) && (paths.length > 0) && ("_mapping".equals(paths[1]))) {
|
||||
|
||||
if (!mappings.containsKey(paths[0])) {
|
||||
// Index does not exist
|
||||
return NOT_FOUND;
|
||||
}
|
||||
|
||||
// Get index mappings
|
||||
if ("GET".equals(request.getMethod())) {
|
||||
try {
|
||||
// Builds a fake mapping response
|
||||
XContentBuilder builder = jsonBuilder().startObject().startObject(paths[0]).startObject("mappings");
|
||||
for (String type : mappings.get(paths[0])) {
|
||||
builder.startObject(type).endObject();
|
||||
}
|
||||
builder.endObject().endObject().endObject();
|
||||
return newResponse(200, builder.bytes().toUtf8());
|
||||
} catch (IOException e) {
|
||||
return newResponse(500, e.getMessage());
|
||||
}
|
||||
|
||||
// Put index mapping
|
||||
} else if ("PUT".equals(request.getMethod()) && paths.length > 2) {
|
||||
Set<String> types = mappings.get(paths[0]);
|
||||
if (types == null) {
|
||||
types = new HashSet<>();
|
||||
}
|
||||
types.add(paths[2]);
|
||||
return OK;
|
||||
switch (requestLine) {
|
||||
// Cluster version
|
||||
case "GET / HTTP/1.1":
|
||||
return newResponse(200, "{\"version\": {\"number\": \"" + Version.CURRENT.number() + "\"}}");
|
||||
// Bulk
|
||||
case "POST /_bulk HTTP/1.1":
|
||||
// Parse the bulk request and extract all index names
|
||||
try {
|
||||
BulkRequest bulk = new BulkRequest();
|
||||
byte[] source = request.getBody().readByteArray();
|
||||
bulk.add(source, 0, source.length);
|
||||
for (ActionRequest docRequest : bulk.requests()) {
|
||||
if (docRequest instanceof IndexRequest) {
|
||||
indices.add(((IndexRequest) docRequest).index());
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
} catch (Exception e) {
|
||||
return newResponse(500, e.getMessage());
|
||||
}
|
||||
return OK;
|
||||
default:
|
||||
String[] paths = Strings.splitStringToArray(request.getPath(), '/');
|
||||
|
||||
return newResponse(500, "MockServerDispatcher does not support: " + request.getRequestLine());
|
||||
// Templates
|
||||
if ((paths != null) && (paths.length > 0) && ("_template".equals(paths[0]))) {
|
||||
String templateName = paths[1];
|
||||
boolean templateExist = templates.containsKey(templateName);
|
||||
|
||||
if ("GET".equals(request.getMethod())) {
|
||||
return templateExist ? newResponse(200, templates.get(templateName).toUtf8()) : NOT_FOUND;
|
||||
}
|
||||
if ("PUT".equals(request.getMethod())) {
|
||||
templates.put(templateName, new BytesArray(request.getBody().readByteArray()));
|
||||
return templateExist ? newResponse(200, "updated") : newResponse(201, "created");
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
return newResponse(500, "MockServerDispatcher does not support: " + request.getRequestLine());
|
||||
}
|
||||
|
||||
MockResponse newResponse(int code, String body) {
|
||||
return new MockResponse().setResponseCode(code).setBody(body);
|
||||
}
|
||||
|
||||
void setTemplate(byte[] template) {
|
||||
synchronized (this) {
|
||||
this.template = template;
|
||||
}
|
||||
}
|
||||
|
||||
byte[] getTemplate() {
|
||||
return template;
|
||||
}
|
||||
|
||||
void addIndex(String index) {
|
||||
synchronized (this) {
|
||||
if (template != null) {
|
||||
// Simulate the use of the index template when creating an index
|
||||
mappings.put(index, new HashSet<>(new PutIndexTemplateRequest().source(template).mappings().keySet()));
|
||||
} else {
|
||||
mappings.put(index, null);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
int countRequests(String method, String path) {
|
||||
int count = 0;
|
||||
for (String request : requests) {
|
||||
|
@ -247,5 +176,9 @@ public class HttpExporterTemplateTests extends AbstractExporterTemplateTestCase
|
|||
boolean hasRequest(String method, String path) {
|
||||
return countRequests(method, path) > 0;
|
||||
}
|
||||
|
||||
boolean hasIndex(String index) {
|
||||
return indices.contains(index);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -46,6 +46,8 @@ import java.util.List;
|
|||
import java.util.Map;
|
||||
|
||||
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
|
||||
import static org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils.dataTemplateName;
|
||||
import static org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils.indexTemplateName;
|
||||
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
|
||||
import static org.hamcrest.Matchers.equalTo;
|
||||
import static org.hamcrest.Matchers.instanceOf;
|
||||
|
@ -53,9 +55,13 @@ import static org.hamcrest.Matchers.is;
|
|||
|
||||
@ESIntegTestCase.ClusterScope(scope = Scope.TEST, numDataNodes = 0, numClientNodes = 0, transportClientRatio = 0.0)
|
||||
public class HttpExporterTests extends MarvelIntegTestCase {
|
||||
|
||||
private int webPort;
|
||||
private MockWebServer webServer;
|
||||
|
||||
private static final byte[] TIMESTAMPED_TEMPLATE = MarvelTemplateUtils.loadTimestampedIndexTemplate();
|
||||
private static final byte[] DATA_TEMPLATE = MarvelTemplateUtils.loadDataIndexTemplate();
|
||||
|
||||
@Before
|
||||
public void startWebservice() throws Exception {
|
||||
for (webPort = 9250; webPort < 9300; webPort++) {
|
||||
|
@ -81,8 +87,10 @@ public class HttpExporterTests extends MarvelIntegTestCase {
|
|||
|
||||
public void testExport() throws Exception {
|
||||
enqueueGetClusterVersionResponse(Version.CURRENT);
|
||||
enqueueResponse(404, "marvel template does not exist");
|
||||
enqueueResponse(201, "marvel template created");
|
||||
enqueueResponse(404, "template for timestamped indices does not exist");
|
||||
enqueueResponse(201, "template for timestamped indices created");
|
||||
enqueueResponse(404, "template for data index does not exist");
|
||||
enqueueResponse(201, "template for data index created");
|
||||
enqueueResponse(200, "successful bulk request ");
|
||||
|
||||
Settings.Builder builder = Settings.builder()
|
||||
|
@ -98,7 +106,7 @@ public class HttpExporterTests extends MarvelIntegTestCase {
|
|||
final int nbDocs = randomIntBetween(1, 25);
|
||||
exporter.export(newRandomMarvelDocs(nbDocs));
|
||||
|
||||
assertThat(webServer.getRequestCount(), equalTo(4));
|
||||
assertThat(webServer.getRequestCount(), equalTo(6));
|
||||
|
||||
RecordedRequest recordedRequest = webServer.takeRequest();
|
||||
assertThat(recordedRequest.getMethod(), equalTo("GET"));
|
||||
|
@ -106,12 +114,21 @@ public class HttpExporterTests extends MarvelIntegTestCase {
|
|||
|
||||
recordedRequest = webServer.takeRequest();
|
||||
assertThat(recordedRequest.getMethod(), equalTo("GET"));
|
||||
assertThat(recordedRequest.getPath(), equalTo("/_template/.marvel-es"));
|
||||
assertThat(recordedRequest.getPath(), equalTo("/_template/" + indexTemplateName()));
|
||||
|
||||
recordedRequest = webServer.takeRequest();
|
||||
assertThat(recordedRequest.getMethod(), equalTo("PUT"));
|
||||
assertThat(recordedRequest.getPath(), equalTo("/_template/.marvel-es"));
|
||||
assertThat(recordedRequest.getBody().readByteArray(), equalTo(MarvelTemplateUtils.loadDefaultTemplate()));
|
||||
assertThat(recordedRequest.getPath(), equalTo("/_template/" + indexTemplateName()));
|
||||
assertThat(recordedRequest.getBody().readByteArray(), equalTo(TIMESTAMPED_TEMPLATE));
|
||||
|
||||
recordedRequest = webServer.takeRequest();
|
||||
assertThat(recordedRequest.getMethod(), equalTo("GET"));
|
||||
assertThat(recordedRequest.getPath(), equalTo("/_template/" + dataTemplateName()));
|
||||
|
||||
recordedRequest = webServer.takeRequest();
|
||||
assertThat(recordedRequest.getMethod(), equalTo("PUT"));
|
||||
assertThat(recordedRequest.getPath(), equalTo("/_template/" + dataTemplateName()));
|
||||
assertThat(recordedRequest.getBody().readByteArray(), equalTo(DATA_TEMPLATE));
|
||||
|
||||
recordedRequest = webServer.takeRequest();
|
||||
assertThat(recordedRequest.getMethod(), equalTo("POST"));
|
||||
|
@ -156,8 +173,10 @@ public class HttpExporterTests extends MarvelIntegTestCase {
|
|||
logger.info("--> starting node");
|
||||
|
||||
enqueueGetClusterVersionResponse(Version.CURRENT);
|
||||
enqueueResponse(404, "marvel template does not exist");
|
||||
enqueueResponse(201, "marvel template created");
|
||||
enqueueResponse(404, "template for timestamped indices does not exist");
|
||||
enqueueResponse(201, "template for timestamped indices created");
|
||||
enqueueResponse(404, "template for data index does not exist");
|
||||
enqueueResponse(201, "template for data index created");
|
||||
enqueueResponse(200, "successful bulk request ");
|
||||
|
||||
String agentNode = internalCluster().startNode(builder);
|
||||
|
@ -168,7 +187,7 @@ public class HttpExporterTests extends MarvelIntegTestCase {
|
|||
exporter.export(Collections.singletonList(newRandomMarvelDoc()));
|
||||
|
||||
assertThat(exporter.supportedClusterVersion, is(true));
|
||||
assertThat(webServer.getRequestCount(), equalTo(4));
|
||||
assertThat(webServer.getRequestCount(), equalTo(6));
|
||||
|
||||
RecordedRequest recordedRequest = webServer.takeRequest();
|
||||
assertThat(recordedRequest.getMethod(), equalTo("GET"));
|
||||
|
@ -176,12 +195,21 @@ public class HttpExporterTests extends MarvelIntegTestCase {
|
|||
|
||||
recordedRequest = webServer.takeRequest();
|
||||
assertThat(recordedRequest.getMethod(), equalTo("GET"));
|
||||
assertThat(recordedRequest.getPath(), equalTo("/_template/.marvel-es"));
|
||||
assertThat(recordedRequest.getPath(), equalTo("/_template/" + indexTemplateName()));
|
||||
|
||||
recordedRequest = webServer.takeRequest();
|
||||
assertThat(recordedRequest.getMethod(), equalTo("PUT"));
|
||||
assertThat(recordedRequest.getPath(), equalTo("/_template/.marvel-es"));
|
||||
assertThat(recordedRequest.getBody().readByteArray(), equalTo(MarvelTemplateUtils.loadDefaultTemplate()));
|
||||
assertThat(recordedRequest.getPath(), equalTo("/_template/" + indexTemplateName()));
|
||||
assertThat(recordedRequest.getBody().readByteArray(), equalTo(TIMESTAMPED_TEMPLATE));
|
||||
|
||||
recordedRequest = webServer.takeRequest();
|
||||
assertThat(recordedRequest.getMethod(), equalTo("GET"));
|
||||
assertThat(recordedRequest.getPath(), equalTo("/_template/" + dataTemplateName()));
|
||||
|
||||
recordedRequest = webServer.takeRequest();
|
||||
assertThat(recordedRequest.getMethod(), equalTo("PUT"));
|
||||
assertThat(recordedRequest.getPath(), equalTo("/_template/" + dataTemplateName()));
|
||||
assertThat(recordedRequest.getBody().readByteArray(), equalTo(DATA_TEMPLATE));
|
||||
|
||||
recordedRequest = webServer.takeRequest();
|
||||
assertThat(recordedRequest.getMethod(), equalTo("POST"));
|
||||
|
@ -214,14 +242,15 @@ public class HttpExporterTests extends MarvelIntegTestCase {
|
|||
exporter = getExporter(agentNode);
|
||||
|
||||
enqueueGetClusterVersionResponse(secondWebServer, Version.CURRENT);
|
||||
enqueueResponse(secondWebServer, 404, "marvel template does not exist");
|
||||
enqueueResponse(secondWebServer, 201, "marvel template created");
|
||||
enqueueResponse(secondWebServer, 404, "template for timestamped indices does not exist");
|
||||
enqueueResponse(secondWebServer, 201, "template for timestamped indices created");
|
||||
enqueueResponse(secondWebServer, 200, "template for data index exist");
|
||||
enqueueResponse(secondWebServer, 200, "successful bulk request ");
|
||||
|
||||
logger.info("--> exporting a second event");
|
||||
exporter.export(Collections.singletonList(newRandomMarvelDoc()));
|
||||
|
||||
assertThat(secondWebServer.getRequestCount(), equalTo(4));
|
||||
assertThat(secondWebServer.getRequestCount(), equalTo(5));
|
||||
|
||||
recordedRequest = secondWebServer.takeRequest();
|
||||
assertThat(recordedRequest.getMethod(), equalTo("GET"));
|
||||
|
@ -229,12 +258,16 @@ public class HttpExporterTests extends MarvelIntegTestCase {
|
|||
|
||||
recordedRequest = secondWebServer.takeRequest();
|
||||
assertThat(recordedRequest.getMethod(), equalTo("GET"));
|
||||
assertThat(recordedRequest.getPath(), equalTo("/_template/.marvel-es"));
|
||||
assertThat(recordedRequest.getPath(), equalTo("/_template/" + indexTemplateName()));
|
||||
|
||||
recordedRequest = secondWebServer.takeRequest();
|
||||
assertThat(recordedRequest.getMethod(), equalTo("PUT"));
|
||||
assertThat(recordedRequest.getPath(), equalTo("/_template/.marvel-es"));
|
||||
assertThat(recordedRequest.getBody().readByteArray(), equalTo(MarvelTemplateUtils.loadDefaultTemplate()));
|
||||
assertThat(recordedRequest.getPath(), equalTo("/_template/" + indexTemplateName()));
|
||||
assertThat(recordedRequest.getBody().readByteArray(), equalTo(TIMESTAMPED_TEMPLATE));
|
||||
|
||||
recordedRequest = secondWebServer.takeRequest();
|
||||
assertThat(recordedRequest.getMethod(), equalTo("GET"));
|
||||
assertThat(recordedRequest.getPath(), equalTo("/_template/" + dataTemplateName()));;
|
||||
|
||||
recordedRequest = secondWebServer.takeRequest();
|
||||
assertThat(recordedRequest.getMethod(), equalTo("POST"));
|
||||
|
@ -287,8 +320,10 @@ public class HttpExporterTests extends MarvelIntegTestCase {
|
|||
logger.info("--> exporting a first event");
|
||||
|
||||
enqueueGetClusterVersionResponse(Version.CURRENT);
|
||||
enqueueResponse(404, "marvel template does not exist");
|
||||
enqueueResponse(201, "marvel template created");
|
||||
enqueueResponse(404, "template for timestamped indices does not exist");
|
||||
enqueueResponse(201, "template for timestamped indices created");
|
||||
enqueueResponse(404, "template for data index does not exist");
|
||||
enqueueResponse(201, "template for data index created");
|
||||
enqueueResponse(200, "successful bulk request ");
|
||||
|
||||
HttpExporter exporter = getExporter(agentNode);
|
||||
|
@ -296,7 +331,7 @@ public class HttpExporterTests extends MarvelIntegTestCase {
|
|||
MarvelDoc doc = newRandomMarvelDoc();
|
||||
exporter.export(Collections.singletonList(doc));
|
||||
|
||||
assertThat(webServer.getRequestCount(), equalTo(4));
|
||||
assertThat(webServer.getRequestCount(), equalTo(6));
|
||||
|
||||
RecordedRequest recordedRequest = webServer.takeRequest();
|
||||
assertThat(recordedRequest.getMethod(), equalTo("GET"));
|
||||
|
@ -304,12 +339,21 @@ public class HttpExporterTests extends MarvelIntegTestCase {
|
|||
|
||||
recordedRequest = webServer.takeRequest();
|
||||
assertThat(recordedRequest.getMethod(), equalTo("GET"));
|
||||
assertThat(recordedRequest.getPath(), equalTo("/_template/.marvel-es"));
|
||||
assertThat(recordedRequest.getPath(), equalTo("/_template/" + indexTemplateName()));
|
||||
|
||||
recordedRequest = webServer.takeRequest();
|
||||
assertThat(recordedRequest.getMethod(), equalTo("PUT"));
|
||||
assertThat(recordedRequest.getPath(), equalTo("/_template/.marvel-es"));
|
||||
assertThat(recordedRequest.getBody().readByteArray(), equalTo(MarvelTemplateUtils.loadDefaultTemplate()));
|
||||
assertThat(recordedRequest.getPath(), equalTo("/_template/" + indexTemplateName()));
|
||||
assertThat(recordedRequest.getBody().readByteArray(), equalTo(TIMESTAMPED_TEMPLATE));
|
||||
|
||||
recordedRequest = webServer.takeRequest();
|
||||
assertThat(recordedRequest.getMethod(), equalTo("GET"));
|
||||
assertThat(recordedRequest.getPath(), equalTo("/_template/" + dataTemplateName()));
|
||||
|
||||
recordedRequest = webServer.takeRequest();
|
||||
assertThat(recordedRequest.getMethod(), equalTo("PUT"));
|
||||
assertThat(recordedRequest.getPath(), equalTo("/_template/" + dataTemplateName()));
|
||||
assertThat(recordedRequest.getBody().readByteArray(), equalTo(DATA_TEMPLATE));
|
||||
|
||||
recordedRequest = webServer.takeRequest();
|
||||
assertThat(recordedRequest.getMethod(), equalTo("POST"));
|
||||
|
@ -332,18 +376,18 @@ public class HttpExporterTests extends MarvelIntegTestCase {
|
|||
logger.info("--> exporting a second event");
|
||||
|
||||
enqueueGetClusterVersionResponse(Version.CURRENT);
|
||||
enqueueResponse(404, "marvel template does not exist");
|
||||
enqueueResponse(201, "marvel template created");
|
||||
enqueueResponse(200, "template for timestamped indices exist");
|
||||
enqueueResponse(200, "template for data index exist");
|
||||
enqueueResponse(200, "successful bulk request ");
|
||||
|
||||
doc = newRandomMarvelDoc();
|
||||
exporter = getExporter(agentNode);
|
||||
exporter.export(Collections.singletonList(doc));
|
||||
|
||||
String expectedMarvelIndex = MarvelSettings.MARVEL_INDICES_PREFIX
|
||||
String expectedMarvelIndex = MarvelSettings.MARVEL_INDICES_PREFIX + MarvelTemplateUtils.TEMPLATE_VERSION + "-"
|
||||
+ DateTimeFormat.forPattern(newTimeFormat).withZoneUTC().print(doc.timestamp());
|
||||
|
||||
assertThat(webServer.getRequestCount(), equalTo(4 + 4));
|
||||
assertThat(webServer.getRequestCount(), equalTo(6 + 4));
|
||||
|
||||
recordedRequest = webServer.takeRequest();
|
||||
assertThat(recordedRequest.getMethod(), equalTo("GET"));
|
||||
|
@ -351,12 +395,11 @@ public class HttpExporterTests extends MarvelIntegTestCase {
|
|||
|
||||
recordedRequest = webServer.takeRequest();
|
||||
assertThat(recordedRequest.getMethod(), equalTo("GET"));
|
||||
assertThat(recordedRequest.getPath(), equalTo("/_template/.marvel-es"));
|
||||
assertThat(recordedRequest.getPath(), equalTo("/_template/" + indexTemplateName()));
|
||||
|
||||
recordedRequest = webServer.takeRequest();
|
||||
assertThat(recordedRequest.getMethod(), equalTo("PUT"));
|
||||
assertThat(recordedRequest.getPath(), equalTo("/_template/.marvel-es"));
|
||||
assertThat(recordedRequest.getBody().readByteArray(), equalTo(MarvelTemplateUtils.loadDefaultTemplate()));
|
||||
assertThat(recordedRequest.getMethod(), equalTo("GET"));
|
||||
assertThat(recordedRequest.getPath(), equalTo("/_template/" + dataTemplateName()));
|
||||
|
||||
recordedRequest = webServer.takeRequest();
|
||||
assertThat(recordedRequest.getMethod(), equalTo("POST"));
|
||||
|
|
|
@ -5,26 +5,12 @@
|
|||
*/
|
||||
package org.elasticsearch.marvel.agent.exporter.local;
|
||||
|
||||
import org.elasticsearch.Version;
|
||||
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
|
||||
import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest;
|
||||
import org.elasticsearch.action.support.IndicesOptions;
|
||||
import org.elasticsearch.cluster.metadata.MappingMetaData;
|
||||
import org.elasticsearch.common.collect.ImmutableOpenMap;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.index.IndexNotFoundException;
|
||||
import org.elasticsearch.marvel.agent.exporter.AbstractExporterTemplateTestCase;
|
||||
import org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils;
|
||||
import org.elasticsearch.marvel.agent.settings.MarvelSettings;
|
||||
|
||||
import java.util.Collections;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
|
||||
import static org.hamcrest.Matchers.equalTo;
|
||||
import static org.hamcrest.Matchers.notNullValue;
|
||||
import static org.hamcrest.core.Is.is;
|
||||
|
||||
public class LocalExporterTemplateTests extends AbstractExporterTemplateTestCase {
|
||||
|
||||
|
@ -34,83 +20,26 @@ public class LocalExporterTemplateTests extends AbstractExporterTemplateTestCase
|
|||
}
|
||||
|
||||
@Override
|
||||
protected Set<String> excludeTemplates() {
|
||||
// Always delete the template between tests
|
||||
return Collections.emptySet();
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void deleteTemplate() throws Exception {
|
||||
protected void deleteTemplates() throws Exception {
|
||||
waitNoPendingTasksOnAll();
|
||||
assertAcked(client().admin().indices().prepareDeleteTemplate(MarvelTemplateUtils.INDEX_TEMPLATE_NAME).get());
|
||||
cluster().wipeAllTemplates(Collections.emptySet());
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void putTemplate(String version) throws Exception {
|
||||
protected void putTemplate(String name, int version) throws Exception {
|
||||
waitNoPendingTasksOnAll();
|
||||
assertAcked(client().admin().indices().preparePutTemplate(MarvelTemplateUtils.INDEX_TEMPLATE_NAME).setSource(generateTemplateSource(version)).get());
|
||||
assertAcked(client().admin().indices().preparePutTemplate(name).setSource(generateTemplateSource(name, version)).get());
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void createMarvelIndex(String index) throws Exception {
|
||||
protected void assertTemplateExist(String name) throws Exception {
|
||||
waitNoPendingTasksOnAll();
|
||||
createIndex(index);
|
||||
waitForMarvelTemplate(name);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void assertTemplateUpdated(Version version) throws Exception {
|
||||
protected void assertTemplateNotUpdated(String name) throws Exception {
|
||||
waitNoPendingTasksOnAll();
|
||||
awaitMarvelTemplateInstalled(version);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void assertTemplateNotUpdated(Version version) throws Exception {
|
||||
waitNoPendingTasksOnAll();
|
||||
awaitMarvelTemplateInstalled(version);
|
||||
}
|
||||
|
||||
private void assertMappings(byte[] reference, String... indices) throws Exception {
|
||||
waitNoPendingTasksOnAll();
|
||||
|
||||
Map<String, String> mappings = new PutIndexTemplateRequest().source(reference).mappings();
|
||||
assertBusy(new Runnable() {
|
||||
@Override
|
||||
public void run() {
|
||||
for (String index : indices) {
|
||||
GetMappingsResponse response = client().admin().indices().prepareGetMappings(index).setIndicesOptions(IndicesOptions.lenientExpandOpen()).get();
|
||||
ImmutableOpenMap<String, MappingMetaData> indexMappings = response.getMappings().get(index);
|
||||
assertNotNull(indexMappings);
|
||||
assertThat(indexMappings.size(), equalTo(mappings.size()));
|
||||
|
||||
for (String mapping : mappings.keySet()) {
|
||||
// We just check that mapping type exists, we don't verify its content
|
||||
assertThat("mapping type " + mapping + " should exist in index " + index, indexMappings.get(mapping), notNullValue());
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void assertMappingsUpdated(String... indices) throws Exception {
|
||||
assertMappings(MarvelTemplateUtils.loadDefaultTemplate(), indices);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void assertMappingsNotUpdated(String... indices) throws Exception {
|
||||
assertMappings(generateTemplateSource(null).toBytes(), indices);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void assertIndicesNotCreated() throws Exception {
|
||||
waitNoPendingTasksOnAll();
|
||||
try {
|
||||
assertThat(client().admin().indices().prepareExists(MarvelSettings.MARVEL_INDICES_PREFIX + "*").get().isExists(), is(false));
|
||||
} catch (IndexNotFoundException e) {
|
||||
// with shield we might get that if wildcards were resolved to no indices
|
||||
if (!shieldEnabled) {
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
assertTemplateExist(name);
|
||||
}
|
||||
}
|
||||
|
|
|
@ -6,9 +6,7 @@
|
|||
package org.elasticsearch.marvel.agent.exporter.local;
|
||||
|
||||
import org.elasticsearch.ElasticsearchException;
|
||||
import org.elasticsearch.Version;
|
||||
import org.elasticsearch.action.admin.indices.recovery.RecoveryResponse;
|
||||
import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse;
|
||||
import org.elasticsearch.action.search.SearchResponse;
|
||||
import org.elasticsearch.cluster.ClusterState;
|
||||
import org.elasticsearch.cluster.health.ClusterHealthStatus;
|
||||
|
@ -36,11 +34,12 @@ import java.util.Map;
|
|||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
|
||||
import static org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils.dataTemplateName;
|
||||
import static org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils.indexTemplateName;
|
||||
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
|
||||
import static org.hamcrest.Matchers.allOf;
|
||||
import static org.hamcrest.Matchers.containsString;
|
||||
import static org.hamcrest.Matchers.equalTo;
|
||||
import static org.hamcrest.Matchers.hasSize;
|
||||
import static org.hamcrest.Matchers.instanceOf;
|
||||
import static org.hamcrest.Matchers.is;
|
||||
import static org.hamcrest.Matchers.notNullValue;
|
||||
|
@ -105,8 +104,8 @@ public class LocalExporterTests extends MarvelIntegTestCase {
|
|||
updateMarvelInterval(3L, TimeUnit.SECONDS);
|
||||
|
||||
// lets wait until the marvel template will be installed
|
||||
awaitMarvelTemplateInstalled();
|
||||
assertThat(getCurrentlyInstalledTemplateVersion(), is(Version.CURRENT));
|
||||
waitForMarvelTemplate(indexTemplateName());
|
||||
waitForMarvelTemplate(dataTemplateName());
|
||||
}
|
||||
|
||||
public void testIndexTimestampFormat() throws Exception {
|
||||
|
@ -122,12 +121,12 @@ public class LocalExporterTests extends MarvelIntegTestCase {
|
|||
LocalExporter exporter = getLocalExporter("_local");
|
||||
|
||||
// first lets test that the index resolver works with time
|
||||
String indexName = MarvelSettings.MARVEL_INDICES_PREFIX + DateTimeFormat.forPattern(timeFormat).withZoneUTC().print(time);
|
||||
String indexName = MarvelSettings.MARVEL_INDICES_PREFIX + MarvelTemplateUtils.TEMPLATE_VERSION + "-" + DateTimeFormat.forPattern(timeFormat).withZoneUTC().print(time);
|
||||
assertThat(exporter.indexNameResolver().resolve(time), equalTo(indexName));
|
||||
|
||||
// now lets test that the index name resolver works with a doc
|
||||
MarvelDoc doc = newRandomMarvelDoc();
|
||||
indexName = MarvelSettings.MARVEL_INDICES_PREFIX + DateTimeFormat.forPattern(timeFormat).withZoneUTC().print(doc.timestamp());
|
||||
indexName = MarvelSettings.MARVEL_INDICES_PREFIX + MarvelTemplateUtils.TEMPLATE_VERSION + "-" + DateTimeFormat.forPattern(timeFormat).withZoneUTC().print(doc.timestamp());
|
||||
assertThat(exporter.indexNameResolver().resolve(doc), equalTo(indexName));
|
||||
|
||||
logger.debug("--> exporting a random marvel document");
|
||||
|
@ -138,7 +137,7 @@ public class LocalExporterTests extends MarvelIntegTestCase {
|
|||
timeFormat = randomFrom("dd", "dd.MM.YYYY", "dd.MM");
|
||||
updateClusterSettings(Settings.builder().put("marvel.agent.exporters._local.index.name.time_format", timeFormat));
|
||||
exporter = getLocalExporter("_local"); // we need to get it again.. as it was rebuilt
|
||||
indexName = MarvelSettings.MARVEL_INDICES_PREFIX + DateTimeFormat.forPattern(timeFormat).withZoneUTC().print(doc.timestamp());
|
||||
indexName = MarvelSettings.MARVEL_INDICES_PREFIX + MarvelTemplateUtils.TEMPLATE_VERSION + "-" + DateTimeFormat.forPattern(timeFormat).withZoneUTC().print(doc.timestamp());
|
||||
assertThat(exporter.indexNameResolver().resolve(doc), equalTo(indexName));
|
||||
|
||||
logger.debug("--> exporting the document again (this time with the the new index name time format [{}], expecting index name [{}]", timeFormat, indexName);
|
||||
|
@ -194,13 +193,4 @@ public class LocalExporterTests extends MarvelIntegTestCase {
|
|||
ClusterStateCollector.TYPE, timeStampGenerator.incrementAndGet(), ClusterState.PROTO, ClusterHealthStatus.GREEN);
|
||||
}
|
||||
}
|
||||
|
||||
private Version getCurrentlyInstalledTemplateVersion() {
|
||||
GetIndexTemplatesResponse response = client().admin().indices().prepareGetTemplates(MarvelTemplateUtils.INDEX_TEMPLATE_NAME).get();
|
||||
assertThat(response, notNullValue());
|
||||
assertThat(response.getIndexTemplates(), notNullValue());
|
||||
assertThat(response.getIndexTemplates(), hasSize(1));
|
||||
assertThat(response.getIndexTemplates().get(0), notNullValue());
|
||||
return MarvelTemplateUtils.templateVersion(response.getIndexTemplates().get(0));
|
||||
}
|
||||
}
|
||||
|
|
|
@ -12,6 +12,7 @@ import org.elasticsearch.common.settings.Settings;
|
|||
import org.elasticsearch.index.query.QueryBuilders;
|
||||
import org.elasticsearch.license.core.License;
|
||||
import org.elasticsearch.marvel.agent.collector.cluster.ClusterInfoCollector;
|
||||
import org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils;
|
||||
import org.elasticsearch.marvel.agent.settings.MarvelSettings;
|
||||
import org.elasticsearch.marvel.test.MarvelIntegTestCase;
|
||||
import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
|
||||
|
@ -21,6 +22,8 @@ import org.junit.Before;
|
|||
import java.util.Map;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import static org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils.dataTemplateName;
|
||||
import static org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils.indexTemplateName;
|
||||
import static org.elasticsearch.test.ESIntegTestCase.Scope.TEST;
|
||||
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
|
||||
import static org.hamcrest.Matchers.equalTo;
|
||||
|
@ -57,20 +60,21 @@ public class ClusterInfoTests extends MarvelIntegTestCase {
|
|||
final String clusterUUID = client().admin().cluster().prepareState().setMetaData(true).get().getState().metaData().clusterUUID();
|
||||
assertTrue(Strings.hasText(clusterUUID));
|
||||
|
||||
logger.debug("--> waiting for the marvel data index to be created (it should have been created by the LicenseCollector)");
|
||||
awaitIndexExists(MarvelSettings.MARVEL_DATA_INDEX_NAME);
|
||||
logger.debug("--> waiting for the marvel data index to be created (it should have been created by the ClusterInfoCollector)");
|
||||
String dataIndex = ".marvel-es-data-" + MarvelTemplateUtils.TEMPLATE_VERSION;
|
||||
awaitIndexExists(dataIndex);
|
||||
|
||||
logger.debug("--> waiting for cluster info collector to collect data");
|
||||
awaitMarvelDocsCount(equalTo(1L), ClusterInfoCollector.TYPE);
|
||||
|
||||
logger.debug("--> retrieving cluster info document");
|
||||
GetResponse response = client().prepareGet(MarvelSettings.MARVEL_DATA_INDEX_NAME, ClusterInfoCollector.TYPE, clusterUUID).get();
|
||||
assertTrue(MarvelSettings.MARVEL_DATA_INDEX_NAME + " document does not exist", response.isExists());
|
||||
GetResponse response = client().prepareGet(dataIndex, ClusterInfoCollector.TYPE, clusterUUID).get();
|
||||
assertTrue("cluster_info document does not exist in data index", response.isExists());
|
||||
|
||||
logger.debug("--> checking that the document contains all required information");
|
||||
|
||||
logger.debug("--> checking that the document contains license information");
|
||||
assertThat(response.getIndex(), equalTo(MarvelSettings.MARVEL_DATA_INDEX_NAME));
|
||||
assertThat(response.getIndex(), equalTo(dataIndex));
|
||||
assertThat(response.getType(), equalTo(ClusterInfoCollector.TYPE));
|
||||
assertThat(response.getId(), equalTo(clusterUUID));
|
||||
|
||||
|
@ -111,14 +115,15 @@ public class ClusterInfoTests extends MarvelIntegTestCase {
|
|||
assertThat(clusterStats, instanceOf(Map.class));
|
||||
assertThat(((Map) clusterStats).size(), greaterThan(0));
|
||||
|
||||
assertMarvelTemplateInstalled();
|
||||
waitForMarvelTemplate(indexTemplateName());
|
||||
waitForMarvelTemplate(dataTemplateName());
|
||||
|
||||
logger.debug("--> check that the cluster_info is not indexed");
|
||||
securedFlush();
|
||||
securedRefresh();
|
||||
|
||||
assertHitCount(client().prepareSearch().setSize(0)
|
||||
.setIndices(MarvelSettings.MARVEL_DATA_INDEX_NAME)
|
||||
.setIndices(dataIndex)
|
||||
.setTypes(ClusterInfoCollector.TYPE)
|
||||
.setQuery(QueryBuilders.boolQuery()
|
||||
.should(QueryBuilders.matchQuery(License.XFields.STATUS.underscore().toString(), License.Status.ACTIVE.label()))
|
||||
|
|
|
@ -11,6 +11,7 @@ import org.elasticsearch.cluster.node.DiscoveryNodes;
|
|||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.index.query.QueryBuilders;
|
||||
import org.elasticsearch.marvel.agent.collector.cluster.ClusterStateCollector;
|
||||
import org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils;
|
||||
import org.elasticsearch.marvel.agent.renderer.AbstractRenderer;
|
||||
import org.elasticsearch.marvel.agent.settings.MarvelSettings;
|
||||
import org.elasticsearch.marvel.test.MarvelIntegTestCase;
|
||||
|
@ -156,9 +157,10 @@ public class ClusterStateTests extends MarvelIntegTestCase {
|
|||
|
||||
for (final String nodeName : internalCluster().getNodeNames()) {
|
||||
final String nodeId = internalCluster().clusterService(nodeName).localNode().getId();
|
||||
final String dataIndex = ".marvel-es-data-" + MarvelTemplateUtils.TEMPLATE_VERSION;
|
||||
|
||||
logger.debug("--> getting marvel document for node id [{}]", nodeId);
|
||||
assertThat(client().prepareGet(MarvelSettings.MARVEL_DATA_INDEX_NAME, ClusterStateCollector.NODE_TYPE, nodeId).get().isExists(), is(true));
|
||||
assertThat(client().prepareGet(dataIndex, ClusterStateCollector.NODE_TYPE, nodeId).get().isExists(), is(true));
|
||||
|
||||
// checks that document is not indexed
|
||||
assertHitCount(client().prepareSearch().setSize(0)
|
||||
|
|
|
@ -36,13 +36,13 @@ public class SecuredClientTests extends MarvelIntegTestCase {
|
|||
assertAccessIsAllowed(securedClient.admin().indices().prepareDelete(MarvelSettings.MARVEL_INDICES_PREFIX));
|
||||
assertAccessIsAllowed(securedClient.admin().indices().prepareCreate(MarvelSettings.MARVEL_INDICES_PREFIX + "test"));
|
||||
|
||||
assertAccessIsAllowed(securedClient.admin().indices().preparePutTemplate("foo").setSource(MarvelTemplateUtils.loadDefaultTemplate()));
|
||||
assertAccessIsAllowed(securedClient.admin().indices().preparePutTemplate("foo").setSource(MarvelTemplateUtils.loadTimestampedIndexTemplate()));
|
||||
assertAccessIsAllowed(securedClient.admin().indices().prepareGetTemplates("foo"));
|
||||
}
|
||||
|
||||
public void testDeniedAccess() {
|
||||
SecuredClient securedClient = internalCluster().getInstance(SecuredClient.class);
|
||||
assertAcked(securedClient.admin().indices().preparePutTemplate("foo").setSource(MarvelTemplateUtils.loadDefaultTemplate()).get());
|
||||
assertAcked(securedClient.admin().indices().preparePutTemplate("foo").setSource(MarvelTemplateUtils.loadDataIndexTemplate()).get());
|
||||
|
||||
if (shieldEnabled) {
|
||||
assertAccessIsDenied(securedClient.admin().indices().prepareDeleteTemplate("foo"));
|
||||
|
|
|
@ -6,14 +6,10 @@
|
|||
package org.elasticsearch.marvel.test;
|
||||
|
||||
import org.elasticsearch.ElasticsearchException;
|
||||
import org.elasticsearch.Version;
|
||||
import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
|
||||
import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse;
|
||||
import org.elasticsearch.cluster.ClusterState;
|
||||
import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
|
||||
import org.elasticsearch.cluster.metadata.MetaData;
|
||||
import org.elasticsearch.common.Strings;
|
||||
import org.elasticsearch.common.io.Streams;
|
||||
import org.elasticsearch.common.regex.Regex;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.common.util.concurrent.CountDown;
|
||||
import org.elasticsearch.index.IndexModule;
|
||||
|
@ -70,11 +66,6 @@ public abstract class MarvelIntegTestCase extends ESIntegTestCase {
|
|||
return super.buildTestCluster(scope, seed);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Set<String> excludeTemplates() {
|
||||
return Collections.singleton(MarvelTemplateUtils.INDEX_TEMPLATE_NAME);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Settings nodeSettings(int nodeOrdinal) {
|
||||
Settings.Builder builder = Settings.builder()
|
||||
|
@ -212,63 +203,29 @@ public abstract class MarvelIntegTestCase extends ESIntegTestCase {
|
|||
}
|
||||
}
|
||||
|
||||
protected void assertMarvelTemplateInstalled() {
|
||||
ClusterStateResponse clusterStateResponse = client().admin().cluster().prepareState().get();
|
||||
if (clusterStateResponse != null) {
|
||||
ClusterState state = clusterStateResponse.getState();
|
||||
MetaData md = state.getMetaData();
|
||||
}
|
||||
GetIndexTemplatesResponse response = client().admin().indices().prepareGetTemplates().get();
|
||||
for (IndexTemplateMetaData template : response.getIndexTemplates()) {
|
||||
if (template.getName().equals(MarvelTemplateUtils.INDEX_TEMPLATE_NAME)) {
|
||||
return;
|
||||
protected void assertTemplateInstalled(String name) {
|
||||
boolean found = false;
|
||||
for (IndexTemplateMetaData template : client().admin().indices().prepareGetTemplates().get().getIndexTemplates()) {
|
||||
if (Regex.simpleMatch(name, template.getName())) {
|
||||
found = true;
|
||||
}
|
||||
}
|
||||
fail("marvel template should exist");
|
||||
assertTrue("failed to find a template matching [" + name + "]", found);
|
||||
}
|
||||
|
||||
protected void assertMarvelTemplateMissing() {
|
||||
for (IndexTemplateMetaData template : client().admin().indices().prepareGetTemplates(MarvelTemplateUtils.INDEX_TEMPLATE_NAME).get().getIndexTemplates()) {
|
||||
if (template.getName().equals(MarvelTemplateUtils.INDEX_TEMPLATE_NAME)) {
|
||||
fail("marvel template shouldn't exist");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
protected void awaitMarvelTemplateInstalled() throws Exception {
|
||||
protected void waitForMarvelTemplate(String name) throws Exception {
|
||||
assertBusy(new Runnable() {
|
||||
@Override
|
||||
public void run() {
|
||||
assertMarvelTemplateInstalled();
|
||||
assertTemplateInstalled(name);
|
||||
}
|
||||
}, 30, TimeUnit.SECONDS);
|
||||
}
|
||||
|
||||
protected void awaitMarvelTemplateInstalled(Version version) throws Exception {
|
||||
assertBusy(new Runnable() {
|
||||
@Override
|
||||
public void run() {
|
||||
assertMarvelTemplateInstalled(version);
|
||||
}
|
||||
}, 30, TimeUnit.SECONDS);
|
||||
}
|
||||
|
||||
protected void assertMarvelTemplateInstalled(Version version) {
|
||||
for (IndexTemplateMetaData template : client().admin().indices().prepareGetTemplates(MarvelTemplateUtils.INDEX_TEMPLATE_NAME).get().getIndexTemplates()) {
|
||||
if (template.getName().equals(MarvelTemplateUtils.INDEX_TEMPLATE_NAME)) {
|
||||
Version templateVersion = MarvelTemplateUtils.templateVersion(template);
|
||||
if (templateVersion != null && templateVersion.id == version.id) {
|
||||
return;
|
||||
}
|
||||
fail("did not find marvel template with expected version [" + version + "]. found version [" + templateVersion + "]");
|
||||
}
|
||||
}
|
||||
fail("marvel template could not be found");
|
||||
}
|
||||
|
||||
protected void waitForMarvelIndices() throws Exception {
|
||||
awaitIndexExists(MarvelSettings.MARVEL_DATA_INDEX_NAME);
|
||||
awaitIndexExists(MarvelSettings.MARVEL_INDICES_PREFIX + "*");
|
||||
String currentVersion = String.valueOf(MarvelTemplateUtils.TEMPLATE_VERSION);
|
||||
awaitIndexExists(MarvelSettings.MARVEL_DATA_INDEX_PREFIX + currentVersion);
|
||||
awaitIndexExists(MarvelSettings.MARVEL_INDICES_PREFIX + currentVersion + "-*");
|
||||
assertBusy(new Runnable() {
|
||||
@Override
|
||||
public void run() {
|
||||
|
|
Loading…
Reference in New Issue