honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[7.x] Add default composable templates for new indexing strategy (#57629) (#58757) 

Backports the following commits to 7.x:

    Add default composable templates for new indexing strategy (#57629)
This commit is contained in:
Lee Hinman 2020-07-01 09:32:32 -06:00 committed by GitHub
parent 7a8da9daa3
commit d3d03fc1c6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
37 changed files with 1389 additions and 98 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
docs/reference/indices/templates.asciidoc
View File

@ -180,7 +180,7 @@ orders overriding them. For example:
--------------------------------------------------
PUT /_template/template_1
{
"index_patterns" : ["*"],
"index_patterns" : ["te*"],
"order" : 0,
"settings" : {
"number_of_shards" : 1
@ -192,7 +192,7 @@ PUT /_template/template_1
PUT /_template/template_2
{
"index_patterns" : ["te*"],
"index_patterns" : ["tes*"],
"order" : 1,
"settings" : {
"number_of_shards" : 1
@ -204,7 +204,7 @@ PUT /_template/template_2
--------------------------------------------------
The above will disable storing the `_source`, but
for indices that start with `te*`, `_source` will still be enabled.
for indices that start with `tes*`, `_source` will still be enabled.
Note, for mappings, the merging is "deep", meaning that specific
object/property based mappings can easily be added/overridden on higher
order templates, with lower order templates providing the basis.
@ -231,7 +231,7 @@ replace the template without specifying one.
--------------------------------------------------
PUT /_template/template_1
{
"index_patterns" : ["*"],
"index_patterns" : ["myindex-*"],
"order" : 0,
"settings" : {
"number_of_shards" : 1

6
rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/10_basic.yml
View File

@ -294,6 +294,7 @@
- do:
cat.aliases:
v: true
name: test*
- match:
$body: |
@ -364,6 +365,7 @@
- do:
cat.aliases:
h: [index, alias]
name: test*
- match:
$body: /^ test \s+ test_1 \s+ $/
@ -373,6 +375,7 @@
cat.aliases:
h: [index, alias]
v: true
name: test*
- match:
$body: |
/^
@ -398,7 +401,8 @@
index: test_index
- do:
cat.aliases: {}
cat.aliases:
name: test*
- match:
$body: |

3
rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/40_hidden.yml
View File

@ -73,7 +73,8 @@
aliases:
test_alias: {}
- do:
cat.aliases: {}
cat.aliases:
name: test*
- match:
$body: |

54
rest-api-spec/src/main/resources/rest-api-spec/test/cluster.stats/10_basic.yml
View File

@ -89,57 +89,3 @@
cluster.stats: {}
- is_true: nodes.packaging_types
---
"get cluster stats returns mapping stats":
- skip:
version: " - 7.6.99"
reason: "mapping stats are added for v7.7.0"
- do:
cluster.stats: {}
- length: { indices.mappings.field_types: 0 }
- do:
indices.create:
index: test-index1
body:
mappings:
properties:
foo:
type: keyword
- do:
indices.create:
index: test-index2
body:
mappings:
properties:
foo:
type: keyword
bar:
properties:
quux:
type: integer
baz:
type: keyword
- do:
cluster.stats: {}
- length: { indices.mappings.field_types: 3 }
- match: { indices.mappings.field_types.0.name: integer }
- match: { indices.mappings.field_types.0.count: 1 }
- match: { indices.mappings.field_types.0.index_count: 1 }
- match: { indices.mappings.field_types.1.name: keyword }
- match: { indices.mappings.field_types.1.count: 3 }
- match: { indices.mappings.field_types.1.index_count: 2 }
- match: { indices.mappings.field_types.2.name: object }
- match: { indices.mappings.field_types.2.count: 1 }
- match: { indices.mappings.field_types.2.index_count: 1 }

5
rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_index_template/10_basic.yml
View File

@ -36,7 +36,7 @@ setup:
- match: {index_templates.0.index_template.template.mappings: {properties: {field: {type: keyword}}}}
---
"Get all tindex emplates":
"Get all index templates":
- skip:
version: " - 7.7.99"
reason: "index template v2 API unavailable before 7.8"
@ -56,7 +56,8 @@ setup:
- do:
indices.get_index_template: {}
- length: {index_templates: 2}
- is_true: index_templates.0.name
- is_true: index_templates.1.name
---
"Get index template with local flag":

10
rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_template/10_basic.yml
View File

@ -142,7 +142,7 @@
body: >
{
"version": 10,
"index_patterns": "*",
"index_patterns": "foo*",
"settings": { "number_of_shards": 1 }
}
- match: { acknowledged: true }
@ -159,7 +159,7 @@
body: >
{
"version": 9,
"index_patterns": "*",
"index_patterns": "foo*",
"settings": { "number_of_shards": 1 }
}
- match: { acknowledged: true }
@ -176,7 +176,7 @@
body: >
{
"version": 6789,
"index_patterns": "*",
"index_patterns": "foo*",
"settings": { "number_of_shards": 1 }
}
- match: { acknowledged: true }
@ -192,7 +192,7 @@
name: "my_template"
body: >
{
"index_patterns": "*",
"index_patterns": "foo*",
"settings": { "number_of_shards": 1 }
}
- match: { acknowledged: true }
@ -209,7 +209,7 @@
body: >
{
"version": 5385,
"index_patterns": "*",
"index_patterns": "foo*",
"settings": { "number_of_shards": 1 }
}
- match: { acknowledged: true }

30
server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsIT.java
View File

@ -29,6 +29,8 @@ import org.elasticsearch.cluster.node.DiscoveryNodeRole;
import org.elasticsearch.common.Priority;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.concurrent.EsExecutors;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.monitor.os.OsStats;
import org.elasticsearch.node.NodeRoleSettings;
import org.elasticsearch.test.ESIntegTestCase;
@ -46,6 +48,7 @@ import java.util.stream.Collectors;
import static org.elasticsearch.action.admin.cluster.node.stats.NodesStatsRequest.Metric.OS;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.greaterThanOrEqualTo;
import static org.hamcrest.Matchers.is;
@ClusterScope(scope = Scope.TEST, numDataNodes = 0)
@ -250,4 +253,31 @@ public class ClusterStatsIT extends ESIntegTestCase {
response = client().admin().cluster().prepareClusterStats().get();
assertThat(response.getStatus(), equalTo(ClusterHealthStatus.GREEN));
}
public void testFieldTypes() {
internalCluster().startNode();
ensureGreen();
ClusterStatsResponse response = client().admin().cluster().prepareClusterStats().get();
assertThat(response.getStatus(), Matchers.equalTo(ClusterHealthStatus.GREEN));
assertTrue(response.getIndicesStats().getMappings().getFieldTypeStats().isEmpty());
client().admin().indices().prepareCreate("test1").addMapping(MapperService.SINGLE_MAPPING_NAME,
"{\"properties\":{\"foo\":{\"type\": \"keyword\"}}}", XContentType.JSON).get();
client().admin().indices().prepareCreate("test2")
.addMapping(MapperService.SINGLE_MAPPING_NAME,
"{\"properties\":{\"foo\":{\"type\": \"keyword\"},\"bar\":{\"properties\":{\"baz\":{\"type\":\"keyword\"}," +
"\"eggplant\":{\"type\":\"integer\"}}}}}", XContentType.JSON).get();
response = client().admin().cluster().prepareClusterStats().get();
assertThat(response.getIndicesStats().getMappings().getFieldTypeStats().size(), equalTo(3));
Set<IndexFeatureStats> stats = response.getIndicesStats().getMappings().getFieldTypeStats();
for (IndexFeatureStats stat : stats) {
if (stat.getName().equals("integer")) {
assertThat(stat.getCount(), greaterThanOrEqualTo(1));
} else if (stat.getName().equals("keyword")) {
assertThat(stat.getCount(), greaterThanOrEqualTo(3));
} else if (stat.getName().equals("object")) {
assertThat(stat.getCount(), greaterThanOrEqualTo(1));
}
}
}
}

3
server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java
View File

@ -498,7 +498,8 @@ public class MetadataIndexTemplateService {
(finalIndexTemplate.composedOf().size() > 0 ? "with component templates " + finalIndexTemplate.composedOf() + " " : "") +
"is invalid", e);
}
logger.info("{} index template [{}]", existing == null ? "adding" : "updating", name);
logger.info("{} index template [{}] for index patterns {}", existing == null ? "adding" : "updating", name,
template.indexPatterns());
return ClusterState.builder(currentState)
.metadata(Metadata.builder(currentState.metadata()).put(name, finalIndexTemplate))
.build();

33
test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
View File

@ -27,6 +27,7 @@ import org.apache.http.message.BasicHeader;
import org.apache.http.nio.conn.ssl.SSLIOSessionStrategy;
import org.apache.http.ssl.SSLContexts;
import org.apache.http.util.EntityUtils;
import org.apache.logging.log4j.message.ParameterizedMessage;
import org.apache.lucene.util.SetOnce;
import org.elasticsearch.Version;
import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksAction;
@ -506,7 +507,8 @@ public abstract class ESRestTestCase extends ESTestCase {
* A set of ILM policies that should be preserved between runs.
*/
protected Set<String> preserveILMPolicyIds() {
return Sets.newHashSet("ilm-history-ilm-policy", "slm-history-ilm-policy", "watch-history-ilm-policy", "ml-size-based-ilm-policy");
return Sets.newHashSet("ilm-history-ilm-policy", "slm-history-ilm-policy",
"watch-history-ilm-policy", "ml-size-based-ilm-policy", "logs", "metrics");
}
/**
@ -604,8 +606,25 @@ public abstract class ESRestTestCase extends ESTestCase {
}
}
try {
adminClient().performRequest(new Request("DELETE", "_component_template/*"));
} catch (ResponseException e) {
Request compReq = new Request("GET", "_component_template");
String componentTemplates = EntityUtils.toString(adminClient().performRequest(compReq).getEntity());
Map<String, Object> cTemplates = XContentHelper.convertToMap(JsonXContent.jsonXContent, componentTemplates, false);
@SuppressWarnings("unchecked")
List<String> names = ((List<Map<String, Object>>) cTemplates.get("component_templates")).stream()
.map(ct -> (String) ct.get("name"))
.collect(Collectors.toList());
for (String componentTemplate : names) {
try {
if (isXPackTemplate(componentTemplate)) {
continue;
}
adminClient().performRequest(new Request("DELETE", "_component_template/" + componentTemplate));
} catch (ResponseException e) {
logger.debug(new ParameterizedMessage("unable to remove component template {}", componentTemplate), e);
}
}
} catch (Exception e) {
logger.info("ignoring exception removing all component templates", e);
// We hit a version of ES that doesn't support index templates v2 yet, so it's safe to ignore
}
} else {
@ -1212,7 +1231,6 @@ public abstract class ESRestTestCase extends ESTestCase {
return true;
}
switch (name) {
case ".triggered_watches":
case ".watches":
case "logstash-index-template":
case ".logstash-management":
@ -1220,6 +1238,13 @@ public abstract class ESRestTestCase extends ESTestCase {
case ".slm-history":
case ".async-search":
case "saml-service-provider":
case "ilm-history":
case "logs":
case "logs-settings":
case "logs-mappings":
case "metrics":
case "metrics-settings":
case "metrics-mappings":
return true;
default:
return false;

9
test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java
View File

@ -30,6 +30,7 @@ import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.Response;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestClientBuilder;
import org.elasticsearch.client.WarningsHandler;
import org.elasticsearch.client.sniff.ElasticsearchNodesSniffer;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.collect.Tuple;
@ -364,6 +365,14 @@ public abstract class ESClientYamlSuiteTestCase extends ESRestTestCase {
&& testCandidate.getTestSection().getSkipSection().getFeatures().contains("default_shards") == false) {
final Request request = new Request("PUT", "/_template/global");
request.setJsonEntity("{\"index_patterns\":[\"*\"],\"settings\":{\"index.number_of_shards\":2}}");
// Because this has not yet transitioned to a composable template, it's possible that
// this can overlap an installed composable template since this is a global (*)
// template. In order to avoid this failing the test, we override the warnings handler
// to be permissive in this case. This can be removed once all tests use composable
// templates instead of legacy templates
RequestOptions.Builder builder = RequestOptions.DEFAULT.toBuilder();
builder.setWarningsHandler(WarningsHandler.PERMISSIVE);
request.setOptions(builder.build());
adminClient().performRequest(request);
}

1
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ClientHelper.java
View File

@ -69,6 +69,7 @@ public final class ClientHelper {
public static final String TRANSFORM_ORIGIN = "transform";
public static final String ASYNC_SEARCH_ORIGIN = "async_search";
public static final String IDP_ORIGIN = "idp";
public static final String STACK_ORIGIN = "stack";
private ClientHelper() {}

4
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/history/SnapshotLifecycleTemplateRegistry.java
View File

@ -69,7 +69,7 @@ public class SnapshotLifecycleTemplateRegistry extends IndexTemplateRegistry {
}
@Override
protected List<IndexTemplateConfig> getTemplateConfigs() {
protected List<IndexTemplateConfig> getLegacyTemplateConfigs() {
if (slmHistoryEnabled == false) {
return Collections.emptyList();
}
@ -90,7 +90,7 @@ public class SnapshotLifecycleTemplateRegistry extends IndexTemplateRegistry {
}
public boolean validate(ClusterState state) {
boolean allTemplatesPresent = getTemplateConfigs().stream()
boolean allTemplatesPresent = getLegacyTemplateConfigs().stream()
.map(IndexTemplateConfig::getTemplateName)
.allMatch(name -> state.metadata().getTemplates().containsKey(name));

204
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/IndexTemplateRegistry.java
View File

@ -9,20 +9,27 @@ package org.elasticsearch.xpack.core.template;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.message.ParameterizedMessage;
import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.admin.indices.template.put.PutComponentTemplateAction;
import org.elasticsearch.action.admin.indices.template.put.PutComposableIndexTemplateAction;
import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.client.Client;
import org.elasticsearch.cluster.ClusterChangedEvent;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.ClusterStateListener;
import org.elasticsearch.cluster.metadata.ComponentTemplate;
import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
import org.elasticsearch.cluster.metadata.IndexTemplateMetadata;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.DeprecationHandler;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.elasticsearch.gateway.GatewayService;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.xpack.core.XPackClient;
@ -30,9 +37,12 @@ import org.elasticsearch.xpack.core.ilm.IndexLifecycleMetadata;
import org.elasticsearch.xpack.core.ilm.LifecyclePolicy;
import org.elasticsearch.xpack.core.ilm.action.PutLifecycleAction;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.Executor;
@ -70,14 +80,38 @@ public abstract class IndexTemplateRegistry implements ClusterStateListener {
* the index templates that should be installed and managed.
* @return The configurations for the templates that should be installed.
*/
protected abstract List<IndexTemplateConfig> getTemplateConfigs();
protected List<IndexTemplateConfig> getLegacyTemplateConfigs() {
return Collections.emptyList();
}
/**
* Retrieves return a list of {@link IndexTemplateConfig} that represents
* the component templates that should be installed and managed. Component
* templates are always installed prior composable templates, so they may
* be referenced by a composable template.
* @return The configurations for the templates that should be installed.
*/
protected List<IndexTemplateConfig> getComponentTemplateConfigs() {
return Collections.emptyList();
}
/**
* Retrieves return a list of {@link IndexTemplateConfig} that represents
* the composable templates that should be installed and managed.
* @return The configurations for the templates that should be installed.
*/
protected List<IndexTemplateConfig> getComposableTemplateConfigs() {
return Collections.emptyList();
}
/**
* Retrieves a list of {@link LifecyclePolicyConfig} that represents the ILM
* policies that should be installed and managed. Only called if ILM is enabled.
* @return The configurations for the lifecycle policies that should be installed.
*/
protected abstract List<LifecyclePolicyConfig> getPolicyConfigs();
protected List<LifecyclePolicyConfig> getPolicyConfigs() {
return Collections.emptyList();
}
/**
* Retrieves an identifier that is used to identify which plugin is asking for this.
@ -147,34 +181,116 @@ public abstract class IndexTemplateRegistry implements ClusterStateListener {
}
private void addTemplatesIfMissing(ClusterState state) {
final List<IndexTemplateConfig> indexTemplates = getTemplateConfigs();
addLegacyTemplatesIfMissing(state);
addComponentTemplatesIfMissing(state);
addComposableTemplatesIfMissing(state);
}
private void addLegacyTemplatesIfMissing(ClusterState state) {
final List<IndexTemplateConfig> indexTemplates = getLegacyTemplateConfigs();
for (IndexTemplateConfig newTemplate : indexTemplates) {
final String templateName = newTemplate.getTemplateName();
final AtomicBoolean creationCheck = templateCreationsInProgress.computeIfAbsent(templateName, key -> new AtomicBoolean(false));
if (creationCheck.compareAndSet(false, true)) {
IndexTemplateMetadata currentTemplate = state.metadata().getTemplates().get(templateName);
if (Objects.isNull(currentTemplate)) {
logger.info("adding index template [{}] for [{}], because it doesn't exist", templateName, getOrigin());
putTemplate(newTemplate, creationCheck);
logger.info("adding legacy template [{}] for [{}], because it doesn't exist", templateName, getOrigin());
putLegacyTemplate(newTemplate, creationCheck);
} else if (Objects.isNull(currentTemplate.getVersion()) || newTemplate.getVersion() > currentTemplate.getVersion()) {
// IndexTemplateConfig now enforces templates contain a `version` property, so if the template doesn't have one we can
// safely assume it's an old version of the template.
logger.info("upgrading index template [{}] for [{}] from version [{}] to version [{}]",
logger.info("upgrading legacy template [{}] for [{}] from version [{}] to version [{}]",
templateName, getOrigin(), currentTemplate.getVersion(), newTemplate.getVersion());
putTemplate(newTemplate, creationCheck);
putLegacyTemplate(newTemplate, creationCheck);
} else {
creationCheck.set(false);
logger.trace("not adding index template [{}] for [{}], because it already exists at version [{}]",
logger.trace("not adding legacy template [{}] for [{}], because it already exists at version [{}]",
templateName, getOrigin(), currentTemplate.getVersion());
}
} else {
logger.trace("skipping the creation of index template [{}] for [{}], because its creation is in progress",
logger.trace("skipping the creation of legacy template [{}] for [{}], because its creation is in progress",
templateName, getOrigin());
}
}
}
private void putTemplate(final IndexTemplateConfig config, final AtomicBoolean creationCheck) {
private void addComponentTemplatesIfMissing(ClusterState state) {
final List<IndexTemplateConfig> indexTemplates = getComponentTemplateConfigs();
for (IndexTemplateConfig newTemplate : indexTemplates) {
final String templateName = newTemplate.getTemplateName();
final AtomicBoolean creationCheck = templateCreationsInProgress.computeIfAbsent(templateName, key -> new AtomicBoolean(false));
if (creationCheck.compareAndSet(false, true)) {
ComponentTemplate currentTemplate = state.metadata().componentTemplates().get(templateName);
if (Objects.isNull(currentTemplate)) {
logger.debug("adding component template [{}] for [{}], because it doesn't exist", templateName, getOrigin());
putComponentTemplate(newTemplate, creationCheck);
} else if (Objects.isNull(currentTemplate.version()) || newTemplate.getVersion() > currentTemplate.version()) {
// IndexTemplateConfig now enforces templates contain a `version` property, so if the template doesn't have one we can
// safely assume it's an old version of the template.
logger.info("upgrading component template [{}] for [{}] from version [{}] to version [{}]",
templateName, getOrigin(), currentTemplate.version(), newTemplate.getVersion());
putComponentTemplate(newTemplate, creationCheck);
} else {
creationCheck.set(false);
logger.trace("not adding component template [{}] for [{}], because it already exists at version [{}]",
templateName, getOrigin(), currentTemplate.version());
}
} else {
logger.trace("skipping the creation of component template [{}] for [{}], because its creation is in progress",
templateName, getOrigin());
}
}
}
private void addComposableTemplatesIfMissing(ClusterState state) {
final List<IndexTemplateConfig> indexTemplates = getComposableTemplateConfigs();
for (IndexTemplateConfig newTemplate : indexTemplates) {
final String templateName = newTemplate.getTemplateName();
final AtomicBoolean creationCheck = templateCreationsInProgress.computeIfAbsent(templateName, key -> new AtomicBoolean(false));
if (creationCheck.compareAndSet(false, true)) {
ComposableIndexTemplate currentTemplate = state.metadata().templatesV2().get(templateName);
boolean componentTemplatesAvailable = componentTemplatesExist(state, newTemplate);
if (componentTemplatesAvailable == false) {
creationCheck.set(false);
logger.trace("not adding composable template [{}] for [{}] because its required component templates do not exist",
templateName, getOrigin());
} else if (Objects.isNull(currentTemplate)) {
logger.debug("adding composable template [{}] for [{}], because it doesn't exist", templateName, getOrigin());
putComposableTemplate(newTemplate, creationCheck);
} else if (Objects.isNull(currentTemplate.version()) || newTemplate.getVersion() > currentTemplate.version()) {
// IndexTemplateConfig now enforces templates contain a `version` property, so if the template doesn't have one we can
// safely assume it's an old version of the template.
logger.info("upgrading composable template [{}] for [{}] from version [{}] to version [{}]",
templateName, getOrigin(), currentTemplate.version(), newTemplate.getVersion());
putComposableTemplate(newTemplate, creationCheck);
} else {
creationCheck.set(false);
logger.trace("not adding composable template [{}] for [{}], because it already exists at version [{}]",
templateName, getOrigin(), currentTemplate.version());
}
} else {
logger.trace("skipping the creation of composable template [{}] for [{}], because its creation is in progress",
templateName, getOrigin());
}
}
}
/**
* Returns true if the cluster state contains all of the component templates needed by the composable template
*/
private static boolean componentTemplatesExist(ClusterState state, IndexTemplateConfig composableTemplate) {
final ComposableIndexTemplate indexTemplate;
try {
indexTemplate = ComposableIndexTemplate.parse(JsonXContent.jsonXContent.createParser(NamedXContentRegistry.EMPTY,
DeprecationHandler.THROW_UNSUPPORTED_OPERATION, composableTemplate.loadBytes()));
} catch (Exception e) {
throw new ElasticsearchParseException("unable to parse composable template " + composableTemplate.getTemplateName(), e);
}
Set<String> neededComponents = new HashSet<>(indexTemplate.composedOf());
return state.metadata().componentTemplates().keySet().containsAll(neededComponents);
}
private void putLegacyTemplate(final IndexTemplateConfig config, final AtomicBoolean creationCheck) {
final Executor executor = threadPool.generic();
executor.execute(() -> {
final String templateName = config.getTemplateName();
@ -187,7 +303,7 @@ public abstract class IndexTemplateRegistry implements ClusterStateListener {
public void onResponse(AcknowledgedResponse response) {
creationCheck.set(false);
if (response.isAcknowledged() == false) {
logger.error("error adding index template [{}] for [{}], request was not acknowledged",
logger.error("error adding legacy template [{}] for [{}], request was not acknowledged",
templateName, getOrigin());
}
}
@ -201,6 +317,72 @@ public abstract class IndexTemplateRegistry implements ClusterStateListener {
});
}
private void putComponentTemplate(final IndexTemplateConfig config, final AtomicBoolean creationCheck) {
final Executor executor = threadPool.generic();
executor.execute(() -> {
final String templateName = config.getTemplateName();
PutComponentTemplateAction.Request request = new PutComponentTemplateAction.Request(templateName);
try {
request.componentTemplate(ComponentTemplate.parse(JsonXContent.jsonXContent.createParser(NamedXContentRegistry.EMPTY,
DeprecationHandler.THROW_UNSUPPORTED_OPERATION, config.loadBytes())));
} catch (Exception e) {
throw new ElasticsearchParseException("unable to parse component template " + config.getTemplateName(), e);
}
request.masterNodeTimeout(TimeValue.timeValueMinutes(1));
executeAsyncWithOrigin(client.threadPool().getThreadContext(), getOrigin(), request,
new ActionListener<AcknowledgedResponse>() {
@Override
public void onResponse(AcknowledgedResponse response) {
creationCheck.set(false);
if (response.isAcknowledged() == false) {
logger.error("error adding component template [{}] for [{}], request was not acknowledged",
templateName, getOrigin());
}
}
@Override
public void onFailure(Exception e) {
creationCheck.set(false);
onPutTemplateFailure(config, e);
}
}, (req, listener) -> client.execute(PutComponentTemplateAction.INSTANCE, req, listener));
});
}
private void putComposableTemplate(final IndexTemplateConfig config, final AtomicBoolean creationCheck) {
final Executor executor = threadPool.generic();
executor.execute(() -> {
final String templateName = config.getTemplateName();
PutComposableIndexTemplateAction.Request request = new PutComposableIndexTemplateAction.Request(templateName);
try {
request.indexTemplate(ComposableIndexTemplate.parse(JsonXContent.jsonXContent.createParser(NamedXContentRegistry.EMPTY,
DeprecationHandler.THROW_UNSUPPORTED_OPERATION, config.loadBytes())));
} catch (Exception e) {
throw new ElasticsearchParseException("unable to parse composable template " + config.getTemplateName(), e);
}
request.masterNodeTimeout(TimeValue.timeValueMinutes(1));
executeAsyncWithOrigin(client.threadPool().getThreadContext(), getOrigin(), request,
new ActionListener<AcknowledgedResponse>() {
@Override
public void onResponse(AcknowledgedResponse response) {
creationCheck.set(false);
if (response.isAcknowledged() == false) {
logger.error("error adding composable template [{}] for [{}], request was not acknowledged",
templateName, getOrigin());
}
}
@Override
public void onFailure(Exception e) {
creationCheck.set(false);
onPutTemplateFailure(config, e);
}
}, (req, listener) -> client.execute(PutComposableIndexTemplateAction.INSTANCE, req, listener));
});
}
private void addIndexLifecyclePoliciesIfMissing(ClusterState state) {
Optional<IndexLifecycleMetadata> maybeMeta = Optional.ofNullable(state.metadata().custom(IndexLifecycleMetadata.TYPE));

60
x-pack/plugin/core/src/main/resources/logs-mappings.json Normal file
View File

@ -0,0 +1,60 @@
{
"template": {
"mappings": {
"dynamic_templates": [
{
"strings_as_keyword": {
"mapping": {
"ignore_above": 1024,
"type": "keyword"
},
"match_mapping_type": "string"
}
}
],
"date_detection": false,
"properties": {
"@timestamp": {
"type": "date"
},
"dataset": {
"properties": {
"type": {
"type": "constant_keyword",
"value": "logs"
},
"name": {
"type": "constant_keyword"
},
"namespace": {
"type": "constant_keyword"
}
}
},
"ecs": {
"properties": {
"version": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"host": {
"properties": {
"ip": {
"type": "ip"
}
}
},
"message": {
"type": "text"
}
}
}
},
"_meta": {
"description": "default mappings for the logs index template installed by x-pack",
"managed": true
},
"version": ${xpack.stack.template.version}
}

12
x-pack/plugin/core/src/main/resources/logs-policy.json Normal file
View File

@ -0,0 +1,12 @@
{
"phases": {
"hot": {
"actions": {
"rollover": {
"max_size": "50gb",
"max_age": "30d"
}
}
}
}
}

20
x-pack/plugin/core/src/main/resources/logs-settings.json Normal file
View File

@ -0,0 +1,20 @@
{
"template": {
"settings": {
"index": {
"lifecycle": {
"name": "logs"
},
"codec": "best_compression",
"query": {
"default_field": ["message"]
}
}
}
},
"_meta": {
"description": "default settings for the logs index template installed by x-pack",
"managed": true
},
"version": ${xpack.stack.template.version}
}

16
x-pack/plugin/core/src/main/resources/logs-template.json Normal file
View File

@ -0,0 +1,16 @@
{
"index_patterns": ["logs-*-*"],
"priority": 100,
"data_stream": {
"timestamp_field": "@timestamp"
},
"composed_of": [
"logs-mappings",
"logs-settings"
],
"_meta": {
"description": "default logs template installed by x-pack",
"managed": true
},
"version": ${xpack.stack.template.version}
}

57
x-pack/plugin/core/src/main/resources/metrics-mappings.json Normal file
View File

@ -0,0 +1,57 @@
{
"template": {
"mappings": {
"dynamic_templates": [
{
"strings_as_keyword": {
"mapping": {
"ignore_above": 1024,
"type": "keyword"
},
"match_mapping_type": "string"
}
}
],
"date_detection": false,
"properties": {
"@timestamp": {
"type": "date"
},
"dataset": {
"properties": {
"type": {
"type": "constant_keyword",
"value": "metrics"
},
"name": {
"type": "constant_keyword"
},
"namespace": {
"type": "constant_keyword"
}
}
},
"ecs": {
"properties": {
"version": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"host": {
"properties": {
"ip": {
"type": "ip"
}
}
}
}
}
},
"_meta": {
"description": "default mappings for the metrics index template installed by x-pack",
"managed": true
},
"version": ${xpack.stack.template.version}
}

12
x-pack/plugin/core/src/main/resources/metrics-policy.json Normal file
View File

@ -0,0 +1,12 @@
{
"phases": {
"hot": {
"actions": {
"rollover": {
"max_size": "50gb",
"max_age": "30d"
}
}
}
}
}

20
x-pack/plugin/core/src/main/resources/metrics-settings.json Normal file
View File

@ -0,0 +1,20 @@
{
"template": {
"settings": {
"index": {
"lifecycle": {
"name": "metrics"
},
"codec": "best_compression",
"query": {
"default_field": ["message"]
}
}
}
},
"_meta": {
"description": "default settings for the metrics index template installed by x-pack",
"managed": true
},
"version": ${xpack.stack.template.version}
}

16
x-pack/plugin/core/src/main/resources/metrics-template.json Normal file
View File

@ -0,0 +1,16 @@
{
"index_patterns": ["metrics-*-*"],
"priority": 100,
"data_stream": {
"timestamp_field": "@timestamp"
},
"composed_of": [
"metrics-mappings",
"metrics-settings"
],
"_meta": {
"description": "default metrics template installed by x-pack",
"managed": true
},
"version": ${xpack.stack.template.version}
}

8
x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/slm/history/SnapshotLifecycleTemplateRegistryTests.java
View File

@ -105,7 +105,7 @@ public class SnapshotLifecycleTemplateRegistryTests extends ESTestCase {
Settings settings = Settings.builder().put(SLM_HISTORY_INDEX_ENABLED_SETTING.getKey(), false).build();
SnapshotLifecycleTemplateRegistry disabledRegistry = new SnapshotLifecycleTemplateRegistry(settings, clusterService, threadPool,
client, xContentRegistry);
assertThat(disabledRegistry.getTemplateConfigs(), hasSize(0));
assertThat(disabledRegistry.getLegacyTemplateConfigs(), hasSize(0));
assertThat(disabledRegistry.getPolicyConfigs(), hasSize(0));
}
@ -119,7 +119,7 @@ public class SnapshotLifecycleTemplateRegistryTests extends ESTestCase {
AtomicInteger calledTimes = new AtomicInteger(0);
client.setVerifier((action, request, listener) -> verifyTemplateInstalled(calledTimes, action, request, listener));
registry.clusterChanged(event);
assertBusy(() -> assertThat(calledTimes.get(), equalTo(registry.getTemplateConfigs().size())));
assertBusy(() -> assertThat(calledTimes.get(), equalTo(registry.getLegacyTemplateConfigs().size())));
calledTimes.set(0);
@ -233,7 +233,7 @@ public class SnapshotLifecycleTemplateRegistryTests extends ESTestCase {
AtomicInteger calledTimes = new AtomicInteger(0);
client.setVerifier((action, request, listener) -> verifyTemplateInstalled(calledTimes, action, request, listener));
registry.clusterChanged(event);
assertBusy(() -> assertThat(calledTimes.get(), equalTo(registry.getTemplateConfigs().size())));
assertBusy(() -> assertThat(calledTimes.get(), equalTo(registry.getLegacyTemplateConfigs().size())));
}
public void testThatUnversionedOldTemplatesAreUpgraded() throws Exception {
@ -244,7 +244,7 @@ public class SnapshotLifecycleTemplateRegistryTests extends ESTestCase {
AtomicInteger calledTimes = new AtomicInteger(0);
client.setVerifier((action, request, listener) -> verifyTemplateInstalled(calledTimes, action, request, listener));
registry.clusterChanged(event);
assertBusy(() -> assertThat(calledTimes.get(), equalTo(registry.getTemplateConfigs().size())));
assertBusy(() -> assertThat(calledTimes.get(), equalTo(registry.getLegacyTemplateConfigs().size())));
}

11
x-pack/plugin/ilm/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ilm/CCRIndexLifecycleIT.java
View File

@ -182,9 +182,9 @@ public class CCRIndexLifecycleIT extends ESCCRRestTestCase {
}
public void testCcrAndIlmWithRollover() throws Exception {
String alias = "metrics";
String indexName = "metrics-000001";
String nextIndexName = "metrics-000002";
String alias = "mymetrics";
String indexName = "mymetrics-000001";
String nextIndexName = "mymetrics-000002";
String policyName = "rollover-test";
if ("leader".equals(targetCluster)) {
@ -197,7 +197,8 @@ public class CCRIndexLifecycleIT extends ESCCRRestTestCase {
.put("index.lifecycle.name", policyName)
.put("index.lifecycle.rollover_alias", alias)
.build();
templateRequest.setJsonEntity("{\"index_patterns\": [\"metrics-*\"], \"settings\": " + Strings.toString(indexSettings) + "}");
templateRequest.setJsonEntity("{\"index_patterns\": [\"mymetrics-*\"], \"settings\": " +
Strings.toString(indexSettings) + "}");
assertOK(client().performRequest(templateRequest));
} else if ("follow".equals(targetCluster)) {
// Policy with the same name must exist in follower cluster too:
@ -205,7 +206,7 @@ public class CCRIndexLifecycleIT extends ESCCRRestTestCase {
// Set up an auto-follow pattern
Request createAutoFollowRequest = new Request("PUT", "/_ccr/auto_follow/my_auto_follow_pattern");
createAutoFollowRequest.setJsonEntity("{\"leader_index_patterns\": [\"metrics-*\"], " +
createAutoFollowRequest.setJsonEntity("{\"leader_index_patterns\": [\"mymetrics-*\"], " +
"\"remote_cluster\": \"leader_cluster\", \"read_poll_timeout\": \"1000ms\"}");
assertOK(client().performRequest(createAutoFollowRequest));

2
x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycle.java
View File

@ -139,7 +139,7 @@ public class IndexLifecycle extends Plugin implements ActionPlugin {
private final SetOnce<SnapshotLifecycleService> snapshotLifecycleService = new SetOnce<>();
private final SetOnce<SnapshotRetentionService> snapshotRetentionService = new SetOnce<>();
private final SetOnce<SnapshotHistoryStore> snapshotHistoryStore = new SetOnce<>();
private Settings settings;
private final Settings settings;
private boolean transportClientMode;
public IndexLifecycle(Settings settings) {

2
x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/history/ILMHistoryTemplateRegistry.java
View File

@ -62,7 +62,7 @@ public class ILMHistoryTemplateRegistry extends IndexTemplateRegistry {
}
@Override
protected List<IndexTemplateConfig> getTemplateConfigs() {
protected List<IndexTemplateConfig> getLegacyTemplateConfigs() {
if (this.ilmHistoryEnabled) {
return Collections.singletonList(TEMPLATE_ILM_HISTORY);
} else {

2
x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlIndexTemplateRegistry.java
View File

@ -132,7 +132,7 @@ public class MlIndexTemplateRegistry extends IndexTemplateRegistry {
}
@Override
protected List<IndexTemplateConfig> getTemplateConfigs() {
protected List<IndexTemplateConfig> getLegacyTemplateConfigs() {
return templatesToUse;
}

2
x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationUtils.java
View File

@ -23,6 +23,7 @@ import static org.elasticsearch.action.admin.cluster.node.tasks.get.GetTaskActio
import static org.elasticsearch.xpack.core.ClientHelper.ASYNC_SEARCH_ORIGIN;
import static org.elasticsearch.xpack.core.ClientHelper.ENRICH_ORIGIN;
import static org.elasticsearch.xpack.core.ClientHelper.IDP_ORIGIN;
import static org.elasticsearch.xpack.core.ClientHelper.STACK_ORIGIN;
import static org.elasticsearch.xpack.core.ClientHelper.TRANSFORM_ORIGIN;
import static org.elasticsearch.xpack.core.ClientHelper.DEPRECATION_ORIGIN;
import static org.elasticsearch.xpack.core.ClientHelper.INDEX_LIFECYCLE_ORIGIN;
@ -117,6 +118,7 @@ public final class AuthorizationUtils {
case INDEX_LIFECYCLE_ORIGIN:
case ENRICH_ORIGIN:
case IDP_ORIGIN:
case STACK_ORIGIN:
case TASKS_ORIGIN: // TODO use a more limited user for tasks
securityContext.executeAsUser(XPackUser.INSTANCE, consumer, Version.CURRENT);
break;

29
x-pack/plugin/stack/build.gradle Normal file
View File

@ -0,0 +1,29 @@
evaluationDependsOn(xpackModule('core'))
apply plugin: 'elasticsearch.esplugin'
esplugin {
name 'x-pack-stack'
description 'Elasticsearch Expanded Pack Plugin - Stack'
classname 'org.elasticsearch.xpack.stack.StackPlugin'
extendedPlugins = ['x-pack-core']
hasNativeController false
requiresKeystore true
}
archivesBaseName = 'x-pack-stack'
dependencies {
compileOnly project(path: xpackModule('core'), configuration: 'default')
testImplementation project(path: xpackModule('core'), configuration: 'testArtifacts')
}
// add all sub-projects of the qa sub-project
gradle.projectsEvaluated {
project.subprojects
.find { it.path == project.path + ":qa" }
.subprojects
.findAll { it.path.startsWith(project.path + ":qa") }
.each { check.dependsOn it.check }
}
integTest.enabled = false

0
x-pack/plugin/stack/qa/build.gradle Normal file
View File

38
x-pack/plugin/stack/qa/rest/build.gradle Normal file
View File

@ -0,0 +1,38 @@
import org.elasticsearch.gradle.test.RestIntegTestTask
apply plugin: 'elasticsearch.testclusters'
apply plugin: 'elasticsearch.standalone-test'
dependencies {
testImplementation project(path: xpackModule('core'), configuration: 'testArtifacts')
testImplementation project(path: xpackModule('stack'), configuration: 'runtime')
}
restResources {
restApi {
includeCore '_common', 'cluster', 'indices', 'index', 'snapshot'
includeXpack 'ilm', 'slm', 'stack'
}
}
def clusterCredentials = [username: System.getProperty('tests.rest.cluster.username', 'test_admin'),
password: System.getProperty('tests.rest.cluster.password', 'x-pack-test-password')]
task restTest(type: RestIntegTestTask) {
mustRunAfter(precommit)
runner {
systemProperty 'tests.rest.cluster.username', clusterCredentials.username
systemProperty 'tests.rest.cluster.password', clusterCredentials.password
}
}
testClusters.restTest {
testDistribution = 'DEFAULT'
setting 'xpack.ml.enabled', 'false'
setting 'xpack.security.enabled', 'true'
setting 'xpack.license.self_generated.type', 'trial'
user clusterCredentials
}
check.dependsOn restTest
test.enabled = false

43
x-pack/plugin/stack/qa/rest/src/test/java/org/elasticsearch/xpack/stack/StackYamlIT.java Normal file
View File

@ -0,0 +1,43 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.stack;
import com.carrotsearch.randomizedtesting.annotations.Name;
import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite;
import org.apache.lucene.util.TimeUnits;
import org.elasticsearch.common.settings.SecureString;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.concurrent.ThreadContext;
import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
import java.util.Objects;
import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
@TimeoutSuite(millis = 30 * TimeUnits.MINUTE) // as default timeout seems not enough on the jenkins VMs
public class StackYamlIT extends ESClientYamlSuiteTestCase {
private static final String USER = Objects.requireNonNull(System.getProperty("tests.rest.cluster.username"));
private static final String PASS = Objects.requireNonNull(System.getProperty("tests.rest.cluster.password"));
public StackYamlIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
super(testCandidate);
}
@ParametersFactory
public static Iterable<Object[]> parameters() throws Exception {
return ESClientYamlSuiteTestCase.createParameters();
}
@Override
protected Settings restClientSettings() {
String token = basicAuthHeaderValue(USER, new SecureString(PASS.toCharArray()));
return Settings.builder().put(super.restClientSettings()).put(ThreadContext.PREFIX + ".Authorization", token).build();
}
}

105
x-pack/plugin/stack/qa/rest/src/test/resources/rest-api-spec/test/stack/10_basic.yml Normal file
View File

@ -0,0 +1,105 @@
---
setup:
- do:
cluster.health:
wait_for_status: yellow
---
"Test stack template installation":
- do:
ilm.get_lifecycle:
policy: "logs"
- do:
ilm.get_lifecycle:
policy: "metrics"
- do:
cluster.get_component_template:
name: logs-mappings
- do:
cluster.get_component_template:
name: logs-settings
- do:
cluster.get_component_template:
name: metrics-mappings
- do:
cluster.get_component_template:
name: metrics-settings
- do:
indices.get_index_template:
name: logs
- do:
indices.get_index_template:
name: metrics
---
"Test logs index auto creation":
- do:
index:
index: logs-foo-bar
body:
"@timestamp": "2020-01-01"
message: "test-log-message"
- do:
indices.get_data_stream:
name: logs-foo-bar
- match: { 0.name: logs-foo-bar }
- match: { 0.timestamp_field.name: '@timestamp' }
- match: { 0.generation: 1 }
- length: { 0.indices: 1 }
- match: { 0.indices.0.index_name: '.ds-logs-foo-bar-000001' }
- do:
indices.get:
index: .ds-logs-foo-bar-000001
- is_true: \.ds-logs-foo-bar-000001.settings
- is_true: \.ds-logs-foo-bar-000001.mappings
- match: { \.ds-logs-foo-bar-000001.settings.index.lifecycle.name: "logs" }
- is_true: \.ds-logs-foo-bar-000001.mappings.properties.message
- match: { \.ds-logs-foo-bar-000001.data_stream: "logs-foo-bar" }
- do:
indices.delete_data_stream:
name: logs-foo-bar
---
"Test metrics index auto creation":
- do:
index:
index: metrics-foo-bar
body:
"@timestamp": "2020-01-01"
message: "test-log-message"
- do:
indices.get_data_stream:
name: metrics-foo-bar
- match: { 0.name: metrics-foo-bar }
- match: { 0.timestamp_field.name: '@timestamp' }
- match: { 0.generation: 1 }
- length: { 0.indices: 1 }
- match: { 0.indices.0.index_name: '.ds-metrics-foo-bar-000001' }
- do:
indices.get:
index: .ds-metrics-foo-bar-000001
- is_true: \.ds-metrics-foo-bar-000001.settings
- is_true: \.ds-metrics-foo-bar-000001.mappings
- match: { \.ds-metrics-foo-bar-000001.settings.index.lifecycle.name: "metrics" }
- is_true: \.ds-metrics-foo-bar-000001.mappings.properties.message
- match: { \.ds-metrics-foo-bar-000001.data_stream: "metrics-foo-bar" }
- do:
indices.delete_data_stream:
name: metrics-foo-bar

66
x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackPlugin.java Normal file
View File

@ -0,0 +1,66 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.stack;
import org.elasticsearch.client.Client;
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
import org.elasticsearch.env.Environment;
import org.elasticsearch.env.NodeEnvironment;
import org.elasticsearch.plugins.ActionPlugin;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.repositories.RepositoriesService;
import org.elasticsearch.script.ScriptService;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.watcher.ResourceWatcherService;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.function.Supplier;
public class StackPlugin extends Plugin implements ActionPlugin {
private final Settings settings;
public static final Setting<Boolean> STACK_TEMPLATES_ENABLED = Setting.boolSetting(
"stack.templates.enabled",
true,
Setting.Property.NodeScope
);
public StackPlugin(Settings settings) {
this.settings = settings;
}
@Override
public List<Setting<?>> getSettings() {
return Collections.singletonList(STACK_TEMPLATES_ENABLED);
}
@Override
public Collection<Object> createComponents(
Client client,
ClusterService clusterService,
ThreadPool threadPool,
ResourceWatcherService resourceWatcherService,
ScriptService scriptService,
NamedXContentRegistry xContentRegistry,
Environment environment,
NodeEnvironment nodeEnvironment,
NamedWriteableRegistry namedWriteableRegistry,
IndexNameExpressionResolver indexNameExpressionResolver,
Supplier<RepositoriesService> repositoriesServiceSupplier
) {
final List<Object> components = new ArrayList<>();
StackTemplateRegistry templateRegistry = new StackTemplateRegistry(settings, clusterService, threadPool, client, xContentRegistry);
return components;
}
}

140
x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java Normal file
View File

@ -0,0 +1,140 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.stack;
import org.elasticsearch.client.Client;
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.xpack.core.ClientHelper;
import org.elasticsearch.xpack.core.template.IndexTemplateConfig;
import org.elasticsearch.xpack.core.template.IndexTemplateRegistry;
import org.elasticsearch.xpack.core.template.LifecyclePolicyConfig;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
public class StackTemplateRegistry extends IndexTemplateRegistry {
// The stack template registry should remain at version 0. This is because templates and
// policies will be changed by the ingest manager once they exist, and ES should only ever put
// the template in place if it does not exist. If this were incremented we could accidentally
// overwrite a template or policy changed by the ingest manager.
public static final int REGISTRY_VERSION = 0;
public static final String TEMPLATE_VERSION_VARIABLE = "xpack.stack.template.version";
private final boolean stackTemplateEnabled;
//////////////////////////////////////////////////////////
// Logs components (for matching logs-*-* indices)
//////////////////////////////////////////////////////////
public static final String LOGS_MAPPINGS_COMPONENT_TEMPLATE_NAME = "logs-mappings";
public static final String LOGS_SETTINGS_COMPONENT_TEMPLATE_NAME = "logs-settings";
public static final String LOGS_ILM_POLICY_NAME = "logs";
public static final String LOGS_INDEX_TEMPLATE_NAME = "logs";
public static final IndexTemplateConfig LOGS_MAPPINGS_COMPONENT_TEMPLATE = new IndexTemplateConfig(
LOGS_MAPPINGS_COMPONENT_TEMPLATE_NAME,
"/logs-mappings.json",
REGISTRY_VERSION,
TEMPLATE_VERSION_VARIABLE
);
public static final IndexTemplateConfig LOGS_SETTINGS_COMPONENT_TEMPLATE = new IndexTemplateConfig(
LOGS_SETTINGS_COMPONENT_TEMPLATE_NAME,
"/logs-settings.json",
REGISTRY_VERSION,
TEMPLATE_VERSION_VARIABLE
);
public static final LifecyclePolicyConfig LOGS_ILM_POLICY = new LifecyclePolicyConfig(LOGS_ILM_POLICY_NAME, "/logs-policy.json");
public static final IndexTemplateConfig LOGS_INDEX_TEMPLATE = new IndexTemplateConfig(
LOGS_INDEX_TEMPLATE_NAME,
"/logs-template.json",
REGISTRY_VERSION,
TEMPLATE_VERSION_VARIABLE
);
//////////////////////////////////////////////////////////
// Metrics components (for matching metric-*-* indices)
//////////////////////////////////////////////////////////
public static final String METRICS_MAPPINGS_COMPONENT_TEMPLATE_NAME = "metrics-mappings";
public static final String METRICS_SETTINGS_COMPONENT_TEMPLATE_NAME = "metrics-settings";
public static final String METRICS_ILM_POLICY_NAME = "metrics";
public static final String METRICS_INDEX_TEMPLATE_NAME = "metrics";
public static final IndexTemplateConfig METRICS_MAPPINGS_COMPONENT_TEMPLATE = new IndexTemplateConfig(
METRICS_MAPPINGS_COMPONENT_TEMPLATE_NAME,
"/metrics-mappings.json",
REGISTRY_VERSION,
TEMPLATE_VERSION_VARIABLE
);
public static final IndexTemplateConfig METRICS_SETTINGS_COMPONENT_TEMPLATE = new IndexTemplateConfig(
METRICS_SETTINGS_COMPONENT_TEMPLATE_NAME,
"/metrics-settings.json",
REGISTRY_VERSION,
TEMPLATE_VERSION_VARIABLE
);
public static final LifecyclePolicyConfig METRICS_ILM_POLICY = new LifecyclePolicyConfig(
METRICS_ILM_POLICY_NAME,
"/metrics-policy.json"
);
public static final IndexTemplateConfig METRICS_INDEX_TEMPLATE = new IndexTemplateConfig(
METRICS_INDEX_TEMPLATE_NAME,
"/metrics-template.json",
REGISTRY_VERSION,
TEMPLATE_VERSION_VARIABLE
);
public StackTemplateRegistry(
Settings nodeSettings,
ClusterService clusterService,
ThreadPool threadPool,
Client client,
NamedXContentRegistry xContentRegistry
) {
super(nodeSettings, clusterService, threadPool, client, xContentRegistry);
this.stackTemplateEnabled = StackPlugin.STACK_TEMPLATES_ENABLED.get(nodeSettings);
}
@Override
protected List<LifecyclePolicyConfig> getPolicyConfigs() {
if (stackTemplateEnabled) {
return Arrays.asList(LOGS_ILM_POLICY, METRICS_ILM_POLICY);
} else {
return Collections.emptyList();
}
}
@Override
protected List<IndexTemplateConfig> getComponentTemplateConfigs() {
if (stackTemplateEnabled) {
return Arrays.asList(
LOGS_MAPPINGS_COMPONENT_TEMPLATE,
LOGS_SETTINGS_COMPONENT_TEMPLATE,
METRICS_MAPPINGS_COMPONENT_TEMPLATE,
METRICS_SETTINGS_COMPONENT_TEMPLATE
);
} else {
return Collections.emptyList();
}
}
@Override
protected List<IndexTemplateConfig> getComposableTemplateConfigs() {
if (stackTemplateEnabled) {
return Arrays.asList(LOGS_INDEX_TEMPLATE, METRICS_INDEX_TEMPLATE);
} else {
return Collections.emptyList();
}
}
@Override
protected String getOrigin() {
return ClientHelper.STACK_ORIGIN;
}
}

453
x-pack/plugin/stack/src/test/java/org/elasticsearch/xpack/stack/StackTemplateRegistryTests.java Normal file
View File

@ -0,0 +1,453 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.stack;
import org.elasticsearch.Version;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.ActionRequest;
import org.elasticsearch.action.ActionResponse;
import org.elasticsearch.action.ActionType;
import org.elasticsearch.action.admin.indices.template.put.PutComponentTemplateAction;
import org.elasticsearch.action.admin.indices.template.put.PutComposableIndexTemplateAction;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.cluster.ClusterChangedEvent;
import org.elasticsearch.cluster.ClusterModule;
import org.elasticsearch.cluster.ClusterName;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.block.ClusterBlocks;
import org.elasticsearch.cluster.metadata.ComponentTemplate;
import org.elasticsearch.cluster.metadata.Metadata;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.cluster.node.DiscoveryNodes;
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.TriFunction;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.test.ClusterServiceUtils;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.test.client.NoOpClient;
import org.elasticsearch.test.junit.annotations.TestLogging;
import org.elasticsearch.threadpool.TestThreadPool;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.xpack.core.ilm.DeleteAction;
import org.elasticsearch.xpack.core.ilm.IndexLifecycleMetadata;
import org.elasticsearch.xpack.core.ilm.LifecycleAction;
import org.elasticsearch.xpack.core.ilm.LifecyclePolicy;
import org.elasticsearch.xpack.core.ilm.LifecyclePolicyMetadata;
import org.elasticsearch.xpack.core.ilm.LifecycleType;
import org.elasticsearch.xpack.core.ilm.OperationMode;
import org.elasticsearch.xpack.core.ilm.RolloverAction;
import org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleType;
import org.elasticsearch.xpack.core.ilm.action.PutLifecycleAction;
import org.junit.After;
import org.junit.Before;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
import static org.elasticsearch.mock.orig.Mockito.when;
import static org.hamcrest.Matchers.anyOf;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.greaterThan;
import static org.hamcrest.Matchers.hasSize;
import static org.hamcrest.Matchers.instanceOf;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy;
public class StackTemplateRegistryTests extends ESTestCase {
private StackTemplateRegistry registry;
private NamedXContentRegistry xContentRegistry;
private ClusterService clusterService;
private ThreadPool threadPool;
private VerifyingClient client;
@Before
public void createRegistryAndClient() {
threadPool = new TestThreadPool(this.getClass().getName());
client = new VerifyingClient(threadPool);
clusterService = ClusterServiceUtils.createClusterService(threadPool);
List<NamedXContentRegistry.Entry> entries = new ArrayList<>(ClusterModule.getNamedXWriteables());
entries.addAll(
Arrays.asList(
new NamedXContentRegistry.Entry(
LifecycleType.class,
new ParseField(TimeseriesLifecycleType.TYPE),
(p) -> TimeseriesLifecycleType.INSTANCE
),
new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(RolloverAction.NAME), RolloverAction::parse),
new NamedXContentRegistry.Entry(LifecycleAction.class, new ParseField(DeleteAction.NAME), DeleteAction::parse)
)
);
xContentRegistry = new NamedXContentRegistry(entries);
registry = new StackTemplateRegistry(Settings.EMPTY, clusterService, threadPool, client, xContentRegistry);
}
@After
@Override
public void tearDown() throws Exception {
super.tearDown();
threadPool.shutdownNow();
}
public void testDisabledDoesNotAddTemplates() {
Settings settings = Settings.builder().put(StackPlugin.STACK_TEMPLATES_ENABLED.getKey(), false).build();
StackTemplateRegistry disabledRegistry = new StackTemplateRegistry(settings, clusterService, threadPool, client, xContentRegistry);
assertThat(disabledRegistry.getComponentTemplateConfigs(), hasSize(0));
assertThat(disabledRegistry.getComposableTemplateConfigs(), hasSize(0));
assertThat(disabledRegistry.getPolicyConfigs(), hasSize(0));
}
public void testThatNonExistingTemplatesAreAddedImmediately() throws Exception {
DiscoveryNode node = new DiscoveryNode("node", ESTestCase.buildNewFakeTransportAddress(), Version.CURRENT);
DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build();
ClusterChangedEvent event = createClusterChangedEvent(Collections.emptyMap(), nodes);
AtomicInteger calledTimes = new AtomicInteger(0);
client.setVerifier((action, request, listener) -> verifyComponentTemplateInstalled(calledTimes, action, request, listener));
registry.clusterChanged(event);
assertBusy(() -> assertThat(calledTimes.get(), equalTo(registry.getComponentTemplateConfigs().size())));
calledTimes.set(0);
// attempting to register the event multiple times as a race condition can yield this test flaky, namely:
// when calling registry.clusterChanged(newEvent) the templateCreationsInProgress state that the IndexTemplateRegistry maintains
// might've not yet been updated to reflect that the first template registration was complete, so a second template registration
// will not be issued anymore, leaving calledTimes to 0
assertBusy(() -> {
// now delete one template from the cluster state and lets retry
ClusterChangedEvent newEvent = createClusterChangedEvent(Collections.emptyMap(), nodes);
registry.clusterChanged(newEvent);
assertThat(calledTimes.get(), greaterThan(1));
});
}
public void testThatNonExistingPoliciesAreAddedImmediately() throws Exception {
DiscoveryNode node = new DiscoveryNode("node", ESTestCase.buildNewFakeTransportAddress(), Version.CURRENT);
DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build();
AtomicInteger calledTimes = new AtomicInteger(0);
client.setVerifier((action, request, listener) -> {
if (action instanceof PutLifecycleAction) {
calledTimes.incrementAndGet();
assertThat(action, instanceOf(PutLifecycleAction.class));
assertThat(request, instanceOf(PutLifecycleAction.Request.class));
final PutLifecycleAction.Request putRequest = (PutLifecycleAction.Request) request;
assertThat(
putRequest.getPolicy().getName(),
anyOf(equalTo(StackTemplateRegistry.LOGS_ILM_POLICY_NAME), equalTo(StackTemplateRegistry.METRICS_ILM_POLICY_NAME))
);
assertNotNull(listener);
return new PutLifecycleAction.Response(true);
} else if (action instanceof PutComponentTemplateAction) {
// Ignore this, it's verified in another test
return new StackTemplateRegistryTests.TestPutIndexTemplateResponse(true);
} else if (action instanceof PutComposableIndexTemplateAction) {
// Ignore this, it's verified in another test
return new AcknowledgedResponse(true);
} else {
fail("client called with unexpected request: " + request.toString());
return null;
}
});
ClusterChangedEvent event = createClusterChangedEvent(Collections.emptyMap(), nodes);
registry.clusterChanged(event);
assertBusy(() -> assertThat(calledTimes.get(), equalTo(2)));
}
public void testPolicyAlreadyExists() {
DiscoveryNode node = new DiscoveryNode("node", ESTestCase.buildNewFakeTransportAddress(), Version.CURRENT);
DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build();
Map<String, LifecyclePolicy> policyMap = new HashMap<>();
List<LifecyclePolicy> policies = registry.getPolicyConfigs()
.stream()
.map(policyConfig -> policyConfig.load(xContentRegistry))
.collect(Collectors.toList());
assertThat(policies, hasSize(2));
policies.forEach(p -> policyMap.put(p.getName(), p.get()));
client.setVerifier((action, request, listener) -> {
if (action instanceof PutComponentTemplateAction) {
// Ignore this, it's verified in another test
return new AcknowledgedResponse(true);
} else if (action instanceof PutLifecycleAction) {
fail("if the policy already exists it should be re-put");
} else {
fail("client called with unexpected request: " + request.toString());
}
return null;
});
ClusterChangedEvent event = createClusterChangedEvent(Collections.emptyMap(), policyMap, nodes);
registry.clusterChanged(event);
}
public void testPolicyAlreadyExistsButDiffers() throws IOException {
DiscoveryNode node = new DiscoveryNode("node", ESTestCase.buildNewFakeTransportAddress(), Version.CURRENT);
DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build();
Map<String, LifecyclePolicy> policyMap = new HashMap<>();
String policyStr = "{\"phases\":{\"delete\":{\"min_age\":\"1m\",\"actions\":{\"delete\":{}}}}}";
List<LifecyclePolicy> policies = registry.getPolicyConfigs()
.stream()
.map(policyConfig -> policyConfig.load(xContentRegistry))
.collect(Collectors.toList());
assertThat(policies, hasSize(2));
policies.forEach(p -> policyMap.put(p.getName(), p.get()));
client.setVerifier((action, request, listener) -> {
if (action instanceof PutComponentTemplateAction) {
// Ignore this, it's verified in another test
return new AcknowledgedResponse(true);
} else if (action instanceof PutLifecycleAction) {
fail("if the policy already exists it should be re-put");
} else {
fail("client called with unexpected request: " + request.toString());
}
return null;
});
try (
XContentParser parser = XContentType.JSON.xContent()
.createParser(xContentRegistry, LoggingDeprecationHandler.THROW_UNSUPPORTED_OPERATION, policyStr)
) {
LifecyclePolicy different = LifecyclePolicy.parse(parser, policies.get(0).getName());
policyMap.put(policies.get(0).getName(), different);
ClusterChangedEvent event = createClusterChangedEvent(Collections.emptyMap(), policyMap, nodes);
registry.clusterChanged(event);
}
}
public void testThatVersionedOldTemplatesAreUpgraded() throws Exception {
DiscoveryNode node = new DiscoveryNode("node", ESTestCase.buildNewFakeTransportAddress(), Version.CURRENT);
DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build();
ClusterChangedEvent event = createClusterChangedEvent(
Collections.singletonMap(
StackTemplateRegistry.LOGS_SETTINGS_COMPONENT_TEMPLATE_NAME,
StackTemplateRegistry.REGISTRY_VERSION - 1
),
nodes
);
AtomicInteger calledTimes = new AtomicInteger(0);
client.setVerifier((action, request, listener) -> verifyComponentTemplateInstalled(calledTimes, action, request, listener));
registry.clusterChanged(event);
assertBusy(() -> assertThat(calledTimes.get(), equalTo(registry.getComponentTemplateConfigs().size())));
}
public void testThatUnversionedOldTemplatesAreUpgraded() throws Exception {
DiscoveryNode node = new DiscoveryNode("node", ESTestCase.buildNewFakeTransportAddress(), Version.CURRENT);
DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build();
ClusterChangedEvent event = createClusterChangedEvent(
Collections.singletonMap(StackTemplateRegistry.LOGS_SETTINGS_COMPONENT_TEMPLATE_NAME, null),
nodes
);
AtomicInteger calledTimes = new AtomicInteger(0);
client.setVerifier((action, request, listener) -> verifyComponentTemplateInstalled(calledTimes, action, request, listener));
registry.clusterChanged(event);
assertBusy(() -> assertThat(calledTimes.get(), equalTo(registry.getComponentTemplateConfigs().size())));
}
@TestLogging(value = "org.elasticsearch.xpack.core.template:DEBUG", reason = "test")
public void testSameOrHigherVersionTemplateNotUpgraded() {
DiscoveryNode node = new DiscoveryNode("node", ESTestCase.buildNewFakeTransportAddress(), Version.CURRENT);
DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build();
Map<String, Integer> versions = new HashMap<>();
versions.put(StackTemplateRegistry.LOGS_SETTINGS_COMPONENT_TEMPLATE_NAME, StackTemplateRegistry.REGISTRY_VERSION);
versions.put(StackTemplateRegistry.LOGS_MAPPINGS_COMPONENT_TEMPLATE_NAME, StackTemplateRegistry.REGISTRY_VERSION);
versions.put(StackTemplateRegistry.METRICS_SETTINGS_COMPONENT_TEMPLATE_NAME, StackTemplateRegistry.REGISTRY_VERSION);
versions.put(StackTemplateRegistry.METRICS_MAPPINGS_COMPONENT_TEMPLATE_NAME, StackTemplateRegistry.REGISTRY_VERSION);
ClusterChangedEvent sameVersionEvent = createClusterChangedEvent(versions, nodes);
client.setVerifier((action, request, listener) -> {
if (action instanceof PutComponentTemplateAction) {
fail("template should not have been re-installed");
return null;
} else if (action instanceof PutLifecycleAction) {
// Ignore this, it's verified in another test
return new PutLifecycleAction.Response(true);
} else if (action instanceof PutComposableIndexTemplateAction) {
// Ignore this, it's verified in another test
return new AcknowledgedResponse(true);
} else {
fail("client called with unexpected request:" + request.toString());
return null;
}
});
registry.clusterChanged(sameVersionEvent);
versions.clear();
versions.put(
StackTemplateRegistry.LOGS_SETTINGS_COMPONENT_TEMPLATE_NAME,
StackTemplateRegistry.REGISTRY_VERSION + randomIntBetween(1, 1000)
);
versions.put(
StackTemplateRegistry.LOGS_MAPPINGS_COMPONENT_TEMPLATE_NAME,
StackTemplateRegistry.REGISTRY_VERSION + randomIntBetween(1, 1000)
);
versions.put(
StackTemplateRegistry.METRICS_SETTINGS_COMPONENT_TEMPLATE_NAME,
StackTemplateRegistry.REGISTRY_VERSION + randomIntBetween(1, 1000)
);
versions.put(
StackTemplateRegistry.METRICS_MAPPINGS_COMPONENT_TEMPLATE_NAME,
StackTemplateRegistry.REGISTRY_VERSION + randomIntBetween(1, 1000)
);
ClusterChangedEvent higherVersionEvent = createClusterChangedEvent(versions, nodes);
registry.clusterChanged(higherVersionEvent);
}
public void testThatMissingMasterNodeDoesNothing() {
DiscoveryNode localNode = new DiscoveryNode("node", ESTestCase.buildNewFakeTransportAddress(), Version.CURRENT);
DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").add(localNode).build();
client.setVerifier((a, r, l) -> {
fail("if the master is missing nothing should happen");
return null;
});
ClusterChangedEvent event = createClusterChangedEvent(
Collections.singletonMap(StackTemplateRegistry.LOGS_INDEX_TEMPLATE_NAME, null),
nodes
);
registry.clusterChanged(event);
}
// -------------
/**
* A client that delegates to a verifying function for action/request/listener
*/
public static class VerifyingClient extends NoOpClient {
private TriFunction<ActionType<?>, ActionRequest, ActionListener<?>, ActionResponse> verifier = (a, r, l) -> {
fail("verifier not set");
return null;
};
VerifyingClient(ThreadPool threadPool) {
super(threadPool);
}
@Override
@SuppressWarnings("unchecked")
protected <Request extends ActionRequest, Response extends ActionResponse> void doExecute(
ActionType<Response> action,
Request request,
ActionListener<Response> listener
) {
try {
listener.onResponse((Response) verifier.apply(action, request, listener));
} catch (Exception e) {
listener.onFailure(e);
}
}
public VerifyingClient setVerifier(TriFunction<ActionType<?>, ActionRequest, ActionListener<?>, ActionResponse> verifier) {
this.verifier = verifier;
return this;
}
}
private ActionResponse verifyComponentTemplateInstalled(
AtomicInteger calledTimes,
ActionType<?> action,
ActionRequest request,
ActionListener<?> listener
) {
if (action instanceof PutComponentTemplateAction) {
calledTimes.incrementAndGet();
assertThat(action, instanceOf(PutComponentTemplateAction.class));
assertThat(request, instanceOf(PutComponentTemplateAction.Request.class));
final PutComponentTemplateAction.Request putRequest = (PutComponentTemplateAction.Request) request;
assertThat(putRequest.componentTemplate().version(), equalTo((long) StackTemplateRegistry.REGISTRY_VERSION));
assertNotNull(listener);
return new TestPutIndexTemplateResponse(true);
} else if (action instanceof PutLifecycleAction) {
// Ignore this, it's verified in another test
return new PutLifecycleAction.Response(true);
} else if (action instanceof PutComposableIndexTemplateAction) {
// Ignore this, it's verified in another test
return new AcknowledgedResponse(true);
} else {
fail("client called with unexpected request:" + request.toString());
return null;
}
}
private ClusterChangedEvent createClusterChangedEvent(Map<String, Integer> existingTemplates, DiscoveryNodes nodes) {
return createClusterChangedEvent(existingTemplates, Collections.emptyMap(), nodes);
}
private ClusterChangedEvent createClusterChangedEvent(
Map<String, Integer> existingTemplates,
Map<String, LifecyclePolicy> existingPolicies,
DiscoveryNodes nodes
) {
ClusterState cs = createClusterState(Settings.EMPTY, existingTemplates, existingPolicies, nodes);
ClusterChangedEvent realEvent = new ClusterChangedEvent(
"created-from-test",
cs,
ClusterState.builder(new ClusterName("test")).build()
);
ClusterChangedEvent event = spy(realEvent);
when(event.localNodeMaster()).thenReturn(nodes.isLocalNodeElectedMaster());
return event;
}
private ClusterState createClusterState(
Settings nodeSettings,
Map<String, Integer> existingComponentTemplates,
Map<String, LifecyclePolicy> existingPolicies,
DiscoveryNodes nodes
) {
Map<String, ComponentTemplate> componentTemplates = new HashMap<>();
for (Map.Entry<String, Integer> template : existingComponentTemplates.entrySet()) {
ComponentTemplate mockTemplate = mock(ComponentTemplate.class);
when(mockTemplate.version()).thenReturn(template.getValue() == null ? null : (long) template.getValue());
componentTemplates.put(template.getKey(), mockTemplate);
}
Map<String, LifecyclePolicyMetadata> existingILMMeta = existingPolicies.entrySet()
.stream()
.collect(Collectors.toMap(Map.Entry::getKey, e -> new LifecyclePolicyMetadata(e.getValue(), Collections.emptyMap(), 1, 1)));
IndexLifecycleMetadata ilmMeta = new IndexLifecycleMetadata(existingILMMeta, OperationMode.RUNNING);
return ClusterState.builder(new ClusterName("test"))
.metadata(
Metadata.builder()
.componentTemplates(componentTemplates)
.transientSettings(nodeSettings)
.putCustom(IndexLifecycleMetadata.TYPE, ilmMeta)
.build()
)
.blocks(new ClusterBlocks.Builder().build())
.nodes(nodes)
.build();
}
private static class TestPutIndexTemplateResponse extends AcknowledgedResponse {
TestPutIndexTemplateResponse(boolean acknowledged) {
super(acknowledged);
}
}
}

2
x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/support/WatcherIndexTemplateRegistry.java
View File

@ -70,7 +70,7 @@ public class WatcherIndexTemplateRegistry extends IndexTemplateRegistry {
}
@Override
protected List<IndexTemplateConfig> getTemplateConfigs() {
protected List<IndexTemplateConfig> getLegacyTemplateConfigs() {
if (clusterService.state().nodes().getMinNodeVersion().onOrAfter(Version.V_7_7_0)) {
return Arrays.asList(
ilmManagementEnabled ? TEMPLATE_CONFIG_WATCH_HISTORY : TEMPLATE_CONFIG_WATCH_HISTORY_NO_ILM,

1
x-pack/qa/core-rest-tests-with-security/build.gradle
View File

@ -31,6 +31,7 @@ testClusters.integTest {
setting 'xpack.watcher.enabled', 'false'
setting 'xpack.ml.enabled', 'false'
setting 'xpack.license.self_generated.type', 'trial'
setting 'indices.lifecycle.history_index_enabled', 'false'
user username: System.getProperty('tests.rest.cluster.username', 'test_user'),
password: System.getProperty('tests.rest.cluster.password', 'x-pack-test-password')