honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[7.x] Add "synthetics-*-*" templates for synthetics fleet data (#62193) (#62346) 

* Add "synthetics-*-*" templates for synthetics fleet data

For the Elastic Agent we currently have `logs` and `metrics`, however, synthetic data doesn't belong
with those and thus we should have a place for it to live. This would be data reported from
heartbeat and under the 'monitoring' category.

This commit adds a composable index template for `synthetics-*-*` indices similar to the work in
 #56709 and #57629.

Resolves #61665
This commit is contained in:
Lee Hinman 2020-09-14 17:14:34 -06:00 committed by GitHub
parent f56ce4f39b
commit 6b2af30a62
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 154 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
docs/reference/data-streams/set-up-a-data-stream.asciidoc
View File

@ -110,7 +110,7 @@ template for a data stream must specify:
[IMPORTANT]
====
{es} has built-in index templates for the `metrics-*-*` and `logs-*-*` index
{es} has built-in index templates for the `metrics-*-*`, `logs-*-*`, and `synthetics-*-*` index
patterns, each with a priority of `100`.
{ingest-guide}/ingest-management-overview.html[{agent}] uses these templates to
create data streams. If you use {agent}, assign your index templates a priority

2
docs/reference/docs/index_.asciidoc
View File

@ -171,7 +171,7 @@ the operation automatically creates the index and applies any matching
[IMPORTANT]
====
{es} has built-in index templates for the `metrics-*-*` and `logs-*-*` index
{es} has built-in index templates for the `metrics-*-*`, `logs-*-*`, and `synthetics-*-*` index
patterns, each with a priority of `100`.
{ingest-guide}/ingest-management-overview.html[{agent}] uses these templates to
create data streams. If you use {agent}, assign your index templates a priority

2
docs/reference/indices/index-templates.asciidoc
View File

@ -22,7 +22,7 @@ If a new data stream or index matches more than one index template, the index te
[IMPORTANT]
====
{es} has built-in index templates for the `metrics-*-*` and `logs-*-*` index
{es} has built-in index templates for the `metrics-*-*`, `logs-*-*`, and `synthetics-*-*` index
patterns, each with a priority of `100`.
{ingest-guide}/ingest-management-overview.html[{agent}] uses these templates to
create data streams. If you use {agent}, assign your index templates a priority

2
docs/reference/indices/put-index-template.asciidoc
View File

@ -86,7 +86,7 @@ used to match the names of data streams and indices during creation.
+
[IMPORTANT]
====
{es} has built-in index templates for the `metrics-*-*` and `logs-*-*` index
{es} has built-in index templates for the `metrics-*-*`, `logs-*-*`, and `synthetics-*-*` index
patterns, each with a priority of `100`.
{ingest-guide}/ingest-management-overview.html[{agent}] uses these templates to
create data streams. If you use {agent}, assign your index templates a priority

3
test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
View File

@ -1314,6 +1314,9 @@ public abstract class ESRestTestCase extends ESTestCase {
case "metrics":
case "metrics-settings":
case "metrics-mappings":
case "synthetics":
case "synthetics-settings":
case "synthetics-mappings":
case ".snapshot-blob-cache":
return true;
default:

50
x-pack/plugin/core/src/main/resources/synthetics-mappings.json Normal file
View File

@ -0,0 +1,50 @@
{
"template": {
"mappings": {
"dynamic_templates": [
{
"strings_as_keyword": {
"mapping": {
"ignore_above": 1024,
"type": "keyword"
},
"match_mapping_type": "string"
}
}
],
"date_detection": false,
"properties": {
"@timestamp": {
"type": "date"
},
"data_stream": {
"properties": {
"type": {
"type": "constant_keyword",
"value": "synthetics"
},
"dataset": {
"type": "constant_keyword"
},
"namespace": {
"type": "constant_keyword"
}
}
},
"ecs": {
"properties": {
"version": {
"ignore_above": 1024,
"type": "keyword"
}
}
}
}
}
},
"_meta": {
"description": "default mappings for the synthetics index template installed by x-pack",
"managed": true
},
"version": ${xpack.stack.template.version}
}

12
x-pack/plugin/core/src/main/resources/synthetics-policy.json Normal file
View File

@ -0,0 +1,12 @@
{
"phases": {
"hot": {
"actions": {
"rollover": {
"max_size": "50gb",
"max_age": "30d"
}
}
}
}
}

17
x-pack/plugin/core/src/main/resources/synthetics-settings.json Normal file
View File

@ -0,0 +1,17 @@
{
"template": {
"settings": {
"index": {
"lifecycle": {
"name": "synthetics"
},
"codec": "best_compression"
}
}
},
"_meta": {
"description": "default settings for the synthetics index template installed by x-pack",
"managed": true
},
"version": ${xpack.stack.template.version}
}

14
x-pack/plugin/core/src/main/resources/synthetics-template.json Normal file
View File

@ -0,0 +1,14 @@
{
"index_patterns": ["synthetics-*-*"],
"priority": 100,
"data_stream": {},
"composed_of": [
"synthetics-mappings",
"synthetics-settings"
],
"_meta": {
"description": "default synthetics template installed by x-pack",
"managed": true
},
"version": ${xpack.stack.template.version}
}

39
x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java
View File

@ -90,6 +90,37 @@ public class StackTemplateRegistry extends IndexTemplateRegistry {
TEMPLATE_VERSION_VARIABLE
);
//////////////////////////////////////////////////////////
// Synthetics components (for matching synthetics-*-* indices)
//////////////////////////////////////////////////////////
public static final String SYNTHETICS_MAPPINGS_COMPONENT_TEMPLATE_NAME = "synthetics-mappings";
public static final String SYNTHETICS_SETTINGS_COMPONENT_TEMPLATE_NAME = "synthetics-settings";
public static final String SYNTHETICS_ILM_POLICY_NAME = "synthetics";
public static final String SYNTHETICS_INDEX_TEMPLATE_NAME = "synthetics";
public static final IndexTemplateConfig SYNTHETICS_MAPPINGS_COMPONENT_TEMPLATE = new IndexTemplateConfig(
SYNTHETICS_MAPPINGS_COMPONENT_TEMPLATE_NAME,
"/synthetics-mappings.json",
REGISTRY_VERSION,
TEMPLATE_VERSION_VARIABLE
);
public static final IndexTemplateConfig SYNTHETICS_SETTINGS_COMPONENT_TEMPLATE = new IndexTemplateConfig(
SYNTHETICS_SETTINGS_COMPONENT_TEMPLATE_NAME,
"/synthetics-settings.json",
REGISTRY_VERSION,
TEMPLATE_VERSION_VARIABLE
);
public static final LifecyclePolicyConfig SYNTHETICS_ILM_POLICY = new LifecyclePolicyConfig(
SYNTHETICS_ILM_POLICY_NAME,
"/synthetics-policy.json"
);
public static final IndexTemplateConfig SYNTHETICS_INDEX_TEMPLATE = new IndexTemplateConfig(
SYNTHETICS_INDEX_TEMPLATE_NAME,
"/synthetics-template.json",
REGISTRY_VERSION,
TEMPLATE_VERSION_VARIABLE
);
public StackTemplateRegistry(
Settings nodeSettings,
ClusterService clusterService,
@ -104,7 +135,7 @@ public class StackTemplateRegistry extends IndexTemplateRegistry {
@Override
protected List<LifecyclePolicyConfig> getPolicyConfigs() {
if (stackTemplateEnabled) {
return Arrays.asList(LOGS_ILM_POLICY, METRICS_ILM_POLICY);
return Arrays.asList(LOGS_ILM_POLICY, METRICS_ILM_POLICY, SYNTHETICS_ILM_POLICY);
} else {
return Collections.emptyList();
}
@ -117,7 +148,9 @@ public class StackTemplateRegistry extends IndexTemplateRegistry {
LOGS_MAPPINGS_COMPONENT_TEMPLATE,
LOGS_SETTINGS_COMPONENT_TEMPLATE,
METRICS_MAPPINGS_COMPONENT_TEMPLATE,
METRICS_SETTINGS_COMPONENT_TEMPLATE
METRICS_SETTINGS_COMPONENT_TEMPLATE,
SYNTHETICS_MAPPINGS_COMPONENT_TEMPLATE,
SYNTHETICS_SETTINGS_COMPONENT_TEMPLATE
);
} else {
return Collections.emptyList();
@ -127,7 +160,7 @@ public class StackTemplateRegistry extends IndexTemplateRegistry {
@Override
protected List<IndexTemplateConfig> getComposableTemplateConfigs() {
if (stackTemplateEnabled) {
return Arrays.asList(LOGS_INDEX_TEMPLATE, METRICS_INDEX_TEMPLATE);
return Arrays.asList(LOGS_INDEX_TEMPLATE, METRICS_INDEX_TEMPLATE, SYNTHETICS_INDEX_TEMPLATE);
} else {
return Collections.emptyList();
}

22
x-pack/plugin/stack/src/test/java/org/elasticsearch/xpack/stack/StackTemplateRegistryTests.java
View File

@ -150,7 +150,11 @@ public class StackTemplateRegistryTests extends ESTestCase {
final PutLifecycleAction.Request putRequest = (PutLifecycleAction.Request) request;
assertThat(
putRequest.getPolicy().getName(),
anyOf(equalTo(StackTemplateRegistry.LOGS_ILM_POLICY_NAME), equalTo(StackTemplateRegistry.METRICS_ILM_POLICY_NAME))
anyOf(
equalTo(StackTemplateRegistry.LOGS_ILM_POLICY_NAME),
equalTo(StackTemplateRegistry.METRICS_ILM_POLICY_NAME),
equalTo(StackTemplateRegistry.SYNTHETICS_ILM_POLICY_NAME)
)
);
assertNotNull(listener);
return new PutLifecycleAction.Response(true);
@ -168,7 +172,7 @@ public class StackTemplateRegistryTests extends ESTestCase {
ClusterChangedEvent event = createClusterChangedEvent(Collections.emptyMap(), nodes);
registry.clusterChanged(event);
assertBusy(() -> assertThat(calledTimes.get(), equalTo(2)));
assertBusy(() -> assertThat(calledTimes.get(), equalTo(3)));
}
public void testPolicyAlreadyExists() {
@ -180,7 +184,7 @@ public class StackTemplateRegistryTests extends ESTestCase {
.stream()
.map(policyConfig -> policyConfig.load(xContentRegistry))
.collect(Collectors.toList());
assertThat(policies, hasSize(2));
assertThat(policies, hasSize(3));
policies.forEach(p -> policyMap.put(p.getName(), p));
client.setVerifier((action, request, listener) -> {
@ -209,7 +213,7 @@ public class StackTemplateRegistryTests extends ESTestCase {
.stream()
.map(policyConfig -> policyConfig.load(xContentRegistry))
.collect(Collectors.toList());
assertThat(policies, hasSize(2));
assertThat(policies, hasSize(3));
policies.forEach(p -> policyMap.put(p.getName(), p));
client.setVerifier((action, request, listener) -> {
@ -276,6 +280,8 @@ public class StackTemplateRegistryTests extends ESTestCase {
versions.put(StackTemplateRegistry.LOGS_MAPPINGS_COMPONENT_TEMPLATE_NAME, StackTemplateRegistry.REGISTRY_VERSION);
versions.put(StackTemplateRegistry.METRICS_SETTINGS_COMPONENT_TEMPLATE_NAME, StackTemplateRegistry.REGISTRY_VERSION);
versions.put(StackTemplateRegistry.METRICS_MAPPINGS_COMPONENT_TEMPLATE_NAME, StackTemplateRegistry.REGISTRY_VERSION);
versions.put(StackTemplateRegistry.SYNTHETICS_SETTINGS_COMPONENT_TEMPLATE_NAME, StackTemplateRegistry.REGISTRY_VERSION);
versions.put(StackTemplateRegistry.SYNTHETICS_MAPPINGS_COMPONENT_TEMPLATE_NAME, StackTemplateRegistry.REGISTRY_VERSION);
ClusterChangedEvent sameVersionEvent = createClusterChangedEvent(versions, nodes);
client.setVerifier((action, request, listener) -> {
if (action instanceof PutComponentTemplateAction) {
@ -311,6 +317,14 @@ public class StackTemplateRegistryTests extends ESTestCase {
StackTemplateRegistry.METRICS_MAPPINGS_COMPONENT_TEMPLATE_NAME,
StackTemplateRegistry.REGISTRY_VERSION + randomIntBetween(1, 1000)
);
versions.put(
StackTemplateRegistry.SYNTHETICS_SETTINGS_COMPONENT_TEMPLATE_NAME,
StackTemplateRegistry.REGISTRY_VERSION + randomIntBetween(1, 1000)
);
versions.put(
StackTemplateRegistry.SYNTHETICS_MAPPINGS_COMPONENT_TEMPLATE_NAME,
StackTemplateRegistry.REGISTRY_VERSION + randomIntBetween(1, 1000)
);
ClusterChangedEvent higherVersionEvent = createClusterChangedEvent(versions, nodes);
registry.clusterChanged(higherVersionEvent);
}