honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Convert ILM and SLM histories into hidden indices (#51456) 

Modifies SLM's and ILM's history indices to be hidden indices for added
protection against accidental querying and deletion, and improves
IndexTemplateRegistry to handle upgrading index templates.

Also modifies the REST test cleanup to delete hidden indices.
This commit is contained in:
Gordon Brown 2020-02-11 14:18:55 -07:00 committed by GitHub
parent bb2e04bc16
commit d48ce12920
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
18 changed files with 234 additions and 99 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java
View File

@ -120,7 +120,6 @@ public class MetaDataCreateIndexService {
* These index patterns will be converted to hidden indices, at which point they should be removed from this list.
*/
private static final CharacterRunAutomaton DOT_INDICES_EXCLUSIONS = new CharacterRunAutomaton(Regex.simpleMatchToAutomaton(
".slm-history-*",
".watch-history-*",
".ml-anomalies-*",
".ml-notifications-*",

1
server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java
View File

@ -641,7 +641,6 @@ public class MetaDataCreateIndexServiceTests extends ESTestCase {
public void testIndexNameExclusionsList() {
// this test case should be removed when DOT_INDICES_EXCLUSIONS is empty
List<String> excludedNames = Arrays.asList(
".slm-history-" + randomAlphaOfLength(5).toLowerCase(Locale.ROOT),
".watch-history-" + randomAlphaOfLength(5).toLowerCase(Locale.ROOT),
".ml-anomalies-" + randomAlphaOfLength(5).toLowerCase(Locale.ROOT),
".ml-notifications-" + randomAlphaOfLength(5).toLowerCase(Locale.ROOT),

7
test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
View File

@ -599,8 +599,11 @@ public abstract class ESRestTestCase extends ESTestCase {
}
protected static void wipeAllIndices() throws IOException {
boolean includeHidden = minimumNodeVersion().onOrAfter(Version.V_7_7_0);
try {
final Response response = adminClient().performRequest(new Request("DELETE", "*"));
final Request deleteReq = new Request("DELETE", "*");
deleteReq.addParameter("expand_wildcards", "open,closed" + (includeHidden ? ",hidden" : ""));
final Response response = adminClient().performRequest(deleteReq);
try (InputStream is = response.getEntity().getContent()) {
assertTrue((boolean) XContentHelper.convertToMap(XContentType.JSON.xContent(), is, true).get("acknowledged"));
}
@ -1235,7 +1238,7 @@ public abstract class ESRestTestCase extends ESTestCase {
final Request request = new Request("GET", "_nodes");
request.addParameter("filter_path", "nodes.*.version");
final Response response = client().performRequest(request);
final Response response = adminClient().performRequest(request);
final Map<String, Object> nodes = ObjectPath.createFromResponse(response).evaluate("nodes");
Version minVersion = null;

3
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/history/SnapshotLifecycleTemplateRegistry.java
View File

@ -35,7 +35,8 @@ import static org.elasticsearch.xpack.core.ilm.LifecycleSettings.SLM_HISTORY_IND
public class SnapshotLifecycleTemplateRegistry extends IndexTemplateRegistry {
// history (please add a comment why you increased the version here)
// version 1: initial
public static final String INDEX_TEMPLATE_VERSION = "1";
// version 2: converted to hidden index
public static final int INDEX_TEMPLATE_VERSION = 2;
public static final String SLM_TEMPLATE_VERSION_VARIABLE = "xpack.slm.template.version";
public static final String SLM_TEMPLATE_NAME = ".slm-history";

19
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/IndexTemplateConfig.java
View File

@ -16,7 +16,7 @@ public class IndexTemplateConfig {
private final String templateName;
private final String fileName;
private final String version;
private final int version;
private final String versionProperty;
/**
@ -27,14 +27,17 @@ public class IndexTemplateConfig {
* {@code {"myTemplateVersion": "${my.version.property}"}}
* With {@code version = "42"; versionProperty = "my.version.property"} will result in {@code {"myTemplateVersion": "42"}}.
*
* @param templateName The name that will be used for the index template. Literal, include the version in this string if
* Note that this code does not automatically insert the {@code version} index template property - include that in the JSON file
* defining the template, preferably using the version variable provided to this constructor.
*
* @param templateName The name that will be used for the index template. Literal, include the version in this string if
* it should be used.
* @param fileName The filename the template should be loaded from. Literal, should include leading {@literal /} and
* extension if necessary.
* @param version The version of the template. Substituted for {@code versionProperty} as described above.
* @param versionProperty The property that will be replaced with the {@code version} string as described above.
*/
public IndexTemplateConfig(String templateName, String fileName, String version, String versionProperty) {
public IndexTemplateConfig(String templateName, String fileName, int version, String versionProperty) {
this.templateName = templateName;
this.fileName = fileName;
this.version = version;
@ -49,14 +52,20 @@ public class IndexTemplateConfig {
return templateName;
}
public int getVersion() {
return version;
}
/**
* Loads the template from disk as a UTF-8 byte array.
* @return The template as a UTF-8 byte array.
*/
public byte[] loadBytes() {
String template = TemplateUtils.loadTemplate(fileName, version,
Pattern.quote("${" + versionProperty + "}"));
final String versionPattern = Pattern.quote("${" + versionProperty + "}");
String template = TemplateUtils.loadTemplate(fileName, Integer.toString(version), versionPattern);
assert template != null && template.length() > 0;
assert Pattern.compile("\"version\"\\s*:\\s*" + version).matcher(template).find()
: "index template must have a version property set to the given version property";
return template.getBytes(StandardCharsets.UTF_8);
}
}

20
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/IndexTemplateRegistry.java
View File

@ -16,6 +16,7 @@ import org.elasticsearch.client.Client;
import org.elasticsearch.cluster.ClusterChangedEvent;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.ClusterStateListener;
import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.settings.Settings;
@ -31,6 +32,7 @@ import org.elasticsearch.xpack.core.ilm.LifecyclePolicy;
import org.elasticsearch.xpack.core.ilm.action.PutLifecycleAction;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
@ -130,16 +132,24 @@ public abstract class IndexTemplateRegistry implements ClusterStateListener {
private void addTemplatesIfMissing(ClusterState state) {
final List<IndexTemplateConfig> indexTemplates = getTemplateConfigs();
for (IndexTemplateConfig template : indexTemplates) {
final String templateName = template.getTemplateName();
for (IndexTemplateConfig newTemplate : indexTemplates) {
final String templateName = newTemplate.getTemplateName();
final AtomicBoolean creationCheck = templateCreationsInProgress.computeIfAbsent(templateName, key -> new AtomicBoolean(false));
if (creationCheck.compareAndSet(false, true)) {
if (!state.metaData().getTemplates().containsKey(templateName)) {
IndexTemplateMetaData currentTemplate = state.metaData().getTemplates().get(templateName);
if (Objects.isNull(currentTemplate)) {
logger.debug("adding index template [{}] for [{}], because it doesn't exist", templateName, getOrigin());
putTemplate(template, creationCheck);
putTemplate(newTemplate, creationCheck);
} else if (Objects.isNull(currentTemplate.getVersion()) || newTemplate.getVersion() > currentTemplate.getVersion()) {
// IndexTemplateConfig now enforces templates contain a `version` property, so if the template doesn't have one we can
// safely assume it's an old version of the template.
logger.info("upgrading index template [{}] for [{}] from version [{}] to version [{}]",
templateName, getOrigin(), currentTemplate.getVersion(), newTemplate.getVersion());
putTemplate(newTemplate, creationCheck);
} else {
creationCheck.set(false);
logger.trace("not adding index template [{}] for [{}], because it already exists", templateName, getOrigin());
logger.trace("not adding index template [{}] for [{}], because it already exists at version [{}]",
templateName, getOrigin(), currentTemplate.getVersion());
}
} else {
logger.trace("skipping the creation of index template [{}] for [{}], because its creation is in progress",

5
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/TemplateUtils.java
View File

@ -59,9 +59,10 @@ public class TemplateUtils {
public static String loadTemplate(String resource, String version, String versionProperty) {
try {
BytesReference source = load(resource);
validate(source);
final String filteredJson = filter(source, version, versionProperty);
validate(new BytesArray(filteredJson));
return filter(source, version, versionProperty);
return filteredJson;
} catch (Exception e) {
throw new IllegalArgumentException("Unable to load template [" + resource + "]", e);
}

2
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/support/WatcherIndexTemplateRegistryField.java
View File

@ -16,7 +16,7 @@ public final class WatcherIndexTemplateRegistryField {
// version 9: add a user field defining which user executed the watch
// version 10: add support for foreach path in actions
// Note: if you change this, also inform the kibana team around the watcher-ui
public static final String INDEX_TEMPLATE_VERSION = "10";
public static final int INDEX_TEMPLATE_VERSION = 10;
public static final String HISTORY_TEMPLATE_NAME = ".watch-history-" + INDEX_TEMPLATE_VERSION;
public static final String HISTORY_TEMPLATE_NAME_NO_ILM = ".watch-history-no-ilm-" + INDEX_TEMPLATE_VERSION;
public static final String TRIGGERED_TEMPLATE_NAME = ".triggered_watches";

4
x-pack/plugin/core/src/main/resources/ilm-history.json
View File

@ -9,6 +9,7 @@
"index.auto_expand_replicas": "0-1",
"index.lifecycle.name": "ilm-history-ilm-policy",
"index.lifecycle.rollover_alias": "ilm-history-${xpack.ilm_history.template.version}",
"index.hidden": true,
"index.format": 1
},
"mappings": {
@ -79,5 +80,6 @@
}
}
}
}
},
"version": ${xpack.ilm_history.template.version}
}

6
x-pack/plugin/core/src/main/resources/slm-history.json
View File

@ -9,6 +9,7 @@
"index.auto_expand_replicas": "0-1",
"index.lifecycle.name": "slm-history-ilm-policy",
"index.lifecycle.rollover_alias": ".slm-history-${xpack.slm.template.version}",
"index.hidden": true,
"index.format": 1
},
"mappings": {
@ -55,5 +56,6 @@
}
}
}
}
}
},
"version": ${xpack.slm.template.version}
}

3
x-pack/plugin/core/src/main/resources/triggered-watches.json
View File

@ -36,5 +36,6 @@
}
}
}
}
},
"version": ${xpack.watcher.template.version}
}

3
x-pack/plugin/core/src/main/resources/watch-history-no-ilm.json
View File

@ -611,5 +611,6 @@
}
}
}
}
},
"version": ${xpack.watcher.template.version}
}

3
x-pack/plugin/core/src/main/resources/watch-history.json
View File

@ -562,5 +562,6 @@
}
}
}
}
},
"version": ${xpack.watcher.template.version}
}

3
x-pack/plugin/core/src/main/resources/watches.json
View File

@ -58,5 +58,6 @@
}
}
}
}
},
"version": ${xpack.watcher.template.version}
}

140
x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/slm/history/SnapshotLifecycleTemplateRegistryTests.java
View File

@ -62,6 +62,7 @@ import java.util.stream.Collectors;
import static org.elasticsearch.mock.orig.Mockito.when;
import static org.elasticsearch.xpack.core.ilm.LifecycleSettings.SLM_HISTORY_INDEX_ENABLED_SETTING;
import static org.elasticsearch.xpack.core.slm.history.SnapshotLifecycleTemplateRegistry.INDEX_TEMPLATE_VERSION;
import static org.elasticsearch.xpack.core.slm.history.SnapshotLifecycleTemplateRegistry.SLM_POLICY_NAME;
import static org.elasticsearch.xpack.core.slm.history.SnapshotLifecycleTemplateRegistry.SLM_TEMPLATE_NAME;
import static org.hamcrest.Matchers.equalTo;
@ -113,27 +114,10 @@ public class SnapshotLifecycleTemplateRegistryTests extends ESTestCase {
DiscoveryNode node = new DiscoveryNode("node", ESTestCase.buildNewFakeTransportAddress(), Version.CURRENT);
DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build();
ClusterChangedEvent event = createClusterChangedEvent(Collections.emptyList(), nodes);
ClusterChangedEvent event = createClusterChangedEvent(Collections.emptyMap(), nodes);
AtomicInteger calledTimes = new AtomicInteger(0);
client.setVerifier((action, request, listener) -> {
if (action instanceof PutIndexTemplateAction) {
calledTimes.incrementAndGet();
assertThat(action, instanceOf(PutIndexTemplateAction.class));
assertThat(request, instanceOf(PutIndexTemplateRequest.class));
final PutIndexTemplateRequest putRequest = (PutIndexTemplateRequest) request;
assertThat(putRequest.name(), equalTo(SLM_TEMPLATE_NAME));
assertThat(putRequest.settings().get("index.lifecycle.name"), equalTo(SLM_POLICY_NAME));
assertNotNull(listener);
return new TestPutIndexTemplateResponse(true);
} else if (action instanceof PutLifecycleAction) {
// Ignore this, it's verified in another test
return new PutLifecycleAction.Response(true);
} else {
fail("client called with unexpected request:" + request.toString());
return null;
}
});
client.setVerifier((action, request, listener) -> verifyTemplateInstalled(calledTimes, action, request, listener));
registry.clusterChanged(event);
assertBusy(() -> assertThat(calledTimes.get(), equalTo(registry.getTemplateConfigs().size())));
@ -145,7 +129,7 @@ public class SnapshotLifecycleTemplateRegistryTests extends ESTestCase {
// will not be issued anymore, leaving calledTimes to 0
assertBusy(() -> {
// now delete one template from the cluster state and lets retry
ClusterChangedEvent newEvent = createClusterChangedEvent(Collections.emptyList(), nodes);
ClusterChangedEvent newEvent = createClusterChangedEvent(Collections.emptyMap(), nodes);
registry.clusterChanged(newEvent);
assertThat(calledTimes.get(), greaterThan(1));
});
@ -174,7 +158,7 @@ public class SnapshotLifecycleTemplateRegistryTests extends ESTestCase {
}
});
ClusterChangedEvent event = createClusterChangedEvent(Collections.emptyList(), nodes);
ClusterChangedEvent event = createClusterChangedEvent(Collections.emptyMap(), nodes);
registry.clusterChanged(event);
assertBusy(() -> assertThat(calledTimes.get(), equalTo(1)));
}
@ -203,7 +187,7 @@ public class SnapshotLifecycleTemplateRegistryTests extends ESTestCase {
return null;
});
ClusterChangedEvent event = createClusterChangedEvent(Collections.emptyList(), policyMap, nodes);
ClusterChangedEvent event = createClusterChangedEvent(Collections.emptyMap(), policyMap, nodes);
registry.clusterChanged(event);
}
@ -235,11 +219,61 @@ public class SnapshotLifecycleTemplateRegistryTests extends ESTestCase {
.createParser(xContentRegistry, LoggingDeprecationHandler.THROW_UNSUPPORTED_OPERATION, policyStr)) {
LifecyclePolicy different = LifecyclePolicy.parse(parser, policy.getName());
policyMap.put(policy.getName(), different);
ClusterChangedEvent event = createClusterChangedEvent(Collections.emptyList(), policyMap, nodes);
ClusterChangedEvent event = createClusterChangedEvent(Collections.emptyMap(), policyMap, nodes);
registry.clusterChanged(event);
}
}
public void testThatVersionedOldTemplatesAreUpgraded() throws Exception {
DiscoveryNode node = new DiscoveryNode("node", ESTestCase.buildNewFakeTransportAddress(), Version.CURRENT);
DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build();
ClusterChangedEvent event = createClusterChangedEvent(Collections.singletonMap(SLM_TEMPLATE_NAME, INDEX_TEMPLATE_VERSION - 1),
nodes);
AtomicInteger calledTimes = new AtomicInteger(0);
client.setVerifier((action, request, listener) -> verifyTemplateInstalled(calledTimes, action, request, listener));
registry.clusterChanged(event);
assertBusy(() -> assertThat(calledTimes.get(), equalTo(registry.getTemplateConfigs().size())));
}
public void testThatUnversionedOldTemplatesAreUpgraded() throws Exception {
DiscoveryNode node = new DiscoveryNode("node", ESTestCase.buildNewFakeTransportAddress(), Version.CURRENT);
DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build();
ClusterChangedEvent event = createClusterChangedEvent(Collections.singletonMap(SLM_TEMPLATE_NAME, null), nodes);
AtomicInteger calledTimes = new AtomicInteger(0);
client.setVerifier((action, request, listener) -> verifyTemplateInstalled(calledTimes, action, request, listener));
registry.clusterChanged(event);
assertBusy(() -> assertThat(calledTimes.get(), equalTo(registry.getTemplateConfigs().size())));
}
public void testSameOrHigherVersionTemplateNotUpgraded() throws Exception {
DiscoveryNode node = new DiscoveryNode("node", ESTestCase.buildNewFakeTransportAddress(), Version.CURRENT);
DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build();
ClusterChangedEvent sameVersionEvent = createClusterChangedEvent(
Collections.singletonMap(SLM_TEMPLATE_NAME, INDEX_TEMPLATE_VERSION), nodes);
AtomicInteger calledTimes = new AtomicInteger(0);
client.setVerifier((action, request, listener) -> {
if (action instanceof PutIndexTemplateAction) {
fail("template should not have been re-installed");
return null;
} else if (action instanceof PutLifecycleAction) {
// Ignore this, it's verified in another test
return new PutLifecycleAction.Response(true);
} else {
fail("client called with unexpected request:" + request.toString());
return null;
}
});
registry.clusterChanged(sameVersionEvent);
ClusterChangedEvent higherVersionEvent = createClusterChangedEvent(
Collections.singletonMap(SLM_TEMPLATE_NAME, INDEX_TEMPLATE_VERSION + randomIntBetween(1, 1000)), nodes);
registry.clusterChanged(higherVersionEvent);
}
public void testThatMissingMasterNodeDoesNothing() {
DiscoveryNode localNode = new DiscoveryNode("node", ESTestCase.buildNewFakeTransportAddress(), Version.CURRENT);
DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").add(localNode).build();
@ -249,20 +283,25 @@ public class SnapshotLifecycleTemplateRegistryTests extends ESTestCase {
return null;
});
ClusterChangedEvent event = createClusterChangedEvent(Arrays.asList(SLM_TEMPLATE_NAME), nodes);
ClusterChangedEvent event = createClusterChangedEvent(Collections.singletonMap(SLM_TEMPLATE_NAME, null), nodes);
registry.clusterChanged(event);
}
public void testValidate() {
assertFalse(registry.validate(createClusterState(Settings.EMPTY, Collections.emptyList(), Collections.emptyMap(), null)));
assertFalse(registry.validate(createClusterState(Settings.EMPTY, Collections.singletonList(SLM_TEMPLATE_NAME),
Collections.emptyMap(), null)));
assertFalse(registry.validate(createClusterState(Settings.EMPTY, Collections.emptyMap(), Collections.emptyMap(), null)));
assertFalse(registry.validate(createClusterState(Settings.EMPTY,
Collections.singletonMap(SLM_TEMPLATE_NAME, null),
Collections.emptyMap(),
null)));
Map<String, LifecyclePolicy> policyMap = new HashMap<>();
policyMap.put(SLM_POLICY_NAME, new LifecyclePolicy(SLM_POLICY_NAME, new HashMap<>()));
assertFalse(registry.validate(createClusterState(Settings.EMPTY, Collections.emptyList(), policyMap, null)));
assertFalse(registry.validate(createClusterState(Settings.EMPTY, Collections.emptyMap(), policyMap, null)));
assertTrue(registry.validate(createClusterState(Settings.EMPTY, Collections.singletonList(SLM_TEMPLATE_NAME), policyMap, null)));
assertTrue(registry.validate(createClusterState(Settings.EMPTY,
Collections.singletonMap(SLM_TEMPLATE_NAME, null),
policyMap,
null)));
}
// -------------
@ -299,14 +338,35 @@ public class SnapshotLifecycleTemplateRegistryTests extends ESTestCase {
}
}
private ClusterChangedEvent createClusterChangedEvent(List<String> existingTemplateNames, DiscoveryNodes nodes) {
return createClusterChangedEvent(existingTemplateNames, Collections.emptyMap(), nodes);
private ActionResponse verifyTemplateInstalled(
AtomicInteger calledTimes, ActionType<?> action, ActionRequest request, ActionListener<?> listener) {
if (action instanceof PutIndexTemplateAction) {
calledTimes.incrementAndGet();
assertThat(action, instanceOf(PutIndexTemplateAction.class));
assertThat(request, instanceOf(PutIndexTemplateRequest.class));
final PutIndexTemplateRequest putRequest = (PutIndexTemplateRequest) request;
assertThat(putRequest.name(), equalTo(SLM_TEMPLATE_NAME));
assertThat(putRequest.settings().get("index.lifecycle.name"), equalTo(SLM_POLICY_NAME));
assertThat(putRequest.version(), equalTo(INDEX_TEMPLATE_VERSION));
assertNotNull(listener);
return new TestPutIndexTemplateResponse(true);
} else if (action instanceof PutLifecycleAction) {
// Ignore this, it's verified in another test
return new PutLifecycleAction.Response(true);
} else {
fail("client called with unexpected request:" + request.toString());
return null;
}
}
private ClusterChangedEvent createClusterChangedEvent(List<String> existingTemplateNames,
private ClusterChangedEvent createClusterChangedEvent(Map<String, Integer> existingTemplates, DiscoveryNodes nodes) {
return createClusterChangedEvent(existingTemplates, Collections.emptyMap(), nodes);
}
private ClusterChangedEvent createClusterChangedEvent(Map<String, Integer> existingTemplates,
Map<String, LifecyclePolicy> existingPolicies,
DiscoveryNodes nodes) {
ClusterState cs = createClusterState(Settings.EMPTY, existingTemplateNames, existingPolicies, nodes);
ClusterState cs = createClusterState(Settings.EMPTY, existingTemplates, existingPolicies, nodes);
ClusterChangedEvent realEvent = new ClusterChangedEvent("created-from-test", cs,
ClusterState.builder(new ClusterName("test")).build());
ClusterChangedEvent event = spy(realEvent);
@ -315,13 +375,15 @@ public class SnapshotLifecycleTemplateRegistryTests extends ESTestCase {
return event;
}
private ClusterState createClusterState(Settings nodeSettings,
List<String> existingTemplateNames,
Map<String, LifecyclePolicy> existingPolicies,
DiscoveryNodes nodes) {
private ClusterState createClusterState(Settings nodeSettings, Map<String, Integer> existingTemplates,
Map<String, LifecyclePolicy> existingPolicies, DiscoveryNodes nodes) {
ImmutableOpenMap.Builder<String, IndexTemplateMetaData> indexTemplates = ImmutableOpenMap.builder();
for (String name : existingTemplateNames) {
indexTemplates.put(name, mock(IndexTemplateMetaData.class));
for (Map.Entry<String, Integer> template : existingTemplates.entrySet()) {
final IndexTemplateMetaData mockTemplate = mock(IndexTemplateMetaData.class);
when(mockTemplate.version()).thenReturn(template.getValue());
when(mockTemplate.getVersion()).thenReturn(template.getValue());
indexTemplates.put(template.getKey(), mockTemplate);
}
Map<String, LifecyclePolicyMetadata> existingILMMeta = existingPolicies.entrySet().stream()

3
x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/history/ILMHistoryTemplateRegistry.java
View File

@ -27,7 +27,8 @@ import java.util.List;
public class ILMHistoryTemplateRegistry extends IndexTemplateRegistry {
// history (please add a comment why you increased the version here)
// version 1: initial
public static final String INDEX_TEMPLATE_VERSION = "1";
// version 2: convert to hidden index
public static final int INDEX_TEMPLATE_VERSION = 2;
public static final String ILM_TEMPLATE_VERSION_VARIABLE = "xpack.ilm_history.template.version";
public static final String ILM_TEMPLATE_NAME = "ilm-history";

2
x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/history/HistoryStoreTests.java
View File

@ -105,7 +105,7 @@ public class HistoryStoreTests extends ESTestCase {
}
public void testIndexNameGeneration() {
String indexTemplateVersion = INDEX_TEMPLATE_VERSION;
String indexTemplateVersion = Integer.toString(INDEX_TEMPLATE_VERSION);
assertThat(getHistoryIndexNameForTime(Instant.ofEpochMilli((long) 0).atZone(ZoneOffset.UTC)),
equalTo(".watcher-history-"+ indexTemplateVersion +"-1970.01.01"));
assertThat(getHistoryIndexNameForTime(Instant.ofEpochMilli(100000000000L).atZone(ZoneOffset.UTC)),

108
x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/support/WatcherIndexTemplateRegistryTests.java
View File

@ -58,6 +58,7 @@ import java.util.stream.Collectors;
import static org.elasticsearch.mock.orig.Mockito.verify;
import static org.elasticsearch.mock.orig.Mockito.when;
import static org.elasticsearch.xpack.core.watcher.support.WatcherIndexTemplateRegistryField.INDEX_TEMPLATE_VERSION;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.hasSize;
import static org.hamcrest.Matchers.is;
@ -111,14 +112,16 @@ public class WatcherIndexTemplateRegistryTests extends ESTestCase {
DiscoveryNode node = new DiscoveryNode("node", ESTestCase.buildNewFakeTransportAddress(), Version.CURRENT);
DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build();
ClusterChangedEvent event = createClusterChangedEvent(Collections.emptyList(), nodes);
ClusterChangedEvent event = createClusterChangedEvent(Collections.emptyMap(), nodes);
registry.clusterChanged(event);
ArgumentCaptor<PutIndexTemplateRequest> argumentCaptor = ArgumentCaptor.forClass(PutIndexTemplateRequest.class);
verify(client.admin().indices(), times(3)).putTemplate(argumentCaptor.capture(), anyObject());
// now delete one template from the cluster state and lets retry
ClusterChangedEvent newEvent = createClusterChangedEvent(Arrays.asList(WatcherIndexTemplateRegistryField.HISTORY_TEMPLATE_NAME,
WatcherIndexTemplateRegistryField.TRIGGERED_TEMPLATE_NAME), nodes);
Map<String, Integer> existingTemplates = new HashMap<>();
existingTemplates.put(WatcherIndexTemplateRegistryField.HISTORY_TEMPLATE_NAME, INDEX_TEMPLATE_VERSION);
existingTemplates.put(WatcherIndexTemplateRegistryField.TRIGGERED_TEMPLATE_NAME, INDEX_TEMPLATE_VERSION);
ClusterChangedEvent newEvent = createClusterChangedEvent(existingTemplates, nodes);
registry.clusterChanged(newEvent);
ArgumentCaptor<PutIndexTemplateRequest> captor = ArgumentCaptor.forClass(PutIndexTemplateRequest.class);
verify(client.admin().indices(), times(4)).putTemplate(captor.capture(), anyObject());
@ -136,14 +139,16 @@ public class WatcherIndexTemplateRegistryTests extends ESTestCase {
registry = new WatcherIndexTemplateRegistry(Settings.builder()
.put(XPackSettings.INDEX_LIFECYCLE_ENABLED.getKey(), false).build(),
clusterService, threadPool, client, xContentRegistry);
ClusterChangedEvent event = createClusterChangedEvent(Settings.EMPTY, Collections.emptyList(), Collections.emptyMap(), nodes);
ClusterChangedEvent event = createClusterChangedEvent(Settings.EMPTY, Collections.emptyMap(), Collections.emptyMap(), nodes);
registry.clusterChanged(event);
ArgumentCaptor<PutIndexTemplateRequest> argumentCaptor = ArgumentCaptor.forClass(PutIndexTemplateRequest.class);
verify(client.admin().indices(), times(3)).putTemplate(argumentCaptor.capture(), anyObject());
// now delete one template from the cluster state and lets retry
ClusterChangedEvent newEvent = createClusterChangedEvent(Arrays.asList(WatcherIndexTemplateRegistryField.HISTORY_TEMPLATE_NAME,
WatcherIndexTemplateRegistryField.TRIGGERED_TEMPLATE_NAME), nodes);
Map<String, Integer> existingTemplates = new HashMap<>();
existingTemplates.put(WatcherIndexTemplateRegistryField.HISTORY_TEMPLATE_NAME, INDEX_TEMPLATE_VERSION);
existingTemplates.put(WatcherIndexTemplateRegistryField.TRIGGERED_TEMPLATE_NAME, INDEX_TEMPLATE_VERSION);
ClusterChangedEvent newEvent = createClusterChangedEvent(existingTemplates, nodes);
registry.clusterChanged(newEvent);
ArgumentCaptor<PutIndexTemplateRequest> captor = ArgumentCaptor.forClass(PutIndexTemplateRequest.class);
verify(client.admin().indices(), times(5)).putTemplate(captor.capture(), anyObject());
@ -155,7 +160,7 @@ public class WatcherIndexTemplateRegistryTests extends ESTestCase {
DiscoveryNode node = new DiscoveryNode("node", ESTestCase.buildNewFakeTransportAddress(), Version.CURRENT);
DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build();
ClusterChangedEvent event = createClusterChangedEvent(Collections.emptyList(), nodes);
ClusterChangedEvent event = createClusterChangedEvent(Collections.emptyMap(), nodes);
registry.clusterChanged(event);
verify(client, times(1)).execute(eq(PutLifecycleAction.INSTANCE), anyObject(), anyObject());
}
@ -171,7 +176,7 @@ public class WatcherIndexTemplateRegistryTests extends ESTestCase {
assertThat(policies, hasSize(1));
LifecyclePolicy policy = policies.get(0);
policyMap.put(policy.getName(), policy);
ClusterChangedEvent event = createClusterChangedEvent(Collections.emptyList(), policyMap, nodes);
ClusterChangedEvent event = createClusterChangedEvent(Collections.emptyMap(), policyMap, nodes);
registry.clusterChanged(event);
verify(client, times(0)).execute(eq(PutLifecycleAction.INSTANCE), anyObject(), anyObject());
}
@ -183,7 +188,7 @@ public class WatcherIndexTemplateRegistryTests extends ESTestCase {
registry = new WatcherIndexTemplateRegistry(Settings.builder()
.put(XPackSettings.INDEX_LIFECYCLE_ENABLED.getKey(), false).build(),
clusterService, threadPool, client, xContentRegistry);
ClusterChangedEvent event = createClusterChangedEvent(Settings.EMPTY, Collections.emptyList(), Collections.emptyMap(), nodes);
ClusterChangedEvent event = createClusterChangedEvent(Settings.EMPTY, Collections.emptyMap(), Collections.emptyMap(), nodes);
registry.clusterChanged(event);
verify(client, times(0)).execute(eq(PutLifecycleAction.INSTANCE), anyObject(), anyObject());
}
@ -203,20 +208,44 @@ public class WatcherIndexTemplateRegistryTests extends ESTestCase {
.createParser(xContentRegistry, LoggingDeprecationHandler.THROW_UNSUPPORTED_OPERATION, policyStr)) {
LifecyclePolicy different = LifecyclePolicy.parse(parser, policy.getName());
policyMap.put(policy.getName(), different);
ClusterChangedEvent event = createClusterChangedEvent(Collections.emptyList(), policyMap, nodes);
ClusterChangedEvent event = createClusterChangedEvent(Collections.emptyMap(), policyMap, nodes);
registry.clusterChanged(event);
verify(client, times(0)).execute(eq(PutLifecycleAction.INSTANCE), anyObject(), anyObject());
}
}
public void testThatTemplatesExist() {
assertThat(WatcherIndexTemplateRegistry.validate(createClusterState(".watch-history")), is(false));
assertThat(WatcherIndexTemplateRegistry.validate(createClusterState(".watch-history", ".triggered_watches", ".watches")),
is(false));
assertThat(WatcherIndexTemplateRegistry.validate(createClusterState(WatcherIndexTemplateRegistryField.HISTORY_TEMPLATE_NAME,
".triggered_watches", ".watches")), is(true));
assertThat(WatcherIndexTemplateRegistry.validate(createClusterState(WatcherIndexTemplateRegistryField.HISTORY_TEMPLATE_NAME,
".triggered_watches", ".watches", "whatever", "else")), is(true));
{
Map<String, Integer> existingTemplates = new HashMap<>();
existingTemplates.put(".watch-history", INDEX_TEMPLATE_VERSION);
assertThat(WatcherIndexTemplateRegistry.validate(createClusterState(existingTemplates)), is(false));
}
{
Map<String, Integer> existingTemplates = new HashMap<>();
existingTemplates.put(".watch-history", INDEX_TEMPLATE_VERSION);
existingTemplates.put(".triggered_watches", INDEX_TEMPLATE_VERSION);
existingTemplates.put(".watches", INDEX_TEMPLATE_VERSION);
assertThat(WatcherIndexTemplateRegistry.validate(createClusterState(existingTemplates)), is(false));
}
{
Map<String, Integer> existingTemplates = new HashMap<>();
existingTemplates.put(WatcherIndexTemplateRegistryField.HISTORY_TEMPLATE_NAME, INDEX_TEMPLATE_VERSION);
existingTemplates.put(".triggered_watches", INDEX_TEMPLATE_VERSION);
existingTemplates.put(".watches", INDEX_TEMPLATE_VERSION);
assertThat(WatcherIndexTemplateRegistry.validate(createClusterState(existingTemplates)), is(true));
}
{
Map<String, Integer> existingTemplates = new HashMap<>();
existingTemplates.put(WatcherIndexTemplateRegistryField.HISTORY_TEMPLATE_NAME, INDEX_TEMPLATE_VERSION);
existingTemplates.put(".triggered_watches", INDEX_TEMPLATE_VERSION);
existingTemplates.put(".watches", INDEX_TEMPLATE_VERSION);
existingTemplates.put("whatever", null);
existingTemplates.put("else", null);
assertThat(WatcherIndexTemplateRegistry.validate(createClusterState(existingTemplates)), is(true));
}
}
// if a node is newer than the master node, the template needs to be applied as well
@ -226,8 +255,11 @@ public class WatcherIndexTemplateRegistryTests extends ESTestCase {
DiscoveryNode masterNode = new DiscoveryNode("master", ESTestCase.buildNewFakeTransportAddress(), Version.V_6_0_0);
DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("master").add(localNode).add(masterNode).build();
ClusterChangedEvent event = createClusterChangedEvent(Arrays.asList(WatcherIndexTemplateRegistryField.TRIGGERED_TEMPLATE_NAME,
WatcherIndexTemplateRegistryField.WATCHES_TEMPLATE_NAME, ".watch-history-6"), nodes);
Map<String, Integer> existingTemplates = new HashMap<>();
existingTemplates.put(WatcherIndexTemplateRegistryField.TRIGGERED_TEMPLATE_NAME, INDEX_TEMPLATE_VERSION);
existingTemplates.put(WatcherIndexTemplateRegistryField.WATCHES_TEMPLATE_NAME, INDEX_TEMPLATE_VERSION);
existingTemplates.put(".watch-history-6", 6);
ClusterChangedEvent event = createClusterChangedEvent(existingTemplates, nodes);
registry.clusterChanged(event);
ArgumentCaptor<PutIndexTemplateRequest> argumentCaptor = ArgumentCaptor.forClass(PutIndexTemplateRequest.class);
@ -240,8 +272,11 @@ public class WatcherIndexTemplateRegistryTests extends ESTestCase {
DiscoveryNode masterNode = new DiscoveryNode("master", ESTestCase.buildNewFakeTransportAddress(), Version.CURRENT);
DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("master").add(localNode).add(masterNode).build();
ClusterChangedEvent event = createClusterChangedEvent(Arrays.asList(WatcherIndexTemplateRegistryField.TRIGGERED_TEMPLATE_NAME,
WatcherIndexTemplateRegistryField.WATCHES_TEMPLATE_NAME, ".watch-history-6"), nodes);
Map<String, Integer> existingTemplates = new HashMap<>();
existingTemplates.put(WatcherIndexTemplateRegistryField.TRIGGERED_TEMPLATE_NAME, null);
existingTemplates.put(WatcherIndexTemplateRegistryField.WATCHES_TEMPLATE_NAME, null);
existingTemplates.put(".watch-history-6", null);
ClusterChangedEvent event = createClusterChangedEvent(existingTemplates, nodes);
registry.clusterChanged(event);
verifyZeroInteractions(client);
@ -251,24 +286,30 @@ public class WatcherIndexTemplateRegistryTests extends ESTestCase {
DiscoveryNode localNode = new DiscoveryNode("node", ESTestCase.buildNewFakeTransportAddress(), Version.CURRENT);
DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").add(localNode).build();
ClusterChangedEvent event = createClusterChangedEvent(Arrays.asList(WatcherIndexTemplateRegistryField.TRIGGERED_TEMPLATE_NAME,
WatcherIndexTemplateRegistryField.WATCHES_TEMPLATE_NAME, ".watch-history-6"), nodes);
Map<String, Integer> existingTemplates = new HashMap<>();
existingTemplates.put(WatcherIndexTemplateRegistryField.TRIGGERED_TEMPLATE_NAME, null);
existingTemplates.put(WatcherIndexTemplateRegistryField.WATCHES_TEMPLATE_NAME, null);
existingTemplates.put(".watch-history-6", null);
ClusterChangedEvent event = createClusterChangedEvent(existingTemplates, nodes);
registry.clusterChanged(event);
verifyZeroInteractions(client);
}
private ClusterChangedEvent createClusterChangedEvent(List<String> existingTemplateNames, DiscoveryNodes nodes) {
private ClusterChangedEvent createClusterChangedEvent(Map<String, Integer> existingTemplateNames, DiscoveryNodes nodes) {
return createClusterChangedEvent(existingTemplateNames, Collections.emptyMap(), nodes);
}
private ClusterState createClusterState(Settings nodeSettings,
List<String> existingTemplateNames,
Map<String, Integer> existingTemplates,
Map<String, LifecyclePolicy> existingPolicies,
DiscoveryNodes nodes) {
ImmutableOpenMap.Builder<String, IndexTemplateMetaData> indexTemplates = ImmutableOpenMap.builder();
for (String name : existingTemplateNames) {
indexTemplates.put(name, mock(IndexTemplateMetaData.class));
for (Map.Entry<String, Integer> template : existingTemplates.entrySet()) {
final IndexTemplateMetaData mockTemplate = mock(IndexTemplateMetaData.class);
when(mockTemplate.version()).thenReturn(template.getValue());
when(mockTemplate.getVersion()).thenReturn(template.getValue());
indexTemplates.put(template.getKey(), mockTemplate);
}
Map<String, LifecyclePolicyMetadata> existingILMMeta = existingPolicies.entrySet().stream()
@ -286,17 +327,17 @@ public class WatcherIndexTemplateRegistryTests extends ESTestCase {
.build();
}
private ClusterChangedEvent createClusterChangedEvent(List<String> existingTemplateNames,
private ClusterChangedEvent createClusterChangedEvent(Map<String, Integer> existingTemplateNames,
Map<String, LifecyclePolicy> existingPolicies,
DiscoveryNodes nodes) {
return createClusterChangedEvent(Settings.EMPTY, existingTemplateNames, existingPolicies, nodes);
}
private ClusterChangedEvent createClusterChangedEvent(Settings nodeSettings,
List<String> existingTemplateNames,
Map<String, Integer> existingTemplates,
Map<String, LifecyclePolicy> existingPolicies,
DiscoveryNodes nodes) {
ClusterState cs = createClusterState(nodeSettings, existingTemplateNames, existingPolicies, nodes);
ClusterState cs = createClusterState(nodeSettings, existingTemplates, existingPolicies, nodes);
ClusterChangedEvent realEvent = new ClusterChangedEvent("created-from-test", cs,
ClusterState.builder(new ClusterName("test")).build());
ClusterChangedEvent event = spy(realEvent);
@ -305,10 +346,11 @@ public class WatcherIndexTemplateRegistryTests extends ESTestCase {
return event;
}
private ClusterState createClusterState(String ... existingTemplates) {
private ClusterState createClusterState(Map<String, Integer> existingTemplates) {
MetaData.Builder metaDataBuilder = MetaData.builder();
for (String templateName : existingTemplates) {
metaDataBuilder.put(IndexTemplateMetaData.builder(templateName)
for (Map.Entry<String, Integer> template : existingTemplates.entrySet()) {
metaDataBuilder.put(IndexTemplateMetaData.builder(template.getKey())
.version(template.getValue())
.patterns(Arrays.asList(generateRandomStringArray(10, 100, false, false))));
}