honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Watcher: Ensure no template deletion race in mixed cluster environment (elastic/x-pack-elasticsearch#1964) 

When having a mixed cluster with 5.6 and 6.0 nodes, the template upgrade
service has a cluster state listener that deletes the old watches and
triggered_watches index template. However during that time the 5.6 node
WatcherIndexTemplateRegistry checks if the templates are missing and
adds them back. This results in a race, because the new .watches index
template does not get added by the WatcherIndexTemplateRegistry when the
6.0 node is node a master node.

This commit circumvents this issue, by only deleting the watches and
triggered watches template during the upgrade process.

Original commit: elastic/x-pack-elasticsearch@71380f460a
This commit is contained in:
Alexander Reelsen 2017-07-11 16:50:41 +02:00 committed by GitHub
parent 5288791f41
commit 22b0560ccb
2 changed files with 35 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
plugin/src/main/java/org/elasticsearch/xpack/upgrade/Upgrade.java
View File

@ -5,10 +5,13 @@
*/
package org.elasticsearch.xpack.upgrade;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.Version;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.ActionRequest;
import org.elasticsearch.action.ActionResponse;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse;
import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplateResponse;
import org.elasticsearch.client.Client;
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
import org.elasticsearch.cluster.node.DiscoveryNodes;
@ -179,16 +182,37 @@ public class Upgrade implements ActionPlugin {
}
private static void postWatcherUpgrade(Client client, Boolean shouldStartWatcher, ActionListener<TransportResponse.Empty> listener) {
client.admin().indices().prepareDelete(".triggered_watches").execute(ActionListener.wrap(deleteIndexResponse -> {
startWatcherIfNeeded(shouldStartWatcher, client, listener);
}, e -> {
if (e instanceof IndexNotFoundException) {
startWatcherIfNeeded(shouldStartWatcher, client, listener);
} else {
listener.onFailure(e);
}
}
));
// if you are confused that these steps are numbered reversed, we are creating the action listeners first
// but only calling the deletion at the end of the method (inception style)
// step 3, after successful deletion of triggered watch index: start watcher
ActionListener<DeleteIndexResponse> deleteTriggeredWatchIndexResponse = ActionListener.wrap(deleteIndexResponse ->
startWatcherIfNeeded(shouldStartWatcher, client, listener), e -> {
if (e instanceof IndexNotFoundException) {
startWatcherIfNeeded(shouldStartWatcher, client, listener);
} else {
listener.onFailure(e);
}
});
// step 2, after acknowledged delete triggered watches template: delete triggered watches index
ActionListener<DeleteIndexTemplateResponse> triggeredWatchIndexTemplateListener = ActionListener.wrap(r -> {
if (r.isAcknowledged()) {
client.admin().indices().prepareDelete(".triggered_watches").execute(deleteTriggeredWatchIndexResponse);
} else {
listener.onFailure(new ElasticsearchException("Deleting triggered_watches template not acknowledged"));
}
}, listener::onFailure);
// step 1, after acknowledged watches template deletion: delete triggered_watches template
ActionListener<DeleteIndexTemplateResponse> watchIndexTemplateListener = ActionListener.wrap(r -> {
if (r.isAcknowledged()) {
client.admin().indices().prepareDeleteTemplate("triggered_watches").execute(triggeredWatchIndexTemplateListener);
} else {
listener.onFailure(new ElasticsearchException("Deleting watches template not acknowledged"));
}
}, listener::onFailure);
client.admin().indices().prepareDeleteTemplate("watches").execute(watchIndexTemplateListener);
}
private static void startWatcherIfNeeded(Boolean shouldStartWatcher, Client client, ActionListener<TransportResponse.Empty> listener) {

4
plugin/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java
View File

@ -514,9 +514,7 @@ public class Watcher implements ActionPlugin {
// These are all old templates from pre 6.0 era, that need to be deleted
public UnaryOperator<Map<String, IndexTemplateMetaData>> getIndexTemplateMetaDataUpgrader() {
return map -> {
map.keySet().removeIf(name -> "watches".equals(name) || "triggered_watches".equals(name)
|| name.startsWith("watch_history_"));
map.keySet().removeIf(name -> name.startsWith("watch_history_"));
return map;
};
}