From bdb1cabe7168887b6a84bb6f5e731d35b3067fef Mon Sep 17 00:00:00 2001 From: Simon Willnauer Date: Mon, 3 Apr 2017 09:52:09 +0200 Subject: [PATCH] Prevent nodes from joining if newer indices exist in the cluster (#23843) Today we prevent nodes from joining when indices exists that are too old. Yet, the opposite can happen too since lucene / elasticsearch is not forward compatible when it gets to indices we won't let nodes join the cluster once there are indices in the clusterstate that are newer than the nodes version. This prevents forward compatibility issues which we never test against. Yet, this will not prevent rolling restarts or anything like this since indices are always created with the minimum node version in the cluster such that an index can only get the version of the higher nodes once all nodes are upgraded to this version. --- .../discovery/zen/MembershipAction.java | 14 +++- .../discovery/zen/NodeJoinController.java | 2 +- .../discovery/zen/ZenDiscovery.java | 2 +- .../discovery/zen/MembershipActionTests.java | 80 +++++++++++++++++++ 4 files changed, 93 insertions(+), 5 deletions(-) create mode 100644 core/src/test/java/org/elasticsearch/discovery/zen/MembershipActionTests.java diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/MembershipAction.java b/core/src/main/java/org/elasticsearch/discovery/zen/MembershipAction.java index 6f56a547d3f..442491e6b13 100644 --- a/core/src/main/java/org/elasticsearch/discovery/zen/MembershipAction.java +++ b/core/src/main/java/org/elasticsearch/discovery/zen/MembershipAction.java @@ -178,20 +178,28 @@ public class MembershipAction extends AbstractComponent { @Override public void messageReceived(ValidateJoinRequest request, TransportChannel channel) throws Exception { - ensureIndexCompatibility(Version.CURRENT.minimumIndexCompatibilityVersion(), request.state.getMetaData()); + ensureIndexCompatibility(Version.CURRENT, request.state.getMetaData()); // for now, the mere fact that we can serialize the cluster state acts as validation.... channel.sendResponse(TransportResponse.Empty.INSTANCE); } } /** - * Ensures that all indices are compatible with the supported index version. + * Ensures that all indices are compatible with the given node version. This will ensure that all indices in the given metadata + * will not be created with a newer version of elasticsearch as well as that all indices are newer or equal to the minimum index + * compatibility version. + * @see Version#minimumIndexCompatibilityVersion() * @throws IllegalStateException if any index is incompatible with the given version */ - static void ensureIndexCompatibility(final Version supportedIndexVersion, MetaData metaData) { + static void ensureIndexCompatibility(final Version nodeVersion, MetaData metaData) { + Version supportedIndexVersion = nodeVersion.minimumIndexCompatibilityVersion(); // we ensure that all indices in the cluster we join are compatible with us no matter if they are // closed or not we can't read mappings of these indices so we need to reject the join... for (IndexMetaData idxMetaData : metaData) { + if (idxMetaData.getCreationVersion().after(nodeVersion)) { + throw new IllegalStateException("index " + idxMetaData.getIndex() + " version not supported: " + + idxMetaData.getCreationVersion() + " the node version is: " + nodeVersion); + } if (idxMetaData.getCreationVersion().before(supportedIndexVersion)) { throw new IllegalStateException("index " + idxMetaData.getIndex() + " version not supported: " + idxMetaData.getCreationVersion() + " minimum compatible index version is: " + supportedIndexVersion); diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/NodeJoinController.java b/core/src/main/java/org/elasticsearch/discovery/zen/NodeJoinController.java index 2d84f5f863d..354425a3dca 100644 --- a/core/src/main/java/org/elasticsearch/discovery/zen/NodeJoinController.java +++ b/core/src/main/java/org/elasticsearch/discovery/zen/NodeJoinController.java @@ -453,7 +453,7 @@ public class NodeJoinController extends AbstractComponent { } // we do this validation quite late to prevent race conditions between nodes joining and importing dangling indices // we have to reject nodes that don't support all indices we have in this cluster - MembershipAction.ensureIndexCompatibility(minNodeVersion.minimumIndexCompatibilityVersion(), currentState.getMetaData()); + MembershipAction.ensureIndexCompatibility(minNodeVersion, currentState.getMetaData()); if (nodesChanged) { newState.nodes(nodesBuilder); return results.build(allocationService.reroute(newState.build(), "node_join")); diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java b/core/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java index be6f52fc22c..7b24536346c 100644 --- a/core/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java +++ b/core/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java @@ -839,7 +839,7 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover } else { // we do this in a couple of places including the cluster update thread. This one here is really just best effort // to ensure we fail as fast as possible. - MembershipAction.ensureIndexCompatibility(node.getVersion().minimumIndexCompatibilityVersion(), state.getMetaData()); + MembershipAction.ensureIndexCompatibility(node.getVersion(), state.getMetaData()); // try and connect to the node, if it fails, we can raise an exception back to the client... transportService.connectToNode(node); diff --git a/core/src/test/java/org/elasticsearch/discovery/zen/MembershipActionTests.java b/core/src/test/java/org/elasticsearch/discovery/zen/MembershipActionTests.java new file mode 100644 index 00000000000..b8d9f175e64 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/discovery/zen/MembershipActionTests.java @@ -0,0 +1,80 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.discovery.zen; + +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.VersionUtils; + +public class MembershipActionTests extends ESTestCase { + + public void testPreventJoinClusterWithNewerIndices() { + Settings.builder().build(); + MetaData.Builder metaBuilder = MetaData.builder(); + IndexMetaData indexMetaData = IndexMetaData.builder("test") + .settings(settings(Version.CURRENT)) + .numberOfShards(1) + .numberOfReplicas(1).build(); + metaBuilder.put(indexMetaData, false); + MetaData metaData = metaBuilder.build(); + MembershipAction.ensureIndexCompatibility(Version.CURRENT, metaData); + + expectThrows(IllegalStateException.class, () -> + MembershipAction.ensureIndexCompatibility(VersionUtils.getPreviousVersion(Version.CURRENT), + metaData)); + } + + public void testPreventJoinClusterWithUnsupportedIndices() { + Settings.builder().build(); + MetaData.Builder metaBuilder = MetaData.builder(); + IndexMetaData indexMetaData = IndexMetaData.builder("test") + .settings(settings(VersionUtils.getPreviousVersion(Version.CURRENT + .minimumIndexCompatibilityVersion()))) + .numberOfShards(1) + .numberOfReplicas(1).build(); + metaBuilder.put(indexMetaData, false); + MetaData metaData = metaBuilder.build(); + expectThrows(IllegalStateException.class, () -> + MembershipAction.ensureIndexCompatibility(Version.CURRENT, + metaData)); + } + + public void testSuccess() { + Settings.builder().build(); + MetaData.Builder metaBuilder = MetaData.builder(); + IndexMetaData indexMetaData = IndexMetaData.builder("test") + .settings(settings(VersionUtils.randomVersionBetween(random(), + Version.CURRENT.minimumIndexCompatibilityVersion(), Version.CURRENT))) + .numberOfShards(1) + .numberOfReplicas(1).build(); + metaBuilder.put(indexMetaData, false); + indexMetaData = IndexMetaData.builder("test1") + .settings(settings(VersionUtils.randomVersionBetween(random(), + Version.CURRENT.minimumIndexCompatibilityVersion(), Version.CURRENT))) + .numberOfShards(1) + .numberOfReplicas(1).build(); + metaBuilder.put(indexMetaData, false); + MetaData metaData = metaBuilder.build(); + MembershipAction.ensureIndexCompatibility(Version.CURRENT, + metaData); + } +}