Index deletes not applied when cluster UUID has changed
If a node was isolated from the cluster while a delete was happening, the node will ignore the deleted operation when rejoining as we couldn't detect whether the new master genuinely deleted the indices or it is a new fresh "reset" master that was started without the old data folder. We can now be smarter and detect these reset masters and actually delete the indices on the node if its not the case of a reset master. Note that this new protection doesn't hold if the node was shut down. In that case it's indices will still be imported as dangling indices. Closes #16825 Closes #11665
This commit is contained in:
Ali Beyad
parent
3d2a50c8ea
commit
83d1e0985d
|
|
@ -25,12 +25,12 @@ import org.elasticsearch.cluster.metadata.MetaData;
|
|||
import org.elasticsearch.cluster.node.DiscoveryNodes;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
|
||||
/**
|
||||
*
|
||||
* An event received by the local node, signaling that the cluster state has changed.
|
||||
*/
|
||||
public class ClusterChangedEvent {
|
||||
|
||||
|
|
@ -43,6 +43,9 @@ public class ClusterChangedEvent {
|
|||
private final DiscoveryNodes.Delta nodesDelta;
|
||||
|
||||
public ClusterChangedEvent(String source, ClusterState state, ClusterState previousState) {
|
||||
Objects.requireNonNull(source, "source must not be null");
|
||||
Objects.requireNonNull(state, "state must not be null");
|
||||
Objects.requireNonNull(previousState, "previousState must not be null");
|
||||
this.source = source;
|
||||
this.state = state;
|
||||
this.previousState = previousState;
|
||||
|
|
@ -56,19 +59,35 @@ public class ClusterChangedEvent {
|
|||
return this.source;
|
||||
}
|
||||
|
||||
/**
|
||||
* The new cluster state that caused this change event.
|
||||
*/
|
||||
public ClusterState state() {
|
||||
return this.state;
|
||||
}
|
||||
|
||||
/**
|
||||
* The previous cluster state for this change event.
|
||||
*/
|
||||
public ClusterState previousState() {
|
||||
return this.previousState;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns <code>true</code> iff the routing tables (for all indices) have
|
||||
* changed between the previous cluster state and the current cluster state.
|
||||
* Note that this is an object reference equality test, not an equals test.
|
||||
*/
|
||||
public boolean routingTableChanged() {
|
||||
return state.routingTable() != previousState.routingTable();
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns <code>true</code> iff the routing table has changed for the given index.
|
||||
* Note that this is an object reference equality test, not an equals test.
|
||||
*/
|
||||
public boolean indexRoutingTableChanged(String index) {
|
||||
Objects.requireNonNull(index, "index must not be null");
|
||||
if (!state.routingTable().hasIndex(index) && !previousState.routingTable().hasIndex(index)) {
|
||||
return false;
|
||||
}
|
||||
|
|
@ -82,9 +101,6 @@ public class ClusterChangedEvent {
|
|||
* Returns the indices created in this event
|
||||
*/
|
||||
public List<String> indicesCreated() {
|
||||
if (previousState == null) {
|
||||
return Arrays.asList(state.metaData().indices().keys().toArray(String.class));
|
||||
}
|
||||
if (!metaDataChanged()) {
|
||||
return Collections.emptyList();
|
||||
}
|
||||
|
|
@ -105,20 +121,14 @@ public class ClusterChangedEvent {
|
|||
* Returns the indices deleted in this event
|
||||
*/
|
||||
public List<String> indicesDeleted() {
|
||||
|
||||
// if the new cluster state has a new master then we cannot know if an index which is not in the cluster state
|
||||
// is actually supposed to be deleted or imported as dangling instead. for example a new master might not have
|
||||
// the index in its cluster state because it was started with an empty data folder and in this case we want to
|
||||
// import as dangling. we check here for new master too to be on the safe side in this case.
|
||||
// This means that under certain conditions deleted indices might be reimported if a master fails while the deletion
|
||||
// request is issued and a node receives the cluster state that would trigger the deletion from the new master.
|
||||
// See test MetaDataWriteDataNodesTests.testIndicesDeleted()
|
||||
// If the new cluster state has a new cluster UUID, the likely scenario is that a node was elected
|
||||
// master that has had its data directory wiped out, in which case we don't want to delete the indices and lose data;
|
||||
// rather we want to import them as dangling indices instead. So we check here if the cluster UUID differs from the previous
|
||||
// cluster UUID, in which case, we don't want to delete indices that the master erroneously believes shouldn't exist.
|
||||
// See test DiscoveryWithServiceDisruptionsIT.testIndicesDeleted()
|
||||
// See discussion on https://github.com/elastic/elasticsearch/pull/9952 and
|
||||
// https://github.com/elastic/elasticsearch/issues/11665
|
||||
if (hasNewMaster() || previousState == null) {
|
||||
return Collections.emptyList();
|
||||
}
|
||||
if (!metaDataChanged()) {
|
||||
if (metaDataChanged() == false || isNewCluster()) {
|
||||
return Collections.emptyList();
|
||||
}
|
||||
List<String> deleted = null;
|
||||
|
|
@ -134,10 +144,20 @@ public class ClusterChangedEvent {
|
|||
return deleted == null ? Collections.<String>emptyList() : deleted;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns <code>true</code> iff the metadata for the cluster has changed between
|
||||
* the previous cluster state and the new cluster state. Note that this is an object
|
||||
* reference equality test, not an equals test.
|
||||
*/
|
||||
public boolean metaDataChanged() {
|
||||
return state.metaData() != previousState.metaData();
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns <code>true</code> iff the {@link IndexMetaData} for a given index
|
||||
* has changed between the previous cluster state and the new cluster state.
|
||||
* Note that this is an object reference equality test, not an equals test.
|
||||
*/
|
||||
public boolean indexMetaDataChanged(IndexMetaData current) {
|
||||
MetaData previousMetaData = previousState.metaData();
|
||||
if (previousMetaData == null) {
|
||||
|
|
@ -152,46 +172,56 @@ public class ClusterChangedEvent {
|
|||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns <code>true</code> iff the cluster level blocks have changed between cluster states.
|
||||
* Note that this is an object reference equality test, not an equals test.
|
||||
*/
|
||||
public boolean blocksChanged() {
|
||||
return state.blocks() != previousState.blocks();
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns <code>true</code> iff the local node is the mater node of the cluster.
|
||||
*/
|
||||
public boolean localNodeMaster() {
|
||||
return state.nodes().localNodeMaster();
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the {@link org.elasticsearch.cluster.node.DiscoveryNodes.Delta} between
|
||||
* the previous cluster state and the new cluster state.
|
||||
*/
|
||||
public DiscoveryNodes.Delta nodesDelta() {
|
||||
return this.nodesDelta;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns <code>true</code> iff nodes have been removed from the cluster since the last cluster state.
|
||||
*/
|
||||
public boolean nodesRemoved() {
|
||||
return nodesDelta.removed();
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns <code>true</code> iff nodes have been added from the cluster since the last cluster state.
|
||||
*/
|
||||
public boolean nodesAdded() {
|
||||
return nodesDelta.added();
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns <code>true</code> iff nodes have been changed (added or removed) from the cluster since the last cluster state.
|
||||
*/
|
||||
public boolean nodesChanged() {
|
||||
return nodesRemoved() || nodesAdded();
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if this cluster state comes from a different master than the previous one.
|
||||
* This is a workaround for the scenario where a node misses a cluster state that has either
|
||||
* no master block or state not recovered flag set. In this case we must make sure that
|
||||
* if an index is missing from the cluster state is not deleted immediately but instead imported
|
||||
* as dangling. See discussion on https://github.com/elastic/elasticsearch/pull/9952
|
||||
*/
|
||||
private boolean hasNewMaster() {
|
||||
String oldMaster = previousState().getNodes().masterNodeId();
|
||||
String newMaster = state().getNodes().masterNodeId();
|
||||
if (oldMaster == null && newMaster == null) {
|
||||
return false;
|
||||
}
|
||||
if (oldMaster == null && newMaster != null) {
|
||||
return true;
|
||||
}
|
||||
return oldMaster.equals(newMaster) == false;
|
||||
// Determines whether or not the current cluster state represents an entirely
|
||||
// different cluster from the previous cluster state, which will happen when a
|
||||
// master node is elected that has never been part of the cluster before.
|
||||
private boolean isNewCluster() {
|
||||
final String prevClusterUUID = previousState.metaData().clusterUUID();
|
||||
final String currClusterUUID = state.metaData().clusterUUID();
|
||||
return prevClusterUUID.equals(currClusterUUID) == false;
|
||||
}
|
||||
}
|
||||
|
|
@ -46,6 +46,11 @@ import static org.elasticsearch.common.transport.TransportAddressSerializers.add
|
|||
*/
|
||||
public class DiscoveryNode implements Streamable, ToXContent {
|
||||
|
||||
public static final String DATA_ATTR = "data";
|
||||
public static final String MASTER_ATTR = "master";
|
||||
public static final String CLIENT_ATTR = "client";
|
||||
public static final String INGEST_ATTR = "ingest";
|
||||
|
||||
public static boolean localNode(Settings settings) {
|
||||
if (Node.NODE_LOCAL_SETTING.exists(settings)) {
|
||||
return Node.NODE_LOCAL_SETTING.get(settings);
|
||||
|
|
@ -274,7 +279,7 @@ public class DiscoveryNode implements Streamable, ToXContent {
|
|||
* Should this node hold data (shards) or not.
|
||||
*/
|
||||
public boolean dataNode() {
|
||||
String data = attributes.get("data");
|
||||
String data = attributes.get(DATA_ATTR);
|
||||
if (data == null) {
|
||||
return !clientNode();
|
||||
}
|
||||
|
|
@ -292,7 +297,7 @@ public class DiscoveryNode implements Streamable, ToXContent {
|
|||
* Is the node a client node or not.
|
||||
*/
|
||||
public boolean clientNode() {
|
||||
String client = attributes.get("client");
|
||||
String client = attributes.get(CLIENT_ATTR);
|
||||
return client != null && Booleans.parseBooleanExact(client);
|
||||
}
|
||||
|
||||
|
|
@ -304,7 +309,7 @@ public class DiscoveryNode implements Streamable, ToXContent {
|
|||
* Can this node become master or not.
|
||||
*/
|
||||
public boolean masterNode() {
|
||||
String master = attributes.get("master");
|
||||
String master = attributes.get(MASTER_ATTR);
|
||||
if (master == null) {
|
||||
return !clientNode();
|
||||
}
|
||||
|
|
@ -322,7 +327,7 @@ public class DiscoveryNode implements Streamable, ToXContent {
|
|||
* Returns a boolean that tells whether this an ingest node or not
|
||||
*/
|
||||
public boolean isIngestNode() {
|
||||
String ingest = attributes.get("ingest");
|
||||
String ingest = attributes.get(INGEST_ATTR);
|
||||
return ingest == null ? true : Booleans.parseBooleanExact(ingest);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,375 @@
|
|||
/*
|
||||
* Licensed to Elasticsearch under one or more contributor
|
||||
* license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright
|
||||
* ownership. Elasticsearch licenses this file to you under
|
||||
* the Apache License, Version 2.0 (the "License"); you may
|
||||
* not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
package org.elasticsearch.cluster;
|
||||
|
||||
import com.carrotsearch.hppc.cursors.ObjectCursor;
|
||||
import org.elasticsearch.Version;
|
||||
import org.elasticsearch.cluster.metadata.IndexMetaData;
|
||||
import org.elasticsearch.cluster.metadata.MetaData;
|
||||
import org.elasticsearch.cluster.node.DiscoveryNode;
|
||||
import org.elasticsearch.cluster.node.DiscoveryNodes;
|
||||
import org.elasticsearch.cluster.routing.RoutingTable;
|
||||
import org.elasticsearch.common.Strings;
|
||||
import org.elasticsearch.common.collect.MapBuilder;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.common.transport.DummyTransportAddress;
|
||||
import org.elasticsearch.test.ESTestCase;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.hamcrest.Matchers.equalTo;
|
||||
|
||||
/**
|
||||
* Tests for the {@link ClusterChangedEvent} class.
|
||||
*/
|
||||
public class ClusterChangedEventTests extends ESTestCase {
|
||||
|
||||
private static final ClusterName TEST_CLUSTER_NAME = new ClusterName("test");
|
||||
private static final int INDICES_CHANGE_NUM_TESTS = 5;
|
||||
private static final String NODE_ID_PREFIX = "node_";
|
||||
private static final String INITIAL_CLUSTER_ID = Strings.randomBase64UUID();
|
||||
// the initial indices which every cluster state test starts out with
|
||||
private static final List<String> initialIndices = Arrays.asList("idx1", "idx2", "idx3");
|
||||
// index settings
|
||||
private static final Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build();
|
||||
|
||||
/**
|
||||
* Test basic properties of the ClusterChangedEvent class:
|
||||
* (1) make sure there are no null values for any of its properties
|
||||
* (2) make sure you can't create a ClusterChangedEvent with any null values
|
||||
*/
|
||||
public void testBasicProperties() {
|
||||
ClusterState newState = createSimpleClusterState();
|
||||
ClusterState previousState = createSimpleClusterState();
|
||||
ClusterChangedEvent event = new ClusterChangedEvent("_na_", newState, previousState);
|
||||
assertThat(event.source(), equalTo("_na_"));
|
||||
assertThat(event.state(), equalTo(newState));
|
||||
assertThat(event.previousState(), equalTo(previousState));
|
||||
assertNotNull("nodesDelta should not be null", event.nodesDelta());
|
||||
|
||||
// should not be able to create a ClusterChangedEvent with null values for any of the constructor args
|
||||
try {
|
||||
event = new ClusterChangedEvent(null, newState, previousState);
|
||||
fail("should not have created a ClusterChangedEvent from a null source: " + event.source());
|
||||
} catch (NullPointerException e) {
|
||||
}
|
||||
try {
|
||||
event = new ClusterChangedEvent("_na_", null, previousState);
|
||||
fail("should not have created a ClusterChangedEvent from a null state: " + event.state());
|
||||
} catch (NullPointerException e) {
|
||||
}
|
||||
try {
|
||||
event = new ClusterChangedEvent("_na_", newState, null);
|
||||
fail("should not have created a ClusterChangedEvent from a null previousState: " + event.previousState());
|
||||
} catch (NullPointerException e) {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Test whether the ClusterChangedEvent returns the correct value for whether the local node is master,
|
||||
* based on what was set on the cluster state.
|
||||
*/
|
||||
public void testLocalNodeIsMaster() {
|
||||
final int numNodesInCluster = 3;
|
||||
ClusterState previousState = createSimpleClusterState();
|
||||
ClusterState newState = createState(numNodesInCluster, true, initialIndices);
|
||||
ClusterChangedEvent event = new ClusterChangedEvent("_na_", newState, previousState);
|
||||
assertTrue("local node should be master", event.localNodeMaster());
|
||||
|
||||
newState = createState(numNodesInCluster, false, initialIndices);
|
||||
event = new ClusterChangedEvent("_na_", newState, previousState);
|
||||
assertFalse("local node should not be master", event.localNodeMaster());
|
||||
}
|
||||
|
||||
/**
|
||||
* Test that the indices created and indices deleted lists between two cluster states
|
||||
* are correct when there is no change in the cluster UUID. Also tests metadata equality
|
||||
* between cluster states.
|
||||
*/
|
||||
public void testMetaDataChangesOnNoMasterChange() {
|
||||
metaDataChangesCheck(false);
|
||||
}
|
||||
|
||||
/**
|
||||
* Test that the indices created and indices deleted lists between two cluster states
|
||||
* are correct when there is a change in the cluster UUID. Also tests metadata equality
|
||||
* between cluster states.
|
||||
*/
|
||||
public void testMetaDataChangesOnNewClusterUUID() {
|
||||
metaDataChangesCheck(true);
|
||||
}
|
||||
|
||||
/**
|
||||
* Test the index metadata change check.
|
||||
*/
|
||||
public void testIndexMetaDataChange() {
|
||||
final int numNodesInCluster = 3;
|
||||
final ClusterState originalState = createState(numNodesInCluster, randomBoolean(), initialIndices);
|
||||
final ClusterState newState = originalState; // doesn't matter for this test, just need a non-null value
|
||||
final ClusterChangedEvent event = new ClusterChangedEvent("_na_", originalState, newState);
|
||||
|
||||
// test when its not the same IndexMetaData
|
||||
final String indexId = initialIndices.get(0);
|
||||
final IndexMetaData originalIndexMeta = originalState.metaData().index(indexId);
|
||||
// make sure the metadata is actually on the cluster state
|
||||
assertNotNull("IndexMetaData for " + indexId + " should exist on the cluster state", originalIndexMeta);
|
||||
IndexMetaData newIndexMeta = createIndexMetadata(indexId, originalIndexMeta.getVersion() + 1);
|
||||
assertTrue("IndexMetaData with different version numbers must be considered changed", event.indexMetaDataChanged(newIndexMeta));
|
||||
|
||||
// test when it doesn't exist
|
||||
newIndexMeta = createIndexMetadata("doesntexist");
|
||||
assertTrue("IndexMetaData that didn't previously exist should be considered changed", event.indexMetaDataChanged(newIndexMeta));
|
||||
|
||||
// test when its the same IndexMetaData
|
||||
assertFalse("IndexMetaData should be the same", event.indexMetaDataChanged(originalIndexMeta));
|
||||
}
|
||||
|
||||
/**
|
||||
* Test nodes added/removed/changed checks.
|
||||
*/
|
||||
public void testNodesAddedAndRemovedAndChanged() {
|
||||
final int numNodesInCluster = 4;
|
||||
final ClusterState originalState = createState(numNodesInCluster, randomBoolean(), initialIndices);
|
||||
|
||||
// test when nodes have not been added or removed between cluster states
|
||||
ClusterState newState = createState(numNodesInCluster, randomBoolean(), initialIndices);
|
||||
ClusterChangedEvent event = new ClusterChangedEvent("_na_", newState, originalState);
|
||||
assertFalse("Nodes should not have been added between cluster states", event.nodesAdded());
|
||||
assertFalse("Nodes should not have been removed between cluster states", event.nodesRemoved());
|
||||
assertFalse("Nodes should not have been changed between cluster states", event.nodesChanged());
|
||||
|
||||
// test when nodes have been removed between cluster states
|
||||
newState = createState(numNodesInCluster - 1, randomBoolean(), initialIndices);
|
||||
event = new ClusterChangedEvent("_na_", newState, originalState);
|
||||
assertTrue("Nodes should have been removed between cluster states", event.nodesRemoved());
|
||||
assertFalse("Nodes should not have been added between cluster states", event.nodesAdded());
|
||||
assertTrue("Nodes should have been changed between cluster states", event.nodesChanged());
|
||||
|
||||
// test when nodes have been added between cluster states
|
||||
newState = createState(numNodesInCluster + 1, randomBoolean(), initialIndices);
|
||||
event = new ClusterChangedEvent("_na_", newState, originalState);
|
||||
assertFalse("Nodes should not have been removed between cluster states", event.nodesRemoved());
|
||||
assertTrue("Nodes should have been added between cluster states", event.nodesAdded());
|
||||
assertTrue("Nodes should have been changed between cluster states", event.nodesChanged());
|
||||
|
||||
// test when nodes both added and removed between cluster states
|
||||
// here we reuse the newState from the previous run which already added extra nodes
|
||||
newState = nextState(newState, randomBoolean(), Collections.emptyList(), Collections.emptyList(), 1);
|
||||
event = new ClusterChangedEvent("_na_", newState, originalState);
|
||||
assertTrue("Nodes should have been removed between cluster states", event.nodesRemoved());
|
||||
assertTrue("Nodes should have been added between cluster states", event.nodesAdded());
|
||||
assertTrue("Nodes should have been changed between cluster states", event.nodesChanged());
|
||||
}
|
||||
|
||||
/**
|
||||
* Test the routing table changes checks.
|
||||
*/
|
||||
public void testRoutingTableChanges() {
|
||||
final int numNodesInCluster = 3;
|
||||
final ClusterState originalState = createState(numNodesInCluster, randomBoolean(), initialIndices);
|
||||
|
||||
// routing tables and index routing tables are same object
|
||||
ClusterState newState = ClusterState.builder(originalState).build();
|
||||
ClusterChangedEvent event = new ClusterChangedEvent("_na_", originalState, newState);
|
||||
assertFalse("routing tables should be the same object", event.routingTableChanged());
|
||||
assertFalse("index routing table should be the same object", event.indexRoutingTableChanged(initialIndices.get(0)));
|
||||
|
||||
// routing tables and index routing tables aren't same object
|
||||
newState = createState(numNodesInCluster, randomBoolean(), initialIndices);
|
||||
event = new ClusterChangedEvent("_na_", originalState, newState);
|
||||
assertTrue("routing tables should not be the same object", event.routingTableChanged());
|
||||
assertTrue("index routing table should not be the same object", event.indexRoutingTableChanged(initialIndices.get(0)));
|
||||
|
||||
// index routing tables are different because they don't exist
|
||||
newState = createState(numNodesInCluster, randomBoolean(), initialIndices.subList(1, initialIndices.size()));
|
||||
event = new ClusterChangedEvent("_na_", originalState, newState);
|
||||
assertTrue("routing tables should not be the same object", event.routingTableChanged());
|
||||
assertTrue("index routing table should not be the same object", event.indexRoutingTableChanged(initialIndices.get(0)));
|
||||
}
|
||||
|
||||
// Tests that the indices change list is correct as well as metadata equality when the metadata has changed.
|
||||
private static void metaDataChangesCheck(final boolean changeClusterUUID) {
|
||||
final int numNodesInCluster = 3;
|
||||
for (int i = 0; i < INDICES_CHANGE_NUM_TESTS; i++) {
|
||||
final ClusterState previousState = createState(numNodesInCluster, randomBoolean(), initialIndices);
|
||||
final int numAdd = randomIntBetween(0, 5); // add random # of indices to the next cluster state
|
||||
final int numDel = randomIntBetween(0, initialIndices.size()); // delete random # of indices from the next cluster state
|
||||
final List<String> addedIndices = addIndices(numAdd);
|
||||
final List<String> delIndices = delIndices(numDel, initialIndices);
|
||||
final ClusterState newState = nextState(previousState, changeClusterUUID, addedIndices, delIndices, 0);
|
||||
final ClusterChangedEvent event = new ClusterChangedEvent("_na_", newState, previousState);
|
||||
final List<String> addsFromEvent = event.indicesCreated();
|
||||
final List<String> delsFromEvent = event.indicesDeleted();
|
||||
Collections.sort(addsFromEvent);
|
||||
Collections.sort(delsFromEvent);
|
||||
assertThat(addsFromEvent, equalTo(addedIndices));
|
||||
assertThat(delsFromEvent, changeClusterUUID ? equalTo(Collections.emptyList()) : equalTo(delIndices));
|
||||
assertThat(event.metaDataChanged(), equalTo(changeClusterUUID || addedIndices.size() > 0 || delIndices.size() > 0));
|
||||
}
|
||||
}
|
||||
|
||||
private static ClusterState createSimpleClusterState() {
|
||||
return ClusterState.builder(TEST_CLUSTER_NAME).build();
|
||||
}
|
||||
|
||||
// Create a basic cluster state with a given set of indices
|
||||
private static ClusterState createState(final int numNodes, final boolean isLocalMaster, final List<String> indices) {
|
||||
final MetaData metaData = createMetaData(indices);
|
||||
return ClusterState.builder(TEST_CLUSTER_NAME)
|
||||
.nodes(createDiscoveryNodes(numNodes, isLocalMaster))
|
||||
.metaData(metaData)
|
||||
.routingTable(createRoutingTable(1, metaData))
|
||||
.build();
|
||||
}
|
||||
|
||||
// Create a modified cluster state from another one, but with some number of indices added and deleted.
|
||||
private static ClusterState nextState(final ClusterState previousState, final boolean changeClusterUUID,
|
||||
final List<String> addedIndices, final List<String> deletedIndices,
|
||||
final int numNodesToRemove) {
|
||||
final ClusterState.Builder builder = ClusterState.builder(previousState);
|
||||
builder.stateUUID(Strings.randomBase64UUID());
|
||||
final MetaData.Builder metaBuilder = MetaData.builder(previousState.metaData());
|
||||
if (changeClusterUUID || addedIndices.size() > 0 || deletedIndices.size() > 0) {
|
||||
// there is some change in metadata cluster state
|
||||
if (changeClusterUUID) {
|
||||
metaBuilder.clusterUUID(Strings.randomBase64UUID());
|
||||
}
|
||||
for (String index : addedIndices) {
|
||||
metaBuilder.put(createIndexMetadata(index), true);
|
||||
}
|
||||
for (String index : deletedIndices) {
|
||||
metaBuilder.remove(index);
|
||||
}
|
||||
builder.metaData(metaBuilder);
|
||||
}
|
||||
if (numNodesToRemove > 0) {
|
||||
final int discoveryNodesSize = previousState.getNodes().size();
|
||||
final DiscoveryNodes.Builder nodesBuilder = DiscoveryNodes.builder(previousState.getNodes());
|
||||
for (int i = 0; i < numNodesToRemove && i < discoveryNodesSize; i++) {
|
||||
nodesBuilder.remove(NODE_ID_PREFIX + i);
|
||||
}
|
||||
builder.nodes(nodesBuilder);
|
||||
}
|
||||
return builder.build();
|
||||
}
|
||||
|
||||
// Create the discovery nodes for a cluster state. For our testing purposes, we want
|
||||
// the first to be master, the second to be master eligible, the third to be a data node,
|
||||
// and the remainder can be any kinds of nodes (master eligible, data, or both).
|
||||
private static DiscoveryNodes createDiscoveryNodes(final int numNodes, final boolean isLocalMaster) {
|
||||
assert (numNodes >= 3) : "the initial cluster state for event change tests should have a minimum of 3 nodes " +
|
||||
"so there are a minimum of 2 master nodes for testing master change events.";
|
||||
final DiscoveryNodes.Builder builder = DiscoveryNodes.builder();
|
||||
final int localNodeIndex = isLocalMaster ? 0 : randomIntBetween(1, numNodes - 1); // randomly assign the local node if not master
|
||||
for (int i = 0; i < numNodes; i++) {
|
||||
final String nodeId = NODE_ID_PREFIX + i;
|
||||
boolean isMasterEligible = false;
|
||||
boolean isData = false;
|
||||
if (i == 0) {
|
||||
// the master node
|
||||
builder.masterNodeId(nodeId);
|
||||
isMasterEligible = true;
|
||||
} else if (i == 1) {
|
||||
// the alternate master node
|
||||
isMasterEligible = true;
|
||||
} else if (i == 2) {
|
||||
// we need at least one data node
|
||||
isData = true;
|
||||
} else {
|
||||
// remaining nodes can be anything (except for master)
|
||||
isMasterEligible = randomBoolean();
|
||||
isData = randomBoolean();
|
||||
}
|
||||
final DiscoveryNode node = newNode(nodeId, isMasterEligible, isData);
|
||||
builder.put(node);
|
||||
if (i == localNodeIndex) {
|
||||
builder.localNodeId(nodeId);
|
||||
}
|
||||
}
|
||||
return builder.build();
|
||||
}
|
||||
|
||||
// Create a new DiscoveryNode
|
||||
private static DiscoveryNode newNode(final String nodeId, boolean isMasterEligible, boolean isData) {
|
||||
final Map<String, String> attributes = MapBuilder.<String, String>newMapBuilder()
|
||||
.put(DiscoveryNode.MASTER_ATTR, isMasterEligible ? "true" : "false")
|
||||
.put(DiscoveryNode.DATA_ATTR, isData ? "true": "false")
|
||||
.immutableMap();
|
||||
return new DiscoveryNode(nodeId, nodeId, DummyTransportAddress.INSTANCE, attributes, Version.CURRENT);
|
||||
}
|
||||
|
||||
// Create the metadata for a cluster state.
|
||||
private static MetaData createMetaData(final List<String> indices) {
|
||||
final MetaData.Builder builder = MetaData.builder();
|
||||
builder.clusterUUID(INITIAL_CLUSTER_ID);
|
||||
for (String index : indices) {
|
||||
builder.put(createIndexMetadata(index), true);
|
||||
}
|
||||
return builder.build();
|
||||
}
|
||||
|
||||
// Create the index metadata for a given index.
|
||||
private static IndexMetaData createIndexMetadata(final String index) {
|
||||
return createIndexMetadata(index, 1);
|
||||
}
|
||||
|
||||
// Create the index metadata for a given index, with the specified version.
|
||||
private static IndexMetaData createIndexMetadata(final String index, final long version) {
|
||||
return IndexMetaData.builder(index)
|
||||
.settings(settings)
|
||||
.numberOfShards(1)
|
||||
.numberOfReplicas(0)
|
||||
.creationDate(System.currentTimeMillis())
|
||||
.version(version)
|
||||
.build();
|
||||
}
|
||||
|
||||
// Create the routing table for a cluster state.
|
||||
private static RoutingTable createRoutingTable(final long version, final MetaData metaData) {
|
||||
final RoutingTable.Builder builder = RoutingTable.builder().version(version);
|
||||
for (ObjectCursor<IndexMetaData> cursor : metaData.indices().values()) {
|
||||
builder.addAsNew(cursor.value);
|
||||
}
|
||||
return builder.build();
|
||||
}
|
||||
|
||||
// Create a list of indices to add
|
||||
private static List<String> addIndices(final int numIndices) {
|
||||
final List<String> list = new ArrayList<>();
|
||||
for (int i = 0; i < numIndices; i++) {
|
||||
list.add("newIdx_" + i);
|
||||
}
|
||||
return list;
|
||||
}
|
||||
|
||||
// Create a list of indices to delete from a list that already belongs to a particular cluster state.
|
||||
private static List<String> delIndices(final int numIndices, final List<String> currIndices) {
|
||||
final List<String> list = new ArrayList<>();
|
||||
for (int i = 0; i < numIndices; i++) {
|
||||
list.add(currIndices.get(i));
|
||||
}
|
||||
return list;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -581,8 +581,7 @@ public class DiscoveryWithServiceDisruptionsIT extends ESIntegTestCase {
|
|||
|
||||
// restore GC
|
||||
masterNodeDisruption.stopDisrupting();
|
||||
ensureStableCluster(3, new TimeValue(DISRUPTION_HEALING_OVERHEAD.millis() + masterNodeDisruption.expectedTimeToHeal().millis()), false,
|
||||
oldNonMasterNodes.get(0));
|
||||
ensureStableCluster(3, new TimeValue(DISRUPTION_HEALING_OVERHEAD.millis() + masterNodeDisruption.expectedTimeToHeal().millis()), false, oldNonMasterNodes.get(0));
|
||||
|
||||
// make sure all nodes agree on master
|
||||
String newMaster = internalCluster().getMasterName();
|
||||
|
|
@ -1072,11 +1071,13 @@ public class DiscoveryWithServiceDisruptionsIT extends ESIntegTestCase {
|
|||
assertTrue(client().prepareGet("index", "doc", "1").get().isExists());
|
||||
}
|
||||
|
||||
// tests if indices are really deleted even if a master transition inbetween
|
||||
@AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/11665")
|
||||
/**
|
||||
* Tests that indices are properly deleted even if there is a master transition in between.
|
||||
* Test for https://github.com/elastic/elasticsearch/issues/11665
|
||||
*/
|
||||
public void testIndicesDeleted() throws Exception {
|
||||
configureUnicastCluster(3, null, 2);
|
||||
InternalTestCluster.Async<List<String>> masterNodes= internalCluster().startMasterOnlyNodesAsync(2);
|
||||
InternalTestCluster.Async<List<String>> masterNodes = internalCluster().startMasterOnlyNodesAsync(2);
|
||||
InternalTestCluster.Async<String> dataNode = internalCluster().startDataOnlyNodeAsync();
|
||||
dataNode.get();
|
||||
masterNodes.get();
|
||||
|
|
|
|||
Loading…
Reference in New Issue