honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[CCR] Only auto follow indices when all primary shards have started (#35814) 

This change adds an extra check that verifies that all primary shards
have been started of an index that is about to be auto followed.

If not all primary shards have been started for an index
then the next auto follow run will try to follow to auto follow
this index again.

Closes #35480
This commit is contained in:
Martijn van Groningen 2018-11-29 09:46:09 +01:00 committed by GitHub
parent fbdfec4305
commit 1390f366d4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 97 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/AutoFollowCoordinator.java
View File

@ -19,6 +19,7 @@ import org.elasticsearch.cluster.ClusterStateApplier;
import org.elasticsearch.cluster.ClusterStateUpdateTask;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.metadata.MetaData;
import org.elasticsearch.cluster.routing.IndexRoutingTable;
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.settings.Settings;
@ -164,6 +165,7 @@ public class AutoFollowCoordinator implements ClusterStateApplier {
final ClusterStateRequest request = new ClusterStateRequest();
request.clear();
request.metaData(true);
request.routingTable(true);
// TODO: set non-compliant status on auto-follow coordination that can be viewed via a stats API
ccrLicenseChecker.checkRemoteClusterLicenseAndFetchClusterState(
client,
@ -367,7 +369,14 @@ public class AutoFollowCoordinator implements ClusterStateApplier {
List<Index> leaderIndicesToFollow = new ArrayList<>();
for (IndexMetaData leaderIndexMetaData : leaderClusterState.getMetaData()) {
if (autoFollowPattern.match(leaderIndexMetaData.getIndex().getName())) {
if (followedIndexUUIDs.contains(leaderIndexMetaData.getIndex().getUUID()) == false) {
IndexRoutingTable indexRoutingTable = leaderClusterState.routingTable().index(leaderIndexMetaData.getIndex());
if (indexRoutingTable != null &&
// Leader indices can be in the cluster state, but not all primary shards may be ready yet.
// This checks ensures all primary shards have started, so that index following does not fail.
// If not all primary shards are ready, then the next time the auto follow coordinator runs
// this index will be auto followed.
indexRoutingTable.allPrimaryShardsActive() &&
followedIndexUUIDs.contains(leaderIndexMetaData.getIndex().getUUID()) == false) {
// TODO: iterate over the indices in the followerClusterState and check whether a IndexMetaData
// has a leader index uuid custom metadata entry that matches with uuid of leaderIndexMetaData variable
// If so then handle it differently: not follow it, but just add an entry to

108
x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/AutoFollowCoordinatorTests.java
View File

@ -11,6 +11,11 @@ import org.elasticsearch.cluster.ClusterName;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.metadata.MetaData;
import org.elasticsearch.cluster.routing.IndexRoutingTable;
import org.elasticsearch.cluster.routing.RoutingTable;
import org.elasticsearch.cluster.routing.ShardRouting;
import org.elasticsearch.cluster.routing.ShardRoutingState;
import org.elasticsearch.cluster.routing.TestShardRouting;
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.settings.Settings;
@ -49,12 +54,7 @@ public class AutoFollowCoordinatorTests extends ESTestCase {
Client client = mock(Client.class);
when(client.getRemoteClusterClient(anyString())).thenReturn(client);
ClusterState leaderState = ClusterState.builder(new ClusterName("remote"))
.metaData(MetaData.builder().put(IndexMetaData.builder("logs-20190101")
.settings(settings(Version.CURRENT).put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true))
.numberOfShards(1)
.numberOfReplicas(0)))
.build();
ClusterState leaderState = createRemoteClusterState("logs-20190101");
AutoFollowPattern autoFollowPattern = new AutoFollowPattern("remote", Collections.singletonList("logs-*"),
null, null, null, null, null, null, null, null, null, null, null);
@ -168,13 +168,7 @@ public class AutoFollowCoordinatorTests extends ESTestCase {
public void testAutoFollowerUpdateClusterStateFailure() {
Client client = mock(Client.class);
when(client.getRemoteClusterClient(anyString())).thenReturn(client);
ClusterState leaderState = ClusterState.builder(new ClusterName("remote"))
.metaData(MetaData.builder().put(IndexMetaData.builder("logs-20190101")
.settings(settings(Version.CURRENT).put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true))
.numberOfShards(1)
.numberOfReplicas(0)))
.build();
ClusterState leaderState = createRemoteClusterState("logs-20190101");
AutoFollowPattern autoFollowPattern = new AutoFollowPattern("remote", Collections.singletonList("logs-*"),
null, null, null, null, null, null, null, null, null, null, null);
@ -230,13 +224,7 @@ public class AutoFollowCoordinatorTests extends ESTestCase {
public void testAutoFollowerCreateAndFollowApiCallFailure() {
Client client = mock(Client.class);
when(client.getRemoteClusterClient(anyString())).thenReturn(client);
ClusterState leaderState = ClusterState.builder(new ClusterName("remote"))
.metaData(MetaData.builder().put(IndexMetaData.builder("logs-20190101")
.settings(settings(Version.CURRENT).put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true))
.numberOfShards(1)
.numberOfReplicas(0)))
.build();
ClusterState leaderState = createRemoteClusterState("logs-20190101");
AutoFollowPattern autoFollowPattern = new AutoFollowPattern("remote", Collections.singletonList("logs-*"),
null, null, null, null, null, null, null, null, null, null, null);
@ -299,24 +287,39 @@ public class AutoFollowCoordinatorTests extends ESTestCase {
new AutoFollowMetadata(Collections.singletonMap("remote", autoFollowPattern), Collections.emptyMap(), headers)))
.build();
RoutingTable.Builder routingTableBuilder = RoutingTable.builder();
MetaData.Builder imdBuilder = MetaData.builder();
for (int i = 0; i < 5; i++) {
String indexName = "metrics-" + i;
Settings.Builder builder = Settings.builder()
.put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
.put(IndexMetaData.SETTING_INDEX_UUID, "metrics-" + i)
.put(IndexMetaData.SETTING_INDEX_UUID, indexName)
.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), i % 2 == 0);
imdBuilder.put(IndexMetaData.builder("metrics-" + i)
.settings(builder)
.numberOfShards(1)
.numberOfReplicas(0));
ShardRouting shardRouting =
TestShardRouting.newShardRouting(indexName, 0, "1", true, ShardRoutingState.INITIALIZING).moveToStarted();
IndexRoutingTable indexRoutingTable = IndexRoutingTable.builder(imdBuilder.get(indexName).getIndex())
.addShard(shardRouting)
.build();
routingTableBuilder.add(indexRoutingTable);
}
imdBuilder.put(IndexMetaData.builder("logs-0")
.settings(settings(Version.CURRENT))
.numberOfShards(1)
.numberOfReplicas(0));
ShardRouting shardRouting =
TestShardRouting.newShardRouting("logs-0", 0, "1", true, ShardRoutingState.INITIALIZING).moveToStarted();
IndexRoutingTable indexRoutingTable = IndexRoutingTable.builder(imdBuilder.get("logs-0").getIndex()).addShard(shardRouting).build();
routingTableBuilder.add(indexRoutingTable);
ClusterState leaderState = ClusterState.builder(new ClusterName("remote"))
.metaData(imdBuilder)
.routingTable(routingTableBuilder.build())
.build();
List<Index> result = AutoFollower.getLeaderIndicesToFollow("remote", autoFollowPattern, leaderState, followerState,
@ -335,6 +338,52 @@ public class AutoFollowCoordinatorTests extends ESTestCase {
assertThat(result.get(1).getName(), equalTo("metrics-4"));
}
public void testGetLeaderIndicesToFollow_shardsNotStarted() {
AutoFollowPattern autoFollowPattern = new AutoFollowPattern("remote", Collections.singletonList("*"),
null, null, null, null, null, null, null, null, null, null, null);
Map<String, Map<String, String>> headers = new HashMap<>();
ClusterState followerState = ClusterState.builder(new ClusterName("remote"))
.metaData(MetaData.builder().putCustom(AutoFollowMetadata.TYPE,
new AutoFollowMetadata(Collections.singletonMap("remote", autoFollowPattern), Collections.emptyMap(), headers)))
.build();
// 1 shard started and another not started:
ClusterState leaderState = createRemoteClusterState("index1");
MetaData.Builder mBuilder= MetaData.builder(leaderState.metaData());
mBuilder.put(IndexMetaData.builder("index2")
.settings(settings(Version.CURRENT).put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true))
.numberOfShards(1)
.numberOfReplicas(0));
ShardRouting shardRouting =
TestShardRouting.newShardRouting("index2", 0, "1", true, ShardRoutingState.INITIALIZING);
IndexRoutingTable indexRoutingTable = IndexRoutingTable.builder(mBuilder.get("index2").getIndex()
).addShard(shardRouting).build();
leaderState = ClusterState.builder(leaderState.getClusterName())
.metaData(mBuilder)
.routingTable(RoutingTable.builder(leaderState.routingTable()).add(indexRoutingTable).build())
.build();
List<Index> result = AutoFollower.getLeaderIndicesToFollow("remote", autoFollowPattern, leaderState, followerState,
Collections.emptyList());
assertThat(result.size(), equalTo(1));
assertThat(result.get(0).getName(), equalTo("index1"));
// Start second shard:
shardRouting = shardRouting.moveToStarted();
indexRoutingTable = IndexRoutingTable.builder(leaderState.metaData().indices().get("index2").getIndex())
.addShard(shardRouting).build();
leaderState = ClusterState.builder(leaderState.getClusterName())
.metaData(leaderState.metaData())
.routingTable(RoutingTable.builder(leaderState.routingTable()).add(indexRoutingTable).build())
.build();
result = AutoFollower.getLeaderIndicesToFollow("remote", autoFollowPattern, leaderState, followerState, Collections.emptyList());
assertThat(result.size(), equalTo(2));
result.sort(Comparator.comparing(Index::getName));
assertThat(result.get(0).getName(), equalTo("index1"));
assertThat(result.get(1).getName(), equalTo("index2"));
}
public void testGetFollowerIndexName() {
AutoFollowPattern autoFollowPattern = new AutoFollowPattern("remote", Collections.singletonList("metrics-*"), null, null,
null, null, null, null, null, null, null, null, null);
@ -408,4 +457,21 @@ public class AutoFollowCoordinatorTests extends ESTestCase {
assertThat(autoFollowStats.getRecentAutoFollowErrors().get("_alias2:index2").getCause().getMessage(), equalTo("error"));
}
private static ClusterState createRemoteClusterState(String indexName) {
IndexMetaData indexMetaData = IndexMetaData.builder(indexName)
.settings(settings(Version.CURRENT).put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true))
.numberOfShards(1)
.numberOfReplicas(0)
.build();
ClusterState.Builder csBuilder = ClusterState.builder(new ClusterName("remote"))
.metaData(MetaData.builder().put(indexMetaData, true));
ShardRouting shardRouting =
TestShardRouting.newShardRouting(indexName, 0, "1", true, ShardRoutingState.INITIALIZING).moveToStarted();
IndexRoutingTable indexRoutingTable = IndexRoutingTable.builder(indexMetaData.getIndex()).addShard(shardRouting).build();
csBuilder.routingTable(RoutingTable.builder().add(indexRoutingTable).build()).build();
return csBuilder.build();
}
}