From be082685626011841602977194b38490a9630a87 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Thu, 18 Jun 2020 10:55:17 -0400 Subject: [PATCH] Allow follower indices to override leader settings (#58103) Today when creating a follower index via the put follow API, or via an auto-follow pattern, it is not possible to specify settings overrides for the follower index. Instead, we copy all of the leader index settings to the follower. Yet, there are cases where a user would want some different settings on the follower index such as the number of replicas, or allocation settings. This commit addresses this by allowing the user to specify settings overrides when creating follower index via manual put follower calls, or via auto-follow patterns. Note that not all settings can be overrode (e.g., index.number_of_shards) so we also have detection that prevents attempting to override settings that must be equal between the leader and follow index. Note that we do not even allow specifying such settings in the overrides, even if they are specified to be equal between the leader and the follower index. Instead, the must be implicitly copied from the leader index, not explicitly set by the user. --- .../client/ccr/FollowConfig.java | 19 + .../ccr/GetAutoFollowPatternResponse.java | 2 + .../java/org/elasticsearch/client/CCRIT.java | 21 + .../GetAutoFollowPatternResponseTests.java | 31 +- .../documentation/CCRDocumentationIT.java | 7 + .../ccr/put_auto_follow_pattern.asciidoc | 1 + .../high-level/ccr/put_follow.asciidoc | 1 + .../put-auto-follow-pattern.asciidoc | 3 + .../ccr/apis/follow-request-body.asciidoc | 4 + .../ccr/apis/follow/put-follow.asciidoc | 3 + .../elasticsearch/xpack/ccr/AutoFollowIT.java | 74 ++- .../xpack/ccr/FollowIndexIT.java | 51 +- .../xpack/ccr/ESCCRRestTestCase.java | 30 +- .../elasticsearch/xpack/ccr/CcrLicenseIT.java | 18 +- .../ccr/action/AutoFollowCoordinator.java | 1 + ...nsportActivateAutoFollowPatternAction.java | 1 + .../TransportPutAutoFollowPatternAction.java | 14 + .../ccr/action/TransportPutFollowAction.java | 21 +- .../action/TransportResumeFollowAction.java | 34 +- .../xpack/ccr/AutoFollowMetadataTests.java | 6 +- .../xpack/ccr/CCRFeatureSetTests.java | 19 +- .../action/AutoFollowCoordinatorTests.java | 597 ++++++++++++++++-- .../GetAutoFollowPatternResponseTests.java | 6 +- .../action/PutFollowActionRequestTests.java | 8 + ...tActivateAutoFollowPatternActionTests.java | 3 + ...ortDeleteAutoFollowPatternActionTests.java | 66 +- ...nsportGetAutoFollowPatternActionTests.java | 45 +- ...nsportPutAutoFollowPatternActionTests.java | 31 +- .../TransportResumeFollowActionTests.java | 2 +- .../xpack/core/ccr/AutoFollowMetadata.java | 94 ++- .../action/PutAutoFollowPatternAction.java | 29 + .../core/ccr/action/PutFollowAction.java | 32 + 32 files changed, 1143 insertions(+), 131 deletions(-) diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ccr/FollowConfig.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ccr/FollowConfig.java index 37b0c5fa2c5..89f59bfc0f9 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ccr/FollowConfig.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ccr/FollowConfig.java @@ -20,6 +20,7 @@ package org.elasticsearch.client.ccr; import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.ObjectParser; @@ -32,6 +33,7 @@ import java.util.Objects; public class FollowConfig { + static final ParseField SETTINGS = new ParseField("settings"); static final ParseField MAX_READ_REQUEST_OPERATION_COUNT = new ParseField("max_read_request_operation_count"); static final ParseField MAX_READ_REQUEST_SIZE = new ParseField("max_read_request_size"); static final ParseField MAX_OUTSTANDING_READ_REQUESTS = new ParseField("max_outstanding_read_requests"); @@ -49,6 +51,7 @@ public class FollowConfig { FollowConfig::new); static { + PARSER.declareObject(FollowConfig::setSettings, (p, c) -> Settings.fromXContent(p), SETTINGS); PARSER.declareInt(FollowConfig::setMaxReadRequestOperationCount, MAX_READ_REQUEST_OPERATION_COUNT); PARSER.declareInt(FollowConfig::setMaxOutstandingReadRequests, MAX_OUTSTANDING_READ_REQUESTS); PARSER.declareField( @@ -81,6 +84,7 @@ public class FollowConfig { return PARSER.apply(parser, null); } + private Settings settings = Settings.EMPTY; private Integer maxReadRequestOperationCount; private Integer maxOutstandingReadRequests; private ByteSizeValue maxReadRequestSize; @@ -95,6 +99,14 @@ public class FollowConfig { FollowConfig() { } + public Settings getSettings() { + return settings; + } + + public void setSettings(final Settings settings) { + this.settings = Objects.requireNonNull(settings); + } + public Integer getMaxReadRequestOperationCount() { return maxReadRequestOperationCount; } @@ -176,6 +188,13 @@ public class FollowConfig { } void toXContentFragment(XContentBuilder builder, ToXContent.Params params) throws IOException { + if (settings.isEmpty() == false) { + builder.startObject(SETTINGS.getPreferredName()); + { + settings.toXContent(builder, params); + } + builder.endObject(); + } if (maxReadRequestOperationCount != null) { builder.field(MAX_READ_REQUEST_OPERATION_COUNT.getPreferredName(), maxReadRequestOperationCount); } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ccr/GetAutoFollowPatternResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ccr/GetAutoFollowPatternResponse.java index d05ab3f3ee3..8db6de0a528 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ccr/GetAutoFollowPatternResponse.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ccr/GetAutoFollowPatternResponse.java @@ -20,6 +20,7 @@ package org.elasticsearch.client.ccr; import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.ConstructingObjectParser; @@ -98,6 +99,7 @@ public final class GetAutoFollowPatternResponse { PARSER.declareString(ConstructingObjectParser.constructorArg(), PutFollowRequest.REMOTE_CLUSTER_FIELD); PARSER.declareStringArray(ConstructingObjectParser.constructorArg(), PutAutoFollowPatternRequest.LEADER_PATTERNS_FIELD); PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), PutAutoFollowPatternRequest.FOLLOW_PATTERN_FIELD); + PARSER.declareObject(Pattern::setSettings, (p, c) -> Settings.fromXContent(p), PutAutoFollowPatternRequest.SETTINGS); PARSER.declareInt(Pattern::setMaxReadRequestOperationCount, FollowConfig.MAX_READ_REQUEST_OPERATION_COUNT); PARSER.declareField( Pattern::setMaxReadRequestSize, diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/CCRIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/CCRIT.java index 9adb9bff9ac..85124be4deb 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/CCRIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/CCRIT.java @@ -20,6 +20,8 @@ package org.elasticsearch.client; import org.apache.logging.log4j.message.ParameterizedMessage; +import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest; +import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; @@ -48,6 +50,8 @@ import org.elasticsearch.client.core.BroadcastResponse; import org.elasticsearch.client.indices.CloseIndexRequest; import org.elasticsearch.client.indices.CreateIndexRequest; import org.elasticsearch.client.indices.CreateIndexResponse; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.seqno.ReplicationTracker; import org.elasticsearch.test.rest.yaml.ObjectPath; @@ -61,6 +65,7 @@ import java.util.Map; import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasEntry; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; @@ -80,6 +85,7 @@ public class CCRIT extends ESRestHighLevelClientTestCase { assertThat(response.isAcknowledged(), is(true)); PutFollowRequest putFollowRequest = new PutFollowRequest("local_cluster", "leader", "follower", ActiveShardCount.ONE); + putFollowRequest.setSettings(Settings.builder().put("index.number_of_replicas", 0L).build()); PutFollowResponse putFollowResponse = execute(putFollowRequest, ccrClient::putFollow, ccrClient::putFollowAsync); assertThat(putFollowResponse.isFollowIndexCreated(), is(true)); assertThat(putFollowResponse.isFollowIndexShardsAcked(), is(true)); @@ -118,6 +124,13 @@ public class CCRIT extends ESRestHighLevelClientTestCase { SearchRequest followerSearchRequest = new SearchRequest("follower"); SearchResponse followerSearchResponse = highLevelClient().search(followerSearchRequest, RequestOptions.DEFAULT); assertThat(followerSearchResponse.getHits().getTotalHits().value, equalTo(1L)); + + GetSettingsRequest followerSettingsRequest = new GetSettingsRequest().indices("follower"); + GetSettingsResponse followerSettingsResponse = + highLevelClient().indices().getSettings(followerSettingsRequest, RequestOptions.DEFAULT); + assertThat( + IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.get(followerSettingsResponse.getIndexToSettings().get("follower")), + equalTo(0)); }); } catch (Exception e) { IndicesFollowStats followStats = ccrClient.getCcrStats(new CcrStatsRequest(), RequestOptions.DEFAULT).getIndicesFollowStats(); @@ -245,6 +258,10 @@ public class CCRIT extends ESRestHighLevelClientTestCase { PutAutoFollowPatternRequest putAutoFollowPatternRequest = new PutAutoFollowPatternRequest("pattern1", "local_cluster", Collections.singletonList("logs-*")); putAutoFollowPatternRequest.setFollowIndexNamePattern("copy-{{leader_index}}"); + final int followerNumberOfReplicas = randomIntBetween(0, 4); + final Settings autoFollowerPatternSettings = + Settings.builder().put("index.number_of_replicas", followerNumberOfReplicas).build(); + putAutoFollowPatternRequest.setSettings(autoFollowerPatternSettings); AcknowledgedResponse putAutoFollowPatternResponse = execute(putAutoFollowPatternRequest, ccrClient::putAutoFollowPattern, ccrClient::putAutoFollowPatternAsync); assertThat(putAutoFollowPatternResponse.isAcknowledged(), is(true)); @@ -260,6 +277,9 @@ public class CCRIT extends ESRestHighLevelClientTestCase { assertThat(ccrStatsResponse.getIndicesFollowStats().getShardFollowStats("copy-logs-20200101"), notNullValue()); }); assertThat(indexExists("copy-logs-20200101"), is(true)); + assertThat( + getIndexSettingsAsMap("copy-logs-20200101"), + hasEntry("index.number_of_replicas", Integer.toString(followerNumberOfReplicas))); GetAutoFollowPatternRequest getAutoFollowPatternRequest = randomBoolean() ? new GetAutoFollowPatternRequest("pattern1") : new GetAutoFollowPatternRequest(); @@ -271,6 +291,7 @@ public class CCRIT extends ESRestHighLevelClientTestCase { assertThat(pattern.getRemoteCluster(), equalTo(putAutoFollowPatternRequest.getRemoteCluster())); assertThat(pattern.getLeaderIndexPatterns(), equalTo(putAutoFollowPatternRequest.getLeaderIndexPatterns())); assertThat(pattern.getFollowIndexNamePattern(), equalTo(putAutoFollowPatternRequest.getFollowIndexNamePattern())); + assertThat(pattern.getSettings(), equalTo(autoFollowerPatternSettings)); // Cleanup: final DeleteAutoFollowPatternRequest deleteAutoFollowPatternRequest = new DeleteAutoFollowPatternRequest("pattern1"); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ccr/GetAutoFollowPatternResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ccr/GetAutoFollowPatternResponseTests.java index c469647f7eb..489713ed05f 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/ccr/GetAutoFollowPatternResponseTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ccr/GetAutoFollowPatternResponseTests.java @@ -20,6 +20,8 @@ package org.elasticsearch.client.ccr; import org.elasticsearch.client.AbstractResponseTestCase; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentParser; @@ -47,8 +49,10 @@ public class GetAutoFollowPatternResponseTests extends AbstractResponseTestCase< NavigableMap patterns = new TreeMap<>(); for (int i = 0; i < numPatterns; i++) { String remoteCluster = randomAlphaOfLength(4); - List leaderIndexPatters = Collections.singletonList(randomAlphaOfLength(4)); + List leaderIndexPatterns = Collections.singletonList(randomAlphaOfLength(4)); String followIndexNamePattern = randomAlphaOfLength(4); + final Settings settings = + Settings.builder().put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), randomIntBetween(0, 4)).build(); boolean active = randomBoolean(); Integer maxOutstandingReadRequests = null; @@ -91,10 +95,26 @@ public class GetAutoFollowPatternResponseTests extends AbstractResponseTestCase< if (randomBoolean()) { readPollTimeout = new TimeValue(randomNonNegativeLong()); } - patterns.put(randomAlphaOfLength(4), new AutoFollowMetadata.AutoFollowPattern(remoteCluster, leaderIndexPatters, - followIndexNamePattern, active, maxReadRequestOperationCount, maxWriteRequestOperationCount, maxOutstandingReadRequests, - maxOutstandingWriteRequests, maxReadRequestSize, maxWriteRequestSize, maxWriteBufferCount, maxWriteBufferSize, - maxRetryDelay, readPollTimeout)); + patterns.put( + randomAlphaOfLength(4), + new AutoFollowMetadata.AutoFollowPattern( + remoteCluster, + leaderIndexPatterns, + followIndexNamePattern, + settings, + active, + maxReadRequestOperationCount, + maxWriteRequestOperationCount, + maxOutstandingReadRequests, + maxOutstandingWriteRequests, + maxReadRequestSize, + maxWriteRequestSize, + maxWriteBufferCount, + maxWriteBufferSize, + maxRetryDelay, + readPollTimeout + ) + ); } return new GetAutoFollowPatternAction.Response(patterns); } @@ -115,6 +135,7 @@ public class GetAutoFollowPatternResponseTests extends AbstractResponseTestCase< assertThat(serverPattern.getRemoteCluster(), equalTo(clientPattern.getRemoteCluster())); assertThat(serverPattern.getLeaderIndexPatterns(), equalTo(clientPattern.getLeaderIndexPatterns())); assertThat(serverPattern.getFollowIndexPattern(), equalTo(clientPattern.getFollowIndexNamePattern())); + assertThat(serverPattern.getSettings(), equalTo(clientPattern.getSettings())); assertThat(serverPattern.getMaxOutstandingReadRequests(), equalTo(clientPattern.getMaxOutstandingReadRequests())); assertThat(serverPattern.getMaxOutstandingWriteRequests(), equalTo(clientPattern.getMaxOutstandingWriteRequests())); assertThat(serverPattern.getMaxReadRequestOperationCount(), equalTo(clientPattern.getMaxReadRequestOperationCount())); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/CCRDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/CCRDocumentationIT.java index 48b1b1aaa95..61615072146 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/CCRDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/CCRDocumentationIT.java @@ -54,6 +54,7 @@ import org.elasticsearch.client.core.BroadcastResponse; import org.elasticsearch.client.indices.CloseIndexRequest; import org.elasticsearch.client.indices.CreateIndexRequest; import org.elasticsearch.client.indices.CreateIndexResponse; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.rest.yaml.ObjectPath; import org.junit.Before; @@ -91,6 +92,9 @@ public class CCRDocumentationIT extends ESRestHighLevelClientTestCase { "follower", // <3> ActiveShardCount.ONE // <4> ); + Settings settings = + Settings.builder().put("index.number_of_replicas", 0L).build(); + putFollowRequest.setSettings(settings); // <5> // end::ccr-put-follow-request // tag::ccr-put-follow-execute @@ -484,6 +488,9 @@ public class CCRDocumentationIT extends ESRestHighLevelClientTestCase { Arrays.asList("logs-*", "metrics-*") // <3> ); request.setFollowIndexNamePattern("copy-{{leader_index}}"); // <4> + Settings settings = + Settings.builder().put("index.number_of_replicas", 0L).build(); + request.setSettings(settings); // <5> // end::ccr-put-auto-follow-pattern-request // tag::ccr-put-auto-follow-pattern-execute diff --git a/docs/java-rest/high-level/ccr/put_auto_follow_pattern.asciidoc b/docs/java-rest/high-level/ccr/put_auto_follow_pattern.asciidoc index 7ee9ccbe9d6..a57b26738a4 100644 --- a/docs/java-rest/high-level/ccr/put_auto_follow_pattern.asciidoc +++ b/docs/java-rest/high-level/ccr/put_auto_follow_pattern.asciidoc @@ -22,6 +22,7 @@ include-tagged::{doc-tests-file}[{api}-request] <2> The name of the remote cluster. <3> The leader index patterns. <4> The pattern used to create the follower index +<5> The settings overrides for the follower index [id="{upid}-{api}-response"] ==== Response diff --git a/docs/java-rest/high-level/ccr/put_follow.asciidoc b/docs/java-rest/high-level/ccr/put_follow.asciidoc index c1991dcf492..68b0d4fbddc 100644 --- a/docs/java-rest/high-level/ccr/put_follow.asciidoc +++ b/docs/java-rest/high-level/ccr/put_follow.asciidoc @@ -22,6 +22,7 @@ include-tagged::{doc-tests-file}[{api}-request] <3> The name of the follower index that gets created as part of the put follow API call. <4> The number of active shard copies to wait for before the put follow API returns a response, as an `ActiveShardCount` +<5> The settings overrides for the follower index. [id="{upid}-{api}-response"] ==== Response diff --git a/docs/reference/ccr/apis/auto-follow/put-auto-follow-pattern.asciidoc b/docs/reference/ccr/apis/auto-follow/put-auto-follow-pattern.asciidoc index 0ccc8c13cf2..cd30494de10 100644 --- a/docs/reference/ccr/apis/auto-follow/put-auto-follow-pattern.asciidoc +++ b/docs/reference/ccr/apis/auto-follow/put-auto-follow-pattern.asciidoc @@ -94,6 +94,9 @@ PUT /_ccr/auto_follow/my_auto_follow_pattern "leader_index*" ], "follow_index_pattern" : "{{leader_index}}-follower", + "settings": { + "index.number_of_replicas": 0 + }, "max_read_request_operation_count" : 1024, "max_outstanding_read_requests" : 16, "max_read_request_size" : "1024k", diff --git a/docs/reference/ccr/apis/follow-request-body.asciidoc b/docs/reference/ccr/apis/follow-request-body.asciidoc index 2a707c56b2b..e474f272246 100644 --- a/docs/reference/ccr/apis/follow-request-body.asciidoc +++ b/docs/reference/ccr/apis/follow-request-body.asciidoc @@ -1,4 +1,8 @@ [testenv="platinum"] +`settings`:: + (object) Settings to override from the leader index. Note that certain + settings can not be overrode (e.g., `index.number_of_shards`). + `max_read_request_operation_count`:: (integer) The maximum number of operations to pull per read from the remote cluster. diff --git a/docs/reference/ccr/apis/follow/put-follow.asciidoc b/docs/reference/ccr/apis/follow/put-follow.asciidoc index 5ca5b09fc75..26a9a362aea 100644 --- a/docs/reference/ccr/apis/follow/put-follow.asciidoc +++ b/docs/reference/ccr/apis/follow/put-follow.asciidoc @@ -90,6 +90,9 @@ PUT /follower_index/_ccr/follow?wait_for_active_shards=1 { "remote_cluster" : "remote_cluster", "leader_index" : "leader_index", + "settings": { + "index.number_of_replicas": 0 + }, "max_read_request_operation_count" : 1024, "max_outstanding_read_requests" : 16, "max_read_request_size" : "1024k", diff --git a/x-pack/plugin/ccr/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ccr/AutoFollowIT.java b/x-pack/plugin/ccr/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ccr/AutoFollowIT.java index cad4ffb314b..2a2fcca41c5 100644 --- a/x-pack/plugin/ccr/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ccr/AutoFollowIT.java +++ b/x-pack/plugin/ccr/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ccr/AutoFollowIT.java @@ -7,13 +7,21 @@ package org.elasticsearch.xpack.ccr; import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.ResponseException; import org.elasticsearch.client.RestClient; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.json.JsonXContent; import java.io.IOException; import java.util.Map; import java.util.concurrent.TimeUnit; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasEntry; +import static org.hamcrest.Matchers.hasKey; +import static org.hamcrest.Matchers.instanceOf; public class AutoFollowIT extends ESCCRRestTestCase { @@ -66,7 +74,27 @@ public class AutoFollowIT extends ESCCRRestTestCase { int initialNumberOfSuccessfulFollowedIndices = getNumberOfSuccessfulFollowedIndices(); Request request = new Request("PUT", "/_ccr/auto_follow/test_pattern"); - request.setJsonEntity("{\"leader_index_patterns\": [\"metrics-*\"], \"remote_cluster\": \"leader_cluster\"}"); + final boolean overrideNumberOfReplicas = randomBoolean(); + try (XContentBuilder bodyBuilder = JsonXContent.contentBuilder()) { + bodyBuilder.startObject(); + { + bodyBuilder.startArray("leader_index_patterns"); + { + bodyBuilder.value("metrics-*"); + } + bodyBuilder.endArray(); + bodyBuilder.field("remote_cluster", "leader_cluster"); + if (overrideNumberOfReplicas) { + bodyBuilder.startObject("settings"); + { + bodyBuilder.field("index.number_of_replicas", 0); + } + bodyBuilder.endObject(); + } + } + bodyBuilder.endObject(); + request.setJsonEntity(Strings.toString(bodyBuilder)); + } assertOK(client().performRequest(request)); try (RestClient leaderClient = buildLeaderClient()) { @@ -84,6 +112,11 @@ public class AutoFollowIT extends ESCCRRestTestCase { assertThat(getNumberOfSuccessfulFollowedIndices(), equalTo(initialNumberOfSuccessfulFollowedIndices + 1)); ensureYellow("metrics-20210101"); verifyDocuments("metrics-20210101", 5, "filtered_field:true"); + if (overrideNumberOfReplicas) { + assertThat(getIndexSettingsAsMap("metrics-20210101"), hasEntry("index.number_of_replicas", "0")); + } else { + assertThat(getIndexSettingsAsMap("metrics-20210101"), hasEntry("index.number_of_replicas", "1")); + } }); assertBusy(() -> { verifyCcrMonitoring("metrics-20210101", "metrics-20210101"); @@ -91,6 +124,45 @@ public class AutoFollowIT extends ESCCRRestTestCase { }, 30, TimeUnit.SECONDS); } + public void testPutAutoFollowPatternThatOverridesRequiredLeaderSetting() throws IOException { + if ("follow".equals(targetCluster) == false) { + logger.info("skipping test, waiting for target cluster [follow]" ); + return; + } + + final Request request = new Request("PUT", "/_ccr/auto_follow/test_pattern"); + try (XContentBuilder bodyBuilder = JsonXContent.contentBuilder()) { + bodyBuilder.startObject(); + { + bodyBuilder.startArray("leader_index_patterns"); + { + bodyBuilder.value("metrics-*"); + } + bodyBuilder.endArray(); + bodyBuilder.field("remote_cluster", "leader_cluster"); + bodyBuilder.startObject("settings"); + { + bodyBuilder.field("index.number_of_shards", 5); + } + bodyBuilder.endObject(); + } + bodyBuilder.endObject(); + request.setJsonEntity(Strings.toString(bodyBuilder)); + } + final ResponseException responseException = expectThrows(ResponseException.class, () -> client().performRequest(request)); + final Response response = responseException.getResponse(); + assertThat(response.getStatusLine().getStatusCode(), equalTo(400)); + final Map responseAsMap = entityAsMap(response); + assertThat(responseAsMap, hasKey("error")); + assertThat(responseAsMap.get("error"), instanceOf(Map.class)); + @SuppressWarnings("unchecked") final Map error = (Map) responseAsMap.get("error"); + assertThat(error, hasEntry("type", "illegal_argument_exception")); + assertThat( + error, + hasEntry("reason", "can not put auto-follow pattern that could override leader settings {\"index.number_of_shards\":\"5\"}") + ); + } + private int getNumberOfSuccessfulFollowedIndices() throws IOException { Request statsRequest = new Request("GET", "/_ccr/stats"); Map response = toMap(client().performRequest(statsRequest)); diff --git a/x-pack/plugin/ccr/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ccr/FollowIndexIT.java b/x-pack/plugin/ccr/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ccr/FollowIndexIT.java index 61b3c8b1186..3880632d36f 100644 --- a/x-pack/plugin/ccr/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ccr/FollowIndexIT.java +++ b/x-pack/plugin/ccr/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ccr/FollowIndexIT.java @@ -6,14 +6,20 @@ package org.elasticsearch.xpack.ccr; import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; import org.elasticsearch.client.RestClient; import org.elasticsearch.common.settings.Settings; +import java.io.IOException; +import java.util.Map; import java.util.concurrent.TimeUnit; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasEntry; +import static org.hamcrest.Matchers.hasKey; +import static org.hamcrest.Matchers.instanceOf; public class FollowIndexIT extends ESCCRRestTestCase { @@ -40,8 +46,26 @@ public class FollowIndexIT extends ESCCRRestTestCase { } else if ("follow".equals(targetCluster)) { logger.info("Running against follow cluster"); final String followIndexName = "test_index2"; - followIndex(leaderIndexName, followIndexName); - assertBusy(() -> verifyDocuments(followIndexName, numDocs, "filtered_field:true")); + final boolean overrideNumberOfReplicas = randomBoolean(); + if (overrideNumberOfReplicas) { + followIndex( + client(), + "leader_cluster", + leaderIndexName, + followIndexName, + Settings.builder().put("index.number_of_replicas", 0).build() + ); + } else { + followIndex(leaderIndexName, followIndexName); + } + assertBusy(() -> { + verifyDocuments(followIndexName, numDocs, "filtered_field:true"); + if (overrideNumberOfReplicas) { + assertThat(getIndexSettingsAsMap("test_index2"), hasEntry("index.number_of_replicas", "0")); + } else { + assertThat(getIndexSettingsAsMap("test_index2"), hasEntry("index.number_of_replicas", "1")); + } + }); // unfollow and then follow and then index a few docs in leader index: pauseFollow(followIndexName); resumeFollow(followIndexName); @@ -62,6 +86,29 @@ public class FollowIndexIT extends ESCCRRestTestCase { } } + public void testFollowThatOverridesRequiredLeaderSetting() throws IOException { + if ("leader".equals(targetCluster)) { + createIndex("override_leader_index", Settings.EMPTY); + } else { + final Settings settings = Settings.builder().put("index.number_of_shards", 5).build(); + final ResponseException responseException = expectThrows( + ResponseException.class, + () -> followIndex(client(), "leader_cluster", "override_leader_index", "override_follow_index", settings) + ); + final Response response = responseException.getResponse(); + assertThat(response.getStatusLine().getStatusCode(), equalTo(400)); + final Map responseAsMap = entityAsMap(response); + assertThat(responseAsMap, hasKey("error")); + assertThat(responseAsMap.get("error"), instanceOf(Map.class)); + @SuppressWarnings("unchecked") final Map error = (Map) responseAsMap.get("error"); + assertThat(error, hasEntry("type", "illegal_argument_exception")); + assertThat( + error, + hasEntry("reason", "can not put follower index that could override leader settings {\"index.number_of_shards\":\"5\"}") + ); + } + } + public void testFollowNonExistingLeaderIndex() throws Exception { if ("follow".equals(targetCluster) == false) { logger.info("skipping test, waiting for target cluster [follow]" ); diff --git a/x-pack/plugin/ccr/qa/src/main/java/org/elasticsearch/xpack/ccr/ESCCRRestTestCase.java b/x-pack/plugin/ccr/qa/src/main/java/org/elasticsearch/xpack/ccr/ESCCRRestTestCase.java index ed6612b18ce..10841dbd7ed 100644 --- a/x-pack/plugin/ccr/qa/src/main/java/org/elasticsearch/xpack/ccr/ESCCRRestTestCase.java +++ b/x-pack/plugin/ccr/qa/src/main/java/org/elasticsearch/xpack/ccr/ESCCRRestTestCase.java @@ -13,6 +13,7 @@ import org.elasticsearch.client.ResponseException; import org.elasticsearch.client.RestClient; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.json.JsonXContent; @@ -74,9 +75,34 @@ public class ESCCRRestTestCase extends ESRestTestCase { } protected static void followIndex(RestClient client, String leaderCluster, String leaderIndex, String followIndex) throws IOException { + followIndex(client, leaderCluster, leaderIndex, followIndex, null); + } + + protected static void followIndex( + final RestClient client, + final String leaderCluster, + final String leaderIndex, + final String followIndex, + final Settings settings + ) throws IOException { final Request request = new Request("PUT", "/" + followIndex + "/_ccr/follow?wait_for_active_shards=1"); - request.setJsonEntity("{\"remote_cluster\": \"" + leaderCluster + "\", \"leader_index\": \"" + leaderIndex + - "\", \"read_poll_timeout\": \"10ms\"}"); + try (XContentBuilder bodyBuilder = JsonXContent.contentBuilder()) { + bodyBuilder.startObject(); + { + bodyBuilder.field("remote_cluster", leaderCluster); + bodyBuilder.field("leader_index", leaderIndex); + bodyBuilder.field("read_poll_timeout", "10ms"); + if (settings != null) { + bodyBuilder.startObject("settings"); + { + settings.toXContent(bodyBuilder, ToXContent.EMPTY_PARAMS); + } + bodyBuilder.endObject(); + } + } + bodyBuilder.endObject(); + request.setJsonEntity(Strings.toString(bodyBuilder)); + } assertOK(client.performRequest(request)); } diff --git a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrLicenseIT.java b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrLicenseIT.java index fe59ce8331b..d24fde89923 100644 --- a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrLicenseIT.java +++ b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrLicenseIT.java @@ -164,8 +164,22 @@ public class CcrLicenseIT extends CcrSingleNodeTestCase { @Override public ClusterState execute(ClusterState currentState) throws Exception { - AutoFollowPattern autoFollowPattern = new AutoFollowPattern("test_alias", Collections.singletonList("logs-*"), - null, true, null, null, null, null, null, null, null, null, null, null); + AutoFollowPattern autoFollowPattern = new AutoFollowPattern( + "test_alias", + Collections.singletonList("logs-*"), + null, + Settings.EMPTY, + true, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null); AutoFollowMetadata autoFollowMetadata = new AutoFollowMetadata( Collections.singletonMap("test_alias", autoFollowPattern), Collections.emptyMap(), diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/AutoFollowCoordinator.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/AutoFollowCoordinator.java index d418f6688ae..ca95fd7e59c 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/AutoFollowCoordinator.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/AutoFollowCoordinator.java @@ -560,6 +560,7 @@ public class AutoFollowCoordinator extends AbstractLifecycleComponent implements request.setRemoteCluster(remoteCluster); request.setLeaderIndex(indexToFollow.getName()); request.setFollowerIndex(followIndexName); + request.setSettings(pattern.getSettings()); request.getParameters().setMaxReadRequestOperationCount(pattern.getMaxReadRequestOperationCount()); request.getParameters().setMaxReadRequestSize(pattern.getMaxReadRequestSize()); request.getParameters().setMaxOutstandingReadRequests(pattern.getMaxOutstandingReadRequests()); diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportActivateAutoFollowPatternAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportActivateAutoFollowPatternAction.java index 259b1aa6ec0..f2dfa59ac25 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportActivateAutoFollowPatternAction.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportActivateAutoFollowPatternAction.java @@ -94,6 +94,7 @@ public class TransportActivateAutoFollowPatternAction extends TransportMasterNod previousAutoFollowPattern.getRemoteCluster(), previousAutoFollowPattern.getLeaderIndexPatterns(), previousAutoFollowPattern.getFollowIndexPattern(), + previousAutoFollowPattern.getSettings(), request.isActive(), previousAutoFollowPattern.getMaxReadRequestOperationCount(), previousAutoFollowPattern.getMaxWriteRequestOperationCount(), diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutAutoFollowPatternAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutAutoFollowPatternAction.java index 74be84b13d5..72b24660937 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutAutoFollowPatternAction.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutAutoFollowPatternAction.java @@ -22,6 +22,7 @@ import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.license.LicenseUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -34,6 +35,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.function.Consumer; @@ -78,6 +80,17 @@ public class TransportPutAutoFollowPatternAction extends listener.onFailure(LicenseUtils.newComplianceException("ccr")); return; } + + final Settings replicatedRequestSettings = TransportResumeFollowAction.filter(request.getSettings()); + if (replicatedRequestSettings.isEmpty() == false) { + final String message = String.format( + Locale.ROOT, + "can not put auto-follow pattern that could override leader settings %s", + replicatedRequestSettings + ); + listener.onFailure(new IllegalArgumentException(message)); + return; + } final Client remoteClient = client.getRemoteClusterClient(request.getRemoteCluster()); final Map filteredHeaders = threadPool.getThreadContext().getHeaders().entrySet().stream() .filter(e -> ShardFollowTask.HEADER_FILTERS.contains(e.getKey())) @@ -160,6 +173,7 @@ public class TransportPutAutoFollowPatternAction extends request.getRemoteCluster(), request.getLeaderIndexPatterns(), request.getFollowIndexNamePattern(), + request.getSettings(), true, request.getParameters().getMaxReadRequestOperationCount(), request.getParameters().getMaxWriteRequestOperationCount(), diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutFollowAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutFollowAction.java index 7fec5e07a2e..996ea8eb493 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutFollowAction.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutFollowAction.java @@ -41,6 +41,7 @@ import org.elasticsearch.xpack.core.ccr.action.PutFollowAction; import org.elasticsearch.xpack.core.ccr.action.ResumeFollowAction; import java.io.IOException; +import java.util.Locale; import java.util.Objects; public final class TransportPutFollowAction @@ -123,14 +124,28 @@ public final class TransportPutFollowAction return; } - final Settings.Builder settingsBuilder = Settings.builder() + final Settings replicatedRequestSettings = TransportResumeFollowAction.filter(request.getSettings()); + if (replicatedRequestSettings.isEmpty() == false) { + final String message = String.format( + Locale.ROOT, + "can not put follower index that could override leader settings %s", + replicatedRequestSettings + ); + listener.onFailure(new IllegalArgumentException(message)); + return; + } + + final Settings overrideSettings = Settings.builder() .put(IndexMetadata.SETTING_INDEX_PROVIDED_NAME, request.getFollowerIndex()) - .put(CcrSettings.CCR_FOLLOWING_INDEX_SETTING.getKey(), true); + .put(CcrSettings.CCR_FOLLOWING_INDEX_SETTING.getKey(), true) + .put(request.getSettings()) + .build(); + final String leaderClusterRepoName = CcrRepository.NAME_PREFIX + request.getRemoteCluster(); final RestoreSnapshotRequest restoreRequest = new RestoreSnapshotRequest(leaderClusterRepoName, CcrRepository.LATEST) .indices(request.getLeaderIndex()).indicesOptions(request.indicesOptions()).renamePattern("^(.*)$") .renameReplacement(request.getFollowerIndex()).masterNodeTimeout(request.masterNodeTimeout()) - .indexSettings(settingsBuilder); + .indexSettings(overrideSettings); final Client clientWithHeaders = CcrLicenseChecker.wrapClient(this.client, threadPool.getThreadContext().getHeaders()); threadPool.executor(ThreadPool.Names.SNAPSHOT).execute(new AbstractRunnable() { diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowAction.java index 13aabfd85cf..aa761c6999d 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowAction.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowAction.java @@ -54,6 +54,7 @@ import java.io.IOException; import java.util.Collections; import java.util.HashSet; import java.util.Iterator; +import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.Set; @@ -237,19 +238,36 @@ public class TransportResumeFollowAction extends TransportMasterNodeAction patterns = new HashMap<>(numAutoFollowPatterns); for (int i = 0; i < numAutoFollowPatterns; i++) { - AutoFollowMetadata.AutoFollowPattern pattern = new AutoFollowMetadata.AutoFollowPattern("remote_cluser", - Collections.singletonList("logs" + i + "*"), null, true, null, null, null, null, null, null, null, null, null, null); + AutoFollowMetadata.AutoFollowPattern pattern = new AutoFollowMetadata.AutoFollowPattern( + "remote_cluser", + Collections.singletonList("logs" + i + "*"), + null, + Settings.EMPTY, + true, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ); patterns.put("pattern" + i, pattern); } metadata.putCustom(AutoFollowMetadata.TYPE, new AutoFollowMetadata(patterns, Collections.emptyMap(), Collections.emptyMap())); diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/AutoFollowCoordinatorTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/AutoFollowCoordinatorTests.java index 99b4ba0d22f..de11c10bf00 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/AutoFollowCoordinatorTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/AutoFollowCoordinatorTests.java @@ -89,8 +89,23 @@ public class AutoFollowCoordinatorTests extends ESTestCase { ClusterState remoteState = createRemoteClusterState("logs-20190101", true); - AutoFollowPattern autoFollowPattern = new AutoFollowPattern("remote", Collections.singletonList("logs-*"), - null, true, null, null, null, null, null, null, null, null, null, null); + AutoFollowPattern autoFollowPattern = new AutoFollowPattern( + "remote", + Collections.singletonList("logs-*"), + null, + Settings.EMPTY, + true, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ); Map patterns = new HashMap<>(); patterns.put("remote", autoFollowPattern); Map> followedLeaderIndexUUIDS = new HashMap<>(); @@ -158,8 +173,23 @@ public class AutoFollowCoordinatorTests extends ESTestCase { Client client = mock(Client.class); when(client.getRemoteClusterClient(anyString())).thenReturn(client); - AutoFollowPattern autoFollowPattern = new AutoFollowPattern("remote", Collections.singletonList("logs-*"), - null, true, null, null, null, null, null, null, null, null, null, null); + AutoFollowPattern autoFollowPattern = new AutoFollowPattern( + "remote", + Collections.singletonList("logs-*"), + null, + Settings.EMPTY, + true, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ); Map patterns = new HashMap<>(); patterns.put("remote", autoFollowPattern); Map> followedLeaderIndexUUIDS = new HashMap<>(); @@ -210,8 +240,23 @@ public class AutoFollowCoordinatorTests extends ESTestCase { when(client.getRemoteClusterClient(anyString())).thenReturn(client); ClusterState remoteState = createRemoteClusterState("logs-20190101", true); - AutoFollowPattern autoFollowPattern = new AutoFollowPattern("remote", Collections.singletonList("logs-*"), - null, true, null, null, null, null, null, null, null, null, null, null); + AutoFollowPattern autoFollowPattern = new AutoFollowPattern( + "remote", + Collections.singletonList("logs-*"), + null, + Settings.EMPTY, + true, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ); Map patterns = new HashMap<>(); patterns.put("remote", autoFollowPattern); Map> followedLeaderIndexUUIDS = new HashMap<>(); @@ -266,10 +311,41 @@ public class AutoFollowCoordinatorTests extends ESTestCase { final String remoteCluster = randomAlphaOfLength(5); final Map autoFollowPatterns = new HashMap<>(2); - autoFollowPatterns.put("pattern_1", new AutoFollowPattern(remoteCluster, Arrays.asList("logs-*", "test-*"), "copy-", false, - null, null, null, null, null, null, null, null, null, null)); - autoFollowPatterns.put("pattern_2", new AutoFollowPattern(remoteCluster, Arrays.asList("users-*"), "copy-", false, null, null, - null, null, null, null, null, null, null, null)); + autoFollowPatterns.put("pattern_1", new AutoFollowPattern( + remoteCluster, + Arrays.asList("logs-*", "test-*"), + "copy-", + Settings.EMPTY, + false, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ) + ); + autoFollowPatterns.put("pattern_2", new AutoFollowPattern( + remoteCluster, + Arrays.asList("users-*"), "copy-", + Settings.EMPTY, + false, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ) + ); final Map> followedLeaderIndexUUIDs = new HashMap<>(2); followedLeaderIndexUUIDs.put("pattern_1", Arrays.asList("uuid1", "uuid2")); @@ -486,8 +562,22 @@ public class AutoFollowCoordinatorTests extends ESTestCase { when(client.getRemoteClusterClient(anyString())).thenReturn(client); ClusterState remoteState = createRemoteClusterState("logs-20190101", true); - AutoFollowPattern autoFollowPattern = new AutoFollowPattern("remote", Collections.singletonList("logs-*"), - null, true, null, null, null, null, null, null, null, null, null, null); + AutoFollowPattern autoFollowPattern = new AutoFollowPattern( + "remote", + Collections.singletonList("logs-*"), + null, + Settings.EMPTY, true, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ); Map patterns = new HashMap<>(); patterns.put("remote", autoFollowPattern); Map> followedLeaderIndexUUIDS = new HashMap<>(); @@ -545,8 +635,23 @@ public class AutoFollowCoordinatorTests extends ESTestCase { } public void testGetLeaderIndicesToFollow() { - AutoFollowPattern autoFollowPattern = new AutoFollowPattern("remote", Collections.singletonList("metrics-*"), null, true, - null, null, null, null, null, null, null, null, null, null); + final AutoFollowPattern autoFollowPattern = new AutoFollowPattern( + "remote", + Collections.singletonList("metrics-*"), + null, + Settings.EMPTY, + true, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ); Map> headers = new HashMap<>(); RoutingTable.Builder routingTableBuilder = RoutingTable.builder(); @@ -601,8 +706,23 @@ public class AutoFollowCoordinatorTests extends ESTestCase { assertThat(result.get(2).getName(), equalTo("metrics-3")); assertThat(result.get(3).getName(), equalTo("metrics-4")); - final AutoFollowPattern inactiveAutoFollowPattern = new AutoFollowPattern("remote", Collections.singletonList("metrics-*"), null, - false, null, null, null, null, null, null, null, null, null, null); + final AutoFollowPattern inactiveAutoFollowPattern = new AutoFollowPattern( + "remote", + Collections.singletonList("metrics-*"), + null, + Settings.EMPTY, + false, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ); result = AutoFollower.getLeaderIndicesToFollow(inactiveAutoFollowPattern, remoteState, Collections.emptyList()); assertThat(result.size(), equalTo(0)); @@ -612,8 +732,23 @@ public class AutoFollowCoordinatorTests extends ESTestCase { } public void testGetLeaderIndicesToFollow_shardsNotStarted() { - AutoFollowPattern autoFollowPattern = new AutoFollowPattern("remote", Collections.singletonList("*"), null, true, - null, null, null, null, null, null, null, null, null, null); + AutoFollowPattern autoFollowPattern = new AutoFollowPattern( + "remote", + Collections.singletonList("*"), + null, + Settings.EMPTY, + true, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ); // 1 shard started and another not started: ClusterState remoteState = createRemoteClusterState("index1", true); @@ -652,8 +787,23 @@ public class AutoFollowCoordinatorTests extends ESTestCase { } public void testGetLeaderIndicesToFollowWithClosedIndices() { - final AutoFollowPattern autoFollowPattern = new AutoFollowPattern("remote", Collections.singletonList("*"), - null, true, null, null, null, null, null, null, null, null, null, null); + final AutoFollowPattern autoFollowPattern = new AutoFollowPattern( + "remote", + Collections.singletonList("*"), + null, + Settings.EMPTY, + true, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ); // index is opened ClusterState remoteState = ClusterStateCreationUtils.stateWithActivePrimary("test-index", true, randomIntBetween(1, 3), 0); @@ -775,16 +925,61 @@ public class AutoFollowCoordinatorTests extends ESTestCase { } public void testGetFollowerIndexName() { - AutoFollowPattern autoFollowPattern = new AutoFollowPattern("remote", Collections.singletonList("metrics-*"), null, true, null, - null, null, null, null, null, null, null, null, null); + AutoFollowPattern autoFollowPattern = new AutoFollowPattern( + "remote", + Collections.singletonList("metrics-*"), + null, + Settings.EMPTY, + true, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ); assertThat(AutoFollower.getFollowerIndexName(autoFollowPattern, "metrics-0"), equalTo("metrics-0")); - autoFollowPattern = new AutoFollowPattern("remote", Collections.singletonList("metrics-*"), "eu-metrics-0", true, null, null, - null, null, null, null, null, null, null, null); + autoFollowPattern = new AutoFollowPattern( + "remote", + Collections.singletonList("metrics-*"), + "eu-metrics-0", + Settings.EMPTY, + true, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ); assertThat(AutoFollower.getFollowerIndexName(autoFollowPattern, "metrics-0"), equalTo("eu-metrics-0")); - autoFollowPattern = new AutoFollowPattern("remote", Collections.singletonList("metrics-*"), "eu-{{leader_index}}", true, null, - null, null, null, null, null, null, null, null, null); + autoFollowPattern = new AutoFollowPattern( + "remote", + Collections.singletonList("metrics-*"), + "eu-{{leader_index}}", + Settings.EMPTY, + true, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ); assertThat(AutoFollower.getFollowerIndexName(autoFollowPattern, "metrics-0"), equalTo("eu-metrics-0")); } @@ -864,12 +1059,65 @@ public class AutoFollowCoordinatorTests extends ESTestCase { Runnable::run); // Add 3 patterns: Map patterns = new HashMap<>(); - patterns.put("pattern1", new AutoFollowPattern("remote1", Collections.singletonList("logs-*"), null, true, null, null, - null, null, null, null, null, null, null, null)); - patterns.put("pattern2", new AutoFollowPattern("remote2", Collections.singletonList("logs-*"), null, true, null, null, - null, null, null, null, null, null, null, null)); - patterns.put("pattern3", new AutoFollowPattern("remote2", Collections.singletonList("metrics-*"), null, true, null, null, - null, null, null, null, null, null, null, null)); + patterns.put( + "pattern1", new AutoFollowPattern( + "remote1", + Collections.singletonList("logs-*"), + null, + Settings.EMPTY, + true, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ) + ); + patterns.put( + "pattern2", + new AutoFollowPattern( + "remote2", + Collections.singletonList("logs-*"), + null, + Settings.EMPTY, + true, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ) + ); + patterns.put( + "pattern3", + new AutoFollowPattern( + "remote2", + Collections.singletonList("metrics-*"), + null, + Settings.EMPTY, + true, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ) + ); ClusterState clusterState = ClusterState.builder(new ClusterName("remote")) .metadata(Metadata.builder().putCustom(AutoFollowMetadata.TYPE, new AutoFollowMetadata(patterns, Collections.emptyMap(), Collections.emptyMap()))) @@ -894,8 +1142,26 @@ public class AutoFollowCoordinatorTests extends ESTestCase { assertThat(autoFollowCoordinator.getStats().getAutoFollowedClusters().get("remote2"), notNullValue()); assertThat(removedAutoFollower1.removed, is(true)); // Add pattern 4: - patterns.put("pattern4", new AutoFollowPattern("remote1", Collections.singletonList("metrics-*"), null, true, null, null, - null, null, null, null, null, null, null, null)); + patterns.put( + "pattern4", + new AutoFollowPattern( + "remote1", + Collections.singletonList("metrics-*"), + null, + Settings.EMPTY, + true, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ) + ); clusterState = ClusterState.builder(new ClusterName("remote")) .metadata(Metadata.builder().putCustom(AutoFollowMetadata.TYPE, new AutoFollowMetadata(patterns, Collections.emptyMap(), Collections.emptyMap()))) @@ -970,12 +1236,64 @@ public class AutoFollowCoordinatorTests extends ESTestCase { // Add 3 patterns: Map patterns = new HashMap<>(); - patterns.put("pattern1", new AutoFollowPattern("remote1", Collections.singletonList("logs-*"), null, true, null, null, - null, null, null, null, null, null, null, null)); - patterns.put("pattern2", new AutoFollowPattern("remote2", Collections.singletonList("logs-*"), null, true, null, null, - null, null, null, null, null, null, null, null)); - patterns.put("pattern3", new AutoFollowPattern("remote2", Collections.singletonList("metrics-*"), null, true, null, null, - null, null, null, null, null, null, null, null)); + patterns.put( + "pattern1", + new AutoFollowPattern( + "remote1", + Collections.singletonList("logs-*"), + null, + Settings.EMPTY, + true, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ) + ); + patterns.put( + "pattern2", + new AutoFollowPattern("remote2", + Collections.singletonList("logs-*"), + null, + Settings.EMPTY, + true, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ) + ); + patterns.put( + "pattern3", + new AutoFollowPattern("remote2", + Collections.singletonList("metrics-*"), + null, + Settings.EMPTY, + true, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ) + ); autoFollowCoordinator.updateAutoFollowers(ClusterState.builder(new ClusterName("remote")) .metadata(Metadata.builder().putCustom(AutoFollowMetadata.TYPE, @@ -991,16 +1309,45 @@ public class AutoFollowCoordinatorTests extends ESTestCase { assertThat(removedAutoFollower2.removed, is(false)); // Make pattern 1 and pattern 3 inactive - patterns.computeIfPresent("pattern1", (name, pattern) -> new AutoFollowPattern(pattern.getRemoteCluster(), - pattern.getLeaderIndexPatterns(), pattern.getFollowIndexPattern(), false, pattern.getMaxReadRequestOperationCount(), - pattern.getMaxWriteRequestOperationCount(), pattern.getMaxOutstandingReadRequests(), pattern.getMaxOutstandingWriteRequests(), - pattern.getMaxReadRequestSize(), pattern.getMaxWriteRequestSize(), pattern.getMaxWriteBufferCount(), - pattern.getMaxWriteBufferSize(), pattern.getMaxRetryDelay(), pattern.getReadPollTimeout())); - patterns.computeIfPresent("pattern3", (name, pattern) -> new AutoFollowPattern(pattern.getRemoteCluster(), - pattern.getLeaderIndexPatterns(), pattern.getFollowIndexPattern(), false, pattern.getMaxReadRequestOperationCount(), - pattern.getMaxWriteRequestOperationCount(), pattern.getMaxOutstandingReadRequests(), pattern.getMaxOutstandingWriteRequests(), - pattern.getMaxReadRequestSize(), pattern.getMaxWriteRequestSize(), pattern.getMaxWriteBufferCount(), - pattern.getMaxWriteBufferSize(), pattern.getMaxRetryDelay(), pattern.getReadPollTimeout())); + patterns.computeIfPresent( + "pattern1", + (name, pattern) -> new AutoFollowPattern( + pattern.getRemoteCluster(), + pattern.getLeaderIndexPatterns(), + pattern.getFollowIndexPattern(), + Settings.EMPTY, + false, + pattern.getMaxReadRequestOperationCount(), + pattern.getMaxWriteRequestOperationCount(), + pattern.getMaxOutstandingReadRequests(), + pattern.getMaxOutstandingWriteRequests(), + pattern.getMaxReadRequestSize(), + pattern.getMaxWriteRequestSize(), + pattern.getMaxWriteBufferCount(), + pattern.getMaxWriteBufferSize(), + pattern.getMaxRetryDelay(), + pattern.getReadPollTimeout() + ) + ); + patterns.computeIfPresent( + "pattern3", + (name, pattern) -> new AutoFollowPattern(pattern.getRemoteCluster(), + pattern.getLeaderIndexPatterns(), + pattern.getFollowIndexPattern(), + Settings.EMPTY, + false, + pattern.getMaxReadRequestOperationCount(), + pattern.getMaxWriteRequestOperationCount(), + pattern.getMaxOutstandingReadRequests(), + pattern.getMaxOutstandingWriteRequests(), + pattern.getMaxReadRequestSize(), + pattern.getMaxWriteRequestSize(), + pattern.getMaxWriteBufferCount(), + pattern.getMaxWriteBufferSize(), + pattern.getMaxRetryDelay(), + pattern.getReadPollTimeout() + ) + ); autoFollowCoordinator.updateAutoFollowers(ClusterState.builder(new ClusterName("remote")) .metadata(Metadata.builder().putCustom(AutoFollowMetadata.TYPE, @@ -1012,13 +1359,45 @@ public class AutoFollowCoordinatorTests extends ESTestCase { assertThat(removedAutoFollower2.removed, is(false)); // Add active pattern 4 and make pattern 2 inactive - patterns.put("pattern4", new AutoFollowPattern("remote1", Collections.singletonList("metrics-*"), null, true, null, null, - null, null, null, null, null, null, null, null)); - patterns.computeIfPresent("pattern2", (name, pattern) -> new AutoFollowPattern(pattern.getRemoteCluster(), - pattern.getLeaderIndexPatterns(), pattern.getFollowIndexPattern(), false, pattern.getMaxReadRequestOperationCount(), - pattern.getMaxWriteRequestOperationCount(), pattern.getMaxOutstandingReadRequests(), pattern.getMaxOutstandingWriteRequests(), - pattern.getMaxReadRequestSize(), pattern.getMaxWriteRequestSize(), pattern.getMaxWriteBufferCount(), - pattern.getMaxWriteBufferSize(), pattern.getMaxRetryDelay(), pattern.getReadPollTimeout())); + patterns.put( + "pattern4", + new AutoFollowPattern( + "remote1", + Collections.singletonList("metrics-*"), + null, + Settings.EMPTY, + true, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ) + ); + patterns.computeIfPresent( + "pattern2", + (name, pattern) -> new AutoFollowPattern(pattern.getRemoteCluster(), + pattern.getLeaderIndexPatterns(), + pattern.getFollowIndexPattern(), + Settings.EMPTY, + false, + pattern.getMaxReadRequestOperationCount(), + pattern.getMaxWriteRequestOperationCount(), + pattern.getMaxOutstandingReadRequests(), + pattern.getMaxOutstandingWriteRequests(), + pattern.getMaxReadRequestSize(), + pattern.getMaxWriteRequestSize(), + pattern.getMaxWriteBufferCount(), + pattern.getMaxWriteBufferSize(), + pattern.getMaxRetryDelay(), + pattern.getReadPollTimeout() + ) + ); autoFollowCoordinator.updateAutoFollowers(ClusterState.builder(new ClusterName("remote")) .metadata(Metadata.builder().putCustom(AutoFollowMetadata.TYPE, @@ -1046,8 +1425,23 @@ public class AutoFollowCoordinatorTests extends ESTestCase { Client client = mock(Client.class); when(client.getRemoteClusterClient(anyString())).thenReturn(client); - AutoFollowPattern autoFollowPattern = new AutoFollowPattern("remote", Collections.singletonList("logs-*"), - null, true, null, null, null, null, null, null, null, null, null, null); + AutoFollowPattern autoFollowPattern = new AutoFollowPattern( + "remote", + Collections.singletonList("logs-*"), + null, + Settings.EMPTY, + true, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ); Map patterns = new HashMap<>(); patterns.put("remote", autoFollowPattern); Map> followedLeaderIndexUUIDS = new HashMap<>(); @@ -1109,8 +1503,23 @@ public class AutoFollowCoordinatorTests extends ESTestCase { Client client = mock(Client.class); when(client.getRemoteClusterClient(anyString())).thenReturn(client); - AutoFollowPattern autoFollowPattern = new AutoFollowPattern("remote", Collections.singletonList("logs-*"), - null, true, null, null, null, null, null, null, null, null, null, null); + AutoFollowPattern autoFollowPattern = new AutoFollowPattern( + "remote", + Collections.singletonList("logs-*"), + null, + Settings.EMPTY, + true, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ); Map patterns = new HashMap<>(); patterns.put("remote", autoFollowPattern); Map> followedLeaderIndexUUIDS = new HashMap<>(); @@ -1168,8 +1577,23 @@ public class AutoFollowCoordinatorTests extends ESTestCase { ClusterState remoteState = randomBoolean() ? createRemoteClusterState("logs-20190101", false) : createRemoteClusterState("logs-20190101", null); - AutoFollowPattern autoFollowPattern = new AutoFollowPattern("remote", Collections.singletonList("logs-*"), - null, true, null, null, null, null, null, null, null, null, null, null); + AutoFollowPattern autoFollowPattern = new AutoFollowPattern( + "remote", + Collections.singletonList("logs-*"), + null, + Settings.EMPTY, + true, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ); Map patterns = new HashMap<>(); patterns.put("remote", autoFollowPattern); Map> followedLeaderIndexUUIDS = new HashMap<>(); @@ -1234,8 +1658,23 @@ public class AutoFollowCoordinatorTests extends ESTestCase { ClusterState remoteState = createRemoteClusterState("logs-20190101", true); - AutoFollowPattern autoFollowPattern = new AutoFollowPattern("remote", Collections.singletonList("logs-*"), - null, true, null, null, null, null, null, null, null, null, null, null); + AutoFollowPattern autoFollowPattern = new AutoFollowPattern( + "remote", + Collections.singletonList("logs-*"), + null, + Settings.EMPTY, + true, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ); Map patterns = new HashMap<>(); patterns.put("remote", autoFollowPattern); Map> followedLeaderIndexUUIDS = new HashMap<>(); @@ -1321,7 +1760,9 @@ public class AutoFollowCoordinatorTests extends ESTestCase { "remote", Collections.singletonList("*"), "{}", - true, 0, + Settings.EMPTY, + true, + 0, 0, 0, 0, @@ -1395,9 +1836,27 @@ public class AutoFollowCoordinatorTests extends ESTestCase { final ClusterState localState = ClusterState.builder(new ClusterName("local")) .metadata(Metadata.builder() .putCustom(AutoFollowMetadata.TYPE, - new AutoFollowMetadata(Collections.singletonMap(pattern, - new AutoFollowPattern("remote", Collections.singletonList("docs-*"), null, true, - null, null, null, null, null, null, null, null, null, null)), + new AutoFollowMetadata( + Collections.singletonMap( + pattern, + new AutoFollowPattern( + "remote", + Collections.singletonList("docs-*"), + null, + Settings.EMPTY, + true, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ) + ), Collections.singletonMap(pattern, Collections.emptyList()), Collections.singletonMap(pattern, Collections.emptyMap())))) .build(); diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/GetAutoFollowPatternResponseTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/GetAutoFollowPatternResponseTests.java index 5fd6381001d..f3edf6d5dd0 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/GetAutoFollowPatternResponseTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/GetAutoFollowPatternResponseTests.java @@ -5,7 +5,9 @@ */ package org.elasticsearch.xpack.ccr.action; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; @@ -33,7 +35,9 @@ public class GetAutoFollowPatternResponseTests extends AbstractWireSerializingTe "remote", Collections.singletonList(randomAlphaOfLength(4)), randomAlphaOfLength(4), - true, randomIntBetween(0, Integer.MAX_VALUE), + Settings.builder().put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), randomIntBetween(0, 4)).build(), + true, + randomIntBetween(0, Integer.MAX_VALUE), randomIntBetween(0, Integer.MAX_VALUE), randomIntBetween(0, Integer.MAX_VALUE), randomIntBetween(0, Integer.MAX_VALUE), diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/PutFollowActionRequestTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/PutFollowActionRequestTests.java index 02b5eca08fa..dbceaec4045 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/PutFollowActionRequestTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/PutFollowActionRequestTests.java @@ -6,7 +6,9 @@ package org.elasticsearch.xpack.ccr.action; import org.elasticsearch.action.support.ActiveShardCount; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.test.AbstractSerializingTestCase; @@ -30,6 +32,9 @@ public class PutFollowActionRequestTests extends AbstractSerializingTestCase existingPatterns = new ArrayList<>(); existingPatterns.add("transactions-*"); - existingAutoFollowPatterns.put("name1", new AutoFollowPattern("eu_cluster", existingPatterns, null, true, null, null, null, - null, null, null, null, null, null, null)); + existingAutoFollowPatterns.put( + "name1", + new AutoFollowPattern( + "eu_cluster", + existingPatterns, + null, + Settings.EMPTY, true, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ) + ); List existingUUIDS = new ArrayList<>(); existingUUIDS.add("_val"); @@ -43,8 +61,26 @@ public class TransportDeleteAutoFollowPatternActionTests extends ESTestCase { { List existingPatterns = new ArrayList<>(); existingPatterns.add("logs-*"); - existingAutoFollowPatterns.put("name2", new AutoFollowPattern("asia_cluster", existingPatterns, null, true, null, null, null, - null, null, null, null, null, null, null)); + existingAutoFollowPatterns.put( + "name2", + new AutoFollowPattern( + "asia_cluster", + existingPatterns, + null, + Settings.EMPTY, + true, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ) + ); List existingUUIDS = new ArrayList<>(); existingUUIDS.add("_val"); @@ -76,8 +112,26 @@ public class TransportDeleteAutoFollowPatternActionTests extends ESTestCase { { List existingPatterns = new ArrayList<>(); existingPatterns.add("transactions-*"); - existingAutoFollowPatterns.put("name1", new AutoFollowPattern("eu_cluster", existingPatterns, null, true, null, null, null, - null, null, null, null, null, null, null)); + existingAutoFollowPatterns.put( + "name1", + new AutoFollowPattern( + "eu_cluster", + existingPatterns, + null, + Settings.EMPTY, + true, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ) + ); existingHeaders.put("key", Collections.singletonMap("key", "val")); } ClusterState clusterState = ClusterState.builder(new ClusterName("us_cluster")) diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportGetAutoFollowPatternActionTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportGetAutoFollowPatternActionTests.java index d4899cd531b..e2d89627a2a 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportGetAutoFollowPatternActionTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportGetAutoFollowPatternActionTests.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.ccr.action; import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.core.ccr.AutoFollowMetadata; import org.elasticsearch.xpack.core.ccr.AutoFollowMetadata.AutoFollowPattern; @@ -23,10 +24,46 @@ public class TransportGetAutoFollowPatternActionTests extends ESTestCase { public void testGetAutoFollowPattern() { Map patterns = new HashMap<>(); - patterns.put("name1", new AutoFollowPattern( - "test_alias1", Collections.singletonList("index-*"), null, true, null, null, null, null, null, null, null, null, null, null)); - patterns.put("name2", new AutoFollowPattern( - "test_alias1", Collections.singletonList("index-*"), null, true, null, null, null, null, null, null, null, null, null, null)); + patterns.put( + "name1", + new AutoFollowPattern( + "test_alias1", + Collections.singletonList("index-*"), + null, + Settings.EMPTY, + true, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ) + ); + patterns.put( + "name2", + new AutoFollowPattern( + "test_alias1", + Collections.singletonList("index-*"), + null, + Settings.EMPTY, + true, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ) + ); Metadata metadata = Metadata.builder() .putCustom(AutoFollowMetadata.TYPE, new AutoFollowMetadata(patterns, Collections.emptyMap(), Collections.emptyMap())) .build(); diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportPutAutoFollowPatternActionTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportPutAutoFollowPatternActionTests.java index 56378beb2bd..b8cbc323316 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportPutAutoFollowPatternActionTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportPutAutoFollowPatternActionTests.java @@ -10,6 +10,7 @@ import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.core.ccr.AutoFollowMetadata; import org.elasticsearch.xpack.core.ccr.AutoFollowMetadata.AutoFollowPattern; @@ -23,6 +24,7 @@ import java.util.List; import java.util.Map; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasItem; import static org.hamcrest.Matchers.notNullValue; public class TransportPutAutoFollowPatternActionTests extends ESTestCase { @@ -32,6 +34,8 @@ public class TransportPutAutoFollowPatternActionTests extends ESTestCase { request.setName("name1"); request.setRemoteCluster("eu_cluster"); request.setLeaderIndexPatterns(Collections.singletonList("logs-*")); + final int numberOfReplicas = randomIntBetween(0, 4); + request.setSettings(Settings.builder().put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), numberOfReplicas).build()); ClusterState localState = ClusterState.builder(new ClusterName("us_cluster")) .metadata(Metadata.builder()) @@ -48,6 +52,13 @@ public class TransportPutAutoFollowPatternActionTests extends ESTestCase { assertThat(autoFollowMetadata.getPatterns().get("name1").getRemoteCluster(), equalTo("eu_cluster")); assertThat(autoFollowMetadata.getPatterns().get("name1").getLeaderIndexPatterns().size(), equalTo(1)); assertThat(autoFollowMetadata.getPatterns().get("name1").getLeaderIndexPatterns().get(0), equalTo("logs-*")); + assertThat( + autoFollowMetadata.getPatterns().get("name1").getSettings().keySet(), + hasItem(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey()) + ); + assertThat( + IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.get(autoFollowMetadata.getPatterns().get("name1").getSettings()), + equalTo(numberOfReplicas)); assertThat(autoFollowMetadata.getFollowedLeaderIndexUUIDs().size(), equalTo(1)); assertThat(autoFollowMetadata.getFollowedLeaderIndexUUIDs().get("name1").size(), equalTo(0)); } @@ -102,8 +113,24 @@ public class TransportPutAutoFollowPatternActionTests extends ESTestCase { Map existingAutoFollowPatterns = new HashMap<>(); List existingPatterns = new ArrayList<>(); existingPatterns.add("transactions-*"); - existingAutoFollowPatterns.put("name1", - new AutoFollowPattern("eu_cluster", existingPatterns, null, true, null, null, null, null, null, null, null, null, null, null)); + existingAutoFollowPatterns.put( + "name1", + new AutoFollowPattern( + "eu_cluster", + existingPatterns, + null, + Settings.EMPTY, + true, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null)); Map> existingAlreadyFollowedIndexUUIDS = new HashMap<>(); List existingUUIDS = new ArrayList<>(); existingUUIDS.add("_val"); diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java index 84179cd4599..1b919cd4e84 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java @@ -154,7 +154,7 @@ public class TransportResumeFollowActionTests extends ESTestCase { .put("index.analysis.analyzer.my_analyzer.type", "custom") .put("index.analysis.analyzer.my_analyzer.tokenizer", "standard").build(), customMetadata); Exception e = expectThrows(IllegalArgumentException.class, () -> validate(request, leaderIMD, followIMD, UUIDs, null)); - assertThat(e.getMessage(), equalTo("the leader index setting[{\"index.analysis.analyzer.my_analyzer.tokenizer\"" + + assertThat(e.getMessage(), equalTo("the leader index settings [{\"index.analysis.analyzer.my_analyzer.tokenizer\"" + ":\"whitespace\",\"index.analysis.analyzer.my_analyzer.type\":\"custom\",\"index.number_of_shards\":\"5\"}] " + "and follower index settings [{\"index.analysis.analyzer.my_analyzer.tokenizer\":\"standard\"," + "\"index.analysis.analyzer.my_analyzer.type\":\"custom\",\"index.number_of_shards\":\"5\"}] must be identical")); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/AutoFollowMetadata.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/AutoFollowMetadata.java index fa8da5bcd51..863404d03bf 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/AutoFollowMetadata.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/AutoFollowMetadata.java @@ -13,6 +13,7 @@ import org.elasticsearch.common.ParseField; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.regex.Regex; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.ConstructingObjectParser; @@ -180,19 +181,38 @@ public class AutoFollowMetadata extends AbstractNamedDiffable i public static final ParseField REMOTE_CLUSTER_FIELD = new ParseField("remote_cluster"); public static final ParseField LEADER_PATTERNS_FIELD = new ParseField("leader_index_patterns"); public static final ParseField FOLLOW_PATTERN_FIELD = new ParseField("follow_index_pattern"); + public static final ParseField SETTINGS_FIELD = new ParseField("settings"); @SuppressWarnings("unchecked") private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("auto_follow_pattern", - args -> new AutoFollowPattern((String) args[0], (List) args[1], (String) args[2], - args[3] == null || (boolean) args[3], (Integer) args[4], (Integer) args[5], (Integer) args[6], (Integer) args[7], - (ByteSizeValue) args[8], (ByteSizeValue) args[9], (Integer) args[10], (ByteSizeValue) args[11], (TimeValue) args[12], - (TimeValue) args[13])); + args -> new AutoFollowPattern( + (String) args[0], + (List) args[1], + (String) args[2], + args[3] == null ? Settings.EMPTY : (Settings) args[3], + args[4] == null || (boolean) args[4], + (Integer) args[5], + (Integer) args[6], + (Integer) args[7], + (Integer) args[8], + (ByteSizeValue) args[9], + (ByteSizeValue) args[10], + (Integer) args[11], + (ByteSizeValue) args[12], + (TimeValue) args[13], + (TimeValue) args[14]) + ); static { PARSER.declareString(ConstructingObjectParser.constructorArg(), REMOTE_CLUSTER_FIELD); PARSER.declareStringArray(ConstructingObjectParser.constructorArg(), LEADER_PATTERNS_FIELD); PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), FOLLOW_PATTERN_FIELD); + PARSER.declareObject( + ConstructingObjectParser.optionalConstructorArg(), + (p, c) -> Settings.fromXContent(p), + SETTINGS_FIELD + ); PARSER.declareBoolean(ConstructingObjectParser.optionalConstructorArg(), ACTIVE); ImmutableFollowParameters.initParser(PARSER); } @@ -200,40 +220,55 @@ public class AutoFollowMetadata extends AbstractNamedDiffable i private final String remoteCluster; private final List leaderIndexPatterns; private final String followIndexPattern; + private final Settings settings; private final boolean active; - public AutoFollowPattern(String remoteCluster, - List leaderIndexPatterns, - String followIndexPattern, - boolean active, - Integer maxReadRequestOperationCount, - Integer maxWriteRequestOperationCount, - Integer maxOutstandingReadRequests, - Integer maxOutstandingWriteRequests, - ByteSizeValue maxReadRequestSize, - ByteSizeValue maxWriteRequestSize, - Integer maxWriteBufferCount, - ByteSizeValue maxWriteBufferSize, - TimeValue maxRetryDelay, - TimeValue pollTimeout) { + public AutoFollowPattern( + String remoteCluster, + List leaderIndexPatterns, + String followIndexPattern, + Settings settings, + boolean active, + Integer maxReadRequestOperationCount, + Integer maxWriteRequestOperationCount, + Integer maxOutstandingReadRequests, + Integer maxOutstandingWriteRequests, + ByteSizeValue maxReadRequestSize, + ByteSizeValue maxWriteRequestSize, + Integer maxWriteBufferCount, + ByteSizeValue maxWriteBufferSize, + TimeValue maxRetryDelay, + TimeValue pollTimeout + ) { super(maxReadRequestOperationCount, maxWriteRequestOperationCount, maxOutstandingReadRequests, maxOutstandingWriteRequests, maxReadRequestSize, maxWriteRequestSize, maxWriteBufferCount, maxWriteBufferSize, maxRetryDelay, pollTimeout); this.remoteCluster = remoteCluster; this.leaderIndexPatterns = leaderIndexPatterns; this.followIndexPattern = followIndexPattern; + this.settings = Objects.requireNonNull(settings); this.active = active; } public static AutoFollowPattern readFrom(StreamInput in) throws IOException { - return new AutoFollowPattern(in.readString(), in.readStringList(), in.readOptionalString(), in); + final String remoteCluster = in.readString(); + final List leaderIndexPatterns = in.readStringList(); + final String followIndexPattern = in.readOptionalString(); + final Settings settings; + if (in.getVersion().onOrAfter(Version.V_7_9_0)) { + settings = Settings.readSettingsFromStream(in); + } else { + settings = Settings.EMPTY; + } + return new AutoFollowPattern(remoteCluster, leaderIndexPatterns, followIndexPattern, settings, in); } private AutoFollowPattern(String remoteCluster, List leaderIndexPatterns, - String followIndexPattern, StreamInput in) throws IOException { + String followIndexPattern, Settings settings, StreamInput in) throws IOException { super(in); this.remoteCluster = remoteCluster; this.leaderIndexPatterns = leaderIndexPatterns; this.followIndexPattern = followIndexPattern; + this.settings = Objects.requireNonNull(settings); if (in.getVersion().onOrAfter(Version.V_7_5_0)) { this.active = in.readBoolean(); } else { @@ -261,6 +296,10 @@ public class AutoFollowMetadata extends AbstractNamedDiffable i return followIndexPattern; } + public Settings getSettings() { + return settings; + } + public boolean isActive() { return active; } @@ -270,6 +309,9 @@ public class AutoFollowMetadata extends AbstractNamedDiffable i out.writeString(remoteCluster); out.writeStringCollection(leaderIndexPatterns); out.writeOptionalString(followIndexPattern); + if (out.getVersion().onOrAfter(Version.V_7_9_0)) { + Settings.writeSettingsToStream(settings, out); + } super.writeTo(out); if (out.getVersion().onOrAfter(Version.V_7_5_0)) { out.writeBoolean(active); @@ -284,6 +326,13 @@ public class AutoFollowMetadata extends AbstractNamedDiffable i if (followIndexPattern != null) { builder.field(FOLLOW_PATTERN_FIELD.getPreferredName(), followIndexPattern); } + if (settings.isEmpty() == false) { + builder.startObject(SETTINGS_FIELD.getPreferredName()); + { + settings.toXContent(builder, params); + } + builder.endObject(); + } toXContentFragment(builder); return builder; } @@ -297,12 +346,13 @@ public class AutoFollowMetadata extends AbstractNamedDiffable i return active == pattern.active && remoteCluster.equals(pattern.remoteCluster) && leaderIndexPatterns.equals(pattern.leaderIndexPatterns) && - followIndexPattern.equals(pattern.followIndexPattern); + followIndexPattern.equals(pattern.followIndexPattern) && + settings.equals(pattern.settings); } @Override public int hashCode() { - return Objects.hash(super.hashCode(), remoteCluster, leaderIndexPatterns, followIndexPattern, active); + return Objects.hash(super.hashCode(), remoteCluster, leaderIndexPatterns, followIndexPattern, settings, active); } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/PutAutoFollowPatternAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/PutAutoFollowPatternAction.java index e1a4b3ffc4c..20524db3f34 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/PutAutoFollowPatternAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/PutAutoFollowPatternAction.java @@ -12,6 +12,7 @@ import org.elasticsearch.action.support.master.AcknowledgedRequest; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.ToXContentObject; @@ -26,6 +27,7 @@ import java.util.Objects; import static org.elasticsearch.action.ValidateActions.addValidationError; import static org.elasticsearch.xpack.core.ccr.AutoFollowMetadata.AutoFollowPattern.REMOTE_CLUSTER_FIELD; +import static org.elasticsearch.xpack.core.ccr.AutoFollowMetadata.AutoFollowPattern.SETTINGS_FIELD; public class PutAutoFollowPatternAction extends ActionType { @@ -49,6 +51,7 @@ public class PutAutoFollowPatternAction extends ActionType PARSER.declareString((params, value) -> params.remoteCluster = value, REMOTE_CLUSTER_FIELD); PARSER.declareStringArray((params, value) -> params.leaderIndexPatterns = value, AutoFollowPattern.LEADER_PATTERNS_FIELD); PARSER.declareString((params, value) -> params.followIndexNamePattern = value, AutoFollowPattern.FOLLOW_PATTERN_FIELD); + PARSER.declareObject((params, value) -> params.settings = value, (p, c) -> Settings.fromXContent(p), SETTINGS_FIELD); FollowParameters.initParser(PARSER); } @@ -59,6 +62,7 @@ public class PutAutoFollowPatternAction extends ActionType request.setRemoteCluster(parameters.remoteCluster); request.setLeaderIndexPatterns(parameters.leaderIndexPatterns); request.setFollowIndexNamePattern(parameters.followIndexNamePattern); + request.setSettings(parameters.settings); request.setParameters(parameters); return request; } @@ -67,6 +71,7 @@ public class PutAutoFollowPatternAction extends ActionType private String remoteCluster; private List leaderIndexPatterns; private String followIndexNamePattern; + private Settings settings = Settings.EMPTY; private FollowParameters parameters = new FollowParameters(); public Request() { @@ -134,6 +139,14 @@ public class PutAutoFollowPatternAction extends ActionType this.followIndexNamePattern = followIndexNamePattern; } + public Settings getSettings() { + return settings; + } + + public void setSettings(final Settings settings) { + this.settings = Objects.requireNonNull(settings); + } + public FollowParameters getParameters() { return parameters; } @@ -149,8 +162,12 @@ public class PutAutoFollowPatternAction extends ActionType leaderIndexPatterns = in.readStringList(); followIndexNamePattern = in.readOptionalString(); if (in.getVersion().onOrAfter(Version.V_6_7_0)) { + if (in.getVersion().onOrAfter(Version.V_7_9_0)) { + settings = Settings.readSettingsFromStream(in); + } parameters = new FollowParameters(in); } else { + settings = Settings.EMPTY; parameters = new FollowParameters(); parameters.maxReadRequestOperationCount = in.readOptionalVInt(); parameters.maxReadRequestSize = in.readOptionalWriteable(ByteSizeValue::new); @@ -173,6 +190,9 @@ public class PutAutoFollowPatternAction extends ActionType out.writeStringCollection(leaderIndexPatterns); out.writeOptionalString(followIndexNamePattern); if (out.getVersion().onOrAfter(Version.V_6_7_0)) { + if (out.getVersion().onOrAfter(Version.V_7_9_0)) { + Settings.writeSettingsToStream(settings, out); + } parameters.writeTo(out); } else { out.writeOptionalVInt(parameters.maxReadRequestOperationCount); @@ -197,6 +217,13 @@ public class PutAutoFollowPatternAction extends ActionType if (followIndexNamePattern != null) { builder.field(AutoFollowPattern.FOLLOW_PATTERN_FIELD.getPreferredName(), followIndexNamePattern); } + if (settings.isEmpty() == false) { + builder.startObject(SETTINGS_FIELD.getPreferredName()); + { + settings.toXContent(builder, params); + } + builder.endObject(); + } parameters.toXContentFragment(builder); } builder.endObject(); @@ -226,6 +253,8 @@ public class PutAutoFollowPatternAction extends ActionType private String remoteCluster; private List leaderIndexPatterns; private String followIndexNamePattern; + private Settings settings = Settings.EMPTY; + } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/PutFollowAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/PutFollowAction.java index 42fec9a6764..bde4c30cfa5 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/PutFollowAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/PutFollowAction.java @@ -17,6 +17,7 @@ import org.elasticsearch.action.support.master.AcknowledgedRequest; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -40,6 +41,7 @@ public final class PutFollowAction extends ActionType private static final ParseField REMOTE_CLUSTER_FIELD = new ParseField("remote_cluster"); private static final ParseField LEADER_INDEX_FIELD = new ParseField("leader_index"); + private static final ParseField SETTINGS_FIELD = new ParseField("settings"); // Note that Request should be the Value class here for this parser with a 'parameters' field that maps to // PutFollowParameters class. But since two minor version are already released with duplicate follow parameters @@ -49,6 +51,11 @@ public final class PutFollowAction extends ActionType static { PARSER.declareString((putFollowParameters, value) -> putFollowParameters.remoteCluster = value, REMOTE_CLUSTER_FIELD); PARSER.declareString((putFollowParameters, value) -> putFollowParameters.leaderIndex = value, LEADER_INDEX_FIELD); + PARSER.declareObject( + (putFollowParameters, value) -> putFollowParameters.settings = value, + (p, c) -> Settings.fromXContent(p), + SETTINGS_FIELD + ); FollowParameters.initParser(PARSER); } @@ -61,12 +68,14 @@ public final class PutFollowAction extends ActionType request.setFollowerIndex(followerIndex); request.setRemoteCluster(parameters.remoteCluster); request.setLeaderIndex(parameters.leaderIndex); + request.setSettings(parameters.settings); request.setParameters(parameters); return request; } private String remoteCluster; private String leaderIndex; + private Settings settings = Settings.EMPTY; private String followerIndex; private FollowParameters parameters = new FollowParameters(); private ActiveShardCount waitForActiveShards = ActiveShardCount.NONE; @@ -98,6 +107,14 @@ public final class PutFollowAction extends ActionType this.leaderIndex = leaderIndex; } + public Settings getSettings() { + return settings; + } + + public void setSettings(final Settings settings) { + this.settings = Objects.requireNonNull(settings); + } + public FollowParameters getParameters() { return parameters; } @@ -157,6 +174,9 @@ public final class PutFollowAction extends ActionType this.remoteCluster = in.readString(); this.leaderIndex = in.readString(); this.followerIndex = in.readString(); + if (in.getVersion().onOrAfter(Version.V_7_9_0)) { + this.settings = Settings.readSettingsFromStream(in); + } this.parameters = new FollowParameters(in); if (in.getVersion().onOrAfter(Version.V_6_7_0)) { waitForActiveShards(ActiveShardCount.readFrom(in)); @@ -169,6 +189,9 @@ public final class PutFollowAction extends ActionType out.writeString(remoteCluster); out.writeString(leaderIndex); out.writeString(followerIndex); + if (out.getVersion().onOrAfter(Version.V_7_9_0)) { + Settings.writeSettingsToStream(settings, out); + } parameters.writeTo(out); if (out.getVersion().onOrAfter(Version.V_6_7_0)) { waitForActiveShards.writeTo(out); @@ -181,6 +204,13 @@ public final class PutFollowAction extends ActionType { builder.field(REMOTE_CLUSTER_FIELD.getPreferredName(), remoteCluster); builder.field(LEADER_INDEX_FIELD.getPreferredName(), leaderIndex); + if (settings.isEmpty() == false) { + builder.startObject(SETTINGS_FIELD.getPreferredName()); + { + settings.toXContent(builder, params); + } + builder.endObject(); + } parameters.toXContentFragment(builder); } builder.endObject(); @@ -209,6 +239,8 @@ public final class PutFollowAction extends ActionType private String remoteCluster; private String leaderIndex; + private Settings settings = Settings.EMPTY; + } }