From 9eddd2bcc9cecd048dfb77cf694d732848d115ed Mon Sep 17 00:00:00 2001 From: Lee Hinman Date: Mon, 20 Apr 2020 12:05:42 -0600 Subject: [PATCH] [7.x] Add prefer_v2_templates flag and index setting (#55411) (#55476) This commit adds a new querystring parameter on the following APIs: - Index - Update - Bulk - Create Index - Rollover These APIs now support a `?prefer_v2_templates=true|false` flag. This flag changes the preference creation to use either V2 index templates or V1 templates. This flag defaults to `false` and will be changed to `true` for 8.0+ in subsequent work. Additionally, setting this flag internally sets the `index.prefer_v2_templates` index-level setting. This setting is used so that actions that automatically create a new index (things like rollover initiated by ILM) will inherit the preference from the original index. This setting is dynamic so that a transition from v1 to v2 templates can occur for long-running indices grouped by an alias performing periodic rollover. This also adds support for sending this parameter to the High Level Rest Client. Relates to #53101 --- .../client/IndicesRequestConverters.java | 3 + .../client/RequestConverters.java | 10 + .../client/indices/CreateIndexRequest.java | 12 + docs/reference/api-conventions.asciidoc | 4 +- .../resources/rest-api-spec/api/bulk.json | 4 + .../resources/rest-api-spec/api/create.json | 4 + .../resources/rest-api-spec/api/index.json | 4 + .../rest-api-spec/api/indices.create.json | 4 + .../rest-api-spec/api/indices.rollover.json | 4 + .../resources/rest-api-spec/api/update.json | 4 + .../15_composition.yml | 33 +++ .../CreateIndexClusterStateUpdateRequest.java | 11 + .../indices/create/CreateIndexRequest.java | 21 +- .../create/TransportCreateIndexAction.java | 1 + .../rollover/MetadataRolloverService.java | 3 +- .../rollover/TransportRolloverAction.java | 8 + .../action/bulk/BulkRequest.java | 40 ++- .../action/bulk/TransportBulkAction.java | 14 +- .../TransportSingleItemBulkWriteAction.java | 4 + .../action/index/IndexRequest.java | 17 ++ .../action/update/TransportUpdateAction.java | 7 +- .../action/update/UpdateRequest.java | 27 +- .../client/IndicesAdminClient.java | 2 +- .../client/support/AbstractClient.java | 2 +- .../cluster/metadata/IndexMetadata.java | 5 +- .../metadata/MetadataCreateIndexService.java | 32 ++- .../common/settings/IndexScopedSettings.java | 1 + .../admin/indices/RestCreateIndexAction.java | 8 + .../indices/RestRolloverIndexAction.java | 1 + .../rest/action/document/RestBulkAction.java | 2 + .../rest/action/document/RestIndexAction.java | 2 + .../action/document/RestUpdateAction.java | 2 + ...ActionIndicesThatCannotBeCreatedTests.java | 3 +- .../bulk/TransportBulkActionIngestTests.java | 4 +- .../action/bulk/TransportBulkActionTests.java | 23 +- .../template/TemplatePreferenceIT.java | 253 ++++++++++++++++++ .../action/TransportResumeFollowAction.java | 1 + 37 files changed, 530 insertions(+), 50 deletions(-) create mode 100644 server/src/test/java/org/elasticsearch/indices/template/TemplatePreferenceIT.java diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesRequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesRequestConverters.java index a18a25736a4..1158ad54aa3 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesRequestConverters.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesRequestConverters.java @@ -116,6 +116,9 @@ final class IndicesRequestConverters { parameters.withTimeout(createIndexRequest.timeout()); parameters.withMasterTimeout(createIndexRequest.masterNodeTimeout()); parameters.withWaitForActiveShards(createIndexRequest.waitForActiveShards()); + if (createIndexRequest.preferV2Templates() != null) { + parameters.putParam(IndexMetadata.PREFER_V2_TEMPLATES_FLAG, Boolean.toString(createIndexRequest.preferV2Templates())); + } request.addParameters(parameters.asMap()); request.setEntity(RequestConverters.createEntity(createIndexRequest, RequestConverters.REQUEST_BODY_CONTENT_TYPE)); return request; diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java index 5fa58675e00..17b7055a522 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java @@ -56,6 +56,7 @@ import org.elasticsearch.client.indices.AnalyzeRequest; import org.elasticsearch.client.security.RefreshPolicy; import org.elasticsearch.client.tasks.TaskId; import org.elasticsearch.cluster.health.ClusterHealthStatus; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Priority; import org.elasticsearch.common.Strings; @@ -132,6 +133,9 @@ final class RequestConverters { parameters.withRefreshPolicy(bulkRequest.getRefreshPolicy()); parameters.withPipeline(bulkRequest.pipeline()); parameters.withRouting(bulkRequest.routing()); + if (bulkRequest.preferV2Templates() != null) { + parameters.putParam(IndexMetadata.PREFER_V2_TEMPLATES_FLAG, Boolean.toString(bulkRequest.preferV2Templates())); + } // Bulk API only supports newline delimited JSON or Smile. Before executing // the bulk, we need to check that all requests have the same content-type // and this content-type is supported by the Bulk API. @@ -345,6 +349,9 @@ final class RequestConverters { parameters.withPipeline(indexRequest.getPipeline()); parameters.withRefreshPolicy(indexRequest.getRefreshPolicy()); parameters.withWaitForActiveShards(indexRequest.waitForActiveShards()); + if (indexRequest.preferV2Templates() != null) { + parameters.putParam(IndexMetadata.PREFER_V2_TEMPLATES_FLAG, Boolean.toString(indexRequest.preferV2Templates())); + } BytesRef source = indexRequest.source().toBytesRef(); ContentType contentType = createContentType(indexRequest.getContentType()); @@ -373,6 +380,9 @@ final class RequestConverters { parameters.withRetryOnConflict(updateRequest.retryOnConflict()); parameters.withVersion(updateRequest.version()); parameters.withVersionType(updateRequest.versionType()); + if (updateRequest.preferV2Templates() != null) { + parameters.putParam(IndexMetadata.PREFER_V2_TEMPLATES_FLAG, Boolean.toString(updateRequest.preferV2Templates())); + } // The Java API allows update requests with different content types // set for the partial document and the upsert document. This client diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/CreateIndexRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/CreateIndexRequest.java index 1a018591dc7..2e55165b574 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/CreateIndexRequest.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/indices/CreateIndexRequest.java @@ -25,6 +25,7 @@ import org.elasticsearch.action.admin.indices.alias.Alias; import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.client.TimedRequest; import org.elasticsearch.client.Validatable; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesArray; @@ -62,6 +63,7 @@ public class CreateIndexRequest extends TimedRequest implements Validatable, ToX private BytesReference mappings; private XContentType mappingsXContentType; + private Boolean preferV2Templates; private final Set aliases = new HashSet<>(); @@ -265,6 +267,16 @@ public class CreateIndexRequest extends TimedRequest implements Validatable, ToX return this; } + public CreateIndexRequest preferV2Templates(Boolean preferV2Templates) { + this.preferV2Templates = preferV2Templates; + return this; + } + + @Nullable + public Boolean preferV2Templates() { + return this.preferV2Templates; + } + /** * Sets the settings and mappings as a single source. * diff --git a/docs/reference/api-conventions.asciidoc b/docs/reference/api-conventions.asciidoc index 7249a597842..5ad56b686b6 100644 --- a/docs/reference/api-conventions.asciidoc +++ b/docs/reference/api-conventions.asciidoc @@ -385,6 +385,7 @@ Returns: "settings": { "index.number_of_replicas": "1", "index.number_of_shards": "1", + "index.prefer_v2_templates": "false", "index.creation_date": "1474389951325", "index.uuid": "n6gzFZTgS664GUfx0Xrpjw", "index.version.created": ..., @@ -421,7 +422,8 @@ Returns: "version": { "created": ... }, - "provided_name" : "twitter" + "provided_name" : "twitter", + "prefer_v2_templates": "false" } } } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/bulk.json b/rest-api-spec/src/main/resources/rest-api-spec/api/bulk.json index 085ff88d0c1..34b4bd9939a 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/bulk.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/bulk.json @@ -87,6 +87,10 @@ "pipeline":{ "type":"string", "description":"The pipeline id to preprocess incoming documents with" + }, + "prefer_v2_templates": { + "type": "boolean", + "description": "favor V2 templates instead of V1 templates during automatic index creation" } }, "body":{ diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/create.json b/rest-api-spec/src/main/resources/rest-api-spec/api/create.json index 171f3da44d3..21eca1bc792 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/create.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/create.json @@ -90,6 +90,10 @@ "pipeline":{ "type":"string", "description":"The pipeline id to preprocess incoming documents with" + }, + "prefer_v2_templates": { + "type": "boolean", + "description": "favor V2 templates instead of V1 templates during automatic index creation" } }, "body":{ diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/index.json b/rest-api-spec/src/main/resources/rest-api-spec/api/index.json index 7ecd7a0e927..2f0cee7b0e1 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/index.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/index.json @@ -139,6 +139,10 @@ "pipeline":{ "type":"string", "description":"The pipeline id to preprocess incoming documents with" + }, + "prefer_v2_templates": { + "type": "boolean", + "description": "favor V2 templates instead of V1 templates during automatic index creation" } }, "body":{ diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.create.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.create.json index 2b9e8617a66..29252f06d48 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.create.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.create.json @@ -37,6 +37,10 @@ "master_timeout":{ "type":"time", "description":"Specify timeout for connection to master" + }, + "prefer_v2_templates": { + "type": "boolean", + "description": "favor V2 templates instead of V1 templates during index creation" } }, "body":{ diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.rollover.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.rollover.json index 4ed1f9b4909..727070de944 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.rollover.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.rollover.json @@ -57,6 +57,10 @@ "wait_for_active_shards":{ "type":"string", "description":"Set the number of active shards to wait for on the newly created rollover index before the operation returns." + }, + "prefer_v2_templates": { + "type": "boolean", + "description": "favor V2 templates instead of V1 templates during automatic index creation" } }, "body":{ diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/update.json b/rest-api-spec/src/main/resources/rest-api-spec/api/update.json index 45b6f764387..479a9111ac2 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/update.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/update.json @@ -99,6 +99,10 @@ "if_primary_term":{ "type":"number", "description":"only perform the update operation if the last operation that has changed the document has the specified primary term" + }, + "prefer_v2_templates": { + "type": "boolean", + "description": "favor V2 templates instead of V1 templates during automatic index creation" } }, "body":{ diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_index_template/15_composition.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_index_template/15_composition.yml index 0099ec96b2e..9ef465bc1a1 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_index_template/15_composition.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_index_template/15_composition.yml @@ -61,6 +61,7 @@ - do: indices.create: index: bar-baz + prefer_v2_templates: true body: settings: index.priority: 17 @@ -120,6 +121,7 @@ - do: indices.create: + prefer_v2_templates: true index: bar-baz - do: @@ -164,6 +166,7 @@ - do: indices.create: + prefer_v2_templates: true index: bazfoo - do: @@ -193,6 +196,7 @@ - do: indices.create: + prefer_v2_templates: true index: eggplant - do: @@ -200,3 +204,32 @@ index: eggplant - match: {eggplant.settings.index.number_of_shards: "3"} + +--- +"Version 1 templates are preferred if the flag is set": + - skip: + version: " - 7.7.99" + reason: "index template v2 API unavailable before 7.8" + features: allowed_warnings + + - do: + allowed_warnings: + - "index template [my-template] has index patterns [eggplant] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template] will take precedence during new index creation" + indices.put_index_template: + name: my-template + body: + index_patterns: ["eggplant"] + template: + settings: + number_of_replicas: 2 + + - do: + indices.create: + prefer_v2_templates: false + index: eggplant + + - do: + indices.get: + index: eggplant + + - match: {eggplant.settings.index.number_of_replicas: "1"} diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexClusterStateUpdateRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexClusterStateUpdateRequest.java index 6cec5d47de5..7df9ae194df 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexClusterStateUpdateRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexClusterStateUpdateRequest.java @@ -25,6 +25,7 @@ import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.cluster.ack.ClusterStateUpdateRequest; import org.elasticsearch.cluster.block.ClusterBlock; import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.Index; @@ -44,6 +45,7 @@ public class CreateIndexClusterStateUpdateRequest extends ClusterStateUpdateRequ private Index recoverFrom; private ResizeType resizeType; private boolean copySettings; + private Boolean preferV2Templates; private Settings settings = Settings.Builder.EMPTY_SETTINGS; @@ -96,6 +98,11 @@ public class CreateIndexClusterStateUpdateRequest extends ClusterStateUpdateRequ return this; } + public CreateIndexClusterStateUpdateRequest preferV2Templates(@Nullable Boolean preferV2Templates) { + this.preferV2Templates = preferV2Templates; + return this; + } + public String cause() { return cause; } @@ -147,4 +154,8 @@ public class CreateIndexClusterStateUpdateRequest extends ClusterStateUpdateRequ return copySettings; } + @Nullable + public Boolean preferV2Templates() { + return preferV2Templates; + } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java index 202ce364ebf..bffbdb78d9a 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java @@ -29,6 +29,7 @@ import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.master.AcknowledgedRequest; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesArray; @@ -83,6 +84,8 @@ public class CreateIndexRequest extends AcknowledgedRequest private final Map mappings = new HashMap<>(); + private Boolean preferV2Templates; + private final Set aliases = new HashSet<>(); private ActiveShardCount waitForActiveShards = ActiveShardCount.DEFAULT; @@ -118,6 +121,9 @@ public class CreateIndexRequest extends AcknowledgedRequest in.readBoolean(); // updateAllTypes } waitForActiveShards = ActiveShardCount.readFrom(in); + if (in.getVersion().onOrAfter(Version.V_7_8_0)) { + preferV2Templates = in.readOptionalBoolean(); + } } public CreateIndexRequest() { @@ -169,6 +175,16 @@ public class CreateIndexRequest extends AcknowledgedRequest return this; } + public CreateIndexRequest preferV2Templates(@Nullable Boolean preferV2Templates) { + this.preferV2Templates = preferV2Templates; + return this; + } + + @Nullable + public Boolean preferV2Templates() { + return this.preferV2Templates; + } + /** * The settings to create the index with. */ @@ -459,7 +475,7 @@ public class CreateIndexRequest extends AcknowledgedRequest public CreateIndexRequest waitForActiveShards(final int waitForActiveShards) { return waitForActiveShards(ActiveShardCount.from(waitForActiveShards)); } - + @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); @@ -483,6 +499,9 @@ public class CreateIndexRequest extends AcknowledgedRequest out.writeBoolean(true); // updateAllTypes } waitForActiveShards.writeTo(out); + if (out.getVersion().onOrAfter(Version.V_7_8_0)) { + out.writeOptionalBoolean(preferV2Templates); + } } @Override diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexAction.java index b6f0aa55582..a086b592954 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexAction.java @@ -81,6 +81,7 @@ public class TransportCreateIndexAction extends TransportMasterNodeAction diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java index 1c32cadb2c3..ab4188b1173 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java @@ -140,7 +140,8 @@ public class MetadataRolloverService { .settings(createIndexRequest.settings()) .aliases(createIndexRequest.aliases()) .waitForActiveShards(ActiveShardCount.NONE) // not waiting for shards here, will wait on the alias switch operation - .mappings(createIndexRequest.mappings()); + .mappings(createIndexRequest.mappings()) + .preferV2Templates(createIndexRequest.preferV2Templates()); } /** diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java index 6b7affbae93..ba4f742c6b1 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java @@ -40,6 +40,7 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.index.shard.DocsStats; import org.elasticsearch.tasks.Task; @@ -133,6 +134,13 @@ public class TransportRolloverAction extends TransportMasterNodeAction() { + createIndex(index, bulkRequest.preferV2Templates(), bulkRequest.timeout(), new ActionListener() { @Override public void onResponse(CreateIndexResponse result) { if (counter.decrementAndGet() == 0) { @@ -274,7 +276,8 @@ public class TransportBulkAction extends HandledTransportAction originalRequest, final IndexRequest indexRequest, final Metadata metadata) { + static boolean resolvePipelines(final DocWriteRequest originalRequest, final IndexRequest indexRequest, + final boolean preferV2Templates, final Metadata metadata) { if (indexRequest.isPipelineResolved() == false) { final String requestPipeline = indexRequest.getPipeline(); indexRequest.setPipeline(IngestService.NOOP_PIPELINE_NAME); @@ -314,7 +317,7 @@ public class TransportBulkAction extends HandledTransportAction listener) { + void createIndex(String index, Boolean preferV2Templates, TimeValue timeout, ActionListener listener) { CreateIndexRequest createIndexRequest = new CreateIndexRequest(); createIndexRequest.index(index); createIndexRequest.cause("auto(bulk api)"); createIndexRequest.masterNodeTimeout(timeout); + createIndexRequest.preferV2Templates(preferV2Templates); client.admin().indices().create(createIndexRequest, listener); } diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportSingleItemBulkWriteAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportSingleItemBulkWriteAction.java index c080006b19d..60f144f5ead 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/TransportSingleItemBulkWriteAction.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportSingleItemBulkWriteAction.java @@ -22,6 +22,7 @@ package org.elasticsearch.action.bulk; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.DocWriteRequest; import org.elasticsearch.action.DocWriteResponse; +import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.HandledTransportAction; import org.elasticsearch.action.support.WriteRequest; @@ -72,6 +73,9 @@ public abstract class TransportSingleItemBulkWriteAction< bulkRequest.setRefreshPolicy(request.getRefreshPolicy()); bulkRequest.timeout(request.timeout()); bulkRequest.waitForActiveShards(request.waitForActiveShards()); + if (request instanceof IndexRequest) { + bulkRequest.preferV2Templates(((IndexRequest) request).preferV2Templates()); + } request.setRefreshPolicy(WriteRequest.RefreshPolicy.NONE); return bulkRequest; } diff --git a/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java b/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java index 7d9eadfe26d..36f7f948774 100644 --- a/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java +++ b/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java @@ -117,6 +117,7 @@ public class IndexRequest extends ReplicatedWriteRequest implement private boolean isRetry = false; private long ifSeqNo = UNASSIGNED_SEQ_NO; private long ifPrimaryTerm = UNASSIGNED_PRIMARY_TERM; + private Boolean preferV2Templates; public IndexRequest(StreamInput in) throws IOException { super(in); @@ -155,6 +156,9 @@ public class IndexRequest extends ReplicatedWriteRequest implement ifSeqNo = UNASSIGNED_SEQ_NO; ifPrimaryTerm = UNASSIGNED_PRIMARY_TERM; } + if (in.getVersion().onOrAfter(Version.V_7_8_0)) { + this.preferV2Templates = in.readOptionalBoolean(); + } } public IndexRequest() { @@ -643,6 +647,16 @@ public class IndexRequest extends ReplicatedWriteRequest implement return ifSeqNo; } + public IndexRequest preferV2Templates(@Nullable Boolean preferV2Templates) { + this.preferV2Templates = preferV2Templates; + return this; + } + + @Nullable + public Boolean preferV2Templates() { + return this.preferV2Templates; + } + /** * If set, only perform this indexing request if the document was last modification was assigned this primary term. * @@ -746,6 +760,9 @@ public class IndexRequest extends ReplicatedWriteRequest implement "sequence number based compare and write is not supported until all nodes are on version 7.0 or higher. " + "Stream version [" + out.getVersion() + "]"); } + if (out.getVersion().onOrAfter(Version.V_7_8_0)) { + out.writeOptionalBoolean(preferV2Templates); + } } @Override diff --git a/server/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java b/server/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java index 22b9a0fb5b4..bc179350f02 100644 --- a/server/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java +++ b/server/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java @@ -117,8 +117,11 @@ public class TransportUpdateAction extends TransportInstanceSingleOperationActio protected void doExecute(Task task, final UpdateRequest request, final ActionListener listener) { // if we don't have a master, we don't have metadata, that's fine, let it find a master using create index API if (autoCreateIndex.shouldAutoCreate(request.index(), clusterService.state())) { - client.admin().indices().create(new CreateIndexRequest().index(request.index()).cause("auto(update api)") - .masterNodeTimeout(request.timeout()), new ActionListener() { + client.admin().indices().create(new CreateIndexRequest() + .index(request.index()) + .cause("auto(update api)") + .preferV2Templates(request.preferV2Templates()) + .masterNodeTimeout(request.timeout()), new ActionListener() { @Override public void onResponse(CreateIndexResponse result) { innerExecute(task, request, listener); diff --git a/server/src/main/java/org/elasticsearch/action/update/UpdateRequest.java b/server/src/main/java/org/elasticsearch/action/update/UpdateRequest.java index 63439bbb571..5f7b9eb9a04 100644 --- a/server/src/main/java/org/elasticsearch/action/update/UpdateRequest.java +++ b/server/src/main/java/org/elasticsearch/action/update/UpdateRequest.java @@ -122,6 +122,7 @@ public class UpdateRequest extends InstanceShardOperationRequest private boolean scriptedUpsert = false; private boolean docAsUpsert = false; private boolean detectNoop = true; + private Boolean preferV2Templates; @Nullable private IndexRequest doc; @@ -168,6 +169,9 @@ public class UpdateRequest extends InstanceShardOperationRequest ifPrimaryTerm = in.readVLong(); detectNoop = in.readBoolean(); scriptedUpsert = in.readBoolean(); + if (in.getVersion().onOrAfter(Version.V_7_8_0)) { + preferV2Templates = in.readOptionalBoolean(); + } } public UpdateRequest(String index, String id) { @@ -235,7 +239,7 @@ public class UpdateRequest extends InstanceShardOperationRequest @Override public String type() { if (type == null) { - return MapperService.SINGLE_MAPPING_NAME; + return MapperService.SINGLE_MAPPING_NAME; } return type; } @@ -264,8 +268,8 @@ public class UpdateRequest extends InstanceShardOperationRequest type = defaultType; } return this; - } - + } + /** * The id of the indexed document. */ @@ -869,12 +873,22 @@ public class UpdateRequest extends InstanceShardOperationRequest return this; } + @Nullable + public Boolean preferV2Templates() { + return this.preferV2Templates; + } + + public UpdateRequest preferV2Templates(@Nullable Boolean preferV2Templates) { + this.preferV2Templates = preferV2Templates; + return this; + } + @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); waitForActiveShards.writeTo(out); - // A 7.x request allows null types but if deserialized in a 6.x node will cause nullpointer exceptions. - // So we use the type accessor method here to make the type non-null (will default it to "_doc"). + // A 7.x request allows null types but if deserialized in a 6.x node will cause nullpointer exceptions. + // So we use the type accessor method here to make the type non-null (will default it to "_doc"). out.writeString(type()); out.writeString(id); out.writeOptionalString(routing); @@ -922,6 +936,9 @@ public class UpdateRequest extends InstanceShardOperationRequest out.writeVLong(ifPrimaryTerm); out.writeBoolean(detectNoop); out.writeBoolean(scriptedUpsert); + if (out.getVersion().onOrAfter(Version.V_7_8_0)) { + out.writeOptionalBoolean(preferV2Templates); + } } @Override diff --git a/server/src/main/java/org/elasticsearch/client/IndicesAdminClient.java b/server/src/main/java/org/elasticsearch/client/IndicesAdminClient.java index c234cf29dee..a4633a04808 100644 --- a/server/src/main/java/org/elasticsearch/client/IndicesAdminClient.java +++ b/server/src/main/java/org/elasticsearch/client/IndicesAdminClient.java @@ -815,7 +815,7 @@ public interface IndicesAdminClient extends ElasticsearchClient { /** * Swaps the index pointed to by an alias given all provided conditions are satisfied */ - ActionFuture rolloversIndex(RolloverRequest request); + ActionFuture rolloverIndex(RolloverRequest request); /** * Swaps the index pointed to by an alias given all provided conditions are satisfied diff --git a/server/src/main/java/org/elasticsearch/client/support/AbstractClient.java b/server/src/main/java/org/elasticsearch/client/support/AbstractClient.java index e53895a55c2..922082a4adb 100644 --- a/server/src/main/java/org/elasticsearch/client/support/AbstractClient.java +++ b/server/src/main/java/org/elasticsearch/client/support/AbstractClient.java @@ -1727,7 +1727,7 @@ public abstract class AbstractClient implements Client { } @Override - public ActionFuture rolloversIndex(RolloverRequest request) { + public ActionFuture rolloverIndex(RolloverRequest request) { return execute(RolloverAction.INSTANCE, request); } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java index 082da02bfbf..39b778f4635 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java @@ -247,6 +247,10 @@ public class IndexMetadata implements Diffable, ToXContentFragmen public static final Setting.AffixSetting INDEX_ROUTING_INITIAL_RECOVERY_GROUP_SETTING = Setting.prefixKeySetting("index.routing.allocation.initial_recovery.", key -> Setting.simpleString(key)); // this is only setable internally not a registered setting!! + public static final String PREFER_V2_TEMPLATES_FLAG = "prefer_v2_templates"; + public static final String SETTING_PREFER_V2_TEMPLATES = "index." + PREFER_V2_TEMPLATES_FLAG; + public static final Setting PREFER_V2_TEMPLATES_SETTING = Setting.boolSetting(SETTING_PREFER_V2_TEMPLATES, false, + Property.Dynamic, Property.IndexScope); /** * The number of active shard copies to check for before proceeding with a write operation. @@ -1162,7 +1166,6 @@ public class IndexMetadata implements Diffable, ToXContentFragmen Arrays.fill(primaryTerms, SequenceNumbers.UNASSIGNED_PRIMARY_TERM); } - public IndexMetadata build() { ImmutableOpenMap.Builder tmpAliases = aliases; Settings tmpSettings = settings; diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java index 77f9f0d7549..1520f29ca50 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java @@ -325,14 +325,19 @@ public class MetadataCreateIndexService { // Check to see if a v2 template matched final String v2Template = MetadataIndexTemplateService.findV2Template(currentState.metadata(), request.index(), isHiddenFromRequest == null ? false : isHiddenFromRequest); + final boolean preferV2Templates = resolvePreferV2Templates(request); - if (v2Template != null) { + if (v2Template != null && preferV2Templates) { // If a v2 template was found, it takes precedence over all v1 templates, so create // the index using that template and the request's specified settings return applyCreateIndexRequestWithV2Template(currentState, request, silent, v2Template, metadataTransformer); } else { - // A v2 template wasn't found, check the v1 templates, in the event no templates are - // found creation still works using the request's specified index settings + if (v2Template != null) { + logger.debug("ignoring matching index template [{}] as [prefer_v2_templates] is set to false", v2Template); + } + // A v2 template wasn't found (or is not preferred), check the v1 templates, in the + // event no templates are found creation still works using the request's specified + // index settings final List v1Templates = MetadataIndexTemplateService.findV1Templates(currentState.metadata(), request.index(), isHiddenFromRequest); @@ -341,6 +346,11 @@ public class MetadataCreateIndexService { } } + private static boolean resolvePreferV2Templates(CreateIndexClusterStateUpdateRequest request) { + return request.preferV2Templates() == null ? + IndexMetadata.PREFER_V2_TEMPLATES_SETTING.get(request.settings()) : request.preferV2Templates(); + } + public ClusterState applyCreateIndexRequest(ClusterState currentState, CreateIndexClusterStateUpdateRequest request, boolean silent) throws Exception { return applyCreateIndexRequest(currentState, request, silent, null); @@ -410,7 +420,8 @@ public class MetadataCreateIndexService { private IndexMetadata buildAndValidateTemporaryIndexMetadata(final ClusterState currentState, final Settings aggregatedIndexSettings, final CreateIndexClusterStateUpdateRequest request, - final int routingNumShards) { + final int routingNumShards, + final boolean preferV2Templates) { final boolean isHiddenAfterTemplates = IndexMetadata.INDEX_HIDDEN_SETTING.get(aggregatedIndexSettings); validateDotIndex(request.index(), currentState, isHiddenAfterTemplates); @@ -418,6 +429,7 @@ public class MetadataCreateIndexService { // remove the setting it's temporary and is only relevant once we create the index final Settings.Builder settingsBuilder = Settings.builder().put(aggregatedIndexSettings); settingsBuilder.remove(IndexMetadata.INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING.getKey()); + settingsBuilder.put(IndexMetadata.PREFER_V2_TEMPLATES_SETTING.getKey(), preferV2Templates); final Settings indexSettings = settingsBuilder.build(); final IndexMetadata.Builder tmpImdBuilder = IndexMetadata.builder(request.index()); @@ -437,7 +449,8 @@ public class MetadataCreateIndexService { final List templates, final BiConsumer metadataTransformer) throws Exception { - logger.info("applying create index request using v1 templates {}", templates); + logger.info("applying create index request using v1 templates {}", + templates.stream().map(IndexTemplateMetadata::name).collect(Collectors.toList())); final Map> mappings = Collections.unmodifiableMap(parseMappings(request.mappings(), templates.stream().map(IndexTemplateMetadata::getMappings) @@ -456,7 +469,8 @@ public class MetadataCreateIndexService { aggregateIndexSettings(currentState, request, MetadataIndexTemplateService.resolveSettings(templates), mappings, null, settings, indexScopedSettings); int routingNumShards = getIndexNumberOfRoutingShards(aggregatedIndexSettings, null); - IndexMetadata tmpImd = buildAndValidateTemporaryIndexMetadata(currentState, aggregatedIndexSettings, request, routingNumShards); + IndexMetadata tmpImd = buildAndValidateTemporaryIndexMetadata(currentState, aggregatedIndexSettings, request, routingNumShards, + resolvePreferV2Templates(request)); return applyCreateIndexWithTemporaryService(currentState, request, silent, null, tmpImd, mappings, indexService -> resolveAndValidateAliases(request.index(), request.aliases(), @@ -486,7 +500,8 @@ public class MetadataCreateIndexService { MetadataIndexTemplateService.resolveSettings(currentState.metadata(), templateName), mappings, null, settings, indexScopedSettings); int routingNumShards = getIndexNumberOfRoutingShards(aggregatedIndexSettings, null); - IndexMetadata tmpImd = buildAndValidateTemporaryIndexMetadata(currentState, aggregatedIndexSettings, request, routingNumShards); + IndexMetadata tmpImd = buildAndValidateTemporaryIndexMetadata(currentState, aggregatedIndexSettings, request, routingNumShards, + resolvePreferV2Templates(request)); return applyCreateIndexWithTemporaryService(currentState, request, silent, null, tmpImd, mappings, indexService -> resolveAndValidateAliases(request.index(), request.aliases(), @@ -518,7 +533,8 @@ public class MetadataCreateIndexService { final Settings aggregatedIndexSettings = aggregateIndexSettings(currentState, request, Settings.EMPTY, mappings, sourceMetadata, settings, indexScopedSettings); final int routingNumShards = getIndexNumberOfRoutingShards(aggregatedIndexSettings, sourceMetadata); - IndexMetadata tmpImd = buildAndValidateTemporaryIndexMetadata(currentState, aggregatedIndexSettings, request, routingNumShards); + IndexMetadata tmpImd = buildAndValidateTemporaryIndexMetadata(currentState, aggregatedIndexSettings, request, routingNumShards, + IndexMetadata.PREFER_V2_TEMPLATES_SETTING.get(sourceMetadata.getSettings())); return applyCreateIndexWithTemporaryService(currentState, request, silent, sourceMetadata, tmpImd, mappings, indexService -> resolveAndValidateAliases(request.index(), request.aliases(), Collections.emptyList(), diff --git a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java index b5454332dc7..1db278d2358 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java +++ b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java @@ -82,6 +82,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings { IndexMetadata.INDEX_DATA_PATH_SETTING, IndexMetadata.INDEX_FORMAT_SETTING, IndexMetadata.INDEX_HIDDEN_SETTING, + IndexMetadata.PREFER_V2_TEMPLATES_SETTING, SearchSlowLog.INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_DEBUG_SETTING, SearchSlowLog.INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_WARN_SETTING, SearchSlowLog.INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_INFO_SETTING, diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCreateIndexAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCreateIndexAction.java index b7ca1ae35d4..8d38e4f1142 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCreateIndexAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCreateIndexAction.java @@ -23,6 +23,8 @@ import org.apache.logging.log4j.LogManager; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.common.xcontent.XContentHelper; @@ -56,6 +58,11 @@ public class RestCreateIndexAction extends BaseRestHandler { return "create_index_action"; } + @Nullable + public static Boolean preferV2Templates(final RestRequest request) { + return request.paramAsBoolean(IndexMetadata.PREFER_V2_TEMPLATES_FLAG, null); + } + @Override public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { final boolean includeTypeName = request.paramAsBoolean(INCLUDE_TYPE_NAME_PARAMETER, @@ -66,6 +73,7 @@ public class RestCreateIndexAction extends BaseRestHandler { } CreateIndexRequest createIndexRequest = new CreateIndexRequest(request.param("index")); + createIndexRequest.preferV2Templates(preferV2Templates(request)); if (request.hasContent()) { Map sourceAsMap = XContentHelper.convertToMap(request.requiredContent(), false, diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java index c09bde69eb9..0fcf86dc4f2 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java @@ -67,6 +67,7 @@ public class RestRolloverIndexAction extends BaseRestHandler { rolloverIndexRequest.masterNodeTimeout(request.paramAsTime("master_timeout", rolloverIndexRequest.masterNodeTimeout())); rolloverIndexRequest.getCreateIndexRequest().waitForActiveShards( ActiveShardCount.parseString(request.param("wait_for_active_shards"))); + rolloverIndexRequest.getCreateIndexRequest().preferV2Templates(RestCreateIndexAction.preferV2Templates(request)); return channel -> client.admin().indices().rolloverIndex(rolloverIndexRequest, new RestToXContentListener<>(channel)); } } diff --git a/server/src/main/java/org/elasticsearch/rest/action/document/RestBulkAction.java b/server/src/main/java/org/elasticsearch/rest/action/document/RestBulkAction.java index 24ee7208b8e..cace954fe5f 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/document/RestBulkAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/document/RestBulkAction.java @@ -31,6 +31,7 @@ import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.RestStatusToXContentListener; +import org.elasticsearch.rest.action.admin.indices.RestCreateIndexAction; import org.elasticsearch.rest.action.search.RestSearchAction; import org.elasticsearch.search.fetch.subphase.FetchSourceContext; @@ -100,6 +101,7 @@ public class RestBulkAction extends BaseRestHandler { bulkRequest.setRefreshPolicy(request.param("refresh")); bulkRequest.add(request.requiredContent(), defaultIndex, defaultType, defaultRouting, defaultFetchSourceContext, defaultPipeline, allowExplicitIndex, request.getXContentType()); + bulkRequest.preferV2Templates(RestCreateIndexAction.preferV2Templates(request)); return channel -> client.bulk(bulkRequest, new RestStatusToXContentListener<>(channel)); } diff --git a/server/src/main/java/org/elasticsearch/rest/action/document/RestIndexAction.java b/server/src/main/java/org/elasticsearch/rest/action/document/RestIndexAction.java index 4c7c0dc8f7e..cf7e9546a99 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/document/RestIndexAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/document/RestIndexAction.java @@ -32,6 +32,7 @@ import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.RestActions; import org.elasticsearch.rest.action.RestStatusToXContentListener; +import org.elasticsearch.rest.action.admin.indices.RestCreateIndexAction; import java.io.IOException; import java.util.List; @@ -145,6 +146,7 @@ public class RestIndexAction extends BaseRestHandler { indexRequest.versionType(VersionType.fromString(request.param("version_type"), indexRequest.versionType())); indexRequest.setIfSeqNo(request.paramAsLong("if_seq_no", indexRequest.ifSeqNo())); indexRequest.setIfPrimaryTerm(request.paramAsLong("if_primary_term", indexRequest.ifPrimaryTerm())); + indexRequest.preferV2Templates(RestCreateIndexAction.preferV2Templates(request)); String sOpType = request.param("op_type"); String waitForActiveShards = request.param("wait_for_active_shards"); if (waitForActiveShards != null) { diff --git a/server/src/main/java/org/elasticsearch/rest/action/document/RestUpdateAction.java b/server/src/main/java/org/elasticsearch/rest/action/document/RestUpdateAction.java index c652e097f03..6caac07416c 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/document/RestUpdateAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/document/RestUpdateAction.java @@ -31,6 +31,7 @@ import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.RestActions; import org.elasticsearch.rest.action.RestStatusToXContentListener; +import org.elasticsearch.rest.action.admin.indices.RestCreateIndexAction; import org.elasticsearch.search.fetch.subphase.FetchSourceContext; import java.io.IOException; @@ -74,6 +75,7 @@ public class RestUpdateAction extends BaseRestHandler { updateRequest.routing(request.param("routing")); updateRequest.timeout(request.paramAsTime("timeout", updateRequest.timeout())); updateRequest.setRefreshPolicy(request.param("refresh")); + updateRequest.preferV2Templates(RestCreateIndexAction.preferV2Templates(request)); String waitForActiveShards = request.param("wait_for_active_shards"); if (waitForActiveShards != null) { updateRequest.waitForActiveShards(ActiveShardCount.parseString(waitForActiveShards)); diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIndicesThatCannotBeCreatedTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIndicesThatCannotBeCreatedTests.java index 29e11669726..6a4431de0d8 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIndicesThatCannotBeCreatedTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIndicesThatCannotBeCreatedTests.java @@ -137,7 +137,8 @@ public class TransportBulkActionIndicesThatCannotBeCreatedTests extends ESTestCa } @Override - void createIndex(String index, TimeValue timeout, ActionListener listener) { + void createIndex(String index, Boolean preferV2Templates, + TimeValue timeout, ActionListener listener) { // If we try to create an index just immediately assume it worked listener.onResponse(new CreateIndexResponse(true, true, index) {}); } diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java index 1fc9fbd6e03..9771085d296 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java @@ -156,7 +156,8 @@ public class TransportBulkActionIngestTests extends ESTestCase { } @Override - void createIndex(String index, TimeValue timeout, ActionListener listener) { + void createIndex(String index, Boolean preferV2Templates, + TimeValue timeout, ActionListener listener) { indexCreated = true; listener.onResponse(null); } @@ -589,6 +590,7 @@ public class TransportBulkActionIngestTests extends ESTestCase { when(state.getMetadata()).thenReturn(metadata); IndexRequest indexRequest = new IndexRequest("missing_index").id("id"); + indexRequest.preferV2Templates(true); indexRequest.source(Collections.emptyMap()); AtomicBoolean responseCalled = new AtomicBoolean(false); AtomicBoolean failureCalled = new AtomicBoolean(false); diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java index 1eba24a1d7e..fa9bd1f836f 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java @@ -79,7 +79,8 @@ public class TransportBulkActionTests extends ESTestCase { } @Override - void createIndex(String index, TimeValue timeout, ActionListener listener) { + void createIndex(String index, Boolean preferV2Templates, + TimeValue timeout, ActionListener listener) { indexCreated = true; listener.onResponse(null); } @@ -173,14 +174,14 @@ public class TransportBulkActionTests extends ESTestCase { // index name matches with IDM: IndexRequest indexRequest = new IndexRequest("idx"); - boolean result = TransportBulkAction.resolvePipelines(indexRequest, indexRequest, metadata); + boolean result = TransportBulkAction.resolvePipelines(indexRequest, indexRequest, false, metadata); assertThat(result, is(true)); assertThat(indexRequest.isPipelineResolved(), is(true)); assertThat(indexRequest.getPipeline(), equalTo("default-pipeline")); // alias name matches with IDM: indexRequest = new IndexRequest("alias"); - result = TransportBulkAction.resolvePipelines(indexRequest, indexRequest, metadata); + result = TransportBulkAction.resolvePipelines(indexRequest, indexRequest, false, metadata); assertThat(result, is(true)); assertThat(indexRequest.isPipelineResolved(), is(true)); assertThat(indexRequest.getPipeline(), equalTo("default-pipeline")); @@ -191,7 +192,7 @@ public class TransportBulkActionTests extends ESTestCase { .settings(settings(Version.CURRENT).put(IndexSettings.DEFAULT_PIPELINE.getKey(), "default-pipeline")); metadata = Metadata.builder().put(templateBuilder).build(); indexRequest = new IndexRequest("idx"); - result = TransportBulkAction.resolvePipelines(indexRequest, indexRequest, metadata); + result = TransportBulkAction.resolvePipelines(indexRequest, indexRequest, false, metadata); assertThat(result, is(true)); assertThat(indexRequest.isPipelineResolved(), is(true)); assertThat(indexRequest.getPipeline(), equalTo("default-pipeline")); @@ -207,7 +208,7 @@ public class TransportBulkActionTests extends ESTestCase { // index name matches with IDM: IndexRequest indexRequest = new IndexRequest("idx"); - boolean result = TransportBulkAction.resolvePipelines(indexRequest, indexRequest, metadata); + boolean result = TransportBulkAction.resolvePipelines(indexRequest, indexRequest, false, metadata); assertThat(result, is(true)); assertThat(indexRequest.isPipelineResolved(), is(true)); assertThat(indexRequest.getPipeline(), equalTo("_none")); @@ -215,7 +216,7 @@ public class TransportBulkActionTests extends ESTestCase { // alias name matches with IDM: indexRequest = new IndexRequest("alias"); - result = TransportBulkAction.resolvePipelines(indexRequest, indexRequest, metadata); + result = TransportBulkAction.resolvePipelines(indexRequest, indexRequest, false, metadata); assertThat(result, is(true)); assertThat(indexRequest.isPipelineResolved(), is(true)); assertThat(indexRequest.getPipeline(), equalTo("_none")); @@ -227,7 +228,7 @@ public class TransportBulkActionTests extends ESTestCase { .settings(settings(Version.CURRENT).put(IndexSettings.FINAL_PIPELINE.getKey(), "final-pipeline")); metadata = Metadata.builder().put(templateBuilder).build(); indexRequest = new IndexRequest("idx"); - result = TransportBulkAction.resolvePipelines(indexRequest, indexRequest, metadata); + result = TransportBulkAction.resolvePipelines(indexRequest, indexRequest, false, metadata); assertThat(result, is(true)); assertThat(indexRequest.isPipelineResolved(), is(true)); assertThat(indexRequest.getPipeline(), equalTo("_none")); @@ -239,7 +240,7 @@ public class TransportBulkActionTests extends ESTestCase { { Metadata metadata = Metadata.builder().build(); IndexRequest indexRequest = new IndexRequest("idx"); - boolean result = TransportBulkAction.resolvePipelines(indexRequest, indexRequest, metadata); + boolean result = TransportBulkAction.resolvePipelines(indexRequest, indexRequest, false, metadata); assertThat(result, is(false)); assertThat(indexRequest.isPipelineResolved(), is(true)); assertThat(indexRequest.getPipeline(), equalTo(IngestService.NOOP_PIPELINE_NAME)); @@ -249,7 +250,7 @@ public class TransportBulkActionTests extends ESTestCase { { Metadata metadata = Metadata.builder().build(); IndexRequest indexRequest = new IndexRequest("idx").setPipeline("request-pipeline"); - boolean result = TransportBulkAction.resolvePipelines(indexRequest, indexRequest, metadata); + boolean result = TransportBulkAction.resolvePipelines(indexRequest, indexRequest, false, metadata); assertThat(result, is(true)); assertThat(indexRequest.isPipelineResolved(), is(true)); assertThat(indexRequest.getPipeline(), equalTo("request-pipeline")); @@ -263,7 +264,7 @@ public class TransportBulkActionTests extends ESTestCase { .numberOfReplicas(0); Metadata metadata = Metadata.builder().put(builder).build(); IndexRequest indexRequest = new IndexRequest("idx").setPipeline("request-pipeline"); - boolean result = TransportBulkAction.resolvePipelines(indexRequest, indexRequest, metadata); + boolean result = TransportBulkAction.resolvePipelines(indexRequest, indexRequest, false, metadata); assertThat(result, is(true)); assertThat(indexRequest.isPipelineResolved(), is(true)); assertThat(indexRequest.getPipeline(), equalTo("request-pipeline")); @@ -277,7 +278,7 @@ public class TransportBulkActionTests extends ESTestCase { .numberOfReplicas(0); Metadata metadata = Metadata.builder().put(builder).build(); IndexRequest indexRequest = new IndexRequest("idx").setPipeline("request-pipeline"); - boolean result = TransportBulkAction.resolvePipelines(indexRequest, indexRequest, metadata); + boolean result = TransportBulkAction.resolvePipelines(indexRequest, indexRequest, false, metadata); assertThat(result, is(true)); assertThat(indexRequest.isPipelineResolved(), is(true)); assertThat(indexRequest.getPipeline(), equalTo("request-pipeline")); diff --git a/server/src/test/java/org/elasticsearch/indices/template/TemplatePreferenceIT.java b/server/src/test/java/org/elasticsearch/indices/template/TemplatePreferenceIT.java new file mode 100644 index 00000000000..2213b98ac66 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/indices/template/TemplatePreferenceIT.java @@ -0,0 +1,253 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.indices.template; + +import org.elasticsearch.action.admin.indices.alias.Alias; +import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; +import org.elasticsearch.action.admin.indices.rollover.RolloverRequest; +import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; +import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplateV2Action; +import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateV2Action; +import org.elasticsearch.action.bulk.BulkRequest; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.update.UpdateRequest; +import org.elasticsearch.cluster.metadata.IndexTemplateV2; +import org.elasticsearch.cluster.metadata.Template; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.test.ESSingleNodeTestCase; +import org.junit.After; +import org.junit.Before; + +import java.util.Collections; + +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.hamcrest.Matchers.equalTo; + +public class TemplatePreferenceIT extends ESSingleNodeTestCase { + + private static final String INDEX = "index"; + + @Before + public void setup() throws Exception { + assertAcked(client().admin().indices().preparePutTemplate("v1") + .setOrder(10) + .setSettings(Settings.builder() + .put("index.priority", 15) + .put("index.soft_deletes.enabled", true) + .build()) + .setPatterns(Collections.singletonList(INDEX + "*")).get()); + + Template v2Settings = new Template(Settings.builder() + .put("index.priority", 23) + .build(), null, null); + IndexTemplateV2 v2template = new IndexTemplateV2(Collections.singletonList(INDEX + "*"), v2Settings, null, null, null, null); + PutIndexTemplateV2Action.Request request = new PutIndexTemplateV2Action.Request("v2"); + request.indexTemplate(v2template); + assertAcked(client().execute(PutIndexTemplateV2Action.INSTANCE, request).get()); + } + + @After + public void cleanup() throws Exception { + assertAcked(client().admin().indices().prepareDeleteTemplate("v1").get()); + assertAcked(client().execute(DeleteIndexTemplateV2Action.INSTANCE, new DeleteIndexTemplateV2Action.Request("v2")).get()); + } + + public void testCreateIndexPreference() throws Exception { + client().admin().indices().prepareCreate(INDEX).get(); + assertUsedV1(); + + client().admin().indices().create(new CreateIndexRequest(INDEX).preferV2Templates(false)).get(); + assertUsedV1(); + + client().admin().indices().create(new CreateIndexRequest(INDEX).preferV2Templates(true)).get(); + assertUsedV2(); + } + + public void testIndexingRequestPreference() throws Exception { + client().index(new IndexRequest(INDEX).source("foo", "bar")).get(); + assertUsedV1(); + + client().index(new IndexRequest(INDEX).source("foo", "bar").preferV2Templates(false)).get(); + assertUsedV1(); + + client().index(new IndexRequest(INDEX).source("foo", "bar").preferV2Templates(true)).get(); + assertUsedV2(); + + client().update(new UpdateRequest(INDEX, "1").doc("foo", "bar").docAsUpsert(true)).get(); + assertUsedV1(); + + client().update(new UpdateRequest(INDEX, "1").doc("foo", "bar").docAsUpsert(true).preferV2Templates(false)).get(); + assertUsedV1(); + + client().update(new UpdateRequest(INDEX, "1").doc("foo", "bar").docAsUpsert(true).preferV2Templates(true)).get(); + assertUsedV2(); + + client().bulk(new BulkRequest(INDEX).add(new IndexRequest(INDEX).source("foo", "bar"))).get(); + assertUsedV1(); + + client().bulk(new BulkRequest(INDEX).add(new IndexRequest(INDEX).source("foo", "bar")).preferV2Templates(false)).get(); + assertUsedV1(); + + client().bulk(new BulkRequest(INDEX).add(new IndexRequest(INDEX).source("foo", "bar")).preferV2Templates(true)).get(); + assertUsedV2(); + } + + public void testRolloverMaintainsSetting() throws Exception { + { + client().admin().indices().prepareCreate(INDEX + "-1") + .addAlias(new Alias("alias").writeIndex(true)) + .get(); + + client().admin().indices().prepareRolloverIndex("alias").get(); + GetSettingsResponse resp = client().admin().indices().prepareGetSettings().setIndices(INDEX + "-000002").get(); + assertThat("expected index to use V1 template and have priority of 15", + resp.getSetting(INDEX + "-000002", "index.priority"), equalTo("15")); + client().admin().indices().prepareDelete(INDEX + "*").get(); + } + + { + client().admin().indices().prepareCreate(INDEX + "-1") + .addAlias(new Alias("alias").writeIndex(true)) + .get(); + + RolloverRequest request = new RolloverRequest("alias", INDEX + "-000002"); + request.getCreateIndexRequest().preferV2Templates(false); + client().admin().indices().rolloverIndex(request).get(); + GetSettingsResponse resp = client().admin().indices().prepareGetSettings().setIndices(INDEX + "-000002").get(); + assertThat("expected index to use V1 template and have priority of 15", + resp.getSetting(INDEX + "-000002", "index.priority"), equalTo("15")); + client().admin().indices().prepareDelete(INDEX + "*").get(); + } + + { + client().admin().indices().prepareCreate(INDEX + "-1") + .addAlias(new Alias("alias").writeIndex(true)) + .get(); + + RolloverRequest request = new RolloverRequest("alias", INDEX + "-000002"); + request.getCreateIndexRequest().preferV2Templates(true); + client().admin().indices().rolloverIndex(request).get(); + GetSettingsResponse resp = client().admin().indices().prepareGetSettings().setIndices(INDEX + "-000002").get(); + assertThat("expected index to use V2 template and have priority of 23", + resp.getSetting(INDEX + "-000002", "index.priority"), equalTo("23")); + client().admin().indices().prepareDelete(INDEX + "*").get(); + } + + { + client().admin().indices().create(new CreateIndexRequest(INDEX + "-1") + .alias(new Alias("alias").writeIndex(true)) + .preferV2Templates(false)) + .get(); + + client().admin().indices().prepareRolloverIndex("alias").get(); + GetSettingsResponse resp = client().admin().indices().prepareGetSettings().setIndices(INDEX + "-000002").get(); + assertThat("expected index to use V1 template and have priority of 15", + resp.getSetting(INDEX + "-000002", "index.priority"), equalTo("15")); + client().admin().indices().prepareDelete(INDEX + "*").get(); + } + + { + client().admin().indices().create(new CreateIndexRequest(INDEX + "-1") + .alias(new Alias("alias").writeIndex(true)) + .preferV2Templates(false)) + .get(); + + RolloverRequest request = new RolloverRequest("alias", INDEX + "-000002"); + request.getCreateIndexRequest().preferV2Templates(false); + client().admin().indices().rolloverIndex(request).get(); + GetSettingsResponse resp = client().admin().indices().prepareGetSettings().setIndices(INDEX + "-000002").get(); + assertThat("expected index to use V1 template and have priority of 15", + resp.getSetting(INDEX + "-000002", "index.priority"), equalTo("15")); + client().admin().indices().prepareDelete(INDEX + "*").get(); + } + + { + client().admin().indices().create(new CreateIndexRequest(INDEX + "-1") + .alias(new Alias("alias").writeIndex(true)) + .preferV2Templates(false)) + .get(); + + RolloverRequest request = new RolloverRequest("alias", INDEX + "-000002"); + request.getCreateIndexRequest().preferV2Templates(true); + client().admin().indices().rolloverIndex(request).get(); + GetSettingsResponse resp = client().admin().indices().prepareGetSettings().setIndices(INDEX + "-000002").get(); + assertThat("expected index to use V2 template and have priority of 23", + resp.getSetting(INDEX + "-000002", "index.priority"), equalTo("23")); + client().admin().indices().prepareDelete(INDEX + "*").get(); + } + + { + client().admin().indices().create(new CreateIndexRequest(INDEX + "-1") + .alias(new Alias("alias").writeIndex(true)) + .preferV2Templates(true)) + .get(); + + client().admin().indices().prepareRolloverIndex("alias").get(); + GetSettingsResponse resp = client().admin().indices().prepareGetSettings().setIndices(INDEX + "-000002").get(); + assertThat("expected index to use V2 template and have priority of 23", + resp.getSetting(INDEX + "-000002", "index.priority"), equalTo("23")); + client().admin().indices().prepareDelete(INDEX + "*").get(); + } + + { + client().admin().indices().create(new CreateIndexRequest(INDEX + "-1") + .alias(new Alias("alias").writeIndex(true)) + .preferV2Templates(true)) + .get(); + + RolloverRequest request = new RolloverRequest("alias", INDEX + "-000002"); + request.getCreateIndexRequest().preferV2Templates(false); + client().admin().indices().rolloverIndex(request).get(); + GetSettingsResponse resp = client().admin().indices().prepareGetSettings().setIndices(INDEX + "-000002").get(); + assertThat("expected index to use V1 template and have priority of 15", + resp.getSetting(INDEX + "-000002", "index.priority"), equalTo("15")); + client().admin().indices().prepareDelete(INDEX + "*").get(); + } + + { + client().admin().indices().create(new CreateIndexRequest(INDEX + "-1") + .alias(new Alias("alias").writeIndex(true)) + .preferV2Templates(true)) + .get(); + + RolloverRequest request = new RolloverRequest("alias", INDEX + "-000002"); + request.getCreateIndexRequest().preferV2Templates(true); + client().admin().indices().rolloverIndex(request).get(); + GetSettingsResponse resp = client().admin().indices().prepareGetSettings().setIndices(INDEX + "-000002").get(); + assertThat("expected index to use V2 template and have priority of 23", + resp.getSetting(INDEX + "-000002", "index.priority"), equalTo("23")); + client().admin().indices().prepareDelete(INDEX + "*").get(); + } + } + + private void assertUsedV1() { + GetSettingsResponse resp = client().admin().indices().prepareGetSettings().setIndices(INDEX).get(); + assertThat("expected index to use V1 template and have priority of 15", + resp.getSetting(INDEX, "index.priority"), equalTo("15")); + client().admin().indices().prepareDelete(INDEX).get(); + } + + private void assertUsedV2() { + GetSettingsResponse resp = client().admin().indices().prepareGetSettings().setIndices(INDEX).get(); + assertThat("expected index to use V2 template and have priority of 23", + resp.getSetting(INDEX, "index.priority"), equalTo("23")); + client().admin().indices().prepareDelete(INDEX).get(); + } +} diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowAction.java index 20595c18dfb..032f199fd64 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowAction.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowAction.java @@ -369,6 +369,7 @@ public class TransportResumeFollowAction extends TransportMasterNodeAction