Add data stream support to put mapping and update index settings APIs. (#58758)

Backport of #58231 to 7.x branch. Change update index setting and put mapping api to execute on all backing indices if data stream is targeted. Relates #53100
2020-07-01 13:32:21 +02:00 · 2020-07-01 13:32:21 +02:00 · a0df96befb
parent b904a60275
commit a0df96befb
5 changed files with 82 additions and 20 deletions
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/20_unsupported_apis.yml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/20_unsupported_apis.yml
@ -45,22 +45,6 @@
      indices.delete:
        index: logs-foobar

-  - do:
-      catch: bad_request
-      indices.put_settings:
-        index: logs-foobar
-        body:
-          number_of_replicas: 1
-
-  - do:
-      catch: bad_request
-      indices.put_mapping:
-        index: logs-foobar
-        body:
-          properties:
-            baz:
-              type: keyword
-
  - do:
      indices.create:
        index: logs-foobarbaz
--- a/server/src/internalClusterTest/java/org/elasticsearch/indices/DataStreamIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/DataStreamIT.java
@ -27,9 +27,11 @@ import org.elasticsearch.action.admin.indices.datastream.DeleteDataStreamAction;
 import org.elasticsearch.action.admin.indices.datastream.GetDataStreamAction;
 import org.elasticsearch.action.admin.indices.get.GetIndexRequest;
 import org.elasticsearch.action.admin.indices.get.GetIndexResponse;
+import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
 import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
 import org.elasticsearch.action.admin.indices.rollover.RolloverRequest;
 import org.elasticsearch.action.admin.indices.rollover.RolloverResponse;
+import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse;
 import org.elasticsearch.action.admin.indices.template.delete.DeleteComposableIndexTemplateAction;
 import org.elasticsearch.action.admin.indices.template.put.PutComposableIndexTemplateAction;
 import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryRequestBuilder;
@ -53,6 +55,7 @@ import org.elasticsearch.cluster.metadata.Template;
 import org.elasticsearch.common.collect.List;
 import org.elasticsearch.common.collect.Map;
 import org.elasticsearch.common.compress.CompressedXContent;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.ObjectPath;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.IndexNotFoundException;
@ -76,9 +79,11 @@ import static org.elasticsearch.indices.IndicesOptionsIntegrationIT.getSettings;
 import static org.elasticsearch.indices.IndicesOptionsIntegrationIT.health;
 import static org.elasticsearch.indices.IndicesOptionsIntegrationIT.indicesStats;
 import static org.elasticsearch.indices.IndicesOptionsIntegrationIT.msearch;
+import static org.elasticsearch.indices.IndicesOptionsIntegrationIT.putMapping;
 import static org.elasticsearch.indices.IndicesOptionsIntegrationIT.refreshBuilder;
 import static org.elasticsearch.indices.IndicesOptionsIntegrationIT.search;
 import static org.elasticsearch.indices.IndicesOptionsIntegrationIT.segments;
+import static org.elasticsearch.indices.IndicesOptionsIntegrationIT.updateSettings;
 import static org.elasticsearch.indices.IndicesOptionsIntegrationIT.validateQuery;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
 import static org.hamcrest.Matchers.equalTo;
@ -386,7 +391,11 @@ public class DataStreamIT extends ESIntegTestCase {
        verifyResolvability(dataStreamName, client().admin().indices().prepareUpgradeStatus(dataStreamName), false);
        verifyResolvability(dataStreamName, getAliases(dataStreamName), true);
        verifyResolvability(dataStreamName, getFieldMapping(dataStreamName), false);
+        verifyResolvability(dataStreamName,
+            putMapping("{\"_doc\":{\"properties\": {\"my_field\":{\"type\":\"keyword\"}}}}", dataStreamName), false);
        verifyResolvability(dataStreamName, getMapping(dataStreamName), false);
+        verifyResolvability(dataStreamName,
+            updateSettings(Settings.builder().put("index.number_of_replicas", 0), dataStreamName), false);
        verifyResolvability(dataStreamName, getSettings(dataStreamName), false);
        verifyResolvability(dataStreamName, health(dataStreamName), false);
        verifyResolvability(dataStreamName, client().admin().cluster().prepareState().setIndices(dataStreamName), false);
@ -417,8 +426,12 @@ public class DataStreamIT extends ESIntegTestCase {
        verifyResolvability(wildcardExpression, client().admin().indices().prepareUpgradeStatus(wildcardExpression), false);
        verifyResolvability(wildcardExpression, getAliases(wildcardExpression), true);
        verifyResolvability(wildcardExpression, getFieldMapping(wildcardExpression), false);
+        verifyResolvability(wildcardExpression,
+            putMapping("{\"_doc\":{\"properties\": {\"my_field\":{\"type\":\"keyword\"}}}}", wildcardExpression), false);
        verifyResolvability(wildcardExpression, getMapping(wildcardExpression), false);
        verifyResolvability(wildcardExpression, getSettings(wildcardExpression), false);
+        verifyResolvability(wildcardExpression,
+            updateSettings(Settings.builder().put("index.number_of_replicas", 0), wildcardExpression), false);
        verifyResolvability(wildcardExpression, health(wildcardExpression), false);
        verifyResolvability(wildcardExpression, client().admin().cluster().prepareState().setIndices(wildcardExpression), false);
        verifyResolvability(wildcardExpression, client().prepareFieldCaps(wildcardExpression).setFields("*"), false);
@ -594,6 +607,58 @@ public class DataStreamIT extends ESIntegTestCase {
        client().admin().indices().deleteDataStream(deleteDataStreamRequest).actionGet();
    }

+    public void testUpdateMappingViaDataStream() throws Exception {
+        putComposableIndexTemplate("id1", "@timestamp", List.of("logs-*"));
+        CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request("logs-foobar");
+        client().admin().indices().createDataStream(createDataStreamRequest).actionGet();
+
+        String backingIndex1 = DataStream.getDefaultBackingIndexName("logs-foobar", 1);
+        String backingIndex2 = DataStream.getDefaultBackingIndexName("logs-foobar", 2);
+
+        RolloverResponse rolloverResponse = client().admin().indices().rolloverIndex(new RolloverRequest("logs-foobar", null)).get();
+        assertThat(rolloverResponse.getNewIndex(), equalTo(backingIndex2));
+        assertTrue(rolloverResponse.isRolledOver());
+
+        java.util.Map<?, ?> expectedMapping = Map.of("properties", Map.of("@timestamp", Map.of("type", "date")));
+        GetMappingsResponse getMappingsResponse = getMapping("logs-foobar").get();
+        assertThat(getMappingsResponse.getMappings().size(), equalTo(2));
+        assertThat(getMappingsResponse.getMappings().get(backingIndex1).get("_doc").getSourceAsMap(), equalTo(expectedMapping));
+        assertThat(getMappingsResponse.getMappings().get(backingIndex2).get("_doc").getSourceAsMap(), equalTo(expectedMapping));
+
+        expectedMapping = Map.of("properties", Map.of("@timestamp", Map.of("type", "date"), "my_field", Map.of("type", "keyword")));
+        putMapping("{\"properties\":{\"my_field\":{\"type\":\"keyword\"}}}", "logs-foobar").get();
+        // The mappings of all backing indices should be updated:
+        getMappingsResponse = getMapping("logs-foobar").get();
+        assertThat(getMappingsResponse.getMappings().size(), equalTo(2));
+        assertThat(getMappingsResponse.getMappings().get(backingIndex1).get("_doc").getSourceAsMap(), equalTo(expectedMapping));
+        assertThat(getMappingsResponse.getMappings().get(backingIndex2).get("_doc").getSourceAsMap(), equalTo(expectedMapping));
+    }
+
+    public void testUpdateIndexSettingsViaDataStream() throws Exception {
+        putComposableIndexTemplate("id1", "@timestamp", List.of("logs-*"));
+        CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request("logs-foobar");
+        client().admin().indices().createDataStream(createDataStreamRequest).actionGet();
+
+        String backingIndex1 = DataStream.getDefaultBackingIndexName("logs-foobar", 1);
+        String backingIndex2 = DataStream.getDefaultBackingIndexName("logs-foobar", 2);
+
+        RolloverResponse rolloverResponse = client().admin().indices().rolloverIndex(new RolloverRequest("logs-foobar", null)).get();
+        assertThat(rolloverResponse.getNewIndex(), equalTo(backingIndex2));
+        assertTrue(rolloverResponse.isRolledOver());
+
+        // The index settings of all backing indices should be updated:
+        GetSettingsResponse getSettingsResponse = getSettings("logs-foobar").get();
+        assertThat(getSettingsResponse.getIndexToSettings().size(), equalTo(2));
+        assertThat(getSettingsResponse.getSetting(backingIndex1, "index.number_of_replicas"), equalTo("1"));
+        assertThat(getSettingsResponse.getSetting(backingIndex2, "index.number_of_replicas"), equalTo("1"));
+
+        updateSettings(Settings.builder().put("index.number_of_replicas", 0), "logs-foobar").get();
+        getSettingsResponse = getSettings("logs-foobar").get();
+        assertThat(getSettingsResponse.getIndexToSettings().size(), equalTo(2));
+        assertThat(getSettingsResponse.getSetting(backingIndex1, "index.number_of_replicas"), equalTo("0"));
+        assertThat(getSettingsResponse.getSetting(backingIndex2, "index.number_of_replicas"), equalTo("0"));
+    }
+
    private static void assertBackingIndex(String backingIndex, String timestampFieldPathInMapping) {
        assertBackingIndex(backingIndex, timestampFieldPathInMapping, Map.of("type", "date"));
    }
--- a/server/src/internalClusterTest/java/org/elasticsearch/indices/IndicesOptionsIntegrationIT.java
+++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/IndicesOptionsIntegrationIT.java
@ -30,10 +30,12 @@ import org.elasticsearch.action.admin.indices.flush.FlushRequestBuilder;
 import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequestBuilder;
 import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsRequestBuilder;
 import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequestBuilder;
+import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequestBuilder;
 import org.elasticsearch.action.admin.indices.refresh.RefreshRequestBuilder;
 import org.elasticsearch.action.admin.indices.segments.IndicesSegmentsRequestBuilder;
 import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequestBuilder;
 import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse;
+import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequestBuilder;
 import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequestBuilder;
 import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryRequestBuilder;
 import org.elasticsearch.action.search.MultiSearchRequestBuilder;
@ -46,6 +48,7 @@ import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
@ -703,10 +706,20 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase {
        return client().admin().indices().prepareGetMappings(indices);
    }

+    static PutMappingRequestBuilder putMapping(String source, String... indices) {
+        return client().admin().indices().preparePutMapping(indices)
+            .setType("_doc")
+            .setSource(source, XContentType.JSON);
+    }
+
    static GetSettingsRequestBuilder getSettings(String... indices) {
        return client().admin().indices().prepareGetSettings(indices);
    }

+    static UpdateSettingsRequestBuilder updateSettings(Settings.Builder settings, String... indices) {
+        return client().admin().indices().prepareUpdateSettings(indices).setSettings(settings);
+    }
+
    private static CreateSnapshotRequestBuilder snapshot(String name, String... indices) {
        return client().admin().cluster().prepareCreateSnapshot("dummy-repo", name).setWaitForCompletion(true).setIndices(indices);
    }
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportPutMappingAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportPutMappingAction.java
@ -86,7 +86,7 @@ public class TransportPutMappingAction extends TransportMasterNodeAction<PutMapp
    protected ClusterBlockException checkBlock(PutMappingRequest request, ClusterState state) {
        String[] indices;
        if (request.getConcreteIndex() == null) {
-            indices = indexNameExpressionResolver.concreteIndexNames(state, request);
+            indices = indexNameExpressionResolver.concreteIndexNames(state, request, true);
        } else {
            indices = new String[] {request.getConcreteIndex().getName()};
        }
@ -98,7 +98,7 @@ public class TransportPutMappingAction extends TransportMasterNodeAction<PutMapp
                                   final ActionListener<AcknowledgedResponse> listener) {
        try {
            final Index[] concreteIndices = request.getConcreteIndex() == null ?
-                indexNameExpressionResolver.concreteIndices(state, request)
+                indexNameExpressionResolver.concreteIndices(state, request, true)
                : new Index[] {request.getConcreteIndex()};
            final Optional<Exception> maybeValidationException = requestValidators.validateRequest(request, state, concreteIndices);
            if (maybeValidationException.isPresent()) {
--- a/server/src/main/java/org/elasticsearch/action/admin/indices/settings/put/TransportUpdateSettingsAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/indices/settings/put/TransportUpdateSettingsAction.java
@ -77,7 +77,7 @@ public class TransportUpdateSettingsAction extends TransportMasterNodeAction<Upd
            return null;
        }
        return state.blocks().indicesBlockedException(ClusterBlockLevel.METADATA_WRITE,
-            indexNameExpressionResolver.concreteIndexNames(state, request));
+            indexNameExpressionResolver.concreteIndexNames(state, request, true));
    }

    @Override
@ -88,7 +88,7 @@ public class TransportUpdateSettingsAction extends TransportMasterNodeAction<Upd
    @Override
    protected void masterOperation(final UpdateSettingsRequest request, final ClusterState state,
                                   final ActionListener<AcknowledgedResponse> listener) {
-        final Index[] concreteIndices = indexNameExpressionResolver.concreteIndices(state, request);
+        final Index[] concreteIndices = indexNameExpressionResolver.concreteIndices(state, request, true);
        UpdateSettingsClusterStateUpdateRequest clusterStateUpdateRequest = new UpdateSettingsClusterStateUpdateRequest()
                .indices(concreteIndices)
                .settings(request.settings())