Allow plugins to upgrade templates and index metadata on startup (#24379)

The UpgraderPlugin adds two additional extension points called during cluster upgrade and when old indices are introduced into the cluster state during initial recovery, restore from a snapshot or as a dangling index. One extension point allows plugin to update old templates and another extension points allows to update/check index metadata.

core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeService.java

@@ -35,11 +35,14 @@ import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.similarity.SimilarityService;
 import org.elasticsearch.indices.mapper.MapperRegistry;
+import org.elasticsearch.plugins.Plugin;
 
 import java.util.AbstractMap;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.Map;
 import java.util.Set;
+import java.util.function.UnaryOperator;
 
 /**
  * This service is responsible for upgrading legacy index metadata to the current version
@@ -54,14 +57,23 @@ public class MetaDataIndexUpgradeService extends AbstractComponent {
     private final NamedXContentRegistry xContentRegistry;
     private final MapperRegistry mapperRegistry;
     private final IndexScopedSettings indexScopedSettings;
+    private final UnaryOperator<IndexMetaData> upgraders;
 
     @Inject
     public MetaDataIndexUpgradeService(Settings settings, NamedXContentRegistry xContentRegistry, MapperRegistry mapperRegistry,
-            IndexScopedSettings indexScopedSettings) {
+                                       IndexScopedSettings indexScopedSettings,
+                                       Collection<UnaryOperator<IndexMetaData>> indexMetaDataUpgraders) {
         super(settings);
         this.xContentRegistry = xContentRegistry;
         this.mapperRegistry = mapperRegistry;
         this.indexScopedSettings = indexScopedSettings;
+        this.upgraders = indexMetaData -> {
+            IndexMetaData newIndexMetaData = indexMetaData;
+            for (UnaryOperator<IndexMetaData> upgrader : indexMetaDataUpgraders) {
+                newIndexMetaData = upgrader.apply(newIndexMetaData);
+            }
+            return newIndexMetaData;
+        };
     }
 
     /**
@@ -84,6 +96,8 @@ public class MetaDataIndexUpgradeService extends AbstractComponent {
         newMetaData = archiveBrokenIndexSettings(newMetaData);
         // only run the check with the upgraded settings!!
         checkMappingsCompatibility(newMetaData);
+        // apply plugin checks
+        newMetaData = upgraders.apply(newMetaData);
         return markAsUpgraded(newMetaData);
     }
 

core/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java

@@ -31,6 +31,7 @@ import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.RoutingNode;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
@@ -50,6 +51,9 @@ import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.function.BiConsumer;
+import java.util.function.Consumer;
+import java.util.function.UnaryOperator;
 
 import static java.util.Collections.emptySet;
 import static java.util.Collections.unmodifiableSet;
@@ -247,23 +251,41 @@ public class GatewayMetaState extends AbstractComponent implements ClusterStateA
             changed |= indexMetaData != newMetaData;
             upgradedMetaData.put(newMetaData, false);
         }
-        // collect current customs
-        Map<String, MetaData.Custom> existingCustoms = new HashMap<>();
-        for (ObjectObjectCursor<String, MetaData.Custom> customCursor : metaData.customs()) {
-            existingCustoms.put(customCursor.key, customCursor.value);
-        }
         // upgrade global custom meta data
-        Map<String, MetaData.Custom> upgradedCustoms = metaDataUpgrader.customMetaDataUpgraders.apply(existingCustoms);
+        if (applyPluginUpgraders(metaData.getCustoms(), metaDataUpgrader.customMetaDataUpgraders,
-        if (upgradedCustoms.equals(existingCustoms) == false) {
+            upgradedMetaData::removeCustom,upgradedMetaData::putCustom)) {
-            existingCustoms.keySet().forEach(upgradedMetaData::removeCustom);
+            changed = true;
-            for (Map.Entry<String, MetaData.Custom> upgradedCustomEntry : upgradedCustoms.entrySet()) {
-                upgradedMetaData.putCustom(upgradedCustomEntry.getKey(), upgradedCustomEntry.getValue());
         }
+        // upgrade current templates
+        if (applyPluginUpgraders(metaData.getTemplates(), metaDataUpgrader.indexTemplateMetaDataUpgraders,
+            upgradedMetaData::removeTemplate, (s, indexTemplateMetaData) -> upgradedMetaData.put(indexTemplateMetaData))) {
             changed = true;
         }
         return changed ? upgradedMetaData.build() : metaData;
     }
 
+    private static <Data> boolean applyPluginUpgraders(ImmutableOpenMap<String, Data> existingData,
+                                                       UnaryOperator<Map<String, Data>> upgrader,
+                                                       Consumer<String> removeData,
+                                                       BiConsumer<String, Data> putData) {
+        // collect current data
+        Map<String, Data> existingMap = new HashMap<>();
+        for (ObjectObjectCursor<String, Data> customCursor : existingData) {
+            existingMap.put(customCursor.key, customCursor.value);
+        }
+        // upgrade global custom meta data
+        Map<String, Data> upgradedCustoms = upgrader.apply(existingMap);
+        if (upgradedCustoms.equals(existingMap) == false) {
+            // remove all data first so a plugin can remove custom metadata or templates if needed
+            existingMap.keySet().forEach(removeData);
+            for (Map.Entry<String, Data> upgradedCustomEntry : upgradedCustoms.entrySet()) {
+                putData.accept(upgradedCustomEntry.getKey(), upgradedCustomEntry.getValue());
+            }
+            return true;
+        }
+        return false;
+    }
+
     // shard state BWC
     private void ensureNoPre019ShardState(NodeEnvironment nodeEnv) throws Exception {
         for (Path dataLocation : nodeEnv.nodeDataPaths()) {

core/src/main/java/org/elasticsearch/node/Node.java

@@ -43,6 +43,8 @@ import org.elasticsearch.cluster.ClusterStateObserver;
 import org.elasticsearch.cluster.InternalClusterInfoService;
 import org.elasticsearch.cluster.NodeConnectionsService;
 import org.elasticsearch.cluster.action.index.MappingUpdatedAction;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.metadata.MetaDataIndexUpgradeService;
 import org.elasticsearch.cluster.node.DiscoveryNode;
@@ -400,14 +402,20 @@ public class Node implements Closeable {
                 .flatMap(p -> p.createComponents(client, clusterService, threadPool, resourceWatcherService,
                                                  scriptModule.getScriptService(), xContentRegistry).stream())
                 .collect(Collectors.toList());
+            final RestController restController = actionModule.getRestController();
+            final NetworkModule networkModule = new NetworkModule(settings, false, pluginsService.filterPlugins(NetworkPlugin.class),
+                    threadPool, bigArrays, circuitBreakerService, namedWriteableRegistry, xContentRegistry, networkService, restController);
             Collection<UnaryOperator<Map<String, MetaData.Custom>>> customMetaDataUpgraders =
                 pluginsService.filterPlugins(Plugin.class).stream()
                     .map(Plugin::getCustomMetaDataUpgrader)
                     .collect(Collectors.toList());
-            final RestController restController = actionModule.getRestController();
+            Collection<UnaryOperator<Map<String, IndexTemplateMetaData>>> indexTemplateMetaDataUpgraders =
-            final NetworkModule networkModule = new NetworkModule(settings, false, pluginsService.filterPlugins(NetworkPlugin.class),
+                pluginsService.filterPlugins(Plugin.class).stream()
-                    threadPool, bigArrays, circuitBreakerService, namedWriteableRegistry, xContentRegistry, networkService, restController);
+                    .map(Plugin::getIndexTemplateMetaDataUpgrader)
-            final MetaDataUpgrader metaDataUpgrader = new MetaDataUpgrader(customMetaDataUpgraders);
+                    .collect(Collectors.toList());
+            Collection<UnaryOperator<IndexMetaData>> indexMetaDataUpgraders = pluginsService.filterPlugins(Plugin.class).stream()
+                    .map(Plugin::getIndexMetaDataUpgrader).collect(Collectors.toList());
+            final MetaDataUpgrader metaDataUpgrader = new MetaDataUpgrader(customMetaDataUpgraders, indexTemplateMetaDataUpgraders);
             final Transport transport = networkModule.getTransportSupplier().get();
             final TransportService transportService = newTransportService(settings, transport, threadPool,
                 networkModule.getTransportInterceptor(), localNodeFactory, settingsModule.getClusterSettings());
@@ -462,8 +470,8 @@ public class Node implements Closeable {
                     b.bind(TransportService.class).toInstance(transportService);
                     b.bind(NetworkService.class).toInstance(networkService);
                     b.bind(UpdateHelper.class).toInstance(new UpdateHelper(settings, scriptModule.getScriptService()));
-                    b.bind(MetaDataIndexUpgradeService.class).toInstance(new MetaDataIndexUpgradeService(settings,
+                    b.bind(MetaDataIndexUpgradeService.class).toInstance(new MetaDataIndexUpgradeService(settings, xContentRegistry,
-                        xContentRegistry, indicesModule.getMapperRegistry(), settingsModule.getIndexScopedSettings()));
+                        indicesModule.getMapperRegistry(), settingsModule.getIndexScopedSettings(), indexMetaDataUpgraders));
                     b.bind(ClusterInfoService.class).toInstance(clusterInfoService);
                     b.bind(Discovery.class).toInstance(discoveryModule.getDiscovery());
                     {

core/src/main/java/org/elasticsearch/plugins/MetaDataUpgrader.java

@@ -19,6 +19,7 @@
 
 package org.elasticsearch.plugins;
 
+import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 
 import java.util.Collection;
@@ -32,7 +33,10 @@ import java.util.function.UnaryOperator;
 public class MetaDataUpgrader {
     public final UnaryOperator<Map<String, MetaData.Custom>> customMetaDataUpgraders;
 
-    public MetaDataUpgrader(Collection<UnaryOperator<Map<String, MetaData.Custom>>> customMetaDataUpgraders) {
+    public final UnaryOperator<Map<String, IndexTemplateMetaData>> indexTemplateMetaDataUpgraders;
+
+    public MetaDataUpgrader(Collection<UnaryOperator<Map<String, MetaData.Custom>>> customMetaDataUpgraders,
+                            Collection<UnaryOperator<Map<String, IndexTemplateMetaData>>> indexTemplateMetaDataUpgraders) {
         this.customMetaDataUpgraders = customs -> {
             Map<String, MetaData.Custom> upgradedCustoms = new HashMap<>(customs);
             for (UnaryOperator<Map<String, MetaData.Custom>> customMetaDataUpgrader : customMetaDataUpgraders) {
@@ -40,5 +44,13 @@ public class MetaDataUpgrader {
             }
             return upgradedCustoms;
         };
+
+        this.indexTemplateMetaDataUpgraders = templates -> {
+            Map<String, IndexTemplateMetaData> upgradedTemplates = new HashMap<>(templates);
+            for (UnaryOperator<Map<String, IndexTemplateMetaData>> upgrader : indexTemplateMetaDataUpgraders) {
+                upgradedTemplates = upgrader.apply(upgradedTemplates);
+            }
+            return upgradedTemplates;
+        };
     }
 }

core/src/main/java/org/elasticsearch/plugins/Plugin.java

@@ -23,6 +23,8 @@ import org.elasticsearch.action.ActionModule;
 import org.elasticsearch.bootstrap.BootstrapCheck;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.ClusterModule;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.component.LifecycleComponent;
@@ -153,6 +155,10 @@ public abstract class Plugin implements Closeable {
      * Provides a function to modify global custom meta data on startup.
      * <p>
      * Plugins should return the input custom map via {@link UnaryOperator#identity()} if no upgrade is required.
+     * <p>
+     * The order of custom meta data upgraders calls is undefined and can change between runs so, it is expected that
+     * plugins will modify only data owned by them to avoid conflicts.
+     * <p>
      * @return Never {@code null}. The same or upgraded {@code MetaData.Custom} map.
      * @throws IllegalStateException if the node should not start because at least one {@code MetaData.Custom}
      *                               is unsupported
@@ -161,6 +167,37 @@ public abstract class Plugin implements Closeable {
         return UnaryOperator.identity();
     }
 
+    /**
+     * Provides a function to modify index template meta data on startup.
+     * <p>
+     * Plugins should return the input template map via {@link UnaryOperator#identity()} if no upgrade is required.
+     * <p>
+     * The order of the template upgrader calls is undefined and can change between runs so, it is expected that
+     * plugins will modify only templates owned by them to avoid conflicts.
+     * <p>
+     * @return Never {@code null}. The same or upgraded {@code IndexTemplateMetaData} map.
+     * @throws IllegalStateException if the node should not start because at least one {@code IndexTemplateMetaData}
+     *                               cannot be upgraded
+     */
+    public UnaryOperator<Map<String, IndexTemplateMetaData>> getIndexTemplateMetaDataUpgrader() {
+        return UnaryOperator.identity();
+    }
+
+    /**
+     * Provides a function to modify index meta data when an index is introduced into the cluster state for the first time.
+     * <p>
+     * Plugins should return the input index metadata via {@link UnaryOperator#identity()} if no upgrade is required.
+     * <p>
+     * The order of the index upgrader calls for the same index is undefined and can change between runs so, it is expected that
+     * plugins will modify only indices owned by them to avoid conflicts.
+     * <p>
+     * @return Never {@code null}. The same or upgraded {@code IndexMetaData}.
+     * @throws IllegalStateException if the node should not start because the index is unsupported
+     */
+    public UnaryOperator<IndexMetaData> getIndexMetaDataUpgrader() {
+        return UnaryOperator.identity();
+    }
+
     /**
      * Provides the list of this plugin's custom thread pools, empty if
      * none.

core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeServiceTests.java

@@ -31,7 +31,8 @@ public class MetaDataIndexUpgradeServiceTests extends ESTestCase {
 
     public void testArchiveBrokenIndexSettings() {
         MetaDataIndexUpgradeService service = new MetaDataIndexUpgradeService(Settings.EMPTY, xContentRegistry(),
-                new MapperRegistry(Collections.emptyMap(), Collections.emptyMap()), IndexScopedSettings.DEFAULT_SCOPED_SETTINGS);
+            new MapperRegistry(Collections.emptyMap(), Collections.emptyMap()), IndexScopedSettings.DEFAULT_SCOPED_SETTINGS,
+            Collections.emptyList());
         IndexMetaData src = newIndexMeta("foo", Settings.EMPTY);
         IndexMetaData indexMetaData = service.archiveBrokenIndexSettings(src);
         assertSame(indexMetaData, src);
@@ -58,7 +59,8 @@ public class MetaDataIndexUpgradeServiceTests extends ESTestCase {
 
     public void testUpgrade() {
         MetaDataIndexUpgradeService service = new MetaDataIndexUpgradeService(Settings.EMPTY, xContentRegistry(),
-                new MapperRegistry(Collections.emptyMap(), Collections.emptyMap()), IndexScopedSettings.DEFAULT_SCOPED_SETTINGS);
+            new MapperRegistry(Collections.emptyMap(), Collections.emptyMap()), IndexScopedSettings.DEFAULT_SCOPED_SETTINGS,
+            Collections.emptyList());
         IndexMetaData src = newIndexMeta("foo", Settings.builder().put("index.refresh_interval", "-200").build());
         assertFalse(service.isUpgraded(src));
         src = service.upgradeIndexMetaData(src, Version.CURRENT.minimumIndexCompatibilityVersion());
@@ -70,7 +72,8 @@ public class MetaDataIndexUpgradeServiceTests extends ESTestCase {
 
     public void testIsUpgraded() {
         MetaDataIndexUpgradeService service = new MetaDataIndexUpgradeService(Settings.EMPTY, xContentRegistry(),
-                new MapperRegistry(Collections.emptyMap(), Collections.emptyMap()), IndexScopedSettings.DEFAULT_SCOPED_SETTINGS);
+            new MapperRegistry(Collections.emptyMap(), Collections.emptyMap()), IndexScopedSettings.DEFAULT_SCOPED_SETTINGS,
+            Collections.emptyList());
         IndexMetaData src = newIndexMeta("foo", Settings.builder().put("index.refresh_interval", "-200").build());
         assertFalse(service.isUpgraded(src));
         Version version = VersionUtils.randomVersionBetween(random(), VersionUtils.getFirstVersion(), VersionUtils.getPreviousVersion());
@@ -82,7 +85,8 @@ public class MetaDataIndexUpgradeServiceTests extends ESTestCase {
 
     public void testFailUpgrade() {
         MetaDataIndexUpgradeService service = new MetaDataIndexUpgradeService(Settings.EMPTY, xContentRegistry(),
-                new MapperRegistry(Collections.emptyMap(), Collections.emptyMap()), IndexScopedSettings.DEFAULT_SCOPED_SETTINGS);
+            new MapperRegistry(Collections.emptyMap(), Collections.emptyMap()), IndexScopedSettings.DEFAULT_SCOPED_SETTINGS,
+            Collections.emptyList());
         final IndexMetaData metaData = newIndexMeta("foo", Settings.builder()
             .put(IndexMetaData.SETTING_VERSION_UPGRADED, Version.V_5_0_0_beta1)
             .put(IndexMetaData.SETTING_VERSION_CREATED, Version.fromString("2.4.0"))
@@ -99,6 +103,38 @@ public class MetaDataIndexUpgradeServiceTests extends ESTestCase {
         service.upgradeIndexMetaData(goodMeta, Version.V_5_0_0.minimumIndexCompatibilityVersion());
     }
 
+    public void testPluginUpgrade() {
+        MetaDataIndexUpgradeService service = new MetaDataIndexUpgradeService(Settings.EMPTY, xContentRegistry(),
+            new MapperRegistry(Collections.emptyMap(), Collections.emptyMap()), IndexScopedSettings.DEFAULT_SCOPED_SETTINGS,
+            Collections.singletonList(
+                indexMetaData -> IndexMetaData.builder(indexMetaData)
+                    .settings(
+                        Settings.builder()
+                            .put(indexMetaData.getSettings())
+                            .put("index.refresh_interval", "10s")
+                    ).build()));
+        IndexMetaData src = newIndexMeta("foo", Settings.builder().put("index.refresh_interval", "200s").build());
+        assertFalse(service.isUpgraded(src));
+        src = service.upgradeIndexMetaData(src, Version.CURRENT.minimumIndexCompatibilityVersion());
+        assertTrue(service.isUpgraded(src));
+        assertEquals("10s", src.getSettings().get("index.refresh_interval"));
+        assertSame(src, service.upgradeIndexMetaData(src, Version.CURRENT.minimumIndexCompatibilityVersion())); // no double upgrade
+    }
+
+    public void testPluginUpgradeFailure() {
+        MetaDataIndexUpgradeService service = new MetaDataIndexUpgradeService(Settings.EMPTY, xContentRegistry(),
+            new MapperRegistry(Collections.emptyMap(), Collections.emptyMap()), IndexScopedSettings.DEFAULT_SCOPED_SETTINGS,
+            Collections.singletonList(
+                indexMetaData -> {
+                    throw new IllegalStateException("Cannot upgrade index " + indexMetaData.getIndex().getName());
+                }
+            ));
+        IndexMetaData src = newIndexMeta("foo", Settings.EMPTY);
+        String message = expectThrows(IllegalStateException.class, () -> service.upgradeIndexMetaData(src,
+            Version.CURRENT.minimumIndexCompatibilityVersion())).getMessage();
+        assertEquals(message, "Cannot upgrade index foo");
+    }
+
     public static IndexMetaData newIndexMeta(String name, Settings indexSettings) {
         Settings build = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
             .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1)

core/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java

@@ -22,7 +22,9 @@ package org.elasticsearch.gateway;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterChangedEvent;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ESAllocationTestCase;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.metadata.MetaDataIndexUpgradeService;
 import org.elasticsearch.cluster.node.DiscoveryNode;
@@ -33,7 +35,6 @@ import org.elasticsearch.cluster.routing.allocation.decider.ClusterRebalanceAllo
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.plugins.MetaDataUpgrader;
-import org.elasticsearch.cluster.ESAllocationTestCase;
 import org.elasticsearch.test.TestCustomMetaData;
 
 import java.util.Arrays;
@@ -258,7 +259,8 @@ public class GatewayMetaStateTests extends ESAllocationTestCase {
             Collections.singletonList(customs -> {
                 customs.put(CustomMetaData1.TYPE, new CustomMetaData1("modified_data1"));
                 return customs;
-            })
+            }),
+            Collections.emptyList()
         );
         MetaData upgrade = GatewayMetaState.upgradeMetaData(metaData, new MockMetaDataIndexUpgradeService(false), metaDataUpgrader);
         assertTrue(upgrade != metaData);
@@ -273,7 +275,8 @@ public class GatewayMetaStateTests extends ESAllocationTestCase {
             Collections.singletonList(customs -> {
                 customs.remove(CustomMetaData1.TYPE);
                 return customs;
-            })
+            }),
+            Collections.emptyList()
         );
         MetaData upgrade = GatewayMetaState.upgradeMetaData(metaData, new MockMetaDataIndexUpgradeService(false), metaDataUpgrader);
         assertTrue(upgrade != metaData);
@@ -287,7 +290,8 @@ public class GatewayMetaStateTests extends ESAllocationTestCase {
             Collections.singletonList(customs -> {
                 customs.put(CustomMetaData1.TYPE, new CustomMetaData1("modified_data1"));
                 return customs;
-            })
+            }),
+            Collections.emptyList()
         );
 
         MetaData upgrade = GatewayMetaState.upgradeMetaData(metaData, new MockMetaDataIndexUpgradeService(false), metaDataUpgrader);
@@ -297,9 +301,27 @@ public class GatewayMetaStateTests extends ESAllocationTestCase {
         assertThat(((TestCustomMetaData) upgrade.custom(CustomMetaData1.TYPE)).getData(), equalTo("modified_data1"));
     }
 
+
+    public void testUpdateTemplateMetaDataOnUpgrade() throws Exception {
+        MetaData metaData = randomMetaData();
+        MetaDataUpgrader metaDataUpgrader = new MetaDataUpgrader(
+            Collections.emptyList(),
+            Collections.singletonList(
+                templates -> {
+                    templates.put("added_test_template", IndexTemplateMetaData.builder("added_test_template").build());
+                    return templates;
+                }
+            ));
+
+        MetaData upgrade = GatewayMetaState.upgradeMetaData(metaData, new MockMetaDataIndexUpgradeService(false), metaDataUpgrader);
+        assertTrue(upgrade != metaData);
+        assertFalse(MetaData.isGlobalStateEquals(upgrade, metaData));
+        assertTrue(upgrade.templates().containsKey("added_test_template"));
+    }
+
     public void testNoMetaDataUpgrade() throws Exception {
         MetaData metaData = randomMetaData(new CustomMetaData1("data"));
-        MetaDataUpgrader metaDataUpgrader = new MetaDataUpgrader(Collections.emptyList());
+        MetaDataUpgrader metaDataUpgrader = new MetaDataUpgrader(Collections.emptyList(), Collections.emptyList());
         MetaData upgrade = GatewayMetaState.upgradeMetaData(metaData, new MockMetaDataIndexUpgradeService(false), metaDataUpgrader);
         assertTrue(upgrade == metaData);
         assertTrue(MetaData.isGlobalStateEquals(upgrade, metaData));
@@ -314,7 +336,7 @@ public class GatewayMetaStateTests extends ESAllocationTestCase {
             customs -> {
                 throw new IllegalStateException("custom meta data too old");
             }
-        ));
+        ), Collections.emptyList());
         try {
             GatewayMetaState.upgradeMetaData(metaData, new MockMetaDataIndexUpgradeService(false), metaDataUpgrader);
         } catch (IllegalStateException e) {
@@ -334,7 +356,8 @@ public class GatewayMetaStateTests extends ESAllocationTestCase {
             case 2:
                 metaData = randomMetaData();
                 break;
-            default: throw new IllegalStateException("should never happen");
+            default:
+                throw new IllegalStateException("should never happen");
         }
         MetaDataUpgrader metaDataUpgrader = new MetaDataUpgrader(
             Arrays.asList(
@@ -345,8 +368,8 @@ public class GatewayMetaStateTests extends ESAllocationTestCase {
                 customs -> {
                     customs.put(CustomMetaData2.TYPE, new CustomMetaData1("modified_data2"));
                     return customs;
-                })
+                }
-        );
+            ), Collections.emptyList());
         MetaData upgrade = GatewayMetaState.upgradeMetaData(metaData, new MockMetaDataIndexUpgradeService(false), metaDataUpgrader);
         assertTrue(upgrade != metaData);
         assertFalse(MetaData.isGlobalStateEquals(upgrade, metaData));
@@ -361,7 +384,7 @@ public class GatewayMetaStateTests extends ESAllocationTestCase {
 
     public void testIndexMetaDataUpgrade() throws Exception {
         MetaData metaData = randomMetaData();
-        MetaDataUpgrader metaDataUpgrader = new MetaDataUpgrader(Collections.emptyList());
+        MetaDataUpgrader metaDataUpgrader = new MetaDataUpgrader(Collections.emptyList(), Collections.emptyList());
         MetaData upgrade = GatewayMetaState.upgradeMetaData(metaData, new MockMetaDataIndexUpgradeService(true), metaDataUpgrader);
         assertTrue(upgrade != metaData);
         assertTrue(MetaData.isGlobalStateEquals(upgrade, metaData));
@@ -372,7 +395,8 @@ public class GatewayMetaStateTests extends ESAllocationTestCase {
 
     public void testCustomMetaDataNoChange() throws Exception {
         MetaData metaData = randomMetaData(new CustomMetaData1("data"));
-        MetaDataUpgrader metaDataUpgrader = new MetaDataUpgrader(Collections.singletonList(HashMap::new));
+        MetaDataUpgrader metaDataUpgrader = new MetaDataUpgrader(Collections.singletonList(HashMap::new),
+            Collections.singletonList(HashMap::new));
         MetaData upgrade = GatewayMetaState.upgradeMetaData(metaData, new MockMetaDataIndexUpgradeService(false), metaDataUpgrader);
         assertTrue(upgrade == metaData);
         assertTrue(MetaData.isGlobalStateEquals(upgrade, metaData));
@@ -381,13 +405,71 @@ public class GatewayMetaStateTests extends ESAllocationTestCase {
         }
     }
 
+    public void testIndexTemplateValidation() throws Exception {
+        MetaData metaData = randomMetaData();
+        MetaDataUpgrader metaDataUpgrader = new MetaDataUpgrader(
+            Collections.emptyList(),
+            Collections.singletonList(
+                customs -> {
+                    throw new IllegalStateException("template is incompatible");
+                }));
+        String message = expectThrows(IllegalStateException.class,
+            () -> GatewayMetaState.upgradeMetaData(metaData, new MockMetaDataIndexUpgradeService(false), metaDataUpgrader)).getMessage();
+        assertThat(message, equalTo("template is incompatible"));
+    }
+
+
+    public void testMultipleIndexTemplateUpgrade() throws Exception {
+        final MetaData metaData;
+        switch (randomIntBetween(0, 2)) {
+            case 0:
+                metaData = randomMetaDataWithIndexTemplates("template1", "template2");
+                break;
+            case 1:
+                metaData = randomMetaDataWithIndexTemplates(randomBoolean() ? "template1" : "template2");
+                break;
+            case 2:
+                metaData = randomMetaData();
+                break;
+            default:
+                throw new IllegalStateException("should never happen");
+        }
+        MetaDataUpgrader metaDataUpgrader = new MetaDataUpgrader(
+            Collections.emptyList(),
+            Arrays.asList(
+                indexTemplateMetaDatas -> {
+                    indexTemplateMetaDatas.put("template1", IndexTemplateMetaData.builder("template1").settings(
+                        Settings.builder().put(IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 20).build()).build());
+                    return indexTemplateMetaDatas;
+
+                },
+                indexTemplateMetaDatas -> {
+                    indexTemplateMetaDatas.put("template2", IndexTemplateMetaData.builder("template2").settings(
+                        Settings.builder().put(IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 10).build()).build());
+                    return indexTemplateMetaDatas;
+
+                }
+            ));
+        MetaData upgrade = GatewayMetaState.upgradeMetaData(metaData, new MockMetaDataIndexUpgradeService(false), metaDataUpgrader);
+        assertTrue(upgrade != metaData);
+        assertFalse(MetaData.isGlobalStateEquals(upgrade, metaData));
+        assertNotNull(upgrade.templates().get("template1"));
+        assertThat(IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.get(upgrade.templates().get("template1").settings()), equalTo(20));
+        assertNotNull(upgrade.templates().get("template2"));
+        assertThat(IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING.get(upgrade.templates().get("template2").settings()), equalTo(10));
+        for (IndexMetaData indexMetaData : upgrade) {
+            assertTrue(metaData.hasIndexMetaData(indexMetaData));
+        }
+    }
+
     private static class MockMetaDataIndexUpgradeService extends MetaDataIndexUpgradeService {
         private final boolean upgrade;
 
         MockMetaDataIndexUpgradeService(boolean upgrade) {
-            super(Settings.EMPTY, null, null, null);
+            super(Settings.EMPTY, null, null, null, null);
             this.upgrade = upgrade;
         }
+
         @Override
         public IndexMetaData upgradeIndexMetaData(IndexMetaData indexMetaData, Version minimumIndexCompatibilityVersion) {
             return upgrade ? IndexMetaData.builder(indexMetaData).build() : indexMetaData;
@@ -445,4 +527,25 @@ public class GatewayMetaStateTests extends ESAllocationTestCase {
         }
         return builder.build();
     }
+
+    private static MetaData randomMetaDataWithIndexTemplates(String... templates) {
+        MetaData.Builder builder = MetaData.builder();
+        for (String template : templates) {
+            IndexTemplateMetaData templateMetaData = IndexTemplateMetaData.builder(template)
+                .settings(settings(Version.CURRENT)
+                    .put(IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), randomIntBetween(0, 3))
+                    .put(IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), randomIntBetween(1, 5)))
+                .build();
+            builder.put(templateMetaData);
+        }
+        for (int i = 0; i < randomIntBetween(1, 5); i++) {
+            builder.put(
+                IndexMetaData.builder(randomAlphaOfLength(10))
+                    .settings(settings(Version.CURRENT))
+                    .numberOfReplicas(randomIntBetween(0, 3))
+                    .numberOfShards(randomIntBetween(1, 5))
+            );
+        }
+        return builder.build();
+    }
 }

core/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java

@@ -162,7 +162,8 @@ public class ClusterStateChanges extends AbstractComponent {
         TransportService transportService = new TransportService(settings, transport, threadPool,
             TransportService.NOOP_TRANSPORT_INTERCEPTOR,
             boundAddress -> DiscoveryNode.createLocal(settings, boundAddress.publishAddress(), UUIDs.randomBase64UUID()), clusterSettings);
-        MetaDataIndexUpgradeService metaDataIndexUpgradeService = new MetaDataIndexUpgradeService(settings, xContentRegistry, null, null) {
+        MetaDataIndexUpgradeService metaDataIndexUpgradeService = new MetaDataIndexUpgradeService(settings, xContentRegistry, null, null,
+            null) {
             // metaData upgrader should do nothing
             @Override
             public IndexMetaData upgradeIndexMetaData(IndexMetaData indexMetaData, Version minimumIndexCompatibilityVersion) {