honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Allow plugins to upgrade templates and index metadata on startup (#24379) 

The UpgraderPlugin adds two additional extension points called during cluster upgrade and when old indices are introduced into the cluster state during initial recovery, restore from a snapshot or as a dangling index. One extension point allows plugin to update old templates and another extension points allows to update/check index metadata.
This commit is contained in:
Igor Motov 2017-05-03 12:51:41 -04:00 committed by GitHub
parent 855b64b0ee
commit cacba6bc46
8 changed files with 273 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeService.java
View File

@ -35,11 +35,14 @@ import org.elasticsearch.index.analysis.NamedAnalyzer;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.similarity.SimilarityService;
import org.elasticsearch.indices.mapper.MapperRegistry;
import org.elasticsearch.plugins.Plugin;
import java.util.AbstractMap;
import java.util.Collection;
import java.util.Collections;
import java.util.Map;
import java.util.Set;
import java.util.function.UnaryOperator;
/**
* This service is responsible for upgrading legacy index metadata to the current version
@ -54,14 +57,23 @@ public class MetaDataIndexUpgradeService extends AbstractComponent {
private final NamedXContentRegistry xContentRegistry;
private final MapperRegistry mapperRegistry;
private final IndexScopedSettings indexScopedSettings;
private final UnaryOperator<IndexMetaData> upgraders;
@Inject
public MetaDataIndexUpgradeService(Settings settings, NamedXContentRegistry xContentRegistry, MapperRegistry mapperRegistry,
IndexScopedSettings indexScopedSettings) {
IndexScopedSettings indexScopedSettings,
Collection<UnaryOperator<IndexMetaData>> indexMetaDataUpgraders) {
super(settings);
this.xContentRegistry = xContentRegistry;
this.mapperRegistry = mapperRegistry;
this.indexScopedSettings = indexScopedSettings;
this.upgraders = indexMetaData -> {
IndexMetaData newIndexMetaData = indexMetaData;
for (UnaryOperator<IndexMetaData> upgrader : indexMetaDataUpgraders) {
newIndexMetaData = upgrader.apply(newIndexMetaData);
}
return newIndexMetaData;
};
}
/**
@ -84,6 +96,8 @@ public class MetaDataIndexUpgradeService extends AbstractComponent {
newMetaData = archiveBrokenIndexSettings(newMetaData);
// only run the check with the upgraded settings!!
checkMappingsCompatibility(newMetaData);
// apply plugin checks
newMetaData = upgraders.apply(newMetaData);
return markAsUpgraded(newMetaData);
}

42
core/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java
View File

@ -31,6 +31,7 @@ import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.cluster.routing.RoutingNode;
import org.elasticsearch.cluster.routing.ShardRouting;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.collect.ImmutableOpenMap;
import org.elasticsearch.common.component.AbstractComponent;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Settings;
@ -50,6 +51,9 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.function.BiConsumer;
import java.util.function.Consumer;
import java.util.function.UnaryOperator;
import static java.util.Collections.emptySet;
import static java.util.Collections.unmodifiableSet;
@ -247,23 +251,41 @@ public class GatewayMetaState extends AbstractComponent implements ClusterStateA
changed |= indexMetaData != newMetaData;
upgradedMetaData.put(newMetaData, false);
}
// collect current customs
Map<String, MetaData.Custom> existingCustoms = new HashMap<>();
for (ObjectObjectCursor<String, MetaData.Custom> customCursor : metaData.customs()) {
existingCustoms.put(customCursor.key, customCursor.value);
}
// upgrade global custom meta data
Map<String, MetaData.Custom> upgradedCustoms = metaDataUpgrader.customMetaDataUpgraders.apply(existingCustoms);
if (upgradedCustoms.equals(existingCustoms) == false) {
existingCustoms.keySet().forEach(upgradedMetaData::removeCustom);
for (Map.Entry<String, MetaData.Custom> upgradedCustomEntry : upgradedCustoms.entrySet()) {
upgradedMetaData.putCustom(upgradedCustomEntry.getKey(), upgradedCustomEntry.getValue());
if (applyPluginUpgraders(metaData.getCustoms(), metaDataUpgrader.customMetaDataUpgraders,
upgradedMetaData::removeCustom,upgradedMetaData::putCustom)) {
changed = true;
}
// upgrade current templates
if (applyPluginUpgraders(metaData.getTemplates(), metaDataUpgrader.indexTemplateMetaDataUpgraders,
upgradedMetaData::removeTemplate, (s, indexTemplateMetaData) -> upgradedMetaData.put(indexTemplateMetaData))) {
changed = true;
}
return changed ? upgradedMetaData.build() : metaData;
}
private static <Data> boolean applyPluginUpgraders(ImmutableOpenMap<String, Data> existingData,
UnaryOperator<Map<String, Data>> upgrader,
Consumer<String> removeData,
BiConsumer<String, Data> putData) {
// collect current data
Map<String, Data> existingMap = new HashMap<>();
for (ObjectObjectCursor<String, Data> customCursor : existingData) {
existingMap.put(customCursor.key, customCursor.value);
}
// upgrade global custom meta data
Map<String, Data> upgradedCustoms = upgrader.apply(existingMap);
if (upgradedCustoms.equals(existingMap) == false) {
// remove all data first so a plugin can remove custom metadata or templates if needed
existingMap.keySet().forEach(removeData);
for (Map.Entry<String, Data> upgradedCustomEntry : upgradedCustoms.entrySet()) {
putData.accept(upgradedCustomEntry.getKey(), upgradedCustomEntry.getValue());
}
return true;
}
return false;
}
// shard state BWC
private void ensureNoPre019ShardState(NodeEnvironment nodeEnv) throws Exception {
for (Path dataLocation : nodeEnv.nodeDataPaths()) {

20
core/src/main/java/org/elasticsearch/node/Node.java
View File

@ -43,6 +43,8 @@ import org.elasticsearch.cluster.ClusterStateObserver;
import org.elasticsearch.cluster.InternalClusterInfoService;
import org.elasticsearch.cluster.NodeConnectionsService;
import org.elasticsearch.cluster.action.index.MappingUpdatedAction;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
import org.elasticsearch.cluster.metadata.MetaData;
import org.elasticsearch.cluster.metadata.MetaDataIndexUpgradeService;
import org.elasticsearch.cluster.node.DiscoveryNode;
@ -400,14 +402,20 @@ public class Node implements Closeable {
.flatMap(p -> p.createComponents(client, clusterService, threadPool, resourceWatcherService,
scriptModule.getScriptService(), xContentRegistry).stream())
.collect(Collectors.toList());
final RestController restController = actionModule.getRestController();
final NetworkModule networkModule = new NetworkModule(settings, false, pluginsService.filterPlugins(NetworkPlugin.class),
threadPool, bigArrays, circuitBreakerService, namedWriteableRegistry, xContentRegistry, networkService, restController);
Collection<UnaryOperator<Map<String, MetaData.Custom>>> customMetaDataUpgraders =
pluginsService.filterPlugins(Plugin.class).stream()
.map(Plugin::getCustomMetaDataUpgrader)
.collect(Collectors.toList());
final RestController restController = actionModule.getRestController();
final NetworkModule networkModule = new NetworkModule(settings, false, pluginsService.filterPlugins(NetworkPlugin.class),
threadPool, bigArrays, circuitBreakerService, namedWriteableRegistry, xContentRegistry, networkService, restController);
final MetaDataUpgrader metaDataUpgrader = new MetaDataUpgrader(customMetaDataUpgraders);
Collection<UnaryOperator<Map<String, IndexTemplateMetaData>>> indexTemplateMetaDataUpgraders =
pluginsService.filterPlugins(Plugin.class).stream()
.map(Plugin::getIndexTemplateMetaDataUpgrader)
.collect(Collectors.toList());
Collection<UnaryOperator<IndexMetaData>> indexMetaDataUpgraders = pluginsService.filterPlugins(Plugin.class).stream()
.map(Plugin::getIndexMetaDataUpgrader).collect(Collectors.toList());
final MetaDataUpgrader metaDataUpgrader = new MetaDataUpgrader(customMetaDataUpgraders, indexTemplateMetaDataUpgraders);
final Transport transport = networkModule.getTransportSupplier().get();
final TransportService transportService = newTransportService(settings, transport, threadPool,
networkModule.getTransportInterceptor(), localNodeFactory, settingsModule.getClusterSettings());
@ -462,8 +470,8 @@ public class Node implements Closeable {
b.bind(TransportService.class).toInstance(transportService);
b.bind(NetworkService.class).toInstance(networkService);
b.bind(UpdateHelper.class).toInstance(new UpdateHelper(settings, scriptModule.getScriptService()));
b.bind(MetaDataIndexUpgradeService.class).toInstance(new MetaDataIndexUpgradeService(settings,
xContentRegistry, indicesModule.getMapperRegistry(), settingsModule.getIndexScopedSettings()));
b.bind(MetaDataIndexUpgradeService.class).toInstance(new MetaDataIndexUpgradeService(settings, xContentRegistry,
indicesModule.getMapperRegistry(), settingsModule.getIndexScopedSettings(), indexMetaDataUpgraders));
b.bind(ClusterInfoService.class).toInstance(clusterInfoService);
b.bind(Discovery.class).toInstance(discoveryModule.getDiscovery());
{

14
core/src/main/java/org/elasticsearch/plugins/MetaDataUpgrader.java
View File

@ -19,6 +19,7 @@
package org.elasticsearch.plugins;
import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
import org.elasticsearch.cluster.metadata.MetaData;
import java.util.Collection;
@ -32,7 +33,10 @@ import java.util.function.UnaryOperator;
public class MetaDataUpgrader {
public final UnaryOperator<Map<String, MetaData.Custom>> customMetaDataUpgraders;
public MetaDataUpgrader(Collection<UnaryOperator<Map<String, MetaData.Custom>>> customMetaDataUpgraders) {
public final UnaryOperator<Map<String, IndexTemplateMetaData>> indexTemplateMetaDataUpgraders;
public MetaDataUpgrader(Collection<UnaryOperator<Map<String, MetaData.Custom>>> customMetaDataUpgraders,
Collection<UnaryOperator<Map<String, IndexTemplateMetaData>>> indexTemplateMetaDataUpgraders) {
this.customMetaDataUpgraders = customs -> {
Map<String, MetaData.Custom> upgradedCustoms = new HashMap<>(customs);
for (UnaryOperator<Map<String, MetaData.Custom>> customMetaDataUpgrader : customMetaDataUpgraders) {
@ -40,5 +44,13 @@ public class MetaDataUpgrader {
}
return upgradedCustoms;
};
this.indexTemplateMetaDataUpgraders = templates -> {
Map<String, IndexTemplateMetaData> upgradedTemplates = new HashMap<>(templates);
for (UnaryOperator<Map<String, IndexTemplateMetaData>> upgrader : indexTemplateMetaDataUpgraders) {
upgradedTemplates = upgrader.apply(upgradedTemplates);
}
return upgradedTemplates;
};
}
}

37
core/src/main/java/org/elasticsearch/plugins/Plugin.java
View File

@ -23,6 +23,8 @@ import org.elasticsearch.action.ActionModule;
import org.elasticsearch.bootstrap.BootstrapCheck;
import org.elasticsearch.client.Client;
import org.elasticsearch.cluster.ClusterModule;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
import org.elasticsearch.cluster.metadata.MetaData;
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.component.LifecycleComponent;
@ -153,6 +155,10 @@ public abstract class Plugin implements Closeable {
* Provides a function to modify global custom meta data on startup.
* <p>
* Plugins should return the input custom map via {@link UnaryOperator#identity()} if no upgrade is required.
* <p>
* The order of custom meta data upgraders calls is undefined and can change between runs so, it is expected that
* plugins will modify only data owned by them to avoid conflicts.
* <p>
* @return Never {@code null}. The same or upgraded {@code MetaData.Custom} map.
* @throws IllegalStateException if the node should not start because at least one {@code MetaData.Custom}
* is unsupported
@ -161,6 +167,37 @@ public abstract class Plugin implements Closeable {
return UnaryOperator.identity();
}
/**
* Provides a function to modify index template meta data on startup.
* <p>
* Plugins should return the input template map via {@link UnaryOperator#identity()} if no upgrade is required.
* <p>
* The order of the template upgrader calls is undefined and can change between runs so, it is expected that
* plugins will modify only templates owned by them to avoid conflicts.
* <p>
* @return Never {@code null}. The same or upgraded {@code IndexTemplateMetaData} map.
* @throws IllegalStateException if the node should not start because at least one {@code IndexTemplateMetaData}
* cannot be upgraded
*/
public UnaryOperator<Map<String, IndexTemplateMetaData>> getIndexTemplateMetaDataUpgrader() {
return UnaryOperator.identity();
}
/**
* Provides a function to modify index meta data when an index is introduced into the cluster state for the first time.
* <p>
* Plugins should return the input index metadata via {@link UnaryOperator#identity()} if no upgrade is required.
* <p>
* The order of the index upgrader calls for the same index is undefined and can change between runs so, it is expected that
* plugins will modify only indices owned by them to avoid conflicts.
* <p>
* @return Never {@code null}. The same or upgraded {@code IndexMetaData}.
* @throws IllegalStateException if the node should not start because the index is unsupported
*/
public UnaryOperator<IndexMetaData> getIndexMetaDataUpgrader() {
return UnaryOperator.identity();
}
/**
* Provides the list of this plugin's custom thread pools, empty if
* none.

44
core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeServiceTests.java
View File

@ -31,7 +31,8 @@ public class MetaDataIndexUpgradeServiceTests extends ESTestCase {
public void testArchiveBrokenIndexSettings() {
MetaDataIndexUpgradeService service = new MetaDataIndexUpgradeService(Settings.EMPTY, xContentRegistry(),
new MapperRegistry(Collections.emptyMap(), Collections.emptyMap()), IndexScopedSettings.DEFAULT_SCOPED_SETTINGS);
new MapperRegistry(Collections.emptyMap(), Collections.emptyMap()), IndexScopedSettings.DEFAULT_SCOPED_SETTINGS,
Collections.emptyList());
IndexMetaData src = newIndexMeta("foo", Settings.EMPTY);
IndexMetaData indexMetaData = service.archiveBrokenIndexSettings(src);
assertSame(indexMetaData, src);
@ -58,7 +59,8 @@ public class MetaDataIndexUpgradeServiceTests extends ESTestCase {
public void testUpgrade() {
MetaDataIndexUpgradeService service = new MetaDataIndexUpgradeService(Settings.EMPTY, xContentRegistry(),
new MapperRegistry(Collections.emptyMap(), Collections.emptyMap()), IndexScopedSettings.DEFAULT_SCOPED_SETTINGS);
new MapperRegistry(Collections.emptyMap(), Collections.emptyMap()), IndexScopedSettings.DEFAULT_SCOPED_SETTINGS,
Collections.emptyList());
IndexMetaData src = newIndexMeta("foo", Settings.builder().put("index.refresh_interval", "-200").build());
assertFalse(service.isUpgraded(src));
src = service.upgradeIndexMetaData(src, Version.CURRENT.minimumIndexCompatibilityVersion());
@ -70,7 +72,8 @@ public class MetaDataIndexUpgradeServiceTests extends ESTestCase {
public void testIsUpgraded() {
MetaDataIndexUpgradeService service = new MetaDataIndexUpgradeService(Settings.EMPTY, xContentRegistry(),
new MapperRegistry(Collections.emptyMap(), Collections.emptyMap()), IndexScopedSettings.DEFAULT_SCOPED_SETTINGS);
new MapperRegistry(Collections.emptyMap(), Collections.emptyMap()), IndexScopedSettings.DEFAULT_SCOPED_SETTINGS,
Collections.emptyList());
IndexMetaData src = newIndexMeta("foo", Settings.builder().put("index.refresh_interval", "-200").build());
assertFalse(service.isUpgraded(src));
Version version = VersionUtils.randomVersionBetween(random(), VersionUtils.getFirstVersion(), VersionUtils.getPreviousVersion());
@ -82,7 +85,8 @@ public class MetaDataIndexUpgradeServiceTests extends ESTestCase {
public void testFailUpgrade() {
MetaDataIndexUpgradeService service = new MetaDataIndexUpgradeService(Settings.EMPTY, xContentRegistry(),
new MapperRegistry(Collections.emptyMap(), Collections.emptyMap()), IndexScopedSettings.DEFAULT_SCOPED_SETTINGS);
new MapperRegistry(Collections.emptyMap(), Collections.emptyMap()), IndexScopedSettings.DEFAULT_SCOPED_SETTINGS,
Collections.emptyList());
final IndexMetaData metaData = newIndexMeta("foo", Settings.builder()
.put(IndexMetaData.SETTING_VERSION_UPGRADED, Version.V_5_0_0_beta1)
.put(IndexMetaData.SETTING_VERSION_CREATED, Version.fromString("2.4.0"))
@ -99,6 +103,38 @@ public class MetaDataIndexUpgradeServiceTests extends ESTestCase {
service.upgradeIndexMetaData(goodMeta, Version.V_5_0_0.minimumIndexCompatibilityVersion());
}
public void testPluginUpgrade() {
MetaDataIndexUpgradeService service = new MetaDataIndexUpgradeService(Settings.EMPTY, xContentRegistry(),
new MapperRegistry(Collections.emptyMap(), Collections.emptyMap()), IndexScopedSettings.DEFAULT_SCOPED_SETTINGS,
Collections.singletonList(
indexMetaData -> IndexMetaData.builder(indexMetaData)
.settings(
Settings.builder()
.put(indexMetaData.getSettings())
.put("index.refresh_interval", "10s")
).build()));
IndexMetaData src = newIndexMeta("foo", Settings.builder().put("index.refresh_interval", "200s").build());
assertFalse(service.isUpgraded(src));
src = service.upgradeIndexMetaData(src, Version.CURRENT.minimumIndexCompatibilityVersion());
assertTrue(service.isUpgraded(src));
assertEquals("10s", src.getSettings().get("index.refresh_interval"));
assertSame(src, service.upgradeIndexMetaData(src, Version.CURRENT.minimumIndexCompatibilityVersion())); // no double upgrade
}
public void testPluginUpgradeFailure() {
MetaDataIndexUpgradeService service = new MetaDataIndexUpgradeService(Settings.EMPTY, xContentRegistry(),
new MapperRegistry(Collections.emptyMap(), Collections.emptyMap()), IndexScopedSettings.DEFAULT_SCOPED_SETTINGS,
Collections.singletonList(
indexMetaData -> {
throw new IllegalStateException("Cannot upgrade index " + indexMetaData.getIndex().getName());
}
));
IndexMetaData src = newIndexMeta("foo", Settings.EMPTY);
String message = expectThrows(IllegalStateException.class, () -> service.upgradeIndexMetaData(src,
Version.CURRENT.minimumIndexCompatibilityVersion())).getMessage();
assertEquals(message, "Cannot upgrade index foo");
}
public static IndexMetaData newIndexMeta(String name, Settings indexSettings) {
Settings build = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1)

127
core/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java
View File

@ -22,7 +22,9 @@ package org.elasticsearch.gateway;
import org.elasticsearch.Version;
import org.elasticsearch.cluster.ClusterChangedEvent;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.ESAllocationTestCase;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
import org.elasticsearch.cluster.metadata.MetaData;
import org.elasticsearch.cluster.metadata.MetaDataIndexUpgradeService;
import org.elasticsearch.cluster.node.DiscoveryNode;
@ -33,7 +35,6 @@ import org.elasticsearch.cluster.routing.allocation.decider.ClusterRebalanceAllo
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.Index;
import org.elasticsearch.plugins.MetaDataUpgrader;
import org.elasticsearch.cluster.ESAllocationTestCase;
import org.elasticsearch.test.TestCustomMetaData;
import java.util.Arrays;
@ -258,7 +259,8 @@ public class GatewayMetaStateTests extends ESAllocationTestCase {
Collections.singletonList(customs -> {
customs.put(CustomMetaData1.TYPE, new CustomMetaData1("modified_data1"));
return customs;
})
}),
Collections.emptyList()
);
MetaData upgrade = GatewayMetaState.upgradeMetaData(metaData, new MockMetaDataIndexUpgradeService(false), metaDataUpgrader);
assertTrue(upgrade != metaData);
@ -273,7 +275,8 @@ public class GatewayMetaStateTests extends ESAllocationTestCase {
Collections.singletonList(customs -> {
customs.remove(CustomMetaData1.TYPE);
return customs;
})
}),
Collections.emptyList()
);
MetaData upgrade = GatewayMetaState.upgradeMetaData(metaData, new MockMetaDataIndexUpgradeService(false), metaDataUpgrader);
assertTrue(upgrade != metaData);
@ -287,7 +290,8 @@ public class GatewayMetaStateTests extends ESAllocationTestCase {
Collections.singletonList(customs -> {
customs.put(CustomMetaData1.TYPE, new CustomMetaData1("modified_data1"));
return customs;
})
}),
Collections.emptyList()
);
MetaData upgrade = GatewayMetaState.upgradeMetaData(metaData, new MockMetaDataIndexUpgradeService(false), metaDataUpgrader);
@ -297,9 +301,27 @@ public class GatewayMetaStateTests extends ESAllocationTestCase {
assertThat(((TestCustomMetaData) upgrade.custom(CustomMetaData1.TYPE)).getData(), equalTo("modified_data1"));
}
public void testUpdateTemplateMetaDataOnUpgrade() throws Exception {
MetaData metaData = randomMetaData();
MetaDataUpgrader metaDataUpgrader = new MetaDataUpgrader(
Collections.emptyList(),
Collections.singletonList(
templates -> {
templates.put("added_test_template", IndexTemplateMetaData.builder("added_test_template").build());
return templates;
}
));
MetaData upgrade = GatewayMetaState.upgradeMetaData(metaData, new MockMetaDataIndexUpgradeService(false), metaDataUpgrader);
assertTrue(upgrade != metaData);
assertFalse(MetaData.isGlobalStateEquals(upgrade, metaData));
assertTrue(upgrade.templates().containsKey("added_test_template"));
}
public void testNoMetaDataUpgrade() throws Exception {
MetaData metaData = randomMetaData(new CustomMetaData1("data"));
MetaDataUpgrader metaDataUpgrader = new MetaDataUpgrader(Collections.emptyList());
MetaDataUpgrader metaDataUpgrader = new MetaDataUpgrader(Collections.emptyList(), Collections.emptyList());
MetaData upgrade = GatewayMetaState.upgradeMetaData(metaData, new MockMetaDataIndexUpgradeService(false), metaDataUpgrader);
assertTrue(upgrade == metaData);
assertTrue(MetaData.isGlobalStateEquals(upgrade, metaData));
@ -314,7 +336,7 @@ public class GatewayMetaStateTests extends ESAllocationTestCase {
customs -> {
throw new IllegalStateException("custom meta data too old");
}
));
), Collections.emptyList());
try {
GatewayMetaState.upgradeMetaData(metaData, new MockMetaDataIndexUpgradeService(false), metaDataUpgrader);
} catch (IllegalStateException e) {
@ -334,7 +356,8 @@ public class GatewayMetaStateTests extends ESAllocationTestCase {
case 2:
metaData = randomMetaData();
break;
default: throw new IllegalStateException("should never happen");
default:
throw new IllegalStateException("should never happen");
}
MetaDataUpgrader metaDataUpgrader = new MetaDataUpgrader(
Arrays.asList(
@ -345,8 +368,8 @@ public class GatewayMetaStateTests extends ESAllocationTestCase {
customs -> {
customs.put(CustomMetaData2.TYPE, new CustomMetaData1("modified_data2"));
return customs;
})
);
}
), Collections.emptyList());
MetaData upgrade = GatewayMetaState.upgradeMetaData(metaData, new MockMetaDataIndexUpgradeService(false), metaDataUpgrader);
assertTrue(upgrade != metaData);
assertFalse(MetaData.isGlobalStateEquals(upgrade, metaData));
@ -361,7 +384,7 @@ public class GatewayMetaStateTests extends ESAllocationTestCase {
public void testIndexMetaDataUpgrade() throws Exception {
MetaData metaData = randomMetaData();
MetaDataUpgrader metaDataUpgrader = new MetaDataUpgrader(Collections.emptyList());
MetaDataUpgrader metaDataUpgrader = new MetaDataUpgrader(Collections.emptyList(), Collections.emptyList());
MetaData upgrade = GatewayMetaState.upgradeMetaData(metaData, new MockMetaDataIndexUpgradeService(true), metaDataUpgrader);
assertTrue(upgrade != metaData);
assertTrue(MetaData.isGlobalStateEquals(upgrade, metaData));
@ -372,7 +395,8 @@ public class GatewayMetaStateTests extends ESAllocationTestCase {
public void testCustomMetaDataNoChange() throws Exception {
MetaData metaData = randomMetaData(new CustomMetaData1("data"));
MetaDataUpgrader metaDataUpgrader = new MetaDataUpgrader(Collections.singletonList(HashMap::new));
MetaDataUpgrader metaDataUpgrader = new MetaDataUpgrader(Collections.singletonList(HashMap::new),
Collections.singletonList(HashMap::new));
MetaData upgrade = GatewayMetaState.upgradeMetaData(metaData, new MockMetaDataIndexUpgradeService(false), metaDataUpgrader);
assertTrue(upgrade == metaData);
assertTrue(MetaData.isGlobalStateEquals(upgrade, metaData));
@ -381,13 +405,71 @@ public class GatewayMetaStateTests extends ESAllocationTestCase {
}
}
public void testIndexTemplateValidation() throws Exception {
MetaData metaData = randomMetaData();
MetaDataUpgrader metaDataUpgrader = new MetaDataUpgrader(
Collections.emptyList(),
Collections.singletonList(
customs -> {
throw new IllegalStateException("template is incompatible");
}));
String message = expectThrows(IllegalStateException.class,
() -> GatewayMetaState.upgradeMetaData(metaData, new MockMetaDataIndexUpgradeService(false), metaDataUpgrader)).getMessage();
assertThat(message, equalTo("template is incompatible"));
}
public void testMultipleIndexTemplateUpgrade() throws Exception {
final MetaData metaData;
switch (randomIntBetween(0, 2)) {
case 0:
metaData = randomMetaDataWithIndexTemplates("template1", "template2");
break;
case 1:
metaData = randomMetaDataWithIndexTemplates(randomBoolean() ? "template1" : "template2");
break;
case 2:
metaData = randomMetaData();
break;
default:
throw new IllegalStateException("should never happen");
}
MetaDataUpgrader metaDataUpgrader = new MetaDataUpgrader(
Collections.emptyList(),
Arrays.asList(
indexTemplateMetaDatas -> {
indexTemplateMetaDatas.put("template1", IndexTemplateMetaData.builder("template1").settings(
Settings.builder().put(IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 20).build()).build());
return indexTemplateMetaDatas;
},
indexTemplateMetaDatas -> {
indexTemplateMetaDatas.put("template2", IndexTemplateMetaData.builder("template2").settings(
Settings.builder().put(IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 10).build()).build());
return indexTemplateMetaDatas;
}
));
MetaData upgrade = GatewayMetaState.upgradeMetaData(metaData, new MockMetaDataIndexUpgradeService(false), metaDataUpgrader);
assertTrue(upgrade != metaData);
assertFalse(MetaData.isGlobalStateEquals(upgrade, metaData));
assertNotNull(upgrade.templates().get("template1"));
assertThat(IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.get(upgrade.templates().get("template1").settings()), equalTo(20));
assertNotNull(upgrade.templates().get("template2"));
assertThat(IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING.get(upgrade.templates().get("template2").settings()), equalTo(10));
for (IndexMetaData indexMetaData : upgrade) {
assertTrue(metaData.hasIndexMetaData(indexMetaData));
}
}
private static class MockMetaDataIndexUpgradeService extends MetaDataIndexUpgradeService {
private final boolean upgrade;
MockMetaDataIndexUpgradeService(boolean upgrade) {
super(Settings.EMPTY, null, null, null);
super(Settings.EMPTY, null, null, null, null);
this.upgrade = upgrade;
}
@Override
public IndexMetaData upgradeIndexMetaData(IndexMetaData indexMetaData, Version minimumIndexCompatibilityVersion) {
return upgrade ? IndexMetaData.builder(indexMetaData).build() : indexMetaData;
@ -445,4 +527,25 @@ public class GatewayMetaStateTests extends ESAllocationTestCase {
}
return builder.build();
}
private static MetaData randomMetaDataWithIndexTemplates(String... templates) {
MetaData.Builder builder = MetaData.builder();
for (String template : templates) {
IndexTemplateMetaData templateMetaData = IndexTemplateMetaData.builder(template)
.settings(settings(Version.CURRENT)
.put(IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), randomIntBetween(0, 3))
.put(IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), randomIntBetween(1, 5)))
.build();
builder.put(templateMetaData);
}
for (int i = 0; i < randomIntBetween(1, 5); i++) {
builder.put(
IndexMetaData.builder(randomAlphaOfLength(10))
.settings(settings(Version.CURRENT))
.numberOfReplicas(randomIntBetween(0, 3))
.numberOfShards(randomIntBetween(1, 5))
);
}
return builder.build();
}
}

3
core/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java
View File

@ -162,7 +162,8 @@ public class ClusterStateChanges extends AbstractComponent {
TransportService transportService = new TransportService(settings, transport, threadPool,
TransportService.NOOP_TRANSPORT_INTERCEPTOR,
boundAddress -> DiscoveryNode.createLocal(settings, boundAddress.publishAddress(), UUIDs.randomBase64UUID()), clusterSettings);
MetaDataIndexUpgradeService metaDataIndexUpgradeService = new MetaDataIndexUpgradeService(settings, xContentRegistry, null, null) {
MetaDataIndexUpgradeService metaDataIndexUpgradeService = new MetaDataIndexUpgradeService(settings, xContentRegistry, null, null,
null) {
// metaData upgrader should do nothing
@Override
public IndexMetaData upgradeIndexMetaData(IndexMetaData indexMetaData, Version minimumIndexCompatibilityVersion) {