From 09bf4e5f007492b5f765941080518c2de2381b35 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Mon, 3 Sep 2018 19:17:57 -0400 Subject: [PATCH] Introduce private settings (#33327) This commit introduces the formal notion of a private setting. This enables us to register some settings that we had previously not registered as fully-fledged settings to avoid them being exposed via APIs such as the create index API. For example, we had hacks in the codebase to allow index.version.created to be passed around inside of settings objects, but was not registered as a setting so that if a user tried to use the setting on any API then they would get an exception. This prevented users from setting index.version.created on index creation, or updating it via the index settings API. By introducing private settings, we can continue to reject these attempts, yet now we can represent these settings as actual settings. In this change, we register index.version.created as an actual setting. We do not cutover all settings that we had been treating as private in this pull request, it is already quite large due to moving some tests around to account for the fact that some tests need to be able to set the index.version.created. This can be done in a follow-up change. --- .../resources/checkstyle_suppressions.xml | 1 - .../main/java/org/elasticsearch/Version.java | 23 +- .../elasticsearch/cluster/ClusterModule.java | 4 +- .../cluster/metadata/AliasValidator.java | 2 - .../cluster/metadata/IndexMetaData.java | 8 +- .../metadata/MetaDataCreateIndexService.java | 54 +++-- .../MetaDataIndexTemplateService.java | 3 +- .../MetaDataUpdateSettingsService.java | 2 +- .../settings/AbstractScopedSettings.java | 53 +++-- .../common/settings/IndexScopedSettings.java | 2 +- .../common/settings/Setting.java | 20 +- .../elasticsearch/index/IndexSettings.java | 2 +- .../java/org/elasticsearch/node/Node.java | 31 ++- .../snapshots/RestoreService.java | 2 +- .../admin/indices/create/ShrinkIndexIT.java | 7 +- .../admin/indices/create/SplitIndexIT.java | 7 +- .../MetaDataIndexTemplateServiceTests.java | 25 ++- .../common/settings/ScopedSettingsTests.java | 27 ++- .../common/settings/SettingTests.java | 9 +- .../org/elasticsearch/get/GetActionIT.java | 38 +--- .../elasticsearch/get/LegacyGetActionIT.java | 88 ++++++++ .../index/analysis/PreBuiltAnalyzerTests.java | 5 + .../index/mapper/DynamicMappingTests.java | 20 -- .../mapper/ExternalFieldMapperTests.java | 21 +- .../mapper/LegacyDynamicMappingTests.java | 67 ++++++ .../mapper/LegacyMapperServiceTests.java | 92 ++++++++ .../mapper/LegacyTypeFieldMapperTests.java | 41 ++++ .../index/mapper/MapperServiceTests.java | 42 ---- .../index/mapper/TypeFieldMapperTests.java | 16 +- .../mapper/UpdateMappingOnClusterIT.java | 71 ------ .../similarity/LegacySimilarityTests.java | 93 ++++++++ .../index/similarity/SimilarityTests.java | 36 --- .../index/store/CorruptedFileIT.java | 7 +- .../PreBuiltAnalyzerIntegrationIT.java | 9 +- .../indices/cluster/ClusterStateChanges.java | 4 +- .../LegacyUpdateMappingIntegrationIT.java | 212 ++++++++++++++++++ .../mapping/UpdateMappingIntegrationIT.java | 70 ------ .../InternalOrPrivateSettingsPlugin.java | 201 +++++++++++++++++ .../indices/settings/InternalSettingsIT.java | 84 +++++++ .../indices/settings/PrivateSettingsIT.java | 81 +++++++ .../indices/settings/UpdateSettingsIT.java | 206 +---------------- .../indices/stats/IndexStatsIT.java | 49 ---- .../indices/stats/LegacyIndexStatsIT.java | 104 +++++++++ .../aggregations/bucket/GeoDistanceIT.java | 10 +- .../aggregations/bucket/GeoHashGridIT.java | 8 +- .../functionscore/DecayFunctionScoreIT.java | 9 +- .../search/geo/GeoBoundingBoxIT.java | 10 +- .../search/geo/GeoDistanceIT.java | 9 +- .../elasticsearch/search/geo/GeoFilterIT.java | 8 +- .../search/geo/GeoPolygonIT.java | 8 +- .../search/query/QueryStringIT.java | 8 - .../search/query/SimpleQueryStringIT.java | 5 +- .../search/sort/GeoDistanceIT.java | 10 +- .../search/sort/GeoDistanceSortBuilderIT.java | 8 +- .../java/org/elasticsearch/node/MockNode.java | 31 ++- .../test/AbstractBuilderTestCase.java | 1 - .../elasticsearch/test/ESIntegTestCase.java | 8 +- .../test/ESSingleNodeTestCase.java | 7 +- .../test/InternalSettingsPlugin.java | 3 - .../test/InternalTestCluster.java | 58 ++++- 60 files changed, 1407 insertions(+), 733 deletions(-) create mode 100644 server/src/test/java/org/elasticsearch/get/LegacyGetActionIT.java create mode 100644 server/src/test/java/org/elasticsearch/index/mapper/LegacyDynamicMappingTests.java create mode 100644 server/src/test/java/org/elasticsearch/index/mapper/LegacyMapperServiceTests.java create mode 100644 server/src/test/java/org/elasticsearch/index/mapper/LegacyTypeFieldMapperTests.java delete mode 100644 server/src/test/java/org/elasticsearch/index/mapper/UpdateMappingOnClusterIT.java create mode 100644 server/src/test/java/org/elasticsearch/index/similarity/LegacySimilarityTests.java create mode 100644 server/src/test/java/org/elasticsearch/indices/mapping/LegacyUpdateMappingIntegrationIT.java create mode 100644 server/src/test/java/org/elasticsearch/indices/settings/InternalOrPrivateSettingsPlugin.java create mode 100644 server/src/test/java/org/elasticsearch/indices/settings/InternalSettingsIT.java create mode 100644 server/src/test/java/org/elasticsearch/indices/settings/PrivateSettingsIT.java create mode 100644 server/src/test/java/org/elasticsearch/indices/stats/LegacyIndexStatsIT.java diff --git a/buildSrc/src/main/resources/checkstyle_suppressions.xml b/buildSrc/src/main/resources/checkstyle_suppressions.xml index 420ed3b10b4..418d366a5cd 100644 --- a/buildSrc/src/main/resources/checkstyle_suppressions.xml +++ b/buildSrc/src/main/resources/checkstyle_suppressions.xml @@ -539,7 +539,6 @@ - diff --git a/server/src/main/java/org/elasticsearch/Version.java b/server/src/main/java/org/elasticsearch/Version.java index 7303e8d34c9..65dc1e2d231 100644 --- a/server/src/main/java/org/elasticsearch/Version.java +++ b/server/src/main/java/org/elasticsearch/Version.java @@ -43,6 +43,8 @@ public class Version implements Comparable, ToXContentFragment { * values below 25 are for alpha builder (since 5.0), and above 25 and below 50 are beta builds, and below 99 are RC builds, with 99 * indicating a release the (internal) format of the id is there so we can easily do after/before checks on the id */ + public static final int V_EMPTY_ID = 0; + public static final Version V_EMPTY = new Version(V_EMPTY_ID, org.apache.lucene.util.Version.LATEST); public static final int V_6_0_0_alpha1_ID = 6000001; public static final Version V_6_0_0_alpha1 = new Version(V_6_0_0_alpha1_ID, org.apache.lucene.util.Version.LUCENE_7_0_0); @@ -167,6 +169,8 @@ public class Version implements Comparable, ToXContentFragment { return V_6_0_0_alpha2; case V_6_0_0_alpha1_ID: return V_6_0_0_alpha1; + case V_EMPTY_ID: + return V_EMPTY; default: return new Version(id, org.apache.lucene.util.Version.LATEST); } @@ -179,11 +183,14 @@ public class Version implements Comparable, ToXContentFragment { * {@value IndexMetaData#SETTING_VERSION_CREATED} */ public static Version indexCreated(Settings indexSettings) { - final Version indexVersion = indexSettings.getAsVersion(IndexMetaData.SETTING_VERSION_CREATED, null); - if (indexVersion == null) { - throw new IllegalStateException( - "[" + IndexMetaData.SETTING_VERSION_CREATED + "] is not present in the index settings for index with uuid: [" - + indexSettings.get(IndexMetaData.SETTING_INDEX_UUID) + "]"); + final Version indexVersion = IndexMetaData.SETTING_INDEX_VERSION_CREATED.get(indexSettings); + if (indexVersion == V_EMPTY) { + final String message = String.format( + Locale.ROOT, + "[%s] is not present in the index settings for index with UUID [%s]", + IndexMetaData.SETTING_INDEX_VERSION_CREATED.getKey(), + indexSettings.get(IndexMetaData.SETTING_INDEX_UUID)); + throw new IllegalStateException(message); } return indexVersion; } @@ -470,8 +477,10 @@ public class Version implements Comparable, ToXContentFragment { if (field.getType() != Version.class) { continue; } - if ("CURRENT".equals(field.getName())) { - continue; + switch (field.getName()) { + case "CURRENT": + case "V_EMPTY": + continue; } assert field.getName().matches("V(_\\d+)+(_(alpha|beta|rc)\\d+)?") : field.getName(); try { diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java b/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java index 8362198a12c..7996059a099 100644 --- a/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java +++ b/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java @@ -25,7 +25,6 @@ import org.elasticsearch.cluster.action.shard.ShardStateAction; import org.elasticsearch.cluster.metadata.IndexGraveyard; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.MetaData; -import org.elasticsearch.cluster.metadata.MetaDataCreateIndexService; import org.elasticsearch.cluster.metadata.MetaDataDeleteIndexService; import org.elasticsearch.cluster.metadata.MetaDataIndexAliasesService; import org.elasticsearch.cluster.metadata.MetaDataIndexStateService; @@ -51,10 +50,10 @@ import org.elasticsearch.cluster.routing.allocation.decider.NodeVersionAllocatio import org.elasticsearch.cluster.routing.allocation.decider.RebalanceOnlyWhenActiveAllocationDecider; import org.elasticsearch.cluster.routing.allocation.decider.ReplicaAfterPrimaryActiveAllocationDecider; import org.elasticsearch.cluster.routing.allocation.decider.ResizeAllocationDecider; +import org.elasticsearch.cluster.routing.allocation.decider.RestoreInProgressAllocationDecider; import org.elasticsearch.cluster.routing.allocation.decider.SameShardAllocationDecider; import org.elasticsearch.cluster.routing.allocation.decider.ShardsLimitAllocationDecider; import org.elasticsearch.cluster.routing.allocation.decider.SnapshotInProgressAllocationDecider; -import org.elasticsearch.cluster.routing.allocation.decider.RestoreInProgressAllocationDecider; import org.elasticsearch.cluster.routing.allocation.decider.ThrottlingAllocationDecider; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.ParseField; @@ -268,7 +267,6 @@ public class ClusterModule extends AbstractModule { bind(AllocationService.class).toInstance(allocationService); bind(ClusterService.class).toInstance(clusterService); bind(NodeConnectionsService.class).asEagerSingleton(); - bind(MetaDataCreateIndexService.class).asEagerSingleton(); bind(MetaDataDeleteIndexService.class).asEagerSingleton(); bind(MetaDataIndexStateService.class).asEagerSingleton(); bind(MetaDataMappingService.class).asEagerSingleton(); diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/AliasValidator.java b/server/src/main/java/org/elasticsearch/cluster/metadata/AliasValidator.java index 33e1687e241..766b35307cd 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/AliasValidator.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/AliasValidator.java @@ -23,7 +23,6 @@ import org.elasticsearch.action.admin.indices.alias.Alias; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Strings; import org.elasticsearch.common.component.AbstractComponent; -import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.common.xcontent.NamedXContentRegistry; @@ -46,7 +45,6 @@ import static org.elasticsearch.index.query.AbstractQueryBuilder.parseInnerQuery */ public class AliasValidator extends AbstractComponent { - @Inject public AliasValidator(Settings settings) { super(settings); } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java index a88ad4418e0..ece729474d1 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java @@ -183,6 +183,10 @@ public class IndexMetaData implements Diffable, ToXContentFragmen Setting.boolSetting(SETTING_READ_ONLY_ALLOW_DELETE, false, Property.Dynamic, Property.IndexScope); public static final String SETTING_VERSION_CREATED = "index.version.created"; + + public static final Setting SETTING_INDEX_VERSION_CREATED = + Setting.versionSetting(SETTING_VERSION_CREATED, Version.V_EMPTY, Property.IndexScope, Property.PrivateIndex); + public static final String SETTING_VERSION_CREATED_STRING = "index.version.created_string"; public static final String SETTING_VERSION_UPGRADED = "index.version.upgraded"; public static final String SETTING_VERSION_UPGRADED_STRING = "index.version.upgraded_string"; @@ -1312,8 +1316,8 @@ public class IndexMetaData implements Diffable, ToXContentFragmen */ public static Settings addHumanReadableSettings(Settings settings) { Settings.Builder builder = Settings.builder().put(settings); - Version version = settings.getAsVersion(SETTING_VERSION_CREATED, null); - if (version != null) { + Version version = SETTING_INDEX_VERSION_CREATED.get(settings); + if (version != Version.V_EMPTY) { builder.put(SETTING_VERSION_CREATED_STRING, version.toString()); } Version versionUpgraded = settings.getAsVersion(SETTING_VERSION_UPGRADED, null); diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java index 3fa77059ed0..e41911d123e 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java @@ -52,7 +52,6 @@ import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.ValidationException; import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.compress.CompressedXContent; -import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.io.PathUtils; import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Setting; @@ -94,7 +93,6 @@ import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_CREATION_ import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_INDEX_UUID; import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS; import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS; -import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_VERSION_CREATED; /** * Service responsible for submitting create index requests @@ -111,13 +109,19 @@ public class MetaDataCreateIndexService extends AbstractComponent { private final IndexScopedSettings indexScopedSettings; private final ActiveShardsObserver activeShardsObserver; private final NamedXContentRegistry xContentRegistry; + private final boolean forbidPrivateIndexSettings; - @Inject - public MetaDataCreateIndexService(Settings settings, ClusterService clusterService, - IndicesService indicesService, AllocationService allocationService, - AliasValidator aliasValidator, Environment env, - IndexScopedSettings indexScopedSettings, ThreadPool threadPool, - NamedXContentRegistry xContentRegistry) { + public MetaDataCreateIndexService( + final Settings settings, + final ClusterService clusterService, + final IndicesService indicesService, + final AllocationService allocationService, + final AliasValidator aliasValidator, + final Environment env, + final IndexScopedSettings indexScopedSettings, + final ThreadPool threadPool, + final NamedXContentRegistry xContentRegistry, + final boolean forbidPrivateIndexSettings) { super(settings); this.clusterService = clusterService; this.indicesService = indicesService; @@ -127,6 +131,7 @@ public class MetaDataCreateIndexService extends AbstractComponent { this.indexScopedSettings = indexScopedSettings; this.activeShardsObserver = new ActiveShardsObserver(settings, clusterService, threadPool); this.xContentRegistry = xContentRegistry; + this.forbidPrivateIndexSettings = forbidPrivateIndexSettings; } /** @@ -348,10 +353,10 @@ public class MetaDataCreateIndexService extends AbstractComponent { } // now, put the request settings, so they override templates indexSettingsBuilder.put(request.settings()); - if (indexSettingsBuilder.get(SETTING_VERSION_CREATED) == null) { - DiscoveryNodes nodes = currentState.nodes(); + if (indexSettingsBuilder.get(IndexMetaData.SETTING_INDEX_VERSION_CREATED.getKey()) == null) { + final DiscoveryNodes nodes = currentState.nodes(); final Version createdVersion = Version.min(Version.CURRENT, nodes.getSmallestNonClientNodeVersion()); - indexSettingsBuilder.put(SETTING_VERSION_CREATED, createdVersion); + indexSettingsBuilder.put(IndexMetaData.SETTING_INDEX_VERSION_CREATED.getKey(), createdVersion); } if (indexSettingsBuilder.get(SETTING_NUMBER_OF_SHARDS) == null) { final int numberOfShards = getNumberOfShards(indexSettingsBuilder); @@ -373,7 +378,7 @@ public class MetaDataCreateIndexService extends AbstractComponent { final Settings idxSettings = indexSettingsBuilder.build(); int numTargetShards = IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.get(idxSettings); final int routingNumShards; - final Version indexVersionCreated = idxSettings.getAsVersion(IndexMetaData.SETTING_VERSION_CREATED, null); + final Version indexVersionCreated = IndexMetaData.SETTING_INDEX_VERSION_CREATED.get(idxSettings); final IndexMetaData sourceMetaData = recoverFromIndex == null ? null : currentState.metaData().getIndexSafe(recoverFromIndex); if (sourceMetaData == null || sourceMetaData.getNumberOfShards() == 1) { @@ -559,7 +564,9 @@ public class MetaDataCreateIndexService extends AbstractComponent { // TODO: this logic can be removed when the current major version is 8 assert Version.CURRENT.major == 7; final int numberOfShards; - if (Version.fromId(Integer.parseInt(indexSettingsBuilder.get(SETTING_VERSION_CREATED))).before(Version.V_7_0_0_alpha1)) { + final Version indexVersionCreated = + Version.fromId(Integer.parseInt(indexSettingsBuilder.get(IndexMetaData.SETTING_INDEX_VERSION_CREATED.getKey()))); + if (indexVersionCreated.before(Version.V_7_0_0_alpha1)) { numberOfShards = 5; } else { numberOfShards = 1; @@ -580,11 +587,12 @@ public class MetaDataCreateIndexService extends AbstractComponent { private void validate(CreateIndexClusterStateUpdateRequest request, ClusterState state) { validateIndexName(request.index(), state); - validateIndexSettings(request.index(), request.settings()); + validateIndexSettings(request.index(), request.settings(), forbidPrivateIndexSettings); } - public void validateIndexSettings(String indexName, Settings settings) throws IndexCreationException { - List validationErrors = getIndexSettingsValidationErrors(settings); + public void validateIndexSettings( + final String indexName, final Settings settings, final boolean forbidPrivateIndexSettings) throws IndexCreationException { + List validationErrors = getIndexSettingsValidationErrors(settings, forbidPrivateIndexSettings); if (validationErrors.isEmpty() == false) { ValidationException validationException = new ValidationException(); validationException.addValidationErrors(validationErrors); @@ -592,7 +600,7 @@ public class MetaDataCreateIndexService extends AbstractComponent { } } - List getIndexSettingsValidationErrors(Settings settings) { + List getIndexSettingsValidationErrors(final Settings settings, final boolean forbidPrivateIndexSettings) { String customPath = IndexMetaData.INDEX_DATA_PATH_SETTING.get(settings); List validationErrors = new ArrayList<>(); if (Strings.isEmpty(customPath) == false && env.sharedDataFile() == null) { @@ -603,6 +611,16 @@ public class MetaDataCreateIndexService extends AbstractComponent { validationErrors.add("custom path [" + customPath + "] is not a sub-path of path.shared_data [" + env.sharedDataFile() + "]"); } } + if (forbidPrivateIndexSettings) { + for (final String key : settings.keySet()) { + final Setting setting = indexScopedSettings.get(key); + if (setting == null) { + assert indexScopedSettings.isPrivateSetting(key); + } else if (setting.isPrivateIndex()) { + validationErrors.add("private index setting [" + key + "] can not be set explicitly"); + } + } + } return validationErrors; } @@ -737,7 +755,7 @@ public class MetaDataCreateIndexService extends AbstractComponent { } indexSettingsBuilder - .put(IndexMetaData.SETTING_VERSION_CREATED, sourceMetaData.getCreationVersion()) + .put(IndexMetaData.SETTING_INDEX_VERSION_CREATED.getKey(), sourceMetaData.getCreationVersion()) .put(IndexMetaData.SETTING_VERSION_UPGRADED, sourceMetaData.getUpgradedVersion()) .put(builder.build()) .put(IndexMetaData.SETTING_ROUTING_PARTITION_SIZE, sourceMetaData.getRoutingPartitionSize()) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java index 1baeb2459f0..40d2a697140 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java @@ -19,7 +19,6 @@ package org.elasticsearch.cluster.metadata; import com.carrotsearch.hppc.cursors.ObjectCursor; - import org.apache.lucene.util.CollectionUtil; import org.elasticsearch.Version; import org.elasticsearch.action.admin.indices.alias.Alias; @@ -301,7 +300,7 @@ public class MetaDataIndexTemplateService extends AbstractComponent { validationErrors.add(t.getMessage()); } } - List indexSettingsValidation = metaDataCreateIndexService.getIndexSettingsValidationErrors(request.settings); + List indexSettingsValidation = metaDataCreateIndexService.getIndexSettingsValidationErrors(request.settings, true); validationErrors.addAll(indexSettingsValidation); if (!validationErrors.isEmpty()) { ValidationException validationException = new ValidationException(); diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataUpdateSettingsService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataUpdateSettingsService.java index 38766c08e08..cc2b13677d1 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataUpdateSettingsService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataUpdateSettingsService.java @@ -85,7 +85,7 @@ public class MetaDataUpdateSettingsService extends AbstractComponent { indexScopedSettings.validate( normalizedSettings.filter(s -> Regex.isSimpleMatchPattern(s) == false), // don't validate wildcards false, // don't validate dependencies here we check it below never allow to change the number of shards - true); // validate internal index settings + true); // validate internal or private index settings for (String key : normalizedSettings.keySet()) { Setting setting = indexScopedSettings.get(key); boolean isWildcard = setting == null && Regex.isSimpleMatchPattern(key); diff --git a/server/src/main/java/org/elasticsearch/common/settings/AbstractScopedSettings.java b/server/src/main/java/org/elasticsearch/common/settings/AbstractScopedSettings.java index 9f1d7d8a395..d8af8d3ead1 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/AbstractScopedSettings.java +++ b/server/src/main/java/org/elasticsearch/common/settings/AbstractScopedSettings.java @@ -352,13 +352,13 @@ public abstract class AbstractScopedSettings extends AbstractComponent { /** * Validates that all settings are registered and valid. * - * @param settings the settings to validate - * @param validateDependencies true if dependent settings should be validated - * @param validateInternalIndex true if internal index settings should be validated + * @param settings the settings to validate + * @param validateDependencies true if dependent settings should be validated + * @param validateInternalOrPrivateIndex true if internal index settings should be validated * @see Setting#getSettingsDependencies(String) */ - public final void validate(final Settings settings, final boolean validateDependencies, final boolean validateInternalIndex) { - validate(settings, validateDependencies, false, false, validateInternalIndex); + public final void validate(final Settings settings, final boolean validateDependencies, final boolean validateInternalOrPrivateIndex) { + validate(settings, validateDependencies, false, false, validateInternalOrPrivateIndex); } /** @@ -381,11 +381,11 @@ public abstract class AbstractScopedSettings extends AbstractComponent { /** * Validates that all settings are registered and valid. * - * @param settings the settings - * @param validateDependencies true if dependent settings should be validated - * @param ignorePrivateSettings true if private settings should be ignored during validation - * @param ignoreArchivedSettings true if archived settings should be ignored during validation - * @param validateInternalIndex true if index internal settings should be validated + * @param settings the settings + * @param validateDependencies true if dependent settings should be validated + * @param ignorePrivateSettings true if private settings should be ignored during validation + * @param ignoreArchivedSettings true if archived settings should be ignored during validation + * @param validateInternalOrPrivateIndex true if index internal settings should be validated * @see Setting#getSettingsDependencies(String) */ public final void validate( @@ -393,17 +393,18 @@ public abstract class AbstractScopedSettings extends AbstractComponent { final boolean validateDependencies, final boolean ignorePrivateSettings, final boolean ignoreArchivedSettings, - final boolean validateInternalIndex) { + final boolean validateInternalOrPrivateIndex) { final List exceptions = new ArrayList<>(); for (final String key : settings.keySet()) { // settings iterate in deterministic fashion - if (isPrivateSetting(key) && ignorePrivateSettings) { + final Setting setting = getRaw(key); + if (((isPrivateSetting(key) || (setting != null && setting.isPrivateIndex())) && ignorePrivateSettings)) { continue; } if (key.startsWith(ARCHIVED_SETTINGS_PREFIX) && ignoreArchivedSettings) { continue; } try { - validate(key, settings, validateDependencies, validateInternalIndex); + validate(key, settings, validateDependencies, validateInternalOrPrivateIndex); } catch (final RuntimeException ex) { exceptions.add(ex); } @@ -426,14 +427,15 @@ public abstract class AbstractScopedSettings extends AbstractComponent { /** * Validates that the settings is valid. * - * @param key the key of the setting to validate - * @param settings the settings - * @param validateDependencies true if dependent settings should be validated - * @param validateInternalIndex true if internal index settings should be validated + * @param key the key of the setting to validate + * @param settings the settings + * @param validateDependencies true if dependent settings should be validated + * @param validateInternalOrPrivateIndex true if internal index settings should be validated * @throws IllegalArgumentException if the setting is invalid */ - void validate(final String key, final Settings settings, final boolean validateDependencies, final boolean validateInternalIndex) { - Setting setting = getRaw(key); + void validate( + final String key, final Settings settings, final boolean validateDependencies, final boolean validateInternalOrPrivateIndex) { + Setting setting = getRaw(key); if (setting == null) { LevensteinDistance ld = new LevensteinDistance(); List> scoredKeys = new ArrayList<>(); @@ -472,10 +474,15 @@ public abstract class AbstractScopedSettings extends AbstractComponent { } } } - // the only time that validateInternalIndex should be true is if this call is coming via the update settings API - if (validateInternalIndex && setting.getProperties().contains(Setting.Property.InternalIndex)) { - throw new IllegalArgumentException( - "can not update internal setting [" + setting.getKey() + "]; this setting is managed via a dedicated API"); + // the only time that validateInternalOrPrivateIndex should be true is if this call is coming via the update settings API + if (validateInternalOrPrivateIndex) { + if (setting.isInternalIndex()) { + throw new IllegalArgumentException( + "can not update internal setting [" + setting.getKey() + "]; this setting is managed via a dedicated API"); + } else if (setting.isPrivateIndex()) { + throw new IllegalArgumentException( + "can not update private setting [" + setting.getKey() + "]; this setting is managed by Elasticsearch"); + } } } setting.get(settings); diff --git a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java index f3de294046c..47625213ddf 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java +++ b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java @@ -61,6 +61,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings { MergeSchedulerConfig.AUTO_THROTTLE_SETTING, MergeSchedulerConfig.MAX_MERGE_COUNT_SETTING, MergeSchedulerConfig.MAX_THREAD_COUNT_SETTING, + IndexMetaData.SETTING_INDEX_VERSION_CREATED, IndexMetaData.INDEX_ROUTING_EXCLUDE_GROUP_SETTING, IndexMetaData.INDEX_ROUTING_INCLUDE_GROUP_SETTING, IndexMetaData.INDEX_ROUTING_REQUIRE_GROUP_SETTING, @@ -200,7 +201,6 @@ public final class IndexScopedSettings extends AbstractScopedSettings { switch (key) { case IndexMetaData.SETTING_CREATION_DATE: case IndexMetaData.SETTING_INDEX_UUID: - case IndexMetaData.SETTING_VERSION_CREATED: case IndexMetaData.SETTING_VERSION_UPGRADED: case IndexMetaData.SETTING_INDEX_PROVIDED_NAME: case MergePolicyConfig.INDEX_MERGE_ENABLED: diff --git a/server/src/main/java/org/elasticsearch/common/settings/Setting.java b/server/src/main/java/org/elasticsearch/common/settings/Setting.java index 7b432c0ed1e..ceeb60f8edd 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/Setting.java +++ b/server/src/main/java/org/elasticsearch/common/settings/Setting.java @@ -21,6 +21,7 @@ package org.elasticsearch.common.settings; import org.apache.logging.log4j.Logger; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.Version; import org.elasticsearch.common.Booleans; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Strings; @@ -126,7 +127,12 @@ public class Setting implements ToXContentObject { * Indicates an index-level setting that is managed internally. Such a setting can only be added to an index on index creation but * can not be updated via the update API. */ - InternalIndex + InternalIndex, + + /** + * Indicates an index-level setting that is privately managed. Such a setting can not even be set on index creation. + */ + PrivateIndex } private final Key key; @@ -160,6 +166,7 @@ public class Setting implements ToXContentObject { } checkPropertyRequiresIndexScope(propertiesAsSet, Property.NotCopyableOnResize); checkPropertyRequiresIndexScope(propertiesAsSet, Property.InternalIndex); + checkPropertyRequiresIndexScope(propertiesAsSet, Property.PrivateIndex); this.properties = propertiesAsSet; } } @@ -284,6 +291,14 @@ public class Setting implements ToXContentObject { return properties.contains(Property.Final); } + public final boolean isInternalIndex() { + return properties.contains(Property.InternalIndex); + } + + public final boolean isPrivateIndex() { + return properties.contains(Property.PrivateIndex); + } + /** * Returns the setting properties * @see Property @@ -968,6 +983,9 @@ public class Setting implements ToXContentObject { } } + public static Setting versionSetting(final String key, final Version defaultValue, Property... properties) { + return new Setting<>(key, s -> Integer.toString(defaultValue.id), s -> Version.fromId(Integer.parseInt(s)), properties); + } public static Setting floatSetting(String key, float defaultValue, Property... properties) { return new Setting<>(key, (s) -> Float.toString(defaultValue), Float::parseFloat, properties); diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index 6a612091b97..0dd3654ce3d 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -396,7 +396,7 @@ public final class IndexSettings { this.nodeSettings = nodeSettings; this.settings = Settings.builder().put(nodeSettings).put(indexMetaData.getSettings()).build(); this.index = indexMetaData.getIndex(); - version = Version.indexCreated(settings); + version = IndexMetaData.SETTING_INDEX_VERSION_CREATED.get(settings); logger = Loggers.getLogger(getClass(), settings, index); nodeName = Node.NODE_NAME_SETTING.get(settings); this.indexMetaData = indexMetaData; diff --git a/server/src/main/java/org/elasticsearch/node/Node.java b/server/src/main/java/org/elasticsearch/node/Node.java index 250ef8f2dd9..eff1a796e31 100644 --- a/server/src/main/java/org/elasticsearch/node/Node.java +++ b/server/src/main/java/org/elasticsearch/node/Node.java @@ -45,9 +45,11 @@ import org.elasticsearch.cluster.ClusterStateObserver; import org.elasticsearch.cluster.InternalClusterInfoService; import org.elasticsearch.cluster.NodeConnectionsService; import org.elasticsearch.cluster.action.index.MappingUpdatedAction; +import org.elasticsearch.cluster.metadata.AliasValidator; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.IndexTemplateMetaData; import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.cluster.metadata.MetaDataCreateIndexService; import org.elasticsearch.cluster.metadata.MetaDataIndexUpgradeService; import org.elasticsearch.cluster.metadata.TemplateUpgradeService; import org.elasticsearch.cluster.node.DiscoveryNode; @@ -260,10 +262,19 @@ public class Node implements Closeable { } public Node(Environment environment) { - this(environment, Collections.emptyList()); + this(environment, Collections.emptyList(), true); } - protected Node(final Environment environment, Collection> classpathPlugins) { + /** + * Constructs a node + * + * @param environment the environment for this node + * @param classpathPlugins the plugins to be loaded from the classpath + * @param forbidPrivateIndexSettings whether or not private index settings are forbidden when creating an index; this is used in the + * test framework for tests that rely on being able to set private settings + */ + protected Node( + final Environment environment, Collection> classpathPlugins, boolean forbidPrivateIndexSettings) { logger = Loggers.getLogger(Node.class); final List resourcesToClose = new ArrayList<>(); // register everything we need to release in the case of an error boolean success = false; @@ -424,6 +435,20 @@ public class Node implements Closeable { threadPool, settingsModule.getIndexScopedSettings(), circuitBreakerService, bigArrays, scriptModule.getScriptService(), client, metaStateService, engineFactoryProviders, indexStoreFactories); + final AliasValidator aliasValidator = new AliasValidator(settings); + + final MetaDataCreateIndexService metaDataCreateIndexService = new MetaDataCreateIndexService( + settings, + clusterService, + indicesService, + clusterModule.getAllocationService(), + aliasValidator, + environment, + settingsModule.getIndexScopedSettings(), + threadPool, + xContentRegistry, + forbidPrivateIndexSettings); + Collection pluginComponents = pluginsService.filterPlugins(Plugin.class).stream() .flatMap(p -> p.createComponents(client, clusterService, threadPool, resourceWatcherService, scriptModule.getScriptService(), xContentRegistry, environment, nodeEnvironment, @@ -512,6 +537,8 @@ public class Node implements Closeable { b.bind(MetaDataUpgrader.class).toInstance(metaDataUpgrader); b.bind(MetaStateService.class).toInstance(metaStateService); b.bind(IndicesService.class).toInstance(indicesService); + b.bind(AliasValidator.class).toInstance(aliasValidator); + b.bind(MetaDataCreateIndexService.class).toInstance(metaDataCreateIndexService); b.bind(SearchService.class).toInstance(searchService); b.bind(SearchTransportService.class).toInstance(searchTransportService); b.bind(SearchPhaseController.class).toInstance(new SearchPhaseController(settings, diff --git a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java index 6acdbad2cce..c820f41b367 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java @@ -270,7 +270,7 @@ public class RestoreService extends AbstractComponent implements ClusterStateApp // Index doesn't exist - create it and start recovery // Make sure that the index we are about to create has a validate name MetaDataCreateIndexService.validateIndexName(renamedIndexName, currentState); - createIndexService.validateIndexSettings(renamedIndexName, snapshotIndexMetaData.getSettings()); + createIndexService.validateIndexSettings(renamedIndexName, snapshotIndexMetaData.getSettings(), false); IndexMetaData.Builder indexMdBuilder = IndexMetaData.builder(snapshotIndexMetaData).state(IndexMetaData.State.OPEN).index(renamedIndexName); indexMdBuilder.settings(Settings.builder().put(snapshotIndexMetaData.getSettings()).put(IndexMetaData.SETTING_INDEX_UUID, UUIDs.randomBase64UUID())); if (!request.includeAliases() && !snapshotIndexMetaData.getAliases().isEmpty()) { diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/create/ShrinkIndexIT.java b/server/src/test/java/org/elasticsearch/action/admin/indices/create/ShrinkIndexIT.java index 0c3cc7e4b15..b9624e3073f 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/create/ShrinkIndexIT.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/create/ShrinkIndexIT.java @@ -59,13 +59,10 @@ import org.elasticsearch.index.query.TermsQueryBuilder; import org.elasticsearch.index.seqno.SeqNoStats; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.indices.IndicesService; -import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; -import org.elasticsearch.test.InternalSettingsPlugin; import org.elasticsearch.test.VersionUtils; import java.util.Arrays; -import java.util.Collection; import java.util.List; import java.util.Map; import java.util.stream.IntStream; @@ -79,8 +76,8 @@ import static org.hamcrest.Matchers.greaterThanOrEqualTo; public class ShrinkIndexIT extends ESIntegTestCase { @Override - protected Collection> nodePlugins() { - return Arrays.asList(InternalSettingsPlugin.class); + protected boolean forbidPrivateIndexSettings() { + return false; } public void testCreateShrinkIndexToN() { diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/create/SplitIndexIT.java b/server/src/test/java/org/elasticsearch/action/admin/indices/create/SplitIndexIT.java index a0fd40a649e..4510956358f 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/create/SplitIndexIT.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/create/SplitIndexIT.java @@ -54,15 +54,12 @@ import org.elasticsearch.index.query.TermsQueryBuilder; import org.elasticsearch.index.seqno.SeqNoStats; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.indices.IndicesService; -import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; -import org.elasticsearch.test.InternalSettingsPlugin; import org.elasticsearch.test.VersionUtils; import java.io.IOException; import java.io.UncheckedIOException; import java.util.Arrays; -import java.util.Collection; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -83,8 +80,8 @@ import static org.hamcrest.Matchers.greaterThanOrEqualTo; public class SplitIndexIT extends ESIntegTestCase { @Override - protected Collection> nodePlugins() { - return Arrays.asList(InternalSettingsPlugin.class); + protected boolean forbidPrivateIndexSettings() { + return false; } public void testCreateSplitIndexToN() throws IOException { diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/template/put/MetaDataIndexTemplateServiceTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/template/put/MetaDataIndexTemplateServiceTests.java index 39f04c6b7b0..892721f8a5c 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/template/put/MetaDataIndexTemplateServiceTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/template/put/MetaDataIndexTemplateServiceTests.java @@ -178,7 +178,11 @@ public class MetaDataIndexTemplateServiceTests extends ESSingleNodeTestCase { null, null, null, - null, null, null, xContentRegistry); + null, + IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, + null, + xContentRegistry, + true); MetaDataIndexTemplateService service = new MetaDataIndexTemplateService(Settings.EMPTY, null, createIndexService, new AliasValidator(Settings.EMPTY), null, new IndexScopedSettings(Settings.EMPTY, IndexScopedSettings.BUILT_IN_INDEX_SETTINGS), xContentRegistry); @@ -202,15 +206,16 @@ public class MetaDataIndexTemplateServiceTests extends ESSingleNodeTestCase { IndicesService indicesService = getInstanceFromNode(IndicesService.class); ClusterService clusterService = getInstanceFromNode(ClusterService.class); MetaDataCreateIndexService createIndexService = new MetaDataCreateIndexService( - Settings.EMPTY, - clusterService, - indicesService, - null, - null, - null, - null, - null, - xContentRegistry()); + Settings.EMPTY, + clusterService, + indicesService, + null, + null, + null, + null, + null, + xContentRegistry(), + true); MetaDataIndexTemplateService service = new MetaDataIndexTemplateService( Settings.EMPTY, clusterService, createIndexService, new AliasValidator(Settings.EMPTY), indicesService, new IndexScopedSettings(Settings.EMPTY, IndexScopedSettings.BUILT_IN_INDEX_SETTINGS), xContentRegistry()); diff --git a/server/src/test/java/org/elasticsearch/common/settings/ScopedSettingsTests.java b/server/src/test/java/org/elasticsearch/common/settings/ScopedSettingsTests.java index 1580c1a3797..af2d874a679 100644 --- a/server/src/test/java/org/elasticsearch/common/settings/ScopedSettingsTests.java +++ b/server/src/test/java/org/elasticsearch/common/settings/ScopedSettingsTests.java @@ -979,19 +979,42 @@ public class ScopedSettingsTests extends ESTestCase { IllegalArgumentException.class, () -> { final Settings settings = Settings.builder().put("index.internal", "internal").build(); - indexScopedSettings.validate(settings, false, /* validateInternalIndex */ true); + indexScopedSettings.validate(settings, false, /* validateInternalOrPrivateIndex */ true); }); final String message = "can not update internal setting [index.internal]; this setting is managed via a dedicated API"; assertThat(e, hasToString(containsString(message))); } + public void testPrivateIndexSettingsFailsValidation() { + final Setting indexInternalSetting = Setting.simpleString("index.private", Property.PrivateIndex, Property.IndexScope); + final IndexScopedSettings indexScopedSettings = + new IndexScopedSettings(Settings.EMPTY, Collections.singleton(indexInternalSetting)); + final IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> { + final Settings settings = Settings.builder().put("index.private", "private").build(); + indexScopedSettings.validate(settings, false, /* validateInternalOrPrivateIndex */ true); + }); + final String message = "can not update private setting [index.private]; this setting is managed by Elasticsearch"; + assertThat(e, hasToString(containsString(message))); + } + public void testInternalIndexSettingsSkipValidation() { final Setting internalIndexSetting = Setting.simpleString("index.internal", Property.InternalIndex, Property.IndexScope); final IndexScopedSettings indexScopedSettings = new IndexScopedSettings(Settings.EMPTY, Collections.singleton(internalIndexSetting)); // nothing should happen, validation should not throw an exception final Settings settings = Settings.builder().put("index.internal", "internal").build(); - indexScopedSettings.validate(settings, false, /* validateInternalIndex */ false); + indexScopedSettings.validate(settings, false, /* validateInternalOrPrivateIndex */ false); + } + + public void testPrivateIndexSettingsSkipValidation() { + final Setting internalIndexSetting = Setting.simpleString("index.private", Property.PrivateIndex, Property.IndexScope); + final IndexScopedSettings indexScopedSettings = + new IndexScopedSettings(Settings.EMPTY, Collections.singleton(internalIndexSetting)); + // nothing should happen, validation should not throw an exception + final Settings settings = Settings.builder().put("index.private", "private").build(); + indexScopedSettings.validate(settings, false, /* validateInternalOrPrivateIndex */ false); } } diff --git a/server/src/test/java/org/elasticsearch/common/settings/SettingTests.java b/server/src/test/java/org/elasticsearch/common/settings/SettingTests.java index 7063e53f789..bedb2857b60 100644 --- a/server/src/test/java/org/elasticsearch/common/settings/SettingTests.java +++ b/server/src/test/java/org/elasticsearch/common/settings/SettingTests.java @@ -735,13 +735,20 @@ public class SettingTests extends ESTestCase { assertThat(e, hasToString(containsString("non-index-scoped setting [foo.bar] can not have property [NotCopyableOnResize]"))); } - public void testRejectNonIndexScopedIndexInternalSetting() { + public void testRejectNonIndexScopedInternalIndexSetting() { final IllegalArgumentException e = expectThrows( IllegalArgumentException.class, () -> Setting.simpleString("foo.bar", Property.InternalIndex)); assertThat(e, hasToString(containsString("non-index-scoped setting [foo.bar] can not have property [InternalIndex]"))); } + public void testRejectNonIndexScopedPrivateIndexSetting() { + final IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> Setting.simpleString("foo.bar", Property.PrivateIndex)); + assertThat(e, hasToString(containsString("non-index-scoped setting [foo.bar] can not have property [PrivateIndex]"))); + } + public void testTimeValue() { final TimeValue random = TimeValue.parseTimeValue(randomTimeValue(), "test"); diff --git a/server/src/test/java/org/elasticsearch/get/GetActionIT.java b/server/src/test/java/org/elasticsearch/get/GetActionIT.java index 829d6ff7c14..edd8660586a 100644 --- a/server/src/test/java/org/elasticsearch/get/GetActionIT.java +++ b/server/src/test/java/org/elasticsearch/get/GetActionIT.java @@ -19,7 +19,6 @@ package org.elasticsearch.get; -import org.elasticsearch.Version; import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.action.admin.indices.alias.Alias; import org.elasticsearch.action.admin.indices.flush.FlushResponse; @@ -214,7 +213,7 @@ public class GetActionIT extends ESIntegTestCase { assertThat(exception.getMessage(), endsWith("can't execute a single index op")); } - private static String indexOrAlias() { + static String indexOrAlias() { return randomBoolean() ? "test" : "alias"; } @@ -524,41 +523,6 @@ public class GetActionIT extends ESIntegTestCase { assertThat(response.getResponses()[2].getResponse().getSourceAsMap().get("field").toString(), equalTo("value2")); } - public void testGetFieldsMetaDataWithRouting() throws Exception { - assertAcked(prepareCreate("test") - .addMapping("_doc", "field1", "type=keyword,store=true") - .addAlias(new Alias("alias")) - .setSettings(Settings.builder().put("index.refresh_interval", -1).put("index.version.created", Version.V_6_0_0.id))); - // multi types in 5.6 - - client().prepareIndex("test", "_doc", "1") - .setRouting("1") - .setSource(jsonBuilder().startObject().field("field1", "value").endObject()) - .get(); - - GetResponse getResponse = client().prepareGet(indexOrAlias(), "_doc", "1") - .setRouting("1") - .setStoredFields("field1") - .get(); - assertThat(getResponse.isExists(), equalTo(true)); - assertThat(getResponse.getField("field1").isMetadataField(), equalTo(false)); - assertThat(getResponse.getField("field1").getValue().toString(), equalTo("value")); - assertThat(getResponse.getField("_routing").isMetadataField(), equalTo(true)); - assertThat(getResponse.getField("_routing").getValue().toString(), equalTo("1")); - - flush(); - - getResponse = client().prepareGet(indexOrAlias(), "_doc", "1") - .setStoredFields("field1") - .setRouting("1") - .get(); - assertThat(getResponse.isExists(), equalTo(true)); - assertThat(getResponse.getField("field1").isMetadataField(), equalTo(false)); - assertThat(getResponse.getField("field1").getValue().toString(), equalTo("value")); - assertThat(getResponse.getField("_routing").isMetadataField(), equalTo(true)); - assertThat(getResponse.getField("_routing").getValue().toString(), equalTo("1")); - } - public void testGetFieldsNonLeafField() throws Exception { assertAcked(prepareCreate("test").addAlias(new Alias("alias")) .addMapping("my-type1", jsonBuilder().startObject().startObject("my-type1").startObject("properties") diff --git a/server/src/test/java/org/elasticsearch/get/LegacyGetActionIT.java b/server/src/test/java/org/elasticsearch/get/LegacyGetActionIT.java new file mode 100644 index 00000000000..4382f677ad6 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/get/LegacyGetActionIT.java @@ -0,0 +1,88 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.get; + +import org.elasticsearch.Version; +import org.elasticsearch.action.admin.indices.alias.Alias; +import org.elasticsearch.action.get.GetResponse; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.test.ESIntegTestCase; + +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.get.GetActionIT.indexOrAlias; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.hamcrest.Matchers.equalTo; + +public class LegacyGetActionIT extends ESIntegTestCase { + + @Override + protected boolean forbidPrivateIndexSettings() { + return false; + } + + public void testGetFieldsMetaDataWithRouting() throws Exception { + assertAcked(prepareCreate("test") + .addMapping("_doc", "field1", "type=keyword,store=true") + .addAlias(new Alias("alias")) + .setSettings( + Settings.builder() + .put("index.refresh_interval", -1) + .put(IndexMetaData.SETTING_INDEX_VERSION_CREATED.getKey(), Version.V_6_0_0))); // multi-types in 6.0.0 + + try (XContentBuilder source = jsonBuilder().startObject().field("field1", "value").endObject()) { + client() + .prepareIndex("test", "_doc", "1") + .setRouting("1") + .setSource(source) + .get(); + } + + { + final GetResponse getResponse = client() + .prepareGet(indexOrAlias(), "_doc", "1") + .setRouting("1") + .setStoredFields("field1") + .get(); + assertThat(getResponse.isExists(), equalTo(true)); + assertThat(getResponse.getField("field1").isMetadataField(), equalTo(false)); + assertThat(getResponse.getField("field1").getValue().toString(), equalTo("value")); + assertThat(getResponse.getField("_routing").isMetadataField(), equalTo(true)); + assertThat(getResponse.getField("_routing").getValue().toString(), equalTo("1")); + } + + flush(); + + { + final GetResponse getResponse = client() + .prepareGet(indexOrAlias(), "_doc", "1") + .setStoredFields("field1") + .setRouting("1") + .get(); + assertThat(getResponse.isExists(), equalTo(true)); + assertThat(getResponse.getField("field1").isMetadataField(), equalTo(false)); + assertThat(getResponse.getField("field1").getValue().toString(), equalTo("value")); + assertThat(getResponse.getField("_routing").isMetadataField(), equalTo(true)); + assertThat(getResponse.getField("_routing").getValue().toString(), equalTo("1")); + } + } + +} diff --git a/server/src/test/java/org/elasticsearch/index/analysis/PreBuiltAnalyzerTests.java b/server/src/test/java/org/elasticsearch/index/analysis/PreBuiltAnalyzerTests.java index 33ec090c61e..a5a4dbd9db8 100644 --- a/server/src/test/java/org/elasticsearch/index/analysis/PreBuiltAnalyzerTests.java +++ b/server/src/test/java/org/elasticsearch/index/analysis/PreBuiltAnalyzerTests.java @@ -41,6 +41,11 @@ import static org.hamcrest.Matchers.is; public class PreBuiltAnalyzerTests extends ESSingleNodeTestCase { + @Override + protected boolean forbidPrivateIndexSettings() { + return false; + } + @Override protected Collection> getPlugins() { return pluginList(InternalSettingsPlugin.class); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DynamicMappingTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DynamicMappingTests.java index b11e4876f9e..95175af5421 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DynamicMappingTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DynamicMappingTests.java @@ -20,7 +20,6 @@ package org.elasticsearch.index.mapper; import org.apache.lucene.index.IndexOptions; import org.elasticsearch.Version; -import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Strings; @@ -190,25 +189,6 @@ public class DynamicMappingTests extends ESSingleNodeTestCase { assertNotNull(fieldType); } - public void testTypeNotCreatedOnIndexFailure() throws IOException, InterruptedException { - XContentBuilder mapping = jsonBuilder().startObject().startObject("_default_") - .field("dynamic", "strict") - .endObject().endObject(); - Settings settings = Settings.builder() - .put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_6_3_0) - .build(); - createIndex("test", settings, "_default_", mapping); - try { - client().prepareIndex().setIndex("test").setType("type").setSource(jsonBuilder().startObject().field("test", "test").endObject()).get(); - fail(); - } catch (StrictDynamicMappingException e) { - - } - - GetMappingsResponse getMappingsResponse = client().admin().indices().prepareGetMappings("test").get(); - assertNull(getMappingsResponse.getMappings().get("test").get("type")); - } - private String serialize(ToXContent mapper) throws Exception { XContentBuilder builder = XContentFactory.jsonBuilder().startObject(); mapper.toXContent(builder, new ToXContent.MapParams(emptyMap())); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ExternalFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ExternalFieldMapperTests.java index 5172e7b0b88..1d65fb27c55 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ExternalFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ExternalFieldMapperTests.java @@ -45,6 +45,7 @@ import java.util.HashMap; import java.util.Map; import java.util.function.Supplier; +import static java.util.Collections.singletonMap; import static org.hamcrest.Matchers.closeTo; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; @@ -56,15 +57,19 @@ public class ExternalFieldMapperTests extends ESSingleNodeTestCase { return pluginList(InternalSettingsPlugin.class); } + @Override + protected boolean forbidPrivateIndexSettings() { + return false; + } + public void testExternalValues() throws Exception { Version version = VersionUtils.randomVersionBetween(random(), Version.V_6_0_0, Version.CURRENT); Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build(); IndexService indexService = createIndex("test", settings); MapperRegistry mapperRegistry = new MapperRegistry( - Collections.singletonMap(ExternalMapperPlugin.EXTERNAL, new ExternalMapper.TypeParser(ExternalMapperPlugin.EXTERNAL, "foo")), - Collections.singletonMap(ExternalMetadataMapper.CONTENT_TYPE, new ExternalMetadataMapper.TypeParser()), - MapperPlugin.NOOP_FIELD_FILTER); + singletonMap(ExternalMapperPlugin.EXTERNAL, new ExternalMapper.TypeParser(ExternalMapperPlugin.EXTERNAL, "foo")), + singletonMap(ExternalMetadataMapper.CONTENT_TYPE, new ExternalMetadataMapper.TypeParser()), MapperPlugin.NOOP_FIELD_FILTER); Supplier queryShardContext = () -> { return indexService.newQueryShardContext(0, null, () -> { throw new UnsupportedOperationException(); }, null); @@ -107,10 +112,7 @@ public class ExternalFieldMapperTests extends ESSingleNodeTestCase { } public void testExternalValuesWithMultifield() throws Exception { - Version version = VersionUtils.randomVersionBetween(random(), Version.V_6_0_0, - Version.CURRENT); - Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build(); - IndexService indexService = createIndex("test", settings); + IndexService indexService = createIndex("test"); Map mapperParsers = new HashMap<>(); mapperParsers.put(ExternalMapperPlugin.EXTERNAL, new ExternalMapper.TypeParser(ExternalMapperPlugin.EXTERNAL, "foo")); mapperParsers.put(TextFieldMapper.CONTENT_TYPE, new TextFieldMapper.TypeParser()); @@ -173,10 +175,7 @@ public class ExternalFieldMapperTests extends ESSingleNodeTestCase { } public void testExternalValuesWithMultifieldTwoLevels() throws Exception { - Version version = VersionUtils.randomVersionBetween(random(), Version.V_6_0_0, - Version.CURRENT); - Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build(); - IndexService indexService = createIndex("test", settings); + IndexService indexService = createIndex("test"); Map mapperParsers = new HashMap<>(); mapperParsers.put(ExternalMapperPlugin.EXTERNAL, new ExternalMapper.TypeParser(ExternalMapperPlugin.EXTERNAL, "foo")); mapperParsers.put(ExternalMapperPlugin.EXTERNAL_BIS, new ExternalMapper.TypeParser(ExternalMapperPlugin.EXTERNAL, "bar")); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/LegacyDynamicMappingTests.java b/server/src/test/java/org/elasticsearch/index/mapper/LegacyDynamicMappingTests.java new file mode 100644 index 00000000000..42d6aa8951c --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/LegacyDynamicMappingTests.java @@ -0,0 +1,67 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.mapper; + +import org.elasticsearch.Version; +import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.test.ESSingleNodeTestCase; + +import java.io.IOException; + +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; + +public class LegacyDynamicMappingTests extends ESSingleNodeTestCase { + + @Override + protected boolean forbidPrivateIndexSettings() { + return false; + } + + public void testTypeNotCreatedOnIndexFailure() throws IOException { + final Settings settings = Settings.builder().put(IndexMetaData.SETTING_INDEX_VERSION_CREATED.getKey(), Version.V_6_3_0).build(); + try (XContentBuilder mapping = jsonBuilder()) { + mapping.startObject(); + { + mapping.startObject("_default_"); + { + mapping.field("dynamic", "strict"); + } + mapping.endObject(); + } + mapping.endObject(); + createIndex("test", settings, "_default_", mapping); + } + try (XContentBuilder sourceBuilder = jsonBuilder().startObject().field("test", "test").endObject()) { + expectThrows(StrictDynamicMappingException.class, () -> client() + .prepareIndex() + .setIndex("test") + .setType("type") + .setSource(sourceBuilder) + .get()); + + GetMappingsResponse getMappingsResponse = client().admin().indices().prepareGetMappings("test").get(); + assertNull(getMappingsResponse.getMappings().get("test").get("type")); + } + } + +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/LegacyMapperServiceTests.java b/server/src/test/java/org/elasticsearch/index/mapper/LegacyMapperServiceTests.java new file mode 100644 index 00000000000..33f9bd51f33 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/LegacyMapperServiceTests.java @@ -0,0 +1,92 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.mapper; + +import org.elasticsearch.Version; +import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.compress.CompressedXContent; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.elasticsearch.index.IndexService; +import org.elasticsearch.test.ESSingleNodeTestCase; + +import java.io.IOException; + +public class LegacyMapperServiceTests extends ESSingleNodeTestCase { + + @Override + protected boolean forbidPrivateIndexSettings() { + return false; + } + + public void testIndexMetaDataUpdateDoesNotLoseDefaultMapper() throws IOException { + final IndexService indexService = + createIndex("test", Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_6_3_0).build()); + try (XContentBuilder builder = JsonXContent.contentBuilder()) { + builder.startObject(); + { + builder.startObject(MapperService.DEFAULT_MAPPING); + { + builder.field("date_detection", false); + } + builder.endObject(); + } + builder.endObject(); + final PutMappingRequest putMappingRequest = new PutMappingRequest(); + putMappingRequest.indices("test"); + putMappingRequest.type(MapperService.DEFAULT_MAPPING); + putMappingRequest.source(builder); + client().admin().indices().preparePutMapping("test").setType(MapperService.DEFAULT_MAPPING).setSource(builder).get(); + } + assertNotNull(indexService.mapperService().documentMapper(MapperService.DEFAULT_MAPPING)); + final Settings zeroReplicasSettings = Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0).build(); + client().admin().indices().prepareUpdateSettings("test").setSettings(zeroReplicasSettings).get(); + /* + * This assertion is a guard against a previous bug that would lose the default mapper when applying a metadata update that did not + * update the default mapping. + */ + assertNotNull(indexService.mapperService().documentMapper(MapperService.DEFAULT_MAPPING)); + } + + public void testDefaultMappingIsDeprecatedOn6() throws IOException { + final Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_6_3_0).build(); + final String mapping; + try (XContentBuilder defaultMapping = XContentFactory.jsonBuilder()) { + defaultMapping.startObject(); + { + defaultMapping.startObject("_default_"); + { + + } + defaultMapping.endObject(); + } + defaultMapping.endObject(); + mapping = Strings.toString(defaultMapping); + } + final MapperService mapperService = createIndex("test", settings).mapperService(); + mapperService.merge("_default_", new CompressedXContent(mapping), MapperService.MergeReason.MAPPING_UPDATE); + assertWarnings("[_default_] mapping is deprecated since it is not useful anymore now that indexes cannot have more than one type"); + } + +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/LegacyTypeFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/LegacyTypeFieldMapperTests.java new file mode 100644 index 00000000000..9566e1afa6d --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/LegacyTypeFieldMapperTests.java @@ -0,0 +1,41 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.mapper; + +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.test.ESSingleNodeTestCase; + +public class LegacyTypeFieldMapperTests extends ESSingleNodeTestCase { + + @Override + protected boolean forbidPrivateIndexSettings() { + return false; + } + + public void testDocValuesMultipleTypes() throws Exception { + TypeFieldMapperTests.testDocValues(index -> { + final Settings settings = Settings.builder().put(IndexMetaData.SETTING_INDEX_VERSION_CREATED.getKey(), Version.V_6_0_0).build(); + return this.createIndex(index, settings); + }); + } + +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java b/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java index e31cee29e67..55f6b44babe 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java @@ -20,17 +20,12 @@ package org.elasticsearch.index.mapper; import org.elasticsearch.ExceptionsHelper; -import org.elasticsearch.Version; -import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; -import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.mapper.KeywordFieldMapper.KeywordFieldType; import org.elasticsearch.index.mapper.MapperService.MergeReason; @@ -122,35 +117,6 @@ public class MapperServiceTests extends ESSingleNodeTestCase { assertNull(indexService.mapperService().documentMapper(MapperService.DEFAULT_MAPPING)); } - public void testIndexMetaDataUpdateDoesNotLoseDefaultMapper() throws IOException { - final IndexService indexService = - createIndex("test", Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_6_3_0).build()); - try (XContentBuilder builder = JsonXContent.contentBuilder()) { - builder.startObject(); - { - builder.startObject(MapperService.DEFAULT_MAPPING); - { - builder.field("date_detection", false); - } - builder.endObject(); - } - builder.endObject(); - final PutMappingRequest putMappingRequest = new PutMappingRequest(); - putMappingRequest.indices("test"); - putMappingRequest.type(MapperService.DEFAULT_MAPPING); - putMappingRequest.source(builder); - client().admin().indices().preparePutMapping("test").setType(MapperService.DEFAULT_MAPPING).setSource(builder).get(); - } - assertNotNull(indexService.mapperService().documentMapper(MapperService.DEFAULT_MAPPING)); - final Settings zeroReplicasSettings = Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0).build(); - client().admin().indices().prepareUpdateSettings("test").setSettings(zeroReplicasSettings).get(); - /* - * This assertion is a guard against a previous bug that would lose the default mapper when applying a metadata update that did not - * update the default mapping. - */ - assertNotNull(indexService.mapperService().documentMapper(MapperService.DEFAULT_MAPPING)); - } - public void testTotalFieldsExceedsLimit() throws Throwable { Function mapping = type -> { try { @@ -370,12 +336,4 @@ public class MapperServiceTests extends ESSingleNodeTestCase { "can have at most one type.", e.getMessage()); } - public void testDefaultMappingIsDeprecatedOn6() throws IOException { - Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_6_3_0).build(); - String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("_default_").endObject().endObject()); - MapperService mapperService = createIndex("test", settings).mapperService(); - mapperService.merge("_default_", new CompressedXContent(mapping), MergeReason.MAPPING_UPDATE); - assertWarnings("[_default_] mapping is deprecated since it is not useful anymore now that indexes " + - "cannot have more than one type"); - } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TypeFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TypeFieldMapperTests.java index 3bec98d33ee..26f2b7fc923 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TypeFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TypeFieldMapperTests.java @@ -25,11 +25,11 @@ import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; -import org.elasticsearch.Version; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.IndexService; import org.elasticsearch.index.fielddata.AtomicOrdinalsFieldData; import org.elasticsearch.index.fielddata.IndexFieldDataCache; import org.elasticsearch.index.fielddata.IndexOrdinalsFieldData; @@ -43,6 +43,7 @@ import java.io.IOException; import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.function.Function; public class TypeFieldMapperTests extends ESSingleNodeTestCase { @@ -51,19 +52,12 @@ public class TypeFieldMapperTests extends ESSingleNodeTestCase { return pluginList(InternalSettingsPlugin.class); } - public void testDocValuesMultipleTypes() throws Exception { - testDocValues(false); - } - public void testDocValuesSingleType() throws Exception { - testDocValues(true); + testDocValues(this::createIndex); } - public void testDocValues(boolean singleType) throws IOException { - Settings indexSettings = singleType ? Settings.EMPTY : Settings.builder() - .put("index.version.created", Version.V_6_0_0) - .build(); - MapperService mapperService = createIndex("test", indexSettings).mapperService(); + public static void testDocValues(Function createIndex) throws IOException { + MapperService mapperService = createIndex.apply("test").mapperService(); DocumentMapper mapper = mapperService.merge("type", new CompressedXContent("{\"type\":{}}"), MergeReason.MAPPING_UPDATE); ParsedDocument document = mapper.parse(SourceToParse.source("index", "type", "id", new BytesArray("{}"), XContentType.JSON)); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/UpdateMappingOnClusterIT.java b/server/src/test/java/org/elasticsearch/index/mapper/UpdateMappingOnClusterIT.java deleted file mode 100644 index 7de97d88a5e..00000000000 --- a/server/src/test/java/org/elasticsearch/index/mapper/UpdateMappingOnClusterIT.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.mapper; - -import org.elasticsearch.Version; -import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; -import org.elasticsearch.client.Client; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.test.ESIntegTestCase; -import org.elasticsearch.test.InternalSettingsPlugin; - -import java.util.Arrays; -import java.util.Collection; - -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; -import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.equalTo; - -public class UpdateMappingOnClusterIT extends ESIntegTestCase { - private static final String INDEX = "index"; - private static final String TYPE = "type"; - - @Override - protected Collection> nodePlugins() { - return Arrays.asList(InternalSettingsPlugin.class); // uses index.version.created - } - - protected void testConflict(String mapping, String mappingUpdate, Version idxVersion, String... errorMessages) throws InterruptedException { - assertAcked(prepareCreate(INDEX).setSource(mapping, XContentType.JSON) - .setSettings(Settings.builder().put("index.version.created", idxVersion.id))); - ensureGreen(INDEX); - GetMappingsResponse mappingsBeforeUpdateResponse = client().admin().indices().prepareGetMappings(INDEX).addTypes(TYPE).get(); - try { - client().admin().indices().preparePutMapping(INDEX).setType(TYPE).setSource(mappingUpdate, XContentType.JSON).get(); - fail(); - } catch (IllegalArgumentException e) { - for (String errorMessage : errorMessages) { - assertThat(e.getMessage(), containsString(errorMessage)); - } - } - compareMappingOnNodes(mappingsBeforeUpdateResponse); - - } - - private void compareMappingOnNodes(GetMappingsResponse previousMapping) { - // make sure all nodes have same cluster state - for (Client client : cluster().getClients()) { - GetMappingsResponse currentMapping = client.admin().indices().prepareGetMappings(INDEX).addTypes(TYPE).setLocal(true).get(); - assertThat(previousMapping.getMappings().get(INDEX).get(TYPE).source(), equalTo(currentMapping.getMappings().get(INDEX).get(TYPE).source())); - } - } -} diff --git a/server/src/test/java/org/elasticsearch/index/similarity/LegacySimilarityTests.java b/server/src/test/java/org/elasticsearch/index/similarity/LegacySimilarityTests.java new file mode 100644 index 00000000000..69ad3bd128f --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/similarity/LegacySimilarityTests.java @@ -0,0 +1,93 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.similarity; + +import org.apache.lucene.search.similarities.BM25Similarity; +import org.apache.lucene.search.similarities.BooleanSimilarity; +import org.apache.lucene.search.similarities.ClassicSimilarity; +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.test.ESSingleNodeTestCase; + +import java.io.IOException; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.instanceOf; + +public class LegacySimilarityTests extends ESSingleNodeTestCase { + + @Override + protected boolean forbidPrivateIndexSettings() { + return false; + } + + public void testResolveDefaultSimilaritiesOn6xIndex() { + final Settings indexSettings = Settings.builder() + .put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_6_3_0) // otherwise classic is forbidden + .build(); + final SimilarityService similarityService = createIndex("foo", indexSettings).similarityService(); + assertThat(similarityService.getSimilarity("classic").get(), instanceOf(ClassicSimilarity.class)); + assertWarnings("The [classic] similarity is now deprecated in favour of BM25, which is generally " + + "accepted as a better alternative. Use the [BM25] similarity or build a custom [scripted] similarity " + + "instead."); + assertThat(similarityService.getSimilarity("BM25").get(), instanceOf(BM25Similarity.class)); + assertThat(similarityService.getSimilarity("boolean").get(), instanceOf(BooleanSimilarity.class)); + assertThat(similarityService.getSimilarity("default"), equalTo(null)); + } + + public void testResolveSimilaritiesFromMappingClassic() throws IOException { + try (XContentBuilder mapping = XContentFactory.jsonBuilder()) { + mapping.startObject(); + { + mapping.startObject("type"); + { + mapping.startObject("properties"); + { + mapping.startObject("field1"); + { + mapping.field("type", "text"); + mapping.field("similarity", "my_similarity"); + } + mapping.endObject(); + } + mapping.endObject(); + } + mapping.endObject(); + } + mapping.endObject(); + + final Settings indexSettings = Settings.builder() + .put(IndexMetaData.SETTING_INDEX_VERSION_CREATED.getKey(), Version.V_6_3_0) // otherwise classic is forbidden + .put("index.similarity.my_similarity.type", "classic") + .put("index.similarity.my_similarity.discount_overlaps", false) + .build(); + final MapperService mapperService = createIndex("foo", indexSettings, "type", mapping).mapperService(); + assertThat(mapperService.fullName("field1").similarity().get(), instanceOf(ClassicSimilarity.class)); + + final ClassicSimilarity similarity = (ClassicSimilarity) mapperService.fullName("field1").similarity().get(); + assertThat(similarity.getDiscountOverlaps(), equalTo(false)); + } + } + +} diff --git a/server/src/test/java/org/elasticsearch/index/similarity/SimilarityTests.java b/server/src/test/java/org/elasticsearch/index/similarity/SimilarityTests.java index 6102a1b55f1..d2244eb8a5a 100644 --- a/server/src/test/java/org/elasticsearch/index/similarity/SimilarityTests.java +++ b/server/src/test/java/org/elasticsearch/index/similarity/SimilarityTests.java @@ -23,7 +23,6 @@ import org.apache.lucene.search.similarities.AfterEffectL; import org.apache.lucene.search.similarities.BM25Similarity; import org.apache.lucene.search.similarities.BasicModelG; import org.apache.lucene.search.similarities.BooleanSimilarity; -import org.apache.lucene.search.similarities.ClassicSimilarity; import org.apache.lucene.search.similarities.DFISimilarity; import org.apache.lucene.search.similarities.DFRSimilarity; import org.apache.lucene.search.similarities.DistributionSPL; @@ -33,8 +32,6 @@ import org.apache.lucene.search.similarities.LMDirichletSimilarity; import org.apache.lucene.search.similarities.LMJelinekMercerSimilarity; import org.apache.lucene.search.similarities.LambdaTTF; import org.apache.lucene.search.similarities.NormalizationH2; -import org.elasticsearch.Version; -import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.Strings; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.settings.Settings; @@ -72,39 +69,6 @@ public class SimilarityTests extends ESSingleNodeTestCase { + "similarity instead.", e.getMessage()); } - public void testResolveDefaultSimilaritiesOn6xIndex() { - Settings indexSettings = Settings.builder() - .put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_6_3_0) // otherwise classic is forbidden - .build(); - SimilarityService similarityService = createIndex("foo", indexSettings).similarityService(); - assertThat(similarityService.getSimilarity("classic").get(), instanceOf(ClassicSimilarity.class)); - assertWarnings("The [classic] similarity is now deprecated in favour of BM25, which is generally " - + "accepted as a better alternative. Use the [BM25] similarity or build a custom [scripted] similarity " - + "instead."); - assertThat(similarityService.getSimilarity("BM25").get(), instanceOf(BM25Similarity.class)); - assertThat(similarityService.getSimilarity("boolean").get(), instanceOf(BooleanSimilarity.class)); - assertThat(similarityService.getSimilarity("default"), equalTo(null)); - } - - public void testResolveSimilaritiesFromMapping_classic() throws IOException { - XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type") - .startObject("properties") - .startObject("field1").field("type", "text").field("similarity", "my_similarity").endObject() - .endObject() - .endObject().endObject(); - - Settings indexSettings = Settings.builder() - .put("index.similarity.my_similarity.type", "classic") - .put("index.similarity.my_similarity.discount_overlaps", false) - .put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_6_3_0) // otherwise classic is forbidden - .build(); - MapperService mapperService = createIndex("foo", indexSettings, "type", mapping).mapperService(); - assertThat(mapperService.fullName("field1").similarity().get(), instanceOf(ClassicSimilarity.class)); - - ClassicSimilarity similarity = (ClassicSimilarity) mapperService.fullName("field1").similarity().get(); - assertThat(similarity.getDiscountOverlaps(), equalTo(false)); - } - public void testResolveSimilaritiesFromMapping_classicIsForbidden() throws IOException { Settings indexSettings = Settings.builder() .put("index.similarity.my_similarity.type", "classic") diff --git a/server/src/test/java/org/elasticsearch/index/store/CorruptedFileIT.java b/server/src/test/java/org/elasticsearch/index/store/CorruptedFileIT.java index fdadb5999f7..14ef65c368c 100644 --- a/server/src/test/java/org/elasticsearch/index/store/CorruptedFileIT.java +++ b/server/src/test/java/org/elasticsearch/index/store/CorruptedFileIT.java @@ -121,8 +121,11 @@ public class CorruptedFileIT extends ESIntegTestCase { @Override protected Collection> nodePlugins() { - return Arrays.asList(MockTransportService.TestPlugin.class, MockIndexEventListener.TestPlugin.class, MockFSIndexStore.TestPlugin.class, - InternalSettingsPlugin.class); // uses index.version.created + return Arrays.asList( + MockTransportService.TestPlugin.class, + MockIndexEventListener.TestPlugin.class, + MockFSIndexStore.TestPlugin.class, + InternalSettingsPlugin.class); } /** diff --git a/server/src/test/java/org/elasticsearch/indices/analysis/PreBuiltAnalyzerIntegrationIT.java b/server/src/test/java/org/elasticsearch/indices/analysis/PreBuiltAnalyzerIntegrationIT.java index 7722795525d..01790a04dc6 100644 --- a/server/src/test/java/org/elasticsearch/indices/analysis/PreBuiltAnalyzerIntegrationIT.java +++ b/server/src/test/java/org/elasticsearch/indices/analysis/PreBuiltAnalyzerIntegrationIT.java @@ -24,14 +24,10 @@ import org.apache.lucene.analysis.TokenStream; import org.elasticsearch.Version; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; -import org.elasticsearch.test.InternalSettingsPlugin; import java.io.IOException; import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Locale; @@ -44,9 +40,10 @@ import static org.hamcrest.Matchers.notNullValue; @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.SUITE) public class PreBuiltAnalyzerIntegrationIT extends ESIntegTestCase { + @Override - protected Collection> nodePlugins() { - return Arrays.asList(InternalSettingsPlugin.class); + protected boolean forbidPrivateIndexSettings() { + return false; } public void testThatPreBuiltAnalyzersAreNotClosedOnIndexClose() throws Exception { diff --git a/server/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java b/server/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java index 1ea15321d5a..77c77baaa11 100644 --- a/server/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java +++ b/server/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java @@ -46,8 +46,8 @@ import org.elasticsearch.cluster.ClusterStateTaskExecutor.ClusterTasksResult; import org.elasticsearch.cluster.ClusterStateUpdateTask; import org.elasticsearch.cluster.EmptyClusterInfoService; import org.elasticsearch.cluster.action.shard.ShardStateAction; -import org.elasticsearch.cluster.action.shard.ShardStateAction.StartedShardEntry; import org.elasticsearch.cluster.action.shard.ShardStateAction.FailedShardEntry; +import org.elasticsearch.cluster.action.shard.ShardStateAction.StartedShardEntry; import org.elasticsearch.cluster.metadata.AliasValidator; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; @@ -183,7 +183,7 @@ public class ClusterStateChanges extends AbstractComponent { allocationService, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, indicesService, threadPool); MetaDataCreateIndexService createIndexService = new MetaDataCreateIndexService(settings, clusterService, indicesService, allocationService, new AliasValidator(settings), environment, - IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, threadPool, xContentRegistry); + IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, threadPool, xContentRegistry, true); transportCloseIndexAction = new TransportCloseIndexAction(settings, transportService, clusterService, threadPool, indexStateService, clusterSettings, actionFilters, indexNameExpressionResolver, destructiveOperations); diff --git a/server/src/test/java/org/elasticsearch/indices/mapping/LegacyUpdateMappingIntegrationIT.java b/server/src/test/java/org/elasticsearch/indices/mapping/LegacyUpdateMappingIntegrationIT.java new file mode 100644 index 00000000000..1bf95f612ce --- /dev/null +++ b/server/src/test/java/org/elasticsearch/indices/mapping/LegacyUpdateMappingIntegrationIT.java @@ -0,0 +1,212 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.indices.mapping; + +import org.elasticsearch.Version; +import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.elasticsearch.index.mapper.MapperParsingException; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.test.ESIntegTestCase; + +import java.util.Map; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasEntry; +import static org.hamcrest.Matchers.hasKey; +import static org.hamcrest.Matchers.not; + +public class LegacyUpdateMappingIntegrationIT extends ESIntegTestCase { + + @Override + protected boolean forbidPrivateIndexSettings() { + return false; + } + + @SuppressWarnings("unchecked") + public void testUpdateDefaultMappingSettings() throws Exception { + logger.info("Creating index with _default_ mappings"); + try (XContentBuilder defaultMapping = JsonXContent.contentBuilder()) { + defaultMapping.startObject(); + { + defaultMapping.startObject(MapperService.DEFAULT_MAPPING); + { + defaultMapping.field("date_detection", false); + } + defaultMapping.endObject(); + } + defaultMapping.endObject(); + client() + .admin() + .indices() + .prepareCreate("test") + .setSettings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_6_3_0).build()) + .addMapping(MapperService.DEFAULT_MAPPING, defaultMapping) + .get(); + } + + { + final GetMappingsResponse getResponse = + client().admin().indices().prepareGetMappings("test").addTypes(MapperService.DEFAULT_MAPPING).get(); + final Map defaultMapping = + getResponse.getMappings().get("test").get(MapperService.DEFAULT_MAPPING).sourceAsMap(); + assertThat(defaultMapping, hasKey("date_detection")); + } + + logger.info("Emptying _default_ mappings"); + // now remove it + try (XContentBuilder mappingBuilder = + JsonXContent.contentBuilder().startObject().startObject(MapperService.DEFAULT_MAPPING).endObject().endObject()) { + final AcknowledgedResponse putResponse = + client() + .admin() + .indices() + .preparePutMapping("test") + .setType(MapperService.DEFAULT_MAPPING) + .setSource(mappingBuilder) + .get(); + assertThat(putResponse.isAcknowledged(), equalTo(true)); + } + logger.info("Done Emptying _default_ mappings"); + + { + final GetMappingsResponse getResponse = + client().admin().indices().prepareGetMappings("test").addTypes(MapperService.DEFAULT_MAPPING).get(); + final Map defaultMapping = + getResponse.getMappings().get("test").get(MapperService.DEFAULT_MAPPING).sourceAsMap(); + assertThat(defaultMapping, not(hasKey("date_detection"))); + } + + // now test you can change stuff that are normally unchangeable + logger.info("Creating _default_ mappings with an analyzed field"); + try (XContentBuilder defaultMapping = JsonXContent.contentBuilder()) { + + defaultMapping.startObject(); + { + defaultMapping.startObject(MapperService.DEFAULT_MAPPING); + { + defaultMapping.startObject("properties"); + { + defaultMapping.startObject("f"); + { + defaultMapping.field("type", "text"); + defaultMapping.field("index", true); + } + defaultMapping.endObject(); + } + defaultMapping.endObject(); + } + defaultMapping.endObject(); + } + defaultMapping.endObject(); + + final AcknowledgedResponse putResponse = + client() + .admin() + .indices() + .preparePutMapping("test") + .setType(MapperService.DEFAULT_MAPPING).setSource(defaultMapping) + .get(); + assertThat(putResponse.isAcknowledged(), equalTo(true)); + } + + logger.info("Changing _default_ mappings field from analyzed to non-analyzed"); + { + try (XContentBuilder mappingBuilder = JsonXContent.contentBuilder()) { + mappingBuilder.startObject(); + { + mappingBuilder.startObject(MapperService.DEFAULT_MAPPING); + { + mappingBuilder.startObject("properties"); + { + mappingBuilder.startObject("f"); + { + mappingBuilder.field("type", "keyword"); + } + mappingBuilder.endObject(); + } + mappingBuilder.endObject(); + } + mappingBuilder.endObject(); + } + mappingBuilder.endObject(); + + final AcknowledgedResponse putResponse = + client() + .admin() + .indices() + .preparePutMapping("test") + .setType(MapperService.DEFAULT_MAPPING) + .setSource(mappingBuilder) + .get(); + assertThat(putResponse.isAcknowledged(), equalTo(true)); + } + } + logger.info("Done changing _default_ mappings field from analyzed to non-analyzed"); + + { + final GetMappingsResponse getResponse = + client().admin().indices().prepareGetMappings("test").addTypes(MapperService.DEFAULT_MAPPING).get(); + final Map defaultMapping = + getResponse.getMappings().get("test").get(MapperService.DEFAULT_MAPPING).sourceAsMap(); + final Map fieldSettings = (Map) ((Map) defaultMapping.get("properties")).get("f"); + assertThat(fieldSettings, hasEntry("type", "keyword")); + } + + // but we still validate the _default_ type + logger.info("Confirming _default_ mappings validation"); + try (XContentBuilder mappingBuilder = JsonXContent.contentBuilder()) { + + mappingBuilder.startObject(); + { + mappingBuilder.startObject(MapperService.DEFAULT_MAPPING); + { + mappingBuilder.startObject("properites"); + { + mappingBuilder.startObject("f"); + { + mappingBuilder.field("type", "non-existent"); + } + mappingBuilder.endObject(); + } + mappingBuilder.endObject(); + } + mappingBuilder.endObject(); + } + mappingBuilder.endObject(); + + expectThrows( + MapperParsingException.class, + () -> client() + .admin() + .indices() + .preparePutMapping("test") + .setType(MapperService.DEFAULT_MAPPING) + .setSource(mappingBuilder) + .get()); + } + + } + +} diff --git a/server/src/test/java/org/elasticsearch/indices/mapping/UpdateMappingIntegrationIT.java b/server/src/test/java/org/elasticsearch/indices/mapping/UpdateMappingIntegrationIT.java index f28d253087d..af9bf9910ec 100644 --- a/server/src/test/java/org/elasticsearch/indices/mapping/UpdateMappingIntegrationIT.java +++ b/server/src/test/java/org/elasticsearch/indices/mapping/UpdateMappingIntegrationIT.java @@ -19,21 +19,18 @@ package org.elasticsearch.indices.mapping; -import org.elasticsearch.Version; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; import org.elasticsearch.action.admin.indices.refresh.RefreshResponse; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.client.Client; -import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MappingMetaData; import org.elasticsearch.common.Priority; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.common.xcontent.json.JsonXContent; -import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; @@ -56,12 +53,8 @@ import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_BLOCKS_WR import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_READ_ONLY; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertBlocked; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertThrows; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.hasEntry; -import static org.hamcrest.Matchers.hasKey; -import static org.hamcrest.Matchers.not; public class UpdateMappingIntegrationIT extends ESIntegTestCase { @@ -200,69 +193,6 @@ public class UpdateMappingIntegrationIT extends ESIntegTestCase { assertThat(putMappingResponse.isAcknowledged(), equalTo(true)); } - @SuppressWarnings("unchecked") - public void testUpdateDefaultMappingSettings() throws Exception { - logger.info("Creating index with _default_ mappings"); - client().admin().indices().prepareCreate("test") - .setSettings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_6_3_0).build()) - .addMapping(MapperService.DEFAULT_MAPPING, - JsonXContent.contentBuilder().startObject().startObject(MapperService.DEFAULT_MAPPING) - .field("date_detection", false) - .endObject().endObject() - ).get(); - - GetMappingsResponse getResponse = client().admin().indices().prepareGetMappings("test").addTypes(MapperService.DEFAULT_MAPPING).get(); - Map defaultMapping = getResponse.getMappings().get("test").get(MapperService.DEFAULT_MAPPING).sourceAsMap(); - assertThat(defaultMapping, hasKey("date_detection")); - - - logger.info("Emptying _default_ mappings"); - // now remove it - AcknowledgedResponse putResponse = client().admin().indices().preparePutMapping("test").setType(MapperService.DEFAULT_MAPPING).setSource( - JsonXContent.contentBuilder().startObject().startObject(MapperService.DEFAULT_MAPPING) - .endObject().endObject() - ).get(); - assertThat(putResponse.isAcknowledged(), equalTo(true)); - logger.info("Done Emptying _default_ mappings"); - - getResponse = client().admin().indices().prepareGetMappings("test").addTypes(MapperService.DEFAULT_MAPPING).get(); - defaultMapping = getResponse.getMappings().get("test").get(MapperService.DEFAULT_MAPPING).sourceAsMap(); - assertThat(defaultMapping, not(hasKey("date_detection"))); - - // now test you can change stuff that are normally unchangeable - logger.info("Creating _default_ mappings with an analyzed field"); - putResponse = client().admin().indices().preparePutMapping("test").setType(MapperService.DEFAULT_MAPPING).setSource( - JsonXContent.contentBuilder().startObject().startObject(MapperService.DEFAULT_MAPPING) - .startObject("properties").startObject("f").field("type", "text").field("index", true).endObject().endObject() - .endObject().endObject() - ).get(); - assertThat(putResponse.isAcknowledged(), equalTo(true)); - - - logger.info("Changing _default_ mappings field from analyzed to non-analyzed"); - putResponse = client().admin().indices().preparePutMapping("test").setType(MapperService.DEFAULT_MAPPING).setSource( - JsonXContent.contentBuilder().startObject().startObject(MapperService.DEFAULT_MAPPING) - .startObject("properties").startObject("f").field("type", "keyword").endObject().endObject() - .endObject().endObject() - ).get(); - assertThat(putResponse.isAcknowledged(), equalTo(true)); - logger.info("Done changing _default_ mappings field from analyzed to non-analyzed"); - - getResponse = client().admin().indices().prepareGetMappings("test").addTypes(MapperService.DEFAULT_MAPPING).get(); - defaultMapping = getResponse.getMappings().get("test").get(MapperService.DEFAULT_MAPPING).sourceAsMap(); - Map fieldSettings = (Map) ((Map) defaultMapping.get("properties")).get("f"); - assertThat(fieldSettings, hasEntry("type", "keyword")); - - // but we still validate the _default_ type - logger.info("Confirming _default_ mappings validation"); - assertThrows(client().admin().indices().preparePutMapping("test").setType(MapperService.DEFAULT_MAPPING).setSource( - JsonXContent.contentBuilder().startObject().startObject(MapperService.DEFAULT_MAPPING) - .startObject("properties").startObject("f").field("type", "DOESNT_EXIST").endObject().endObject() - .endObject().endObject() - ), MapperParsingException.class); - - } - public void testUpdateMappingConcurrently() throws Throwable { createIndex("test1", "test2"); diff --git a/server/src/test/java/org/elasticsearch/indices/settings/InternalOrPrivateSettingsPlugin.java b/server/src/test/java/org/elasticsearch/indices/settings/InternalOrPrivateSettingsPlugin.java new file mode 100644 index 00000000000..8792232b381 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/indices/settings/InternalOrPrivateSettingsPlugin.java @@ -0,0 +1,201 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.indices.settings; + +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.master.MasterNodeRequest; +import org.elasticsearch.action.support.master.TransportMasterNodeAction; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.ClusterStateUpdateTask; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.plugins.ActionPlugin; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +public class InternalOrPrivateSettingsPlugin extends Plugin implements ActionPlugin { + + static final Setting INDEX_INTERNAL_SETTING = + Setting.simpleString("index.internal", Setting.Property.IndexScope, Setting.Property.InternalIndex); + + static final Setting INDEX_PRIVATE_SETTING = + Setting.simpleString("index.private", Setting.Property.IndexScope, Setting.Property.PrivateIndex); + + @Override + public List> getSettings() { + return Arrays.asList(INDEX_INTERNAL_SETTING, INDEX_PRIVATE_SETTING); + } + + public static class UpdateInternalOrPrivateAction extends Action { + + static final UpdateInternalOrPrivateAction INSTANCE = new UpdateInternalOrPrivateAction(); + private static final String NAME = "indices:admin/settings/update-internal-or-private-index"; + + public UpdateInternalOrPrivateAction() { + super(NAME); + } + + static class Request extends MasterNodeRequest { + + private String index; + private String key; + private String value; + + Request() { + + } + + Request(final String index, final String key, final String value) { + this.index = index; + this.key = key; + this.value = value; + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public void readFrom(final StreamInput in) throws IOException { + super.readFrom(in); + index = in.readString(); + key = in.readString(); + value = in.readString(); + } + + @Override + public void writeTo(final StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(index); + out.writeString(key); + out.writeString(value); + } + + } + + static class Response extends ActionResponse { + + } + + @Override + public UpdateInternalOrPrivateAction.Response newResponse() { + return new UpdateInternalOrPrivateAction.Response(); + } + + } + + public static class TransportUpdateInternalOrPrivateAction + extends TransportMasterNodeAction { + + @Inject + public TransportUpdateInternalOrPrivateAction( + final Settings settings, + final TransportService transportService, + final ClusterService clusterService, + final ThreadPool threadPool, + final ActionFilters actionFilters, + final IndexNameExpressionResolver indexNameExpressionResolver) { + super( + settings, + UpdateInternalOrPrivateAction.NAME, + transportService, + clusterService, + threadPool, + actionFilters, + indexNameExpressionResolver, + UpdateInternalOrPrivateAction.Request::new); + } + + @Override + protected String executor() { + return ThreadPool.Names.SAME; + } + + @Override + protected UpdateInternalOrPrivateAction.Response newResponse() { + return new UpdateInternalOrPrivateAction.Response(); + } + + @Override + protected void masterOperation( + final UpdateInternalOrPrivateAction.Request request, + final ClusterState state, + final ActionListener listener) throws Exception { + clusterService.submitStateUpdateTask("update-index-internal-or-private", new ClusterStateUpdateTask() { + @Override + public ClusterState execute(final ClusterState currentState) throws Exception { + final MetaData.Builder builder = MetaData.builder(currentState.metaData()); + final IndexMetaData.Builder imdBuilder = IndexMetaData.builder(currentState.metaData().index(request.index)); + final Settings.Builder settingsBuilder = + Settings.builder() + .put(currentState.metaData().index(request.index).getSettings()) + .put(request.key, request.value); + imdBuilder.settings(settingsBuilder); + builder.put(imdBuilder.build(), true); + return ClusterState.builder(currentState).metaData(builder).build(); + } + + @Override + public void clusterStateProcessed(final String source, final ClusterState oldState, final ClusterState newState) { + listener.onResponse(new UpdateInternalOrPrivateAction.Response()); + } + + @Override + public void onFailure(final String source, final Exception e) { + listener.onFailure(e); + } + + }); + } + + @Override + protected ClusterBlockException checkBlock(final UpdateInternalOrPrivateAction.Request request, final ClusterState state) { + return null; + } + + } + + @Override + public List> getActions() { + return Collections.singletonList( + new ActionHandler<>(UpdateInternalOrPrivateAction.INSTANCE, TransportUpdateInternalOrPrivateAction.class)); + } + +} diff --git a/server/src/test/java/org/elasticsearch/indices/settings/InternalSettingsIT.java b/server/src/test/java/org/elasticsearch/indices/settings/InternalSettingsIT.java new file mode 100644 index 00000000000..1d11fbc79fc --- /dev/null +++ b/server/src/test/java/org/elasticsearch/indices/settings/InternalSettingsIT.java @@ -0,0 +1,84 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.indices.settings; + +import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.ESIntegTestCase; + +import java.util.Collection; +import java.util.Collections; + +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasToString; + +public class InternalSettingsIT extends ESIntegTestCase { + + @Override + protected Collection> nodePlugins() { + return Collections.singleton(InternalOrPrivateSettingsPlugin.class); + } + + @Override + protected Collection> transportClientPlugins() { + return Collections.singletonList(InternalOrPrivateSettingsPlugin.class); + } + + public void testSetInternalIndexSettingOnCreate() { + final Settings settings = Settings.builder().put("index.internal", "internal").build(); + createIndex("index", settings); + final GetSettingsResponse response = client().admin().indices().prepareGetSettings("index").get(); + assertThat(response.getSetting("index", "index.internal"), equalTo("internal")); + } + + public void testUpdateInternalIndexSettingViaSettingsAPI() { + final Settings settings = Settings.builder().put("index.internal", "internal").build(); + createIndex("test", settings); + final GetSettingsResponse response = client().admin().indices().prepareGetSettings("test").get(); + assertThat(response.getSetting("test", "index.internal"), equalTo("internal")); + // we can not update the setting via the update settings API + final IllegalArgumentException e = expectThrows(IllegalArgumentException.class, + () -> client().admin() + .indices() + .prepareUpdateSettings("test") + .setSettings(Settings.builder().put("index.internal", "internal-update")) + .get()); + final String message = "can not update internal setting [index.internal]; this setting is managed via a dedicated API"; + assertThat(e, hasToString(containsString(message))); + final GetSettingsResponse responseAfterAttemptedUpdate = client().admin().indices().prepareGetSettings("test").get(); + assertThat(responseAfterAttemptedUpdate.getSetting("test", "index.internal"), equalTo("internal")); + } + + public void testUpdateInternalIndexSettingViaDedicatedAPI() { + final Settings settings = Settings.builder().put("index.internal", "internal").build(); + createIndex("test", settings); + final GetSettingsResponse response = client().admin().indices().prepareGetSettings("test").get(); + assertThat(response.getSetting("test", "index.internal"), equalTo("internal")); + client().execute( + InternalOrPrivateSettingsPlugin.UpdateInternalOrPrivateAction.INSTANCE, + new InternalOrPrivateSettingsPlugin.UpdateInternalOrPrivateAction.Request("test", "index.internal", "internal-update")) + .actionGet(); + final GetSettingsResponse responseAfterUpdate = client().admin().indices().prepareGetSettings("test").get(); + assertThat(responseAfterUpdate.getSetting("test", "index.internal"), equalTo("internal-update")); + } + +} diff --git a/server/src/test/java/org/elasticsearch/indices/settings/PrivateSettingsIT.java b/server/src/test/java/org/elasticsearch/indices/settings/PrivateSettingsIT.java new file mode 100644 index 00000000000..08f45eac5be --- /dev/null +++ b/server/src/test/java/org/elasticsearch/indices/settings/PrivateSettingsIT.java @@ -0,0 +1,81 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.indices.settings; + +import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; +import org.elasticsearch.common.ValidationException; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.ESIntegTestCase; + +import java.util.Collection; +import java.util.Collections; + +import static org.hamcrest.Matchers.anyOf; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasToString; +import static org.hamcrest.Matchers.instanceOf; + +public class PrivateSettingsIT extends ESIntegTestCase { + + @Override + protected Collection> nodePlugins() { + return Collections.singletonList(InternalOrPrivateSettingsPlugin.class); + } + + @Override + protected Collection> transportClientPlugins() { + return Collections.singletonList(InternalOrPrivateSettingsPlugin.class); + } + + public void testSetPrivateIndexSettingOnCreate() { + final Settings settings = Settings.builder().put("index.private", "private").build(); + final Exception e = expectThrows(Exception.class, () -> createIndex("index", settings)); + assertThat(e, anyOf(instanceOf(IllegalArgumentException.class), instanceOf(ValidationException.class))); + assertThat(e, hasToString(containsString("private index setting [index.private] can not be set explicitly"))); + } + + public void testUpdatePrivateIndexSettingViaSettingsAPI() { + createIndex("test"); + // we can not update the setting via the update settings API + final IllegalArgumentException e = expectThrows(IllegalArgumentException.class, + () -> client().admin() + .indices() + .prepareUpdateSettings("test") + .setSettings(Settings.builder().put("index.private", "private-update")) + .get()); + final String message = "can not update private setting [index.private]; this setting is managed by Elasticsearch"; + assertThat(e, hasToString(containsString(message))); + final GetSettingsResponse responseAfterAttemptedUpdate = client().admin().indices().prepareGetSettings("test").get(); + assertNull(responseAfterAttemptedUpdate.getSetting("test", "index.private")); + } + + public void testUpdatePrivatelIndexSettingViaDedicatedAPI() { + createIndex("test"); + client().execute( + InternalOrPrivateSettingsPlugin.UpdateInternalOrPrivateAction.INSTANCE, + new InternalOrPrivateSettingsPlugin.UpdateInternalOrPrivateAction.Request("test", "index.private", "private-update")) + .actionGet(); + final GetSettingsResponse responseAfterUpdate = client().admin().indices().prepareGetSettings("test").get(); + assertThat(responseAfterUpdate.getSetting("test", "index.private"), equalTo("private-update")); + } + +} diff --git a/server/src/test/java/org/elasticsearch/indices/settings/UpdateSettingsIT.java b/server/src/test/java/org/elasticsearch/indices/settings/UpdateSettingsIT.java index 069f965ac6b..fbf1dcd5b33 100644 --- a/server/src/test/java/org/elasticsearch/indices/settings/UpdateSettingsIT.java +++ b/server/src/test/java/org/elasticsearch/indices/settings/UpdateSettingsIT.java @@ -19,40 +19,19 @@ package org.elasticsearch.indices.settings; -import org.elasticsearch.action.Action; -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.ActionRequest; -import org.elasticsearch.action.ActionRequestValidationException; -import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; -import org.elasticsearch.action.support.ActionFilters; -import org.elasticsearch.action.support.master.MasterNodeRequest; -import org.elasticsearch.action.support.master.TransportMasterNodeAction; -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.ClusterStateUpdateTask; -import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.metadata.IndexMetaData; -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; -import org.elasticsearch.cluster.metadata.MetaData; -import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Priority; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexModule; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.engine.VersionConflictEngineException; import org.elasticsearch.indices.IndicesService; -import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; -import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.transport.TransportService; -import java.io.IOException; import java.util.Arrays; import java.util.Collection; import java.util.Collections; @@ -67,7 +46,6 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertBloc import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertThrows; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.hasToString; import static org.hamcrest.Matchers.nullValue; public class UpdateSettingsIT extends ESIntegTestCase { @@ -101,12 +79,8 @@ public class UpdateSettingsIT extends ESIntegTestCase { @Override protected Collection> nodePlugins() { - return Arrays.asList(DummySettingPlugin.class, FinalSettingPlugin.class, InternalIndexSettingsPlugin.class); - } - - @Override - protected Collection> transportClientPlugins() { - return Collections.singletonList(InternalIndexSettingsPlugin.class); + return Arrays.asList( + DummySettingPlugin.class, FinalSettingPlugin.class); } public static class DummySettingPlugin extends Plugin { @@ -151,151 +125,6 @@ public class UpdateSettingsIT extends ESIntegTestCase { } } - public static class InternalIndexSettingsPlugin extends Plugin implements ActionPlugin { - - public static final Setting INDEX_INTERNAL_SETTING = - Setting.simpleString("index.internal", Setting.Property.IndexScope, Setting.Property.InternalIndex); - - @Override - public List> getSettings() { - return Collections.singletonList(INDEX_INTERNAL_SETTING); - } - - public static class UpdateInternalIndexAction - extends Action { - - private static final UpdateInternalIndexAction INSTANCE = new UpdateInternalIndexAction(); - private static final String NAME = "indices:admin/settings/update-internal-index"; - - public UpdateInternalIndexAction() { - super(NAME); - } - - static class Request extends MasterNodeRequest { - - private String index; - private String value; - - Request() { - - } - - Request(final String index, final String value) { - this.index = index; - this.value = value; - } - - @Override - public ActionRequestValidationException validate() { - return null; - } - - @Override - public void readFrom(final StreamInput in) throws IOException { - super.readFrom(in); - index = in.readString(); - value = in.readString(); - } - - @Override - public void writeTo(final StreamOutput out) throws IOException { - super.writeTo(out); - out.writeString(index); - out.writeString(value); - } - - } - - static class Response extends ActionResponse { - - } - - @Override - public Response newResponse() { - return new Response(); - } - - } - - public static class TransportUpdateInternalIndexAction - extends TransportMasterNodeAction { - - @Inject - public TransportUpdateInternalIndexAction( - final Settings settings, - final TransportService transportService, - final ClusterService clusterService, - final ThreadPool threadPool, - final ActionFilters actionFilters, - final IndexNameExpressionResolver indexNameExpressionResolver) { - super( - settings, - UpdateInternalIndexAction.NAME, - transportService, - clusterService, - threadPool, - actionFilters, - indexNameExpressionResolver, - UpdateInternalIndexAction.Request::new); - } - - @Override - protected String executor() { - return ThreadPool.Names.SAME; - } - - @Override - protected UpdateInternalIndexAction.Response newResponse() { - return new UpdateInternalIndexAction.Response(); - } - - @Override - protected void masterOperation( - final UpdateInternalIndexAction.Request request, - final ClusterState state, - final ActionListener listener) throws Exception { - clusterService.submitStateUpdateTask("update-index-internal", new ClusterStateUpdateTask() { - @Override - public ClusterState execute(final ClusterState currentState) throws Exception { - final MetaData.Builder builder = MetaData.builder(currentState.metaData()); - final IndexMetaData.Builder imdBuilder = IndexMetaData.builder(currentState.metaData().index(request.index)); - final Settings.Builder settingsBuilder = - Settings.builder() - .put(currentState.metaData().index(request.index).getSettings()) - .put("index.internal", request.value); - imdBuilder.settings(settingsBuilder); - builder.put(imdBuilder.build(), true); - return ClusterState.builder(currentState).metaData(builder).build(); - } - - @Override - public void clusterStateProcessed(final String source, final ClusterState oldState, final ClusterState newState) { - listener.onResponse(new UpdateInternalIndexAction.Response()); - } - - @Override - public void onFailure(final String source, final Exception e) { - listener.onFailure(e); - } - - }); - } - - @Override - protected ClusterBlockException checkBlock(UpdateInternalIndexAction.Request request, ClusterState state) { - return null; - } - - } - - @Override - public List> getActions() { - return Collections.singletonList( - new ActionHandler<>(UpdateInternalIndexAction.INSTANCE, TransportUpdateInternalIndexAction.class)); - } - - } - public void testUpdateDependentClusterSettings() { IllegalArgumentException iae = expectThrows(IllegalArgumentException.class, () -> client().admin().cluster().prepareUpdateSettings().setPersistentSettings(Settings.builder() @@ -646,35 +475,4 @@ public class UpdateSettingsIT extends ESIntegTestCase { } } - public void testUpdateInternalIndexSettingViaSettingsAPI() { - final Settings settings = Settings.builder().put("index.internal", "internal").build(); - createIndex("test", settings); - final GetSettingsResponse response = client().admin().indices().prepareGetSettings("test").get(); - assertThat(response.getSetting("test", "index.internal"), equalTo("internal")); - // we can not update the setting via the update settings API - final IllegalArgumentException e = expectThrows(IllegalArgumentException.class, - () -> client().admin() - .indices() - .prepareUpdateSettings("test") - .setSettings(Settings.builder().put("index.internal", "internal-update")) - .get()); - final String message = "can not update internal setting [index.internal]; this setting is managed via a dedicated API"; - assertThat(e, hasToString(containsString(message))); - final GetSettingsResponse responseAfterAttemptedUpdate = client().admin().indices().prepareGetSettings("test").get(); - assertThat(responseAfterAttemptedUpdate.getSetting("test", "index.internal"), equalTo("internal")); - } - - public void testUpdateInternalIndexSettingViaDedicatedAPI() { - final Settings settings = Settings.builder().put("index.internal", "internal").build(); - createIndex("test", settings); - final GetSettingsResponse response = client().admin().indices().prepareGetSettings("test").get(); - assertThat(response.getSetting("test", "index.internal"), equalTo("internal")); - client().execute( - InternalIndexSettingsPlugin.UpdateInternalIndexAction.INSTANCE, - new InternalIndexSettingsPlugin.UpdateInternalIndexAction.Request("test", "internal-update")) - .actionGet(); - final GetSettingsResponse responseAfterUpdate = client().admin().indices().prepareGetSettings("test").get(); - assertThat(responseAfterUpdate.getSetting("test", "index.internal"), equalTo("internal-update")); - } - } diff --git a/server/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java b/server/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java index ce162b9600c..44ca66d571d 100644 --- a/server/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java +++ b/server/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java @@ -20,7 +20,6 @@ package org.elasticsearch.indices.stats; import org.apache.lucene.util.LuceneTestCase.SuppressCodecs; -import org.elasticsearch.Version; import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; @@ -727,54 +726,6 @@ public class IndexStatsIT extends ESIntegTestCase { } - public void testFieldDataFieldsParam() throws Exception { - assertAcked(client().admin().indices().prepareCreate("test1") - .setSettings(Settings.builder().put("index.version.created", Version.V_6_0_0.id)) - .addMapping("_doc", "bar", "type=text,fielddata=true", - "baz", "type=text,fielddata=true").get()); - - ensureGreen(); - - client().prepareIndex("test1", "_doc", Integer.toString(1)).setSource("{\"bar\":\"bar\",\"baz\":\"baz\"}", XContentType.JSON).get(); - client().prepareIndex("test1", "_doc", Integer.toString(2)).setSource("{\"bar\":\"bar\",\"baz\":\"baz\"}", XContentType.JSON).get(); - refresh(); - - client().prepareSearch("_all").addSort("bar", SortOrder.ASC).addSort("baz", SortOrder.ASC).execute().actionGet(); - - IndicesStatsRequestBuilder builder = client().admin().indices().prepareStats(); - IndicesStatsResponse stats = builder.execute().actionGet(); - - assertThat(stats.getTotal().fieldData.getMemorySizeInBytes(), greaterThan(0L)); - assertThat(stats.getTotal().fieldData.getFields(), is(nullValue())); - - stats = builder.setFieldDataFields("bar").execute().actionGet(); - assertThat(stats.getTotal().fieldData.getMemorySizeInBytes(), greaterThan(0L)); - assertThat(stats.getTotal().fieldData.getFields().containsField("bar"), is(true)); - assertThat(stats.getTotal().fieldData.getFields().get("bar"), greaterThan(0L)); - assertThat(stats.getTotal().fieldData.getFields().containsField("baz"), is(false)); - - stats = builder.setFieldDataFields("bar", "baz").execute().actionGet(); - assertThat(stats.getTotal().fieldData.getMemorySizeInBytes(), greaterThan(0L)); - assertThat(stats.getTotal().fieldData.getFields().containsField("bar"), is(true)); - assertThat(stats.getTotal().fieldData.getFields().get("bar"), greaterThan(0L)); - assertThat(stats.getTotal().fieldData.getFields().containsField("baz"), is(true)); - assertThat(stats.getTotal().fieldData.getFields().get("baz"), greaterThan(0L)); - - stats = builder.setFieldDataFields("*").execute().actionGet(); - assertThat(stats.getTotal().fieldData.getMemorySizeInBytes(), greaterThan(0L)); - assertThat(stats.getTotal().fieldData.getFields().containsField("bar"), is(true)); - assertThat(stats.getTotal().fieldData.getFields().get("bar"), greaterThan(0L)); - assertThat(stats.getTotal().fieldData.getFields().containsField("baz"), is(true)); - assertThat(stats.getTotal().fieldData.getFields().get("baz"), greaterThan(0L)); - - stats = builder.setFieldDataFields("*r").execute().actionGet(); - assertThat(stats.getTotal().fieldData.getMemorySizeInBytes(), greaterThan(0L)); - assertThat(stats.getTotal().fieldData.getFields().containsField("bar"), is(true)); - assertThat(stats.getTotal().fieldData.getFields().get("bar"), greaterThan(0L)); - assertThat(stats.getTotal().fieldData.getFields().containsField("baz"), is(false)); - - } - public void testCompletionFieldsParam() throws Exception { assertAcked(prepareCreate("test1") .addMapping( diff --git a/server/src/test/java/org/elasticsearch/indices/stats/LegacyIndexStatsIT.java b/server/src/test/java/org/elasticsearch/indices/stats/LegacyIndexStatsIT.java new file mode 100644 index 00000000000..c8ae3edb886 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/indices/stats/LegacyIndexStatsIT.java @@ -0,0 +1,104 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.indices.stats; + +import org.elasticsearch.Version; +import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequestBuilder; +import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.search.sort.SortOrder; +import org.elasticsearch.test.ESIntegTestCase; + +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.nullValue; + +public class LegacyIndexStatsIT extends ESIntegTestCase { + + @Override + protected boolean forbidPrivateIndexSettings() { + return false; + } + + public void testFieldDataFieldsParam() { + assertAcked(client() + .admin() + .indices() + .prepareCreate("test1") + .setSettings(Settings.builder().put(IndexMetaData.SETTING_INDEX_VERSION_CREATED.getKey(), Version.V_6_0_0)) + .addMapping("_doc", "bar", "type=text,fielddata=true", "baz", "type=text,fielddata=true") + .get()); + + ensureGreen(); + + client().prepareIndex("test1", "_doc", Integer.toString(1)).setSource("{\"bar\":\"bar\",\"baz\":\"baz\"}", XContentType.JSON).get(); + client().prepareIndex("test1", "_doc", Integer.toString(2)).setSource("{\"bar\":\"bar\",\"baz\":\"baz\"}", XContentType.JSON).get(); + refresh(); + + client().prepareSearch("_all").addSort("bar", SortOrder.ASC).addSort("baz", SortOrder.ASC).execute().actionGet(); + + final IndicesStatsRequestBuilder builder = client().admin().indices().prepareStats(); + + { + final IndicesStatsResponse stats = builder.execute().actionGet(); + assertThat(stats.getTotal().fieldData.getMemorySizeInBytes(), greaterThan(0L)); + assertThat(stats.getTotal().fieldData.getFields(), is(nullValue())); + } + + { + final IndicesStatsResponse stats = builder.setFieldDataFields("bar").execute().actionGet(); + assertThat(stats.getTotal().fieldData.getMemorySizeInBytes(), greaterThan(0L)); + assertThat(stats.getTotal().fieldData.getFields().containsField("bar"), is(true)); + assertThat(stats.getTotal().fieldData.getFields().get("bar"), greaterThan(0L)); + assertThat(stats.getTotal().fieldData.getFields().containsField("baz"), is(false)); + } + + { + final IndicesStatsResponse stats = builder.setFieldDataFields("bar", "baz").execute().actionGet(); + assertThat(stats.getTotal().fieldData.getMemorySizeInBytes(), greaterThan(0L)); + assertThat(stats.getTotal().fieldData.getFields().containsField("bar"), is(true)); + assertThat(stats.getTotal().fieldData.getFields().get("bar"), greaterThan(0L)); + assertThat(stats.getTotal().fieldData.getFields().containsField("baz"), is(true)); + assertThat(stats.getTotal().fieldData.getFields().get("baz"), greaterThan(0L)); + } + + { + final IndicesStatsResponse stats = builder.setFieldDataFields("*").execute().actionGet(); + assertThat(stats.getTotal().fieldData.getMemorySizeInBytes(), greaterThan(0L)); + assertThat(stats.getTotal().fieldData.getFields().containsField("bar"), is(true)); + assertThat(stats.getTotal().fieldData.getFields().get("bar"), greaterThan(0L)); + assertThat(stats.getTotal().fieldData.getFields().containsField("baz"), is(true)); + assertThat(stats.getTotal().fieldData.getFields().get("baz"), greaterThan(0L)); + } + + { + final IndicesStatsResponse stats = builder.setFieldDataFields("*r").execute().actionGet(); + assertThat(stats.getTotal().fieldData.getMemorySizeInBytes(), greaterThan(0L)); + assertThat(stats.getTotal().fieldData.getFields().containsField("bar"), is(true)); + assertThat(stats.getTotal().fieldData.getFields().get("bar"), greaterThan(0L)); + assertThat(stats.getTotal().fieldData.getFields().containsField("baz"), is(false)); + } + + } + +} diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/GeoDistanceIT.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/GeoDistanceIT.java index ce45d222dd7..ef0651a21c7 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/GeoDistanceIT.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/GeoDistanceIT.java @@ -27,7 +27,6 @@ import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.DistanceUnit; import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.plugins.Plugin; import org.elasticsearch.search.aggregations.Aggregator.SubAggCollectionMode; import org.elasticsearch.search.aggregations.InternalAggregation; import org.elasticsearch.search.aggregations.bucket.histogram.Histogram; @@ -35,13 +34,11 @@ import org.elasticsearch.search.aggregations.bucket.range.Range; import org.elasticsearch.search.aggregations.bucket.range.Range.Bucket; import org.elasticsearch.search.aggregations.bucket.terms.Terms; import org.elasticsearch.test.ESIntegTestCase; -import org.elasticsearch.test.InternalSettingsPlugin; import org.elasticsearch.test.VersionUtils; import org.hamcrest.Matchers; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collection; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -63,12 +60,11 @@ import static org.hamcrest.core.IsNull.nullValue; public class GeoDistanceIT extends ESIntegTestCase { @Override - protected Collection> nodePlugins() { - return Arrays.asList(InternalSettingsPlugin.class); // uses index.version.created + protected boolean forbidPrivateIndexSettings() { + return false; } - private Version version = VersionUtils.randomVersionBetween(random(), Version.V_6_0_0, - Version.CURRENT); + private Version version = VersionUtils.randomVersionBetween(random(), Version.V_6_0_0, Version.CURRENT); private IndexRequestBuilder indexCity(String idx, String name, String... latLons) throws Exception { XContentBuilder source = jsonBuilder().startObject().field("city", name); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/GeoHashGridIT.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/GeoHashGridIT.java index 971742aec2d..2b63a2ca633 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/GeoHashGridIT.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/GeoHashGridIT.java @@ -21,7 +21,6 @@ package org.elasticsearch.search.aggregations.bucket; import com.carrotsearch.hppc.ObjectIntHashMap; import com.carrotsearch.hppc.ObjectIntMap; import com.carrotsearch.hppc.cursors.ObjectIntCursor; - import org.elasticsearch.Version; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchResponse; @@ -30,19 +29,16 @@ import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.query.GeoBoundingBoxQueryBuilder; -import org.elasticsearch.plugins.Plugin; import org.elasticsearch.search.aggregations.AggregationBuilders; import org.elasticsearch.search.aggregations.InternalAggregation; import org.elasticsearch.search.aggregations.bucket.filter.Filter; import org.elasticsearch.search.aggregations.bucket.geogrid.GeoHashGrid; import org.elasticsearch.search.aggregations.bucket.geogrid.GeoHashGrid.Bucket; import org.elasticsearch.test.ESIntegTestCase; -import org.elasticsearch.test.InternalSettingsPlugin; import org.elasticsearch.test.VersionUtils; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collection; import java.util.HashSet; import java.util.List; import java.util.Random; @@ -61,8 +57,8 @@ import static org.hamcrest.Matchers.equalTo; public class GeoHashGridIT extends ESIntegTestCase { @Override - protected Collection> nodePlugins() { - return Arrays.asList(InternalSettingsPlugin.class); // uses index.version.created + protected boolean forbidPrivateIndexSettings() { + return false; } private Version version = VersionUtils.randomVersionBetween(random(), Version.V_6_0_0, diff --git a/server/src/test/java/org/elasticsearch/search/functionscore/DecayFunctionScoreIT.java b/server/src/test/java/org/elasticsearch/search/functionscore/DecayFunctionScoreIT.java index 0a860a636d4..b9b0aa5b201 100644 --- a/server/src/test/java/org/elasticsearch/search/functionscore/DecayFunctionScoreIT.java +++ b/server/src/test/java/org/elasticsearch/search/functionscore/DecayFunctionScoreIT.java @@ -37,18 +37,14 @@ import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder; import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder.FilterFunctionBuilder; import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders; -import org.elasticsearch.plugins.Plugin; import org.elasticsearch.search.MultiValueMode; import org.elasticsearch.search.SearchHits; import org.elasticsearch.test.ESIntegTestCase; -import org.elasticsearch.test.InternalSettingsPlugin; import org.elasticsearch.test.VersionUtils; import java.time.ZoneOffset; import java.time.ZonedDateTime; import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; import java.util.List; import java.util.Locale; @@ -74,9 +70,10 @@ import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.lessThan; public class DecayFunctionScoreIT extends ESIntegTestCase { + @Override - protected Collection> nodePlugins() { - return Arrays.asList(InternalSettingsPlugin.class); // uses index.version.created + protected boolean forbidPrivateIndexSettings() { + return false; } private final QueryBuilder baseQuery = constantScoreQuery(termQuery("test", "value")); diff --git a/server/src/test/java/org/elasticsearch/search/geo/GeoBoundingBoxIT.java b/server/src/test/java/org/elasticsearch/search/geo/GeoBoundingBoxIT.java index 80b40042801..32b80089c20 100644 --- a/server/src/test/java/org/elasticsearch/search/geo/GeoBoundingBoxIT.java +++ b/server/src/test/java/org/elasticsearch/search/geo/GeoBoundingBoxIT.java @@ -26,15 +26,10 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.query.GeoValidationMethod; -import org.elasticsearch.plugins.Plugin; import org.elasticsearch.search.SearchHit; import org.elasticsearch.test.ESIntegTestCase; -import org.elasticsearch.test.InternalSettingsPlugin; import org.elasticsearch.test.VersionUtils; -import java.util.Arrays; -import java.util.Collection; - import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.index.query.QueryBuilders.boolQuery; @@ -45,9 +40,10 @@ import static org.hamcrest.Matchers.anyOf; import static org.hamcrest.Matchers.equalTo; public class GeoBoundingBoxIT extends ESIntegTestCase { + @Override - protected Collection> nodePlugins() { - return Arrays.asList(InternalSettingsPlugin.class); // uses index.version.created + protected boolean forbidPrivateIndexSettings() { + return false; } public void testSimpleBoundingBoxTest() throws Exception { diff --git a/server/src/test/java/org/elasticsearch/search/geo/GeoDistanceIT.java b/server/src/test/java/org/elasticsearch/search/geo/GeoDistanceIT.java index 143fd611c3f..d78e24a05cd 100644 --- a/server/src/test/java/org/elasticsearch/search/geo/GeoDistanceIT.java +++ b/server/src/test/java/org/elasticsearch/search/geo/GeoDistanceIT.java @@ -41,12 +41,10 @@ import org.elasticsearch.search.aggregations.Aggregations; import org.elasticsearch.search.aggregations.bucket.range.InternalGeoDistance; import org.elasticsearch.search.aggregations.bucket.range.Range; import org.elasticsearch.test.ESIntegTestCase; -import org.elasticsearch.test.InternalSettingsPlugin; import org.elasticsearch.test.VersionUtils; import org.junit.Before; import java.io.IOException; -import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashMap; @@ -68,7 +66,7 @@ public class GeoDistanceIT extends ESIntegTestCase { @Override protected Collection> nodePlugins() { - return Arrays.asList(CustomScriptPlugin.class, InternalSettingsPlugin.class); + return Collections.singletonList(CustomScriptPlugin.class); } public static class CustomScriptPlugin extends MockScriptPlugin { @@ -99,6 +97,11 @@ public class GeoDistanceIT extends ESIntegTestCase { } } + @Override + protected boolean forbidPrivateIndexSettings() { + return false; + } + @Before public void setupTestIndex() throws IOException { Version version = VersionUtils.randomVersionBetween(random(), Version.V_6_0_0, diff --git a/server/src/test/java/org/elasticsearch/search/geo/GeoFilterIT.java b/server/src/test/java/org/elasticsearch/search/geo/GeoFilterIT.java index bf02f6eb22e..8d2f7cd6993 100644 --- a/server/src/test/java/org/elasticsearch/search/geo/GeoFilterIT.java +++ b/server/src/test/java/org/elasticsearch/search/geo/GeoFilterIT.java @@ -49,10 +49,8 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.core.internal.io.Streams; import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.plugins.Plugin; import org.elasticsearch.search.SearchHit; import org.elasticsearch.test.ESIntegTestCase; -import org.elasticsearch.test.InternalSettingsPlugin; import org.elasticsearch.test.VersionUtils; import org.junit.BeforeClass; import org.locationtech.spatial4j.context.SpatialContext; @@ -65,8 +63,6 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; import java.util.Random; import java.util.zip.GZIPInputStream; @@ -87,8 +83,8 @@ import static org.hamcrest.Matchers.lessThanOrEqualTo; public class GeoFilterIT extends ESIntegTestCase { @Override - protected Collection> nodePlugins() { - return Arrays.asList(InternalSettingsPlugin.class); // uses index.version.created + protected boolean forbidPrivateIndexSettings() { + return false; } private static boolean intersectSupport; diff --git a/server/src/test/java/org/elasticsearch/search/geo/GeoPolygonIT.java b/server/src/test/java/org/elasticsearch/search/geo/GeoPolygonIT.java index c4ccb219030..741efa6c595 100644 --- a/server/src/test/java/org/elasticsearch/search/geo/GeoPolygonIT.java +++ b/server/src/test/java/org/elasticsearch/search/geo/GeoPolygonIT.java @@ -24,15 +24,11 @@ import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.plugins.Plugin; import org.elasticsearch.search.SearchHit; import org.elasticsearch.test.ESIntegTestCase; -import org.elasticsearch.test.InternalSettingsPlugin; import org.elasticsearch.test.VersionUtils; import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; import java.util.List; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; @@ -47,8 +43,8 @@ import static org.hamcrest.Matchers.equalTo; public class GeoPolygonIT extends ESIntegTestCase { @Override - protected Collection> nodePlugins() { - return Arrays.asList(InternalSettingsPlugin.class); // uses index.version.created + protected boolean forbidPrivateIndexSettings() { + return false; } @Override diff --git a/server/src/test/java/org/elasticsearch/search/query/QueryStringIT.java b/server/src/test/java/org/elasticsearch/search/query/QueryStringIT.java index a90e98a38ee..f2d69fc377d 100644 --- a/server/src/test/java/org/elasticsearch/search/query/QueryStringIT.java +++ b/server/src/test/java/org/elasticsearch/search/query/QueryStringIT.java @@ -31,17 +31,13 @@ import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.query.Operator; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.query.QueryStringQueryBuilder; -import org.elasticsearch.plugins.Plugin; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; import org.elasticsearch.test.ESIntegTestCase; -import org.elasticsearch.test.InternalSettingsPlugin; import org.junit.Before; import java.io.IOException; import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -58,10 +54,6 @@ import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; public class QueryStringIT extends ESIntegTestCase { - @Override - protected Collection> nodePlugins() { - return Arrays.asList(InternalSettingsPlugin.class); // uses index.version.created - } @Before public void setup() throws Exception { diff --git a/server/src/test/java/org/elasticsearch/search/query/SimpleQueryStringIT.java b/server/src/test/java/org/elasticsearch/search/query/SimpleQueryStringIT.java index 5176c327ac7..598f5625588 100644 --- a/server/src/test/java/org/elasticsearch/search/query/SimpleQueryStringIT.java +++ b/server/src/test/java/org/elasticsearch/search/query/SimpleQueryStringIT.java @@ -44,12 +44,11 @@ import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.test.ESIntegTestCase; -import org.elasticsearch.test.InternalSettingsPlugin; import java.io.IOException; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -79,7 +78,7 @@ import static org.hamcrest.Matchers.equalTo; public class SimpleQueryStringIT extends ESIntegTestCase { @Override protected Collection> nodePlugins() { - return Arrays.asList(MockAnalysisPlugin.class, InternalSettingsPlugin.class); // uses index.version.created + return Collections.singletonList(MockAnalysisPlugin.class); } public void testSimpleQueryString() throws ExecutionException, InterruptedException { diff --git a/server/src/test/java/org/elasticsearch/search/sort/GeoDistanceIT.java b/server/src/test/java/org/elasticsearch/search/sort/GeoDistanceIT.java index e134b20c309..6925e80ca27 100644 --- a/server/src/test/java/org/elasticsearch/search/sort/GeoDistanceIT.java +++ b/server/src/test/java/org/elasticsearch/search/sort/GeoDistanceIT.java @@ -29,14 +29,10 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; -import org.elasticsearch.test.InternalSettingsPlugin; import org.elasticsearch.test.VersionUtils; import java.io.IOException; -import java.util.Arrays; -import java.util.Collection; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; @@ -51,11 +47,11 @@ import static org.hamcrest.Matchers.closeTo; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; - public class GeoDistanceIT extends ESIntegTestCase { + @Override - protected Collection> nodePlugins() { - return Arrays.asList(InternalSettingsPlugin.class); + protected boolean forbidPrivateIndexSettings() { + return false; } public void testDistanceSortingMVFields() throws Exception { diff --git a/server/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderIT.java b/server/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderIT.java index cac5fede848..a0a919b5a62 100644 --- a/server/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderIT.java +++ b/server/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderIT.java @@ -28,16 +28,13 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.DistanceUnit; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.query.GeoValidationMethod; -import org.elasticsearch.plugins.Plugin; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.test.ESIntegTestCase; -import org.elasticsearch.test.InternalSettingsPlugin; import org.elasticsearch.test.VersionUtils; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collection; import java.util.List; import java.util.concurrent.ExecutionException; @@ -50,11 +47,12 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSort import static org.hamcrest.Matchers.closeTo; public class GeoDistanceSortBuilderIT extends ESIntegTestCase { + private static final String LOCATION_FIELD = "location"; @Override - protected Collection> nodePlugins() { - return Arrays.asList(InternalSettingsPlugin.class); + protected boolean forbidPrivateIndexSettings() { + return false; } public void testManyToManyGeoPoints() throws ExecutionException, InterruptedException, IOException { diff --git a/test/framework/src/main/java/org/elasticsearch/node/MockNode.java b/test/framework/src/main/java/org/elasticsearch/node/MockNode.java index 79694f8050a..0e7e35e88a9 100644 --- a/test/framework/src/main/java/org/elasticsearch/node/MockNode.java +++ b/test/framework/src/main/java/org/elasticsearch/node/MockNode.java @@ -65,18 +65,36 @@ import java.util.function.Function; * */ public class MockNode extends Node { + private final Collection> classpathPlugins; - public MockNode(Settings settings, Collection> classpathPlugins) { - this(settings, classpathPlugins, null); + public MockNode(final Settings settings, final Collection> classpathPlugins) { + this(settings, classpathPlugins, true); } - public MockNode(Settings settings, Collection> classpathPlugins, Path configPath) { - this(InternalSettingsPreparer.prepareEnvironment(settings, Collections.emptyMap(), configPath), classpathPlugins); + public MockNode( + final Settings settings, + final Collection> classpathPlugins, + final boolean forbidPrivateIndexSettings) { + this(settings, classpathPlugins, null, forbidPrivateIndexSettings); } - public MockNode(Environment environment, Collection> classpathPlugins) { - super(environment, classpathPlugins); + public MockNode( + final Settings settings, + final Collection> classpathPlugins, + final Path configPath, + final boolean forbidPrivateIndexSettings) { + this( + InternalSettingsPreparer.prepareEnvironment(settings, Collections.emptyMap(), configPath), + classpathPlugins, + forbidPrivateIndexSettings); + } + + private MockNode( + final Environment environment, + final Collection> classpathPlugins, + final boolean forbidPrivateIndexSettings) { + super(environment, classpathPlugins, forbidPrivateIndexSettings); this.classpathPlugins = classpathPlugins; } @@ -156,5 +174,6 @@ public class MockNode extends Node { return new MockHttpTransport(); } } + } diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java index 58593cbe2fd..cc390289341 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java @@ -340,7 +340,6 @@ public abstract class AbstractBuilderTestCase extends ESTestCase { clientInvocationHandler); ScriptModule scriptModule = createScriptModule(pluginsService.filterPlugins(ScriptPlugin.class)); List> additionalSettings = pluginsService.getPluginSettings(); - additionalSettings.add(InternalSettingsPlugin.VERSION_CREATED); SettingsModule settingsModule = new SettingsModule(nodeSettings, additionalSettings, pluginsService.getPluginSettingsFilter()); searchModule = new SearchModule(nodeSettings, false, pluginsService.filterPlugins(SearchPlugin.class)); IndicesModule indicesModule = new IndicesModule(pluginsService.filterPlugins(MapperPlugin.class)); diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java index be9e40ab420..68a862c109d 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java @@ -1910,7 +1910,7 @@ public abstract class ESIntegTestCase extends ESTestCase { return new InternalTestCluster(seed, createTempDir(), supportsDedicatedMasters, getAutoMinMasterNodes(), minNumDataNodes, maxNumDataNodes, InternalTestCluster.clusterName(scope.name(), seed) + "-cluster", nodeConfigurationSource, getNumClientNodes(), - nodePrefix, mockPlugins, getClientWrapper()); + nodePrefix, mockPlugins, getClientWrapper(), forbidPrivateIndexSettings()); } protected NodeConfigurationSource getNodeConfigSource() { @@ -2243,8 +2243,6 @@ public abstract class ESIntegTestCase extends ESTestCase { return internalCluster().routingKeyForShard(resolveIndex(index), shard, random()); } - - @Override protected NamedXContentRegistry xContentRegistry() { if (isInternalCluster() && cluster().size() > 0) { @@ -2256,6 +2254,10 @@ public abstract class ESIntegTestCase extends ESTestCase { } } + protected boolean forbidPrivateIndexSettings() { + return true; + } + /** * Returns an instance of {@link RestClient} pointing to the current test cluster. * Creates a new client if the method is invoked for the first time in the context of the current test scope. diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java index 19290f8cf11..d73520f91b3 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java @@ -211,7 +211,7 @@ public abstract class ESSingleNodeTestCase extends ESTestCase { if (addMockHttpTransport()) { plugins.add(MockHttpTransport.TestPlugin.class); } - Node build = new MockNode(settings, plugins); + Node build = new MockNode(settings, plugins, forbidPrivateIndexSettings()); try { build.start(); } catch (NodeValidationException e) { @@ -341,4 +341,9 @@ public abstract class ESSingleNodeTestCase extends ESTestCase { protected NamedXContentRegistry xContentRegistry() { return getInstanceFromNode(NamedXContentRegistry.class); } + + protected boolean forbidPrivateIndexSettings() { + return true; + } + } diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalSettingsPlugin.java b/test/framework/src/main/java/org/elasticsearch/test/InternalSettingsPlugin.java index be8c824f0f7..1f4a35a29c2 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/InternalSettingsPlugin.java +++ b/test/framework/src/main/java/org/elasticsearch/test/InternalSettingsPlugin.java @@ -32,8 +32,6 @@ import java.util.concurrent.TimeUnit; public final class InternalSettingsPlugin extends Plugin { - public static final Setting VERSION_CREATED = - Setting.intSetting("index.version.created", 0, Property.IndexScope, Property.NodeScope); public static final Setting PROVIDED_NAME_SETTING = Setting.simpleString("index.provided_name",Property.IndexScope, Property.NodeScope); public static final Setting MERGE_ENABLED = @@ -47,7 +45,6 @@ public final class InternalSettingsPlugin extends Plugin { @Override public List> getSettings() { return Arrays.asList( - VERSION_CREATED, MERGE_ENABLED, INDEX_CREATION_DATE_SETTING, PROVIDED_NAME_SETTING, diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java index 4c813372fae..3c46acd0fbe 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java +++ b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java @@ -26,7 +26,6 @@ import com.carrotsearch.randomizedtesting.generators.RandomPicks; import com.carrotsearch.randomizedtesting.generators.RandomStrings; import org.apache.logging.log4j.Logger; import org.apache.lucene.store.AlreadyClosedException; -import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.admin.cluster.node.stats.NodeStats; import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags; @@ -64,6 +63,7 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.PageCacheRecycler; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.discovery.DiscoveryModule; import org.elasticsearch.discovery.zen.ElectMasterService; import org.elasticsearch.discovery.zen.ZenDiscovery; @@ -205,6 +205,8 @@ public final class InternalTestCluster extends TestCluster { private final Collection> mockPlugins; + private final boolean forbidPrivateIndexSettings; + /** * All nodes started by the cluster will have their name set to nodePrefix followed by a positive number */ @@ -214,13 +216,53 @@ public final class InternalTestCluster extends TestCluster { private ServiceDisruptionScheme activeDisruptionScheme; private Function clientWrapper; - public InternalTestCluster(long clusterSeed, Path baseDir, - boolean randomlyAddDedicatedMasters, - boolean autoManageMinMasterNodes, int minNumDataNodes, int maxNumDataNodes, String clusterName, NodeConfigurationSource nodeConfigurationSource, int numClientNodes, - String nodePrefix, Collection> mockPlugins, Function clientWrapper) { + public InternalTestCluster( + final long clusterSeed, + final Path baseDir, + final boolean randomlyAddDedicatedMasters, + final boolean autoManageMinMasterNodes, + final int minNumDataNodes, + final int maxNumDataNodes, + final String clusterName, + final NodeConfigurationSource nodeConfigurationSource, + final int numClientNodes, + final String nodePrefix, + final Collection> mockPlugins, + final Function clientWrapper) { + this( + clusterSeed, + baseDir, + randomlyAddDedicatedMasters, + autoManageMinMasterNodes, + minNumDataNodes, + maxNumDataNodes, + clusterName, + nodeConfigurationSource, + numClientNodes, + nodePrefix, + mockPlugins, + clientWrapper, + true); + } + + public InternalTestCluster( + final long clusterSeed, + final Path baseDir, + final boolean randomlyAddDedicatedMasters, + final boolean autoManageMinMasterNodes, + final int minNumDataNodes, + final int maxNumDataNodes, + final String clusterName, + final NodeConfigurationSource nodeConfigurationSource, + final int numClientNodes, + final String nodePrefix, + final Collection> mockPlugins, + final Function clientWrapper, + final boolean forbidPrivateIndexSettings) { super(clusterSeed); this.autoManageMinMasterNodes = autoManageMinMasterNodes; this.clientWrapper = clientWrapper; + this.forbidPrivateIndexSettings = forbidPrivateIndexSettings; this.baseDir = baseDir; this.clusterName = clusterName; if (minNumDataNodes < 0 || maxNumDataNodes < 0) { @@ -583,7 +625,11 @@ public final class InternalTestCluster extends TestCluster { // we clone this here since in the case of a node restart we might need it again secureSettings = ((MockSecureSettings) secureSettings).clone(); } - MockNode node = new MockNode(finalSettings.build(), plugins, nodeConfigurationSource.nodeConfigPath(nodeId)); + MockNode node = new MockNode( + finalSettings.build(), + plugins, + nodeConfigurationSource.nodeConfigPath(nodeId), + forbidPrivateIndexSettings); try { IOUtils.close(secureSettings); } catch (IOException e) {