honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[CCR] Adjust list of dynamic index settings that should be replicated (#35195) 

Adjust list of dynamic index settings that should be replicated
and added a test that verifies whether builtin dynamic index settings
are classified as replicated or non replicated (whitelisted).
This commit is contained in:
Martijn van Groningen 2018-11-08 03:59:58 +01:00 committed by Jason Tedor
parent 4f4fc3b8f8
commit 2f2090f562
2 changed files with 90 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

62
x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowAction.java
View File

@ -15,8 +15,10 @@ import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.block.ClusterBlockException;
import org.elasticsearch.cluster.block.ClusterBlockLevel;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.routing.UnassignedInfo;
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider;
import org.elasticsearch.cluster.routing.allocation.decider.MaxRetryAllocationDecider;
import org.elasticsearch.cluster.routing.allocation.decider.ShardsLimitAllocationDecider;
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.inject.Inject;
@ -28,8 +30,11 @@ import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.index.IndexNotFoundException;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.IndexingSlowLog;
import org.elasticsearch.index.MergePolicyConfig;
import org.elasticsearch.index.MergeSchedulerConfig;
import org.elasticsearch.index.SearchSlowLog;
import org.elasticsearch.index.cache.bitset.BitsetFilterCache;
import org.elasticsearch.index.engine.EngineConfig;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.indices.IndicesRequestCache;
@ -335,21 +340,37 @@ public class TransportResumeFollowAction extends TransportMasterNodeAction<Resum
return historyUUIDs.split(",");
}
private static final Set<Setting<?>> WHITE_LISTED_SETTINGS;
/**
* These are settings that are not replicated to the follower index and
* therefor these settings are not validated whether they have the same
* value between leader and follower index.
*
* These dynamic settings don't affect how documents are indexed (affect index time text analysis) and / or
* are inconvenient if they were replicated (e.g. changing number of replicas).
*/
static final Set<Setting<?>> WHITE_LISTED_SETTINGS;
static {
final Set<Setting<?>> whiteListedSettings = new HashSet<>();
whiteListedSettings.add(IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING);
whiteListedSettings.add(IndexMetaData.INDEX_AUTO_EXPAND_REPLICAS_SETTING);
whiteListedSettings.add(IndexMetaData.INDEX_ROUTING_EXCLUDE_GROUP_SETTING);
whiteListedSettings.add(IndexMetaData.INDEX_ROUTING_INCLUDE_GROUP_SETTING);
whiteListedSettings.add(IndexMetaData.INDEX_ROUTING_REQUIRE_GROUP_SETTING);
whiteListedSettings.add(IndexMetaData.INDEX_READ_ONLY_SETTING);
whiteListedSettings.add(IndexMetaData.INDEX_BLOCKS_READ_SETTING);
whiteListedSettings.add(IndexMetaData.INDEX_BLOCKS_WRITE_SETTING);
whiteListedSettings.add(IndexMetaData.INDEX_BLOCKS_METADATA_SETTING);
whiteListedSettings.add(IndexMetaData.INDEX_BLOCKS_READ_ONLY_ALLOW_DELETE_SETTING);
whiteListedSettings.add(IndexMetaData.INDEX_PRIORITY_SETTING);
whiteListedSettings.add(IndexMetaData.SETTING_WAIT_FOR_ACTIVE_SHARDS);
whiteListedSettings.add(EnableAllocationDecider.INDEX_ROUTING_REBALANCE_ENABLE_SETTING);
whiteListedSettings.add(EnableAllocationDecider.INDEX_ROUTING_ALLOCATION_ENABLE_SETTING);
whiteListedSettings.add(ShardsLimitAllocationDecider.INDEX_TOTAL_SHARDS_PER_NODE_SETTING);
whiteListedSettings.add(MaxRetryAllocationDecider.SETTING_ALLOCATION_MAX_RETRY);
whiteListedSettings.add(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING);
whiteListedSettings.add(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING);
whiteListedSettings.add(IndexSettings.MAX_RESULT_WINDOW_SETTING);
whiteListedSettings.add(IndexSettings.INDEX_WARMER_ENABLED_SETTING);
whiteListedSettings.add(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING);
@ -361,6 +382,26 @@ public class TransportResumeFollowAction extends TransportMasterNodeAction<Resum
whiteListedSettings.add(IndexSettings.QUERY_STRING_ALLOW_LEADING_WILDCARD);
whiteListedSettings.add(IndexSettings.ALLOW_UNMAPPED);
whiteListedSettings.add(IndexSettings.INDEX_SEARCH_IDLE_AFTER);
whiteListedSettings.add(IndexSettings.INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING);
whiteListedSettings.add(IndexSettings.MAX_SCRIPT_FIELDS_SETTING);
whiteListedSettings.add(IndexSettings.MAX_REGEX_LENGTH_SETTING);
whiteListedSettings.add(IndexSettings.MAX_TERMS_COUNT_SETTING);
whiteListedSettings.add(IndexSettings.MAX_ANALYZED_OFFSET_SETTING);
whiteListedSettings.add(IndexSettings.MAX_DOCVALUE_FIELDS_SEARCH_SETTING);
whiteListedSettings.add(IndexSettings.MAX_TOKEN_COUNT_SETTING);
whiteListedSettings.add(IndexSettings.MAX_SLICES_PER_SCROLL);
whiteListedSettings.add(IndexSettings.MAX_ADJACENCY_MATRIX_FILTERS_SETTING);
whiteListedSettings.add(IndexSettings.DEFAULT_PIPELINE);
whiteListedSettings.add(IndexSettings.INDEX_SEARCH_THROTTLED);
whiteListedSettings.add(IndexSettings.INDEX_TRANSLOG_RETENTION_AGE_SETTING);
whiteListedSettings.add(IndexSettings.INDEX_TRANSLOG_RETENTION_SIZE_SETTING);
whiteListedSettings.add(IndexSettings.INDEX_TRANSLOG_GENERATION_THRESHOLD_SIZE_SETTING);
whiteListedSettings.add(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING);
whiteListedSettings.add(IndexSettings.INDEX_TRANSLOG_DURABILITY_SETTING);
whiteListedSettings.add(IndexSettings.INDEX_GC_DELETES_SETTING);
whiteListedSettings.add(IndexSettings.MAX_REFRESH_LISTENERS_PER_SHARD);
whiteListedSettings.add(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING);
whiteListedSettings.add(BitsetFilterCache.INDEX_LOAD_RANDOM_ACCESS_FILTERS_EAGERLY_SETTING);
whiteListedSettings.add(SearchSlowLog.INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_DEBUG_SETTING);
@ -380,7 +421,20 @@ public class TransportResumeFollowAction extends TransportMasterNodeAction<Resum
whiteListedSettings.add(IndexingSlowLog.INDEX_INDEXING_SLOWLOG_REFORMAT_SETTING);
whiteListedSettings.add(IndexingSlowLog.INDEX_INDEXING_SLOWLOG_MAX_SOURCE_CHARS_TO_LOG_SETTING);
whiteListedSettings.add(IndexSettings.INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING);
whiteListedSettings.add(MergePolicyConfig.INDEX_COMPOUND_FORMAT_SETTING);
whiteListedSettings.add(MergePolicyConfig.INDEX_MERGE_POLICY_MAX_MERGE_AT_ONCE_SETTING);
whiteListedSettings.add(MergePolicyConfig.INDEX_MERGE_POLICY_SEGMENTS_PER_TIER_SETTING);
whiteListedSettings.add(MergePolicyConfig.INDEX_MERGE_POLICY_DELETES_PCT_ALLOWED_SETTING);
whiteListedSettings.add(MergePolicyConfig.INDEX_MERGE_POLICY_EXPUNGE_DELETES_ALLOWED_SETTING);
whiteListedSettings.add(MergePolicyConfig.INDEX_MERGE_POLICY_FLOOR_SEGMENT_SETTING);
whiteListedSettings.add(MergePolicyConfig.INDEX_MERGE_POLICY_MAX_MERGE_AT_ONCE_EXPLICIT_SETTING);
whiteListedSettings.add(MergePolicyConfig.INDEX_MERGE_POLICY_MAX_MERGED_SEGMENT_SETTING);
whiteListedSettings.add(MergePolicyConfig.INDEX_MERGE_POLICY_RECLAIM_DELETES_WEIGHT_SETTING);
whiteListedSettings.add(MergeSchedulerConfig.AUTO_THROTTLE_SETTING);
whiteListedSettings.add(MergeSchedulerConfig.MAX_MERGE_COUNT_SETTING);
whiteListedSettings.add(MergeSchedulerConfig.MAX_THREAD_COUNT_SETTING);
whiteListedSettings.add(EngineConfig.INDEX_CODEC_SETTING);
WHITE_LISTED_SETTINGS = Collections.unmodifiableSet(whiteListedSettings);
}

32
x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java
View File

@ -9,9 +9,12 @@ package org.elasticsearch.xpack.ccr.action;
import org.elasticsearch.Version;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.metadata.IndexMetaData.State;
import org.elasticsearch.common.settings.IndexScopedSettings;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.MapperTestUtils;
import org.elasticsearch.index.engine.EngineConfig;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.xpack.ccr.Ccr;
@ -21,11 +24,14 @@ import org.elasticsearch.xpack.core.ccr.action.ResumeFollowAction;
import java.io.IOException;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import static java.util.Collections.singletonMap;
import static org.elasticsearch.xpack.ccr.action.TransportResumeFollowAction.validate;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.is;
public class TransportResumeFollowActionTests extends ESTestCase {
@ -203,6 +209,32 @@ public class TransportResumeFollowActionTests extends ESTestCase {
}
}
public void testDynamicIndexSettingsAreClassified() {
// We should be conscious which dynamic settings are replicated from leader to follower index.
// This is the list of settings that should be replicated:
Set<Setting<?>> replicatedSettings = new HashSet<>();
// These fields need to be replicated otherwise documents that can be indexed in the leader index cannot
// be indexed in the follower index:
replicatedSettings.add(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING);
replicatedSettings.add(MapperService.INDEX_MAPPING_NESTED_DOCS_LIMIT_SETTING);
replicatedSettings.add(MapperService.INDEX_MAPPING_NESTED_FIELDS_LIMIT_SETTING);
replicatedSettings.add(MapperService.INDEX_MAPPING_DEPTH_LIMIT_SETTING);
replicatedSettings.add(MapperService.INDEX_MAPPER_DYNAMIC_SETTING);
replicatedSettings.add(IndexSettings.MAX_NGRAM_DIFF_SETTING);
replicatedSettings.add(IndexSettings.MAX_SHINGLE_DIFF_SETTING);
replicatedSettings.add(EngineConfig.INDEX_OPTIMIZE_AUTO_GENERATED_IDS);
for (Setting<?> setting : IndexScopedSettings.BUILT_IN_INDEX_SETTINGS) {
if (setting.isDynamic()) {
boolean notReplicated = TransportResumeFollowAction.WHITE_LISTED_SETTINGS.contains(setting);
boolean replicated = replicatedSettings.contains(setting);
assertThat("setting [" + setting.getKey() + "] is not classified as replicated xor not replicated",
notReplicated ^ replicated, is(true));
}
}
}
private static IndexMetaData createIMD(String index,
int numberOfShards,
Settings settings,