honeymoose/OpenSearch
1
0
Fork 0
mirror of https://github.com/honeymoose/OpenSearch.git synced 2025-02-25 22:36:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

Check shard limit after applying index templates (#44619)

Browse Source 
Today when creating an index and checking cluster shard limits, we check
the number of shards before applying index templates. At this point, we
do not know the actual number of shards that will be used to create the
index. In a case when the defaults are used and a template would
override, we could be grossly underestimating the number of shards that
would be created, and thus incorrectly applying the limits. This commit
addresses this by checking the shard limits after applying index
templates.
This commit is contained in:
Jason Tedor 2019-07-23 00:49:16 -07:00
parent e2c8f8dfa3
commit 6928a315c4
No known key found for this signature in database
GPG Key ID: FA89F05560F16BC5
5 changed files with 95 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java
View File

@ -20,8 +20,8 @@
package org.elasticsearch.cluster.metadata;
import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.message.ParameterizedMessage;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.ResourceAlreadyExistsException;
@ -438,6 +438,13 @@ public class MetaDataCreateIndexService {
indexScopedSettings);
}
final Settings actualIndexSettings = indexSettingsBuilder.build();
/*
* We can not check the shard limit until we have applied templates, otherwise we do not know the actual number of shards
* that will be used to create this index.
*/
checkShardLimit(actualIndexSettings, currentState);
tmpImdBuilder.settings(actualIndexSettings);
if (recoverFromIndex != null) {
@ -593,7 +600,7 @@ public class MetaDataCreateIndexService {
assert Version.CURRENT.major == 7;
final int numberOfShards;
final Version indexVersionCreated =
Version.fromId(Integer.parseInt(indexSettingsBuilder.get(IndexMetaData.SETTING_INDEX_VERSION_CREATED.getKey())));
Version.fromId(Integer.parseInt(indexSettingsBuilder.get(IndexMetaData.SETTING_INDEX_VERSION_CREATED.getKey())));
if (indexVersionCreated.before(Version.V_7_0_0)) {
numberOfShards = 5;
} else {
@ -602,6 +609,10 @@ public class MetaDataCreateIndexService {
return numberOfShards;
}
protected void checkShardLimit(final Settings settings, final ClusterState clusterState) {
MetaDataCreateIndexService.checkShardLimit(settings, clusterState);
}
@Override
public void onFailure(String source, Exception e) {
if (e instanceof ResourceAlreadyExistsException) {
@ -622,9 +633,6 @@ public class MetaDataCreateIndexService {
final boolean forbidPrivateIndexSettings) throws IndexCreationException {
List<String> validationErrors = getIndexSettingsValidationErrors(settings, forbidPrivateIndexSettings);
Optional<String> shardAllocation = checkShardLimit(settings, clusterState);
shardAllocation.ifPresent(validationErrors::add);
if (validationErrors.isEmpty() == false) {
ValidationException validationException = new ValidationException();
validationException.addValidationErrors(validationErrors);
@ -635,15 +643,21 @@ public class MetaDataCreateIndexService {
/**
* Checks whether an index can be created without going over the cluster shard limit.
*
* @param settings The settings of the index to be created.
* @param clusterState The current cluster state.
* @return If present, an error message to be used to reject index creation. If empty, a signal that this operation may be carried out.
* @param settings the settings of the index to be created
* @param clusterState the current cluster state
* @throws ValidationException if creating this index would put the cluster over the cluster shard limit
*/
static Optional<String> checkShardLimit(Settings settings, ClusterState clusterState) {
int shardsToCreate = IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.get(settings)
* (1 + IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING.get(settings));
public static void checkShardLimit(final Settings settings, final ClusterState clusterState) {
final int numberOfShards = IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.get(settings);
final int numberOfReplicas = IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING.get(settings);
final int shardsToCreate = numberOfShards * (1 + numberOfReplicas);
return IndicesService.checkShardLimit(shardsToCreate, clusterState);
final Optional<String> shardLimit = IndicesService.checkShardLimit(shardsToCreate, clusterState);
if (shardLimit.isPresent()) {
final ValidationException e = new ValidationException();
e.addValidationError(shardLimit.get());
throw e;
}
}
List<String> getIndexSettingsValidationErrors(final Settings settings, final boolean forbidPrivateIndexSettings) {

1
server/src/main/java/org/elasticsearch/snapshots/RestoreService.java
View File

@ -283,6 +283,7 @@ public class RestoreService implements ClusterStateApplier {
indexMdBuilder.settings(Settings.builder()
.put(snapshotIndexMetaData.getSettings())
.put(IndexMetaData.SETTING_INDEX_UUID, UUIDs.randomBase64UUID()));
MetaDataCreateIndexService.checkShardLimit(snapshotIndexMetaData.getSettings(), currentState);
if (!request.includeAliases() && !snapshotIndexMetaData.getAliases().isEmpty()) {
// Remove all aliases - they shouldn't be restored
indexMdBuilder.removeAllAliases();

9
server/src/test/java/org/elasticsearch/cluster/metadata/IndexCreationTaskTests.java
View File

@ -429,7 +429,14 @@ public class IndexCreationTaskTests extends ESTestCase {
setupRequest();
final MetaDataCreateIndexService.IndexCreationTask task = new MetaDataCreateIndexService.IndexCreationTask(
logger, allocationService, request, listener, indicesService, aliasValidator, xContentRegistry, clusterStateSettings.build(),
validator, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS);
validator, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS) {
@Override
protected void checkShardLimit(final Settings settings, final ClusterState clusterState) {
// we have to make this a no-op since we are not mocking enough for this method to be able to execute
}
};
return task.execute(state);
}

21
server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java
View File

@ -37,7 +37,7 @@ import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
import org.elasticsearch.cluster.routing.allocation.decider.MaxRetryAllocationDecider;
import org.elasticsearch.cluster.shards.ClusterShardLimitIT;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.logging.DeprecationLogger;
import org.elasticsearch.common.ValidationException;
import org.elasticsearch.common.settings.IndexScopedSettings;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Settings;
@ -53,7 +53,7 @@ import java.util.Collections;
import java.util.Comparator;
import java.util.HashSet;
import java.util.List;
import java.util.Optional;
import java.util.Locale;
import java.util.Set;
import java.util.function.Consumer;
import java.util.stream.Collectors;
@ -65,8 +65,10 @@ import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF
import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_VERSION_CREATED;
import static org.elasticsearch.cluster.shards.ClusterShardLimitIT.ShardCounts.forDataNodeCount;
import static org.elasticsearch.indices.IndicesServiceTests.createClusterForShardLimitTest;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.endsWith;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.hasToString;
public class MetaDataCreateIndexServiceTests extends ESTestCase {
@ -486,14 +488,19 @@ public class MetaDataCreateIndexServiceTests extends ESTestCase {
.put(SETTING_NUMBER_OF_REPLICAS, counts.getFailingIndexReplicas())
.build();
DeprecationLogger deprecationLogger = new DeprecationLogger(logger);
Optional<String> errorMessage = MetaDataCreateIndexService.checkShardLimit(indexSettings, state);
final ValidationException e = expectThrows(
ValidationException.class,
() -> MetaDataCreateIndexService.checkShardLimit(indexSettings, state));
int totalShards = counts.getFailingIndexShards() * (1 + counts.getFailingIndexReplicas());
int currentShards = counts.getFirstIndexShards() * (1 + counts.getFirstIndexReplicas());
int maxShards = counts.getShardsPerNode() * nodesInCluster;
assertTrue(errorMessage.isPresent());
assertEquals("this action would add [" + totalShards + "] total shards, but this cluster currently has [" + currentShards
+ "]/[" + maxShards + "] maximum shards open", errorMessage.get());
final String expectedMessage = String.format(
Locale.ROOT,
"this action would add [%d] total shards, but this cluster currently has [%d]/[%d] maximum shards open",
totalShards,
currentShards,
maxShards);
assertThat(e, hasToString(containsString(expectedMessage)));
}
}

46
server/src/test/java/org/elasticsearch/cluster/shards/ClusterShardLimitIT.java
View File

@ -37,6 +37,7 @@ import org.elasticsearch.snapshots.SnapshotInfo;
import org.elasticsearch.snapshots.SnapshotState;
import org.elasticsearch.test.ESIntegTestCase;
import java.util.Collections;
import java.util.List;
import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS;
@ -101,6 +102,51 @@ public class ClusterShardLimitIT extends ESIntegTestCase {
assertFalse(clusterState.getMetaData().hasIndex("should-fail"));
}
public void testIndexCreationOverLimitFromTemplate() {
int dataNodes = client().admin().cluster().prepareState().get().getState().getNodes().getDataNodes().size();
final ShardCounts counts;
{
final ShardCounts temporaryCounts = ShardCounts.forDataNodeCount(dataNodes);
/*
* We are going to create an index that will bring us up to one below the limit; we go one below the limit to ensure the
* template is used instead of one shard.
*/
counts = new ShardCounts(
temporaryCounts.shardsPerNode,
temporaryCounts.firstIndexShards - 1,
temporaryCounts.firstIndexReplicas,
temporaryCounts.failingIndexShards + 1,
temporaryCounts.failingIndexReplicas);
}
setShardsPerNode(counts.getShardsPerNode());
if (counts.firstIndexShards > 0) {
createIndex(
"test",
Settings.builder()
.put(indexSettings())
.put(SETTING_NUMBER_OF_SHARDS, counts.getFirstIndexShards())
.put(SETTING_NUMBER_OF_REPLICAS, counts.getFirstIndexReplicas()).build());
}
assertAcked(client().admin()
.indices()
.preparePutTemplate("should-fail*")
.setPatterns(Collections.singletonList("should-fail"))
.setOrder(1)
.setSettings(Settings.builder()
.put(SETTING_NUMBER_OF_SHARDS, counts.getFailingIndexShards())
.put(SETTING_NUMBER_OF_REPLICAS, counts.getFailingIndexReplicas()))
.get());
final IllegalArgumentException e =
expectThrows(IllegalArgumentException.class, () -> client().admin().indices().prepareCreate("should-fail").get());
verifyException(dataNodes, counts, e);
ClusterState clusterState = client().admin().cluster().prepareState().get().getState();
assertFalse(clusterState.getMetaData().hasIndex("should-fail"));
}
public void testIncreaseReplicasOverLimit() {
int dataNodes = client().admin().cluster().prepareState().get().getState().getNodes().getDataNodes().size();