honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Make default number of shards configurable (#625) 

* Make default number of shards configurable

The default number of primary shards for a new index, when the number of shards are not provided in the request, can be configured for the cluster.

Signed-off-by: Arunabh Singh <arunabs@amazon.com>

* Address PR comments

Signed-off-by: Arunabh Singh <arunabs@amazon.com>

Co-authored-by: Arunabh Singh <arunabs@amazon.com>
This commit is contained in:
arunabh23 2021-05-06 00:53:49 +05:30 committed by GitHub
parent 6d1e6a0720
commit 41e0ff9d6d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 41 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
qa/evil-tests/src/test/java/org/opensearch/cluster/metadata/EvilSystemPropertyTests.java
View File

@ -35,26 +35,47 @@ import org.opensearch.common.SuppressForbidden;
import org.opensearch.common.settings.Settings;
import org.opensearch.test.OpenSearchTestCase;
import static org.opensearch.cluster.metadata.IndexMetadata.DEFAULT_NUMBER_OF_SHARDS;
import static org.opensearch.cluster.metadata.IndexMetadata.MAX_NUMBER_OF_SHARDS;
import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS;
public class EvilSystemPropertyTests extends OpenSearchTestCase {
@SuppressForbidden(reason = "manipulates system properties for testing")
public void testMaxNumShards() {
public void testNumShards() {
IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () ->
IndexMetadata.buildNumberOfShardsSetting()
.get(Settings.builder().put("index.number_of_shards", 1025).build()));
assertEquals("Failed to parse value [1025] for setting [index.number_of_shards] must be <= 1024", exception.getMessage());
.get(Settings.builder().put(SETTING_NUMBER_OF_SHARDS, 1025).build()));
assertEquals("Failed to parse value [1025] for setting [" + SETTING_NUMBER_OF_SHARDS + "] must be <= 1024", exception.getMessage());
Integer numShards = IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.get(Settings.builder().put("index.number_of_shards", 100).build());
Integer numShards = IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.get(Settings.builder().put(SETTING_NUMBER_OF_SHARDS, 100).build());
assertEquals(100, numShards.intValue());
int limit = randomIntBetween(1, 10);
System.setProperty("opensearch.index.max_number_of_shards", Integer.toString(limit));
System.setProperty(MAX_NUMBER_OF_SHARDS, Integer.toString(limit));
try {
IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () ->
IndexMetadata.buildNumberOfShardsSetting()
.get(Settings.builder().put("index.number_of_shards", 11).build()));
assertEquals("Failed to parse value [11] for setting [index.number_of_shards] must be <= " + limit, e.getMessage());
System.clearProperty(MAX_NUMBER_OF_SHARDS);
Integer defaultFromSetting = IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getDefault(Settings.EMPTY);
assertEquals(1, defaultFromSetting.intValue());
int randomDefault = randomIntBetween(1, 10);
System.setProperty(DEFAULT_NUMBER_OF_SHARDS, Integer.toString(randomDefault));
defaultFromSetting = IndexMetadata.buildNumberOfShardsSetting().getDefault(Settings.EMPTY);
assertEquals(randomDefault, defaultFromSetting.intValue());
randomDefault = randomIntBetween(1, 10);
System.setProperty(MAX_NUMBER_OF_SHARDS, Integer.toString(randomDefault));
System.setProperty(DEFAULT_NUMBER_OF_SHARDS, Integer.toString(randomDefault + 1));
e = expectThrows(IllegalArgumentException.class, IndexMetadata::buildNumberOfShardsSetting);
assertEquals(DEFAULT_NUMBER_OF_SHARDS + " value [" + (randomDefault + 1) + "] must between " +
"1 and " + MAX_NUMBER_OF_SHARDS + " [" + randomDefault + "]", e.getMessage());
} finally {
System.clearProperty("opensearch.index.max_number_of_shards");
System.clearProperty(MAX_NUMBER_OF_SHARDS);
System.clearProperty(DEFAULT_NUMBER_OF_SHARDS);
}
}
}

16
server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java
View File

@ -152,15 +152,25 @@ public class IndexMetadata implements Diffable<IndexMetadata>, ToXContentFragmen
* if a cluster should allow to create more than 1024 shards per index. NOTE: this does not limit the number of shards
* per cluster. this also prevents creating stuff like a new index with millions of shards by accident which essentially
* kills the entire cluster with OOM on the spot.*/
final int maxNumShards = Integer.parseInt(System.getProperty("opensearch.index.max_number_of_shards", "1024"));
final int maxNumShards = Integer.parseInt(System.getProperty(MAX_NUMBER_OF_SHARDS, "1024"));
if (maxNumShards < 1) {
throw new IllegalArgumentException("opensearch.index.max_number_of_shards must be > 0");
throw new IllegalArgumentException(MAX_NUMBER_OF_SHARDS + " must be > 0");
}
return Setting.intSetting(SETTING_NUMBER_OF_SHARDS, 1, 1, maxNumShards, Property.IndexScope, Property.Final);
/* This property should be provided if the default number of shards for any new index has to be changed to a value
* other than 1. This value is applicable only if the index.number_of_shards setting is not provided as part of the
* index creation request. */
final int defaultNumShards = Integer.parseInt(System.getProperty(DEFAULT_NUMBER_OF_SHARDS, "1"));
if (defaultNumShards < 1 || defaultNumShards > maxNumShards) {
throw new IllegalArgumentException(DEFAULT_NUMBER_OF_SHARDS + " value [" + defaultNumShards + "] must between " +
"1 and " + MAX_NUMBER_OF_SHARDS + " [" + maxNumShards + "]");
}
return Setting.intSetting(SETTING_NUMBER_OF_SHARDS, defaultNumShards, 1, maxNumShards, Property.IndexScope, Property.Final);
}
public static final String INDEX_SETTING_PREFIX = "index.";
public static final String SETTING_NUMBER_OF_SHARDS = "index.number_of_shards";
static final String DEFAULT_NUMBER_OF_SHARDS = "opensearch.index.default_number_of_shards";
static final String MAX_NUMBER_OF_SHARDS = "opensearch.index.max_number_of_shards";
public static final Setting<Integer> INDEX_NUMBER_OF_SHARDS_SETTING = buildNumberOfShardsSetting();
public static final String SETTING_NUMBER_OF_REPLICAS = "index.number_of_replicas";
public static final Setting<Integer> INDEX_NUMBER_OF_REPLICAS_SETTING =

2
server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java
View File

@ -795,7 +795,7 @@ public class MetadataCreateIndexService {
if (indexVersionCreated.before(LegacyESVersion.V_7_0_0)) {
numberOfShards = 5;
} else {
numberOfShards = 1;
numberOfShards = INDEX_NUMBER_OF_SHARDS_SETTING.getDefault(Settings.EMPTY);
}
return numberOfShards;
}