Make default number of shards configurable (#625)

* Make default number of shards configurable

The default number of primary shards for a new index, when the number of shards are not provided in the request, can be configured for the cluster.

Signed-off-by: Arunabh Singh <arunabs@amazon.com>

* Address PR comments

Signed-off-by: Arunabh Singh <arunabs@amazon.com>

Co-authored-by: Arunabh Singh <arunabs@amazon.com>

qa/evil-tests/src/test/java/org/opensearch/cluster/metadata/EvilSystemPropertyTests.java

@@ -35,26 +35,47 @@ import org.opensearch.common.SuppressForbidden;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.test.OpenSearchTestCase;
 
+import static org.opensearch.cluster.metadata.IndexMetadata.DEFAULT_NUMBER_OF_SHARDS;
+import static org.opensearch.cluster.metadata.IndexMetadata.MAX_NUMBER_OF_SHARDS;
+import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS;
+
 public class EvilSystemPropertyTests extends OpenSearchTestCase {
 
     @SuppressForbidden(reason = "manipulates system properties for testing")
-    public void testMaxNumShards() {
+    public void testNumShards() {
         IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () ->
             IndexMetadata.buildNumberOfShardsSetting()
-                .get(Settings.builder().put("index.number_of_shards", 1025).build()));
+                .get(Settings.builder().put(SETTING_NUMBER_OF_SHARDS, 1025).build()));
-        assertEquals("Failed to parse value [1025] for setting [index.number_of_shards] must be <= 1024", exception.getMessage());
+        assertEquals("Failed to parse value [1025] for setting [" + SETTING_NUMBER_OF_SHARDS + "] must be <= 1024", exception.getMessage());
 
-        Integer numShards = IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.get(Settings.builder().put("index.number_of_shards", 100).build());
+        Integer numShards = IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.get(Settings.builder().put(SETTING_NUMBER_OF_SHARDS, 100).build());
         assertEquals(100, numShards.intValue());
         int limit = randomIntBetween(1, 10);
-        System.setProperty("opensearch.index.max_number_of_shards", Integer.toString(limit));
+        System.setProperty(MAX_NUMBER_OF_SHARDS, Integer.toString(limit));
         try {
             IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () ->
                 IndexMetadata.buildNumberOfShardsSetting()
                     .get(Settings.builder().put("index.number_of_shards", 11).build()));
             assertEquals("Failed to parse value [11] for setting [index.number_of_shards] must be <= " + limit, e.getMessage());
+            System.clearProperty(MAX_NUMBER_OF_SHARDS);
+
+            Integer defaultFromSetting = IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getDefault(Settings.EMPTY);
+            assertEquals(1, defaultFromSetting.intValue());
+
+            int randomDefault = randomIntBetween(1, 10);
+            System.setProperty(DEFAULT_NUMBER_OF_SHARDS, Integer.toString(randomDefault));
+            defaultFromSetting = IndexMetadata.buildNumberOfShardsSetting().getDefault(Settings.EMPTY);
+            assertEquals(randomDefault, defaultFromSetting.intValue());
+
+            randomDefault = randomIntBetween(1, 10);
+            System.setProperty(MAX_NUMBER_OF_SHARDS, Integer.toString(randomDefault));
+            System.setProperty(DEFAULT_NUMBER_OF_SHARDS, Integer.toString(randomDefault + 1));
+            e = expectThrows(IllegalArgumentException.class, IndexMetadata::buildNumberOfShardsSetting);
+            assertEquals(DEFAULT_NUMBER_OF_SHARDS + " value [" + (randomDefault + 1) + "] must between " +
+                "1 and " + MAX_NUMBER_OF_SHARDS + " [" + randomDefault + "]", e.getMessage());
         } finally {
-            System.clearProperty("opensearch.index.max_number_of_shards");
+            System.clearProperty(MAX_NUMBER_OF_SHARDS);
+            System.clearProperty(DEFAULT_NUMBER_OF_SHARDS);
         }
     }
 }

server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java

@@ -152,15 +152,25 @@ public class IndexMetadata implements Diffable<IndexMetadata>, ToXContentFragmen
          * if a cluster should allow to create more than 1024 shards per index. NOTE: this does not limit the number of shards
          * per cluster. this also prevents creating stuff like a new index with millions of shards by accident which essentially
          * kills the entire cluster with OOM on the spot.*/
-        final int maxNumShards = Integer.parseInt(System.getProperty("opensearch.index.max_number_of_shards", "1024"));
+        final int maxNumShards = Integer.parseInt(System.getProperty(MAX_NUMBER_OF_SHARDS, "1024"));
         if (maxNumShards < 1) {
-            throw new IllegalArgumentException("opensearch.index.max_number_of_shards must be > 0");
+            throw new IllegalArgumentException(MAX_NUMBER_OF_SHARDS + " must be > 0");
         }
-        return Setting.intSetting(SETTING_NUMBER_OF_SHARDS, 1, 1, maxNumShards, Property.IndexScope, Property.Final);
+        /* This property should be provided if the default number of shards for any new index has to be changed to a value
+         * other than 1. This value is applicable only if the index.number_of_shards setting is not provided as part of the
+         * index creation request. */
+        final int defaultNumShards = Integer.parseInt(System.getProperty(DEFAULT_NUMBER_OF_SHARDS, "1"));
+        if (defaultNumShards < 1 || defaultNumShards > maxNumShards) {
+            throw new IllegalArgumentException(DEFAULT_NUMBER_OF_SHARDS + " value [" + defaultNumShards + "] must between " +
+                "1 and " + MAX_NUMBER_OF_SHARDS + " [" + maxNumShards + "]");
+        }
+        return Setting.intSetting(SETTING_NUMBER_OF_SHARDS, defaultNumShards, 1, maxNumShards, Property.IndexScope, Property.Final);
     }
 
     public static final String INDEX_SETTING_PREFIX = "index.";
     public static final String SETTING_NUMBER_OF_SHARDS = "index.number_of_shards";
+    static final String DEFAULT_NUMBER_OF_SHARDS = "opensearch.index.default_number_of_shards";
+    static final String MAX_NUMBER_OF_SHARDS = "opensearch.index.max_number_of_shards";
     public static final Setting<Integer> INDEX_NUMBER_OF_SHARDS_SETTING = buildNumberOfShardsSetting();
     public static final String SETTING_NUMBER_OF_REPLICAS = "index.number_of_replicas";
     public static final Setting<Integer> INDEX_NUMBER_OF_REPLICAS_SETTING =

server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java

@@ -795,7 +795,7 @@ public class MetadataCreateIndexService {
         if (indexVersionCreated.before(LegacyESVersion.V_7_0_0)) {
             numberOfShards = 5;
         } else {
-            numberOfShards = 1;
+            numberOfShards = INDEX_NUMBER_OF_SHARDS_SETTING.getDefault(Settings.EMPTY);
         }
         return numberOfShards;
     }