First pass at validating similarities insite the Settings infrastructure.

core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java

@@ -35,6 +35,7 @@ import org.elasticsearch.index.fielddata.IndexFieldDataService;
 import org.elasticsearch.index.mapper.FieldMapper;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.percolator.PercolatorQueriesRegistry;
+import org.elasticsearch.index.similarity.SimilarityService;
 import org.elasticsearch.index.store.FsDirectoryService;
 import org.elasticsearch.index.store.IndexStore;
 import org.elasticsearch.index.store.Store;
@@ -133,8 +134,15 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
         FsDirectoryService.INDEX_LOCK_FACTOR_SETTING,
         EngineConfig.INDEX_CODEC_SETTING,
         IndexWarmer.INDEX_NORMS_LOADING_SETTING,
-        // this sucks but we can't really validate all the analyzers/similarity in here
-        Setting.groupSetting("index.similarity.", false, Setting.Scope.INDEX), // this allows similarity settings to be passed
+        // validate that built-in similarities don't get redefined
+        Setting.groupSetting("index.similarity.", false, Setting.Scope.INDEX, (s) -> {
+          boolean valid = true;
+          String similarityName = s.substring(0, s.indexOf("."));
+          if(SimilarityService.BUILT_IN.keySet().contains(similarityName)) {
+            throw new IllegalArgumentException("Cannot redefine built-in Similarity [" + similarityName + "]");
+          }
+          return valid;
+        }), // this allows similarity settings to be passed
         Setting.groupSetting("index.analysis.", false, Setting.Scope.INDEX) // this allows analysis settings to be passed
 
     )));

core/src/main/java/org/elasticsearch/common/settings/Setting.java

@@ -40,6 +40,7 @@ import java.util.Objects;
 import java.util.function.BiConsumer;
 import java.util.function.Consumer;
 import java.util.function.Function;
+import java.util.function.Predicate;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
@@ -486,6 +487,10 @@ public class Setting<T> extends ToXContentToBytes {
     }
 
     public static Setting<Settings> groupSetting(String key, boolean dynamic, Scope scope) {
+      return groupSetting(key, dynamic, scope, (s) -> true);
+    }
+
+    public static Setting<Settings> groupSetting(String key, boolean dynamic, Scope scope, Predicate<String> settingsValidator) {
         if (key.endsWith(".") == false) {
             throw new IllegalArgumentException("key must end with a '.'");
         }
@@ -498,7 +503,7 @@ public class Setting<T> extends ToXContentToBytes {
 
             @Override
             public Settings get(Settings settings) {
-                return settings.getByPrefix(key);
+                return settings.getByPrefix(key).filter(settingsValidator);
             }
 
             @Override

core/src/test/java/org/elasticsearch/common/settings/ScopedSettingsTests.java

@@ -213,6 +213,13 @@ public class ScopedSettingsTests extends ESTestCase {
         } catch (IllegalArgumentException e) {
             assertEquals("Failed to parse value [true] for setting [index.number_of_replicas]", e.getMessage());
         }
+
+        try {
+          settings.validate("index.similarity.classic.type", Settings.builder().put("index.similarity.classic.type", "mine").build());
+          fail();
+        } catch (IllegalArgumentException e) {
+          assertEquals("Cannot redefine built-in Similarity [classic]", e.getMessage());
+        }
     }