Snapshot/Restore: separate repository registration

Separate repository registration to make sure that failure in registering one repository doesn't cause failures to register other repositories.

Closes #10351

src/main/java/org/elasticsearch/repositories/RepositoriesService.java

@@ -286,10 +286,19 @@ public class RepositoriesService extends AbstractComponent implements ClusterSta
                             // Previous version is different from the version in settings
                             logger.debug("updating repository [{}]", repositoryMetaData.name());
                             closeRepository(repositoryMetaData.name(), holder);
-                            holder = createRepositoryHolder(repositoryMetaData);
+                            holder = null;
+                            try {
+                                holder = createRepositoryHolder(repositoryMetaData);
+                            } catch (RepositoryException ex) {
+                                logger.warn("failed to change repository [{}]", ex, repositoryMetaData.name());
+                            }
                         }
                     } else {
-                        holder = createRepositoryHolder(repositoryMetaData);
+                        try {
+                            holder = createRepositoryHolder(repositoryMetaData);
+                        } catch (RepositoryException ex) {
+                            logger.warn("failed to create repository [{}]", ex, repositoryMetaData.name());
+                        }
                     }
                     if (holder != null) {
                         logger.debug("registering repository [{}]", repositoryMetaData.name());

src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreTests.java

@@ -55,6 +55,7 @@ import org.elasticsearch.index.store.support.AbstractIndexStore;
 import org.elasticsearch.indices.ttl.IndicesTTLService;
 import org.elasticsearch.repositories.RepositoryMissingException;
 import org.elasticsearch.snapshots.mockstore.MockRepositoryModule;
+import org.elasticsearch.snapshots.mockstore.MockRepositoryPlugin;
 import org.elasticsearch.test.InternalTestCluster;
 import org.junit.Ignore;
 import org.junit.Test;
@@ -575,6 +576,27 @@ public class DedicatedClusterSnapshotRestoreTests extends AbstractSnapshotTests
         assertThat(reusedShards.size(), greaterThanOrEqualTo(numberOfShards / 2));
     }
 
+
+    @Test
+    public void registrationFailureTest() {
+        logger.info("--> start first node");
+        internalCluster().startNode(settingsBuilder().put("plugin.types", MockRepositoryPlugin.class.getName()));
+        logger.info("--> start second node");
+        // Make sure the first node is elected as master
+        internalCluster().startNode(settingsBuilder().put("node.master", false));
+        // Register mock repositories
+        for (int i = 0; i < 5; i++) {
+            client().admin().cluster().preparePutRepository("test-repo" + i)
+                    .setType("mock").setSettings(ImmutableSettings.settingsBuilder()
+                    .put("location", newTempDir(LifecycleScope.SUITE))).setVerify(false).get();
+        }
+        logger.info("--> make sure that properly setup repository can be registered on all nodes");
+        client().admin().cluster().preparePutRepository("test-repo-0")
+                .setType("fs").setSettings(ImmutableSettings.settingsBuilder()
+                .put("location", newTempDir(LifecycleScope.SUITE))).get();
+
+    }
+
     @Test
     @Ignore
     public void chaosSnapshotTest() throws Exception {

src/test/java/org/elasticsearch/snapshots/mockstore/MockRepositoryPlugin.java

@@ -0,0 +1,40 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.snapshots.mockstore;
+
+import org.elasticsearch.plugins.AbstractPlugin;
+import org.elasticsearch.repositories.RepositoriesModule;
+
+public class MockRepositoryPlugin extends AbstractPlugin {
+
+    @Override
+    public String name() {
+        return "mock-repository";
+    }
+
+    @Override
+    public String description() {
+        return "Mock Repository";
+    }
+
+    public void onModule(RepositoriesModule repositoriesModule) {
+        repositoriesModule.registerRepository("mock", MockRepositoryModule.class);
+    }
+}