SOLR-4710: You cannot delete a collection fully from ZooKeeper unless all nodes are up and functioning correctly.

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1468247 13f79535-47bb-0310-9956-ffa450edef68

solr/CHANGES.txt

@@ -185,6 +185,8 @@ Bug Fixes
 
 * SOLR-4702: Fix example /browse "Did you mean?" suggestion feature. (ehatcher, Mark Bennett)
 
+* SOLR-4710: You cannot delete a collection fully from ZooKeeper unless all nodes are up and 
+  functioning correctly. (Mark Miller)
 
 Optimizations
 ----------------------

solr/core/src/java/org/apache/solr/cloud/Overseer.java

@@ -28,7 +28,6 @@ import java.util.Map.Entry;
 import java.util.Set;
 
 import org.apache.solr.common.SolrException;
-import org.apache.solr.common.SolrException.ErrorCode;
 import org.apache.solr.common.cloud.ClosableThread;
 import org.apache.solr.common.cloud.ClusterState;
 import org.apache.solr.common.cloud.DocCollection;
@@ -51,7 +50,8 @@ import org.slf4j.LoggerFactory;
  */
 public class Overseer {
   public static final String QUEUE_OPERATION = "operation";
-
+  public static final String REMOVECOLLECTION = "removecollection";
+  
   private static final int STATE_UPDATE_DELAY = 1500;  // delay between cloud state updates
 
   private static Logger log = LoggerFactory.getLogger(Overseer.class);
@@ -178,6 +178,8 @@ public class Overseer {
         clusterState = updateState(clusterState, message);
       } else if (DELETECORE.equals(operation)) {
         clusterState = removeCore(clusterState, message);
+      } else if (REMOVECOLLECTION.equals(operation)) {
+        clusterState = removeCollection(clusterState, message);
       } else if (ZkStateReader.LEADER_PROP.equals(operation)) {
 
         StringBuilder sb = new StringBuilder();
@@ -504,6 +506,20 @@ public class Overseer {
         return new ClusterState(state.getLiveNodes(), newCollections);
       }
 
+      /*
+       * Remove collection from cloudstate
+       */
+      private ClusterState removeCollection(final ClusterState clusterState, ZkNodeProps message) {
+
+        final String collection = message.getStr("name");
+
+        final Map<String, DocCollection> newCollections = new LinkedHashMap<String,DocCollection>(clusterState.getCollectionStates()); // shallow copy
+        newCollections.remove(collection);
+
+        ClusterState newState = new ClusterState(clusterState.getLiveNodes(), newCollections);
+        return newState;
+      }
+      
       /*
        * Remove core from cloudstate
        */

solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java

@@ -168,6 +168,26 @@ public class OverseerCollectionProcessor implements Runnable, ClosableThread {
         params.set(CoreAdminParams.ACTION, CoreAdminAction.UNLOAD.toString());
         params.set(CoreAdminParams.DELETE_INSTANCE_DIR, true);
         collectionCmd(zkStateReader.getClusterState(), message, params, results, null);
+        
+        ZkNodeProps m = new ZkNodeProps(Overseer.QUEUE_OPERATION,
+            Overseer.REMOVECOLLECTION, "name", message.getStr("name"));
+          Overseer.getInQueue(zkStateReader.getZkClient()).offer(ZkStateReader.toJSON(m));
+          
+        // wait for a while until we don't see the collection
+        long now = System.currentTimeMillis();
+        long timeout = now + 30000;
+        boolean removed = false;
+        while (System.currentTimeMillis() < timeout) {
+          Thread.sleep(100);
+          removed = !zkStateReader.getClusterState().getCollections().contains(message.getStr("name"));
+          if (removed) {
+            Thread.sleep(100); // just a bit of time so it's more likely other readers see on return
+            break;
+          }
+        }
+        if (!removed) {
+          throw new SolrException(ErrorCode.SERVER_ERROR, "Could not fully remove collection: " + message.getStr("name"));
+        }
       } else if (RELOADCOLLECTION.equals(operation)) {
         ModifiableSolrParams params = new ModifiableSolrParams();
         params.set(CoreAdminParams.ACTION, CoreAdminAction.RELOAD.toString());

solr/core/src/test/org/apache/solr/cloud/CollectionsAPIDistributedZkTest.java

@@ -141,12 +141,11 @@ public class CollectionsAPIDistributedZkTest extends AbstractFullDistribZkTestBa
   
   @Override
   public void doTest() throws Exception {
-    
     testNodesUsedByCreate();
     testCollectionsAPI();
-    deletePartiallyCreatedCollection();
     testErrorHandling();
-
+    deletePartiallyCreatedCollection();
+    deleteCollectionWithDownNodes();
     if (DEBUG) {
       super.printLayout();
     }
@@ -182,7 +181,34 @@ public class CollectionsAPIDistributedZkTest extends AbstractFullDistribZkTestBa
     request = new QueryRequest(params);
     request.setPath("/admin/collections");
     resp = createNewSolrServer("", baseUrl).request(request);
-
+  }
+  
+  
+  private void deleteCollectionWithDownNodes() throws Exception {
+    String baseUrl = getBaseUrl((HttpSolrServer) clients.get(0));
+    // now try to remove a collection when a couple of it's nodes are down
+    createCollection(null, "halfdeletedcollection2", 3, 2, 6,
+        createNewSolrServer("", baseUrl), null);
+    
+    // stop a couple nodes
+    ChaosMonkey.stop(jettys.get(0));
+    ChaosMonkey.stop(jettys.get(1));
+    
+    baseUrl = getBaseUrl((HttpSolrServer) clients.get(2));
+    
+    // remove a collection
+    ModifiableSolrParams params = new ModifiableSolrParams();
+    params.set("action", CollectionAction.DELETE.toString());
+    params.set("name", "halfdeletedcollection2");
+    QueryRequest request = new QueryRequest(params);
+    request.setPath("/admin/collections");
+    
+    createNewSolrServer("", baseUrl).request(request);
+    
+    cloudClient.getZkStateReader().updateClusterState(true);
+    assertFalse(cloudClient.getZkStateReader().getClusterState()
+        .getCollections().contains("halfdeletedcollection2"));
+    
   }
 
   private void testErrorHandling() throws Exception {