HDDS-369. Remove the containers of a dead node from the container state map. Contributed by Elek, Marton

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DeadNodeHandler.java

@@ -6,9 +6,9 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- *
+ * <p>
- *     http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -18,11 +18,19 @@
 
 package org.apache.hadoop.hdds.scm.node;
 
+import java.util.Set;
+
 import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerStateManager;
+import org.apache.hadoop.hdds.scm.exceptions.SCMException;
 import org.apache.hadoop.hdds.scm.node.states.Node2ContainerMap;
 import org.apache.hadoop.hdds.server.events.EventHandler;
 import org.apache.hadoop.hdds.server.events.EventPublisher;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 /**
  * Handles Dead Node event.
  */
@@ -30,13 +38,34 @@ public class DeadNodeHandler implements EventHandler<DatanodeDetails> {
 
   private final Node2ContainerMap node2ContainerMap;
 
-  public DeadNodeHandler(Node2ContainerMap node2ContainerMap) {
+  private final ContainerStateManager containerStateManager;
+
+  private static final Logger LOG =
+      LoggerFactory.getLogger(DeadNodeHandler.class);
+
+  public DeadNodeHandler(
+      Node2ContainerMap node2ContainerMap,
+      ContainerStateManager containerStateManager) {
     this.node2ContainerMap = node2ContainerMap;
+    this.containerStateManager = containerStateManager;
   }
 
   @Override
   public void onMessage(DatanodeDetails datanodeDetails,
-                        EventPublisher publisher) {
+      EventPublisher publisher) {
-    //TODO: add logic to handle dead node.
+    Set<ContainerID> containers =
+        node2ContainerMap.getContainers(datanodeDetails.getUuid());
+    LOG.info(
+        "Datanode {}  is dead. Removing replications from the in-memory state.",
+        datanodeDetails.getUuid());
+    for (ContainerID container : containers) {
+      try {
+        containerStateManager.removeContainerReplica(container,
+            datanodeDetails);
+      } catch (SCMException e) {
+        LOG.error("Can't remove container from containerStateMap {}", container
+            .getId(), e);
+      }
+    }
   }
 }

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/states/Node2ContainerMap.java

@@ -18,10 +18,6 @@
 
 package org.apache.hadoop.hdds.scm.node.states;
 
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.hdds.scm.container.ContainerID;
-import org.apache.hadoop.hdds.scm.exceptions.SCMException;
-
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.Map;
@@ -30,8 +26,15 @@ import java.util.TreeSet;
 import java.util.UUID;
 import java.util.concurrent.ConcurrentHashMap;
 
-import static org.apache.hadoop.hdds.scm.exceptions.SCMException.ResultCodes.DUPLICATE_DATANODE;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
-import static org.apache.hadoop.hdds.scm.exceptions.SCMException.ResultCodes.NO_SUCH_DATANODE;
+import org.apache.hadoop.hdds.scm.exceptions.SCMException;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import static org.apache.hadoop.hdds.scm.exceptions.SCMException.ResultCodes
+    .DUPLICATE_DATANODE;
+import static org.apache.hadoop.hdds.scm.exceptions.SCMException.ResultCodes
+    .NO_SUCH_DATANODE;
 
 /**
  * This data structure maintains the list of containers that is on a datanode.
@@ -62,7 +65,7 @@ public class Node2ContainerMap {
   /**
    * Insert a new datanode into Node2Container Map.
    *
-   * @param datanodeID -- Datanode UUID
+   * @param datanodeID   -- Datanode UUID
    * @param containerIDs - List of ContainerIDs.
    */
   public void insertNewDatanode(UUID datanodeID, Set<ContainerID> containerIDs)
@@ -72,7 +75,7 @@ public class Node2ContainerMap {
     if (dn2ContainerMap.putIfAbsent(datanodeID, new HashSet<>(containerIDs))
         != null) {
       throw new SCMException("Node already exists in the map",
-                  DUPLICATE_DATANODE);
+          DUPLICATE_DATANODE);
     }
   }
 
@@ -97,6 +100,7 @@ public class Node2ContainerMap {
 
   /**
    * Removes datanode Entry from the map.
+   *
    * @param datanodeID - Datanode ID.
    */
   public void removeDatanode(UUID datanodeID) {
@@ -170,10 +174,6 @@ public class Node2ContainerMap {
         .build();
   }
 
-
-
-
-
   /**
    * Results possible from processing a container report by
    * Node2ContainerMapper.
@@ -185,4 +185,9 @@ public class Node2ContainerMap {
     MISSING_AND_NEW_CONTAINERS_FOUND,
     NEW_DATANODE_FOUND
   }
+
+  @VisibleForTesting
+  public int size() {
+    return dn2ContainerMap.size();
+  }
 }

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java

@@ -218,7 +218,8 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
 
     NewNodeHandler newNodeHandler = new NewNodeHandler(node2ContainerMap);
     StaleNodeHandler staleNodeHandler = new StaleNodeHandler(node2ContainerMap);
-    DeadNodeHandler deadNodeHandler = new DeadNodeHandler(node2ContainerMap);
+    DeadNodeHandler deadNodeHandler = new DeadNodeHandler(node2ContainerMap,
+        getScmContainerManager().getStateManager());
     ContainerActionsHandler actionsHandler = new ContainerActionsHandler();
     PendingDeleteHandler pendingDeleteHandler =
         new PendingDeleteHandler(scmBlockManager.getSCMBlockDeletingService());

hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDeadNodeHandler.java

@@ -0,0 +1,112 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.scm.node;
+
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto
+    .StorageContainerDatanodeProtocolProtos.ContainerInfo;
+import org.apache.hadoop.hdds.scm.TestUtils;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerStateManager;
+import org.apache.hadoop.hdds.scm.container.Mapping;
+import org.apache.hadoop.hdds.scm.exceptions.SCMException;
+import org.apache.hadoop.hdds.scm.node.states.Node2ContainerMap;
+import org.apache.hadoop.hdds.server.events.EventPublisher;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+/**
+ * Test DeadNodeHandler.
+ */
+public class TestDeadNodeHandler {
+  @Test
+  public void testOnMessage() throws SCMException {
+    //GIVEN
+    DatanodeDetails datanode1 = TestUtils.randomDatanodeDetails();
+    DatanodeDetails datanode2 = TestUtils.randomDatanodeDetails();
+
+    ContainerInfo container1 = TestUtils.getRandomContainerInfo(1);
+    ContainerInfo container2 = TestUtils.getRandomContainerInfo(2);
+    ContainerInfo container3 = TestUtils.getRandomContainerInfo(3);
+
+    Node2ContainerMap node2ContainerMap = new Node2ContainerMap();
+    ContainerStateManager containerStateManager = new ContainerStateManager(
+        new OzoneConfiguration(),
+        Mockito.mock(Mapping.class)
+    );
+    DeadNodeHandler handler =
+        new DeadNodeHandler(node2ContainerMap, containerStateManager);
+
+    node2ContainerMap
+        .insertNewDatanode(datanode1.getUuid(), new HashSet<ContainerID>() {{
+            add(new ContainerID(container1.getContainerID()));
+            add(new ContainerID(container2.getContainerID()));
+          }});
+
+    node2ContainerMap
+        .insertNewDatanode(datanode2.getUuid(), new HashSet<ContainerID>() {{
+            add(new ContainerID(container1.getContainerID()));
+            add(new ContainerID(container3.getContainerID()));
+          }});
+
+    containerStateManager.getContainerStateMap()
+        .addContainerReplica(new ContainerID(container1.getContainerID()),
+            datanode1, datanode2);
+
+    containerStateManager.getContainerStateMap()
+        .addContainerReplica(new ContainerID(container2.getContainerID()),
+            datanode1);
+
+    containerStateManager.getContainerStateMap()
+        .addContainerReplica(new ContainerID(container3.getContainerID()),
+            datanode2);
+
+    //WHEN datanode1 is dead
+    handler.onMessage(datanode1, Mockito.mock(EventPublisher.class));
+
+    //THEN
+
+    //node2ContainerMap has not been changed
+    Assert.assertEquals(2, node2ContainerMap.size());
+
+    Set<DatanodeDetails> container1Replicas =
+        containerStateManager.getContainerStateMap()
+            .getContainerReplicas(new ContainerID(container1.getContainerID()));
+    Assert.assertEquals(1, container1Replicas.size());
+    Assert.assertEquals(datanode2, container1Replicas.iterator().next());
+
+    Set<DatanodeDetails> container2Replicas =
+        containerStateManager.getContainerStateMap()
+            .getContainerReplicas(new ContainerID(container2.getContainerID()));
+    Assert.assertEquals(0, container2Replicas.size());
+
+    Set<DatanodeDetails> container3Replicas =
+        containerStateManager.getContainerStateMap()
+            .getContainerReplicas(new ContainerID(container3.getContainerID()));
+    Assert.assertEquals(1, container3Replicas.size());
+    Assert.assertEquals(datanode2, container3Replicas.iterator().next());
+
+  }
+}