From 9964e33e8df1a6574d106c22fcaf339db8d48750 Mon Sep 17 00:00:00 2001 From: Hanisha Koneru Date: Tue, 4 Sep 2018 14:57:54 -0700 Subject: [PATCH] HDDS-369. Remove the containers of a dead node from the container state map. Contributed by Elek, Marton --- .../hadoop/hdds/scm/node/DeadNodeHandler.java | 41 ++++++- .../scm/node/states/Node2ContainerMap.java | 29 +++-- .../scm/server/StorageContainerManager.java | 3 +- .../hdds/scm/node/TestDeadNodeHandler.java | 112 ++++++++++++++++++ 4 files changed, 166 insertions(+), 19 deletions(-) create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDeadNodeHandler.java diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DeadNodeHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DeadNodeHandler.java index 427aef88ac1..c853b3b61c5 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DeadNodeHandler.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/DeadNodeHandler.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,11 +18,19 @@ package org.apache.hadoop.hdds.scm.node; +import java.util.Set; + import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.scm.container.ContainerID; +import org.apache.hadoop.hdds.scm.container.ContainerStateManager; +import org.apache.hadoop.hdds.scm.exceptions.SCMException; import org.apache.hadoop.hdds.scm.node.states.Node2ContainerMap; import org.apache.hadoop.hdds.server.events.EventHandler; import org.apache.hadoop.hdds.server.events.EventPublisher; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + /** * Handles Dead Node event. */ @@ -30,13 +38,34 @@ public class DeadNodeHandler implements EventHandler { private final Node2ContainerMap node2ContainerMap; - public DeadNodeHandler(Node2ContainerMap node2ContainerMap) { + private final ContainerStateManager containerStateManager; + + private static final Logger LOG = + LoggerFactory.getLogger(DeadNodeHandler.class); + + public DeadNodeHandler( + Node2ContainerMap node2ContainerMap, + ContainerStateManager containerStateManager) { this.node2ContainerMap = node2ContainerMap; + this.containerStateManager = containerStateManager; } @Override public void onMessage(DatanodeDetails datanodeDetails, - EventPublisher publisher) { - //TODO: add logic to handle dead node. + EventPublisher publisher) { + Set containers = + node2ContainerMap.getContainers(datanodeDetails.getUuid()); + LOG.info( + "Datanode {} is dead. Removing replications from the in-memory state.", + datanodeDetails.getUuid()); + for (ContainerID container : containers) { + try { + containerStateManager.removeContainerReplica(container, + datanodeDetails); + } catch (SCMException e) { + LOG.error("Can't remove container from containerStateMap {}", container + .getId(), e); + } + } } } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/states/Node2ContainerMap.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/states/Node2ContainerMap.java index d4d475e84ef..97c254be316 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/states/Node2ContainerMap.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/states/Node2ContainerMap.java @@ -18,10 +18,6 @@ package org.apache.hadoop.hdds.scm.node.states; -import com.google.common.base.Preconditions; -import org.apache.hadoop.hdds.scm.container.ContainerID; -import org.apache.hadoop.hdds.scm.exceptions.SCMException; - import java.util.Collections; import java.util.HashSet; import java.util.Map; @@ -30,8 +26,15 @@ import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; -import static org.apache.hadoop.hdds.scm.exceptions.SCMException.ResultCodes.DUPLICATE_DATANODE; -import static org.apache.hadoop.hdds.scm.exceptions.SCMException.ResultCodes.NO_SUCH_DATANODE; +import org.apache.hadoop.hdds.scm.container.ContainerID; +import org.apache.hadoop.hdds.scm.exceptions.SCMException; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import static org.apache.hadoop.hdds.scm.exceptions.SCMException.ResultCodes + .DUPLICATE_DATANODE; +import static org.apache.hadoop.hdds.scm.exceptions.SCMException.ResultCodes + .NO_SUCH_DATANODE; /** * This data structure maintains the list of containers that is on a datanode. @@ -62,7 +65,7 @@ public boolean isKnownDatanode(UUID datanodeID) { /** * Insert a new datanode into Node2Container Map. * - * @param datanodeID -- Datanode UUID + * @param datanodeID -- Datanode UUID * @param containerIDs - List of ContainerIDs. */ public void insertNewDatanode(UUID datanodeID, Set containerIDs) @@ -72,7 +75,7 @@ public void insertNewDatanode(UUID datanodeID, Set containerIDs) if (dn2ContainerMap.putIfAbsent(datanodeID, new HashSet<>(containerIDs)) != null) { throw new SCMException("Node already exists in the map", - DUPLICATE_DATANODE); + DUPLICATE_DATANODE); } } @@ -97,6 +100,7 @@ public void setContainersForDatanode(UUID datanodeID, /** * Removes datanode Entry from the map. + * * @param datanodeID - Datanode ID. */ public void removeDatanode(UUID datanodeID) { @@ -170,10 +174,6 @@ public ReportResult processReport(UUID datanodeID, Set .build(); } - - - - /** * Results possible from processing a container report by * Node2ContainerMapper. @@ -185,4 +185,9 @@ public enum ReportStatus { MISSING_AND_NEW_CONTAINERS_FOUND, NEW_DATANODE_FOUND } + + @VisibleForTesting + public int size() { + return dn2ContainerMap.size(); + } } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java index 178e2bde17b..7e2bc23f209 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java @@ -218,7 +218,8 @@ private StorageContainerManager(OzoneConfiguration conf) throws IOException { NewNodeHandler newNodeHandler = new NewNodeHandler(node2ContainerMap); StaleNodeHandler staleNodeHandler = new StaleNodeHandler(node2ContainerMap); - DeadNodeHandler deadNodeHandler = new DeadNodeHandler(node2ContainerMap); + DeadNodeHandler deadNodeHandler = new DeadNodeHandler(node2ContainerMap, + getScmContainerManager().getStateManager()); ContainerActionsHandler actionsHandler = new ContainerActionsHandler(); PendingDeleteHandler pendingDeleteHandler = new PendingDeleteHandler(scmBlockManager.getSCMBlockDeletingService()); diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDeadNodeHandler.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDeadNodeHandler.java new file mode 100644 index 00000000000..4be10e102fb --- /dev/null +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDeadNodeHandler.java @@ -0,0 +1,112 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdds.scm.node; + +import java.util.HashSet; +import java.util.Set; + +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.protocol.proto + .StorageContainerDatanodeProtocolProtos.ContainerInfo; +import org.apache.hadoop.hdds.scm.TestUtils; +import org.apache.hadoop.hdds.scm.container.ContainerID; +import org.apache.hadoop.hdds.scm.container.ContainerStateManager; +import org.apache.hadoop.hdds.scm.container.Mapping; +import org.apache.hadoop.hdds.scm.exceptions.SCMException; +import org.apache.hadoop.hdds.scm.node.states.Node2ContainerMap; +import org.apache.hadoop.hdds.server.events.EventPublisher; + +import org.junit.Assert; +import org.junit.Test; +import org.mockito.Mockito; + +/** + * Test DeadNodeHandler. + */ +public class TestDeadNodeHandler { + @Test + public void testOnMessage() throws SCMException { + //GIVEN + DatanodeDetails datanode1 = TestUtils.randomDatanodeDetails(); + DatanodeDetails datanode2 = TestUtils.randomDatanodeDetails(); + + ContainerInfo container1 = TestUtils.getRandomContainerInfo(1); + ContainerInfo container2 = TestUtils.getRandomContainerInfo(2); + ContainerInfo container3 = TestUtils.getRandomContainerInfo(3); + + Node2ContainerMap node2ContainerMap = new Node2ContainerMap(); + ContainerStateManager containerStateManager = new ContainerStateManager( + new OzoneConfiguration(), + Mockito.mock(Mapping.class) + ); + DeadNodeHandler handler = + new DeadNodeHandler(node2ContainerMap, containerStateManager); + + node2ContainerMap + .insertNewDatanode(datanode1.getUuid(), new HashSet() {{ + add(new ContainerID(container1.getContainerID())); + add(new ContainerID(container2.getContainerID())); + }}); + + node2ContainerMap + .insertNewDatanode(datanode2.getUuid(), new HashSet() {{ + add(new ContainerID(container1.getContainerID())); + add(new ContainerID(container3.getContainerID())); + }}); + + containerStateManager.getContainerStateMap() + .addContainerReplica(new ContainerID(container1.getContainerID()), + datanode1, datanode2); + + containerStateManager.getContainerStateMap() + .addContainerReplica(new ContainerID(container2.getContainerID()), + datanode1); + + containerStateManager.getContainerStateMap() + .addContainerReplica(new ContainerID(container3.getContainerID()), + datanode2); + + //WHEN datanode1 is dead + handler.onMessage(datanode1, Mockito.mock(EventPublisher.class)); + + //THEN + + //node2ContainerMap has not been changed + Assert.assertEquals(2, node2ContainerMap.size()); + + Set container1Replicas = + containerStateManager.getContainerStateMap() + .getContainerReplicas(new ContainerID(container1.getContainerID())); + Assert.assertEquals(1, container1Replicas.size()); + Assert.assertEquals(datanode2, container1Replicas.iterator().next()); + + Set container2Replicas = + containerStateManager.getContainerStateMap() + .getContainerReplicas(new ContainerID(container2.getContainerID())); + Assert.assertEquals(0, container2Replicas.size()); + + Set container3Replicas = + containerStateManager.getContainerStateMap() + .getContainerReplicas(new ContainerID(container3.getContainerID())); + Assert.assertEquals(1, container3Replicas.size()); + Assert.assertEquals(datanode2, container3Replicas.iterator().next()); + + } +} \ No newline at end of file