From 469165e6f29a6e7788f218bdbbc3f7bacf26628b Mon Sep 17 00:00:00 2001 From: Shashikant Banerjee Date: Mon, 9 Sep 2019 22:43:20 +0530 Subject: [PATCH] HDDS-1843. Undetectable corruption after restart of a datanode. Contributed by Shashikant Banerjee(#1364). --- .../proto/DatanodeContainerProtocol.proto | 5 +- .../container/common/impl/ContainerSet.java | 44 ++++++++-- .../container/common/impl/HddsDispatcher.java | 45 +++++++--- .../common/interfaces/Container.java | 7 ++ .../interfaces/ContainerDispatcher.java | 6 +- .../server/ratis/ContainerStateMachine.java | 37 ++++---- .../server/ratis/DispatcherContext.java | 24 ++--- .../container/keyvalue/KeyValueContainer.java | 5 ++ .../keyvalue/impl/BlockManagerImpl.java | 3 +- .../TestContainerStateMachineFailures.java | 88 +++++++++++++++++++ .../server/ratis/TestCSMMetrics.java | 5 +- .../container/server/TestContainerServer.java | 4 +- .../server/TestSecureContainerServer.java | 6 +- 13 files changed, 218 insertions(+), 61 deletions(-) diff --git a/hadoop-hdds/common/src/main/proto/DatanodeContainerProtocol.proto b/hadoop-hdds/common/src/main/proto/DatanodeContainerProtocol.proto index 9a27a8c89b7..1bfe4d1247c 100644 --- a/hadoop-hdds/common/src/main/proto/DatanodeContainerProtocol.proto +++ b/hadoop-hdds/common/src/main/proto/DatanodeContainerProtocol.proto @@ -248,8 +248,9 @@ message ContainerDataProto { optional ContainerType containerType = 10 [default = KeyValueContainer]; } -message ContainerIdSetProto { - repeated int64 containerId = 1; +message Container2BCSIDMapProto { + // repeated Container2BCSIDMapEntryProto container2BCSID = 1; + map container2BCSID = 1; } enum ContainerType { diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerSet.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerSet.java index 266371dd8e4..784f56c29f2 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerSet.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerSet.java @@ -240,14 +240,46 @@ public class ContainerSet { } /** - * Builds the missing container set by taking a diff total no containers - * actually found and number of containers which actually got created. + * Builds the missing container set by taking a diff between total no + * containers actually found and number of containers which actually + * got created. It also validates the BCSID stored in the snapshot file + * for each container as against what is reported in containerScan. * This will only be called during the initialization of Datanode Service * when it still not a part of any write Pipeline. - * @param createdContainerSet ContainerId set persisted in the Ratis snapshot + * @param container2BCSIDMap Map of containerId to BCSID persisted in the + * Ratis snapshot */ - public void buildMissingContainerSet(Set createdContainerSet) { - missingContainerSet.addAll(createdContainerSet); - missingContainerSet.removeAll(containerMap.keySet()); + public void buildMissingContainerSetAndValidate( + Map container2BCSIDMap) { + container2BCSIDMap.entrySet().parallelStream().forEach((mapEntry) -> { + long id = mapEntry.getKey(); + if (!containerMap.containsKey(id)) { + LOG.warn("Adding container {} to missing container set.", id); + missingContainerSet.add(id); + } else { + Container container = containerMap.get(id); + long containerBCSID = container.getBlockCommitSequenceId(); + long snapshotBCSID = mapEntry.getValue(); + if (containerBCSID < snapshotBCSID) { + LOG.warn( + "Marking container {} unhealthy as reported BCSID {} is smaller" + + " than ratis snapshot recorded value {}", id, + containerBCSID, snapshotBCSID); + // just mark the container unhealthy. Once the DatanodeStateMachine + // thread starts it will send container report to SCM where these + // unhealthy containers would be detected + try { + container.markContainerUnhealthy(); + } catch (StorageContainerException sce) { + // The container will still be marked unhealthy in memory even if + // exception occurs. It won't accept any new transactions and will + // be handled by SCM. Eve if dn restarts, it will still be detected + // as unheathy as its BCSID won't change. + LOG.error("Unable to persist unhealthy state for container {}", id); + } + } + } + }); + } } diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java index 39e163e54e3..e95d899db44 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java @@ -135,8 +135,10 @@ public class HddsDispatcher implements ContainerDispatcher, Auditor { } @Override - public void buildMissingContainerSet(Set createdContainerSet) { - containerSet.buildMissingContainerSet(createdContainerSet); + public void buildMissingContainerSetAndValidate( + Map container2BCSIDMap) { + containerSet + .buildMissingContainerSetAndValidate(container2BCSIDMap); } @Override @@ -185,9 +187,9 @@ public class HddsDispatcher implements ContainerDispatcher, Auditor { cmdType == ContainerProtos.Type.WriteChunk && (dispatcherContext == null || dispatcherContext.getStage() == DispatcherContext.WriteChunkStage.COMBINED); - Set containerIdSet = null; + Map container2BCSIDMap = null; if (dispatcherContext != null) { - containerIdSet = dispatcherContext.getCreateContainerSet(); + container2BCSIDMap = dispatcherContext.getContainer2BCSIDMap(); } if (isWriteCommitStage) { // check if the container Id exist in the loaded snapshot file. if @@ -196,9 +198,10 @@ public class HddsDispatcher implements ContainerDispatcher, Auditor { // snapshot. // just add it to the list, and remove it from missing container set // as it might have been added in the list during "init". - Preconditions.checkNotNull(containerIdSet); - if (!containerIdSet.contains(containerID)) { - containerIdSet.add(containerID); + Preconditions.checkNotNull(container2BCSIDMap); + if (container2BCSIDMap.get(containerID) == null) { + container2BCSIDMap + .put(containerID, container.getBlockCommitSequenceId()); containerSet.getMissingContainerSet().remove(containerID); } } @@ -228,11 +231,14 @@ public class HddsDispatcher implements ContainerDispatcher, Auditor { audit(action, eventType, params, AuditEventStatus.FAILURE, sce); return ContainerUtils.logAndReturnError(LOG, sce, msg); } - Preconditions.checkArgument(isWriteStage && containerIdSet != null + Preconditions.checkArgument(isWriteStage && container2BCSIDMap != null || dispatcherContext == null); - if (containerIdSet != null) { + if (container2BCSIDMap != null) { // adds this container to list of containers created in the pipeline - containerIdSet.add(containerID); + // with initial BCSID recorded as 0. + Preconditions + .checkArgument(!container2BCSIDMap.containsKey(containerID)); + container2BCSIDMap.put(containerID, Long.valueOf(0)); } container = getContainer(containerID); } @@ -313,7 +319,8 @@ public class HddsDispatcher implements ContainerDispatcher, Auditor { sendCloseContainerActionIfNeeded(container); } - if(result == Result.SUCCESS) { + if (result == Result.SUCCESS) { + updateBCSID(container, dispatcherContext, cmdType); audit(action, eventType, params, AuditEventStatus.SUCCESS, null); } else { audit(action, eventType, params, AuditEventStatus.FAILURE, @@ -329,6 +336,22 @@ public class HddsDispatcher implements ContainerDispatcher, Auditor { } } + private void updateBCSID(Container container, + DispatcherContext dispatcherContext, ContainerProtos.Type cmdType) { + if (dispatcherContext != null && (cmdType == ContainerProtos.Type.PutBlock + || cmdType == ContainerProtos.Type.PutSmallFile)) { + Preconditions.checkNotNull(container); + long bcsID = container.getBlockCommitSequenceId(); + long containerId = container.getContainerData().getContainerID(); + Map container2BCSIDMap; + container2BCSIDMap = dispatcherContext.getContainer2BCSIDMap(); + Preconditions.checkNotNull(container2BCSIDMap); + Preconditions.checkArgument(container2BCSIDMap.containsKey(containerId)); + // updates the latest BCSID on every putBlock or putSmallFile + // transaction over Ratis. + container2BCSIDMap.computeIfPresent(containerId, (u, v) -> v = bcsID); + } + } /** * Create a container using the input container request. * @param containerRequest - the container request which requires container diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Container.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Container.java index 05ff93fb9fc..7f7deaf9206 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Container.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Container.java @@ -155,6 +155,13 @@ public interface Container extends RwLock { void updateBlockCommitSequenceId(long blockCommitSequenceId); /** + * Returns the blockCommitSequenceId. + */ + long getBlockCommitSequenceId(); + + /** + * check and report the structural integrity of the container. + * @return true if the integrity checks pass * Scan the container metadata to detect corruption. */ boolean scanMetaData(); diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerDispatcher.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerDispatcher.java index e5a74cb10ec..ee0b6bcb200 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerDispatcher.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerDispatcher.java @@ -26,7 +26,7 @@ import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException; import org.apache.hadoop.ozone.container.common.transport.server.ratis.DispatcherContext; -import java.util.Set; +import java.util.Map; /** * Dispatcher acts as the bridge between the transport layer and @@ -62,9 +62,9 @@ public interface ContainerDispatcher { /** * finds and builds the missing containers in case of a lost disk etc - * in the ContainerSet. + * in the ContainerSet. It also validates the BCSID of the containers found. */ - void buildMissingContainerSet(Set createdContainers); + void buildMissingContainerSetAndValidate(Map container2BCSIDMap); /** * Shutdown Dispatcher services. diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java index 66a9d5857f4..cee9741c4fb 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java @@ -46,7 +46,7 @@ import org.apache.ratis.thirdparty.com.google.protobuf .InvalidProtocolBufferException; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Type; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos. - ContainerIdSetProto; + Container2BCSIDMapProto; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos .ContainerCommandRequestProto; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos @@ -88,8 +88,6 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.ExecutionException; import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; -import java.util.Set; -import java.util.concurrent.ConcurrentSkipListSet; import java.util.concurrent.Executors; import java.io.FileOutputStream; import java.io.FileInputStream; @@ -146,7 +144,7 @@ public class ContainerStateMachine extends BaseStateMachine { CompletableFuture> writeChunkFutureMap; // keeps track of the containers created per pipeline - private final Set createContainerSet; + private final Map container2BCSIDMap; private ExecutorService[] executors; private final Map applyTransactionCompletionMap; private final Cache stateMachineDataCache; @@ -181,7 +179,7 @@ public class ContainerStateMachine extends BaseStateMachine { .maximumSize(chunkExecutor.getCorePoolSize()).build(); this.isBlockTokenEnabled = isBlockTokenEnabled; this.tokenVerifier = tokenVerifier; - this.createContainerSet = new ConcurrentSkipListSet<>(); + this.container2BCSIDMap = new ConcurrentHashMap<>(); final int numContainerOpExecutors = conf.getInt( OzoneConfigKeys.DFS_CONTAINER_RATIS_NUM_CONTAINER_OP_EXECUTORS_KEY, @@ -244,14 +242,15 @@ public class ContainerStateMachine extends BaseStateMachine { // initialize the dispatcher with snapshot so that it build the missing // container list try (FileInputStream fin = new FileInputStream(snapshotFile)) { - byte[] containerIds = IOUtils.toByteArray(fin); - ContainerProtos.ContainerIdSetProto proto = - ContainerProtos.ContainerIdSetProto.parseFrom(containerIds); + byte[] container2BCSIDData = IOUtils.toByteArray(fin); + ContainerProtos.Container2BCSIDMapProto proto = + ContainerProtos.Container2BCSIDMapProto + .parseFrom(container2BCSIDData); // read the created containers list from the snapshot file and add it to - // the createContainerSet here. - // createContainerSet will further grow as and when containers get created - createContainerSet.addAll(proto.getContainerIdList()); - dispatcher.buildMissingContainerSet(createContainerSet); + // the container2BCSIDMap here. + // container2BCSIDMap will further grow as and when containers get created + container2BCSIDMap.putAll(proto.getContainer2BCSIDMap()); + dispatcher.buildMissingContainerSetAndValidate(container2BCSIDMap); } return last.getIndex(); } @@ -263,8 +262,9 @@ public class ContainerStateMachine extends BaseStateMachine { * @throws IOException */ public void persistContainerSet(OutputStream out) throws IOException { - ContainerIdSetProto.Builder builder = ContainerIdSetProto.newBuilder(); - builder.addAllContainerId(createContainerSet); + Container2BCSIDMapProto.Builder builder = + Container2BCSIDMapProto.newBuilder(); + builder.putAllContainer2BCSID(container2BCSIDMap); // TODO : while snapshot is being taken, deleteContainer call should not // should not happen. Lock protection will be required if delete // container happens outside of Ratis. @@ -433,7 +433,7 @@ public class ContainerStateMachine extends BaseStateMachine { .setTerm(term) .setLogIndex(entryIndex) .setStage(DispatcherContext.WriteChunkStage.WRITE_DATA) - .setCreateContainerSet(createContainerSet) + .setContainer2BCSIDMap(container2BCSIDMap) .build(); // ensure the write chunk happens asynchronously in writeChunkExecutor pool // thread. @@ -697,8 +697,9 @@ public class ContainerStateMachine extends BaseStateMachine { builder .setStage(DispatcherContext.WriteChunkStage.COMMIT_DATA); } - if (cmdType == Type.WriteChunk || cmdType ==Type.PutSmallFile) { - builder.setCreateContainerSet(createContainerSet); + if (cmdType == Type.WriteChunk || cmdType == Type.PutSmallFile + || cmdType == Type.PutBlock) { + builder.setContainer2BCSIDMap(container2BCSIDMap); } CompletableFuture applyTransactionFuture = new CompletableFuture<>(); @@ -811,7 +812,7 @@ public class ContainerStateMachine extends BaseStateMachine { // Make best effort to quasi-close all the containers on group removal. // Containers already in terminal state like CLOSED or UNHEALTHY will not // be affected. - for (Long cid : createContainerSet) { + for (Long cid : container2BCSIDMap.keySet()) { try { containerController.markContainerForClose(cid); containerController.quasiCloseContainer(cid); diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/DispatcherContext.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/DispatcherContext.java index 446f19f05ea..7d46910164e 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/DispatcherContext.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/DispatcherContext.java @@ -20,7 +20,7 @@ package org.apache.hadoop.ozone.container.common.transport.server.ratis; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; -import java.util.Set; +import java.util.Map; /** * DispatcherContext class holds transport protocol specific context info @@ -45,15 +45,15 @@ public final class DispatcherContext { // the log index in Ratis log to which the request belongs to private final long logIndex; - private final Set createContainerSet; + private final Map container2BCSIDMap; private DispatcherContext(long term, long index, WriteChunkStage stage, - boolean readFromTmpFile, Set containerSet) { + boolean readFromTmpFile, Map container2BCSIDMap) { this.term = term; this.logIndex = index; this.stage = stage; this.readFromTmpFile = readFromTmpFile; - this.createContainerSet = containerSet; + this.container2BCSIDMap = container2BCSIDMap; } public long getLogIndex() { @@ -72,8 +72,8 @@ public final class DispatcherContext { return stage; } - public Set getCreateContainerSet() { - return createContainerSet; + public Map getContainer2BCSIDMap() { + return container2BCSIDMap; } /** @@ -84,7 +84,7 @@ public final class DispatcherContext { private boolean readFromTmpFile = false; private long term; private long logIndex; - private Set createContainerSet; + private Map container2BCSIDMap; /** * Sets the WriteChunkStage. @@ -131,13 +131,13 @@ public final class DispatcherContext { } /** - * Sets the createContainerSet to contain all the containerIds per + * Sets the container2BCSIDMap to contain all the containerIds per * RaftGroup. - * @param set createContainerSet + * @param map container2BCSIDMap * @return Builder */ - public Builder setCreateContainerSet(Set set) { - this.createContainerSet = set; + public Builder setContainer2BCSIDMap(Map map) { + this.container2BCSIDMap = map; return this; } /** @@ -147,7 +147,7 @@ public final class DispatcherContext { */ public DispatcherContext build() { return new DispatcherContext(term, logIndex, stage, readFromTmpFile, - createContainerSet); + container2BCSIDMap); } } diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java index b7f46c94167..ff570376a54 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java @@ -611,6 +611,11 @@ public class KeyValueContainer implements Container { containerData.updateBlockCommitSequenceId(blockCommitSequenceId); } + @Override + public long getBlockCommitSequenceId() { + return containerData.getBlockCommitSequenceId(); + } + /** * Returns KeyValueContainerReport for the KeyValueContainer. diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/BlockManagerImpl.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/BlockManagerImpl.java index dadd2afc3e5..53715c84982 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/BlockManagerImpl.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/BlockManagerImpl.java @@ -282,5 +282,4 @@ public class BlockManagerImpl implements BlockManager { public void shutdown() { BlockUtils.shutdownCache(ContainerCache.getInstance(config)); } - -} +} \ No newline at end of file diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineFailures.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineFailures.java index 86621d6b16c..7b908151f48 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineFailures.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineFailures.java @@ -17,6 +17,7 @@ package org.apache.hadoop.ozone.client.rpc; +import com.google.common.primitives.Longs; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.hdds.client.ReplicationFactor; import org.apache.hadoop.hdds.client.ReplicationType; @@ -26,6 +27,7 @@ import org.apache.hadoop.hdds.scm.XceiverClientManager; import org.apache.hadoop.hdds.scm.XceiverClientSpi; import org.apache.hadoop.hdds.scm.client.HddsClientUtils; import org.apache.hadoop.hdds.scm.pipeline.Pipeline; +import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.ozone.OzoneConsts; @@ -39,7 +41,9 @@ import org.apache.hadoop.ozone.container.common.impl.ContainerData; import org.apache.hadoop.ozone.container.common.impl.ContainerDataYaml; import org.apache.hadoop.ozone.container.common.impl.HddsDispatcher; import org.apache.hadoop.ozone.container.common.transport.server.ratis.ContainerStateMachine; +import org.apache.hadoop.ozone.container.common.utils.ReferenceCountedDB; import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData; +import org.apache.hadoop.ozone.container.keyvalue.helpers.BlockUtils; import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; @@ -348,4 +352,88 @@ public class TestContainerStateMachineFailures { FileInfo latestSnapshot = storage.findLatestSnapshot().getFile(); Assert.assertTrue(snapshot.getPath().equals(latestSnapshot.getPath())); } + + @Test + public void testValidateBCSIDOnDnRestart() throws Exception { + OzoneOutputStream key = + objectStore.getVolume(volumeName).getBucket(bucketName) + .createKey("ratis", 1024, ReplicationType.RATIS, + ReplicationFactor.ONE, new HashMap<>()); + // First write and flush creates a container in the datanode + key.write("ratis".getBytes()); + key.flush(); + key.write("ratis".getBytes()); + KeyOutputStream groupOutputStream = (KeyOutputStream) key.getOutputStream(); + List locationInfoList = + groupOutputStream.getLocationInfoList(); + Assert.assertEquals(1, locationInfoList.size()); + OmKeyLocationInfo omKeyLocationInfo = locationInfoList.get(0); + ContainerData containerData = + cluster.getHddsDatanodes().get(0).getDatanodeStateMachine() + .getContainer().getContainerSet() + .getContainer(omKeyLocationInfo.getContainerID()) + .getContainerData(); + Assert.assertTrue(containerData instanceof KeyValueContainerData); + KeyValueContainerData keyValueContainerData = + (KeyValueContainerData) containerData; + key.close(); + + long containerID = omKeyLocationInfo.getContainerID(); + cluster.shutdownHddsDatanode( + cluster.getHddsDatanodes().get(0).getDatanodeDetails()); + // delete the container db file + FileUtil.fullyDelete(new File(keyValueContainerData.getContainerPath())); + cluster.restartHddsDatanode( + cluster.getHddsDatanodes().get(0).getDatanodeDetails(), true); + OzoneContainer ozoneContainer = + cluster.getHddsDatanodes().get(0).getDatanodeStateMachine() + .getContainer(); + // make sure the missing containerSet is not empty + HddsDispatcher dispatcher = (HddsDispatcher) ozoneContainer.getDispatcher(); + Assert.assertTrue(!dispatcher.getMissingContainerSet().isEmpty()); + Assert + .assertTrue(dispatcher.getMissingContainerSet().contains(containerID)); + // write a new key + key = objectStore.getVolume(volumeName).getBucket(bucketName) + .createKey("ratis", 1024, ReplicationType.RATIS, ReplicationFactor.ONE, + new HashMap<>()); + // First write and flush creates a container in the datanode + key.write("ratis1".getBytes()); + key.flush(); + groupOutputStream = (KeyOutputStream) key.getOutputStream(); + locationInfoList = groupOutputStream.getLocationInfoList(); + Assert.assertEquals(1, locationInfoList.size()); + omKeyLocationInfo = locationInfoList.get(0); + key.close(); + containerID = omKeyLocationInfo.getContainerID(); + containerData = cluster.getHddsDatanodes().get(0).getDatanodeStateMachine() + .getContainer().getContainerSet() + .getContainer(omKeyLocationInfo.getContainerID()).getContainerData(); + Assert.assertTrue(containerData instanceof KeyValueContainerData); + keyValueContainerData = (KeyValueContainerData) containerData; + ReferenceCountedDB db = BlockUtils. + getDB(keyValueContainerData, conf); + byte[] blockCommitSequenceIdKey = + DFSUtil.string2Bytes(OzoneConsts.BLOCK_COMMIT_SEQUENCE_ID_PREFIX); + + // modify the bcsid for the container in the ROCKS DB tereby inducing + // corruption + db.getStore().put(blockCommitSequenceIdKey, Longs.toByteArray(0)); + db.decrementReference(); + // shutdown of dn will take a snapsot which will persist the valid BCSID + // recorded in the container2BCSIDMap in ContainerStateMachine + cluster.shutdownHddsDatanode( + cluster.getHddsDatanodes().get(0).getDatanodeDetails()); + // after the restart, there will be a mismatch in BCSID of what is recorded + // in the and what is there in RockSDB and hence the container would be + // marked unhealthy + cluster.restartHddsDatanode( + cluster.getHddsDatanodes().get(0).getDatanodeDetails(), true); + // Make sure the container is marked unhealthy + Assert.assertTrue( + cluster.getHddsDatanodes().get(0).getDatanodeStateMachine() + .getContainer().getContainerSet().getContainer(containerID) + .getContainerState() + == ContainerProtos.ContainerDataProto.State.UNHEALTHY); + } } \ No newline at end of file diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/TestCSMMetrics.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/TestCSMMetrics.java index 7eecadfa414..3967c0ccf72 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/TestCSMMetrics.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/TestCSMMetrics.java @@ -58,7 +58,7 @@ import static org.junit.Assert.assertTrue; import org.apache.ratis.protocol.RaftGroupId; import org.apache.ratis.util.function.CheckedBiConsumer; -import java.util.Set; +import java.util.Map; import java.util.function.BiConsumer; import org.junit.Test; @@ -230,7 +230,8 @@ public class TestCSMMetrics { } @Override - public void buildMissingContainerSet(Set createdContainerSet) { + public void buildMissingContainerSetAndValidate( + Map container2BCSIDMap) { } } } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestContainerServer.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestContainerServer.java index c95b8eadcb7..59d741d16ad 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestContainerServer.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestContainerServer.java @@ -71,7 +71,6 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.UUID; -import java.util.Set; import static org.apache.ratis.rpc.SupportedRpcType.GRPC; import static org.apache.ratis.rpc.SupportedRpcType.NETTY; @@ -293,7 +292,8 @@ public class TestContainerServer { } @Override - public void buildMissingContainerSet(Set createdContainerSet) { + public void buildMissingContainerSetAndValidate( + Map container2BCSIDMap) { } } } \ No newline at end of file diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestSecureContainerServer.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestSecureContainerServer.java index 2f6b91f396f..cfee1a63788 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestSecureContainerServer.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestSecureContainerServer.java @@ -71,7 +71,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.EnumSet; import java.util.List; -import java.util.Set; +import java.util.Map; import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_BLOCK_TOKEN_ENABLED; import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result.SUCCESS; @@ -284,8 +284,8 @@ public class TestSecureContainerServer { } @Override - public void buildMissingContainerSet(Set createdContainerSet) { + public void buildMissingContainerSetAndValidate( + Map container2BCSIDMap) { } } - } \ No newline at end of file