diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/KeyOutputStream.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/KeyOutputStream.java
index 042aceef1a3..af3963100d9 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/KeyOutputStream.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/KeyOutputStream.java
@@ -338,8 +338,9 @@ public class KeyOutputStream extends OutputStream {
       ListIterator<BlockOutputStreamEntry> streamEntryIterator =
           streamEntries.listIterator(currentStreamIndex);
       while (streamEntryIterator.hasNext()) {
-        if (streamEntryIterator.next().getBlockID().getContainerID()
-            == containerID) {
+        BlockOutputStreamEntry streamEntry = streamEntryIterator.next();
+        if (streamEntry.getBlockID().getContainerID()
+            == containerID && streamEntry.getCurrentPosition() == 0) {
           streamEntryIterator.remove();
         }
       }