diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/statistics/DistinctKeyCollector.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/statistics/DistinctKeyCollector.java
index b22a0990332..0e042b68b2c 100644
--- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/statistics/DistinctKeyCollector.java
+++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/statistics/DistinctKeyCollector.java
@@ -120,8 +120,9 @@ public class DistinctKeyCollector implements KeyCollector<DistinctKeyCollector>
     if (isNewMin || isKeySelected(key)) {
       if (isNewMin && !retainedKeys.isEmpty() && !isKeySelected(retainedKeys.firstKey())) {
         // Old min should be kicked out.
-        totalWeightUnadjusted -= retainedKeys.removeLong(retainedKeys.firstKey());
-        retainedBytes -= retainedKeys.firstKey().estimatedObjectSizeBytes();
+        RowKey rowKey = retainedKeys.firstKey();
+        totalWeightUnadjusted -= retainedKeys.removeLong(rowKey);
+        retainedBytes -= rowKey.estimatedObjectSizeBytes();
       }
 
       if (retainedKeys.putIfAbsent(key, weight) == MISSING_KEY_WEIGHT) {
diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/statistics/DistinctKeyCollectorTest.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/statistics/DistinctKeyCollectorTest.java
index 697005934e5..37ac2a10811 100644
--- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/statistics/DistinctKeyCollectorTest.java
+++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/statistics/DistinctKeyCollectorTest.java
@@ -20,6 +20,7 @@
 package org.apache.druid.msq.statistics;
 
 import com.google.common.collect.ImmutableList;
+import it.unimi.dsi.fastutil.objects.Object2LongRBTreeMap;
 import org.apache.druid.common.config.NullHandling;
 import org.apache.druid.frame.key.ClusterBy;
 import org.apache.druid.frame.key.ClusterByPartition;
@@ -90,6 +91,21 @@ public class DistinctKeyCollectorTest
     );
   }
 
+  @Test
+  public void test_single_key_addition()
+  {
+    DistinctKeyCollector distinctKeyCollector = new DistinctKeyCollector(
+        clusterBy.keyComparator(signature),
+        new Object2LongRBTreeMap<>(comparator),
+        2
+    );
+    List<Pair<RowKey, Integer>> pairs = KeyCollectorTestUtils.sequentialKeys(2);
+
+    distinctKeyCollector.add(pairs.get(1).lhs, pairs.get(1).rhs);
+    distinctKeyCollector.downSample();
+    distinctKeyCollector.add(pairs.get(0).lhs, pairs.get(0).rhs);
+  }
+
   @Test
   public void test_uniformRandomKeys_unweighted()
   {