diff --git a/lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java b/lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java
index 69c1b3f7720..86798c05340 100644
--- a/lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java
+++ b/lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java
@@ -1026,7 +1026,7 @@ public class TestPackedInts extends LuceneTestCase {
 
         for (int i = 0; i < arr.length; ++i) {
           buf.add(arr[i]);
-          if (rarely()) {
+          if (rarely() && !TEST_NIGHTLY) {
             final long expectedBytesUsed = RamUsageTester.sizeOf(buf);
             final long computedBytesUsed = buf.ramBytesUsed();
             assertEquals(expectedBytesUsed, computedBytesUsed);