diff --git a/app/assets/javascripts/discourse/tests/test-boot-ember-cli.js b/app/assets/javascripts/discourse/tests/test-boot-ember-cli.js
index 690702f1b18..3ada4484ab8 100644
--- a/app/assets/javascripts/discourse/tests/test-boot-ember-cli.js
+++ b/app/assets/javascripts/discourse/tests/test-boot-ember-cli.js
@@ -41,7 +41,10 @@ document.addEventListener("discourse-booted", () => {
   if (QUnit.config.seed === undefined) {
     // If we're running in browser, default to random order. Otherwise, let Ember Exam
     // handle randomization.
-    QUnit.config.seed = true;
+    QUnit.config.seed = Math.random().toString(36).slice(2);
+  } else {
+    // Don't reorder when specifying a seed
+    QUnit.config.reorder = false;
   }
 
   loader.loadModules();
@@ -53,4 +56,5 @@ document.addEventListener("discourse-booted", () => {
     setupEmberOnerrorValidation: !skipCore,
   });
 });
+
 window.EmberENV.TESTS_FILE_LOADED = true;