DEV: Debug AR connection pool queue on CI (#25687)

Why this change?

On CI, we have been seeing flaky system tests because ActiveRecord is
unable to checkout a connection. This patch is meant to help us debug
which thread is not returning the connection to the queue.

.github/workflows/tests.yml

@@ -244,6 +244,7 @@ jobs:
         if: matrix.build_type == 'system' && matrix.target == 'plugins'
         env:
           CHECKOUT_TIMEOUT: 10
+          DEBUG_AR_CONNECTION_QUEUE: 1
         run: |
           GLOBIGNORE="plugins/chat/*";
           LOAD_PLUGINS=1 RAILS_ENABLE_TEST_LOG=1 RAILS_TEST_LOG_LEVEL=error PARALLEL_TEST_PROCESSORS=4 bin/turbo_rspec --use-runtime-info --profile=50 --verbose --format documentation plugins/*/spec/system
@@ -254,6 +255,7 @@ jobs:
         if: matrix.build_type == 'system' && matrix.target == 'chat'
         env:
           CHECKOUT_TIMEOUT: 10
+          DEBUG_AR_CONNECTION_QUEUE: 1
         run: LOAD_PLUGINS=1 RAILS_ENABLE_TEST_LOG=1 RAILS_TEST_LOG_LEVEL=error PARALLEL_TEST_PROCESSORS=4 bin/turbo_rspec --use-runtime-info --profile=50 --verbose --format documentation plugins/chat/spec/system
         timeout-minutes: 30
 

lib/freedom_patches/debug_connection_queue.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+if ENV["DEBUG_AR_CONNECTION_QUEUE"] == "1"
+  module QueuePatch
+    # Add +element+ to the queue.  Never blocks.
+    def add(element)
+      puts "::group::##{Process.pid} Adding element to the queue"
+      puts Thread.current.backtrace.first(30).join("\n")
+      puts "::endgroup::"
+      super
+    end
+
+    # If +element+ is in the queue, remove and return it, or +nil+.
+    def delete(element)
+      puts "::group::##{Process.pid} Delete element from the queue"
+      puts Thread.current.backtrace.first(30).join("\n")
+      puts "::endgroup::"
+      super
+    end
+
+    # Remove all elements from the queue.
+    def clear
+      puts "::group::##{Process.pid} Clear all elements from the queue"
+      puts Thread.current.backtrace.first(30).join("\n")
+      puts "::endgroup::"
+      super
+    end
+
+    private
+
+    def remove
+      puts "::group::##{Process.pid} Removing element from the queue"
+      puts Thread.current.backtrace.first(30).join("\n")
+      puts "::endgroup::"
+      super
+    end
+  end
+
+  ActiveRecord::ConnectionAdapters::ConnectionPool::Queue.prepend(QueuePatch)
+end

spec/rails_helper.rb

@@ -663,6 +663,7 @@ RSpec.configure do |config|
     end
 
     page.execute_script("if (typeof MessageBus !== 'undefined') { MessageBus.stop(); }")
+    Capybara.reset_session!
     MessageBus.backend_instance.reset! # Clears all existing backlog from memory backend
     Discourse.redis.flushdb
   end