DEV: Improve multisite db scripts in dev (#17337)

## Without multisite.yml config

No change. `bin/rails db:create` / `db:migrate` / `db:drop` should work the same.

## With multisite.yml config

### db:create

`bin/rails db:create` creates development, test, and all databases from the multisite config

`RAILS_DB=[site] bin/rails db:create` creates the database for the specified site from the multisite config

### db:migrate

`bin/rails db:migrate` migrates the development database and all databases from the multisite config

`RAILS_ENV=test bin/rails db:migrate` migrates the test database and `discourse_test_multisite`

`RAILS_DB=[site] bin/rails db:migrate` migrates the database for the specified site from the multisite config

### db:drop

`bin/rails db:drop` drops development, test, and all databases from the multisite config

`RAILS_DB=[site] bin/rails db:create` drops the database for the specified site from the multisite config

config/boot.rb

@@ -45,7 +45,7 @@ if ENV['RAILS_ENV'] == "test" && ENV['TEST_ENV_NUMBER']
   pid = Process.spawn("redis-server --dir tmp/test_data_#{n}/redis --port #{port}", out: "/dev/null")
 
   ENV["DISCOURSE_REDIS_PORT"] = port.to_s
-  ENV["RAILS_DB"] = "discourse_test_#{n}"
+  ENV["RAILS_TEST_DB"] = "discourse_test_#{n}"
 
   at_exit do
     Process.kill("SIGTERM", pid)

config/database.yml

@@ -22,7 +22,7 @@ development:
 # Do not set this db to the same as development or production.
 
 <%
-  test_db = ENV["RAILS_DB"]
+  test_db = ENV["RAILS_TEST_DB"]
   if !test_db.present?
     test_db = "discourse_test"
 

lib/tasks/db.rake

@@ -11,17 +11,17 @@ end
 
 module MultisiteTestHelpers
   def self.load_multisite?
-    Rails.env.test? && !ENV["RAILS_DB"] && !ENV["SKIP_MULTISITE"]
+    Rails.env.test? && !ENV["RAILS_TEST_DB"] && !ENV["SKIP_MULTISITE"]
   end
 
   def self.create_multisite?
-    (ENV["RAILS_ENV"] == "test" || !ENV["RAILS_ENV"]) && !ENV["RAILS_DB"] && !ENV["SKIP_MULTISITE"]
+    (Rails.env.test? || Rails.env.development?) && !ENV["RAILS_TEST_DB"] && !ENV["DATABASE_URL"] && !ENV["SKIP_MULTISITE"]
   end
 end
 
 task 'db:environment:set' => [:load_config]  do |_, args|
   if MultisiteTestHelpers.load_multisite?
-    system("RAILS_ENV=test RAILS_DB=discourse_test_multisite rake db:environment:set")
+    system("RAILS_ENV=test RAILS_TEST_DB=discourse_test_multisite rake db:environment:set")
   end
 end
 
@@ -30,46 +30,154 @@ task 'db:force_skip_persist' do
   GlobalSetting.skip_redis = true
 end
 
-task 'db:create' => [:load_config] do |_, args|
+def config_to_url(config)
-  if MultisiteTestHelpers.create_multisite?
+  if config[:username] || config[:password]
-    unless system("RAILS_ENV=test RAILS_DB=discourse_test_multisite rake db:create")
+    userinfo = [config[:username], config[:password]].join(":")
+  end
 
-      STDERR.puts "-" * 80
+  URI::Generic.new(
-      STDERR.puts "ERROR: Could not create multisite DB. A common cause of this is a plugin"
+    config[:adapter],
-      STDERR.puts "checking the column structure when initializing, which raises an error."
+    userinfo,
-      STDERR.puts "-" * 80
+    config[:hostname] || "localhost",
-      raise "Could not initialize discourse_test_multisite"
+    config[:port],
+    nil,
+    "/#{config[:database]}",
+    nil,
+    nil,
+    nil
+  ).to_s
+end
+
+# db:create and related tasks
+
+task "multisite:create" => ["db:load_config"] do
+  next if !Rails.env.development? || !ENV["RAILS_DB"] || ENV["DATABASE_URL"]
+
+  spec = RailsMultisite::ConnectionManagement.connection_spec(db: ENV["RAILS_DB"])
+  database_url = config_to_url(spec.config)
+  system("DATABASE_URL=#{database_url} rake db:create")
+  exit 0
+end
+
+task "multisite:create:all" => ["db:load_config"] do
+  next if !MultisiteTestHelpers.create_multisite?
+
+  unless system("RAILS_ENV=test RAILS_TEST_DB=discourse_test_multisite rake db:create")
+    STDERR.puts "-" * 80
+    STDERR.puts "ERROR: Could not create multisite DB. A common cause of this is a plugin"
+    STDERR.puts "checking the column structure when initializing, which raises an error."
+    STDERR.puts "-" * 80
+    raise "Could not initialize discourse_test_multisite"
+  end
+
+  RailsMultisite::ConnectionManagement.all_dbs.each do |db|
+    spec = RailsMultisite::ConnectionManagement.connection_spec(db: db)
+    next unless spec
+
+    database_url = config_to_url(spec.config)
+    system("DATABASE_URL=#{database_url} rake db:create")
+  end
+end
+
+task "db:create" => :load_config do
+  Rake::Task["multisite:create:all"].invoke
+end
+
+begin
+  prerequisites = Rake::Task['db:create'].prerequisites.map(&:to_sym)
+  Rake::Task['db:create'].clear_prerequisites
+  Rake::Task['db:create'].enhance(["db:force_skip_persist", "multisite:create"] + prerequisites)
+end
+
+# db:drop and related tasks
+
+task "multisite:drop" => ["db:load_config"] do
+  next if !Rails.env.development? || !ENV["RAILS_DB"] || ENV["DATABASE_URL"]
+
+  spec = RailsMultisite::ConnectionManagement.connection_spec(db: ENV["RAILS_DB"])
+  database_url = config_to_url(spec.config)
+  system("DATABASE_URL=#{database_url} rake db:drop")
+  exit 0
+end
+
+task "multisite:drop:all" => ["db:load_config"] do
+  next if !MultisiteTestHelpers.create_multisite?
+
+  system("RAILS_TEST_DB=discourse_test_multisite RAILS_ENV=test rake db:drop")
+
+  RailsMultisite::ConnectionManagement.all_dbs.each do |db|
+    spec = RailsMultisite::ConnectionManagement.connection_spec(db: db)
+    next unless spec
+
+    database_url = config_to_url(spec.config)
+    system("DATABASE_URL=#{database_url} rake db:drop")
+  end
+end
+
+task "db:drop" => :load_config do
+  Rake::Task["multisite:drop:all"].invoke
+end
+
+begin
+  prerequisites = Rake::Task['db:drop'].prerequisites.map(&:to_sym)
+  Rake::Task['db:drop'].clear_prerequisites
+  Rake::Task['db:drop'].enhance(["db:force_skip_persist", "multisite:drop"] + prerequisites)
+end
+
+# db:migrate and related tasks
+
+# we need to run seed_fu every time we run rake db:migrate
+Rake::Task["db:migrate"].clear
+task "db:migrate" => ["load_config", "environment", "set_locale"] do |_, args|
+  DistributedMutex.synchronize('db_migration', redis: Discourse.redis.without_namespace, validity: 300) do
+    migrations = ActiveRecord::Base.connection.migration_context.migrations
+    now_timestamp = Time.now.utc.strftime("%Y%m%d%H%M%S").to_i
+    epoch_timestamp = Time.at(0).utc.strftime("%Y%m%d%H%M%S").to_i
+
+    raise "Migration #{migrations.last.version} is timestamped in the future" if migrations.last.version > now_timestamp
+    raise "Migration #{migrations.first.version} is timestamped before the epoch" if migrations.first.version < epoch_timestamp
+
+    %i[pg_trgm unaccent].each do |extension|
+      begin
+        DB.exec "CREATE EXTENSION IF NOT EXISTS #{extension}"
+      rescue => e
+        STDERR.puts "Cannot enable database extension #{extension}"
+        STDERR.puts e
+      end
+    end
+
+    ActiveRecord::Tasks::DatabaseTasks.migrate
+
+    SeedFu.quiet = true
+    SeedFu.seed(SeedHelper.paths, SeedHelper.filter)
+
+    if Rails.env.development? && !ENV["RAILS_TEST_DB"] && !ENV["RAILS_DB"]
+      Rake::Task["db:schema:cache:dump"].invoke
+    end
+
+    if !Discourse.skip_post_deployment_migrations? && ENV["SKIP_OPTIMIZE_ICONS"] != "1"
+      SiteIconManager.ensure_optimized!
     end
   end
-end
 
-begin
+  next if Discourse.is_parallel_test?
-  reqs = Rake::Task['db:create'].prerequisites.map(&:to_sym)
-  Rake::Task['db:create'].clear_prerequisites
-  Rake::Task['db:create'].enhance(["db:force_skip_persist"] + reqs)
-end
 
-task 'db:drop' => [:load_config] do |_, args|
+  if MultisiteTestHelpers.load_multisite?
-  if MultisiteTestHelpers.create_multisite?
+    system("RAILS_TEST_DB=discourse_test_multisite rake db:migrate")
-    system("RAILS_DB=discourse_test_multisite RAILS_ENV=test rake db:drop")
+    next
+  end
+
+  next if !Rails.env.development? || !ENV["RAILS_DB"] || ENV["DATABASE_URL"]
+
+  RailsMultisite::ConnectionManagement.all_dbs.each do |db|
+    spec = RailsMultisite::ConnectionManagement.connection_spec(db: db)
+    next unless spec
+
+    database_url = config_to_url(spec.config)
+    system("DATABASE_URL=#{database_url} rake db:migrate")
   end
 end
 
-begin
-  Rake::Task["db:migrate"].clear
-  Rake::Task["db:rollback"].clear
-end
-
-task 'db:rollback' => ['environment', 'set_locale'] do |_, args|
-  step = ENV["STEP"] ? ENV["STEP"].to_i : 1
-  ActiveRecord::Base.connection.migration_context.rollback(step)
-  Rake::Task['db:_dump'].invoke
-end
-
-# our optimized version of multisite migrate, we have many sites and we have seeds
-# this ensures we can run migrations concurrently to save huge amounts of time
-Rake::Task['multisite:migrate'].clear
-
 class StdOutDemux
   def initialize(stdout)
     @stdout = stdout
@@ -109,6 +217,9 @@ class SeedHelper
   end
 end
 
+# our optimized version of multisite migrate, we have many sites and we have seeds
+# this ensures we can run migrations concurrently to save huge amounts of time
+Rake::Task['multisite:migrate'].clear
 task 'multisite:migrate' => ['db:load_config', 'environment', 'set_locale'] do |_, args|
   if ENV["RAILS_ENV"] != "production"
     raise "Multisite migrate is only supported in production"
@@ -209,42 +320,13 @@ task 'multisite:migrate' => ['db:load_config', 'environment', 'set_locale'] do |
   end
 end
 
-# we need to run seed_fu every time we run rake db:migrate
+# Other tasks
-task 'db:migrate' => ['load_config', 'environment', 'set_locale'] do |_, args|
-  DistributedMutex.synchronize('db_migration', redis: Discourse.redis.without_namespace, validity: 300) do
-    migrations = ActiveRecord::Base.connection.migration_context.migrations
-    now_timestamp = Time.now.utc.strftime('%Y%m%d%H%M%S').to_i
-    epoch_timestamp = Time.at(0).utc.strftime('%Y%m%d%H%M%S').to_i
 
-    raise "Migration #{migrations.last.version} is timestamped in the future" if migrations.last.version > now_timestamp
+Rake::Task["db:rollback"].clear
-    raise "Migration #{migrations.first.version} is timestamped before the epoch" if migrations.first.version < epoch_timestamp
+task 'db:rollback' => ['environment', 'set_locale'] do |_, args|
-
+  step = ENV["STEP"] ? ENV["STEP"].to_i : 1
-    %i[pg_trgm unaccent].each do |extension|
+  ActiveRecord::Base.connection.migration_context.rollback(step)
-      begin
+  Rake::Task['db:_dump'].invoke
-        DB.exec "CREATE EXTENSION IF NOT EXISTS #{extension}"
-      rescue => e
-        STDERR.puts "Cannot enable database extension #{extension}"
-        STDERR.puts e
-      end
-    end
-
-    ActiveRecord::Tasks::DatabaseTasks.migrate
-
-    SeedFu.quiet = true
-    SeedFu.seed(SeedHelper.paths, SeedHelper.filter)
-
-    if Rails.env.development? && !ENV["RAILS_DB"]
-      Rake::Task['db:schema:cache:dump'].invoke
-    end
-
-    if !Discourse.skip_post_deployment_migrations? && ENV['SKIP_OPTIMIZE_ICONS'] != '1'
-      SiteIconManager.ensure_optimized!
-    end
-  end
-
-  if !Discourse.is_parallel_test? && MultisiteTestHelpers.load_multisite?
-    system("RAILS_DB=discourse_test_multisite rake db:migrate")
-  end
 end
 
 task 'test:prepare' => 'environment' do
@@ -292,7 +374,6 @@ end
 
 desc 'Statistics about database'
 task 'db:stats' => 'environment' do
-
   sql = <<~SQL
     select table_name,
     (
@@ -343,7 +424,6 @@ end
 
 desc 'Validate indexes'
 task 'db:validate_indexes', [:arg] => ['db:ensure_post_migrations', 'environment'] do |_, args|
-
   db = TemporaryDb.new
   db.start
   db.migrate

lib/temporary_db.rb

@@ -104,13 +104,13 @@ class TemporaryDb
     old_user = ENV["PGUSER"]
     old_port = ENV["PGPORT"]
     old_dev_db = ENV["DISCOURSE_DEV_DB"]
-    old_rails_db = ENV["RAILS_DB"]
+    old_rails_test_db = ENV["RAILS_TEST_DB"]
 
     ENV["PGHOST"] = "localhost"
     ENV["PGUSER"] = "discourse"
     ENV["PGPORT"] = pg_port.to_s
     ENV["DISCOURSE_DEV_DB"] = "discourse"
-    ENV["RAILS_DB"] = "discourse"
+    ENV["RAILS_TEST_DB"] = "discourse"
 
     yield
   ensure
@@ -118,7 +118,7 @@ class TemporaryDb
     ENV["PGUSER"] = old_user
     ENV["PGPORT"] = old_port
     ENV["DISCOURSE_DEV_DB"] = old_dev_db
-    ENV["RAILS_DB"] = old_rails_db
+    ENV["RAILS_TEST_DB"] = old_rails_test_db
   end
 
   def remove