diff --git a/config/boot.rb b/config/boot.rb index 6866506f24b..11767b09139 100644 --- a/config/boot.rb +++ b/config/boot.rb @@ -45,7 +45,7 @@ if ENV['RAILS_ENV'] == "test" && ENV['TEST_ENV_NUMBER'] pid = Process.spawn("redis-server --dir tmp/test_data_#{n}/redis --port #{port}", out: "/dev/null") ENV["DISCOURSE_REDIS_PORT"] = port.to_s - ENV["RAILS_DB"] = "discourse_test_#{n}" + ENV["RAILS_TEST_DB"] = "discourse_test_#{n}" at_exit do Process.kill("SIGTERM", pid) diff --git a/config/database.yml b/config/database.yml index 0644f3d527c..83133681d27 100644 --- a/config/database.yml +++ b/config/database.yml @@ -22,7 +22,7 @@ development: # Do not set this db to the same as development or production. <% - test_db = ENV["RAILS_DB"] + test_db = ENV["RAILS_TEST_DB"] if !test_db.present? test_db = "discourse_test" diff --git a/lib/tasks/db.rake b/lib/tasks/db.rake index c14b5666c5b..620e24e7409 100644 --- a/lib/tasks/db.rake +++ b/lib/tasks/db.rake @@ -11,17 +11,17 @@ end module MultisiteTestHelpers def self.load_multisite? - Rails.env.test? && !ENV["RAILS_DB"] && !ENV["SKIP_MULTISITE"] + Rails.env.test? && !ENV["RAILS_TEST_DB"] && !ENV["SKIP_MULTISITE"] end def self.create_multisite? - (ENV["RAILS_ENV"] == "test" || !ENV["RAILS_ENV"]) && !ENV["RAILS_DB"] && !ENV["SKIP_MULTISITE"] + (Rails.env.test? || Rails.env.development?) && !ENV["RAILS_TEST_DB"] && !ENV["DATABASE_URL"] && !ENV["SKIP_MULTISITE"] end end task 'db:environment:set' => [:load_config] do |_, args| if MultisiteTestHelpers.load_multisite? - system("RAILS_ENV=test RAILS_DB=discourse_test_multisite rake db:environment:set") + system("RAILS_ENV=test RAILS_TEST_DB=discourse_test_multisite rake db:environment:set") end end @@ -30,46 +30,154 @@ task 'db:force_skip_persist' do GlobalSetting.skip_redis = true end -task 'db:create' => [:load_config] do |_, args| - if MultisiteTestHelpers.create_multisite? - unless system("RAILS_ENV=test RAILS_DB=discourse_test_multisite rake db:create") +def config_to_url(config) + if config[:username] || config[:password] + userinfo = [config[:username], config[:password]].join(":") + end - STDERR.puts "-" * 80 - STDERR.puts "ERROR: Could not create multisite DB. A common cause of this is a plugin" - STDERR.puts "checking the column structure when initializing, which raises an error." - STDERR.puts "-" * 80 - raise "Could not initialize discourse_test_multisite" + URI::Generic.new( + config[:adapter], + userinfo, + config[:hostname] || "localhost", + config[:port], + nil, + "/#{config[:database]}", + nil, + nil, + nil + ).to_s +end + +# db:create and related tasks + +task "multisite:create" => ["db:load_config"] do + next if !Rails.env.development? || !ENV["RAILS_DB"] || ENV["DATABASE_URL"] + + spec = RailsMultisite::ConnectionManagement.connection_spec(db: ENV["RAILS_DB"]) + database_url = config_to_url(spec.config) + system("DATABASE_URL=#{database_url} rake db:create") + exit 0 +end + +task "multisite:create:all" => ["db:load_config"] do + next if !MultisiteTestHelpers.create_multisite? + + unless system("RAILS_ENV=test RAILS_TEST_DB=discourse_test_multisite rake db:create") + STDERR.puts "-" * 80 + STDERR.puts "ERROR: Could not create multisite DB. A common cause of this is a plugin" + STDERR.puts "checking the column structure when initializing, which raises an error." + STDERR.puts "-" * 80 + raise "Could not initialize discourse_test_multisite" + end + + RailsMultisite::ConnectionManagement.all_dbs.each do |db| + spec = RailsMultisite::ConnectionManagement.connection_spec(db: db) + next unless spec + + database_url = config_to_url(spec.config) + system("DATABASE_URL=#{database_url} rake db:create") + end +end + +task "db:create" => :load_config do + Rake::Task["multisite:create:all"].invoke +end + +begin + prerequisites = Rake::Task['db:create'].prerequisites.map(&:to_sym) + Rake::Task['db:create'].clear_prerequisites + Rake::Task['db:create'].enhance(["db:force_skip_persist", "multisite:create"] + prerequisites) +end + +# db:drop and related tasks + +task "multisite:drop" => ["db:load_config"] do + next if !Rails.env.development? || !ENV["RAILS_DB"] || ENV["DATABASE_URL"] + + spec = RailsMultisite::ConnectionManagement.connection_spec(db: ENV["RAILS_DB"]) + database_url = config_to_url(spec.config) + system("DATABASE_URL=#{database_url} rake db:drop") + exit 0 +end + +task "multisite:drop:all" => ["db:load_config"] do + next if !MultisiteTestHelpers.create_multisite? + + system("RAILS_TEST_DB=discourse_test_multisite RAILS_ENV=test rake db:drop") + + RailsMultisite::ConnectionManagement.all_dbs.each do |db| + spec = RailsMultisite::ConnectionManagement.connection_spec(db: db) + next unless spec + + database_url = config_to_url(spec.config) + system("DATABASE_URL=#{database_url} rake db:drop") + end +end + +task "db:drop" => :load_config do + Rake::Task["multisite:drop:all"].invoke +end + +begin + prerequisites = Rake::Task['db:drop'].prerequisites.map(&:to_sym) + Rake::Task['db:drop'].clear_prerequisites + Rake::Task['db:drop'].enhance(["db:force_skip_persist", "multisite:drop"] + prerequisites) +end + +# db:migrate and related tasks + +# we need to run seed_fu every time we run rake db:migrate +Rake::Task["db:migrate"].clear +task "db:migrate" => ["load_config", "environment", "set_locale"] do |_, args| + DistributedMutex.synchronize('db_migration', redis: Discourse.redis.without_namespace, validity: 300) do + migrations = ActiveRecord::Base.connection.migration_context.migrations + now_timestamp = Time.now.utc.strftime("%Y%m%d%H%M%S").to_i + epoch_timestamp = Time.at(0).utc.strftime("%Y%m%d%H%M%S").to_i + + raise "Migration #{migrations.last.version} is timestamped in the future" if migrations.last.version > now_timestamp + raise "Migration #{migrations.first.version} is timestamped before the epoch" if migrations.first.version < epoch_timestamp + + %i[pg_trgm unaccent].each do |extension| + begin + DB.exec "CREATE EXTENSION IF NOT EXISTS #{extension}" + rescue => e + STDERR.puts "Cannot enable database extension #{extension}" + STDERR.puts e + end + end + + ActiveRecord::Tasks::DatabaseTasks.migrate + + SeedFu.quiet = true + SeedFu.seed(SeedHelper.paths, SeedHelper.filter) + + if Rails.env.development? && !ENV["RAILS_TEST_DB"] && !ENV["RAILS_DB"] + Rake::Task["db:schema:cache:dump"].invoke + end + + if !Discourse.skip_post_deployment_migrations? && ENV["SKIP_OPTIMIZE_ICONS"] != "1" + SiteIconManager.ensure_optimized! end end -end -begin - reqs = Rake::Task['db:create'].prerequisites.map(&:to_sym) - Rake::Task['db:create'].clear_prerequisites - Rake::Task['db:create'].enhance(["db:force_skip_persist"] + reqs) -end + next if Discourse.is_parallel_test? -task 'db:drop' => [:load_config] do |_, args| - if MultisiteTestHelpers.create_multisite? - system("RAILS_DB=discourse_test_multisite RAILS_ENV=test rake db:drop") + if MultisiteTestHelpers.load_multisite? + system("RAILS_TEST_DB=discourse_test_multisite rake db:migrate") + next + end + + next if !Rails.env.development? || !ENV["RAILS_DB"] || ENV["DATABASE_URL"] + + RailsMultisite::ConnectionManagement.all_dbs.each do |db| + spec = RailsMultisite::ConnectionManagement.connection_spec(db: db) + next unless spec + + database_url = config_to_url(spec.config) + system("DATABASE_URL=#{database_url} rake db:migrate") end end -begin - Rake::Task["db:migrate"].clear - Rake::Task["db:rollback"].clear -end - -task 'db:rollback' => ['environment', 'set_locale'] do |_, args| - step = ENV["STEP"] ? ENV["STEP"].to_i : 1 - ActiveRecord::Base.connection.migration_context.rollback(step) - Rake::Task['db:_dump'].invoke -end - -# our optimized version of multisite migrate, we have many sites and we have seeds -# this ensures we can run migrations concurrently to save huge amounts of time -Rake::Task['multisite:migrate'].clear - class StdOutDemux def initialize(stdout) @stdout = stdout @@ -109,6 +217,9 @@ class SeedHelper end end +# our optimized version of multisite migrate, we have many sites and we have seeds +# this ensures we can run migrations concurrently to save huge amounts of time +Rake::Task['multisite:migrate'].clear task 'multisite:migrate' => ['db:load_config', 'environment', 'set_locale'] do |_, args| if ENV["RAILS_ENV"] != "production" raise "Multisite migrate is only supported in production" @@ -209,42 +320,13 @@ task 'multisite:migrate' => ['db:load_config', 'environment', 'set_locale'] do | end end -# we need to run seed_fu every time we run rake db:migrate -task 'db:migrate' => ['load_config', 'environment', 'set_locale'] do |_, args| - DistributedMutex.synchronize('db_migration', redis: Discourse.redis.without_namespace, validity: 300) do - migrations = ActiveRecord::Base.connection.migration_context.migrations - now_timestamp = Time.now.utc.strftime('%Y%m%d%H%M%S').to_i - epoch_timestamp = Time.at(0).utc.strftime('%Y%m%d%H%M%S').to_i +# Other tasks - raise "Migration #{migrations.last.version} is timestamped in the future" if migrations.last.version > now_timestamp - raise "Migration #{migrations.first.version} is timestamped before the epoch" if migrations.first.version < epoch_timestamp - - %i[pg_trgm unaccent].each do |extension| - begin - DB.exec "CREATE EXTENSION IF NOT EXISTS #{extension}" - rescue => e - STDERR.puts "Cannot enable database extension #{extension}" - STDERR.puts e - end - end - - ActiveRecord::Tasks::DatabaseTasks.migrate - - SeedFu.quiet = true - SeedFu.seed(SeedHelper.paths, SeedHelper.filter) - - if Rails.env.development? && !ENV["RAILS_DB"] - Rake::Task['db:schema:cache:dump'].invoke - end - - if !Discourse.skip_post_deployment_migrations? && ENV['SKIP_OPTIMIZE_ICONS'] != '1' - SiteIconManager.ensure_optimized! - end - end - - if !Discourse.is_parallel_test? && MultisiteTestHelpers.load_multisite? - system("RAILS_DB=discourse_test_multisite rake db:migrate") - end +Rake::Task["db:rollback"].clear +task 'db:rollback' => ['environment', 'set_locale'] do |_, args| + step = ENV["STEP"] ? ENV["STEP"].to_i : 1 + ActiveRecord::Base.connection.migration_context.rollback(step) + Rake::Task['db:_dump'].invoke end task 'test:prepare' => 'environment' do @@ -292,7 +374,6 @@ end desc 'Statistics about database' task 'db:stats' => 'environment' do - sql = <<~SQL select table_name, ( @@ -343,7 +424,6 @@ end desc 'Validate indexes' task 'db:validate_indexes', [:arg] => ['db:ensure_post_migrations', 'environment'] do |_, args| - db = TemporaryDb.new db.start db.migrate diff --git a/lib/temporary_db.rb b/lib/temporary_db.rb index 1bcffa570d8..04068cb5064 100644 --- a/lib/temporary_db.rb +++ b/lib/temporary_db.rb @@ -104,13 +104,13 @@ class TemporaryDb old_user = ENV["PGUSER"] old_port = ENV["PGPORT"] old_dev_db = ENV["DISCOURSE_DEV_DB"] - old_rails_db = ENV["RAILS_DB"] + old_rails_test_db = ENV["RAILS_TEST_DB"] ENV["PGHOST"] = "localhost" ENV["PGUSER"] = "discourse" ENV["PGPORT"] = pg_port.to_s ENV["DISCOURSE_DEV_DB"] = "discourse" - ENV["RAILS_DB"] = "discourse" + ENV["RAILS_TEST_DB"] = "discourse" yield ensure @@ -118,7 +118,7 @@ class TemporaryDb ENV["PGUSER"] = old_user ENV["PGPORT"] = old_port ENV["DISCOURSE_DEV_DB"] = old_dev_db - ENV["RAILS_DB"] = old_rails_db + ENV["RAILS_TEST_DB"] = old_rails_test_db end def remove