DEV: Improve multisite db scripts in dev (#17337)
## Without multisite.yml config No change. `bin/rails db:create` / `db:migrate` / `db:drop` should work the same. ## With multisite.yml config ### db:create `bin/rails db:create` creates development, test, and all databases from the multisite config `RAILS_DB=[site] bin/rails db:create` creates the database for the specified site from the multisite config ### db:migrate `bin/rails db:migrate` migrates the development database and all databases from the multisite config `RAILS_ENV=test bin/rails db:migrate` migrates the test database and `discourse_test_multisite` `RAILS_DB=[site] bin/rails db:migrate` migrates the database for the specified site from the multisite config ### db:drop `bin/rails db:drop` drops development, test, and all databases from the multisite config `RAILS_DB=[site] bin/rails db:create` drops the database for the specified site from the multisite config
This commit is contained in:
Jarek Radosz
GitHub
parent
c3fd91670e
commit
2e4056d185
|
|
@ -45,7 +45,7 @@ if ENV['RAILS_ENV'] == "test" && ENV['TEST_ENV_NUMBER']
|
|||
pid = Process.spawn("redis-server --dir tmp/test_data_#{n}/redis --port #{port}", out: "/dev/null")
|
||||
|
||||
ENV["DISCOURSE_REDIS_PORT"] = port.to_s
|
||||
ENV["RAILS_DB"] = "discourse_test_#{n}"
|
||||
ENV["RAILS_TEST_DB"] = "discourse_test_#{n}"
|
||||
|
||||
at_exit do
|
||||
Process.kill("SIGTERM", pid)
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ development:
|
|||
# Do not set this db to the same as development or production.
|
||||
|
||||
<%
|
||||
test_db = ENV["RAILS_DB"]
|
||||
test_db = ENV["RAILS_TEST_DB"]
|
||||
if !test_db.present?
|
||||
test_db = "discourse_test"
|
||||
|
||||
|
|
|
|||
|
|
@ -11,17 +11,17 @@ end
|
|||
|
||||
module MultisiteTestHelpers
|
||||
def self.load_multisite?
|
||||
Rails.env.test? && !ENV["RAILS_DB"] && !ENV["SKIP_MULTISITE"]
|
||||
Rails.env.test? && !ENV["RAILS_TEST_DB"] && !ENV["SKIP_MULTISITE"]
|
||||
end
|
||||
|
||||
def self.create_multisite?
|
||||
(ENV["RAILS_ENV"] == "test" || !ENV["RAILS_ENV"]) && !ENV["RAILS_DB"] && !ENV["SKIP_MULTISITE"]
|
||||
(Rails.env.test? || Rails.env.development?) && !ENV["RAILS_TEST_DB"] && !ENV["DATABASE_URL"] && !ENV["SKIP_MULTISITE"]
|
||||
end
|
||||
end
|
||||
|
||||
task 'db:environment:set' => [:load_config] do |_, args|
|
||||
if MultisiteTestHelpers.load_multisite?
|
||||
system("RAILS_ENV=test RAILS_DB=discourse_test_multisite rake db:environment:set")
|
||||
system("RAILS_ENV=test RAILS_TEST_DB=discourse_test_multisite rake db:environment:set")
|
||||
end
|
||||
end
|
||||
|
||||
|
|
@ -30,46 +30,154 @@ task 'db:force_skip_persist' do
|
|||
GlobalSetting.skip_redis = true
|
||||
end
|
||||
|
||||
task 'db:create' => [:load_config] do |_, args|
|
||||
if MultisiteTestHelpers.create_multisite?
|
||||
unless system("RAILS_ENV=test RAILS_DB=discourse_test_multisite rake db:create")
|
||||
def config_to_url(config)
|
||||
if config[:username] || config[:password]
|
||||
userinfo = [config[:username], config[:password]].join(":")
|
||||
end
|
||||
|
||||
STDERR.puts "-" * 80
|
||||
STDERR.puts "ERROR: Could not create multisite DB. A common cause of this is a plugin"
|
||||
STDERR.puts "checking the column structure when initializing, which raises an error."
|
||||
STDERR.puts "-" * 80
|
||||
raise "Could not initialize discourse_test_multisite"
|
||||
URI::Generic.new(
|
||||
config[:adapter],
|
||||
userinfo,
|
||||
config[:hostname] || "localhost",
|
||||
config[:port],
|
||||
nil,
|
||||
"/#{config[:database]}",
|
||||
nil,
|
||||
nil,
|
||||
nil
|
||||
).to_s
|
||||
end
|
||||
|
||||
# db:create and related tasks
|
||||
|
||||
task "multisite:create" => ["db:load_config"] do
|
||||
next if !Rails.env.development? || !ENV["RAILS_DB"] || ENV["DATABASE_URL"]
|
||||
|
||||
spec = RailsMultisite::ConnectionManagement.connection_spec(db: ENV["RAILS_DB"])
|
||||
database_url = config_to_url(spec.config)
|
||||
system("DATABASE_URL=#{database_url} rake db:create")
|
||||
exit 0
|
||||
end
|
||||
|
||||
task "multisite:create:all" => ["db:load_config"] do
|
||||
next if !MultisiteTestHelpers.create_multisite?
|
||||
|
||||
unless system("RAILS_ENV=test RAILS_TEST_DB=discourse_test_multisite rake db:create")
|
||||
STDERR.puts "-" * 80
|
||||
STDERR.puts "ERROR: Could not create multisite DB. A common cause of this is a plugin"
|
||||
STDERR.puts "checking the column structure when initializing, which raises an error."
|
||||
STDERR.puts "-" * 80
|
||||
raise "Could not initialize discourse_test_multisite"
|
||||
end
|
||||
|
||||
RailsMultisite::ConnectionManagement.all_dbs.each do |db|
|
||||
spec = RailsMultisite::ConnectionManagement.connection_spec(db: db)
|
||||
next unless spec
|
||||
|
||||
database_url = config_to_url(spec.config)
|
||||
system("DATABASE_URL=#{database_url} rake db:create")
|
||||
end
|
||||
end
|
||||
|
||||
task "db:create" => :load_config do
|
||||
Rake::Task["multisite:create:all"].invoke
|
||||
end
|
||||
|
||||
begin
|
||||
prerequisites = Rake::Task['db:create'].prerequisites.map(&:to_sym)
|
||||
Rake::Task['db:create'].clear_prerequisites
|
||||
Rake::Task['db:create'].enhance(["db:force_skip_persist", "multisite:create"] + prerequisites)
|
||||
end
|
||||
|
||||
# db:drop and related tasks
|
||||
|
||||
task "multisite:drop" => ["db:load_config"] do
|
||||
next if !Rails.env.development? || !ENV["RAILS_DB"] || ENV["DATABASE_URL"]
|
||||
|
||||
spec = RailsMultisite::ConnectionManagement.connection_spec(db: ENV["RAILS_DB"])
|
||||
database_url = config_to_url(spec.config)
|
||||
system("DATABASE_URL=#{database_url} rake db:drop")
|
||||
exit 0
|
||||
end
|
||||
|
||||
task "multisite:drop:all" => ["db:load_config"] do
|
||||
next if !MultisiteTestHelpers.create_multisite?
|
||||
|
||||
system("RAILS_TEST_DB=discourse_test_multisite RAILS_ENV=test rake db:drop")
|
||||
|
||||
RailsMultisite::ConnectionManagement.all_dbs.each do |db|
|
||||
spec = RailsMultisite::ConnectionManagement.connection_spec(db: db)
|
||||
next unless spec
|
||||
|
||||
database_url = config_to_url(spec.config)
|
||||
system("DATABASE_URL=#{database_url} rake db:drop")
|
||||
end
|
||||
end
|
||||
|
||||
task "db:drop" => :load_config do
|
||||
Rake::Task["multisite:drop:all"].invoke
|
||||
end
|
||||
|
||||
begin
|
||||
prerequisites = Rake::Task['db:drop'].prerequisites.map(&:to_sym)
|
||||
Rake::Task['db:drop'].clear_prerequisites
|
||||
Rake::Task['db:drop'].enhance(["db:force_skip_persist", "multisite:drop"] + prerequisites)
|
||||
end
|
||||
|
||||
# db:migrate and related tasks
|
||||
|
||||
# we need to run seed_fu every time we run rake db:migrate
|
||||
Rake::Task["db:migrate"].clear
|
||||
task "db:migrate" => ["load_config", "environment", "set_locale"] do |_, args|
|
||||
DistributedMutex.synchronize('db_migration', redis: Discourse.redis.without_namespace, validity: 300) do
|
||||
migrations = ActiveRecord::Base.connection.migration_context.migrations
|
||||
now_timestamp = Time.now.utc.strftime("%Y%m%d%H%M%S").to_i
|
||||
epoch_timestamp = Time.at(0).utc.strftime("%Y%m%d%H%M%S").to_i
|
||||
|
||||
raise "Migration #{migrations.last.version} is timestamped in the future" if migrations.last.version > now_timestamp
|
||||
raise "Migration #{migrations.first.version} is timestamped before the epoch" if migrations.first.version < epoch_timestamp
|
||||
|
||||
%i[pg_trgm unaccent].each do |extension|
|
||||
begin
|
||||
DB.exec "CREATE EXTENSION IF NOT EXISTS #{extension}"
|
||||
rescue => e
|
||||
STDERR.puts "Cannot enable database extension #{extension}"
|
||||
STDERR.puts e
|
||||
end
|
||||
end
|
||||
|
||||
ActiveRecord::Tasks::DatabaseTasks.migrate
|
||||
|
||||
SeedFu.quiet = true
|
||||
SeedFu.seed(SeedHelper.paths, SeedHelper.filter)
|
||||
|
||||
if Rails.env.development? && !ENV["RAILS_TEST_DB"] && !ENV["RAILS_DB"]
|
||||
Rake::Task["db:schema:cache:dump"].invoke
|
||||
end
|
||||
|
||||
if !Discourse.skip_post_deployment_migrations? && ENV["SKIP_OPTIMIZE_ICONS"] != "1"
|
||||
SiteIconManager.ensure_optimized!
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
begin
|
||||
reqs = Rake::Task['db:create'].prerequisites.map(&:to_sym)
|
||||
Rake::Task['db:create'].clear_prerequisites
|
||||
Rake::Task['db:create'].enhance(["db:force_skip_persist"] + reqs)
|
||||
end
|
||||
next if Discourse.is_parallel_test?
|
||||
|
||||
task 'db:drop' => [:load_config] do |_, args|
|
||||
if MultisiteTestHelpers.create_multisite?
|
||||
system("RAILS_DB=discourse_test_multisite RAILS_ENV=test rake db:drop")
|
||||
if MultisiteTestHelpers.load_multisite?
|
||||
system("RAILS_TEST_DB=discourse_test_multisite rake db:migrate")
|
||||
next
|
||||
end
|
||||
|
||||
next if !Rails.env.development? || !ENV["RAILS_DB"] || ENV["DATABASE_URL"]
|
||||
|
||||
RailsMultisite::ConnectionManagement.all_dbs.each do |db|
|
||||
spec = RailsMultisite::ConnectionManagement.connection_spec(db: db)
|
||||
next unless spec
|
||||
|
||||
database_url = config_to_url(spec.config)
|
||||
system("DATABASE_URL=#{database_url} rake db:migrate")
|
||||
end
|
||||
end
|
||||
|
||||
begin
|
||||
Rake::Task["db:migrate"].clear
|
||||
Rake::Task["db:rollback"].clear
|
||||
end
|
||||
|
||||
task 'db:rollback' => ['environment', 'set_locale'] do |_, args|
|
||||
step = ENV["STEP"] ? ENV["STEP"].to_i : 1
|
||||
ActiveRecord::Base.connection.migration_context.rollback(step)
|
||||
Rake::Task['db:_dump'].invoke
|
||||
end
|
||||
|
||||
# our optimized version of multisite migrate, we have many sites and we have seeds
|
||||
# this ensures we can run migrations concurrently to save huge amounts of time
|
||||
Rake::Task['multisite:migrate'].clear
|
||||
|
||||
class StdOutDemux
|
||||
def initialize(stdout)
|
||||
@stdout = stdout
|
||||
|
|
@ -109,6 +217,9 @@ class SeedHelper
|
|||
end
|
||||
end
|
||||
|
||||
# our optimized version of multisite migrate, we have many sites and we have seeds
|
||||
# this ensures we can run migrations concurrently to save huge amounts of time
|
||||
Rake::Task['multisite:migrate'].clear
|
||||
task 'multisite:migrate' => ['db:load_config', 'environment', 'set_locale'] do |_, args|
|
||||
if ENV["RAILS_ENV"] != "production"
|
||||
raise "Multisite migrate is only supported in production"
|
||||
|
|
@ -209,42 +320,13 @@ task 'multisite:migrate' => ['db:load_config', 'environment', 'set_locale'] do |
|
|||
end
|
||||
end
|
||||
|
||||
# we need to run seed_fu every time we run rake db:migrate
|
||||
task 'db:migrate' => ['load_config', 'environment', 'set_locale'] do |_, args|
|
||||
DistributedMutex.synchronize('db_migration', redis: Discourse.redis.without_namespace, validity: 300) do
|
||||
migrations = ActiveRecord::Base.connection.migration_context.migrations
|
||||
now_timestamp = Time.now.utc.strftime('%Y%m%d%H%M%S').to_i
|
||||
epoch_timestamp = Time.at(0).utc.strftime('%Y%m%d%H%M%S').to_i
|
||||
# Other tasks
|
||||
|
||||
raise "Migration #{migrations.last.version} is timestamped in the future" if migrations.last.version > now_timestamp
|
||||
raise "Migration #{migrations.first.version} is timestamped before the epoch" if migrations.first.version < epoch_timestamp
|
||||
|
||||
%i[pg_trgm unaccent].each do |extension|
|
||||
begin
|
||||
DB.exec "CREATE EXTENSION IF NOT EXISTS #{extension}"
|
||||
rescue => e
|
||||
STDERR.puts "Cannot enable database extension #{extension}"
|
||||
STDERR.puts e
|
||||
end
|
||||
end
|
||||
|
||||
ActiveRecord::Tasks::DatabaseTasks.migrate
|
||||
|
||||
SeedFu.quiet = true
|
||||
SeedFu.seed(SeedHelper.paths, SeedHelper.filter)
|
||||
|
||||
if Rails.env.development? && !ENV["RAILS_DB"]
|
||||
Rake::Task['db:schema:cache:dump'].invoke
|
||||
end
|
||||
|
||||
if !Discourse.skip_post_deployment_migrations? && ENV['SKIP_OPTIMIZE_ICONS'] != '1'
|
||||
SiteIconManager.ensure_optimized!
|
||||
end
|
||||
end
|
||||
|
||||
if !Discourse.is_parallel_test? && MultisiteTestHelpers.load_multisite?
|
||||
system("RAILS_DB=discourse_test_multisite rake db:migrate")
|
||||
end
|
||||
Rake::Task["db:rollback"].clear
|
||||
task 'db:rollback' => ['environment', 'set_locale'] do |_, args|
|
||||
step = ENV["STEP"] ? ENV["STEP"].to_i : 1
|
||||
ActiveRecord::Base.connection.migration_context.rollback(step)
|
||||
Rake::Task['db:_dump'].invoke
|
||||
end
|
||||
|
||||
task 'test:prepare' => 'environment' do
|
||||
|
|
@ -292,7 +374,6 @@ end
|
|||
|
||||
desc 'Statistics about database'
|
||||
task 'db:stats' => 'environment' do
|
||||
|
||||
sql = <<~SQL
|
||||
select table_name,
|
||||
(
|
||||
|
|
@ -343,7 +424,6 @@ end
|
|||
|
||||
desc 'Validate indexes'
|
||||
task 'db:validate_indexes', [:arg] => ['db:ensure_post_migrations', 'environment'] do |_, args|
|
||||
|
||||
db = TemporaryDb.new
|
||||
db.start
|
||||
db.migrate
|
||||
|
|
|
|||
|
|
@ -104,13 +104,13 @@ class TemporaryDb
|
|||
old_user = ENV["PGUSER"]
|
||||
old_port = ENV["PGPORT"]
|
||||
old_dev_db = ENV["DISCOURSE_DEV_DB"]
|
||||
old_rails_db = ENV["RAILS_DB"]
|
||||
old_rails_test_db = ENV["RAILS_TEST_DB"]
|
||||
|
||||
ENV["PGHOST"] = "localhost"
|
||||
ENV["PGUSER"] = "discourse"
|
||||
ENV["PGPORT"] = pg_port.to_s
|
||||
ENV["DISCOURSE_DEV_DB"] = "discourse"
|
||||
ENV["RAILS_DB"] = "discourse"
|
||||
ENV["RAILS_TEST_DB"] = "discourse"
|
||||
|
||||
yield
|
||||
ensure
|
||||
|
|
@ -118,7 +118,7 @@ class TemporaryDb
|
|||
ENV["PGUSER"] = old_user
|
||||
ENV["PGPORT"] = old_port
|
||||
ENV["DISCOURSE_DEV_DB"] = old_dev_db
|
||||
ENV["RAILS_DB"] = old_rails_db
|
||||
ENV["RAILS_TEST_DB"] = old_rails_test_db
|
||||
end
|
||||
|
||||
def remove
|
||||
|
|
|
|||
Loading…
Reference in New Issue