PERF: speed up migrations on multisite

Previously we were migrating multisites serially, this is extremely slow
especially when 200 dbs are involved.

The new implementation defaults to running 20 migrations concurrently, leading
to a 20x speedup.

We also amended it so errors are printed out last, something that makes
debugging failures easier.

This is code specific to Discourse cause we integrate SeedFu with our
migrations and can not include this in the multisite gem.

lib/tasks/db.rake

@@ -66,6 +66,71 @@ task 'db:rollback' => ['environment', 'set_locale'] do |_, args|
   Rake::Task['db:_dump'].invoke
 end
 
+# our optimized version of multisite migrate, we have many sites and we have seeds
+# this ensures we can run migrations concurrently to save huge amounts of time
+Rake::Task['multisite:migrate'].clear
+
+task 'multisite:migrate' => ['db:load_config', 'environment', 'set_locale'] do |_, args|
+  if ENV["RAILS_ENV"] != "production"
+    raise "Multisite migrate is only supported in production"
+  end
+
+  concurrency = (ENV['MIGRATE_CONCURRENCY'].presence || "20").to_i
+
+  puts "Multisite migrator is running using #{concurrency} threads"
+  puts
+
+  queue = Queue.new
+  exceptions = Queue.new
+
+  RailsMultisite::ConnectionManagement.each_connection do |db|
+    queue << db
+  end
+
+  concurrency.times { queue << :done }
+
+  SeedFu.quiet = true
+
+  (1..concurrency).map do
+    Thread.new {
+      while true
+        db = queue.pop
+        break if db == :done
+
+        RailsMultisite::ConnectionManagement.with_connection(db) do
+          begin
+            puts "Migrating #{db}"
+            ActiveRecord::Tasks::DatabaseTasks.migrate
+            SeedFu.seed(DiscoursePluginRegistry.seed_paths)
+            if !Discourse.skip_post_deployment_migrations? && ENV['SKIP_OPTIMIZE_ICONS'] != '1'
+              SiteIconManager.ensure_optimized!
+            end
+          rescue => e
+            exceptions << [db, e]
+          end
+        end
+      end
+    }
+  end.each(&:join)
+
+  if exceptions.length > 0
+    STDERR.puts
+    STDERR.puts "-" * 80
+    STDERR.puts "#{exceptions.length} migrations failed!"
+    while !exceptions.empty?
+      db, e = exceptions.pop
+      STDERR.puts
+      STDERR.puts "Failed to migrate #{db}"
+      STDERR.puts e.inspect
+      STDERR.puts e.backtrace
+      STDERR.puts
+    end
+    exit 1
+  end
+
+  Rake::Task['db:_dump'].invoke
+end
+
 # we need to run seed_fu every time we run rake db:migrate
 task 'db:migrate' => ['load_config', 'environment', 'set_locale'] do |_, args|