diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 6236a7fd5ec..5155c8a03a3 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -78,11 +78,3 @@ updates: types: patterns: - "@types/*" -# - package-ecosystem: "bundler" -# directory: "migrations/config/gemfiles/convert" -# schedule: -# interval: "weekly" -# day: "wednesday" -# time: "10:00" -# timezone: "Europe/Vienna" -# versioning-strategy: "increase" diff --git a/.github/workflows/migration-tests.yml b/.github/workflows/migration-tests.yml index 436fe691ff0..188d154093f 100644 --- a/.github/workflows/migration-tests.yml +++ b/.github/workflows/migration-tests.yml @@ -37,7 +37,7 @@ jobs: fail-fast: false matrix: - ruby: ["3.2"] + ruby: ["3.3"] steps: - name: Set working directory owner @@ -78,7 +78,7 @@ jobs: ${{ steps.container-envs.outputs.ruby_version }}- ${{ steps.container-envs.outputs.debian_release }}- ${{ hashFiles('**/Gemfile.lock') }}- - ${{ hashFiles('migrations/config/gemfiles/**/Gemfile') }} + migrations-tooling - name: Setup gems run: | @@ -86,8 +86,9 @@ jobs: bundle config --local path vendor/bundle bundle config --local deployment true bundle config --local without development + bundle config --local with migrations bundle install --jobs $(($(nproc) - 1)) - # don't call `bundle clean` clean, we need the gems for the migrations + bundle clean - name: pnpm install run: pnpm install --frozen-lockfile @@ -133,36 +134,3 @@ jobs: - name: RSpec run: bin/rspec --default-path migrations/spec - - runtime: - if: github.event_name == 'pull_request' || github.repository != 'discourse/discourse-private-mirror' - name: Runs on ${{ matrix.os }}, Ruby ${{ matrix.ruby }} - timeout-minutes: 20 - - strategy: - fail-fast: false - - matrix: - os: ["ubuntu-latest", "macos-latest"] - ruby: ["3.2", "3.3"] - - runs-on: ${{ matrix.os }} - - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 1 - - - name: Modify path for libpq - if: matrix.os == 'macos-latest' - run: echo "/opt/homebrew/opt/libpq/bin" >> $GITHUB_PATH - - - name: Setup Ruby - uses: ruby/setup-ruby@v1 - with: - ruby-version: ${{ matrix.ruby }} - bundler-cache: true - - - name: Run converter - working-directory: migrations - run: bin/convert version diff --git a/Gemfile b/Gemfile index f7537b8fd09..d27542cbc56 100644 --- a/Gemfile +++ b/Gemfile @@ -274,3 +274,19 @@ gem "csv", require: false # dependencies for the automation plugin gem "iso8601" gem "rrule" + +group :migrations, optional: true do + gem "extralite-bundle", require: "extralite" + + # auto-loading + gem "zeitwerk" + + # databases + gem "trilogy" + + # CLI + gem "ruby-progressbar" + + # additional Gemfiles from converters + Dir[File.expand_path("migrations/**/Gemfile", __dir__)].each { |path| eval_gemfile(path) } +end diff --git a/Gemfile.lock b/Gemfile.lock index e2e4bf66870..28eb1002cf5 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -137,6 +137,7 @@ GEM excon (0.111.0) execjs (2.9.1) exifr (1.4.0) + extralite-bundle (2.8.2) fabrication (2.31.0) faker (2.23.0) i18n (>= 1.8.11, < 2) @@ -547,6 +548,7 @@ GEM test-prof (1.4.1) thor (1.3.2) timeout (0.4.1) + trilogy (2.8.1) tzinfo (2.0.6) concurrent-ruby (~> 1.0) tzinfo-data (1.2024.2) @@ -624,6 +626,7 @@ DEPENDENCIES email_reply_trimmer excon execjs + extralite-bundle fabrication faker (~> 2.16) fakeweb @@ -704,6 +707,7 @@ DEPENDENCIES rtlcss rubocop-discourse ruby-prof + ruby-progressbar ruby-readability rubyzip sanitize @@ -722,6 +726,7 @@ DEPENDENCIES syntax_tree-disable_ternary test-prof thor + trilogy tzinfo-data uglifier unf @@ -730,6 +735,7 @@ DEPENDENCIES webmock yaml-lint yard + zeitwerk BUNDLED WITH - 2.5.9 + 2.5.18 diff --git a/migrations/.gitignore b/migrations/.gitignore index e12bca7ffbf..702cfb5955d 100644 --- a/migrations/.gitignore +++ b/migrations/.gitignore @@ -1,4 +1,6 @@ -!/db/schema/*.sql +!/db/**/*.sql +!/spec/support/fixtures/**/*.sql tmp/* +private/ Gemfile.lock diff --git a/migrations/README.md b/migrations/README.md index 6f8017faa7d..46cfd4fd8d2 100644 --- a/migrations/README.md +++ b/migrations/README.md @@ -1,7 +1,31 @@ # Migrations Tooling +## Command line interface + +```bash +./bin/cli help +``` + +## Converters + +Public converters are stored in `lib/converters/`. +If you need to run a private converter, put its code into a subdirectory of `private/converters/` + ## Development +### Installing gems + +```bash +bundle config set --local with migrations +bundle install +``` + +### Updating gems + +```bash +bundle update --group migrations +``` + ### Running tests You need to execute `rspec` in the root of the project. diff --git a/migrations/bin/cli b/migrations/bin/cli new file mode 100755 index 00000000000..e6a5ea96b37 --- /dev/null +++ b/migrations/bin/cli @@ -0,0 +1,23 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require "thor" +require_relative "../lib/migrations" + +module Migrations + load_rails_environment + configure_zeitwerk + enable_i18n + + class CommandLineInterface < Thor + include ::Migrations::CLI::ConvertCommand + include ::Migrations::CLI::ImportCommand + include ::Migrations::CLI::UploadCommand + + def self.exit_on_failure? + true + end + end + + CommandLineInterface.start +end diff --git a/migrations/bin/convert b/migrations/bin/convert deleted file mode 100755 index f257698c153..00000000000 --- a/migrations/bin/convert +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env ruby -# frozen_string_literal: true - -require_relative "../lib/migrations" - -module Migrations - load_gemfiles("common") - configure_zeitwerk("lib/common", "lib/converters") - - module Convert - class CLI < Thor - desc "execute", "Run the conversion" - - def execute - FileUtils.mkdir_p("/tmp/converter") - - ::Migrations::IntermediateDatabaseMigrator.reset!("/tmp/converter/intermediate.db") - ::Migrations::IntermediateDatabaseMigrator.migrate("/tmp/converter/intermediate.db") - - # require_relative "converters/pepper/main" - end - - desc "version", "Print the version" - - def version - puts "0.0.1" - end - end - end -end - -Migrations::Convert::CLI.start(ARGV) diff --git a/migrations/bin/import b/migrations/bin/import deleted file mode 100755 index 374223b68f4..00000000000 --- a/migrations/bin/import +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env ruby -# frozen_string_literal: true - -require_relative "../lib/migrations" - -module Migrations - load_rails_environment - - load_gemfiles("common") - configure_zeitwerk("lib/common") - - module Import - class << self - def run - puts "Importing into Discourse #{Discourse::VERSION::STRING}" - puts "Extralite SQLite version: #{Extralite.sqlite3_version}" - end - end - end -end - -Migrations::Import.run diff --git a/migrations/config/gemfiles/README.md b/migrations/config/gemfiles/README.md deleted file mode 100644 index 6080404cf7b..00000000000 --- a/migrations/config/gemfiles/README.md +++ /dev/null @@ -1,22 +0,0 @@ -## Gemfiles for migrations-tooling - -This directory contains Gemfiles for the migration related tools. - -Those tools use `bundler/inline`, so this isn't strictly needed. However, we use GitHub's Dependabot to keep the -dependencies up-to-date, and it requires a Gemfile to work. Also, it's easier to test the tools with a Gemfile. - -Please add an entry in the `.github/workflows/dependabot.yml` file when you add a new Gemfile to enable Dependabot for -the Gemfile. - -#### Example - -```yaml - - package-ecosystem: "bundler" - directory: "migrations/config/gemfiles/convert" - schedule: - interval: "weekly" - day: "wednesday" - time: "10:00" - timezone: "Europe/Vienna" - versioning-strategy: "increase" -``` diff --git a/migrations/config/gemfiles/common/Gemfile b/migrations/config/gemfiles/common/Gemfile deleted file mode 100644 index 1f1de5e271d..00000000000 --- a/migrations/config/gemfiles/common/Gemfile +++ /dev/null @@ -1,30 +0,0 @@ -# frozen_string_literal: true - -source "https://rubygems.org" - -# the minimal Ruby version required by migration-tooling -ruby ">= 3.2.2" - -# `activesupport` gem needs to be in sync with the Rails version of Discourse, see `/Gemfile` -lock_file = %w[Gemfile.lock ../Gemfile.lock].detect { File.exist?(_1) } -activesupport_version = - Bundler::LockfileParser - .new(Bundler.read_file(lock_file)) - .specs - .detect { _1.name == "activesupport" } - .version -gem "activesupport", "= #{activesupport_version}", require: "active_support" - -# for SQLite -gem "extralite-bundle", "~> 2.8", require: "extralite", github: "digital-fabric/extralite" -gem "lru_redux", "~> 1.1", require: false - -# for communication between process forks -gem "msgpack", "~> 1.7" - -# for CLI -gem "colored2", "~> 4.0" -gem "thor", "~> 1.3" - -# auto-loading -gem "zeitwerk", "~> 2.6" diff --git a/migrations/config/locales/migrations.en.yml b/migrations/config/locales/migrations.en.yml new file mode 100644 index 00000000000..3d529b64c64 --- /dev/null +++ b/migrations/config/locales/migrations.en.yml @@ -0,0 +1,19 @@ +en: + progressbar: + warnings: + one: "%{count} warning" + other: "%{count} warnings" + errors: + one: "%{count} error" + other: "%{count} errors" + estimated: "ETA: %{duration}" + elapsed: "Time: %{duration}" + processed: + percentage: "Processed: %{percentage}" + progress: "Processed: %{current}" + progress_with_max: "Processed: %{current} / %{max}" + + converter: + default_step_title: "Converting %{type}" + max_progress_calculation: "Calculating items took %{duration}" + diff --git a/migrations/db/schema/002-config.sql b/migrations/db/intermediate_db_schema/001-config.sql similarity index 100% rename from migrations/db/schema/002-config.sql rename to migrations/db/intermediate_db_schema/001-config.sql diff --git a/migrations/db/schema/003-log_entries.sql b/migrations/db/intermediate_db_schema/002-log_entries.sql similarity index 100% rename from migrations/db/schema/003-log_entries.sql rename to migrations/db/intermediate_db_schema/002-log_entries.sql diff --git a/migrations/db/schema/100-base-schema.sql b/migrations/db/intermediate_db_schema/100-base-schema.sql similarity index 100% rename from migrations/db/schema/100-base-schema.sql rename to migrations/db/intermediate_db_schema/100-base-schema.sql diff --git a/migrations/db/schema/001-schema_migrations.sql b/migrations/db/schema/001-schema_migrations.sql deleted file mode 100644 index 560a70953d3..00000000000 --- a/migrations/db/schema/001-schema_migrations.sql +++ /dev/null @@ -1,6 +0,0 @@ -CREATE TABLE schema_migrations -( - path TEXT NOT NULL PRIMARY KEY, - created_at DATETIME NOT NULL, - sql_hash TEXT NOT NULL -); diff --git a/migrations/lib/converters/.gitkeep b/migrations/db/uploads_db_schema/100-base-schema.sql similarity index 100% rename from migrations/lib/converters/.gitkeep rename to migrations/db/uploads_db_schema/100-base-schema.sql diff --git a/migrations/lib/cli/convert_command.rb b/migrations/lib/cli/convert_command.rb new file mode 100644 index 00000000000..258f6d417a3 --- /dev/null +++ b/migrations/lib/cli/convert_command.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +module Migrations::CLI::ConvertCommand + def self.included(thor) + thor.class_eval do + desc "convert [FROM]", "Convert a file" + option :settings, type: :string, desc: "Path of settings file", banner: "path" + option :reset, type: :boolean, desc: "Reset database before converting data" + def convert(converter_type) + converter_type = converter_type.downcase + validate_converter_type!(converter_type) + + settings = load_settings(converter_type) + + ::Migrations::Database.reset!(settings[:intermediate_db][:path]) if options[:reset] + + converter = "migrations/converters/#{converter_type}/converter".camelize.constantize + converter.new(settings).run + end + + private + + def validate_converter_type!(type) + converter_names = ::Migrations::Converters.names + + raise Thor::Error, <<~MSG if !converter_names.include?(type) + Unknown converter name: #{type} + Valid names are: #{converter_names.join(", ")} + MSG + end + + def validate_settings_path!(settings_path) + if !File.exist?(settings_path) + raise Thor::Error, "Settings file not found: #{settings_path}" + end + end + + def load_settings(converter_type) + settings_path = calculate_settings_path(converter_type) + validate_settings_path!(settings_path) + + YAML.safe_load(File.read(settings_path), symbolize_names: true) + end + + def calculate_settings_path(converter_type) + settings_path = + options[:settings] || ::Migrations::Converters.default_settings_path(converter_type) + File.expand_path(settings_path, Dir.pwd) + end + end + end +end diff --git a/migrations/lib/cli/import_command.rb b/migrations/lib/cli/import_command.rb new file mode 100644 index 00000000000..2709397f87f --- /dev/null +++ b/migrations/lib/cli/import_command.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Migrations::CLI::ImportCommand + def self.included(thor) + thor.class_eval do + desc "import", "Import a file" + def import + require "extralite" + + puts "Importing into Discourse #{Discourse::VERSION::STRING}" + puts "Extralite SQLite version: #{Extralite.sqlite3_version}" + end + end + end +end diff --git a/migrations/lib/cli/upload_command.rb b/migrations/lib/cli/upload_command.rb new file mode 100644 index 00000000000..f85777518ce --- /dev/null +++ b/migrations/lib/cli/upload_command.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +module Migrations::CLI::UploadCommand + def self.included(thor) + thor.class_eval do + desc "upload", "Upload a file" + def upload + puts "Uploading..." + end + end + end +end diff --git a/migrations/lib/common/date_helper.rb b/migrations/lib/common/date_helper.rb new file mode 100644 index 00000000000..9cf8fdc8fd3 --- /dev/null +++ b/migrations/lib/common/date_helper.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +module Migrations + module DateHelper + # based on code from https://gist.github.com/emmahsax/af285a4b71d8506a1625a3e591dc993b + def self.human_readable_time(secs) + return "< 1 second" if secs < 1 + + [[60, :seconds], [60, :minutes], [24, :hours], [Float::INFINITY, :days]].map do |count, name| + next if secs <= 0 + + secs, number = secs.divmod(count) + unless number.to_i == 0 + "#{number.to_i} #{number == 1 ? name.to_s.delete_suffix("s") : name}" + end + end + .compact + .reverse + .join(", ") + end + end +end diff --git a/migrations/lib/common/extended_progress_bar.rb b/migrations/lib/common/extended_progress_bar.rb new file mode 100644 index 00000000000..80b908cee5b --- /dev/null +++ b/migrations/lib/common/extended_progress_bar.rb @@ -0,0 +1,119 @@ +# frozen_string_literal: true + +require "ruby-progressbar" + +module Migrations + class ExtendedProgressBar + def initialize( + max_progress: nil, + report_progress_in_percent: false, + use_custom_progress_increment: false + ) + @max_progress = max_progress + @report_progress_in_percent = report_progress_in_percent + @use_custom_progress_increment = use_custom_progress_increment + + @warning_count = 0 + @error_count = 0 + @extra_information = "" + + @base_format = nil + @progressbar = nil + end + + def run + raise "ProgressBar already started" if @progressbar + + format = setup_progressbar + yield self + finalize_progressbar(format) + + nil + end + + def update(stats) + extra_information_changed = false + + if stats.warning_count > 0 + @warning_count += stats.warning_count + extra_information_changed = true + end + + if stats.error_count > 0 + @error_count += stats.error_count + extra_information_changed = true + end + + if extra_information_changed + @extra_information = +"" + + if @warning_count > 0 + @extra_information << " | " << + I18n.t("progressbar.warnings", count: @warning_count).yellow + end + + if @error_count > 0 + @extra_information << " | " << I18n.t("progressbar.errors", count: @error_count).red + end + + @progressbar.format = "#{@base_format}#{@extra_information}" + end + + if @use_custom_progress_increment + @progressbar.progress += stats.progress + else + @progressbar.increment + end + end + + private + + def setup_progressbar + format = + if @report_progress_in_percent + I18n.t("progressbar.processed.percentage", percentage: "%J%") + elsif @max_progress + I18n.t("progressbar.processed.progress_with_max", current: "%c", max: "%C") + else + I18n.t("progressbar.processed.progress", current: "%c") + end + + @base_format = @max_progress ? " %a |%E | #{format}" : " %a | #{format}" + + @progressbar = + ::ProgressBar.create( + total: @max_progress, + autofinish: false, + projector: { + type: "smoothing", + strength: 0.5, + }, + format: @base_format, + throttle_rate: 0.5, + ) + + format + end + + def finalize_progressbar(format) + print "\033[K" # delete the output of progressbar, because it doesn't overwrite longer lines + final_format = @max_progress ? " %a | #{format}" : " %a | #{format}" + @progressbar.format = "#{final_format}#{@extra_information}" + @progressbar.finish + end + end +end + +class ProgressBar + module Components + class Time + def estimated_with_label(out_of_bounds_time_format = nil) + I18n.t("progressbar.estimated", duration: estimated(out_of_bounds_time_format)) + end + + def elapsed_with_label + I18n.t("progressbar.elapsed", duration: elapsed) + end + end + end +end diff --git a/migrations/lib/common/fork_manager.rb b/migrations/lib/common/fork_manager.rb new file mode 100644 index 00000000000..fa8dddbbe51 --- /dev/null +++ b/migrations/lib/common/fork_manager.rb @@ -0,0 +1,105 @@ +# frozen_string_literal: true + +module Migrations + module ForkManager + @before_fork_hooks = [] + @after_fork_parent_hooks = [] + @after_fork_child_hooks = [] + @execute_parent_forks = true + + class << self + def batch_forks + @execute_parent_forks = false + run_before_fork_hooks + + yield + + run_after_fork_parent_hooks + @execute_parent_forks = true + end + + def before_fork(run_once: false, &block) + if block + @before_fork_hooks << { run_once:, block: } + block + end + end + + def remove_before_fork_hook(block) + @before_fork_hooks.delete_if { |hook| hook[:block] == block } + end + + def after_fork_parent(run_once: false, &block) + if block + @after_fork_parent_hooks << { run_once:, block: } + block + end + end + + def remove_after_fork_parent_hook(block) + @after_fork_parent_hooks.delete_if { |hook| hook[:block] == block } + end + + def after_fork_child(&block) + if block + @after_fork_child_hooks << { run_once: true, block: } + block + end + end + + def remove_after_fork_child_hook(block) + @after_fork_child_hooks.delete_if { |hook| hook[:block] == block } + end + + def fork + run_before_fork_hooks if @execute_parent_forks + + pid = + Process.fork do + run_after_fork_child_hooks + yield + end + + @after_fork_child_hooks.clear + + run_after_fork_parent_hooks if @execute_parent_forks + + pid + end + + def size + @before_fork_hooks.size + @after_fork_parent_hooks.size + @after_fork_child_hooks.size + end + + def clear! + @before_fork_hooks.clear + @after_fork_parent_hooks.clear + @after_fork_child_hooks.clear + end + + private + + def run_before_fork_hooks + run_hooks(@before_fork_hooks) + end + + def run_after_fork_parent_hooks + run_hooks(@after_fork_parent_hooks) + cleanup_run_once_hooks(@after_fork_child_hooks) + end + + def run_after_fork_child_hooks + run_hooks(@after_fork_child_hooks) + end + + def run_hooks(hooks) + hooks.each { |hook| hook[:block].call } + cleanup_run_once_hooks(hooks) + end + + def cleanup_run_once_hooks(hooks) + hooks.delete_if { |hook| hook[:run_once] } + end + end + end +end diff --git a/migrations/lib/common/intermediate_database.rb b/migrations/lib/common/intermediate_database.rb deleted file mode 100644 index b7411ee5708..00000000000 --- a/migrations/lib/common/intermediate_database.rb +++ /dev/null @@ -1,121 +0,0 @@ -# frozen_string_literal: true - -require "extralite" -require "lru_redux" - -module Migrations - class IntermediateDatabase - DEFAULT_JOURNAL_MODE = "wal" - TRANSACTION_BATCH_SIZE = 1000 - PREPARED_STATEMENT_CACHE_SIZE = 5 - - def self.create_connection(path:, journal_mode: DEFAULT_JOURNAL_MODE) - db = ::Extralite::Database.new(path) - db.pragma( - busy_timeout: 60_000, # 60 seconds - journal_mode: journal_mode, - synchronous: "off", - temp_store: "memory", - locking_mode: journal_mode == "wal" ? "normal" : "exclusive", - cache_size: -10_000, # 10_000 pages - ) - db - end - - def self.connect - db = self.class.new - yield(db) - ensure - db.close if db - end - - attr_reader :connection - attr_reader :path - - def initialize(path:, journal_mode: DEFAULT_JOURNAL_MODE) - @path = path - @journal_mode = journal_mode - @connection = self.class.create_connection(path: path, journal_mode: journal_mode) - @statement_counter = 0 - - # don't cache too many prepared statements - @statement_cache = PreparedStatementCache.new(PREPARED_STATEMENT_CACHE_SIZE) - end - - def close - if @connection - commit_transaction - @statement_cache.clear - @connection.close - end - - @connection = nil - @statement_counter = 0 - end - - def reconnect - close - @connection = self.class.create_connection(path: @path, journal_mode: @journal_mode) - end - - def copy_from(source_db_paths) - commit_transaction - @statement_counter = 0 - - table_names = get_table_names - insert_actions = { "config" => "OR REPLACE", "uploads" => "OR IGNORE" } - - source_db_paths.each do |source_db_path| - @connection.execute("ATTACH DATABASE ? AS source", source_db_path) - - table_names.each do |table_name| - or_action = insert_actions[table_name] || "" - @connection.execute( - "INSERT #{or_action} INTO #{table_name} SELECT * FROM source.#{table_name}", - ) - end - - @connection.execute("DETACH DATABASE source") - end - end - - def begin_transaction - return if @connection.transaction_active? - @connection.execute("BEGIN DEFERRED TRANSACTION") - end - - def commit_transaction - return unless @connection.transaction_active? - @connection.execute("COMMIT") - end - - private - - def insert(sql, *parameters) - begin_transaction if @statement_counter == 0 - - stmt = @statement_cache.getset(sql) { @connection.prepare(sql) } - stmt.execute(*parameters) - - if (@statement_counter += 1) > TRANSACTION_BATCH_SIZE - commit_transaction - @statement_counter = 0 - end - end - - def iso8601(column_name, alias_name = nil) - alias_name ||= column_name.split(".").last - "strftime('%Y-%m-%dT%H:%M:%SZ', #{column_name}) AS #{alias_name}" - end - - def get_table_names - @connection.query_splat(<<~SQL) - SELECT name - FROM sqlite_schema - WHERE type = 'table' - AND name NOT LIKE 'sqlite_%' - AND name NOT IN ('schema_migrations', 'config') - SQL - end - end -end diff --git a/migrations/lib/common/intermediate_database_migrator.rb b/migrations/lib/common/intermediate_database_migrator.rb deleted file mode 100644 index 860a6ceecaa..00000000000 --- a/migrations/lib/common/intermediate_database_migrator.rb +++ /dev/null @@ -1,58 +0,0 @@ -# frozen_string_literal: true - -module Migrations - class IntermediateDatabaseMigrator - class << self - def reset!(path) - [path, "#{path}-wal", "#{path}-shm"].each { |p| FileUtils.rm_f(p) if File.exist?(p) } - end - - def migrate(path) - connection = IntermediateDatabase.create_connection(path: path) - performed_migrations = find_performed_migrations(connection) - - path = File.join(::Migrations.root_path, "db", "schema") - migrate_from_path(connection, path, performed_migrations) - - connection.close - end - - private - - def new_database?(connection) - connection.query_single_splat(<<~SQL) == 0 - SELECT COUNT(*) - FROM sqlite_schema - WHERE type = 'table' AND name = 'schema_migrations' - SQL - end - - def find_performed_migrations(connection) - return Set.new if new_database?(connection) - - connection.query_splat(<<~SQL).to_set - SELECT path - FROM schema_migrations - SQL - end - - def migrate_from_path(connection, migration_path, performed_migrations) - file_pattern = File.join(migration_path, "*.sql") - Dir[file_pattern].sort.each do |path| - relative_path = Pathname(path).relative_path_from(Migrations.root_path).to_s - - if performed_migrations.exclude?(relative_path) - sql = File.read(path) - sql_hash = Digest::SHA1.hexdigest(sql) - connection.execute(sql) - - connection.execute(<<~SQL, path: relative_path, sql_hash: sql_hash) - INSERT INTO schema_migrations (path, created_at, sql_hash) - VALUES (:path, datetime('now'), :sql_hash) - SQL - end - end - end - end - end -end diff --git a/migrations/lib/converters.rb b/migrations/lib/converters.rb new file mode 100644 index 00000000000..3881d8f518e --- /dev/null +++ b/migrations/lib/converters.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +module Migrations + module Converters + def self.all + @all_converters ||= + begin + base_path = File.join(::Migrations.root_path, "lib", "converters", "base") + core_paths = Dir[File.join(::Migrations.root_path, "lib", "converters", "*")] + private_paths = Dir[File.join(::Migrations.root_path, "private", "converters", "*")] + all_paths = core_paths - [base_path] + private_paths + + all_paths.each_with_object({}) do |path, hash| + next unless File.directory?(path) + + name = File.basename(path).downcase + existing_path = hash[name] + + raise <<~MSG if existing_path + Duplicate converter name found: #{name} + * #{existing_path} + * #{path} + MSG + + hash[name] = path + end + end + end + + def self.names + self.all.keys.sort + end + + def self.path_of(converter_name) + converter_name = converter_name.downcase + path = self.all[converter_name] + raise "Could not find a converter named '#{converter_name}'" unless path + path + end + + def self.default_settings_path(converter_name) + File.join(path_of(converter_name), "settings.yml") + end + end +end diff --git a/migrations/lib/converters/base/converter.rb b/migrations/lib/converters/base/converter.rb new file mode 100644 index 00000000000..baa9928f9a2 --- /dev/null +++ b/migrations/lib/converters/base/converter.rb @@ -0,0 +1,79 @@ +# frozen_string_literal: true + +module Migrations::Converters::Base + class Converter + attr_accessor :settings + + def initialize(settings) + @settings = settings + end + + def run + if respond_to?(:setup) + puts "Initializing..." + setup + end + + create_database + + steps.each do |step_class| + step = create_step(step_class) + before_step_execution(step) + execute_step(step) + after_step_execution(step) + end + rescue SignalException + STDERR.puts "\nAborted" + exit(1) + ensure + ::Migrations::Database::IntermediateDB.close + end + + def steps + raise NotImplementedError + end + + def before_step_execution(step) + # do nothing + end + + def execute_step(step) + executor = + if step.is_a?(ProgressStep) + ProgressStepExecutor + else + StepExecutor + end + + executor.new(step).execute + end + + def after_step_execution(step) + # do nothing + end + + def step_args(step_class) + {} + end + + private + + def create_database + db_path = File.expand_path(settings[:intermediate_db][:path], ::Migrations.root_path) + ::Migrations::Database.migrate( + db_path, + migrations_path: ::Migrations::Database::INTERMEDIATE_DB_SCHEMA_PATH, + ) + + db = ::Migrations::Database.connect(db_path) + ::Migrations::Database::IntermediateDB.setup(db) + end + + def create_step(step_class) + default_args = { settings: settings } + + args = default_args.merge(step_args(step_class)) + step_class.new(args) + end + end +end diff --git a/migrations/lib/converters/base/parallel_job.rb b/migrations/lib/converters/base/parallel_job.rb new file mode 100644 index 00000000000..478d183f861 --- /dev/null +++ b/migrations/lib/converters/base/parallel_job.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Migrations::Converters::Base + class ParallelJob + def initialize(step) + @step = step + @stats = ProgressStats.new + + @offline_connection = ::Migrations::Database::OfflineConnection.new + + ::Migrations::ForkManager.after_fork_child do + ::Migrations::Database::IntermediateDB.setup(@offline_connection) + end + end + + def run(item) + @stats.reset! + @offline_connection.clear! + + begin + @step.process_item(item, @stats) + rescue StandardError => e + @stats.log_error("Failed to process item", exception: e, details: item) + end + + [@offline_connection.parametrized_insert_statements, @stats] + end + + def cleanup + end + end +end diff --git a/migrations/lib/converters/base/progress_stats.rb b/migrations/lib/converters/base/progress_stats.rb new file mode 100644 index 00000000000..b0cc432bcf3 --- /dev/null +++ b/migrations/lib/converters/base/progress_stats.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +module Migrations::Converters::Base + class ProgressStats + attr_accessor :progress, :warning_count, :error_count + + def initialize + reset! + end + + def reset! + @progress = 1 + @warning_count = 0 + @error_count = 0 + end + + def log_info(message, details: nil) + log(::Migrations::Database::IntermediateDB::LogEntry::INFO, message, details:) + end + + def log_warning(message, exception: nil, details: nil) + @warning_count += 1 + log(::Migrations::Database::IntermediateDB::LogEntry::WARNING, message, exception:, details:) + end + + def log_error(message, exception: nil, details: nil) + @error_count += 1 + log(::Migrations::Database::IntermediateDB::LogEntry::ERROR, message, exception:, details:) + end + + def ==(other) + other.is_a?(ProgressStats) && progress == other.progress && + warning_count == other.warning_count && error_count == other.error_count + end + + private + + def log(type, message, exception: nil, details: nil) + ::Migrations::Database::IntermediateDB::LogEntry.create!( + type:, + message:, + exception:, + details:, + ) + end + end +end diff --git a/migrations/lib/converters/base/progress_step.rb b/migrations/lib/converters/base/progress_step.rb new file mode 100644 index 00000000000..08772f20dc3 --- /dev/null +++ b/migrations/lib/converters/base/progress_step.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +module Migrations::Converters::Base + class ProgressStep < Step + def max_progress + nil + end + + def items + raise NotImplementedError + end + + def process_item(item, stats) + raise NotImplementedError + end + + class << self + def run_in_parallel(value) + @run_in_parallel = !!value + end + + def run_in_parallel? + @run_in_parallel == true + end + + def report_progress_in_percent(value) + @report_progress_in_percent = !!value + end + + def report_progress_in_percent? + @report_progress_in_percent == true + end + + def use_custom_progress_increment(value) + @use_custom_progress_increment = !!value + end + + def use_custom_progress_increment? + @use_custom_progress_increment == true + end + end + end +end diff --git a/migrations/lib/converters/base/progress_step_executor.rb b/migrations/lib/converters/base/progress_step_executor.rb new file mode 100644 index 00000000000..959c7eabac8 --- /dev/null +++ b/migrations/lib/converters/base/progress_step_executor.rb @@ -0,0 +1,123 @@ +# frozen_string_literal: true + +require "etc" +require "colored2" + +module Migrations::Converters::Base + class ProgressStepExecutor + WORKER_COUNT = Etc.nprocessors - 1 # leave 1 CPU free to do other work + MIN_PARALLEL_ITEMS = WORKER_COUNT * 10 + MAX_QUEUE_SIZE = WORKER_COUNT * 100 + PRINT_RUNTIME_AFTER_SECONDS = 5 + + def initialize(step) + @step = step + end + + def execute + @max_progress = calculate_max_progress + + puts @step.class.title + @step.execute + + if execute_in_parallel? + execute_parallel + else + execute_serially + end + end + + private + + def execute_in_parallel? + @step.class.run_in_parallel? && (@max_progress.nil? || @max_progress > MIN_PARALLEL_ITEMS) + end + + def execute_serially + job = SerialJob.new(@step) + + with_progressbar do |progressbar| + @step.items.each do |item| + stats = job.run(item) + progressbar.update(stats) + end + end + end + + def execute_parallel + worker_output_queue = SizedQueue.new(MAX_QUEUE_SIZE) + work_queue = SizedQueue.new(MAX_QUEUE_SIZE) + + workers = start_workers(work_queue, worker_output_queue) + writer_thread = start_db_writer(worker_output_queue) + push_work(work_queue) + + workers.each(&:wait) + worker_output_queue.close + writer_thread.join + end + + def calculate_max_progress + start_time = Time.now + max_progress = @step.max_progress + duration = Time.now - start_time + + if duration > PRINT_RUNTIME_AFTER_SECONDS + message = + I18n.t( + "converter.max_progress_calculation", + duration: ::Migrations::DateHelper.human_readable_time(duration), + ) + puts " #{message}" + end + + max_progress + end + + def with_progressbar + ::Migrations::ExtendedProgressBar + .new( + max_progress: @max_progress, + report_progress_in_percent: @step.class.report_progress_in_percent?, + use_custom_progress_increment: @step.class.use_custom_progress_increment?, + ) + .run { |progressbar| yield progressbar } + end + + def start_db_writer(worker_output_queue) + Thread.new do + Thread.current.name = "writer_thread" + + with_progressbar do |progressbar| + while (parametrized_insert_statements, stats = worker_output_queue.pop) + parametrized_insert_statements.each do |sql, parameters| + ::Migrations::Database::IntermediateDB.insert(sql, *parameters) + end + + progressbar.update(stats) + end + end + end + end + + def start_workers(work_queue, worker_output_queue) + workers = [] + + Process.warmup + + ::Migrations::ForkManager.batch_forks do + WORKER_COUNT.times do |index| + job = ParallelJob.new(@step) + workers << Worker.new(index, work_queue, worker_output_queue, job).start + end + end + + workers + end + + def push_work(work_queue) + @step.items.each { |item| work_queue.push(item) } + work_queue.close + end + end +end diff --git a/migrations/lib/converters/base/serial_job.rb b/migrations/lib/converters/base/serial_job.rb new file mode 100644 index 00000000000..dab7dbee6e3 --- /dev/null +++ b/migrations/lib/converters/base/serial_job.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Migrations::Converters::Base + class SerialJob + def initialize(step) + @step = step + @stats = ProgressStats.new + end + + def run(item) + @stats.reset! + + begin + @step.process_item(item, @stats) + rescue StandardError => e + @stats.log_error("Failed to process item", exception: e, details: item) + end + + @stats + end + + def cleanup + end + end +end diff --git a/migrations/lib/converters/base/step.rb b/migrations/lib/converters/base/step.rb new file mode 100644 index 00000000000..9dc3ba712a1 --- /dev/null +++ b/migrations/lib/converters/base/step.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Migrations::Converters::Base + class Step + IntermediateDB = ::Migrations::Database::IntermediateDB + + attr_accessor :settings + + def initialize(args = {}) + args.each { |arg, value| instance_variable_set("@#{arg}", value) if respond_to?(arg, true) } + end + + def execute + # do nothing + end + + class << self + def title( + value = ( + getter = true + nil + ) + ) + @title = value unless getter + @title.presence || + I18n.t( + "converter.default_step_title", + type: name&.demodulize&.underscore&.humanize(capitalize: false), + ) + end + end + end +end diff --git a/migrations/lib/converters/base/step_executor.rb b/migrations/lib/converters/base/step_executor.rb new file mode 100644 index 00000000000..c97e784252e --- /dev/null +++ b/migrations/lib/converters/base/step_executor.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module Migrations::Converters::Base + class StepExecutor + def initialize(step) + @step = step + end + + def execute + puts @step.class.title + @step.execute + end + end +end diff --git a/migrations/lib/converters/base/worker.rb b/migrations/lib/converters/base/worker.rb new file mode 100644 index 00000000000..83d2b6a659d --- /dev/null +++ b/migrations/lib/converters/base/worker.rb @@ -0,0 +1,101 @@ +# frozen_string_literal: true + +require "oj" + +module Migrations::Converters::Base + class Worker + OJ_SETTINGS = { + mode: :custom, + create_id: "^o", + create_additions: true, + cache_keys: true, + class_cache: true, + symbol_keys: true, + } + + def initialize(index, input_queue, output_queue, job) + @index = index + @input_queue = input_queue + @output_queue = output_queue + @job = job + + @threads = [] + @mutex = Mutex.new + @data_processed = ConditionVariable.new + end + + def start + parent_input_stream, parent_output_stream = IO.pipe + fork_input_stream, fork_output_stream = IO.pipe + + worker_pid = + start_fork(parent_input_stream, parent_output_stream, fork_input_stream, fork_output_stream) + + fork_output_stream.close + parent_input_stream.close + + start_input_thread(parent_output_stream, worker_pid) + start_output_thread(fork_input_stream) + + self + end + + def wait + @threads.each(&:join) + end + + private + + def start_fork(parent_input_stream, parent_output_stream, fork_input_stream, fork_output_stream) + ::Migrations::ForkManager.fork do + begin + Process.setproctitle("worker_process#{@index}") + + parent_output_stream.close + fork_input_stream.close + + Oj.load(parent_input_stream, OJ_SETTINGS) do |data| + result = @job.run(data) + Oj.to_stream(fork_output_stream, result, OJ_SETTINGS) + end + rescue SignalException + exit(1) + ensure + @job.cleanup + end + end + end + + def start_input_thread(output_stream, worker_pid) + @threads << Thread.new do + Thread.current.name = "worker_#{@index}_input" + + begin + while (data = @input_queue.pop) + Oj.to_stream(output_stream, data, OJ_SETTINGS) + @mutex.synchronize { @data_processed.wait(@mutex) } + end + ensure + output_stream.close + Process.waitpid(worker_pid) + end + end + end + + def start_output_thread(input_stream) + @threads << Thread.new do + Thread.current.name = "worker_#{@index}_output" + + begin + Oj.load(input_stream, OJ_SETTINGS) do |data| + @output_queue.push(data) + @mutex.synchronize { @data_processed.signal } + end + ensure + input_stream.close + @mutex.synchronize { @data_processed.signal } + end + end + end + end +end diff --git a/migrations/lib/converters/example/converter.rb b/migrations/lib/converters/example/converter.rb new file mode 100644 index 00000000000..a7fc9de124d --- /dev/null +++ b/migrations/lib/converters/example/converter.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +module Migrations::Converters::Example + class Converter < ::Migrations::Converters::Base::Converter + def steps + [Step1, Step2, Step3, Step4] + end + end +end diff --git a/migrations/lib/converters/example/settings.yml b/migrations/lib/converters/example/settings.yml new file mode 100644 index 00000000000..9b443f611c2 --- /dev/null +++ b/migrations/lib/converters/example/settings.yml @@ -0,0 +1,2 @@ +intermediate_db: + path: "~/intermediate.db" diff --git a/migrations/lib/converters/example/steps/step1.rb b/migrations/lib/converters/example/steps/step1.rb new file mode 100644 index 00000000000..e7cda29538a --- /dev/null +++ b/migrations/lib/converters/example/steps/step1.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +module Migrations::Converters::Example + class Step1 < ::Migrations::Converters::Base::Step + title "Hello world" + + def execute + super + IntermediateDB::LogEntry.create!(type: "info", message: "This is a test") + end + end +end diff --git a/migrations/lib/converters/example/steps/step2.rb b/migrations/lib/converters/example/steps/step2.rb new file mode 100644 index 00000000000..216f1dc067c --- /dev/null +++ b/migrations/lib/converters/example/steps/step2.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +module Migrations::Converters::Example + class Step2 < ::Migrations::Converters::Base::ProgressStep + run_in_parallel false + + def items + [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + end + + def process_item(item, stats) + sleep(0.5) + + stats.warning_count += 1 if item.in?([3, 7, 9]) + stats.error_count += 1 if item.in?([6, 10]) + + IntermediateDB::LogEntry.create!(type: "info", message: "Step2 - #{item}") + end + end +end diff --git a/migrations/lib/converters/example/steps/step3.rb b/migrations/lib/converters/example/steps/step3.rb new file mode 100644 index 00000000000..a6ff4ac9573 --- /dev/null +++ b/migrations/lib/converters/example/steps/step3.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Migrations::Converters::Example + class Step3 < ::Migrations::Converters::Base::ProgressStep + run_in_parallel true + + def max_progress + 1000 + end + + def items + (1..1000).map { |i| { counter: i } } + end + + def process_item(item, stats) + sleep(0.5) + + IntermediateDB::LogEntry.create!(type: "info", message: "Step3 - #{item[:counter]}") + end + end +end diff --git a/migrations/lib/converters/example/steps/subdirectory/step4.rb b/migrations/lib/converters/example/steps/subdirectory/step4.rb new file mode 100644 index 00000000000..c1868f24840 --- /dev/null +++ b/migrations/lib/converters/example/steps/subdirectory/step4.rb @@ -0,0 +1,6 @@ +# frozen_string_literal: true + +module Migrations::Converters::Example + class Step4 < ::Migrations::Converters::Base::Step + end +end diff --git a/migrations/lib/converters/pepper/main.rb b/migrations/lib/converters/pepper/main.rb deleted file mode 100644 index 9b978400d58..00000000000 --- a/migrations/lib/converters/pepper/main.rb +++ /dev/null @@ -1,7 +0,0 @@ -# frozen_string_literal: true - -gemfile do - source "https://rubygems.org" - - gem "hashids" -end diff --git a/migrations/lib/database.rb b/migrations/lib/database.rb new file mode 100644 index 00000000000..aad2d9324dc --- /dev/null +++ b/migrations/lib/database.rb @@ -0,0 +1,59 @@ +# frozen_string_literal: true + +require "date" +require "extralite" +require "ipaddr" + +module Migrations + module Database + INTERMEDIATE_DB_SCHEMA_PATH = File.join(::Migrations.root_path, "db", "intermediate_db_schema") + UPLOADS_DB_SCHEMA_PATH = File.join(::Migrations.root_path, "db", "uploads_db_schema") + + module_function + + def migrate(db_path, migrations_path:) + Migrator.new(db_path).migrate(migrations_path) + end + + def reset!(db_path) + Migrator.new(db_path).reset! + end + + def connect(path) + connection = Connection.new(path:) + return connection unless block_given? + + begin + yield(connection) + ensure + connection.close + end + nil + end + + def format_datetime(value) + value&.utc&.iso8601 + end + + def format_date(value) + value&.to_date&.iso8601 + end + + def format_boolean(value) + return nil if value.nil? + value ? 1 : 0 + end + + def format_ip_address(value) + return nil if value.blank? + IPAddr.new(value).to_s + rescue ArgumentError + nil + end + + def to_blob(value) + return nil if value.blank? + Extralite::Blob.new(value) + end + end +end diff --git a/migrations/lib/database/connection.rb b/migrations/lib/database/connection.rb new file mode 100644 index 00000000000..ca31f7120d6 --- /dev/null +++ b/migrations/lib/database/connection.rb @@ -0,0 +1,101 @@ +# frozen_string_literal: true + +require "extralite" +require "lru_redux" + +module Migrations::Database + class Connection + TRANSACTION_BATCH_SIZE = 1000 + PREPARED_STATEMENT_CACHE_SIZE = 5 + + def self.open_database(path:) + FileUtils.mkdir_p(File.dirname(path)) + + db = Extralite::Database.new(path) + db.pragma( + busy_timeout: 60_000, # 60 seconds + journal_mode: "wal", + synchronous: "off", + temp_store: "memory", + locking_mode: "normal", + cache_size: -10_000, # 10_000 pages + ) + db + end + + attr_reader :db, :path + + def initialize(path:, transaction_batch_size: TRANSACTION_BATCH_SIZE) + @path = path + @transaction_batch_size = transaction_batch_size + @db = self.class.open_database(path:) + @statement_counter = 0 + + # don't cache too many prepared statements + @statement_cache = PreparedStatementCache.new(PREPARED_STATEMENT_CACHE_SIZE) + + @fork_hooks = setup_fork_handling + end + + def close + close_connection(keep_path: false) + + before_hook, after_hook = @fork_hooks + ::Migrations::ForkManager.remove_before_fork_hook(before_hook) + ::Migrations::ForkManager.remove_after_fork_parent_hook(after_hook) + end + + def closed? + !@db || @db.closed? + end + + def insert(sql, parameters = []) + begin_transaction if @statement_counter == 0 + + stmt = @statement_cache.getset(sql) { @db.prepare(sql) } + stmt.execute(parameters) + + if (@statement_counter += 1) >= @transaction_batch_size + commit_transaction + @statement_counter = 0 + end + end + + private + + def begin_transaction + return if @db.transaction_active? + + @db.execute("BEGIN DEFERRED TRANSACTION") + end + + def commit_transaction + return unless @db.transaction_active? + + @db.execute("COMMIT") + end + + def close_connection(keep_path:) + return if !@db + + commit_transaction + @statement_cache.clear + @db.close + + @path = nil unless keep_path + @db = nil + @statement_counter = 0 + end + + def setup_fork_handling + before_hook = ::Migrations::ForkManager.before_fork { close_connection(keep_path: true) } + + after_hook = + ::Migrations::ForkManager.after_fork_parent do + @db = self.class.open_database(path: @path) if @path + end + + [before_hook, after_hook] + end + end +end diff --git a/migrations/lib/database/intermediate_db.rb b/migrations/lib/database/intermediate_db.rb new file mode 100644 index 00000000000..9c88cefe06e --- /dev/null +++ b/migrations/lib/database/intermediate_db.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +require "singleton" + +module Migrations::Database + module IntermediateDB + def self.setup(db_connection) + close + @db = db_connection + end + + def self.insert(sql, *parameters) + @db.insert(sql, parameters) + end + + def self.close + @db.close if @db + end + end +end diff --git a/migrations/lib/database/intermediate_db/log_entry.rb b/migrations/lib/database/intermediate_db/log_entry.rb new file mode 100644 index 00000000000..733272745c3 --- /dev/null +++ b/migrations/lib/database/intermediate_db/log_entry.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Migrations::Database::IntermediateDB + module LogEntry + INFO = "info" + WARNING = "warning" + ERROR = "error" + + SQL = <<~SQL + INSERT INTO log_entries (created_at, type, message, exception, details) + VALUES (?, ?, ?, ?, ?) + SQL + + def self.create!(created_at: Time.now, type:, message:, exception: nil, details: nil) + ::Migrations::Database::IntermediateDB.insert( + SQL, + ::Migrations::Database.format_datetime(created_at), + type, + message, + exception&.full_message(highlight: false), + details, + ) + end + end +end diff --git a/migrations/lib/database/migrator.rb b/migrations/lib/database/migrator.rb new file mode 100644 index 00000000000..abb22057685 --- /dev/null +++ b/migrations/lib/database/migrator.rb @@ -0,0 +1,82 @@ +# frozen_string_literal: true + +module Migrations::Database + class Migrator + def initialize(db_path) + @db_path = db_path + @db = nil + end + + def migrate(migrations_path) + @migrations_path = migrations_path + @db = Connection.open_database(path: @db_path) + + if new_database? + create_schema_migrations_table + performed_migrations = Set.new + else + performed_migrations = find_performed_migrations + end + + migrate_from_path(@migrations_path, performed_migrations) + + @db.close + end + + def reset! + [@db_path, "#{@db_path}-wal", "#{@db_path}-shm"].each do |path| + FileUtils.remove_file(path, force: true) if File.exist?(path) + end + end + + private + + def new_database? + @db.query_single_splat(<<~SQL) == 0 + SELECT COUNT(*) + FROM sqlite_schema + WHERE type = 'table' AND name = 'schema_migrations' + SQL + end + + def find_performed_migrations + @db.query_splat(<<~SQL).to_set + SELECT path + FROM schema_migrations + SQL + end + + def create_schema_migrations_table + @db.execute(<<~SQL) + CREATE TABLE schema_migrations + ( + path TEXT NOT NULL PRIMARY KEY, + created_at DATETIME NOT NULL, + sql_hash TEXT NOT NULL + ); + SQL + end + + def migrate_from_path(migration_path, performed_migrations) + file_pattern = File.join(migration_path, "*.sql") + root_path = @migrations_path || ::Migrations.root_path + + Dir[file_pattern].sort.each do |path| + relative_path = Pathname(path).relative_path_from(root_path).to_s + + if performed_migrations.exclude?(relative_path) + sql = File.read(path) + sql_hash = Digest::SHA1.hexdigest(sql) + + @db.transaction do + @db.execute(sql) + @db.execute(<<~SQL, path: relative_path, sql_hash: sql_hash) + INSERT INTO schema_migrations (path, created_at, sql_hash) + VALUES (:path, datetime('now'), :sql_hash) + SQL + end + end + end + end + end +end diff --git a/migrations/lib/database/offline_connection.rb b/migrations/lib/database/offline_connection.rb new file mode 100644 index 00000000000..dfafe1d498b --- /dev/null +++ b/migrations/lib/database/offline_connection.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module Migrations::Database + class OfflineConnection + def initialize + @parametrized_insert_statements = [] + end + + def close + @parametrized_insert_statements = nil + end + + def closed? + @parametrized_insert_statements.nil? + end + + def insert(sql, parameters = []) + @parametrized_insert_statements << [sql, parameters] + end + + def parametrized_insert_statements + @parametrized_insert_statements + end + + def clear! + @parametrized_insert_statements.clear if @parametrized_insert_statements + end + end +end diff --git a/migrations/lib/common/prepared_statement_cache.rb b/migrations/lib/database/prepared_statement_cache.rb similarity index 82% rename from migrations/lib/common/prepared_statement_cache.rb rename to migrations/lib/database/prepared_statement_cache.rb index 9207e03e46d..1de4a4e6d37 100644 --- a/migrations/lib/common/prepared_statement_cache.rb +++ b/migrations/lib/database/prepared_statement_cache.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true -class Migrations - class PreparedStatementCache < ::LruRedux::Cache +module Migrations::Database + class PreparedStatementCache < LruRedux::Cache class PreparedStatementHash < Hash def shift result = super diff --git a/migrations/lib/migrations.rb b/migrations/lib/migrations.rb index a51170fb3e5..ebc969a6088 100644 --- a/migrations/lib/migrations.rb +++ b/migrations/lib/migrations.rb @@ -1,61 +1,75 @@ # frozen_string_literal: true -require "bundler/inline" -require "bundler/ui" +require "bundler/setup" +Bundler.setup + +require "active_support" +require "active_support/core_ext" +require "zeitwerk" + +require_relative "converters" module Migrations def self.root_path @root_path ||= File.expand_path("..", __dir__) end - def self.load_gemfiles(*relative_paths) - gemfiles_root_path = File.join(Migrations.root_path, "config/gemfiles") - - relative_paths.each do |relative_path| - path = File.join(File.expand_path(relative_path, gemfiles_root_path), "Gemfile") - - unless File.exist?(path) - warn "Could not find Gemfile at #{path}" - exit 1 - end - - gemfile_content = File.read(path) - - # Create new UI and set level to confirm to avoid printing unnecessary messages - bundler_ui = Bundler::UI::Shell.new - bundler_ui.level = "confirm" - - begin - gemfile(true, ui: bundler_ui) do - # rubocop:disable Security/Eval - eval(gemfile_content, nil, path, 1) - # rubocop:enable Security/Eval - end - rescue Bundler::BundlerError => e - warn "\e[31m#{e.message}\e[0m" - exit 1 - end - end - end - def self.load_rails_environment(quiet: false) - puts "Loading application..." unless quiet + message = "Loading Rails environment ..." + print message unless quiet rails_root = File.expand_path("../..", __dir__) # rubocop:disable Discourse/NoChdir - Dir.chdir(rails_root) { require File.join(rails_root, "config/environment") } + Dir.chdir(rails_root) do + begin + require File.join(rails_root, "config/environment") + rescue LoadError => e + $stderr.puts e.message + raise + end + end # rubocop:enable Discourse/NoChdir + + print "\r" + print " " * message.length + print "\r" end - def self.configure_zeitwerk(*directories) - require "zeitwerk" - - root_path = Migrations.root_path - + def self.configure_zeitwerk loader = Zeitwerk::Loader.new - directories.each do |dir| - loader.push_dir(File.expand_path(dir, root_path), namespace: Migrations) + loader.log! if ENV["DEBUG"] + + loader.inflector.inflect( + { "cli" => "CLI", "intermediate_db" => "IntermediateDB", "uploads_db" => "UploadsDB" }, + ) + + loader.push_dir(File.join(::Migrations.root_path, "lib"), namespace: ::Migrations) + loader.push_dir(File.join(::Migrations.root_path, "lib", "common"), namespace: ::Migrations) + + # All sub-directories of a converter should have the same namespace. + # Unfortunately `loader.collapse` doesn't work recursively. + Converters.all.each do |name, converter_path| + module_name = name.camelize.to_sym + namespace = ::Migrations::Converters.const_set(module_name, Module.new) + + Dir[File.join(converter_path, "**", "*")].each do |subdirectory| + next unless File.directory?(subdirectory) + loader.push_dir(subdirectory, namespace: namespace) + end end + loader.setup end + + def self.enable_i18n + require "i18n" + + locale_glob = File.join(::Migrations.root_path, "config", "locales", "**", "migrations.*.yml") + I18n.load_path += Dir[locale_glob] + I18n.backend.load_translations + + # always use English for now + I18n.default_locale = :en + I18n.locale = :en + end end diff --git a/migrations/scripts/benchmarks/RESULTS.md b/migrations/scripts/benchmarks/RESULTS.md new file mode 100644 index 00000000000..3c67ee54758 --- /dev/null +++ b/migrations/scripts/benchmarks/RESULTS.md @@ -0,0 +1,82 @@ +# Benchmark Results + +Here are the latest benchmark results. All benchmarks ran with `ruby 3.2.3 (2024-01-18 revision 52bb2ac0a6) [x86_64-linux]` + +## database_write.rb + +Compares the INSERT speed of SQLite and DuckDB + +| Database | User Time | System Time | Total Time | Real Time | +|-----------|----------:|------------:|-----------:|-----------:| +| SQLite3 | 99.212731 | 4.883932 | 104.096663 | 104.233575 | +| Extralite | 45.396666 | 4.457247 | 49.853913 | 50.065680 | +| DuckDB | 49.012140 | 10.210994 | 59.223134 | 52.095679 | + +## hash_vs_data.rb + +Compares the INSERT speed when the data is bound as Hash or Data class + +``` + Extralite regular 868.703k (± 2.2%) i/s - 8.744M in 10.070511s + Extralite hash 579.753k (± 1.2%) i/s - 5.838M in 10.071266s + Extralite data 672.752k (± 0.8%) i/s - 6.790M in 10.093191s +Extralite data/array 826.296k (± 0.9%) i/s - 8.318M in 10.067518s + SQLite3 regular 362.037k (± 0.7%) i/s - 3.628M in 10.021699s + SQLite3 hash 308.647k (± 1.1%) i/s - 3.111M in 10.081159s + SQLite3 data/hash 288.747k (± 2.7%) i/s - 2.890M in 10.018335s + +Comparison: + Extralite regular: 868702.8 i/s +Extralite data/array: 826295.7 i/s - 1.05x slower + Extralite data: 672752.0 i/s - 1.29x slower + Extralite hash: 579753.5 i/s - 1.50x slower + SQLite3 regular: 362037.0 i/s - 2.40x slower + SQLite3 hash: 308646.7 i/s - 2.81x slower + SQLite3 data/hash: 288747.1 i/s - 3.01x slower +``` + +## parameter_binding.rb + +A similar benchmark that looks at various parameter binding styles, especially in Extralite + +``` + Extralite regular 825.159 (± 0.6%) i/s - 8.316k in 10.078450s + Extralite named 571.135 (± 0.4%) i/s - 5.742k in 10.053796s + Extralite index 769.273 (± 1.0%) i/s - 7.742k in 10.065238s + Extralite array 860.549 (± 0.5%) i/s - 8.624k in 10.021749s + SQLite3 regular 361.745 (± 0.6%) i/s - 3.636k in 10.051588s + SQLite3 named 307.875 (± 0.6%) i/s - 3.090k in 10.036954s + +Comparison: + Extralite array: 860.5 i/s + Extralite regular: 825.2 i/s - 1.04x slower + Extralite index: 769.3 i/s - 1.12x slower + Extralite named: 571.1 i/s - 1.51x slower + SQLite3 regular: 361.7 i/s - 2.38x slower + SQLite3 named: 307.9 i/s - 2.80x slower +``` + +## time_formatting.rb + +Fastest way of converting `Time` into `String`? + +``` + Time#iso8601 1.084M (± 0.9%) i/s - 10.875M in 10.033905s + Time#strftime 1.213M (± 1.4%) i/s - 12.200M in 10.056764s + DateTime#iso8601 2.419M (± 1.8%) i/s - 24.296M in 10.046295s + +Comparison: + DateTime#iso8601: 2419162.1 i/s + Time#strftime: 1213390.0 i/s - 1.99x slower + Time#iso8601: 1083922.8 i/s - 2.23x slower +``` + +## write.rb + +Compares writing lots of data into a single SQLite database. + +``` +single writer 43.9766 seconds +forked writer - same DB 53.5112 seconds +forked writer - multi DB 3.0815 seconds +``` diff --git a/migrations/scripts/benchmarks/hash_vs_data.rb b/migrations/scripts/benchmarks/hash_vs_data.rb index e961dff2f4e..20bb209bcc7 100755 --- a/migrations/scripts/benchmarks/hash_vs_data.rb +++ b/migrations/scripts/benchmarks/hash_vs_data.rb @@ -6,7 +6,7 @@ require "bundler/inline" gemfile(true) do source "https://rubygems.org" gem "benchmark-ips" - gem "extralite-bundle", github: "digital-fabric/extralite" + gem "extralite-bundle" gem "sqlite3" end diff --git a/migrations/scripts/benchmarks/parameter_binding.rb b/migrations/scripts/benchmarks/parameter_binding.rb index 4ac90a73043..a3cc17b4d97 100755 --- a/migrations/scripts/benchmarks/parameter_binding.rb +++ b/migrations/scripts/benchmarks/parameter_binding.rb @@ -6,7 +6,7 @@ require "bundler/inline" gemfile(true) do source "https://rubygems.org" gem "benchmark-ips" - gem "extralite-bundle", github: "digital-fabric/extralite" + gem "extralite-bundle" gem "sqlite3" end diff --git a/migrations/scripts/benchmarks/write.rb b/migrations/scripts/benchmarks/write.rb index f9764bfd209..a05e1d6239f 100755 --- a/migrations/scripts/benchmarks/write.rb +++ b/migrations/scripts/benchmarks/write.rb @@ -5,7 +5,7 @@ require "bundler/inline" gemfile(true) do source "https://rubygems.org" - gem "extralite-bundle", github: "digital-fabric/extralite" + gem "extralite-bundle" end require "etc" @@ -43,7 +43,7 @@ def with_db_path yield tempfile.path db = create_extralite_db(tempfile.path) - row_count = db.query_single_value("SELECT COUNT(*) FROM users") + row_count = db.query_single_splat("SELECT COUNT(*) FROM users") puts "Row count: #{row_count}" if row_count != ROW_COUNT db.close ensure diff --git a/migrations/scripts/generate_schema b/migrations/scripts/generate_schema index fc37df6dbba..13b98a40c7a 100755 --- a/migrations/scripts/generate_schema +++ b/migrations/scripts/generate_schema @@ -11,7 +11,6 @@ require_relative "../lib/migrations" module Migrations load_rails_environment - load_gemfiles("common") class SchemaGenerator def initialize(opts = {}) @@ -305,4 +304,4 @@ module Migrations end end -Migrations::SchemaGenerator.new(output_file_path: ARGV.first).run +::Migrations::SchemaGenerator.new(output_file_path: ARGV.first).run diff --git a/migrations/spec/.gitkeep b/migrations/spec/.gitkeep deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/migrations/spec/bin/import_spec.rb b/migrations/spec/bin/import_spec.rb deleted file mode 100644 index 5496c2c78f8..00000000000 --- a/migrations/spec/bin/import_spec.rb +++ /dev/null @@ -1,15 +0,0 @@ -# frozen_string_literal: true - -RSpec.describe "Migrations::Import" do - subject(:cli) do - # rubocop:disable Discourse/NoChdir - Dir.chdir("migrations") { system("bin/import", exception: true) } - # rubocop:enable Discourse/NoChdir - end - - it "works" do - expect { cli }.to output( - include("Importing into Discourse #{Discourse::VERSION::STRING}"), - ).to_stdout_from_any_process - end -end diff --git a/migrations/spec/lib/converters/base/parallel_job_spec.rb b/migrations/spec/lib/converters/base/parallel_job_spec.rb new file mode 100644 index 00000000000..a1ddb574706 --- /dev/null +++ b/migrations/spec/lib/converters/base/parallel_job_spec.rb @@ -0,0 +1,83 @@ +# frozen_string_literal: true + +RSpec.describe ::Migrations::Converters::Base::ParallelJob do + subject(:job) { described_class.new(step) } + + let(:step) { instance_double(::Migrations::Converters::Base::ProgressStep) } + let(:item) { { key: "value" } } + let(:stats) { instance_double(::Migrations::Converters::Base::ProgressStats) } + let(:intermediate_db) { class_double(::Migrations::Database::IntermediateDB).as_stubbed_const } + + before do + allow(::Migrations::Converters::Base::ProgressStats).to receive(:new).and_return(stats) + + allow(stats).to receive(:reset!) + allow(stats).to receive(:log_error) + allow(intermediate_db).to receive(:setup) + allow(intermediate_db).to receive(:close) + end + + after do + ::Migrations::Database::IntermediateDB.setup(nil) + ::Migrations::ForkManager.clear! + end + + describe "#initialize" do + it "sets up `OfflineConnection` as `IntermediateDB` connection" do + described_class.new(step) + + ::Migrations::ForkManager.fork do + expect(intermediate_db).to have_received(:setup).with( + an_instance_of(::Migrations::Database::OfflineConnection), + ) + end + end + end + + describe "#run" do + let(:offline_connection) { instance_double(::Migrations::Database::OfflineConnection) } + + before do + allow(::Migrations::Database::OfflineConnection).to receive(:new).and_return( + offline_connection, + ) + allow(offline_connection).to receive(:clear!) + + allow(step).to receive(:process_item) + allow(offline_connection).to receive(:parametrized_insert_statements).and_return( + [["SQL", [1, 2]], ["SQL", [2, 3]]], + ) + end + + it "resets stats and clears the offline connection" do + job.run(item) + + expect(stats).to have_received(:reset!) + expect(offline_connection).to have_received(:clear!) + end + + it "processes an item and logs errors if exceptions occur" do + allow(step).to receive(:process_item).and_raise(StandardError.new("error")) + + job.run(item) + + expect(stats).to have_received(:log_error).with( + "Failed to process item", + exception: an_instance_of(StandardError), + details: item, + ) + end + + it "returns the parametrized insert statements and stats" do + result = job.run(item) + + expect(result).to eq([[["SQL", [1, 2]], ["SQL", [2, 3]]], stats]) + end + end + + describe "#cleanup" do + it "can be called without errors" do + expect { job.cleanup }.not_to raise_error + end + end +end diff --git a/migrations/spec/lib/converters/base/progress_stats_spec.rb b/migrations/spec/lib/converters/base/progress_stats_spec.rb new file mode 100644 index 00000000000..94d4c02463b --- /dev/null +++ b/migrations/spec/lib/converters/base/progress_stats_spec.rb @@ -0,0 +1,156 @@ +# frozen_string_literal: true + +RSpec.describe ::Migrations::Converters::Base::ProgressStats do + subject(:stats) { described_class.new } + + describe "#initialize" do + it "starts at the correct values" do + expect(stats.progress).to eq(1) + expect(stats.warning_count).to eq(0) + expect(stats.error_count).to eq(0) + end + end + + describe "attribute accessors" do + it "allows reading and writing for :progress" do + stats.progress = 10 + expect(stats.progress).to eq(10) + end + + it "allows reading and writing for :warning_count" do + stats.warning_count = 5 + expect(stats.warning_count).to eq(5) + end + + it "allows reading and writing for :error_count" do + stats.error_count = 3 + expect(stats.error_count).to eq(3) + end + end + + describe "#reset!" do + before do + stats.progress = 5 + stats.warning_count = 2 + stats.error_count = 3 + stats.reset! + end + + it "resets progress to 1" do + expect(stats.progress).to eq(1) + end + + it "resets warning_count to 0" do + expect(stats.warning_count).to eq(0) + end + + it "resets error_count to 0" do + expect(stats.error_count).to eq(0) + end + end + + describe "#log_info" do + before { allow(::Migrations::Database::IntermediateDB::LogEntry).to receive(:create!) } + + it "logs an info message" do + stats.log_info("Info message") + + expect(::Migrations::Database::IntermediateDB::LogEntry).to have_received(:create!).with( + type: ::Migrations::Database::IntermediateDB::LogEntry::INFO, + message: "Info message", + exception: nil, + details: nil, + ) + end + + it "logs an info message with details" do + stats.log_info("Info message", details: { key: "value" }) + + expect(::Migrations::Database::IntermediateDB::LogEntry).to have_received(:create!).with( + type: ::Migrations::Database::IntermediateDB::LogEntry::INFO, + message: "Info message", + exception: nil, + details: { + key: "value", + }, + ) + end + end + + describe "#log_warning" do + before { allow(::Migrations::Database::IntermediateDB::LogEntry).to receive(:create!) } + + it "logs a warning message and increments warning_count" do + expect { stats.log_warning("Warning message") }.to change { stats.warning_count }.by(1) + + expect(::Migrations::Database::IntermediateDB::LogEntry).to have_received(:create!).with( + type: ::Migrations::Database::IntermediateDB::LogEntry::WARNING, + message: "Warning message", + exception: nil, + details: nil, + ) + end + + it "logs a warning message with exception and details and increments warning_count" do + exception = StandardError.new("Warning exception") + + expect { + stats.log_warning("Warning message", exception: exception, details: { key: "value" }) + }.to change { stats.warning_count }.by(1) + + expect(::Migrations::Database::IntermediateDB::LogEntry).to have_received(:create!).with( + type: ::Migrations::Database::IntermediateDB::LogEntry::WARNING, + message: "Warning message", + exception: exception, + details: { + key: "value", + }, + ) + end + end + + describe "#log_error" do + before { allow(::Migrations::Database::IntermediateDB::LogEntry).to receive(:create!) } + + it "logs an error message and increments error_count" do + expect { stats.log_error("Error message") }.to change { stats.error_count }.by(1) + + expect(::Migrations::Database::IntermediateDB::LogEntry).to have_received(:create!).with( + type: ::Migrations::Database::IntermediateDB::LogEntry::ERROR, + message: "Error message", + exception: nil, + details: nil, + ) + end + + it "logs an error message with exception and details and increments error_count" do + exception = StandardError.new("Error exception") + + expect { + stats.log_error("Error message", exception: exception, details: { key: "value" }) + }.to change { stats.error_count }.by(1) + + expect(::Migrations::Database::IntermediateDB::LogEntry).to have_received(:create!).with( + type: ::Migrations::Database::IntermediateDB::LogEntry::ERROR, + message: "Error message", + exception: exception, + details: { + key: "value", + }, + ) + end + end + + describe "#==" do + let(:other_stats) { described_class.new } + + it "returns true for objects with the same values" do + expect(stats).to eq(other_stats) + end + + it "returns false for objects with different values" do + other_stats.progress = 2 + expect(stats).not_to eq(other_stats) + end + end +end diff --git a/migrations/spec/lib/converters/base/serial_job_spec.rb b/migrations/spec/lib/converters/base/serial_job_spec.rb new file mode 100644 index 00000000000..5910dc72117 --- /dev/null +++ b/migrations/spec/lib/converters/base/serial_job_spec.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +RSpec.describe ::Migrations::Converters::Base::SerialJob do + subject(:job) { described_class.new(step) } + + let(:step) { instance_double(::Migrations::Converters::Base::ProgressStep) } + let(:item) { "Item" } + let(:stats) do + instance_double(::Migrations::Converters::Base::ProgressStats, reset!: nil, log_error: nil) + end + + before { allow(::Migrations::Converters::Base::ProgressStats).to receive(:new).and_return(stats) } + + describe "#run" do + it "resets stats and processes item" do + allow(step).to receive(:process_item).and_return(stats) + + job.run(item) + + expect(stats).to have_received(:reset!) + expect(step).to have_received(:process_item).with(item, stats) + end + + it "logs error if processing item raises an exception" do + allow(step).to receive(:process_item).and_raise(StandardError) + + job.run(item) + + expect(stats).to have_received(:log_error).with( + "Failed to process item", + exception: an_instance_of(StandardError), + details: item, + ) + end + end + + describe "#cleanup" do + it "can be called without errors" do + expect { job.cleanup }.not_to raise_error + end + end +end diff --git a/migrations/spec/lib/converters/base/step_spec.rb b/migrations/spec/lib/converters/base/step_spec.rb new file mode 100644 index 00000000000..ae32a913ce0 --- /dev/null +++ b/migrations/spec/lib/converters/base/step_spec.rb @@ -0,0 +1,59 @@ +# frozen_string_literal: true + +RSpec.describe ::Migrations::Converters::Base::Step do + before do + Object.const_set( + "TemporaryModule", + Module.new do + const_set("TopicUsers", Class.new(::Migrations::Converters::Base::Step) {}) + const_set("Users", Class.new(::Migrations::Converters::Base::Step) {}) + end, + ) + end + + after do + TemporaryModule.send(:remove_const, "TopicUsers") + TemporaryModule.send(:remove_const, "Users") + Object.send(:remove_const, "TemporaryModule") + end + + describe ".title" do + it "uses the classname within title" do + expect(TemporaryModule::TopicUsers.title).to eq("Converting topic users") + expect(TemporaryModule::Users.title).to eq("Converting users") + end + + it "uses the `title` attribute if it has been set" do + TemporaryModule::Users.title "Foo bar" + expect(TemporaryModule::Users.title).to eq("Foo bar") + end + end + + describe "#initialize" do + it "works when no arguments are supplied" do + step = nil + expect { step = TemporaryModule::Users.new }.not_to raise_error + expect(step.settings).to be_nil + end + + it "initializes the `settings` attribute if given" do + settings = { a: 1, b: 2 } + step = TemporaryModule::Users.new(settings:) + expect(step.settings).to eq(settings) + end + + it "initializes additional attributes if they exist" do + TemporaryModule::Users.class_eval { attr_accessor :foo, :bar } + + settings = { a: 1, b: 2 } + foo = "a string" + bar = false + + step = TemporaryModule::Users.new(settings:, foo:, bar:, non_existent: 123) + expect(step.settings).to eq(settings) + expect(step.foo).to eq(foo) + expect(step.bar).to eq(bar) + expect(step).to_not respond_to(:non_existent) + end + end +end diff --git a/migrations/spec/lib/converters/base/worker_spec.rb b/migrations/spec/lib/converters/base/worker_spec.rb new file mode 100644 index 00000000000..1cd29d5b611 --- /dev/null +++ b/migrations/spec/lib/converters/base/worker_spec.rb @@ -0,0 +1,114 @@ +# frozen_string_literal: true + +RSpec.describe ::Migrations::Converters::Base::Worker do + subject(:worker) { described_class.new(index, input_queue, output_queue, job) } + + let(:index) { 1 } + let(:input_queue) { Queue.new } + let(:output_queue) { Queue.new } + let(:job) do + instance_double(::Migrations::Converters::Base::ParallelJob, run: "result", cleanup: nil) + end + + after do + input_queue.close if !input_queue.closed? + output_queue.close if !output_queue.closed? + end + + describe "#start" do + it "works when `input_queue` is empty" do + expect do + worker.start + input_queue.close + worker.wait + output_queue.close + end.not_to raise_error + end + + it "uses `ForkManager.fork`" do + allow(::Migrations::ForkManager).to receive(:fork).and_call_original + + worker.start + input_queue.close + worker.wait + output_queue.close + + expect(::Migrations::ForkManager).to have_received(:fork) + end + + it "writes the output of `job.run` into `output_queue`" do + allow(job).to receive(:run) { |data| "run: #{data[:text]}" } + + worker.start + input_queue << { text: "Item 1" } << { text: "Item 2" } << { text: "Item 3" } + input_queue.close + worker.wait + output_queue.close + + expect(output_queue).to have_queue_contents("run: Item 1", "run: Item 2", "run: Item 3") + end + + def create_progress_stats(progress: 1, warning_count: 0, error_count: 0) + stats = ::Migrations::Converters::Base::ProgressStats.new + stats.progress = progress + stats.warning_count = warning_count + stats.error_count = error_count + stats + end + + it "writes objects to the `output_queue`" do + all_stats = [ + create_progress_stats, + create_progress_stats(warning_count: 1), + create_progress_stats(warning_count: 1, error_count: 1), + create_progress_stats(warning_count: 2, error_count: 1), + ] + + allow(job).to receive(:run) do |data| + index = data[:index] + [index, all_stats[index]] + end + + worker.start + input_queue << { index: 0 } << { index: 1 } << { index: 2 } << { index: 3 } + input_queue.close + worker.wait + output_queue.close + + expect(output_queue).to have_queue_contents( + [0, all_stats[0]], + [1, all_stats[1]], + [2, all_stats[2]], + [3, all_stats[3]], + ) + end + + it "runs `job.cleanup` at the end" do + temp_file = Tempfile.new("method_call_check") + temp_file_path = temp_file.path + + allow(job).to receive(:run) do |data| + File.write(temp_file_path, "run: #{data[:text]}\n", mode: "a+") + data[:text] + end + allow(job).to receive(:cleanup) do + File.write(temp_file_path, "cleanup\n", mode: "a+") + end + + worker.start + input_queue << { text: "Item 1" } << { text: "Item 2" } << { text: "Item 3" } + input_queue.close + worker.wait + output_queue.close + + expect(File.read(temp_file_path)).to eq <<~LOG + run: Item 1 + run: Item 2 + run: Item 3 + cleanup + LOG + ensure + temp_file.unlink + end + end +end diff --git a/migrations/spec/lib/converters_spec.rb b/migrations/spec/lib/converters_spec.rb new file mode 100644 index 00000000000..054235e949b --- /dev/null +++ b/migrations/spec/lib/converters_spec.rb @@ -0,0 +1,105 @@ +# frozen_string_literal: true + +RSpec.describe ::Migrations::Converters do + let(:root_path) { Dir.mktmpdir } + let(:core_path) { File.join(root_path, "lib", "converters") } + let(:private_path) { File.join(root_path, "private", "converters") } + + before do + allow(::Migrations).to receive(:root_path).and_return(root_path) + reset_memoization(described_class, :@all_converters) + end + after do + FileUtils.remove_dir(root_path, force: true) + reset_memoization(described_class, :@all_converters) + end + + def create_converters(core_names: [], private_names: []) + core_names.each { |dir| FileUtils.mkdir_p(File.join(core_path, dir)) } + private_names.each { |dir| FileUtils.mkdir_p(File.join(private_path, dir)) } + end + + describe ".all" do + subject(:all) { described_class.all } + + it "returns all the converters except for 'base'" do + create_converters(core_names: %w[base foo bar]) + + expect(all).to eq( + { "foo" => File.join(core_path, "foo"), "bar" => File.join(core_path, "bar") }, + ) + end + + it "returns converters from core and private directory" do + create_converters(core_names: %w[base foo bar], private_names: %w[baz qux]) + + expect(all).to eq( + { + "foo" => File.join(core_path, "foo"), + "bar" => File.join(core_path, "bar"), + "baz" => File.join(private_path, "baz"), + "qux" => File.join(private_path, "qux"), + }, + ) + end + + it "raises an error if there a duplicate names" do + create_converters(core_names: %w[base foo bar], private_names: %w[foo baz qux]) + + expect { all }.to raise_error(StandardError, /Duplicate converter name found: foo/) + end + end + + describe ".names" do + subject(:names) { described_class.names } + + it "returns a sorted array of converter names" do + create_converters(core_names: %w[base foo bar], private_names: %w[baz qux]) + + expect(names).to eq(%w[bar baz foo qux]) + end + end + + describe ".path_of" do + it "returns the path of a converter" do + create_converters(core_names: %w[base foo bar]) + + expect(described_class.path_of("foo")).to eq(File.join(core_path, "foo")) + end + + it "raises an error if there is no converter" do + create_converters(core_names: %w[base foo bar]) + + expect { described_class.path_of("baz") }.to raise_error( + StandardError, + "Could not find a converter named 'baz'", + ) + expect { described_class.path_of("base") }.to raise_error( + StandardError, + "Could not find a converter named 'base'", + ) + end + end + + describe ".default_settings_path" do + it "returns the path of the default settings file" do + create_converters(core_names: %w[foo bar]) + + expect(described_class.default_settings_path("foo")).to eq( + File.join(core_path, "foo", "settings.yml"), + ) + expect(described_class.default_settings_path("bar")).to eq( + File.join(core_path, "bar", "settings.yml"), + ) + end + + it "raises an error if there is no converter" do + create_converters(core_names: %w[foo bar]) + + expect { described_class.default_settings_path("baz") }.to raise_error( + StandardError, + "Could not find a converter named 'baz'", + ) + end + end +end diff --git a/migrations/spec/lib/database/connection_spec.rb b/migrations/spec/lib/database/connection_spec.rb new file mode 100644 index 00000000000..7edb096df33 --- /dev/null +++ b/migrations/spec/lib/database/connection_spec.rb @@ -0,0 +1,176 @@ +# frozen_string_literal: true + +RSpec.describe ::Migrations::Database::Connection do + def create_connection(**params) + Dir.mktmpdir do |storage_path| + db_path = File.join(storage_path, "test.db") + connection = described_class.new(path: db_path, **params) + + return connection if !block_given? + + begin + yield connection + ensure + connection.close if connection + end + end + end + + describe "class" do + subject(:connection) { create_connection } + + after { connection.close } + + it_behaves_like "a database connection" + end + + describe ".open_database" do + it "creates a database at the given path " do + Dir.mktmpdir do |storage_path| + db_path = File.join(storage_path, "test.db") + db = described_class.open_database(path: db_path) + + expect(File.exist?(db_path)).to be true + expect(db.pragma("journal_mode")).to eq("wal") + expect(db.pragma("locking_mode")).to eq("normal") + ensure + db.close if db + end + end + end + + describe "#close" do + it "closes the underlying database" do + create_connection do |connection| + db = connection.db + connection.close + expect(db).to be_closed + end + end + + it "closes cached prepared statements" do + cache_class = ::Migrations::Database::PreparedStatementCache + cache_double = instance_spy(cache_class) + allow(cache_class).to receive(:new).and_return(cache_double) + + create_connection do |connection| + expect(cache_double).not_to have_received(:clear) + connection.close + expect(cache_double).to have_received(:clear).once + end + end + + it "commits an active transaction" do + create_connection do |connection| + db = described_class.open_database(path: connection.path) + db.execute("CREATE TABLE foo (id INTEGER)") + + connection.insert("INSERT INTO foo (id) VALUES (?)", [1]) + connection.insert("INSERT INTO foo (id) VALUES (?)", [2]) + expect(db.query_single_splat("SELECT COUNT(*) FROM foo")).to eq(0) + + connection.close + expect(db.query_single_splat("SELECT COUNT(*) FROM foo")).to eq(2) + + db.close + end + end + end + + describe "#closed?" do + it "correctly reports if connection is closed" do + create_connection do |connection| + expect(connection.closed?).to be false + connection.close + expect(connection.closed?).to be true + end + end + end + + describe "#insert" do + it "commits inserted rows when reaching `batch_size`" do + transaction_batch_size = 3 + + create_connection(transaction_batch_size:) do |connection| + db = described_class.open_database(path: connection.path) + db.execute("CREATE TABLE foo (id INTEGER)") + + 1.upto(10) do |index| + connection.insert("INSERT INTO foo (id) VALUES (?)", [index]) + + expected_count = index / transaction_batch_size * transaction_batch_size + expect(db.query_single_splat("SELECT COUNT(*) FROM foo")).to eq(expected_count) + end + + db.close + end + end + + it "works with one and more parameters" do + transaction_batch_size = 1 + + create_connection(transaction_batch_size:) do |connection| + db = described_class.open_database(path: connection.path) + db.execute("CREATE TABLE foo (id INTEGER)") + db.execute("CREATE TABLE bar (id INTEGER, name TEXT)") + + connection.insert("INSERT INTO foo (id) VALUES (?)", [1]) + connection.insert("INSERT INTO bar (id, name) VALUES (?, ?)", [1, "Alice"]) + + expect(db.query_splat("SELECT id FROM foo")).to contain_exactly(1) + expect(db.query("SELECT id, name FROM bar")).to contain_exactly({ id: 1, name: "Alice" }) + + db.close + end + end + end + + context "when `::Migrations::ForkManager.fork` is used" do + it "temporarily closes the connection while a process fork is created" do + create_connection do |connection| + expect(connection.closed?).to be false + + connection.db.execute("CREATE TABLE foo (id INTEGER)") + connection.insert("INSERT INTO foo (id) VALUES (?)", [1]) + expect(connection.db.query_splat("SELECT id FROM foo")).to contain_exactly(1) + + db_before_fork = connection.db + + ::Migrations::ForkManager.fork do + expect(connection.closed?).to be true + expect(connection.db).to be_nil + end + + expect(connection.closed?).to be false + expect(connection.db).to_not eq(db_before_fork) + + connection.insert("INSERT INTO foo (id) VALUES (?)", [2]) + expect(connection.db.query_splat("SELECT id FROM foo")).to contain_exactly(1, 2) + end + end + + it "works with multiple forks" do + create_connection do |connection| + expect(connection.closed?).to be false + + ::Migrations::ForkManager.fork { expect(connection.closed?).to be true } + + expect(connection.closed?).to be false + + ::Migrations::ForkManager.fork { expect(connection.closed?).to be true } + + expect(connection.closed?).to be false + end + end + + it "cleans up fork hooks when connection gets closed" do + expect(::Migrations::ForkManager.size).to eq(0) + + create_connection do |connection| + expect(::Migrations::ForkManager.size).to eq(2) + connection.close + expect(::Migrations::ForkManager.size).to eq(0) + end + end + end +end diff --git a/migrations/spec/lib/database/intermediate_db/log_entry_spec.rb b/migrations/spec/lib/database/intermediate_db/log_entry_spec.rb new file mode 100644 index 00000000000..e7e288c3dd0 --- /dev/null +++ b/migrations/spec/lib/database/intermediate_db/log_entry_spec.rb @@ -0,0 +1,5 @@ +# frozen_string_literal: true + +RSpec.describe ::Migrations::Database::IntermediateDB::LogEntry do + it_behaves_like "a database entity" +end diff --git a/migrations/spec/lib/database/intermediate_db_spec.rb b/migrations/spec/lib/database/intermediate_db_spec.rb new file mode 100644 index 00000000000..fd429eef9be --- /dev/null +++ b/migrations/spec/lib/database/intermediate_db_spec.rb @@ -0,0 +1,97 @@ +# frozen_string_literal: true + +RSpec.describe ::Migrations::Database::IntermediateDB do + before { reset_memoization(described_class, :@db) } + after { reset_memoization(described_class, :@db) } + + def create_connection_double + connection = instance_double(::Migrations::Database::Connection) + allow(connection).to receive(:insert) + allow(connection).to receive(:close) + connection + end + + describe ".setup" do + it "works with `::Migrations::Database::Connection`" do + Dir.mktmpdir do |storage_path| + db_path = File.join(storage_path, "test.db") + connection = ::Migrations::Database::Connection.new(path: db_path) + + connection.db.execute("CREATE TABLE foo (id INTEGER)") + + described_class.setup(connection) + described_class.insert("INSERT INTO foo (id) VALUES (?)", 1) + described_class.insert("INSERT INTO foo (id) VALUES (?)", 2) + + expect(connection.db.query_splat("SELECT id FROM foo")).to contain_exactly(1, 2) + + connection.close + end + end + + it "works with `::Migrations::Database::OfflineConnection`" do + connection = ::Migrations::Database::OfflineConnection.new + + described_class.setup(connection) + described_class.insert("INSERT INTO foo (id, name) VALUES (?, ?)", 1, "Alice") + described_class.insert("INSERT INTO foo (id, name) VALUES (?, ?)", 2, "Bob") + + expect(connection.parametrized_insert_statements).to eq( + [ + ["INSERT INTO foo (id, name) VALUES (?, ?)", [1, "Alice"]], + ["INSERT INTO foo (id, name) VALUES (?, ?)", [2, "Bob"]], + ], + ) + + connection.close + end + + it "switches the connection" do + old_connection = create_connection_double + new_connection = create_connection_double + + sql = "INSERT INTO foo (id) VALUES (?)" + + described_class.setup(old_connection) + described_class.insert(sql, 1) + expect(old_connection).to have_received(:insert).with(sql, [1]) + expect(new_connection).to_not have_received(:insert) + + described_class.setup(new_connection) + described_class.insert(sql, 2) + expect(old_connection).to_not have_received(:insert).with(sql, [2]) + expect(new_connection).to have_received(:insert).with(sql, [2]) + end + + it "closes a previous connection" do + old_connection = create_connection_double + new_connection = create_connection_double + + described_class.setup(old_connection) + described_class.setup(new_connection) + expect(old_connection).to have_received(:close) + expect(new_connection).to_not have_received(:close) + end + end + + context "with fake connection" do + let(:connection) { create_connection_double } + let!(:sql) { "INSERT INTO foo (id, name) VALUES (?, ?)" } + + before { described_class.setup(connection) } + + describe ".insert" do + it "calls `#insert` on the connection" do + described_class.insert(sql, 1, "Alice") + expect(connection).to have_received(:insert).with(sql, [1, "Alice"]) + end + end + + describe ".close" do + it "closes the underlying connection" do + described_class.close + expect(connection).to have_received(:close).with(no_args) + end + end + end +end diff --git a/migrations/spec/lib/database/migrator_spec.rb b/migrations/spec/lib/database/migrator_spec.rb new file mode 100644 index 00000000000..04985f25199 --- /dev/null +++ b/migrations/spec/lib/database/migrator_spec.rb @@ -0,0 +1,104 @@ +# frozen_string_literal: true + +RSpec.describe ::Migrations::Database::Migrator do + def migrate( + migrations_directory: nil, + migrations_path: nil, + storage_path: nil, + db_filename: "intermediate.db", + ignore_errors: false + ) + if migrations_directory + migrations_path = + File.join( + ::Migrations.root_path, + "spec", + "support", + "fixtures", + "schema", + migrations_directory, + ) + end + + temp_path = storage_path = Dir.mktmpdir if storage_path.nil? + db_path = File.join(storage_path, db_filename) + + begin + described_class.new(db_path).migrate(migrations_path) + rescue StandardError + raise unless ignore_errors + end + + yield db_path, storage_path + ensure + FileUtils.remove_dir(temp_path, force: true) if temp_path + end + + describe "#migrate" do + it "works with the IntermediateDB schema" do + migrate( + migrations_path: ::Migrations::Database::INTERMEDIATE_DB_SCHEMA_PATH, + db_filename: "intermediate.db", + ) do |db_path, storage_path| + expect(Dir.children(storage_path)).to contain_exactly("intermediate.db") + + db = Extralite::Database.new(db_path) + expect(db.tables).not_to be_empty + db.close + end + end + + it "works with the UploadsDB schema" do + migrate( + migrations_path: ::Migrations::Database::UPLOADS_DB_SCHEMA_PATH, + db_filename: "uploads.db", + ) do |db_path, storage_path| + expect(Dir.children(storage_path)).to contain_exactly("uploads.db") + + db = Extralite::Database.new(db_path) + expect(db.tables).not_to be_empty + db.close + end + end + + it "executes schema files" do + Dir.mktmpdir do |storage_path| + migrate(migrations_directory: "one", storage_path:) do |db_path| + db = Extralite::Database.new(db_path) + expect(db.tables).to contain_exactly("first_table", "schema_migrations") + db.close + end + + migrate(migrations_directory: "one", storage_path:) do |db_path| + db = Extralite::Database.new(db_path) + expect(db.tables).to contain_exactly("first_table", "schema_migrations") + db.close + end + + migrate(migrations_directory: "two", storage_path:) do |db_path| + db = Extralite::Database.new(db_path) + expect(db.tables).to contain_exactly("first_table", "second_table", "schema_migrations") + db.close + end + end + end + end + + describe "#reset!" do + it "deletes all DB related files" do + migrate(migrations_directory: "invalid", ignore_errors: true) do |db_path, storage_path| + File.write(File.join(storage_path, "hello_world.txt"), "Hello World!") + + expect(Dir.children(storage_path)).to contain_exactly( + "intermediate.db", + "intermediate.db-shm", + "intermediate.db-wal", + "hello_world.txt", + ) + + described_class.new(db_path).reset! + expect(Dir.children(storage_path)).to contain_exactly("hello_world.txt") + end + end + end +end diff --git a/migrations/spec/lib/database/offline_connection_spec.rb b/migrations/spec/lib/database/offline_connection_spec.rb new file mode 100644 index 00000000000..0cf43c33592 --- /dev/null +++ b/migrations/spec/lib/database/offline_connection_spec.rb @@ -0,0 +1,67 @@ +# frozen_string_literal: true + +RSpec.describe ::Migrations::Database::OfflineConnection do + subject(:connection) { described_class.new } + + let!(:sql) { "INSERT INTO foo (id, name) VALUES (?, ?)" } + + it_behaves_like "a database connection" + + describe "#close" do + it "removes the cached statements" do + connection.insert(sql, [1, "Alice"]) + connection.insert(sql, [2, "Bob"]) + + expect(connection.parametrized_insert_statements).to_not be_empty + + connection.close + expect(connection.parametrized_insert_statements).to be_nil + end + end + + describe "#closed?" do + it "correctly reports if connection is closed" do + expect(connection.closed?).to be false + connection.close + expect(connection.closed?).to be true + end + end + + describe "#insert" do + it "can be called without errors" do + expect { connection.insert(sql, [1, "Alice"]) }.not_to raise_error + end + end + + describe "#parametrized_insert_statements" do + it "returns an empty array if nothing has been cached" do + expect(connection.parametrized_insert_statements).to eq([]) + end + + it "returns the cached INSERT statements and parameters in original order" do + connection.insert(sql, [1, "Alice"]) + connection.insert(sql, [2, "Bob"]) + connection.insert(sql, [3, "Carol"]) + + expected_data = [[sql, [1, "Alice"]], [sql, [2, "Bob"]], [sql, [3, "Carol"]]] + expect(connection.parametrized_insert_statements).to eq(expected_data) + + # multiple calls return the same data + expect(connection.parametrized_insert_statements).to eq(expected_data) + expect(connection.parametrized_insert_statements).to eq(expected_data) + end + end + + describe "#clear!" do + it "clears all cached data" do + connection.insert(sql, [1, "Alice"]) + connection.insert(sql, [2, "Bob"]) + connection.insert(sql, [3, "Carol"]) + + expect(connection.parametrized_insert_statements).to_not be_empty + + connection.clear! + expect(connection.parametrized_insert_statements).to eq([]) + end + end +end diff --git a/migrations/spec/lib/database/prepared_statement_cache_spec.rb b/migrations/spec/lib/database/prepared_statement_cache_spec.rb new file mode 100644 index 00000000000..007d5ff95e1 --- /dev/null +++ b/migrations/spec/lib/database/prepared_statement_cache_spec.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +require "extralite" + +RSpec.describe ::Migrations::Database::PreparedStatementCache do + let(:cache) { described_class.new(3) } + + def create_statement_double + instance_double(Extralite::Query, close: nil) + end + + it "should inherit behavior from LruRedux::Cache" do + expect(described_class).to be < LruRedux::Cache + end + + it "closes the statement when an old entry is removed" do + cache["a"] = a_statement = create_statement_double + cache["b"] = b_statement = create_statement_double + cache["c"] = c_statement = create_statement_double + + # this should remove the oldest entry "a" from the cache and call #close on the statement + cache["d"] = d_statement = create_statement_double + + expect(a_statement).to have_received(:close) + expect(b_statement).not_to have_received(:close) + expect(c_statement).not_to have_received(:close) + expect(d_statement).not_to have_received(:close) + end + + it "closes all statements when the cache is cleared" do + cache["a"] = a_statement = create_statement_double + cache["b"] = b_statement = create_statement_double + cache["c"] = c_statement = create_statement_double + + cache.clear + + expect(a_statement).to have_received(:close) + expect(b_statement).to have_received(:close) + expect(c_statement).to have_received(:close) + end +end diff --git a/migrations/spec/lib/database_spec.rb b/migrations/spec/lib/database_spec.rb new file mode 100644 index 00000000000..e3fec57d456 --- /dev/null +++ b/migrations/spec/lib/database_spec.rb @@ -0,0 +1,143 @@ +# frozen_string_literal: true + +RSpec.describe ::Migrations::Database do + context "with `Migrator`" do + let(:db_path) { "path/to/db" } + let(:migrations_path) { "path/to/migrations" } + let(:migrator_instance) { instance_double(::Migrations::Database::Migrator) } + + before do + allow(::Migrations::Database::Migrator).to receive(:new).with(db_path).and_return( + migrator_instance, + ) + + allow(::Migrations::Database::Migrator).to receive(:new).with(db_path).and_return( + migrator_instance, + ) + end + + describe ".migrate" do + it "migrates the database" do + allow(migrator_instance).to receive(:migrate) + + described_class.migrate(db_path, migrations_path:) + + expect(::Migrations::Database::Migrator).to have_received(:new).with(db_path) + expect(migrator_instance).to have_received(:migrate).with(migrations_path) + end + end + + describe ".reset!" do + it "resets the database" do + allow(migrator_instance).to receive(:reset!) + + described_class.reset!(db_path) + + expect(::Migrations::Database::Migrator).to have_received(:new).with(db_path) + expect(migrator_instance).to have_received(:reset!) + end + end + end + + describe ".connect" do + it "yields a new connection and closes it after the block" do + Dir.mktmpdir do |storage_path| + db_path = File.join(storage_path, "test.db") + db = nil + + described_class.connect(db_path) do |connection| + expect(connection).to be_a(::Migrations::Database::Connection) + expect(connection.path).to eq(db_path) + + db = connection.db + expect(db).not_to be_closed + end + + expect(db).to be_closed + end + end + + it "closes the connection even if an exception is raised within block" do + Dir.mktmpdir do |storage_path| + db_path = File.join(storage_path, "test.db") + db = nil + + expect { + described_class.connect(db_path) do |connection| + db = connection.db + expect(db).not_to be_closed + raise "boom" + end + }.to raise_error(StandardError) + + expect(db).to be_closed + end + end + end + + describe ".format_datetime" do + it "formats a DateTime object to ISO 8601 string" do + datetime = DateTime.new(2023, 10, 5, 17, 30, 0) + expect(described_class.format_datetime(datetime)).to eq("2023-10-05T17:30:00Z") + end + + it "returns nil for nil input" do + expect(described_class.format_datetime(nil)).to be_nil + end + end + + describe ".format_date" do + it "formats a Date object to ISO 8601 string" do + date = Date.new(2023, 10, 5) + expect(described_class.format_date(date)).to eq("2023-10-05") + end + + it "returns nil for nil input" do + expect(described_class.format_date(nil)).to be_nil + end + end + + describe ".format_boolean" do + it "returns 1 for true" do + expect(described_class.format_boolean(true)).to eq(1) + end + + it "returns 0 for false" do + expect(described_class.format_boolean(false)).to eq(0) + end + + it "returns nil for nil input" do + expect(described_class.format_boolean(nil)).to be_nil + end + end + + describe ".format_ip_address" do + it "formats a valid IPv4 address" do + expect(described_class.format_ip_address("192.168.1.1")).to eq("192.168.1.1") + end + + it "formats a valid IPv6 address" do + expect(described_class.format_ip_address("2001:0db8:85a3:0000:0000:8a2e:0370:7334")).to eq( + "2001:db8:85a3::8a2e:370:7334", + ) + end + + it "returns nil for an invalid IP address" do + expect(described_class.format_ip_address("invalid_ip")).to be_nil + end + + it "returns nil for nil input" do + expect(described_class.format_ip_address(nil)).to be_nil + end + end + + describe ".to_blob" do + it "converts a string to a `Extralite::Blob`" do + expect(described_class.to_blob("Hello, 世界!")).to be_a(Extralite::Blob) + end + + it "returns nil for nil input" do + expect(described_class.to_blob(nil)).to be_nil + end + end +end diff --git a/migrations/spec/lib/migrations_spec.rb b/migrations/spec/lib/migrations_spec.rb index 86d8588f54d..d6bc987d2a9 100644 --- a/migrations/spec/lib/migrations_spec.rb +++ b/migrations/spec/lib/migrations_spec.rb @@ -1,43 +1,9 @@ # frozen_string_literal: true -require_relative "../../lib/migrations" - -RSpec.describe Migrations do +RSpec.describe ::Migrations do describe ".root_path" do it "returns the root path" do expect(described_class.root_path).to eq(File.expand_path("../..", __dir__)) end end - - describe ".load_gemfiles" do - it "exits with error if the gemfile does not exist" do - relative_path = "does_not_exist" - - expect { described_class.load_gemfiles(relative_path) }.to output( - include("Could not find Gemfile").and include(relative_path) - ).to_stderr.and raise_error(SystemExit) { |error| expect(error.status).to eq(1) } - end - - def with_temporary_root_path - Dir.mktmpdir do |temp_dir| - described_class.stubs(:root_path).returns(temp_dir) - yield temp_dir - end - end - - it "exits with an error if the required Ruby version isn't found" do - with_temporary_root_path do |root_path| - gemfile_path = File.join(root_path, "config/gemfiles/test/Gemfile") - FileUtils.mkdir_p(File.dirname(gemfile_path)) - File.write(gemfile_path, <<~GEMFILE) - source "http://localhost" - ruby "~> 100.0.0" - GEMFILE - - expect { described_class.load_gemfiles("test") }.to output( - include("your Gemfile specified ~> 100.0.0"), - ).to_stderr.and raise_error(SystemExit) { |error| expect(error.status).to eq(1) } - end - end - end end diff --git a/migrations/spec/rails_helper.rb b/migrations/spec/rails_helper.rb index d167b6ffecd..f1d6b987b46 100644 --- a/migrations/spec/rails_helper.rb +++ b/migrations/spec/rails_helper.rb @@ -3,18 +3,13 @@ # we need to require the rails_helper from core to load the Rails environment require_relative "../../spec/rails_helper" -require "bundler/inline" -require "bundler/ui" +require_relative "../lib/migrations" -# this is a hack to allow us to load Gemfiles for converters -Dir[File.expand_path("../config/gemfiles/**/Gemfile", __dir__)].each do |path| - # Create new UI and set level to confirm to avoid printing unnecessary messages - bundler_ui = Bundler::UI::Shell.new - bundler_ui.level = "confirm" +::Migrations.configure_zeitwerk +::Migrations.enable_i18n - gemfile(true, ui: bundler_ui) do - # rubocop:disable Security/Eval - eval(File.read(path), nil, path, 1) - # rubocop:enable Security/Eval - end -end +require "rspec-multi-mock" + +Dir[File.expand_path("./support/**/*.rb", __dir__)].each { |f| require f } + +RSpec.configure { |config| config.mock_with MultiMock::Adapter.for(:rspec, :mocha) } diff --git a/migrations/spec/support/fixtures/schema/copy/schema.sql b/migrations/spec/support/fixtures/schema/copy/schema.sql new file mode 100644 index 00000000000..e15177cc1e4 --- /dev/null +++ b/migrations/spec/support/fixtures/schema/copy/schema.sql @@ -0,0 +1,17 @@ +CREATE TABLE users +( + id INTEGER NOT NULL PRIMARY KEY, + username TEXT NOT NULL UNIQUE +); + +CREATE TABLE config +( + name TEXT NOT NULL PRIMARY KEY, + value TEXT NOT NULL +); + +CREATE TABLE uploads +( + id INTEGER NOT NULL PRIMARY KEY, + url TEXT NOT NULL +); diff --git a/migrations/spec/support/fixtures/schema/invalid/001-invalid.sql b/migrations/spec/support/fixtures/schema/invalid/001-invalid.sql new file mode 100644 index 00000000000..1eda88bff30 --- /dev/null +++ b/migrations/spec/support/fixtures/schema/invalid/001-invalid.sql @@ -0,0 +1 @@ +CREATE TABLE defect diff --git a/migrations/spec/support/fixtures/schema/one/001-first-table.sql b/migrations/spec/support/fixtures/schema/one/001-first-table.sql new file mode 100644 index 00000000000..8e007574b38 --- /dev/null +++ b/migrations/spec/support/fixtures/schema/one/001-first-table.sql @@ -0,0 +1,4 @@ +CREATE TABLE first_table +( + id INTEGER TEXT NOT NULL PRIMARY KEY +); diff --git a/migrations/spec/support/fixtures/schema/two/001-first-table.sql b/migrations/spec/support/fixtures/schema/two/001-first-table.sql new file mode 100644 index 00000000000..8e007574b38 --- /dev/null +++ b/migrations/spec/support/fixtures/schema/two/001-first-table.sql @@ -0,0 +1,4 @@ +CREATE TABLE first_table +( + id INTEGER TEXT NOT NULL PRIMARY KEY +); diff --git a/migrations/spec/support/fixtures/schema/two/002-second-table.sql b/migrations/spec/support/fixtures/schema/two/002-second-table.sql new file mode 100644 index 00000000000..39a7fb470a1 --- /dev/null +++ b/migrations/spec/support/fixtures/schema/two/002-second-table.sql @@ -0,0 +1,4 @@ +CREATE TABLE second_table +( + id INTEGER TEXT NOT NULL PRIMARY KEY +); diff --git a/migrations/spec/support/helpers.rb b/migrations/spec/support/helpers.rb new file mode 100644 index 00000000000..dfb1d6c4d98 --- /dev/null +++ b/migrations/spec/support/helpers.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +def reset_memoization(instance, *variables) + variables.each do |var| + instance.remove_instance_variable(var) if instance.instance_variable_defined?(var) + end +end diff --git a/migrations/spec/support/matchers/have_constant.rb b/migrations/spec/support/matchers/have_constant.rb new file mode 100644 index 00000000000..a53261fdcde --- /dev/null +++ b/migrations/spec/support/matchers/have_constant.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +RSpec::Matchers.define :have_constant do |const| + match { |owner| owner.const_defined?(const) } + + failure_message { |owner| "expected #{owner} to have a constant #{const}" } + + failure_message_when_negated do |owner| + "expected #{owner} not to have a constant #{const}, but it does" + end +end diff --git a/migrations/spec/support/matchers/have_queue_contents.rb b/migrations/spec/support/matchers/have_queue_contents.rb new file mode 100644 index 00000000000..2708984bc20 --- /dev/null +++ b/migrations/spec/support/matchers/have_queue_contents.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +RSpec::Matchers.define :have_queue_contents do |*expected| + match do |queue| + @actual = [] + @actual << queue.pop(true) until queue.empty? + @actual == expected + rescue ThreadError + @actual == expected + end + + failure_message do + "expected queue to have contents #{expected.inspect}, but got #{@actual.inspect}" + end + + failure_message_when_negated do + "expected queue not to have contents #{expected.inspect}, but it did" + end +end diff --git a/migrations/spec/support/shared_examples/database_connection.rb b/migrations/spec/support/shared_examples/database_connection.rb new file mode 100644 index 00000000000..3a9b9c3b53d --- /dev/null +++ b/migrations/spec/support/shared_examples/database_connection.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +RSpec.shared_examples "a database connection" do + it "responds to #insert" do + expect(subject).to respond_to(:insert).with(1..2).arguments + end + + it "responds to #close" do + expect(subject).to respond_to(:close).with(0).arguments + end + + it "responds to #closed?" do + expect(subject).to respond_to(:closed?).with(0).arguments + end +end diff --git a/migrations/spec/support/shared_examples/database_entity.rb b/migrations/spec/support/shared_examples/database_entity.rb new file mode 100644 index 00000000000..c0d3f053484 --- /dev/null +++ b/migrations/spec/support/shared_examples/database_entity.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +RSpec.shared_examples "a database entity" do + it "has SQL constant" do + expect(subject).to have_constant(:SQL) + end + + it "responds to .create!" do + expect(subject).to respond_to(:create!) + end +end