From 93264da206d225bee6f893ef858c55a0bf1b9204 Mon Sep 17 00:00:00 2001 From: Selase Krakani <849886+s3lase@users.noreply.github.com> Date: Mon, 22 Jan 2024 17:45:52 +0000 Subject: [PATCH] DEV: Add script to generate schema for intermediate DB (#24484) This will be used by migration scripts. --- .github/workflows/migration-tests.yml | 6 + migrations/.gitignore | 2 + .../000_base_schema.sql | 375 ++++++++++++++ migrations/scripts/schema.yml | 481 ++++++++++++++++++ migrations/scripts/schema_generator | 263 ++++++++++ 5 files changed, 1127 insertions(+) create mode 100644 migrations/.gitignore create mode 100644 migrations/common/intermediate_db_schema/000_base_schema.sql create mode 100644 migrations/scripts/schema.yml create mode 100755 migrations/scripts/schema_generator diff --git a/.github/workflows/migration-tests.yml b/.github/workflows/migration-tests.yml index 177e72eaf3d..258ab0a096b 100644 --- a/.github/workflows/migration-tests.yml +++ b/.github/workflows/migration-tests.yml @@ -122,5 +122,11 @@ jobs: if: steps.app-cache.outputs.cache-hit != 'true' run: rm -rf tmp/app-cache/uploads && cp -r public/uploads tmp/app-cache/uploads + - name: Check core database drift + run: | + mkdir /tmp/intermediate_db + ./migrations/scripts/schema_generator /tmp/intermediate_db/base_migration.sql + diff -u migrations/common/intermediate_db_schema/000_base_schema.sql /tmp/intermediate_db/base_migration.sql + - name: RSpec run: bin/rspec migrations/spec/ diff --git a/migrations/.gitignore b/migrations/.gitignore new file mode 100644 index 00000000000..11d154d1c30 --- /dev/null +++ b/migrations/.gitignore @@ -0,0 +1,2 @@ +tmp/* +!/common/intermediate_db_schema/*.sql \ No newline at end of file diff --git a/migrations/common/intermediate_db_schema/000_base_schema.sql b/migrations/common/intermediate_db_schema/000_base_schema.sql new file mode 100644 index 00000000000..7e366738d0f --- /dev/null +++ b/migrations/common/intermediate_db_schema/000_base_schema.sql @@ -0,0 +1,375 @@ +/* + This file is auto-generated from the Discourse core database schema. Instead of editing it directly, + please update the `schema.yml` configuration file and re-run the `generate_schema` script to update it. +*/ + +CREATE TABLE badges ( + id INTEGER NOT NULL PRIMARY KEY, + name TEXT NOT NULL, + description TEXT, + badge_type_id INTEGER NOT NULL, + created_at DATETIME NOT NULL, + multiple_grant BOOLEAN NOT NULL, + query TEXT, + long_description TEXT, + image_upload_id INTEGER, + bage_group TEXT +); + +CREATE TABLE categories ( + id INTEGER NOT NULL PRIMARY KEY, + name TEXT NOT NULL, + color TEXT NOT NULL, + created_at DATETIME NOT NULL, + slug TEXT NOT NULL, + description TEXT, + text_color TEXT NOT NULL, + read_restricted BOOLEAN NOT NULL, + position INTEGER, + parent_category_id INTEGER, + about_topic_title TEXT, + old_relative_url TEXT, + existing_id INTEGER, + permissions JSON_TEXT, + logo_upload_id TEXT, + tag_group_ids JSON_TEXT +); + +CREATE TABLE category_custom_fields ( + category_id INTEGER NOT NULL, + name TEXT NOT NULL, + value TEXT, + PRIMARY KEY (category_id, name) +); + +CREATE TABLE config ( + name TEXT NOT NULL PRIMARY KEY, + value TEXT NOT NULL +); + +CREATE TABLE group_members ( + group_id INTEGER, + user_id INTEGER, + owner BOOLEAN, + PRIMARY KEY (group_id, user_id) +); + +CREATE TABLE groups ( + id INTEGER NOT NULL PRIMARY KEY, + name TEXT NOT NULL, + full_name TEXT, + visibility_level INTEGER NOT NULL, + messageable_level INTEGER, + mentionable_level INTEGER, + members_visibility_level INTEGER NOT NULL, + description TEXT +); + +CREATE TABLE likes ( + post_id INTEGER NOT NULL, + user_id INTEGER NOT NULL, + created_at DATETIME NOT NULL, + PRIMARY KEY (user_id, post_id) +); + +CREATE TABLE log_entries ( + created_at DATETIME NOT NULL, + type TEXT NOT NULL, + message TEXT NOT NULL, + exception TEXT, + details TEXT +); + +CREATE TABLE muted_users ( + user_id INTEGER NOT NULL, + muted_user_id INTEGER NOT NULL, + PRIMARY KEY (user_id, muted_user_id) +); + +CREATE TABLE permalink_normalizations ( + normalization TEXT NOT NULL PRIMARY KEY +); + +CREATE TABLE poll_options ( + id INTEGER NOT NULL PRIMARY KEY, + poll_id INTEGER NOT NULL, + created_at DATETIME, + text TEXT NOT NULL, + position INTEGER +); + +CREATE TABLE poll_votes ( + poll_option_id INTEGER NOT NULL, + user_id INTEGER NOT NULL, + created_at DATETIME, + PRIMARY KEY (poll_option_id, user_id) +); + +CREATE TABLE polls ( + id INTEGER NOT NULL PRIMARY KEY, + post_id INTEGER, + name TEXT NOT NULL, + close_at DATETIME, + type INTEGER NOT NULL, + status INTEGER NOT NULL, + results INTEGER NOT NULL, + visibility INTEGER NOT NULL, + min INTEGER, + max INTEGER, + step INTEGER, + anonymous_voters INTEGER, + created_at DATETIME NOT NULL, + chart_type INTEGER NOT NULL, + groups TEXT, + title TEXT +); + +CREATE TABLE post_custom_fields ( + post_id INTEGER NOT NULL, + name TEXT NOT NULL, + value TEXT, + PRIMARY KEY (post_id, name) +); + +CREATE TABLE posts ( + id INTEGER NOT NULL PRIMARY KEY, + user_id INTEGER, + topic_id INTEGER NOT NULL, + post_number INTEGER NOT NULL, + raw TEXT NOT NULL, + created_at DATETIME NOT NULL, + like_count INTEGER NOT NULL, + reply_to_post_id TEXT, + original_raw TEXT, + upload_ids JSON_TEXT, + old_relative_url TEXT, + accepted_answer BOOLEAN, + small_action TEXT, + whisper BOOLEAN, + placeholders JSON_TEXT +); + +CREATE INDEX posts_by_topic_post_number ON posts (topic_id, post_number); + +CREATE TABLE schema_migrations ( + path TEXT NOT NULL PRIMARY KEY, + created_at DATETIME +); + +CREATE TABLE site_settings ( + name TEXT NOT NULL, + value TEXT, + action TEXT +); + +CREATE TABLE tag_groups ( + id INTEGER NOT NULL PRIMARY KEY, + name TEXT NOT NULL +); + +CREATE TABLE tag_users ( + tag_id INTEGER NOT NULL, + user_id INTEGER NOT NULL, + notification_level INTEGER NOT NULL, + PRIMARY KEY (tag_id, user_id) +); + +CREATE TABLE tags ( + id INTEGER NOT NULL PRIMARY KEY, + name TEXT NOT NULL, + tag_group_id INTEGER +); + +CREATE TABLE topic_tags ( + topic_id INTEGER NOT NULL, + tag_id INTEGER NOT NULL, + PRIMARY KEY (topic_id, tag_id) +); + +CREATE TABLE topic_users ( + user_id INTEGER NOT NULL, + topic_id INTEGER NOT NULL, + last_read_post_number INTEGER, + last_visited_at DATETIME, + first_visited_at DATETIME, + notification_level INTEGER NOT NULL, + notifications_changed_at DATETIME, + notifications_reason_id INTEGER, + total_msecs_viewed INTEGER NOT NULL, + PRIMARY KEY (user_id, topic_id) +); + +CREATE TABLE topics ( + id INTEGER NOT NULL PRIMARY KEY, + title TEXT NOT NULL, + created_at DATETIME NOT NULL, + views INTEGER NOT NULL, + user_id INTEGER, + category_id INTEGER, + visible BOOLEAN NOT NULL, + closed BOOLEAN NOT NULL, + archived BOOLEAN NOT NULL, + pinned_at DATETIME, + subtype TEXT, + pinned_globally BOOLEAN NOT NULL, + pinned_until DATETIME, + old_relative_url TEXT, + private_message TEXT +); + +CREATE TABLE uploads ( + id INTEGER NOT NULL PRIMARY KEY, + user_id INTEGER NOT NULL, + filename TEXT NOT NULL, + relative_path TEXT, + type TEXT, + data BLOB +); + +CREATE TABLE user_badges ( + badge_id INTEGER NOT NULL, + user_id INTEGER NOT NULL, + granted_at DATETIME NOT NULL +); + +CREATE TABLE user_field_values ( + user_id INTEGER NOT NULL, + field_id INTEGER NOT NULL, + is_multiselect_field BOOLEAN NOT NULL, + value TEXT +); + +CREATE UNIQUE INDEX user_field_values_multiselect ON user_field_values (user_id, field_id, value) WHERE is_multiselect_field = TRUE; +CREATE UNIQUE INDEX user_field_values_not_multiselect ON user_field_values (user_id, field_id) WHERE is_multiselect_field = FALSE; + +CREATE TABLE user_fields ( + id INTEGER NOT NULL PRIMARY KEY, + name TEXT NOT NULL, + field_type TEXT NOT NULL, + editable BOOLEAN NOT NULL, + description TEXT NOT NULL, + required BOOLEAN NOT NULL, + show_on_profile BOOLEAN NOT NULL, + position INTEGER, + show_on_user_card BOOLEAN NOT NULL, + searchable BOOLEAN NOT NULL, + options JSON_TEXT +); + +CREATE TABLE users ( + id INTEGER NOT NULL PRIMARY KEY, + username TEXT NOT NULL, + created_at DATETIME, + name TEXT, + last_seen_at DATETIME, + admin BOOLEAN NOT NULL, + trust_level INTEGER, + approved BOOLEAN NOT NULL, + approved_at DATETIME, + date_of_birth DATE, + moderator BOOLEAN, + registration_ip_address TEXT, + staged BOOLEAN, + email TEXT, + avatar_path TEXT, + avatar_url TEXT, + avatar_upload_id TEXT, + bio TEXT, + password TEXT, + suspension TEXT, + location TEXT, + website TEXT, + old_relative_url TEXT, + sso_record TEXT, + anonymized BOOLEAN, + original_username TEXT, + timezone TEXT, + email_level INTEGER, + email_messages_level INTEGER, + email_digests BOOLEAN +); + + +/* +Core table columns implicitly excluded from the generated schema above via the `include` configuration option +in `schema.yml`. This serves as an inventory of these columns, allowing new core additions to be tracked and, +if necessary, synchronized with the intermediate database schema. + +Table: groups +-------------- + created_at datetime false + automatic boolean false + user_count integer false + automatic_membership_email_domains text true + primary_group boolean false + title string true + grant_trust_level integer true + incoming_email string true + has_messages boolean false + flair_bg_color string true + flair_color string true + bio_raw text true + bio_cooked text true + allow_membership_requests boolean false + default_notification_level integer false + public_exit boolean false + public_admission boolean false + membership_request_template text true + smtp_server string true + smtp_port integer true + smtp_ssl boolean true + imap_server string true + imap_port integer true + imap_ssl boolean true + imap_mailbox_name string false + imap_uid_validity integer false + imap_last_uid integer false + email_username string true + email_password string true + publish_read_state boolean false + imap_last_error text true + imap_old_emails integer true + imap_new_emails integer true + flair_icon string true + flair_upload_id integer true + allow_unknown_sender_topic_replies boolean false + smtp_enabled boolean true + smtp_updated_at datetime true + smtp_updated_by_id integer true + imap_enabled boolean true + imap_updated_at datetime true + imap_updated_by_id integer true + email_from_alias string true + +Table: site_settings +--------------------- + id integer false + data_type integer false + created_at datetime false + +Table: tag_groups +------------------ + created_at datetime false + parent_tag_id integer true + one_per_topic boolean true + +Table: tags +------------ + created_at datetime false + pm_topic_count integer false + target_tag_id integer true + description string true + public_topic_count integer false + staff_topic_count integer false + +Table: user_badges +------------------- + id integer false + granted_by_id integer false + post_id integer true + notification_id integer true + seq integer false + featured_rank integer true + created_at datetime false + is_favorite boolean true +*/ \ No newline at end of file diff --git a/migrations/scripts/schema.yml b/migrations/scripts/schema.yml new file mode 100644 index 00000000000..1cd4ca8a1b9 --- /dev/null +++ b/migrations/scripts/schema.yml @@ -0,0 +1,481 @@ +## Configuration options for the base intermediate schema generator +## +## After modifying this file, regenerate the base intermediate schema +## by running the `generate_schema` script. + +# Default relative path for generated base schema file. +# An absolute path can also be provided to the script as the first CLI argument. +# If the CLI argument is present, it takes precedence over the value specified here. +output_file_path: ../common/intermediate_db_schema/000_base_schema.sql + +## Tables to include in the generated base intermediate schema. +## +## Available table options: +## virtual: Boolean. Enables the inclusion of a table in the schema solely based. +## on the provided configuration. A virtual table does not need to be available in the core schema. +## ignore: List of columns to ignore. Convenient if most of the table's column are needed. +## Usage is mutually exclusive with the `include` option. Only one should be used at a time. +## include: List of columns to include. Convenient if only a few columns are needed. +## Usage is mutually exclusive with the `include`` option. Only one should be used at a time. +## primary_key: Literal or list of columns to use as primary key. +## extend: List of objects describing columns to be added/extended. +## The following options are available for an "extend" object: +## name: Required. The name of the column being extended. +## is_null: Specifies if the column can be null. +## type: Column type. Defaults to TEXT. +## indexes: List of indexes to create. The following options are available for an "index" object: +## name: Index name. +## columns: List of column(s) to index. +tables: + schema_migrations: + virtual: true + primary_key: path + extend: + - name: path + is_null: false + - name: created_at + type: datetime + config: + virtual: true + primary_key: name + extend: + - name: name + is_null: false + - name: value + is_null: false + log_entries: + virtual: true + extend: + - name: created_at + type: datetime + is_null: false + - name: type + is_null: false + - name: message + is_null: false + - name: exception + - name: details + users: + ignore: + - seen_notification_id + - last_posted_at + - password_hash + - salt + - active + - last_emailed_at + - approved_by_id + - previous_visit_at + - suspended_at + - suspended_till + - views + - flag_level + - ip_address + - title + - uploaded_avatar_id + - locale + - primary_group_id + - first_seen_at + - silenced_till + - group_locked_trust_level + - manual_locked_trust_level + - secure_identifier + - flair_group_id + - last_seen_reviewable_id + - password_algorithm + - username_lower + extend: + - name: email + - name: created_at + is_null: true + - name: staged + is_null: true + - name: avatar_path + - name: avatar_url + - name: avatar_upload_id + - name: bio + - name: password + is_null: true + - name: trust_level + is_null: true + - name: suspension + - name: location + - name: website + - name: old_relative_url + - name: sso_record + - name: anonymized + type: boolean + - name: original_username + - name: timezone + - name: email_level + type: integer + - name: email_messages_level + type: integer + - name: email_digests + type: boolean + categories: + ignore: + - topic_id + - topic_count + - user_id + - topics_year + - topics_month + - topics_week + - auto_close_hours + - post_count + - latest_post_id + - latest_topic_id + - posts_year + - posts_month + - posts_week + - email_in + - email_in_allow_strangers + - topics_day + - posts_day + - allow_badges + - name_lower + - auto_close_based_on_last_post + - topic_template + - contains_messages + - sort_order + - sort_ascending + - uploaded_logo_id + - uploaded_background_id + - topic_featured_link_allowed + - all_topics_wiki + - show_subcategory_list + - num_featured_topics + - default_view + - subcategory_list_style + - default_top_period + - mailinglist_mirror + - minimum_required_tags + - navigate_to_first_post_after_read + - search_priority + - allow_global_tags + - reviewable_by_group_id + - read_only_banner + - default_list_filter + - allow_unlimited_owner_edits_on_first_post + - default_slow_mode_seconds + - uploaded_logo_dark_id + - uploaded_background_dark_id + extend: + - name: about_topic_title + - name: old_relative_url + - name: existing_id + type: integer + - name: permissions + type: json_text # JSON_TEXT ??? + - name: logo_upload_id + - name: tag_group_ids + type: json_text # JSON_TEXT ??? + topics: + ignore: + - last_posted_at + - posts_count + - last_post_user_id + - reply_count + - featured_user1_id + - featured_user2_id + - featured_user3_id + - featured_user4_id + - deleted_at + - highest_post_number + - like_count + - incoming_link_count + - moderator_posts_count + - bumped_at + - has_summary + - archetype + - notify_moderators_count + - spam_count + - score + - percent_rank + - slug + - deleted_by_id + - participant_count + - word_count + - excerpt + - fancy_title + - highest_staff_post_number + - featured_link + - reviewable_score + - image_upload_id + - slow_mode_seconds + - bannered_until + - external_id + extend: + - name: old_relative_url + - name: private_message + posts: + ignore: + - cooked + - reply_to_post_number + - reply_count + - quote_count + - deleted_at + - off_topic_count + - incoming_link_count + - bookmark_count + - score + - reads + - post_type + - sort_order + - last_editor_id + - hidden + - hidden_reason_id + - notify_moderators_count + - spam_count + - illegal_count + - inappropriate_count + - last_version_at + - user_deleted + - reply_to_user_id + - percent_rank + - notify_user_count + - like_score + - deleted_by_id + - edit_reason + - word_count + - version + - cook_method + - wiki + - baked_at + - baked_version + - hidden_at + - self_edits + - reply_quoted + - via_email + - raw_email + - public_version + - action_code + - locked_by_id + - image_upload_id + - outbound_message_id + - qa_vote_count # TODO: added from plugin, maybe skip these automatically for core schema? + extend: + - name: reply_to_post_id # NOTE: should this be text?? + - name: original_raw + - name: upload_ids + type: json_text + - name: post_number + type: integer + - name: old_relative_url + - name: accepted_answer + type: boolean + - name: small_action + - name: whisper + type: boolean + - name: placeholders + type: json_text + indexes: + - name: posts_by_topic_post_number + columns: [topic_id, post_number] + uploads: + ignore: + - original_filename + - filesize + - width + - height + - url + - created_at + - sha1 + - origin + - retain_hours + - extension + - thumbnail_width + - thumbnail_height + - etag + - secure + - access_control_post_id + - original_sha1 + - animated + - verification_status + - security_last_changed_at + - security_last_changed_reason + - dominant_color + extend: + - name: filename + is_null: false + - name: relative_path + - name: type + - name: data + type: blob + groups: + include: + - id + - name + - full_name + - visibility_level + - members_visibility_level + - mentionable_level + - messageable_level + extend: + - name: description + group_members: + virtual: true + primary_key: [group_id, user_id] + extend: + - name: group_id + type: integer + - name: user_id + type: integer + - name: owner + type: boolean + likes: + virtual: true + primary_key: [user_id, post_id] + extend: + - name: post_id + type: integer + is_null: false + - name: user_id + type: integer + is_null: false + - name: created_at + type: datetime + is_null: false + # TODO: Pending default values & auto incrementing id column + user_fields: + ignore: + - created_at + - external_name + - external_type + extend: + - name: options + type: json_text + muted_users: + primary_key: [user_id, muted_user_id] + ignore: + - id + - created_at + # NOTE: Perhaps use core's user_field_options instead? + user_field_values: + virtual: true + extend: + - name: user_id + type: integer + is_null: false + - name: field_id + type: integer + is_null: false + - name: is_multiselect_field + type: boolean + is_null: false + - name: value + indexes: + - name: user_field_values_multiselect + columns: [user_id, field_id, value] + unique: true + condition: WHERE is_multiselect_field = TRUE + - name: user_field_values_not_multiselect + columns: [user_id, field_id] + unique: true + condition: WHERE is_multiselect_field = FALSE + tags: + include: + - id + - name + extend: + - name: tag_group_id + type: integer + tag_groups: + include: + - id + - name + topic_tags: + primary_key: [topic_id, tag_id] + ignore: + - id + - created_at + tag_users: + primary_key: [tag_id, user_id] + ignore: + - id + - created_at + badges: + ignore: + - grant_count + - allow_title + - icon + - listable + - target_posts + - enabled + - auto_revoke + - trigger + - show_posts + - system + - image + - badge_grouping_id + extend: + - name: bage_group + user_badges: + include: + - user_id + - badge_id + - granted_at + topic_users: + primary_key: [user_id, topic_id] + ignore: + - id + - posted + - cleared_pinned_at + - last_emailed_post_number + - liked + - bookmarked + - last_posted_at + permalink_normalizations: + virtual: true + primary_key: normalization + extend: + - name: normalization + is_null: false + site_settings: + include: + - name + - value + extend: + - name: action + category_custom_fields: + primary_key: [category_id, name] + ignore: + - id + - created_at + post_custom_fields: + primary_key: [post_id, name] + ignore: + - id + - created_at + polls: {} + poll_options: + ignore: + - digest + - html + - anonymous_votes + extend: + - name: poll_id + is_null: false + - name: text + is_null: false + - name: position + type: integer + - name: created_at + is_null: true + poll_votes: + primary_key: [poll_option_id, user_id] + ignore: [poll_id] + extend: + - name: created_at + is_null: true + - name: poll_option_id + is_null: false + - name: user_id + is_null: false +## Schema-wide column configuration options. These options apply to all tables. +## See table specific column configuration options above. +## +## Available Options: +## ignore: List of core/plugin table columns to ignore and exclude from intermediate schema. +columns: + ignore: + - updated_at \ No newline at end of file diff --git a/migrations/scripts/schema_generator b/migrations/scripts/schema_generator new file mode 100755 index 00000000000..5228514ced9 --- /dev/null +++ b/migrations/scripts/schema_generator @@ -0,0 +1,263 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +# Generate the converter's base intermediate database migration file from +# the core database state and YAML configuration in schema.yml +# Invoke from core root directory as `./migrations/scripts/generate_schema` +# It accepts an optional command line argument for the output file path which +# overrides the path configured in schema.yml + +puts "Loading application..." +require_relative "../../config/environment" + +module Migrations + class SchemaGenerator + def initialize(opts = {}) + config = YAML.load_file(File.join(__dir__, "schema.yml"), symbolize_names: true) + + @core_db_connection = ActiveRecord::Base.connection + @output_stream = StringIO.new + @indirectly_ignored_columns = Hash.new { |h, k| h[k] = [] } + + @output_file_path = opts[:output_file_path] || config[:output_file_path] + + @table_configs = config[:tables] + @column_configs = config[:columns] + + @configured_table_names = @table_configs&.keys&.sort || [] + @global_column_ignore_list = @column_configs[:ignore] || [] + end + + def run + puts "Generating base converter migration file for Discourse #{Discourse::VERSION::STRING}" + + generate_header + generate_tables + generate_indirectly_ignored_columns_log + generate_migration_file + + puts "", "Done" + end + + private + + def generate_header + @output_stream.puts <<~HEADER + /* + This file is auto-generated from the Discourse core database schema. Instead of editing it directly, + please update the `schema.yml` configuration file and re-run the `generate_schema` script to update it. + */ + HEADER + end + + def generate_tables + puts "Generating tables..." + + @configured_table_names.each do |name| + raise "Core table named '#{name}' not found" unless valid_table?(name) + + generate_table(name) + end + end + + def generate_indirectly_ignored_columns_log + puts "Generating indirectly ignored column list..." + + @output_stream.puts "\n\n/*" + + @output_stream.puts <<~NOTE + Core table columns implicitly excluded from the generated schema above via the `include` configuration option + in `schema.yml`. This serves as an inventory of these columns, allowing new core additions to be tracked and, + if necessary, synchronized with the intermediate database schema.\n + NOTE + + @indirectly_ignored_columns.each_with_index do |(table_name, columns), index| + next if virtual_table?(table_name) || columns.blank? + + @output_stream.puts "" if index.positive? + @output_stream.puts "Table: #{table_name}" + @output_stream.puts "--------#{"-" * table_name.length}" + + columns.each do |column| + @output_stream.puts " #{column.name} #{column.type} #{column.null}" + end + end + + @output_stream.puts "*/" + end + + def generate_migration_file + file_path = File.expand_path(@output_file_path, __dir__) + + puts "Generating base migration file '#{file_path}'..." + + File.open(file_path, "w") { |f| f << @output_stream.string.chomp } + end + + def generate_column_definition(column) + definition = " #{column.name} #{type(column)}" + definition << " NOT NULL" unless column.null + + definition + end + + def generate_index(table_name, index) + @output_stream.print "CREATE " + @output_stream.print "UNIQUE " if index[:unique] + @output_stream.print "INDEX #{index[:name]} ON #{table_name} (#{index[:columns].join(", ")})" + @output_stream.print " #{index[:condition]}" if index[:condition].present? + @output_stream.puts ";" + end + + def column_list_for(table_name) + ignore_columns = @table_configs.dig(table_name, :ignore) || [] + include_columns = @table_configs.dig(table_name, :include) || [] + + include_columns.present? ? [:include, include_columns] : [:ignore, ignore_columns] + end + + def generate_table(name) + puts "Generating #{name}..." + + column_definitions = [] + column_records = columns(name) + mode, column_list = column_list_for(name) + indexes = indexes(name) + configured_primary_key = primary_key(name) + + primary_key, composite_key = + if configured_primary_key.present? + [configured_primary_key].flatten.each do |pk| + if column_records.map(&:name).exclude?(pk) + raise "Column named '#{pk}' does not exist in table '#{name}'" + end + end + + [ + configured_primary_key, + configured_primary_key.is_a?(Array) && configured_primary_key.length > 1, + ] + else + virtual_table?(name) ? [] : [@core_db_connection.primary_key(name), false] + end + + @output_stream.puts "" + @output_stream.puts "CREATE TABLE #{name} (" + + if !composite_key && primary_key.present? + primary_key_column = column_records.find { |c| c.name == primary_key } + + if (mode == :include && column_list.include?(primary_key_column.name)) || + (mode == :ignore && column_list.exclude?(primary_key_column.name)) + column_definitions << " #{primary_key_column.name} #{type(primary_key_column)} NOT NULL PRIMARY KEY" + end + end + + column_records.each do |column| + next if @global_column_ignore_list.include?(column.name) + next if (mode == :ignore) && column_list.include?(column.name) + if !column.is_a?(CustomColumn) && (mode == :include) && column_list.exclude?(column.name) + @indirectly_ignored_columns[name] << column + + next + end + next if !composite_key && (column.name == primary_key) + + column_definitions << generate_column_definition(column) + end + + column_definitions << " PRIMARY KEY (#{primary_key.join(", ")})" if composite_key + + @output_stream.puts column_definitions.join(",\n") + @output_stream.puts ");" + @output_stream.puts "" if indexes.present? + + indexes.each { |index| generate_index(name, index) } + end + + class CustomColumn + attr_reader :name + + def initialize(name, type, null) + @name = name + @raw_type = type + @raw_null = null + end + + def type + @raw_type&.to_sym || :text + end + + def null + @raw_null.nil? ? true : @raw_null + end + + def merge!(other_column) + @raw_null = other_column.null if @raw_null.nil? + @raw_type ||= other_column.type + + self + end + end + + def columns(name) + extensions = column_extensions(name) + + return extensions if virtual_table?(name) + + default_columns = @core_db_connection.columns(name) + + return default_columns if extensions.blank? + + extended_columns = + default_columns.map do |default_column| + extension = extensions.find { |ext| ext.name == default_column.name } + + if extension + extensions.delete(extension) + + extension.merge!(default_column) + else + default_column + end + end + + extended_columns + extensions + end + + def column_extensions(name) + extensions = @table_configs.dig(name, :extend) + + return [] if extensions.nil? + + extensions.map { |column| CustomColumn.new(column[:name], column[:type], column[:is_null]) } + end + + def type(column) + case column.type + when :string, :inet + "TEXT" + else + column.type.to_s.upcase + end + end + + def valid_table?(name) + @core_db_connection.tables.include?(name.to_s) || virtual_table?(name) + end + + def virtual_table?(name) + !!@table_configs.dig(name, :virtual) + end + + def indexes(table_name) + @table_configs.dig(table_name, :indexes) || [] + end + + def primary_key(table_name) + @table_configs.dig(table_name, :primary_key) + end + end +end + +Migrations::SchemaGenerator.new(output_file_path: ARGV.first).run