# frozen_string_literal: true require "yaml" require "zip" class ZippedSiteStructure attr_reader :zip def initialize(path, create: false) @zip = Zip::File.open(path, create) @uploads = {} end def close @zip.close end def set(name, data) @zip.get_output_stream("#{name}.json") { |file| file.write(data.to_json) } end def get(name) data = @zip.get_input_stream("#{name}.json").read JSON.parse(data) end def set_upload(upload_or_id_or_url) return nil if upload_or_id_or_url.blank? if Integer === upload_or_id_or_url upload = Upload.find_by(id: upload_or_id_or_url) elsif String === upload_or_id_or_url upload = Upload.get_from_url(upload_or_id_or_url) elsif Upload === upload_or_id_or_url upload = upload_or_id_or_url end if !upload STDERR.puts "ERROR: Could not find upload #{upload_or_id_or_url.inspect}" return nil end if @uploads[upload.id].present? puts " - Already exported upload #{upload_or_id_or_url} to #{@uploads[upload.id][:path]}" return @uploads[upload.id] end local_path = upload.local? ? Discourse.store.path_for(upload) : Discourse.store.download(upload).path zip_path = File.join("uploads", File.basename(local_path)) zip_path = get_unique_path(zip_path) puts " - Exporting upload #{upload_or_id_or_url} to #{zip_path}" @zip.add(zip_path, local_path) @uploads[upload.id] ||= { filename: upload.original_filename, path: zip_path } end def get_upload(upload, opts = {}) return nil if upload.blank? if @uploads[upload["path"]].present? puts " - Already imported upload #{upload["filename"]} from #{upload["path"]}" return @uploads[upload["path"]] end puts " - Importing upload #{upload["filename"]} from #{upload["path"]}" tempfile = Tempfile.new(upload["filename"], binmode: true) tempfile.write(@zip.get_input_stream(upload["path"]).read) tempfile.rewind @uploads[upload["path"]] ||= UploadCreator.new(tempfile, upload["filename"], opts).create_for( Discourse::SYSTEM_USER_ID, ) end private def get_unique_path(path) return path if @zip.find_entry(path).blank? extname = File.extname(path) basename = File.basename(path, extname) dirname = File.dirname(path) i = 0 loop do i += 1 path = File.join(dirname, "#{basename}_#{i}#{extname}") return path if @zip.find_entry(path).blank? end end end desc "Exports site structure (settings, groups, categories, tags, themes, etc) to a ZIP file" task "site:export_structure", [:zip_path] => :environment do |task, args| if args[:zip_path].blank? STDERR.puts "ERROR: rake site:export_structure[]" exit 1 elsif File.exist?(args[:zip_path]) STDERR.puts "ERROR: File '#{args[:zip_path]}' already exists" exit 2 end data = ZippedSiteStructure.new(args[:zip_path], create: true) puts puts "Exporting site settings" puts settings = {} SiteSetting .all_settings(include_hidden: true) .each do |site_setting| next if site_setting[:default] == site_setting[:value] puts "- Site setting #{site_setting[:setting]} -> #{site_setting[:value].inspect}" settings[site_setting[:setting]] = if site_setting[:type] == "upload" data.set_upload(site_setting[:value]) else site_setting[:value] end end data.set("site_settings", settings) puts puts "Exporting users" puts users = [] User .real .where(admin: true) .each do |u| puts "- User #{u.username}" users << { username: u.username, name: u.name, email: u.email, active: u.active, admin: u.admin, } end data.set("users", users) puts puts "Exporting groups" puts groups = [] Group .where(automatic: false) .each do |g| puts "- Group #{g.name}" groups << { name: g.name, automatic_membership_email_domains: g.automatic_membership_email_domains, primary_group: g.primary_group, title: g.title, grant_trust_level: g.grant_trust_level, incoming_email: g.incoming_email, has_messages: g.has_messages, flair_bg_color: g.flair_bg_color, flair_color: g.flair_color, bio_raw: g.bio_raw, allow_membership_requests: g.allow_membership_requests, full_name: g.full_name, default_notification_level: g.default_notification_level, visibility_level: g.visibility_level, public_exit: g.public_exit, public_admission: g.public_admission, membership_request_template: g.membership_request_template, messageable_level: g.messageable_level, mentionable_level: g.mentionable_level, publish_read_state: g.publish_read_state, members_visibility_level: g.members_visibility_level, flair_icon: g.flair_icon, flair_upload_id: data.set_upload(g.flair_upload_id), allow_unknown_sender_topic_replies: g.allow_unknown_sender_topic_replies, } end data.set("groups", groups) puts puts "Exporting categories" puts categories = [] Category.find_each do |c| puts "- Category #{c.name} (#{c.slug})" categories << { name: c.name, color: c.color, slug: c.slug, description: c.description, text_color: c.text_color, read_restricted: c.read_restricted, auto_close_hours: c.auto_close_hours, parent_category: c.parent_category&.slug, position: c.position, email_in: c.email_in, email_in_allow_strangers: c.email_in_allow_strangers, allow_badges: c.allow_badges, auto_close_based_on_last_post: c.auto_close_based_on_last_post, topic_template: c.topic_template, sort_order: c.sort_order, sort_ascending: c.sort_ascending, uploaded_logo_id: data.set_upload(c.uploaded_logo_id), uploaded_logo_dark_id: data.set_upload(c.uploaded_logo_dark_id), uploaded_background_id: data.set_upload(c.uploaded_background_id), topic_featured_link_allowed: c.topic_featured_link_allowed, all_topics_wiki: c.all_topics_wiki, show_subcategory_list: c.show_subcategory_list, default_view: c.default_view, subcategory_list_style: c.subcategory_list_style, default_top_period: c.default_top_period, mailinglist_mirror: c.mailinglist_mirror, minimum_required_tags: c.minimum_required_tags, navigate_to_first_post_after_read: c.navigate_to_first_post_after_read, search_priority: c.search_priority, allow_global_tags: c.allow_global_tags, read_only_banner: c.read_only_banner, default_list_filter: c.default_list_filter, permissions: c.permissions_params, } end data.set("categories", categories) puts puts "Exporting tag groups" puts tag_groups = [] TagGroup.all.each do |tg| puts "- Tag group #{tg.name}" tag_groups << { name: tg.name, tag_names: tg.tags.map(&:name) } end data.set("tag_groups", tag_groups) puts puts "Exporting tags" puts tags = [] Tag.find_each do |t| puts "- Tag #{t.name}" tag = { name: t.name } tag[:target_tag] = t.target_tag.name if t.target_tag.present? tags << tag end data.set("tags", tags) puts puts "Exporting themes and theme components" puts themes = [] Theme.find_each do |theme| puts "- Theme #{theme.name}" if theme.remote_theme.present? themes << { name: theme.name, url: theme.remote_theme.remote_url, private_key: theme.remote_theme.private_key, branch: theme.remote_theme.branch, } else exporter = ThemeStore::ZipExporter.new(theme) file_path = exporter.package_filename file_zip_path = File.join("themes", File.basename(file_path)) data.zip.add(file_zip_path, file_path) themes << { name: theme.name, filename: File.basename(file_path), path: file_zip_path } end end data.set("themes", themes) puts puts "Exporting theme settings" puts theme_settings = [] ThemeSetting.find_each do |theme_setting| puts "- Theme setting #{theme_setting.name} -> #{theme_setting.value}" value = if theme_setting.data_type == ThemeSetting.types[:upload] data.set_upload(theme_setting.value) else theme_setting.value end theme_settings << { name: theme_setting.name, data_type: theme_setting.data_type, value: value, theme: theme_setting.theme.name, } end data.set("theme_settings", theme_settings) puts puts "Done" puts data.close end desc "Imports site structure from a ZIP file exported by site:export_structure" task "site:import_structure", [:zip_path] => :environment do |task, args| if args[:zip_path].blank? STDERR.puts "ERROR: rake site:import_structure[]" exit 1 elsif !File.exist?(args[:zip_path]) STDERR.puts "ERROR: File '#{args[:zip_path]}' does not exist" exit 2 end data = ZippedSiteStructure.new(args[:zip_path]) puts puts "Importing site settings" puts settings = data.get("site_settings") imported_settings = Set.new 3.times.each do |try| puts "Loading site settings (try ##{try})" settings.each do |key, value| next if imported_settings.include?(key) begin if SiteSetting.type_supervisor.get_type(key) == :upload value = data.get_upload(value, for_site_setting: true) end if SiteSetting.public_send(key) != value puts "- Site setting #{key} -> #{value}" SiteSetting.set_and_log(key, value) end imported_settings << key rescue => e next if try < 2 STDERR.puts "ERROR: Cannot set #{key} to #{value}" puts e.backtrace end end end puts puts "Importing users" puts data .get("users") .each do |u| puts "- User #{u["username"]}" begin user = User.find_or_initialize_by(username: u.delete("username")) user.update!(u) rescue => e STDERR.puts "ERROR: Cannot import user: #{e.message}" puts e.backtrace end end puts puts "Importing groups" puts data .get("groups") .each do |g| puts "- Group #{g["name"]}" begin group = Group.find_or_initialize_by(name: g.delete("name")) group.update!(g) rescue => e STDERR.puts "ERROR: Cannot import group: #{e.message}" puts e.backtrace end end puts puts "Importing categories" puts data .get("categories") .each do |c| puts "- Category #{c["name"]} (#{c["slug"]})" begin category = Category.find_or_initialize_by(slug: c.delete("slug")) category.user ||= Discourse.system_user category.parent_category = Category.find_by(slug: c.delete("parent_category")) category.permissions = c.delete("permissions") category.update!(c) rescue => e STDERR.puts "ERROR: Cannot import category: #{e.message}" puts e.backtrace end end puts puts "Importing tag groups" puts data .get("tag_groups") .each do |tg| puts "- Tag group #{tg["name"]}" tag_group = TagGroup.find_or_initialize_by(name: tg.delete("name")) tag_group.update!(tg) end puts puts "Importing tags" puts data .get("tags") .each do |t| puts "- Tag #{t["name"]}" if t["target_tag"].present? begin t["target_tag"] = Tag.find_or_create_by!(name: t.delete("target_tag")) rescue => e STDERR.puts "ERROR: Cannot import target tag: #{e.message}" puts e.backtrace end end begin tag = Tag.find_or_initialize_by(name: t.delete("name")) tag.update!(t) rescue => e STDERR.puts "ERROR: Cannot import tag: #{e.message}" puts e.backtrace end end puts puts "Importing themes and theme components" puts data .get("themes") .each do |t| puts "- Theme #{t["name"]}" begin if t["url"].present? next if Theme.find_by(name: t["name"]).present? RemoteTheme.import_theme( t["url"], Discourse.system_user, private_key: t["private_key"], branch: t["branch"], ) elsif t["filename"].present? tempfile = Tempfile.new(t["filename"], binmode: true) tempfile.write(data.zip.get_input_stream(t["path"]).read) tempfile.flush RemoteTheme.update_zipped_theme( tempfile.path, t["filename"], user: Discourse.system_user, theme_id: Theme.find_by(name: t["name"])&.id, ) end rescue => e STDERR.puts "ERROR: Cannot import theme: #{e.message}" puts e.backtrace end end puts puts "Importing theme settings" puts data .get("theme_settings") .each do |ts| puts "- Theme setting #{ts["name"]} -> #{ts["value"]}" begin if ts["data_type"] == ThemeSetting.types[:upload] ts["value"] = data.get_upload(ts["value"], for_theme: true) end ThemeSetting.find_or_initialize_by( name: ts["name"], theme: Theme.find_by(name: ts["theme"]), ).update!(data_type: ts["data_type"], value: ts["value"]) rescue => e STDERR.puts "ERROR: Cannot import theme setting: #{e.message}" puts e.backtrace end end puts puts "Done" puts data.close end