From 7c29ff3d85606d55feb8fd0255a63283100d8678 Mon Sep 17 00:00:00 2001
From: Bianca Nenciu <nbianca@users.noreply.github.com>
Date: Mon, 24 May 2021 05:38:42 +0300
Subject: [PATCH] FEATURE: Add tasks to export and import site structure
 (#12584)

---
 lib/tasks/site.rake | 491 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 491 insertions(+)
 create mode 100644 lib/tasks/site.rake

diff --git a/lib/tasks/site.rake b/lib/tasks/site.rake
new file mode 100644
index 00000000000..4f168ada132
--- /dev/null
+++ b/lib/tasks/site.rake
@@ -0,0 +1,491 @@
+# frozen_string_literal: true
+
+require 'yaml'
+require 'zip'
+
+class ZippedSiteStructure
+  attr_reader :zip
+
+  def initialize(path, create: false)
+    @zip = Zip::File.open(path, create)
+  end
+
+  def close
+    @zip.close
+  end
+
+  def set(name, data)
+    @zip.get_output_stream("#{name}.json") do |file|
+      file.write(data.to_json)
+    end
+  end
+
+  def get(name)
+    data = @zip.get_input_stream("#{name}.json").read
+    JSON.parse(data)
+  end
+
+  def set_upload(upload_or_id_or_url)
+    return nil if upload_or_id_or_url.blank?
+
+    if Integer === upload_or_id_or_url
+      upload = Upload.find_by(id: upload_or_id_or_url)
+    elsif String === upload_or_id_or_url
+      upload = Upload.get_from_url(upload_or_id_or_url)
+    elsif Upload === upload_or_id_or_url
+      upload = upload_or_id_or_url
+    end
+
+    if !upload
+      STDERR.puts "ERROR: Could not find upload #{upload_or_id_or_url.inspect}"
+      return nil
+    end
+
+    local_path = upload.local? ? Discourse.store.path_for(upload) : Discourse.store.download(upload).path
+    zip_path = File.join('uploads', File.basename(local_path))
+
+    puts "  - Exporting upload #{upload_or_id_or_url} to #{zip_path}"
+    @zip.add(zip_path, local_path)
+
+    { filename: upload.original_filename, path: file_zip_path }
+  end
+
+  def get_upload(upload, opts = {})
+    return nil if upload.blank?
+
+    puts "  - Importing upload #{upload['filename']} from #{upload['path']}"
+
+    tempfile = Tempfile.new(upload['filename'], binmode: true)
+    tempfile.write(@zip.get_input_stream(upload['path']).read)
+    tempfile.rewind
+
+    UploadCreator.new(tempfile, upload['filename'], opts)
+      .create_for(Discourse::SYSTEM_USER_ID)
+  end
+end
+
+desc 'Exports site structure (settings, groups, categories, tags, themes, etc) to a ZIP file'
+task 'site:export_structure', [:zip_path] => :environment do |task, args|
+  if args[:zip_path].blank?
+    STDERR.puts "ERROR: rake site:export_structure[<path to ZIP file>]"
+    exit 1
+  elsif File.exists?(args[:zip_path])
+    STDERR.puts "ERROR: File '#{args[:zip_path]}' already exists"
+    exit 2
+  end
+
+  data = ZippedSiteStructure.new(args[:zip_path], create: true)
+
+  puts
+  puts "Exporting site settings"
+  puts
+
+  settings = {}
+
+  SiteSetting.all_settings(include_hidden: true).each do |site_setting|
+    next if site_setting[:default] == site_setting[:value]
+
+    puts "- Site setting #{site_setting[:setting]} -> #{site_setting[:value].inspect}"
+
+    settings[site_setting[:setting]] = if site_setting[:type] == 'upload'
+      data.set_upload(site_setting[:value])
+    else
+      site_setting[:value]
+    end
+  end
+
+  data.set('site_settings', settings)
+
+  puts
+  puts "Exporting users"
+  puts
+
+  users = []
+
+  User.real.where(admin: true).each do |u|
+    puts "- User #{u.username}"
+
+    users << {
+      username: u.username,
+      name: u.name,
+      email: u.email,
+      active: u.active,
+      admin: u.admin,
+    }
+  end
+
+  data.set('users', users)
+
+  puts
+  puts "Exporting groups"
+  puts
+
+  groups = []
+
+  Group.where(automatic: false).each do |g|
+    puts "- Group #{g.name}"
+
+    groups << {
+      name: g.name,
+      automatic_membership_email_domains: g.automatic_membership_email_domains,
+      primary_group: g.primary_group,
+      title: g.title,
+      grant_trust_level: g.grant_trust_level,
+      incoming_email: g.incoming_email,
+      has_messages: g.has_messages,
+      flair_bg_color: g.flair_bg_color,
+      flair_color: g.flair_color,
+      bio_raw: g.bio_raw,
+      allow_membership_requests: g.allow_membership_requests,
+      full_name: g.full_name,
+      default_notification_level: g.default_notification_level,
+      visibility_level: g.visibility_level,
+      public_exit: g.public_exit,
+      public_admission: g.public_admission,
+      membership_request_template: g.membership_request_template,
+      messageable_level: g.messageable_level,
+      mentionable_level: g.mentionable_level,
+      publish_read_state: g.publish_read_state,
+      members_visibility_level: g.members_visibility_level,
+      flair_icon: g.flair_icon,
+      flair_upload_id: data.set_upload(g.flair_upload_id),
+      allow_unknown_sender_topic_replies: g.allow_unknown_sender_topic_replies,
+    }
+  end
+
+  data.set('groups', groups)
+
+  puts
+  puts "Exporting categories"
+  puts
+
+  categories = []
+
+  Category.find_each do |c|
+    puts "- Category #{c.name} (#{c.slug})"
+
+    categories << {
+      name: c.name,
+      color: c.color,
+      slug: c.slug,
+      description: c.description,
+      text_color: c.text_color,
+      read_restricted: c.read_restricted,
+      auto_close_hours: c.auto_close_hours,
+      parent_category: c.parent_category&.slug,
+      position: c.position,
+      email_in: c.email_in,
+      email_in_allow_strangers: c.email_in_allow_strangers,
+      allow_badges: c.allow_badges,
+      auto_close_based_on_last_post: c.auto_close_based_on_last_post,
+      topic_template: c.topic_template,
+      sort_order: c.sort_order,
+      sort_ascending: c.sort_ascending,
+      uploaded_logo_id: data.set_upload(c.uploaded_logo_id),
+      uploaded_background_id: data.set_upload(c.uploaded_background_id),
+      topic_featured_link_allowed: c.topic_featured_link_allowed,
+      all_topics_wiki: c.all_topics_wiki,
+      show_subcategory_list: c.show_subcategory_list,
+      default_view: c.default_view,
+      subcategory_list_style: c.subcategory_list_style,
+      default_top_period: c.default_top_period,
+      mailinglist_mirror: c.mailinglist_mirror,
+      minimum_required_tags: c.minimum_required_tags,
+      navigate_to_first_post_after_read: c.navigate_to_first_post_after_read,
+      search_priority: c.search_priority,
+      allow_global_tags: c.allow_global_tags,
+      read_only_banner: c.read_only_banner,
+      default_list_filter: c.default_list_filter,
+      permissions: c.permissions_params,
+    }
+  end
+
+  data.set('categories', categories)
+
+  puts
+  puts "Exporting tag groups"
+  puts
+
+  tag_groups = []
+
+  TagGroup.all.each do |tg|
+    puts "- Tag group #{tg.name}"
+
+    tag_groups << {
+      name: tg.name,
+      tag_names: tg.tags.map(&:name)
+    }
+  end
+
+  data.set('tag_groups', tag_groups)
+
+  puts
+  puts "Exporting tags"
+  puts
+
+  tags = []
+
+  Tag.find_each do |t|
+    puts "- Tag #{t.name}"
+
+    tag = { name: t.name }
+    tag[:target_tag] = t.target_tag.name if t.target_tag.present?
+
+    tags << tag
+  end
+
+  data.set('tags', tags)
+
+  puts
+  puts "Exporting themes and theme components"
+  puts
+
+  themes = []
+
+  Theme.find_each do |theme|
+    puts "- Theme #{theme.name}"
+
+    if theme.remote_theme.present?
+      themes << {
+        name: theme.name,
+        url: theme.remote_theme.remote_url,
+        private_key: theme.remote_theme.private_key,
+        branch: theme.remote_theme.branch
+      }
+    else
+      exporter = ThemeStore::ZipExporter.new(theme)
+      file_path = exporter.package_filename
+      file_zip_path = File.join('themes', File.basename(file_path))
+      data.zip.add(file_zip_path, file_path)
+      themes << { name: theme.name, filename: File.basename(file_path), path: file_zip_path }
+    end
+  end
+
+  data.set('themes', themes)
+
+  puts
+  puts "Exporting theme settings"
+  puts
+
+  theme_settings = []
+
+  ThemeSetting.find_each do |theme_setting|
+    puts "- Theme setting #{theme_setting.name} -> #{theme_setting.value}"
+
+    value = if theme_setting.data_type == ThemeSetting.types[:upload]
+      data.set_upload(theme_setting.value)
+    else
+      theme_setting.value
+    end
+
+    theme_settings << {
+      name: theme_setting.name,
+      data_type: theme_setting.data_type,
+      value: value,
+      theme: theme_setting.theme.name,
+    }
+  end
+
+  data.set('theme_settings', theme_settings)
+
+  puts
+  puts "Done"
+  puts
+
+  data.close
+end
+
+desc 'Imports site structure from a ZIP file exported by site:export_structure'
+task 'site:import_structure', [:zip_path] => :environment do |task, args|
+  if args[:zip_path].blank?
+    STDERR.puts "ERROR: rake site:import_structure[<path to ZIP file>]"
+    exit 1
+  elsif !File.exists?(args[:zip_path])
+    STDERR.puts "ERROR: File '#{args[:zip_path]}' does not exist"
+    exit 2
+  end
+
+  data = ZippedSiteStructure.new(args[:zip_path])
+
+  puts
+  puts "Importing site settings"
+  puts
+
+  settings = data.get('site_settings')
+  imported_settings = Set.new
+
+  3.times.each do |try|
+    puts "Loading site settings (try ##{try})"
+
+    settings.each do |key, value|
+      next if imported_settings.include?(key)
+
+      begin
+        if SiteSetting.type_supervisor.get_type(key) == :upload
+          value = data.get_upload(value, for_site_setting: true)
+        end
+
+        if SiteSetting.public_send(key) != value
+          puts "- Site setting #{key} -> #{value}"
+          SiteSetting.set_and_log(key, value)
+        end
+
+        imported_settings << key
+      rescue => e
+        next if try < 2
+
+        STDERR.puts "ERROR: Cannot set #{key} to #{value}"
+        puts e.backtrace
+      end
+    end
+  end
+
+  puts
+  puts "Importing users"
+  puts
+
+  data.get('users').each do |u|
+    puts "- User #{u['username']}"
+
+    begin
+      user = User.find_or_initialize_by(username: u.delete('username'))
+      user.update!(u)
+    rescue => e
+      STDERR.puts "ERROR: Cannot import user: #{e.message}"
+      puts e.backtrace
+    end
+  end
+
+  puts
+  puts "Importing groups"
+  puts
+
+  data.get('groups').each do |g|
+    puts "- Group #{g['name']}"
+
+    begin
+      group = Group.find_or_initialize_by(name: g.delete('name'))
+      group.update!(g)
+    rescue => e
+      STDERR.puts "ERROR: Cannot import group: #{e.message}"
+      puts e.backtrace
+    end
+  end
+
+  puts
+  puts "Importing categories"
+  puts
+
+  data.get('categories').each do |c|
+    puts "- Category #{c['name']} (#{c['slug']})"
+
+    begin
+      category = Category.find_or_initialize_by(slug: c.delete('slug'))
+      category.user ||= Discourse.system_user
+      category.parent_category = Category.find_by(slug: c.delete('parent_category'))
+      category.permissions = c.delete('permissions')
+      category.update!(c)
+    rescue => e
+      STDERR.puts "ERROR: Cannot import category: #{e.message}"
+      puts e.backtrace
+    end
+  end
+
+  puts
+  puts "Importing tag groups"
+  puts
+
+  data.get('tag_groups').each do |tg|
+    puts "- Tag group #{tg['name']}"
+
+    tag_group = TagGroup.find_or_initialize_by(name: tg.delete('name'))
+    tag_group.update!(tg)
+  end
+
+  puts
+  puts "Importing tags"
+  puts
+
+  data.get('tags').each do |t|
+    puts "- Tag #{t['name']}"
+
+    if t['target_tag'].present?
+      begin
+        t['target_tag'] = Tag.find_or_create_by!(name: t.delete('target_tag'))
+      rescue => e
+        STDERR.puts "ERROR: Cannot import target tag: #{e.message}"
+        puts e.backtrace
+      end
+    end
+
+    begin
+      tag = Tag.find_or_initialize_by(name: t.delete('name'))
+      tag.update!(t)
+    rescue => e
+      STDERR.puts "ERROR: Cannot import tag: #{e.message}"
+      puts e.backtrace
+    end
+  end
+
+  puts
+  puts "Importing themes and theme components"
+  puts
+
+  data.get('themes').each do |t|
+    puts "- Theme #{t['name']}"
+
+    begin
+      if t['url'].present?
+        next if Theme.find_by(name: t['name']).present?
+
+        RemoteTheme.import_theme(
+          t['url'],
+          Discourse.system_user,
+          private_key: t['private_key'],
+          branch: t['branch']
+        )
+      elsif t['filename'].present?
+        tempfile = Tempfile.new(t['filename'], binmode: true)
+        tempfile.write(data.zip.get_input_stream(t['path']).read)
+        tempfile.flush
+
+        RemoteTheme.update_zipped_theme(
+          tempfile.path,
+          t['filename'],
+          user: Discourse.system_user,
+          theme_id: Theme.find_by(name: t['name'])&.id,
+        )
+      end
+    rescue => e
+      STDERR.puts "ERROR: Cannot import theme: #{e.message}"
+      puts e.backtrace
+    end
+  end
+
+  puts
+  puts "Importing theme settings"
+  puts
+
+  data.get('theme_settings').each do |ts|
+    puts "- Theme setting #{ts['name']} -> #{ts['value']}"
+
+    begin
+      if ts['data_type'] == ThemeSetting.types[:upload]
+        ts['value'] = data.get_upload(ts['value'], for_theme: true)
+      end
+
+      ThemeSetting
+        .find_or_initialize_by(name: ts['name'], theme: Theme.find_by(name: ts['theme']))
+        .update!(data_type: ts['data_type'], value: ts['value'])
+    rescue => e
+      STDERR.puts "ERROR: Cannot import theme setting: #{e.message}"
+      puts e.backtrace
+    end
+  end
+
+  puts
+  puts "Done"
+  puts
+
+  data.close
+end