From c11e8c9a64a93dcc9469639f40e37497228d093c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Hanol?= Date: Tue, 24 Apr 2018 11:04:41 +0200 Subject: [PATCH] add support for importing attachments in bbPress importer --- script/import_scripts/bbpress.rb | 150 +++++++++++++++++++++++++------ 1 file changed, 125 insertions(+), 25 deletions(-) diff --git a/script/import_scripts/bbpress.rb b/script/import_scripts/bbpress.rb index 9059270f6f4..1538de4af8f 100644 --- a/script/import_scripts/bbpress.rb +++ b/script/import_scripts/bbpress.rb @@ -1,23 +1,15 @@ require 'mysql2' require File.expand_path(File.dirname(__FILE__) + "/base.rb") -# Before running this script, paste these lines into your shell, -# then use arrow keys to edit the values -=begin -export BBPRESS_HOST="localhost" -export BBPRESS_USER="root" -export BBPRESS_DB="bbpress" -export BBPRESS_PW="" -=end - class ImportScripts::Bbpress < ImportScripts::Base - BB_PRESS_HOST ||= ENV['BBPRESS_HOST'] || "localhost" - BB_PRESS_DB ||= ENV['BBPRESS_DB'] || "bbpress" - BATCH_SIZE ||= 1000 - BB_PRESS_PW ||= ENV['BBPRESS_PW'] || "" - BB_PRESS_USER ||= ENV['BBPRESS_USER'] || "root" - BB_PRESS_PREFIX ||= ENV['BBPRESS_PREFIX'] || "wp_" + BB_PRESS_HOST ||= ENV['BBPRESS_HOST'] || "localhost" + BB_PRESS_DB ||= ENV['BBPRESS_DB'] || "bbpress" + BATCH_SIZE ||= 1000 + BB_PRESS_PW ||= ENV['BBPRESS_PW'] || "" + BB_PRESS_USER ||= ENV['BBPRESS_USER'] || "root" + BB_PRESS_PREFIX ||= ENV['BBPRESS_PREFIX'] || "wp_" + BB_PRESS_ATTACHMENTS_DIR ||= ENV['BBPRESS_ATTACHMENTS_DIR'] || "/path/to/attachments" def initialize super @@ -38,6 +30,7 @@ class ImportScripts::Bbpress < ImportScripts::Base import_categories import_topics_and_posts import_private_messages + import_attachments create_permalinks end @@ -246,10 +239,15 @@ class ImportScripts::Bbpress < ImportScripts::Base create_posts(posts, total: total_posts, offset: offset) do |p| skip = false + user_id = user_id_from_imported_user_id(p["post_author"]) || + find_user_by_import_id(p["post_author"]).try(:id) || + user_id_from_imported_user_id(anon_names[p['id']]) || + find_user_by_import_id(anon_names[p['id']]).try(:id) || + -1 + post = { id: p["id"], - user_id: user_id_from_imported_user_id(p["post_author"]) || find_user_by_import_id(p["post_author"]).try(:id) || - user_id_from_imported_user_id(anon_names[p['id']]) || find_user_by_import_id(anon_names[p['id']]).try(:id) || -1, + user_id: user_id, raw: p["post_content"], created_at: p["post_date"], like_count: posts_likes[p["id"]], @@ -277,17 +275,118 @@ class ImportScripts::Bbpress < ImportScripts::Base end end + def import_attachments + import_attachments_from_postmeta + import_attachments_from_bb_attachments + end + + def import_attachments_from_postmeta + puts "", "Importing attachments from 'postmeta'..." + + count = 0 + last_attachment_id = -1 + + total_attachments = bbpress_query(<<-SQL + SELECT COUNT(*) count + FROM #{BB_PRESS_PREFIX}postmeta pm + JOIN #{BB_PRESS_PREFIX}posts p ON p.id = pm.post_id + WHERE pm.meta_key = '_wp_attached_file' + AND p.post_parent > 0 + SQL + ).first["count"] + + batches(BATCH_SIZE) do |offset| + attachments = bbpress_query(<<-SQL + SELECT pm.meta_id id, pm.meta_value, p.post_parent post_id + FROM #{BB_PRESS_PREFIX}postmeta pm + JOIN #{BB_PRESS_PREFIX}posts p ON p.id = pm.post_id + WHERE pm.meta_key = '_wp_attached_file' + AND p.post_parent > 0 + AND pm.meta_id > #{last_attachment_id} + ORDER BY pm.meta_id + LIMIT #{BATCH_SIZE} + SQL + ).to_a + + break if attachments.empty? + last_attachment_id = attachments[-1]["id"].to_i + + attachments.each do |a| + print_status(count += 1, total_attachments, get_start_time("attachments_from_postmeta")) + path = File.join(BB_PRESS_ATTACHMENTS_DIR, a["meta_value"]) + if File.exists?(path) + if post = Post.find_by(id: post_id_from_imported_post_id(a["post_id"])) + filename = File.basename(a["meta_value"]) + upload = create_upload(post.user.id, path, filename) + if upload&.persisted? + html = html_for_upload(upload, filename) + if !post.raw[html] + post.raw << "\n\n" << html + post.save! + PostUpload.create!(post: post, upload: upload) unless PostUpload.where(post: post, upload: upload).exists? + end + end + end + end + end + end + end + + def import_attachments_from_bb_attachments + puts "", "Importing attachments from 'bb_attachments'..." + + count = 0 + last_attachment_id = -1 + + total_attachments = bbpress_query(<<-SQL + SELECT COUNT(*) count + FROM #{BB_PRESS_PREFIX}bb_attachments + WHERE post_id IN (SELECT id FROM #{BB_PRESS_PREFIX}posts WHERE post_status <> 'spam' AND post_type IN ('topic', 'reply')) + SQL + ).first["count"] + + batches(BATCH_SIZE) do |offset| + attachments = bbpress_query(<<-SQL + SELECT id, filename, post_id + FROM #{BB_PRESS_PREFIX}bb_attachments + WHERE post_id IN (SELECT id FROM #{BB_PRESS_PREFIX}posts WHERE post_status <> 'spam' AND post_type IN ('topic', 'reply')) + AND id > #{last_attachment_id} + ORDER BY id + LIMIT #{BATCH_SIZE} + SQL + ).to_a + + break if attachments.empty? + last_attachment_id = attachments[-1]["id"].to_i + + attachments.each do |a| + print_status(count += 1, total_attachments, get_start_time("attachments_from_bb_attachments")) + if path = find_attachment(a["filename"], a["id"]) + if post = Post.find_by(id: post_id_from_imported_post_id(a["post_id"])) + upload = create_upload(post.user.id, path, a["filename"]) + if upload&.persisted? + html = html_for_upload(upload, a["filename"]) + if !post.raw[html] + post.raw << "\n\n" << html + post.save! + PostUpload.create!(post: post, upload: upload) unless PostUpload.where(post: post, upload: upload).exists? + end + end + end + end + end + end + end + + def find_attachment(filename, id) + @attachments ||= Dir[File.join(BB_PRESS_ATTACHMENTS_DIR, "vf-attachs", "**", "*.*")] + @attachments.find { |p| p.end_with?("/#{id}.#{filename}") } + end + def create_permalinks puts "", "creating permalinks..." last_topic_id = -1 - total_topics = bbpress_query(<<-SQL - SELECT COUNT(*) count - FROM #{BB_PRESS_PREFIX}posts - WHERE post_status <> 'spam' - AND post_type IN ('topic') - SQL - ).first["count"] batches(BATCH_SIZE) do |offset| topics = bbpress_query(<<-SQL @@ -301,13 +400,14 @@ class ImportScripts::Bbpress < ImportScripts::Base LIMIT #{BATCH_SIZE} SQL ).to_a + break if topics.empty? + last_topic_id = topics[-1]["id"].to_i topics.each do |t| topic = topic_lookup_from_imported_post_id(t['id']) Permalink.create(url: URI.parse(t['guid']).path.chomp('/'), topic_id: topic[:topic_id]) rescue nil end - last_topic_id = topics[-1]["id"].to_i end end