diff --git a/script/import_scripts/mylittleforum.rb b/script/import_scripts/mylittleforum.rb new file mode 100644 index 00000000000..795148c7d4d --- /dev/null +++ b/script/import_scripts/mylittleforum.rb @@ -0,0 +1,453 @@ +require "mysql2" +require File.expand_path(File.dirname(__FILE__) + "/base.rb") +require 'htmlentities' + + +# Before running this script, paste these lines into your shell, +# then use arrow keys to edit the values +=begin +export DB_HOST="localhost" +export DB_NAME="mylittleforum" +export DB_PW="" +export DB_USER="root" +export TABLE_PREFIX="forum_" +export IMPORT_AFTER="1970-01-01" +export IMAGE_BASE="http://www.example.com/forum" +export BASE="forum" +=end + + +class ImportScripts::MylittleforumSQL < ImportScripts::Base + + DB_HOST ||= ENV['DB_HOST'] || "localhost" + DB_NAME ||= ENV['DB_NAME'] || "mylittleforum" + DB_PW ||= ENV['DB_PW'] || "" + DB_USER ||= ENV['DB_USER'] || "root" + TABLE_PREFIX ||= ENV['TABLE_PREFIX'] || "forum_" + IMPORT_AFTER ||= ENV['IMPORT_AFTER'] || "1970-01-01" + IMAGE_BASE ||= ENV['IMAGE_BASE'] || "" + BASE ||= ENV['BASE'] || "forum/" + BATCH_SIZE = 1000 + CONVERT_HTML = true + QUIET = nil || ENV['VERBOSE'] == "TRUE" + FORCE_HOSTNAME = nil || ENV['FORCE_HOSTNAME'] + + QUIET = true + + # Site settings + SiteSetting.disable_emails = true + if FORCE_HOSTNAME + SiteSetting.force_hostname=FORCE_HOSTNAME + end + + + def initialize + + if IMPORT_AFTER > "1970-01-01" + print_warning("Importing data after #{IMPORT_AFTER}") + end + + super + @htmlentities = HTMLEntities.new + begin + @client = Mysql2::Client.new( + host: DB_HOST, + username: DB_USER, + password: DB_PW, + database: DB_NAME + ) + rescue Exception => e + puts '='*50 + puts e.message + puts < '#{IMPORT_AFTER}';").first['count'] + + batches(BATCH_SIZE) do |offset| + results = mysql_query(" + SELECT user_id as UserID, user_name as username, + user_real_name as Name, + user_email as Email, + user_hp as website, + user_place as Location, + profile as bio_raw, + last_login as DateLastActive, + user_ip as InsertIPAddress, + user_pw as password, + logins as days_visited, # user_stats + registered as DateInserted, + user_pw as password, + user_type + FROM #{TABLE_PREFIX}userdata + WHERE last_login > '#{IMPORT_AFTER}' + order by UserID ASC + LIMIT #{BATCH_SIZE} + OFFSET #{offset};") + + break if results.size < 1 + + next if all_records_exist? :users, results.map {|u| u['UserID'].to_i} + + create_users(results, total: total_count, offset: offset) do |user| + next if user['Email'].blank? + next if @lookup.user_id_from_imported_user_id(user['UserID']) + + # username = fix_username(user['username']) + + { id: user['UserID'], + email: user['Email'], + username: user['username'], + name: user['Name'], + created_at: user['DateInserted'] == nil ? 0 : Time.zone.at(user['DateInserted']), + bio_raw: user['bio_raw'], + registration_ip_address: user['InsertIPAddress'], + website: user['user_hp'], + password: user['password'], + last_seen_at: user['DateLastActive'] == nil ? 0 : Time.zone.at(user['DateLastActive']), + location: user['Location'], + admin: user['user_type'] == "admin", + moderator: user['user_type'] == "mod", + } + end + end + end + + def fix_username(username) + olduser = username.dup + username.gsub!(/Dr\. /,"Dr") # no & + username.gsub!(/[ +!\/,*()?]/,"_") # can't have these + username.gsub!(/&/,"_and_") # no & + username.gsub!(/@/,"_at_") # no @ + username.gsub!(/#/,"_hash_") # no & + username.gsub!(/\'/,"") # seriously? + username.gsub!(/[._]+/,"_") # can't have 2 special in a row + username.gsub!(/_+/,"_") # could result in dupes, but wtf? + username.gsub!(/_$/,"") # could result in dupes, but wtf? + if olduser != username + print_warning ("#{olduser} --> #{username}") + end + username + end + + def import_categories + puts "", "importing categories..." + + categories = mysql_query(" + SELECT id as CategoryID, + category as Name, + description as Description + FROM #{TABLE_PREFIX}categories + ORDER BY CategoryID ASC + ").to_a + + create_categories(categories) do |category| + { + id: category['CategoryID'], + name: CGI.unescapeHTML(category['Name']), + description: CGI.unescapeHTML(category['Description']) + } + end + end + + def import_topics + puts "", "importing topics..." + + total_count = mysql_query("SELECT count(*) count FROM #{TABLE_PREFIX}entries + WHERE time > '#{IMPORT_AFTER}' + AND pid = 0;").first['count'] + + batches(BATCH_SIZE) do |offset| + discussions = mysql_query( + "SELECT id as DiscussionID, + category as CategoryID, + subject as Name, + text as Body, + time as DateInserted, + youtube_link as youtube, + user_id as InsertUserID + FROM #{TABLE_PREFIX}entries + WHERE pid = 0 + AND time > '#{IMPORT_AFTER}' + ORDER BY time ASC + LIMIT #{BATCH_SIZE} + OFFSET #{offset};") + + break if discussions.size < 1 + next if all_records_exist? :posts, discussions.map {|t| "discussion#" + t['DiscussionID'].to_s} + + create_posts(discussions, total: total_count, offset: offset) do |discussion| + + raw = clean_up(discussion['Body']) + + youtube = nil + unless discussion['youtube'].blank? + youtube = clean_youtube(discussion['youtube']) + raw += "\n#{youtube}\n" + print_warning(raw) + end + + { + id: "discussion#" + discussion['DiscussionID'].to_s, + user_id: user_id_from_imported_user_id(discussion['InsertUserID']) || Discourse::SYSTEM_USER_ID, + title: discussion['Name'].gsub('\\"','"'), + category: category_id_from_imported_category_id(discussion['CategoryID']), + raw: raw, + created_at: Time.zone.at(discussion['DateInserted']), + } + end + end + end + + def import_posts + puts "", "importing posts..." + + total_count = mysql_query( + "SELECT count(*) count + FROM #{TABLE_PREFIX}entries + WHERE pid > 0 + AND time > '#{IMPORT_AFTER}';").first['count'] + + batches(BATCH_SIZE) do |offset| + comments = mysql_query( + "SELECT id as CommentID, + tid as DiscussionID, + text as Body, + time as DateInserted, + youtube_link as youtube, + user_id as InsertUserID + FROM #{TABLE_PREFIX}entries + WHERE pid > 0 + AND time > '#{IMPORT_AFTER}' + ORDER BY time ASC + LIMIT #{BATCH_SIZE} + OFFSET #{offset};") + + break if comments.size < 1 + next if all_records_exist? :posts, comments.map {|comment| "comment#" + comment['CommentID'].to_s} + + create_posts(comments, total: total_count, offset: offset) do |comment| + next unless t = topic_lookup_from_imported_post_id("discussion#" + comment['DiscussionID'].to_s) + next if comment['Body'].blank? + raw = clean_up(comment['Body']) + youtube = nil + unless comment['youtube'].blank? + youtube = clean_youtube(comment['youtube']) + raw += "\n#{youtube}\n" + end + { + id: "comment#" + comment['CommentID'].to_s, + user_id: user_id_from_imported_user_id(comment['InsertUserID']) || Discourse::SYSTEM_USER_ID, + topic_id: t[:topic_id], + raw: clean_up(raw), + created_at: Time.zone.at(comment['DateInserted']) + } + end + end + end + + def clean_youtube(youtube_raw) + youtube_cooked = clean_up(youtube_raw.dup.to_s) + # get just src from