Discource-C
/
discourse
mirror of https://github.com/discourse/discourse.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Remove old mbox import script

This commit is contained in:
Gerhard Schlager 2018-10-30 20:44:27 +01:00
parent e0ccd36dbe
commit edbc004a9a
1 changed files with 0 additions and 555 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

555
script/import_scripts/mbox.rb
View File

@ -1,555 +0,0 @@
require 'sqlite3'
require File.expand_path(File.dirname(__FILE__) + "/base.rb")
# Paste these lines into your shell before running this:
=begin
export MBOX_SUBDIR="messages" # subdirectory with mbox files
export LIST_NAME=LIST_NAME
export DEFAULT_TRUST_LEVEL=1
export DATA_DIR=~/data/import
export SPLIT_AT="^From " # or "^From (.*)"
=end
# If you change the functionality of this script, please consider updating this HOWTO:
# https://meta.discourse.org/t/howto-import-mbox-mailing-list-files/51233
class ImportScripts::Mbox < ImportScripts::Base
include ActiveModel::Validations
# CHANGE THESE BEFORE RUNNING THE IMPORTER
MBOX_SUBDIR = ENV['MBOX_SUBDIR'] || "messages" # subdirectory with mbox files
LIST_NAME = ENV['LIST_NAME'] || "" # Will remove [LIST_NAME] from Subjects
DEFAULT_TRUST_LEVEL = ENV['DEFAULT_TRUST_LEVEL'] || 1
DATA_DIR = ENV['DATA_DIR'] || "~/data/import"
MBOX_DIR = File.expand_path(DATA_DIR) # where index.db will be created
BATCH_SIZE = 1000
# Site settings
SiteSetting.disable_emails = "non-staff"
# Comment out if each file contains a single message
# Use formail to split yourself: http://linuxcommand.org/man_pages/formail1.html
# SPLIT_AT = /^From (.*) at/ # for Google Groups?
SPLIT_AT = /#{ENV['SPLIT_AT']}/ || /^From / # for standard MBOX files
# Will create a category if it doesn't exist
# create subdirectories in MBOX_SUBDIR with categories
CATEGORY_MAPPINGS = {
"default" => "uncategorized",
# ex: "jobs-folder" => "jobs"
}
unless File.directory?(MBOX_DIR)
puts "Cannot find import directory #{MBOX_DIR}. Giving up."
exit
end
validates_format_of :email, with: /\A([^@\s]+)@((?:[-a-z0-9]+\.)+[a-z]{2,})\Z/i, on: :create
def execute
import_categories
create_email_indices
create_user_indices
massage_indices
import_users
create_forum_topics
import_replies
# replace_email_addresses # uncomment to replace all email address with @username
end
def import_categories
mappings = CATEGORY_MAPPINGS.values - ['uncategorized']
create_categories(mappings) do |c|
{ id: c, name: c }
end
end
def open_db
SQLite3::Database.new("#{MBOX_DIR}/index.db")
end
def each_line(f)
infile = File.open(f, 'r')
if f.ends_with?('.gz')
gz = Zlib::GzipReader.new(infile)
gz.each_line do |line|
yield line
end
else
infile.each_line do |line|
yield line
end
end
ensure
infile.close
end
def all_messages
files = Dir["#{MBOX_DIR}/#{MBOX_SUBDIR}/*"]
CATEGORY_MAPPINGS.keys.each do |k|
files << Dir["#{MBOX_DIR}/#{k}/*"]
end
files.flatten!
files.sort!
files.each_with_index do |f, idx|
print_warning "\nProcessing: #{f}"
start_time = Time.now
if SPLIT_AT.present?
msg = ""
message_count = 0
each_line(f) do |line|
line = line.scrub
if line =~ SPLIT_AT
p message_count += 1
if !msg.empty?
mail = Mail.read_from_string(msg)
yield mail, f
print_status(idx, files.size, start_time)
msg = ""
end
end
msg << line
end
if !msg.empty?
mail = Mail.read_from_string(msg)
yield mail, f
print_status(idx, files.size, start_time)
msg = ""
end
else
raw = File.read(f)
mail = Mail.read_from_string(raw)
yield mail, f
print_status(idx, files.size, start_time)
end
end
end
def massage_indices
db = open_db
db.execute "UPDATE emails SET reply_to = null WHERE reply_to = ''"
rows = db.execute "SELECT msg_id, title, reply_to FROM emails ORDER BY datetime(email_date) ASC"
msg_ids = {}
titles = {}
rows.each do |row|
msg_ids[row[0]] = true
if titles[row[1]].nil?
titles[row[1]] = row[0]
end
end
# First, any replies where the parent doesn't exist should have that field cleared
not_found = []
rows.each do |row|
msg_id, _, reply_to = row
if reply_to.present?
not_found << msg_id if msg_ids[reply_to].blank?
end
end
puts "#{not_found.size} records couldn't be associated with parents"
if not_found.present?
db.execute "UPDATE emails SET reply_to = NULL WHERE msg_id IN (#{not_found.map { |nf| "'#{nf}'" }.join(',')})"
end
dupe_titles = db.execute "SELECT title, COUNT(*) FROM emails GROUP BY title HAVING count(*) > 1"
puts "#{dupe_titles.size} replies to wire up"
dupe_titles.each do |t|
title = t[0]
first = titles[title]
db.execute "UPDATE emails SET reply_to = ? WHERE title = ? and msg_id <> ?", [first, title, first]
end
ensure
db.close
end
def extract_name(mail)
from_name = nil
from = mail[:from]
from_email = nil
if mail.from.present?
from_email = mail.from.dup
if from_email.kind_of?(Array)
if from_email[0].nil?
print_warning "Cannot find email address (ignoring)!\n#{mail}"
else
from_email = from_email.first.dup
from_email.gsub!(/ at /, '@')
from_email.gsub!(/ [at] /, '@')
# strip real names in ()s. Todo: read into name
from_email.gsub!(/ \(.*$/, '')
from_email.gsub!(/ /, '')
end
end
end
display_names = from.try(:display_names)
if display_names.present?
from_name = display_names.first
end
if from_name.blank? && from.to_s =~ /\(([^\)]+)\)/
from_name = Regexp.last_match[1]
end
from_name = from.to_s if from_name.blank?
[from_email, from_name]
end
def print_warning(message)
$stderr.puts "#{message}"
end
def create_email_indices
db = open_db
db.execute "DROP TABLE IF EXISTS emails"
db.execute <<-SQL
CREATE TABLE emails (
msg_id VARCHAR(995) PRIMARY KEY,
from_email VARCHAR(255) NOT NULL,
from_name VARCHAR(255) NOT NULL,
title VARCHAR(255) NOT NULL,
reply_to VARCHAR(955) NULL,
email_date DATETIME NOT NULL,
message TEXT NOT NULL,
category VARCHAR(255) NOT NULL
);
SQL
db.execute "CREATE INDEX by_title ON emails (title)"
db.execute "CREATE INDEX by_email ON emails (from_email)"
puts "", "creating indices"
all_messages do |mail, filename|
directory = filename.sub("#{MBOX_DIR}/", '').split("/")[0]
category = CATEGORY_MAPPINGS[directory] || CATEGORY_MAPPINGS['default'] || 'uncategorized'
msg_id = mail['Message-ID'].to_s
# Many ways to get a name
from_email, from_name = extract_name(mail)
title = clean_title(mail['Subject'].to_s)
reply_to = mail['In-Reply-To'].to_s
email_date = mail['date'].to_s
email_date = DateTime.parse(email_date).to_s unless email_date.blank?
if from_email.kind_of?(String)
unless from_email.match(/\A[\w+\-.]+@[a-z\d\-]+(\.[a-z\d\-]+)*\.[a-z]+\z/i)
print_warning "Ignoring bad email address #{from_email} in #{msg_id}"
else
db.execute "INSERT OR IGNORE INTO emails (msg_id,
from_email,
from_name,
title,
reply_to,
email_date,
message,
category)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
[msg_id, from_email, from_name, title, reply_to, email_date, mail.to_s, category]
end
end
end
ensure
db.close
end
def create_user_indices
db = open_db
db.execute "DROP TABLE IF EXISTS users"
db.execute <<-SQL
CREATE TABLE users (
email VARCHAR(995) PRIMARY KEY,
name VARCHAR(255) NOT NULL
);
SQL
db.execute "INSERT OR IGNORE INTO users (email, name) SELECT from_email, from_name FROM emails"
ensure
db.close
end
def clean_title(title)
title ||= ""
#Strip mailing list name from subject
title = title.gsub(/\[#{Regexp.escape(LIST_NAME)}\]/, '').strip
original_length = title.length
#Strip Reply prefix from title (Standard and localized)
title = title.gsub(/^Re: */i, '')
title = title.gsub(/^R: */i, '') #Italian
title = title.gsub(/^RIF: */i, '') #Italian
#Strip Forward prefix from title (Standard and localized)
title = title.gsub(/^Fwd: */i, '')
title = title.gsub(/^I: */i, '') #Italian
title.strip
#In case of mixed localized prefixes there could be many of them if the mail client didn't strip the localized ones
if original_length > title.length
clean_title(title)
else
title
end
end
def clean_raw(input)
raw = input.dup
raw.scrub!
raw.gsub!(/-- \nYou received this message because you are subscribed to the Google Groups "[^"]*" group.\nTo unsubscribe from this group and stop receiving emails from it, send an email to [^+@]+\+unsubscribe@googlegroups.com\.\nFor more options, visit https:\/\/groups\.google\.com\/groups\/opt_out\./, '')
raw
end
def import_users
puts "", "importing users"
db = open_db
all_users = db.execute("SELECT name, email FROM users")
total_count = all_users.size
batches(BATCH_SIZE) do |offset|
users = all_users[offset..offset + BATCH_SIZE - 1]
break if users.nil?
next if all_records_exist? :users, users.map { |u| u[1] }
create_users(users, total: total_count, offset: offset) do |u|
{
id: u[1],
email: u[1],
name: u[0],
trust_level: DEFAULT_TRUST_LEVEL,
}
end
end
ensure
db.close
end
def replace_email_addresses
puts "", "replacing email addresses with @usernames"
post = Post.new
total_count = User.real.count
progress_count = 0
start_time = Time.now
# from: https://meta.discourse.org/t/replace-a-string-in-all-posts/48729/17
# and https://github.com/discourse/discourse/blob/master/lib/tasks/posts.rake#L114-L136
User.find_each do |u|
i = 0
find = u.email.dup
replace = "@#{u.username}"
if !replace.include? "@"
puts "Skipping #{replace}"
end
found = Post.where("raw ILIKE ?", "%#{find}%")
next if found.nil?
next if found.count < 1
found.each do |p|
new_raw = p.raw.dup
new_raw = new_raw.gsub!(/#{Regexp.escape(find)}/i, replace) || new_raw
if new_raw != p.raw
p.revise(Discourse.system_user, { raw: new_raw }, bypass_bump: true)
print_warning "\nReplaced #{find} with #{replace} in topic #{p.topic_id}"
end
end
progress_count += 1
puts ""
print_status(progress_count, total_count, start_time)
end
end
def parse_email(msg)
receiver = Email::Receiver.new(msg)
mail = Mail.read_from_string(msg)
mail.body
selected = receiver.select_body
selected.force_encoding(selected.encoding).encode("UTF-8")
end
def create_forum_topics
puts "", "creating forum topics"
db = open_db
all_topics = db.execute("SELECT msg_id,
from_email,
from_name,
title,
email_date,
message,
category
FROM emails
WHERE reply_to IS NULL
ORDER BY DATE(email_date)")
topic_count = all_topics.size
batches(BATCH_SIZE) do |offset|
topics = all_topics[offset..offset + BATCH_SIZE - 1]
break if topics.nil?
next if all_records_exist? :posts, topics.map { |t| t[0] }
create_posts(topics, total: topic_count, offset: offset) do |t|
raw_email = t[5]
receiver = Email::Receiver.new(raw_email)
mail = Mail.read_from_string(raw_email)
mail.body
from_email, _ = extract_name(mail)
selected = receiver.select_body
next unless selected
selected = selected.join('') if selected.kind_of?(Array)
title = mail.subject
username = User.find_by_email(from_email).username
# import the attachments
raw = ""
mail.attachments.each do |attachment|
tmp = Tempfile.new("discourse-email-attachment")
begin
# read attachment
File.open(tmp.path, "w+b") { |f| f.write attachment.body.decoded }
# create the upload for the user
upload = UploadCreator.new(tmp, attachment.filename).create_for(user_id_from_imported_user_id(from_email) || Discourse::SYSTEM_USER_ID)
if upload && upload.errors.empty?
raw << "\n\n#{receiver.attachment_markdown(upload)}\n\n"
end
ensure
tmp.try(:close!) rescue nil
end
end
user_id = user_id_from_imported_user_id(from_email) || Discourse::SYSTEM_USER_ID
raw = selected.force_encoding(selected.encoding).encode("UTF-8")
raw = clean_raw(raw)
raw = raw.dup.to_s
raw.gsub!(/#{from_email}/, "@#{username}")
cleaned_email = from_email.dup.sub(/@/, ' at ')
raw.gsub!(/#{cleaned_email}/, "@#{username}")
{ id: t[0],
title: clean_title(title),
user_id: user_id,
created_at: mail.date,
category: t[6],
raw: clean_raw(raw),
cook_method: Post.cook_methods[:email] }
end
end
ensure
db.close
end
def import_replies
puts "", "creating topic replies"
db = open_db
replies = db.execute("SELECT msg_id,
from_email,
from_name,
title,
email_date,
message,
reply_to
FROM emails
WHERE reply_to IS NOT NULL
ORDER BY DATE(email_date)
")
post_count = replies.size
puts "Replies: #{post_count}"
batches(BATCH_SIZE) do |offset|
posts = replies[offset..offset + BATCH_SIZE - 1]
break if posts.nil?
break if posts.count < 1
next if all_records_exist? :posts, posts.map { |p| p[0] }
create_posts(posts, total: post_count, offset: offset) do |p|
parent_id = p[6]
id = p[0]
topic = topic_lookup_from_imported_post_id(parent_id)
topic_id = topic[:topic_id] if topic
next unless topic_id
raw_email = p[5]
receiver = Email::Receiver.new(raw_email)
mail = Mail.read_from_string(raw_email)
mail.body
from_email, _ = extract_name(mail)
selected = receiver.select_body
selected = selected.join('') if selected.kind_of?(Array)
next unless selected
raw = selected.force_encoding(selected.encoding).encode("UTF-8")
username = User.find_by_email(from_email).username
user_id = user_id_from_imported_user_id(from_email) || Discourse::SYSTEM_USER_ID
raw = clean_raw(raw).to_s
raw.gsub!(/#{from_email}/, "@#{username}")
cleaned_email = from_email.dup.sub(/@/, ' at ')
raw.gsub!(/#{cleaned_email}/, "@#{username}")
# import the attachments
mail.attachments.each do |attachment|
tmp = Tempfile.new("discourse-email-attachment")
begin
# read attachment
File.open(tmp.path, "w+b") { |f| f.write attachment.body.decoded }
# create the upload for the user
upload = UploadCreator.new(tmp, attachment.filename).create_for(user_id_from_imported_user_id(from_email) || Discourse::SYSTEM_USER_ID)
if upload && upload.errors.empty?
raw << "\n\n#{receiver.attachment_markdown(upload)}\n\n"
end
ensure
tmp.try(:close!) rescue nil
end
end
{ id: id,
topic_id: topic_id,
user_id: user_id_from_imported_user_id(from_email) || Discourse::SYSTEM_USER_ID,
created_at: mail.date,
raw: clean_raw(raw),
cook_method: Post.cook_methods[:email] }
end
end
ensure
db.close
end
end
ImportScripts::Mbox.new.perform