diff --git a/script/import_scripts/askbot.rb b/script/import_scripts/askbot.rb index c6af92b4eae..ec0cfc7cb8c 100644 --- a/script/import_scripts/askbot.rb +++ b/script/import_scripts/askbot.rb @@ -73,7 +73,7 @@ class ImportScripts::MyAskBot < ImportScripts::Base LIMIT #{BATCH_SIZE} OFFSET #{offset} SQL - ) + ) break if tags.ntuples() < 1 tags.each do |tag| tid = tag["thread_id"].to_i @@ -110,6 +110,8 @@ class ImportScripts::MyAskBot < ImportScripts::Base break if users.ntuples() < 1 + next if all_records_exist? :users, users.map {|u| u["id"].to_i} + create_users(users, total: total_count, offset: offset) do |user| { id: user["id"], @@ -153,6 +155,8 @@ class ImportScripts::MyAskBot < ImportScripts::Base break if posts.ntuples() < 1 + next if all_records_exist? :posts, posts.map {|p| p["id"].to_i} + create_posts(posts, total: post_count, offset: offset) do |post| pid = post["id"] tid = post["thread_id"].to_i @@ -206,6 +210,8 @@ class ImportScripts::MyAskBot < ImportScripts::Base break if posts.ntuples() < 1 + next if all_records_exist? :posts, posts.map {|p| p["id"].to_i} + create_posts(posts, total: post_count, offset: offset) do |post| tid = post["thread_id"].to_i next unless thread = @thread_parents[tid] @@ -267,4 +273,4 @@ class ImportScripts::MyAskBot < ImportScripts::Base end end -ImportScripts::MyAskBot.new.perform +ImportScripts::MyAskBot.new.perform diff --git a/script/import_scripts/base.rb b/script/import_scripts/base.rb index 0bc796c7301..57bbc24e9d9 100644 --- a/script/import_scripts/base.rb +++ b/script/import_scripts/base.rb @@ -194,6 +194,23 @@ class ImportScripts::Base g.tap(&:save) end + def all_records_exist?(type, import_ids) + return false if import_ids.empty? + + existing = "#{type.to_s.classify}CustomField".constantize.where(name: 'import_id') + + if Fixnum === import_ids.first + existing = existing.where('cast(value as int) in (?)', import_ids) + else + existing = existing.where('value in (?)', import_ids) + end + + if existing.count == import_ids.length + # puts "Skipping #{import_ids.length} already imported #{type}" + true + end + end + # Iterate through a list of user records to be imported. # Takes a collection, and yields to the block for each element. # Block should return a hash with the attributes for the User model. @@ -258,9 +275,8 @@ class ImportScripts::Base if opts[:username].blank? || opts[:username].length < User.username_length.begin || opts[:username].length > User.username_length.end || - opts[:username] =~ /[^A-Za-z0-9_]/ || - opts[:username][0] =~ /[^A-Za-z0-9]/ || - !User.username_available?(opts[:username]) + !User.username_available?(opts[:username]) || + !UsernameValidator.new(opts[:username]).valid_format? opts[:username] = UserNameSuggester.suggest(opts[:username] || opts[:name] || opts[:email]) end opts[:email] = opts[:email].downcase @@ -289,13 +305,18 @@ class ImportScripts::Base if opts[:active] && opts[:password].present? u.activate end - rescue + rescue => e # try based on email - existing = User.find_by(email: opts[:email].downcase) - if existing - existing.custom_fields["import_id"] = import_id - existing.save! - u = existing + if e.record.errors.messages[:email].present? + existing = User.find_by(email: opts[:email].downcase) + if existing + existing.custom_fields["import_id"] = import_id + existing.save! + u = existing + end + else + puts "Error on record: #{opts}" + raise e end end post_create_action.try(:call, u) if u.persisted? diff --git a/script/import_scripts/bbpress.rb b/script/import_scripts/bbpress.rb index 066c9c23d5b..aba552d11e0 100644 --- a/script/import_scripts/bbpress.rb +++ b/script/import_scripts/bbpress.rb @@ -85,6 +85,8 @@ class ImportScripts::Bbpress < ImportScripts::Base break if results.size < 1 + next if all_records_exist? :posts, results.map {|p| p["id"].to_i} + create_posts(results, total: total_count, offset: offset) do |post| skip = false mapped = {} diff --git a/script/import_scripts/discuz_x.rb b/script/import_scripts/discuz_x.rb index 68f1738061d..f252ddb861d 100644 --- a/script/import_scripts/discuz_x.rb +++ b/script/import_scripts/discuz_x.rb @@ -98,6 +98,8 @@ class ImportScripts::DiscuzX < ImportScripts::Base break if results.size < 1 + next if all_records_exist? :users, users.map {|u| u["id"].to_i} + create_users(results, total: total_count, offset: offset) do |user| { id: user['id'], email: user['email'], @@ -205,6 +207,8 @@ class ImportScripts::DiscuzX < ImportScripts::Base break if results.size < 1 + next if all_records_exist? :posts, results.map {|p| p["id"].to_i} + create_posts(results, total: total_count, offset: offset) do |m| skip = false mapped = {} @@ -281,6 +285,8 @@ class ImportScripts::DiscuzX < ImportScripts::Base break if results.size < 1 + next if all_records_exist? :posts, results.map {|m| "pm:#{m['id']}"} + create_posts(results, total: total_count, offset: offset) do |m| skip = false mapped = {} diff --git a/script/import_scripts/drupal.rb b/script/import_scripts/drupal.rb index 82266d77ca8..a15933b10c1 100644 --- a/script/import_scripts/drupal.rb +++ b/script/import_scripts/drupal.rb @@ -121,6 +121,8 @@ class ImportScripts::Drupal < ImportScripts::Base break if results.size < 1 + next if all_records_exist? :posts, results.map {|p| "nid:#{p['nid']}"} + create_posts(results, total: total_count, offset: offset) do |row| { id: "nid:#{row['nid']}", @@ -167,6 +169,8 @@ class ImportScripts::Drupal < ImportScripts::Base break if results.size < 1 + next if all_records_exist? :posts, results.map {|p| "cid:#{p['cid']}"} + create_posts(results, total: total_count, offset: offset) do |row| topic_mapping = topic_lookup_from_imported_post_id("nid:#{row['nid']}") if topic_mapping && topic_id = topic_mapping[:topic_id] diff --git a/script/import_scripts/drupal_qa.rb b/script/import_scripts/drupal_qa.rb index 9ce0b556e81..9899f73ad5e 100644 --- a/script/import_scripts/drupal_qa.rb +++ b/script/import_scripts/drupal_qa.rb @@ -56,6 +56,8 @@ class ImportScripts::DrupalQA < ImportScripts::Drupal break if results.size < 1 + next if all_records_exist? :posts, results.map {|p| "nid:#{p['nid']}"} + create_posts(results, total: total_count, offset: offset) do |row| { id: "nid:#{row['nid']}", @@ -100,6 +102,8 @@ class ImportScripts::DrupalQA < ImportScripts::Drupal break if results.size < 1 + next if all_records_exist? :posts, results.map {|p| "cid:#{p['cid']}"} + create_posts(results, total: total_count, offset: offset) do |row| topic_mapping = topic_lookup_from_imported_post_id("nid:#{row['nid']}") if topic_mapping && topic_id = topic_mapping[:topic_id] @@ -151,6 +155,8 @@ class ImportScripts::DrupalQA < ImportScripts::Drupal break if results.size < 1 + next if all_records_exist? :posts, results.map {|p| "cid:#{p['cid']}"} + create_posts(results, total: total_count, offset: offset) do |row| topic_mapping = topic_lookup_from_imported_post_id("nid:#{row['nid']}") if topic_mapping && topic_id = topic_mapping[:topic_id] @@ -201,6 +207,8 @@ class ImportScripts::DrupalQA < ImportScripts::Drupal break if results.size < 1 + next if all_records_exist? :posts, results.map {|p| "cid:#{p['cid']}"} + create_posts(results, total: total_count, offset: offset) do |row| topic_mapping = topic_lookup_from_imported_post_id("nid:#{row['nid']}") if topic_mapping && topic_id = topic_mapping[:topic_id] diff --git a/script/import_scripts/kunena.rb b/script/import_scripts/kunena.rb index 8e1f6275b38..3cb52e7b38a 100644 --- a/script/import_scripts/kunena.rb +++ b/script/import_scripts/kunena.rb @@ -109,6 +109,8 @@ class ImportScripts::Kunena < ImportScripts::Base break if results.size < 1 + next if all_records_exist? :posts, results.map {|p| p['id'].to_i} + create_posts(results, total: total_count, offset: offset) do |m| skip = false mapped = {} diff --git a/script/import_scripts/kunena3.rb b/script/import_scripts/kunena3.rb index 56103861fd4..49b3f24e917 100644 --- a/script/import_scripts/kunena3.rb +++ b/script/import_scripts/kunena3.rb @@ -109,6 +109,8 @@ class ImportScripts::Kunena < ImportScripts::Base break if results.size < 1 + next if all_records_exist? :posts, results.map {|p| p['id'].to_i} + create_posts(results, total: total_count, offset: offset) do |m| skip = false mapped = {} diff --git a/script/import_scripts/lithium.rb b/script/import_scripts/lithium.rb index 4d7a84366c8..795f04f366a 100644 --- a/script/import_scripts/lithium.rb +++ b/script/import_scripts/lithium.rb @@ -101,6 +101,8 @@ class ImportScripts::Lithium < ImportScripts::Base break if users.size < 1 + next if all_records_exist? :users, users.map {|u| u["id"].to_i} + create_users(users, total: user_count, offset: offset) do |user| { @@ -274,9 +276,10 @@ class ImportScripts::Lithium < ImportScripts::Base OFFSET #{offset} SQL - break if topics.size < 1 + next if all_records_exist? :posts, topics.map {|topic| "#{topic["node_id"]} #{topic["id"]}"} + create_posts(topics, total: topic_count, offset: offset) do |topic| category_id = category_id_from_imported_category_id(topic["node_id"]) @@ -322,6 +325,8 @@ class ImportScripts::Lithium < ImportScripts::Base break if posts.size < 1 + next if all_records_exist? :posts, posts.map {|post| "#{post["node_id"]} #{post["root_id"]} #{post["id"]}"} + create_posts(posts, total: post_count, offset: offset) do |post| raw = post["raw"] next unless topic = topic_lookup_from_imported_post_id("#{post["node_id"]} #{post["root_id"]}") @@ -593,6 +598,8 @@ class ImportScripts::Lithium < ImportScripts::Base break if topics.size < 1 + next if all_records_exist? :posts, topics.map {|topic| "pm_#{topic["note_id"]}"} + create_posts(topics, total: topic_count, offset: offset) do |topic| user_id = user_id_from_imported_user_id(topic["sender_user_id"]) || Discourse::SYSTEM_USER_ID diff --git a/script/import_scripts/mbox.rb b/script/import_scripts/mbox.rb index 78549ae909a..d47b5553fc7 100755 --- a/script/import_scripts/mbox.rb +++ b/script/import_scripts/mbox.rb @@ -69,6 +69,7 @@ class ImportScripts::Mbox < ImportScripts::Base batches(BATCH_SIZE) do |offset| users = user_keys[offset..offset+BATCH_SIZE-1] break if users.nil? + next if all_records_exist? :users, users create_users(users, total: total_count, offset: offset) do |email| { @@ -99,6 +100,8 @@ class ImportScripts::Mbox < ImportScripts::Base topics = all_topics[offset..offset+BATCH_SIZE-1] break if topics.nil? + next if all_records_exist? :posts, topics.map {|t| t['id'].to_i} + create_posts(topics, total: topic_count, offset: offset) do |t| raw_email = File.read(t['file']) receiver = Email::Receiver.new(raw_email, skip_sanity_check: true) @@ -136,6 +139,8 @@ class ImportScripts::Mbox < ImportScripts::Base posts = replies[offset..offset+BATCH_SIZE-1] break if posts.nil? + next if all_records_exist? :posts, posts.map {|p| p['id'].to_i} + create_posts(posts, total: post_count, offset: offset) do |p| parent_id = p['topic'] id = p['id'] diff --git a/script/import_scripts/mybb.rb b/script/import_scripts/mybb.rb index c2337d56a14..b76e1e0d919 100644 --- a/script/import_scripts/mybb.rb +++ b/script/import_scripts/mybb.rb @@ -48,6 +48,8 @@ class ImportScripts::MyBB < ImportScripts::Base break if results.size < 1 + next if all_records_exist? :users, users.map {|u| u["id"].to_i} + create_users(results, total: total_count, offset: offset) do |user| { id: user['id'], email: user['email'], @@ -100,6 +102,8 @@ class ImportScripts::MyBB < ImportScripts::Base break if results.size < 1 + next if all_records_exist? :posts, results.map {|m| m['id'].to_i} + create_posts(results, total: total_count, offset: offset) do |m| skip = false mapped = {} diff --git a/script/import_scripts/nabble.rb b/script/import_scripts/nabble.rb index 607772da418..78e48497509 100644 --- a/script/import_scripts/nabble.rb +++ b/script/import_scripts/nabble.rb @@ -40,6 +40,8 @@ class ImportScripts::Nabble < ImportScripts::Base break if users.ntuples() < 1 + next if all_records_exist? :users, users.map {|u| u["user_id"].to_i} + create_users(users, total: total_count, offset: offset) do |user| { id: user["user_id"], @@ -80,6 +82,8 @@ class ImportScripts::Nabble < ImportScripts::Base break if topics.ntuples() < 1 + next if all_records_exist? :posts, topics.map {|t| t['node_id'].to_i} + create_posts(topics, total: topic_count, offset: offset) do |t| raw = body_from(t) next unless raw @@ -122,6 +126,8 @@ class ImportScripts::Nabble < ImportScripts::Base break if posts.ntuples() < 1 + next if all_records_exist? :posts, posts.map {|p| p['node_id'].to_i} + create_posts(posts, total: post_count, offset: offset) do |p| parent_id = p['parent_id'] id = p['node_id'] diff --git a/script/import_scripts/phpbb3/importer.rb b/script/import_scripts/phpbb3/importer.rb index 28c2ed6fa63..11592b40d18 100644 --- a/script/import_scripts/phpbb3/importer.rb +++ b/script/import_scripts/phpbb3/importer.rb @@ -56,6 +56,8 @@ module ImportScripts::PhpBB3 rows = @database.fetch_users(offset) break if rows.size < 1 + next if all_records_exist? :users, importer.map_to_import_ids(rows) + create_users(rows, total: total_count, offset: offset) do |row| importer.map_user(row) end diff --git a/script/import_scripts/phpbb3/importers/user_importer.rb b/script/import_scripts/phpbb3/importers/user_importer.rb index be0321d48ef..aeef5ec8627 100644 --- a/script/import_scripts/phpbb3/importers/user_importer.rb +++ b/script/import_scripts/phpbb3/importers/user_importer.rb @@ -9,6 +9,10 @@ module ImportScripts::PhpBB3 @settings = settings end + def map_to_import_ids(array) + array.map {|u| u[:user_id]} + end + def map_user(row) is_active_user = row[:user_inactive_reason] != Constants::INACTIVE_REGISTER diff --git a/script/import_scripts/punbb.rb b/script/import_scripts/punbb.rb index 08ecc883130..a5d54372f1e 100644 --- a/script/import_scripts/punbb.rb +++ b/script/import_scripts/punbb.rb @@ -43,6 +43,8 @@ class ImportScripts::PunBB < ImportScripts::Base break if results.size < 1 + next if all_records_exist? :users, users.map {|u| u["id"].to_i} + create_users(results, total: total_count, offset: offset) do |user| { id: user['id'], email: user['email'], @@ -118,6 +120,7 @@ class ImportScripts::PunBB < ImportScripts::Base ").to_a break if results.size < 1 + next if all_records_exist? :posts, results.map {|m| m['id'].to_i} create_posts(results, total: total_count, offset: offset) do |m| skip = false diff --git a/script/import_scripts/sfn.rb b/script/import_scripts/sfn.rb index f8a14f00d93..14e54196cdb 100644 --- a/script/import_scripts/sfn.rb +++ b/script/import_scripts/sfn.rb @@ -77,6 +77,7 @@ class ImportScripts::Sfn < ImportScripts::Base SQL break if users.size < 1 + next if all_records_exist? :users, users.map {|u| u["id"].to_i} create_users(users, total: user_count, offset: offset) do |user| external_user = @external_users[user["id"]] @@ -231,6 +232,7 @@ class ImportScripts::Sfn < ImportScripts::Base SQL break if topics.size < 1 + next if all_records_exist? :posts, topics.map {|t| t['id'].to_i} create_posts(topics, total: topic_count, offset: offset) do |topic| next unless category_id = CATEGORY_MAPPING[topic["category_id"]] @@ -282,6 +284,8 @@ class ImportScripts::Sfn < ImportScripts::Base break if posts.size < 1 + next if all_records_exist? :posts, posts.map {|p| p['id'].to_i} + create_posts(posts, total: posts_count, offset: offset) do |post| next unless parent = topic_lookup_from_imported_post_id(post["topic_id"]) diff --git a/script/import_scripts/tnation.rb b/script/import_scripts/tnation.rb index b8563997e71..438327c7b0e 100644 --- a/script/import_scripts/tnation.rb +++ b/script/import_scripts/tnation.rb @@ -173,6 +173,8 @@ class ImportScripts::Tnation < ImportScripts::Base break if users.size < 1 + next if all_records_exist? :users, users.map {|u| u["id"].to_i} + user_bios = {} user_avatars = {} user_properties = {} @@ -317,6 +319,7 @@ class ImportScripts::Tnation < ImportScripts::Base posts = posts.to_a break if posts.size < 1 + next if all_records_exist? :posts, posts.map {|p| p['id'].to_i} # load images forum_images = {} diff --git a/script/import_scripts/vanilla_mysql.rb b/script/import_scripts/vanilla_mysql.rb index 841bf1478a7..e472ce525b7 100644 --- a/script/import_scripts/vanilla_mysql.rb +++ b/script/import_scripts/vanilla_mysql.rb @@ -42,6 +42,8 @@ class ImportScripts::VanillaSQL < ImportScripts::Base break if results.size < 1 + next if all_records_exist? :users, users.map {|u| u['UserID'].to_i} + create_users(results, total: total_count, offset: offset) do |user| next if user['Email'].blank? next if user['Name'].blank? @@ -92,6 +94,7 @@ class ImportScripts::VanillaSQL < ImportScripts::Base OFFSET #{offset};") break if discussions.size < 1 + next if all_records_exist? :posts, discussions.map {|t| "discussion#" + t['DiscussionID'].to_s} create_posts(discussions, total: total_count, offset: offset) do |discussion| { @@ -121,6 +124,7 @@ class ImportScripts::VanillaSQL < ImportScripts::Base OFFSET #{offset};") break if comments.size < 1 + next if all_records_exist? :posts, comments.map {|comment| "comment#" + comment['CommentID'].to_s} create_posts(comments, total: total_count, offset: offset) do |comment| next unless t = topic_lookup_from_imported_post_id("discussion#" + comment['DiscussionID'].to_s) diff --git a/script/import_scripts/vbulletin.rb b/script/import_scripts/vbulletin.rb index 6d3ac4415f8..22e5883db38 100644 --- a/script/import_scripts/vbulletin.rb +++ b/script/import_scripts/vbulletin.rb @@ -71,6 +71,8 @@ class ImportScripts::VBulletin < ImportScripts::Base break if users.size < 1 + next if all_records_exist? :users, users.map {|u| u["userid"].to_i} + create_users(users, total: user_count, offset: offset) do |user| username = @htmlentities.decode(user["username"]).strip @@ -208,6 +210,7 @@ class ImportScripts::VBulletin < ImportScripts::Base SQL break if topics.size < 1 + next if all_records_exist? :posts, topics.map {|t| "thread-#{topic["threadid"]}" } create_posts(topics, total: topic_count, offset: offset) do |topic| raw = preprocess_post_raw(topic["raw"]) rescue nil @@ -249,6 +252,7 @@ class ImportScripts::VBulletin < ImportScripts::Base SQL break if posts.size < 1 + next if all_records_exist? :posts, posts.map {|p| p["postid"] } create_posts(posts, total: post_count, offset: offset) do |post| raw = preprocess_post_raw(post["raw"]) rescue nil diff --git a/script/import_scripts/xenforo.rb b/script/import_scripts/xenforo.rb index 536c03ec144..41f7cd15874 100644 --- a/script/import_scripts/xenforo.rb +++ b/script/import_scripts/xenforo.rb @@ -42,6 +42,8 @@ class ImportScripts::XenForo < ImportScripts::Base break if results.size < 1 + next if all_records_exist? :users, users.map {|u| u["id"].to_i} + create_users(results, total: total_count, offset: offset) do |user| next if user['username'].blank? { id: user['id'], @@ -98,6 +100,7 @@ class ImportScripts::XenForo < ImportScripts::Base ").to_a break if results.size < 1 + next if all_records_exist? :posts, results.map {|p| p['id'] } create_posts(results, total: total_count, offset: offset) do |m| skip = false