From 29d27ec2ef157c71d27f8b3eef5b0136a3995b58 Mon Sep 17 00:00:00 2001 From: Gerhard Schlager Date: Wed, 30 Sep 2015 16:38:06 +0200 Subject: [PATCH 1/2] FIX: Prevent cast error when import_id contains strings and numbers Some importers insert numbers and strings as import_id into the same custom_fields table. This change prevents cast errors during imports. --- script/import_scripts/base.rb | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/script/import_scripts/base.rb b/script/import_scripts/base.rb index 57bbc24e9d9..dda270aacee 100644 --- a/script/import_scripts/base.rb +++ b/script/import_scripts/base.rb @@ -198,12 +198,7 @@ class ImportScripts::Base return false if import_ids.empty? existing = "#{type.to_s.classify}CustomField".constantize.where(name: 'import_id') - - if Fixnum === import_ids.first - existing = existing.where('cast(value as int) in (?)', import_ids) - else - existing = existing.where('value in (?)', import_ids) - end + existing = existing.where('value in (?)', import_ids.map(&:to_s)) if existing.count == import_ids.length # puts "Skipping #{import_ids.length} already imported #{type}" From a03ead97670cb6a6b6324bc63f6bdc764d0eb1e6 Mon Sep 17 00:00:00 2001 From: Gerhard Schlager Date: Wed, 30 Sep 2015 17:56:37 +0200 Subject: [PATCH 2/2] FIX: phpBB3 importer failed to import users FEATURE: Skip batches if posts or messages exists --- .../import_scripts/phpbb3/database/database_base.rb | 2 +- script/import_scripts/phpbb3/importer.rb | 8 +++++++- .../phpbb3/importers/message_importer.rb | 11 ++++++++++- .../import_scripts/phpbb3/importers/post_importer.rb | 4 ++++ .../import_scripts/phpbb3/importers/user_importer.rb | 8 ++++++-- 5 files changed, 28 insertions(+), 5 deletions(-) diff --git a/script/import_scripts/phpbb3/database/database_base.rb b/script/import_scripts/phpbb3/database/database_base.rb index 3c8b4b37181..a63cecf4c2e 100644 --- a/script/import_scripts/phpbb3/database/database_base.rb +++ b/script/import_scripts/phpbb3/database/database_base.rb @@ -13,7 +13,7 @@ module ImportScripts::PhpBB3 # Executes a database query. def query(sql) - @database_client.query(sql, cache_rows: false, symbolize_keys: true) + @database_client.query(sql, cache_rows: true, symbolize_keys: true) end # Executes a database query and returns the value of the 'count' column. diff --git a/script/import_scripts/phpbb3/importer.rb b/script/import_scripts/phpbb3/importer.rb index 11592b40d18..f2c1dc9de16 100644 --- a/script/import_scripts/phpbb3/importer.rb +++ b/script/import_scripts/phpbb3/importer.rb @@ -56,7 +56,7 @@ module ImportScripts::PhpBB3 rows = @database.fetch_users(offset) break if rows.size < 1 - next if all_records_exist? :users, importer.map_to_import_ids(rows) + next if all_records_exist?(:users, importer.map_users_to_import_ids(rows)) create_users(rows, total: total_count, offset: offset) do |row| importer.map_user(row) @@ -73,6 +73,8 @@ module ImportScripts::PhpBB3 rows = @database.fetch_anonymous_users(offset) break if rows.size < 1 + next if all_records_exist?(:users, importer.map_anonymous_users_to_import_ids(rows)) + create_users(rows, total: total_count, offset: offset) do |row| importer.map_anonymous_user(row) end @@ -98,6 +100,8 @@ module ImportScripts::PhpBB3 rows = @database.fetch_posts(offset) break if rows.size < 1 + next if all_records_exist?(:posts, importer.map_to_import_ids(rows)) + create_posts(rows, total: total_count, offset: offset) do |row| importer.map_post(row) end @@ -118,6 +122,8 @@ module ImportScripts::PhpBB3 rows = @database.fetch_messages(@settings.fix_private_messages, offset) break if rows.size < 1 + next if all_records_exist?(:posts, importer.map_to_import_ids(rows)) + create_posts(rows, total: total_count, offset: offset) do |row| importer.map_message(row) end diff --git a/script/import_scripts/phpbb3/importers/message_importer.rb b/script/import_scripts/phpbb3/importers/message_importer.rb index 6200b0b0230..0ebab7d242a 100644 --- a/script/import_scripts/phpbb3/importers/message_importer.rb +++ b/script/import_scripts/phpbb3/importers/message_importer.rb @@ -13,12 +13,17 @@ module ImportScripts::PhpBB3 @settings = settings end + def map_to_import_ids(rows) + rows.map { |row| get_import_id(row) } + end + + def map_message(row) user_id = @lookup.user_id_from_imported_user_id(row[:author_id]) || Discourse.system_user.id attachments = import_attachments(row, user_id) mapped = { - id: "pm:#{row[:msg_id]}", + id: get_import_id(row), user_id: user_id, created_at: Time.zone.at(row[:message_time]), raw: @text_processor.process_private_msg(row[:message_text], attachments) @@ -79,5 +84,9 @@ module ImportScripts::PhpBB3 import_user_id.to_s == author_id.to_s ? nil : @lookup.find_user_by_import_id(import_user_id).try(:username) end.compact end + + def get_import_id(row) + "pm:#{row[:msg_id]}" + end end end diff --git a/script/import_scripts/phpbb3/importers/post_importer.rb b/script/import_scripts/phpbb3/importers/post_importer.rb index be0daebbf22..b6de98622fa 100644 --- a/script/import_scripts/phpbb3/importers/post_importer.rb +++ b/script/import_scripts/phpbb3/importers/post_importer.rb @@ -13,6 +13,10 @@ module ImportScripts::PhpBB3 @settings = settings end + def map_to_import_ids(rows) + rows.map { |row| row[:post_id] } + end + def map_post(row) imported_user_id = row[:post_username].blank? ? row[:poster_id] : row[:post_username] user_id = @lookup.user_id_from_imported_user_id(imported_user_id) || Discourse.system_user.id diff --git a/script/import_scripts/phpbb3/importers/user_importer.rb b/script/import_scripts/phpbb3/importers/user_importer.rb index aeef5ec8627..c1cd914998b 100644 --- a/script/import_scripts/phpbb3/importers/user_importer.rb +++ b/script/import_scripts/phpbb3/importers/user_importer.rb @@ -9,8 +9,8 @@ module ImportScripts::PhpBB3 @settings = settings end - def map_to_import_ids(array) - array.map {|u| u[:user_id]} + def map_users_to_import_ids(rows) + rows.map { |row| row[:user_id] } end def map_user(row) @@ -42,6 +42,10 @@ module ImportScripts::PhpBB3 } end + def map_anonymous_users_to_import_ids(rows) + rows.map { |row| row[:post_username] } + end + def map_anonymous_user(row) username = row[:post_username]