IntenseWebs/discourse
3
0
Fork 0
mirror of https://github.com/discourse/discourse.git synced 2025-02-25 18:55:32 -06:00
Code Issues Packages Projects Releases Wiki Activity

DEV: various bugfixes in bulk importer

Browse Source
This commit is contained in:
Régis Hanol 2020-06-19 17:53:06 +02:00
parent 0b58414de2
commit 47a1157458
1 changed files with 11 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
script/bulk_import/base.rb
View File

@ -258,10 +258,13 @@ class BulkImport::Base
end end
def post_number_from_imported_id(id) def post_number_from_imported_id(id)
@post_number_by_post_id[post_id_from_imported_id(id)] post_id = post_id_from_imported_id(id)
post_id && @post_number_by_post_id[post_id]
end end
def topic_id_from_imported_post_id(id) def topic_id_from_imported_post_id(id)
@topic_id_by_post_id[post_id_from_imported_id(id)] post_id = post_id_from_imported_id(id)
post_id && @topic_id_by_post_id[post_id]
end end
GROUP_COLUMNS ||= %i{ GROUP_COLUMNS ||= %i{
@ -641,9 +644,8 @@ class BulkImport::Base
imported_username, imported_postid = $1, $2 imported_username, imported_postid = $1, $2
username = @mapped_usernames[imported_username] || imported_username username = @mapped_usernames[imported_username] || imported_username
post_id = post_id_from_imported_id(imported_postid) post_number = post_number_from_imported_id(imported_postid)
post_number = @post_number_by_post_id[post_id] topic_id = topic_id_from_imported_post_id(imported_post_id)
topic_id = @topic_id_by_post_id[post_id]
if post_number && topic_id if post_number && topic_id
"\n[quote=\"#{username}, post:#{post_number}, topic:#{topic_id}\"]\n" "\n[quote=\"#{username}, post:#{post_number}, topic:#{topic_id}\"]\n"
@ -688,8 +690,7 @@ class BulkImport::Base
@raw_connection.copy_data(sql, @encoder) do @raw_connection.copy_data(sql, @encoder) do
rows.each do |row| rows.each do |row|
begin begin
mapped = yield(row) next unless mapped = yield(row)
next unless mapped
processed = send(process_method_name, mapped) processed = send(process_method_name, mapped)
imported_ids << mapped[:imported_id] unless mapped[:imported_id].nil? imported_ids << mapped[:imported_id] unless mapped[:imported_id].nil?
imported_ids |= mapped[:imported_ids] unless mapped[:imported_ids].nil? imported_ids |= mapped[:imported_ids] unless mapped[:imported_ids].nil?
@ -697,7 +698,8 @@ class BulkImport::Base
print "\r%7d - %6d/sec" % [imported_ids.size, imported_ids.size.to_f / (Time.now - start)] if imported_ids.size % 5000 == 0 print "\r%7d - %6d/sec" % [imported_ids.size, imported_ids.size.to_f / (Time.now - start)] if imported_ids.size % 5000 == 0
rescue => e rescue => e
puts "\n" puts "\n"
puts "ERROR: #{e.inspect}" puts "ERROR: #{e.message}"
puts backtrace.join("\n")
end end
end end
end end
@ -725,8 +727,7 @@ class BulkImport::Base
sql = "COPY #{table}_custom_fields (#{table}_id, name, value, created_at, updated_at) FROM STDIN" sql = "COPY #{table}_custom_fields (#{table}_id, name, value, created_at, updated_at) FROM STDIN"
@raw_connection.copy_data(sql, @encoder) do @raw_connection.copy_data(sql, @encoder) do
rows.each do |row| rows.each do |row|
cf = yield row next unless cf = yield(row)
next unless cf
@raw_connection.put_copy_data [cf[:record_id], name, cf[:value], NOW, NOW] @raw_connection.put_copy_data [cf[:record_id], name, cf[:value], NOW, NOW]
end end
end end