IntenseWebs/discourse
3
0
Fork 0
mirror of https://github.com/discourse/discourse.git synced 2026-08-11 05:25:16 -05:00
Code Issues Packages Projects Releases Wiki Activity

DEV: Add support for converting and importing topics (#34767)

Browse Source 
This adds converter(Discourse-only, for now) and importer steps for
`topics`.
This commit is contained in:
Selase Krakani
2025-09-30 11:52:22 +00:00
committed by GitHub
parent c909c79b28
commit e70b60ff7f
6 changed files with 323 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
migrations/config/intermediate_db.yml
+31 -1
View File
@@ -128,6 +128,37 @@ schema:
exclude:
- "id"
primary_key_column_names: [ "user_id" , "name", "value" ]
topics:
columns:
exclude:
- "last_posted_at"
- "last_post_user_id"
- "reply_count"
- "featured_user1_id"
- "featured_user2_id"
- "featured_user3_id"
- "featured_user4_id"
- "image_upload_id"
- "highest_post_number"
- "like_count"
- "locale"
- "incoming_link_count"
- "moderator_posts_count"
- "bumped_at"
- "has_summary"
- "notify_moderators_count"
- "spam_count"
- "percent_rank"
- "posts_count"
- "score"
- "slug"
- "participant_count"
- "word_count"
- "excerpt"
- "fancy_title"
- "highest_staff_post_number"
- "reviewable_score"
- "slow_mode_seconds"
user_emails:
columns:
include:
@@ -436,7 +467,6 @@ schema:
- "topic_voting_category_settings"
- "topic_voting_topic_vote_count"
- "topic_voting_votes"
- "topics"
- "translation_overrides"
- "unsubscribe_keys"
- "upload_references"
migrations/db/intermediate_db_schema/100-base-schema.sql
+24
View File
@@ -191,6 +191,30 @@ CREATE TABLE tags
name TEXT NOT NULL
);
CREATE TABLE topics
(
original_id NUMERIC NOT NULL PRIMARY KEY,
archetype TEXT,
archived BOOLEAN,
bannered_until DATETIME,
category_id NUMERIC,
closed BOOLEAN,
created_at DATETIME,
deleted_at DATETIME,
deleted_by_id NUMERIC,
external_id NUMERIC,
featured_link TEXT,
pinned_at DATETIME,
pinned_globally BOOLEAN,
pinned_until DATETIME,
subtype TEXT,
title TEXT NOT NULL,
user_id NUMERIC,
views INTEGER,
visibility_reason_id NUMERIC,
visible BOOLEAN
);
CREATE TABLE user_associated_accounts
(
provider_name TEXT NOT NULL,
migrations/lib/converters/discourse/steps/topics.rb
+44
View File
@@ -0,0 +1,44 @@
# frozen_string_literal: true
module Migrations::Converters::Discourse
class Topics < ::Migrations::Converters::Base::ProgressStep
attr_accessor :source_db
def max_progress
@source_db.count <<~SQL
SELECT COUNT(*) FROM topics
SQL
end
def items
@source_db.query <<~SQL
SELECT * FROM topics
SQL
end
def process_item(item)
IntermediateDB::Topic.create(
original_id: item[:id],
archetype: item[:archetype],
archived: item[:archived],
bannered_until: item[:bannered_until],
category_id: item[:category_id],
closed: item[:closed],
created_at: item[:created_at],
deleted_at: item[:deleted_at],
deleted_by_id: item[:deleted_by_id],
external_id: item[:external_id],
featured_link: item[:featured_link],
pinned_at: item[:pinned_at],
pinned_globally: item[:pinned_globally],
pinned_until: item[:pinned_until],
subtype: item[:subtype],
title: item[:title],
user_id: item[:user_id],
views: item[:views],
visibility_reason_id: item[:visibility_reason_id],
visible: item[:visible],
)
end
end
end
migrations/lib/database/intermediate_db/topic.rb
+85
View File
@@ -0,0 +1,85 @@
# frozen_string_literal: true
# This file is auto-generated from the IntermediateDB schema. To make changes,
# update the "config/intermediate_db.yml" configuration file and then run
# `bin/cli schema generate` to regenerate this file.
module Migrations::Database::IntermediateDB
module Topic
SQL = <<~SQL
INSERT INTO topics (
original_id,
archetype,
archived,
bannered_until,
category_id,
closed,
created_at,
deleted_at,
deleted_by_id,
external_id,
featured_link,
pinned_at,
pinned_globally,
pinned_until,
subtype,
title,
user_id,
views,
visibility_reason_id,
visible
)
VALUES (
?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?
)
SQL
private_constant :SQL
def self.create(
original_id:,
archetype: nil,
archived: nil,
bannered_until: nil,
category_id: nil,
closed: nil,
created_at: nil,
deleted_at: nil,
deleted_by_id: nil,
external_id: nil,
featured_link: nil,
pinned_at: nil,
pinned_globally: nil,
pinned_until: nil,
subtype: nil,
title:,
user_id: nil,
views: nil,
visibility_reason_id: nil,
visible: nil
)
::Migrations::Database::IntermediateDB.insert(
SQL,
original_id,
archetype,
::Migrations::Database.format_boolean(archived),
::Migrations::Database.format_datetime(bannered_until),
category_id,
::Migrations::Database.format_boolean(closed),
::Migrations::Database.format_datetime(created_at),
::Migrations::Database.format_datetime(deleted_at),
deleted_by_id,
external_id,
featured_link,
::Migrations::Database.format_datetime(pinned_at),
::Migrations::Database.format_boolean(pinned_globally),
::Migrations::Database.format_datetime(pinned_until),
subtype,
title,
user_id,
views,
visibility_reason_id,
::Migrations::Database.format_boolean(visible),
)
end
end
end
migrations/lib/importer/mapping_type.rb
+1
View File
@@ -10,6 +10,7 @@ module Migrations::Importer
TAG_GROUPS = 6
BADGE_GROUPINGS = 7
USER_FIELDS = 8
TOPICS = 9
UPLOADS = 10
end
end
migrations/lib/importer/steps/topics.rb
+138
View File
@@ -0,0 +1,138 @@
# frozen_string_literal: true
module Migrations::Importer::Steps
class Topics < ::Migrations::Importer::CopyStep
ARCHETYPES = Archetype.list.map(&:id).to_set.freeze
DEFAULT_ARCHETYPE = Archetype.default
SUBTYPES = TopicSubtype.instance_variable_get(:@subtypes).keys.to_set.freeze
VISIBILITY_REASONS = Topic.visibility_reasons.values.to_set.freeze
DEFAULT_VIEWS = 0
EXTERNAL_ID_FORMAT = /\A[\w-]+\z/
UNCATEGORIZED_ID = SiteSetting.uncategorized_category_id
MAX_TOPIC_TITLE_LENGTH = SiteSetting.max_topic_title_length
depends_on :categories, :users, :uploads
store_mapped_ids true
requires_set :existing_external_ids, "SELECT LOWER(external_id) FROM topics"
column_names %i[
id
archetype
archived
bannered_until
bumped_at
category_id
closed
created_at
deleted_at
deleted_by_id
external_id
featured_link
last_post_user_id
pinned_at
pinned_globally
pinned_until
subtype
slug
title
updated_at
user_id
views
visibility_reason_id
visible
]
total_rows_query <<~SQL, MappingType::TOPICS
SELECT COUNT(*)
FROM topics
LEFT JOIN mapped.ids mapped_topic
ON topics.original_id = mapped_topic.original_id AND mapped_topic.type = ?1
WHERE mapped_topic.original_id IS NULL
SQL
rows_query <<~SQL, MappingType::TOPICS, MappingType::CATEGORIES, MappingType::USERS
SELECT topics.*,
mapped_category.discourse_id AS discourse_category_id,
mapped_user.discourse_id AS discourse_user_id,
mapped_deleted_by_user.discourse_id AS discourse_deleted_by_user_id
FROM topics
LEFT JOIN mapped.ids mapped_topic
ON topics.original_id = mapped_topic.original_id AND mapped_topic.type = ?1
LEFT JOIN mapped.ids mapped_category
ON topics.category_id = mapped_category.original_id AND mapped_category.type = ?2
LEFT JOIN mapped.ids mapped_user
ON topics.user_id = mapped_user.original_id AND mapped_user.type = ?3
LEFT JOIN mapped.ids mapped_deleted_by_user
ON topics.deleted_by_id = mapped_deleted_by_user.original_id AND mapped_deleted_by_user.type = ?3
WHERE mapped_topic.original_id IS NULL
SQL
private
def transform_row(row)
return nil if row[:archetype] != Archetype.private_message && row[:discourse_category_id].nil?
if (external_id = row[:external_id])
return nil unless process_external_id(external_id)
end
row[:archived] ||= false
row[:closed] ||= false
row[:visible] = true if row[:visible].nil?
row[:views] ||= DEFAULT_VIEWS
row[:category_id] = row[:discourse_category_id] ||
(UNCATEGORIZED_ID if row[:archetype] != Archetype.private_message)
row[:deleted_by_id] = row[:discourse_deleted_by_user_id]
row[:user_id] = row[:discourse_user_id] || SYSTEM_USER_ID
row[:title] = row[:title][0, MAX_TOPIC_TITLE_LENGTH].scrub.strip
row[:last_post_user_id] ||= row[:user_id]
row[:slug] = Slug.for(row[:title])
row[:bumped_at] = row[:created_at]
row[:archetype] = ensure_valid_value(
value: row[:archetype],
allowed_set: ARCHETYPES,
default_value: DEFAULT_ARCHETYPE,
) do |_, default_value|
puts " #{row[:id]}: Topic archetype is invalid, defaulting to #{default_value}"
end
row[:subtype] = ensure_valid_value(
value: row[:subtype],
allowed_set: SUBTYPES,
default_value: nil,
) if row[:subtype]
row[:visibility_reason_id] = ensure_valid_value(
value: row[:visibility_reason_id],
allowed_set: VISIBILITY_REASONS,
default_value: nil,
) if row[:visibility_reason_id]
super
end
def process_external_id(external_id)
return nil if external_id.blank?
external_id = external_id.strip
if external_id.length > Topic::EXTERNAL_ID_MAX_LENGTH ||
!external_id.match?(EXTERNAL_ID_FORMAT)
puts " Invalid format or length for external_id '#{external_id}'"
return false
end
unless @existing_external_ids.add?(external_id.downcase)
puts " Duplicate external_id '#{external_id}'"
return false
end
true
end
end
end