DEV: remove exec_sql and replace with mini_sql

Introduce new patterns for direct sql that are safe and fast.

MiniSql is not prone to memory bloat that can happen with direct PG usage.
It also has an extremely fast materializer and very a convenient API

- DB.exec(sql, *params) => runs sql returns row count
- DB.query(sql, *params) => runs sql returns usable objects (not a hash)
- DB.query_hash(sql, *params) => runs sql returns an array of hashes
- DB.query_single(sql, *params) => runs sql and returns a flat one dimensional array
- DB.build(sql) => returns a sql builder

See more at: https://github.com/discourse/mini_sql
This commit is contained in:
Sam
2018-06-19 16:13:14 +10:00
parent cc3fc87dd7
commit 5f64fd0a21
112 changed files with 782 additions and 763 deletions

View File

@@ -29,7 +29,7 @@ MS_SPEND_CREATING_POST ||= 5000
def insert_post_timings
log "Inserting post timings..."
exec_sql <<-SQL
DB.exec <<-SQL
INSERT INTO post_timings (topic_id, post_number, user_id, msecs)
SELECT topic_id, post_number, user_id, #{MS_SPEND_CREATING_POST}
FROM posts
@@ -41,7 +41,7 @@ end
def insert_post_replies
log "Inserting post replies..."
exec_sql <<-SQL
DB.exec <<-SQL
INSERT INTO post_replies (post_id, reply_id, created_at, updated_at)
SELECT p2.id, p.id, p.created_at, p.created_at
FROM posts p
@@ -53,7 +53,7 @@ end
def insert_topic_users
log "Inserting topic users..."
exec_sql <<-SQL
DB.exec <<-SQL
INSERT INTO topic_users (user_id, topic_id, posted, last_read_post_number, highest_seen_post_number, first_visited_at, last_visited_at, total_msecs_viewed)
SELECT user_id, topic_id, 't' , MAX(post_number), MAX(post_number), MIN(created_at), MAX(created_at), COUNT(id) * #{MS_SPEND_CREATING_POST}
FROM posts
@@ -66,7 +66,7 @@ end
def insert_topic_views
log "Inserting topic views..."
exec_sql <<-SQL
DB.exec <<-SQL
WITH X AS (
SELECT topic_id, user_id, DATE(p.created_at) posted_at
FROM posts p
@@ -86,7 +86,7 @@ end
def insert_user_actions
log "Inserting user actions for NEW_TOPIC = 4..."
exec_sql <<-SQL
DB.exec <<-SQL
INSERT INTO user_actions (action_type, user_id, target_topic_id, target_post_id, acting_user_id, created_at, updated_at)
SELECT 4, p.user_id, topic_id, p.id, p.user_id, p.created_at, p.created_at
FROM posts p
@@ -100,7 +100,7 @@ def insert_user_actions
log "Inserting user actions for REPLY = 5..."
exec_sql <<-SQL
DB.exec <<-SQL
INSERT INTO user_actions (action_type, user_id, target_topic_id, target_post_id, acting_user_id, created_at, updated_at)
SELECT 5, p.user_id, topic_id, p.id, p.user_id, p.created_at, p.created_at
FROM posts p
@@ -114,7 +114,7 @@ def insert_user_actions
log "Inserting user actions for RESPONSE = 6..."
exec_sql <<-SQL
DB.exec <<-SQL
INSERT INTO user_actions (action_type, user_id, target_topic_id, target_post_id, acting_user_id, created_at, updated_at)
SELECT 6, p.user_id, p.topic_id, p.id, p2.user_id, p.created_at, p.created_at
FROM posts p
@@ -137,7 +137,7 @@ end
def insert_user_options
log "Inserting user options..."
exec_sql <<-SQL
DB.exec <<-SQL
INSERT INTO user_options (
user_id,
email_always,
@@ -189,7 +189,7 @@ end
def insert_user_stats
log "Inserting user stats..."
exec_sql <<-SQL
DB.exec <<-SQL
INSERT INTO user_stats (user_id, new_since)
SELECT id, created_at
FROM users
@@ -200,7 +200,7 @@ end
def insert_user_visits
log "Inserting user visits..."
exec_sql <<-SQL
DB.exec <<-SQL
INSERT INTO user_visits (user_id, visited_at, posts_read)
SELECT user_id, DATE(created_at), COUNT(*)
FROM posts
@@ -213,7 +213,7 @@ end
def insert_draft_sequences
log "Inserting draft sequences..."
exec_sql <<-SQL
DB.exec <<-SQL
INSERT INTO draft_sequences (user_id, draft_key, sequence)
SELECT user_id, CONCAT('#{Draft::EXISTING_TOPIC}', id), 1
FROM topics
@@ -226,7 +226,7 @@ end
def update_user_stats
log "Updating user stats..."
exec_sql <<-SQL
DB.exec <<-SQL
WITH X AS (
SELECT p.user_id
, COUNT(p.id) posts
@@ -283,7 +283,7 @@ end
def update_posts
log "Updating posts..."
exec_sql <<-SQL
DB.exec <<-SQL
WITH Y AS (
SELECT post_id, COUNT(*) replies FROM post_replies GROUP BY post_id
)
@@ -310,7 +310,7 @@ end
def update_topics
log "Updating topics..."
exec_sql <<-SQL
DB.exec <<-SQL
WITH X AS (
SELECT topic_id
, COUNT(*) posts
@@ -350,7 +350,7 @@ end
def update_categories
log "Updating categories..."
exec_sql <<-SQL
DB.exec <<-SQL
WITH X AS (
SELECT category_id
, MAX(p.id) post_id
@@ -382,7 +382,7 @@ end
def update_users
log "Updating users..."
exec_sql <<-SQL
DB.exec <<-SQL
WITH X AS (
SELECT user_id
, MIN(created_at) min_created_at
@@ -406,7 +406,7 @@ end
def update_groups
log "Updating groups..."
exec_sql <<-SQL
DB.exec <<-SQL
WITH X AS (
SELECT group_id, COUNT(*) count
FROM group_users
@@ -428,12 +428,6 @@ def log(message)
puts "[#{DateTime.now.strftime("%Y-%m-%d %H:%M:%S")}] #{message}"
end
def exec_sql(sql)
ActiveRecord::Base.transaction do
ActiveRecord::Base.exec_sql(sql)
end
end
task "import:create_phpbb_permalinks" => :environment do
log 'Creating Permalinks...'
@@ -477,7 +471,6 @@ task "import:remap_old_phpbb_permalinks" => :environment do
# skip
end
end
i
log "Done! #{i} posts remapped."
end