IntenseWebs/discourse
3
0
Fork 0
mirror of https://github.com/discourse/discourse.git synced 2026-09-05 04:40:41 -05:00
Code Issues Packages Projects Releases Wiki Activity

DEV: remove exec_sql and replace with mini_sql

Browse Source 
Introduce new patterns for direct sql that are safe and fast.

MiniSql is not prone to memory bloat that can happen with direct PG usage.
It also has an extremely fast materializer and very a convenient API

- DB.exec(sql, *params) => runs sql returns row count
- DB.query(sql, *params) => runs sql returns usable objects (not a hash)
- DB.query_hash(sql, *params) => runs sql returns an array of hashes
- DB.query_single(sql, *params) => runs sql and returns a flat one dimensional array
- DB.build(sql) => returns a sql builder

See more at: https://github.com/discourse/mini_sql
This commit is contained in:
Sam
2018-06-19 16:13:36 +10:00
parent cc3fc87dd7
commit 5f64fd0a21
112 changed files with 782 additions and 763 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/services/badge_granter.rb
+58 -51
View File
@@ -204,17 +204,18 @@ class BadgeGranter
end
query_plan = nil
# HACK: active record is weird, force it to go down the sanitization path that cares not for % stuff
query_plan = ActiveRecord::Base.exec_sql("EXPLAIN #{sql} /*:backfill*/", params) if opts[:explain]
# HACK: active record sanitization too flexible, force it to go down the sanitization path that cares not for % stuff
# note mini_sql uses AR sanitizer at the moment (review if changed)
query_plan = DB.query_hash("EXPLAIN #{sql} /*:backfill*/", params) if opts[:explain]
sample = SqlBuilder.map_exec(OpenStruct, grants_sql, params).map(&:to_h)
sample = DB.query(grants_sql, params)
sample.each do |result|
raise "Query returned a non-existent user ID:\n#{result[:id]}" unless User.find(result[:id]).present?
raise "Query did not return a badge grant time\n(Try using 'current_timestamp granted_at')" unless result[:granted_at]
raise "Query returned a non-existent user ID:\n#{result.id}" unless User.exists?(id: result.id)
raise "Query did not return a badge grant time\n(Try using 'current_timestamp granted_at')" unless result.granted_at
if opts[:target_posts]
raise "Query did not return a post ID" unless result[:post_id]
raise "Query returned a non-existent post ID:\n#{result[:post_id]}" unless Post.find(result[:post_id]).present?
raise "Query did not return a post ID" unless result.post_id
raise "Query returned a non-existent post ID:\n#{result.post_id}" unless Post.exists?(result.post_id).present?
end
end
@@ -258,28 +259,31 @@ class BadgeGranter
WHERE ub.badge_id = :id AND q.user_id IS NULL
)"
Badge.exec_sql(sql, id: badge.id,
post_ids: [-1],
user_ids: [-2],
backfill: true,
multiple_grant: true # cheat here, cause we only run on backfill and are deleting
) if badge.auto_revoke && full_backfill
DB.exec(
sql,
id: badge.id,
post_ids: [-1],
user_ids: [-2],
backfill: true,
multiple_grant: true # cheat here, cause we only run on backfill and are deleting
) if badge.auto_revoke && full_backfill
sql = " WITH w as (
INSERT INTO user_badges(badge_id, user_id, granted_at, granted_by_id, post_id)
SELECT :id, q.user_id, q.granted_at, -1, #{post_id_field}
FROM ( #{badge.query} ) q
LEFT JOIN user_badges ub ON
ub.badge_id = :id AND ub.user_id = q.user_id
#{post_clause}
/*where*/
RETURNING id, user_id, granted_at
)
select w.*, username, locale, (u.admin OR u.moderator) AS staff FROM w
JOIN users u on u.id = w.user_id
"
sql = <<~SQL
WITH w as (
INSERT INTO user_badges(badge_id, user_id, granted_at, granted_by_id, post_id)
SELECT :id, q.user_id, q.granted_at, -1, #{post_id_field}
FROM ( #{badge.query} ) q
LEFT JOIN user_badges ub ON
ub.badge_id = :id AND ub.user_id = q.user_id
#{post_clause}
/*where*/
RETURNING id, user_id, granted_at
)
select w.*, username, locale, (u.admin OR u.moderator) AS staff FROM w
JOIN users u on u.id = w.user_id
SQL
builder = SqlBuilder.new(sql)
builder = DB.build(sql)
builder.where("ub.badge_id IS NULL AND q.user_id <> -1")
if (post_ids || user_ids) && !badge.query.include?(":backfill")
@@ -297,11 +301,12 @@ class BadgeGranter
return
end
builder.map_exec(OpenStruct, id: badge.id,
multiple_grant: badge.multiple_grant,
backfill: full_backfill,
post_ids: post_ids || [-2],
user_ids: user_ids || [-2]).each do |row|
builder.query(
id: badge.id,
multiple_grant: badge.multiple_grant,
backfill: full_backfill,
post_ids: post_ids || [-2],
user_ids: user_ids || [-2]).each do |row|
# old bronze badges do not matter
next if badge.badge_type_id == (BadgeType::Bronze) && row.granted_at < (2.days.ago)
@@ -332,10 +337,11 @@ class BadgeGranter
}.to_json)
end
Badge.exec_sql("UPDATE user_badges SET notification_id = :notification_id WHERE id = :id",
notification_id: notification.id,
id: row.id
)
DB.exec(
"UPDATE user_badges SET notification_id = :notification_id WHERE id = :id",
notification_id: notification.id,
id: row.id
)
end
badge.reset_grant_count!
@@ -345,21 +351,22 @@ class BadgeGranter
end
def self.revoke_ungranted_titles!
Badge.exec_sql("UPDATE users SET title = ''
WHERE NOT title IS NULL AND
title <> '' AND
EXISTS (
SELECT 1
FROM user_profiles
WHERE user_id = users.id AND badge_granted_title
) AND
title NOT IN (
SELECT name
FROM badges
WHERE allow_title AND enabled AND
badges.id IN (SELECT badge_id FROM user_badges ub where ub.user_id = users.id)
)
")
DB.exec <<~SQL
UPDATE users SET title = ''
WHERE NOT title IS NULL AND
title <> '' AND
EXISTS (
SELECT 1
FROM user_profiles
WHERE user_id = users.id AND badge_granted_title
) AND
title NOT IN (
SELECT name
FROM badges
WHERE allow_title AND enabled AND
badges.id IN (SELECT badge_id FROM user_badges ub where ub.user_id = users.id)
)
SQL
end
end
app/services/post_alerter.rb
+8 -6
View File
@@ -194,16 +194,18 @@ class PostAlerter
}
def group_stats(topic)
sql = <<~SQL
SELECT COUNT(*) FROM topics t
JOIN topic_allowed_groups g ON g.group_id = :group_id AND g.topic_id = t.id
LEFT JOIN group_archived_messages a ON a.topic_id = t.id AND a.group_id = g.group_id
WHERE a.id IS NULL AND t.deleted_at is NULL AND t.archetype = 'private_message'
SQL
topic.allowed_groups.map do |g|
{
group_id: g.id,
group_name: g.name.downcase,
inbox_count: Topic.exec_sql(
"SELECT COUNT(*) FROM topics t
JOIN topic_allowed_groups g ON g.group_id = :group_id AND g.topic_id = t.id
LEFT JOIN group_archived_messages a ON a.topic_id = t.id AND a.group_id = g.group_id
WHERE a.id IS NULL AND t.deleted_at is NULL AND t.archetype = 'private_message'",
group_id: g.id).values[0][0].to_i
inbox_count: DB.query_single(sql, group_id: g.id).first.to_i
}
end
end
app/services/search_indexer.rb
+3 -3
View File
@@ -61,7 +61,7 @@ class SearchIndexer
# Would be nice to use AR here but not sure how to execut Postgres functions
# when inserting data like this.
rows = Post.exec_sql_row_count(<<~SQL, params)
rows = DB.exec(<<~SQL, params)
UPDATE #{table_name}
SET
raw_data = :raw_data,
@@ -72,7 +72,7 @@ class SearchIndexer
SQL
if rows == 0
Post.exec_sql(<<~SQL, params)
DB.exec(<<~SQL, params)
INSERT INTO #{table_name}
(#{foreign_key}, search_data, locale, raw_data, version)
VALUES (:id, #{ranked_index}, :locale, :raw_data, :version)
@@ -111,7 +111,7 @@ class SearchIndexer
def self.queue_post_reindex(topic_id)
return if @disabled
ActiveRecord::Base.exec_sql(<<~SQL, topic_id: topic_id)
DB.exec(<<~SQL, topic_id: topic_id)
UPDATE post_search_data
SET version = 0
WHERE post_id IN (SELECT id FROM posts WHERE topic_id = :topic_id)
app/services/user_merger.rb
+15 -13
View File
@@ -89,11 +89,13 @@ class UserMerger
limit_reached = EXCLUDED.limit_reached
SQL
GivenDailyLike.exec_sql(sql,
source_user_id: @source_user.id,
target_user_id: @target_user.id,
max_likes_per_day: SiteSetting.max_likes_per_day,
action_type_id: PostActionType.types[:like])
DB.exec(
sql,
source_user_id: @source_user.id,
target_user_id: @target_user.id,
max_likes_per_day: SiteSetting.max_likes_per_day,
action_type_id: PostActionType.types[:like]
)
end
def merge_post_timings
@@ -107,7 +109,7 @@ class UserMerger
AND t.topic_id = s.topic_id AND t.post_number = s.post_number
SQL
PostTiming.exec_sql(sql, source_user_id: @source_user.id, target_user_id: @target_user.id)
DB.exec(sql, source_user_id: @source_user.id, target_user_id: @target_user.id)
end
def merge_user_visits
@@ -123,7 +125,7 @@ class UserMerger
AND t.visited_at = s.visited_at
SQL
UserVisit.exec_sql(sql, source_user_id: @source_user.id, target_user_id: @target_user.id)
DB.exec(sql, source_user_id: @source_user.id, target_user_id: @target_user.id)
end
def update_site_settings
@@ -136,7 +138,7 @@ class UserMerger
def update_user_stats
# topics_entered
UserStat.exec_sql(<<~SQL, target_user_id: @target_user.id)
DB.exec(<<~SQL, target_user_id: @target_user.id)
UPDATE user_stats
SET topics_entered = (
SELECT COUNT(topic_id)
@@ -147,7 +149,7 @@ class UserMerger
SQL
# time_read and days_visited
UserStat.exec_sql(<<~SQL, target_user_id: @target_user.id)
DB.exec(<<~SQL, target_user_id: @target_user.id)
UPDATE user_stats
SET time_read = COALESCE(x.time_read, 0),
days_visited = COALESCE(x.days_visited, 0)
@@ -162,7 +164,7 @@ class UserMerger
SQL
# posts_read_count
UserStat.exec_sql(<<~SQL, target_user_id: @target_user.id)
DB.exec(<<~SQL, target_user_id: @target_user.id)
UPDATE user_stats
SET posts_read_count = (
SELECT COUNT(1)
@@ -176,7 +178,7 @@ class UserMerger
SQL
# likes_given, likes_received, new_since, read_faq, first_post_created_at
UserStat.exec_sql(<<~SQL, source_user_id: @source_user.id, target_user_id: @target_user.id)
DB.exec(<<~SQL, source_user_id: @source_user.id, target_user_id: @target_user.id)
UPDATE user_stats AS t
SET likes_given = t.likes_given + s.likes_given,
likes_received = t.likes_received + s.likes_received,
@@ -189,7 +191,7 @@ class UserMerger
end
def merge_user_attributes
User.exec_sql(<<~SQL, source_user_id: @source_user.id, target_user_id: @target_user.id)
DB.exec(<<~SQL, source_user_id: @source_user.id, target_user_id: @target_user.id)
UPDATE users AS t
SET created_at = LEAST(t.created_at, s.created_at),
updated_at = LEAST(t.updated_at, s.updated_at),
@@ -213,7 +215,7 @@ class UserMerger
WHERE t.id = :target_user_id AND s.id = :source_user_id
SQL
UserProfile.exec_sql(<<~SQL, source_user_id: @source_user.id, target_user_id: @target_user.id)
DB.exec(<<~SQL, source_user_id: @source_user.id, target_user_id: @target_user.id)
UPDATE user_profiles AS t
SET location = COALESCE(t.location, s.location),
website = COALESCE(t.website, s.website),
app/services/user_updater.rb
+12 -11
View File
@@ -143,17 +143,18 @@ class UserUpdater
MutedUser.where('user_id = ? AND muted_user_id not in (?)', user.id, desired_ids).destroy_all
# SQL is easier here than figuring out how to do the same in AR
MutedUser.exec_sql("INSERT into muted_users(user_id, muted_user_id, created_at, updated_at)
SELECT :user_id, id, :now, :now
FROM users
WHERE
id in (:desired_ids) AND
id NOT IN (
SELECT muted_user_id
FROM muted_users
WHERE user_id = :user_id
)",
now: Time.now, user_id: user.id, desired_ids: desired_ids)
DB.exec(<<~SQL, now: Time.now, user_id: user.id, desired_ids: desired_ids)
INSERT into muted_users(user_id, muted_user_id, created_at, updated_at)
SELECT :user_id, id, :now, :now
FROM users
WHERE
id in (:desired_ids) AND
id NOT IN (
SELECT muted_user_id
FROM muted_users
WHERE user_id = :user_id
)
SQL
end
end