From 953d3f2151d4b30961186a0d6a7dcc937d1c03df Mon Sep 17 00:00:00 2001 From: Sam Date: Fri, 8 Aug 2014 10:00:10 +1000 Subject: [PATCH] PERF: add optional delta badge filtering This is tricky and optional, some badges are very expensive to calculate pass in :backfill , :post_ids and :user_ids to all badge queries so they can do pre-filtering, if you do it after the sub-query it is too late --- app/models/badge.rb | 26 ++++++++++++++++++-------- app/services/badge_granter.rb | 24 +++++++++++++++++++----- 2 files changed, 37 insertions(+), 13 deletions(-) diff --git a/app/models/badge.rb b/app/models/badge.rb index 9b19b23b7be..460f8c1ade8 100644 --- a/app/models/badge.rb +++ b/app/models/badge.rb @@ -66,6 +66,7 @@ SQL FROM quoted_posts q1 JOIN badge_posts p1 ON p1.id = q1.post_id JOIN badge_posts p2 ON p2.id = q1.quoted_post_id + WHERE (:backfill OR ( p1.id IN (:post_ids) )) GROUP BY p1.user_id ) ids JOIN quoted_posts q ON q.id = ids.id @@ -79,7 +80,8 @@ SQL FROM topic_links l1 JOIN badge_posts p1 ON p1.id = l1.post_id JOIN badge_posts p2 ON p2.id = l1.link_post_id - WHERE NOT reflection AND p1.topic_id <> p2.topic_id AND not quote + WHERE NOT reflection AND p1.topic_id <> p2.topic_id AND not quote AND + (:backfill OR ( p1.id in (:post_ids) )) GROUP BY l1.user_id ) ids JOIN topic_links l ON l.id = ids.id @@ -104,7 +106,8 @@ SQL SELECT pa.user_id, min(pa.id) id FROM post_actions pa JOIN badge_posts p on p.id = pa.post_id - WHERE post_action_type_id IN (#{PostActionType.flag_types.values.join(",")}) + WHERE post_action_type_id IN (#{PostActionType.flag_types.values.join(",")}) AND + (:backfill OR pa.post_id IN (:post_ids) ) GROUP BY pa.user_id ) x JOIN post_actions pa1 on pa1.id = x.id @@ -116,7 +119,8 @@ SQL SELECT pa.user_id, min(pa.id) id FROM post_actions pa JOIN badge_posts p on p.id = pa.post_id - WHERE post_action_type_id = 2 + WHERE post_action_type_id = 2 AND + (:backfill OR pa.post_id IN (:post_ids) ) GROUP BY pa.user_id ) x JOIN post_actions pa1 on pa1.id = x.id @@ -126,7 +130,8 @@ SQL Editor = < 0 + WHERE p.self_edits > 0 AND + (:backfill OR p.id IN (:post_ids) ) GROUP BY p.user_id SQL @@ -134,7 +139,8 @@ SQL SELECT p.user_id, min(post_id) post_id, min(pa.created_at) granted_at FROM post_actions pa JOIN badge_posts p on p.id = pa.post_id - WHERE post_action_type_id = 2 + WHERE post_action_type_id = 2 AND + (:backfill OR pa.post_id IN (:post_ids) ) GROUP BY p.user_id SQL @@ -143,7 +149,8 @@ SQL FROM users u JOIN user_profiles up on u.id = up.user_id WHERE bio_raw IS NOT NULL AND LENGTH(TRIM(bio_raw)) > #{Badge::AutobiographerMinBioLength} AND - uploaded_avatar_id IS NOT NULL + uploaded_avatar_id IS NOT NULL AND + (:backfill OR u.id IN (:user_ids) ) SQL def self.like_badge(count) @@ -151,7 +158,8 @@ SQL " SELECT p.user_id, p.id post_id, p.updated_at granted_at FROM badge_posts p - WHERE p.like_count >= #{count.to_i} + WHERE p.like_count >= #{count.to_i} AND + (:backfill OR p.id IN (:post_ids) ) " end @@ -159,7 +167,9 @@ SQL # we can do better with dates, but its hard work figuring this out historically " SELECT u.id user_id, current_timestamp granted_at, NULL post_id FROM users u - WHERE trust_level >= #{level.to_i} + WHERE trust_level >= #{level.to_i} AND ( + :backfill OR u.id IN (:user_ids) + ) " end end diff --git a/app/services/badge_granter.rb b/app/services/badge_granter.rb index 3a75db3c792..8a87123fbf2 100644 --- a/app/services/badge_granter.rb +++ b/app/services/badge_granter.rb @@ -154,7 +154,12 @@ class BadgeGranter post_ids = opts[:post_ids] if opts user_ids = opts[:user_ids] if opts - post_clause = badge.target_posts ? "AND q.post_id = ub.post_id" : "" + post_ids = nil unless post_ids.present? + user_ids = nil unless user_ids.present? + + full_backfill = !user_ids && !post_ids + + post_clause = badge.target_posts ? "AND (q.post_id = ub.post_id OR NOT :multiple_grant)" : "" post_id_field = badge.target_posts ? "q.post_id" : "NULL" sql = "DELETE FROM user_badges @@ -167,7 +172,12 @@ class BadgeGranter WHERE ub.badge_id = :id AND q.user_id IS NULL )" - Badge.exec_sql(sql, id: badge.id) if badge.auto_revoke && !post_ids && !user_ids + Badge.exec_sql(sql, id: badge.id, + post_ids: [-1], + user_ids: [-2], + backfill: true, + multiple_grant: true # cheat here, cause we only run on backfill and are deleting + ) if badge.auto_revoke && full_backfill sql = "INSERT INTO user_badges(badge_id, user_id, granted_at, granted_by_id, post_id) SELECT :id, q.user_id, q.granted_at, -1, #{post_id_field} @@ -181,10 +191,14 @@ class BadgeGranter builder = SqlBuilder.new(sql) builder.where("ub.badge_id IS NULL AND q.user_id <> -1") - builder.where("q.post_id in (:post_ids)", post_ids: post_ids) if post_ids.present? - builder.where("q.user_id in (:user_ids)", user_ids: user_ids) if user_ids.present? + builder.where("q.post_id in (:post_ids)") if post_ids + builder.where("q.user_id in (:user_ids)") if user_ids - builder.map_exec(OpenStruct, id: badge.id).each do |row| + builder.map_exec(OpenStruct, id: badge.id, + multiple_grant: badge.multiple_grant, + backfill: full_backfill, + post_ids: post_ids || [-2], + user_ids: user_ids || [-2]).each do |row| # old bronze badges do not matter next if badge.badge_type_id == BadgeType::Bronze and row.granted_at < 2.days.ago