OPTIM: change Top Topic's score computation algorithm

New algorithm will not wipe the entire table anymore and will only remove
invisible topics and add new visible topics.

app/jobs/scheduled/daily.rb

@@ -1,19 +0,0 @@
-require_dependency 'score_calculator'
-
-module Jobs
-
-  # This job will run on a regular basis to update statistics and denormalized data.
-  # If it does not run, the site will not function properly.
-  class Daily < Jobs::Scheduled
-    every 1.day
-
-    def execute(args)
-      # TODO: optimise this against a big site before doing this any more
-      # frequently
-      #
-      # current implementation wipes an entire table and rebuilds causing huge
-      # amounts of IO
-      TopTopic.refresh!
-    end
-  end
-end

app/jobs/scheduled/periodical_updates.rb

@@ -21,6 +21,9 @@ module Jobs
       # Update the scores of posts
       ScoreCalculator.new.calculate(1.day.ago)
 
+      # Update the scores of topics
+      TopTopic.refresh!
+
       # Automatically close stuff that we missed
       Topic.auto_close
     end

app/models/top_topic.rb

@@ -12,20 +12,11 @@ class TopTopic < ActiveRecord::Base
 
   def self.refresh!
     transaction do
-      # clean up the table
+      # update the topics list
-      exec_sql("DELETE FROM top_topics")
+      remove_invisible_topics
-      # insert the list of all the visible topics
+      add_new_visible_topics
-      exec_sql("INSERT INTO top_topics (topic_id)
+      # update the denormalized data
-                SELECT id
-                FROM topics
-                WHERE deleted_at IS NULL
-                AND visible
-                AND archetype <> :private_message
-                AND NOT archived",
-                private_message: Archetype::private_message)
-
       TopTopic.periods.each do |period|
-        # update all the counter caches
         TopTopic.sort_orders.each do |sort|
           TopTopic.send("update_#{sort}_count_for", period)
         end
@@ -35,14 +26,45 @@ class TopTopic < ActiveRecord::Base
     end
   end
 
+  def self.remove_invisible_topics
+    exec_sql("WITH invisible_topics AS (
+                SELECT id
+                FROM topics
+                WHERE deleted_at IS NOT NULL
+                   OR NOT visible
+                   OR archetype = :private_message
+                   OR archived
+              )
+              DELETE FROM top_topics
+              WHERE topic_id IN (SELECT id FROM invisible_topics)",
+              private_message: Archetype::private_message)
+  end
+
+  def self.add_new_visible_topics
+    exec_sql("WITH visible_topics AS (
+              SELECT t.id
+              FROM topics t
+              LEFT JOIN top_topics tt ON t.id = tt.topic_id
+              WHERE t.deleted_at IS NULL
+                AND t.visible
+                AND t.archetype <> :private_message
+                AND NOT t.archived
+                AND tt.topic_id IS NULL
+            )
+            INSERT INTO top_topics (topic_id)
+            SELECT id
+            FROM visible_topics",
+              private_message: Archetype::private_message)
+  end
+
   def self.update_posts_count_for(period)
     sql = "SELECT topic_id, GREATEST(COUNT(*), 1) AS count
            FROM posts
            WHERE created_at >= :from
-           AND deleted_at IS NULL
+             AND deleted_at IS NULL
-           AND NOT hidden
+             AND NOT hidden
-           AND post_type = #{Post.types[:regular]}
+             AND post_type = #{Post.types[:regular]}
-           AND user_id <> #{Discourse.system_user.id}
+             AND user_id <> #{Discourse.system_user.id}
            GROUP BY topic_id"
 
     TopTopic.update_top_topics(period, "posts", sql)
@@ -61,8 +83,8 @@ class TopTopic < ActiveRecord::Base
     sql = "SELECT topic_id, GREATEST(SUM(like_count), 1) AS count
            FROM posts
            WHERE created_at >= :from
-           AND deleted_at IS NULL
+             AND deleted_at IS NULL
-           AND NOT hidden
+             AND NOT hidden
            GROUP BY topic_id"
 
     TopTopic.update_top_topics(period, "likes", sql)