FEATURE: New 'Reviewable' model to make reviewable items generic

Includes support for flags, reviewable users and queued posts, with REST API
backwards compatibility.

Co-Authored-By: romanrizzi <romanalejandro@gmail.com>
Co-Authored-By: jjaffeux <j.jaffeux@gmail.com>

db/fixtures/007_web_hook_event_types.rb

@@ -37,3 +37,8 @@ WebHookEventType.seed do |b|
   b.id = WebHookEventType::QUEUED_POST
   b.name = "queued_post"
 end
+
+WebHookEventType.seed do |b|
+  b.id = WebHookEventType::REVIEWABLE
+  b.name = "reviewable"
+end

db/migrate/20190103160533_create_reviewables.rb

@@ -0,0 +1,41 @@
+class CreateReviewables < ActiveRecord::Migration[5.2]
+  def change
+    create_table :reviewables do |t|
+      t.string :type, null: false
+      t.integer :status, null: false, default: 0
+      t.integer :created_by_id, null: false
+
+      # Who can review this item? Moderators always can
+      t.boolean :reviewable_by_moderator, null: false, default: false
+      t.integer :reviewable_by_group_id, null: true
+
+      # On some high traffic sites they want things in review to be claimed
+      # so that two people don't work on the same thing.
+      t.integer :claimed_by_id, null: true
+
+      # For filtering
+      t.integer :category_id, null: true
+      t.integer :topic_id, null: true
+      t.float :score, null: false, default: 0
+      t.boolean :potential_spam, null: false, default: false
+
+      # Polymorphic relation of reviewable thing
+      t.integer :target_id, null: true
+      t.string :target_type, null: true
+      t.integer :target_created_by_id, null: true
+
+      t.json :payload, null: true
+
+      # Helps us prevent simultaneous updates
+      t.integer :version, null: false, default: 0
+
+      t.datetime :latest_score, null: true
+      t.timestamps
+    end
+
+    add_index :reviewables, :status
+    add_index :reviewables, [:status, :type]
+    add_index :reviewables, [:status, :score]
+    add_index :reviewables, [:type, :target_id], unique: true
+  end
+end

db/migrate/20190103185626_create_reviewable_users.rb

@@ -0,0 +1,56 @@
+class CreateReviewableUsers < ActiveRecord::Migration[5.2]
+  def up
+    # Create reviewables for approved users
+    if DB.query_single("SELECT 1 FROM site_settings WHERE name = 'must_approve_users' AND value = 't'").first
+      execute(<<~SQL)
+        INSERT INTO reviewables (
+          type,
+          status,
+          created_by_id,
+          reviewable_by_moderator,
+          target_type,
+          target_id,
+          created_at,
+          updated_at
+        )
+        SELECT 'ReviewableUser',
+          0,
+          #{Discourse::SYSTEM_USER_ID},
+          true,
+          'User',
+          id,
+          created_at,
+          created_at
+        FROM users
+        WHERE approved = false
+      SQL
+
+      # Migrate Created History
+      execute(<<~SQL)
+        INSERT INTO reviewable_histories (
+          reviewable_id,
+          reviewable_history_type,
+          status,
+          created_by_id,
+          created_at,
+          updated_at
+        )
+        SELECT r.id,
+          1,
+          1,
+          r.created_by_id,
+          r.created_at,
+          r.created_at
+        FROM reviewables AS r
+        WHERE r.type = 'ReviewableUser'
+      SQL
+    end
+  end
+
+  def down
+    execute(<<~SQL)
+      DELETE FROM reviewables
+      WHERE type = 'ReviewableUser'
+    SQL
+  end
+end

db/migrate/20190110212005_create_reviewable_histories.rb

@@ -0,0 +1,14 @@
+class CreateReviewableHistories < ActiveRecord::Migration[5.2]
+  def change
+    create_table :reviewable_histories do |t|
+      t.integer :reviewable_id, null: false
+      t.integer :reviewable_history_type, null: false
+      t.integer :status, null: false
+      t.integer :created_by_id, null: false
+      t.json    :edited, null: true
+      t.timestamps
+    end
+
+    add_index :reviewable_histories, :reviewable_id
+  end
+end

db/migrate/20190111170824_migrate_reviewable_queued_posts.rb

@@ -0,0 +1,104 @@
+class MigrateReviewableQueuedPosts < ActiveRecord::Migration[5.2]
+  def up
+    execute(<<~SQL)
+      INSERT INTO reviewables (
+        type,
+        status,
+        created_by_id,
+        reviewable_by_moderator,
+        topic_id,
+        category_id,
+        payload,
+        created_at,
+        updated_at
+      )
+      SELECT 'ReviewableQueuedPost',
+        state - 1,
+        user_id,
+        true,
+        topic_id,
+        nullif(post_options->>'category', '')::int,
+        json_build_object(
+          'old_queued_post_id', id,
+          'raw', raw
+        )::jsonb || post_options::jsonb,
+        created_at,
+        updated_at
+      FROM queued_posts
+    SQL
+
+    # Migrate Created History
+    execute(<<~SQL)
+      INSERT INTO reviewable_histories (
+        reviewable_id,
+        reviewable_history_type,
+        status,
+        created_by_id,
+        created_at,
+        updated_at
+      )
+      SELECT r.id,
+        0,
+        0,
+        qp.user_id,
+        qp.created_at,
+        qp.created_at
+      FROM reviewables AS r
+      INNER JOIN queued_posts AS qp ON qp.id = (payload->>'old_queued_post_id')::int
+    SQL
+
+    # Migrate Approved History
+    execute(<<~SQL)
+      INSERT INTO reviewable_histories (
+        reviewable_id,
+        reviewable_history_type,
+        status,
+        created_by_id,
+        created_at,
+        updated_at
+      )
+      SELECT r.id,
+        1,
+        1,
+        qp.approved_by_id,
+        qp.approved_at,
+        qp.approved_at
+      FROM reviewables AS r
+      INNER JOIN queued_posts AS qp ON qp.id = (payload->>'old_queued_post_id')::int
+      WHERE qp.state = 2
+    SQL
+
+    # Migrate Rejected History
+    execute(<<~SQL)
+      INSERT INTO reviewable_histories (
+        reviewable_id,
+        reviewable_history_type,
+        status,
+        created_by_id,
+        created_at,
+        updated_at
+      )
+      SELECT r.id,
+        1,
+        2,
+        qp.rejected_by_id,
+        qp.rejected_at,
+        qp.rejected_at
+      FROM reviewables AS r
+      INNER JOIN queued_posts AS qp ON qp.id = (payload->>'old_queued_post_id')::int
+      WHERE qp.state = 3
+    SQL
+  end
+
+  def down
+    execute(<<~SQL)
+      DELETE FROM reviewable_histories
+      WHERE reviewable_id IN (SELECT id FROM reviewables WHERE type = 'ReviewableQueuedPost')
+    SQL
+
+    execute(<<~SQL)
+      DELETE FROM reviewables
+      WHERE type = 'ReviewableQueuedPost'
+    SQL
+  end
+end

db/migrate/20190121202656_remove_user_action_pending.rb

@@ -0,0 +1,8 @@
+class RemoveUserActionPending < ActiveRecord::Migration[5.2]
+  def up
+    execute "DELETE FROM user_actions WHERE action_type = 14"
+  end
+
+  def down
+  end
+end

db/migrate/20190130163000_create_reviewable_scores.rb

@@ -0,0 +1,19 @@
+class CreateReviewableScores < ActiveRecord::Migration[5.2]
+  def change
+    create_table :reviewable_scores do |t|
+      t.integer :reviewable_id, null: false
+      t.integer :user_id, null: false
+      t.integer :reviewable_score_type, null: false
+      t.integer :status, null: false
+      t.float :score, null: false, default: 0
+      t.float :take_action_bonus, null: false, default: 0
+      t.integer :reviewed_by_id, null: true
+      t.datetime :reviewed_at, null: true
+      t.integer :meta_topic_id, null: true
+      t.timestamps
+    end
+
+    add_index :reviewable_scores, :reviewable_id
+    add_index :reviewable_scores, :user_id
+  end
+end

db/migrate/20190130163001_migrate_reviewable_flagged_posts.rb

@@ -0,0 +1,117 @@
+class MigrateReviewableFlaggedPosts < ActiveRecord::Migration[5.2]
+  def up
+
+    # for the migration we'll do 1.0 + trust_level and not take into account user flagging accuracy
+    # It should be good enough for old flags whose scores are not as important as pending flags.
+    execute(<<~SQL)
+      INSERT INTO reviewables (
+        type,
+        status,
+        topic_id,
+        category_id,
+        payload,
+        target_type,
+        target_id,
+        target_created_by_id,
+        score,
+        created_by_id,
+        created_at,
+        updated_at
+      )
+      SELECT 'ReviewableFlaggedPost',
+        CASE
+          WHEN MAX(pa.agreed_at) IS NOT NULL THEN 1
+          WHEN MAX(pa.disagreed_at) IS NOT NULL THEN 2
+          WHEN MAX(pa.deferred_at) IS NOT NULL THEN 3
+          WHEN MAX(pa.deleted_at) IS NOT NULL THEN 4
+          ELSE 0
+        END,
+        t.id,
+        t.category_id,
+        json_build_object(),
+        'Post',
+        pa.post_id,
+        p.user_id,
+        0,
+        MAX(pa.user_id),
+        MIN(pa.created_at),
+        MAX(pa.updated_at)
+      FROM post_actions AS pa
+      INNER JOIN posts AS p ON pa.post_id = p.id
+      INNER JOIN topics AS t ON t.id = p.topic_id
+      INNER JOIN post_action_types AS pat ON pat.id = pa.post_action_type_id
+      WHERE pat.is_flag
+        AND pat.name_key <> 'notify_user'
+        AND p.user_id > 0
+        AND p.deleted_at IS NULL
+        AND t.deleted_at IS NULL
+      GROUP BY pa.post_id,
+        t.id,
+        t.category_id,
+        p.user_id
+    SQL
+
+    execute(<<~SQL)
+      INSERT INTO reviewable_scores (
+        reviewable_id,
+        user_id,
+        reviewable_score_type,
+        status,
+        score,
+        meta_topic_id,
+        created_at,
+        updated_at
+      )
+      SELECT r.id,
+        pa.user_id,
+        pa.post_action_type_id,
+        CASE
+          WHEN pa.agreed_at IS NOT NULL THEN 1
+          WHEN pa.disagreed_at IS NOT NULL THEN 2
+          WHEN pa.deferred_at IS NOT NULL THEN 3
+          WHEN pa.deleted_at IS NOT NULL THEN 3
+          ELSE 0
+        END,
+        1.0 +
+        (CASE
+          WHEN pau.moderator OR pau.admin THEN 5.0
+          ELSE pau.trust_level
+        END) +
+        (CASE
+          WHEN pa.staff_took_action THEN 5.0
+          ELSE 0.0
+        END),
+        rp.topic_id,
+        pa.created_at,
+        pa.updated_at
+      FROM post_actions AS pa
+      INNER JOIN post_action_types AS pat ON pat.id = pa.post_action_type_id
+      INNER JOIN users AS pau ON pa.user_id = pau.id
+      INNER JOIN reviewables AS r ON pa.post_id = r.target_id
+      LEFT OUTER JOIN posts AS rp ON rp.id = pa.related_post_id
+      WHERE pat.is_flag
+        AND r.type = 'ReviewableFlaggedPost'
+    SQL
+
+    execute(<<~SQL)
+      UPDATE reviewables
+      SET score = COALESCE((
+        SELECT sum(score)
+        FROM reviewable_scores AS rs
+        WHERE rs.reviewable_id = reviewables.id
+          AND rs.status = 0
+      ), 0),
+      potential_spam = EXISTS(
+        SELECT 1
+        FROM reviewable_scores AS rs
+        WHERE rs.reviewable_id = reviewables.id
+          AND rs.reviewable_score_type = 8
+      )
+    SQL
+  end
+
+  def down
+    execute "DELETE FROM reviewables WHERE type = 'ReviewableFlaggedPost'"
+    execute "DELETE FROM reviewable_scores"
+  end
+end

db/migrate/20190215204033_add_score_bonus_to_post_action_types.rb

@@ -0,0 +1,5 @@
+class AddScoreBonusToPostActionTypes < ActiveRecord::Migration[5.2]
+  def change
+    add_column :post_action_types, :score_bonus, :float, default: 0.0, null: false
+  end
+end

db/migrate/20190306184409_add_reviewable_score_to_topics.rb

@@ -0,0 +1,22 @@
+class AddReviewableScoreToTopics < ActiveRecord::Migration[5.2]
+  def up
+    add_column :topics, :reviewable_score, :float, null: false, default: 0
+
+    execute(<<~SQL)
+      UPDATE topics
+      SET reviewable_score = sums.score
+      FROM (
+         SELECT SUM(r.score) AS score,
+           r.topic_id
+         FROM reviewables AS r
+         WHERE r.status = 0
+         GROUP BY r.topic_id
+      ) AS sums
+      WHERE sums.topic_id = topics.id
+    SQL
+  end
+
+  def down
+    remove_column :topics, :reviewable_score
+  end
+end

db/migrate/20190313171338_add_indexes_to_reviewables.rb

@@ -0,0 +1,11 @@
+class AddIndexesToReviewables < ActiveRecord::Migration[5.2]
+  def up
+    remove_index :reviewables, :status
+    add_index :reviewables, [:status, :created_at]
+  end
+
+  def down
+    remove_index :reviewables, [:status, :created_at]
+    add_index :reviewables, :status
+  end
+end

db/migrate/20190315170411_add_index_to_reviewable_histories.rb

@@ -0,0 +1,5 @@
+class AddIndexToReviewableHistories < ActiveRecord::Migration[5.2]
+  def change
+    add_index :reviewable_histories, :created_by_id
+  end
+end

db/migrate/20190315174428_migrate_flag_history.rb

@@ -0,0 +1,131 @@
+class MigrateFlagHistory < ActiveRecord::Migration[5.2]
+  def up
+
+    # Migrate Created History
+    execute(<<~SQL)
+      INSERT INTO reviewable_histories (
+        reviewable_id,
+        reviewable_history_type,
+        status,
+        created_by_id,
+        created_at,
+        updated_at
+      )
+      SELECT r.id,
+        0,
+        0,
+        r.created_by_id,
+        r.created_at,
+        r.created_at
+      FROM reviewables AS r
+      WHERE r.type = 'ReviewableFlaggedPost'
+        AND (
+          NOT EXISTS(
+            SELECT 1
+            FROM reviewable_histories AS rh
+            WHERE rh.reviewable_id = r.id
+              AND rh.reviewable_history_type = 0
+          )
+        )
+    SQL
+
+    # Approved
+    execute(<<~SQL)
+      INSERT INTO reviewable_histories (
+        reviewable_id,
+        reviewable_history_type,
+        status,
+        created_by_id,
+        created_at,
+        updated_at
+      )
+      SELECT r.id,
+        1,
+        1,
+        pa.agreed_by_id,
+        pa.agreed_at,
+        pa.agreed_at
+      FROM reviewables AS r
+      INNER JOIN post_actions AS pa ON pa.post_id = r.target_id
+      WHERE r.type = 'ReviewableFlaggedPost'
+        AND pa.agreed_at IS NOT NULL
+        AND pa.agreed_by_id IS NOT NULL
+    SQL
+
+    # Rejected
+    execute(<<~SQL)
+      INSERT INTO reviewable_histories (
+        reviewable_id,
+        reviewable_history_type,
+        status,
+        created_by_id,
+        created_at,
+        updated_at
+      )
+      SELECT r.id,
+        1,
+        2,
+        pa.disagreed_by_id,
+        pa.disagreed_at,
+        pa.disagreed_at
+      FROM reviewables AS r
+      INNER JOIN post_actions AS pa ON pa.post_id = r.target_id
+      WHERE r.type = 'ReviewableFlaggedPost'
+        AND pa.disagreed_at IS NOT NULL
+        AND pa.disagreed_by_id IS NOT NULL
+    SQL
+
+    # Ignored
+    execute(<<~SQL)
+      INSERT INTO reviewable_histories (
+        reviewable_id,
+        reviewable_history_type,
+        status,
+        created_by_id,
+        created_at,
+        updated_at
+      )
+      SELECT r.id,
+        1,
+        3,
+        pa.deferred_by_id,
+        pa.deferred_at,
+        pa.deferred_at
+      FROM reviewables AS r
+      INNER JOIN post_actions AS pa ON pa.post_id = r.target_id
+      WHERE r.type = 'ReviewableFlaggedPost'
+        AND pa.deferred_at IS NOT NULL
+        AND pa.deferred_by_id IS NOT NULL
+    SQL
+
+    # Deleted
+    execute(<<~SQL)
+      INSERT INTO reviewable_histories (
+        reviewable_id,
+        reviewable_history_type,
+        status,
+        created_by_id,
+        created_at,
+        updated_at
+      )
+      SELECT r.id,
+        1,
+        4,
+        pa.deleted_by_id,
+        pa.deleted_at,
+        pa.deleted_at
+      FROM reviewables AS r
+      INNER JOIN post_actions AS pa ON pa.post_id = r.target_id
+      WHERE r.type = 'ReviewableFlaggedPost'
+        AND pa.deleted_at IS NOT NULL
+        AND pa.deleted_by_id IS NOT NULL
+    SQL
+  end
+
+  def down
+    execute(<<~SQL)
+      DELETE FROM reviewable_histories
+      WHERE reviewable_id IN (SELECT id FROM reviewables WHERE type = 'ReviewableFlaggedPost')
+    SQL
+  end
+end

db/migrate/20190327205525_require_reviewable_scores.rb

@@ -0,0 +1,43 @@
+class RequireReviewableScores < ActiveRecord::Migration[5.2]
+  def up
+    min_score = DB.query_single("SELECT value FROM site_settings WHERE name = 'min_score_default_visibility'")[0].to_f
+    min_score = 1.0 if (min_score < 1.0)
+
+    execute(<<~SQL)
+      INSERT INTO reviewable_scores (
+        reviewable_id,
+        user_id,
+        reviewable_score_type,
+        score,
+        status,
+        created_at,
+        updated_at
+      )
+      SELECT r.id,
+        -1,
+        9,
+        #{min_score},
+        r.status,
+        r.created_at,
+        r.created_at
+      FROM reviewables AS r
+      WHERE r.type IN ('ReviewableQueuedPost', 'ReviewableUser')
+    SQL
+
+    execute(<<~SQL)
+      UPDATE reviewables SET score = (
+        SELECT SUM(score)
+        FROM reviewable_scores
+        WHERE reviewable_scores.reviewable_id = reviewables.id
+      )
+    SQL
+  end
+
+  def down
+    execute(<<~SQL)
+      DELETE FROM reviewable_scores WHERE reviewable_id IN (
+        SELECT id FROM reviewables WHERE type IN ('ReviewableQueuedPost', 'ReviewableUser')
+      )
+    SQL
+  end
+end

db/post_migrate/20190121203023_drop_queued_post_id_from_user_actions.rb

@@ -0,0 +1,9 @@
+class DropQueuedPostIdFromUserActions < ActiveRecord::Migration[5.2]
+  def up
+    remove_column :user_actions, :queued_post_id
+  end
+
+  def down
+    add_column :user_actions, :queued_post_id, :integer
+  end
+end

db/post_migrate/20190123171817_drop_queued_posts.rb

@@ -0,0 +1,5 @@
+class DropQueuedPosts < ActiveRecord::Migration[5.2]
+  def up
+    drop_table :queued_posts
+  end
+end