Merge pull request #5700 from discourse/crawl-block

FEATURE: control web crawlers access with white/blacklist

app/assets/javascripts/admin/models/report.js.es6

@@ -2,6 +2,7 @@ import { ajax } from 'discourse/lib/ajax';
 import round from "discourse/lib/round";
 import { fmt } from 'discourse/lib/computed';
 import { fillMissingDates } from 'discourse/lib/utilities';
+import computed from 'ember-addons/ember-computed-decorators';
 
 const Report = Discourse.Model.extend({
   reportUrl: fmt("type", "/admin/reports/%@"),
@@ -42,7 +43,8 @@ const Report = Discourse.Model.extend({
   lastSevenDaysCount:  function() { return this.valueFor(1, 7); }.property("data"),
   lastThirtyDaysCount: function() { return this.valueFor(1, 30); }.property("data"),
 
-  yesterdayTrend: function() {
+  @computed('data')
+  yesterdayTrend() {
     const yesterdayVal = this.valueAt(1);
     const twoDaysAgoVal = this.valueAt(2);
     if (yesterdayVal > twoDaysAgoVal) {
@@ -52,9 +54,10 @@ const Report = Discourse.Model.extend({
     } else {
       return "no-change";
     }
-  }.property("data"),
+  },
 
-  sevenDayTrend: function() {
+  @computed('data')
+  sevenDayTrend() {
     const currentPeriod = this.valueFor(1, 7);
     const prevPeriod = this.valueFor(8, 14);
     if (currentPeriod > prevPeriod) {
@@ -64,36 +67,39 @@ const Report = Discourse.Model.extend({
     } else {
       return "no-change";
     }
-  }.property("data"),
+  },
 
-  thirtyDayTrend: function() {
-    if (this.get("prev30Days")) {
+  @computed('prev30Days', 'data')
+  thirtyDayTrend(prev30Days) {
+    if (prev30Days) {
       const currentPeriod = this.valueFor(1, 30);
       if (currentPeriod > this.get("prev30Days")) {
         return "trending-up";
-      } else if (currentPeriod < this.get("prev30Days")) {
+      } else if (currentPeriod < prev30Days) {
         return "trending-down";
       }
     }
     return "no-change";
-  }.property("data", "prev30Days"),
+  },
 
-  icon: function() {
-    switch (this.get("type")) {
+  @computed('type')
+  icon(type) {
+    switch (type) {
       case "flags": return "flag";
       case "likes": return "heart";
       case "bookmarks": return "bookmark";
       default: return null;
     }
-  }.property("type"),
+  },
 
-  method: function() {
-    if (this.get("type") === "time_to_first_response") {
+  @computed('type')
+  method(type) {
+    if (type === "time_to_first_response") {
       return "average";
     } else {
       return "sum";
     }
-  }.property("type"),
+  },
 
   percentChangeString(val1, val2) {
     const val = ((val1 - val2) / val2) * 100;
@@ -114,21 +120,31 @@ const Report = Discourse.Model.extend({
     return title;
   },
 
-  yesterdayCountTitle: function() {
+  @computed('data')
+  yesterdayCountTitle() {
     return this.changeTitle(this.valueAt(1), this.valueAt(2), "two days ago");
-  }.property("data"),
+  },
 
-  sevenDayCountTitle: function() {
+  @computed('data')
+  sevenDayCountTitle() {
     return this.changeTitle(this.valueFor(1, 7), this.valueFor(8, 14), "two weeks ago");
-  }.property("data"),
+  },
 
-  thirtyDayCountTitle: function() {
-    return this.changeTitle(this.valueFor(1, 30), this.get("prev30Days"), "in the previous 30 day period");
-  }.property("data"),
+  @computed('prev30Days', 'data')
+  thirtyDayCountTitle(prev30Days) {
+    return this.changeTitle(this.valueFor(1, 30), prev30Days, "in the previous 30 day period");
+  },
 
-  dataReversed: function() {
-    return this.get("data").toArray().reverse();
-  }.property("data")
+  @computed('data')
+  sortedData(data) {
+    return this.get('xAxisIsDate') ? data.toArray().reverse() : data.toArray();
+  },
+
+  @computed('data')
+  xAxisIsDate() {
+    if (!this.data[0]) return false;
+    return this.data && this.data[0].x.match(/\d{4}-\d{1,2}-\d{1,2}/);
+  }
 
 });
 
@@ -152,6 +168,14 @@ Report.reopenClass({
 
       const model = Report.create({ type: type });
       model.setProperties(json.report);
+
+      if (json.report.related_report) {
+        // TODO: fillMissingDates if xaxis is date
+        const related = Report.create({ type: json.report.related_report.type });
+        related.setProperties(json.report.related_report);
+        model.set('relatedReport', related);
+      }
+
       return model;
     });
   }

app/assets/javascripts/admin/templates/components/admin-table-report.hbs

@@ -0,0 +1,17 @@
+{{#if model.sortedData}}
+  <table class="table report {{model.type}}">
+    <tr>
+      <th>{{model.xaxis}}</th>
+      <th>{{model.yaxis}}</th>
+    </tr>
+
+    {{#each model.sortedData as |row|}}
+      <tr>
+        <td class="x-value">{{row.x}}</td>
+        <td>
+          {{row.y}}
+        </td>
+      </tr>
+    {{/each}}
+  </table>
+{{/if}}

app/assets/javascripts/admin/templates/reports.hbs

@@ -31,20 +31,10 @@
   {{#if viewingGraph}}
     {{admin-graph model=model}}
   {{else}}
-  <table class='table report'>
-    <tr>
-      <th>{{model.xaxis}}</th>
-      <th>{{model.yaxis}}</th>
-    </tr>
+    {{admin-table-report model=model}}
+  {{/if}}
 
-    {{#each model.dataReversed as |row|}}
-      <tr>
-        <td>{{row.x}}</td>
-        <td>
-          {{row.y}}
-        </td>
-      </tr>
-    {{/each}}
-  </table>
+  {{#if model.relatedReport}}
+    {{admin-table-report model=model.relatedReport}}
   {{/if}}
 {{/conditional-loading-spinner}}

app/assets/stylesheets/common/admin/admin_base.scss

@@ -167,6 +167,20 @@ $mobile-breakpoint: 700px;
       }
     }
 
+    &.web_crawlers {
+      tr {
+        th:nth-of-type(1) {
+          width: 60%;
+        }
+      }
+      td.x-value {
+        max-width: 0;
+        overflow: hidden;
+        text-overflow: ellipsis;
+        white-space: nowrap;
+      }
+    }
+
     .bar-container {
       float: left;
       width: 300px;

app/controllers/robots_txt_controller.rb

@@ -3,7 +3,21 @@ class RobotsTxtController < ApplicationController
   skip_before_action :preload_json, :check_xhr, :redirect_to_login_if_required
 
   def index
-    path = SiteSetting.allow_index_in_robots_txt ? :index : :no_index
+    if SiteSetting.allow_index_in_robots_txt
+      path = :index
+      if SiteSetting.whitelisted_crawler_user_agents.present?
+        @allowed_user_agents = SiteSetting.whitelisted_crawler_user_agents.split('|')
+        @disallowed_user_agents = ['*']
+      elsif SiteSetting.blacklisted_crawler_user_agents.present?
+        @allowed_user_agents = ['*']
+        @disallowed_user_agents = SiteSetting.blacklisted_crawler_user_agents.split('|')
+      else
+        @allowed_user_agents = ['*']
+      end
+    else
+      path = :no_index
+    end
+
     render path, content_type: 'text/plain'
   end
 end

app/jobs/scheduled/clean_up_crawler_stats.rb

@@ -0,0 +1,26 @@
+module Jobs
+
+  class CleanUpCrawlerStats < Jobs::Scheduled
+    every 1.day
+
+    def execute(args)
+      WebCrawlerRequest.where('date < ?', WebCrawlerRequest.max_record_age.ago).delete_all
+
+      # keep count of only the top user agents
+      WebCrawlerRequest.exec_sql <<~SQL
+        WITH ranked_requests AS (
+          SELECT row_number() OVER (ORDER BY count DESC) as row_number, id
+            FROM web_crawler_requests
+           WHERE date = '#{1.day.ago.strftime("%Y-%m-%d")}'
+        )
+        DELETE FROM web_crawler_requests
+        WHERE id IN (
+          SELECT ranked_requests.id
+            FROM ranked_requests
+           WHERE row_number > #{WebCrawlerRequest.max_records_per_day}
+        )
+      SQL
+    end
+  end
+
+end

app/models/application_request.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 class ApplicationRequest < ActiveRecord::Base
+
   enum req_type: %i(http_total
                     http_2xx
                     http_background
@@ -12,41 +13,12 @@ class ApplicationRequest < ActiveRecord::Base
                     page_view_logged_in_mobile
                     page_view_anon_mobile)
 
-  cattr_accessor :autoflush, :autoflush_seconds, :last_flush
-  # auto flush if backlog is larger than this
-  self.autoflush = 2000
-
-  # auto flush if older than this
-  self.autoflush_seconds = 5.minutes
-  self.last_flush = Time.now.utc
+  include CachedCounting
 
   def self.increment!(type, opts = nil)
-    key = redis_key(type)
-    val = $redis.incr(key).to_i
-
-    # readonly mode it is going to be 0, skip
-    return if val == 0
-
-    # 3.days, see: https://github.com/rails/rails/issues/21296
-    $redis.expire(key, 259200)
-
-    autoflush = (opts && opts[:autoflush]) || self.autoflush
-    if autoflush > 0 && val >= autoflush
-      write_cache!
-      return
-    end
-
-    if (Time.now.utc - last_flush).to_i > autoflush_seconds
-      write_cache!
-    end
+    perform_increment!(redis_key(type), opts)
   end
 
-  GET_AND_RESET = <<~LUA
-    local val = redis.call('get', KEYS[1])
-    redis.call('set', KEYS[1], '0')
-    return val
-  LUA
-
   def self.write_cache!(date = nil)
     if date.nil?
       write_cache!(Time.now.utc)
@@ -58,13 +30,9 @@ class ApplicationRequest < ActiveRecord::Base
 
     date = date.to_date
 
-    # this may seem a bit fancy but in so it allows
-    # for concurrent calls without double counting
     req_types.each do |req_type, _|
-      key = redis_key(req_type, date)
+      val = get_and_reset(redis_key(req_type, date))
 
-      namespaced_key = $redis.namespace_key(key)
-      val = $redis.without_namespace.eval(GET_AND_RESET, keys: [namespaced_key]).to_i
       next if val == 0
 
       id = req_id(date, req_type)

app/models/concerns/cached_counting.rb

@@ -0,0 +1,67 @@
+module CachedCounting
+  extend ActiveSupport::Concern
+
+  included do
+    class << self
+      attr_accessor :autoflush, :autoflush_seconds, :last_flush
+    end
+
+    # auto flush if backlog is larger than this
+    self.autoflush = 2000
+
+    # auto flush if older than this
+    self.autoflush_seconds = 5.minutes
+
+    self.last_flush = Time.now.utc
+  end
+
+  class_methods do
+    def perform_increment!(key, opts = nil)
+      val = $redis.incr(key).to_i
+
+      # readonly mode it is going to be 0, skip
+      return if val == 0
+
+      # 3.days, see: https://github.com/rails/rails/issues/21296
+      $redis.expire(key, 259200)
+
+      autoflush = (opts && opts[:autoflush]) || self.autoflush
+      if autoflush > 0 && val >= autoflush
+        write_cache!
+        return
+      end
+
+      if (Time.now.utc - last_flush).to_i > autoflush_seconds
+        write_cache!
+      end
+    end
+
+    def write_cache!(date = nil)
+      raise NotImplementedError
+    end
+
+    GET_AND_RESET = <<~LUA
+      local val = redis.call('get', KEYS[1])
+      redis.call('set', KEYS[1], '0')
+      return val
+    LUA
+
+    # this may seem a bit fancy but in so it allows
+    # for concurrent calls without double counting
+    def get_and_reset(key)
+      namespaced_key = $redis.namespace_key(key)
+      $redis.without_namespace.eval(GET_AND_RESET, keys: [namespaced_key]).to_i
+    end
+
+    def request_id(query_params, retries = 0)
+      id = where(query_params).pluck(:id).first
+      id ||= create!(query_params.merge(count: 0)).id
+    rescue # primary key violation
+      if retries == 0
+        request_id(query_params, 1)
+      else
+        raise
+      end
+    end
+  end
+end

app/models/report.rb

@@ -27,7 +27,11 @@ class Report
      category_id: category_id,
      group_id: group_id,
      prev30Days: self.prev30Days
-    }
+    }.tap do |json|
+      if type == 'page_view_crawler_reqs'
+        json[:related_report] = Report.find('web_crawlers', start_date: start_date, end_date: end_date)&.as_json
+      end
+    end
   end
 
   def Report.add_report(name, &block)
@@ -231,4 +235,12 @@ class Report
   def self.report_notify_user_private_messages(report)
     private_messages_report report, TopicSubtype.notify_user
   end
+
+  def self.report_web_crawlers(report)
+    report.data = WebCrawlerRequest.where('date >= ? and date <= ?', report.start_date, report.end_date)
+      .limit(200)
+      .order('sum_count DESC')
+      .group(:user_agent).sum(:count)
+      .map { |ua, count| { x: ua, y: count } }
+  end
 end

app/models/web_crawler_request.rb

@@ -0,0 +1,76 @@
+class WebCrawlerRequest < ActiveRecord::Base
+  include CachedCounting
+
+  # auto flush if older than this
+  self.autoflush_seconds = 1.hour
+
+  cattr_accessor :max_record_age, :max_records_per_day
+
+  # only keep the top records based on request count
+  self.max_records_per_day = 200
+
+  # delete records older than this
+  self.max_record_age = 30.days
+
+  def self.increment!(user_agent, opts = nil)
+    ua_list_key = user_agent_list_key
+    $redis.sadd(ua_list_key, user_agent)
+    $redis.expire(ua_list_key, 259200) # 3.days
+
+    perform_increment!(redis_key(user_agent), opts)
+  end
+
+  def self.write_cache!(date = nil)
+    if date.nil?
+      write_cache!(Time.now.utc)
+      write_cache!(Time.now.utc.yesterday)
+      return
+    end
+
+    self.last_flush = Time.now.utc
+
+    date = date.to_date
+    ua_list_key = user_agent_list_key(date)
+
+    while user_agent = $redis.spop(ua_list_key)
+      val = get_and_reset(redis_key(user_agent, date))
+
+      next if val == 0
+
+      self.where(id: req_id(date, user_agent)).update_all(["count = count + ?", val])
+    end
+  rescue Redis::CommandError => e
+    raise unless e.message =~ /READONLY/
+    nil
+  end
+
+  def self.clear_cache!(date = nil)
+    if date.nil?
+      clear_cache!(Time.now.utc)
+      clear_cache!(Time.now.utc.yesterday)
+      return
+    end
+
+    list_key = user_agent_list_key(date)
+
+    $redis.smembers(list_key).each do |user_agent, _|
+      $redis.del redis_key(user_agent, date)
+    end
+
+    $redis.del(list_key)
+  end
+
+  protected
+
+  def self.user_agent_list_key(time = Time.now.utc)
+    "crawl_ua_list:#{time.strftime('%Y%m%d')}"
+  end
+
+  def self.redis_key(user_agent, time = Time.now.utc)
+    "crawl_req:#{time.strftime('%Y%m%d')}:#{user_agent}"
+  end
+
+  def self.req_id(date, user_agent)
+    request_id(date: date, user_agent: user_agent)
+  end
+end

app/views/robots_txt/index.erb

@@ -1,6 +1,8 @@
 # See http://www.robotstxt.org/wc/norobots.html for documentation on how to use the robots.txt file
 #
-User-agent: *
+<% @allowed_user_agents.each do |user_agent| %>
+User-agent: <%= user_agent %>
+<% end %>
 Disallow: /auth/cas
 Disallow: /auth/facebook/callback
 Disallow: /auth/twitter/callback
@@ -29,4 +31,12 @@ Disallow: /groups
 Disallow: /groups/
 Disallow: /uploads/
 
+<% if @disallowed_user_agents %>
+  <% @disallowed_user_agents.each do |user_agent| %>
+User-agent: <%= user_agent %>
+Disallow: /
+
+  <% end %>
+<% end %>
+
 <%= server_plugin_outlet "robots_txt_index" %>