FEATURE: Allow customization of robots.txt (#7884)

* FEATURE: Allow customization of robots.txt

This allows admins to customize/override the content of the robots.txt
file at /admin/customize/robots. That page is not linked to anywhere in
the UI -- admins have to manually type the URL to access that page.

* use Ember.computed.not

* Jeff feedback

* Feedback

* Remove unused import

app/controllers/admin/robots_txt_controller.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+class Admin::RobotsTxtController < Admin::AdminController
+
+  def show
+    render json: { robots_txt: current_robots_txt, overridden: @overridden }
+  end
+
+  def update
+    params.require(:robots_txt)
+    SiteSetting.overridden_robots_txt = params[:robots_txt]
+
+    render json: { robots_txt: current_robots_txt, overridden: @overridden }
+  end
+
+  def reset
+    SiteSetting.overridden_robots_txt = ""
+    render json: { robots_txt: original_robots_txt, overridden: false }
+  end
+
+  private
+
+  def current_robots_txt
+    robots_txt = SiteSetting.overridden_robots_txt.presence
+    @overridden = robots_txt.present?
+    robots_txt ||= original_robots_txt
+    robots_txt
+  end
+
+  def original_robots_txt
+    if SiteSetting.allow_index_in_robots_txt?
+      @robots_info = ::RobotsTxtController.fetch_default_robots_info
+      render_to_string "robots_txt/index"
+    else
+      render_to_string "robots_txt/no_index"
+    end
+  end
+end

app/controllers/robots_txt_controller.rb

@@ -4,6 +4,8 @@ class RobotsTxtController < ApplicationController
   layout false
   skip_before_action :preload_json, :check_xhr, :redirect_to_login_if_required
 
+  OVERRIDDEN_HEADER = "# This robots.txt file has been customized at /admin/customize/robots\n"
+
   # NOTE: order is important!
   DISALLOWED_PATHS ||= %w{
     /auth/
@@ -33,8 +35,13 @@ class RobotsTxtController < ApplicationController
   }
 
   def index
+    if (overridden = SiteSetting.overridden_robots_txt.dup).present?
+      overridden.prepend(OVERRIDDEN_HEADER) if guardian.is_admin? && !is_api?
+      render plain: overridden
+      return
+    end
     if SiteSetting.allow_index_in_robots_txt?
-      @robots_info = fetch_robots_info
+      @robots_info = self.class.fetch_default_robots_info
       render :index, content_type: 'text/plain'
     else
       render :no_index, content_type: 'text/plain'
@@ -46,12 +53,13 @@ class RobotsTxtController < ApplicationController
   # JSON that can be used by a script to create a robots.txt that works well with your
   # existing site.
   def builder
-    render json: fetch_robots_info
+    result = self.class.fetch_default_robots_info
+    overridden = SiteSetting.overridden_robots_txt
+    result[:overridden] = overridden if overridden.present?
+    render json: result
   end
 
-protected
-
-  def fetch_robots_info
+  def self.fetch_default_robots_info
     deny_paths = DISALLOWED_PATHS.map { |p| Discourse.base_uri + p }
     deny_all = [ "#{Discourse.base_uri}/" ]
 
@@ -87,5 +95,4 @@ protected
 
     result
   end
-
 end