IntenseWebs/discourse
3
0
Fork 0
mirror of https://github.com/discourse/discourse.git synced 2025-02-25 18:55:32 -06:00
Code Issues Packages Projects Releases Wiki Activity

FEATURE: control which web crawlers can access using a whitelist or blacklist

Browse Source
This commit is contained in:
Neil Lalonde
2018-03-15 17:10:45 -04:00
parent cbbeedf53b
commit ced7e9a691
22 changed files with 722 additions and 97 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
app/controllers/robots_txt_controller.rb
View File

@@ -3,7 +3,21 @@ class RobotsTxtController < ApplicationController
skip_before_action :preload_json, :check_xhr, :redirect_to_login_if_required
def index
path = SiteSetting.allow_index_in_robots_txt ? :index : :no_index
if SiteSetting.allow_index_in_robots_txt
path = :index
if SiteSetting.whitelisted_crawler_user_agents.present?
@allowed_user_agents = SiteSetting.whitelisted_crawler_user_agents.split('|')
@disallowed_user_agents = ['*']
elsif SiteSetting.blacklisted_crawler_user_agents.present?
@allowed_user_agents = ['*']
@disallowed_user_agents = SiteSetting.blacklisted_crawler_user_agents.split('|')
else
@allowed_user_agents = ['*']
end
else
path = :no_index
end
render path, content_type: 'text/plain'
end
end