diff --git a/app/views/robots_txt/index.erb b/app/views/robots_txt/index.erb index 2d35051cf3c..2e9105c902f 100644 --- a/app/views/robots_txt/index.erb +++ b/app/views/robots_txt/index.erb @@ -23,4 +23,8 @@ Disallow: <%= Discourse.base_uri + "/" %> <% @crawler_delayed_agents.each do |agent, delay| %> User-agent: <%= agent %> Crawl-delay: <%= delay %> +<% RobotsTxtController::DISALLOWED_PATHS.each do |path| %> +Disallow: <%= Discourse.base_uri + path %> +<% end %> + <% end %>