FEATURE: add Noindex to robots.txt for disallowed routes

This strips pages out of indexes that should not exist see:

https://meta.discourse.org/t/pages-listed-in-the-robots-txt-are-crawled-and-indexed-by-google/100309/11?u=sam

app/views/robots_txt/index.erb

@@ -10,6 +10,7 @@ Crawl-delay: <%= agent[:delay] %>
 <%- end -%>
 <% agent[:disallow].each do |path| %>
 Disallow: <%= path %>
+Noindex: <%= path %>
 <% end %>
 
 

spec/requests/robots_txt_controller_spec.rb

@@ -18,6 +18,7 @@ RSpec.describe RobotsTxtController do
         Discourse.stubs(:base_uri).returns('/forum')
         get '/robots.txt'
         expect(response.body).to include("\nDisallow: /forum/admin")
+        expect(response.body).to include("\nNoindex: /forum/admin")
       end
     end