FEATURE: Let sites add a sitemap.xml file. (#16357)

* FEATURE: Let sites add a sitemap.xml file.

This PR adds the same features discourse-sitemap provides to core. Sitemaps are only added to the robots.txt file if the `enable_sitemap` setting is enabled and `login_required` disabled.

After merging discourse/discourse-sitemap#34, this change will take priority over the sitemap plugin because it will disable itself. We're also using the same sitemaps table, so our migration won't try to create it
again using `if_not_exists: true`.

spec/requests/robots_txt_controller_spec.rb

@@ -130,5 +130,36 @@ RSpec.describe RobotsTxtController do
       expect(response.status).to eq(200)
       expect(response.body).to eq(SiteSetting.overridden_robots_txt)
     end
+
+    describe 'sitemap' do
+      let(:sitemap_line) { "Sitemap: #{Discourse.base_protocol}://#{Discourse.current_hostname}/sitemap.xml" }
+
+      it 'include sitemap location when enabled' do
+        SiteSetting.enable_sitemap = true
+        SiteSetting.login_required = false
+
+        get '/robots.txt'
+
+        expect(response.body).to include(sitemap_line)
+      end
+
+      it "doesn't include sitemap location when disabled" do
+        SiteSetting.enable_sitemap = false
+        SiteSetting.login_required = false
+
+        get '/robots.txt'
+
+        expect(response.body).not_to include(sitemap_line)
+      end
+
+      it "doesn't include sitemap location when site has login_required enabled" do
+        SiteSetting.enable_sitemap = true
+        SiteSetting.login_required = true
+
+        get '/robots.txt'
+
+        expect(response.body).not_to include(sitemap_line)
+      end
+    end
   end
 end