DEV: Fix custom homepage crawler display and override (#31841)

Fixes the custom homepage crawler output to include links to the site's
top menu.

![CleanShot 2025-03-15 at 16 27
16@2x](https://github.com/user-attachments/assets/57f25b65-a218-4811-b7d4-211e3d60e586)

This also provides a way to override that content via a plugin. Example
usage in a `plugin.rb` file:

```
register_html_builder("server:custom-homepage-crawler-view") do |c|
  "<div>override</div>"
end
```

app/controllers/custom_homepage_controller.rb

@@ -1,7 +1,9 @@
 # frozen_string_literal: true
 
 class CustomHomepageController < ApplicationController
+  skip_before_action :check_xhr, only: [:index]
+
   def index
-    render "default/custom"
+    respond_to { |format| format.html { render :index } }
   end
 end

app/views/custom_homepage/index.erb

@@ -0,0 +1,11 @@
+<%= replace_plugin_html('server:custom-homepage-crawler-view') do %>
+  <ul class="crawler-view-anon-menu">
+  <% anonymous_top_menu_items.each_with_index do |item, index| %>
+    <li>
+      <a href='<%= Discourse.base_url %>/<%= item %>'>
+        <%= I18n.t("js.filters.#{item}.title") %>
+      </a>
+    </li>
+  <% end %>
+  </ul>
+<% end %>

app/views/default/custom.html.erb

@@ -1,9 +0,0 @@
-<ul>
-<% anonymous_top_menu_items.each_with_index do |item, index| %>
-  <li>
-    <a href='<%= Discourse.base_url %>/<%= item %>'>
-      <%= I18n.t("js.filters.#{item}.title") %>
-    </a>
-  </li>
-<% end %>
-</ul>

spec/requests/custom_homepage_controller_spec.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+RSpec.describe CustomHomepageController do
+  describe "#index" do
+    context "with crawler view" do
+      it "should display the menu by default" do
+        get "/custom", headers: { "HTTP_USER_AGENT" => "Googlebot" }
+
+        expect(response.status).to eq(200)
+        expect(response.body).to include("<ul class=\"crawler-view-anon-menu\">")
+      end
+
+      context "with plugin override" do
+        let(:plugin_class) do
+          Class.new(Plugin::Instance) do
+            attr_accessor :enabled
+
+            def enabled?
+              @enabled
+            end
+          end
+        end
+
+        it "should allow plugin to override output" do
+          plugin =
+            plugin_class.new(nil, "#{Rails.root}/spec/fixtures/plugins/csp_extension/plugin.rb")
+
+          plugin.register_html_builder("server:custom-homepage-crawler-view") do |c|
+            "<div>override</div>"
+          end
+          plugin.activate!
+          Discourse.plugins << plugin
+          plugin.enabled = true
+
+          get "/custom", headers: { "HTTP_USER_AGENT" => "Googlebot" }
+
+          expect(response.status).to eq(200)
+          expect(response.body).not_to include("<ul class=\"crawler-view-anon-menu\">")
+          expect(response.body).to include("override")
+
+          plugin.enabled = false
+          Discourse.plugins.delete plugin
+          DiscoursePluginRegistry.reset!
+        end
+      end
+    end
+  end
+end