FEATURE: Allow fallback home routes for crawlers for custom homepage sites (#41234)

When using a custom homepage, the crawler view currently is limited, it
only shows the site's top menu. Instead, let's add a default fallback
route (/latest) and an admin site setting where the admin can pick
another fallback route. This should be better for SEO and discovery.

app/controllers/application_controller.rb

@@ -78,13 +78,7 @@ class ApplicationController < ActionController::Base
   helper_method :show_browser_update?
 
   def use_crawler_layout?
-    @use_crawler_layout ||=
-      request.user_agent && (request.media_type.blank? || request.media_type.include?("html")) &&
-        !%w[json rss].include?(params[:format]) &&
-        (
-          has_escaped_fragment? || params.key?("print") || show_browser_update? ||
-            CrawlerDetection.crawler?(request.user_agent, request.headers["HTTP_VIA"])
-        )
+    @use_crawler_layout ||= CrawlerDetection.crawler_layout_request?(request)
   end
   helper_method :use_crawler_layout?
 

config/locales/server.en.yml

@@ -2066,6 +2066,7 @@ en:
     include_secure_categories_in_tag_counts: "When enabled, count of topics for a tag will include topics that are in read restricted categories for all users. When disabled, normal users are only shown a count of topics for a tag where all the topics are in public categories."
     display_personal_messages_tag_counts: "When enabled, count of personal messages tagged with a given tag will be displayed."
     top_menu: "Determine which items appear in the homepage navigation, and in what order."
+    custom_homepage_crawler_route: "Route shown to web crawlers when a custom homepage is enabled."
     post_menu: "Configure the visibility and order of default post menu items. Additional items added by plugins or themes are managed separately and won’t appear in this list."
     post_menu_hidden_items: "The menu items to hide by default in the post menu unless an expansion ellipsis is clicked on. Additional items added by plugins or themes are managed separately and won’t appear in this list."
     share_links: "Determine which items appear on the share dialog, and in what order."

config/site_settings.yml

@@ -306,6 +306,11 @@ basic:
     upcoming_change_default_override:
       upcoming_change: "enable_unified_new"
       new_default: "latest|new|hot|categories"
+  custom_homepage_crawler_route:
+    type: enum
+    default: "latest"
+    choices: "TopMenu.crawler_homepage_choices"
+    area: "interface"
   post_menu:
     client: true
     type: list

lib/crawler_detection.rb

@@ -69,6 +69,17 @@ module CrawlerDetection
     user_agent.match?(matcher)
   end
 
+  def self.crawler_layout_request?(request)
+    return false if request.blank?
+    return false if request.user_agent.blank?
+    return false if request.media_type.present? && !request.media_type.include?("html")
+    return false if %w[json rss].include?(request.params[:format].to_s)
+
+    (SiteSetting.enable_escaped_fragments? && request.params.key?("_escaped_fragment_")) ||
+      request.params.key?("print") || show_browser_update?(request.user_agent) ||
+      crawler?(request.user_agent, request.headers["HTTP_VIA"])
+  end
+
   # Given a user_agent that returns true from crawler?, should its request be allowed?
   def self.allow_crawler?(user_agent)
     if SiteSetting.allowed_crawler_user_agents.blank? &&

lib/homepage_helper.rb

@@ -4,18 +4,28 @@ class HomepageHelper
   def self.resolve(request = nil, current_user = nil)
     return "blank" if !current_user && SiteSetting.login_required?
 
-    return "custom" if ThemeModifierHelper.new(request: request).custom_homepage
+    if ThemeModifierHelper.new(request: request).custom_homepage
+      return custom_homepage_route(request)
+    end
 
-    enabled = false
     enabled =
       DiscoursePluginRegistry.apply_modifier(
         :custom_homepage_enabled,
-        enabled,
+        false,
         request: request,
         current_user: current_user,
       )
-    return "custom" if enabled
+
+    return custom_homepage_route(request) if enabled
 
     current_user ? SiteSetting.homepage : SiteSetting.anonymous_homepage
   end
+
+  def self.custom_homepage_route(request)
+    if CrawlerDetection.crawler_layout_request?(request)
+      return SiteSetting.custom_homepage_crawler_route
+    end
+
+    "custom"
+  end
 end

lib/top_menu.rb

@@ -12,4 +12,8 @@ module TopMenu
     end
     base
   end
+
+  def self.crawler_homepage_choices
+    choices & (Discourse.anonymous_filters.map(&:to_s) + %w[new categories])
+  end
 end

spec/lib/crawler_detection_spec.rb

@@ -94,6 +94,42 @@ RSpec.describe CrawlerDetection do
     end
   end
 
+  describe ".crawler_layout_request?" do
+    it "returns true for crawler HTML requests" do
+      request = ActionDispatch::TestRequest.create("HTTP_USER_AGENT" => "Googlebot")
+
+      expect(CrawlerDetection.crawler_layout_request?(request)).to eq(true)
+    end
+
+    it "returns false for JSON requests" do
+      request =
+        ActionDispatch::TestRequest.create(
+          "HTTP_USER_AGENT" => "Googlebot",
+          "action_dispatch.request.path_parameters" => {
+            format: "json",
+          },
+        )
+
+      expect(CrawlerDetection.crawler_layout_request?(request)).to eq(false)
+    end
+
+    it "honors the escaped fragments site setting" do
+      SiteSetting.enable_escaped_fragments = false
+      request =
+        ActionDispatch::TestRequest.create(
+          "HTTP_USER_AGENT" =>
+            "Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1667.0 Safari/537.36",
+        )
+      request.params["_escaped_fragment_"] = ""
+
+      expect(CrawlerDetection.crawler_layout_request?(request)).to eq(false)
+
+      SiteSetting.enable_escaped_fragments = true
+
+      expect(CrawlerDetection.crawler_layout_request?(request)).to eq(true)
+    end
+  end
+
   describe ".allow_crawler?" do
     it "returns true if allowlist and blocklist are blank" do
       expect(

spec/lib/homepage_helper_spec.rb

@@ -8,12 +8,19 @@ RSpec.describe HomepageHelper do
       expect(HomepageHelper.resolve).to eq("latest")
     end
 
-    context "when theme has a custom homepage" do
-      before { ThemeModifierHelper.any_instance.expects(:custom_homepage).returns(true) }
+    context "when a theme has a custom homepage" do
+      before { ThemeModifierHelper.any_instance.stubs(:custom_homepage).returns(true) }
 
       it "returns custom" do
         expect(HomepageHelper.resolve).to eq("custom")
       end
+
+      it "returns the configured crawler route for crawler requests" do
+        SiteSetting.custom_homepage_crawler_route = "categories"
+        request = ActionDispatch::TestRequest.create("HTTP_USER_AGENT" => "Googlebot")
+
+        expect(HomepageHelper.resolve(request)).to eq("categories")
+      end
     end
 
     context "when a plugin modifies the custom_homepage_enabled to true" do

spec/models/site_setting_spec.rb

@@ -80,6 +80,20 @@ RSpec.describe SiteSetting do
     end
   end
 
+  describe "custom_homepage_crawler_route" do
+    it "allows public top menu routes" do
+      SiteSetting.custom_homepage_crawler_route = "categories"
+
+      expect(SiteSetting.custom_homepage_crawler_route).to eq("categories")
+    end
+
+    it "does not allow authenticated-only routes" do
+      expect { SiteSetting.custom_homepage_crawler_route = "bookmarks" }.to raise_error(
+        Discourse::InvalidParameters,
+      )
+    end
+  end
+
   describe "min_redirected_to_top_period" do
     context "when has_enough_top_topics" do
       before do

spec/requests/home_page_controller_spec.rb

@@ -62,6 +62,18 @@ RSpec.describe HomePageController do
         )
       end
 
+      it "uses the configured crawler route when a custom homepage is enabled" do
+        ThemeModifierHelper.any_instance.stubs(:custom_homepage).returns(true)
+        SiteSetting.custom_homepage_crawler_route = "categories"
+        category = Fabricate(:category, name: "Crawler Category")
+
+        get "/", headers: { "HTTP_USER_AGENT" => "Googlebot" }
+
+        expect(response.status).to eq(200)
+        expect(response.body).to include(category.name)
+        expect(response.body).not_to include("crawler-view-anon-menu")
+      end
+
       it "should not display the site description on another route" do
         get "/top", headers: { "HTTP_USER_AGENT" => "Googlebot" }