diff --git a/config/site_settings.yml b/config/site_settings.yml index 8fbb3f3fa8d..1878e2ce41a 100644 --- a/config/site_settings.yml +++ b/config/site_settings.yml @@ -929,7 +929,7 @@ security: allow_index_in_robots_txt: true allow_moderators_to_create_categories: false crawler_user_agents: - default: 'Googlebot|Mediapartners|AdsBot|curl|HTTrack|Twitterbot|facebookexternalhit|bingbot|Baiduspider|ia_archiver|Wayback Save Page|360Spider|Swiftbot|YandexBot' + default: 'Googlebot|Mediapartners|AdsBot|curl|HTTrack|Twitterbot|facebookexternalhit|bingbot|Baiduspider|ia_archiver|archive.org_bot|Wayback Save Page|360Spider|Swiftbot|YandexBot' type: list cors_origins: default: ''