IntenseWebs/discourse
3
0
Fork 0
mirror of https://github.com/discourse/discourse.git synced 2026-09-05 04:40:41 -05:00
Code Issues Packages Projects Releases Wiki Activity

add alt 'archive.org_bot' user-agent (#5273)

Browse Source 
add `archive.org_bot' – another user-agent used by Internet Archive when crawling for Wayback Machine
This commit is contained in:
Gordon Mohr
2017-10-29 10:30:29 +01:00
committed by Régis Hanol
parent bf00ab5d4a
commit da72ecc3fc
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
config/site_settings.yml
+1 -1
View File
@@ -929,7 +929,7 @@ security:
allow_index_in_robots_txt: true
allow_moderators_to_create_categories: false
crawler_user_agents:
default: 'Googlebot|Mediapartners|AdsBot|curl|HTTrack|Twitterbot|facebookexternalhit|bingbot|Baiduspider|ia_archiver|Wayback Save Page|360Spider|Swiftbot|YandexBot'
default: 'Googlebot|Mediapartners|AdsBot|curl|HTTrack|Twitterbot|facebookexternalhit|bingbot|Baiduspider|ia_archiver|archive.org_bot|Wayback Save Page|360Spider|Swiftbot|YandexBot'
type: list
cors_origins:
default: ''