diff --git a/config/site_settings.yml b/config/site_settings.yml
index e26f171ac94..68dded8cace 100644
--- a/config/site_settings.yml
+++ b/config/site_settings.yml
@@ -1063,6 +1063,9 @@ security:
     hidden: true
     default: 'rss|bot|spider|crawler|facebook|archive|wayback|ping|monitor'
     type: list
+  crawler_check_bypass_agents:
+    hidden: true
+    default: 'cubot'
   cors_origins:
     default: ''
     type: list
diff --git a/lib/crawler_detection.rb b/lib/crawler_detection.rb
index bf28c6c1aa8..0c4a91573bd 100644
--- a/lib/crawler_detection.rb
+++ b/lib/crawler_detection.rb
@@ -22,7 +22,12 @@ module CrawlerDetection
 
     if user_agent.match?(possibly_real)
       known_bots = (@matchers[SiteSetting.crawler_user_agents] ||= to_matcher(SiteSetting.crawler_user_agents))
-      user_agent.match?(known_bots)
+      if user_agent.match?(known_bots)
+        bypass = (@matchers[SiteSetting.crawler_check_bypass_agents] ||= to_matcher(SiteSetting.crawler_check_bypass_agents))
+        !user_agent.match?(bypass)
+      else
+        false
+      end
     else
       true
     end
diff --git a/spec/components/crawler_detection_spec.rb b/spec/components/crawler_detection_spec.rb
index 65f811ef518..7fff4e80cc2 100644
--- a/spec/components/crawler_detection_spec.rb
+++ b/spec/components/crawler_detection_spec.rb
@@ -2,47 +2,61 @@ require 'rails_helper'
 require_dependency 'crawler_detection'
 
 describe CrawlerDetection do
+
+  def crawler!(s)
+    if (!CrawlerDetection.crawler?(s))
+      raise "#{s} should be a crawler!"
+    end
+  end
+
+  def not_crawler!(s)
+    if CrawlerDetection.crawler?(s)
+      raise "#{s} should not be a crawler!"
+    end
+  end
+
   describe "crawler?" do
 
     it "can be amended via site settings" do
       SiteSetting.crawler_user_agents = 'Mooble|Kaboodle+*'
-      expect(CrawlerDetection.crawler?("Mozilla/5.0 Safari (compatible; Kaboodle+*/2.1; +http://www.google.com/bot.html)")).to eq(true)
-      expect(CrawlerDetection.crawler?("Mozilla/5.0 Safari (compatible; Mooble+*/2.1; +http://www.google.com/bot.html)")).to eq(true)
-      expect(CrawlerDetection.crawler?("Mozilla/5.0 Safari (compatible; Gooble+*/2.1; +http://www.google.com/bot.html)")).to eq(false)
+
+      crawler! "Mozilla/5.0 Safari (compatible; Kaboodle+*/2.1; +http://www.google.com/bot.html)"
+      crawler! "Mozilla/5.0 Safari (compatible; Mooble+*/2.1; +http://www.google.com/bot.html)"
+      not_crawler! "Mozilla/5.0 Safari (compatible; Gooble+*/2.1; +http://www.google.com/bot.html)"
     end
 
     it "returns true for crawler user agents" do
       # https://support.google.com/webmasters/answer/1061943?hl=en
-      expect(described_class.crawler?("Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)")).to eq(true)
-      expect(described_class.crawler?("Googlebot/2.1 (+http://www.google.com/bot.html)")).to eq(true)
-      expect(described_class.crawler?("Googlebot-News")).to eq(true)
-      expect(described_class.crawler?("Googlebot-Image/1.0")).to eq(true)
-      expect(described_class.crawler?("Googlebot-Video/1.0")).to eq(true)
-      expect(described_class.crawler?("(compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)")).to eq(true)
-      expect(described_class.crawler?("Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)")).to eq(true)
-      expect(described_class.crawler?("(compatible; Mediapartners-Google/2.1; +http://www.google.com/bot.html)")).to eq(true)
-      expect(described_class.crawler?("Mediapartners-Google")).to eq(true)
-      expect(described_class.crawler?("AdsBot-Google (+http://www.google.com/adsbot.html)")).to eq(true)
-      expect(described_class.crawler?("Twitterbot")).to eq(true)
-      expect(described_class.crawler?("facebookexternalhit/1.1 (+http(s)://www.facebook.com/externalhit_uatext.php)")).to eq(true)
-      expect(described_class.crawler?("Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)")).to eq(true)
-      expect(described_class.crawler?("Baiduspider+(+http://www.baidu.com/search/spider.htm)")).to eq(true)
-      expect(described_class.crawler?("Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)")).to eq(true)
-
-      expect(described_class.crawler?("DiscourseAPI Ruby Gem 0.19.0")).to eq(true)
-      expect(described_class.crawler?("Pingdom.com_bot_version_1.4_(http://www.pingdom.com/)")).to eq(true)
-      expect(described_class.crawler?("LogicMonitor SiteMonitor/1.0")).to eq(true)
-      expect(described_class.crawler?("Java/1.8.0_151")).to eq(true)
-      expect(described_class.crawler?("Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)")).to eq(true)
+      crawler! "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"
+      crawler! "Googlebot/2.1 (+http://www.google.com/bot.html)"
+      crawler! "Googlebot-News"
+      crawler! "Googlebot-Image/1.0"
+      crawler! "Googlebot-Video/1.0"
+      crawler! "(compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)"
+      crawler! "Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"
+      crawler! "(compatible; Mediapartners-Google/2.1; +http://www.google.com/bot.html)"
+      crawler! "Mediapartners-Google"
+      crawler! "AdsBot-Google (+http://www.google.com/adsbot.html)"
+      crawler! "Twitterbot"
+      crawler! "facebookexternalhit/1.1 (+http(s)://www.facebook.com/externalhit_uatext.php)"
+      crawler! "Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)"
+      crawler! "Baiduspider+(+http://www.baidu.com/search/spider.htm)"
+      crawler! "Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)"
+      crawler! "DiscourseAPI Ruby Gem 0.19.0"
+      crawler! "Pingdom.com_bot_version_1.4_(http://www.pingdom.com/)"
+      crawler! "LogicMonitor SiteMonitor/1.0"
+      crawler! "Java/1.8.0_151"
+      crawler! "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)"
     end
 
     it "returns false for non-crawler user agents" do
-      expect(described_class.crawler?("Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1667.0 Safari/537.36")).to eq(false)
-      expect(described_class.crawler?("Mozilla/5.0 (Windows NT 6.3; Trident/7.0; rv:11.0) like Gecko")).to eq(false)
-      expect(described_class.crawler?("Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Trident/6.0)")).to eq(false)
-      expect(described_class.crawler?("Mozilla/5.0 (iPad; CPU OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5355d Safari/8536.25")).to eq(false)
-      expect(described_class.crawler?("Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:25.0) Gecko/20100101 Firefox/25.0")).to eq(false)
-      expect(described_class.crawler?("Mozilla/5.0 (Linux; U; Android 4.0.3; ko-kr; LG-L160L Build/IML74K) AppleWebkit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30")).to eq(false)
+      not_crawler! "Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1667.0 Safari/537.36"
+      not_crawler! "Mozilla/5.0 (Windows NT 6.3; Trident/7.0; rv:11.0) like Gecko"
+      not_crawler! "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Trident/6.0)"
+      not_crawler! "Mozilla/5.0 (iPad; CPU OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5355d Safari/8536.25"
+      not_crawler! "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:25.0) Gecko/20100101 Firefox/25.0"
+      not_crawler! "Mozilla/5.0 (Linux; U; Android 4.0.3; ko-kr; LG-L160L Build/IML74K) AppleWebkit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30"
+      not_crawler! "Mozilla/5.0 (Linux; Android 6.0; CUBOT DINOSAUR Build/MRA58K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.87 Mobile Safari/537.36+"
     end
 
   end