FEATURE: much improved and simplified crawler detection

- phase one does it match 'trident|webkit|gecko|chrome|safari|msie|opera'
    yes- well it is possibly a browser

- phase two does it match 'rss|bot|spider|crawler|facebook|archive|wayback|ping|monitor'
    probably a crawler then

Based off: https://gist.github.com/SamSaffron/6cfad7ea3e6df321ffb7a84f93720a53
This commit is contained in:
Sam 2018-01-16 15:41:13 +11:00
parent eaca2cb049
commit 7b562d2f46
3 changed files with 26 additions and 10 deletions

View File

@ -939,9 +939,13 @@ security:
enable_escaped_fragments: true
allow_index_in_robots_txt: true
allow_moderators_to_create_categories: false
non_crawler_user_agents:
hidden: true
default: 'trident|webkit|gecko|chrome|safari|msie|opera'
type: list
crawler_user_agents:
hidden: true
default: 'Googlebot|Mediapartners|AdsBot|curl|HTTrack|Twitterbot|facebookexternalhit|bingbot|Baiduspider|ia_archiver|archive.org_bot|Wayback Save Page|360Spider|Swiftbot|YandexBot'
default: 'rss|bot|spider|crawler|facebook|archive|wayback|ping|monitor'
type: list
cors_origins:
default: ''

View File

@ -1,17 +1,23 @@
module CrawlerDetection
# added 'ia_archiver' based on https://meta.discourse.org/t/unable-to-archive-discourse-pages-with-the-internet-archive/21232
# added 'Wayback Save Page' based on https://meta.discourse.org/t/unable-to-archive-discourse-with-the-internet-archive-save-page-now-button/22875
# added 'Swiftbot' based on https://meta.discourse.org/t/how-to-add-html-markup-or-meta-tags-for-external-search-engine/28220
def self.to_matcher(string)
escaped = string.split('|').map { |agent| Regexp.escape(agent) }.join('|')
Regexp.new(escaped)
Regexp.new(escaped, Regexp::IGNORECASE)
end
def self.crawler?(user_agent)
# this is done to avoid regenerating regexes
@non_crawler_matchers ||= {}
@matchers ||= {}
matcher = (@matchers[SiteSetting.crawler_user_agents] ||= to_matcher(SiteSetting.crawler_user_agents))
matcher.match?(user_agent)
possibly_real = (@non_crawler_matchers[SiteSetting.non_crawler_user_agents] ||= to_matcher(SiteSetting.non_crawler_user_agents))
if user_agent.match?(possibly_real)
known_bots = (@matchers[SiteSetting.crawler_user_agents] ||= to_matcher(SiteSetting.crawler_user_agents))
user_agent.match?(known_bots)
else
true
end
end
end

View File

@ -6,9 +6,9 @@ describe CrawlerDetection do
it "can be amended via site settings" do
SiteSetting.crawler_user_agents = 'Mooble|Kaboodle+*'
expect(CrawlerDetection.crawler?("Mozilla/5.0 (compatible; Kaboodle+*/2.1; +http://www.google.com/bot.html)")).to eq(true)
expect(CrawlerDetection.crawler?("Mozilla/5.0 (compatible; Mooble+*/2.1; +http://www.google.com/bot.html)")).to eq(true)
expect(CrawlerDetection.crawler?("Mozilla/5.0 (compatible; Gooble+*/2.1; +http://www.google.com/bot.html)")).to eq(false)
expect(CrawlerDetection.crawler?("Mozilla/5.0 Safari (compatible; Kaboodle+*/2.1; +http://www.google.com/bot.html)")).to eq(true)
expect(CrawlerDetection.crawler?("Mozilla/5.0 Safari (compatible; Mooble+*/2.1; +http://www.google.com/bot.html)")).to eq(true)
expect(CrawlerDetection.crawler?("Mozilla/5.0 Safari (compatible; Gooble+*/2.1; +http://www.google.com/bot.html)")).to eq(false)
end
it "returns true for crawler user agents" do
@ -37,6 +37,12 @@ describe CrawlerDetection do
expect(described_class.crawler?("Mozilla/5.0 (iPad; CPU OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5355d Safari/8536.25")).to eq(false)
expect(described_class.crawler?("Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:25.0) Gecko/20100101 Firefox/25.0")).to eq(false)
expect(described_class.crawler?("Mozilla/5.0 (Linux; U; Android 4.0.3; ko-kr; LG-L160L Build/IML74K) AppleWebkit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30")).to eq(false)
expect(described_class.crawler?("DiscourseAPI Ruby Gem 0.19.0")).to eq(true)
expect(described_class.crawler?("Pingdom.com_bot_version_1.4_(http://www.pingdom.com/)")).to eq(true)
expect(described_class.crawler?("LogicMonitor SiteMonitor/1.0")).to eq(true)
expect(described_class.crawler?("Java/1.8.0_151")).to eq(true)
expect(described_class.crawler?("Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)")).to eq(true)
end
end