FEATURE: much improved and simplified crawler detection
- phase one does it match 'trident|webkit|gecko|chrome|safari|msie|opera' yes- well it is possibly a browser - phase two does it match 'rss|bot|spider|crawler|facebook|archive|wayback|ping|monitor' probably a crawler then Based off: https://gist.github.com/SamSaffron/6cfad7ea3e6df321ffb7a84f93720a53
This commit is contained in:
parent
eaca2cb049
commit
7b562d2f46
|
@ -939,9 +939,13 @@ security:
|
|||
enable_escaped_fragments: true
|
||||
allow_index_in_robots_txt: true
|
||||
allow_moderators_to_create_categories: false
|
||||
non_crawler_user_agents:
|
||||
hidden: true
|
||||
default: 'trident|webkit|gecko|chrome|safari|msie|opera'
|
||||
type: list
|
||||
crawler_user_agents:
|
||||
hidden: true
|
||||
default: 'Googlebot|Mediapartners|AdsBot|curl|HTTrack|Twitterbot|facebookexternalhit|bingbot|Baiduspider|ia_archiver|archive.org_bot|Wayback Save Page|360Spider|Swiftbot|YandexBot'
|
||||
default: 'rss|bot|spider|crawler|facebook|archive|wayback|ping|monitor'
|
||||
type: list
|
||||
cors_origins:
|
||||
default: ''
|
||||
|
|
|
@ -1,17 +1,23 @@
|
|||
module CrawlerDetection
|
||||
|
||||
# added 'ia_archiver' based on https://meta.discourse.org/t/unable-to-archive-discourse-pages-with-the-internet-archive/21232
|
||||
# added 'Wayback Save Page' based on https://meta.discourse.org/t/unable-to-archive-discourse-with-the-internet-archive-save-page-now-button/22875
|
||||
# added 'Swiftbot' based on https://meta.discourse.org/t/how-to-add-html-markup-or-meta-tags-for-external-search-engine/28220
|
||||
def self.to_matcher(string)
|
||||
escaped = string.split('|').map { |agent| Regexp.escape(agent) }.join('|')
|
||||
Regexp.new(escaped)
|
||||
Regexp.new(escaped, Regexp::IGNORECASE)
|
||||
end
|
||||
|
||||
def self.crawler?(user_agent)
|
||||
# this is done to avoid regenerating regexes
|
||||
@non_crawler_matchers ||= {}
|
||||
@matchers ||= {}
|
||||
matcher = (@matchers[SiteSetting.crawler_user_agents] ||= to_matcher(SiteSetting.crawler_user_agents))
|
||||
matcher.match?(user_agent)
|
||||
|
||||
possibly_real = (@non_crawler_matchers[SiteSetting.non_crawler_user_agents] ||= to_matcher(SiteSetting.non_crawler_user_agents))
|
||||
|
||||
if user_agent.match?(possibly_real)
|
||||
known_bots = (@matchers[SiteSetting.crawler_user_agents] ||= to_matcher(SiteSetting.crawler_user_agents))
|
||||
user_agent.match?(known_bots)
|
||||
else
|
||||
true
|
||||
end
|
||||
|
||||
end
|
||||
end
|
||||
|
|
|
@ -6,9 +6,9 @@ describe CrawlerDetection do
|
|||
|
||||
it "can be amended via site settings" do
|
||||
SiteSetting.crawler_user_agents = 'Mooble|Kaboodle+*'
|
||||
expect(CrawlerDetection.crawler?("Mozilla/5.0 (compatible; Kaboodle+*/2.1; +http://www.google.com/bot.html)")).to eq(true)
|
||||
expect(CrawlerDetection.crawler?("Mozilla/5.0 (compatible; Mooble+*/2.1; +http://www.google.com/bot.html)")).to eq(true)
|
||||
expect(CrawlerDetection.crawler?("Mozilla/5.0 (compatible; Gooble+*/2.1; +http://www.google.com/bot.html)")).to eq(false)
|
||||
expect(CrawlerDetection.crawler?("Mozilla/5.0 Safari (compatible; Kaboodle+*/2.1; +http://www.google.com/bot.html)")).to eq(true)
|
||||
expect(CrawlerDetection.crawler?("Mozilla/5.0 Safari (compatible; Mooble+*/2.1; +http://www.google.com/bot.html)")).to eq(true)
|
||||
expect(CrawlerDetection.crawler?("Mozilla/5.0 Safari (compatible; Gooble+*/2.1; +http://www.google.com/bot.html)")).to eq(false)
|
||||
end
|
||||
|
||||
it "returns true for crawler user agents" do
|
||||
|
@ -37,6 +37,12 @@ describe CrawlerDetection do
|
|||
expect(described_class.crawler?("Mozilla/5.0 (iPad; CPU OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5355d Safari/8536.25")).to eq(false)
|
||||
expect(described_class.crawler?("Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:25.0) Gecko/20100101 Firefox/25.0")).to eq(false)
|
||||
expect(described_class.crawler?("Mozilla/5.0 (Linux; U; Android 4.0.3; ko-kr; LG-L160L Build/IML74K) AppleWebkit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30")).to eq(false)
|
||||
|
||||
expect(described_class.crawler?("DiscourseAPI Ruby Gem 0.19.0")).to eq(true)
|
||||
expect(described_class.crawler?("Pingdom.com_bot_version_1.4_(http://www.pingdom.com/)")).to eq(true)
|
||||
expect(described_class.crawler?("LogicMonitor SiteMonitor/1.0")).to eq(true)
|
||||
expect(described_class.crawler?("Java/1.8.0_151")).to eq(true)
|
||||
expect(described_class.crawler?("Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)")).to eq(true)
|
||||
end
|
||||
|
||||
end
|
||||
|
|
Loading…
Reference in New Issue