From a307ad65172ad21ba59c4b82377440dcf39469fd Mon Sep 17 00:00:00 2001 From: mcmcclur Date: Tue, 16 May 2017 11:17:05 -0400 Subject: [PATCH] Update crawler_detection.rb Add HTTrack to the list of detected crawlers so that Discourse will serve vanilla HTML per https://meta.discourse.org/t/a-basic-discourse-archival-tool/62614/25 --- lib/crawler_detection.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/crawler_detection.rb b/lib/crawler_detection.rb index 075375ebc90..a8892fdc769 100644 --- a/lib/crawler_detection.rb +++ b/lib/crawler_detection.rb @@ -4,6 +4,6 @@ module CrawlerDetection # added 'Swiftbot' based on https://meta.discourse.org/t/how-to-add-html-markup-or-meta-tags-for-external-search-engine/28220 def self.crawler?(user_agent) - !/Googlebot|Mediapartners|AdsBot|curl|Twitterbot|facebookexternalhit|bingbot|Baiduspider|ia_archiver|Wayback Save Page|360Spider|Swiftbot|YandexBot/.match(user_agent).nil? + !/Googlebot|Mediapartners|AdsBot|curl|HTTrack|Twitterbot|facebookexternalhit|bingbot|Baiduspider|ia_archiver|Wayback Save Page|360Spider|Swiftbot|YandexBot/.match(user_agent).nil? end end