Discource-C
/
discourse
mirror of https://github.com/discourse/discourse.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #2128 from vikhyat/crawler-detection 

Improved crawler detection
This commit is contained in:
Sam 2014-03-17 10:02:33 +11:00
parent 0f359d3f62 5c1fe30cd6
commit 82e34eabb7
3 changed files with 6 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
app/views/layouts/crawler.html.erb
View File

@ -16,6 +16,7 @@
<%= render :partial => "common/discourse_stylesheet" %> <%= render :partial => "common/discourse_stylesheet" %>
<%= discourse_csrf_tags %> <%= discourse_csrf_tags %>
<%= raw SiteContent.content_for(:head) %> <%= raw SiteContent.content_for(:head) %>
<%= yield :head %>
</head> </head>
<body> <body>
<%- unless customization_disabled? %> <%- unless customization_disabled? %>

2
lib/crawler_detection.rb
View File

@ -1,5 +1,5 @@
module CrawlerDetection module CrawlerDetection
def self.crawler?(user_agent) def self.crawler?(user_agent)
!/Googlebot|Mediapartners|AdsBot/.match(user_agent).nil? !/Googlebot|Mediapartners|AdsBot|curl|Twitterbot|facebookexternalhit|bingbot|Baiduspider/.match(user_agent).nil?
end end
end end

4
spec/components/crawler_detection_spec.rb
View File

@ -15,6 +15,10 @@ describe CrawlerDetection do
described_class.crawler?("(compatible; Mediapartners-Google/2.1; +http://www.google.com/bot.html)").should == true described_class.crawler?("(compatible; Mediapartners-Google/2.1; +http://www.google.com/bot.html)").should == true
described_class.crawler?("Mediapartners-Google").should == true described_class.crawler?("Mediapartners-Google").should == true
described_class.crawler?("AdsBot-Google (+http://www.google.com/adsbot.html)").should == true described_class.crawler?("AdsBot-Google (+http://www.google.com/adsbot.html)").should == true
described_class.crawler?("Twitterbot").should == true
described_class.crawler?("facebookexternalhit/1.1 (+http(s)://www.facebook.com/externalhit_uatext.php)").should == true
described_class.crawler?("Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)").should == true
described_class.crawler?("Baiduspider+(+http://www.baidu.com/search/spider.htm)").should == true
end end
it "returns false for non-crawler user agents" do it "returns false for non-crawler user agents" do