Track Discourse user agent pageviews as crawler

Since 5bfe051e, Discourse user agents are marked as non-crawlers (to avoid accidental blacklisting). This makes sure pageviews for these agents are tracked as crawler hits.
This commit is contained in:
Penar Musaraj 2019-05-08 10:38:55 -04:00
parent 4aaee7ee35
commit a4eb523af6
2 changed files with 15 additions and 1 deletions

View File

@ -62,7 +62,11 @@ module Middleware
@is_crawler ||=
begin
user_agent = @env[USER_AGENT]
CrawlerDetection.crawler?(user_agent) ? :true : :false
if CrawlerDetection.crawler?(user_agent)
:true
else
user_agent.downcase.include?("discourse") ? :true : :false
end
end
@is_crawler == :true
end

View File

@ -68,6 +68,16 @@ describe Middleware::RequestTracker do
expect(ApplicationRequest.page_view_anon.first.count).to eq(2)
expect(ApplicationRequest.page_view_crawler.first.count).to eq(1)
expect(ApplicationRequest.page_view_anon_mobile.first.count).to eq(1)
# log discourse User Agent requests as crawler for page views
data = Middleware::RequestTracker.get_data(env(
"HTTP_USER_AGENT" => "DiscourseAPI Ruby Gem 0.19.0"
), ["200", { "Content-Type" => 'text/html' }], 0.1)
Middleware::RequestTracker.log_request(data)
ApplicationRequest.write_cache!
expect(ApplicationRequest.page_view_crawler.first.count).to eq(2)
end
end