correct specs, ensure crawler layout only applies to html

app/controllers/application_controller.rb

@@ -60,7 +60,11 @@ class ApplicationController < ActionController::Base
   end
 
   def use_crawler_layout?
-    @use_crawler_layout ||= (has_escaped_fragment? || CrawlerDetection.crawler?(request.user_agent) || params.key?("print"))
+    @use_crawler_layout ||=
+      request.user_agent &&
+      (request.content_type.blank? || request.content_type.include?('html')) &&
+      !['json', 'rss'].include?(params[:format]) &&
+      (has_escaped_fragment? || CrawlerDetection.crawler?(request.user_agent) || params.key?("print"))
   end
 
   def add_readonly_header

lib/crawler_detection.rb

@@ -1,16 +1,24 @@
 module CrawlerDetection
 
-  def self.to_matcher(string)
+  def self.to_matcher(string, type: nil)
     escaped = string.split('|').map { |agent| Regexp.escape(agent) }.join('|')
+
+    if type == :real && Rails.env == "test"
+      # we need this bypass so we properly render views
+      escaped << "|Rails Testing"
+    end
+
     Regexp.new(escaped, Regexp::IGNORECASE)
   end
 
   def self.crawler?(user_agent)
+    return true if user_agent.nil?
+
     # this is done to avoid regenerating regexes
     @non_crawler_matchers ||= {}
     @matchers ||= {}
 
-    possibly_real = (@non_crawler_matchers[SiteSetting.non_crawler_user_agents] ||= to_matcher(SiteSetting.non_crawler_user_agents))
+    possibly_real = (@non_crawler_matchers[SiteSetting.non_crawler_user_agents] ||= to_matcher(SiteSetting.non_crawler_user_agents, type: :real))
 
     if user_agent.match?(possibly_real)
       known_bots = (@matchers[SiteSetting.crawler_user_agents] ||= to_matcher(SiteSetting.crawler_user_agents))

spec/components/middleware/request_tracker_spec.rb

@@ -6,6 +6,7 @@ describe Middleware::RequestTracker do
   def env(opts = {})
     {
       "HTTP_HOST" => "http://test.com",
+      "HTTP_USER_AGENT" => "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36",
       "REQUEST_URI" => "/path?bla=1",
       "REQUEST_METHOD" => "GET",
       "rack.input" => ""