FIX: Correctly render 403 errors to crawlers using basic-html (#26287)

Previously, when crawlers triggered a Discourse::InvalidAccess exception, they would be served the full Ember SPA. The SPA is not optimized for crawling, and so this is likely to cause problems for sites. This issue is particularly problematic when user profiles are hidden from the public via the `hide_user_profiles_from_public` setting, because the crawler would end up being 'soft-redirected' to the homepage in the SPA.

app/controllers/application_controller.rb

@@ -339,7 +339,7 @@ class ApplicationController < ActionController::Base
         return render plain: message, status: status_code
       end
       with_resolved_locale do
-        error_page_opts[:layout] = (opts[:include_ember] && @preloaded) ? "application" : "no_ember"
+        error_page_opts[:layout] = (opts[:include_ember] && @preloaded) ? set_layout : "no_ember"
         render html: build_not_found_page(error_page_opts)
       end
     end

spec/requests/users_controller_spec.rb

@@ -4542,7 +4542,7 @@ RSpec.describe UsersController do
         expect(parsed["trust_level"]).to be_blank
       end
 
-      it "should redirect to login page for anonymous user when profiles are hidden" do
+      it "should 403 for anonymous user when profiles are hidden" do
         SiteSetting.hide_user_profiles_from_public = true
         get "/u/#{user.username}.json"
         expect(response).to have_http_status(:forbidden)
@@ -4550,6 +4550,13 @@ RSpec.describe UsersController do
         expect(response).to have_http_status(:forbidden)
       end
 
+      it "should 403 correctly for crawlers when profiles are hidden" do
+        SiteSetting.hide_user_profiles_from_public = true
+        get "/u/#{user.username}", headers: { "User-Agent" => "Googlebot" }
+        expect(response).to have_http_status(:forbidden)
+        expect(response.body).to have_tag("body.crawler")
+      end
+
       describe "user profile views" do
         it "should track a user profile view for an anon user" do
           get "/"