FEATURE: add no_index header if robots is disabled

Expand SiteSetting.allow_index_in_robots_txt so it also adds a
noindex header if set to false.

This makes sure that nothing is indexed even if it somehow reaches
Google.

app/controllers/application_controller.rb

@@ -44,7 +44,7 @@ class ApplicationController < ActionController::Base
   after_action  :perform_refresh_session
   after_action  :dont_cache_page
   after_action  :conditionally_allow_site_embedding
-  after_action  :add_noindex_header, if: -> { is_feed_request? }
+  after_action  :add_noindex_header, if: -> { is_feed_request? || !SiteSetting.allow_index_in_robots_txt }
 
   layout :set_layout
 

spec/requests/topics_controller_spec.rb

@@ -1843,12 +1843,19 @@ RSpec.describe TopicsController do
     end
 
     it "is not included for normal topics" do
-      topic = Fabricate(:topic, visible: true)
       get "/t/#{topic.slug}/#{topic.id}.json"
 
       expect(response.headers['X-Robots-Tag']).to eq(nil)
     end
 
+    it "is included when allow_index_in_robots_txt is set to false" do
+      SiteSetting.allow_index_in_robots_txt = false
+
+      get "/t/#{topic.slug}/#{topic.id}.json"
+
+      expect(response.headers['X-Robots-Tag']).to eq('noindex')
+    end
+
     it "doesn't store an incoming link when there's no referer" do
       expect {
         get "/t/#{topic.id}.json"