FEATURE: allow search engines to index tag pages. (#12248)

Previously, we blocked search engines in tag pages since they may get marked as a duplicate content.

* DEV: block tag inner pages from search engines crawling.

app/controllers/robots_txt_controller.rb

@@ -23,7 +23,7 @@ class RobotsTxtController < ApplicationController
     /u
     /my
     /search
-    /tag
+    /tag/*/l
     /g
     /t/*/*.rss
     /c/*.rss

app/controllers/tags_controller.rb

@@ -23,7 +23,7 @@ class TagsController < ::ApplicationController
 
   before_action :fetch_tag, only: [:info, :create_synonyms, :destroy_synonym]
 
-  after_action :add_noindex_header
+  after_action :add_noindex_header, except: [:index, :show]
 
   def index
     @description_meta = I18n.t("tags.title")

spec/requests/tags_controller_spec.rb

@@ -29,7 +29,6 @@ describe TagsController do
         tags = response.parsed_body["tags"]
         expect(tags.length).to eq(1)
         expect(tags[0]['text']).to eq("topic-test")
-        expect(response.headers['X-Robots-Tag']).to eq('noindex')
       end
     end