FEATURE: Block indexing the embed topic list (#16495)

This adds a robots tag header to disallow indexing the topic list that
powers our embed topic list feature. It also uses a new tag that allows
the content to be indexed in the parent page.

See https://developers.google.com/search/blog/2022/01/robots-meta-tag-indexifembedded

https://meta.discourse.org/t/-/125911/127

app/controllers/embed_controller.rb

@@ -35,6 +35,8 @@ class EmbedController < ApplicationController
       raise Discourse::InvalidParameters.new(:embed_class) unless @embed_class =~ /^[a-zA-Z0-9\-_]+$/
     end
 
+    response.headers['X-Robots-Tag'] = 'noindex, indexifembedded'
+
     if params.has_key?(:template) && params[:template] == "complete"
       @template = "complete"
     else

spec/requests/embed_controller_spec.rb

@@ -156,6 +156,14 @@ describe EmbedController do
         expect(response.body).to match("data-referer=\"\\*\"")
       end
 
+      it "disallows indexing the embed topic list" do
+        topic = Fabricate(:topic)
+        get '/embed/topics?discourse_embed_id=de-1234', headers: {
+          'REFERER' => 'https://example.com/evil-trout'
+        }
+        expect(response.status).to eq(200)
+        expect(response.headers['X-Robots-Tag']).to match(/noindex/)
+      end
     end
   end