DEV: Add plugin API to add to robots.txt (#17378)

This plugin API can be used to add to robots.txt. The event handler
receives the complete robots information before it is converted into
robots.txt.

app/controllers/robots_txt_controller.rb

@@ -84,6 +84,8 @@ class RobotsTxtController < ApplicationController
       result[:agents] << { name: 'Googlebot', disallow: deny_paths_googlebot }
     end
 
+    DiscourseEvent.trigger(:robots_info, result)
+
     result
   end
 end

spec/requests/robots_txt_controller_spec.rb

@@ -161,5 +161,25 @@ RSpec.describe RobotsTxtController do
         expect(response.body).not_to include(sitemap_line)
       end
     end
+
+    describe 'plugins' do
+      let(:event_handler) do
+        Proc.new { |robots_info| robots_info[:agents] << { name: 'Test', disallow: ['/test/'] } }
+      end
+
+      before do
+        DiscourseEvent.on(:robots_info, &event_handler)
+      end
+
+      after do
+        DiscourseEvent.off(:robots_info, &event_handler)
+      end
+
+      it 'can add to robots.txt' do
+        get '/robots.txt'
+
+        expect(response.parsed_body).to include("User-agent: Test\nDisallow: /test/")
+      end
+    end
   end
 end