per spec we need to repeat disallow paths per agent

2018-04-16 15:38:10 +10:00 · 2018-04-16 15:38:10 +10:00 · 223379e21a
parent a1ef455c78
commit 223379e21a
1 changed files with 4 additions and 0 deletions
--- a/app/views/robots_txt/index.erb
+++ b/app/views/robots_txt/index.erb
@ -23,4 +23,8 @@ Disallow: <%= Discourse.base_uri + "/" %>
 <% @crawler_delayed_agents.each do |agent, delay| %>
 User-agent: <%= agent %>
 Crawl-delay: <%= delay %>
+<% RobotsTxtController::DISALLOWED_PATHS.each do |path| %>
+Disallow: <%= Discourse.base_uri + path %>
+<% end %>
+
 <% end %>