diff --git a/app/views/robots_txt/no_index.erb b/app/views/robots_txt/no_index.erb index 88d46ae9edb..44f3503f95d 100644 --- a/app/views/robots_txt/no_index.erb +++ b/app/views/robots_txt/no_index.erb @@ -1,9 +1,3 @@ -# See http://www.robotstxt.org/wc/norobots.html for documentation on how to use the robots.txt file -# - -# Googlebot must be allowed to index so it can remove items from the index -# we return the X-Robots-Tag with noindex, nofollow which will ensure -# indexing is minimized and nothing shows up in Google search results User-agent: googlebot Allow: <%= Discourse.base_path + "/" %> Disallow: <%= Discourse.base_path + "/uploads/*" %>