From b52143feff8c32f21ed53033b6a0a65ee45dce0e Mon Sep 17 00:00:00 2001
From: Joshua Rosenfeld <jomaxro@discourse.org>
Date: Thu, 25 Jun 2020 13:55:06 -0400
Subject: [PATCH] FIX: Remove paths from robots.txt in favor of noindex header

Google no longer supports the use of robots.txt to block indexing.
See https://support.google.com/webmasters/answer/6062608 and
https://support.google.com/webmasters/answer/93710

Previous commits have added the `noindex` header to appropriate pages,
now we need to remove the paths from robots.txt so the pages can be
crawled.

Follow up to:
13f229808a22db9e1032832a313ab701b66614c8
b6765aac4b532c026418a7ffd9effd0741ab8a37
676be3a853454a33cf627c3d570feb37d3bb0bfd
07b728c5e557c9aae91c51f3eaac5c32d479f2a2
c94e6a9a66757ea48d99e3ee8d880523871cb6f4
---
 app/controllers/robots_txt_controller.rb | 13 -------------
 1 file changed, 13 deletions(-)

diff --git a/app/controllers/robots_txt_controller.rb b/app/controllers/robots_txt_controller.rb
index 9c99cb5c1dc..3af15d9174a 100644
--- a/app/controllers/robots_txt_controller.rb
+++ b/app/controllers/robots_txt_controller.rb
@@ -10,14 +10,6 @@ class RobotsTxtController < ApplicationController
   DISALLOWED_PATHS ||= %w{
     /auth/
     /assets/browser-update*.js
-    /users/
-    /u/
-    /my/
-    /badges/
-    /search
-    /search/
-    /tags
-    /tags/
     /email/
     /session
     /session/
@@ -27,11 +19,6 @@ class RobotsTxtController < ApplicationController
     /user-api-key/
     /*?api_key*
     /*?*api_key*
-    /groups
-    /groups/
-    /t/*/*.rss
-    /tags/*.rss
-    /c/*.rss
   }
 
   def index