Disallow auth callbacks in robots.txt

2014-01-14 10:42:22 -05:00 · 2014-01-14 10:42:22 -05:00 · 88d9f3a786
parent 2a89f51d19
commit 88d9f3a786
1 changed files with 9 additions and 0 deletions
--- a/app/views/robots_txt/index.erb
+++ b/app/views/robots_txt/index.erb
@ -1,2 +1,11 @@
 # See http://www.robotstxt.org/wc/norobots.html for documentation on how to use the robots.txt file
 #
+User-agent: *
+Disallow: /auth/cas
+Disallow: /auth/facebook/callback
+Disallow: /auth/twitter/callback
+Disallow: /auth/google/callback
+Disallow: /auth/yahoo/callback
+Disallow: /auth/github/callback
+Disallow: /auth/persona/callback
+Disallow: /auth/cas/callback