diff --git a/app/controllers/robots_txt_controller.rb b/app/controllers/robots_txt_controller.rb new file mode 100644 index 00000000000..785cc8646c6 --- /dev/null +++ b/app/controllers/robots_txt_controller.rb @@ -0,0 +1,14 @@ +class RobotsTxtController < ApplicationController + layout false + skip_before_filter :check_xhr + + def index + path = if SiteSetting.allow_index_in_robots_txt && !SiteSetting.restrict_access + :index + else + :no_index + end + + render path, content_type: 'text/plain' + end +end diff --git a/app/models/site_setting.rb b/app/models/site_setting.rb index 6a0720b1616..65c3b93b79c 100644 --- a/app/models/site_setting.rb +++ b/app/models/site_setting.rb @@ -102,6 +102,8 @@ class SiteSetting < ActiveRecord::Base # we need to think of a way to force users to enter certain settings, this is a minimal config thing setting(:notification_email, 'info@discourse.org') + setting(:allow_index_in_robots_txt, true) + setting(:send_welcome_message, true) setting(:twitter_consumer_key, '') diff --git a/public/robots.txt b/app/views/robots_txt/index.erb similarity index 50% rename from public/robots.txt rename to app/views/robots_txt/index.erb index 085187fa58b..376f3ae7184 100644 --- a/public/robots.txt +++ b/app/views/robots_txt/index.erb @@ -1,5 +1,2 @@ # See http://www.robotstxt.org/wc/norobots.html for documentation on how to use the robots.txt file # -# To ban all spiders from the entire site uncomment the next two lines: -# User-Agent: * -# Disallow: / diff --git a/app/views/robots_txt/no_index.erb b/app/views/robots_txt/no_index.erb new file mode 100644 index 00000000000..867036ff28a --- /dev/null +++ b/app/views/robots_txt/no_index.erb @@ -0,0 +1,6 @@ +# See http://www.robotstxt.org/wc/norobots.html for documentation on how to use the robots.txt file +# +User-Agent: * +Disallow: / + + diff --git a/config/locales/en.yml b/config/locales/en.yml index 51510682d4f..0a3331ef46e 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -257,6 +257,9 @@ en: posts_per_page: "How many posts are returned on a topic page" system_username: "Username that sends system messages" send_welcome_message: "Do new users get a welcome private message?" + + allow_index_in_robots_txt: "Site should be indexed by search engines (update robots.txt)" + port: "If you'd like to specify a port in the URL. Useful in development mode. Leave blank for none." force_hostname: "If you'd like to specify a hostname in the URL. Useful in development mode. Leave blank for none." diff --git a/config/routes.rb b/config/routes.rb index c81a86d1776..9da1c576503 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -207,6 +207,9 @@ Discourse::Application.routes.draw do post 'draft' => 'draft#update' delete 'draft' => 'draft#destroy' + + get 'robots.txt' => 'robots_txt#index' + # You can have the root of your site routed with "root" # just remember to delete public/index.html. root :to => 'list#index' diff --git a/spec/controllers/robots_txt_controller_spec.rb b/spec/controllers/robots_txt_controller_spec.rb new file mode 100644 index 00000000000..6bd739c062d --- /dev/null +++ b/spec/controllers/robots_txt_controller_spec.rb @@ -0,0 +1,26 @@ +require 'spec_helper' + +describe RobotsTxtController do + + context '.index' do + it "returns noindex when indexing is disallowed" do + SiteSetting.stubs(:allow_index_in_robots_txt).returns(true) + get :index + response.should render_template :index + end + + it "returns index when indexing is allowed" do + SiteSetting.stubs(:allow_index_in_robots_txt).returns(false) + get :index + response.should render_template :no_index + end + + it "serves it regardless if a site is in private mode" do + SiteSetting.stubs(:allow_index_in_robots_txt).returns(true) + SiteSetting.stubs(:restrict_access).returns(true) + get :index + response.should render_template :no_index + end + + end +end