From d554a591020edb2fb6448cc962c4bb7bfcaca5b4 Mon Sep 17 00:00:00 2001
From: Robin Ward <robin.ward@gmail.com>
Date: Fri, 10 May 2013 16:58:23 -0400
Subject: [PATCH] Support for a new site setting:
 `newuser_spam_host_threshold`. If a new user posts a link to the same host
 enough tiles, they will not be able to post the same link again.

Additionally, the site will flag all their previous posts with links as spam and they will
be instantly hidden via the auto hide workflow.
---
 .../preferences_username_controller.js        |  16 +--
 .../discourse/views/post_link_view.js         |   4 +-
 app/controllers/posts_controller.rb           |   5 +-
 app/models/post.rb                            |  61 +++++++++-
 app/models/post_action.rb                     |  34 ++++--
 app/models/site_setting.rb                    |   2 +
 app/models/topic_link.rb                      |  12 +-
 app/models/user.rb                            |  14 +++
 config/locales/server.en.yml                  |   3 +
 lib/discourse.rb                              |   6 +
 lib/post_creator.rb                           |  17 +++
 lib/system_message.rb                         |  10 +-
 spec/components/discourse_spec.rb             |  21 +++-
 spec/components/post_creator_spec.rb          |  25 +++-
 spec/components/system_message_spec.rb        |  13 --
 spec/controllers/posts_controller_spec.rb     |  25 ++++
 spec/models/post_action_spec.rb               |  23 ++++
 spec/models/post_spec.rb                      | 112 +++++++++++++++---
 spec/models/user_spec.rb                      |  27 ++++-
 19 files changed, 355 insertions(+), 75 deletions(-)

diff --git a/app/assets/javascripts/discourse/controllers/preferences_username_controller.js b/app/assets/javascripts/discourse/controllers/preferences_username_controller.js
index 1f325a19b7a..b2c31b56a75 100644
--- a/app/assets/javascripts/discourse/controllers/preferences_username_controller.js
+++ b/app/assets/javascripts/discourse/controllers/preferences_username_controller.js
@@ -12,20 +12,20 @@ Discourse.PreferencesUsernameController = Discourse.ObjectController.extend({
   error: false,
   errorMessage: null,
 
-  saveDisabled: (function() {
+  saveDisabled: function() {
     if (this.get('saving')) return true;
     if (this.blank('newUsername')) return true;
     if (this.get('taken')) return true;
     if (this.get('unchanged')) return true;
     if (this.get('errorMessage')) return true;
     return false;
-  }).property('newUsername', 'taken', 'errorMessage', 'unchanged', 'saving'),
+  }.property('newUsername', 'taken', 'errorMessage', 'unchanged', 'saving'),
 
-  unchanged: (function() {
+  unchanged: function() {
     return this.get('newUsername') === this.get('content.username');
-  }).property('newUsername', 'content.username'),
+  }.property('newUsername', 'content.username'),
 
-  checkTaken: (function() {
+  checkTaken: function() {
     if( this.get('newUsername') && this.get('newUsername').length < 3 ) {
       this.set('errorMessage', Em.String.i18n('user.name.too_short'));
     } else {
@@ -42,12 +42,12 @@ Discourse.PreferencesUsernameController = Discourse.ObjectController.extend({
         }
       });
     }
-  }).observes('newUsername'),
+  }.observes('newUsername'),
 
-  saveButtonText: (function() {
+  saveButtonText: function() {
     if (this.get('saving')) return Em.String.i18n("saving");
     return Em.String.i18n("user.change_username.action");
-  }).property('saving'),
+  }.property('saving'),
 
   changeUsername: function() {
     var preferencesUsernameController = this;
diff --git a/app/assets/javascripts/discourse/views/post_link_view.js b/app/assets/javascripts/discourse/views/post_link_view.js
index 4023a26c245..c4f74b789f6 100644
--- a/app/assets/javascripts/discourse/views/post_link_view.js
+++ b/app/assets/javascripts/discourse/views/post_link_view.js
@@ -10,10 +10,10 @@ Discourse.PostLinkView = Discourse.View.extend({
   tagName: 'li',
   classNameBindings: ['direction'],
 
-  direction: (function() {
+  direction: function() {
     if (this.get('content.reflection')) return 'incoming';
     return null;
-  }).property('content.reflection'),
+  }.property('content.reflection'),
 
   render: function(buffer) {
     var clicks;
diff --git a/app/controllers/posts_controller.rb b/app/controllers/posts_controller.rb
index b6c78eb38e1..f3800fe476a 100644
--- a/app/controllers/posts_controller.rb
+++ b/app/controllers/posts_controller.rb
@@ -39,8 +39,11 @@ class PostsController < ApplicationController
                                    meta_data: params[:meta_data],
                                    auto_close_days: params[:auto_close_days])
     post = post_creator.create
-
     if post_creator.errors.present?
+
+      # If the post was spam, flag all the user's posts as spam
+      current_user.flag_linked_posts_as_spam if post_creator.spam?
+
       render_json_error(post_creator)
     else
       post_serializer = PostSerializer.new(post, scope: guardian, root: false)
diff --git a/app/models/post.rb b/app/models/post.rb
index d5c1724ea32..b1c929a995f 100644
--- a/app/models/post.rb
+++ b/app/models/post.rb
@@ -99,6 +99,7 @@ class Post < ActiveRecord::Base
     @white_listed_image_classes ||= ['avatar', 'favicon', 'thumbnail']
   end
 
+  # How many images are present in the post
   def image_count
     return 0 unless raw.present?
 
@@ -110,19 +111,28 @@ class Post < ActiveRecord::Base
     end.count
   end
 
-  def link_count
-    return 0 unless raw.present?
+  # Returns an array of all links in a post
+  def raw_links
+    return [] unless raw.present?
+
+    return @raw_links if @raw_links.present?
 
     # Don't include @mentions in the link count
-    total = 0
+    @raw_links = []
     cooked_document.search("a[href]").each do |l|
       html_class = l.attributes['class']
+      url = l.attributes['href'].to_s
       if html_class.present?
         next if html_class.to_s == 'mention' && l.attributes['href'].to_s =~ /^\/users\//
       end
-      total +=1
+      @raw_links << url
     end
-    total
+    @raw_links
+  end
+
+  # How many links are present in the post
+  def link_count
+    raw_links.size
   end
 
   # Sometimes the post is being edited by someone else, for example, a mod.
@@ -136,6 +146,7 @@ class Post < ActiveRecord::Base
     @acting_user = pu
   end
 
+  # Ensure maximum amount of mentions in a post
   def max_mention_validator
     if acting_user.present? && acting_user.has_trust_level?(:basic)
       errors.add(:base, I18n.t(:too_many_mentions, count: SiteSetting.max_mentions_per_post)) if raw_mentions.size > SiteSetting.max_mentions_per_post
@@ -144,17 +155,57 @@ class Post < ActiveRecord::Base
     end
   end
 
+  # Ensure new users can not put too many images in a post
   def max_images_validator
     return if acting_user.present? && acting_user.has_trust_level?(:basic)
     errors.add(:base, I18n.t(:too_many_images, count: SiteSetting.newuser_max_images)) if image_count > SiteSetting.newuser_max_images
   end
 
+  # Ensure new users can not put too many links in a post
   def max_links_validator
     return if acting_user.present? && acting_user.has_trust_level?(:basic)
     errors.add(:base, I18n.t(:too_many_links, count: SiteSetting.newuser_max_links)) if link_count > SiteSetting.newuser_max_links
   end
 
 
+  # Count how many hosts are linked in the post
+  def linked_hosts
+    return {} if raw_links.blank?
+
+    return @linked_hosts if @linked_hosts.present?
+
+    @linked_hosts = {}
+    raw_links.each do |u|
+      uri = URI.parse(u)
+      host = uri.host
+      @linked_hosts[host] = (@linked_hosts[host] || 0) + 1
+    end
+    @linked_hosts
+  end
+
+  def total_hosts_usage
+    hosts = linked_hosts.clone
+
+    # Count hosts in previous posts the user has made, PLUS these new ones
+    TopicLink.where(domain: hosts.keys, user_id: acting_user.id).each do |tl|
+      hosts[tl.domain] = (hosts[tl.domain] || 0) + 1
+    end
+
+    hosts
+  end
+
+  # Prevent new users from posting the same hosts too many times.
+  def has_host_spam?
+    return false if acting_user.present? && acting_user.has_trust_level?(:basic)
+
+    total_hosts_usage.each do |host, count|
+      return true if count >= SiteSetting.newuser_spam_host_threshold
+    end
+
+    false
+  end
+
+
   def raw_mentions
     return [] if raw.blank?
 
diff --git a/app/models/post_action.rb b/app/models/post_action.rb
index ac3f687797d..9cec12945da 100644
--- a/app/models/post_action.rb
+++ b/app/models/post_action.rb
@@ -64,11 +64,8 @@ class PostAction < ActiveRecord::Base
     end
 
     PostAction.update_all({ deleted_at: Time.zone.now, deleted_by: moderator_id }, { post_id: post.id, post_action_type_id: actions })
-
     f = actions.map{|t| ["#{PostActionType.types[t]}_count", 0]}
-
     Post.with_deleted.update_all(Hash[*f.flatten], id: post.id)
-
     update_flagged_posts_count
   end
 
@@ -145,6 +142,7 @@ class PostAction < ActiveRecord::Base
     post_action_type_id == PostActionType.types[:notify_user] ||
     post_action_type_id == PostActionType.types[:notify_moderators]
   end
+
   # A custom rate limiter for this model
   def post_action_rate_limiter
     return unless is_flag? || is_bookmark? || is_like?
@@ -174,6 +172,30 @@ class PostAction < ActiveRecord::Base
                                     .exists?
   end
 
+  # Returns the flag counts for a post, taking into account that some users
+  # can weigh flags differently.
+  def self.flag_counts_for(post_id)
+    flag_counts = exec_sql("SELECT SUM(CASE
+                                         WHEN pa.deleted_at IS NULL AND u.admin THEN :flags_required_to_hide_post
+                                         WHEN pa.deleted_at IS NULL AND (NOT u.admin) THEN 1
+                                         ELSE 0
+                                       END) AS new_flags,
+                                   SUM(CASE
+                                         WHEN pa.deleted_at IS NOT NULL AND u.admin THEN :flags_required_to_hide_post
+                                         WHEN pa.deleted_at IS NOT NULL AND (NOT u.admin) THEN 1
+                                         ELSE 0
+                                       END) AS old_flags
+                            FROM post_actions AS pa
+                              INNER JOIN users AS u ON u.id = pa.user_id
+                            WHERE pa.post_id = :post_id AND
+                              pa.post_action_type_id IN (:post_action_types)",
+                            post_id: post_id,
+                            post_action_types: PostActionType.auto_action_flag_types.values,
+                            flags_required_to_hide_post: SiteSetting.flags_required_to_hide_post).first
+
+    [flag_counts['old_flags'].to_i, flag_counts['new_flags'].to_i]
+  end
+
   after_save do
     # Update denormalized counts
     post_action_type = PostActionType.types[post_action_type_id]
@@ -195,11 +217,7 @@ class PostAction < ActiveRecord::Base
 
     if PostActionType.auto_action_flag_types.include?(post_action_type) && SiteSetting.flags_required_to_hide_post > 0
       # automatic hiding of posts
-      flag_counts = exec_sql("SELECT SUM(CASE WHEN deleted_at IS NULL THEN 1 ELSE 0 END) AS new_flags,
-                                     SUM(CASE WHEN deleted_at IS NOT NULL THEN 1 ELSE 0 END) AS old_flags
-                              FROM post_actions
-                              WHERE post_id = ? AND post_action_type_id IN (?)", post.id, PostActionType.auto_action_flag_types.values).first
-      old_flags, new_flags = flag_counts['old_flags'].to_i, flag_counts['new_flags'].to_i
+      old_flags, new_flags = PostAction.flag_counts_for(post.id)
 
       if new_flags >= SiteSetting.flags_required_to_hide_post
         reason = old_flags > 0 ? Post.hidden_reasons[:flag_threshold_reached_again] : Post.hidden_reasons[:flag_threshold_reached]
diff --git a/app/models/site_setting.rb b/app/models/site_setting.rb
index eb5442bb841..5a770f0d2e2 100644
--- a/app/models/site_setting.rb
+++ b/app/models/site_setting.rb
@@ -178,6 +178,8 @@ class SiteSetting < ActiveRecord::Base
   setting(:newuser_max_links, 2)
   setting(:newuser_max_images, 0)
 
+  setting(:newuser_spam_host_threshold, 3)
+
   setting(:title_fancy_entities, true)
 
   # The default locale for the site
diff --git a/app/models/topic_link.rb b/app/models/topic_link.rb
index 6133e87c035..433ee111e69 100644
--- a/app/models/topic_link.rb
+++ b/app/models/topic_link.rb
@@ -68,12 +68,12 @@ class TopicLink < ActiveRecord::Base
 
           added_urls << url
           TopicLink.create(post_id: post.id,
-                                 user_id: post.user_id,
-                                 topic_id: post.topic_id,
-                                 url: url,
-                                 domain: parsed.host || Discourse.current_hostname,
-                                 internal: internal,
-                                 link_topic_id: topic_id)
+                           user_id: post.user_id,
+                           topic_id: post.topic_id,
+                           url: url,
+                           domain: parsed.host || Discourse.current_hostname,
+                           internal: internal,
+                           link_topic_id: topic_id)
 
           # Create the reflection if we can
           if topic_id.present?
diff --git a/app/models/user.rb b/app/models/user.rb
index fae98a183a0..572a1e7084a 100644
--- a/app/models/user.rb
+++ b/app/models/user.rb
@@ -3,6 +3,7 @@ require_dependency 'email_token'
 require_dependency 'trust_level'
 require_dependency 'pbkdf2'
 require_dependency 'summarize'
+require_dependency 'discourse'
 
 class User < ActiveRecord::Base
   attr_accessible :name, :username, :password, :email, :bio_raw, :website
@@ -22,6 +23,8 @@ class User < ActiveRecord::Base
   has_many :views
   has_many :user_visits
   has_many :invites
+  has_many :topic_links
+
   has_one :twitter_user_info, dependent: :destroy
   has_one :github_user_info, dependent: :destroy
   belongs_to :approved_by, class_name: 'User'
@@ -570,6 +573,17 @@ class User < ActiveRecord::Base
     cats.map{|c| c.id}.sort
   end
 
+  # Flag all posts from a user as spam
+  def flag_linked_posts_as_spam
+    admin = Discourse.system_user
+    topic_links.includes(:post).each do |tl|
+      begin
+        PostAction.act(admin, tl.post, PostActionType.types[:spam])
+      rescue PostAction::AlreadyActed
+        # If the user has already acted, just ignore it
+      end
+    end
+  end
 
   protected
 
diff --git a/config/locales/server.en.yml b/config/locales/server.en.yml
index 495d0c94c80..e139fb21578 100644
--- a/config/locales/server.en.yml
+++ b/config/locales/server.en.yml
@@ -32,6 +32,7 @@ en:
     zero: "Sorry, new users can't put links in posts."
     one: "Sorry, new users can only put one link in a post."
     other: "Sorry, new users can only put %{count} links in a post."
+  spamming_host: "Sorry you cannot post a link to that host."
 
   just_posted_that: "is too similar to what you recently posted"
   has_already_been_used: "has already been used"
@@ -574,6 +575,8 @@ en:
     tos_url: "If you have a Terms of Service document hosted elsewhere that you want to use, provide the full URL here."
     privacy_policy_url: "If you have a Privacy Policy document hosted elsewhere that you want to use, provide the full URL here."
 
+    newuser_spam_host_threshold: "How many times a new user can post a link to the same host within their `newuser_spam_host_posts` posts before being considered spam."
+
   notification_types:
     mentioned: "%{display_username} mentioned you in %{link}"
     liked: "%{display_username} liked your post in %{link}"
diff --git a/lib/discourse.rb b/lib/discourse.rb
index d621cb12e0d..456856b77c0 100644
--- a/lib/discourse.rb
+++ b/lib/discourse.rb
@@ -73,6 +73,12 @@ module Discourse
     end
   end
 
+  # Either returns the system_username user or the first admin.
+  def self.system_user
+    user = User.where(username_lower: SiteSetting.system_username).first if SiteSetting.system_username.present?
+    user = User.admins.order(:id).first if user.blank?
+    user
+  end
 
 private
 
diff --git a/lib/post_creator.rb b/lib/post_creator.rb
index d71cbbbb228..3285b771af1 100644
--- a/lib/post_creator.rb
+++ b/lib/post_creator.rb
@@ -31,9 +31,16 @@ class PostCreator
     # If we don't do this we introduce a rather risky dependency
     @user = user
     @opts = opts
+    @spam = false
+
     raise Discourse::InvalidParameters.new(:raw) if @opts[:raw].blank?
   end
 
+  # True if the post was considered spam
+  def spam?
+    @spam
+  end
+
   def guardian
     @guardian ||= Guardian.new(@user)
   end
@@ -63,6 +70,16 @@ class PostCreator
 
       post.image_sizes = @opts[:image_sizes] if @opts[:image_sizes].present?
       post.invalidate_oneboxes = @opts[:invalidate_oneboxes] if @opts[:invalidate_oneboxes].present?
+
+
+      # If the post has host spam, roll it back.
+      if post.has_host_spam?
+        post.errors.add(:base, I18n.t(:spamming_host))
+        @errors = post.errors
+        @spam = true
+        raise ActiveRecord::Rollback.new
+      end
+
       unless post.save
         @errors = post.errors
         raise ActiveRecord::Rollback.new
diff --git a/lib/system_message.rb b/lib/system_message.rb
index 162c3f37bef..8e5ed4e5d4c 100644
--- a/lib/system_message.rb
+++ b/lib/system_message.rb
@@ -1,6 +1,7 @@
 # Handle sending a message to a user from the system.
 require_dependency 'post_creator'
 require_dependency 'topic_subtype'
+require_dependency 'discourse'
 
 class SystemMessage
 
@@ -30,7 +31,7 @@ class SystemMessage
     title = I18n.t("system_messages.#{type}.subject_template", params)
     raw_body = I18n.t("system_messages.#{type}.text_body_template", params)
 
-    PostCreator.create(SystemMessage.system_user,
+    PostCreator.create(Discourse.system_user,
                        raw: raw_body,
                        title: title,
                        archetype: Archetype.private_message,
@@ -39,11 +40,6 @@ class SystemMessage
   end
 
 
-  # Either returns the system_username user or the first admin.
-  def self.system_user
-    user = User.where(username_lower: SiteSetting.system_username).first if SiteSetting.system_username.present?
-    user = User.admins.order(:id).first if user.blank?
-    user
-  end
+
 
 end
diff --git a/spec/components/discourse_spec.rb b/spec/components/discourse_spec.rb
index 0059511a8b8..dad0b8e17f9 100644
--- a/spec/components/discourse_spec.rb
+++ b/spec/components/discourse_spec.rb
@@ -16,7 +16,6 @@ describe Discourse do
   end
 
   context 'base_url' do
-
     context 'when ssl is off' do
       before do
         SiteSetting.expects(:use_ssl?).returns(false)
@@ -45,12 +44,26 @@ describe Discourse do
       it "returns the non standart port in the base url" do
         Discourse.base_url.should == "http://foo.com:3000"
       end
-
     end
-
-
   end
 
 
+  context '#system_user' do
+
+    let!(:admin) { Fabricate(:admin) }
+    let!(:another_admin) { Fabricate(:another_admin) }
+
+    it 'returns the user specified by the site setting system_username' do
+      SiteSetting.stubs(:system_username).returns(another_admin.username)
+      Discourse.system_user.should == another_admin
+    end
+
+    it 'returns the first admin user otherwise' do
+      SiteSetting.stubs(:system_username).returns(nil)
+      Discourse.system_user.should == admin
+    end
+
+  end
+
 end
 
diff --git a/spec/components/post_creator_spec.rb b/spec/components/post_creator_spec.rb
index f7c9dba1c1c..ba6557a3603 100644
--- a/spec/components/post_creator_spec.rb
+++ b/spec/components/post_creator_spec.rb
@@ -32,6 +32,11 @@ describe PostCreator do
 
     context 'success' do
 
+      it "doesn't return true for spam" do
+        creator.create
+        creator.spam?.should be_false
+      end
+
       it 'generates the correct messages for a secure topic' do
 
         admin = Fabricate(:admin)
@@ -60,7 +65,6 @@ describe PostCreator do
                                                    ].sort
         admin_ids = [Group[:admins].id]
         messages.any?{|m| m.group_ids != admin_ids}.should be_false
-
       end
 
       it 'generates the correct messages for a normal topic' do
@@ -187,6 +191,25 @@ describe PostCreator do
 
   end
 
+
+  context "host spam" do
+
+    let!(:topic) { Fabricate(:topic, user: user) }
+    let(:basic_topic_params) { { raw: 'test reply', topic_id: topic.id, reply_to_post_number: 4} }
+    let(:creator) { PostCreator.new(user, basic_topic_params) }
+
+    before do
+      Post.any_instance.expects(:has_host_spam?).returns(true)
+    end
+
+    it "does not create the post" do
+      creator.create
+      creator.errors.should be_present
+      creator.spam?.should be_true
+    end
+
+  end
+
   # more integration testing ... maximise our testing
   context 'existing topic' do
     let!(:topic) { Fabricate(:topic, user: user) }
diff --git a/spec/components/system_message_spec.rb b/spec/components/system_message_spec.rb
index c326568166c..7db9728b71a 100644
--- a/spec/components/system_message_spec.rb
+++ b/spec/components/system_message_spec.rb
@@ -21,18 +21,5 @@ describe SystemMessage do
     end
   end
 
-  context '#system_user' do
-
-    it 'returns the user specified by the site setting system_username' do
-      SiteSetting.stubs(:system_username).returns(admin.username)
-      SystemMessage.system_user.should == admin
-    end
-
-    it 'returns the first admin user otherwise' do
-      SiteSetting.stubs(:system_username).returns(nil)
-      SystemMessage.system_user.should == admin
-    end
-
-  end
 
 end
diff --git a/spec/controllers/posts_controller_spec.rb b/spec/controllers/posts_controller_spec.rb
index 2456d952a7c..35f9c59d736 100644
--- a/spec/controllers/posts_controller_spec.rb
+++ b/spec/controllers/posts_controller_spec.rb
@@ -273,6 +273,31 @@ describe PostsController do
         ::JSON.parse(response.body).should be_present
       end
 
+      context "errors" do
+
+        let(:post_with_errors) { Fabricate.build(:post, user: user)}
+
+        before do
+          post_with_errors.errors.add(:base, I18n.t(:spamming_host))
+          PostCreator.any_instance.stubs(:errors).returns(post_with_errors.errors)
+          PostCreator.any_instance.expects(:create).returns(post_with_errors)
+        end
+
+        it "does not succeed" do
+          xhr :post, :create, post: {raw: 'test'}
+          User.any_instance.expects(:flag_linked_posts_as_spam).never
+          response.should_not be_success
+        end
+
+        it "it triggers flag_linked_posts_as_spam when the post creator returns spam" do
+          PostCreator.any_instance.expects(:spam?).returns(true)
+          User.any_instance.expects(:flag_linked_posts_as_spam)
+          xhr :post, :create, post: {raw: 'test'}
+        end
+
+      end
+
+
       context "parameters" do
 
         let(:post_creator) { mock }
diff --git a/spec/models/post_action_spec.rb b/spec/models/post_action_spec.rb
index 8183efc4cd7..4b93d8b8d11 100644
--- a/spec/models/post_action_spec.rb
+++ b/spec/models/post_action_spec.rb
@@ -155,6 +155,29 @@ describe PostAction do
 
   describe 'flagging' do
 
+    context "flag_counts_for" do
+      it "returns the correct flag counts" do
+        post = Fabricate(:post)
+
+        SiteSetting.stubs(:flags_required_to_hide_post).returns(7)
+
+        # A post with no flags has 0 for flag counts
+        PostAction.flag_counts_for(post.id).should == [0, 0]
+
+        flag = PostAction.act(Fabricate(:evil_trout), post, PostActionType.types[:spam])
+        PostAction.flag_counts_for(post.id).should == [0, 1]
+
+        # If an admin flags the post, it is counted higher
+        admin = Fabricate(:admin)
+        PostAction.act(admin, post, PostActionType.types[:spam])
+        PostAction.flag_counts_for(post.id).should == [0, 8]
+
+        # If a flag is dismissed
+        PostAction.clear_flags!(post, admin)
+        PostAction.flag_counts_for(post.id).should == [8, 0]
+      end
+    end
+
     it 'does not allow you to flag stuff with 2 reasons' do
       post = Fabricate(:post)
       u1 = Fabricate(:evil_trout)
diff --git a/spec/models/post_spec.rb b/spec/models/post_spec.rb
index 785d6a7f051..f93f37cbf13 100644
--- a/spec/models/post_spec.rb
+++ b/spec/models/post_spec.rb
@@ -7,6 +7,13 @@ describe Post do
     ImageSorcery.any_instance.stubs(:convert).returns(false)
   end
 
+  # Help us build a post with a raw body
+  def post_with_body(body, user=nil)
+    args = post_args.merge(raw: body)
+    args[:user] = user if user.present?
+    Fabricate.build(:post, args)
+  end
+
   it { should belong_to :user }
   it { should belong_to :topic }
   it { should validate_presence_of :raw }
@@ -89,12 +96,12 @@ describe Post do
   describe "maximum images" do
     let(:newuser) { Fabricate(:user, trust_level: TrustLevel.levels[:newuser]) }
     let(:post_no_images) { Fabricate.build(:post, post_args.merge(user: newuser)) }
-    let(:post_one_image) { Fabricate.build(:post, post_args.merge(raw: "![sherlock](http://bbc.co.uk/sherlock.jpg)", user: newuser)) }
-    let(:post_two_images) { Fabricate.build(:post, post_args.merge(raw: "<img src='http://discourse.org/logo.png'> <img src='http://bbc.co.uk/sherlock.jpg'>", user: newuser)) }
-    let(:post_with_avatars) { Fabricate.build(:post, post_args.merge(raw: '<img alt="smiley" title=":smiley:" src="/assets/emoji/smiley.png" class="avatar"> <img alt="wink" title=":wink:" src="/assets/emoji/wink.png" class="avatar">', user: newuser)) }
-    let(:post_with_favicon) { Fabricate.build(:post, post_args.merge(raw: '<img src="/assets/favicons/wikipedia.png" class="favicon">', user: newuser)) }
-    let(:post_with_thumbnail) { Fabricate.build(:post, post_args.merge(raw: '<img src="/assets/emoji/smiley.png" class="thumbnail">', user: newuser)) }
-    let(:post_with_two_classy_images) { Fabricate.build(:post, post_args.merge(raw: "<img src='http://discourse.org/logo.png' class='classy'> <img src='http://bbc.co.uk/sherlock.jpg' class='classy'>", user: newuser)) }
+    let(:post_one_image) { post_with_body("![sherlock](http://bbc.co.uk/sherlock.jpg)", newuser) }
+    let(:post_two_images) { post_with_body("<img src='http://discourse.org/logo.png'> <img src='http://bbc.co.uk/sherlock.jpg'>", newuser) }
+    let(:post_with_avatars) { post_with_body('<img alt="smiley" title=":smiley:" src="/assets/emoji/smiley.png" class="avatar"> <img alt="wink" title=":wink:" src="/assets/emoji/wink.png" class="avatar">', newuser) }
+    let(:post_with_favicon) { post_with_body('<img src="/assets/favicons/wikipedia.png" class="favicon">', newuser) }
+    let(:post_with_thumbnail) { post_with_body('<img src="/assets/emoji/smiley.png" class="thumbnail">', newuser) }
+    let(:post_with_two_classy_images) { post_with_body("<img src='http://discourse.org/logo.png' class='classy'> <img src='http://bbc.co.uk/sherlock.jpg' class='classy'>", newuser) }
 
     it "returns 0 images for an empty post" do
       Fabricate.build(:post).image_count.should == 0
@@ -159,11 +166,78 @@ describe Post do
 
   end
 
+  context "links" do
+    let(:newuser) { Fabricate(:user, trust_level: TrustLevel.levels[:newuser]) }
+    let(:no_links) { post_with_body("hello world my name is evil trout", newuser) }
+    let(:one_link) { post_with_body("[jlawr](http://www.imdb.com/name/nm2225369)", newuser) }
+    let(:two_links) { post_with_body("<a href='http://disneyland.disney.go.com/'>disney</a> <a href='http://reddit.com'>reddit</a>", newuser)}
+    let(:three_links) { post_with_body("http://discourse.org and http://discourse.org/another_url and http://www.imdb.com/name/nm2225369", newuser)}
+
+    describe "raw_links" do
+      it "returns a blank collection for a post with no links" do
+        no_links.raw_links.should be_blank
+      end
+
+      it "finds a link within markdown" do
+        one_link.raw_links.should == ["http://www.imdb.com/name/nm2225369"]
+      end
+
+      it "can find two links from html" do
+        two_links.raw_links.should == ["http://disneyland.disney.go.com/", "http://reddit.com"]
+      end
+
+      it "can find three links without markup" do
+        three_links.raw_links.should == ["http://discourse.org", "http://discourse.org/another_url", "http://www.imdb.com/name/nm2225369"]
+      end
+    end
+
+    describe "linked_hosts" do
+      it "returns blank with no links" do
+        no_links.linked_hosts.should be_blank
+      end
+
+      it "returns the host and a count for links" do
+        two_links.linked_hosts.should == {"disneyland.disney.go.com" => 1, "reddit.com" => 1}
+      end
+
+      it "it counts properly with more than one link on the same host" do
+        three_links.linked_hosts.should == {"discourse.org" => 2, "www.imdb.com" => 1}
+      end
+    end
+
+    describe "total host usage" do
+
+      it "has none for a regular post" do
+        no_links.total_hosts_usage.should be_blank
+      end
+
+      context "with a previous host" do
+
+        let(:user) { old_post.newuser }
+        let(:another_disney_link) { post_with_body("[radiator springs](http://disneyland.disney.go.com/disney-california-adventure/radiator-springs-racers/)", newuser) }
+
+        before do
+          another_disney_link.save
+          TopicLink.extract_from(another_disney_link)
+        end
+
+        it "contains the new post's links, PLUS the previous one" do
+          two_links.total_hosts_usage.should == {'disneyland.disney.go.com' => 2, 'reddit.com' => 1}
+        end
+
+      end
+
+    end
+
+
+  end
+
+
   describe "maximum links" do
     let(:newuser) { Fabricate(:user, trust_level: TrustLevel.levels[:newuser]) }
-    let(:post_one_link) { Fabricate.build(:post, post_args.merge(raw: "[sherlock](http://www.bbc.co.uk/programmes/b018ttws)", user: newuser)) }
-    let(:post_two_links) { Fabricate.build(:post, post_args.merge(raw: "<a href='http://discourse.org'>discourse</a> <a href='http://twitter.com'>twitter</a>", user: newuser)) }
-    let(:post_with_mentions) { Fabricate.build(:post, post_args.merge(raw: "hello @#{newuser.username} how are you doing?") )}
+    let(:post_one_link) { post_with_body("[sherlock](http://www.bbc.co.uk/programmes/b018ttws)", newuser) }
+    let(:post_two_links) { post_with_body("<a href='http://discourse.org'>discourse</a> <a href='http://twitter.com'>twitter</a>", newuser) }
+    let(:post_with_mentions) { post_with_body("hello @#{newuser.username} how are you doing?", newuser) }
 
     it "returns 0 links for an empty post" do
       Fabricate.build(:post).link_count.should == 0
@@ -251,8 +325,8 @@ describe Post do
     context "max mentions" do
 
       let(:newuser) { Fabricate(:user, trust_level: TrustLevel.levels[:newuser]) }
-      let(:post_with_one_mention) { Fabricate.build(:post, post_args.merge(raw: "@Jake is the person I'm mentioning", user: newuser)) }
-      let(:post_with_two_mentions) { Fabricate.build(:post, post_args.merge(raw: "@Jake @Finn are the people I'm mentioning", user: newuser)) }
+      let(:post_with_one_mention) { post_with_body("@Jake is the person I'm mentioning", newuser) }
+      let(:post_with_two_mentions) { post_with_body("@Jake @Finn are the people I'm mentioning", newuser) }
 
       context 'new user' do
         before do
@@ -298,7 +372,7 @@ describe Post do
   context "raw_hash" do
 
     let(:raw) { "this is our test post body"}
-    let(:post) { Fabricate.build(:post, raw: raw) }
+    let(:post) { post_with_body(raw) }
 
     it "returns a value" do
       post.raw_hash.should be_present
@@ -310,19 +384,19 @@ describe Post do
     end
 
     it "returns the same value for the same raw" do
-      post.raw_hash.should == Fabricate.build(:post, raw: raw).raw_hash
+      post.raw_hash.should == post_with_body(raw).raw_hash
     end
 
     it "returns a different value for a different raw" do
-      post.raw_hash.should_not == Fabricate.build(:post, raw: "something else").raw_hash
+      post.raw_hash.should_not == post_with_body("something else").raw_hash
     end
 
     it "returns the same hash even with different white space" do
-      post.raw_hash.should == Fabricate.build(:post, raw: " thisis ourt est postbody").raw_hash
+      post.raw_hash.should == post_with_body(" thisis ourt est postbody").raw_hash
     end
 
     it "returns the same hash even with different text case" do
-      post.raw_hash.should == Fabricate.build(:post, raw: "THIS is OUR TEST post BODy").raw_hash
+      post.raw_hash.should == post_with_body("THIS is OUR TEST post BODy").raw_hash
     end
   end
 
@@ -600,12 +674,12 @@ describe Post do
   end
 
   describe 'urls' do
-    it 'no-ops for empty list' do 
+    it 'no-ops for empty list' do
       Post.urls([]).should == {}
     end
 
-    # integration test -> should move to centralized integration test 
-    it 'finds urls for posts presented' do 
+    # integration test -> should move to centralized integration test
+    it 'finds urls for posts presented' do
       p1 = Fabricate(:post)
       p2 = Fabricate(:post)
       Post.urls([p1.id, p2.id]).should == {p1.id => p1.url, p2.id => p2.url}
diff --git a/spec/models/user_spec.rb b/spec/models/user_spec.rb
index b83513ff538..061f87b1f41 100644
--- a/spec/models/user_spec.rb
+++ b/spec/models/user_spec.rb
@@ -266,7 +266,7 @@ describe User do
 
   describe "trust levels" do
 
-    # NOTE be sure to use build to avoid db calls 
+    # NOTE be sure to use build to avoid db calls
     let(:user) { Fabricate.build(:user, trust_level: TrustLevel.levels[:newuser]) }
 
     it "sets to the default trust level setting" do
@@ -770,6 +770,31 @@ describe User do
     end
   end
 
+  describe "flag_linked_posts_as_spam" do
+    let(:user) { Fabricate(:user) }
+    let!(:admin) { Fabricate(:admin) }
+    let!(:post) { PostCreator.new(user, title: "this topic contains spam", raw: "this post has a link: http://discourse.org").create }
+    let!(:another_post) { PostCreator.new(user, title: "this topic also contains spam", raw: "this post has a link: http://discourse.org/asdfa").create }
+    let!(:post_without_link) { PostCreator.new(user, title: "this topic shouldn't be spam", raw: "this post has no links in it.").create }
+
+    it "has flagged all the user's posts as spam" do
+      user.flag_linked_posts_as_spam
+
+      post.reload
+      post.spam_count.should == 1
+
+      another_post.reload
+      another_post.spam_count.should == 1
+
+      post_without_link.reload
+      post_without_link.spam_count.should == 0
+
+      # It doesn't raise an exception if called again
+      user.flag_linked_posts_as_spam
+
+    end
+
+  end
 
   describe 'update_time_read!' do
     let(:user) { Fabricate(:user) }