FEATURE: automatic daily roll-up for screened IP addresses

app/controllers/admin/screened_ip_addresses_controller.rb

@@ -44,86 +44,8 @@ class Admin::ScreenedIpAddressesController < Admin::AdminController
     render json: success_json
   end
 
-  def star_subnets_query
-    @star_subnets_query ||= <<-SQL
-      SELECT network(inet(host(ip_address) || '/24')) AS ip_range
-        FROM screened_ip_addresses
-       WHERE action_type = #{ScreenedIpAddress.actions[:block]}
-         AND family(ip_address) = 4
-         AND masklen(ip_address) = 32
-    GROUP BY ip_range
-      HAVING COUNT(*) >= :min_count
-    SQL
-  end
-
-  def star_star_subnets_query
-    @star_star_subnets_query ||= <<-SQL
-      WITH weighted_subnets AS (
-        SELECT network(inet(host(ip_address) || '/16')) AS ip_range,
-               CASE masklen(ip_address)
-                 WHEN 32 THEN 1
-                 WHEN 24 THEN :roll_up_weight
-                 ELSE 0
-               END AS weight
-          FROM screened_ip_addresses
-         WHERE action_type = #{ScreenedIpAddress.actions[:block]}
-           AND family(ip_address) = 4
-      )
-      SELECT ip_range
-        FROM weighted_subnets
-    GROUP BY ip_range
-      HAVING SUM(weight) >= :min_count
-    SQL
-  end
-
-  def star_subnets
-    min_count = SiteSetting.min_ban_entries_for_roll_up
-    ScreenedIpAddress.exec_sql(star_subnets_query, min_count: min_count).values.flatten
-  end
-
-  def star_star_subnets
-    weight = SiteSetting.min_ban_entries_for_roll_up
-    ScreenedIpAddress.exec_sql(star_star_subnets_query, min_count: 10, roll_up_weight: weight).values.flatten
-  end
-
   def roll_up
-    # 1 - retrieve all subnets that needs roll up
+    subnets = ScreenedIpAddress.roll_up(current_user)
-    subnets = [star_subnets, star_star_subnets].flatten
-
-    # 2 - log the call
-    StaffActionLogger.new(current_user).log_roll_up(subnets) unless subnets.blank?
-
-    subnets.each do |subnet|
-      # 3 - create subnet if not already exists
-      ScreenedIpAddress.new(ip_address: subnet).save unless ScreenedIpAddress.where(ip_address: subnet).first
-
-      # 4 - update stats
-      sql = <<-SQL
-        UPDATE screened_ip_addresses
-           SET match_count   = sum_match_count,
-               created_at    = min_created_at,
-               last_match_at = max_last_match_at
-          FROM (
-            SELECT SUM(match_count)   AS sum_match_count,
-                   MIN(created_at)    AS min_created_at,
-                   MAX(last_match_at) AS max_last_match_at
-              FROM screened_ip_addresses
-             WHERE action_type = #{ScreenedIpAddress.actions[:block]}
-               AND family(ip_address) = 4
-               AND ip_address << :ip_address
-          ) s
-         WHERE ip_address = :ip_address
-      SQL
-
-      ScreenedIpAddress.exec_sql(sql, ip_address: subnet)
-
-      # 5 - remove old matches
-      ScreenedIpAddress.where(action_type: ScreenedIpAddress.actions[:block])
-                       .where("family(ip_address) = 4")
-                       .where("ip_address << ?", subnet)
-                       .delete_all
-    end
-
     render json: success_json.merge!({ subnets: subnets })
   end
 

app/jobs/scheduled/clean_up_unmatched_ips.rb

@@ -4,8 +4,12 @@ module Jobs
     every 1.day
 
     def execute(args)
+      # roll-up IP addresses first
+      ScreenedIpAddress.roll_up
+
       last_match_threshold = SiteSetting.max_age_unmatched_ips.days.ago
 
+      # remove old unmatched IP addresses
       ScreenedIpAddress.where(action_type: ScreenedIpAddress.actions[:block])
                        .where("last_match_at < ?", last_match_threshold)
                        .destroy_all

app/models/screened_email.rb

@@ -57,7 +57,7 @@ class ScreenedEmail < ActiveRecord::Base
         end
       end
     end
-    return matrix.last.last
+    matrix.last.last
   end
 
 end

app/models/screened_ip_address.rb

@@ -81,6 +81,91 @@ class ScreenedIpAddress < ActiveRecord::Base
     return true if ip_address.nil?
     !exists_for_ip_address_and_action?(ip_address, actions[:allow_admin], record_match: false)
   end
+
+  def self.star_subnets_query
+    @star_subnets_query ||= <<-SQL
+      SELECT network(inet(host(ip_address) || '/24')) AS ip_range
+        FROM screened_ip_addresses
+       WHERE action_type = #{ScreenedIpAddress.actions[:block]}
+         AND family(ip_address) = 4
+         AND masklen(ip_address) = 32
+    GROUP BY ip_range
+      HAVING COUNT(*) >= :min_count
+    SQL
+  end
+
+  def self.star_star_subnets_query
+    @star_star_subnets_query ||= <<-SQL
+      WITH weighted_subnets AS (
+        SELECT network(inet(host(ip_address) || '/16')) AS ip_range,
+               CASE masklen(ip_address)
+                 WHEN 32 THEN 1
+                 WHEN 24 THEN :roll_up_weight
+                 ELSE 0
+               END AS weight
+          FROM screened_ip_addresses
+         WHERE action_type = #{ScreenedIpAddress.actions[:block]}
+           AND family(ip_address) = 4
+      )
+      SELECT ip_range
+        FROM weighted_subnets
+    GROUP BY ip_range
+      HAVING SUM(weight) >= :min_count
+    SQL
+  end
+
+  def self.star_subnets
+    min_count = SiteSetting.min_ban_entries_for_roll_up
+    ScreenedIpAddress.exec_sql(star_subnets_query, min_count: min_count).values.flatten
+  end
+
+  def self.star_star_subnets
+    weight = SiteSetting.min_ban_entries_for_roll_up
+    ScreenedIpAddress.exec_sql(star_star_subnets_query, min_count: 10, roll_up_weight: weight).values.flatten
+  end
+
+  def self.roll_up(current_user=Discourse.system_user)
+    # 1 - retrieve all subnets that needs roll up
+    subnets = [star_subnets, star_star_subnets].flatten
+
+    # 2 - log the call
+    StaffActionLogger.new(current_user).log_roll_up(subnets) unless subnets.blank?
+
+    subnets.each do |subnet|
+      # 3 - create subnet if not already exists
+      ScreenedIpAddress.new(ip_address: subnet).save unless ScreenedIpAddress.where(ip_address: subnet).exists?
+
+      # 4 - update stats
+      sql = <<-SQL
+        UPDATE screened_ip_addresses
+           SET match_count   = sum_match_count,
+               created_at    = min_created_at,
+               last_match_at = max_last_match_at
+          FROM (
+            SELECT SUM(match_count)   AS sum_match_count,
+                   MIN(created_at)    AS min_created_at,
+                   MAX(last_match_at) AS max_last_match_at
+              FROM screened_ip_addresses
+             WHERE action_type = #{ScreenedIpAddress.actions[:block]}
+               AND family(ip_address) = 4
+               AND ip_address << :ip_address
+          ) s
+         WHERE ip_address = :ip_address
+      SQL
+
+      ScreenedIpAddress.exec_sql(sql, ip_address: subnet)
+
+      # 5 - remove old matches
+      ScreenedIpAddress.where(action_type: ScreenedIpAddress.actions[:block])
+                       .where("family(ip_address) = 4")
+                       .where("ip_address << ?", subnet)
+                       .delete_all
+    end
+
+    # return the subnets
+    subnets
+  end
+
 end
 
 # == Schema Information