FEATURE: Add WebHookEventsDailyAggregate report page (#27573)

* FEATURE: Add WebHookEventsDailyAggregate

Add WebHookEventsDailyAggregate model to store daily aggregates of web hook events.
Add AggregateWebHooksEvents job to aggregate web hook events daily.
Add spec for WebHookEventsDailyAggregate model.

* DEV: Update annotations for web_hook_events_daily_aggregate.rb

* DEV: Update app/jobs/scheduled/aggregate_web_hooks_events.rb

Co-authored-by: Martin Brennan <martin@discourse.org>

* DEV: Address review feedback

Solves:
- https://github.com/discourse/discourse/pull/27542#discussion_r1646961101
- https://github.com/discourse/discourse/pull/27542#discussion_r1646958890
- https://github.com/discourse/discourse/pull/27542#discussion_r1646976808
- https://github.com/discourse/discourse/pull/27542#discussion_r1646979846
- https://github.com/discourse/discourse/pull/27542#discussion_r1646981036

* A11Y: Add translation to retain_web_hook_events_aggregate_days key

* FEATURE: Purge old web hook events daily aggregate

Solves: https://github.com/discourse/discourse/pull/27542#discussion_r1646961101

* DEV:  Update tests for web_hook_events_daily_aggregate

Update WebHookEventsDailyAggregate to not use save! at the end
Solves: https://github.com/discourse/discourse/pull/27542#discussion_r1646984601

* PERF: Change job query to use WebHook table instead of WebHookEvent table

* DEV: Update tests to use `fab!`

* DEV: Address code review feedback.

Add idempotency to job
Add has_many to WebHook

* FEATURE: Add WebHookEventsDailyAggregate report page

Add locales for the new report page
Reorder imports in the report.rb

* DEV: Remove logger line

* FEATURE: Add `auto_insert_none_item` option to `report-filters` components

---------

Co-authored-by: Martin Brennan <martin@discourse.org>

app/assets/javascripts/admin/addon/components/report-filters/group.hbs

@@ -5,6 +5,7 @@
   @onChange={{this.onChange}}
   @options={{hash
     allowAny=this.filter.allow_any
+    autoInsertNoneItem=this.filter.auto_insert_none_item
     filterable=true
     none="admin.dashboard.reports.groups"
   }}

app/assets/javascripts/admin/addon/components/report-filters/list.hbs

@@ -4,6 +4,7 @@
   @onChange={{this.onChange}}
   @options={{hash
     allowAny=this.filter.allow_any
+    autoInsertNoneItem=this.filter.auto_insert_none_item
     filterable=true
     none="admin.dashboard.report_filter_any"
   }}

app/models/concerns/reports/web_hook_events_daily_aggregate.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+module Reports::WebHookEventsDailyAggregate
+  extend ActiveSupport::Concern
+  class_methods do
+    def report_web_hook_events_daily_aggregate(report)
+      report.labels = [
+        { type: :date, property: :x, title: I18n.t("reports.default.labels.day") },
+        { type: :percent, property: :y, title: I18n.t("reports.default.labels.percent") },
+      ]
+
+      type = report.filters.dig(:type_of_web_hook_event)
+      # all = failed_events + successful_events per day;
+      # failed events = failed events per day;
+      # successful events = successful events per day;
+      # mean duration = mean duration of all events per day;
+      report.add_filter(
+        "type_of_web_hook_event",
+        type: "list",
+        default: type || "all",
+        choices:
+          ["all", "failed events", "successful events", "mean duration"].map do |t|
+            { id: t, name: t }
+          end,
+        allow_any: false,
+        auto_insert_none_item: false,
+      )
+
+      report.average = true
+      report.percent = true
+
+      data_points = WebHookEventsDailyAggregate.by_day(report.start_date, report.end_date)
+
+      report.data = []
+
+      data_points.each do |data_point|
+        case type
+        when "failed events"
+          report.data << { x: data_point["date"], y: data_point["failed_event_count"] }
+        when "successful events"
+          report.data << { x: data_point["date"], y: data_point["successful_event_count"] }
+        when "mean duration"
+          report.data << { x: data_point["date"], y: data_point["mean_duration"] }
+        else
+          report.data << {
+            x: data_point["date"],
+            y: data_point["successful_event_count"] + data_point["failed_event_count"],
+          }
+        end
+      end
+
+      if report.facets.include?(:prev_period)
+        report.prev_period =
+          WebHookEventsDailyAggregate.by_day(report.prev_start_date, report.prev_end_date)
+      end
+
+      if report.facets.include?(:prev30Days)
+        report.prev30Days =
+          WebHookEventsDailyAggregate.by_day(report.start_date - 30.days, report.start_date)
+      end
+    end
+  end
+end

app/models/report.rb

@@ -16,50 +16,51 @@ class Report
     include_subcategories
   ]
 
-  include Reports::PostEdits
-  include Reports::TopTrafficSources
-  include Reports::TopicsWithNoResponse
-  include Reports::DauByMau
-  include Reports::FlagsStatus
-  include Reports::Emails
-  include Reports::Likes
-  include Reports::SystemPrivateMessages
-  include Reports::UsersByType
-  include Reports::StorageStats
-  include Reports::NotifyModeratorsPrivateMessages
-  include Reports::SuspiciousLogins
-  include Reports::TopReferredTopics
-  include Reports::Signups
-  include Reports::NotifyUserPrivateMessages
-  include Reports::NewContributors
-  include Reports::TrendingSearch
-  include Reports::UserToUserPrivateMessages
-  include Reports::Flags
-  include Reports::Topics
-  include Reports::Posts
   include Reports::Bookmarks
-  include Reports::StaffLogins
+  include Reports::ConsolidatedApiRequests
-  include Reports::DailyEngagedUsers
-  include Reports::UserToUserPrivateMessagesWithReplies
-  include Reports::MobileVisits
-  include Reports::TopReferrers
-  include Reports::WebCrawlers
-  include Reports::ModeratorsActivity
-  include Reports::TopIgnoredUsers
-  include Reports::UserFlaggingRatio
-  include Reports::TrustLevelGrowth
   include Reports::ConsolidatedPageViews
   include Reports::ConsolidatedPageViewsBrowserDetection
-  include Reports::ConsolidatedApiRequests
+  include Reports::DailyEngagedUsers
-  include Reports::Visits
+  include Reports::DauByMau
-  include Reports::TimeToFirstResponse
+  include Reports::Emails
-  include Reports::UsersByTrustLevel
+  include Reports::Flags
+  include Reports::FlagsStatus
+  include Reports::Likes
+  include Reports::MobileVisits
   include Reports::ModeratorWarningPrivateMessages
+  include Reports::ModeratorsActivity
+  include Reports::NewContributors
+  include Reports::NotifyModeratorsPrivateMessages
+  include Reports::NotifyUserPrivateMessages
+  include Reports::PostEdits
+  include Reports::Posts
   include Reports::ProfileViews
+  include Reports::Signups
+  include Reports::StaffLogins
+  include Reports::StorageStats
+  include Reports::SuspiciousLogins
+  include Reports::SystemPrivateMessages
+  include Reports::TimeToFirstResponse
+  include Reports::TopIgnoredUsers
+  include Reports::TopReferredTopics
+  include Reports::TopReferrers
+  include Reports::TopTrafficSources
   include Reports::TopUploads
   include Reports::TopUsersByLikesReceived
-  include Reports::TopUsersByLikesReceivedFromInferiorTrustLevel
   include Reports::TopUsersByLikesReceivedFromAVarietyOfPeople
+  include Reports::TopUsersByLikesReceivedFromInferiorTrustLevel
+  include Reports::Topics
+  include Reports::TopicsWithNoResponse
+  include Reports::TrendingSearch
+  include Reports::TrustLevelGrowth
+  include Reports::UserFlaggingRatio
+  include Reports::UserToUserPrivateMessages
+  include Reports::UserToUserPrivateMessagesWithReplies
+  include Reports::UsersByTrustLevel
+  include Reports::UsersByType
+  include Reports::Visits
+  include Reports::WebCrawlers
+  include Reports::WebHookEventsDailyAggregate
 
   attr_accessor :type,
                 :data,

config/locales/client.en.yml

@@ -5038,6 +5038,8 @@ en:
               label: Category
             include_subcategories:
               label: "Include Subcategories"
+            type_of_web_hook_event:
+              label: "Type of event"
       flags:
         title: "Moderation Flags"
         description: "Description"

config/locales/server.en.yml

@@ -1570,6 +1570,11 @@ en:
         user_agent: "User Agent"
         page_views: "Pageviews"
       description: "List of web crawler user agents, sorted by pageviews."
+    web_hook_events_daily_aggregate:
+      title: "Web Hook Events"
+      xaxis: "Day"
+      yaxis: "Number of events"
+      description: "Number of web hook events triggered or their mean duration."
     suspicious_logins:
       title: "Suspicious Logins"
       labels: