diff --git a/app/assets/javascripts/discourse/app/components/about-page.gjs b/app/assets/javascripts/discourse/app/components/about-page.gjs index 5d929aaa44c..42be96476b9 100644 --- a/app/assets/javascripts/discourse/app/components/about-page.gjs +++ b/app/assets/javascripts/discourse/app/components/about-page.gjs @@ -1,5 +1,6 @@ import Component from "@glimmer/component"; import { hash } from "@ember/helper"; +import { service } from "@ember/service"; import { htmlSafe } from "@ember/template"; import AboutPageUsers from "discourse/components/about-page-users"; import PluginOutlet from "discourse/components/plugin-outlet"; @@ -20,6 +21,8 @@ export function clearAboutPageActivities() { } export default class AboutPage extends Component { + @service siteSettings; + get moderatorsCount() { return this.args.model.moderators.length; } @@ -115,6 +118,20 @@ export default class AboutPage extends Component { }, ]; + if (this.siteSettings.display_eu_visitor_stats) { + list.splice(2, 0, { + icon: "user-secret", + class: "visitors", + activityText: I18n.messageFormat("about.activities.visitors_MF", { + total_count: this.args.model.stats.visitors_7_days, + eu_count: this.args.model.stats.eu_visitors_7_days, + total_formatted_number: number(this.args.model.stats.visitors_7_days), + eu_formatted_number: number(this.args.model.stats.eu_visitors_7_days), + }), + period: I18n.t("about.activities.periods.last_7_days"), + }); + } + return list.concat(this.siteActivitiesFromPlugins()); } diff --git a/app/assets/javascripts/discourse/app/controllers/about.js b/app/assets/javascripts/discourse/app/controllers/about.js index aa203fc2932..6d211c17ed8 100644 --- a/app/assets/javascripts/discourse/app/controllers/about.js +++ b/app/assets/javascripts/discourse/app/controllers/about.js @@ -23,4 +23,15 @@ export default class AboutController extends Controller { return null; } } + + @discourseComputed( + "model.stats.visitors_30_days", + "model.stats.eu_visitors_30_days" + ) + statsTableFooter(all, eu) { + return I18n.messageFormat("about.traffic_info_footer_MF", { + total_visitors: all, + eu_visitors: eu, + }); + } } diff --git a/app/assets/javascripts/discourse/app/templates/about.hbs b/app/assets/javascripts/discourse/app/templates/about.hbs index 023ebf79d27..135aa123cc5 100644 --- a/app/assets/javascripts/discourse/app/templates/about.hbs +++ b/app/assets/javascripts/discourse/app/templates/about.hbs @@ -139,6 +139,22 @@ {{number this.model.stats.active_users_30_days}} — + {{#if this.siteSettings.display_eu_visitor_stats}} + + {{i18n "about.visitor_count"}} + {{number this.model.stats.visitors_last_day}} + {{number this.model.stats.visitors_7_days}} + {{number this.model.stats.visitors_30_days}} + — + + + {{i18n "about.eu_visitor_count"}} + {{number this.model.stats.eu_visitors_last_day}} + {{number this.model.stats.eu_visitors_7_days}} + {{number this.model.stats.eu_visitors_30_days}} + — + + {{/if}} {{i18n "about.like_count"}} {{number this.model.stats.likes_last_day}} @@ -170,6 +186,10 @@ {{/each}} + {{#if this.siteSettings.display_eu_visitor_stats}} +

{{this.statsTableFooter}}

+ {{/if}} {{/if}} diff --git a/app/assets/stylesheets/common/base/about.scss b/app/assets/stylesheets/common/base/about.scss index b70f50a5510..e3a015c5e34 100644 --- a/app/assets/stylesheets/common/base/about.scss +++ b/app/assets/stylesheets/common/base/about.scss @@ -113,5 +113,9 @@ section.about { } } } + + .stats-table-footer { + max-width: 30em; + } } } diff --git a/app/models/application_request.rb b/app/models/application_request.rb index a789ceb3346..b471314a1a0 100644 --- a/app/models/application_request.rb +++ b/app/models/application_request.rb @@ -59,6 +59,17 @@ class ApplicationRequest < ActiveRecord::Base s end + + def self.request_type_count_for_period(type, since) + id = self.req_types[type] + if !id + raise ArgumentError.new( + "unknown request type #{type.inspect} in ApplicationRequest.req_types", + ) + end + + self.where(req_type: id).where("date >= ?", since).sum(:count) + end end # == Schema Information diff --git a/app/models/stat.rb b/app/models/stat.rb index e425f7e5e26..0ab2a5c09d2 100644 --- a/app/models/stat.rb +++ b/app/models/stat.rb @@ -40,7 +40,7 @@ class Stat end def self.core_stats - [ + list = [ Stat.new("topics", show_in_ui: true, expose_via_api: true) { Statistics.topics }, Stat.new("posts", show_in_ui: true, expose_via_api: true) { Statistics.posts }, Stat.new("users", show_in_ui: true, expose_via_api: true) { Statistics.users }, @@ -50,6 +50,19 @@ class Stat Statistics.participating_users end, ] + + if SiteSetting.display_eu_visitor_stats + list.concat( + [ + Stat.new("visitors", show_in_ui: true, expose_via_api: true) { Statistics.visitors }, + Stat.new("eu_visitors", show_in_ui: true, expose_via_api: true) do + Statistics.eu_visitors + end, + ], + ) + end + + list end def self._api_stats diff --git a/config/locales/client.en.yml b/config/locales/client.en.yml index 5ac999b5a54..4692c81bb36 100644 --- a/config/locales/client.en.yml +++ b/config/locales/client.en.yml @@ -342,6 +342,16 @@ en: post_count: "Posts" user_count: "Sign-ups" active_user_count: "Active users" + visitor_count: "Visitors" + eu_visitor_count: "Visitors from European Union" + traffic_info_footer_MF: | + In the last 6 months, this site has served content to an average of approximately { total_visitors, plural, + one {# people} + other {# people} + } each month, with an average of approximately { eu_visitors, plural, + one {# people} + other {# people} + } from the European Union. contact: "Contact us" contact_info: "In the event of a critical issue or urgent matter affecting this site, please contact %{contact_info}." site_activity: "Site activity" @@ -363,6 +373,14 @@ en: likes: one: "%{formatted_number} like" other: "%{formatted_number} likes" + visitors_MF: | + { total_count, plural, + one {{total_formatted_number} visitor} + other {{total_formatted_number} visitors} + }, about { eu_count, plural, + one {{eu_formatted_number}} + other {{eu_formatted_number}} + } from the EU periods: last_7_days: "in the last 7 days" today: "today" diff --git a/config/locales/server.en.yml b/config/locales/server.en.yml index 88b44f595ba..8a8a48a0539 100644 --- a/config/locales/server.en.yml +++ b/config/locales/server.en.yml @@ -2209,6 +2209,7 @@ en: tos_url: "If you have a Terms of Service document hosted elsewhere that you want to use, provide the full URL here." privacy_policy_url: "If you have a Privacy Policy document hosted elsewhere that you want to use, provide the full URL here." log_anonymizer_details: "Whether to keep a user's details in the log after being anonymized." + display_eu_visitor_stats: "Show number of global and EU visitors on the /about page." newuser_spam_host_threshold: "How many times a new user can post a link to the same host within their `newuser_spam_host_threshold` posts before being considered spam." diff --git a/config/site_settings.yml b/config/site_settings.yml index ccd5a440256..3d5c6e80375 100644 --- a/config/site_settings.yml +++ b/config/site_settings.yml @@ -2503,6 +2503,10 @@ legal: default: false hidden: true client: true + display_eu_visitor_stats: + default: false + client: true + hidden: true backups: enable_backups: diff --git a/lib/statistics.rb b/lib/statistics.rb index c366cb13366..1beda7cd6b7 100644 --- a/lib/statistics.rb +++ b/lib/statistics.rb @@ -1,6 +1,36 @@ # frozen_string_literal: true class Statistics + EU_COUNTRIES = %w[ + AT + BE + BG + CY + CZ + DE + DK + EE + ES + FI + FR + GR + HR + HU + IE + IT + LT + LU + LV + MT + NL + PL + PT + RO + SE + SI + SK + ] + def self.active_users { last_day: User.where("last_seen_at > ?", 1.day.ago).count, @@ -56,6 +86,58 @@ class Statistics } end + def self.visitors + periods = [[1.day.ago, :last_day], [7.days.ago, :"7_days"], [30.days.ago, :"30_days"]] + + periods + .map do |(period, key)| + anon_page_views = + ApplicationRequest.request_type_count_for_period(:page_view_anon_browser, period) + + logged_in_visitors = logged_in_visitors_count(period) + next key, anon_page_views if logged_in_visitors == 0 + + logged_in_page_views = + ApplicationRequest.request_type_count_for_period(:page_view_logged_in_browser, period) + next key, anon_page_views + logged_in_visitors if logged_in_page_views == 0 + + total_visitors = logged_in_visitors + avg_logged_in_page_view_per_user = logged_in_page_views.to_f / logged_in_visitors + anon_visitors = (anon_page_views / avg_logged_in_page_view_per_user).round + total_visitors += anon_visitors + [key, total_visitors] + end + .to_h + end + + def self.eu_visitors + periods = [[1.day.ago, :last_day], [7.days.ago, :"7_days"], [30.days.ago, :"30_days"]] + + periods + .map do |(period, key)| + logged_in_page_views = + ApplicationRequest.request_type_count_for_period(:page_view_logged_in_browser, period) + anon_page_views = + ApplicationRequest.request_type_count_for_period(:page_view_anon_browser, period) + + all_logged_in_visitors = logged_in_visitors_count(period) + eu_logged_in_visitors = eu_logged_in_visitors_count(period) + + next key, 0 if all_logged_in_visitors == 0 || eu_logged_in_visitors == 0 + next key, eu_logged_in_visitors if logged_in_page_views == 0 + + avg_logged_in_page_view_per_user = logged_in_page_views / all_logged_in_visitors.to_f + + eu_logged_in_visitors_ratio = eu_logged_in_visitors / all_logged_in_visitors.to_f + + eu_anon_visitors = + ((anon_page_views / avg_logged_in_page_view_per_user) * eu_logged_in_visitors_ratio).round + eu_visitors = eu_logged_in_visitors + eu_anon_visitors + [key, eu_visitors] + end + .to_h + end + private def self.participating_users_count(date) @@ -75,4 +157,27 @@ class Statistics DB.query_single(sql, date: date, action_types: UserAction::USER_ACTED_TYPES).first end + + def self.logged_in_visitors_count(since) + DB.query_single(<<~SQL, since:).first + SELECT COUNT(DISTINCT user_id) + FROM user_visits + WHERE visited_at >= :since + SQL + end + + def self.eu_logged_in_visitors_count(since) + results = DB.query_hash(<<~SQL, since:) + SELECT DISTINCT(user_id), ip_address + FROM user_visits uv + INNER JOIN users u + ON u.id = uv.user_id + WHERE visited_at >= :since AND ip_address IS NOT NULL + SQL + + results.reduce(0) do |sum, hash| + ip_info = DiscourseIpInfo.get(hash["ip_address"].to_s) + sum + (EU_COUNTRIES.include?(ip_info[:country_code]) ? 1 : 0) + end + end end diff --git a/spec/lib/statistics_spec.rb b/spec/lib/statistics_spec.rb index 657eec99abf..02c568c8d1c 100644 --- a/spec/lib/statistics_spec.rb +++ b/spec/lib/statistics_spec.rb @@ -1,7 +1,59 @@ # frozen_string_literal: true RSpec.describe Statistics do - describe "#participating_users" do + def create_page_views_and_user_visit_records(date, users) + freeze_time(date - 50.minutes) do + 2.times { ApplicationRequest.increment!(:page_view_anon_browser) } + ApplicationRequest.increment!(:page_view_logged_in_browser) + end + + freeze_time(date - 3.days) do + ApplicationRequest.increment!(:page_view_anon_browser) + 5.times { ApplicationRequest.increment!(:page_view_logged_in_browser) } + end + + freeze_time(date - 6.days) do + 3.times { ApplicationRequest.increment!(:page_view_anon_browser) } + 4.times { ApplicationRequest.increment!(:page_view_logged_in_browser) } + end + + freeze_time(date - 8.days) do + ApplicationRequest.increment!(:page_view_anon_browser) + ApplicationRequest.increment!(:page_view_logged_in_browser) + end + + freeze_time(date - 15.days) do + 4.times { ApplicationRequest.increment!(:page_view_anon_browser) } + 3.times { ApplicationRequest.increment!(:page_view_logged_in_browser) } + end + + freeze_time(date - 31.days) do + ApplicationRequest.increment!(:page_view_anon_browser) + ApplicationRequest.increment!(:page_view_logged_in_browser) + end + + UserVisit.create!(user_id: users[0].id, visited_at: date - 50.minute) + + UserVisit.create!(user_id: users[0].id, visited_at: date - 36.hours) + UserVisit.create!(user_id: users[1].id, visited_at: date - 2.day) + UserVisit.create!(user_id: users[0].id, visited_at: date - 4.days) + UserVisit.create!(user_id: users[2].id, visited_at: date - 6.days) + UserVisit.create!(user_id: users[3].id, visited_at: date - 3.days) + UserVisit.create!(user_id: users[3].id, visited_at: date - 5.days) + UserVisit.create!(user_id: users[1].id, visited_at: date - 66.hours) + + UserVisit.create!(user_id: users[2].id, visited_at: date - 8.days) + UserVisit.create!(user_id: users[3].id, visited_at: date - 13.days) + UserVisit.create!(user_id: users[0].id, visited_at: date - 24.days) + UserVisit.create!(user_id: users[4].id, visited_at: date - 19.days) + + UserVisit.create!(user_id: users[2].id, visited_at: date - 31.days) + end + + fab!(:users) { Fabricate.times(5, :user) } + let(:date) { DateTime.parse("2024-03-01 13:00") } + + describe ".participating_users" do it "returns no participating users by default" do pu = described_class.participating_users expect(pu[:last_day]).to eq(0) @@ -29,4 +81,138 @@ RSpec.describe Statistics do expect(described_class.participating_users[:last_day]).to eq(1) end end + + describe ".visitors" do + before do + ApplicationRequest.enable + create_page_views_and_user_visit_records(date, users) + end + + after { ApplicationRequest.disable } + + it "estimates the number of visitors for each of the previous 1 day, 7 days and 30 days periods" do + freeze_time(date) do + visitors = described_class.visitors + + # anon page views: 2 + # logged-in page views: 1 + # logged-in visitors: 1 + # we can estimate the number of unique anon visitors by dividing the + # number of anon page views by the average number of logged-in page + # views per logged-in visitor. + # in this case, the estimated number of anon visitors is 2 / (1 / 1) = 2. + # total visitors = logged-in visitors (1) + estimated anon visitors (2) = 3 + expect(visitors[:last_day]).to eq(3) + + # anon page views: 6 + # logged-in page views: 10 + # logged-in visitors: 4 + # we can estimate the number of unique anon visitors by dividing the + # number of anon page views by the average number of logged-in page + # views per logged-in visitor. + # in this case, the estimated number of anon visitors is 6 / (10 / 4) ~= 2. + # total visitors = logged-in visitors (4) + estimated anon visitors (2) = 6 + expect(visitors[:"7_days"]).to eq(6) + + # anon page views: 11 + # logged-in page views: 14 + # logged-in visitors: 5 + # we can estimate the number of unique anon visitors by dividing the + # number of anon page views by the average number of logged-in page + # views per logged-in visitor. + # in this case, the estimated number of anon visitors is 11 / (14 / 5) ~= 4. + # total visitors = logged-in visitors (5) + estimated anon visitors (4) = 9 + expect(visitors[:"30_days"]).to eq(9) + end + end + + it "is the same as the number of anon page views when there are no logged in visitors" do + freeze_time(date) do + UserVisit.delete_all + + visitors = described_class.visitors + + expect(visitors[:last_day]).to eq(2) + expect(visitors[:"7_days"]).to eq(6) + expect(visitors[:"30_days"]).to eq(11) + end + end + end + + describe ".eu_visitors" do + before do + ApplicationRequest.enable + create_page_views_and_user_visit_records(date, users) + + users[0].update!(ip_address: IPAddr.new("60.23.1.42")) + users[1].update!(ip_address: IPAddr.new("90.19.255.63")) + users[2].update!(ip_address: IPAddr.new("8.33.134.244")) + users[3].update!(ip_address: IPAddr.new("2.74.0.98")) + users[4].update!(ip_address: IPAddr.new("88.82.3.101")) + + # EU IP addresses + DiscourseIpInfo.stubs(:get).with("60.23.1.42").returns({ country_code: "FR" }) # users[0] + DiscourseIpInfo.stubs(:get).with("2.74.0.98").returns({ country_code: "NL" }) # users[3] + DiscourseIpInfo.stubs(:get).with("88.82.3.101").returns({ country_code: "DE" }) # users[4] + + # non-EU IP addresses + DiscourseIpInfo.stubs(:get).with("90.19.255.63").returns({ country_code: "US" }) # users[1] + DiscourseIpInfo.stubs(:get).with("8.33.134.244").returns({ country_code: "SA" }) # users[2] + end + + after { ApplicationRequest.disable } + + it "estimates the number of EU visitors for each of the previous 1 day, 7 days and 30 days periods" do + freeze_time(date) do + eu_visitors = described_class.eu_visitors + + # anon page views: 2 + # logged-in page views: 1 + # logged-in visitors: 1 + # EU logged-in visitors: 1 (users[0]) + # we can estimate the number of unique EU anon visitors by dividing the + # number of anon page views by the average number of logged-in page + # views per logged-in visitor, then multiplying the result by the ratio + # of EU logged-in visitors to all logged-in visitors. + # in this case, the estimated number of EU anon visitors is 2 / (1 / 1) * (1 / 1) = 2 + # total EU visitors = EU logged-in visitors (1) + estimated EU anon visitors (2) = 3 + expect(eu_visitors[:last_day]).to eq(3) + + # anon page views: 6 + # logged-in page views: 10 + # logged-in visitors: 4 + # EU logged-in visitors: 2 (users[0], users[3]) + # we can estimate the number of unique EU anon visitors by dividing the + # number of anon page views by the average number of logged-in page + # views per logged-in visitor, then multiplying the result by the ratio + # of EU logged-in visitors to all logged-in visitors. + # in this case, the estimated number of EU anon visitors is 6 / (10 / 4) * (2 / 4) ~= 1 + # total EU visitors = EU logged-in visitors (2) + estimated EU anon visitors (1) = 3 + expect(eu_visitors[:"7_days"]).to eq(3) + + # anon page views: 11 + # logged-in page views: 14 + # logged-in visitors: 5 + # EU logged-in visitors: 3 (users[0], users[3], users[4]) + # we can estimate the number of unique EU anon visitors by dividing the + # number of anon page views by the average number of logged-in page + # views per logged-in visitor, then multiplying the result by the ratio + # of EU logged-in visitors to all logged-in visitors. + # in this case, the estimated number of EU anon visitors is 11 / (14 / 5) * (3 / 5) ~= 1 + # total EU visitors = EU logged-in visitors (3) + estimated EU anon visitors (2) = 5 + expect(eu_visitors[:"30_days"]).to eq(5) + end + end + + it "returns 0 for EU visitors when there are no logged-in users" do + freeze_time(date) do + UserVisit.delete_all + + eu_visitors = described_class.eu_visitors + expect(eu_visitors[:last_day]).to eq(0) + expect(eu_visitors[:"7_days"]).to eq(0) + expect(eu_visitors[:"30_days"]).to eq(0) + end + end + end end diff --git a/spec/system/about_page_spec.rb b/spec/system/about_page_spec.rb index fd1c17440be..02c92f0c7bf 100644 --- a/spec/system/about_page_spec.rb +++ b/spec/system/about_page_spec.rb @@ -127,6 +127,31 @@ describe "About page", type: :system do end end + describe "visitors" do + context "when the display_eu_visitor_stats setting is disabled" do + before { SiteSetting.display_eu_visitor_stats = false } + + it "doesn't show the row" do + about_page.visit + + expect(about_page.site_activities).to have_no_activity_item("visitors") + end + end + + context "when the display_eu_visitor_stats setting is enabled" do + before { SiteSetting.display_eu_visitor_stats = true } + + it "shows the row" do + about_page.visit + + expect(about_page.site_activities).to have_activity_item("visitors") + expect(about_page.site_activities.visitors).to have_text( + "1 visitor, about 0 from the EU", + ) + end + end + end + describe "active users" do before do User.update_all(last_seen_at: 1.month.ago) diff --git a/spec/system/page_objects/components/about_page_site_activity.rb b/spec/system/page_objects/components/about_page_site_activity.rb index 5e0a9a5d7d0..794aca0a25c 100644 --- a/spec/system/page_objects/components/about_page_site_activity.rb +++ b/spec/system/page_objects/components/about_page_site_activity.rb @@ -23,6 +23,13 @@ module PageObjects ) end + def visitors + AboutPageSiteActivityItem.new( + container.find(".about__activities-item.visitors"), + translation_key: nil, + ) + end + def active_users AboutPageSiteActivityItem.new( container.find(".about__activities-item.active-users"), @@ -51,6 +58,14 @@ module PageObjects translation_key:, ) end + + def has_activity_item?(name) + container.has_css?(".about__activities-item.#{name}") + end + + def has_no_activity_item?(name) + container.has_no_css?(".about__activities-item.#{name}") + end end end end diff --git a/spec/system/page_objects/components/about_page_site_activity_item.rb b/spec/system/page_objects/components/about_page_site_activity_item.rb index b0f9b07c644..7ae821fd304 100644 --- a/spec/system/page_objects/components/about_page_site_activity_item.rb +++ b/spec/system/page_objects/components/about_page_site_activity_item.rb @@ -16,6 +16,10 @@ module PageObjects ) end + def has_text?(text) + container.find(".about__activities-item-count").has_text?(text) + end + def has_1_day_period? period_element.has_text?(I18n.t("js.about.activities.periods.today")) end