Discource-C
/
discourse
mirror of https://github.com/discourse/discourse.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

FEATURE: Add estimated number of global and EU visitors to the about page (#28382) 

This commit implements 2 new metrics/stats in the /about page for the _estimated_ numbers of unique visitors from the EU and the rest of the world. This new feature is currently off by default, but it can be enabled by turning on the hidden `display_eu_visitor_stats` site settings via the rails console.

There are a number of assumptions that we're making here in order to estimate the number of unique visitors, specifically:

1. we're assuming that the average of page views per anonymous visitor is similar to the average number of page views that a logged-in visitor makes, and
2. we're assuming that the ratio of logged in visitors from the EU is similar to the ratio of anonymous visitors from the EU

Discourse keeps track of the number of both logged-in and anonymous page views, and also the number of unique logged-in visitors and where they're from. So with those numbers and the assumptions above, we can estimate the number of unique anonymous visitors from the EU and the rest of the world.

Internal topic: t/128480.
This commit is contained in:
Osama Sayegh 2024-08-21 00:03:42 +03:00 committed by GitHub
parent 9802d4040f
commit 10ae7ef44a
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
14 changed files with 436 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
app/assets/javascripts/discourse/app/components/about-page.gjs
View File

@ -1,5 +1,6 @@
import Component from "@glimmer/component";
import { hash } from "@ember/helper";
import { service } from "@ember/service";
import { htmlSafe } from "@ember/template";
import AboutPageUsers from "discourse/components/about-page-users";
import PluginOutlet from "discourse/components/plugin-outlet";
@ -20,6 +21,8 @@ export function clearAboutPageActivities() {
}
export default class AboutPage extends Component {
@service siteSettings;
get moderatorsCount() {
return this.args.model.moderators.length;
}
@ -115,6 +118,20 @@ export default class AboutPage extends Component {
},
];
if (this.siteSettings.display_eu_visitor_stats) {
list.splice(2, 0, {
icon: "user-secret",
class: "visitors",
activityText: I18n.messageFormat("about.activities.visitors_MF", {
total_count: this.args.model.stats.visitors_7_days,
eu_count: this.args.model.stats.eu_visitors_7_days,
total_formatted_number: number(this.args.model.stats.visitors_7_days),
eu_formatted_number: number(this.args.model.stats.eu_visitors_7_days),
}),
period: I18n.t("about.activities.periods.last_7_days"),
});
}
return list.concat(this.siteActivitiesFromPlugins());
}

11
app/assets/javascripts/discourse/app/controllers/about.js
View File

@ -23,4 +23,15 @@ export default class AboutController extends Controller {
return null;
}
}
@discourseComputed(
"model.stats.visitors_30_days",
"model.stats.eu_visitors_30_days"
)
statsTableFooter(all, eu) {
return I18n.messageFormat("about.traffic_info_footer_MF", {
total_visitors: all,
eu_visitors: eu,
});
}
}

20
app/assets/javascripts/discourse/app/templates/about.hbs
View File

@ -139,6 +139,22 @@
<td>{{number this.model.stats.active_users_30_days}}</td>
<td>&mdash;</td>
</tr>
{{#if this.siteSettings.display_eu_visitor_stats}}
<tr class="about-visitor-count">
<td>{{i18n "about.visitor_count"}}</td>
<td>{{number this.model.stats.visitors_last_day}}</td>
<td>{{number this.model.stats.visitors_7_days}}</td>
<td>{{number this.model.stats.visitors_30_days}}</td>
<td>&mdash;</td>
</tr>
<tr class="about-eu-visitor-count">
<td>{{i18n "about.eu_visitor_count"}}</td>
<td>{{number this.model.stats.eu_visitors_last_day}}</td>
<td>{{number this.model.stats.eu_visitors_7_days}}</td>
<td>{{number this.model.stats.eu_visitors_30_days}}</td>
<td>&mdash;</td>
</tr>
{{/if}}
<tr class="about-like-count">
<td>{{i18n "about.like_count"}}</td>
<td>{{number this.model.stats.likes_last_day}}</td>
@ -170,6 +186,10 @@
{{/each}}
</tbody>
</table>
{{#if this.siteSettings.display_eu_visitor_stats}}
<p class="about stats-table-footer"><small
>{{this.statsTableFooter}}</small></p>
{{/if}}
</section>
{{/if}}

4
app/assets/stylesheets/common/base/about.scss
View File

@ -113,5 +113,9 @@ section.about {
}
}
}
.stats-table-footer {
max-width: 30em;
}
}
}

11
app/models/application_request.rb
View File

@ -59,6 +59,17 @@ class ApplicationRequest < ActiveRecord::Base
s
end
def self.request_type_count_for_period(type, since)
id = self.req_types[type]
if !id
raise ArgumentError.new(
"unknown request type #{type.inspect} in ApplicationRequest.req_types",
)
end
self.where(req_type: id).where("date >= ?", since).sum(:count)
end
end
# == Schema Information

15
app/models/stat.rb
View File

@ -40,7 +40,7 @@ class Stat
end
def self.core_stats
[
list = [
Stat.new("topics", show_in_ui: true, expose_via_api: true) { Statistics.topics },
Stat.new("posts", show_in_ui: true, expose_via_api: true) { Statistics.posts },
Stat.new("users", show_in_ui: true, expose_via_api: true) { Statistics.users },
@ -50,6 +50,19 @@ class Stat
Statistics.participating_users
end,
]
if SiteSetting.display_eu_visitor_stats
list.concat(
[
Stat.new("visitors", show_in_ui: true, expose_via_api: true) { Statistics.visitors },
Stat.new("eu_visitors", show_in_ui: true, expose_via_api: true) do
Statistics.eu_visitors
end,
],
)
end
list
end
def self._api_stats

18
config/locales/client.en.yml
View File

@ -342,6 +342,16 @@ en:
post_count: "Posts"
user_count: "Sign-ups"
active_user_count: "Active users"
visitor_count: "Visitors"
eu_visitor_count: "Visitors from European Union"
traffic_info_footer_MF: |
In the last 6 months, this site has served content to an average of approximately { total_visitors, plural,
one {# people}
other {# people}
} each month, with an average of approximately { eu_visitors, plural,
one {# people}
other {# people}
} from the European Union.
contact: "Contact us"
contact_info: "In the event of a critical issue or urgent matter affecting this site, please contact %{contact_info}."
site_activity: "Site activity"
@ -363,6 +373,14 @@ en:
likes:
one: "%{formatted_number} like"
other: "%{formatted_number} likes"
visitors_MF: |
{ total_count, plural,
one {{total_formatted_number} visitor}
other {{total_formatted_number} visitors}
}, about { eu_count, plural,
one {{eu_formatted_number}}
other {{eu_formatted_number}}
} from the EU
periods:
last_7_days: "in the last 7 days"
today: "today"

1
config/locales/server.en.yml
View File

@ -2209,6 +2209,7 @@ en:
tos_url: "If you have a Terms of Service document hosted elsewhere that you want to use, provide the full URL here."
privacy_policy_url: "If you have a Privacy Policy document hosted elsewhere that you want to use, provide the full URL here."
log_anonymizer_details: "Whether to keep a user's details in the log after being anonymized."
display_eu_visitor_stats: "Show number of global and EU visitors on the /about page."
newuser_spam_host_threshold: "How many times a new user can post a link to the same host within their `newuser_spam_host_threshold` posts before being considered spam."

4
config/site_settings.yml
View File

@ -2503,6 +2503,10 @@ legal:
default: false
hidden: true
client: true
display_eu_visitor_stats:
default: false
client: true
hidden: true
backups:
enable_backups:

105
lib/statistics.rb
View File

@ -1,6 +1,36 @@
# frozen_string_literal: true
class Statistics
EU_COUNTRIES = %w[
AT
BE
BG
CY
CZ
DE
DK
EE
ES
FI
FR
GR
HR
HU
IE
IT
LT
LU
LV
MT
NL
PL
PT
RO
SE
SI
SK
]
def self.active_users
{
last_day: User.where("last_seen_at > ?", 1.day.ago).count,
@ -56,6 +86,58 @@ class Statistics
}
end
def self.visitors
periods = [[1.day.ago, :last_day], [7.days.ago, :"7_days"], [30.days.ago, :"30_days"]]
periods
.map do |(period, key)|
anon_page_views =
ApplicationRequest.request_type_count_for_period(:page_view_anon_browser, period)
logged_in_visitors = logged_in_visitors_count(period)
next key, anon_page_views if logged_in_visitors == 0
logged_in_page_views =
ApplicationRequest.request_type_count_for_period(:page_view_logged_in_browser, period)
next key, anon_page_views + logged_in_visitors if logged_in_page_views == 0
total_visitors = logged_in_visitors
avg_logged_in_page_view_per_user = logged_in_page_views.to_f / logged_in_visitors
anon_visitors = (anon_page_views / avg_logged_in_page_view_per_user).round
total_visitors += anon_visitors
[key, total_visitors]
end
.to_h
end
def self.eu_visitors
periods = [[1.day.ago, :last_day], [7.days.ago, :"7_days"], [30.days.ago, :"30_days"]]
periods
.map do |(period, key)|
logged_in_page_views =
ApplicationRequest.request_type_count_for_period(:page_view_logged_in_browser, period)
anon_page_views =
ApplicationRequest.request_type_count_for_period(:page_view_anon_browser, period)
all_logged_in_visitors = logged_in_visitors_count(period)
eu_logged_in_visitors = eu_logged_in_visitors_count(period)
next key, 0 if all_logged_in_visitors == 0 || eu_logged_in_visitors == 0
next key, eu_logged_in_visitors if logged_in_page_views == 0
avg_logged_in_page_view_per_user = logged_in_page_views / all_logged_in_visitors.to_f
eu_logged_in_visitors_ratio = eu_logged_in_visitors / all_logged_in_visitors.to_f
eu_anon_visitors =
((anon_page_views / avg_logged_in_page_view_per_user) * eu_logged_in_visitors_ratio).round
eu_visitors = eu_logged_in_visitors + eu_anon_visitors
[key, eu_visitors]
end
.to_h
end
private
def self.participating_users_count(date)
@ -75,4 +157,27 @@ class Statistics
DB.query_single(sql, date: date, action_types: UserAction::USER_ACTED_TYPES).first
end
def self.logged_in_visitors_count(since)
DB.query_single(<<~SQL, since:).first
SELECT COUNT(DISTINCT user_id)
FROM user_visits
WHERE visited_at >= :since
SQL
end
def self.eu_logged_in_visitors_count(since)
results = DB.query_hash(<<~SQL, since:)
SELECT DISTINCT(user_id), ip_address
FROM user_visits uv
INNER JOIN users u
ON u.id = uv.user_id
WHERE visited_at >= :since AND ip_address IS NOT NULL
SQL
results.reduce(0) do |sum, hash|
ip_info = DiscourseIpInfo.get(hash["ip_address"].to_s)
sum + (EU_COUNTRIES.include?(ip_info[:country_code]) ? 1 : 0)
end
end
end

188
spec/lib/statistics_spec.rb
View File

@ -1,7 +1,59 @@
# frozen_string_literal: true
RSpec.describe Statistics do
describe "#participating_users" do
def create_page_views_and_user_visit_records(date, users)
freeze_time(date - 50.minutes) do
2.times { ApplicationRequest.increment!(:page_view_anon_browser) }
ApplicationRequest.increment!(:page_view_logged_in_browser)
end
freeze_time(date - 3.days) do
ApplicationRequest.increment!(:page_view_anon_browser)
5.times { ApplicationRequest.increment!(:page_view_logged_in_browser) }
end
freeze_time(date - 6.days) do
3.times { ApplicationRequest.increment!(:page_view_anon_browser) }
4.times { ApplicationRequest.increment!(:page_view_logged_in_browser) }
end
freeze_time(date - 8.days) do
ApplicationRequest.increment!(:page_view_anon_browser)
ApplicationRequest.increment!(:page_view_logged_in_browser)
end
freeze_time(date - 15.days) do
4.times { ApplicationRequest.increment!(:page_view_anon_browser) }
3.times { ApplicationRequest.increment!(:page_view_logged_in_browser) }
end
freeze_time(date - 31.days) do
ApplicationRequest.increment!(:page_view_anon_browser)
ApplicationRequest.increment!(:page_view_logged_in_browser)
end
UserVisit.create!(user_id: users[0].id, visited_at: date - 50.minute)
UserVisit.create!(user_id: users[0].id, visited_at: date - 36.hours)
UserVisit.create!(user_id: users[1].id, visited_at: date - 2.day)
UserVisit.create!(user_id: users[0].id, visited_at: date - 4.days)
UserVisit.create!(user_id: users[2].id, visited_at: date - 6.days)
UserVisit.create!(user_id: users[3].id, visited_at: date - 3.days)
UserVisit.create!(user_id: users[3].id, visited_at: date - 5.days)
UserVisit.create!(user_id: users[1].id, visited_at: date - 66.hours)
UserVisit.create!(user_id: users[2].id, visited_at: date - 8.days)
UserVisit.create!(user_id: users[3].id, visited_at: date - 13.days)
UserVisit.create!(user_id: users[0].id, visited_at: date - 24.days)
UserVisit.create!(user_id: users[4].id, visited_at: date - 19.days)
UserVisit.create!(user_id: users[2].id, visited_at: date - 31.days)
end
fab!(:users) { Fabricate.times(5, :user) }
let(:date) { DateTime.parse("2024-03-01 13:00") }
describe ".participating_users" do
it "returns no participating users by default" do
pu = described_class.participating_users
expect(pu[:last_day]).to eq(0)
@ -29,4 +81,138 @@ RSpec.describe Statistics do
expect(described_class.participating_users[:last_day]).to eq(1)
end
end
describe ".visitors" do
before do
ApplicationRequest.enable
create_page_views_and_user_visit_records(date, users)
end
after { ApplicationRequest.disable }
it "estimates the number of visitors for each of the previous 1 day, 7 days and 30 days periods" do
freeze_time(date) do
visitors = described_class.visitors
# anon page views: 2
# logged-in page views: 1
# logged-in visitors: 1
# we can estimate the number of unique anon visitors by dividing the
# number of anon page views by the average number of logged-in page
# views per logged-in visitor.
# in this case, the estimated number of anon visitors is 2 / (1 / 1) = 2.
# total visitors = logged-in visitors (1) + estimated anon visitors (2) = 3
expect(visitors[:last_day]).to eq(3)
# anon page views: 6
# logged-in page views: 10
# logged-in visitors: 4
# we can estimate the number of unique anon visitors by dividing the
# number of anon page views by the average number of logged-in page
# views per logged-in visitor.
# in this case, the estimated number of anon visitors is 6 / (10 / 4) ~= 2.
# total visitors = logged-in visitors (4) + estimated anon visitors (2) = 6
expect(visitors[:"7_days"]).to eq(6)
# anon page views: 11
# logged-in page views: 14
# logged-in visitors: 5
# we can estimate the number of unique anon visitors by dividing the
# number of anon page views by the average number of logged-in page
# views per logged-in visitor.
# in this case, the estimated number of anon visitors is 11 / (14 / 5) ~= 4.
# total visitors = logged-in visitors (5) + estimated anon visitors (4) = 9
expect(visitors[:"30_days"]).to eq(9)
end
end
it "is the same as the number of anon page views when there are no logged in visitors" do
freeze_time(date) do
UserVisit.delete_all
visitors = described_class.visitors
expect(visitors[:last_day]).to eq(2)
expect(visitors[:"7_days"]).to eq(6)
expect(visitors[:"30_days"]).to eq(11)
end
end
end
describe ".eu_visitors" do
before do
ApplicationRequest.enable
create_page_views_and_user_visit_records(date, users)
users[0].update!(ip_address: IPAddr.new("60.23.1.42"))
users[1].update!(ip_address: IPAddr.new("90.19.255.63"))
users[2].update!(ip_address: IPAddr.new("8.33.134.244"))
users[3].update!(ip_address: IPAddr.new("2.74.0.98"))
users[4].update!(ip_address: IPAddr.new("88.82.3.101"))
# EU IP addresses
DiscourseIpInfo.stubs(:get).with("60.23.1.42").returns({ country_code: "FR" }) # users[0]
DiscourseIpInfo.stubs(:get).with("2.74.0.98").returns({ country_code: "NL" }) # users[3]
DiscourseIpInfo.stubs(:get).with("88.82.3.101").returns({ country_code: "DE" }) # users[4]
# non-EU IP addresses
DiscourseIpInfo.stubs(:get).with("90.19.255.63").returns({ country_code: "US" }) # users[1]
DiscourseIpInfo.stubs(:get).with("8.33.134.244").returns({ country_code: "SA" }) # users[2]
end
after { ApplicationRequest.disable }
it "estimates the number of EU visitors for each of the previous 1 day, 7 days and 30 days periods" do
freeze_time(date) do
eu_visitors = described_class.eu_visitors
# anon page views: 2
# logged-in page views: 1
# logged-in visitors: 1
# EU logged-in visitors: 1 (users[0])
# we can estimate the number of unique EU anon visitors by dividing the
# number of anon page views by the average number of logged-in page
# views per logged-in visitor, then multiplying the result by the ratio
# of EU logged-in visitors to all logged-in visitors.
# in this case, the estimated number of EU anon visitors is 2 / (1 / 1) * (1 / 1) = 2
# total EU visitors = EU logged-in visitors (1) + estimated EU anon visitors (2) = 3
expect(eu_visitors[:last_day]).to eq(3)
# anon page views: 6
# logged-in page views: 10
# logged-in visitors: 4
# EU logged-in visitors: 2 (users[0], users[3])
# we can estimate the number of unique EU anon visitors by dividing the
# number of anon page views by the average number of logged-in page
# views per logged-in visitor, then multiplying the result by the ratio
# of EU logged-in visitors to all logged-in visitors.
# in this case, the estimated number of EU anon visitors is 6 / (10 / 4) * (2 / 4) ~= 1
# total EU visitors = EU logged-in visitors (2) + estimated EU anon visitors (1) = 3
expect(eu_visitors[:"7_days"]).to eq(3)
# anon page views: 11
# logged-in page views: 14
# logged-in visitors: 5
# EU logged-in visitors: 3 (users[0], users[3], users[4])
# we can estimate the number of unique EU anon visitors by dividing the
# number of anon page views by the average number of logged-in page
# views per logged-in visitor, then multiplying the result by the ratio
# of EU logged-in visitors to all logged-in visitors.
# in this case, the estimated number of EU anon visitors is 11 / (14 / 5) * (3 / 5) ~= 1
# total EU visitors = EU logged-in visitors (3) + estimated EU anon visitors (2) = 5
expect(eu_visitors[:"30_days"]).to eq(5)
end
end
it "returns 0 for EU visitors when there are no logged-in users" do
freeze_time(date) do
UserVisit.delete_all
eu_visitors = described_class.eu_visitors
expect(eu_visitors[:last_day]).to eq(0)
expect(eu_visitors[:"7_days"]).to eq(0)
expect(eu_visitors[:"30_days"]).to eq(0)
end
end
end
end

25
spec/system/about_page_spec.rb
View File

@ -127,6 +127,31 @@ describe "About page", type: :system do
end
end
describe "visitors" do
context "when the display_eu_visitor_stats setting is disabled" do
before { SiteSetting.display_eu_visitor_stats = false }
it "doesn't show the row" do
about_page.visit
expect(about_page.site_activities).to have_no_activity_item("visitors")
end
end
context "when the display_eu_visitor_stats setting is enabled" do
before { SiteSetting.display_eu_visitor_stats = true }
it "shows the row" do
about_page.visit
expect(about_page.site_activities).to have_activity_item("visitors")
expect(about_page.site_activities.visitors).to have_text(
"1 visitor, about 0 from the EU",
)
end
end
end
describe "active users" do
before do
User.update_all(last_seen_at: 1.month.ago)

15
spec/system/page_objects/components/about_page_site_activity.rb
View File

@ -23,6 +23,13 @@ module PageObjects
)
end
def visitors
AboutPageSiteActivityItem.new(
container.find(".about__activities-item.visitors"),
translation_key: nil,
)
end
def active_users
AboutPageSiteActivityItem.new(
container.find(".about__activities-item.active-users"),
@ -51,6 +58,14 @@ module PageObjects
translation_key:,
)
end
def has_activity_item?(name)
container.has_css?(".about__activities-item.#{name}")
end
def has_no_activity_item?(name)
container.has_no_css?(".about__activities-item.#{name}")
end
end
end
end

4
spec/system/page_objects/components/about_page_site_activity_item.rb
View File

@ -16,6 +16,10 @@ module PageObjects
)
end
def has_text?(text)
container.find(".about__activities-item-count").has_text?(text)
end
def has_1_day_period?
period_element.has_text?(I18n.t("js.about.activities.periods.today"))
end