FIX: Use MaxMind supplied permalinks to download MaxMind databases (#26847)

This commit switches `DiscourseIpInfo.mmdb_download` to use the
permalinks supplied by MaxMind to download the MaxMind databases as
specified in
https://dev.maxmind.com/geoip/updating-databases#directly-downloading-databases
which states:

```
To directly download databases, follow these steps:

1. In the "Download Links" column, click "Get Permalink(s)" for the desired database.
2. Copy the permalink(s) provided in the modal window.
3. Provide your account ID and your license key using Basic Authentication to authenticate.
```

Previously we are downloading from `https://download.maxmind.com/app/geoip_download` but this is not
documented anyway on MaxMind's docs so this URL can in theory break
in the future without warning. Therefore, we are taking a proactive
approach to download the databases from MaxMind the recommended way
instead of relying on a hidden URL. This old way of downloading the
databases with only a license key will be deprecated in 3.3 and be
removed in 3.4.

app/services/problem_check/maxmind_db_configuration.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+class ProblemCheck::MaxmindDbConfiguration < ProblemCheck
+  self.priority = "low"
+
+  def call
+    if GlobalSetting.maxmind_license_key.present? && GlobalSetting.maxmind_account_id.blank?
+      problem
+    else
+      no_problem
+    end
+  end
+
+  private
+
+  def translation_key
+    "dashboard.maxmind_db_configuration_warning"
+  end
+end

config/discourse_defaults.conf

@@ -294,7 +294,8 @@ refresh_maxmind_db_during_precompile_days = 2
 maxmind_backup_path =
 
 # register an account at: https://www.maxmind.com/en/geolite2/signup
-# then head to profile and get your license key
+# then head to profile and get your account ID and license key
+maxmind_account_id =
 maxmind_license_key =
 
 # Configures a URL mirror to download the MaxMind databases from.

config/locales/server.en.yml

@@ -1622,6 +1622,7 @@ en:
     sidekiq_warning: 'Sidekiq is not running. Many tasks, like sending emails, are executed asynchronously by Sidekiq. Please ensure at least one Sidekiq process is running. <a href="https://github.com/mperham/sidekiq" target="_blank">Learn about Sidekiq here</a>.'
     queue_size_warning: "The number of queued jobs is %{queue_size}, which is high. This could indicate a problem with the Sidekiq process(es), or you may need to add more Sidekiq workers."
     memory_warning: "Your server is running with less than 1 GB of total memory. At least 1 GB of memory is recommended."
+    maxmind_db_configuration_warning: 'The server has been configured to use MaxMind databases for reverse IP lookups but a valid MaxMind account ID has not been configured which may result in MaxMind databases failing to download in the future. <a href="https://meta.discourse.org/t/configure-maxmind-for-reverse-ip-lookups/173941" target="_blank">See this guide to learn more</a>.'
     google_oauth2_config_warning: 'The server is configured to allow signup and login with Google OAuth2 (enable_google_oauth2_logins), but the client id and client secret values are not set. Go to <a href="%{base_path}/admin/site_settings">the Site Settings</a> and update the settings. <a href="https://meta.discourse.org/t/configuring-google-login-for-discourse/15858" target="_blank">See this guide to learn more</a>.'
     facebook_config_warning: 'The server is configured to allow signup and login with Facebook (enable_facebook_logins), but the app id and app secret values are not set. Go to <a href="%{base_path}/admin/site_settings">the Site Settings</a> and update the settings. <a href="https://meta.discourse.org/t/configuring-facebook-login-for-discourse/13394" target="_blank">See this guide to learn more</a>.'
     twitter_config_warning: 'The server is configured to allow signup and login with Twitter (enable_twitter_logins), but the key and secret values are not set. Go to <a href="%{base_path}/admin/site_settings">the Site Settings</a> and update the settings. <a href="https://meta.discourse.org/t/configuring-twitter-login-for-discourse/13395" target="_blank">See this guide to learn more</a>.'

lib/discourse_ip_info.rb

@@ -25,17 +25,34 @@ class DiscourseIpInfo
   end
 
   def self.mmdb_download(name)
+    extra_headers = {}
+
     url =
       if GlobalSetting.maxmind_mirror_url.present?
         File.join(GlobalSetting.maxmind_mirror_url, "#{name}.tar.gz").to_s
       else
-        if GlobalSetting.maxmind_license_key.blank?
+        license_key = GlobalSetting.maxmind_license_key
-          STDERR.puts "MaxMind IP database updates require a license"
+
-          STDERR.puts "Please set DISCOURSE_MAXMIND_LICENSE_KEY to one you generated at https://www.maxmind.com"
+        if license_key.blank?
+          STDERR.puts "MaxMind IP database download requires an account ID and a license key"
+          STDERR.puts "Please set DISCOURSE_MAXMIND_ACCOUNT_ID and DISCOURSE_MAXMIND_LICENSE_KEY. See https://meta.discourse.org/t/configure-maxmind-for-reverse-ip-lookups/173941 for more details."
           return
         end
 
-        "https://download.maxmind.com/app/geoip_download?license_key=#{GlobalSetting.maxmind_license_key}&edition_id=#{name}&suffix=tar.gz"
+        account_id = GlobalSetting.maxmind_account_id
+
+        if account_id.present?
+          extra_headers[
+            "Authorization"
+          ] = "Basic #{Base64.strict_encode64("#{account_id}:#{license_key}")}"
+
+          "https://download.maxmind.com/geoip/databases/#{name}/download?suffix=tar.gz"
+        else
+          # This URL is not documented by MaxMind, but it works but we don't know when it will stop working. Therefore,
+          # we are deprecating this in 3.3 and will remove it in 3.4. An admin dashboard warning has been added to inform
+          # site admins about this deprecation. See `ProblemCheck::MaxmindDbConfiguration` for more information.
+          "https://download.maxmind.com/app/geoip_download?license_key=#{license_key}&edition_id=#{name}&suffix=tar.gz"
+        end
       end
 
     gz_file =
@@ -45,6 +62,7 @@ class DiscourseIpInfo
         tmp_file_name: "#{name}.gz",
         validate_uri: false,
         follow_redirect: true,
+        extra_headers:,
       )
 
     filename = File.basename(gz_file.path)

lib/discourse_ip_info_spec.rb

@@ -2,6 +2,29 @@
 
 RSpec.describe DiscourseIpInfo do
   describe ".mmdb_download" do
+    it "should download the MaxMind databases from MaxMind's download permalinks when `maxmind_license_key` and `maxmind_account_id` global setting has been set" do
+      global_setting :maxmind_license_key, "license_key"
+      global_setting :maxmind_account_id, "account_id"
+
+      stub_request(
+        :get,
+        "https://download.maxmind.com/geoip/databases/GeoLite2-City/download?suffix=tar.gz",
+      ).with(basic_auth: %w[account_id license_key]).to_return(status: 200, body: "", headers: {})
+
+      described_class.mmdb_download("GeoLite2-City")
+    end
+
+    it "should download the MaxMind databases from MaxMind's undocumented download URL when `maxmind_license_key` global setting has been set but not `maxmind_account_id` for backwards compatibility reasons" do
+      global_setting :maxmind_license_key, "license_key"
+
+      stub_request(
+        :get,
+        "https://download.maxmind.com/app/geoip_download?license_key=license_key&edition_id=GeoLite2-City&suffix=tar.gz",
+      ).to_return(status: 200, body: "", headers: {})
+
+      described_class.mmdb_download("GeoLite2-City")
+    end
+
     it "should download the MaxMind databases from the right URL when `maxmind_mirror_url` global setting has been configured" do
       global_setting :maxmind_mirror_url, "https://b.www.example.com/mirror"
 

lib/file_helper.rb

@@ -50,7 +50,8 @@ class FileHelper
     verbose: false,
     validate_uri: true,
     retain_on_max_file_size_exceeded: false,
-    include_port_in_host_header: false
+    include_port_in_host_header: false,
+    extra_headers: {}
   )
     url = "https:" + url if url.start_with?("//")
     raise Discourse::InvalidParameters.new(:url) unless url =~ %r{\Ahttps?://}
@@ -66,6 +67,7 @@ class FileHelper
         validate_uri: validate_uri,
         timeout: read_timeout,
         include_port_in_host_header: include_port_in_host_header,
+        headers: extra_headers,
       )
 
     fd.get do |response, chunk, uri|

spec/services/problem_check/maxmind_db_configuration_spec.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+RSpec.describe ProblemCheck::MaxmindDbConfiguration do
+  subject(:check) { described_class.new }
+
+  context "when `maxmind_license_key` and `maxmind_account_id` global settings are not set" do
+    it "should not raise any warning message" do
+      expect(check).to be_chill_about_it
+    end
+  end
+
+  context "when `maxmind_license_key` and `maxmind_account_id` global settings are set" do
+    it "should not raise any warning message" do
+      expect(check).to be_chill_about_it
+    end
+  end
+
+  context "when `maxmind_license_key` global setting is set but not `maxmind_account_id`" do
+    it "should raise the right warning" do
+      global_setting :maxmind_license_key, "license_key"
+
+      expect(check).to have_a_problem.with_priority("low").with_message(
+        I18n.t("dashboard.maxmind_db_configuration_warning"),
+      )
+    end
+  end
+end