FEATURE: Add custom S3 Endpoint and DigitalOcean Spaces/Minio support for Backups (#6045)

- Add custom S3 Endpoints and DigitalOcean Spaces support
- Add Minio support using 'force_path_style' option and fix uploads to custom endpoint

app/models/site_setting.rb

@@ -128,6 +128,14 @@ class SiteSetting < ActiveRecord::Base
       SiteSetting.enable_s3_uploads ? SiteSetting.s3_upload_bucket : GlobalSetting.s3_bucket
     end
 
+    def self.s3_endpoint
+      SiteSetting.enable_s3_uploads ? SiteSetting.s3_endpoint : GlobalSetting.s3_endpoint
+    end
+
+    def self.s3_force_path_style
+      SiteSetting.enable_s3_uploads ? SiteSetting.s3_force_path_style : GlobalSetting.s3_force_path_style
+    end
+
     def self.enable_s3_uploads
       SiteSetting.enable_s3_uploads || GlobalSetting.use_s3?
     end
@@ -138,15 +146,22 @@ class SiteSetting < ActiveRecord::Base
     end
 
     def self.absolute_base_url
+      url_basename = SiteSetting.s3_endpoint.split('/')[-1]
       bucket = SiteSetting.enable_s3_uploads ? Discourse.store.s3_bucket_name : GlobalSetting.s3_bucket_name
 
       # cf. http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region
-      if SiteSetting.Upload.s3_region == "us-east-1"
-        "//#{bucket}.s3.amazonaws.com"
-      elsif SiteSetting.Upload.s3_region == 'cn-north-1'
-        "//#{bucket}.s3.cn-north-1.amazonaws.com.cn"
+      if SiteSetting.s3_endpoint == "https://s3.amazonaws.com"
+        if SiteSetting.Upload.s3_region == "us-east-1"
+          "//#{bucket}.s3.amazonaws.com"
+        elsif SiteSetting.Upload.s3_region == 'cn-north-1'
+          "//#{bucket}.s3.cn-north-1.amazonaws.com.cn"
+        else
+          "//#{bucket}.s3-#{SiteSetting.Upload.s3_region}.amazonaws.com"
+        end
+      elsif SiteSetting.s3_force_path_style
+        "//#{url_basename}/#{bucket}"
       else
-        "//#{bucket}.s3-#{SiteSetting.Upload.s3_region}.amazonaws.com"
+        "//#{bucket}.#{url_basename}"
       end
     end
   end

config/discourse_defaults.conf

@@ -174,7 +174,8 @@ s3_access_key_id =
 s3_secret_access_key =
 s3_use_iam_profile = false
 s3_cdn_url =
-
+s3_endpoint =
+s3_force_path_style =
 
 ### rate limits apply to all sites
 max_user_api_reqs_per_minute = 20

config/locales/server.en.yml

@@ -1277,6 +1277,8 @@ en:
     backup_frequency: "The number of days between backups."
     enable_s3_backups: "Upload backups to S3 when complete. IMPORTANT: requires valid S3 credentials entered in Files settings."
     s3_backup_bucket: "The remote bucket to hold backups. WARNING: Make sure it is a private bucket."
+    s3_endpoint: "The endpoint can be modified to backup to an S3 compatible service like DigitalOcean Spaces or Minio. WARNING: Use default if using AWS S3"
+    s3_force_path_style: "Enforce path-style addressing for your custom endpoint. IMPORTANT: Required for using Minio uploads and backups."
     s3_disable_cleanup: "Disable the removal of backups from S3 when removed locally."
     backup_time_of_day: "Time of day UTC when the backup should occur."
     backup_with_uploads: "Include uploads in scheduled backups. Disabling this will only backup the database."

config/site_settings.yml

@@ -924,9 +924,14 @@ files:
   s3_upload_bucket:
     default: ''
     regex: '^[a-z0-9\-\/]+$' # can't use '.' when using HTTPS
+  s3_endpoint:
+    default: 'https://s3.amazonaws.com'
+    regex: '^https?:\/\/.+[^\/]$'
   s3_cdn_url:
     default: ''
     regex: '^https?:\/\/.+[^\/]$'
+  s3_force_path_style:
+    default: false
   allow_profile_backgrounds:
     client: true
     default: true

lib/s3_helper.rb

@@ -150,7 +150,9 @@ class S3Helper
   end
 
   def self.s3_options(obj)
-    opts = { region: obj.s3_region }
+    opts = { region: obj.s3_region,
+             endpoint: SiteSetting.s3_endpoint,
+             force_path_style: SiteSetting.s3_force_path_style }
 
     unless obj.s3_use_iam_profile
       opts[:access_key_id] = obj.s3_access_key_id