PERF: remove "fog" dependency
This commit is contained in:
Sam
parent
ff55bcb4d0
commit
f079dd63ed
3
Gemfile
3
Gemfile
|
|
@ -57,7 +57,8 @@ gem 'fast_xor'
|
|||
|
||||
# while we sort out https://github.com/sdsykes/fastimage/pull/46
|
||||
gem 'fastimage_discourse', require: 'fastimage'
|
||||
gem 'fog', '1.26.0', require: false
|
||||
gem 'aws-sdk', require: false
|
||||
gem 'excon', require: false
|
||||
gem 'unf', require: false
|
||||
|
||||
gem 'email_reply_parser'
|
||||
|
|
|
|||
89
Gemfile.lock
89
Gemfile.lock
|
|
@ -6,7 +6,6 @@ PATH
|
|||
GEM
|
||||
remote: https://rubygems.org/
|
||||
specs:
|
||||
CFPropertyList (2.2.8)
|
||||
actionmailer (4.1.10)
|
||||
actionpack (= 4.1.10)
|
||||
actionview (= 4.1.10)
|
||||
|
|
@ -41,6 +40,14 @@ GEM
|
|||
activerecord (>= 2.3.0)
|
||||
rake (~> 10.4.2, >= 10.4.2)
|
||||
arel (5.0.1.20140414130214)
|
||||
aws-sdk (2.0.45)
|
||||
aws-sdk-resources (= 2.0.45)
|
||||
aws-sdk-core (2.0.45)
|
||||
builder (~> 3.0)
|
||||
jmespath (~> 1.0)
|
||||
multi_json (~> 1.0)
|
||||
aws-sdk-resources (2.0.45)
|
||||
aws-sdk-core (= 2.0.45)
|
||||
babel-source (4.6.6)
|
||||
babel-transpiler (0.6.0)
|
||||
babel-source (>= 4.0, < 5)
|
||||
|
|
@ -84,7 +91,7 @@ GEM
|
|||
ember-source (1.11.3.1)
|
||||
erubis (2.7.0)
|
||||
eventmachine (1.0.7)
|
||||
excon (0.44.4)
|
||||
excon (0.45.3)
|
||||
execjs (2.5.2)
|
||||
exifr (1.1.3)
|
||||
fabrication (2.9.8)
|
||||
|
|
@ -101,79 +108,11 @@ GEM
|
|||
fast_xs (0.8.0)
|
||||
fastimage_discourse (1.6.6)
|
||||
ffi (1.9.6)
|
||||
fission (0.5.0)
|
||||
CFPropertyList (~> 2.2)
|
||||
flamegraph (0.1.0)
|
||||
fast_stack
|
||||
fog (1.26.0)
|
||||
fog-atmos
|
||||
fog-brightbox (~> 0.4)
|
||||
fog-core (~> 1.27, >= 1.27.1)
|
||||
fog-ecloud
|
||||
fog-json
|
||||
fog-profitbricks
|
||||
fog-radosgw (>= 0.0.2)
|
||||
fog-sakuracloud (>= 0.0.4)
|
||||
fog-softlayer
|
||||
fog-storm_on_demand
|
||||
fog-terremark
|
||||
fog-vmfusion
|
||||
fog-voxel
|
||||
fog-xml (~> 0.1.1)
|
||||
ipaddress (~> 0.5)
|
||||
nokogiri (~> 1.5, >= 1.5.11)
|
||||
fog-atmos (0.1.0)
|
||||
fog-core
|
||||
fog-xml
|
||||
fog-brightbox (0.7.1)
|
||||
fog-core (~> 1.22)
|
||||
fog-json
|
||||
inflecto (~> 0.0.2)
|
||||
fog-core (1.27.2)
|
||||
builder
|
||||
excon (~> 0.38)
|
||||
formatador (~> 0.2)
|
||||
mime-types
|
||||
net-scp (~> 1.1)
|
||||
net-ssh (>= 2.1.3)
|
||||
fog-ecloud (0.0.2)
|
||||
fog-core
|
||||
fog-xml
|
||||
fog-json (1.0.0)
|
||||
multi_json (~> 1.0)
|
||||
fog-profitbricks (0.0.1)
|
||||
fog-core
|
||||
fog-xml
|
||||
nokogiri
|
||||
fog-radosgw (0.0.3)
|
||||
fog-core (>= 1.21.0)
|
||||
fog-json
|
||||
fog-xml (>= 0.0.1)
|
||||
fog-sakuracloud (0.1.1)
|
||||
fog-core
|
||||
fog-json
|
||||
fog-softlayer (0.3.26)
|
||||
fog-core
|
||||
fog-json
|
||||
fog-storm_on_demand (0.1.0)
|
||||
fog-core
|
||||
fog-json
|
||||
fog-terremark (0.0.3)
|
||||
fog-core
|
||||
fog-xml
|
||||
fog-vmfusion (0.0.1)
|
||||
fission
|
||||
fog-core
|
||||
fog-voxel (0.0.2)
|
||||
fog-core
|
||||
fog-xml
|
||||
fog-xml (0.1.1)
|
||||
fog-core
|
||||
nokogiri (~> 1.5, >= 1.5.11)
|
||||
foreman (0.77.0)
|
||||
dotenv (~> 1.0.2)
|
||||
thor (~> 0.19.1)
|
||||
formatador (0.2.5)
|
||||
fspath (2.0.6)
|
||||
gctools (0.2.3)
|
||||
given_core (3.5.4)
|
||||
|
|
@ -195,8 +134,8 @@ GEM
|
|||
progress (~> 3.0.0)
|
||||
image_size (1.1.5)
|
||||
in_threads (1.2.2)
|
||||
inflecto (0.0.2)
|
||||
ipaddress (0.8.0)
|
||||
jmespath (1.0.2)
|
||||
multi_json (~> 1.0)
|
||||
jquery-rails (3.1.2)
|
||||
railties (>= 3.0, < 5.0)
|
||||
thor (>= 0.14, < 2.0)
|
||||
|
|
@ -231,9 +170,6 @@ GEM
|
|||
multi_xml (0.5.5)
|
||||
multipart-post (2.0.0)
|
||||
mustache (0.99.8)
|
||||
net-scp (1.2.1)
|
||||
net-ssh (>= 2.6.5)
|
||||
net-ssh (2.9.2)
|
||||
netrc (0.10.3)
|
||||
nokogiri (1.6.6.2)
|
||||
mini_portile (~> 0.6.0)
|
||||
|
|
@ -459,6 +395,7 @@ DEPENDENCIES
|
|||
actionpack-action_caching
|
||||
active_model_serializers (~> 0.8.3)
|
||||
annotate
|
||||
aws-sdk
|
||||
babel-transpiler
|
||||
barber
|
||||
better_errors
|
||||
|
|
@ -468,6 +405,7 @@ DEPENDENCIES
|
|||
email_reply_parser
|
||||
ember-rails
|
||||
ember-source (= 1.11.3.1)
|
||||
excon
|
||||
fabrication (= 2.9.8)
|
||||
fakeweb (~> 1.3.0)
|
||||
fast_blank
|
||||
|
|
@ -475,7 +413,6 @@ DEPENDENCIES
|
|||
fast_xs
|
||||
fastimage_discourse
|
||||
flamegraph
|
||||
fog (= 1.26.0)
|
||||
foreman
|
||||
gctools
|
||||
handlebars-source (= 2.0.0)
|
||||
|
|
|
|||
|
|
@ -80,7 +80,7 @@ module FileStore
|
|||
|
||||
def store_file(file, path, filename=nil, content_type=nil)
|
||||
# stored uploaded are public by default
|
||||
options = { public: true }
|
||||
options = { acl: 'public-read' }
|
||||
# add a "content disposition" header for "attachments"
|
||||
options[:content_disposition] = "attachment; filename=\"#{filename}\"" if filename && !FileHelper.is_image?(filename)
|
||||
# add a "content type" header when provided (ie. for "attachments")
|
||||
|
|
|
|||
111
lib/s3_helper.rb
111
lib/s3_helper.rb
|
|
@ -1,50 +1,76 @@
|
|||
require "fog"
|
||||
require "aws-sdk"
|
||||
|
||||
class S3Helper
|
||||
|
||||
def initialize(s3_bucket, tombstone_prefix=nil, fog=nil)
|
||||
def initialize(s3_bucket, tombstone_prefix=nil)
|
||||
raise Discourse::InvalidParameters.new("s3_bucket") if s3_bucket.blank?
|
||||
|
||||
@s3_bucket = s3_bucket
|
||||
@tombstone_prefix = tombstone_prefix
|
||||
|
||||
check_missing_site_settings
|
||||
|
||||
@fog = fog || Fog::Storage.new(s3_options)
|
||||
end
|
||||
|
||||
def upload(file, unique_filename, options={})
|
||||
args = {
|
||||
body: file,
|
||||
key: unique_filename,
|
||||
public: false,
|
||||
}
|
||||
|
||||
args.merge!(options)
|
||||
|
||||
directory = get_or_create_directory(@s3_bucket)
|
||||
directory.files.create(args)
|
||||
obj = s3_bucket.object(unique_filename)
|
||||
obj.upload_file(file, options)
|
||||
end
|
||||
|
||||
def remove(unique_filename, copy_to_tombstone=false)
|
||||
bucket = s3_bucket
|
||||
|
||||
# copy the file in tombstone
|
||||
if copy_to_tombstone && @tombstone_prefix.present?
|
||||
@fog.copy_object(unique_filename, @s3_bucket, @tombstone_prefix + unique_filename, @s3_bucket)
|
||||
bucket.object(@tombstone_prefix + unique_filename).copy_from(copy_source: "#{@s3_bucket}/#{unique_filename}")
|
||||
end
|
||||
# delete the file
|
||||
@fog.delete_object(@s3_bucket, unique_filename)
|
||||
rescue Excon::Errors::NotFound
|
||||
# if the file cannot be found, don't raise an error
|
||||
bucket.object(unique_filename).delete
|
||||
rescue Aws::S3::Errors::NoSuchKey
|
||||
end
|
||||
|
||||
def update_tombstone_lifecycle(grace_period)
|
||||
|
||||
return if @tombstone_prefix.blank?
|
||||
# cf. http://docs.aws.amazon.com/AmazonS3/latest/dev/object-lifecycle-mgmt.html
|
||||
@fog.put_bucket_lifecycle(@s3_bucket, lifecycle(grace_period))
|
||||
s3_resource.client.put_bucket_lifecycle({
|
||||
bucket: @s3_bucket,
|
||||
lifecycle_configuration: {
|
||||
rules: [
|
||||
{
|
||||
id: 'purge-tombstone',
|
||||
status: 'Enabled',
|
||||
expiration: {
|
||||
days: grace_period
|
||||
},
|
||||
prefix: @tombstone_prefix
|
||||
}
|
||||
]
|
||||
}
|
||||
})
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def s3_resource
|
||||
opts = {}
|
||||
|
||||
opts = {
|
||||
access_key_id: SiteSetting.s3_access_key_id,
|
||||
secret_access_key: SiteSetting.s3_secret_access_key
|
||||
} unless SiteSetting.s3_use_iam_profile
|
||||
|
||||
opts[:region] = SiteSetting.s3_region unless SiteSetting.s3_region.blank?
|
||||
|
||||
Aws::S3::Resource.new(opts)
|
||||
end
|
||||
|
||||
def s3_bucket
|
||||
bucket = s3_resource.bucket(@s3_bucket)
|
||||
bucket.create unless bucket.exists?
|
||||
bucket
|
||||
end
|
||||
|
||||
|
||||
def check_missing_site_settings
|
||||
unless SiteSetting.s3_use_iam_profile
|
||||
raise Discourse::SiteSettingMissing.new("s3_access_key_id") if SiteSetting.s3_access_key_id.blank?
|
||||
|
|
@ -52,51 +78,4 @@ class S3Helper
|
|||
end
|
||||
end
|
||||
|
||||
def s3_options
|
||||
options = { provider: 'AWS', scheme: SiteSetting.scheme }
|
||||
|
||||
# cf. https://github.com/fog/fog/issues/2381
|
||||
options[:path_style] = dns_compatible?(@s3_bucket, SiteSetting.use_https?)
|
||||
|
||||
options[:region] = SiteSetting.s3_region unless SiteSetting.s3_region.blank?
|
||||
|
||||
if SiteSetting.s3_use_iam_profile
|
||||
options.merge!(use_iam_profile: true)
|
||||
else
|
||||
options.merge!(aws_access_key_id: SiteSetting.s3_access_key_id,
|
||||
aws_secret_access_key: SiteSetting.s3_secret_access_key)
|
||||
end
|
||||
|
||||
options
|
||||
end
|
||||
|
||||
def get_or_create_directory(bucket)
|
||||
directory = @fog.directories.get(bucket)
|
||||
directory = @fog.directories.create(key: bucket) unless directory
|
||||
directory
|
||||
end
|
||||
|
||||
def lifecycle(grace_period)
|
||||
{
|
||||
"Rules" => [{
|
||||
"Prefix" => @tombstone_prefix,
|
||||
"Enabled" => true,
|
||||
"Expiration" => { "Days" => grace_period }
|
||||
}]
|
||||
}
|
||||
end
|
||||
|
||||
# cf. https://github.com/aws/aws-sdk-core-ruby/blob/master/aws-sdk-core/lib/aws-sdk-core/plugins/s3_bucket_dns.rb#L65-L80
|
||||
def dns_compatible?(bucket_name, ssl)
|
||||
return false unless valid_subdomain?(bucket_name)
|
||||
bucket_name.match(/\./) && ssl ? false : true
|
||||
end
|
||||
|
||||
def valid_subdomain?(bucket_name)
|
||||
bucket_name.size < 64 &&
|
||||
bucket_name =~ /^[a-z0-9][a-z0-9.-]+[a-z0-9]$/ &&
|
||||
bucket_name !~ /(\d+\.){3}\d+/ &&
|
||||
bucket_name !~ /[.-]{2}/
|
||||
end
|
||||
|
||||
end
|
||||
|
|
|
|||
Loading…
Reference in New Issue