Discource-C
/
discourse
mirror of https://github.com/discourse/discourse.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1275 from ZogStriP/enable-thumbnailing-on-s3 

Enable thumbnailing on s3
This commit is contained in:
Robin Ward 2013-08-01 07:35:35 -07:00
parent ba1472d99c ed9417fa3b
commit 4f0713b9da
25 changed files with 528 additions and 505 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
Gemfile.lock
View File

@ -168,7 +168,7 @@ GEM
handlebars-source (= 1.0.12)
erubis (2.7.0)
eventmachine (1.0.3)
excon (0.20.1)
excon (0.25.3)
execjs (1.4.0)
multi_json (~> 1.0)
fabrication (2.6.5)
@ -181,15 +181,15 @@ GEM
fast_xs (0.8.0)
fastimage (1.3.0)
ffi (1.8.1)
fog (1.10.1)
fog (1.14.0)
builder
excon (~> 0.20)
excon (~> 0.25.0)
formatador (~> 0.2.0)
mime-types
multi_json (~> 1.0)
net-scp (~> 1.1)
net-ssh (>= 2.1.3)
nokogiri (~> 1.5.0)
nokogiri (~> 1.5)
ruby-hmac
formatador (0.2.4)
fspath (2.0.4)
@ -247,9 +247,9 @@ GEM
multi_json (1.7.7)
multipart-post (1.2.0)
mustache (0.99.4)
net-scp (1.1.0)
net-scp (1.1.2)
net-ssh (>= 2.6.5)
net-ssh (2.6.7)
net-ssh (2.6.8)
nokogiri (1.5.9)
oauth (0.4.7)
oauth2 (0.8.1)

54
app/models/optimized_image.rb
View File

@ -8,62 +8,46 @@ class OptimizedImage < ActiveRecord::Base
@image_sorcery_loaded ||= require "image_sorcery"
original_path = "#{Rails.root}/public#{upload.url}"
external_copy = Discourse.store.download(upload) if Discourse.store.external?
original_path = if Discourse.store.external?
external_copy.path
else
Discourse.store.path_for(upload)
end
# create a temp file with the same extension as the original
temp_file = Tempfile.new(["discourse", File.extname(original_path)])
temp_file = Tempfile.new(["discourse-thumbnail", File.extname(original_path)])
temp_path = temp_file.path
if ImageSorcery.new(original_path).convert(temp_path, resize: "#{width}x#{height}")
thumbnail = OptimizedImage.new({
thumbnail = OptimizedImage.create!(
upload_id: upload.id,
sha1: Digest::SHA1.file(temp_path).hexdigest,
extension: File.extname(temp_path),
width: width,
height: height
})
# make sure the directory exists
FileUtils.mkdir_p Pathname.new(thumbnail.path).dirname
# move the temp file to the right location
File.open(thumbnail.path, "wb") do |f|
f.write temp_file.read
end
height: height,
url: "",
)
# store the optimized image and update its url
thumbnail.url = Discourse.store.store_optimized_image(temp_file, thumbnail)
thumbnail.save
end
# close && remove temp file
temp_file.close
temp_file.unlink
temp_file.close!
# make sure we remove the cached copy from external stores
external_copy.close! if Discourse.store.external?
thumbnail
end
def destroy
OptimizedImage.transaction do
remove_file
Discourse.store.remove_file(url)
super
end
end
def remove_file
File.delete path
rescue Errno::ENOENT
end
def url
"#{LocalStore.base_url}/#{optimized_path}/#{filename}"
end
def path
"#{LocalStore.base_path}/#{optimized_path}/#{filename}"
end
def optimized_path
"_optimized/#{sha1[0..2]}/#{sha1[3..5]}"
end
def filename
"#{sha1[6..16]}_#{width}x#{height}#{extension}"
end
end
# == Schema Information

2
app/models/post.rb
View File

@ -1,7 +1,5 @@
require_dependency 'jobs'
require_dependency 'pretty_text'
require_dependency 'local_store'
require_dependency 's3_store'
require_dependency 'rate_limiter'
require_dependency 'post_revisor'
require_dependency 'enum'

10
app/models/post_analyzer.rb
View File

@ -35,13 +35,9 @@ class PostAnalyzer
# How many attachments are present in the post
def attachment_count
return 0 unless @raw.present?
if SiteSetting.enable_s3_uploads?
cooked_document.css("a.attachment[href^=\"#{S3Store.base_url}\"]")
else
cooked_document.css("a.attachment[href^=\"#{LocalStore.directory}\"]") +
cooked_document.css("a.attachment[href^=\"#{LocalStore.base_url}\"]")
end.count
attachments = cooked_document.css("a.attachment[href^=\"#{Discourse.store.absolute_base_url}\"]")
attachments += cooked_document.css("a.attachment[href^=\"#{Discourse.store.relative_base_url}\"]") if Discourse.store.internal?
attachments.count
end
def raw_mentions

4
app/models/topic_link.rb
View File

@ -103,8 +103,8 @@ class TopicLink < ActiveRecord::Base
topic_id = nil
post_number = nil
if Upload.has_been_uploaded?(url)
internal = !Upload.is_on_s3?(url)
if Discourse.store.has_been_uploaded?(url)
internal = Discourse.store.internal?
elsif parsed.host == Discourse.current_hostname || !parsed.host
internal = true

46
app/models/upload.rb
View File

@ -2,8 +2,6 @@ require 'digest/sha1'
require 'image_sizer'
require 'tempfile'
require 'pathname'
require_dependency 's3_store'
require_dependency 'local_store'
class Upload < ActiveRecord::Base
belongs_to :user
@ -20,17 +18,12 @@ class Upload < ActiveRecord::Base
optimized_images.where(width: width, height: height).first
end
def thumbnail_url
thumbnail.url if has_thumbnail?
end
def has_thumbnail?
thumbnail.present?
end
def create_thumbnail!
return unless SiteSetting.create_thumbnails?
return if SiteSetting.enable_s3_uploads?
return if has_thumbnail?
thumbnail = OptimizedImage.create_for(self, width, height)
optimized_images << thumbnail if thumbnail
@ -38,7 +31,7 @@ class Upload < ActiveRecord::Base
def destroy
Upload.transaction do
Upload.remove_file url
Discourse.store.remove_file(url)
super
end
end
@ -58,7 +51,7 @@ class Upload < ActiveRecord::Base
file.rewind
end
# create a db record (so we can use the id)
upload = Upload.create!({
upload = Upload.create!(
user_id: user_id,
original_filename: file.original_filename,
filesize: filesize,
@ -66,9 +59,9 @@ class Upload < ActiveRecord::Base
url: "",
width: width,
height: height,
})
)
# store the file and update its url
upload.url = Upload.store_file(file, sha1, upload.id)
upload.url = Discourse.store.store_upload(file, upload)
# save the url
upload.save
end
@ -76,36 +69,11 @@ class Upload < ActiveRecord::Base
upload
end
def self.store_file(file, sha1, upload_id)
return S3Store.store_file(file, sha1, upload_id) if SiteSetting.enable_s3_uploads?
return LocalStore.store_file(file, sha1, upload_id)
end
def self.remove_file(url)
return S3Store.remove_file(url) if SiteSetting.enable_s3_uploads?
return LocalStore.remove_file(url)
end
def self.has_been_uploaded?(url)
is_relative?(url) || is_local?(url) || is_on_s3?(url)
end
def self.is_relative?(url)
url.start_with?(LocalStore.directory)
end
def self.is_local?(url)
!SiteSetting.enable_s3_uploads? && url.start_with?(LocalStore.base_url)
end
def self.is_on_s3?(url)
SiteSetting.enable_s3_uploads? && url.start_with?(S3Store.base_url)
end
def self.get_from_url(url)
# we store relative urls, so we need to remove any host/cdn
url = url.gsub(/^#{LocalStore.asset_host}/i, "") if LocalStore.asset_host.present?
Upload.where(url: url).first if has_been_uploaded?(url)
asset_host = Rails.configuration.action_controller.asset_host
url = url.gsub(/^#{asset_host}/i, "") if asset_host.present?
Upload.where(url: url).first if Discourse.store.has_been_uploaded?(url)
end
end

22
db/migrate/20130728172550_add_url_to_optimized_images.rb Normal file
View File

@ -0,0 +1,22 @@
class AddUrlToOptimizedImages < ActiveRecord::Migration
def up
# add a nullable url column
add_column :optimized_images, :url, :string
# compute the url for existing images
execute "UPDATE optimized_images
SET url = substring(u.url from '^\/uploads\/[^/]+\/')
|| '_optimized/'
|| substring(oi.sha1 for 3) || '/'
|| substring(oi.sha1 from 4 for 3) || '/'
|| substring(oi.sha1 from 7 for 11) || oi.extension
FROM optimized_images oi
JOIN uploads u ON u.id = oi.upload_id
WHERE optimized_images.id = oi.id;"
# change the column to be non nullable
change_column :optimized_images, :url, :string, null: false
end
def down
remove_column :optimized_images, :url
end
end

28
lib/cooked_post_processor.rb
View File

@ -77,7 +77,7 @@ class CookedPostProcessor
end
def relative_to_absolute(src)
if src =~ /\A\/[^\/]/
if src =~ /^\/[^\/]/
Discourse.base_url_no_prefix + src
else
src
@ -98,7 +98,7 @@ class CookedPostProcessor
def associate_to_post(upload)
return if PostUpload.where(post_id: @post.id, upload_id: upload.id).count > 0
PostUpload.create({ post_id: @post.id, upload_id: upload.id })
PostUpload.create(post_id: @post.id, upload_id: upload.id)
rescue ActiveRecord::RecordNotUnique
# do not care if it's already associated
end
@ -155,7 +155,7 @@ class CookedPostProcessor
a.add_child(img)
# replace the image by its thumbnail
img['src'] = upload.thumbnail_url if upload && upload.has_thumbnail?
img['src'] = relative_to_absolute(upload.thumbnail.url) if upload && upload.has_thumbnail?
# then, some overlay informations
meta = Nokogiri::XML::Node.new("div", @doc)
@ -206,12 +206,13 @@ class CookedPostProcessor
end
def get_size(url)
# make sure s3 urls have a scheme (otherwise, FastImage will fail)
url = "http:" + url if Upload.is_on_s3?(url)
return unless is_valid_image_uri?(url)
uri = url
# make sure urls have a scheme (otherwise, FastImage will fail)
uri = (SiteSetting.use_ssl? ? "https:" : "http:") + url if url.start_with?("//")
return unless is_valid_image_uri?(uri)
# we can *always* crawl our own images
return unless SiteSetting.crawl_images? || Upload.has_been_uploaded?(url)
@size_cache[url] ||= FastImage.size(url)
return unless SiteSetting.crawl_images? || Discourse.store.has_been_uploaded?(url)
@size_cache[url] ||= FastImage.size(uri)
rescue Zlib::BufError # FastImage.size raises BufError for some gifs
end
@ -222,14 +223,9 @@ class CookedPostProcessor
end
def attachments
if SiteSetting.enable_s3_uploads?
@doc.css("a.attachment[href^=\"#{S3Store.base_url}\"]")
else
# local uploads are identified using a relative uri
@doc.css("a.attachment[href^=\"#{LocalStore.directory}\"]") +
# when cdn is enabled, we have the whole url
@doc.css("a.attachment[href^=\"#{LocalStore.base_url}\"]")
end
attachments = @doc.css("a.attachment[href^=\"#{Discourse.store.absolute_base_url}\"]")
attachments += @doc.css("a.attachment[href^=\"#{Discourse.store.relative_base_url}\"]") if Discourse.store.internal?
attachments
end
def dirty?

12
lib/discourse.rb
View File

@ -105,7 +105,7 @@ module Discourse
def self.git_version
return $git_version if $git_version
f = Rails.root.to_s + "/config/version"
f = Rails.root.to_s + "/lib/version"
require f if File.exists?("#{f}.rb")
begin
@ -122,6 +122,16 @@ module Discourse
user
end
def self.store
if SiteSetting.enable_s3_uploads?
@s3_store_loaded ||= require 'file_store/s3_store'
S3Store.new
else
@local_store_loaded ||= require 'file_store/local_store'
LocalStore.new
end
end
private
def self.maintenance_mode_key

93
lib/file_store/local_store.rb Normal file
View File

@ -0,0 +1,93 @@
class LocalStore
def store_upload(file, upload)
unique_sha1 = Digest::SHA1.hexdigest("#{Time.now.to_s}#{file.original_filename}")[0,16]
extension = File.extname(file.original_filename)
clean_name = "#{unique_sha1}#{extension}"
path = "#{relative_base_url}/#{upload.id}/#{clean_name}"
# copy the file to the right location
copy_file(file, "#{public_dir}#{path}")
# url
Discourse.base_uri + path
end
def store_optimized_image(file, optimized_image)
# 1234567890ABCDEF_100x200.jpg
filename = [
optimized_image.sha1[6..16],
"_#{optimized_image.width}x#{optimized_image.height}",
optimized_image.extension,
].join
# <rails>/public/uploads/site/_optimized/123/456/<filename>
path = File.join(
relative_base_url,
"_optimized",
optimized_image.sha1[0..2],
optimized_image.sha1[3..5],
filename
)
# copy the file to the right location
copy_file(file, "#{public_dir}#{path}")
# url
Discourse.base_uri + path
end
def remove_file(url)
File.delete("#{public_dir}#{url}") if has_been_uploaded?(url)
rescue Errno::ENOENT
# don't care if the file isn't there
end
def has_been_uploaded?(url)
is_relative?(url) || is_local?(url)
end
def absolute_base_url
url = asset_host.present? ? asset_host : Discourse.base_url_no_prefix
"#{url}#{relative_base_url}"
end
def relative_base_url
"/uploads/#{RailsMultisite::ConnectionManagement.current_db}"
end
def external?
!internal?
end
def internal?
true
end
def path_for(upload)
"#{public_dir}#{upload.url}"
end
private
def copy_file(file, path)
FileUtils.mkdir_p Pathname.new(path).dirname
# move the file to the right location
# not using cause mv, cause permissions are no good on move
File.open(path, "wb") do |f|
f.write(file.read)
end
end
def is_relative?(url)
url.start_with?(relative_base_url)
end
def is_local?(url)
url.start_with?(absolute_base_url)
end
def public_dir
"#{Rails.root}/public"
end
def asset_host
Rails.configuration.action_controller.asset_host
end
end

119
lib/file_store/s3_store.rb Normal file
View File

@ -0,0 +1,119 @@
require 'digest/sha1'
require 'open-uri'
class S3Store
def store_upload(file, upload)
extension = File.extname(file.original_filename)
remote_filename = "#{upload.id}#{upload.sha1}#{extension}"
# if this fails, it will throw an exception
upload(file.tempfile, remote_filename, file.content_type)
# returns the url of the uploaded file
"#{absolute_base_url}/#{remote_filename}"
end
def store_optimized_image(file, optimized_image)
extension = File.extname(file.path)
remote_filename = [
optimized_image.id,
optimized_image.sha1,
"_#{optimized_image.width}x#{optimized_image.height}",
extension
].join
# if this fails, it will throw an exception
upload(file, remote_filename)
# returns the url of the uploaded file
"#{absolute_base_url}/#{remote_filename}"
end
def remove_file(url)
check_missing_site_settings
return unless has_been_uploaded?(url)
name = File.basename(url)
remove(name)
end
def has_been_uploaded?(url)
url.start_with?(absolute_base_url)
end
def absolute_base_url
"//#{s3_bucket}.s3.amazonaws.com"
end
def external?
true
end
def internal?
!external?
end
def download(upload)
temp_file = Tempfile.new(["discourse-s3", File.extname(upload.original_filename)])
url = (SiteSetting.use_ssl? ? "https:" : "http:") + upload.url
File.open(temp_file.path, "wb") do |f|
f.write open(url, "rb", read_timeout: 20).read
end
temp_file
end
private
def s3_bucket
SiteSetting.s3_upload_bucket.downcase
end
def check_missing_site_settings
raise Discourse::SiteSettingMissing.new("s3_upload_bucket") if SiteSetting.s3_upload_bucket.blank?
raise Discourse::SiteSettingMissing.new("s3_access_key_id") if SiteSetting.s3_access_key_id.blank?
raise Discourse::SiteSettingMissing.new("s3_secret_access_key") if SiteSetting.s3_secret_access_key.blank?
end
def get_or_create_directory(name)
check_missing_site_settings
@fog_loaded ||= require 'fog'
fog = Fog::Storage.new generate_options
directory = fog.directories.get(name)
directory = fog.directories.create(key: name) unless directory
directory
end
def generate_options
options = {
provider: 'AWS',
aws_access_key_id: SiteSetting.s3_access_key_id,
aws_secret_access_key: SiteSetting.s3_secret_access_key,
}
options[:region] = SiteSetting.s3_region unless SiteSetting.s3_region.empty?
options
end
def upload(file, name, content_type=nil)
args = {
key: name,
public: true,
body: file,
}
args[:content_type] = content_type if content_type
directory.files.create(args)
end
def remove(name)
directory.files.destroy(key: name)
end
def directory
get_or_create_directory(s3_bucket)
end
end

96
lib/image_optimizer.rb
View File

@ -1,96 +0,0 @@
#
# This class is used to download and optimize images.
#
require 'image_sorcery'
require 'digest/sha1'
require 'open-uri'
class ImageOptimizer
attr_accessor :url
# url is a url of an image ex:
# 'http://site.com/image.png'
# '/uploads/site/image.png'
def initialize(url)
@url = url
# make sure directories exists
FileUtils.mkdir_p downloads_dir
FileUtils.mkdir_p optimized_dir
end
# return the path of an optimized image,
# if already cached return cached, else download and cache
# at the original size.
# if size is specified return a resized image
# if height or width are nil maintain aspect ratio
#
# Optimised image is the "most efficient" storage for an image
# at the basic level it runs through image_optim https://github.com/toy/image_optim
# it also has a failsafe that converts jpg to png or the opposite. if jpg size is 1.5*
# as efficient as png it flips formats.
def optimized_image_url (width = nil, height = nil)
begin
unless has_been_uploaded?
return @url unless SiteSetting.crawl_images?
# download the file if it hasn't been cached yet
download! unless File.exists?(cached_path)
end
# resize the image using Image Magick
result = ImageSorcery.new(cached_path).convert(optimized_path, resize: "#{width}x#{height}")
return optimized_url if result
@url
rescue
@url
end
end
private
def public_dir
@public_dir ||= "#{Rails.root}/public"
end
def downloads_dir
@downloads_dir ||= "#{public_dir}/downloads/#{RailsMultisite::ConnectionManagement.current_db}"
end
def optimized_dir
@optimized_dir ||= "#{public_dir}/images/#{RailsMultisite::ConnectionManagement.current_db}"
end
def has_been_uploaded?
@url.start_with?(Discourse.base_url_no_prefix)
end
def cached_path
@cached_path ||= if has_been_uploaded?
"#{public_dir}#{@url[Discourse.base_url_no_prefix.length..-1]}"
else
"#{downloads_dir}/#{file_name(@url)}"
end
end
def optimized_path
@optimized_path ||= "#{optimized_dir}/#{file_name(cached_path)}"
end
def file_name (uri)
image_info = FastImage.new(uri)
name = Digest::SHA1.hexdigest(uri)[0,16]
name << ".#{image_info.type}"
name
end
def download!
File.open(cached_path, "wb") do |f|
f.write open(@url, "rb", read_timeout: 20).read
end
end
def optimized_url
@optimized_url ||= Discourse.base_url_no_prefix + "/images/#{RailsMultisite::ConnectionManagement.current_db}/#{file_name(cached_path)}"
end
end

47
lib/local_store.rb
View File

@ -1,47 +0,0 @@
module LocalStore
def self.store_file(file, sha1, upload_id)
unique_sha1 = Digest::SHA1.hexdigest("#{Time.now.to_s}#{file.original_filename}")[0,16]
extension = File.extname(file.original_filename)
clean_name = "#{unique_sha1}#{extension}"
url_root = "#{directory}/#{upload_id}"
path = "#{Rails.root}/public#{url_root}"
FileUtils.mkdir_p path
# not using cause mv, cause permissions are no good on move
File.open("#{path}/#{clean_name}", "wb") do |f|
f.write File.read(file.tempfile)
end
# url
Discourse::base_uri + "#{url_root}/#{clean_name}"
end
def self.remove_file(url)
File.delete("#{Rails.root}/public#{url}")
rescue Errno::ENOENT
end
def self.uploaded_regex
/\/uploads\/#{RailsMultisite::ConnectionManagement.current_db}\/(?<upload_id>\d+)\/[0-9a-f]{16}\.(png|jpg|jpeg|gif|tif|tiff|bmp)/
end
def self.base_url
url = asset_host.present? ? asset_host : Discourse.base_url_no_prefix
"#{url}#{directory}"
end
def self.base_path
"#{Rails.root}/public#{directory}"
end
def self.directory
"/uploads/#{RailsMultisite::ConnectionManagement.current_db}"
end
def self.asset_host
Rails.configuration.action_controller.asset_host
end
end

70
lib/s3_store.rb
View File

@ -1,70 +0,0 @@
module S3Store
def self.store_file(file, sha1, upload_id)
S3Store.check_missing_site_settings
directory = S3Store.get_or_create_directory(SiteSetting.s3_upload_bucket)
extension = File.extname(file.original_filename)
remote_filename = "#{upload_id}#{sha1}#{extension}"
# if this fails, it will throw an exception
file = S3Store.upload(file, remote_filename, directory)
"#{S3Store.base_url}/#{remote_filename}"
end
def self.base_url
"//#{SiteSetting.s3_upload_bucket.downcase}.s3.amazonaws.com"
end
def self.remove_file(url)
S3Store.check_missing_site_settings
directory = S3Store.get_or_create_directory(SiteSetting.s3_upload_bucket)
file = S3Store.destroy(url, directory)
end
def self.check_missing_site_settings
raise Discourse::SiteSettingMissing.new("s3_upload_bucket") if SiteSetting.s3_upload_bucket.blank?
raise Discourse::SiteSettingMissing.new("s3_access_key_id") if SiteSetting.s3_access_key_id.blank?
raise Discourse::SiteSettingMissing.new("s3_secret_access_key") if SiteSetting.s3_secret_access_key.blank?
end
def self.get_or_create_directory(name)
@fog_loaded = require 'fog' unless @fog_loaded
options = S3Store.generate_options
fog = Fog::Storage.new(options)
directory = fog.directories.get(name)
directory = fog.directories.create(key: name) unless directory
directory
end
def self.generate_options
options = {
provider: 'AWS',
aws_access_key_id: SiteSetting.s3_access_key_id,
aws_secret_access_key: SiteSetting.s3_secret_access_key
}
options[:region] = SiteSetting.s3_region unless SiteSetting.s3_region.empty?
options
end
def self.upload(file, name, directory)
directory.files.create(
key: name,
public: true,
body: file.tempfile,
content_type: file.content_type
)
end
def self.destroy(name, directory)
directory.files.destroy(key: name)
end
end

21
lib/tasks/images.rake
View File

@ -10,27 +10,6 @@ task "images:compress" => :environment do
end
end
desc "updates reverse index of image uploads"
task "images:reindex" => :environment do
RailsMultisite::ConnectionManagement.each_connection do |db|
puts "Reindexing #{db}"
Post.select([:id, :cooked]).find_each do |p|
doc = Nokogiri::HTML::fragment(p.cooked)
doc.search("img").each do |img|
src = img['src']
if src.present? && Upload.has_been_uploaded?(src) && m = Upload.uploaded_regex.match(src)
begin
PostUpload.create({ post_id: p.id, upload_id: m[:upload_id] })
rescue ActiveRecord::RecordNotUnique
end
end
end
putc "."
end
end
puts "\ndone."
end
desc "clean orphan uploaded files"
task "images:clean_orphans" => :environment do
RailsMultisite::ConnectionManagement.each_connection do |db|

21
spec/components/cooked_post_processor_spec.rb
View File

@ -31,9 +31,7 @@ describe CookedPostProcessor do
Upload.expects(:get_from_url).returns(upload)
cpp.post_process_attachments
# ensures absolute urls on attachment
cpp.html.should =~ /#{LocalStore.base_url}/
# ensure name is present
cpp.html.should =~ /archive.zip/
cpp.html.should =~ /#{Discourse.store.absolute_base_url}/
# keeps the reverse index up to date
post.uploads.reload
post.uploads.count.should == 1
@ -74,7 +72,7 @@ describe CookedPostProcessor do
Upload.expects(:get_from_url).returns(upload)
cpp.post_process_images
# ensures absolute urls on uploaded images
cpp.html.should =~ /#{LocalStore.base_url}/
cpp.html.should =~ /#{LocalStore.new.absolute_base_url}/
# dirty
cpp.should be_dirty
# keeps the reverse index up to date
@ -227,7 +225,6 @@ describe CookedPostProcessor do
let(:cpp) { CookedPostProcessor.new(post) }
it "ensures s3 urls have a default scheme" do
Upload.stubs(:is_on_s3?).returns(true)
FastImage.stubs(:size)
cpp.expects(:is_valid_image_uri?).with("http://bucket.s3.aws.amazon.com/image.jpg")
cpp.get_size("//bucket.s3.aws.amazon.com/image.jpg")
@ -239,13 +236,15 @@ describe CookedPostProcessor do
it "doesn't call FastImage" do
FastImage.expects(:size).never
cpp.get_size("http://foo.bar/image.png").should == nil
cpp.get_size("http://foo.bar/image1.png").should == nil
end
it "is always allowed to crawled our own images" do
Upload.expects(:has_been_uploaded?).returns(true)
it "is always allowed to crawl our own images" do
store = {}
Discourse.expects(:store).returns(store)
store.expects(:has_been_uploaded?).returns(true)
FastImage.expects(:size).returns([100, 200])
cpp.get_size("http://foo.bar/image.png").should == [100, 200]
cpp.get_size("http://foo.bar/image2.png").should == [100, 200]
end
end
@ -253,8 +252,8 @@ describe CookedPostProcessor do
it "caches the results" do
SiteSetting.stubs(:crawl_images?).returns(true)
FastImage.expects(:size).returns([200, 400])
cpp.get_size("http://foo.bar/image.png")
cpp.get_size("http://foo.bar/image.png").should == [200, 400]
cpp.get_size("http://foo.bar/image3.png")
cpp.get_size("http://foo.bar/image3.png").should == [200, 400]
end
end

13
spec/components/discourse_spec.rb
View File

@ -65,5 +65,18 @@ describe Discourse do
end
context "#store" do
it "returns LocalStore by default" do
Discourse.store.should be_a(LocalStore)
end
it "returns S3Store when S3 is enabled" do
SiteSetting.expects(:enable_s3_uploads?).returns(true)
Discourse.store.should be_a(S3Store)
end
end
end

76
spec/components/file_store/local_store_spec.rb Normal file
View File

@ -0,0 +1,76 @@
require 'spec_helper'
require 'file_store/local_store'
describe LocalStore do
let(:store) { LocalStore.new }
let(:upload) { build(:upload) }
let(:uploaded_file) do
ActionDispatch::Http::UploadedFile.new({
filename: 'logo.png',
tempfile: File.new("#{Rails.root}/spec/fixtures/images/logo.png")
})
end
let(:optimized_image) { build(:optimized_image) }
it "is internal" do
store.internal?.should == true
store.external?.should == false
end
describe "store_upload" do
it "returns a relative url" do
Time.stubs(:now).returns(Time.utc(2013, 2, 17, 12, 0, 0, 0))
upload.stubs(:id).returns(42)
store.expects(:copy_file)
store.store_upload(uploaded_file, upload).should == "/uploads/default/42/253dc8edf9d4ada1.png"
end
end
describe "store_optimized_image" do
it "returns a relative url" do
store.expects(:copy_file)
store.store_optimized_image({}, optimized_image).should == "/uploads/default/_optimized/86f/7e4/37faa5a7fce_100x200.png"
end
end
describe "remove_file" do
it "does not delete any file" do
File.expects(:delete).never
store.remove_file("/path/to/file")
end
it "deletes the file locally" do
File.expects(:delete)
store.remove_file("/uploads/default/42/253dc8edf9d4ada1.png")
end
end
describe "has_been_uploaded?" do
it "identifies local or relatives urls" do
Discourse.expects(:base_url_no_prefix).returns("http://discuss.site.com")
store.has_been_uploaded?("http://discuss.site.com/uploads/default/42/0123456789ABCDEF.jpg").should == true
store.has_been_uploaded?("/uploads/default/42/0123456789ABCDEF.jpg").should == true
end
it "identifies local urls when using a CDN" do
Rails.configuration.action_controller.stubs(:asset_host).returns("http://my.cdn.com")
store.has_been_uploaded?("http://my.cdn.com/uploads/default/42/0123456789ABCDEF.jpg").should == true
end
it "does not match dummy urls" do
store.has_been_uploaded?("http://domain.com/uploads/default/42/0123456789ABCDEF.jpg").should == false
end
end
end

84
spec/components/file_store/s3_store_spec.rb Normal file
View File

@ -0,0 +1,84 @@
require 'spec_helper'
require 'fog'
require 'file_store/s3_store'
describe S3Store do
let(:store) { S3Store.new }
let(:upload) { build(:upload) }
let(:uploaded_file) do
ActionDispatch::Http::UploadedFile.new({
filename: 'logo.png',
tempfile: File.new("#{Rails.root}/spec/fixtures/images/logo.png")
})
end
let(:optimized_image) { build(:optimized_image) }
let(:optimized_image_file) do
ActionDispatch::Http::UploadedFile.new({
filename: 'logo.png',
tempfile: File.new("#{Rails.root}/spec/fixtures/images/logo.png")
})
end
before(:each) do
SiteSetting.stubs(:s3_upload_bucket).returns("S3_Upload_Bucket")
SiteSetting.stubs(:s3_access_key_id).returns("s3_access_key_id")
SiteSetting.stubs(:s3_secret_access_key).returns("s3_secret_access_key")
Fog.mock!
end
after(:each) { Fog.unmock! }
it "is internal" do
store.external?.should == true
store.internal?.should == false
end
describe "store_upload" do
it "returns a relative url" do
upload.stubs(:id).returns(42)
store.store_upload(uploaded_file, upload).should == "//s3_upload_bucket.s3.amazonaws.com/42e9d71f5ee7c92d6dc9e92ffdad17b8bd49418f98.png"
end
end
describe "store_optimized_image" do
it "returns a relative url" do
optimized_image.stubs(:id).returns(42)
store.store_optimized_image(optimized_image_file, optimized_image).should == "//s3_upload_bucket.s3.amazonaws.com/4286f7e437faa5a7fce15d1ddcb9eaeaea377667b8_100x200.png"
end
end
describe "remove_file" do
it "does not delete any file" do
store.expects(:remove).never
store.remove_file("//other_bucket.s3.amazonaws.com/42.png")
end
it "deletes the file on s3" do
store.expects(:remove)
store.remove_file("//s3_upload_bucket.s3.amazonaws.com/42.png")
end
end
describe "has_been_uploaded?" do
it "identifies S3 uploads" do
SiteSetting.stubs(:enable_s3_uploads).returns(true)
store.has_been_uploaded?("//s3_upload_bucket.s3.amazonaws.com/1337.png").should == true
end
it "does not match other s3 urls" do
store.has_been_uploaded?("//s3.amazonaws.com/Bucket/1337.png").should == false
end
end
end

30
spec/components/local_store_spec.rb
View File

@ -1,30 +0,0 @@
require 'spec_helper'
require 'local_store'
describe LocalStore do
describe "store_file" do
let(:file) do
ActionDispatch::Http::UploadedFile.new({
filename: 'logo.png',
content_type: 'image/png',
tempfile: File.new("#{Rails.root}/spec/fixtures/images/logo.png")
})
end
let(:image_info) { FastImage.new(file) }
it 'returns the url of the uploaded file if successful' do
# prevent the tests from creating directories & files...
FileUtils.stubs(:mkdir_p)
File.stubs(:open)
# The Time needs to be frozen as it is used to generate a clean & unique name
Time.stubs(:now).returns(Time.utc(2013, 2, 17, 12, 0, 0, 0))
#
LocalStore.store_file(file, "", 1).should == '/uploads/default/1/253dc8edf9d4ada1.png'
end
end
end

36
spec/components/s3_store_spec.rb
View File

@ -1,36 +0,0 @@
require 'spec_helper'
require 'fog'
require 's3_store'
describe S3Store do
describe "store_file" do
let(:file) do
ActionDispatch::Http::UploadedFile.new({
filename: 'logo.png',
content_type: 'image/png',
tempfile: File.new("#{Rails.root}/spec/fixtures/images/logo.png")
})
end
let(:image_info) { FastImage.new(file) }
before(:each) do
SiteSetting.stubs(:s3_upload_bucket).returns("S3_Upload_Bucket")
SiteSetting.stubs(:s3_access_key_id).returns("s3_access_key_id")
SiteSetting.stubs(:s3_secret_access_key).returns("s3_secret_access_key")
Fog.mock!
end
it 'returns the url of the S3 upload if successful' do
S3Store.store_file(file, "SHA", 1).should == '//s3_upload_bucket.s3.amazonaws.com/1SHA.png'
end
after(:each) do
Fog.unmock!
end
end
end

3
spec/fabricators/optimized_image_fabricator.rb
View File

@ -1,7 +1,8 @@
Fabricator(:optimized_image) do
upload
sha1 "abcdef"
sha1 "86f7e437faa5a7fce15d1ddcb9eaeaea377667b8"
extension ".png"
width 100
height 200
url "138569_100x200.png"
end

1
spec/fabricators/upload_fabricator.rb
View File

@ -1,5 +1,6 @@
Fabricator(:upload) do
user
sha1 "e9d71f5ee7c92d6dc9e92ffdad17b8bd49418f98"
original_filename "uploaded.jpg"
filesize 1234
width 100

58
spec/models/optimized_image_spec.rb
View File

@ -4,25 +4,55 @@ describe OptimizedImage do
it { should belong_to :upload }
let(:upload) { build(:upload) }
let(:oi) { OptimizedImage.create_for(upload, 100, 100) }
let(:upload) { Fabricate(:upload) }
let(:oi) { OptimizedImage.create_for(upload, 100, 200) }
describe ".create_for" do
before(:each) do
ImageSorcery.any_instance.expects(:convert).returns(true)
# make sure we don't hit the filesystem
FileUtils.stubs(:mkdir_p)
File.stubs(:open)
before { ImageSorcery.any_instance.expects(:convert).returns(true) }
describe "internal store" do
it "works" do
Tempfile.any_instance.expects(:close!)
oi.sha1.should == "da39a3ee5e6b4b0d3255bfef95601890afd80709"
oi.extension.should == ".jpg"
oi.width.should == 100
oi.height.should == 200
oi.url.should == "/uploads/default/_optimized/da3/9a3/ee5e6b4b0d3_100x200.jpg"
end
end
it "works" do
Tempfile.any_instance.expects(:close)
Tempfile.any_instance.expects(:unlink)
oi.sha1.should == "da39a3ee5e6b4b0d3255bfef95601890afd80709"
oi.extension.should == ".jpg"
oi.width.should == 100
oi.height.should == 100
describe "external store" do
require 'file_store/s3_store'
require 'fog'
let(:store) { S3Store.new }
before do
Discourse.stubs(:store).returns(store)
SiteSetting.stubs(:s3_upload_bucket).returns("S3_Upload_Bucket")
SiteSetting.stubs(:s3_access_key_id).returns("s3_access_key_id")
SiteSetting.stubs(:s3_secret_access_key).returns("s3_secret_access_key")
Fog.mock!
end
it "works" do
# fake downloaded file
downloaded_file = {}
downloaded_file.expects(:path).returns("/path/to/fake.png")
downloaded_file.expects(:close!)
store.expects(:download).returns(downloaded_file)
# assertions
oi.sha1.should == "da39a3ee5e6b4b0d3255bfef95601890afd80709"
oi.extension.should == ".png"
oi.width.should == 100
oi.height.should == 200
oi.url.should =~ /^\/\/s3_upload_bucket.s3.amazonaws.com\/[0-9a-f]+_100x200.png/
end
end
end

75
spec/models/upload_spec.rb
View File

@ -42,20 +42,12 @@ describe Upload do
it "does not create a thumbnail when disabled" do
SiteSetting.stubs(:create_thumbnails?).returns(false)
SiteSetting.expects(:enable_s3_uploads?).never
upload.create_thumbnail!
end
it "does not create a thumbnail when using S3" do
SiteSetting.expects(:create_thumbnails?).returns(true)
SiteSetting.expects(:enable_s3_uploads?).returns(true)
upload.expects(:has_thumbnail?).never
OptimizedImage.expects(:create_for).never
upload.create_thumbnail!
end
it "does not create another thumbnail" do
SiteSetting.expects(:create_thumbnails?).returns(true)
SiteSetting.expects(:enable_s3_uploads?).returns(false)
upload.expects(:has_thumbnail?).returns(true)
OptimizedImage.expects(:create_for).never
upload.create_thumbnail!
@ -65,7 +57,6 @@ describe Upload do
upload = Fabricate(:upload)
thumbnail = Fabricate(:optimized_image, upload: upload)
SiteSetting.expects(:create_thumbnails?).returns(true)
SiteSetting.expects(:enable_s3_uploads?).returns(false)
upload.expects(:has_thumbnail?).returns(false)
OptimizedImage.expects(:create_for).returns(thumbnail)
upload.create_thumbnail!
@ -104,7 +95,9 @@ describe Upload do
end
it "saves proper information" do
Upload.expects(:store_file).returns(url)
store = {}
Discourse.expects(:store).returns(store)
store.expects(:store_upload).returns(url)
upload = Upload.create_for(user_id, image, image_filesize)
upload.user_id.should == user_id
upload.original_filename.should == image.original_filename
@ -117,66 +110,6 @@ describe Upload do
end
context ".store_file" do
it "store files on s3 when enabled" do
SiteSetting.expects(:enable_s3_uploads?).returns(true)
LocalStore.expects(:store_file).never
S3Store.expects(:store_file)
Upload.store_file(image, image_sha1, 1)
end
it "store files locally by default" do
S3Store.expects(:store_file).never
LocalStore.expects(:store_file)
Upload.store_file(image, image_sha1, 1)
end
end
context ".remove_file" do
it "remove files on s3 when enabled" do
SiteSetting.expects(:enable_s3_uploads?).returns(true)
LocalStore.expects(:remove_file).never
S3Store.expects(:remove_file)
Upload.remove_file(upload.url)
end
it "remove files locally by default" do
S3Store.expects(:remove_file).never
LocalStore.expects(:remove_file)
Upload.remove_file(upload.url)
end
end
context ".has_been_uploaded?" do
it "identifies internal or relatives urls" do
Discourse.expects(:base_url_no_prefix).returns("http://discuss.site.com")
Upload.has_been_uploaded?("http://discuss.site.com/uploads/default/42/0123456789ABCDEF.jpg").should == true
Upload.has_been_uploaded?("/uploads/default/42/0123456789ABCDEF.jpg").should == true
end
it "identifies internal urls when using a CDN" do
Rails.configuration.action_controller.expects(:asset_host).returns("http://my.cdn.com").twice
Upload.has_been_uploaded?("http://my.cdn.com/uploads/default/42/0123456789ABCDEF.jpg").should == true
end
it "identifies S3 uploads" do
SiteSetting.stubs(:enable_s3_uploads).returns(true)
SiteSetting.stubs(:s3_upload_bucket).returns("Bucket")
Upload.has_been_uploaded?("//bucket.s3.amazonaws.com/1337.png").should == true
end
it "identifies external urls" do
Upload.has_been_uploaded?("http://domain.com/uploads/default/42/0123456789ABCDEF.jpg").should == false
Upload.has_been_uploaded?("//s3.amazonaws.com/Bucket/1337.png").should == false
end
end
context ".get_from_url" do
it "works when the file has been uploaded" do