diff --git a/app/jobs/onceoff/migrate_upload_extensions.rb b/app/jobs/onceoff/migrate_upload_extensions.rb new file mode 100644 index 00000000000..38bcd3a1eab --- /dev/null +++ b/app/jobs/onceoff/migrate_upload_extensions.rb @@ -0,0 +1,11 @@ +module Jobs + + class MigrateUploadExtensions < Jobs::Onceoff + def execute_onceoff(args) + Upload.find_each do |upload| + upload.extension = File.extname(upload.original_filename)[1..10] + upload.save + end + end + end +end diff --git a/app/models/topic_link.rb b/app/models/topic_link.rb index f5925960a27..4576a0a601c 100644 --- a/app/models/topic_link.rb +++ b/app/models/topic_link.rb @@ -161,6 +161,7 @@ SQL added_urls << url unless TopicLink.exists?(topic_id: post.topic_id, post_id: post.id, url: url) + file_extension = File.extname(parsed.path)[1..10].downcase unless File.extname(parsed.path).empty? begin TopicLink.create!(post_id: post.id, user_id: post.user_id, @@ -170,7 +171,8 @@ SQL internal: internal, link_topic_id: topic_id, link_post_id: reflected_post.try(:id), - quote: link.is_quote) + quote: link.is_quote, + extension: file_extension) rescue ActiveRecord::RecordNotUnique, PG::UniqueViolation # it's fine end diff --git a/app/models/upload.rb b/app/models/upload.rb index 514748f2d7e..f9886d8998d 100644 --- a/app/models/upload.rb +++ b/app/models/upload.rb @@ -53,10 +53,6 @@ class Upload < ActiveRecord::Base end end - def extension - File.extname(original_filename) - end - def self.generate_digest(path) Digest::SHA1.file(path).hexdigest end diff --git a/db/migrate/20170609115401_add_extension_to_topic_links.rb b/db/migrate/20170609115401_add_extension_to_topic_links.rb new file mode 100644 index 00000000000..7c05a12d1a1 --- /dev/null +++ b/db/migrate/20170609115401_add_extension_to_topic_links.rb @@ -0,0 +1,6 @@ +class AddExtensionToTopicLinks < ActiveRecord::Migration + def change + add_column :topic_links, :extension, :string, limit: 10 + add_index :topic_links, :extension + end +end diff --git a/db/migrate/20170703115216_add_extension_to_uploads.rb b/db/migrate/20170703115216_add_extension_to_uploads.rb new file mode 100644 index 00000000000..76aa08fa113 --- /dev/null +++ b/db/migrate/20170703115216_add_extension_to_uploads.rb @@ -0,0 +1,11 @@ +class AddExtensionToUploads < ActiveRecord::Migration + def up + add_column :uploads, :extension, :string, limit: 10 + execute "CREATE INDEX index_uploads_on_extension ON uploads(lower(extension))" + end + + def down + remove_column :uploads, :extension + execute "DROP INDEX index_uploads_on_extension" + end +end diff --git a/lib/file_store/base_store.rb b/lib/file_store/base_store.rb index b79282fe681..7b17fb2d852 100644 --- a/lib/file_store/base_store.rb +++ b/lib/file_store/base_store.rb @@ -95,7 +95,7 @@ module FileStore end def get_path_for_upload(upload) - get_path_for("original".freeze, upload.id, upload.sha1, upload.extension) + get_path_for("original".freeze, upload.id, upload.sha1, "."+upload.extension) end def get_path_for_optimized_image(optimized_image) diff --git a/lib/search.rb b/lib/search.rb index 21eb7aeffa8..5aa0e372433 100644 --- a/lib/search.rb +++ b/lib/search.rb @@ -491,6 +491,19 @@ class Search end end + advanced_filter(/filetypes?:([a-zA-Z0-9,\-_]+)/) do |posts, match| + file_extensions = match.split(",").map(&:downcase) + + posts.where("posts.id IN ( + SELECT post_id FROM topic_links + WHERE extension IN (:file_extensions) + UNION + SELECT post_uploads.post_id FROM uploads + JOIN post_uploads ON post_uploads.upload_id = uploads.id + WHERE lower(uploads.extension) IN (:file_extensions) + )", {file_extensions: file_extensions}) + end + private def process_advanced_search!(term) diff --git a/lib/upload_creator.rb b/lib/upload_creator.rb index 67efcd32704..847f932aff6 100644 --- a/lib/upload_creator.rb +++ b/lib/upload_creator.rb @@ -75,6 +75,7 @@ class UploadCreator @upload.sha1 = sha1 @upload.url = "" @upload.origin = @opts[:origin][0...1000] if @opts[:origin] + @upload.extension = File.extname(@filename)[1..10] if FileHelper.is_image?(@filename) @upload.width, @upload.height = ImageSizer.resize(*@image_info.size) diff --git a/spec/components/search_spec.rb b/spec/components/search_spec.rb index 1a4aa45f87b..7d0f86cff4a 100644 --- a/spec/components/search_spec.rb +++ b/spec/components/search_spec.rb @@ -738,6 +738,23 @@ describe Search do end end + it "can find posts which contains filetypes" do + post1 = Fabricate(:post, + raw: "http://example.com/image.png") + post2 = Fabricate(:post, + raw: "Discourse logo\n"\ + "http://example.com/logo.png\n"\ + "http://example.com/vector_image.svg") + post_with_upload = Fabricate(:post, uploads: [Fabricate(:upload)]) + Fabricate(:post) + + TopicLink.extract_from(post1) + TopicLink.extract_from(post2) + + expect(Search.execute('filetype:svg').posts).to eq([post2]) + expect(Search.execute('filetype:png').posts.map(&:id)).to contain_exactly(post1.id, post2.id, post_with_upload.id) + expect(Search.execute('logo filetype:png').posts.map(&:id)).to eq([post2.id]) + end end it 'can parse complex strings using ts_query helper' do diff --git a/spec/fabricators/upload_fabricator.rb b/spec/fabricators/upload_fabricator.rb index 968eaa8d7a3..d9df6633ce1 100644 --- a/spec/fabricators/upload_fabricator.rb +++ b/spec/fabricators/upload_fabricator.rb @@ -6,6 +6,7 @@ Fabricator(:upload) do width 100 height 200 url { sequence(:url) { |n| "/uploads/default/#{n}/1234567890123456.png" } } + extension "png" end Fabricator(:upload_s3, from: :upload) do diff --git a/spec/models/topic_link_spec.rb b/spec/models/topic_link_spec.rb index 570ddf32928..c42371a8bfb 100644 --- a/spec/models/topic_link_spec.rb +++ b/spec/models/topic_link_spec.rb @@ -192,7 +192,7 @@ http://b.com/#{'a' * 500} end context "link to a local attachments" do - let(:post) { topic.posts.create(user: user, raw: 'ruby.rb') } + let(:post) { topic.posts.create(user: user, raw: 'ruby.rb') } it "extracts the link" do TopicLink.extract_from(post) @@ -202,9 +202,11 @@ http://b.com/#{'a' * 500} # is set to internal expect(link).to be_internal # has the correct url - expect(link.url).to eq("/uploads/default/208/87bb3d8428eb4783.rb") + expect(link.url).to eq("/uploads/default/208/87bb3d8428eb4783.rb?foo=bar") # should not be the reflection expect(link).not_to be_reflection + # should have file extension + expect(link.extension).to eq('rb') end end @@ -223,6 +225,8 @@ http://b.com/#{'a' * 500} expect(link.url).to eq("//s3.amazonaws.com/bucket/2104a0211c9ce41ed67989a1ed62e9a394c1fbd1446.rb") # should not be the reflection expect(link).not_to be_reflection + # should have file extension + expect(link.extension).to eq('rb') end end diff --git a/spec/models/upload_spec.rb b/spec/models/upload_spec.rb index d6eaedee761..6625cc5fd89 100644 --- a/spec/models/upload_spec.rb +++ b/spec/models/upload_spec.rb @@ -46,6 +46,11 @@ describe Upload do end + it "extracts file extension" do + created_upload = UploadCreator.new(image, image_filename).create_for(user_id) + expect(created_upload.extension).to eq("png") + end + context ".get_from_url" do let(:url) { "/uploads/default/original/3X/1/0/10f73034616a796dfd70177dc54b6def44c4ba6f.png" } let(:upload) { Fabricate(:upload, url: url) }