Merge pull request #4958 from dmacjam/search_posts_by_filetype

FEATURE: Search posts by filetype

app/jobs/onceoff/migrate_upload_extensions.rb

@@ -0,0 +1,11 @@
+module Jobs
+
+  class MigrateUploadExtensions < Jobs::Onceoff
+    def execute_onceoff(args)
+      Upload.find_each do |upload|
+        upload.extension = File.extname(upload.original_filename)[1..10]
+        upload.save
+      end
+    end
+  end
+end

app/models/topic_link.rb

@@ -161,6 +161,7 @@ SQL
           added_urls << url
 
           unless TopicLink.exists?(topic_id: post.topic_id, post_id: post.id, url: url)
+            file_extension = File.extname(parsed.path)[1..10].downcase unless File.extname(parsed.path).empty?
             begin
               TopicLink.create!(post_id: post.id,
                                 user_id: post.user_id,
@@ -170,7 +171,8 @@ SQL
                                 internal: internal,
                                 link_topic_id: topic_id,
                                 link_post_id: reflected_post.try(:id),
-                                quote: link.is_quote)
+                                quote: link.is_quote,
+                                extension: file_extension)
             rescue ActiveRecord::RecordNotUnique, PG::UniqueViolation
               # it's fine
             end

app/models/upload.rb

@@ -53,10 +53,6 @@ class Upload < ActiveRecord::Base
     end
   end
 
-  def extension
-    File.extname(original_filename)
-  end
-
   def self.generate_digest(path)
     Digest::SHA1.file(path).hexdigest
   end

db/migrate/20170609115401_add_extension_to_topic_links.rb

@@ -0,0 +1,6 @@
+class AddExtensionToTopicLinks < ActiveRecord::Migration
+  def change
+    add_column :topic_links, :extension, :string, limit: 10
+    add_index :topic_links, :extension
+  end
+end

db/migrate/20170703115216_add_extension_to_uploads.rb

@@ -0,0 +1,11 @@
+class AddExtensionToUploads < ActiveRecord::Migration
+  def up
+    add_column :uploads, :extension, :string, limit: 10
+    execute "CREATE INDEX index_uploads_on_extension ON uploads(lower(extension))"
+  end
+
+  def down
+    remove_column :uploads, :extension
+    execute "DROP INDEX index_uploads_on_extension"
+  end
+end

lib/file_store/base_store.rb

@@ -95,7 +95,7 @@ module FileStore
     end
 
     def get_path_for_upload(upload)
-      get_path_for("original".freeze, upload.id, upload.sha1, upload.extension)
+      get_path_for("original".freeze, upload.id, upload.sha1, "."+upload.extension)
     end
 
     def get_path_for_optimized_image(optimized_image)

lib/search.rb

@@ -491,6 +491,19 @@ class Search
     end
   end
 
+  advanced_filter(/filetypes?:([a-zA-Z0-9,\-_]+)/) do |posts, match|
+    file_extensions = match.split(",").map(&:downcase)
+
+    posts.where("posts.id IN (
+      SELECT post_id FROM topic_links
+      WHERE extension IN (:file_extensions)
+      UNION
+      SELECT post_uploads.post_id FROM uploads
+      JOIN post_uploads ON post_uploads.upload_id = uploads.id
+      WHERE lower(uploads.extension) IN (:file_extensions)
+      )", {file_extensions: file_extensions})
+  end
+
   private
 
     def process_advanced_search!(term)

lib/upload_creator.rb

@@ -75,6 +75,7 @@ class UploadCreator
       @upload.sha1              = sha1
       @upload.url               = ""
       @upload.origin            = @opts[:origin][0...1000] if @opts[:origin]
+      @upload.extension         = File.extname(@filename)[1..10]
 
       if FileHelper.is_image?(@filename)
         @upload.width, @upload.height = ImageSizer.resize(*@image_info.size)

spec/components/search_spec.rb

@@ -738,6 +738,23 @@ describe Search do
       end
     end
 
+    it "can find posts which contains filetypes" do
+      post1 = Fabricate(:post,
+                        raw: "http://example.com/image.png")
+      post2 = Fabricate(:post,
+                         raw: "Discourse logo\n"\
+                              "http://example.com/logo.png\n"\
+                              "http://example.com/vector_image.svg")
+      post_with_upload = Fabricate(:post, uploads: [Fabricate(:upload)])
+      Fabricate(:post)
+
+      TopicLink.extract_from(post1)
+      TopicLink.extract_from(post2)
+
+      expect(Search.execute('filetype:svg').posts).to eq([post2])
+      expect(Search.execute('filetype:png').posts.map(&:id)).to contain_exactly(post1.id, post2.id, post_with_upload.id)
+      expect(Search.execute('logo filetype:png').posts.map(&:id)).to eq([post2.id])
+    end
   end
 
   it 'can parse complex strings using ts_query helper' do

spec/fabricators/upload_fabricator.rb

@@ -6,6 +6,7 @@ Fabricator(:upload) do
   width 100
   height 200
   url { sequence(:url) { |n| "/uploads/default/#{n}/1234567890123456.png" } }
+  extension "png"
 end
 
 Fabricator(:upload_s3, from: :upload) do

spec/models/topic_link_spec.rb

@@ -192,7 +192,7 @@ http://b.com/#{'a' * 500}
     end
 
     context "link to a local attachments" do
-      let(:post) { topic.posts.create(user: user, raw: '<a class="attachment" href="/uploads/default/208/87bb3d8428eb4783.rb">ruby.rb</a>') }
+      let(:post) { topic.posts.create(user: user, raw: '<a class="attachment" href="/uploads/default/208/87bb3d8428eb4783.rb?foo=bar">ruby.rb</a>') }
 
       it "extracts the link" do
         TopicLink.extract_from(post)
@@ -202,9 +202,11 @@ http://b.com/#{'a' * 500}
         # is set to internal
         expect(link).to be_internal
         # has the correct url
-        expect(link.url).to eq("/uploads/default/208/87bb3d8428eb4783.rb")
+        expect(link.url).to eq("/uploads/default/208/87bb3d8428eb4783.rb?foo=bar")
         # should not be the reflection
         expect(link).not_to be_reflection
+        # should have file extension
+        expect(link.extension).to eq('rb')
       end
 
     end
@@ -223,6 +225,8 @@ http://b.com/#{'a' * 500}
         expect(link.url).to eq("//s3.amazonaws.com/bucket/2104a0211c9ce41ed67989a1ed62e9a394c1fbd1446.rb")
         # should not be the reflection
         expect(link).not_to be_reflection
+        # should have file extension
+        expect(link.extension).to eq('rb')
       end
 
     end

spec/models/upload_spec.rb

@@ -46,6 +46,11 @@ describe Upload do
 
   end
 
+  it "extracts file extension" do
+    created_upload = UploadCreator.new(image, image_filename).create_for(user_id)
+    expect(created_upload.extension).to eq("png")
+  end
+
   context ".get_from_url" do
     let(:url) { "/uploads/default/original/3X/1/0/10f73034616a796dfd70177dc54b6def44c4ba6f.png" }
     let(:upload) { Fabricate(:upload, url: url) }