FIX: remove the tmp inventory files after the s3 uploads check.

lib/s3_inventory.rb

@@ -31,8 +31,13 @@ class S3Inventory
     end
 
     DistributedMutex.synchronize("s3_inventory_list_missing_#{type}") do
-      download_inventory_files_to_tmp_directory
-      decompress_inventory_files
+      begin
+        files.each do |file|
+          next if File.exists?(file[:filename][0...-3])
+
+          download_inventory_file_to_tmp_directory(file)
+          decompress_inventory_file(file)
+        end
 
         multisite_prefix = "uploads/#{RailsMultisite::ConnectionManagement.current_db}/"
         ActiveRecord::Base.transaction do
@@ -76,6 +81,9 @@ class S3Inventory
             connection.exec("DROP TABLE #{table_name}") unless connection.nil?
           end
         end
+      ensure
+        cleanup!
+      end
     end
   end
 
@@ -118,23 +126,19 @@ class S3Inventory
     log "#{missing[:count]} post uploads are missing."
   end
 
-  def download_inventory_files_to_tmp_directory
-    files.each do |file|
-      next if File.exists?(file[:filename])
+  def download_inventory_file_to_tmp_directory(file)
+    return if File.exists?(file[:filename])
 
     log "Downloading inventory file '#{file[:key]}' to tmp directory..."
     failure_message = "Failed to inventory file '#{file[:key]}' to tmp directory."
 
     @s3_helper.download_file(file[:key], file[:filename], failure_message)
   end
-  end
 
-  def decompress_inventory_files
-    files.each do |file|
+  def decompress_inventory_file(file)
     log "Decompressing inventory file '#{file[:filename]}', this may take a while..."
     Discourse::Utils.execute_command('gzip', '--decompress', file[:filename], failure_message: "Failed to decompress inventory file '#{file[:filename]}'.", chdir: tmp_directory)
   end
-  end
 
   def update_bucket_policy
     @s3_helper.s3_client.put_bucket_policy(
@@ -173,6 +177,13 @@ class S3Inventory
 
   private
 
+  def cleanup!
+    files.each do |file|
+      File.delete(file[:filename]) if File.exists?(file[:filename])
+      File.delete(file[:filename][0...-3]) if File.exists?(file[:filename][0...-3])
+    end
+  end
+
   def connection
     @connection ||= ActiveRecord::Base.connection.raw_connection
   end
@@ -202,8 +213,7 @@ class S3Inventory
   def tmp_directory
     @tmp_directory ||= begin
       current_db = RailsMultisite::ConnectionManagement.current_db
-      timestamp = Time.now.strftime("%Y-%m-%d-%H%M%S")
-      directory = File.join(Rails.root, "tmp", INVENTORY_PREFIX, current_db, timestamp)
+      directory = File.join(Rails.root, "tmp", INVENTORY_PREFIX, current_db)
       FileUtils.mkdir_p(directory)
       directory
     end

spec/components/s3_inventory_spec.rb

@@ -48,6 +48,8 @@ describe "S3Inventory" do
         next_marker: "eyJNYXJrZXIiOiBudWxsLCAiYm90b190cnVuY2F0ZV9hbW91bnQiOiAyfQ=="
       }
     })
+
+    inventory.stubs(:cleanup!)
   end
 
   it "should raise error if an inventory file is not found" do
@@ -67,9 +69,7 @@ describe "S3Inventory" do
     Fabricate(:upload, etag: "ETag2", created_at: Time.now)
     Fabricate(:upload, created_at: 2.days.ago)
 
-    inventory.expects(:download_inventory_files_to_tmp_directory)
-    inventory.expects(:decompress_inventory_files)
-    inventory.expects(:files).returns([{ key: "Key", filename: "#{csv_filename}.gz" }]).times(2)
+    inventory.expects(:files).returns([{ key: "Key", filename: "#{csv_filename}.gz" }]).times(3)
     inventory.expects(:inventory_date).returns(Time.now)
 
     output = capture_stdout do
@@ -87,9 +87,7 @@ describe "S3Inventory" do
     ]
     files.each { |file| Fabricate(:upload, url: file[0]) }
 
-    inventory.expects(:download_inventory_files_to_tmp_directory)
-    inventory.expects(:decompress_inventory_files)
-    inventory.expects(:files).returns([{ key: "Key", filename: "#{csv_filename}.gz" }]).times(2)
+    inventory.expects(:files).returns([{ key: "Key", filename: "#{csv_filename}.gz" }]).times(3)
 
     output = capture_stdout do
       expect { inventory.backfill_etags_and_list_missing }.to change { Upload.where(etag: nil).count }.by(-2)
@@ -111,9 +109,7 @@ describe "S3Inventory" do
     post.link_post_uploads
     upload.delete
 
-    inventory.expects(:download_inventory_files_to_tmp_directory)
-    inventory.expects(:decompress_inventory_files)
-    inventory.expects(:files).returns([{ key: "Key", filename: "#{csv_filename}.gz" }]).times(2)
+    inventory.expects(:files).returns([{ key: "Key", filename: "#{csv_filename}.gz" }]).times(3)
 
     output = capture_stdout do
       inventory.backfill_etags_and_list_missing