[#2180] Iterate over storage keys rather than blobs

To create less confusion on dev

lib/cellar/cellar_adapter.rb

@@ -81,12 +81,24 @@ module Cellar
       end
 
       def list_prefixed(prefix)
-        request = Net::HTTP::Get.new("/?prefix=#{prefix}")
+        result = []
-        @signer.sign(request, "")
+        marker = ''
-        response = @http.request(request)
+
-        if response.is_a?(Net::HTTPSuccess)
+        begin
-          parse_bucket_listing(response.body)
+          request = Net::HTTP::Get.new("/?prefix=#{prefix}&marker=#{marker}")
-        end
+          @signer.sign(request, "")
+          response = @http.request(request)
+          if response.is_a?(Net::HTTPSuccess)
+            (listing, truncated) = parse_bucket_listing(response.body)
+            result += listing
+            marker = listing.last
+          else
+            # TODO: error handling
+            return nil
+          end
+        end while truncated
+
+        result
       end
 
       def delete_keys(keys)
@@ -126,9 +138,11 @@ module Cellar
 
       def parse_bucket_listing(bucket_listing_xml)
         doc = Nokogiri::XML(bucket_listing_xml)
-        doc
+        listing = doc
           .xpath('//xmlns:Contents/xmlns:Key')
           .map(&:text)
+        truncated = doc.xpath('//xmlns:IsTruncated').text == 'true'
+        [listing, truncated]
       end
 
       def bulk_deletion_request_body(keys)

lib/tasks/2018_12_03_finish_piece_jointe_transfer.rake

@@ -69,35 +69,36 @@ namespace :'2018_12_03_finish_piece_jointe_transfer' do
       def refresh_outdated_files
         rake_puts "Refresh outdated attachments"
 
-        progress = ProgressReport.new(ActiveStorage::Blob.count)
         refreshed_keys = []
         missing_keys = []
         old_pj_adapter.session do |old_pjs|
-          ActiveStorage::Blob.find_each do |blob|
+          keys = old_pjs.list_prefixed('')
-            new_pj_metadata = new_pjs.files.head(blob.key)
+          progress = ProgressReport.new(keys.count)
+          keys.each do |key|
+            new_pj_metadata = new_pjs.files.head(key)
 
             refresh_needed = new_pj_metadata.nil?
             if !refresh_needed
               new_pj_last_modified = new_pj_metadata.last_modified.in_time_zone
-              old_pj_last_modified = old_pjs.last_modified(blob.key)
+              old_pj_last_modified = old_pjs.last_modified(key)
               if old_pj_last_modified.nil?
-                missing_keys.push(blob.key)
+                missing_keys.push(key)
               else
                 refresh_needed = new_pj_last_modified < old_pj_last_modified
               end
             end
 
             if refresh_needed
-              refreshed_keys.push(blob.key)
+              refreshed_keys.push(key)
               if force?
-                file = Tempfile.new(blob.key)
+                file = Tempfile.new(key)
                 file.binmode
-                old_pjs.download(blob.key) do |chunk|
+                old_pjs.download(key) do |chunk|
                   file.write(chunk)
                 end
                 file.rewind
                 new_pjs.files.create(
-                  :key    => blob.key,
+                  :key    => key,
                   :body   => file,
                   :public => false
                 )
@@ -107,8 +108,8 @@ namespace :'2018_12_03_finish_piece_jointe_transfer' do
             end
             progress.inc
           end
+          progress.finish
         end
-        progress.finish
 
         if verbose?
           rake_puts "Refreshed #{refreshed_keys.count} attachments\n#{refreshed_keys.join(', ')}"
@@ -132,15 +133,18 @@ namespace :'2018_12_03_finish_piece_jointe_transfer' do
         end
         rake_puts "Fix MIME types"
 
-        progress = ProgressReport.new(ActiveStorage::Blob.count)
+        progress = ProgressReport.new(new_pjs.count.to_i)
         failed_keys = []
         updated_keys = []
-        ActiveStorage::Blob.find_each do |blob|
+        new_pjs.files.each do |file|
-          if blob.identified? && blob.content_type.present?
+          blob = ActiveStorage::Blob.find_by(key: file.key)
-            updated_keys.push(blob.key)
+          if blob.nil?
+            failed_keys.push(file.key)
+          elsif blob.identified? && blob.content_type.present?
+            updated_keys.push(file.key)
             if force?
-              if !blob.service.change_content_type(blob.key, blob.content_type)
+              if !blob.service.change_content_type(file.key, blob.content_type)
-                failed_keys.push(blob.key)
+                failed_keys.push(file.key)
               end
             end
           end
@@ -152,7 +156,7 @@ namespace :'2018_12_03_finish_piece_jointe_transfer' do
           rake_puts "Updated MIME Type for #{updated_keys.count} keys\n#{updated_keys.join(', ')}"
         end
         if failed_keys.present?
-          rake_puts "failed to update #{failed_keys.count} keys (dangling blob?)\n#{failed_keys.join(', ')}"
+          rake_puts "failed to update #{failed_keys.count} keys\n#{failed_keys.join(', ')}"
         end
       end
 

spec/lib/cellar/cellar_adapter_spec.rb

@@ -54,7 +54,7 @@ describe 'CellarAdapter' do
 
     subject { session.send(:parse_bucket_listing, response) }
 
-    it { is_expected.to eq(["sample1.jpg", "sample2.jpg"]) }
+    it { is_expected.to eq([["sample1.jpg", "sample2.jpg"], false]) }
   end
 
   describe 'bulk_deletion_request_body' do