diff --git a/app/lib/api_entreprise/api.rb b/app/lib/api_entreprise/api.rb index 117e73beb..79b2cc766 100644 --- a/app/lib/api_entreprise/api.rb +++ b/app/lib/api_entreprise/api.rb @@ -22,6 +22,8 @@ class ApiEntreprise::API call(RNA_RESOURCE_NAME, siret, procedure_id) end + private + def self.call(resource_name, siret_or_siren, procedure_id) url = url(resource_name, siret_or_siren) params = params(siret_or_siren, procedure_id) @@ -38,7 +40,9 @@ class ApiEntreprise::API end def self.url(resource_name, siret_or_siren) - [API_ENTREPRISE_URL, resource_name, siret_or_siren].join("/") + base_url = [API_ENTREPRISE_URL, resource_name, siret_or_siren].join("/") + + "#{base_url}?with_insee_v3=true" end def self.params(siret_or_siren, procedure_id) @@ -50,8 +54,6 @@ class ApiEntreprise::API } end - private - def self.token Rails.application.secrets.api_entreprise[:key] end diff --git a/lib/active_storage/service/cellar_service.rb b/lib/active_storage/service/cellar_service.rb index 43f2d0632..56b00e24b 100644 --- a/lib/active_storage/service/cellar_service.rb +++ b/lib/active_storage/service/cellar_service.rb @@ -37,7 +37,7 @@ module ActiveStorage def delete_prefixed(prefix) instrument :delete_prefixed, prefix: prefix do @adapter.session do |s| - keys = s.list_prefixed(prefix) + keys = s.list_prefixed(prefix).map(&:first) s.delete_keys(keys) end end diff --git a/lib/cellar/cellar_adapter.rb b/lib/cellar/cellar_adapter.rb index 1a1ed400c..0cd7e88d0 100644 --- a/lib/cellar/cellar_adapter.rb +++ b/lib/cellar/cellar_adapter.rb @@ -91,7 +91,7 @@ module Cellar if response.is_a?(Net::HTTPSuccess) (listing, truncated) = parse_bucket_listing(response.body) result += listing - marker = listing.last + marker = listing.last.first else # TODO: error handling return nil @@ -139,8 +139,13 @@ module Cellar def parse_bucket_listing(bucket_listing_xml) doc = Nokogiri::XML(bucket_listing_xml) listing = doc - .xpath('//xmlns:Contents/xmlns:Key') - .map(&:text) + .xpath('//xmlns:Contents') + .map do |node| + [ + node.xpath('xmlns:Key').text, + DateTime.iso8601(node.xpath('xmlns:LastModified').text) + ] + end truncated = doc.xpath('//xmlns:IsTruncated').text == 'true' [listing, truncated] end diff --git a/lib/tasks/2018_12_03_finish_piece_jointe_transfer.rake b/lib/tasks/2018_12_03_finish_piece_jointe_transfer.rake index fd8991ef3..c11db190f 100644 --- a/lib/tasks/2018_12_03_finish_piece_jointe_transfer.rake +++ b/lib/tasks/2018_12_03_finish_piece_jointe_transfer.rake @@ -67,23 +67,32 @@ namespace :'2018_12_03_finish_piece_jointe_transfer' do # This task ports them to the new storage after the switch, while being careful not to # overwrite attachments that may have changed in the new storage after the switch. def refresh_outdated_files - rake_puts "Refresh outdated attachments" - refreshed_keys = [] - missing_keys = [] old_pj_adapter.session do |old_pjs| - keys = old_pjs.list_prefixed('') - progress = ProgressReport.new(keys.count) - keys.each do |key| - new_pj_metadata = new_pjs.files.head(key) + rake_puts "List old PJs" + old_pj_listing = old_pjs.list_prefixed('') - refresh_needed = new_pj_metadata.nil? - if !refresh_needed - new_pj_last_modified = new_pj_metadata.last_modified.in_time_zone - old_pj_last_modified = old_pjs.last_modified(key) - if old_pj_last_modified.nil? - missing_keys.push(key) - else + rake_puts "List new PJs" + new_pj_listing = {} + progress = ProgressReport.new(new_pjs.count.to_i) + new_pjs.files.each do |f| + new_pj_listing[f.key] = f.last_modified.in_time_zone + progress.inc + end + progress.finish + + rake_puts "Refresh outdated attachments" + progress = ProgressReport.new(old_pj_listing.count) + old_pj_listing.each do |key, old_pj_last_modified| + new_pj_last_modified = new_pj_listing[key] + + if new_pj_last_modified.nil? || new_pj_last_modified < old_pj_last_modified + # Looks like we need to refresh this PJ. + # Fetch fresh metadata to avoid overwriting a last-minute change + new_pj_metadata = new_pjs.files.head(key) + refresh_needed = new_pj_metadata.nil? + if !refresh_needed + new_pj_last_modified = new_pj_metadata.last_modified.in_time_zone refresh_needed = new_pj_last_modified < old_pj_last_modified end end @@ -114,9 +123,6 @@ namespace :'2018_12_03_finish_piece_jointe_transfer' do if verbose? rake_puts "Refreshed #{refreshed_keys.count} attachments\n#{refreshed_keys.join(', ')}" end - if missing_keys.present? - rake_puts "Failed to refresh #{missing_keys.count} attachments\n#{missing_keys.join(', ')}" - end end # For OpenStack, the content type cannot be forced dynamically from a direct download URL. diff --git a/spec/lib/cellar/cellar_adapter_spec.rb b/spec/lib/cellar/cellar_adapter_spec.rb index 312a8cb77..c385a5ecf 100644 --- a/spec/lib/cellar/cellar_adapter_spec.rb +++ b/spec/lib/cellar/cellar_adapter_spec.rb @@ -43,7 +43,7 @@ describe 'CellarAdapter' do sample2.jpg - 2011-02-26T01:56:20.000Z + 2014-03-21T17:44:07.000Z "bf1d737a4d46a19f3bced6905cc8b902" 142863 STANDARD @@ -54,7 +54,17 @@ describe 'CellarAdapter' do subject { session.send(:parse_bucket_listing, response) } - it { is_expected.to eq([["sample1.jpg", "sample2.jpg"], false]) } + it do + is_expected.to eq( + [ + [ + ["sample1.jpg", DateTime.new(2011, 2, 26, 1, 56, 20, 0)], + ["sample2.jpg", DateTime.new(2014, 3, 21, 17, 44, 7, 0)] + ], + false + ] + ) + end end describe 'bulk_deletion_request_body' do