DGNum/demarches-normaliennes
13
1
Fork 0
Code Issues 1 Pull requests Projects Releases Packages Wiki Activity

[#2180] Avoid fetching old PJ modification times separately

Browse source 
Speeds up transfer
This commit is contained in:
Frederic Merizen 2018-12-08 15:55:40 +01:00
parent 7b18c51483
commit 270c7568fa
4 changed files with 25 additions and 19 deletions
Download patch file Download diff file Expand all files Collapse all files

17
lib/tasks/2018_12_03_finish_piece_jointe_transfer.rake
View file

@ -68,25 +68,19 @@ namespace :'2018_12_03_finish_piece_jointe_transfer' do
# overwrite attachments that may have changed in the new storage after the switch.
def refresh_outdated_files
refreshed_keys = []
missing_keys = []
old_pj_adapter.session do |old_pjs|
rake_puts "List old PJs"
keys = old_pjs.list_prefixed('')
old_pj_listing = old_pjs.list_prefixed('')
rake_puts "Refresh outdated attachments"
progress = ProgressReport.new(keys.count)
keys.each do |key|
progress = ProgressReport.new(old_pj_listing.count)
old_pj_listing.each do |key, old_pj_last_modified|
new_pj_metadata = new_pjs.files.head(key)
refresh_needed = new_pj_metadata.nil?
if !refresh_needed
new_pj_last_modified = new_pj_metadata.last_modified.in_time_zone
old_pj_last_modified = old_pjs.last_modified(key)
if old_pj_last_modified.nil?
missing_keys.push(key)
else
refresh_needed = new_pj_last_modified < old_pj_last_modified
end
refresh_needed = new_pj_last_modified < old_pj_last_modified
end
if refresh_needed
@ -115,9 +109,6 @@ namespace :'2018_12_03_finish_piece_jointe_transfer' do
if verbose?
rake_puts "Refreshed #{refreshed_keys.count} attachments\n#{refreshed_keys.join(', ')}"
end
if missing_keys.present?
rake_puts "Failed to refresh #{missing_keys.count} attachments\n#{missing_keys.join(', ')}"
end
end
# For OpenStack, the content type cannot be forced dynamically from a direct download URL.