Browse code

Fixes #525 - Fixed no invalidate of remotely copied files during sync

Fixed a bug which didn't invalidate any files which were copied remotely
during sync
Based on work of kgritesh #939.
A little bit more efficient version

Florent Viard authored on 2017/12/08 08:41:59
Showing 1 changed files
... ...
@@ -1119,7 +1119,7 @@ def cmd_sync_remote2remote(args):
1119 1119
     total_size_copied += size
1120 1120
 
1121 1121
 
1122
-    n_copied, bytes_saved, failed_copy_files = remote_copy(s3, copy_pairs, destination_base)
1122
+    n_copied, bytes_saved, failed_copy_files = remote_copy(s3, copy_pairs, destination_base, None)
1123 1123
     total_files_copied += n_copied
1124 1124
     total_size_copied += bytes_saved
1125 1125
 
... ...
@@ -1527,7 +1527,7 @@ def local_copy(copy_pairs, destination_base):
1527 1527
             failed_copy_list[relative_file] = src_obj
1528 1528
     return len(copy_pairs), saved_bytes, failed_copy_list
1529 1529
 
1530
-def remote_copy(s3, copy_pairs, destination_base):
1530
+def remote_copy(s3, copy_pairs, destination_base, uploaded_objects_list=None):
1531 1531
     cfg = Config()
1532 1532
     saved_bytes = 0
1533 1533
     failed_copy_list = FileDict()
... ...
@@ -1540,6 +1540,8 @@ def remote_copy(s3, copy_pairs, destination_base):
1540 1540
             s3.object_copy(dst1_uri, dst2_uri, extra_headers)
1541 1541
             saved_bytes += src_obj.get(u'size', 0)
1542 1542
             output(u"remote copy: '%s' -> '%s'" % (dst1, dst2))
1543
+            if uploaded_objects_list is not None:
1544
+                uploaded_objects_list.append(dst2)
1543 1545
         except:
1544 1546
             warning(u"Unable to remote copy files '%s' -> '%s'" % (dst1_uri, dst2_uri))
1545 1547
             failed_copy_list[dst2] = src_obj
... ...
@@ -1770,7 +1772,12 @@ def cmd_sync_local2remote(args):
1770 1770
         status, n, size_transferred = _upload(update_list, n, upload_count, size_transferred)
1771 1771
         if ret == EX_OK:
1772 1772
             ret = status
1773
-        n_copies, saved_bytes, failed_copy_files  = remote_copy(s3, copy_pairs, destination_base)
1773
+        # uploaded_objects_list reference is passed so it can be filled with
1774
+        # destination object of succcessful copies so that they can be
1775
+        # invalidated by cf
1776
+        n_copies, saved_bytes, failed_copy_files  = remote_copy(s3, copy_pairs,
1777
+                                                                destination_base,
1778
+                                                                uploaded_objects_list)
1774 1779
 
1775 1780
         #upload file that could not be copied
1776 1781
         debug("Process files that were not remote copied")