Fixed a bug which didn't invalidate any files which were copied remotely
during sync
Based on work of kgritesh #939.
A little bit more efficient version
... | ... |
@@ -1119,7 +1119,7 @@ def cmd_sync_remote2remote(args): |
1119 | 1119 |
total_size_copied += size |
1120 | 1120 |
|
1121 | 1121 |
|
1122 |
- n_copied, bytes_saved, failed_copy_files = remote_copy(s3, copy_pairs, destination_base) |
|
1122 |
+ n_copied, bytes_saved, failed_copy_files = remote_copy(s3, copy_pairs, destination_base, None) |
|
1123 | 1123 |
total_files_copied += n_copied |
1124 | 1124 |
total_size_copied += bytes_saved |
1125 | 1125 |
|
... | ... |
@@ -1527,7 +1527,7 @@ def local_copy(copy_pairs, destination_base): |
1527 | 1527 |
failed_copy_list[relative_file] = src_obj |
1528 | 1528 |
return len(copy_pairs), saved_bytes, failed_copy_list |
1529 | 1529 |
|
1530 |
-def remote_copy(s3, copy_pairs, destination_base): |
|
1530 |
+def remote_copy(s3, copy_pairs, destination_base, uploaded_objects_list=None): |
|
1531 | 1531 |
cfg = Config() |
1532 | 1532 |
saved_bytes = 0 |
1533 | 1533 |
failed_copy_list = FileDict() |
... | ... |
@@ -1540,6 +1540,8 @@ def remote_copy(s3, copy_pairs, destination_base): |
1540 | 1540 |
s3.object_copy(dst1_uri, dst2_uri, extra_headers) |
1541 | 1541 |
saved_bytes += src_obj.get(u'size', 0) |
1542 | 1542 |
output(u"remote copy: '%s' -> '%s'" % (dst1, dst2)) |
1543 |
+ if uploaded_objects_list is not None: |
|
1544 |
+ uploaded_objects_list.append(dst2) |
|
1543 | 1545 |
except: |
1544 | 1546 |
warning(u"Unable to remote copy files '%s' -> '%s'" % (dst1_uri, dst2_uri)) |
1545 | 1547 |
failed_copy_list[dst2] = src_obj |
... | ... |
@@ -1770,7 +1772,12 @@ def cmd_sync_local2remote(args): |
1770 | 1770 |
status, n, size_transferred = _upload(update_list, n, upload_count, size_transferred) |
1771 | 1771 |
if ret == EX_OK: |
1772 | 1772 |
ret = status |
1773 |
- n_copies, saved_bytes, failed_copy_files = remote_copy(s3, copy_pairs, destination_base) |
|
1773 |
+ # uploaded_objects_list reference is passed so it can be filled with |
|
1774 |
+ # destination object of succcessful copies so that they can be |
|
1775 |
+ # invalidated by cf |
|
1776 |
+ n_copies, saved_bytes, failed_copy_files = remote_copy(s3, copy_pairs, |
|
1777 |
+ destination_base, |
|
1778 |
+ uploaded_objects_list) |
|
1774 | 1779 |
|
1775 | 1780 |
#upload file that could not be copied |
1776 | 1781 |
debug("Process files that were not remote copied") |