...
|
...
|
@@ -751,7 +751,14 @@ def cmd_sync_remote2remote(args):
|
751
|
751
|
seq = 0
|
752
|
752
|
seq = _upload(src_list, seq, src_count + update_count)
|
753
|
753
|
seq = _upload(update_list, seq, src_count + update_count)
|
754
|
|
- n_copied, bytes_saved = remote_copy(s3, copy_pairs, destination_base)
|
|
754
|
+ n_copied, bytes_saved, failed_copy_files = remote_copy(s3, copy_pairs, destination_base)
|
|
755
|
+
|
|
756
|
+ #process files not copied
|
|
757
|
+ output("Process files that was not remote copied")
|
|
758
|
+ failed_copy_count = len (failed_copy_files)
|
|
759
|
+ for key in failed_copy_files:
|
|
760
|
+ failed_copy_files[key]['target_uri'] = destination_base + key
|
|
761
|
+ seq = _upload(failed_copy_files, seq, failed_copy_count)
|
755
|
762
|
|
756
|
763
|
total_elapsed = time.time() - timestamp_start
|
757
|
764
|
outstr = "Done. Copied %d files in %0.1f seconds, %0.2f files/s" % (seq, total_elapsed, seq/total_elapsed)
|
...
|
...
|
@@ -986,6 +993,7 @@ def local_copy(copy_pairs, destination_base):
|
986
|
986
|
|
987
|
987
|
def remote_copy(s3, copy_pairs, destination_base):
|
988
|
988
|
saved_bytes = 0
|
|
989
|
+ failed_copy_list = FileDict()
|
989
|
990
|
for (src_obj, dst1, dst2) in copy_pairs:
|
990
|
991
|
debug(u"Remote Copying from %s to %s" % (dst1, dst2))
|
991
|
992
|
dst1_uri = S3Uri(destination_base + dst1)
|
...
|
...
|
@@ -997,8 +1005,11 @@ def remote_copy(s3, copy_pairs, destination_base):
|
997
|
997
|
saved_bytes = saved_bytes + int(info['headers']['content-length'])
|
998
|
998
|
output(u"remote copy: %s -> %s" % (dst1, dst2))
|
999
|
999
|
except:
|
1000
|
|
- raise
|
1001
|
|
- return (len(copy_pairs), saved_bytes)
|
|
1000
|
+ #raise
|
|
1001
|
+ ## don't raise error but add file that can not be copied into array for further process
|
|
1002
|
+ warning(u'Unable to remote copy files %s -> %s' % (dst1_uri, dst2_uri))
|
|
1003
|
+ failed_copy_list[dst2_uri] = src_obj
|
|
1004
|
+ return (len(copy_pairs), saved_bytes, failed_copy_list)
|
1002
|
1005
|
|
1003
|
1006
|
def _build_attr_header(local_list, src):
|
1004
|
1007
|
import pwd, grp
|
...
|
...
|
@@ -1188,7 +1199,14 @@ def cmd_sync_local2remote(args):
|
1188
|
1188
|
timestamp_start = time.time()
|
1189
|
1189
|
n, total_size = _upload(local_list, 0, local_count, total_size)
|
1190
|
1190
|
n, total_size = _upload(update_list, n, local_count, total_size)
|
1191
|
|
- n_copies, saved_bytes = remote_copy(s3, copy_pairs, destination_base)
|
|
1191
|
+ n_copies, saved_bytes, failed_copy_files = remote_copy(s3, copy_pairs, destination_base)
|
|
1192
|
+
|
|
1193
|
+ #upload file that could not be copied
|
|
1194
|
+ output("Process files that was not remote copied")
|
|
1195
|
+ failed_copy_count = len(failed_copy_files)
|
|
1196
|
+ _set_remote_uri(failed_copy_files, destination_base, single_file_local)
|
|
1197
|
+ n, total_size = _upload(failed_copy_files, n, failed_copy_count, total_size)
|
|
1198
|
+
|
1192
|
1199
|
if cfg.delete_removed and cfg.delete_after:
|
1193
|
1200
|
_do_deletes(s3, remote_list)
|
1194
|
1201
|
total_elapsed = time.time() - timestamp_start
|