...
|
...
|
@@ -757,7 +757,14 @@ def cmd_sync_remote2remote(args):
|
757
|
757
|
seq = 0
|
758
|
758
|
seq = _upload(src_list, seq, src_count + update_count)
|
759
|
759
|
seq = _upload(update_list, seq, src_count + update_count)
|
760
|
|
- n_copied, bytes_saved = remote_copy(s3, copy_pairs, destination_base)
|
|
760
|
+ n_copied, bytes_saved, failed_copy_files = remote_copy(s3, copy_pairs, destination_base)
|
|
761
|
+
|
|
762
|
+ #process files not copied
|
|
763
|
+ output("Process files that was not remote copied")
|
|
764
|
+ failed_copy_count = len (failed_copy_files)
|
|
765
|
+ for key in failed_copy_files:
|
|
766
|
+ failed_copy_files[key]['target_uri'] = destination_base + key
|
|
767
|
+ seq = _upload(failed_copy_files, seq, failed_copy_count)
|
761
|
768
|
|
762
|
769
|
total_elapsed = time.time() - timestamp_start
|
763
|
770
|
outstr = "Done. Copied %d files in %0.1f seconds, %0.2f files/s" % (seq, total_elapsed, seq/total_elapsed)
|
...
|
...
|
@@ -1001,6 +1008,7 @@ def local_copy(copy_pairs, destination_base):
|
1001
|
1001
|
|
1002
|
1002
|
def remote_copy(s3, copy_pairs, destination_base):
|
1003
|
1003
|
saved_bytes = 0
|
|
1004
|
+ failed_copy_list = FileDict()
|
1004
|
1005
|
for (src_obj, dst1, dst2) in copy_pairs:
|
1005
|
1006
|
debug(u"Remote Copying from %s to %s" % (dst1, dst2))
|
1006
|
1007
|
dst1_uri = S3Uri(destination_base + dst1)
|
...
|
...
|
@@ -1012,8 +1020,11 @@ def remote_copy(s3, copy_pairs, destination_base):
|
1012
|
1012
|
saved_bytes = saved_bytes + int(info['headers']['content-length'])
|
1013
|
1013
|
output(u"remote copy: %s -> %s" % (dst1, dst2))
|
1014
|
1014
|
except:
|
1015
|
|
- raise
|
1016
|
|
- return (len(copy_pairs), saved_bytes)
|
|
1015
|
+ #raise
|
|
1016
|
+ ## don't raise error but add file that can not be copied into array for further process
|
|
1017
|
+ warning(u'Unable to remote copy files %s -> %s' % (dst1_uri, dst2_uri))
|
|
1018
|
+ failed_copy_list[dst2_uri] = src_obj
|
|
1019
|
+ return (len(copy_pairs), saved_bytes, failed_copy_list)
|
1017
|
1020
|
|
1018
|
1021
|
def _build_attr_header(local_list, src):
|
1019
|
1022
|
import pwd, grp
|
...
|
...
|
@@ -1203,7 +1214,14 @@ def cmd_sync_local2remote(args):
|
1203
|
1203
|
timestamp_start = time.time()
|
1204
|
1204
|
n, total_size = _upload(local_list, 0, local_count, total_size)
|
1205
|
1205
|
n, total_size = _upload(update_list, n, local_count, total_size)
|
1206
|
|
- n_copies, saved_bytes = remote_copy(s3, copy_pairs, destination_base)
|
|
1206
|
+ n_copies, saved_bytes, failed_copy_files = remote_copy(s3, copy_pairs, destination_base)
|
|
1207
|
+
|
|
1208
|
+ #upload file that could not be copied
|
|
1209
|
+ output("Process files that was not remote copied")
|
|
1210
|
+ failed_copy_count = len(failed_copy_files)
|
|
1211
|
+ _set_remote_uri(failed_copy_files, destination_base, single_file_local)
|
|
1212
|
+ n, total_size = _upload(failed_copy_files, n, failed_copy_count, total_size)
|
|
1213
|
+
|
1207
|
1214
|
if cfg.delete_removed and cfg.delete_after:
|
1208
|
1215
|
_do_deletes(s3, remote_list)
|
1209
|
1216
|
total_elapsed = time.time() - timestamp_start
|