Browse code

Merge remote-tracking branch 'spartman/remote-copy-fix' into merge

Matt Domsch authored on 2013/11/26 10:48:23
Showing 1 changed files
... ...
@@ -757,7 +757,14 @@ def cmd_sync_remote2remote(args):
757 757
     seq = 0
758 758
     seq = _upload(src_list, seq, src_count + update_count)
759 759
     seq = _upload(update_list, seq, src_count + update_count)
760
-    n_copied, bytes_saved = remote_copy(s3, copy_pairs, destination_base)
760
+    n_copied, bytes_saved, failed_copy_files = remote_copy(s3, copy_pairs, destination_base)
761
+
762
+    #process files not copied
763
+    output("Process files that was not remote copied")
764
+    failed_copy_count = len (failed_copy_files) 
765
+    for key in failed_copy_files:
766
+        failed_copy_files[key]['target_uri'] = destination_base + key
767
+    seq = _upload(failed_copy_files, seq, failed_copy_count)
761 768
 
762 769
     total_elapsed = time.time() - timestamp_start
763 770
     outstr = "Done. Copied %d files in %0.1f seconds, %0.2f files/s" % (seq, total_elapsed, seq/total_elapsed)
... ...
@@ -1001,6 +1008,7 @@ def local_copy(copy_pairs, destination_base):
1001 1001
 
1002 1002
 def remote_copy(s3, copy_pairs, destination_base):
1003 1003
     saved_bytes = 0
1004
+    failed_copy_list = FileDict()
1004 1005
     for (src_obj, dst1, dst2) in copy_pairs:
1005 1006
         debug(u"Remote Copying from %s to %s" % (dst1, dst2))
1006 1007
         dst1_uri = S3Uri(destination_base + dst1)
... ...
@@ -1012,8 +1020,11 @@ def remote_copy(s3, copy_pairs, destination_base):
1012 1012
             saved_bytes = saved_bytes + int(info['headers']['content-length'])
1013 1013
             output(u"remote copy: %s -> %s" % (dst1, dst2))
1014 1014
         except:
1015
-            raise
1016
-    return (len(copy_pairs), saved_bytes)
1015
+            #raise
1016
+            ## don't raise error but add file that can not be copied into array for further process
1017
+            warning(u'Unable to remote copy files %s -> %s' % (dst1_uri, dst2_uri))
1018
+            failed_copy_list[dst2_uri] = src_obj    
1019
+    return (len(copy_pairs), saved_bytes, failed_copy_list)
1017 1020
 
1018 1021
 def _build_attr_header(local_list, src):
1019 1022
     import pwd, grp
... ...
@@ -1203,7 +1214,14 @@ def cmd_sync_local2remote(args):
1203 1203
         timestamp_start = time.time()
1204 1204
         n, total_size = _upload(local_list, 0, local_count, total_size)
1205 1205
         n, total_size = _upload(update_list, n, local_count, total_size)
1206
-        n_copies, saved_bytes = remote_copy(s3, copy_pairs, destination_base)
1206
+        n_copies, saved_bytes, failed_copy_files  = remote_copy(s3, copy_pairs, destination_base)
1207
+
1208
+        #upload file that could not be copied
1209
+        output("Process files that was not remote copied")
1210
+        failed_copy_count = len(failed_copy_files)
1211
+        _set_remote_uri(failed_copy_files, destination_base, single_file_local)
1212
+        n, total_size = _upload(failed_copy_files, n, failed_copy_count, total_size)
1213
+
1207 1214
         if cfg.delete_removed and cfg.delete_after:
1208 1215
             _do_deletes(s3, remote_list)
1209 1216
         total_elapsed = time.time() - timestamp_start