Browse code

managed files that was not remote copy to upload them directly

Emmanuel authored on 2013/06/30 12:23:11
Showing 1 changed files
... ...
@@ -751,7 +751,14 @@ def cmd_sync_remote2remote(args):
751 751
     seq = 0
752 752
     seq = _upload(src_list, seq, src_count + update_count)
753 753
     seq = _upload(update_list, seq, src_count + update_count)
754
-    n_copied, bytes_saved = remote_copy(s3, copy_pairs, destination_base)
754
+    n_copied, bytes_saved, failed_copy_files = remote_copy(s3, copy_pairs, destination_base)
755
+
756
+    #process files not copied
757
+    output("Process files that was not remote copied")
758
+    failed_copy_count = len (failed_copy_files) 
759
+    for key in failed_copy_files:
760
+        failed_copy_files[key]['target_uri'] = destination_base + key
761
+    seq = _upload(failed_copy_files, seq, failed_copy_count)
755 762
 
756 763
     total_elapsed = time.time() - timestamp_start
757 764
     outstr = "Done. Copied %d files in %0.1f seconds, %0.2f files/s" % (seq, total_elapsed, seq/total_elapsed)
... ...
@@ -986,6 +993,7 @@ def local_copy(copy_pairs, destination_base):
986 986
 
987 987
 def remote_copy(s3, copy_pairs, destination_base):
988 988
     saved_bytes = 0
989
+    failed_copy_list = FileDict()
989 990
     for (src_obj, dst1, dst2) in copy_pairs:
990 991
         debug(u"Remote Copying from %s to %s" % (dst1, dst2))
991 992
         dst1_uri = S3Uri(destination_base + dst1)
... ...
@@ -997,8 +1005,11 @@ def remote_copy(s3, copy_pairs, destination_base):
997 997
             saved_bytes = saved_bytes + int(info['headers']['content-length'])
998 998
             output(u"remote copy: %s -> %s" % (dst1, dst2))
999 999
         except:
1000
-            raise
1001
-    return (len(copy_pairs), saved_bytes)
1000
+            #raise
1001
+            ## don't raise error but add file that can not be copied into array for further process
1002
+            warning(u'Unable to remote copy files %s -> %s' % (dst1_uri, dst2_uri))
1003
+            failed_copy_list[dst2_uri] = src_obj    
1004
+    return (len(copy_pairs), saved_bytes, failed_copy_list)
1002 1005
 
1003 1006
 def _build_attr_header(local_list, src):
1004 1007
     import pwd, grp
... ...
@@ -1188,7 +1199,14 @@ def cmd_sync_local2remote(args):
1188 1188
         timestamp_start = time.time()
1189 1189
         n, total_size = _upload(local_list, 0, local_count, total_size)
1190 1190
         n, total_size = _upload(update_list, n, local_count, total_size)
1191
-        n_copies, saved_bytes = remote_copy(s3, copy_pairs, destination_base)
1191
+        n_copies, saved_bytes, failed_copy_files  = remote_copy(s3, copy_pairs, destination_base)
1192
+
1193
+        #upload file that could not be copied
1194
+        output("Process files that was not remote copied")
1195
+        failed_copy_count = len(failed_copy_files)
1196
+        _set_remote_uri(failed_copy_files, destination_base, single_file_local)
1197
+        n, total_size = _upload(failed_copy_files, n, failed_copy_count, total_size)
1198
+
1192 1199
         if cfg.delete_removed and cfg.delete_after:
1193 1200
             _do_deletes(s3, remote_list)
1194 1201
         total_elapsed = time.time() - timestamp_start