Browse code

Enable multipart for [sync] - do not check MD5

Multipart-uploaded files don't have a valid MD5 sum in their ETag.
We can detect it and disable MD5 comparison when deciding whether
to sync these files. In such a case only the size (and later on a
timestamp) is compared.

Michal Ludvig authored on 2012/01/12 11:30:52
Showing 4 changed files
... ...
@@ -1,5 +1,7 @@
1 1
 s3cmd 1.1.0   -   ???
2 2
 ===========
3
+* MultiPart upload enabled for both [put] and [sync]. Default chunk
4
+  size is 15MB.
3 5
 * CloudFront invalidation via [sync --cf-invalidate] and [cfinvalinfo].
4 6
 * Increased socket_timeout from 10 secs to 5 mins.
5 7
 * Added "Static WebSite" support [ws-create / ws-delete / ws-info]
... ...
@@ -301,8 +301,13 @@ def compare_filelists(src_list, dst_list, src_remote, dst_remote):
301 301
                 debug(u"XFER: %s (size mismatch: src=%s dst=%s)" % (file, src_list[file]['size'], dst_list[file]['size']))
302 302
                 attribs_match = False
303 303
 
304
-            if attribs_match and 'md5' in cfg.sync_checks:
305
-                ## ... same size, check MD5
304
+            ## Check MD5
305
+            compare_md5 = 'md5' in cfg.sync_checks
306
+            # Multipart-uploaded files don't have a valid MD5 sum - it ends with "...-NN"
307
+            if compare_md5 and (src_remote == True and src_list[file]['md5'].find("-") >= 0) or (dst_remote == True and dst_list[file]['md5'].find("-") >= 0):
308
+                compare_md5 = False
309
+                info(u"Disabled MD5 check for %s" % file)
310
+            if attribs_match and compare_md5:
306 311
                 try:
307 312
                     if src_remote == False and dst_remote == True:
308 313
                         src_md5 = hash_file_md5(src_list[file]['full_name'])
... ...
@@ -756,6 +756,7 @@ class S3(object):
756 756
         upload.upload_all_parts()
757 757
         response = upload.complete_multipart_upload()
758 758
         response["speed"] = 0 # XXX
759
+        response["size"] = size
759 760
         return response
760 761
 
761 762
     def recv_file(self, request, stream, labels, start_position = 0, retries = _max_retries):
... ...
@@ -837,10 +837,6 @@ def cmd_sync_local2remote(args):
837 837
 
838 838
     s3 = S3(cfg)
839 839
 
840
-    ## FIXME
841
-    cfg.multipart_enabled = False
842
-    warning(u"MultiPart: disabled for 'sync' command. Don't panic, we'll fix it!")
843
-
844 840
     if cfg.encrypt:
845 841
         error(u"S3cmd 'sync' doesn't yet support GPG encryption, sorry.")
846 842
         error(u"Either use unconditional 's3cmd put --recursive'")