Multipart-uploaded files don't have a valid MD5 sum in their ETag.
We can detect it and disable MD5 comparison when deciding whether
to sync these files. In such a case only the size (and later on a
timestamp) is compared.
... | ... |
@@ -1,5 +1,7 @@ |
1 | 1 |
s3cmd 1.1.0 - ??? |
2 | 2 |
=========== |
3 |
+* MultiPart upload enabled for both [put] and [sync]. Default chunk |
|
4 |
+ size is 15MB. |
|
3 | 5 |
* CloudFront invalidation via [sync --cf-invalidate] and [cfinvalinfo]. |
4 | 6 |
* Increased socket_timeout from 10 secs to 5 mins. |
5 | 7 |
* Added "Static WebSite" support [ws-create / ws-delete / ws-info] |
... | ... |
@@ -301,8 +301,13 @@ def compare_filelists(src_list, dst_list, src_remote, dst_remote): |
301 | 301 |
debug(u"XFER: %s (size mismatch: src=%s dst=%s)" % (file, src_list[file]['size'], dst_list[file]['size'])) |
302 | 302 |
attribs_match = False |
303 | 303 |
|
304 |
- if attribs_match and 'md5' in cfg.sync_checks: |
|
305 |
- ## ... same size, check MD5 |
|
304 |
+ ## Check MD5 |
|
305 |
+ compare_md5 = 'md5' in cfg.sync_checks |
|
306 |
+ # Multipart-uploaded files don't have a valid MD5 sum - it ends with "...-NN" |
|
307 |
+ if compare_md5 and (src_remote == True and src_list[file]['md5'].find("-") >= 0) or (dst_remote == True and dst_list[file]['md5'].find("-") >= 0): |
|
308 |
+ compare_md5 = False |
|
309 |
+ info(u"Disabled MD5 check for %s" % file) |
|
310 |
+ if attribs_match and compare_md5: |
|
306 | 311 |
try: |
307 | 312 |
if src_remote == False and dst_remote == True: |
308 | 313 |
src_md5 = hash_file_md5(src_list[file]['full_name']) |
... | ... |
@@ -756,6 +756,7 @@ class S3(object): |
756 | 756 |
upload.upload_all_parts() |
757 | 757 |
response = upload.complete_multipart_upload() |
758 | 758 |
response["speed"] = 0 # XXX |
759 |
+ response["size"] = size |
|
759 | 760 |
return response |
760 | 761 |
|
761 | 762 |
def recv_file(self, request, stream, labels, start_position = 0, retries = _max_retries): |
... | ... |
@@ -837,10 +837,6 @@ def cmd_sync_local2remote(args): |
837 | 837 |
|
838 | 838 |
s3 = S3(cfg) |
839 | 839 |
|
840 |
- ## FIXME |
|
841 |
- cfg.multipart_enabled = False |
|
842 |
- warning(u"MultiPart: disabled for 'sync' command. Don't panic, we'll fix it!") |
|
843 |
- |
|
844 | 840 |
if cfg.encrypt: |
845 | 841 |
error(u"S3cmd 'sync' doesn't yet support GPG encryption, sorry.") |
846 | 842 |
error(u"Either use unconditional 's3cmd put --recursive'") |