Multipart-uploaded files don't have a valid MD5 sum in their ETag.
We can detect it and disable MD5 comparison when deciding whether
to sync these files. In such a case only the size (and later on a
timestamp) is compared.
| ... | ... |
@@ -1,5 +1,7 @@ |
| 1 | 1 |
s3cmd 1.1.0 - ??? |
| 2 | 2 |
=========== |
| 3 |
+* MultiPart upload enabled for both [put] and [sync]. Default chunk |
|
| 4 |
+ size is 15MB. |
|
| 3 | 5 |
* CloudFront invalidation via [sync --cf-invalidate] and [cfinvalinfo]. |
| 4 | 6 |
* Increased socket_timeout from 10 secs to 5 mins. |
| 5 | 7 |
* Added "Static WebSite" support [ws-create / ws-delete / ws-info] |
| ... | ... |
@@ -301,8 +301,13 @@ def compare_filelists(src_list, dst_list, src_remote, dst_remote): |
| 301 | 301 |
debug(u"XFER: %s (size mismatch: src=%s dst=%s)" % (file, src_list[file]['size'], dst_list[file]['size'])) |
| 302 | 302 |
attribs_match = False |
| 303 | 303 |
|
| 304 |
- if attribs_match and 'md5' in cfg.sync_checks: |
|
| 305 |
- ## ... same size, check MD5 |
|
| 304 |
+ ## Check MD5 |
|
| 305 |
+ compare_md5 = 'md5' in cfg.sync_checks |
|
| 306 |
+ # Multipart-uploaded files don't have a valid MD5 sum - it ends with "...-NN" |
|
| 307 |
+ if compare_md5 and (src_remote == True and src_list[file]['md5'].find("-") >= 0) or (dst_remote == True and dst_list[file]['md5'].find("-") >= 0):
|
|
| 308 |
+ compare_md5 = False |
|
| 309 |
+ info(u"Disabled MD5 check for %s" % file) |
|
| 310 |
+ if attribs_match and compare_md5: |
|
| 306 | 311 |
try: |
| 307 | 312 |
if src_remote == False and dst_remote == True: |
| 308 | 313 |
src_md5 = hash_file_md5(src_list[file]['full_name']) |
| ... | ... |
@@ -756,6 +756,7 @@ class S3(object): |
| 756 | 756 |
upload.upload_all_parts() |
| 757 | 757 |
response = upload.complete_multipart_upload() |
| 758 | 758 |
response["speed"] = 0 # XXX |
| 759 |
+ response["size"] = size |
|
| 759 | 760 |
return response |
| 760 | 761 |
|
| 761 | 762 |
def recv_file(self, request, stream, labels, start_position = 0, retries = _max_retries): |
| ... | ... |
@@ -837,10 +837,6 @@ def cmd_sync_local2remote(args): |
| 837 | 837 |
|
| 838 | 838 |
s3 = S3(cfg) |
| 839 | 839 |
|
| 840 |
- ## FIXME |
|
| 841 |
- cfg.multipart_enabled = False |
|
| 842 |
- warning(u"MultiPart: disabled for 'sync' command. Don't panic, we'll fix it!") |
|
| 843 |
- |
|
| 844 | 840 |
if cfg.encrypt: |
| 845 | 841 |
error(u"S3cmd 'sync' doesn't yet support GPG encryption, sorry.") |
| 846 | 842 |
error(u"Either use unconditional 's3cmd put --recursive'") |