We were not getting the md5 value from the attribute header of each
file for the recursive case, only for the non-recursive case. That
was stupid. We were certainly _setting_ the md5value into the
attribute header.
This should eliminate the
INFO: disabled md5 check for...
messages on subsequent syncs when s3cmd 1.5.0-alpha+ is used to upload
content in the first place.
| ... | ... |
@@ -257,6 +257,20 @@ def fetch_local_list(args, recursive = None): |
| 257 | 257 |
return local_list, single_file |
| 258 | 258 |
|
| 259 | 259 |
def fetch_remote_list(args, require_attribs = False, recursive = None): |
| 260 |
+ def _get_remote_attribs(uri, remote_item): |
|
| 261 |
+ response = S3(cfg).object_info(uri) |
|
| 262 |
+ remote_item.update({
|
|
| 263 |
+ 'size': int(response['headers']['content-length']), |
|
| 264 |
+ 'md5': response['headers']['etag'].strip('"\''),
|
|
| 265 |
+ 'timestamp' : dateRFC822toUnix(response['headers']['date']) |
|
| 266 |
+ }) |
|
| 267 |
+ # get md5 from header if it's present. We would have set that during upload |
|
| 268 |
+ if response['headers'].has_key('x-amz-meta-s3cmd-attrs'):
|
|
| 269 |
+ attrs = parse_attrs_header(response['headers']['x-amz-meta-s3cmd-attrs']) |
|
| 270 |
+ if attrs.has_key('md5'):
|
|
| 271 |
+ remote_item.update({'md5': attrs['md5']})
|
|
| 272 |
+ debug(u"retreived md5=%s from headers" % (attrs['md5'])) |
|
| 273 |
+ |
|
| 260 | 274 |
def _get_filelist_remote(remote_uri, recursive = True): |
| 261 | 275 |
## If remote_uri ends with '/' then all remote files will have |
| 262 | 276 |
## the remote_uri prefix removed in the relative path. |
| ... | ... |
@@ -306,7 +320,9 @@ def fetch_remote_list(args, require_attribs = False, recursive = None): |
| 306 | 306 |
'dev' : None, |
| 307 | 307 |
'inode' : None, |
| 308 | 308 |
} |
| 309 |
- md5 = object['ETag'][1:-1] |
|
| 309 |
+ if require_attribs or rem_list[key]['md5'].find("-"): # always get it for multipart uploads
|
|
| 310 |
+ _get_remote_attribs(S3Uri(object_uri_str), rem_list[key]) |
|
| 311 |
+ md5 = rem_list[key]['md5'] |
|
| 310 | 312 |
rem_list.record_md5(key, md5) |
| 311 | 313 |
if break_now: |
| 312 | 314 |
break |
| ... | ... |
@@ -365,19 +381,12 @@ def fetch_remote_list(args, require_attribs = False, recursive = None): |
| 365 | 365 |
'object_key': uri.object() |
| 366 | 366 |
} |
| 367 | 367 |
if require_attribs: |
| 368 |
- response = S3(cfg).object_info(uri) |
|
| 369 |
- remote_item.update({
|
|
| 370 |
- 'size': int(response['headers']['content-length']), |
|
| 371 |
- 'md5': response['headers']['etag'].strip('"\''),
|
|
| 372 |
- 'timestamp' : dateRFC822toUnix(response['headers']['date']) |
|
| 373 |
- }) |
|
| 374 |
- # get md5 from header if it's present. We would have set that during upload |
|
| 375 |
- if response['headers'].has_key('x-amz-meta-s3cmd-attrs'):
|
|
| 376 |
- attrs = parse_attrs_header(response['headers']['x-amz-meta-s3cmd-attrs']) |
|
| 377 |
- if attrs.has_key('md5'):
|
|
| 378 |
- remote_item.update({'md5': attrs['md5']})
|
|
| 368 |
+ _get_remote_attribs(uri, remote_item) |
|
| 379 | 369 |
|
| 380 | 370 |
remote_list[key] = remote_item |
| 371 |
+ md5 = remote_item.get('md5')
|
|
| 372 |
+ if md5: |
|
| 373 |
+ remote_list.record_md5(key, md5) |
|
| 381 | 374 |
return remote_list |
| 382 | 375 |
|
| 383 | 376 |
def parse_attrs_header(attrs_header): |