|
...
|
...
|
@@ -300,6 +300,20 @@ def fetch_local_list(args, is_src = False, recursive = None):
|
|
300
|
300
|
return local_list, single_file
|
|
301
|
301
|
|
|
302
|
302
|
def fetch_remote_list(args, require_attribs = False, recursive = None):
|
|
|
303
|
+ def _get_remote_attribs(uri, remote_item):
|
|
|
304
|
+ response = S3(cfg).object_info(uri)
|
|
|
305
|
+ remote_item.update({
|
|
|
306
|
+ 'size': int(response['headers']['content-length']),
|
|
|
307
|
+ 'md5': response['headers']['etag'].strip('"\''),
|
|
|
308
|
+ 'timestamp' : dateRFC822toUnix(response['headers']['date'])
|
|
|
309
|
+ })
|
|
|
310
|
+ # get md5 from header if it's present. We would have set that during upload
|
|
|
311
|
+ if response['headers'].has_key('x-amz-meta-s3cmd-attrs'):
|
|
|
312
|
+ attrs = parse_attrs_header(response['headers']['x-amz-meta-s3cmd-attrs'])
|
|
|
313
|
+ if attrs.has_key('md5'):
|
|
|
314
|
+ remote_item.update({'md5': attrs['md5']})
|
|
|
315
|
+ debug(u"retreived md5=%s from headers" % (attrs['md5']))
|
|
|
316
|
+
|
|
303
|
317
|
def _get_filelist_remote(remote_uri, recursive = True):
|
|
304
|
318
|
## If remote_uri ends with '/' then all remote files will have
|
|
305
|
319
|
## the remote_uri prefix removed in the relative path.
|
|
...
|
...
|
@@ -349,7 +363,9 @@ def fetch_remote_list(args, require_attribs = False, recursive = None):
|
|
349
|
349
|
'dev' : None,
|
|
350
|
350
|
'inode' : None,
|
|
351
|
351
|
}
|
|
352
|
|
- md5 = object['ETag'][1:-1]
|
|
|
352
|
+ if require_attribs or rem_list[key]['md5'].find("-"): # always get it for multipart uploads
|
|
|
353
|
+ _get_remote_attribs(S3Uri(object_uri_str), rem_list[key])
|
|
|
354
|
+ md5 = rem_list[key]['md5']
|
|
353
|
355
|
rem_list.record_md5(key, md5)
|
|
354
|
356
|
if break_now:
|
|
355
|
357
|
break
|
|
...
|
...
|
@@ -408,19 +424,12 @@ def fetch_remote_list(args, require_attribs = False, recursive = None):
|
|
408
|
408
|
'object_key': uri.object()
|
|
409
|
409
|
}
|
|
410
|
410
|
if require_attribs:
|
|
411
|
|
- response = S3(cfg).object_info(uri)
|
|
412
|
|
- remote_item.update({
|
|
413
|
|
- 'size': int(response['headers']['content-length']),
|
|
414
|
|
- 'md5': response['headers']['etag'].strip('"\''),
|
|
415
|
|
- 'timestamp' : dateRFC822toUnix(response['headers']['date'])
|
|
416
|
|
- })
|
|
417
|
|
- # get md5 from header if it's present. We would have set that during upload
|
|
418
|
|
- if response['headers'].has_key('x-amz-meta-s3cmd-attrs'):
|
|
419
|
|
- attrs = parse_attrs_header(response['headers']['x-amz-meta-s3cmd-attrs'])
|
|
420
|
|
- if attrs.has_key('md5'):
|
|
421
|
|
- remote_item.update({'md5': attrs['md5']})
|
|
|
411
|
+ _get_remote_attribs(uri, remote_item)
|
|
422
|
412
|
|
|
423
|
413
|
remote_list[key] = remote_item
|
|
|
414
|
+ md5 = remote_item.get('md5')
|
|
|
415
|
+ if md5:
|
|
|
416
|
+ remote_list.record_md5(key, md5)
|
|
424
|
417
|
return remote_list
|
|
425
|
418
|
|
|
426
|
419
|
def parse_attrs_header(attrs_header):
|