Browse code

Merge branch 'bug/remote-md5sum' into merge

Matt Domsch authored on 2013/05/21 05:38:36
Showing 1 changed files
... ...
@@ -300,6 +300,20 @@ def fetch_local_list(args, is_src = False, recursive = None):
300 300
     return local_list, single_file
301 301
 
302 302
 def fetch_remote_list(args, require_attribs = False, recursive = None):
303
+    def _get_remote_attribs(uri, remote_item):
304
+        response = S3(cfg).object_info(uri)
305
+        remote_item.update({
306
+        'size': int(response['headers']['content-length']),
307
+        'md5': response['headers']['etag'].strip('"\''),
308
+        'timestamp' : dateRFC822toUnix(response['headers']['date'])
309
+        })
310
+        # get md5 from header if it's present.  We would have set that during upload
311
+        if response['headers'].has_key('x-amz-meta-s3cmd-attrs'):
312
+            attrs = parse_attrs_header(response['headers']['x-amz-meta-s3cmd-attrs'])
313
+            if attrs.has_key('md5'):
314
+                remote_item.update({'md5': attrs['md5']})
315
+                debug(u"retreived md5=%s from headers" % (attrs['md5']))
316
+
303 317
     def _get_filelist_remote(remote_uri, recursive = True):
304 318
         ## If remote_uri ends with '/' then all remote files will have
305 319
         ## the remote_uri prefix removed in the relative path.
... ...
@@ -349,7 +363,9 @@ def fetch_remote_list(args, require_attribs = False, recursive = None):
349 349
                 'dev' : None,
350 350
                 'inode' : None,
351 351
             }
352
-            md5 = object['ETag'][1:-1]
352
+            if require_attribs or rem_list[key]['md5'].find("-"): # always get it for multipart uploads
353
+                _get_remote_attribs(S3Uri(object_uri_str), rem_list[key])
354
+            md5 = rem_list[key]['md5']
353 355
             rem_list.record_md5(key, md5)
354 356
             if break_now:
355 357
                 break
... ...
@@ -408,19 +424,12 @@ def fetch_remote_list(args, require_attribs = False, recursive = None):
408 408
                     'object_key': uri.object()
409 409
                 }
410 410
                 if require_attribs:
411
-                    response = S3(cfg).object_info(uri)
412
-                    remote_item.update({
413
-                    'size': int(response['headers']['content-length']),
414
-                    'md5': response['headers']['etag'].strip('"\''),
415
-                    'timestamp' : dateRFC822toUnix(response['headers']['date'])
416
-                    })
417
-                    # get md5 from header if it's present.  We would have set that during upload
418
-                    if response['headers'].has_key('x-amz-meta-s3cmd-attrs'):
419
-                        attrs = parse_attrs_header(response['headers']['x-amz-meta-s3cmd-attrs'])
420
-                        if attrs.has_key('md5'):
421
-                            remote_item.update({'md5': attrs['md5']})
411
+                    _get_remote_attribs(uri, remote_item)
422 412
 
423 413
                 remote_list[key] = remote_item
414
+                md5 = remote_item.get('md5')
415
+                if md5:
416
+                    remote_list.record_md5(key, md5)
424 417
     return remote_list
425 418
 
426 419
 def parse_attrs_header(attrs_header):