Browse code

Get md5 from remote attribute header in recursive syncs too

We were not getting the md5 value from the attribute header of each
file for the recursive case, only for the non-recursive case. That
was stupid. We were certainly _setting_ the md5value into the
attribute header.

This should eliminate the
INFO: disabled md5 check for...
messages on subsequent syncs when s3cmd 1.5.0-alpha+ is used to upload
content in the first place.

Matt Domsch authored on 2013/05/21 05:33:46
Showing 1 changed files
... ...
@@ -257,6 +257,20 @@ def fetch_local_list(args, recursive = None):
257 257
     return local_list, single_file
258 258
 
259 259
 def fetch_remote_list(args, require_attribs = False, recursive = None):
260
+    def _get_remote_attribs(uri, remote_item):
261
+        response = S3(cfg).object_info(uri)
262
+        remote_item.update({
263
+        'size': int(response['headers']['content-length']),
264
+        'md5': response['headers']['etag'].strip('"\''),
265
+        'timestamp' : dateRFC822toUnix(response['headers']['date'])
266
+        })
267
+        # get md5 from header if it's present.  We would have set that during upload
268
+        if response['headers'].has_key('x-amz-meta-s3cmd-attrs'):
269
+            attrs = parse_attrs_header(response['headers']['x-amz-meta-s3cmd-attrs'])
270
+            if attrs.has_key('md5'):
271
+                remote_item.update({'md5': attrs['md5']})
272
+                debug(u"retreived md5=%s from headers" % (attrs['md5']))
273
+
260 274
     def _get_filelist_remote(remote_uri, recursive = True):
261 275
         ## If remote_uri ends with '/' then all remote files will have
262 276
         ## the remote_uri prefix removed in the relative path.
... ...
@@ -306,7 +320,9 @@ def fetch_remote_list(args, require_attribs = False, recursive = None):
306 306
                 'dev' : None,
307 307
                 'inode' : None,
308 308
             }
309
-            md5 = object['ETag'][1:-1]
309
+            if require_attribs or rem_list[key]['md5'].find("-"): # always get it for multipart uploads
310
+                _get_remote_attribs(S3Uri(object_uri_str), rem_list[key])
311
+            md5 = rem_list[key]['md5']
310 312
             rem_list.record_md5(key, md5)
311 313
             if break_now:
312 314
                 break
... ...
@@ -365,19 +381,12 @@ def fetch_remote_list(args, require_attribs = False, recursive = None):
365 365
                     'object_key': uri.object()
366 366
                 }
367 367
                 if require_attribs:
368
-                    response = S3(cfg).object_info(uri)
369
-                    remote_item.update({
370
-                    'size': int(response['headers']['content-length']),
371
-                    'md5': response['headers']['etag'].strip('"\''),
372
-                    'timestamp' : dateRFC822toUnix(response['headers']['date'])
373
-                    })
374
-                    # get md5 from header if it's present.  We would have set that during upload
375
-                    if response['headers'].has_key('x-amz-meta-s3cmd-attrs'):
376
-                        attrs = parse_attrs_header(response['headers']['x-amz-meta-s3cmd-attrs'])
377
-                        if attrs.has_key('md5'):
378
-                            remote_item.update({'md5': attrs['md5']})
368
+                    _get_remote_attribs(uri, remote_item)
379 369
 
380 370
                 remote_list[key] = remote_item
371
+                md5 = remote_item.get('md5')
372
+                if md5:
373
+                    remote_list.record_md5(key, md5)
381 374
     return remote_list
382 375
 
383 376
 def parse_attrs_header(attrs_header):