Browse code

Fixes #1088 - Handle servers indicating (lying) that listObject result is truncated, when it is not the case

For unknown reasons, it happened with a Dell EMC ECS Object Storage that
a ListObject for a single object returned with the Truncated flag set to
True, despite not being any other result to fetch.
Before this fix, the code was not expecting that and so crashing in that
case.

Florent Viard authored on 2020/04/20 10:43:07
Showing 1 changed files
... ...
@@ -358,7 +358,8 @@ class S3(object):
358 358
         num_prefixes = 0
359 359
         max_keys = limit
360 360
         while truncated:
361
-            response = self.bucket_list_noparse(bucket, prefix, recursive, uri_params, max_keys)
361
+            response = self.bucket_list_noparse(bucket, prefix, recursive,
362
+                                                uri_params, max_keys)
362 363
             current_list = _get_contents(response["data"])
363 364
             current_prefixes = _get_common_prefixes(response["data"])
364 365
             num_objects += len(current_list)
... ...
@@ -369,9 +370,15 @@ class S3(object):
369 369
             if truncated:
370 370
                 if limit == -1 or num_objects + num_prefixes < limit:
371 371
                     if current_list:
372
-                        uri_params['marker'] = _get_next_marker(response["data"], current_list)
373
-                    else:
372
+                        uri_params['marker'] = \
373
+                            _get_next_marker(response["data"], current_list)
374
+                    elif current_prefixes:
374 375
                         uri_params['marker'] = current_prefixes[-1]["Prefix"]
376
+                    else:
377
+                        # Unexpectedly, the server lied, and so the previous
378
+                        # response was not truncated. So, no new key to get.
379
+                        yield False, current_prefixes, current_list
380
+                        break
375 381
                     debug("Listing continues after '%s'" % uri_params['marker'])
376 382
                 else:
377 383
                     yield truncated, current_prefixes, current_list