For unknown reasons, it happened with a Dell EMC ECS Object Storage that
a ListObject for a single object returned with the Truncated flag set to
True, despite not being any other result to fetch.
Before this fix, the code was not expecting that and so crashing in that
case.
... | ... |
@@ -358,7 +358,8 @@ class S3(object): |
358 | 358 |
num_prefixes = 0 |
359 | 359 |
max_keys = limit |
360 | 360 |
while truncated: |
361 |
- response = self.bucket_list_noparse(bucket, prefix, recursive, uri_params, max_keys) |
|
361 |
+ response = self.bucket_list_noparse(bucket, prefix, recursive, |
|
362 |
+ uri_params, max_keys) |
|
362 | 363 |
current_list = _get_contents(response["data"]) |
363 | 364 |
current_prefixes = _get_common_prefixes(response["data"]) |
364 | 365 |
num_objects += len(current_list) |
... | ... |
@@ -369,9 +370,15 @@ class S3(object): |
369 | 369 |
if truncated: |
370 | 370 |
if limit == -1 or num_objects + num_prefixes < limit: |
371 | 371 |
if current_list: |
372 |
- uri_params['marker'] = _get_next_marker(response["data"], current_list) |
|
373 |
- else: |
|
372 |
+ uri_params['marker'] = \ |
|
373 |
+ _get_next_marker(response["data"], current_list) |
|
374 |
+ elif current_prefixes: |
|
374 | 375 |
uri_params['marker'] = current_prefixes[-1]["Prefix"] |
376 |
+ else: |
|
377 |
+ # Unexpectedly, the server lied, and so the previous |
|
378 |
+ # response was not truncated. So, no new key to get. |
|
379 |
+ yield False, current_prefixes, current_list |
|
380 |
+ break |
|
375 | 381 |
debug("Listing continues after '%s'" % uri_params['marker']) |
376 | 382 |
else: |
377 | 383 |
yield truncated, current_prefixes, current_list |