Browse code

Fixes #913 - Fix limitrate to recover of sudden changes in bandwidth and to not slow down too much the transfers

A "maximum" throttle was also added to avoid excessive
throttling/limitrate.
Transfer performances should be improved when using the limitrate
feature.

Florent Viard authored on 2018/03/02 10:06:05
Showing 2 changed files
... ...
@@ -164,6 +164,9 @@ class Config(object):
164 164
     # expected for every send file requests.
165 165
     use_http_expect = False
166 166
     signurl_use_https = False
167
+    # Maximum sleep duration for throtte / limitrate.
168
+    # s3 will timeout if a request/transfer is stuck for more than a short time
169
+    throttle_max = 100
167 170
 
168 171
     ## Creating a singleton
169 172
     def __new__(self, configfile = None, access_key=None, secret_key=None, access_token=None):
... ...
@@ -1427,6 +1427,7 @@ class S3(object):
1427 1427
                     # CONTINUE case. Reset the response
1428 1428
                     http_response.read()
1429 1429
                     conn.c._HTTPConnection__state = ConnMan._CS_REQ_SENT
1430
+
1430 1431
                 while (size_left > 0):
1431 1432
                     #debug("SendFile: Reading up to %d bytes from '%s' - remaining bytes: %s" % (self.config.send_chunk, filename, size_left))
1432 1433
                     l = min(self.config.send_chunk, size_left)
... ...
@@ -1439,20 +1440,22 @@ class S3(object):
1439 1439
                         start_time = time.time()
1440 1440
 
1441 1441
                     md5_hash.update(data)
1442
+
1442 1443
                     conn.c.wrapper_send_body(data)
1443 1444
                     if self.config.progress_meter:
1444 1445
                         progress.update(delta_position = len(data))
1445 1446
                     size_left -= len(data)
1446 1447
 
1447 1448
                     #throttle
1449
+                    limitrate_throttle = throttle
1448 1450
                     if self.config.limitrate > 0:
1449 1451
                         real_duration = time.time() - start_time
1450
-                        expected_duration = float(l)/self.config.limitrate
1451
-                        throttle = max(expected_duration - real_duration, throttle)
1452
-                    if throttle:
1453
-                        time.sleep(throttle)
1454
-                md5_computed = md5_hash.hexdigest()
1452
+                        expected_duration = float(l) / self.config.limitrate
1453
+                        limitrate_throttle = max(expected_duration - real_duration, limitrate_throttle)
1454
+                    if limitrate_throttle:
1455
+                        time.sleep(min(limitrate_throttle, self.config.throttle_max))
1455 1456
 
1457
+                md5_computed = md5_hash.hexdigest()
1456 1458
                 http_response = conn.c.getresponse()
1457 1459
 
1458 1460
             response = {}