A "maximum" throttle was also added to avoid excessive
throttling/limitrate.
Transfer performances should be improved when using the limitrate
feature.
... | ... |
@@ -164,6 +164,9 @@ class Config(object): |
164 | 164 |
# expected for every send file requests. |
165 | 165 |
use_http_expect = False |
166 | 166 |
signurl_use_https = False |
167 |
+ # Maximum sleep duration for throtte / limitrate. |
|
168 |
+ # s3 will timeout if a request/transfer is stuck for more than a short time |
|
169 |
+ throttle_max = 100 |
|
167 | 170 |
|
168 | 171 |
## Creating a singleton |
169 | 172 |
def __new__(self, configfile = None, access_key=None, secret_key=None, access_token=None): |
... | ... |
@@ -1427,6 +1427,7 @@ class S3(object): |
1427 | 1427 |
# CONTINUE case. Reset the response |
1428 | 1428 |
http_response.read() |
1429 | 1429 |
conn.c._HTTPConnection__state = ConnMan._CS_REQ_SENT |
1430 |
+ |
|
1430 | 1431 |
while (size_left > 0): |
1431 | 1432 |
#debug("SendFile: Reading up to %d bytes from '%s' - remaining bytes: %s" % (self.config.send_chunk, filename, size_left)) |
1432 | 1433 |
l = min(self.config.send_chunk, size_left) |
... | ... |
@@ -1439,20 +1440,22 @@ class S3(object): |
1439 | 1439 |
start_time = time.time() |
1440 | 1440 |
|
1441 | 1441 |
md5_hash.update(data) |
1442 |
+ |
|
1442 | 1443 |
conn.c.wrapper_send_body(data) |
1443 | 1444 |
if self.config.progress_meter: |
1444 | 1445 |
progress.update(delta_position = len(data)) |
1445 | 1446 |
size_left -= len(data) |
1446 | 1447 |
|
1447 | 1448 |
#throttle |
1449 |
+ limitrate_throttle = throttle |
|
1448 | 1450 |
if self.config.limitrate > 0: |
1449 | 1451 |
real_duration = time.time() - start_time |
1450 |
- expected_duration = float(l)/self.config.limitrate |
|
1451 |
- throttle = max(expected_duration - real_duration, throttle) |
|
1452 |
- if throttle: |
|
1453 |
- time.sleep(throttle) |
|
1454 |
- md5_computed = md5_hash.hexdigest() |
|
1452 |
+ expected_duration = float(l) / self.config.limitrate |
|
1453 |
+ limitrate_throttle = max(expected_duration - real_duration, limitrate_throttle) |
|
1454 |
+ if limitrate_throttle: |
|
1455 |
+ time.sleep(min(limitrate_throttle, self.config.throttle_max)) |
|
1455 | 1456 |
|
1457 |
+ md5_computed = md5_hash.hexdigest() |
|
1456 | 1458 |
http_response = conn.c.getresponse() |
1457 | 1459 |
|
1458 | 1460 |
response = {} |