Make connection pooling configurable
Florent Viard authored on 2020/03/22 00:48:34... | ... |
@@ -206,6 +206,7 @@ class Config(object): |
206 | 206 |
# s3 will timeout if a request/transfer is stuck for more than a short time |
207 | 207 |
throttle_max = 100 |
208 | 208 |
public_url_use_https = False |
209 |
+ connection_pooling = True |
|
209 | 210 |
|
210 | 211 |
## Creating a singleton |
211 | 212 |
def __new__(self, configfile = None, access_key=None, secret_key=None, access_token=None): |
... | ... |
@@ -257,16 +257,27 @@ class ConnMan(object): |
257 | 257 |
@staticmethod |
258 | 258 |
def put(conn): |
259 | 259 |
if conn.id.startswith("proxy://"): |
260 |
- conn.c.close() |
|
260 |
+ ConnMan.close(conn) |
|
261 | 261 |
debug("ConnMan.put(): closing proxy connection (keep-alive not yet supported)") |
262 | 262 |
return |
263 | 263 |
|
264 | 264 |
if conn.counter >= ConnMan.conn_max_counter: |
265 |
- conn.c.close() |
|
265 |
+ ConnMan.close(conn) |
|
266 | 266 |
debug("ConnMan.put(): closing over-used connection") |
267 | 267 |
return |
268 | 268 |
|
269 |
+ cfg = Config() |
|
270 |
+ if not cfg.connection_pooling: |
|
271 |
+ ConnMan.close(conn) |
|
272 |
+ debug("ConnMan.put(): closing connection (connection pooling disabled)") |
|
273 |
+ return |
|
274 |
+ |
|
269 | 275 |
ConnMan.conn_pool_sem.acquire() |
270 | 276 |
ConnMan.conn_pool[conn.id].append(conn) |
271 | 277 |
ConnMan.conn_pool_sem.release() |
272 | 278 |
debug("ConnMan.put(): connection put back to pool (%s#%d)" % (conn.id, conn.counter)) |
279 |
+ |
|
280 |
+ @staticmethod |
|
281 |
+ def close(conn): |
|
282 |
+ if conn: |
|
283 |
+ conn.c.close() |
... | ... |
@@ -1307,10 +1307,9 @@ class S3(object): |
1307 | 1307 |
# When the connection is broken, BadStatusLine is raised with py2 |
1308 | 1308 |
# and RemoteDisconnected is raised by py3 with a trap: |
1309 | 1309 |
# RemoteDisconnected has an errno field with a None value. |
1310 |
- if conn: |
|
1311 |
- # close the connection and re-establish |
|
1312 |
- conn.counter = ConnMan.conn_max_counter |
|
1313 |
- ConnMan.put(conn) |
|
1310 |
+ |
|
1311 |
+ # close the connection and re-establish |
|
1312 |
+ ConnMan.close(conn) |
|
1314 | 1313 |
if retries: |
1315 | 1314 |
warning("Retrying failed request: %s (%s)" % (resource['uri'], e)) |
1316 | 1315 |
warning("Waiting %d sec..." % self._fail_wait(retries)) |
... | ... |
@@ -1664,10 +1663,9 @@ class S3(object): |
1664 | 1664 |
or "[Errno 104]" in str(e) or "[Errno 32]" in str(e) |
1665 | 1665 |
) and not isinstance(e, SocketTimeoutException): |
1666 | 1666 |
raise |
1667 |
- if conn: |
|
1668 |
- # close the connection and re-establish |
|
1669 |
- conn.counter = ConnMan.conn_max_counter |
|
1670 |
- ConnMan.put(conn) |
|
1667 |
+ |
|
1668 |
+ # close the connection and re-establish |
|
1669 |
+ ConnMan.close(conn) |
|
1671 | 1670 |
|
1672 | 1671 |
if retries: |
1673 | 1672 |
warning("Retrying failed request: %s (%s)" % (resource['uri'], e)) |
... | ... |
@@ -1761,8 +1759,7 @@ class S3(object): |
1761 | 1761 |
) and not isinstance(e, SocketTimeoutException): |
1762 | 1762 |
raise |
1763 | 1763 |
# close the connection and re-establish |
1764 |
- conn.counter = ConnMan.conn_max_counter |
|
1765 |
- ConnMan.put(conn) |
|
1764 |
+ ConnMan.close(conn) |
|
1766 | 1765 |
|
1767 | 1766 |
if retries: |
1768 | 1767 |
warning("Retrying failed request: %s (%s)" % (resource['uri'], e)) |
... | ... |
@@ -2760,6 +2760,7 @@ def main(): |
2760 | 2760 |
optparser.add_option( "--no-check-hostname", dest="check_ssl_hostname", action="store_false", help="Do not check SSL certificate hostname validity") |
2761 | 2761 |
optparser.add_option( "--signature-v2", dest="signature_v2", action="store_true", help="Use AWS Signature version 2 instead of newer signature methods. Helpful for S3-like systems that don't have AWS Signature v4 yet.") |
2762 | 2762 |
optparser.add_option( "--limit-rate", dest="limitrate", action="store", type="string", help="Limit the upload or download speed to amount bytes per second. Amount may be expressed in bytes, kilobytes with the k suffix, or megabytes with the m suffix") |
2763 |
+ optparser.add_option( "--no-connection-pooling", dest="connection_pooling", action="store_false", help="Disable connection re-use") |
|
2763 | 2764 |
optparser.add_option( "--requester-pays", dest="requester_pays", action="store_true", help="Set the REQUESTER PAYS flag for operations") |
2764 | 2765 |
optparser.add_option("-l", "--long-listing", dest="long_listing", action="store_true", help="Produce long listing [ls]") |
2765 | 2766 |
optparser.add_option( "--stop-on-error", dest="stop_on_error", action="store_true", help="stop if error in transfer") |