Browse code

Merge pull request #1032 from vainu-arto/make-connection-pooling-configurable

Make connection pooling configurable

Florent Viard authored on 2020/03/22 00:48:34
Showing 4 changed files
... ...
@@ -206,6 +206,7 @@ class Config(object):
206 206
     # s3 will timeout if a request/transfer is stuck for more than a short time
207 207
     throttle_max = 100
208 208
     public_url_use_https = False
209
+    connection_pooling = True
209 210
 
210 211
     ## Creating a singleton
211 212
     def __new__(self, configfile = None, access_key=None, secret_key=None, access_token=None):
... ...
@@ -257,16 +257,27 @@ class ConnMan(object):
257 257
     @staticmethod
258 258
     def put(conn):
259 259
         if conn.id.startswith("proxy://"):
260
-            conn.c.close()
260
+            ConnMan.close(conn)
261 261
             debug("ConnMan.put(): closing proxy connection (keep-alive not yet supported)")
262 262
             return
263 263
 
264 264
         if conn.counter >= ConnMan.conn_max_counter:
265
-            conn.c.close()
265
+            ConnMan.close(conn)
266 266
             debug("ConnMan.put(): closing over-used connection")
267 267
             return
268 268
 
269
+        cfg = Config()
270
+        if not cfg.connection_pooling:
271
+            ConnMan.close(conn)
272
+            debug("ConnMan.put(): closing connection (connection pooling disabled)")
273
+            return
274
+
269 275
         ConnMan.conn_pool_sem.acquire()
270 276
         ConnMan.conn_pool[conn.id].append(conn)
271 277
         ConnMan.conn_pool_sem.release()
272 278
         debug("ConnMan.put(): connection put back to pool (%s#%d)" % (conn.id, conn.counter))
279
+
280
+    @staticmethod
281
+    def close(conn):
282
+        if conn:
283
+            conn.c.close()
... ...
@@ -1307,10 +1307,9 @@ class S3(object):
1307 1307
             # When the connection is broken, BadStatusLine is raised with py2
1308 1308
             # and RemoteDisconnected is raised by py3 with a trap:
1309 1309
             # RemoteDisconnected has an errno field with a None value.
1310
-            if conn:
1311
-                # close the connection and re-establish
1312
-                conn.counter = ConnMan.conn_max_counter
1313
-                ConnMan.put(conn)
1310
+
1311
+            # close the connection and re-establish
1312
+            ConnMan.close(conn)
1314 1313
             if retries:
1315 1314
                 warning("Retrying failed request: %s (%s)" % (resource['uri'], e))
1316 1315
                 warning("Waiting %d sec..." % self._fail_wait(retries))
... ...
@@ -1664,10 +1663,9 @@ class S3(object):
1664 1664
                 or "[Errno 104]" in str(e) or "[Errno 32]" in str(e)
1665 1665
                ) and not isinstance(e, SocketTimeoutException):
1666 1666
                 raise
1667
-            if conn:
1668
-                # close the connection and re-establish
1669
-                conn.counter = ConnMan.conn_max_counter
1670
-                ConnMan.put(conn)
1667
+
1668
+            # close the connection and re-establish
1669
+            ConnMan.close(conn)
1671 1670
 
1672 1671
             if retries:
1673 1672
                 warning("Retrying failed request: %s (%s)" % (resource['uri'], e))
... ...
@@ -1761,8 +1759,7 @@ class S3(object):
1761 1761
                ) and not isinstance(e, SocketTimeoutException):
1762 1762
                 raise
1763 1763
             # close the connection and re-establish
1764
-            conn.counter = ConnMan.conn_max_counter
1765
-            ConnMan.put(conn)
1764
+            ConnMan.close(conn)
1766 1765
 
1767 1766
             if retries:
1768 1767
                 warning("Retrying failed request: %s (%s)" % (resource['uri'], e))
... ...
@@ -2760,6 +2760,7 @@ def main():
2760 2760
     optparser.add_option(      "--no-check-hostname", dest="check_ssl_hostname", action="store_false", help="Do not check SSL certificate hostname validity")
2761 2761
     optparser.add_option(      "--signature-v2", dest="signature_v2", action="store_true", help="Use AWS Signature version 2 instead of newer signature methods. Helpful for S3-like systems that don't have AWS Signature v4 yet.")
2762 2762
     optparser.add_option(      "--limit-rate", dest="limitrate", action="store", type="string", help="Limit the upload or download speed to amount bytes per second.  Amount may be expressed in bytes, kilobytes with the k suffix, or megabytes with the m suffix")
2763
+    optparser.add_option(      "--no-connection-pooling", dest="connection_pooling", action="store_false", help="Disable connection re-use")
2763 2764
     optparser.add_option(      "--requester-pays", dest="requester_pays", action="store_true", help="Set the REQUESTER PAYS flag for operations")
2764 2765
     optparser.add_option("-l", "--long-listing", dest="long_listing", action="store_true", help="Produce long listing [ls]")
2765 2766
     optparser.add_option(      "--stop-on-error", dest="stop_on_error", action="store_true", help="stop if error in transfer")