Also add a new "connection_max_age" config entry to define the maximum
idle time for a connection in the pool.
| ... | ... |
@@ -218,6 +218,10 @@ class Config(object): |
| 218 | 218 |
throttle_max = 100 |
| 219 | 219 |
public_url_use_https = False |
| 220 | 220 |
connection_pooling = True |
| 221 |
+ # How long a connection can be kept idle in the pool and still be alive |
|
| 222 |
+ # Currently, aws s3 closes connections that are idle for 20 seconds or |
|
| 223 |
+ # longer, but 16s is used here by default to be safe. |
|
| 224 |
+ connection_max_age = 16 |
|
| 221 | 225 |
|
| 222 | 226 |
## Creating a singleton |
| 223 | 227 |
def __new__(self, configfile = None, access_key=None, secret_key=None, access_token=None): |
| ... | ... |
@@ -14,8 +14,10 @@ if sys.version_info >= (3, 0): |
| 14 | 14 |
else: |
| 15 | 15 |
from .Custom_httplib27 import httplib |
| 16 | 16 |
import ssl |
| 17 |
-from threading import Semaphore |
|
| 17 |
+ |
|
| 18 | 18 |
from logging import debug |
| 19 |
+from threading import Semaphore |
|
| 20 |
+from time import time |
|
| 19 | 21 |
try: |
| 20 | 22 |
# python 3 support |
| 21 | 23 |
from urlparse import urlparse |
| ... | ... |
@@ -238,18 +240,19 @@ class http_connection(object): |
| 238 | 238 |
debug(u'proxied HTTPConnection(%s, %s)', cfg.proxy_host, cfg.proxy_port) |
| 239 | 239 |
# No tunnel here for the moment |
| 240 | 240 |
|
| 241 |
+ self.last_used_time = time() |
|
| 241 | 242 |
|
| 242 | 243 |
class ConnMan(object): |
| 243 | 244 |
_CS_REQ_SENT = httplib._CS_REQ_SENT |
| 244 | 245 |
CONTINUE = httplib.CONTINUE |
| 245 | 246 |
conn_pool_sem = Semaphore() |
| 246 | 247 |
conn_pool = {}
|
| 247 |
- conn_max_counter = 800 ## AWS closes connection after some ~90 requests |
|
| 248 |
+ conn_max_counter = 800 ## AWS closes connection after some ~90 requests |
|
| 248 | 249 |
|
| 249 | 250 |
@staticmethod |
| 250 |
- def get(hostname, ssl = None): |
|
| 251 |
+ def get(hostname, ssl=None): |
|
| 251 | 252 |
cfg = Config() |
| 252 |
- if ssl == None: |
|
| 253 |
+ if ssl is None: |
|
| 253 | 254 |
ssl = cfg.use_https |
| 254 | 255 |
conn = None |
| 255 | 256 |
if cfg.proxy_host != "": |
| ... | ... |
@@ -261,9 +264,19 @@ class ConnMan(object): |
| 261 | 261 |
ConnMan.conn_pool_sem.acquire() |
| 262 | 262 |
if conn_id not in ConnMan.conn_pool: |
| 263 | 263 |
ConnMan.conn_pool[conn_id] = [] |
| 264 |
- if len(ConnMan.conn_pool[conn_id]): |
|
| 264 |
+ while ConnMan.conn_pool[conn_id]: |
|
| 265 | 265 |
conn = ConnMan.conn_pool[conn_id].pop() |
| 266 |
- debug("ConnMan.get(): re-using connection: %s#%d" % (conn.id, conn.counter))
|
|
| 266 |
+ cur_time = time() |
|
| 267 |
+ if cur_time < conn.last_used_time + cfg.connection_max_age \ |
|
| 268 |
+ and cur_time >= conn.last_used_time: |
|
| 269 |
+ debug("ConnMan.get(): re-using connection: %s#%d"
|
|
| 270 |
+ % (conn.id, conn.counter)) |
|
| 271 |
+ break |
|
| 272 |
+ # Conn is too old or wall clock went back in the past |
|
| 273 |
+ debug("ConnMan.get(): closing expired connection")
|
|
| 274 |
+ ConnMan.close(conn) |
|
| 275 |
+ conn = None |
|
| 276 |
+ |
|
| 267 | 277 |
ConnMan.conn_pool_sem.release() |
| 268 | 278 |
if not conn: |
| 269 | 279 |
debug("ConnMan.get(): creating new connection: %s" % conn_id)
|
| ... | ... |
@@ -278,7 +291,8 @@ class ConnMan(object): |
| 278 | 278 |
def put(conn): |
| 279 | 279 |
if conn.id.startswith("proxy://"):
|
| 280 | 280 |
ConnMan.close(conn) |
| 281 |
- debug("ConnMan.put(): closing proxy connection (keep-alive not yet supported)")
|
|
| 281 |
+ debug("ConnMan.put(): closing proxy connection (keep-alive not yet"
|
|
| 282 |
+ " supported)") |
|
| 282 | 283 |
return |
| 283 | 284 |
|
| 284 | 285 |
if conn.counter >= ConnMan.conn_max_counter: |
| ... | ... |
@@ -292,10 +306,14 @@ class ConnMan(object): |
| 292 | 292 |
debug("ConnMan.put(): closing connection (connection pooling disabled)")
|
| 293 | 293 |
return |
| 294 | 294 |
|
| 295 |
+ # Update timestamp of conn to record when was its last use |
|
| 296 |
+ conn.last_used_time = time() |
|
| 297 |
+ |
|
| 295 | 298 |
ConnMan.conn_pool_sem.acquire() |
| 296 | 299 |
ConnMan.conn_pool[conn.id].append(conn) |
| 297 | 300 |
ConnMan.conn_pool_sem.release() |
| 298 |
- debug("ConnMan.put(): connection put back to pool (%s#%d)" % (conn.id, conn.counter))
|
|
| 301 |
+ debug("ConnMan.put(): connection put back to pool (%s#%d)"
|
|
| 302 |
+ % (conn.id, conn.counter)) |
|
| 299 | 303 |
|
| 300 | 304 |
@staticmethod |
| 301 | 305 |
def close(conn): |