We need a more generic approach to connection pooling.
Michal Ludvig authored on 2013/03/10 19:06:35... | ... |
@@ -246,10 +246,9 @@ class S3(object): |
246 | 246 |
truncated = True |
247 | 247 |
list = [] |
248 | 248 |
prefixes = [] |
249 |
- conn = self.get_connection(bucket) |
|
250 | 249 |
|
251 | 250 |
while truncated: |
252 |
- response = self.bucket_list_noparse(conn, bucket, prefix, recursive, uri_params) |
|
251 |
+ response = self.bucket_list_noparse(bucket, prefix, recursive, uri_params) |
|
253 | 252 |
current_list = _get_contents(response["data"]) |
254 | 253 |
current_prefixes = _get_common_prefixes(response["data"]) |
255 | 254 |
truncated = _list_truncated(response["data"]) |
... | ... |
@@ -263,19 +262,17 @@ class S3(object): |
263 | 263 |
list += current_list |
264 | 264 |
prefixes += current_prefixes |
265 | 265 |
|
266 |
- conn.close() |
|
267 |
- |
|
268 | 266 |
response['list'] = list |
269 | 267 |
response['common_prefixes'] = prefixes |
270 | 268 |
return response |
271 | 269 |
|
272 |
- def bucket_list_noparse(self, connection, bucket, prefix = None, recursive = None, uri_params = {}): |
|
270 |
+ def bucket_list_noparse(self, bucket, prefix = None, recursive = None, uri_params = {}): |
|
273 | 271 |
if prefix: |
274 | 272 |
uri_params['prefix'] = self.urlencode_string(prefix) |
275 | 273 |
if not self.config.recursive and not recursive: |
276 | 274 |
uri_params['delimiter'] = "/" |
277 | 275 |
request = self.create_request("BUCKET_LIST", bucket = bucket, **uri_params) |
278 |
- response = self.send_request(request, conn = connection) |
|
276 |
+ response = self.send_request(request) |
|
279 | 277 |
#debug(response) |
280 | 278 |
return response |
281 | 279 |
|
... | ... |
@@ -662,7 +659,7 @@ class S3(object): |
662 | 662 |
# Wait a few seconds. The more it fails the more we wait. |
663 | 663 |
return (self._max_retries - retries + 1) * 3 |
664 | 664 |
|
665 |
- def send_request(self, request, body = None, retries = _max_retries, conn = None): |
|
665 |
+ def send_request(self, request, body = None, retries = _max_retries): |
|
666 | 666 |
method_string, resource, headers = request.get_triplet() |
667 | 667 |
debug("Processing request, please wait...") |
668 | 668 |
if not headers.has_key('content-length'): |
... | ... |
@@ -671,13 +668,7 @@ class S3(object): |
671 | 671 |
# "Stringify" all headers |
672 | 672 |
for header in headers.keys(): |
673 | 673 |
headers[header] = str(headers[header]) |
674 |
- if conn is None: |
|
675 |
- debug("Establishing connection") |
|
676 |
- conn = self.get_connection(resource['bucket']) |
|
677 |
- close_conn = True |
|
678 |
- else: |
|
679 |
- debug("Using existing connection") |
|
680 |
- close_conn = False |
|
674 |
+ conn = self.get_connection(resource['bucket']) |
|
681 | 675 |
uri = self.format_uri(resource) |
682 | 676 |
debug("Sending request method_string=%r, uri=%r, headers=%r, body=(%i bytes)" % (method_string, uri, headers, len(body or ""))) |
683 | 677 |
conn.request(method_string, uri, body, headers) |
... | ... |
@@ -688,8 +679,7 @@ class S3(object): |
688 | 688 |
response["headers"] = convertTupleListToDict(http_response.getheaders()) |
689 | 689 |
response["data"] = http_response.read() |
690 | 690 |
debug("Response: " + str(response)) |
691 |
- if close_conn is True: |
|
692 |
- conn.close() |
|
691 |
+ conn.close() |
|
693 | 692 |
except Exception, e: |
694 | 693 |
if retries: |
695 | 694 |
warning("Retrying failed request: %s (%s)" % (resource['uri'], e)) |