Browse code

Partial fix for #941 - Retry for all kind of requests in case of OS network timeout events

Florent Viard authored on 2018/03/01 03:17:29
Showing 1 changed files
... ...
@@ -17,6 +17,7 @@ import mimetypes
17 17
 import io
18 18
 import pprint
19 19
 from xml.sax import saxutils
20
+from socket import timeout as SocketTimeoutException
20 21
 from logging import debug, info, warning, error
21 22
 from stat import ST_SIZE
22 23
 try:
... ...
@@ -1248,8 +1249,9 @@ class S3(object):
1248 1248
         response = {}
1249 1249
         debug("Processing request, please wait...")
1250 1250
 
1251
-        conn = ConnMan.get(self.get_hostname(resource['bucket']))
1251
+        conn = None
1252 1252
         try:
1253
+            conn = ConnMan.get(self.get_hostname(resource['bucket']))
1253 1254
             # TODO: Check what was supposed to be the usage of conn.path here
1254 1255
             # Currently this is always "None" all the time as not defined in ConnMan
1255 1256
             uri = self.format_uri(resource, conn.path)
... ...
@@ -1266,11 +1268,13 @@ class S3(object):
1266 1266
             ConnMan.put(conn)
1267 1267
         except (IOError, Exception) as e:
1268 1268
             debug("Response:\n" + pprint.pformat(response))
1269
-            if hasattr(e, 'errno') and e.errno not in (errno.EPIPE, errno.ECONNRESET):
1269
+            if ((hasattr(e, 'errno') and e.errno not in (errno.EPIPE, errno.ECONNRESET, errno.ETIMEDOUT))
1270
+               or "[Errno 104]" in str(e) or "[Errno 32]" in str(e)) and not isinstance(e, SocketTimeoutException):
1270 1271
                 raise
1271
-            # close the connection and re-establish
1272
-            conn.counter = ConnMan.conn_max_counter
1273
-            ConnMan.put(conn)
1272
+            if conn:
1273
+                # close the connection and re-establish
1274
+                conn.counter = ConnMan.conn_max_counter
1275
+                ConnMan.put(conn)
1274 1276
             if retries:
1275 1277
                 warning("Retrying failed request: %s (%s)" % (resource['uri'], e))
1276 1278
                 warning("Waiting %d sec..." % self._fail_wait(retries))
... ...
@@ -1454,8 +1458,8 @@ class S3(object):
1454 1454
                 if retries < self._max_retries:
1455 1455
                     throttle = throttle and throttle * 5 or 0.01
1456 1456
                 known_error = False
1457
-                if ((hasattr(e, 'errno') and e.errno not in (errno.EPIPE, errno.ECONNRESET))
1458
-                   or "[Errno 104]" in str(e) or "[Errno 32]" in str(e)):
1457
+                if ((hasattr(e, 'errno') and e.errno not in (errno.EPIPE, errno.ECONNRESET, errno.ETIMEDOUT))
1458
+                   or "[Errno 104]" in str(e) or "[Errno 32]" in str(e)) and not isinstance(e, SocketTimeoutException):
1459 1459
                     # We have to detect these errors by looking at the error string
1460 1460
                     # Connection reset by peer and Broken pipe
1461 1461
                     # The server broke the connection early with an error like
... ...
@@ -1601,8 +1605,9 @@ class S3(object):
1601 1601
             info("Receiving file '%s', please wait..." % filename)
1602 1602
         timestamp_start = time.time()
1603 1603
 
1604
-        conn = ConnMan.get(self.get_hostname(resource['bucket']))
1604
+        conn = None
1605 1605
         try:
1606
+            conn = ConnMan.get(self.get_hostname(resource['bucket']))
1606 1607
             conn.c.putrequest(method_string, self.format_uri(resource, conn.path))
1607 1608
             for header in headers.keys():
1608 1609
                 conn.c.putheader(encode_to_s3(header), encode_to_s3(headers[header]))
... ...
@@ -1626,11 +1631,13 @@ class S3(object):
1626 1626
         except (IOError, Exception) as e:
1627 1627
             if self.config.progress_meter:
1628 1628
                 progress.done("failed")
1629
-            if hasattr(e, 'errno') and e.errno not in (errno.EPIPE, errno.ECONNRESET):
1629
+            if ((hasattr(e, 'errno') and e.errno not in (errno.EPIPE, errno.ECONNRESET, errno.ETIMEDOUT))
1630
+               or "[Errno 104]" in str(e) or "[Errno 32]" in str(e)) and not isinstance(e, SocketTimeoutException):
1630 1631
                 raise
1631
-            # close the connection and re-establish
1632
-            conn.counter = ConnMan.conn_max_counter
1633
-            ConnMan.put(conn)
1632
+            if conn:
1633
+                # close the connection and re-establish
1634
+                conn.counter = ConnMan.conn_max_counter
1635
+                ConnMan.put(conn)
1634 1636
 
1635 1637
             if retries:
1636 1638
                 warning("Retrying failed request: %s (%s)" % (resource['uri'], e))
... ...
@@ -1715,7 +1722,8 @@ class S3(object):
1715 1715
         except (IOError, Exception) as e:
1716 1716
             if self.config.progress_meter:
1717 1717
                 progress.done("failed")
1718
-            if hasattr(e, 'errno') and e.errno not in (errno.EPIPE, errno.ECONNRESET):
1718
+            if ((hasattr(e, 'errno') and e.errno not in (errno.EPIPE, errno.ECONNRESET, errno.ETIMEDOUT))
1719
+               or "[Errno 104]" in str(e) or "[Errno 32]" in str(e)) and not isinstance(e, SocketTimeoutException):
1719 1720
                 raise
1720 1721
             # close the connection and re-establish
1721 1722
             conn.counter = ConnMan.conn_max_counter