from __future__ import absolute_import, print_function
import os
import sys
import http.client as httplib
from http.client import (_CS_REQ_SENT, _CS_REQ_STARTED, CONTINUE, UnknownProtocol,
CannotSendHeader, NO_CONTENT, NOT_MODIFIED, EXPECTATION_FAILED,
HTTPMessage, HTTPException)
from io import StringIO
from .Utils import encode_to_s3
_METHODS_EXPECTING_BODY = ['PATCH', 'POST', 'PUT']
# Fixed python 2.X httplib to be able to support
# Expect: 100-Continue http feature
# Inspired by:
# http://bugs.python.org/file26357/issue1346874-273.patch
def _encode(data, name='data'):
"""Call data.encode("latin-1") but show a better error message."""
try:
return data.encode("latin-1")
except UnicodeEncodeError as err:
# The following is equivalent to raise Exception() from None
# but is still byte-compilable compatible with python 2.
exc = UnicodeEncodeError(
err.encoding,
err.object,
err.start,
err.end,
"%s (%.20r) is not valid Latin-1. Use %s.encode('utf-8') "
"if you want to send it encoded in UTF-8." %
(name.title(), data[err.start:err.end], name))
exc.__cause__ = None
raise exc
def httpresponse_patched_begin(self):
""" Re-implemented httplib begin function
to not loop over "100 CONTINUE" status replies
but to report it to higher level so it can be processed.
"""
if self.headers is not None:
# we've already started reading the response
return
# read only one status even if we get a non-100 response
version, status, reason = self._read_status()
self.code = self.status = status
self.reason = reason.strip()
if version in ('HTTP/1.0', 'HTTP/0.9'):
# Some servers might still return "0.9", treat it as 1.0 anyway
self.version = 10
elif version.startswith('HTTP/1.'):
self.version = 11 # use HTTP/1.1 code for HTTP/1.x where x>=1
else:
raise UnknownProtocol(version)
self.headers = self.msg = httplib.parse_headers(self.fp)
if self.debuglevel > 0:
for hdr in self.headers:
print("header:", hdr, end=" ")
# are we using the chunked-style of transfer encoding?
tr_enc = self.headers.get('transfer-encoding')
if tr_enc and tr_enc.lower() == "chunked":
self.chunked = True
self.chunk_left = None
else:
self.chunked = False
# will the connection close at the end of the response?
self.will_close = self._check_close()
# do we have a Content-Length?
# NOTE: RFC 2616, S4.4, #3 says we ignore this if tr_enc is "chunked"
self.length = None
length = self.headers.get('content-length')
if length and not self.chunked:
try:
self.length = int(length)
except ValueError:
self.length = None
else:
if self.length < 0: # ignore nonsensical negative lengths
self.length = None
else:
self.length = None
# does the body have a fixed length? (of zero)
if (status == NO_CONTENT or status == NOT_MODIFIED or
100 <= status < 200 or # 1xx codes
self._method == 'HEAD'):
self.length = 0
# if the connection remains open, and we aren't using chunked, and
# a content-length was not provided, then assume that the connection
# WILL close.
if (not self.will_close and
not self.chunked and
self.length is None):
self.will_close = True
# No need to override httplib with this one, as it is only used by send_request
def httpconnection_patched_get_content_length(body, method):
"""## REIMPLEMENTED because new in last httplib but needed by send_request"""
"""Get the content-length based on the body.
If the body is None, we set Content-Length: 0 for methods that expect
a body (RFC 7230, Section 3.3.2). We also set the Content-Length for
any method if the body is a str or bytes-like object and not a file.
"""
if body is None:
# do an explicit check for not None here to distinguish
# between unset and set but empty
if method.upper() in _METHODS_EXPECTING_BODY:
return 0
else:
return None
if hasattr(body, 'read'):
# file-like object.
return None
try:
# does it implement the buffer protocol (bytes, bytearray, array)?
mv = memoryview(body)
return mv.nbytes
except TypeError:
pass
if isinstance(body, str):
return len(body)
return None
def httpconnection_patched_send_request(self, method, url, body, headers,
encode_chunked=False):
# Honor explicitly requested Host: and Accept-Encoding: headers.
header_names = dict.fromkeys([k.lower() for k in headers])
skips = {}
if 'host' in header_names:
skips['skip_host'] = 1
if 'accept-encoding' in header_names:
skips['skip_accept_encoding'] = 1
expect_continue = False
for hdr, value in headers.items():
if 'expect' == hdr.lower() and '100-continue' in value.lower():
expect_continue = True
self.putrequest(method, url, **skips)
# chunked encoding will happen if HTTP/1.1 is used and either
# the caller passes encode_chunked=True or the following
# conditions hold:
# 1. content-length has not been explicitly set
# 2. the body is a file or iterable, but not a str or bytes-like
# 3. Transfer-Encoding has NOT been explicitly set by the caller
if 'content-length' not in header_names:
# only chunk body if not explicitly set for backwards
# compatibility, assuming the client code is already handling the
# chunking
if 'transfer-encoding' not in header_names:
# if content-length cannot be automatically determined, fall
# back to chunked encoding
encode_chunked = False
content_length = httpconnection_patched_get_content_length(body, method)
if content_length is None:
if body is not None:
if self.debuglevel > 0:
print('Unable to determine size of %r' % body)
encode_chunked = True
self.putheader('Transfer-Encoding', 'chunked')
else:
self.putheader('Content-Length', str(content_length))
else:
encode_chunked = False
for hdr, value in headers.items():
self.putheader(encode_to_s3(hdr), encode_to_s3(value))
if isinstance(body, str):
# RFC 2616 Section 3.7.1 says that text default has a
# default charset of iso-8859-1.
body = _encode(body, 'body')
# If an Expect: 100-continue was sent, we need to check for a 417
# Expectation Failed to avoid unnecessarily sending the body
# See RFC 2616 8.2.3
if not expect_continue:
self.endheaders(body, encode_chunked=encode_chunked)
else:
if not body:
raise HTTPException("A body is required when expecting "
"100-continue")
self.endheaders()
resp = self.getresponse()
resp.read()
self._HTTPConnection__state = _CS_REQ_SENT
if resp.status == EXPECTATION_FAILED:
raise ExpectationFailed()
elif resp.status == CONTINUE:
self.wrapper_send_body(body, encode_chunked)
def httpconnection_patched_endheaders(self, message_body=None, encode_chunked=False):
"""REIMPLEMENTED because new argument encode_chunked added after py 3.4"""
"""Indicate that the last header line has been sent to the server.
This method sends the request to the server. The optional message_body
argument can be used to pass a message body associated with the
request.
"""
if self._HTTPConnection__state == _CS_REQ_STARTED:
self._HTTPConnection__state = _CS_REQ_SENT
else:
raise CannotSendHeader()
self._send_output(message_body, encode_chunked=encode_chunked)
def httpconnection_patched_read_readable(self, readable):
"""REIMPLEMENTED because needed by send_output and added after py 3.4
"""
blocksize = 8192
if self.debuglevel > 0:
print("sendIng a read()able")
encode = self._is_textIO(readable)
if encode and self.debuglevel > 0:
print("encoding file using iso-8859-1")
while True:
datablock = readable.read(blocksize)
if not datablock:
break
if encode:
datablock = datablock.encode("iso-8859-1")
yield datablock
def httpconnection_patched_send_output(self, message_body=None,
encode_chunked=False):
"""REIMPLEMENTED because needed by endheaders and parameter
encode_chunked was added"""
"""Send the currently buffered request and clear the buffer.
Appends an extra \\r\\n to the buffer.
A message_body may be specified, to be appended to the request.
"""
self._buffer.extend((b"", b""))
msg = b"\r\n".join(self._buffer)
del self._buffer[:]
self.send(msg)
if message_body is not None:
self.wrapper_send_body(message_body, encode_chunked)
class ExpectationFailed(HTTPException):
pass
# Wrappers #
def httpconnection_patched_wrapper_send_body(self, message_body, encode_chunked=False):
# create a consistent interface to message_body
if hasattr(message_body, 'read'):
# Let file-like take precedence over byte-like. This
# is needed to allow the current position of mmap'ed
# files to be taken into account.
chunks = self._read_readable(message_body)
else:
try:
# this is solely to check to see if message_body
# implements the buffer API. it /would/ be easier
# to capture if PyObject_CheckBuffer was exposed
# to Python.
memoryview(message_body)
except TypeError:
try:
chunks = iter(message_body)
except TypeError:
raise TypeError("message_body should be a bytes-like "
"object or an iterable, got %r"
% type(message_body))
else:
# the object implements the buffer interface and
# can be passed directly into socket methods
chunks = (message_body,)
for chunk in chunks:
if not chunk:
if self.debuglevel > 0:
print('Zero length chunk ignored')
continue
if encode_chunked and self._http_vsn == 11:
# chunked encoding
chunk = '{:X}\r\n'.format(len(chunk)).encode('ascii') + chunk \
+ b'\r\n'
self.send(chunk)
if encode_chunked and self._http_vsn == 11:
# end chunked transfer
self.send(b'0\r\n\r\n')
httplib.HTTPResponse.begin = httpresponse_patched_begin
httplib.HTTPConnection.endheaders = httpconnection_patched_endheaders
httplib.HTTPConnection._send_readable = httpconnection_patched_read_readable
httplib.HTTPConnection._send_output = httpconnection_patched_send_output
httplib.HTTPConnection._send_request = httpconnection_patched_send_request
# Interfaces added to httplib.HTTPConnection:
httplib.HTTPConnection.wrapper_send_body = httpconnection_patched_wrapper_send_body