awsrequest.py 23 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623
  1. # Copyright (c) 2012-2013 Mitch Garnaat http://garnaat.org/
  2. # Copyright 2012-2014 Amazon.com, Inc. or its affiliates. All Rights Reserved.
  3. #
  4. # Licensed under the Apache License, Version 2.0 (the "License"). You
  5. # may not use this file except in compliance with the License. A copy of
  6. # the License is located at
  7. #
  8. # http://aws.amazon.com/apache2.0/
  9. #
  10. # or in the "license" file accompanying this file. This file is
  11. # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
  12. # ANY KIND, either express or implied. See the License for the specific
  13. # language governing permissions and limitations under the License.
  14. import io
  15. import sys
  16. import logging
  17. import functools
  18. import socket
  19. import urllib3.util
  20. from urllib3.connection import VerifiedHTTPSConnection
  21. from urllib3.connection import HTTPConnection
  22. from urllib3.connectionpool import HTTPConnectionPool
  23. from urllib3.connectionpool import HTTPSConnectionPool
  24. import botocore.utils
  25. from botocore.compat import six
  26. from botocore.compat import HTTPHeaders, HTTPResponse, urlunsplit, urlsplit, \
  27. urlencode, MutableMapping
  28. from botocore.exceptions import UnseekableStreamError
  29. logger = logging.getLogger(__name__)
  30. class AWSHTTPResponse(HTTPResponse):
  31. # The *args, **kwargs is used because the args are slightly
  32. # different in py2.6 than in py2.7/py3.
  33. def __init__(self, *args, **kwargs):
  34. self._status_tuple = kwargs.pop('status_tuple')
  35. HTTPResponse.__init__(self, *args, **kwargs)
  36. def _read_status(self):
  37. if self._status_tuple is not None:
  38. status_tuple = self._status_tuple
  39. self._status_tuple = None
  40. return status_tuple
  41. else:
  42. return HTTPResponse._read_status(self)
  43. class AWSConnection(object):
  44. """Mixin for HTTPConnection that supports Expect 100-continue.
  45. This when mixed with a subclass of httplib.HTTPConnection (though
  46. technically we subclass from urllib3, which subclasses
  47. httplib.HTTPConnection) and we only override this class to support Expect
  48. 100-continue, which we need for S3. As far as I can tell, this is
  49. general purpose enough to not be specific to S3, but I'm being
  50. tentative and keeping it in botocore because I've only tested
  51. this against AWS services.
  52. """
  53. def __init__(self, *args, **kwargs):
  54. super(AWSConnection, self).__init__(*args, **kwargs)
  55. self._original_response_cls = self.response_class
  56. # We'd ideally hook into httplib's states, but they're all
  57. # __mangled_vars so we use our own state var. This variable is set
  58. # when we receive an early response from the server. If this value is
  59. # set to True, any calls to send() are noops. This value is reset to
  60. # false every time _send_request is called. This is to workaround the
  61. # fact that py2.6 (and only py2.6) has a separate send() call for the
  62. # body in _send_request, as opposed to endheaders(), which is where the
  63. # body is sent in all versions > 2.6.
  64. self._response_received = False
  65. self._expect_header_set = False
  66. def close(self):
  67. super(AWSConnection, self).close()
  68. # Reset all of our instance state we were tracking.
  69. self._response_received = False
  70. self._expect_header_set = False
  71. self.response_class = self._original_response_cls
  72. def _send_request(self, method, url, body, headers, *args, **kwargs):
  73. self._response_received = False
  74. if headers.get('Expect', b'') == b'100-continue':
  75. self._expect_header_set = True
  76. else:
  77. self._expect_header_set = False
  78. self.response_class = self._original_response_cls
  79. rval = super(AWSConnection, self)._send_request(
  80. method, url, body, headers, *args, **kwargs)
  81. self._expect_header_set = False
  82. return rval
  83. def _convert_to_bytes(self, mixed_buffer):
  84. # Take a list of mixed str/bytes and convert it
  85. # all into a single bytestring.
  86. # Any six.text_types will be encoded as utf-8.
  87. bytes_buffer = []
  88. for chunk in mixed_buffer:
  89. if isinstance(chunk, six.text_type):
  90. bytes_buffer.append(chunk.encode('utf-8'))
  91. else:
  92. bytes_buffer.append(chunk)
  93. msg = b"\r\n".join(bytes_buffer)
  94. return msg
  95. def _send_output(self, message_body=None, *args, **kwargs):
  96. self._buffer.extend((b"", b""))
  97. msg = self._convert_to_bytes(self._buffer)
  98. del self._buffer[:]
  99. # If msg and message_body are sent in a single send() call,
  100. # it will avoid performance problems caused by the interaction
  101. # between delayed ack and the Nagle algorithm.
  102. if isinstance(message_body, bytes):
  103. msg += message_body
  104. message_body = None
  105. self.send(msg)
  106. if self._expect_header_set:
  107. # This is our custom behavior. If the Expect header was
  108. # set, it will trigger this custom behavior.
  109. logger.debug("Waiting for 100 Continue response.")
  110. # Wait for 1 second for the server to send a response.
  111. if urllib3.util.wait_for_read(self.sock, 1):
  112. self._handle_expect_response(message_body)
  113. return
  114. else:
  115. # From the RFC:
  116. # Because of the presence of older implementations, the
  117. # protocol allows ambiguous situations in which a client may
  118. # send "Expect: 100-continue" without receiving either a 417
  119. # (Expectation Failed) status or a 100 (Continue) status.
  120. # Therefore, when a client sends this header field to an origin
  121. # server (possibly via a proxy) from which it has never seen a
  122. # 100 (Continue) status, the client SHOULD NOT wait for an
  123. # indefinite period before sending the request body.
  124. logger.debug("No response seen from server, continuing to "
  125. "send the response body.")
  126. if message_body is not None:
  127. # message_body was not a string (i.e. it is a file), and
  128. # we must run the risk of Nagle.
  129. self.send(message_body)
  130. def _consume_headers(self, fp):
  131. # Most servers (including S3) will just return
  132. # the CLRF after the 100 continue response. However,
  133. # some servers (I've specifically seen this for squid when
  134. # used as a straight HTTP proxy) will also inject a
  135. # Connection: keep-alive header. To account for this
  136. # we'll read until we read '\r\n', and ignore any headers
  137. # that come immediately after the 100 continue response.
  138. current = None
  139. while current != b'\r\n':
  140. current = fp.readline()
  141. def _handle_expect_response(self, message_body):
  142. # This is called when we sent the request headers containing
  143. # an Expect: 100-continue header and received a response.
  144. # We now need to figure out what to do.
  145. fp = self.sock.makefile('rb', 0)
  146. try:
  147. maybe_status_line = fp.readline()
  148. parts = maybe_status_line.split(None, 2)
  149. if self._is_100_continue_status(maybe_status_line):
  150. self._consume_headers(fp)
  151. logger.debug("100 Continue response seen, "
  152. "now sending request body.")
  153. self._send_message_body(message_body)
  154. elif len(parts) == 3 and parts[0].startswith(b'HTTP/'):
  155. # From the RFC:
  156. # Requirements for HTTP/1.1 origin servers:
  157. #
  158. # - Upon receiving a request which includes an Expect
  159. # request-header field with the "100-continue"
  160. # expectation, an origin server MUST either respond with
  161. # 100 (Continue) status and continue to read from the
  162. # input stream, or respond with a final status code.
  163. #
  164. # So if we don't get a 100 Continue response, then
  165. # whatever the server has sent back is the final response
  166. # and don't send the message_body.
  167. logger.debug("Received a non 100 Continue response "
  168. "from the server, NOT sending request body.")
  169. status_tuple = (parts[0].decode('ascii'),
  170. int(parts[1]), parts[2].decode('ascii'))
  171. response_class = functools.partial(
  172. AWSHTTPResponse, status_tuple=status_tuple)
  173. self.response_class = response_class
  174. self._response_received = True
  175. finally:
  176. fp.close()
  177. def _send_message_body(self, message_body):
  178. if message_body is not None:
  179. self.send(message_body)
  180. def send(self, str):
  181. if self._response_received:
  182. logger.debug("send() called, but reseponse already received. "
  183. "Not sending data.")
  184. return
  185. return super(AWSConnection, self).send(str)
  186. def _is_100_continue_status(self, maybe_status_line):
  187. parts = maybe_status_line.split(None, 2)
  188. # Check for HTTP/<version> 100 Continue\r\n
  189. return (
  190. len(parts) >= 3 and parts[0].startswith(b'HTTP/') and
  191. parts[1] == b'100')
  192. class AWSHTTPConnection(AWSConnection, HTTPConnection):
  193. """ An HTTPConnection that supports 100 Continue behavior. """
  194. class AWSHTTPSConnection(AWSConnection, VerifiedHTTPSConnection):
  195. """ An HTTPSConnection that supports 100 Continue behavior. """
  196. class AWSHTTPConnectionPool(HTTPConnectionPool):
  197. ConnectionCls = AWSHTTPConnection
  198. class AWSHTTPSConnectionPool(HTTPSConnectionPool):
  199. ConnectionCls = AWSHTTPSConnection
  200. def prepare_request_dict(request_dict, endpoint_url, context=None,
  201. user_agent=None):
  202. """
  203. This method prepares a request dict to be created into an
  204. AWSRequestObject. This prepares the request dict by adding the
  205. url and the user agent to the request dict.
  206. :type request_dict: dict
  207. :param request_dict: The request dict (created from the
  208. ``serialize`` module).
  209. :type user_agent: string
  210. :param user_agent: The user agent to use for this request.
  211. :type endpoint_url: string
  212. :param endpoint_url: The full endpoint url, which contains at least
  213. the scheme, the hostname, and optionally any path components.
  214. """
  215. r = request_dict
  216. if user_agent is not None:
  217. headers = r['headers']
  218. headers['User-Agent'] = user_agent
  219. host_prefix = r.get('host_prefix')
  220. url = _urljoin(endpoint_url, r['url_path'], host_prefix)
  221. if r['query_string']:
  222. # NOTE: This is to avoid circular import with utils. This is being
  223. # done to avoid moving classes to different modules as to not cause
  224. # breaking chainges.
  225. percent_encode_sequence = botocore.utils.percent_encode_sequence
  226. encoded_query_string = percent_encode_sequence(r['query_string'])
  227. if '?' not in url:
  228. url += '?%s' % encoded_query_string
  229. else:
  230. url += '&%s' % encoded_query_string
  231. r['url'] = url
  232. r['context'] = context
  233. if context is None:
  234. r['context'] = {}
  235. def create_request_object(request_dict):
  236. """
  237. This method takes a request dict and creates an AWSRequest object
  238. from it.
  239. :type request_dict: dict
  240. :param request_dict: The request dict (created from the
  241. ``prepare_request_dict`` method).
  242. :rtype: ``botocore.awsrequest.AWSRequest``
  243. :return: An AWSRequest object based on the request_dict.
  244. """
  245. r = request_dict
  246. request_object = AWSRequest(
  247. method=r['method'], url=r['url'], data=r['body'], headers=r['headers'])
  248. request_object.context = r['context']
  249. return request_object
  250. def _urljoin(endpoint_url, url_path, host_prefix):
  251. p = urlsplit(endpoint_url)
  252. # <part> - <index>
  253. # scheme - p[0]
  254. # netloc - p[1]
  255. # path - p[2]
  256. # query - p[3]
  257. # fragment - p[4]
  258. if not url_path or url_path == '/':
  259. # If there's no path component, ensure the URL ends with
  260. # a '/' for backwards compatibility.
  261. if not p[2]:
  262. new_path = '/'
  263. else:
  264. new_path = p[2]
  265. elif p[2].endswith('/') and url_path.startswith('/'):
  266. new_path = p[2][:-1] + url_path
  267. else:
  268. new_path = p[2] + url_path
  269. new_netloc = p[1]
  270. if host_prefix is not None:
  271. new_netloc = host_prefix + new_netloc
  272. reconstructed = urlunsplit((p[0], new_netloc, new_path, p[3], p[4]))
  273. return reconstructed
  274. class AWSRequestPreparer(object):
  275. """
  276. This class performs preparation on AWSRequest objects similar to that of
  277. the PreparedRequest class does in the requests library. However, the logic
  278. has been boiled down to meet the specific use cases in botocore. Of note
  279. there are the following differences:
  280. This class does not heavily prepare the URL. Requests performed many
  281. validations and corrections to ensure the URL is properly formatted.
  282. Botocore either performs these validations elsewhere or otherwise
  283. consistently provides well formatted URLs.
  284. This class does not heavily prepare the body. Body preperation is
  285. simple and supports only the cases that we document: bytes and
  286. file-like objects to determine the content-length. This will also
  287. additionally prepare a body that is a dict to be url encoded params
  288. string as some signers rely on this. Finally, this class does not
  289. support multipart file uploads.
  290. This class does not prepare the method, auth or cookies.
  291. """
  292. def prepare(self, original):
  293. method = original.method
  294. url = self._prepare_url(original)
  295. body = self._prepare_body(original)
  296. headers = self._prepare_headers(original, body)
  297. stream_output = original.stream_output
  298. return AWSPreparedRequest(method, url, headers, body, stream_output)
  299. def _prepare_url(self, original):
  300. url = original.url
  301. if original.params:
  302. params = urlencode(list(original.params.items()), doseq=True)
  303. url = '%s?%s' % (url, params)
  304. return url
  305. def _prepare_headers(self, original, prepared_body=None):
  306. headers = HeadersDict(original.headers.items())
  307. # If the transfer encoding or content length is already set, use that
  308. if 'Transfer-Encoding' in headers or 'Content-Length' in headers:
  309. return headers
  310. # Ensure we set the content length when it is expected
  311. if original.method not in ('GET', 'HEAD', 'OPTIONS'):
  312. length = self._determine_content_length(prepared_body)
  313. if length is not None:
  314. headers['Content-Length'] = str(length)
  315. else:
  316. # Failed to determine content length, using chunked
  317. # NOTE: This shouldn't ever happen in practice
  318. body_type = type(prepared_body)
  319. logger.debug('Failed to determine length of %s', body_type)
  320. headers['Transfer-Encoding'] = 'chunked'
  321. return headers
  322. def _to_utf8(self, item):
  323. key, value = item
  324. if isinstance(key, six.text_type):
  325. key = key.encode('utf-8')
  326. if isinstance(value, six.text_type):
  327. value = value.encode('utf-8')
  328. return key, value
  329. def _prepare_body(self, original):
  330. """Prepares the given HTTP body data."""
  331. body = original.data
  332. if body == b'':
  333. body = None
  334. if isinstance(body, dict):
  335. params = [self._to_utf8(item) for item in body.items()]
  336. body = urlencode(params, doseq=True)
  337. return body
  338. def _determine_content_length(self, body):
  339. # No body, content length of 0
  340. if not body:
  341. return 0
  342. # Try asking the body for it's length
  343. try:
  344. return len(body)
  345. except (AttributeError, TypeError) as e:
  346. pass
  347. # Try getting the length from a seekable stream
  348. if hasattr(body, 'seek') and hasattr(body, 'tell'):
  349. try:
  350. orig_pos = body.tell()
  351. body.seek(0, 2)
  352. end_file_pos = body.tell()
  353. body.seek(orig_pos)
  354. return end_file_pos - orig_pos
  355. except io.UnsupportedOperation:
  356. # in case when body is, for example, io.BufferedIOBase object
  357. # it has "seek" method which throws "UnsupportedOperation"
  358. # exception in such case we want to fall back to "chunked"
  359. # encoding
  360. pass
  361. # Failed to determine the length
  362. return None
  363. class AWSRequest(object):
  364. """Represents the elements of an HTTP request.
  365. This class was originally inspired by requests.models.Request, but has been
  366. boiled down to meet the specific use cases in botocore. That being said this
  367. class (even in requests) is effectively a named-tuple.
  368. """
  369. _REQUEST_PREPARER_CLS = AWSRequestPreparer
  370. def __init__(self,
  371. method=None,
  372. url=None,
  373. headers=None,
  374. data=None,
  375. params=None,
  376. auth_path=None,
  377. stream_output=False):
  378. self._request_preparer = self._REQUEST_PREPARER_CLS()
  379. # Default empty dicts for dict params.
  380. params = {} if params is None else params
  381. self.method = method
  382. self.url = url
  383. self.headers = HTTPHeaders()
  384. self.data = data
  385. self.params = params
  386. self.auth_path = auth_path
  387. self.stream_output = stream_output
  388. if headers is not None:
  389. for key, value in headers.items():
  390. self.headers[key] = value
  391. # This is a dictionary to hold information that is used when
  392. # processing the request. What is inside of ``context`` is open-ended.
  393. # For example, it may have a timestamp key that is used for holding
  394. # what the timestamp is when signing the request. Note that none
  395. # of the information that is inside of ``context`` is directly
  396. # sent over the wire; the information is only used to assist in
  397. # creating what is sent over the wire.
  398. self.context = {}
  399. def prepare(self):
  400. """Constructs a :class:`AWSPreparedRequest <AWSPreparedRequest>`."""
  401. return self._request_preparer.prepare(self)
  402. @property
  403. def body(self):
  404. body = self.prepare().body
  405. if isinstance(body, six.text_type):
  406. body = body.encode('utf-8')
  407. return body
  408. class AWSPreparedRequest(object):
  409. """A data class representing a finalized request to be sent over the wire.
  410. Requests at this stage should be treated as final, and the properties of
  411. the request should not be modified.
  412. :ivar method: The HTTP Method
  413. :ivar url: The full url
  414. :ivar headers: The HTTP headers to send.
  415. :ivar body: The HTTP body.
  416. :ivar stream_output: If the response for this request should be streamed.
  417. """
  418. def __init__(self, method, url, headers, body, stream_output):
  419. self.method = method
  420. self.url = url
  421. self.headers = headers
  422. self.body = body
  423. self.stream_output = stream_output
  424. def __repr__(self):
  425. fmt = (
  426. '<AWSPreparedRequest stream_output=%s, method=%s, url=%s, '
  427. 'headers=%s>'
  428. )
  429. return fmt % (self.stream_output, self.method, self.url, self.headers)
  430. def reset_stream(self):
  431. """Resets the streaming body to it's initial position.
  432. If the request contains a streaming body (a streamable file-like object)
  433. seek to the object's initial position to ensure the entire contents of
  434. the object is sent. This is a no-op for static bytes-like body types.
  435. """
  436. # Trying to reset a stream when there is a no stream will
  437. # just immediately return. It's not an error, it will produce
  438. # the same result as if we had actually reset the stream (we'll send
  439. # the entire body contents again if we need to).
  440. # Same case if the body is a string/bytes/bytearray type.
  441. non_seekable_types = (six.binary_type, six.text_type, bytearray)
  442. if self.body is None or isinstance(self.body, non_seekable_types):
  443. return
  444. try:
  445. logger.debug("Rewinding stream: %s", self.body)
  446. self.body.seek(0)
  447. except Exception as e:
  448. logger.debug("Unable to rewind stream: %s", e)
  449. raise UnseekableStreamError(stream_object=self.body)
  450. class AWSResponse(object):
  451. """A data class representing an HTTP response.
  452. This class was originally inspired by requests.models.Response, but has
  453. been boiled down to meet the specific use cases in botocore. This has
  454. effectively been reduced to a named tuple.
  455. :ivar url: The full url.
  456. :ivar status_code: The status code of the HTTP response.
  457. :ivar headers: The HTTP headers received.
  458. :ivar body: The HTTP response body.
  459. """
  460. def __init__(self, url, status_code, headers, raw):
  461. self.url = url
  462. self.status_code = status_code
  463. self.headers = HeadersDict(headers)
  464. self.raw = raw
  465. self._content = None
  466. @property
  467. def content(self):
  468. """Content of the response as bytes."""
  469. if self._content is None:
  470. # Read the contents.
  471. # NOTE: requests would attempt to call stream and fall back
  472. # to a custom generator that would call read in a loop, but
  473. # we don't rely on this behavior
  474. self._content = bytes().join(self.raw.stream()) or bytes()
  475. return self._content
  476. @property
  477. def text(self):
  478. """Content of the response as a proper text type.
  479. Uses the encoding type provided in the reponse headers to decode the
  480. response content into a proper text type. If the encoding is not
  481. present in the headers, UTF-8 is used as a default.
  482. """
  483. encoding = botocore.utils.get_encoding_from_headers(self.headers)
  484. if encoding:
  485. return self.content.decode(encoding)
  486. else:
  487. return self.content.decode('utf-8')
  488. class _HeaderKey(object):
  489. def __init__(self, key):
  490. self._key = key
  491. self._lower = key.lower()
  492. def __hash__(self):
  493. return hash(self._lower)
  494. def __eq__(self, other):
  495. return isinstance(other, _HeaderKey) and self._lower == other._lower
  496. def __str__(self):
  497. return self._key
  498. def __repr__(self):
  499. return repr(self._key)
  500. class HeadersDict(MutableMapping):
  501. """A case-insenseitive dictionary to represent HTTP headers. """
  502. def __init__(self, *args, **kwargs):
  503. self._dict = {}
  504. self.update(*args, **kwargs)
  505. def __setitem__(self, key, value):
  506. self._dict[_HeaderKey(key)] = value
  507. def __getitem__(self, key):
  508. return self._dict[_HeaderKey(key)]
  509. def __delitem__(self, key):
  510. del self._dict[_HeaderKey(key)]
  511. def __iter__(self):
  512. return (str(key) for key in self._dict)
  513. def __len__(self):
  514. return len(self._dict)
  515. def __repr__(self):
  516. return repr(self._dict)
  517. def copy(self):
  518. return HeadersDict(self.items())