parsers.py 41 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024
  1. # Copyright 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved.
  2. #
  3. # Licensed under the Apache License, Version 2.0 (the "License"). You
  4. # may not use this file except in compliance with the License. A copy of
  5. # the License is located at
  6. #
  7. # http://aws.amazon.com/apache2.0/
  8. #
  9. # or in the "license" file accompanying this file. This file is
  10. # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
  11. # ANY KIND, either express or implied. See the License for the specific
  12. # language governing permissions and limitations under the License.
  13. """Response parsers for the various protocol types.
  14. The module contains classes that can take an HTTP response, and given
  15. an output shape, parse the response into a dict according to the
  16. rules in the output shape.
  17. There are many similarities amongst the different protocols with regard
  18. to response parsing, and the code is structured in a way to avoid
  19. code duplication when possible. The diagram below is a diagram
  20. showing the inheritance hierarchy of the response classes.
  21. ::
  22. +--------------+
  23. |ResponseParser|
  24. +--------------+
  25. ^ ^ ^
  26. +--------------------+ | +-------------------+
  27. | | |
  28. +----------+----------+ +------+-------+ +-------+------+
  29. |BaseXMLResponseParser| |BaseRestParser| |BaseJSONParser|
  30. +---------------------+ +--------------+ +--------------+
  31. ^ ^ ^ ^ ^ ^
  32. | | | | | |
  33. | | | | | |
  34. | ++----------+-+ +-+-----------++ |
  35. | |RestXMLParser| |RestJSONParser| |
  36. +-----+-----+ +-------------+ +--------------+ +----+-----+
  37. |QueryParser| |JSONParser|
  38. +-----------+ +----------+
  39. The diagram above shows that there is a base class, ``ResponseParser`` that
  40. contains logic that is similar amongst all the different protocols (``query``,
  41. ``json``, ``rest-json``, ``rest-xml``). Amongst the various services there
  42. is shared logic that can be grouped several ways:
  43. * The ``query`` and ``rest-xml`` both have XML bodies that are parsed in the
  44. same way.
  45. * The ``json`` and ``rest-json`` protocols both have JSON bodies that are
  46. parsed in the same way.
  47. * The ``rest-json`` and ``rest-xml`` protocols have additional attributes
  48. besides body parameters that are parsed the same (headers, query string,
  49. status code).
  50. This is reflected in the class diagram above. The ``BaseXMLResponseParser``
  51. and the BaseJSONParser contain logic for parsing the XML/JSON body,
  52. and the BaseRestParser contains logic for parsing out attributes that
  53. come from other parts of the HTTP response. Classes like the
  54. ``RestXMLParser`` inherit from the ``BaseXMLResponseParser`` to get the
  55. XML body parsing logic and the ``BaseRestParser`` to get the HTTP
  56. header/status code/query string parsing.
  57. Additionally, there are event stream parsers that are used by the other parsers
  58. to wrap streaming bodies that represent a stream of events. The
  59. BaseEventStreamParser extends from ResponseParser and defines the logic for
  60. parsing values from the headers and payload of a message from the underlying
  61. binary encoding protocol. Currently, event streams support parsing bodies
  62. encoded as JSON and XML through the following hierarchy.
  63. +--------------+
  64. |ResponseParser|
  65. +--------------+
  66. ^ ^ ^
  67. +--------------------+ | +------------------+
  68. | | |
  69. +----------+----------+ +----------+----------+ +-------+------+
  70. |BaseXMLResponseParser| |BaseEventStreamParser| |BaseJSONParser|
  71. +---------------------+ +---------------------+ +--------------+
  72. ^ ^ ^ ^
  73. | | | |
  74. | | | |
  75. +-+----------------+-+ +-+-----------------+-+
  76. |EventStreamXMLParser| |EventStreamJSONParser|
  77. +--------------------+ +---------------------+
  78. Return Values
  79. =============
  80. Each call to ``parse()`` returns a dict has this form::
  81. Standard Response
  82. {
  83. "ResponseMetadata": {"RequestId": <requestid>}
  84. <response keys>
  85. }
  86. Error response
  87. {
  88. "ResponseMetadata": {"RequestId": <requestid>}
  89. "Error": {
  90. "Code": <string>,
  91. "Message": <string>,
  92. "Type": <string>,
  93. <additional keys>
  94. }
  95. }
  96. """
  97. import re
  98. import base64
  99. import json
  100. import logging
  101. from botocore.compat import six, ETree, XMLParseError
  102. from botocore.eventstream import EventStream, NoInitialResponseError
  103. from botocore.utils import parse_timestamp, merge_dicts, \
  104. is_json_value_header, lowercase_dict
  105. LOG = logging.getLogger(__name__)
  106. DEFAULT_TIMESTAMP_PARSER = parse_timestamp
  107. class ResponseParserFactory(object):
  108. def __init__(self):
  109. self._defaults = {}
  110. def set_parser_defaults(self, **kwargs):
  111. """Set default arguments when a parser instance is created.
  112. You can specify any kwargs that are allowed by a ResponseParser
  113. class. There are currently two arguments:
  114. * timestamp_parser - A callable that can parse a timestamp string
  115. * blob_parser - A callable that can parse a blob type
  116. """
  117. self._defaults.update(kwargs)
  118. def create_parser(self, protocol_name):
  119. parser_cls = PROTOCOL_PARSERS[protocol_name]
  120. return parser_cls(**self._defaults)
  121. def create_parser(protocol):
  122. return ResponseParserFactory().create_parser(protocol)
  123. def _text_content(func):
  124. # This decorator hides the difference between
  125. # an XML node with text or a plain string. It's used
  126. # to ensure that scalar processing operates only on text
  127. # strings, which allows the same scalar handlers to be used
  128. # for XML nodes from the body and HTTP headers.
  129. def _get_text_content(self, shape, node_or_string):
  130. if hasattr(node_or_string, 'text'):
  131. text = node_or_string.text
  132. if text is None:
  133. # If an XML node is empty <foo></foo>,
  134. # we want to parse that as an empty string,
  135. # not as a null/None value.
  136. text = ''
  137. else:
  138. text = node_or_string
  139. return func(self, shape, text)
  140. return _get_text_content
  141. class ResponseParserError(Exception):
  142. pass
  143. class ResponseParser(object):
  144. """Base class for response parsing.
  145. This class represents the interface that all ResponseParsers for the
  146. various protocols must implement.
  147. This class will take an HTTP response and a model shape and parse the
  148. HTTP response into a dictionary.
  149. There is a single public method exposed: ``parse``. See the ``parse``
  150. docstring for more info.
  151. """
  152. DEFAULT_ENCODING = 'utf-8'
  153. EVENT_STREAM_PARSER_CLS = None
  154. def __init__(self, timestamp_parser=None, blob_parser=None):
  155. if timestamp_parser is None:
  156. timestamp_parser = DEFAULT_TIMESTAMP_PARSER
  157. self._timestamp_parser = timestamp_parser
  158. if blob_parser is None:
  159. blob_parser = self._default_blob_parser
  160. self._blob_parser = blob_parser
  161. self._event_stream_parser = None
  162. if self.EVENT_STREAM_PARSER_CLS is not None:
  163. self._event_stream_parser = self.EVENT_STREAM_PARSER_CLS(
  164. timestamp_parser, blob_parser)
  165. def _default_blob_parser(self, value):
  166. # Blobs are always returned as bytes type (this matters on python3).
  167. # We don't decode this to a str because it's entirely possible that the
  168. # blob contains binary data that actually can't be decoded.
  169. return base64.b64decode(value)
  170. def parse(self, response, shape):
  171. """Parse the HTTP response given a shape.
  172. :param response: The HTTP response dictionary. This is a dictionary
  173. that represents the HTTP request. The dictionary must have the
  174. following keys, ``body``, ``headers``, and ``status_code``.
  175. :param shape: The model shape describing the expected output.
  176. :return: Returns a dictionary representing the parsed response
  177. described by the model. In addition to the shape described from
  178. the model, each response will also have a ``ResponseMetadata``
  179. which contains metadata about the response, which contains at least
  180. two keys containing ``RequestId`` and ``HTTPStatusCode``. Some
  181. responses may populate additional keys, but ``RequestId`` will
  182. always be present.
  183. """
  184. LOG.debug('Response headers: %s', response['headers'])
  185. LOG.debug('Response body:\n%s', response['body'])
  186. if response['status_code'] >= 301:
  187. if self._is_generic_error_response(response):
  188. parsed = self._do_generic_error_parse(response)
  189. elif self._is_modeled_error_shape(shape):
  190. parsed = self._do_modeled_error_parse(response, shape)
  191. # We don't want to decorate the modeled fields with metadata
  192. return parsed
  193. else:
  194. parsed = self._do_error_parse(response, shape)
  195. else:
  196. parsed = self._do_parse(response, shape)
  197. # We don't want to decorate event stream responses with metadata
  198. if shape and shape.serialization.get('eventstream'):
  199. return parsed
  200. # Add ResponseMetadata if it doesn't exist and inject the HTTP
  201. # status code and headers from the response.
  202. if isinstance(parsed, dict):
  203. response_metadata = parsed.get('ResponseMetadata', {})
  204. response_metadata['HTTPStatusCode'] = response['status_code']
  205. # Ensure that the http header keys are all lower cased. Older
  206. # versions of urllib3 (< 1.11) would unintentionally do this for us
  207. # (see urllib3#633). We need to do this conversion manually now.
  208. headers = response['headers']
  209. response_metadata['HTTPHeaders'] = lowercase_dict(headers)
  210. parsed['ResponseMetadata'] = response_metadata
  211. return parsed
  212. def _is_modeled_error_shape(self, shape):
  213. return shape is not None and shape.metadata.get('exception', False)
  214. def _is_generic_error_response(self, response):
  215. # There are times when a service will respond with a generic
  216. # error response such as:
  217. # '<html><body><b>Http/1.1 Service Unavailable</b></body></html>'
  218. #
  219. # This can also happen if you're going through a proxy.
  220. # In this case the protocol specific _do_error_parse will either
  221. # fail to parse the response (in the best case) or silently succeed
  222. # and treat the HTML above as an XML response and return
  223. # non sensical parsed data.
  224. # To prevent this case from happening we first need to check
  225. # whether or not this response looks like the generic response.
  226. if response['status_code'] >= 500:
  227. if 'body' not in response or response['body'] is None:
  228. return True
  229. body = response['body'].strip()
  230. return body.startswith(b'<html>') or not body
  231. def _do_generic_error_parse(self, response):
  232. # There's not really much we can do when we get a generic
  233. # html response.
  234. LOG.debug("Received a non protocol specific error response from the "
  235. "service, unable to populate error code and message.")
  236. return {
  237. 'Error': {'Code': str(response['status_code']),
  238. 'Message': six.moves.http_client.responses.get(
  239. response['status_code'], '')},
  240. 'ResponseMetadata': {},
  241. }
  242. def _do_parse(self, response, shape):
  243. raise NotImplementedError("%s._do_parse" % self.__class__.__name__)
  244. def _do_error_parse(self, response, shape):
  245. raise NotImplementedError(
  246. "%s._do_error_parse" % self.__class__.__name__)
  247. def _do_modeled_error_parse(self, response, shape, parsed):
  248. raise NotImplementedError(
  249. "%s._do_modeled_error_parse" % self.__class__.__name__)
  250. def _parse_shape(self, shape, node):
  251. handler = getattr(self, '_handle_%s' % shape.type_name,
  252. self._default_handle)
  253. return handler(shape, node)
  254. def _handle_list(self, shape, node):
  255. # Enough implementations share list serialization that it's moved
  256. # up here in the base class.
  257. parsed = []
  258. member_shape = shape.member
  259. for item in node:
  260. parsed.append(self._parse_shape(member_shape, item))
  261. return parsed
  262. def _default_handle(self, shape, value):
  263. return value
  264. def _create_event_stream(self, response, shape):
  265. parser = self._event_stream_parser
  266. name = response['context'].get('operation_name')
  267. return EventStream(response['body'], shape, parser, name)
  268. class BaseXMLResponseParser(ResponseParser):
  269. def __init__(self, timestamp_parser=None, blob_parser=None):
  270. super(BaseXMLResponseParser, self).__init__(timestamp_parser,
  271. blob_parser)
  272. self._namespace_re = re.compile('{.*}')
  273. def _handle_map(self, shape, node):
  274. parsed = {}
  275. key_shape = shape.key
  276. value_shape = shape.value
  277. key_location_name = key_shape.serialization.get('name') or 'key'
  278. value_location_name = value_shape.serialization.get('name') or 'value'
  279. if shape.serialization.get('flattened') and not isinstance(node, list):
  280. node = [node]
  281. for keyval_node in node:
  282. for single_pair in keyval_node:
  283. # Within each <entry> there's a <key> and a <value>
  284. tag_name = self._node_tag(single_pair)
  285. if tag_name == key_location_name:
  286. key_name = self._parse_shape(key_shape, single_pair)
  287. elif tag_name == value_location_name:
  288. val_name = self._parse_shape(value_shape, single_pair)
  289. else:
  290. raise ResponseParserError("Unknown tag: %s" % tag_name)
  291. parsed[key_name] = val_name
  292. return parsed
  293. def _node_tag(self, node):
  294. return self._namespace_re.sub('', node.tag)
  295. def _handle_list(self, shape, node):
  296. # When we use _build_name_to_xml_node, repeated elements are aggregated
  297. # into a list. However, we can't tell the difference between a scalar
  298. # value and a single element flattened list. So before calling the
  299. # real _handle_list, we know that "node" should actually be a list if
  300. # it's flattened, and if it's not, then we make it a one element list.
  301. if shape.serialization.get('flattened') and not isinstance(node, list):
  302. node = [node]
  303. return super(BaseXMLResponseParser, self)._handle_list(shape, node)
  304. def _handle_structure(self, shape, node):
  305. parsed = {}
  306. members = shape.members
  307. if shape.metadata.get('exception', False):
  308. node = self._get_error_root(node)
  309. xml_dict = self._build_name_to_xml_node(node)
  310. for member_name in members:
  311. member_shape = members[member_name]
  312. if 'location' in member_shape.serialization or \
  313. member_shape.serialization.get('eventheader'):
  314. # All members with locations have already been handled,
  315. # so we don't need to parse these members.
  316. continue
  317. xml_name = self._member_key_name(member_shape, member_name)
  318. member_node = xml_dict.get(xml_name)
  319. if member_node is not None:
  320. parsed[member_name] = self._parse_shape(
  321. member_shape, member_node)
  322. elif member_shape.serialization.get('xmlAttribute'):
  323. attribs = {}
  324. location_name = member_shape.serialization['name']
  325. for key, value in node.attrib.items():
  326. new_key = self._namespace_re.sub(
  327. location_name.split(':')[0] + ':', key)
  328. attribs[new_key] = value
  329. if location_name in attribs:
  330. parsed[member_name] = attribs[location_name]
  331. return parsed
  332. def _get_error_root(self, original_root):
  333. if self._node_tag(original_root) == 'ErrorResponse':
  334. for child in original_root:
  335. if self._node_tag(child) == 'Error':
  336. return child
  337. return original_root
  338. def _member_key_name(self, shape, member_name):
  339. # This method is needed because we have to special case flattened list
  340. # with a serialization name. If this is the case we use the
  341. # locationName from the list's member shape as the key name for the
  342. # surrounding structure.
  343. if shape.type_name == 'list' and shape.serialization.get('flattened'):
  344. list_member_serialized_name = shape.member.serialization.get(
  345. 'name')
  346. if list_member_serialized_name is not None:
  347. return list_member_serialized_name
  348. serialized_name = shape.serialization.get('name')
  349. if serialized_name is not None:
  350. return serialized_name
  351. return member_name
  352. def _build_name_to_xml_node(self, parent_node):
  353. # If the parent node is actually a list. We should not be trying
  354. # to serialize it to a dictionary. Instead, return the first element
  355. # in the list.
  356. if isinstance(parent_node, list):
  357. return self._build_name_to_xml_node(parent_node[0])
  358. xml_dict = {}
  359. for item in parent_node:
  360. key = self._node_tag(item)
  361. if key in xml_dict:
  362. # If the key already exists, the most natural
  363. # way to handle this is to aggregate repeated
  364. # keys into a single list.
  365. # <foo>1</foo><foo>2</foo> -> {'foo': [Node(1), Node(2)]}
  366. if isinstance(xml_dict[key], list):
  367. xml_dict[key].append(item)
  368. else:
  369. # Convert from a scalar to a list.
  370. xml_dict[key] = [xml_dict[key], item]
  371. else:
  372. xml_dict[key] = item
  373. return xml_dict
  374. def _parse_xml_string_to_dom(self, xml_string):
  375. try:
  376. parser = ETree.XMLParser(
  377. target=ETree.TreeBuilder(),
  378. encoding=self.DEFAULT_ENCODING)
  379. parser.feed(xml_string)
  380. root = parser.close()
  381. except XMLParseError as e:
  382. raise ResponseParserError(
  383. "Unable to parse response (%s), "
  384. "invalid XML received. Further retries may succeed:\n%s" %
  385. (e, xml_string))
  386. return root
  387. def _replace_nodes(self, parsed):
  388. for key, value in parsed.items():
  389. if list(value):
  390. sub_dict = self._build_name_to_xml_node(value)
  391. parsed[key] = self._replace_nodes(sub_dict)
  392. else:
  393. parsed[key] = value.text
  394. return parsed
  395. @_text_content
  396. def _handle_boolean(self, shape, text):
  397. if text == 'true':
  398. return True
  399. else:
  400. return False
  401. @_text_content
  402. def _handle_float(self, shape, text):
  403. return float(text)
  404. @_text_content
  405. def _handle_timestamp(self, shape, text):
  406. return self._timestamp_parser(text)
  407. @_text_content
  408. def _handle_integer(self, shape, text):
  409. return int(text)
  410. @_text_content
  411. def _handle_string(self, shape, text):
  412. return text
  413. @_text_content
  414. def _handle_blob(self, shape, text):
  415. return self._blob_parser(text)
  416. _handle_character = _handle_string
  417. _handle_double = _handle_float
  418. _handle_long = _handle_integer
  419. class QueryParser(BaseXMLResponseParser):
  420. def _do_error_parse(self, response, shape):
  421. xml_contents = response['body']
  422. root = self._parse_xml_string_to_dom(xml_contents)
  423. parsed = self._build_name_to_xml_node(root)
  424. self._replace_nodes(parsed)
  425. # Once we've converted xml->dict, we need to make one or two
  426. # more adjustments to extract nested errors and to be consistent
  427. # with ResponseMetadata for non-error responses:
  428. # 1. {"Errors": {"Error": {...}}} -> {"Error": {...}}
  429. # 2. {"RequestId": "id"} -> {"ResponseMetadata": {"RequestId": "id"}}
  430. if 'Errors' in parsed:
  431. parsed.update(parsed.pop('Errors'))
  432. if 'RequestId' in parsed:
  433. parsed['ResponseMetadata'] = {'RequestId': parsed.pop('RequestId')}
  434. return parsed
  435. def _do_modeled_error_parse(self, response, shape):
  436. return self._parse_body_as_xml(response, shape, inject_metadata=False)
  437. def _do_parse(self, response, shape):
  438. return self._parse_body_as_xml(response, shape, inject_metadata=True)
  439. def _parse_body_as_xml(self, response, shape, inject_metadata=True):
  440. xml_contents = response['body']
  441. root = self._parse_xml_string_to_dom(xml_contents)
  442. parsed = {}
  443. if shape is not None:
  444. start = root
  445. if 'resultWrapper' in shape.serialization:
  446. start = self._find_result_wrapped_shape(
  447. shape.serialization['resultWrapper'],
  448. root)
  449. parsed = self._parse_shape(shape, start)
  450. if inject_metadata:
  451. self._inject_response_metadata(root, parsed)
  452. return parsed
  453. def _find_result_wrapped_shape(self, element_name, xml_root_node):
  454. mapping = self._build_name_to_xml_node(xml_root_node)
  455. return mapping[element_name]
  456. def _inject_response_metadata(self, node, inject_into):
  457. mapping = self._build_name_to_xml_node(node)
  458. child_node = mapping.get('ResponseMetadata')
  459. if child_node is not None:
  460. sub_mapping = self._build_name_to_xml_node(child_node)
  461. for key, value in sub_mapping.items():
  462. sub_mapping[key] = value.text
  463. inject_into['ResponseMetadata'] = sub_mapping
  464. class EC2QueryParser(QueryParser):
  465. def _inject_response_metadata(self, node, inject_into):
  466. mapping = self._build_name_to_xml_node(node)
  467. child_node = mapping.get('requestId')
  468. if child_node is not None:
  469. inject_into['ResponseMetadata'] = {'RequestId': child_node.text}
  470. def _do_error_parse(self, response, shape):
  471. # EC2 errors look like:
  472. # <Response>
  473. # <Errors>
  474. # <Error>
  475. # <Code>InvalidInstanceID.Malformed</Code>
  476. # <Message>Invalid id: "1343124"</Message>
  477. # </Error>
  478. # </Errors>
  479. # <RequestID>12345</RequestID>
  480. # </Response>
  481. # This is different from QueryParser in that it's RequestID,
  482. # not RequestId
  483. original = super(EC2QueryParser, self)._do_error_parse(response, shape)
  484. if 'RequestID' in original:
  485. original['ResponseMetadata'] = {
  486. 'RequestId': original.pop('RequestID')
  487. }
  488. return original
  489. def _get_error_root(self, original_root):
  490. for child in original_root:
  491. if self._node_tag(child) == 'Errors':
  492. for errors_child in child:
  493. if self._node_tag(errors_child) == 'Error':
  494. return errors_child
  495. return original_root
  496. class BaseJSONParser(ResponseParser):
  497. def _handle_structure(self, shape, value):
  498. final_parsed = {}
  499. if shape.is_document_type:
  500. final_parsed = value
  501. else:
  502. member_shapes = shape.members
  503. if value is None:
  504. # If the comes across the wire as "null" (None in python),
  505. # we should be returning this unchanged, instead of as an
  506. # empty dict.
  507. return None
  508. final_parsed = {}
  509. for member_name in member_shapes:
  510. member_shape = member_shapes[member_name]
  511. json_name = member_shape.serialization.get('name', member_name)
  512. raw_value = value.get(json_name)
  513. if raw_value is not None:
  514. final_parsed[member_name] = self._parse_shape(
  515. member_shapes[member_name],
  516. raw_value)
  517. return final_parsed
  518. def _handle_map(self, shape, value):
  519. parsed = {}
  520. key_shape = shape.key
  521. value_shape = shape.value
  522. for key, value in value.items():
  523. actual_key = self._parse_shape(key_shape, key)
  524. actual_value = self._parse_shape(value_shape, value)
  525. parsed[actual_key] = actual_value
  526. return parsed
  527. def _handle_blob(self, shape, value):
  528. return self._blob_parser(value)
  529. def _handle_timestamp(self, shape, value):
  530. return self._timestamp_parser(value)
  531. def _do_error_parse(self, response, shape):
  532. body = self._parse_body_as_json(response['body'])
  533. error = {"Error": {"Message": '', "Code": ''}, "ResponseMetadata": {}}
  534. # Error responses can have slightly different structures for json.
  535. # The basic structure is:
  536. #
  537. # {"__type":"ConnectClientException",
  538. # "message":"The error message."}
  539. # The error message can either come in the 'message' or 'Message' key
  540. # so we need to check for both.
  541. error['Error']['Message'] = body.get('message',
  542. body.get('Message', ''))
  543. # if the message did not contain an error code
  544. # include the response status code
  545. response_code = response.get('status_code')
  546. code = body.get('__type', response_code and str(response_code))
  547. if code is not None:
  548. # code has a couple forms as well:
  549. # * "com.aws.dynamodb.vAPI#ProvisionedThroughputExceededException"
  550. # * "ResourceNotFoundException"
  551. if '#' in code:
  552. code = code.rsplit('#', 1)[1]
  553. error['Error']['Code'] = code
  554. self._inject_response_metadata(error, response['headers'])
  555. return error
  556. def _inject_response_metadata(self, parsed, headers):
  557. if 'x-amzn-requestid' in headers:
  558. parsed.setdefault('ResponseMetadata', {})['RequestId'] = (
  559. headers['x-amzn-requestid'])
  560. def _parse_body_as_json(self, body_contents):
  561. if not body_contents:
  562. return {}
  563. body = body_contents.decode(self.DEFAULT_ENCODING)
  564. try:
  565. original_parsed = json.loads(body)
  566. return original_parsed
  567. except ValueError:
  568. # if the body cannot be parsed, include
  569. # the literal string as the message
  570. return { 'message': body }
  571. class BaseEventStreamParser(ResponseParser):
  572. def _do_parse(self, response, shape):
  573. final_parsed = {}
  574. if shape.serialization.get('eventstream'):
  575. event_type = response['headers'].get(':event-type')
  576. event_shape = shape.members.get(event_type)
  577. if event_shape:
  578. final_parsed[event_type] = self._do_parse(response, event_shape)
  579. else:
  580. self._parse_non_payload_attrs(response, shape,
  581. shape.members, final_parsed)
  582. self._parse_payload(response, shape, shape.members, final_parsed)
  583. return final_parsed
  584. def _do_error_parse(self, response, shape):
  585. exception_type = response['headers'].get(':exception-type')
  586. exception_shape = shape.members.get(exception_type)
  587. if exception_shape is not None:
  588. original_parsed = self._initial_body_parse(response['body'])
  589. body = self._parse_shape(exception_shape, original_parsed)
  590. error = {
  591. 'Error': {
  592. 'Code': exception_type,
  593. 'Message': body.get('Message', body.get('message', ''))
  594. }
  595. }
  596. else:
  597. error = {
  598. 'Error': {
  599. 'Code': response['headers'].get(':error-code', ''),
  600. 'Message': response['headers'].get(':error-message', ''),
  601. }
  602. }
  603. return error
  604. def _parse_payload(self, response, shape, member_shapes, final_parsed):
  605. if shape.serialization.get('event'):
  606. for name in member_shapes:
  607. member_shape = member_shapes[name]
  608. if member_shape.serialization.get('eventpayload'):
  609. body = response['body']
  610. if member_shape.type_name == 'blob':
  611. parsed_body = body
  612. elif member_shape.type_name == 'string':
  613. parsed_body = body.decode(self.DEFAULT_ENCODING)
  614. else:
  615. raw_parse = self._initial_body_parse(body)
  616. parsed_body = self._parse_shape(member_shape, raw_parse)
  617. final_parsed[name] = parsed_body
  618. return
  619. # If we didn't find an explicit payload, use the current shape
  620. original_parsed = self._initial_body_parse(response['body'])
  621. body_parsed = self._parse_shape(shape, original_parsed)
  622. final_parsed.update(body_parsed)
  623. def _parse_non_payload_attrs(self, response, shape,
  624. member_shapes, final_parsed):
  625. headers = response['headers']
  626. for name in member_shapes:
  627. member_shape = member_shapes[name]
  628. if member_shape.serialization.get('eventheader'):
  629. if name in headers:
  630. value = headers[name]
  631. if member_shape.type_name == 'timestamp':
  632. # Event stream timestamps are an in milleseconds so we
  633. # divide by 1000 to convert to seconds.
  634. value = self._timestamp_parser(value / 1000.0)
  635. final_parsed[name] = value
  636. def _initial_body_parse(self, body_contents):
  637. # This method should do the initial xml/json parsing of the
  638. # body. We we still need to walk the parsed body in order
  639. # to convert types, but this method will do the first round
  640. # of parsing.
  641. raise NotImplementedError("_initial_body_parse")
  642. class EventStreamJSONParser(BaseEventStreamParser, BaseJSONParser):
  643. def _initial_body_parse(self, body_contents):
  644. return self._parse_body_as_json(body_contents)
  645. class EventStreamXMLParser(BaseEventStreamParser, BaseXMLResponseParser):
  646. def _initial_body_parse(self, xml_string):
  647. if not xml_string:
  648. return ETree.Element('')
  649. return self._parse_xml_string_to_dom(xml_string)
  650. class JSONParser(BaseJSONParser):
  651. EVENT_STREAM_PARSER_CLS = EventStreamJSONParser
  652. """Response parser for the "json" protocol."""
  653. def _do_parse(self, response, shape):
  654. parsed = {}
  655. if shape is not None:
  656. event_name = shape.event_stream_name
  657. if event_name:
  658. parsed = self._handle_event_stream(response, shape, event_name)
  659. else:
  660. parsed = self._handle_json_body(response['body'], shape)
  661. self._inject_response_metadata(parsed, response['headers'])
  662. return parsed
  663. def _do_modeled_error_parse(self, response, shape):
  664. return self._handle_json_body(response['body'], shape)
  665. def _handle_event_stream(self, response, shape, event_name):
  666. event_stream_shape = shape.members[event_name]
  667. event_stream = self._create_event_stream(response, event_stream_shape)
  668. try:
  669. event = event_stream.get_initial_response()
  670. except NoInitialResponseError:
  671. error_msg = 'First event was not of type initial-response'
  672. raise ResponseParserError(error_msg)
  673. parsed = self._handle_json_body(event.payload, shape)
  674. parsed[event_name] = event_stream
  675. return parsed
  676. def _handle_json_body(self, raw_body, shape):
  677. # The json.loads() gives us the primitive JSON types,
  678. # but we need to traverse the parsed JSON data to convert
  679. # to richer types (blobs, timestamps, etc.
  680. parsed_json = self._parse_body_as_json(raw_body)
  681. return self._parse_shape(shape, parsed_json)
  682. class BaseRestParser(ResponseParser):
  683. def _do_parse(self, response, shape):
  684. final_parsed = {}
  685. final_parsed['ResponseMetadata'] = self._populate_response_metadata(
  686. response)
  687. self._add_modeled_parse(response, shape, final_parsed)
  688. return final_parsed
  689. def _add_modeled_parse(self, response, shape, final_parsed):
  690. if shape is None:
  691. return final_parsed
  692. member_shapes = shape.members
  693. self._parse_non_payload_attrs(response, shape,
  694. member_shapes, final_parsed)
  695. self._parse_payload(response, shape, member_shapes, final_parsed)
  696. def _do_modeled_error_parse(self, response, shape):
  697. final_parsed = {}
  698. self._add_modeled_parse(response, shape, final_parsed)
  699. return final_parsed
  700. def _populate_response_metadata(self, response):
  701. metadata = {}
  702. headers = response['headers']
  703. if 'x-amzn-requestid' in headers:
  704. metadata['RequestId'] = headers['x-amzn-requestid']
  705. elif 'x-amz-request-id' in headers:
  706. metadata['RequestId'] = headers['x-amz-request-id']
  707. # HostId is what it's called whenever this value is returned
  708. # in an XML response body, so to be consistent, we'll always
  709. # call is HostId.
  710. metadata['HostId'] = headers.get('x-amz-id-2', '')
  711. return metadata
  712. def _parse_payload(self, response, shape, member_shapes, final_parsed):
  713. if 'payload' in shape.serialization:
  714. # If a payload is specified in the output shape, then only that
  715. # shape is used for the body payload.
  716. payload_member_name = shape.serialization['payload']
  717. body_shape = member_shapes[payload_member_name]
  718. if body_shape.serialization.get('eventstream'):
  719. body = self._create_event_stream(response, body_shape)
  720. final_parsed[payload_member_name] = body
  721. elif body_shape.type_name in ['string', 'blob']:
  722. # This is a stream
  723. body = response['body']
  724. if isinstance(body, bytes):
  725. body = body.decode(self.DEFAULT_ENCODING)
  726. final_parsed[payload_member_name] = body
  727. else:
  728. original_parsed = self._initial_body_parse(response['body'])
  729. final_parsed[payload_member_name] = self._parse_shape(
  730. body_shape, original_parsed)
  731. else:
  732. original_parsed = self._initial_body_parse(response['body'])
  733. body_parsed = self._parse_shape(shape, original_parsed)
  734. final_parsed.update(body_parsed)
  735. def _parse_non_payload_attrs(self, response, shape,
  736. member_shapes, final_parsed):
  737. headers = response['headers']
  738. for name in member_shapes:
  739. member_shape = member_shapes[name]
  740. location = member_shape.serialization.get('location')
  741. if location is None:
  742. continue
  743. elif location == 'statusCode':
  744. final_parsed[name] = self._parse_shape(
  745. member_shape, response['status_code'])
  746. elif location == 'headers':
  747. final_parsed[name] = self._parse_header_map(member_shape,
  748. headers)
  749. elif location == 'header':
  750. header_name = member_shape.serialization.get('name', name)
  751. if header_name in headers:
  752. final_parsed[name] = self._parse_shape(
  753. member_shape, headers[header_name])
  754. def _parse_header_map(self, shape, headers):
  755. # Note that headers are case insensitive, so we .lower()
  756. # all header names and header prefixes.
  757. parsed = {}
  758. prefix = shape.serialization.get('name', '').lower()
  759. for header_name in headers:
  760. if header_name.lower().startswith(prefix):
  761. # The key name inserted into the parsed hash
  762. # strips off the prefix.
  763. name = header_name[len(prefix):]
  764. parsed[name] = headers[header_name]
  765. return parsed
  766. def _initial_body_parse(self, body_contents):
  767. # This method should do the initial xml/json parsing of the
  768. # body. We we still need to walk the parsed body in order
  769. # to convert types, but this method will do the first round
  770. # of parsing.
  771. raise NotImplementedError("_initial_body_parse")
  772. def _handle_string(self, shape, value):
  773. parsed = value
  774. if is_json_value_header(shape):
  775. decoded = base64.b64decode(value).decode(self.DEFAULT_ENCODING)
  776. parsed = json.loads(decoded)
  777. return parsed
  778. class RestJSONParser(BaseRestParser, BaseJSONParser):
  779. EVENT_STREAM_PARSER_CLS = EventStreamJSONParser
  780. def _initial_body_parse(self, body_contents):
  781. return self._parse_body_as_json(body_contents)
  782. def _do_error_parse(self, response, shape):
  783. error = super(RestJSONParser, self)._do_error_parse(response, shape)
  784. self._inject_error_code(error, response)
  785. return error
  786. def _inject_error_code(self, error, response):
  787. # The "Code" value can come from either a response
  788. # header or a value in the JSON body.
  789. body = self._initial_body_parse(response['body'])
  790. if 'x-amzn-errortype' in response['headers']:
  791. code = response['headers']['x-amzn-errortype']
  792. # Could be:
  793. # x-amzn-errortype: ValidationException:
  794. code = code.split(':')[0]
  795. error['Error']['Code'] = code
  796. elif 'code' in body or 'Code' in body:
  797. error['Error']['Code'] = body.get(
  798. 'code', body.get('Code', ''))
  799. class RestXMLParser(BaseRestParser, BaseXMLResponseParser):
  800. EVENT_STREAM_PARSER_CLS = EventStreamXMLParser
  801. def _initial_body_parse(self, xml_string):
  802. if not xml_string:
  803. return ETree.Element('')
  804. return self._parse_xml_string_to_dom(xml_string)
  805. def _do_error_parse(self, response, shape):
  806. # We're trying to be service agnostic here, but S3 does have a slightly
  807. # different response structure for its errors compared to other
  808. # rest-xml serivces (route53/cloudfront). We handle this by just
  809. # trying to parse both forms.
  810. # First:
  811. # <ErrorResponse xmlns="...">
  812. # <Error>
  813. # <Type>Sender</Type>
  814. # <Code>InvalidInput</Code>
  815. # <Message>Invalid resource type: foo</Message>
  816. # </Error>
  817. # <RequestId>request-id</RequestId>
  818. # </ErrorResponse>
  819. if response['body']:
  820. # If the body ends up being invalid xml, the xml parser should not
  821. # blow up. It should at least try to pull information about the
  822. # the error response from other sources like the HTTP status code.
  823. try:
  824. return self._parse_error_from_body(response)
  825. except ResponseParserError as e:
  826. LOG.debug(
  827. 'Exception caught when parsing error response body:',
  828. exc_info=True)
  829. return self._parse_error_from_http_status(response)
  830. def _parse_error_from_http_status(self, response):
  831. return {
  832. 'Error': {
  833. 'Code': str(response['status_code']),
  834. 'Message': six.moves.http_client.responses.get(
  835. response['status_code'], ''),
  836. },
  837. 'ResponseMetadata': {
  838. 'RequestId': response['headers'].get('x-amz-request-id', ''),
  839. 'HostId': response['headers'].get('x-amz-id-2', ''),
  840. }
  841. }
  842. def _parse_error_from_body(self, response):
  843. xml_contents = response['body']
  844. root = self._parse_xml_string_to_dom(xml_contents)
  845. parsed = self._build_name_to_xml_node(root)
  846. self._replace_nodes(parsed)
  847. if root.tag == 'Error':
  848. # This is an S3 error response. First we'll populate the
  849. # response metadata.
  850. metadata = self._populate_response_metadata(response)
  851. # The RequestId and the HostId are already in the
  852. # ResponseMetadata, but are also duplicated in the XML
  853. # body. We don't need these values in both places,
  854. # we'll just remove them from the parsed XML body.
  855. parsed.pop('RequestId', '')
  856. parsed.pop('HostId', '')
  857. return {'Error': parsed, 'ResponseMetadata': metadata}
  858. elif 'RequestId' in parsed:
  859. # Other rest-xml serivces:
  860. parsed['ResponseMetadata'] = {'RequestId': parsed.pop('RequestId')}
  861. default = {'Error': {'Message': '', 'Code': ''}}
  862. merge_dicts(default, parsed)
  863. return default
  864. @_text_content
  865. def _handle_string(self, shape, text):
  866. text = super(RestXMLParser, self)._handle_string(shape, text)
  867. return text
  868. PROTOCOL_PARSERS = {
  869. 'ec2': EC2QueryParser,
  870. 'query': QueryParser,
  871. 'json': JSONParser,
  872. 'rest-json': RestJSONParser,
  873. 'rest-xml': RestXMLParser,
  874. }