handlers.py 45 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136
  1. # Copyright 2012-2014 Amazon.com, Inc. or its affiliates. All Rights Reserved.
  2. #
  3. # Licensed under the Apache License, Version 2.0 (the "License"). You
  4. # may not use this file except in compliance with the License. A copy of
  5. # the License is located at
  6. #
  7. # http://aws.amazon.com/apache2.0/
  8. #
  9. # or in the "license" file accompanying this file. This file is
  10. # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
  11. # ANY KIND, either express or implied. See the License for the specific
  12. # language governing permissions and limitations under the License.
  13. """Builtin event handlers.
  14. This module contains builtin handlers for events emitted by botocore.
  15. """
  16. import base64
  17. import logging
  18. import copy
  19. import re
  20. import warnings
  21. import uuid
  22. from botocore.compat import (
  23. unquote, json, six, unquote_str, ensure_bytes, get_md5,
  24. MD5_AVAILABLE, OrderedDict, urlsplit, urlunsplit, XMLParseError,
  25. ETree,
  26. )
  27. from botocore.docs.utils import AutoPopulatedParam
  28. from botocore.docs.utils import HideParamFromOperations
  29. from botocore.docs.utils import AppendParamDocumentation
  30. from botocore.signers import add_generate_presigned_url
  31. from botocore.signers import add_generate_presigned_post
  32. from botocore.signers import add_generate_db_auth_token
  33. from botocore.exceptions import ParamValidationError
  34. from botocore.exceptions import AliasConflictParameterError
  35. from botocore.exceptions import UnsupportedTLSVersionWarning
  36. from botocore.exceptions import MissingServiceIdError
  37. from botocore.utils import percent_encode, SAFE_CHARS
  38. from botocore.utils import switch_host_with_param
  39. from botocore.utils import hyphenize_service_id
  40. from botocore.utils import conditionally_calculate_md5
  41. from botocore import retryhandler
  42. from botocore import utils
  43. from botocore import translate
  44. import botocore
  45. import botocore.auth
  46. logger = logging.getLogger(__name__)
  47. REGISTER_FIRST = object()
  48. REGISTER_LAST = object()
  49. # From the S3 docs:
  50. # The rules for bucket names in the US Standard region allow bucket names
  51. # to be as long as 255 characters, and bucket names can contain any
  52. # combination of uppercase letters, lowercase letters, numbers, periods
  53. # (.), hyphens (-), and underscores (_).
  54. VALID_BUCKET = re.compile(r'^[a-zA-Z0-9.\-_]{1,255}$')
  55. _ACCESSPOINT_ARN = (
  56. r'^arn:(aws).*:(s3|s3-object-lambda):[a-z\-0-9]+:[0-9]{12}:accesspoint[/:]'
  57. r'[a-zA-Z0-9\-]{1,63}$'
  58. )
  59. _OUTPOST_ARN = (
  60. r'^arn:(aws).*:s3-outposts:[a-z\-0-9]+:[0-9]{12}:outpost[/:]'
  61. r'[a-zA-Z0-9\-]{1,63}[/:]accesspoint[/:][a-zA-Z0-9\-]{1,63}$'
  62. )
  63. VALID_S3_ARN = re.compile('|'.join([_ACCESSPOINT_ARN, _OUTPOST_ARN]))
  64. VERSION_ID_SUFFIX = re.compile(r'\?versionId=[^\s]+$')
  65. SERVICE_NAME_ALIASES = {
  66. 'runtime.sagemaker': 'sagemaker-runtime'
  67. }
  68. def handle_service_name_alias(service_name, **kwargs):
  69. return SERVICE_NAME_ALIASES.get(service_name, service_name)
  70. def escape_xml_payload(params, **kwargs):
  71. # Replace \r and \n with the escaped sequence over the whole XML document
  72. # to avoid linebreak normalization modifying customer input when the
  73. # document is parsed. Ideally, we would do this in ElementTree.tostring,
  74. # but it doesn't allow us to override entity escaping for text fields. For
  75. # this operation \r and \n can only appear in the XML document if they were
  76. # passed as part of the customer input.
  77. body = params['body']
  78. replaced = False
  79. if b'\r' in body:
  80. replaced = True
  81. body = body.replace(b'\r', b'
')
  82. if b'\n' in body:
  83. replaced = True
  84. body = body.replace(b'\n', b'
')
  85. if not replaced:
  86. return
  87. params['body'] = body
  88. if 'Content-MD5' in params['headers']:
  89. # The Content-MD5 is now wrong, so we'll need to recalculate it
  90. del params['headers']['Content-MD5']
  91. conditionally_calculate_md5(params, **kwargs)
  92. def check_for_200_error(response, **kwargs):
  93. # From: http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectCOPY.html
  94. # There are two opportunities for a copy request to return an error. One
  95. # can occur when Amazon S3 receives the copy request and the other can
  96. # occur while Amazon S3 is copying the files. If the error occurs before
  97. # the copy operation starts, you receive a standard Amazon S3 error. If the
  98. # error occurs during the copy operation, the error response is embedded in
  99. # the 200 OK response. This means that a 200 OK response can contain either
  100. # a success or an error. Make sure to design your application to parse the
  101. # contents of the response and handle it appropriately.
  102. #
  103. # So this handler checks for this case. Even though the server sends a
  104. # 200 response, conceptually this should be handled exactly like a
  105. # 500 response (with respect to raising exceptions, retries, etc.)
  106. # We're connected *before* all the other retry logic handlers, so as long
  107. # as we switch the error code to 500, we'll retry the error as expected.
  108. if response is None:
  109. # A None response can happen if an exception is raised while
  110. # trying to retrieve the response. See Endpoint._get_response().
  111. return
  112. http_response, parsed = response
  113. if _looks_like_special_case_error(http_response):
  114. logger.debug("Error found for response with 200 status code, "
  115. "errors: %s, changing status code to "
  116. "500.", parsed)
  117. http_response.status_code = 500
  118. def _looks_like_special_case_error(http_response):
  119. if http_response.status_code == 200:
  120. try:
  121. parser = ETree.XMLParser(
  122. target=ETree.TreeBuilder(),
  123. encoding='utf-8')
  124. parser.feed(http_response.content)
  125. root = parser.close()
  126. except XMLParseError:
  127. # In cases of network disruptions, we may end up with a partial
  128. # streamed response from S3. We need to treat these cases as
  129. # 500 Service Errors and try again.
  130. return True
  131. if root.tag == 'Error':
  132. return True
  133. return False
  134. def set_operation_specific_signer(context, signing_name, **kwargs):
  135. """ Choose the operation-specific signer.
  136. Individual operations may have a different auth type than the service as a
  137. whole. This will most often manifest as operations that should not be
  138. authenticated at all, but can include other auth modes such as sigv4
  139. without body signing.
  140. """
  141. auth_type = context.get('auth_type')
  142. # Auth type will be None if the operation doesn't have a configured auth
  143. # type.
  144. if not auth_type:
  145. return
  146. # Auth type will be the string value 'none' if the operation should not
  147. # be signed at all.
  148. if auth_type == 'none':
  149. return botocore.UNSIGNED
  150. if auth_type.startswith('v4'):
  151. signature_version = 'v4'
  152. if signing_name == 's3':
  153. signature_version = 's3v4'
  154. # If the operation needs an unsigned body, we set additional context
  155. # allowing the signer to be aware of this.
  156. if auth_type == 'v4-unsigned-body':
  157. context['payload_signing_enabled'] = False
  158. return signature_version
  159. def decode_console_output(parsed, **kwargs):
  160. if 'Output' in parsed:
  161. try:
  162. # We're using 'replace' for errors because it is
  163. # possible that console output contains non string
  164. # chars we can't utf-8 decode.
  165. value = base64.b64decode(six.b(parsed['Output'])).decode(
  166. 'utf-8', 'replace')
  167. parsed['Output'] = value
  168. except (ValueError, TypeError, AttributeError):
  169. logger.debug('Error decoding base64', exc_info=True)
  170. def generate_idempotent_uuid(params, model, **kwargs):
  171. for name in model.idempotent_members:
  172. if name not in params:
  173. params[name] = str(uuid.uuid4())
  174. logger.debug("injecting idempotency token (%s) into param '%s'." %
  175. (params[name], name))
  176. def decode_quoted_jsondoc(value):
  177. try:
  178. value = json.loads(unquote(value))
  179. except (ValueError, TypeError):
  180. logger.debug('Error loading quoted JSON', exc_info=True)
  181. return value
  182. def json_decode_template_body(parsed, **kwargs):
  183. if 'TemplateBody' in parsed:
  184. try:
  185. value = json.loads(
  186. parsed['TemplateBody'], object_pairs_hook=OrderedDict)
  187. parsed['TemplateBody'] = value
  188. except (ValueError, TypeError):
  189. logger.debug('error loading JSON', exc_info=True)
  190. def validate_bucket_name(params, **kwargs):
  191. if 'Bucket' not in params:
  192. return
  193. bucket = params['Bucket']
  194. if not VALID_BUCKET.search(bucket) and not VALID_S3_ARN.search(bucket):
  195. error_msg = (
  196. 'Invalid bucket name "%s": Bucket name must match '
  197. 'the regex "%s" or be an ARN matching the regex "%s"' % (
  198. bucket, VALID_BUCKET.pattern, VALID_S3_ARN.pattern))
  199. raise ParamValidationError(report=error_msg)
  200. def sse_md5(params, **kwargs):
  201. """
  202. S3 server-side encryption requires the encryption key to be sent to the
  203. server base64 encoded, as well as a base64-encoded MD5 hash of the
  204. encryption key. This handler does both if the MD5 has not been set by
  205. the caller.
  206. """
  207. _sse_md5(params, 'SSECustomer')
  208. def copy_source_sse_md5(params, **kwargs):
  209. """
  210. S3 server-side encryption requires the encryption key to be sent to the
  211. server base64 encoded, as well as a base64-encoded MD5 hash of the
  212. encryption key. This handler does both if the MD5 has not been set by
  213. the caller specifically if the parameter is for the copy-source sse-c key.
  214. """
  215. _sse_md5(params, 'CopySourceSSECustomer')
  216. def _sse_md5(params, sse_member_prefix='SSECustomer'):
  217. if not _needs_s3_sse_customization(params, sse_member_prefix):
  218. return
  219. sse_key_member = sse_member_prefix + 'Key'
  220. sse_md5_member = sse_member_prefix + 'KeyMD5'
  221. key_as_bytes = params[sse_key_member]
  222. if isinstance(key_as_bytes, six.text_type):
  223. key_as_bytes = key_as_bytes.encode('utf-8')
  224. key_md5_str = base64.b64encode(
  225. get_md5(key_as_bytes).digest()).decode('utf-8')
  226. key_b64_encoded = base64.b64encode(key_as_bytes).decode('utf-8')
  227. params[sse_key_member] = key_b64_encoded
  228. params[sse_md5_member] = key_md5_str
  229. def _needs_s3_sse_customization(params, sse_member_prefix):
  230. return (params.get(sse_member_prefix + 'Key') is not None and
  231. sse_member_prefix + 'KeyMD5' not in params)
  232. def disable_signing(**kwargs):
  233. """
  234. This handler disables request signing by setting the signer
  235. name to a special sentinel value.
  236. """
  237. return botocore.UNSIGNED
  238. def add_expect_header(model, params, **kwargs):
  239. if model.http.get('method', '') not in ['PUT', 'POST']:
  240. return
  241. if 'body' in params:
  242. body = params['body']
  243. if hasattr(body, 'read'):
  244. # Any file like object will use an expect 100-continue
  245. # header regardless of size.
  246. logger.debug("Adding expect 100 continue header to request.")
  247. params['headers']['Expect'] = '100-continue'
  248. class DeprecatedServiceDocumenter(object):
  249. def __init__(self, replacement_service_name):
  250. self._replacement_service_name = replacement_service_name
  251. def inject_deprecation_notice(self, section, event_name, **kwargs):
  252. section.style.start_important()
  253. section.write('This service client is deprecated. Please use ')
  254. section.style.ref(
  255. self._replacement_service_name,
  256. self._replacement_service_name,
  257. )
  258. section.write(' instead.')
  259. section.style.end_important()
  260. def document_copy_source_form(section, event_name, **kwargs):
  261. if 'request-example' in event_name:
  262. parent = section.get_section('structure-value')
  263. param_line = parent.get_section('CopySource')
  264. value_portion = param_line.get_section('member-value')
  265. value_portion.clear_text()
  266. value_portion.write("'string' or {'Bucket': 'string', "
  267. "'Key': 'string', 'VersionId': 'string'}")
  268. elif 'request-params' in event_name:
  269. param_section = section.get_section('CopySource')
  270. type_section = param_section.get_section('param-type')
  271. type_section.clear_text()
  272. type_section.write(':type CopySource: str or dict')
  273. doc_section = param_section.get_section('param-documentation')
  274. doc_section.clear_text()
  275. doc_section.write(
  276. "The name of the source bucket, key name of the source object, "
  277. "and optional version ID of the source object. You can either "
  278. "provide this value as a string or a dictionary. The "
  279. "string form is {bucket}/{key} or "
  280. "{bucket}/{key}?versionId={versionId} if you want to copy a "
  281. "specific version. You can also provide this value as a "
  282. "dictionary. The dictionary format is recommended over "
  283. "the string format because it is more explicit. The dictionary "
  284. "format is: {'Bucket': 'bucket', 'Key': 'key', 'VersionId': 'id'}."
  285. " Note that the VersionId key is optional and may be omitted."
  286. " To specify an S3 access point, provide the access point"
  287. " ARN for the ``Bucket`` key in the copy source dictionary. If you"
  288. " want to provide the copy source for an S3 access point as a"
  289. " string instead of a dictionary, the ARN provided must be the"
  290. " full S3 access point object ARN"
  291. " (i.e. {accesspoint_arn}/object/{key})"
  292. )
  293. def handle_copy_source_param(params, **kwargs):
  294. """Convert CopySource param for CopyObject/UploadPartCopy.
  295. This handler will deal with two cases:
  296. * CopySource provided as a string. We'll make a best effort
  297. to URL encode the key name as required. This will require
  298. parsing the bucket and version id from the CopySource value
  299. and only encoding the key.
  300. * CopySource provided as a dict. In this case we're
  301. explicitly given the Bucket, Key, and VersionId so we're
  302. able to encode the key and ensure this value is serialized
  303. and correctly sent to S3.
  304. """
  305. source = params.get('CopySource')
  306. if source is None:
  307. # The call will eventually fail but we'll let the
  308. # param validator take care of this. It will
  309. # give a better error message.
  310. return
  311. if isinstance(source, six.string_types):
  312. params['CopySource'] = _quote_source_header(source)
  313. elif isinstance(source, dict):
  314. params['CopySource'] = _quote_source_header_from_dict(source)
  315. def _quote_source_header_from_dict(source_dict):
  316. try:
  317. bucket = source_dict['Bucket']
  318. key = source_dict['Key']
  319. version_id = source_dict.get('VersionId')
  320. if VALID_S3_ARN.search(bucket):
  321. final = '%s/object/%s' % (bucket, key)
  322. else:
  323. final = '%s/%s' % (bucket, key)
  324. except KeyError as e:
  325. raise ParamValidationError(
  326. report='Missing required parameter: %s' % str(e))
  327. final = percent_encode(final, safe=SAFE_CHARS + '/')
  328. if version_id is not None:
  329. final += '?versionId=%s' % version_id
  330. return final
  331. def _quote_source_header(value):
  332. result = VERSION_ID_SUFFIX.search(value)
  333. if result is None:
  334. return percent_encode(value, safe=SAFE_CHARS + '/')
  335. else:
  336. first, version_id = value[:result.start()], value[result.start():]
  337. return percent_encode(first, safe=SAFE_CHARS + '/') + version_id
  338. def _get_cross_region_presigned_url(request_signer, request_dict, model,
  339. source_region, destination_region):
  340. # The better way to do this is to actually get the
  341. # endpoint_resolver and get the endpoint_url given the
  342. # source region. In this specific case, we know that
  343. # we can safely replace the dest region with the source
  344. # region because of the supported EC2 regions, but in
  345. # general this is not a safe assumption to make.
  346. # I think eventually we should try to plumb through something
  347. # that allows us to resolve endpoints from regions.
  348. request_dict_copy = copy.deepcopy(request_dict)
  349. request_dict_copy['body']['DestinationRegion'] = destination_region
  350. request_dict_copy['url'] = request_dict['url'].replace(
  351. destination_region, source_region)
  352. request_dict_copy['method'] = 'GET'
  353. request_dict_copy['headers'] = {}
  354. return request_signer.generate_presigned_url(
  355. request_dict_copy, region_name=source_region,
  356. operation_name=model.name)
  357. def _get_presigned_url_source_and_destination_regions(request_signer, params):
  358. # Gets the source and destination regions to be used
  359. destination_region = request_signer._region_name
  360. source_region = params.get('SourceRegion')
  361. return source_region, destination_region
  362. def inject_presigned_url_ec2(params, request_signer, model, **kwargs):
  363. # The customer can still provide this, so we should pass if they do.
  364. if 'PresignedUrl' in params['body']:
  365. return
  366. src, dest = _get_presigned_url_source_and_destination_regions(
  367. request_signer, params['body'])
  368. url = _get_cross_region_presigned_url(
  369. request_signer, params, model, src, dest)
  370. params['body']['PresignedUrl'] = url
  371. # EC2 Requires that the destination region be sent over the wire in
  372. # addition to the source region.
  373. params['body']['DestinationRegion'] = dest
  374. def inject_presigned_url_rds(params, request_signer, model, **kwargs):
  375. # SourceRegion is not required for RDS operations, so it's possible that
  376. # it isn't set. In that case it's probably a local copy so we don't need
  377. # to do anything else.
  378. if 'SourceRegion' not in params['body']:
  379. return
  380. src, dest = _get_presigned_url_source_and_destination_regions(
  381. request_signer, params['body'])
  382. # Since SourceRegion isn't actually modeled for RDS, it needs to be
  383. # removed from the request params before we send the actual request.
  384. del params['body']['SourceRegion']
  385. if 'PreSignedUrl' in params['body']:
  386. return
  387. url = _get_cross_region_presigned_url(
  388. request_signer, params, model, src, dest)
  389. params['body']['PreSignedUrl'] = url
  390. def json_decode_policies(parsed, model, **kwargs):
  391. # Any time an IAM operation returns a policy document
  392. # it is a string that is json that has been urlencoded,
  393. # i.e urlencode(json.dumps(policy_document)).
  394. # To give users something more useful, we will urldecode
  395. # this value and json.loads() the result so that they have
  396. # the policy document as a dictionary.
  397. output_shape = model.output_shape
  398. if output_shape is not None:
  399. _decode_policy_types(parsed, model.output_shape)
  400. def _decode_policy_types(parsed, shape):
  401. # IAM consistently uses the policyDocumentType shape to indicate
  402. # strings that have policy documents.
  403. shape_name = 'policyDocumentType'
  404. if shape.type_name == 'structure':
  405. for member_name, member_shape in shape.members.items():
  406. if member_shape.type_name == 'string' and \
  407. member_shape.name == shape_name and \
  408. member_name in parsed:
  409. parsed[member_name] = decode_quoted_jsondoc(
  410. parsed[member_name])
  411. elif member_name in parsed:
  412. _decode_policy_types(parsed[member_name], member_shape)
  413. if shape.type_name == 'list':
  414. shape_member = shape.member
  415. for item in parsed:
  416. _decode_policy_types(item, shape_member)
  417. def parse_get_bucket_location(parsed, http_response, **kwargs):
  418. # s3.GetBucketLocation cannot be modeled properly. To
  419. # account for this we just manually parse the XML document.
  420. # The "parsed" passed in only has the ResponseMetadata
  421. # filled out. This handler will fill in the LocationConstraint
  422. # value.
  423. if http_response.raw is None:
  424. return
  425. response_body = http_response.content
  426. parser = ETree.XMLParser(
  427. target=ETree.TreeBuilder(),
  428. encoding='utf-8')
  429. parser.feed(response_body)
  430. root = parser.close()
  431. region = root.text
  432. parsed['LocationConstraint'] = region
  433. def base64_encode_user_data(params, **kwargs):
  434. if 'UserData' in params:
  435. if isinstance(params['UserData'], six.text_type):
  436. # Encode it to bytes if it is text.
  437. params['UserData'] = params['UserData'].encode('utf-8')
  438. params['UserData'] = base64.b64encode(
  439. params['UserData']).decode('utf-8')
  440. def document_base64_encoding(param):
  441. description = ('**This value will be base64 encoded automatically. Do '
  442. 'not base64 encode this value prior to performing the '
  443. 'operation.**')
  444. append = AppendParamDocumentation(param, description)
  445. return append.append_documentation
  446. def validate_ascii_metadata(params, **kwargs):
  447. """Verify S3 Metadata only contains ascii characters.
  448. From: http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html
  449. "Amazon S3 stores user-defined metadata in lowercase. Each name, value pair
  450. must conform to US-ASCII when using REST and UTF-8 when using SOAP or
  451. browser-based uploads via POST."
  452. """
  453. metadata = params.get('Metadata')
  454. if not metadata or not isinstance(metadata, dict):
  455. # We have to at least type check the metadata as a dict type
  456. # because this handler is called before param validation.
  457. # We'll go ahead and return because the param validator will
  458. # give a descriptive error message for us.
  459. # We might need a post-param validation event.
  460. return
  461. for key, value in metadata.items():
  462. try:
  463. key.encode('ascii')
  464. value.encode('ascii')
  465. except UnicodeEncodeError as e:
  466. error_msg = (
  467. 'Non ascii characters found in S3 metadata '
  468. 'for key "%s", value: "%s". \nS3 metadata can only '
  469. 'contain ASCII characters. ' % (key, value)
  470. )
  471. raise ParamValidationError(
  472. report=error_msg)
  473. def fix_route53_ids(params, model, **kwargs):
  474. """
  475. Check for and split apart Route53 resource IDs, setting
  476. only the last piece. This allows the output of one operation
  477. (e.g. ``'foo/1234'``) to be used as input in another
  478. operation (e.g. it expects just ``'1234'``).
  479. """
  480. input_shape = model.input_shape
  481. if not input_shape or not hasattr(input_shape, 'members'):
  482. return
  483. members = [name for (name, shape) in input_shape.members.items()
  484. if shape.name in ['ResourceId', 'DelegationSetId']]
  485. for name in members:
  486. if name in params:
  487. orig_value = params[name]
  488. params[name] = orig_value.split('/')[-1]
  489. logger.debug('%s %s -> %s', name, orig_value, params[name])
  490. def inject_account_id(params, **kwargs):
  491. if params.get('accountId') is None:
  492. # Glacier requires accountId, but allows you
  493. # to specify '-' for the current owners account.
  494. # We add this default value if the user does not
  495. # provide the accountId as a convenience.
  496. params['accountId'] = '-'
  497. def add_glacier_version(model, params, **kwargs):
  498. request_dict = params
  499. request_dict['headers']['x-amz-glacier-version'] = model.metadata[
  500. 'apiVersion']
  501. def add_accept_header(model, params, **kwargs):
  502. if params['headers'].get('Accept', None) is None:
  503. request_dict = params
  504. request_dict['headers']['Accept'] = 'application/json'
  505. def add_glacier_checksums(params, **kwargs):
  506. """Add glacier checksums to the http request.
  507. This will add two headers to the http request:
  508. * x-amz-content-sha256
  509. * x-amz-sha256-tree-hash
  510. These values will only be added if they are not present
  511. in the HTTP request.
  512. """
  513. request_dict = params
  514. headers = request_dict['headers']
  515. body = request_dict['body']
  516. if isinstance(body, six.binary_type):
  517. # If the user provided a bytes type instead of a file
  518. # like object, we're temporarily create a BytesIO object
  519. # so we can use the util functions to calculate the
  520. # checksums which assume file like objects. Note that
  521. # we're not actually changing the body in the request_dict.
  522. body = six.BytesIO(body)
  523. starting_position = body.tell()
  524. if 'x-amz-content-sha256' not in headers:
  525. headers['x-amz-content-sha256'] = utils.calculate_sha256(
  526. body, as_hex=True)
  527. body.seek(starting_position)
  528. if 'x-amz-sha256-tree-hash' not in headers:
  529. headers['x-amz-sha256-tree-hash'] = utils.calculate_tree_hash(body)
  530. body.seek(starting_position)
  531. def document_glacier_tree_hash_checksum():
  532. doc = '''
  533. This is a required field.
  534. Ideally you will want to compute this value with checksums from
  535. previous uploaded parts, using the algorithm described in
  536. `Glacier documentation <http://docs.aws.amazon.com/amazonglacier/latest/dev/checksum-calculations.html>`_.
  537. But if you prefer, you can also use botocore.utils.calculate_tree_hash()
  538. to compute it from raw file by::
  539. checksum = calculate_tree_hash(open('your_file.txt', 'rb'))
  540. '''
  541. return AppendParamDocumentation('checksum', doc).append_documentation
  542. def document_cloudformation_get_template_return_type(section, event_name, **kwargs):
  543. if 'response-params' in event_name:
  544. template_body_section = section.get_section('TemplateBody')
  545. type_section = template_body_section.get_section('param-type')
  546. type_section.clear_text()
  547. type_section.write('(*dict*) --')
  548. elif 'response-example' in event_name:
  549. parent = section.get_section('structure-value')
  550. param_line = parent.get_section('TemplateBody')
  551. value_portion = param_line.get_section('member-value')
  552. value_portion.clear_text()
  553. value_portion.write('{}')
  554. def switch_host_machinelearning(request, **kwargs):
  555. switch_host_with_param(request, 'PredictEndpoint')
  556. def check_openssl_supports_tls_version_1_2(**kwargs):
  557. import ssl
  558. try:
  559. openssl_version_tuple = ssl.OPENSSL_VERSION_INFO
  560. if openssl_version_tuple < (1, 0, 1):
  561. warnings.warn(
  562. 'Currently installed openssl version: %s does not '
  563. 'support TLS 1.2, which is required for use of iot-data. '
  564. 'Please use python installed with openssl version 1.0.1 or '
  565. 'higher.' % (ssl.OPENSSL_VERSION),
  566. UnsupportedTLSVersionWarning
  567. )
  568. # We cannot check the openssl version on python2.6, so we should just
  569. # pass on this conveniency check.
  570. except AttributeError:
  571. pass
  572. def change_get_to_post(request, **kwargs):
  573. # This is useful when we need to change a potentially large GET request
  574. # into a POST with x-www-form-urlencoded encoding.
  575. if request.method == 'GET' and '?' in request.url:
  576. request.headers['Content-Type'] = 'application/x-www-form-urlencoded'
  577. request.method = 'POST'
  578. request.url, request.data = request.url.split('?', 1)
  579. def set_list_objects_encoding_type_url(params, context, **kwargs):
  580. if 'EncodingType' not in params:
  581. # We set this context so that we know it wasn't the customer that
  582. # requested the encoding.
  583. context['encoding_type_auto_set'] = True
  584. params['EncodingType'] = 'url'
  585. def decode_list_object(parsed, context, **kwargs):
  586. # This is needed because we are passing url as the encoding type. Since the
  587. # paginator is based on the key, we need to handle it before it can be
  588. # round tripped.
  589. #
  590. # From the documentation: If you specify encoding-type request parameter,
  591. # Amazon S3 includes this element in the response, and returns encoded key
  592. # name values in the following response elements:
  593. # Delimiter, Marker, Prefix, NextMarker, Key.
  594. _decode_list_object(
  595. top_level_keys=['Delimiter', 'Marker', 'NextMarker'],
  596. nested_keys=[('Contents', 'Key'), ('CommonPrefixes', 'Prefix')],
  597. parsed=parsed,
  598. context=context
  599. )
  600. def decode_list_object_v2(parsed, context, **kwargs):
  601. # From the documentation: If you specify encoding-type request parameter,
  602. # Amazon S3 includes this element in the response, and returns encoded key
  603. # name values in the following response elements:
  604. # Delimiter, Prefix, ContinuationToken, Key, and StartAfter.
  605. _decode_list_object(
  606. top_level_keys=['Delimiter', 'Prefix', 'StartAfter'],
  607. nested_keys=[('Contents', 'Key'), ('CommonPrefixes', 'Prefix')],
  608. parsed=parsed,
  609. context=context
  610. )
  611. def decode_list_object_versions(parsed, context, **kwargs):
  612. # From the documentation: If you specify encoding-type request parameter,
  613. # Amazon S3 includes this element in the response, and returns encoded key
  614. # name values in the following response elements:
  615. # KeyMarker, NextKeyMarker, Prefix, Key, and Delimiter.
  616. _decode_list_object(
  617. top_level_keys=[
  618. 'KeyMarker',
  619. 'NextKeyMarker',
  620. 'Prefix',
  621. 'Delimiter',
  622. ],
  623. nested_keys=[
  624. ('Versions', 'Key'),
  625. ('DeleteMarkers', 'Key'),
  626. ('CommonPrefixes', 'Prefix'),
  627. ],
  628. parsed=parsed,
  629. context=context
  630. )
  631. def _decode_list_object(top_level_keys, nested_keys, parsed, context):
  632. if parsed.get('EncodingType') == 'url' and \
  633. context.get('encoding_type_auto_set'):
  634. # URL decode top-level keys in the response if present.
  635. for key in top_level_keys:
  636. if key in parsed:
  637. parsed[key] = unquote_str(parsed[key])
  638. # URL decode nested keys from the response if present.
  639. for (top_key, child_key) in nested_keys:
  640. if top_key in parsed:
  641. for member in parsed[top_key]:
  642. member[child_key] = unquote_str(member[child_key])
  643. def convert_body_to_file_like_object(params, **kwargs):
  644. if 'Body' in params:
  645. if isinstance(params['Body'], six.string_types):
  646. params['Body'] = six.BytesIO(ensure_bytes(params['Body']))
  647. elif isinstance(params['Body'], six.binary_type):
  648. params['Body'] = six.BytesIO(params['Body'])
  649. def _add_parameter_aliases(handler_list):
  650. # Mapping of original parameter to parameter alias.
  651. # The key is <service>.<operation>.parameter
  652. # The first part of the key is used for event registration.
  653. # The last part is the original parameter name and the value is the
  654. # alias to expose in documentation.
  655. aliases = {
  656. 'ec2.*.Filter': 'Filters',
  657. 'logs.CreateExportTask.from': 'fromTime',
  658. 'cloudsearchdomain.Search.return': 'returnFields'
  659. }
  660. for original, new_name in aliases.items():
  661. event_portion, original_name = original.rsplit('.', 1)
  662. parameter_alias = ParameterAlias(original_name, new_name)
  663. # Add the handlers to the list of handlers.
  664. # One handler is to handle when users provide the alias.
  665. # The other handler is to update the documentation to show only
  666. # the alias.
  667. parameter_build_event_handler_tuple = (
  668. 'before-parameter-build.' + event_portion,
  669. parameter_alias.alias_parameter_in_call,
  670. REGISTER_FIRST
  671. )
  672. docs_event_handler_tuple = (
  673. 'docs.*.' + event_portion + '.complete-section',
  674. parameter_alias.alias_parameter_in_documentation)
  675. handler_list.append(parameter_build_event_handler_tuple)
  676. handler_list.append(docs_event_handler_tuple)
  677. class ParameterAlias(object):
  678. def __init__(self, original_name, alias_name):
  679. self._original_name = original_name
  680. self._alias_name = alias_name
  681. def alias_parameter_in_call(self, params, model, **kwargs):
  682. if model.input_shape:
  683. # Only consider accepting the alias if it is modeled in the
  684. # input shape.
  685. if self._original_name in model.input_shape.members:
  686. if self._alias_name in params:
  687. if self._original_name in params:
  688. raise AliasConflictParameterError(
  689. original=self._original_name,
  690. alias=self._alias_name,
  691. operation=model.name
  692. )
  693. # Remove the alias parameter value and use the old name
  694. # instead.
  695. params[self._original_name] = params.pop(self._alias_name)
  696. def alias_parameter_in_documentation(self, event_name, section, **kwargs):
  697. if event_name.startswith('docs.request-params'):
  698. if self._original_name not in section.available_sections:
  699. return
  700. # Replace the name for parameter type
  701. param_section = section.get_section(self._original_name)
  702. param_type_section = param_section.get_section('param-type')
  703. self._replace_content(param_type_section)
  704. # Replace the name for the parameter description
  705. param_name_section = param_section.get_section('param-name')
  706. self._replace_content(param_name_section)
  707. elif event_name.startswith('docs.request-example'):
  708. section = section.get_section('structure-value')
  709. if self._original_name not in section.available_sections:
  710. return
  711. # Replace the name for the example
  712. param_section = section.get_section(self._original_name)
  713. self._replace_content(param_section)
  714. def _replace_content(self, section):
  715. content = section.getvalue().decode('utf-8')
  716. updated_content = content.replace(
  717. self._original_name, self._alias_name)
  718. section.clear_text()
  719. section.write(updated_content)
  720. class ClientMethodAlias(object):
  721. def __init__(self, actual_name):
  722. """ Aliases a non-extant method to an existing method.
  723. :param actual_name: The name of the method that actually exists on
  724. the client.
  725. """
  726. self._actual = actual_name
  727. def __call__(self, client, **kwargs):
  728. return getattr(client, self._actual)
  729. # TODO: Remove this class as it is no longer used
  730. class HeaderToHostHoister(object):
  731. """Takes a header and moves it to the front of the hoststring.
  732. """
  733. _VALID_HOSTNAME = re.compile(r'(?!-)[a-z\d-]{1,63}(?<!-)$', re.IGNORECASE)
  734. def __init__(self, header_name):
  735. self._header_name = header_name
  736. def hoist(self, params, **kwargs):
  737. """Hoist a header to the hostname.
  738. Hoist a header to the beginning of the hostname with a suffix "." after
  739. it. The original header should be removed from the header map. This
  740. method is intended to be used as a target for the before-call event.
  741. """
  742. if self._header_name not in params['headers']:
  743. return
  744. header_value = params['headers'][self._header_name]
  745. self._ensure_header_is_valid_host(header_value)
  746. original_url = params['url']
  747. new_url = self._prepend_to_host(original_url, header_value)
  748. params['url'] = new_url
  749. def _ensure_header_is_valid_host(self, header):
  750. match = self._VALID_HOSTNAME.match(header)
  751. if not match:
  752. raise ParamValidationError(report=(
  753. 'Hostnames must contain only - and alphanumeric characters, '
  754. 'and between 1 and 63 characters long.'
  755. ))
  756. def _prepend_to_host(self, url, prefix):
  757. url_components = urlsplit(url)
  758. parts = url_components.netloc.split('.')
  759. parts = [prefix] + parts
  760. new_netloc = '.'.join(parts)
  761. new_components = (
  762. url_components.scheme,
  763. new_netloc,
  764. url_components.path,
  765. url_components.query,
  766. ''
  767. )
  768. new_url = urlunsplit(new_components)
  769. return new_url
  770. def inject_api_version_header_if_needed(model, params, **kwargs):
  771. if not model.is_endpoint_discovery_operation:
  772. return
  773. params['headers']['x-amz-api-version'] = model.service_model.api_version
  774. def remove_lex_v2_start_conversation(class_attributes, **kwargs):
  775. """Operation requires h2 which is currently unsupported in Python"""
  776. if 'start_conversation' in class_attributes:
  777. del class_attributes['start_conversation']
  778. # This is a list of (event_name, handler).
  779. # When a Session is created, everything in this list will be
  780. # automatically registered with that Session.
  781. BUILTIN_HANDLERS = [
  782. ('choose-service-name', handle_service_name_alias),
  783. ('getattr.mturk.list_hi_ts_for_qualification_type',
  784. ClientMethodAlias('list_hits_for_qualification_type')),
  785. ('before-parameter-build.s3.UploadPart',
  786. convert_body_to_file_like_object, REGISTER_LAST),
  787. ('before-parameter-build.s3.PutObject',
  788. convert_body_to_file_like_object, REGISTER_LAST),
  789. ('creating-client-class', add_generate_presigned_url),
  790. ('creating-client-class.s3', add_generate_presigned_post),
  791. ('creating-client-class.iot-data', check_openssl_supports_tls_version_1_2),
  792. ('creating-client-class.lex-runtime-v2', remove_lex_v2_start_conversation),
  793. ('after-call.iam', json_decode_policies),
  794. ('after-call.ec2.GetConsoleOutput', decode_console_output),
  795. ('after-call.cloudformation.GetTemplate', json_decode_template_body),
  796. ('after-call.s3.GetBucketLocation', parse_get_bucket_location),
  797. ('before-parameter-build', generate_idempotent_uuid),
  798. ('before-parameter-build.s3', validate_bucket_name),
  799. ('before-parameter-build.s3.ListObjects',
  800. set_list_objects_encoding_type_url),
  801. ('before-parameter-build.s3.ListObjectsV2',
  802. set_list_objects_encoding_type_url),
  803. ('before-parameter-build.s3.ListObjectVersions',
  804. set_list_objects_encoding_type_url),
  805. ('before-parameter-build.s3.CopyObject',
  806. handle_copy_source_param),
  807. ('before-parameter-build.s3.UploadPartCopy',
  808. handle_copy_source_param),
  809. ('before-parameter-build.s3.CopyObject', validate_ascii_metadata),
  810. ('before-parameter-build.s3.PutObject', validate_ascii_metadata),
  811. ('before-parameter-build.s3.CreateMultipartUpload',
  812. validate_ascii_metadata),
  813. ('docs.*.s3.CopyObject.complete-section', document_copy_source_form),
  814. ('docs.*.s3.UploadPartCopy.complete-section', document_copy_source_form),
  815. ('before-call.s3', add_expect_header),
  816. ('before-call.glacier', add_glacier_version),
  817. ('before-call.apigateway', add_accept_header),
  818. ('before-call.s3.PutObject', conditionally_calculate_md5),
  819. ('before-call.s3.UploadPart', conditionally_calculate_md5),
  820. ('before-call.s3.DeleteObjects', escape_xml_payload),
  821. ('before-call.s3.PutBucketLifecycleConfiguration', escape_xml_payload),
  822. ('before-call.glacier.UploadArchive', add_glacier_checksums),
  823. ('before-call.glacier.UploadMultipartPart', add_glacier_checksums),
  824. ('before-call.ec2.CopySnapshot', inject_presigned_url_ec2),
  825. ('request-created.machinelearning.Predict', switch_host_machinelearning),
  826. ('needs-retry.s3.UploadPartCopy', check_for_200_error, REGISTER_FIRST),
  827. ('needs-retry.s3.CopyObject', check_for_200_error, REGISTER_FIRST),
  828. ('needs-retry.s3.CompleteMultipartUpload', check_for_200_error,
  829. REGISTER_FIRST),
  830. ('choose-signer.cognito-identity.GetId', disable_signing),
  831. ('choose-signer.cognito-identity.GetOpenIdToken', disable_signing),
  832. ('choose-signer.cognito-identity.UnlinkIdentity', disable_signing),
  833. ('choose-signer.cognito-identity.GetCredentialsForIdentity',
  834. disable_signing),
  835. ('choose-signer.sts.AssumeRoleWithSAML', disable_signing),
  836. ('choose-signer.sts.AssumeRoleWithWebIdentity', disable_signing),
  837. ('choose-signer', set_operation_specific_signer),
  838. ('before-parameter-build.s3.HeadObject', sse_md5),
  839. ('before-parameter-build.s3.GetObject', sse_md5),
  840. ('before-parameter-build.s3.PutObject', sse_md5),
  841. ('before-parameter-build.s3.CopyObject', sse_md5),
  842. ('before-parameter-build.s3.CopyObject', copy_source_sse_md5),
  843. ('before-parameter-build.s3.CreateMultipartUpload', sse_md5),
  844. ('before-parameter-build.s3.UploadPart', sse_md5),
  845. ('before-parameter-build.s3.UploadPartCopy', sse_md5),
  846. ('before-parameter-build.s3.UploadPartCopy', copy_source_sse_md5),
  847. ('before-parameter-build.ec2.RunInstances', base64_encode_user_data),
  848. ('before-parameter-build.autoscaling.CreateLaunchConfiguration',
  849. base64_encode_user_data),
  850. ('before-parameter-build.route53', fix_route53_ids),
  851. ('before-parameter-build.glacier', inject_account_id),
  852. ('after-call.s3.ListObjects', decode_list_object),
  853. ('after-call.s3.ListObjectsV2', decode_list_object_v2),
  854. ('after-call.s3.ListObjectVersions', decode_list_object_versions),
  855. # Cloudsearchdomain search operation will be sent by HTTP POST
  856. ('request-created.cloudsearchdomain.Search',
  857. change_get_to_post),
  858. # Glacier documentation customizations
  859. ('docs.*.glacier.*.complete-section',
  860. AutoPopulatedParam('accountId', 'Note: this parameter is set to "-" by'
  861. 'default if no value is not specified.')
  862. .document_auto_populated_param),
  863. ('docs.*.glacier.UploadArchive.complete-section',
  864. AutoPopulatedParam('checksum').document_auto_populated_param),
  865. ('docs.*.glacier.UploadMultipartPart.complete-section',
  866. AutoPopulatedParam('checksum').document_auto_populated_param),
  867. ('docs.request-params.glacier.CompleteMultipartUpload.complete-section',
  868. document_glacier_tree_hash_checksum()),
  869. # Cloudformation documentation customizations
  870. ('docs.*.cloudformation.GetTemplate.complete-section',
  871. document_cloudformation_get_template_return_type),
  872. # UserData base64 encoding documentation customizations
  873. ('docs.*.ec2.RunInstances.complete-section',
  874. document_base64_encoding('UserData')),
  875. ('docs.*.autoscaling.CreateLaunchConfiguration.complete-section',
  876. document_base64_encoding('UserData')),
  877. # EC2 CopySnapshot documentation customizations
  878. ('docs.*.ec2.CopySnapshot.complete-section',
  879. AutoPopulatedParam('PresignedUrl').document_auto_populated_param),
  880. ('docs.*.ec2.CopySnapshot.complete-section',
  881. AutoPopulatedParam('DestinationRegion').document_auto_populated_param),
  882. # S3 SSE documentation modifications
  883. ('docs.*.s3.*.complete-section',
  884. AutoPopulatedParam('SSECustomerKeyMD5').document_auto_populated_param),
  885. # S3 SSE Copy Source documentation modifications
  886. ('docs.*.s3.*.complete-section',
  887. AutoPopulatedParam(
  888. 'CopySourceSSECustomerKeyMD5').document_auto_populated_param),
  889. # Add base64 information to Lambda
  890. ('docs.*.lambda.UpdateFunctionCode.complete-section',
  891. document_base64_encoding('ZipFile')),
  892. # The following S3 operations cannot actually accept a ContentMD5
  893. ('docs.*.s3.*.complete-section',
  894. HideParamFromOperations(
  895. 's3', 'ContentMD5',
  896. ['DeleteObjects', 'PutBucketAcl', 'PutBucketCors',
  897. 'PutBucketLifecycle', 'PutBucketLogging', 'PutBucketNotification',
  898. 'PutBucketPolicy', 'PutBucketReplication', 'PutBucketRequestPayment',
  899. 'PutBucketTagging', 'PutBucketVersioning', 'PutBucketWebsite',
  900. 'PutObjectAcl']).hide_param),
  901. #############
  902. # RDS
  903. #############
  904. ('creating-client-class.rds', add_generate_db_auth_token),
  905. ('before-call.rds.CopyDBClusterSnapshot',
  906. inject_presigned_url_rds),
  907. ('before-call.rds.CreateDBCluster',
  908. inject_presigned_url_rds),
  909. ('before-call.rds.CopyDBSnapshot',
  910. inject_presigned_url_rds),
  911. ('before-call.rds.CreateDBInstanceReadReplica',
  912. inject_presigned_url_rds),
  913. ('before-call.rds.StartDBInstanceAutomatedBackupsReplication',
  914. inject_presigned_url_rds),
  915. # RDS PresignedUrl documentation customizations
  916. ('docs.*.rds.CopyDBClusterSnapshot.complete-section',
  917. AutoPopulatedParam('PreSignedUrl').document_auto_populated_param),
  918. ('docs.*.rds.CreateDBCluster.complete-section',
  919. AutoPopulatedParam('PreSignedUrl').document_auto_populated_param),
  920. ('docs.*.rds.CopyDBSnapshot.complete-section',
  921. AutoPopulatedParam('PreSignedUrl').document_auto_populated_param),
  922. ('docs.*.rds.CreateDBInstanceReadReplica.complete-section',
  923. AutoPopulatedParam('PreSignedUrl').document_auto_populated_param),
  924. ('docs.*.rds.StartDBInstanceAutomatedBackupsReplication.complete-section',
  925. AutoPopulatedParam('PreSignedUrl').document_auto_populated_param),
  926. #############
  927. # Neptune
  928. #############
  929. ('before-call.neptune.CopyDBClusterSnapshot',
  930. inject_presigned_url_rds),
  931. ('before-call.neptune.CreateDBCluster',
  932. inject_presigned_url_rds),
  933. # Neptune PresignedUrl documentation customizations
  934. ('docs.*.neptune.CopyDBClusterSnapshot.complete-section',
  935. AutoPopulatedParam('PreSignedUrl').document_auto_populated_param),
  936. ('docs.*.neptune.CreateDBCluster.complete-section',
  937. AutoPopulatedParam('PreSignedUrl').document_auto_populated_param),
  938. #############
  939. # DocDB
  940. #############
  941. ('before-call.docdb.CopyDBClusterSnapshot',
  942. inject_presigned_url_rds),
  943. ('before-call.docdb.CreateDBCluster',
  944. inject_presigned_url_rds),
  945. # DocDB PresignedUrl documentation customizations
  946. ('docs.*.docdb.CopyDBClusterSnapshot.complete-section',
  947. AutoPopulatedParam('PreSignedUrl').document_auto_populated_param),
  948. ('docs.*.docdb.CreateDBCluster.complete-section',
  949. AutoPopulatedParam('PreSignedUrl').document_auto_populated_param),
  950. ###########
  951. # SMS Voice
  952. ##########
  953. ('docs.title.sms-voice',
  954. DeprecatedServiceDocumenter(
  955. 'pinpoint-sms-voice').inject_deprecation_notice),
  956. ('before-call', inject_api_version_header_if_needed),
  957. ]
  958. _add_parameter_aliases(BUILTIN_HANDLERS)