12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136 |
- # Copyright 2012-2014 Amazon.com, Inc. or its affiliates. All Rights Reserved.
- #
- # Licensed under the Apache License, Version 2.0 (the "License"). You
- # may not use this file except in compliance with the License. A copy of
- # the License is located at
- #
- # http://aws.amazon.com/apache2.0/
- #
- # or in the "license" file accompanying this file. This file is
- # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
- # ANY KIND, either express or implied. See the License for the specific
- # language governing permissions and limitations under the License.
- """Builtin event handlers.
- This module contains builtin handlers for events emitted by botocore.
- """
- import base64
- import logging
- import copy
- import re
- import warnings
- import uuid
- from botocore.compat import (
- unquote, json, six, unquote_str, ensure_bytes, get_md5,
- MD5_AVAILABLE, OrderedDict, urlsplit, urlunsplit, XMLParseError,
- ETree,
- )
- from botocore.docs.utils import AutoPopulatedParam
- from botocore.docs.utils import HideParamFromOperations
- from botocore.docs.utils import AppendParamDocumentation
- from botocore.signers import add_generate_presigned_url
- from botocore.signers import add_generate_presigned_post
- from botocore.signers import add_generate_db_auth_token
- from botocore.exceptions import ParamValidationError
- from botocore.exceptions import AliasConflictParameterError
- from botocore.exceptions import UnsupportedTLSVersionWarning
- from botocore.exceptions import MissingServiceIdError
- from botocore.utils import percent_encode, SAFE_CHARS
- from botocore.utils import switch_host_with_param
- from botocore.utils import hyphenize_service_id
- from botocore.utils import conditionally_calculate_md5
- from botocore import retryhandler
- from botocore import utils
- from botocore import translate
- import botocore
- import botocore.auth
- logger = logging.getLogger(__name__)
- REGISTER_FIRST = object()
- REGISTER_LAST = object()
- # From the S3 docs:
- # The rules for bucket names in the US Standard region allow bucket names
- # to be as long as 255 characters, and bucket names can contain any
- # combination of uppercase letters, lowercase letters, numbers, periods
- # (.), hyphens (-), and underscores (_).
- VALID_BUCKET = re.compile(r'^[a-zA-Z0-9.\-_]{1,255}$')
- _ACCESSPOINT_ARN = (
- r'^arn:(aws).*:(s3|s3-object-lambda):[a-z\-0-9]+:[0-9]{12}:accesspoint[/:]'
- r'[a-zA-Z0-9\-]{1,63}$'
- )
- _OUTPOST_ARN = (
- r'^arn:(aws).*:s3-outposts:[a-z\-0-9]+:[0-9]{12}:outpost[/:]'
- r'[a-zA-Z0-9\-]{1,63}[/:]accesspoint[/:][a-zA-Z0-9\-]{1,63}$'
- )
- VALID_S3_ARN = re.compile('|'.join([_ACCESSPOINT_ARN, _OUTPOST_ARN]))
- VERSION_ID_SUFFIX = re.compile(r'\?versionId=[^\s]+$')
- SERVICE_NAME_ALIASES = {
- 'runtime.sagemaker': 'sagemaker-runtime'
- }
- def handle_service_name_alias(service_name, **kwargs):
- return SERVICE_NAME_ALIASES.get(service_name, service_name)
- def escape_xml_payload(params, **kwargs):
- # Replace \r and \n with the escaped sequence over the whole XML document
- # to avoid linebreak normalization modifying customer input when the
- # document is parsed. Ideally, we would do this in ElementTree.tostring,
- # but it doesn't allow us to override entity escaping for text fields. For
- # this operation \r and \n can only appear in the XML document if they were
- # passed as part of the customer input.
- body = params['body']
- replaced = False
- if b'\r' in body:
- replaced = True
- body = body.replace(b'\r', b'
')
- if b'\n' in body:
- replaced = True
- body = body.replace(b'\n', b'
')
- if not replaced:
- return
- params['body'] = body
- if 'Content-MD5' in params['headers']:
- # The Content-MD5 is now wrong, so we'll need to recalculate it
- del params['headers']['Content-MD5']
- conditionally_calculate_md5(params, **kwargs)
- def check_for_200_error(response, **kwargs):
- # From: http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectCOPY.html
- # There are two opportunities for a copy request to return an error. One
- # can occur when Amazon S3 receives the copy request and the other can
- # occur while Amazon S3 is copying the files. If the error occurs before
- # the copy operation starts, you receive a standard Amazon S3 error. If the
- # error occurs during the copy operation, the error response is embedded in
- # the 200 OK response. This means that a 200 OK response can contain either
- # a success or an error. Make sure to design your application to parse the
- # contents of the response and handle it appropriately.
- #
- # So this handler checks for this case. Even though the server sends a
- # 200 response, conceptually this should be handled exactly like a
- # 500 response (with respect to raising exceptions, retries, etc.)
- # We're connected *before* all the other retry logic handlers, so as long
- # as we switch the error code to 500, we'll retry the error as expected.
- if response is None:
- # A None response can happen if an exception is raised while
- # trying to retrieve the response. See Endpoint._get_response().
- return
- http_response, parsed = response
- if _looks_like_special_case_error(http_response):
- logger.debug("Error found for response with 200 status code, "
- "errors: %s, changing status code to "
- "500.", parsed)
- http_response.status_code = 500
- def _looks_like_special_case_error(http_response):
- if http_response.status_code == 200:
- try:
- parser = ETree.XMLParser(
- target=ETree.TreeBuilder(),
- encoding='utf-8')
- parser.feed(http_response.content)
- root = parser.close()
- except XMLParseError:
- # In cases of network disruptions, we may end up with a partial
- # streamed response from S3. We need to treat these cases as
- # 500 Service Errors and try again.
- return True
- if root.tag == 'Error':
- return True
- return False
- def set_operation_specific_signer(context, signing_name, **kwargs):
- """ Choose the operation-specific signer.
- Individual operations may have a different auth type than the service as a
- whole. This will most often manifest as operations that should not be
- authenticated at all, but can include other auth modes such as sigv4
- without body signing.
- """
- auth_type = context.get('auth_type')
- # Auth type will be None if the operation doesn't have a configured auth
- # type.
- if not auth_type:
- return
- # Auth type will be the string value 'none' if the operation should not
- # be signed at all.
- if auth_type == 'none':
- return botocore.UNSIGNED
- if auth_type.startswith('v4'):
- signature_version = 'v4'
- if signing_name == 's3':
- signature_version = 's3v4'
- # If the operation needs an unsigned body, we set additional context
- # allowing the signer to be aware of this.
- if auth_type == 'v4-unsigned-body':
- context['payload_signing_enabled'] = False
- return signature_version
- def decode_console_output(parsed, **kwargs):
- if 'Output' in parsed:
- try:
- # We're using 'replace' for errors because it is
- # possible that console output contains non string
- # chars we can't utf-8 decode.
- value = base64.b64decode(six.b(parsed['Output'])).decode(
- 'utf-8', 'replace')
- parsed['Output'] = value
- except (ValueError, TypeError, AttributeError):
- logger.debug('Error decoding base64', exc_info=True)
- def generate_idempotent_uuid(params, model, **kwargs):
- for name in model.idempotent_members:
- if name not in params:
- params[name] = str(uuid.uuid4())
- logger.debug("injecting idempotency token (%s) into param '%s'." %
- (params[name], name))
- def decode_quoted_jsondoc(value):
- try:
- value = json.loads(unquote(value))
- except (ValueError, TypeError):
- logger.debug('Error loading quoted JSON', exc_info=True)
- return value
- def json_decode_template_body(parsed, **kwargs):
- if 'TemplateBody' in parsed:
- try:
- value = json.loads(
- parsed['TemplateBody'], object_pairs_hook=OrderedDict)
- parsed['TemplateBody'] = value
- except (ValueError, TypeError):
- logger.debug('error loading JSON', exc_info=True)
- def validate_bucket_name(params, **kwargs):
- if 'Bucket' not in params:
- return
- bucket = params['Bucket']
- if not VALID_BUCKET.search(bucket) and not VALID_S3_ARN.search(bucket):
- error_msg = (
- 'Invalid bucket name "%s": Bucket name must match '
- 'the regex "%s" or be an ARN matching the regex "%s"' % (
- bucket, VALID_BUCKET.pattern, VALID_S3_ARN.pattern))
- raise ParamValidationError(report=error_msg)
- def sse_md5(params, **kwargs):
- """
- S3 server-side encryption requires the encryption key to be sent to the
- server base64 encoded, as well as a base64-encoded MD5 hash of the
- encryption key. This handler does both if the MD5 has not been set by
- the caller.
- """
- _sse_md5(params, 'SSECustomer')
- def copy_source_sse_md5(params, **kwargs):
- """
- S3 server-side encryption requires the encryption key to be sent to the
- server base64 encoded, as well as a base64-encoded MD5 hash of the
- encryption key. This handler does both if the MD5 has not been set by
- the caller specifically if the parameter is for the copy-source sse-c key.
- """
- _sse_md5(params, 'CopySourceSSECustomer')
- def _sse_md5(params, sse_member_prefix='SSECustomer'):
- if not _needs_s3_sse_customization(params, sse_member_prefix):
- return
- sse_key_member = sse_member_prefix + 'Key'
- sse_md5_member = sse_member_prefix + 'KeyMD5'
- key_as_bytes = params[sse_key_member]
- if isinstance(key_as_bytes, six.text_type):
- key_as_bytes = key_as_bytes.encode('utf-8')
- key_md5_str = base64.b64encode(
- get_md5(key_as_bytes).digest()).decode('utf-8')
- key_b64_encoded = base64.b64encode(key_as_bytes).decode('utf-8')
- params[sse_key_member] = key_b64_encoded
- params[sse_md5_member] = key_md5_str
- def _needs_s3_sse_customization(params, sse_member_prefix):
- return (params.get(sse_member_prefix + 'Key') is not None and
- sse_member_prefix + 'KeyMD5' not in params)
- def disable_signing(**kwargs):
- """
- This handler disables request signing by setting the signer
- name to a special sentinel value.
- """
- return botocore.UNSIGNED
- def add_expect_header(model, params, **kwargs):
- if model.http.get('method', '') not in ['PUT', 'POST']:
- return
- if 'body' in params:
- body = params['body']
- if hasattr(body, 'read'):
- # Any file like object will use an expect 100-continue
- # header regardless of size.
- logger.debug("Adding expect 100 continue header to request.")
- params['headers']['Expect'] = '100-continue'
- class DeprecatedServiceDocumenter(object):
- def __init__(self, replacement_service_name):
- self._replacement_service_name = replacement_service_name
- def inject_deprecation_notice(self, section, event_name, **kwargs):
- section.style.start_important()
- section.write('This service client is deprecated. Please use ')
- section.style.ref(
- self._replacement_service_name,
- self._replacement_service_name,
- )
- section.write(' instead.')
- section.style.end_important()
- def document_copy_source_form(section, event_name, **kwargs):
- if 'request-example' in event_name:
- parent = section.get_section('structure-value')
- param_line = parent.get_section('CopySource')
- value_portion = param_line.get_section('member-value')
- value_portion.clear_text()
- value_portion.write("'string' or {'Bucket': 'string', "
- "'Key': 'string', 'VersionId': 'string'}")
- elif 'request-params' in event_name:
- param_section = section.get_section('CopySource')
- type_section = param_section.get_section('param-type')
- type_section.clear_text()
- type_section.write(':type CopySource: str or dict')
- doc_section = param_section.get_section('param-documentation')
- doc_section.clear_text()
- doc_section.write(
- "The name of the source bucket, key name of the source object, "
- "and optional version ID of the source object. You can either "
- "provide this value as a string or a dictionary. The "
- "string form is {bucket}/{key} or "
- "{bucket}/{key}?versionId={versionId} if you want to copy a "
- "specific version. You can also provide this value as a "
- "dictionary. The dictionary format is recommended over "
- "the string format because it is more explicit. The dictionary "
- "format is: {'Bucket': 'bucket', 'Key': 'key', 'VersionId': 'id'}."
- " Note that the VersionId key is optional and may be omitted."
- " To specify an S3 access point, provide the access point"
- " ARN for the ``Bucket`` key in the copy source dictionary. If you"
- " want to provide the copy source for an S3 access point as a"
- " string instead of a dictionary, the ARN provided must be the"
- " full S3 access point object ARN"
- " (i.e. {accesspoint_arn}/object/{key})"
- )
- def handle_copy_source_param(params, **kwargs):
- """Convert CopySource param for CopyObject/UploadPartCopy.
- This handler will deal with two cases:
- * CopySource provided as a string. We'll make a best effort
- to URL encode the key name as required. This will require
- parsing the bucket and version id from the CopySource value
- and only encoding the key.
- * CopySource provided as a dict. In this case we're
- explicitly given the Bucket, Key, and VersionId so we're
- able to encode the key and ensure this value is serialized
- and correctly sent to S3.
- """
- source = params.get('CopySource')
- if source is None:
- # The call will eventually fail but we'll let the
- # param validator take care of this. It will
- # give a better error message.
- return
- if isinstance(source, six.string_types):
- params['CopySource'] = _quote_source_header(source)
- elif isinstance(source, dict):
- params['CopySource'] = _quote_source_header_from_dict(source)
- def _quote_source_header_from_dict(source_dict):
- try:
- bucket = source_dict['Bucket']
- key = source_dict['Key']
- version_id = source_dict.get('VersionId')
- if VALID_S3_ARN.search(bucket):
- final = '%s/object/%s' % (bucket, key)
- else:
- final = '%s/%s' % (bucket, key)
- except KeyError as e:
- raise ParamValidationError(
- report='Missing required parameter: %s' % str(e))
- final = percent_encode(final, safe=SAFE_CHARS + '/')
- if version_id is not None:
- final += '?versionId=%s' % version_id
- return final
- def _quote_source_header(value):
- result = VERSION_ID_SUFFIX.search(value)
- if result is None:
- return percent_encode(value, safe=SAFE_CHARS + '/')
- else:
- first, version_id = value[:result.start()], value[result.start():]
- return percent_encode(first, safe=SAFE_CHARS + '/') + version_id
- def _get_cross_region_presigned_url(request_signer, request_dict, model,
- source_region, destination_region):
- # The better way to do this is to actually get the
- # endpoint_resolver and get the endpoint_url given the
- # source region. In this specific case, we know that
- # we can safely replace the dest region with the source
- # region because of the supported EC2 regions, but in
- # general this is not a safe assumption to make.
- # I think eventually we should try to plumb through something
- # that allows us to resolve endpoints from regions.
- request_dict_copy = copy.deepcopy(request_dict)
- request_dict_copy['body']['DestinationRegion'] = destination_region
- request_dict_copy['url'] = request_dict['url'].replace(
- destination_region, source_region)
- request_dict_copy['method'] = 'GET'
- request_dict_copy['headers'] = {}
- return request_signer.generate_presigned_url(
- request_dict_copy, region_name=source_region,
- operation_name=model.name)
- def _get_presigned_url_source_and_destination_regions(request_signer, params):
- # Gets the source and destination regions to be used
- destination_region = request_signer._region_name
- source_region = params.get('SourceRegion')
- return source_region, destination_region
- def inject_presigned_url_ec2(params, request_signer, model, **kwargs):
- # The customer can still provide this, so we should pass if they do.
- if 'PresignedUrl' in params['body']:
- return
- src, dest = _get_presigned_url_source_and_destination_regions(
- request_signer, params['body'])
- url = _get_cross_region_presigned_url(
- request_signer, params, model, src, dest)
- params['body']['PresignedUrl'] = url
- # EC2 Requires that the destination region be sent over the wire in
- # addition to the source region.
- params['body']['DestinationRegion'] = dest
- def inject_presigned_url_rds(params, request_signer, model, **kwargs):
- # SourceRegion is not required for RDS operations, so it's possible that
- # it isn't set. In that case it's probably a local copy so we don't need
- # to do anything else.
- if 'SourceRegion' not in params['body']:
- return
- src, dest = _get_presigned_url_source_and_destination_regions(
- request_signer, params['body'])
- # Since SourceRegion isn't actually modeled for RDS, it needs to be
- # removed from the request params before we send the actual request.
- del params['body']['SourceRegion']
- if 'PreSignedUrl' in params['body']:
- return
- url = _get_cross_region_presigned_url(
- request_signer, params, model, src, dest)
- params['body']['PreSignedUrl'] = url
- def json_decode_policies(parsed, model, **kwargs):
- # Any time an IAM operation returns a policy document
- # it is a string that is json that has been urlencoded,
- # i.e urlencode(json.dumps(policy_document)).
- # To give users something more useful, we will urldecode
- # this value and json.loads() the result so that they have
- # the policy document as a dictionary.
- output_shape = model.output_shape
- if output_shape is not None:
- _decode_policy_types(parsed, model.output_shape)
- def _decode_policy_types(parsed, shape):
- # IAM consistently uses the policyDocumentType shape to indicate
- # strings that have policy documents.
- shape_name = 'policyDocumentType'
- if shape.type_name == 'structure':
- for member_name, member_shape in shape.members.items():
- if member_shape.type_name == 'string' and \
- member_shape.name == shape_name and \
- member_name in parsed:
- parsed[member_name] = decode_quoted_jsondoc(
- parsed[member_name])
- elif member_name in parsed:
- _decode_policy_types(parsed[member_name], member_shape)
- if shape.type_name == 'list':
- shape_member = shape.member
- for item in parsed:
- _decode_policy_types(item, shape_member)
- def parse_get_bucket_location(parsed, http_response, **kwargs):
- # s3.GetBucketLocation cannot be modeled properly. To
- # account for this we just manually parse the XML document.
- # The "parsed" passed in only has the ResponseMetadata
- # filled out. This handler will fill in the LocationConstraint
- # value.
- if http_response.raw is None:
- return
- response_body = http_response.content
- parser = ETree.XMLParser(
- target=ETree.TreeBuilder(),
- encoding='utf-8')
- parser.feed(response_body)
- root = parser.close()
- region = root.text
- parsed['LocationConstraint'] = region
- def base64_encode_user_data(params, **kwargs):
- if 'UserData' in params:
- if isinstance(params['UserData'], six.text_type):
- # Encode it to bytes if it is text.
- params['UserData'] = params['UserData'].encode('utf-8')
- params['UserData'] = base64.b64encode(
- params['UserData']).decode('utf-8')
- def document_base64_encoding(param):
- description = ('**This value will be base64 encoded automatically. Do '
- 'not base64 encode this value prior to performing the '
- 'operation.**')
- append = AppendParamDocumentation(param, description)
- return append.append_documentation
- def validate_ascii_metadata(params, **kwargs):
- """Verify S3 Metadata only contains ascii characters.
- From: http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html
- "Amazon S3 stores user-defined metadata in lowercase. Each name, value pair
- must conform to US-ASCII when using REST and UTF-8 when using SOAP or
- browser-based uploads via POST."
- """
- metadata = params.get('Metadata')
- if not metadata or not isinstance(metadata, dict):
- # We have to at least type check the metadata as a dict type
- # because this handler is called before param validation.
- # We'll go ahead and return because the param validator will
- # give a descriptive error message for us.
- # We might need a post-param validation event.
- return
- for key, value in metadata.items():
- try:
- key.encode('ascii')
- value.encode('ascii')
- except UnicodeEncodeError as e:
- error_msg = (
- 'Non ascii characters found in S3 metadata '
- 'for key "%s", value: "%s". \nS3 metadata can only '
- 'contain ASCII characters. ' % (key, value)
- )
- raise ParamValidationError(
- report=error_msg)
- def fix_route53_ids(params, model, **kwargs):
- """
- Check for and split apart Route53 resource IDs, setting
- only the last piece. This allows the output of one operation
- (e.g. ``'foo/1234'``) to be used as input in another
- operation (e.g. it expects just ``'1234'``).
- """
- input_shape = model.input_shape
- if not input_shape or not hasattr(input_shape, 'members'):
- return
- members = [name for (name, shape) in input_shape.members.items()
- if shape.name in ['ResourceId', 'DelegationSetId']]
- for name in members:
- if name in params:
- orig_value = params[name]
- params[name] = orig_value.split('/')[-1]
- logger.debug('%s %s -> %s', name, orig_value, params[name])
- def inject_account_id(params, **kwargs):
- if params.get('accountId') is None:
- # Glacier requires accountId, but allows you
- # to specify '-' for the current owners account.
- # We add this default value if the user does not
- # provide the accountId as a convenience.
- params['accountId'] = '-'
- def add_glacier_version(model, params, **kwargs):
- request_dict = params
- request_dict['headers']['x-amz-glacier-version'] = model.metadata[
- 'apiVersion']
- def add_accept_header(model, params, **kwargs):
- if params['headers'].get('Accept', None) is None:
- request_dict = params
- request_dict['headers']['Accept'] = 'application/json'
- def add_glacier_checksums(params, **kwargs):
- """Add glacier checksums to the http request.
- This will add two headers to the http request:
- * x-amz-content-sha256
- * x-amz-sha256-tree-hash
- These values will only be added if they are not present
- in the HTTP request.
- """
- request_dict = params
- headers = request_dict['headers']
- body = request_dict['body']
- if isinstance(body, six.binary_type):
- # If the user provided a bytes type instead of a file
- # like object, we're temporarily create a BytesIO object
- # so we can use the util functions to calculate the
- # checksums which assume file like objects. Note that
- # we're not actually changing the body in the request_dict.
- body = six.BytesIO(body)
- starting_position = body.tell()
- if 'x-amz-content-sha256' not in headers:
- headers['x-amz-content-sha256'] = utils.calculate_sha256(
- body, as_hex=True)
- body.seek(starting_position)
- if 'x-amz-sha256-tree-hash' not in headers:
- headers['x-amz-sha256-tree-hash'] = utils.calculate_tree_hash(body)
- body.seek(starting_position)
- def document_glacier_tree_hash_checksum():
- doc = '''
- This is a required field.
- Ideally you will want to compute this value with checksums from
- previous uploaded parts, using the algorithm described in
- `Glacier documentation <http://docs.aws.amazon.com/amazonglacier/latest/dev/checksum-calculations.html>`_.
- But if you prefer, you can also use botocore.utils.calculate_tree_hash()
- to compute it from raw file by::
- checksum = calculate_tree_hash(open('your_file.txt', 'rb'))
- '''
- return AppendParamDocumentation('checksum', doc).append_documentation
- def document_cloudformation_get_template_return_type(section, event_name, **kwargs):
- if 'response-params' in event_name:
- template_body_section = section.get_section('TemplateBody')
- type_section = template_body_section.get_section('param-type')
- type_section.clear_text()
- type_section.write('(*dict*) --')
- elif 'response-example' in event_name:
- parent = section.get_section('structure-value')
- param_line = parent.get_section('TemplateBody')
- value_portion = param_line.get_section('member-value')
- value_portion.clear_text()
- value_portion.write('{}')
- def switch_host_machinelearning(request, **kwargs):
- switch_host_with_param(request, 'PredictEndpoint')
- def check_openssl_supports_tls_version_1_2(**kwargs):
- import ssl
- try:
- openssl_version_tuple = ssl.OPENSSL_VERSION_INFO
- if openssl_version_tuple < (1, 0, 1):
- warnings.warn(
- 'Currently installed openssl version: %s does not '
- 'support TLS 1.2, which is required for use of iot-data. '
- 'Please use python installed with openssl version 1.0.1 or '
- 'higher.' % (ssl.OPENSSL_VERSION),
- UnsupportedTLSVersionWarning
- )
- # We cannot check the openssl version on python2.6, so we should just
- # pass on this conveniency check.
- except AttributeError:
- pass
- def change_get_to_post(request, **kwargs):
- # This is useful when we need to change a potentially large GET request
- # into a POST with x-www-form-urlencoded encoding.
- if request.method == 'GET' and '?' in request.url:
- request.headers['Content-Type'] = 'application/x-www-form-urlencoded'
- request.method = 'POST'
- request.url, request.data = request.url.split('?', 1)
- def set_list_objects_encoding_type_url(params, context, **kwargs):
- if 'EncodingType' not in params:
- # We set this context so that we know it wasn't the customer that
- # requested the encoding.
- context['encoding_type_auto_set'] = True
- params['EncodingType'] = 'url'
- def decode_list_object(parsed, context, **kwargs):
- # This is needed because we are passing url as the encoding type. Since the
- # paginator is based on the key, we need to handle it before it can be
- # round tripped.
- #
- # From the documentation: If you specify encoding-type request parameter,
- # Amazon S3 includes this element in the response, and returns encoded key
- # name values in the following response elements:
- # Delimiter, Marker, Prefix, NextMarker, Key.
- _decode_list_object(
- top_level_keys=['Delimiter', 'Marker', 'NextMarker'],
- nested_keys=[('Contents', 'Key'), ('CommonPrefixes', 'Prefix')],
- parsed=parsed,
- context=context
- )
- def decode_list_object_v2(parsed, context, **kwargs):
- # From the documentation: If you specify encoding-type request parameter,
- # Amazon S3 includes this element in the response, and returns encoded key
- # name values in the following response elements:
- # Delimiter, Prefix, ContinuationToken, Key, and StartAfter.
- _decode_list_object(
- top_level_keys=['Delimiter', 'Prefix', 'StartAfter'],
- nested_keys=[('Contents', 'Key'), ('CommonPrefixes', 'Prefix')],
- parsed=parsed,
- context=context
- )
- def decode_list_object_versions(parsed, context, **kwargs):
- # From the documentation: If you specify encoding-type request parameter,
- # Amazon S3 includes this element in the response, and returns encoded key
- # name values in the following response elements:
- # KeyMarker, NextKeyMarker, Prefix, Key, and Delimiter.
- _decode_list_object(
- top_level_keys=[
- 'KeyMarker',
- 'NextKeyMarker',
- 'Prefix',
- 'Delimiter',
- ],
- nested_keys=[
- ('Versions', 'Key'),
- ('DeleteMarkers', 'Key'),
- ('CommonPrefixes', 'Prefix'),
- ],
- parsed=parsed,
- context=context
- )
- def _decode_list_object(top_level_keys, nested_keys, parsed, context):
- if parsed.get('EncodingType') == 'url' and \
- context.get('encoding_type_auto_set'):
- # URL decode top-level keys in the response if present.
- for key in top_level_keys:
- if key in parsed:
- parsed[key] = unquote_str(parsed[key])
- # URL decode nested keys from the response if present.
- for (top_key, child_key) in nested_keys:
- if top_key in parsed:
- for member in parsed[top_key]:
- member[child_key] = unquote_str(member[child_key])
- def convert_body_to_file_like_object(params, **kwargs):
- if 'Body' in params:
- if isinstance(params['Body'], six.string_types):
- params['Body'] = six.BytesIO(ensure_bytes(params['Body']))
- elif isinstance(params['Body'], six.binary_type):
- params['Body'] = six.BytesIO(params['Body'])
- def _add_parameter_aliases(handler_list):
- # Mapping of original parameter to parameter alias.
- # The key is <service>.<operation>.parameter
- # The first part of the key is used for event registration.
- # The last part is the original parameter name and the value is the
- # alias to expose in documentation.
- aliases = {
- 'ec2.*.Filter': 'Filters',
- 'logs.CreateExportTask.from': 'fromTime',
- 'cloudsearchdomain.Search.return': 'returnFields'
- }
- for original, new_name in aliases.items():
- event_portion, original_name = original.rsplit('.', 1)
- parameter_alias = ParameterAlias(original_name, new_name)
- # Add the handlers to the list of handlers.
- # One handler is to handle when users provide the alias.
- # The other handler is to update the documentation to show only
- # the alias.
- parameter_build_event_handler_tuple = (
- 'before-parameter-build.' + event_portion,
- parameter_alias.alias_parameter_in_call,
- REGISTER_FIRST
- )
- docs_event_handler_tuple = (
- 'docs.*.' + event_portion + '.complete-section',
- parameter_alias.alias_parameter_in_documentation)
- handler_list.append(parameter_build_event_handler_tuple)
- handler_list.append(docs_event_handler_tuple)
- class ParameterAlias(object):
- def __init__(self, original_name, alias_name):
- self._original_name = original_name
- self._alias_name = alias_name
- def alias_parameter_in_call(self, params, model, **kwargs):
- if model.input_shape:
- # Only consider accepting the alias if it is modeled in the
- # input shape.
- if self._original_name in model.input_shape.members:
- if self._alias_name in params:
- if self._original_name in params:
- raise AliasConflictParameterError(
- original=self._original_name,
- alias=self._alias_name,
- operation=model.name
- )
- # Remove the alias parameter value and use the old name
- # instead.
- params[self._original_name] = params.pop(self._alias_name)
- def alias_parameter_in_documentation(self, event_name, section, **kwargs):
- if event_name.startswith('docs.request-params'):
- if self._original_name not in section.available_sections:
- return
- # Replace the name for parameter type
- param_section = section.get_section(self._original_name)
- param_type_section = param_section.get_section('param-type')
- self._replace_content(param_type_section)
- # Replace the name for the parameter description
- param_name_section = param_section.get_section('param-name')
- self._replace_content(param_name_section)
- elif event_name.startswith('docs.request-example'):
- section = section.get_section('structure-value')
- if self._original_name not in section.available_sections:
- return
- # Replace the name for the example
- param_section = section.get_section(self._original_name)
- self._replace_content(param_section)
- def _replace_content(self, section):
- content = section.getvalue().decode('utf-8')
- updated_content = content.replace(
- self._original_name, self._alias_name)
- section.clear_text()
- section.write(updated_content)
- class ClientMethodAlias(object):
- def __init__(self, actual_name):
- """ Aliases a non-extant method to an existing method.
- :param actual_name: The name of the method that actually exists on
- the client.
- """
- self._actual = actual_name
- def __call__(self, client, **kwargs):
- return getattr(client, self._actual)
- # TODO: Remove this class as it is no longer used
- class HeaderToHostHoister(object):
- """Takes a header and moves it to the front of the hoststring.
- """
- _VALID_HOSTNAME = re.compile(r'(?!-)[a-z\d-]{1,63}(?<!-)$', re.IGNORECASE)
- def __init__(self, header_name):
- self._header_name = header_name
- def hoist(self, params, **kwargs):
- """Hoist a header to the hostname.
- Hoist a header to the beginning of the hostname with a suffix "." after
- it. The original header should be removed from the header map. This
- method is intended to be used as a target for the before-call event.
- """
- if self._header_name not in params['headers']:
- return
- header_value = params['headers'][self._header_name]
- self._ensure_header_is_valid_host(header_value)
- original_url = params['url']
- new_url = self._prepend_to_host(original_url, header_value)
- params['url'] = new_url
- def _ensure_header_is_valid_host(self, header):
- match = self._VALID_HOSTNAME.match(header)
- if not match:
- raise ParamValidationError(report=(
- 'Hostnames must contain only - and alphanumeric characters, '
- 'and between 1 and 63 characters long.'
- ))
- def _prepend_to_host(self, url, prefix):
- url_components = urlsplit(url)
- parts = url_components.netloc.split('.')
- parts = [prefix] + parts
- new_netloc = '.'.join(parts)
- new_components = (
- url_components.scheme,
- new_netloc,
- url_components.path,
- url_components.query,
- ''
- )
- new_url = urlunsplit(new_components)
- return new_url
- def inject_api_version_header_if_needed(model, params, **kwargs):
- if not model.is_endpoint_discovery_operation:
- return
- params['headers']['x-amz-api-version'] = model.service_model.api_version
- def remove_lex_v2_start_conversation(class_attributes, **kwargs):
- """Operation requires h2 which is currently unsupported in Python"""
- if 'start_conversation' in class_attributes:
- del class_attributes['start_conversation']
- # This is a list of (event_name, handler).
- # When a Session is created, everything in this list will be
- # automatically registered with that Session.
- BUILTIN_HANDLERS = [
- ('choose-service-name', handle_service_name_alias),
- ('getattr.mturk.list_hi_ts_for_qualification_type',
- ClientMethodAlias('list_hits_for_qualification_type')),
- ('before-parameter-build.s3.UploadPart',
- convert_body_to_file_like_object, REGISTER_LAST),
- ('before-parameter-build.s3.PutObject',
- convert_body_to_file_like_object, REGISTER_LAST),
- ('creating-client-class', add_generate_presigned_url),
- ('creating-client-class.s3', add_generate_presigned_post),
- ('creating-client-class.iot-data', check_openssl_supports_tls_version_1_2),
- ('creating-client-class.lex-runtime-v2', remove_lex_v2_start_conversation),
- ('after-call.iam', json_decode_policies),
- ('after-call.ec2.GetConsoleOutput', decode_console_output),
- ('after-call.cloudformation.GetTemplate', json_decode_template_body),
- ('after-call.s3.GetBucketLocation', parse_get_bucket_location),
- ('before-parameter-build', generate_idempotent_uuid),
- ('before-parameter-build.s3', validate_bucket_name),
- ('before-parameter-build.s3.ListObjects',
- set_list_objects_encoding_type_url),
- ('before-parameter-build.s3.ListObjectsV2',
- set_list_objects_encoding_type_url),
- ('before-parameter-build.s3.ListObjectVersions',
- set_list_objects_encoding_type_url),
- ('before-parameter-build.s3.CopyObject',
- handle_copy_source_param),
- ('before-parameter-build.s3.UploadPartCopy',
- handle_copy_source_param),
- ('before-parameter-build.s3.CopyObject', validate_ascii_metadata),
- ('before-parameter-build.s3.PutObject', validate_ascii_metadata),
- ('before-parameter-build.s3.CreateMultipartUpload',
- validate_ascii_metadata),
- ('docs.*.s3.CopyObject.complete-section', document_copy_source_form),
- ('docs.*.s3.UploadPartCopy.complete-section', document_copy_source_form),
- ('before-call.s3', add_expect_header),
- ('before-call.glacier', add_glacier_version),
- ('before-call.apigateway', add_accept_header),
- ('before-call.s3.PutObject', conditionally_calculate_md5),
- ('before-call.s3.UploadPart', conditionally_calculate_md5),
- ('before-call.s3.DeleteObjects', escape_xml_payload),
- ('before-call.s3.PutBucketLifecycleConfiguration', escape_xml_payload),
- ('before-call.glacier.UploadArchive', add_glacier_checksums),
- ('before-call.glacier.UploadMultipartPart', add_glacier_checksums),
- ('before-call.ec2.CopySnapshot', inject_presigned_url_ec2),
- ('request-created.machinelearning.Predict', switch_host_machinelearning),
- ('needs-retry.s3.UploadPartCopy', check_for_200_error, REGISTER_FIRST),
- ('needs-retry.s3.CopyObject', check_for_200_error, REGISTER_FIRST),
- ('needs-retry.s3.CompleteMultipartUpload', check_for_200_error,
- REGISTER_FIRST),
- ('choose-signer.cognito-identity.GetId', disable_signing),
- ('choose-signer.cognito-identity.GetOpenIdToken', disable_signing),
- ('choose-signer.cognito-identity.UnlinkIdentity', disable_signing),
- ('choose-signer.cognito-identity.GetCredentialsForIdentity',
- disable_signing),
- ('choose-signer.sts.AssumeRoleWithSAML', disable_signing),
- ('choose-signer.sts.AssumeRoleWithWebIdentity', disable_signing),
- ('choose-signer', set_operation_specific_signer),
- ('before-parameter-build.s3.HeadObject', sse_md5),
- ('before-parameter-build.s3.GetObject', sse_md5),
- ('before-parameter-build.s3.PutObject', sse_md5),
- ('before-parameter-build.s3.CopyObject', sse_md5),
- ('before-parameter-build.s3.CopyObject', copy_source_sse_md5),
- ('before-parameter-build.s3.CreateMultipartUpload', sse_md5),
- ('before-parameter-build.s3.UploadPart', sse_md5),
- ('before-parameter-build.s3.UploadPartCopy', sse_md5),
- ('before-parameter-build.s3.UploadPartCopy', copy_source_sse_md5),
- ('before-parameter-build.ec2.RunInstances', base64_encode_user_data),
- ('before-parameter-build.autoscaling.CreateLaunchConfiguration',
- base64_encode_user_data),
- ('before-parameter-build.route53', fix_route53_ids),
- ('before-parameter-build.glacier', inject_account_id),
- ('after-call.s3.ListObjects', decode_list_object),
- ('after-call.s3.ListObjectsV2', decode_list_object_v2),
- ('after-call.s3.ListObjectVersions', decode_list_object_versions),
- # Cloudsearchdomain search operation will be sent by HTTP POST
- ('request-created.cloudsearchdomain.Search',
- change_get_to_post),
- # Glacier documentation customizations
- ('docs.*.glacier.*.complete-section',
- AutoPopulatedParam('accountId', 'Note: this parameter is set to "-" by'
- 'default if no value is not specified.')
- .document_auto_populated_param),
- ('docs.*.glacier.UploadArchive.complete-section',
- AutoPopulatedParam('checksum').document_auto_populated_param),
- ('docs.*.glacier.UploadMultipartPart.complete-section',
- AutoPopulatedParam('checksum').document_auto_populated_param),
- ('docs.request-params.glacier.CompleteMultipartUpload.complete-section',
- document_glacier_tree_hash_checksum()),
- # Cloudformation documentation customizations
- ('docs.*.cloudformation.GetTemplate.complete-section',
- document_cloudformation_get_template_return_type),
- # UserData base64 encoding documentation customizations
- ('docs.*.ec2.RunInstances.complete-section',
- document_base64_encoding('UserData')),
- ('docs.*.autoscaling.CreateLaunchConfiguration.complete-section',
- document_base64_encoding('UserData')),
- # EC2 CopySnapshot documentation customizations
- ('docs.*.ec2.CopySnapshot.complete-section',
- AutoPopulatedParam('PresignedUrl').document_auto_populated_param),
- ('docs.*.ec2.CopySnapshot.complete-section',
- AutoPopulatedParam('DestinationRegion').document_auto_populated_param),
- # S3 SSE documentation modifications
- ('docs.*.s3.*.complete-section',
- AutoPopulatedParam('SSECustomerKeyMD5').document_auto_populated_param),
- # S3 SSE Copy Source documentation modifications
- ('docs.*.s3.*.complete-section',
- AutoPopulatedParam(
- 'CopySourceSSECustomerKeyMD5').document_auto_populated_param),
- # Add base64 information to Lambda
- ('docs.*.lambda.UpdateFunctionCode.complete-section',
- document_base64_encoding('ZipFile')),
- # The following S3 operations cannot actually accept a ContentMD5
- ('docs.*.s3.*.complete-section',
- HideParamFromOperations(
- 's3', 'ContentMD5',
- ['DeleteObjects', 'PutBucketAcl', 'PutBucketCors',
- 'PutBucketLifecycle', 'PutBucketLogging', 'PutBucketNotification',
- 'PutBucketPolicy', 'PutBucketReplication', 'PutBucketRequestPayment',
- 'PutBucketTagging', 'PutBucketVersioning', 'PutBucketWebsite',
- 'PutObjectAcl']).hide_param),
- #############
- # RDS
- #############
- ('creating-client-class.rds', add_generate_db_auth_token),
- ('before-call.rds.CopyDBClusterSnapshot',
- inject_presigned_url_rds),
- ('before-call.rds.CreateDBCluster',
- inject_presigned_url_rds),
- ('before-call.rds.CopyDBSnapshot',
- inject_presigned_url_rds),
- ('before-call.rds.CreateDBInstanceReadReplica',
- inject_presigned_url_rds),
- ('before-call.rds.StartDBInstanceAutomatedBackupsReplication',
- inject_presigned_url_rds),
- # RDS PresignedUrl documentation customizations
- ('docs.*.rds.CopyDBClusterSnapshot.complete-section',
- AutoPopulatedParam('PreSignedUrl').document_auto_populated_param),
- ('docs.*.rds.CreateDBCluster.complete-section',
- AutoPopulatedParam('PreSignedUrl').document_auto_populated_param),
- ('docs.*.rds.CopyDBSnapshot.complete-section',
- AutoPopulatedParam('PreSignedUrl').document_auto_populated_param),
- ('docs.*.rds.CreateDBInstanceReadReplica.complete-section',
- AutoPopulatedParam('PreSignedUrl').document_auto_populated_param),
- ('docs.*.rds.StartDBInstanceAutomatedBackupsReplication.complete-section',
- AutoPopulatedParam('PreSignedUrl').document_auto_populated_param),
- #############
- # Neptune
- #############
- ('before-call.neptune.CopyDBClusterSnapshot',
- inject_presigned_url_rds),
- ('before-call.neptune.CreateDBCluster',
- inject_presigned_url_rds),
- # Neptune PresignedUrl documentation customizations
- ('docs.*.neptune.CopyDBClusterSnapshot.complete-section',
- AutoPopulatedParam('PreSignedUrl').document_auto_populated_param),
- ('docs.*.neptune.CreateDBCluster.complete-section',
- AutoPopulatedParam('PreSignedUrl').document_auto_populated_param),
- #############
- # DocDB
- #############
- ('before-call.docdb.CopyDBClusterSnapshot',
- inject_presigned_url_rds),
- ('before-call.docdb.CreateDBCluster',
- inject_presigned_url_rds),
- # DocDB PresignedUrl documentation customizations
- ('docs.*.docdb.CopyDBClusterSnapshot.complete-section',
- AutoPopulatedParam('PreSignedUrl').document_auto_populated_param),
- ('docs.*.docdb.CreateDBCluster.complete-section',
- AutoPopulatedParam('PreSignedUrl').document_auto_populated_param),
- ###########
- # SMS Voice
- ##########
- ('docs.title.sms-voice',
- DeprecatedServiceDocumenter(
- 'pinpoint-sms-voice').inject_deprecation_notice),
- ('before-call', inject_api_version_header_if_needed),
- ]
- _add_parameter_aliases(BUILTIN_HANDLERS)
|