utils.py 89 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365
  1. # Copyright 2012-2014 Amazon.com, Inc. or its affiliates. All Rights Reserved.
  2. #
  3. # Licensed under the Apache License, Version 2.0 (the "License"). You
  4. # may not use this file except in compliance with the License. A copy of
  5. # the License is located at
  6. #
  7. # http://aws.amazon.com/apache2.0/
  8. #
  9. # or in the "license" file accompanying this file. This file is
  10. # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
  11. # ANY KIND, either express or implied. See the License for the specific
  12. # language governing permissions and limitations under the License.
  13. import base64
  14. import re
  15. import time
  16. import logging
  17. import datetime
  18. import hashlib
  19. import binascii
  20. import functools
  21. import weakref
  22. import random
  23. import os
  24. import socket
  25. import cgi
  26. import warnings
  27. import dateutil.parser
  28. from dateutil.tz import tzutc
  29. import botocore
  30. import botocore.awsrequest
  31. import botocore.httpsession
  32. from botocore.compat import (
  33. json, quote, zip_longest, urlsplit, urlunsplit, OrderedDict,
  34. six, urlparse, get_tzinfo_options, get_md5, MD5_AVAILABLE
  35. )
  36. from botocore.vendored.six.moves.urllib.request import getproxies, proxy_bypass
  37. from botocore.exceptions import (
  38. InvalidExpressionError, ConfigNotFound, InvalidDNSNameError, ClientError,
  39. MetadataRetrievalError, EndpointConnectionError, ReadTimeoutError,
  40. ConnectionClosedError, ConnectTimeoutError, UnsupportedS3ArnError,
  41. UnsupportedS3AccesspointConfigurationError, SSOTokenLoadError,
  42. InvalidRegionError, InvalidIMDSEndpointError, UnsupportedOutpostResourceError,
  43. UnsupportedS3ControlConfigurationError, UnsupportedS3ControlArnError,
  44. InvalidHostLabelError, HTTPClientError, UnsupportedS3ConfigurationError,
  45. )
  46. from urllib3.exceptions import LocationParseError
  47. logger = logging.getLogger(__name__)
  48. DEFAULT_METADATA_SERVICE_TIMEOUT = 1
  49. METADATA_BASE_URL = 'http://169.254.169.254/'
  50. METADATA_BASE_URL_IPv6 = 'http://[fe80:ec2::254%eth0]/'
  51. # These are chars that do not need to be urlencoded.
  52. # Based on rfc2986, section 2.3
  53. SAFE_CHARS = '-._~'
  54. LABEL_RE = re.compile(r'[a-z0-9][a-z0-9\-]*[a-z0-9]')
  55. RETRYABLE_HTTP_ERRORS = (
  56. ReadTimeoutError, EndpointConnectionError, ConnectionClosedError,
  57. ConnectTimeoutError,
  58. )
  59. S3_ACCELERATE_WHITELIST = ['dualstack']
  60. # In switching events from using service name / endpoint prefix to service
  61. # id, we have to preserve compatibility. This maps the instances where either
  62. # is different than the transformed service id.
  63. EVENT_ALIASES = {
  64. "a4b": "alexa-for-business",
  65. "alexaforbusiness": "alexa-for-business",
  66. "api.mediatailor": "mediatailor",
  67. "api.pricing": "pricing",
  68. "api.sagemaker": "sagemaker",
  69. "apigateway": "api-gateway",
  70. "application-autoscaling": "application-auto-scaling",
  71. "appstream2": "appstream",
  72. "autoscaling": "auto-scaling",
  73. "autoscaling-plans": "auto-scaling-plans",
  74. "ce": "cost-explorer",
  75. "cloudhsmv2": "cloudhsm-v2",
  76. "cloudsearchdomain": "cloudsearch-domain",
  77. "cognito-idp": "cognito-identity-provider",
  78. "config": "config-service",
  79. "cur": "cost-and-usage-report-service",
  80. "data.iot": "iot-data-plane",
  81. "data.jobs.iot": "iot-jobs-data-plane",
  82. "data.mediastore": "mediastore-data",
  83. "datapipeline": "data-pipeline",
  84. "devicefarm": "device-farm",
  85. "devices.iot1click": "iot-1click-devices-service",
  86. "directconnect": "direct-connect",
  87. "discovery": "application-discovery-service",
  88. "dms": "database-migration-service",
  89. "ds": "directory-service",
  90. "dynamodbstreams": "dynamodb-streams",
  91. "elasticbeanstalk": "elastic-beanstalk",
  92. "elasticfilesystem": "efs",
  93. "elasticloadbalancing": "elastic-load-balancing",
  94. "elasticmapreduce": "emr",
  95. "elastictranscoder": "elastic-transcoder",
  96. "elb": "elastic-load-balancing",
  97. "elbv2": "elastic-load-balancing-v2",
  98. "email": "ses",
  99. "entitlement.marketplace": "marketplace-entitlement-service",
  100. "es": "elasticsearch-service",
  101. "events": "eventbridge",
  102. "cloudwatch-events": "eventbridge",
  103. "iot-data": "iot-data-plane",
  104. "iot-jobs-data": "iot-jobs-data-plane",
  105. "iot1click-devices": "iot-1click-devices-service",
  106. "iot1click-projects": "iot-1click-projects",
  107. "kinesisanalytics": "kinesis-analytics",
  108. "kinesisvideo": "kinesis-video",
  109. "lex-models": "lex-model-building-service",
  110. "lex-runtime": "lex-runtime-service",
  111. "logs": "cloudwatch-logs",
  112. "machinelearning": "machine-learning",
  113. "marketplace-entitlement": "marketplace-entitlement-service",
  114. "marketplacecommerceanalytics": "marketplace-commerce-analytics",
  115. "metering.marketplace": "marketplace-metering",
  116. "meteringmarketplace": "marketplace-metering",
  117. "mgh": "migration-hub",
  118. "models.lex": "lex-model-building-service",
  119. "monitoring": "cloudwatch",
  120. "mturk-requester": "mturk",
  121. "opsworks-cm": "opsworkscm",
  122. "projects.iot1click": "iot-1click-projects",
  123. "resourcegroupstaggingapi": "resource-groups-tagging-api",
  124. "route53": "route-53",
  125. "route53domains": "route-53-domains",
  126. "runtime.lex": "lex-runtime-service",
  127. "runtime.sagemaker": "sagemaker-runtime",
  128. "sdb": "simpledb",
  129. "secretsmanager": "secrets-manager",
  130. "serverlessrepo": "serverlessapplicationrepository",
  131. "servicecatalog": "service-catalog",
  132. "states": "sfn",
  133. "stepfunctions": "sfn",
  134. "storagegateway": "storage-gateway",
  135. "streams.dynamodb": "dynamodb-streams",
  136. "tagging": "resource-groups-tagging-api"
  137. }
  138. # Vendoring IPv6 validation regex patterns from urllib3
  139. # https://github.com/urllib3/urllib3/blob/7e856c0/src/urllib3/util/url.py
  140. IPV4_PAT = r"(?:[0-9]{1,3}\.){3}[0-9]{1,3}"
  141. HEX_PAT = "[0-9A-Fa-f]{1,4}"
  142. LS32_PAT = "(?:{hex}:{hex}|{ipv4})".format(hex=HEX_PAT, ipv4=IPV4_PAT)
  143. _subs = {"hex": HEX_PAT, "ls32": LS32_PAT}
  144. _variations = [
  145. # 6( h16 ":" ) ls32
  146. "(?:%(hex)s:){6}%(ls32)s",
  147. # "::" 5( h16 ":" ) ls32
  148. "::(?:%(hex)s:){5}%(ls32)s",
  149. # [ h16 ] "::" 4( h16 ":" ) ls32
  150. "(?:%(hex)s)?::(?:%(hex)s:){4}%(ls32)s",
  151. # [ *1( h16 ":" ) h16 ] "::" 3( h16 ":" ) ls32
  152. "(?:(?:%(hex)s:)?%(hex)s)?::(?:%(hex)s:){3}%(ls32)s",
  153. # [ *2( h16 ":" ) h16 ] "::" 2( h16 ":" ) ls32
  154. "(?:(?:%(hex)s:){0,2}%(hex)s)?::(?:%(hex)s:){2}%(ls32)s",
  155. # [ *3( h16 ":" ) h16 ] "::" h16 ":" ls32
  156. "(?:(?:%(hex)s:){0,3}%(hex)s)?::%(hex)s:%(ls32)s",
  157. # [ *4( h16 ":" ) h16 ] "::" ls32
  158. "(?:(?:%(hex)s:){0,4}%(hex)s)?::%(ls32)s",
  159. # [ *5( h16 ":" ) h16 ] "::" h16
  160. "(?:(?:%(hex)s:){0,5}%(hex)s)?::%(hex)s",
  161. # [ *6( h16 ":" ) h16 ] "::"
  162. "(?:(?:%(hex)s:){0,6}%(hex)s)?::",
  163. ]
  164. UNRESERVED_PAT = r"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789._!\-~"
  165. IPV6_PAT = "(?:" + "|".join([x % _subs for x in _variations]) + ")"
  166. ZONE_ID_PAT = "(?:%25|%)(?:[" + UNRESERVED_PAT + "]|%[a-fA-F0-9]{2})+"
  167. IPV6_ADDRZ_PAT = r"\[" + IPV6_PAT + r"(?:" + ZONE_ID_PAT + r")?\]"
  168. IPV6_ADDRZ_RE = re.compile("^" + IPV6_ADDRZ_PAT + "$")
  169. # These are the characters that are stripped by post-bpo-43882 urlparse().
  170. UNSAFE_URL_CHARS = frozenset('\t\r\n')
  171. def ensure_boolean(val):
  172. """Ensures a boolean value if a string or boolean is provided
  173. For strings, the value for True/False is case insensitive
  174. """
  175. if isinstance(val, bool):
  176. return val
  177. else:
  178. return val.lower() == 'true'
  179. def is_json_value_header(shape):
  180. """Determines if the provided shape is the special header type jsonvalue.
  181. :type shape: botocore.shape
  182. :param shape: Shape to be inspected for the jsonvalue trait.
  183. :return: True if this type is a jsonvalue, False otherwise
  184. :rtype: Bool
  185. """
  186. return (hasattr(shape, 'serialization') and
  187. shape.serialization.get('jsonvalue', False) and
  188. shape.serialization.get('location') == 'header' and
  189. shape.type_name == 'string')
  190. def get_service_module_name(service_model):
  191. """Returns the module name for a service
  192. This is the value used in both the documentation and client class name
  193. """
  194. name = service_model.metadata.get(
  195. 'serviceAbbreviation',
  196. service_model.metadata.get(
  197. 'serviceFullName', service_model.service_name))
  198. name = name.replace('Amazon', '')
  199. name = name.replace('AWS', '')
  200. name = re.sub(r'\W+', '', name)
  201. return name
  202. def normalize_url_path(path):
  203. if not path:
  204. return '/'
  205. return remove_dot_segments(path)
  206. def normalize_boolean(val):
  207. """Returns None if val is None, otherwise ensure value
  208. converted to boolean"""
  209. if val is None:
  210. return val
  211. else:
  212. return ensure_boolean(val)
  213. def remove_dot_segments(url):
  214. # RFC 3986, section 5.2.4 "Remove Dot Segments"
  215. # Also, AWS services require consecutive slashes to be removed,
  216. # so that's done here as well
  217. if not url:
  218. return ''
  219. input_url = url.split('/')
  220. output_list = []
  221. for x in input_url:
  222. if x and x != '.':
  223. if x == '..':
  224. if output_list:
  225. output_list.pop()
  226. else:
  227. output_list.append(x)
  228. if url[0] == '/':
  229. first = '/'
  230. else:
  231. first = ''
  232. if url[-1] == '/' and output_list:
  233. last = '/'
  234. else:
  235. last = ''
  236. return first + '/'.join(output_list) + last
  237. def validate_jmespath_for_set(expression):
  238. # Validates a limited jmespath expression to determine if we can set a
  239. # value based on it. Only works with dotted paths.
  240. if not expression or expression == '.':
  241. raise InvalidExpressionError(expression=expression)
  242. for invalid in ['[', ']', '*']:
  243. if invalid in expression:
  244. raise InvalidExpressionError(expression=expression)
  245. def set_value_from_jmespath(source, expression, value, is_first=True):
  246. # This takes a (limited) jmespath-like expression & can set a value based
  247. # on it.
  248. # Limitations:
  249. # * Only handles dotted lookups
  250. # * No offsets/wildcards/slices/etc.
  251. if is_first:
  252. validate_jmespath_for_set(expression)
  253. bits = expression.split('.', 1)
  254. current_key, remainder = bits[0], bits[1] if len(bits) > 1 else ''
  255. if not current_key:
  256. raise InvalidExpressionError(expression=expression)
  257. if remainder:
  258. if current_key not in source:
  259. # We've got something in the expression that's not present in the
  260. # source (new key). If there's any more bits, we'll set the key
  261. # with an empty dictionary.
  262. source[current_key] = {}
  263. return set_value_from_jmespath(
  264. source[current_key],
  265. remainder,
  266. value,
  267. is_first=False
  268. )
  269. # If we're down to a single key, set it.
  270. source[current_key] = value
  271. class _RetriesExceededError(Exception):
  272. """Internal exception used when the number of retries are exceeded."""
  273. pass
  274. class BadIMDSRequestError(Exception):
  275. def __init__(self, request):
  276. self.request = request
  277. class IMDSFetcher(object):
  278. _RETRIES_EXCEEDED_ERROR_CLS = _RetriesExceededError
  279. _TOKEN_PATH = 'latest/api/token'
  280. _TOKEN_TTL = '21600'
  281. def __init__(self, timeout=DEFAULT_METADATA_SERVICE_TIMEOUT,
  282. num_attempts=1, base_url=METADATA_BASE_URL,
  283. env=None, user_agent=None, config=None):
  284. self._timeout = timeout
  285. self._num_attempts = num_attempts
  286. self._base_url = self._select_base_url(base_url, config)
  287. if env is None:
  288. env = os.environ.copy()
  289. self._disabled = env.get('AWS_EC2_METADATA_DISABLED', 'false').lower()
  290. self._disabled = self._disabled == 'true'
  291. self._user_agent = user_agent
  292. self._session = botocore.httpsession.URLLib3Session(
  293. timeout=self._timeout,
  294. proxies=get_environ_proxies(self._base_url),
  295. )
  296. def get_base_url(self):
  297. return self._base_url
  298. def _select_base_url(self, base_url, config):
  299. if config is None:
  300. config = {}
  301. requires_ipv6 = ensure_boolean(config.get('imds_use_ipv6', False))
  302. custom_metadata_endpoint = config.get('ec2_metadata_service_endpoint')
  303. if requires_ipv6 and custom_metadata_endpoint:
  304. logger.warn("Custom endpoint and IMDS_USE_IPV6 are both set. Using custom endpoint.")
  305. chosen_base_url = None
  306. if base_url != METADATA_BASE_URL:
  307. chosen_base_url = base_url
  308. elif custom_metadata_endpoint:
  309. chosen_base_url = custom_metadata_endpoint
  310. elif requires_ipv6:
  311. chosen_base_url = METADATA_BASE_URL_IPv6
  312. else:
  313. chosen_base_url = METADATA_BASE_URL
  314. logger.debug("IMDS ENDPOINT: %s" % chosen_base_url)
  315. if not is_valid_uri(chosen_base_url):
  316. raise InvalidIMDSEndpointError(endpoint=chosen_base_url)
  317. return chosen_base_url
  318. def _fetch_metadata_token(self):
  319. self._assert_enabled()
  320. url = self._base_url + self._TOKEN_PATH
  321. headers = {
  322. 'x-aws-ec2-metadata-token-ttl-seconds': self._TOKEN_TTL,
  323. }
  324. self._add_user_agent(headers)
  325. request = botocore.awsrequest.AWSRequest(
  326. method='PUT', url=url, headers=headers)
  327. for i in range(self._num_attempts):
  328. try:
  329. response = self._session.send(request.prepare())
  330. if response.status_code == 200:
  331. return response.text
  332. elif response.status_code in (404, 403, 405):
  333. return None
  334. elif response.status_code in (400,):
  335. raise BadIMDSRequestError(request)
  336. except ReadTimeoutError:
  337. return None
  338. except RETRYABLE_HTTP_ERRORS as e:
  339. logger.debug(
  340. "Caught retryable HTTP exception while making metadata "
  341. "service request to %s: %s", url, e, exc_info=True)
  342. except HTTPClientError as e:
  343. if isinstance(e.kwargs.get('error'), LocationParseError):
  344. raise InvalidIMDSEndpointError(endpoint=url, error=e)
  345. else:
  346. raise
  347. return None
  348. def _get_request(self, url_path, retry_func, token=None):
  349. """Make a get request to the Instance Metadata Service.
  350. :type url_path: str
  351. :param url_path: The path component of the URL to make a get request.
  352. This arg is appended to the base_url that was provided in the
  353. initializer.
  354. :type retry_func: callable
  355. :param retry_func: A function that takes the response as an argument
  356. and determines if it needs to retry. By default empty and non
  357. 200 OK responses are retried.
  358. :type token: str
  359. :param token: Metadata token to send along with GET requests to IMDS.
  360. """
  361. self._assert_enabled()
  362. if retry_func is None:
  363. retry_func = self._default_retry
  364. url = self._base_url + url_path
  365. headers = {}
  366. if token is not None:
  367. headers['x-aws-ec2-metadata-token'] = token
  368. self._add_user_agent(headers)
  369. for i in range(self._num_attempts):
  370. try:
  371. request = botocore.awsrequest.AWSRequest(
  372. method='GET', url=url, headers=headers)
  373. response = self._session.send(request.prepare())
  374. if not retry_func(response):
  375. return response
  376. except RETRYABLE_HTTP_ERRORS as e:
  377. logger.debug(
  378. "Caught retryable HTTP exception while making metadata "
  379. "service request to %s: %s", url, e, exc_info=True)
  380. raise self._RETRIES_EXCEEDED_ERROR_CLS()
  381. def _add_user_agent(self, headers):
  382. if self._user_agent is not None:
  383. headers['User-Agent'] = self._user_agent
  384. def _assert_enabled(self):
  385. if self._disabled:
  386. logger.debug("Access to EC2 metadata has been disabled.")
  387. raise self._RETRIES_EXCEEDED_ERROR_CLS()
  388. def _default_retry(self, response):
  389. return (
  390. self._is_non_ok_response(response) or
  391. self._is_empty(response)
  392. )
  393. def _is_non_ok_response(self, response):
  394. if response.status_code != 200:
  395. self._log_imds_response(response, 'non-200', log_body=True)
  396. return True
  397. return False
  398. def _is_empty(self, response):
  399. if not response.content:
  400. self._log_imds_response(response, 'no body', log_body=True)
  401. return True
  402. return False
  403. def _log_imds_response(self, response, reason_to_log, log_body=False):
  404. statement = (
  405. "Metadata service returned %s response "
  406. "with status code of %s for url: %s"
  407. )
  408. logger_args = [
  409. reason_to_log, response.status_code, response.url
  410. ]
  411. if log_body:
  412. statement += ", content body: %s"
  413. logger_args.append(response.content)
  414. logger.debug(statement, *logger_args)
  415. class InstanceMetadataFetcher(IMDSFetcher):
  416. _URL_PATH = 'latest/meta-data/iam/security-credentials/'
  417. _REQUIRED_CREDENTIAL_FIELDS = [
  418. 'AccessKeyId', 'SecretAccessKey', 'Token', 'Expiration'
  419. ]
  420. def retrieve_iam_role_credentials(self):
  421. try:
  422. token = self._fetch_metadata_token()
  423. role_name = self._get_iam_role(token)
  424. credentials = self._get_credentials(role_name, token)
  425. if self._contains_all_credential_fields(credentials):
  426. return {
  427. 'role_name': role_name,
  428. 'access_key': credentials['AccessKeyId'],
  429. 'secret_key': credentials['SecretAccessKey'],
  430. 'token': credentials['Token'],
  431. 'expiry_time': credentials['Expiration'],
  432. }
  433. else:
  434. # IMDS can return a 200 response that has a JSON formatted
  435. # error message (i.e. if ec2 is not trusted entity for the
  436. # attached role). We do not necessarily want to retry for
  437. # these and we also do not necessarily want to raise a key
  438. # error. So at least log the problematic response and return
  439. # an empty dictionary to signal that it was not able to
  440. # retrieve credentials. These error will contain both a
  441. # Code and Message key.
  442. if 'Code' in credentials and 'Message' in credentials:
  443. logger.debug('Error response received when retrieving'
  444. 'credentials: %s.', credentials)
  445. return {}
  446. except self._RETRIES_EXCEEDED_ERROR_CLS:
  447. logger.debug("Max number of attempts exceeded (%s) when "
  448. "attempting to retrieve data from metadata service.",
  449. self._num_attempts)
  450. except BadIMDSRequestError as e:
  451. logger.debug("Bad IMDS request: %s", e.request)
  452. return {}
  453. def _get_iam_role(self, token=None):
  454. return self._get_request(
  455. url_path=self._URL_PATH,
  456. retry_func=self._needs_retry_for_role_name,
  457. token=token,
  458. ).text
  459. def _get_credentials(self, role_name, token=None):
  460. r = self._get_request(
  461. url_path=self._URL_PATH + role_name,
  462. retry_func=self._needs_retry_for_credentials,
  463. token=token,
  464. )
  465. return json.loads(r.text)
  466. def _is_invalid_json(self, response):
  467. try:
  468. json.loads(response.text)
  469. return False
  470. except ValueError:
  471. self._log_imds_response(response, 'invalid json')
  472. return True
  473. def _needs_retry_for_role_name(self, response):
  474. return (
  475. self._is_non_ok_response(response) or
  476. self._is_empty(response)
  477. )
  478. def _needs_retry_for_credentials(self, response):
  479. return (
  480. self._is_non_ok_response(response) or
  481. self._is_empty(response) or
  482. self._is_invalid_json(response)
  483. )
  484. def _contains_all_credential_fields(self, credentials):
  485. for field in self._REQUIRED_CREDENTIAL_FIELDS:
  486. if field not in credentials:
  487. logger.debug(
  488. 'Retrieved credentials is missing required field: %s',
  489. field)
  490. return False
  491. return True
  492. def merge_dicts(dict1, dict2, append_lists=False):
  493. """Given two dict, merge the second dict into the first.
  494. The dicts can have arbitrary nesting.
  495. :param append_lists: If true, instead of clobbering a list with the new
  496. value, append all of the new values onto the original list.
  497. """
  498. for key in dict2:
  499. if isinstance(dict2[key], dict):
  500. if key in dict1 and key in dict2:
  501. merge_dicts(dict1[key], dict2[key])
  502. else:
  503. dict1[key] = dict2[key]
  504. # If the value is a list and the ``append_lists`` flag is set,
  505. # append the new values onto the original list
  506. elif isinstance(dict2[key], list) and append_lists:
  507. # The value in dict1 must be a list in order to append new
  508. # values onto it.
  509. if key in dict1 and isinstance(dict1[key], list):
  510. dict1[key].extend(dict2[key])
  511. else:
  512. dict1[key] = dict2[key]
  513. else:
  514. # At scalar types, we iterate and merge the
  515. # current dict that we're on.
  516. dict1[key] = dict2[key]
  517. def lowercase_dict(original):
  518. """Copies the given dictionary ensuring all keys are lowercase strings. """
  519. copy = {}
  520. for key in original:
  521. copy[key.lower()] = original[key]
  522. return copy
  523. def parse_key_val_file(filename, _open=open):
  524. try:
  525. with _open(filename) as f:
  526. contents = f.read()
  527. return parse_key_val_file_contents(contents)
  528. except OSError:
  529. raise ConfigNotFound(path=filename)
  530. def parse_key_val_file_contents(contents):
  531. # This was originally extracted from the EC2 credential provider, which was
  532. # fairly lenient in its parsing. We only try to parse key/val pairs if
  533. # there's a '=' in the line.
  534. final = {}
  535. for line in contents.splitlines():
  536. if '=' not in line:
  537. continue
  538. key, val = line.split('=', 1)
  539. key = key.strip()
  540. val = val.strip()
  541. final[key] = val
  542. return final
  543. def percent_encode_sequence(mapping, safe=SAFE_CHARS):
  544. """Urlencode a dict or list into a string.
  545. This is similar to urllib.urlencode except that:
  546. * It uses quote, and not quote_plus
  547. * It has a default list of safe chars that don't need
  548. to be encoded, which matches what AWS services expect.
  549. If any value in the input ``mapping`` is a list type,
  550. then each list element wil be serialized. This is the equivalent
  551. to ``urlencode``'s ``doseq=True`` argument.
  552. This function should be preferred over the stdlib
  553. ``urlencode()`` function.
  554. :param mapping: Either a dict to urlencode or a list of
  555. ``(key, value)`` pairs.
  556. """
  557. encoded_pairs = []
  558. if hasattr(mapping, 'items'):
  559. pairs = mapping.items()
  560. else:
  561. pairs = mapping
  562. for key, value in pairs:
  563. if isinstance(value, list):
  564. for element in value:
  565. encoded_pairs.append('%s=%s' % (percent_encode(key),
  566. percent_encode(element)))
  567. else:
  568. encoded_pairs.append('%s=%s' % (percent_encode(key),
  569. percent_encode(value)))
  570. return '&'.join(encoded_pairs)
  571. def percent_encode(input_str, safe=SAFE_CHARS):
  572. """Urlencodes a string.
  573. Whereas percent_encode_sequence handles taking a dict/sequence and
  574. producing a percent encoded string, this function deals only with
  575. taking a string (not a dict/sequence) and percent encoding it.
  576. If given the binary type, will simply URL encode it. If given the
  577. text type, will produce the binary type by UTF-8 encoding the
  578. text. If given something else, will convert it to the text type
  579. first.
  580. """
  581. # If its not a binary or text string, make it a text string.
  582. if not isinstance(input_str, (six.binary_type, six.text_type)):
  583. input_str = six.text_type(input_str)
  584. # If it's not bytes, make it bytes by UTF-8 encoding it.
  585. if not isinstance(input_str, six.binary_type):
  586. input_str = input_str.encode('utf-8')
  587. return quote(input_str, safe=safe)
  588. def _parse_timestamp_with_tzinfo(value, tzinfo):
  589. """Parse timestamp with pluggable tzinfo options."""
  590. if isinstance(value, (int, float)):
  591. # Possibly an epoch time.
  592. return datetime.datetime.fromtimestamp(value, tzinfo())
  593. else:
  594. try:
  595. return datetime.datetime.fromtimestamp(float(value), tzinfo())
  596. except (TypeError, ValueError):
  597. pass
  598. try:
  599. # In certain cases, a timestamp marked with GMT can be parsed into a
  600. # different time zone, so here we provide a context which will
  601. # enforce that GMT == UTC.
  602. return dateutil.parser.parse(value, tzinfos={'GMT': tzutc()})
  603. except (TypeError, ValueError) as e:
  604. raise ValueError('Invalid timestamp "%s": %s' % (value, e))
  605. def parse_timestamp(value):
  606. """Parse a timestamp into a datetime object.
  607. Supported formats:
  608. * iso8601
  609. * rfc822
  610. * epoch (value is an integer)
  611. This will return a ``datetime.datetime`` object.
  612. """
  613. for tzinfo in get_tzinfo_options():
  614. try:
  615. return _parse_timestamp_with_tzinfo(value, tzinfo)
  616. except OSError as e:
  617. logger.debug('Unable to parse timestamp with "%s" timezone info.',
  618. tzinfo.__name__, exc_info=e)
  619. raise RuntimeError('Unable to calculate correct timezone offset for '
  620. '"%s"' % value)
  621. def parse_to_aware_datetime(value):
  622. """Converted the passed in value to a datetime object with tzinfo.
  623. This function can be used to normalize all timestamp inputs. This
  624. function accepts a number of different types of inputs, but
  625. will always return a datetime.datetime object with time zone
  626. information.
  627. The input param ``value`` can be one of several types:
  628. * A datetime object (both naive and aware)
  629. * An integer representing the epoch time (can also be a string
  630. of the integer, i.e '0', instead of 0). The epoch time is
  631. considered to be UTC.
  632. * An iso8601 formatted timestamp. This does not need to be
  633. a complete timestamp, it can contain just the date portion
  634. without the time component.
  635. The returned value will be a datetime object that will have tzinfo.
  636. If no timezone info was provided in the input value, then UTC is
  637. assumed, not local time.
  638. """
  639. # This is a general purpose method that handles several cases of
  640. # converting the provided value to a string timestamp suitable to be
  641. # serialized to an http request. It can handle:
  642. # 1) A datetime.datetime object.
  643. if isinstance(value, datetime.datetime):
  644. datetime_obj = value
  645. else:
  646. # 2) A string object that's formatted as a timestamp.
  647. # We document this as being an iso8601 timestamp, although
  648. # parse_timestamp is a bit more flexible.
  649. datetime_obj = parse_timestamp(value)
  650. if datetime_obj.tzinfo is None:
  651. # I think a case would be made that if no time zone is provided,
  652. # we should use the local time. However, to restore backwards
  653. # compat, the previous behavior was to assume UTC, which is
  654. # what we're going to do here.
  655. datetime_obj = datetime_obj.replace(tzinfo=tzutc())
  656. else:
  657. datetime_obj = datetime_obj.astimezone(tzutc())
  658. return datetime_obj
  659. def datetime2timestamp(dt, default_timezone=None):
  660. """Calculate the timestamp based on the given datetime instance.
  661. :type dt: datetime
  662. :param dt: A datetime object to be converted into timestamp
  663. :type default_timezone: tzinfo
  664. :param default_timezone: If it is provided as None, we treat it as tzutc().
  665. But it is only used when dt is a naive datetime.
  666. :returns: The timestamp
  667. """
  668. epoch = datetime.datetime(1970, 1, 1)
  669. if dt.tzinfo is None:
  670. if default_timezone is None:
  671. default_timezone = tzutc()
  672. dt = dt.replace(tzinfo=default_timezone)
  673. d = dt.replace(tzinfo=None) - dt.utcoffset() - epoch
  674. if hasattr(d, "total_seconds"):
  675. return d.total_seconds() # Works in Python 2.7+
  676. return (d.microseconds + (d.seconds + d.days * 24 * 3600) * 10**6) / 10**6
  677. def calculate_sha256(body, as_hex=False):
  678. """Calculate a sha256 checksum.
  679. This method will calculate the sha256 checksum of a file like
  680. object. Note that this method will iterate through the entire
  681. file contents. The caller is responsible for ensuring the proper
  682. starting position of the file and ``seek()``'ing the file back
  683. to its starting location if other consumers need to read from
  684. the file like object.
  685. :param body: Any file like object. The file must be opened
  686. in binary mode such that a ``.read()`` call returns bytes.
  687. :param as_hex: If True, then the hex digest is returned.
  688. If False, then the digest (as binary bytes) is returned.
  689. :returns: The sha256 checksum
  690. """
  691. checksum = hashlib.sha256()
  692. for chunk in iter(lambda: body.read(1024 * 1024), b''):
  693. checksum.update(chunk)
  694. if as_hex:
  695. return checksum.hexdigest()
  696. else:
  697. return checksum.digest()
  698. def calculate_tree_hash(body):
  699. """Calculate a tree hash checksum.
  700. For more information see:
  701. http://docs.aws.amazon.com/amazonglacier/latest/dev/checksum-calculations.html
  702. :param body: Any file like object. This has the same constraints as
  703. the ``body`` param in calculate_sha256
  704. :rtype: str
  705. :returns: The hex version of the calculated tree hash
  706. """
  707. chunks = []
  708. required_chunk_size = 1024 * 1024
  709. sha256 = hashlib.sha256
  710. for chunk in iter(lambda: body.read(required_chunk_size), b''):
  711. chunks.append(sha256(chunk).digest())
  712. if not chunks:
  713. return sha256(b'').hexdigest()
  714. while len(chunks) > 1:
  715. new_chunks = []
  716. for first, second in _in_pairs(chunks):
  717. if second is not None:
  718. new_chunks.append(sha256(first + second).digest())
  719. else:
  720. # We're at the end of the list and there's no pair left.
  721. new_chunks.append(first)
  722. chunks = new_chunks
  723. return binascii.hexlify(chunks[0]).decode('ascii')
  724. def _in_pairs(iterable):
  725. # Creates iterator that iterates over the list in pairs:
  726. # for a, b in _in_pairs([0, 1, 2, 3, 4]):
  727. # print(a, b)
  728. #
  729. # will print:
  730. # 0, 1
  731. # 2, 3
  732. # 4, None
  733. shared_iter = iter(iterable)
  734. # Note that zip_longest is a compat import that uses
  735. # the itertools izip_longest. This creates an iterator,
  736. # this call below does _not_ immediately create the list
  737. # of pairs.
  738. return zip_longest(shared_iter, shared_iter)
  739. class CachedProperty(object):
  740. """A read only property that caches the initially computed value.
  741. This descriptor will only call the provided ``fget`` function once.
  742. Subsequent access to this property will return the cached value.
  743. """
  744. def __init__(self, fget):
  745. self._fget = fget
  746. def __get__(self, obj, cls):
  747. if obj is None:
  748. return self
  749. else:
  750. computed_value = self._fget(obj)
  751. obj.__dict__[self._fget.__name__] = computed_value
  752. return computed_value
  753. class ArgumentGenerator(object):
  754. """Generate sample input based on a shape model.
  755. This class contains a ``generate_skeleton`` method that will take
  756. an input/output shape (created from ``botocore.model``) and generate
  757. a sample dictionary corresponding to the input/output shape.
  758. The specific values used are place holder values. For strings either an
  759. empty string or the member name can be used, for numbers 0 or 0.0 is used.
  760. The intended usage of this class is to generate the *shape* of the input
  761. structure.
  762. This can be useful for operations that have complex input shapes.
  763. This allows a user to just fill in the necessary data instead of
  764. worrying about the specific structure of the input arguments.
  765. Example usage::
  766. s = botocore.session.get_session()
  767. ddb = s.get_service_model('dynamodb')
  768. arg_gen = ArgumentGenerator()
  769. sample_input = arg_gen.generate_skeleton(
  770. ddb.operation_model('CreateTable').input_shape)
  771. print("Sample input for dynamodb.CreateTable: %s" % sample_input)
  772. """
  773. def __init__(self, use_member_names=False):
  774. self._use_member_names = use_member_names
  775. def generate_skeleton(self, shape):
  776. """Generate a sample input.
  777. :type shape: ``botocore.model.Shape``
  778. :param shape: The input shape.
  779. :return: The generated skeleton input corresponding to the
  780. provided input shape.
  781. """
  782. stack = []
  783. return self._generate_skeleton(shape, stack)
  784. def _generate_skeleton(self, shape, stack, name=''):
  785. stack.append(shape.name)
  786. try:
  787. if shape.type_name == 'structure':
  788. return self._generate_type_structure(shape, stack)
  789. elif shape.type_name == 'list':
  790. return self._generate_type_list(shape, stack)
  791. elif shape.type_name == 'map':
  792. return self._generate_type_map(shape, stack)
  793. elif shape.type_name == 'string':
  794. if self._use_member_names:
  795. return name
  796. if shape.enum:
  797. return random.choice(shape.enum)
  798. return ''
  799. elif shape.type_name in ['integer', 'long']:
  800. return 0
  801. elif shape.type_name in ['float', 'double']:
  802. return 0.0
  803. elif shape.type_name == 'boolean':
  804. return True
  805. elif shape.type_name == 'timestamp':
  806. return datetime.datetime(1970, 1, 1, 0, 0, 0)
  807. finally:
  808. stack.pop()
  809. def _generate_type_structure(self, shape, stack):
  810. if stack.count(shape.name) > 1:
  811. return {}
  812. skeleton = OrderedDict()
  813. for member_name, member_shape in shape.members.items():
  814. skeleton[member_name] = self._generate_skeleton(
  815. member_shape, stack, name=member_name)
  816. return skeleton
  817. def _generate_type_list(self, shape, stack):
  818. # For list elements we've arbitrarily decided to
  819. # return two elements for the skeleton list.
  820. name = ''
  821. if self._use_member_names:
  822. name = shape.member.name
  823. return [
  824. self._generate_skeleton(shape.member, stack, name),
  825. ]
  826. def _generate_type_map(self, shape, stack):
  827. key_shape = shape.key
  828. value_shape = shape.value
  829. assert key_shape.type_name == 'string'
  830. return OrderedDict([
  831. ('KeyName', self._generate_skeleton(value_shape, stack)),
  832. ])
  833. def is_valid_ipv6_endpoint_url(endpoint_url):
  834. if UNSAFE_URL_CHARS.intersection(endpoint_url):
  835. return False
  836. netloc = urlparse(endpoint_url).netloc
  837. return IPV6_ADDRZ_RE.match(netloc) is not None
  838. def is_valid_endpoint_url(endpoint_url):
  839. """Verify the endpoint_url is valid.
  840. :type endpoint_url: string
  841. :param endpoint_url: An endpoint_url. Must have at least a scheme
  842. and a hostname.
  843. :return: True if the endpoint url is valid. False otherwise.
  844. """
  845. # post-bpo-43882 urlsplit() strips unsafe characters from URL, causing
  846. # it to pass hostname validation below. Detect them early to fix that.
  847. if UNSAFE_URL_CHARS.intersection(endpoint_url):
  848. return False
  849. parts = urlsplit(endpoint_url)
  850. hostname = parts.hostname
  851. if hostname is None:
  852. return False
  853. if len(hostname) > 255:
  854. return False
  855. if hostname[-1] == ".":
  856. hostname = hostname[:-1]
  857. allowed = re.compile(
  858. r"^((?!-)[A-Z\d-]{1,63}(?<!-)\.)*((?!-)[A-Z\d-]{1,63}(?<!-))$",
  859. re.IGNORECASE)
  860. return allowed.match(hostname)
  861. def is_valid_uri(endpoint_url):
  862. return is_valid_endpoint_url(endpoint_url) or is_valid_ipv6_endpoint_url(endpoint_url)
  863. def validate_region_name(region_name):
  864. """Provided region_name must be a valid host label."""
  865. if region_name is None:
  866. return
  867. valid_host_label = re.compile(r'^(?![0-9]+$)(?!-)[a-zA-Z0-9-]{,63}(?<!-)$')
  868. valid = valid_host_label.match(region_name)
  869. if not valid:
  870. raise InvalidRegionError(region_name=region_name)
  871. def check_dns_name(bucket_name):
  872. """
  873. Check to see if the ``bucket_name`` complies with the
  874. restricted DNS naming conventions necessary to allow
  875. access via virtual-hosting style.
  876. Even though "." characters are perfectly valid in this DNS
  877. naming scheme, we are going to punt on any name containing a
  878. "." character because these will cause SSL cert validation
  879. problems if we try to use virtual-hosting style addressing.
  880. """
  881. if '.' in bucket_name:
  882. return False
  883. n = len(bucket_name)
  884. if n < 3 or n > 63:
  885. # Wrong length
  886. return False
  887. match = LABEL_RE.match(bucket_name)
  888. if match is None or match.end() != len(bucket_name):
  889. return False
  890. return True
  891. def fix_s3_host(request, signature_version, region_name,
  892. default_endpoint_url=None, **kwargs):
  893. """
  894. This handler looks at S3 requests just before they are signed.
  895. If there is a bucket name on the path (true for everything except
  896. ListAllBuckets) it checks to see if that bucket name conforms to
  897. the DNS naming conventions. If it does, it alters the request to
  898. use ``virtual hosting`` style addressing rather than ``path-style``
  899. addressing.
  900. """
  901. if request.context.get('use_global_endpoint', False):
  902. default_endpoint_url = 's3.amazonaws.com'
  903. try:
  904. switch_to_virtual_host_style(
  905. request, signature_version, default_endpoint_url)
  906. except InvalidDNSNameError as e:
  907. bucket_name = e.kwargs['bucket_name']
  908. logger.debug('Not changing URI, bucket is not DNS compatible: %s',
  909. bucket_name)
  910. def switch_to_virtual_host_style(request, signature_version,
  911. default_endpoint_url=None, **kwargs):
  912. """
  913. This is a handler to force virtual host style s3 addressing no matter
  914. the signature version (which is taken in consideration for the default
  915. case). If the bucket is not DNS compatible an InvalidDNSName is thrown.
  916. :param request: A AWSRequest object that is about to be sent.
  917. :param signature_version: The signature version to sign with
  918. :param default_endpoint_url: The endpoint to use when switching to a
  919. virtual style. If None is supplied, the virtual host will be
  920. constructed from the url of the request.
  921. """
  922. if request.auth_path is not None:
  923. # The auth_path has already been applied (this may be a
  924. # retried request). We don't need to perform this
  925. # customization again.
  926. return
  927. elif _is_get_bucket_location_request(request):
  928. # For the GetBucketLocation response, we should not be using
  929. # the virtual host style addressing so we can avoid any sigv4
  930. # issues.
  931. logger.debug("Request is GetBucketLocation operation, not checking "
  932. "for DNS compatibility.")
  933. return
  934. parts = urlsplit(request.url)
  935. request.auth_path = parts.path
  936. path_parts = parts.path.split('/')
  937. # Retrieve what the endpoint we will be prepending the bucket name to.
  938. if default_endpoint_url is None:
  939. default_endpoint_url = parts.netloc
  940. if len(path_parts) > 1:
  941. bucket_name = path_parts[1]
  942. if not bucket_name:
  943. # If the bucket name is empty we should not be checking for
  944. # dns compatibility.
  945. return
  946. logger.debug('Checking for DNS compatible bucket for: %s',
  947. request.url)
  948. if check_dns_name(bucket_name):
  949. # If the operation is on a bucket, the auth_path must be
  950. # terminated with a '/' character.
  951. if len(path_parts) == 2:
  952. if request.auth_path[-1] != '/':
  953. request.auth_path += '/'
  954. path_parts.remove(bucket_name)
  955. # At the very least the path must be a '/', such as with the
  956. # CreateBucket operation when DNS style is being used. If this
  957. # is not used you will get an empty path which is incorrect.
  958. path = '/'.join(path_parts) or '/'
  959. global_endpoint = default_endpoint_url
  960. host = bucket_name + '.' + global_endpoint
  961. new_tuple = (parts.scheme, host, path,
  962. parts.query, '')
  963. new_uri = urlunsplit(new_tuple)
  964. request.url = new_uri
  965. logger.debug('URI updated to: %s', new_uri)
  966. else:
  967. raise InvalidDNSNameError(bucket_name=bucket_name)
  968. def _is_get_bucket_location_request(request):
  969. return request.url.endswith('?location')
  970. def instance_cache(func):
  971. """Method decorator for caching method calls to a single instance.
  972. **This is not a general purpose caching decorator.**
  973. In order to use this, you *must* provide an ``_instance_cache``
  974. attribute on the instance.
  975. This decorator is used to cache method calls. The cache is only
  976. scoped to a single instance though such that multiple instances
  977. will maintain their own cache. In order to keep things simple,
  978. this decorator requires that you provide an ``_instance_cache``
  979. attribute on your instance.
  980. """
  981. func_name = func.__name__
  982. @functools.wraps(func)
  983. def _cache_guard(self, *args, **kwargs):
  984. cache_key = (func_name, args)
  985. if kwargs:
  986. kwarg_items = tuple(sorted(kwargs.items()))
  987. cache_key = (func_name, args, kwarg_items)
  988. result = self._instance_cache.get(cache_key)
  989. if result is not None:
  990. return result
  991. result = func(self, *args, **kwargs)
  992. self._instance_cache[cache_key] = result
  993. return result
  994. return _cache_guard
  995. def switch_host_s3_accelerate(request, operation_name, **kwargs):
  996. """Switches the current s3 endpoint with an S3 Accelerate endpoint"""
  997. # Note that when registered the switching of the s3 host happens
  998. # before it gets changed to virtual. So we are not concerned with ensuring
  999. # that the bucket name is translated to the virtual style here and we
  1000. # can hard code the Accelerate endpoint.
  1001. parts = urlsplit(request.url).netloc.split('.')
  1002. parts = [p for p in parts if p in S3_ACCELERATE_WHITELIST]
  1003. endpoint = 'https://s3-accelerate.'
  1004. if len(parts) > 0:
  1005. endpoint += '.'.join(parts) + '.'
  1006. endpoint += 'amazonaws.com'
  1007. if operation_name in ['ListBuckets', 'CreateBucket', 'DeleteBucket']:
  1008. return
  1009. _switch_hosts(request, endpoint, use_new_scheme=False)
  1010. def switch_host_with_param(request, param_name):
  1011. """Switches the host using a parameter value from a JSON request body"""
  1012. request_json = json.loads(request.data.decode('utf-8'))
  1013. if request_json.get(param_name):
  1014. new_endpoint = request_json[param_name]
  1015. _switch_hosts(request, new_endpoint)
  1016. def _switch_hosts(request, new_endpoint, use_new_scheme=True):
  1017. final_endpoint = _get_new_endpoint(
  1018. request.url, new_endpoint, use_new_scheme)
  1019. request.url = final_endpoint
  1020. def _get_new_endpoint(original_endpoint, new_endpoint, use_new_scheme=True):
  1021. new_endpoint_components = urlsplit(new_endpoint)
  1022. original_endpoint_components = urlsplit(original_endpoint)
  1023. scheme = original_endpoint_components.scheme
  1024. if use_new_scheme:
  1025. scheme = new_endpoint_components.scheme
  1026. final_endpoint_components = (
  1027. scheme,
  1028. new_endpoint_components.netloc,
  1029. original_endpoint_components.path,
  1030. original_endpoint_components.query,
  1031. ''
  1032. )
  1033. final_endpoint = urlunsplit(final_endpoint_components)
  1034. logger.debug('Updating URI from %s to %s' % (
  1035. original_endpoint, final_endpoint))
  1036. return final_endpoint
  1037. def deep_merge(base, extra):
  1038. """Deeply two dictionaries, overriding existing keys in the base.
  1039. :param base: The base dictionary which will be merged into.
  1040. :param extra: The dictionary to merge into the base. Keys from this
  1041. dictionary will take precedence.
  1042. """
  1043. for key in extra:
  1044. # If the key represents a dict on both given dicts, merge the sub-dicts
  1045. if key in base and isinstance(base[key], dict)\
  1046. and isinstance(extra[key], dict):
  1047. deep_merge(base[key], extra[key])
  1048. continue
  1049. # Otherwise, set the key on the base to be the value of the extra.
  1050. base[key] = extra[key]
  1051. def hyphenize_service_id(service_id):
  1052. """Translate the form used for event emitters.
  1053. :param service_id: The service_id to convert.
  1054. """
  1055. return service_id.replace(' ', '-').lower()
  1056. class S3RegionRedirector(object):
  1057. def __init__(self, endpoint_bridge, client, cache=None):
  1058. self._endpoint_resolver = endpoint_bridge
  1059. self._cache = cache
  1060. if self._cache is None:
  1061. self._cache = {}
  1062. # This needs to be a weak ref in order to prevent memory leaks on
  1063. # python 2.6
  1064. self._client = weakref.proxy(client)
  1065. def register(self, event_emitter=None):
  1066. emitter = event_emitter or self._client.meta.events
  1067. emitter.register('needs-retry.s3', self.redirect_from_error)
  1068. emitter.register('before-call.s3', self.set_request_url)
  1069. emitter.register('before-parameter-build.s3',
  1070. self.redirect_from_cache)
  1071. def redirect_from_error(self, request_dict, response, operation, **kwargs):
  1072. """
  1073. An S3 request sent to the wrong region will return an error that
  1074. contains the endpoint the request should be sent to. This handler
  1075. will add the redirect information to the signing context and then
  1076. redirect the request.
  1077. """
  1078. if response is None:
  1079. # This could be none if there was a ConnectionError or other
  1080. # transport error.
  1081. return
  1082. if self._is_s3_accesspoint(request_dict.get('context', {})):
  1083. logger.debug(
  1084. 'S3 request was previously to an accesspoint, not redirecting.'
  1085. )
  1086. return
  1087. if request_dict.get('context', {}).get('s3_redirected'):
  1088. logger.debug(
  1089. 'S3 request was previously redirected, not redirecting.')
  1090. return
  1091. error = response[1].get('Error', {})
  1092. error_code = error.get('Code')
  1093. response_metadata = response[1].get('ResponseMetadata', {})
  1094. # We have to account for 400 responses because
  1095. # if we sign a Head* request with the wrong region,
  1096. # we'll get a 400 Bad Request but we won't get a
  1097. # body saying it's an "AuthorizationHeaderMalformed".
  1098. is_special_head_object = (
  1099. error_code in ['301', '400'] and
  1100. operation.name == 'HeadObject'
  1101. )
  1102. is_special_head_bucket = (
  1103. error_code in ['301', '400'] and
  1104. operation.name == 'HeadBucket' and
  1105. 'x-amz-bucket-region' in response_metadata.get('HTTPHeaders', {})
  1106. )
  1107. is_wrong_signing_region = (
  1108. error_code == 'AuthorizationHeaderMalformed' and
  1109. 'Region' in error
  1110. )
  1111. is_redirect_status = response[0] is not None and \
  1112. response[0].status_code in [301, 302, 307]
  1113. is_permanent_redirect = error_code == 'PermanentRedirect'
  1114. if not any([is_special_head_object, is_wrong_signing_region,
  1115. is_permanent_redirect, is_special_head_bucket,
  1116. is_redirect_status]):
  1117. return
  1118. bucket = request_dict['context']['signing']['bucket']
  1119. client_region = request_dict['context'].get('client_region')
  1120. new_region = self.get_bucket_region(bucket, response)
  1121. if new_region is None:
  1122. logger.debug(
  1123. "S3 client configured for region %s but the bucket %s is not "
  1124. "in that region and the proper region could not be "
  1125. "automatically determined." % (client_region, bucket))
  1126. return
  1127. logger.debug(
  1128. "S3 client configured for region %s but the bucket %s is in region"
  1129. " %s; Please configure the proper region to avoid multiple "
  1130. "unnecessary redirects and signing attempts." % (
  1131. client_region, bucket, new_region))
  1132. endpoint = self._endpoint_resolver.resolve('s3', new_region)
  1133. endpoint = endpoint['endpoint_url']
  1134. signing_context = {
  1135. 'region': new_region,
  1136. 'bucket': bucket,
  1137. 'endpoint': endpoint
  1138. }
  1139. request_dict['context']['signing'] = signing_context
  1140. self._cache[bucket] = signing_context
  1141. self.set_request_url(request_dict, request_dict['context'])
  1142. request_dict['context']['s3_redirected'] = True
  1143. # Return 0 so it doesn't wait to retry
  1144. return 0
  1145. def get_bucket_region(self, bucket, response):
  1146. """
  1147. There are multiple potential sources for the new region to redirect to,
  1148. but they aren't all universally available for use. This will try to
  1149. find region from response elements, but will fall back to calling
  1150. HEAD on the bucket if all else fails.
  1151. :param bucket: The bucket to find the region for. This is necessary if
  1152. the region is not available in the error response.
  1153. :param response: A response representing a service request that failed
  1154. due to incorrect region configuration.
  1155. """
  1156. # First try to source the region from the headers.
  1157. service_response = response[1]
  1158. response_headers = service_response['ResponseMetadata']['HTTPHeaders']
  1159. if 'x-amz-bucket-region' in response_headers:
  1160. return response_headers['x-amz-bucket-region']
  1161. # Next, check the error body
  1162. region = service_response.get('Error', {}).get('Region', None)
  1163. if region is not None:
  1164. return region
  1165. # Finally, HEAD the bucket. No other choice sadly.
  1166. try:
  1167. response = self._client.head_bucket(Bucket=bucket)
  1168. headers = response['ResponseMetadata']['HTTPHeaders']
  1169. except ClientError as e:
  1170. headers = e.response['ResponseMetadata']['HTTPHeaders']
  1171. region = headers.get('x-amz-bucket-region', None)
  1172. return region
  1173. def set_request_url(self, params, context, **kwargs):
  1174. endpoint = context.get('signing', {}).get('endpoint', None)
  1175. if endpoint is not None:
  1176. params['url'] = _get_new_endpoint(params['url'], endpoint, False)
  1177. def redirect_from_cache(self, params, context, **kwargs):
  1178. """
  1179. This handler retrieves a given bucket's signing context from the cache
  1180. and adds it into the request context.
  1181. """
  1182. if self._is_s3_accesspoint(context):
  1183. return
  1184. bucket = params.get('Bucket')
  1185. signing_context = self._cache.get(bucket)
  1186. if signing_context is not None:
  1187. context['signing'] = signing_context
  1188. else:
  1189. context['signing'] = {'bucket': bucket}
  1190. def _is_s3_accesspoint(self, context):
  1191. return 's3_accesspoint' in context
  1192. class InvalidArnException(ValueError):
  1193. pass
  1194. class ArnParser(object):
  1195. def parse_arn(self, arn):
  1196. arn_parts = arn.split(':', 5)
  1197. if len(arn_parts) < 6:
  1198. raise InvalidArnException(
  1199. 'Provided ARN: %s must be of the format: '
  1200. 'arn:partition:service:region:account:resource' % arn
  1201. )
  1202. return {
  1203. 'partition': arn_parts[1],
  1204. 'service': arn_parts[2],
  1205. 'region': arn_parts[3],
  1206. 'account': arn_parts[4],
  1207. 'resource': arn_parts[5],
  1208. }
  1209. class S3ArnParamHandler(object):
  1210. _RESOURCE_REGEX = re.compile(
  1211. r'^(?P<resource_type>accesspoint|outpost)[/:](?P<resource_name>.+)$'
  1212. )
  1213. _OUTPOST_RESOURCE_REGEX = re.compile(
  1214. r'^(?P<outpost_name>[a-zA-Z0-9\-]{1,63})[/:]accesspoint[/:]'
  1215. r'(?P<accesspoint_name>[a-zA-Z0-9\-]{1,63}$)'
  1216. )
  1217. _BLACKLISTED_OPERATIONS = [
  1218. 'CreateBucket'
  1219. ]
  1220. def __init__(self, arn_parser=None):
  1221. self._arn_parser = arn_parser
  1222. if arn_parser is None:
  1223. self._arn_parser = ArnParser()
  1224. def register(self, event_emitter):
  1225. event_emitter.register('before-parameter-build.s3', self.handle_arn)
  1226. def handle_arn(self, params, model, context, **kwargs):
  1227. if model.name in self._BLACKLISTED_OPERATIONS:
  1228. return
  1229. arn_details = self._get_arn_details_from_bucket_param(params)
  1230. if arn_details is None:
  1231. return
  1232. if arn_details['resource_type'] == 'accesspoint':
  1233. self._store_accesspoint(params, context, arn_details)
  1234. elif arn_details['resource_type'] == 'outpost':
  1235. self._store_outpost(params, context, arn_details)
  1236. def _get_arn_details_from_bucket_param(self, params):
  1237. if 'Bucket' in params:
  1238. try:
  1239. arn = params['Bucket']
  1240. arn_details = self._arn_parser.parse_arn(arn)
  1241. self._add_resource_type_and_name(arn, arn_details)
  1242. return arn_details
  1243. except InvalidArnException:
  1244. pass
  1245. return None
  1246. def _add_resource_type_and_name(self, arn, arn_details):
  1247. match = self._RESOURCE_REGEX.match(arn_details['resource'])
  1248. if match:
  1249. arn_details['resource_type'] = match.group('resource_type')
  1250. arn_details['resource_name'] = match.group('resource_name')
  1251. else:
  1252. raise UnsupportedS3ArnError(arn=arn)
  1253. def _store_accesspoint(self, params, context, arn_details):
  1254. # Ideally the access-point would be stored as a parameter in the
  1255. # request where the serializer would then know how to serialize it,
  1256. # but access-points are not modeled in S3 operations so it would fail
  1257. # validation. Instead, we set the access-point to the bucket parameter
  1258. # to have some value set when serializing the request and additional
  1259. # information on the context from the arn to use in forming the
  1260. # access-point endpoint.
  1261. params['Bucket'] = arn_details['resource_name']
  1262. context['s3_accesspoint'] = {
  1263. 'name': arn_details['resource_name'],
  1264. 'account': arn_details['account'],
  1265. 'partition': arn_details['partition'],
  1266. 'region': arn_details['region'],
  1267. 'service': arn_details['service'],
  1268. }
  1269. def _store_outpost(self, params, context, arn_details):
  1270. resource_name = arn_details['resource_name']
  1271. match = self._OUTPOST_RESOURCE_REGEX.match(resource_name)
  1272. if not match:
  1273. raise UnsupportedOutpostResourceError(resource_name=resource_name)
  1274. # Because we need to set the bucket name to something to pass
  1275. # validation we're going to use the access point name to be consistent
  1276. # with normal access point arns.
  1277. accesspoint_name = match.group('accesspoint_name')
  1278. params['Bucket'] = accesspoint_name
  1279. context['s3_accesspoint'] = {
  1280. 'outpost_name': match.group('outpost_name'),
  1281. 'name': accesspoint_name,
  1282. 'account': arn_details['account'],
  1283. 'partition': arn_details['partition'],
  1284. 'region': arn_details['region'],
  1285. 'service': arn_details['service'],
  1286. }
  1287. class S3EndpointSetter(object):
  1288. _DEFAULT_PARTITION = 'aws'
  1289. _DEFAULT_DNS_SUFFIX = 'amazonaws.com'
  1290. def __init__(self, endpoint_resolver, region=None,
  1291. s3_config=None, endpoint_url=None, partition=None):
  1292. self._endpoint_resolver = endpoint_resolver
  1293. self._region = region
  1294. self._s3_config = s3_config
  1295. if s3_config is None:
  1296. self._s3_config = {}
  1297. self._endpoint_url = endpoint_url
  1298. self._partition = partition
  1299. if partition is None:
  1300. self._partition = self._DEFAULT_PARTITION
  1301. def register(self, event_emitter):
  1302. event_emitter.register('before-sign.s3', self.set_endpoint)
  1303. event_emitter.register(
  1304. 'before-call.s3.WriteGetObjectResponse',
  1305. self.update_endpoint_to_s3_object_lambda
  1306. )
  1307. def update_endpoint_to_s3_object_lambda(self, params, context, **kwargs):
  1308. if self._use_accelerate_endpoint:
  1309. raise UnsupportedS3ConfigurationError(
  1310. msg='S3 client does not support accelerate endpoints for S3 Object Lambda operations',
  1311. )
  1312. self._override_signing_name(context, 's3-object-lambda')
  1313. if self._endpoint_url:
  1314. # Only update the url if an explicit url was not provided
  1315. return
  1316. resolver = self._endpoint_resolver
  1317. resolved = resolver.construct_endpoint('s3-object-lambda', self._region)
  1318. # Ideally we would be able to replace the endpoint before
  1319. # serialization but there's no event to do that currently
  1320. new_endpoint = 'https://{host_prefix}{hostname}'.format(
  1321. host_prefix=params['host_prefix'],
  1322. hostname=resolved['hostname'],
  1323. )
  1324. params['url'] = _get_new_endpoint(params['url'], new_endpoint, False)
  1325. def set_endpoint(self, request, **kwargs):
  1326. if self._use_accesspoint_endpoint(request):
  1327. self._validate_accesspoint_supported(request)
  1328. self._validate_fips_supported(request)
  1329. self._validate_global_regions(request)
  1330. region_name = self._resolve_region_for_accesspoint_endpoint(
  1331. request)
  1332. self._resolve_signing_name_for_accesspoint_endpoint(
  1333. request)
  1334. self._switch_to_accesspoint_endpoint(request, region_name)
  1335. return
  1336. if self._use_accelerate_endpoint:
  1337. switch_host_s3_accelerate(request=request, **kwargs)
  1338. if self._s3_addressing_handler:
  1339. self._s3_addressing_handler(request=request, **kwargs)
  1340. def _use_accesspoint_endpoint(self, request):
  1341. return 's3_accesspoint' in request.context
  1342. def _validate_fips_supported(self, request):
  1343. if 'fips' not in self._region:
  1344. return
  1345. if 'outpost_name' in request.context['s3_accesspoint']:
  1346. raise UnsupportedS3AccesspointConfigurationError(
  1347. msg=(
  1348. 'Client is configured to use the FIPS psuedo-region "%s", '
  1349. 'but outpost ARNs do not support FIPS endpoints.' % (
  1350. self._region)
  1351. )
  1352. )
  1353. client_region = self._region.replace('fips-', '').replace('-fips', '')
  1354. accesspoint_region = request.context['s3_accesspoint']['region']
  1355. if accesspoint_region != client_region:
  1356. if self._s3_config.get('use_arn_region', True):
  1357. raise UnsupportedS3AccesspointConfigurationError(
  1358. msg=(
  1359. 'Client is configured to use the FIPS psuedo-region '
  1360. '"%s", but the access-point ARN provided is for the '
  1361. '"%s" region. The use_arn_region configuration does '
  1362. 'not allow for cross-region calls when a FIPS '
  1363. 'pseudo-region is configured.' % (
  1364. self._region, accesspoint_region)
  1365. )
  1366. )
  1367. else:
  1368. raise UnsupportedS3AccesspointConfigurationError(
  1369. msg=(
  1370. 'Client is configured to use the FIPS psuedo-region '
  1371. '"%s", but the access-point ARN provided is for the '
  1372. '"%s" region. For clients using a FIPS psuedo-region '
  1373. 'calls to access-point ARNs in another region are not '
  1374. 'allowed.' % (self._region, accesspoint_region)
  1375. )
  1376. )
  1377. def _validate_global_regions(self, request):
  1378. if self._s3_config.get('use_arn_region', True):
  1379. return
  1380. if self._region in ['aws-global', 's3-external-1']:
  1381. raise UnsupportedS3AccesspointConfigurationError(
  1382. msg=(
  1383. 'Client is configured to use the global psuedo-region '
  1384. '"%s". When providing access-point ARNs a regional '
  1385. 'endpoint must be specified.' % self._region
  1386. )
  1387. )
  1388. def _validate_accesspoint_supported(self, request):
  1389. if self._use_accelerate_endpoint:
  1390. raise UnsupportedS3AccesspointConfigurationError(
  1391. msg=(
  1392. 'Client does not support s3 accelerate configuration '
  1393. 'when an access-point ARN is specified.'
  1394. )
  1395. )
  1396. request_partition = request.context['s3_accesspoint']['partition']
  1397. if request_partition != self._partition:
  1398. raise UnsupportedS3AccesspointConfigurationError(
  1399. msg=(
  1400. 'Client is configured for "%s" partition, but access-point'
  1401. ' ARN provided is for "%s" partition. The client and '
  1402. ' access-point partition must be the same.' % (
  1403. self._partition, request_partition)
  1404. )
  1405. )
  1406. s3_service = request.context['s3_accesspoint'].get('service')
  1407. if s3_service == 's3-object-lambda' and self._s3_config.get('use_dualstack_endpoint'):
  1408. raise UnsupportedS3AccesspointConfigurationError(
  1409. msg=(
  1410. 'Client does not support s3 dualstack configuration '
  1411. 'when an S3 Object Lambda access point ARN is specified.'
  1412. )
  1413. )
  1414. outpost_name = request.context['s3_accesspoint'].get('outpost_name')
  1415. if outpost_name and self._s3_config.get('use_dualstack_endpoint'):
  1416. raise UnsupportedS3AccesspointConfigurationError(
  1417. msg=(
  1418. 'Client does not support s3 dualstack configuration '
  1419. 'when an outpost ARN is specified.'
  1420. )
  1421. )
  1422. def _resolve_region_for_accesspoint_endpoint(self, request):
  1423. if self._s3_config.get('use_arn_region', True):
  1424. accesspoint_region = request.context['s3_accesspoint']['region']
  1425. # If we are using the region from the access point,
  1426. # we will also want to make sure that we set it as the
  1427. # signing region as well
  1428. self._override_signing_region(request, accesspoint_region)
  1429. return accesspoint_region
  1430. return self._region
  1431. def _resolve_signing_name_for_accesspoint_endpoint(self, request):
  1432. accesspoint_service = request.context['s3_accesspoint']['service']
  1433. self._override_signing_name(request.context, accesspoint_service)
  1434. def _switch_to_accesspoint_endpoint(self, request, region_name):
  1435. original_components = urlsplit(request.url)
  1436. accesspoint_endpoint = urlunsplit((
  1437. original_components.scheme,
  1438. self._get_accesspoint_netloc(request.context, region_name),
  1439. self._get_accesspoint_path(
  1440. original_components.path, request.context),
  1441. original_components.query,
  1442. ''
  1443. ))
  1444. logger.debug(
  1445. 'Updating URI from %s to %s' % (request.url, accesspoint_endpoint))
  1446. request.url = accesspoint_endpoint
  1447. def _get_accesspoint_netloc(self, request_context, region_name):
  1448. s3_accesspoint = request_context['s3_accesspoint']
  1449. accesspoint_netloc_components = [
  1450. '%s-%s' % (s3_accesspoint['name'], s3_accesspoint['account']),
  1451. ]
  1452. outpost_name = s3_accesspoint.get('outpost_name')
  1453. if self._endpoint_url:
  1454. if outpost_name:
  1455. accesspoint_netloc_components.append(outpost_name)
  1456. endpoint_url_netloc = urlsplit(self._endpoint_url).netloc
  1457. accesspoint_netloc_components.append(endpoint_url_netloc)
  1458. else:
  1459. if outpost_name:
  1460. outpost_host = [outpost_name, 's3-outposts']
  1461. accesspoint_netloc_components.extend(outpost_host)
  1462. elif s3_accesspoint['service'] == 's3-object-lambda':
  1463. component = self._inject_fips_if_needed(
  1464. 's3-object-lambda', request_context)
  1465. accesspoint_netloc_components.append(component)
  1466. else:
  1467. component = self._inject_fips_if_needed(
  1468. 's3-accesspoint', request_context)
  1469. accesspoint_netloc_components.append(component)
  1470. if self._s3_config.get('use_dualstack_endpoint'):
  1471. accesspoint_netloc_components.append('dualstack')
  1472. accesspoint_netloc_components.extend(
  1473. [
  1474. region_name,
  1475. self._get_dns_suffix(region_name)
  1476. ]
  1477. )
  1478. return '.'.join(accesspoint_netloc_components)
  1479. def _inject_fips_if_needed(self, component, request_context):
  1480. if 'fips' in request_context.get('client_region', ''):
  1481. return '%s-fips' % component
  1482. return component
  1483. def _get_accesspoint_path(self, original_path, request_context):
  1484. # The Bucket parameter was substituted with the access-point name as
  1485. # some value was required in serializing the bucket name. Now that
  1486. # we are making the request directly to the access point, we will
  1487. # want to remove that access-point name from the path.
  1488. name = request_context['s3_accesspoint']['name']
  1489. # All S3 operations require at least a / in their path.
  1490. return original_path.replace('/' + name, '', 1) or '/'
  1491. def _get_dns_suffix(self, region_name):
  1492. resolved = self._endpoint_resolver.construct_endpoint(
  1493. 's3', region_name)
  1494. dns_suffix = self._DEFAULT_DNS_SUFFIX
  1495. if resolved and 'dnsSuffix' in resolved:
  1496. dns_suffix = resolved['dnsSuffix']
  1497. return dns_suffix
  1498. def _override_signing_region(self, request, region_name):
  1499. signing_context = request.context.get('signing', {})
  1500. # S3SigV4Auth will use the context['signing']['region'] value to
  1501. # sign with if present. This is used by the Bucket redirector
  1502. # as well but we should be fine because the redirector is never
  1503. # used in combination with the accesspoint setting logic.
  1504. signing_context['region'] = region_name
  1505. request.context['signing'] = signing_context
  1506. def _override_signing_name(self, context, signing_name):
  1507. signing_context = context.get('signing', {})
  1508. # S3SigV4Auth will use the context['signing']['signing_name'] value to
  1509. # sign with if present. This is used by the Bucket redirector
  1510. # as well but we should be fine because the redirector is never
  1511. # used in combination with the accesspoint setting logic.
  1512. signing_context['signing_name'] = signing_name
  1513. context['signing'] = signing_context
  1514. @CachedProperty
  1515. def _use_accelerate_endpoint(self):
  1516. # Enable accelerate if the configuration is set to to true or the
  1517. # endpoint being used matches one of the accelerate endpoints.
  1518. # Accelerate has been explicitly configured.
  1519. if self._s3_config.get('use_accelerate_endpoint'):
  1520. return True
  1521. # Accelerate mode is turned on automatically if an endpoint url is
  1522. # provided that matches the accelerate scheme.
  1523. if self._endpoint_url is None:
  1524. return False
  1525. # Accelerate is only valid for Amazon endpoints.
  1526. netloc = urlsplit(self._endpoint_url).netloc
  1527. if not netloc.endswith('amazonaws.com'):
  1528. return False
  1529. # The first part of the url should always be s3-accelerate.
  1530. parts = netloc.split('.')
  1531. if parts[0] != 's3-accelerate':
  1532. return False
  1533. # Url parts between 's3-accelerate' and 'amazonaws.com' which
  1534. # represent different url features.
  1535. feature_parts = parts[1:-2]
  1536. # There should be no duplicate url parts.
  1537. if len(feature_parts) != len(set(feature_parts)):
  1538. return False
  1539. # Remaining parts must all be in the whitelist.
  1540. return all(p in S3_ACCELERATE_WHITELIST for p in feature_parts)
  1541. @CachedProperty
  1542. def _addressing_style(self):
  1543. # Use virtual host style addressing if accelerate is enabled or if
  1544. # the given endpoint url is an accelerate endpoint.
  1545. if self._use_accelerate_endpoint:
  1546. return 'virtual'
  1547. # If a particular addressing style is configured, use it.
  1548. configured_addressing_style = self._s3_config.get('addressing_style')
  1549. if configured_addressing_style:
  1550. return configured_addressing_style
  1551. @CachedProperty
  1552. def _s3_addressing_handler(self):
  1553. # If virtual host style was configured, use it regardless of whether
  1554. # or not the bucket looks dns compatible.
  1555. if self._addressing_style == 'virtual':
  1556. logger.debug("Using S3 virtual host style addressing.")
  1557. return switch_to_virtual_host_style
  1558. # If path style is configured, no additional steps are needed. If
  1559. # endpoint_url was specified, don't default to virtual. We could
  1560. # potentially default provided endpoint urls to virtual hosted
  1561. # style, but for now it is avoided.
  1562. if self._addressing_style == 'path' or self._endpoint_url is not None:
  1563. logger.debug("Using S3 path style addressing.")
  1564. return None
  1565. logger.debug("Defaulting to S3 virtual host style addressing with "
  1566. "path style addressing fallback.")
  1567. # By default, try to use virtual style with path fallback.
  1568. return fix_s3_host
  1569. class S3ControlEndpointSetter(object):
  1570. _DEFAULT_PARTITION = 'aws'
  1571. _DEFAULT_DNS_SUFFIX = 'amazonaws.com'
  1572. _HOST_LABEL_REGEX = re.compile(r'^[a-zA-Z0-9\-]{1,63}$')
  1573. def __init__(self, endpoint_resolver, region=None,
  1574. s3_config=None, endpoint_url=None, partition=None):
  1575. self._endpoint_resolver = endpoint_resolver
  1576. self._region = region
  1577. self._s3_config = s3_config
  1578. if s3_config is None:
  1579. self._s3_config = {}
  1580. self._endpoint_url = endpoint_url
  1581. self._partition = partition
  1582. if partition is None:
  1583. self._partition = self._DEFAULT_PARTITION
  1584. def register(self, event_emitter):
  1585. event_emitter.register('before-sign.s3-control', self.set_endpoint)
  1586. def set_endpoint(self, request, **kwargs):
  1587. if self._use_endpoint_from_arn_details(request):
  1588. self._validate_endpoint_from_arn_details_supported(request)
  1589. region_name = self._resolve_region_from_arn_details(request)
  1590. self._resolve_signing_name_from_arn_details(request)
  1591. self._resolve_endpoint_from_arn_details(request, region_name)
  1592. self._add_headers_from_arn_details(request)
  1593. elif self._use_endpoint_from_outpost_id(request):
  1594. self._validate_outpost_redirection_valid(request)
  1595. outpost_id = request.context['outpost_id']
  1596. self._override_signing_name(request, 's3-outposts')
  1597. new_netloc = self._construct_outpost_endpoint(self._region)
  1598. self._update_request_netloc(request, new_netloc)
  1599. def _use_endpoint_from_arn_details(self, request):
  1600. return 'arn_details' in request.context
  1601. def _use_endpoint_from_outpost_id(self, request):
  1602. return 'outpost_id' in request.context
  1603. def _validate_endpoint_from_arn_details_supported(self, request):
  1604. if not self._s3_config.get('use_arn_region', False):
  1605. arn_region = request.context['arn_details']['region']
  1606. if arn_region != self._region:
  1607. error_msg = (
  1608. 'The use_arn_region configuration is disabled but '
  1609. 'received arn for "%s" when the client is configured '
  1610. 'to use "%s"'
  1611. ) % (arn_region, self._region)
  1612. raise UnsupportedS3ControlConfigurationError(msg=error_msg)
  1613. request_partion = request.context['arn_details']['partition']
  1614. if request_partion != self._partition:
  1615. raise UnsupportedS3ControlConfigurationError(
  1616. msg=(
  1617. 'Client is configured for "%s" partition, but arn '
  1618. 'provided is for "%s" partition. The client and '
  1619. 'arn partition must be the same.' % (
  1620. self._partition, request_partion)
  1621. )
  1622. )
  1623. if self._s3_config.get('use_accelerate_endpoint'):
  1624. raise UnsupportedS3ControlConfigurationError(
  1625. msg='S3 control client does not support accelerate endpoints',
  1626. )
  1627. if 'outpost_name' in request.context['arn_details']:
  1628. self._validate_outpost_redirection_valid(request)
  1629. def _validate_outpost_redirection_valid(self, request):
  1630. if self._s3_config.get('use_dualstack_endpoint'):
  1631. raise UnsupportedS3ControlConfigurationError(
  1632. msg=(
  1633. 'Client does not support s3 dualstack configuration '
  1634. 'when an outpost is specified.'
  1635. )
  1636. )
  1637. def _resolve_region_from_arn_details(self, request):
  1638. if self._s3_config.get('use_arn_region', False):
  1639. arn_region = request.context['arn_details']['region']
  1640. # If we are using the region from the expanded arn, we will also
  1641. # want to make sure that we set it as the signing region as well
  1642. self._override_signing_region(request, arn_region)
  1643. return arn_region
  1644. return self._region
  1645. def _resolve_signing_name_from_arn_details(self, request):
  1646. arn_service = request.context['arn_details']['service']
  1647. self._override_signing_name(request, arn_service)
  1648. return arn_service
  1649. def _resolve_endpoint_from_arn_details(self, request, region_name):
  1650. new_netloc = self._resolve_netloc_from_arn_details(request, region_name)
  1651. self._update_request_netloc(request, new_netloc)
  1652. def _update_request_netloc(self, request, new_netloc):
  1653. original_components = urlsplit(request.url)
  1654. arn_details_endpoint = urlunsplit((
  1655. original_components.scheme,
  1656. new_netloc,
  1657. original_components.path,
  1658. original_components.query,
  1659. ''
  1660. ))
  1661. logger.debug(
  1662. 'Updating URI from %s to %s' % (request.url, arn_details_endpoint)
  1663. )
  1664. request.url = arn_details_endpoint
  1665. def _resolve_netloc_from_arn_details(self, request, region_name):
  1666. arn_details = request.context['arn_details']
  1667. if 'outpost_name' in arn_details:
  1668. return self._construct_outpost_endpoint(region_name)
  1669. account = arn_details['account']
  1670. return self._construct_s3_control_endpoint(region_name, account)
  1671. def _is_valid_host_label(self, label):
  1672. return self._HOST_LABEL_REGEX.match(label)
  1673. def _validate_host_labels(self, *labels):
  1674. for label in labels:
  1675. if not self._is_valid_host_label(label):
  1676. raise InvalidHostLabelError(label=label)
  1677. def _construct_s3_control_endpoint(self, region_name, account):
  1678. self._validate_host_labels(region_name, account)
  1679. if self._endpoint_url:
  1680. endpoint_url_netloc = urlsplit(self._endpoint_url).netloc
  1681. netloc = [account, endpoint_url_netloc]
  1682. else:
  1683. netloc = [
  1684. account,
  1685. 's3-control',
  1686. ]
  1687. self._add_dualstack(netloc)
  1688. dns_suffix = self._get_dns_suffix(region_name)
  1689. netloc.extend([region_name, dns_suffix])
  1690. return self._construct_netloc(netloc)
  1691. def _construct_outpost_endpoint(self, region_name):
  1692. self._validate_host_labels(region_name)
  1693. if self._endpoint_url:
  1694. return urlsplit(self._endpoint_url).netloc
  1695. else:
  1696. netloc = [
  1697. 's3-outposts',
  1698. region_name,
  1699. self._get_dns_suffix(region_name),
  1700. ]
  1701. return self._construct_netloc(netloc)
  1702. def _construct_netloc(self, netloc):
  1703. return '.'.join(netloc)
  1704. def _add_dualstack(self, netloc):
  1705. if self._s3_config.get('use_dualstack_endpoint'):
  1706. netloc.append('dualstack')
  1707. def _get_dns_suffix(self, region_name):
  1708. resolved = self._endpoint_resolver.construct_endpoint(
  1709. 's3', region_name)
  1710. dns_suffix = self._DEFAULT_DNS_SUFFIX
  1711. if resolved and 'dnsSuffix' in resolved:
  1712. dns_suffix = resolved['dnsSuffix']
  1713. return dns_suffix
  1714. def _override_signing_region(self, request, region_name):
  1715. signing_context = request.context.get('signing', {})
  1716. # S3SigV4Auth will use the context['signing']['region'] value to
  1717. # sign with if present. This is used by the Bucket redirector
  1718. # as well but we should be fine because the redirector is never
  1719. # used in combination with the accesspoint setting logic.
  1720. signing_context['region'] = region_name
  1721. request.context['signing'] = signing_context
  1722. def _override_signing_name(self, request, signing_name):
  1723. signing_context = request.context.get('signing', {})
  1724. # S3SigV4Auth will use the context['signing']['signing_name'] value to
  1725. # sign with if present. This is used by the Bucket redirector
  1726. # as well but we should be fine because the redirector is never
  1727. # used in combination with the accesspoint setting logic.
  1728. signing_context['signing_name'] = signing_name
  1729. request.context['signing'] = signing_context
  1730. def _add_headers_from_arn_details(self, request):
  1731. arn_details = request.context['arn_details']
  1732. outpost_name = arn_details.get('outpost_name')
  1733. if outpost_name:
  1734. self._add_outpost_id_header(request, outpost_name)
  1735. def _add_outpost_id_header(self, request, outpost_name):
  1736. request.headers['x-amz-outpost-id'] = outpost_name
  1737. class S3ControlArnParamHandler(object):
  1738. _RESOURCE_SPLIT_REGEX = re.compile(r'[/:]')
  1739. def __init__(self, arn_parser=None):
  1740. self._arn_parser = arn_parser
  1741. if arn_parser is None:
  1742. self._arn_parser = ArnParser()
  1743. def register(self, event_emitter):
  1744. event_emitter.register(
  1745. 'before-parameter-build.s3-control',
  1746. self.handle_arn,
  1747. )
  1748. def handle_arn(self, params, model, context, **kwargs):
  1749. if model.name in ('CreateBucket', 'ListRegionalBuckets'):
  1750. # CreateBucket and ListRegionalBuckets are special cases that do
  1751. # not obey ARN based redirection but will redirect based off of the
  1752. # presence of the OutpostId parameter
  1753. self._handle_outpost_id_param(params, model, context)
  1754. else:
  1755. self._handle_name_param(params, model, context)
  1756. self._handle_bucket_param(params, model, context)
  1757. def _get_arn_details_from_param(self, params, param_name):
  1758. if param_name not in params:
  1759. return None
  1760. try:
  1761. arn = params[param_name]
  1762. arn_details = self._arn_parser.parse_arn(arn)
  1763. arn_details['original'] = arn
  1764. arn_details['resources'] = self._split_resource(arn_details)
  1765. return arn_details
  1766. except InvalidArnException:
  1767. return None
  1768. def _split_resource(self, arn_details):
  1769. return self._RESOURCE_SPLIT_REGEX.split(arn_details['resource'])
  1770. def _override_account_id_param(self, params, arn_details):
  1771. account_id = arn_details['account']
  1772. if 'AccountId' in params and params['AccountId'] != account_id:
  1773. error_msg = (
  1774. 'Account ID in arn does not match the AccountId parameter '
  1775. 'provided: "%s"'
  1776. ) % params['AccountId']
  1777. raise UnsupportedS3ControlArnError(
  1778. arn=arn_details['original'],
  1779. msg=error_msg,
  1780. )
  1781. params['AccountId'] = account_id
  1782. def _handle_outpost_id_param(self, params, model, context):
  1783. if 'OutpostId' not in params:
  1784. return
  1785. context['outpost_id'] = params['OutpostId']
  1786. def _handle_name_param(self, params, model, context):
  1787. # CreateAccessPoint is a special case that does not expand Name
  1788. if model.name == 'CreateAccessPoint':
  1789. return
  1790. arn_details = self._get_arn_details_from_param(params, 'Name')
  1791. if arn_details is None:
  1792. return
  1793. if self._is_outpost_accesspoint(arn_details):
  1794. self._store_outpost_accesspoint(params, context, arn_details)
  1795. else:
  1796. error_msg = 'The Name parameter does not support the provided ARN'
  1797. raise UnsupportedS3ControlArnError(
  1798. arn=arn_details['original'],
  1799. msg=error_msg,
  1800. )
  1801. def _is_outpost_accesspoint(self, arn_details):
  1802. if arn_details['service'] != 's3-outposts':
  1803. return False
  1804. resources = arn_details['resources']
  1805. if len(resources) != 4:
  1806. return False
  1807. # Resource must be of the form outpost/op-123/accesspoint/name
  1808. return resources[0] == 'outpost' and resources[2] == 'accesspoint'
  1809. def _store_outpost_accesspoint(self, params, context, arn_details):
  1810. self._override_account_id_param(params, arn_details)
  1811. accesspoint_name = arn_details['resources'][3]
  1812. params['Name'] = accesspoint_name
  1813. arn_details['accesspoint_name'] = accesspoint_name
  1814. arn_details['outpost_name'] = arn_details['resources'][1]
  1815. context['arn_details'] = arn_details
  1816. def _handle_bucket_param(self, params, model, context):
  1817. arn_details = self._get_arn_details_from_param(params, 'Bucket')
  1818. if arn_details is None:
  1819. return
  1820. if self._is_outpost_bucket(arn_details):
  1821. self._store_outpost_bucket(params, context, arn_details)
  1822. else:
  1823. error_msg = (
  1824. 'The Bucket parameter does not support the provided ARN'
  1825. )
  1826. raise UnsupportedS3ControlArnError(
  1827. arn=arn_details['original'],
  1828. msg=error_msg,
  1829. )
  1830. def _is_outpost_bucket(self, arn_details):
  1831. if arn_details['service'] != 's3-outposts':
  1832. return False
  1833. resources = arn_details['resources']
  1834. if len(resources) != 4:
  1835. return False
  1836. # Resource must be of the form outpost/op-123/bucket/name
  1837. return resources[0] == 'outpost' and resources[2] == 'bucket'
  1838. def _store_outpost_bucket(self, params, context, arn_details):
  1839. self._override_account_id_param(params, arn_details)
  1840. bucket_name = arn_details['resources'][3]
  1841. params['Bucket'] = bucket_name
  1842. arn_details['bucket_name'] = bucket_name
  1843. arn_details['outpost_name'] = arn_details['resources'][1]
  1844. context['arn_details'] = arn_details
  1845. class ContainerMetadataFetcher(object):
  1846. TIMEOUT_SECONDS = 2
  1847. RETRY_ATTEMPTS = 3
  1848. SLEEP_TIME = 1
  1849. IP_ADDRESS = '169.254.170.2'
  1850. _ALLOWED_HOSTS = [IP_ADDRESS, 'localhost', '127.0.0.1']
  1851. def __init__(self, session=None, sleep=time.sleep):
  1852. if session is None:
  1853. session = botocore.httpsession.URLLib3Session(
  1854. timeout=self.TIMEOUT_SECONDS
  1855. )
  1856. self._session = session
  1857. self._sleep = sleep
  1858. def retrieve_full_uri(self, full_url, headers=None):
  1859. """Retrieve JSON metadata from container metadata.
  1860. :type full_url: str
  1861. :param full_url: The full URL of the metadata service.
  1862. This should include the scheme as well, e.g
  1863. "http://localhost:123/foo"
  1864. """
  1865. self._validate_allowed_url(full_url)
  1866. return self._retrieve_credentials(full_url, headers)
  1867. def _validate_allowed_url(self, full_url):
  1868. parsed = botocore.compat.urlparse(full_url)
  1869. is_whitelisted_host = self._check_if_whitelisted_host(
  1870. parsed.hostname)
  1871. if not is_whitelisted_host:
  1872. raise ValueError(
  1873. "Unsupported host '%s'. Can only "
  1874. "retrieve metadata from these hosts: %s" %
  1875. (parsed.hostname, ', '.join(self._ALLOWED_HOSTS)))
  1876. def _check_if_whitelisted_host(self, host):
  1877. if host in self._ALLOWED_HOSTS:
  1878. return True
  1879. return False
  1880. def retrieve_uri(self, relative_uri):
  1881. """Retrieve JSON metadata from ECS metadata.
  1882. :type relative_uri: str
  1883. :param relative_uri: A relative URI, e.g "/foo/bar?id=123"
  1884. :return: The parsed JSON response.
  1885. """
  1886. full_url = self.full_url(relative_uri)
  1887. return self._retrieve_credentials(full_url)
  1888. def _retrieve_credentials(self, full_url, extra_headers=None):
  1889. headers = {'Accept': 'application/json'}
  1890. if extra_headers is not None:
  1891. headers.update(extra_headers)
  1892. attempts = 0
  1893. while True:
  1894. try:
  1895. return self._get_response(
  1896. full_url, headers, self.TIMEOUT_SECONDS)
  1897. except MetadataRetrievalError as e:
  1898. logger.debug("Received error when attempting to retrieve "
  1899. "container metadata: %s", e, exc_info=True)
  1900. self._sleep(self.SLEEP_TIME)
  1901. attempts += 1
  1902. if attempts >= self.RETRY_ATTEMPTS:
  1903. raise
  1904. def _get_response(self, full_url, headers, timeout):
  1905. try:
  1906. AWSRequest = botocore.awsrequest.AWSRequest
  1907. request = AWSRequest(method='GET', url=full_url, headers=headers)
  1908. response = self._session.send(request.prepare())
  1909. response_text = response.content.decode('utf-8')
  1910. if response.status_code != 200:
  1911. raise MetadataRetrievalError(
  1912. error_msg=(
  1913. "Received non 200 response (%s) from ECS metadata: %s"
  1914. ) % (response.status_code, response_text))
  1915. try:
  1916. return json.loads(response_text)
  1917. except ValueError:
  1918. error_msg = (
  1919. "Unable to parse JSON returned from ECS metadata services"
  1920. )
  1921. logger.debug('%s:%s', error_msg, response_text)
  1922. raise MetadataRetrievalError(error_msg=error_msg)
  1923. except RETRYABLE_HTTP_ERRORS as e:
  1924. error_msg = ("Received error when attempting to retrieve "
  1925. "ECS metadata: %s" % e)
  1926. raise MetadataRetrievalError(error_msg=error_msg)
  1927. def full_url(self, relative_uri):
  1928. return 'http://%s%s' % (self.IP_ADDRESS, relative_uri)
  1929. def get_environ_proxies(url):
  1930. if should_bypass_proxies(url):
  1931. return {}
  1932. else:
  1933. return getproxies()
  1934. def should_bypass_proxies(url):
  1935. """
  1936. Returns whether we should bypass proxies or not.
  1937. """
  1938. # NOTE: requests allowed for ip/cidr entries in no_proxy env that we don't
  1939. # support current as urllib only checks DNS suffix
  1940. # If the system proxy settings indicate that this URL should be bypassed,
  1941. # don't proxy.
  1942. # The proxy_bypass function is incredibly buggy on OS X in early versions
  1943. # of Python 2.6, so allow this call to fail. Only catch the specific
  1944. # exceptions we've seen, though: this call failing in other ways can reveal
  1945. # legitimate problems.
  1946. try:
  1947. if proxy_bypass(urlparse(url).netloc):
  1948. return True
  1949. except (TypeError, socket.gaierror):
  1950. pass
  1951. return False
  1952. def get_encoding_from_headers(headers, default='ISO-8859-1'):
  1953. """Returns encodings from given HTTP Header Dict.
  1954. :param headers: dictionary to extract encoding from.
  1955. :param default: default encoding if the content-type is text
  1956. """
  1957. content_type = headers.get('content-type')
  1958. if not content_type:
  1959. return None
  1960. content_type, params = cgi.parse_header(content_type)
  1961. if 'charset' in params:
  1962. return params['charset'].strip("'\"")
  1963. if 'text' in content_type:
  1964. return default
  1965. def calculate_md5(body, **kwargs):
  1966. if isinstance(body, (bytes, bytearray)):
  1967. binary_md5 = _calculate_md5_from_bytes(body)
  1968. else:
  1969. binary_md5 = _calculate_md5_from_file(body)
  1970. return base64.b64encode(binary_md5).decode('ascii')
  1971. def _calculate_md5_from_bytes(body_bytes):
  1972. md5 = get_md5(body_bytes)
  1973. return md5.digest()
  1974. def _calculate_md5_from_file(fileobj):
  1975. start_position = fileobj.tell()
  1976. md5 = get_md5()
  1977. for chunk in iter(lambda: fileobj.read(1024 * 1024), b''):
  1978. md5.update(chunk)
  1979. fileobj.seek(start_position)
  1980. return md5.digest()
  1981. def conditionally_calculate_md5(params, **kwargs):
  1982. """Only add a Content-MD5 if the system supports it."""
  1983. headers = params['headers']
  1984. body = params['body']
  1985. if MD5_AVAILABLE and body is not None and 'Content-MD5' not in headers:
  1986. md5_digest = calculate_md5(body, **kwargs)
  1987. params['headers']['Content-MD5'] = md5_digest
  1988. class FileWebIdentityTokenLoader(object):
  1989. def __init__(self, web_identity_token_path, _open=open):
  1990. self._web_identity_token_path = web_identity_token_path
  1991. self._open = _open
  1992. def __call__(self):
  1993. with self._open(self._web_identity_token_path) as token_file:
  1994. return token_file.read()
  1995. class SSOTokenLoader(object):
  1996. def __init__(self, cache=None):
  1997. if cache is None:
  1998. cache = {}
  1999. self._cache = cache
  2000. def _generate_cache_key(self, start_url):
  2001. return hashlib.sha1(start_url.encode('utf-8')).hexdigest()
  2002. def __call__(self, start_url):
  2003. cache_key = self._generate_cache_key(start_url)
  2004. try:
  2005. token = self._cache[cache_key]
  2006. return token['accessToken']
  2007. except KeyError:
  2008. logger.debug('Failed to load SSO token:', exc_info=True)
  2009. error_msg = (
  2010. 'The SSO access token has either expired or is otherwise '
  2011. 'invalid.'
  2012. )
  2013. raise SSOTokenLoadError(error_msg=error_msg)