args.py 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370
  1. # Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved.
  2. #
  3. # Licensed under the Apache License, Version 2.0 (the "License"). You
  4. # may not use this file except in compliance with the License. A copy of
  5. # the License is located at
  6. #
  7. # http://aws.amazon.com/apache2.0/
  8. #
  9. # or in the "license" file accompanying this file. This file is
  10. # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
  11. # ANY KIND, either express or implied. See the License for the specific
  12. # language governing permissions and limitations under the License.
  13. """Internal module to help with normalizing botocore client args.
  14. This module (and all function/classes within this module) should be
  15. considered internal, and *not* a public API.
  16. """
  17. import copy
  18. import logging
  19. import socket
  20. import botocore.exceptions
  21. import botocore.serialize
  22. import botocore.utils
  23. from botocore.signers import RequestSigner
  24. from botocore.config import Config
  25. from botocore.endpoint import EndpointCreator
  26. logger = logging.getLogger(__name__)
  27. VALID_REGIONAL_ENDPOINTS_CONFIG = [
  28. 'legacy',
  29. 'regional',
  30. ]
  31. LEGACY_GLOBAL_STS_REGIONS = [
  32. 'ap-northeast-1',
  33. 'ap-south-1',
  34. 'ap-southeast-1',
  35. 'ap-southeast-2',
  36. 'aws-global',
  37. 'ca-central-1',
  38. 'eu-central-1',
  39. 'eu-north-1',
  40. 'eu-west-1',
  41. 'eu-west-2',
  42. 'eu-west-3',
  43. 'sa-east-1',
  44. 'us-east-1',
  45. 'us-east-2',
  46. 'us-west-1',
  47. 'us-west-2',
  48. ]
  49. class ClientArgsCreator(object):
  50. def __init__(self, event_emitter, user_agent, response_parser_factory,
  51. loader, exceptions_factory, config_store):
  52. self._event_emitter = event_emitter
  53. self._user_agent = user_agent
  54. self._response_parser_factory = response_parser_factory
  55. self._loader = loader
  56. self._exceptions_factory = exceptions_factory
  57. self._config_store = config_store
  58. def get_client_args(self, service_model, region_name, is_secure,
  59. endpoint_url, verify, credentials, scoped_config,
  60. client_config, endpoint_bridge):
  61. final_args = self.compute_client_args(
  62. service_model, client_config, endpoint_bridge, region_name,
  63. endpoint_url, is_secure, scoped_config)
  64. service_name = final_args['service_name']
  65. parameter_validation = final_args['parameter_validation']
  66. endpoint_config = final_args['endpoint_config']
  67. protocol = final_args['protocol']
  68. config_kwargs = final_args['config_kwargs']
  69. s3_config = final_args['s3_config']
  70. partition = endpoint_config['metadata'].get('partition', None)
  71. socket_options = final_args['socket_options']
  72. signing_region = endpoint_config['signing_region']
  73. endpoint_region_name = endpoint_config['region_name']
  74. event_emitter = copy.copy(self._event_emitter)
  75. signer = RequestSigner(
  76. service_model.service_id, signing_region,
  77. endpoint_config['signing_name'],
  78. endpoint_config['signature_version'],
  79. credentials, event_emitter
  80. )
  81. config_kwargs['s3'] = s3_config
  82. new_config = Config(**config_kwargs)
  83. endpoint_creator = EndpointCreator(event_emitter)
  84. endpoint = endpoint_creator.create_endpoint(
  85. service_model, region_name=endpoint_region_name,
  86. endpoint_url=endpoint_config['endpoint_url'], verify=verify,
  87. response_parser_factory=self._response_parser_factory,
  88. max_pool_connections=new_config.max_pool_connections,
  89. proxies=new_config.proxies,
  90. timeout=(new_config.connect_timeout, new_config.read_timeout),
  91. socket_options=socket_options,
  92. client_cert=new_config.client_cert,
  93. proxies_config=new_config.proxies_config)
  94. serializer = botocore.serialize.create_serializer(
  95. protocol, parameter_validation)
  96. response_parser = botocore.parsers.create_parser(protocol)
  97. return {
  98. 'serializer': serializer,
  99. 'endpoint': endpoint,
  100. 'response_parser': response_parser,
  101. 'event_emitter': event_emitter,
  102. 'request_signer': signer,
  103. 'service_model': service_model,
  104. 'loader': self._loader,
  105. 'client_config': new_config,
  106. 'partition': partition,
  107. 'exceptions_factory': self._exceptions_factory
  108. }
  109. def compute_client_args(self, service_model, client_config,
  110. endpoint_bridge, region_name, endpoint_url,
  111. is_secure, scoped_config):
  112. service_name = service_model.endpoint_prefix
  113. protocol = service_model.metadata['protocol']
  114. parameter_validation = True
  115. if client_config and not client_config.parameter_validation:
  116. parameter_validation = False
  117. elif scoped_config:
  118. raw_value = scoped_config.get('parameter_validation')
  119. if raw_value is not None:
  120. parameter_validation = botocore.utils.ensure_boolean(raw_value)
  121. # Override the user agent if specified in the client config.
  122. user_agent = self._user_agent
  123. if client_config is not None:
  124. if client_config.user_agent is not None:
  125. user_agent = client_config.user_agent
  126. if client_config.user_agent_extra is not None:
  127. user_agent += ' %s' % client_config.user_agent_extra
  128. s3_config = self.compute_s3_config(client_config)
  129. endpoint_config = self._compute_endpoint_config(
  130. service_name=service_name,
  131. region_name=region_name,
  132. endpoint_url=endpoint_url,
  133. is_secure=is_secure,
  134. endpoint_bridge=endpoint_bridge,
  135. s3_config=s3_config,
  136. )
  137. # Create a new client config to be passed to the client based
  138. # on the final values. We do not want the user to be able
  139. # to try to modify an existing client with a client config.
  140. config_kwargs = dict(
  141. region_name=endpoint_config['region_name'],
  142. signature_version=endpoint_config['signature_version'],
  143. user_agent=user_agent)
  144. if client_config is not None:
  145. config_kwargs.update(
  146. connect_timeout=client_config.connect_timeout,
  147. read_timeout=client_config.read_timeout,
  148. max_pool_connections=client_config.max_pool_connections,
  149. proxies=client_config.proxies,
  150. proxies_config=client_config.proxies_config,
  151. retries=client_config.retries,
  152. client_cert=client_config.client_cert,
  153. inject_host_prefix=client_config.inject_host_prefix,
  154. )
  155. self._compute_retry_config(config_kwargs)
  156. s3_config = self.compute_s3_config(client_config)
  157. return {
  158. 'service_name': service_name,
  159. 'parameter_validation': parameter_validation,
  160. 'user_agent': user_agent,
  161. 'endpoint_config': endpoint_config,
  162. 'protocol': protocol,
  163. 'config_kwargs': config_kwargs,
  164. 's3_config': s3_config,
  165. 'socket_options': self._compute_socket_options(scoped_config)
  166. }
  167. def compute_s3_config(self, client_config):
  168. s3_configuration = self._config_store.get_config_variable('s3')
  169. # Next specific client config values takes precedence over
  170. # specific values in the scoped config.
  171. if client_config is not None:
  172. if client_config.s3 is not None:
  173. if s3_configuration is None:
  174. s3_configuration = client_config.s3
  175. else:
  176. # The current s3_configuration dictionary may be
  177. # from a source that only should be read from so
  178. # we want to be safe and just make a copy of it to modify
  179. # before it actually gets updated.
  180. s3_configuration = s3_configuration.copy()
  181. s3_configuration.update(client_config.s3)
  182. return s3_configuration
  183. def _compute_endpoint_config(self, service_name, region_name, endpoint_url,
  184. is_secure, endpoint_bridge, s3_config):
  185. resolve_endpoint_kwargs = {
  186. 'service_name': service_name,
  187. 'region_name': region_name,
  188. 'endpoint_url': endpoint_url,
  189. 'is_secure': is_secure,
  190. 'endpoint_bridge': endpoint_bridge,
  191. }
  192. if service_name == 's3':
  193. return self._compute_s3_endpoint_config(
  194. s3_config=s3_config, **resolve_endpoint_kwargs)
  195. if service_name == 'sts':
  196. return self._compute_sts_endpoint_config(**resolve_endpoint_kwargs)
  197. return self._resolve_endpoint(**resolve_endpoint_kwargs)
  198. def _compute_s3_endpoint_config(self, s3_config,
  199. **resolve_endpoint_kwargs):
  200. force_s3_global = self._should_force_s3_global(
  201. resolve_endpoint_kwargs['region_name'], s3_config)
  202. if force_s3_global:
  203. resolve_endpoint_kwargs['region_name'] = None
  204. endpoint_config = self._resolve_endpoint(**resolve_endpoint_kwargs)
  205. self._set_region_if_custom_s3_endpoint(
  206. endpoint_config, resolve_endpoint_kwargs['endpoint_bridge'])
  207. # For backwards compatibility reasons, we want to make sure the
  208. # client.meta.region_name will remain us-east-1 if we forced the
  209. # endpoint to be the global region. Specifically, if this value
  210. # changes to aws-global, it breaks logic where a user is checking
  211. # for us-east-1 as the global endpoint such as in creating buckets.
  212. if force_s3_global and endpoint_config['region_name'] == 'aws-global':
  213. endpoint_config['region_name'] = 'us-east-1'
  214. return endpoint_config
  215. def _should_force_s3_global(self, region_name, s3_config):
  216. s3_regional_config = 'legacy'
  217. if s3_config and 'us_east_1_regional_endpoint' in s3_config:
  218. s3_regional_config = s3_config['us_east_1_regional_endpoint']
  219. self._validate_s3_regional_config(s3_regional_config)
  220. return (
  221. s3_regional_config == 'legacy' and
  222. region_name in ['us-east-1', None]
  223. )
  224. def _validate_s3_regional_config(self, config_val):
  225. if config_val not in VALID_REGIONAL_ENDPOINTS_CONFIG:
  226. raise botocore.exceptions.\
  227. InvalidS3UsEast1RegionalEndpointConfigError(
  228. s3_us_east_1_regional_endpoint_config=config_val)
  229. def _set_region_if_custom_s3_endpoint(self, endpoint_config,
  230. endpoint_bridge):
  231. # If a user is providing a custom URL, the endpoint resolver will
  232. # refuse to infer a signing region. If we want to default to s3v4,
  233. # we have to account for this.
  234. if endpoint_config['signing_region'] is None \
  235. and endpoint_config['region_name'] is None:
  236. endpoint = endpoint_bridge.resolve('s3')
  237. endpoint_config['signing_region'] = endpoint['signing_region']
  238. endpoint_config['region_name'] = endpoint['region_name']
  239. def _compute_sts_endpoint_config(self, **resolve_endpoint_kwargs):
  240. endpoint_config = self._resolve_endpoint(**resolve_endpoint_kwargs)
  241. if self._should_set_global_sts_endpoint(
  242. resolve_endpoint_kwargs['region_name'],
  243. resolve_endpoint_kwargs['endpoint_url']):
  244. self._set_global_sts_endpoint(
  245. endpoint_config, resolve_endpoint_kwargs['is_secure'])
  246. return endpoint_config
  247. def _should_set_global_sts_endpoint(self, region_name, endpoint_url):
  248. if endpoint_url:
  249. return False
  250. return (
  251. self._get_sts_regional_endpoints_config() == 'legacy' and
  252. region_name in LEGACY_GLOBAL_STS_REGIONS
  253. )
  254. def _get_sts_regional_endpoints_config(self):
  255. sts_regional_endpoints_config = self._config_store.get_config_variable(
  256. 'sts_regional_endpoints')
  257. if not sts_regional_endpoints_config:
  258. sts_regional_endpoints_config = 'legacy'
  259. if sts_regional_endpoints_config not in \
  260. VALID_REGIONAL_ENDPOINTS_CONFIG:
  261. raise botocore.exceptions.InvalidSTSRegionalEndpointsConfigError(
  262. sts_regional_endpoints_config=sts_regional_endpoints_config)
  263. return sts_regional_endpoints_config
  264. def _set_global_sts_endpoint(self, endpoint_config, is_secure):
  265. scheme = 'https' if is_secure else 'http'
  266. endpoint_config['endpoint_url'] = '%s://sts.amazonaws.com' % scheme
  267. endpoint_config['signing_region'] = 'us-east-1'
  268. def _resolve_endpoint(self, service_name, region_name,
  269. endpoint_url, is_secure, endpoint_bridge):
  270. return endpoint_bridge.resolve(
  271. service_name, region_name, endpoint_url, is_secure)
  272. def _compute_socket_options(self, scoped_config):
  273. # This disables Nagle's algorithm and is the default socket options
  274. # in urllib3.
  275. socket_options = [(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)]
  276. if scoped_config:
  277. # Enables TCP Keepalive if specified in shared config file.
  278. if self._ensure_boolean(scoped_config.get('tcp_keepalive', False)):
  279. socket_options.append(
  280. (socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1))
  281. return socket_options
  282. def _compute_retry_config(self, config_kwargs):
  283. self._compute_retry_max_attempts(config_kwargs)
  284. self._compute_retry_mode(config_kwargs)
  285. def _compute_retry_max_attempts(self, config_kwargs):
  286. # There's a pre-existing max_attempts client config value that actually
  287. # means max *retry* attempts. There's also a `max_attempts` we pull
  288. # from the config store that means *total attempts*, which includes the
  289. # intitial request. We can't change what `max_attempts` means in
  290. # client config so we try to normalize everything to a new
  291. # "total_max_attempts" variable. We ensure that after this, the only
  292. # configuration for "max attempts" is the 'total_max_attempts' key.
  293. # An explicitly provided max_attempts in the client config
  294. # overrides everything.
  295. retries = config_kwargs.get('retries')
  296. if retries is not None:
  297. if 'total_max_attempts' in retries:
  298. retries.pop('max_attempts', None)
  299. return
  300. if 'max_attempts' in retries:
  301. value = retries.pop('max_attempts')
  302. # client config max_attempts means total retries so we
  303. # have to add one for 'total_max_attempts' to account
  304. # for the initial request.
  305. retries['total_max_attempts'] = value + 1
  306. return
  307. # Otherwise we'll check the config store which checks env vars,
  308. # config files, etc. There is no default value for max_attempts
  309. # so if this returns None and we don't set a default value here.
  310. max_attempts = self._config_store.get_config_variable('max_attempts')
  311. if max_attempts is not None:
  312. if retries is None:
  313. retries = {}
  314. config_kwargs['retries'] = retries
  315. retries['total_max_attempts'] = max_attempts
  316. def _compute_retry_mode(self, config_kwargs):
  317. retries = config_kwargs.get('retries')
  318. if retries is None:
  319. retries = {}
  320. config_kwargs['retries'] = retries
  321. elif 'mode' in retries:
  322. # If there's a retry mode explicitly set in the client config
  323. # that overrides everything.
  324. return
  325. retry_mode = self._config_store.get_config_variable('retry_mode')
  326. if retry_mode is None:
  327. retry_mode = 'legacy'
  328. retries['mode'] = retry_mode
  329. def _ensure_boolean(self, val):
  330. if isinstance(val, bool):
  331. return val
  332. else:
  333. return val.lower() == 'true'