client.py 42 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972
  1. # Copyright 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved.
  2. #
  3. # Licensed under the Apache License, Version 2.0 (the "License"). You
  4. # may not use this file except in compliance with the License. A copy of
  5. # the License is located at
  6. #
  7. # http://aws.amazon.com/apache2.0/
  8. #
  9. # or in the "license" file accompanying this file. This file is
  10. # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
  11. # ANY KIND, either express or implied. See the License for the specific
  12. # language governing permissions and limitations under the License.
  13. import logging
  14. import functools
  15. from botocore import waiter, xform_name
  16. from botocore.args import ClientArgsCreator
  17. from botocore.auth import AUTH_TYPE_MAPS
  18. from botocore.awsrequest import prepare_request_dict
  19. from botocore.docs.docstring import ClientMethodDocstring
  20. from botocore.docs.docstring import PaginatorDocstring
  21. from botocore.exceptions import (
  22. ClientError, DataNotFoundError, OperationNotPageableError,
  23. UnknownSignatureVersionError, InvalidEndpointDiscoveryConfigurationError,
  24. UnknownFIPSEndpointError,
  25. )
  26. from botocore.hooks import first_non_none_response
  27. from botocore.model import ServiceModel
  28. from botocore.paginate import Paginator
  29. from botocore.utils import (
  30. CachedProperty, get_service_module_name, S3RegionRedirector,
  31. S3ArnParamHandler, S3EndpointSetter, ensure_boolean,
  32. S3ControlArnParamHandler, S3ControlEndpointSetter,
  33. )
  34. from botocore.args import ClientArgsCreator
  35. from botocore import UNSIGNED
  36. # Keep this imported. There's pre-existing code that uses
  37. # "from botocore.client import Config".
  38. from botocore.config import Config
  39. from botocore.history import get_global_history_recorder
  40. from botocore.discovery import (
  41. EndpointDiscoveryHandler, EndpointDiscoveryManager,
  42. block_endpoint_discovery_required_operations
  43. )
  44. from botocore.retries import standard
  45. from botocore.retries import adaptive
  46. logger = logging.getLogger(__name__)
  47. history_recorder = get_global_history_recorder()
  48. class ClientCreator(object):
  49. """Creates client objects for a service."""
  50. def __init__(self, loader, endpoint_resolver, user_agent, event_emitter,
  51. retry_handler_factory, retry_config_translator,
  52. response_parser_factory=None, exceptions_factory=None,
  53. config_store=None):
  54. self._loader = loader
  55. self._endpoint_resolver = endpoint_resolver
  56. self._user_agent = user_agent
  57. self._event_emitter = event_emitter
  58. self._retry_handler_factory = retry_handler_factory
  59. self._retry_config_translator = retry_config_translator
  60. self._response_parser_factory = response_parser_factory
  61. self._exceptions_factory = exceptions_factory
  62. # TODO: Migrate things away from scoped_config in favor of the
  63. # config_store. The config store can pull things from both the scoped
  64. # config and environment variables (and potentially more in the
  65. # future).
  66. self._config_store = config_store
  67. def create_client(self, service_name, region_name, is_secure=True,
  68. endpoint_url=None, verify=None,
  69. credentials=None, scoped_config=None,
  70. api_version=None,
  71. client_config=None):
  72. responses = self._event_emitter.emit(
  73. 'choose-service-name', service_name=service_name)
  74. service_name = first_non_none_response(responses, default=service_name)
  75. service_model = self._load_service_model(service_name, api_version)
  76. cls = self._create_client_class(service_name, service_model)
  77. endpoint_bridge = ClientEndpointBridge(
  78. self._endpoint_resolver, scoped_config, client_config,
  79. service_signing_name=service_model.metadata.get('signingName'))
  80. client_args = self._get_client_args(
  81. service_model, region_name, is_secure, endpoint_url,
  82. verify, credentials, scoped_config, client_config, endpoint_bridge)
  83. service_client = cls(**client_args)
  84. self._register_retries(service_client)
  85. self._register_s3_events(
  86. service_client, endpoint_bridge, endpoint_url, client_config,
  87. scoped_config)
  88. self._register_s3_control_events(
  89. service_client, endpoint_bridge, endpoint_url, client_config,
  90. scoped_config)
  91. self._register_endpoint_discovery(
  92. service_client, endpoint_url, client_config
  93. )
  94. self._register_lazy_block_unknown_fips_pseudo_regions(service_client)
  95. return service_client
  96. def create_client_class(self, service_name, api_version=None):
  97. service_model = self._load_service_model(service_name, api_version)
  98. return self._create_client_class(service_name, service_model)
  99. def _create_client_class(self, service_name, service_model):
  100. class_attributes = self._create_methods(service_model)
  101. py_name_to_operation_name = self._create_name_mapping(service_model)
  102. class_attributes['_PY_TO_OP_NAME'] = py_name_to_operation_name
  103. bases = [BaseClient]
  104. service_id = service_model.service_id.hyphenize()
  105. self._event_emitter.emit(
  106. 'creating-client-class.%s' % service_id,
  107. class_attributes=class_attributes,
  108. base_classes=bases)
  109. class_name = get_service_module_name(service_model)
  110. cls = type(str(class_name), tuple(bases), class_attributes)
  111. return cls
  112. def _load_service_model(self, service_name, api_version=None):
  113. json_model = self._loader.load_service_model(service_name, 'service-2',
  114. api_version=api_version)
  115. service_model = ServiceModel(json_model, service_name=service_name)
  116. return service_model
  117. def _register_retries(self, client):
  118. retry_mode = client.meta.config.retries['mode']
  119. if retry_mode == 'standard':
  120. self._register_v2_standard_retries(client)
  121. elif retry_mode == 'adaptive':
  122. self._register_v2_standard_retries(client)
  123. self._register_v2_adaptive_retries(client)
  124. elif retry_mode == 'legacy':
  125. self._register_legacy_retries(client)
  126. def _register_v2_standard_retries(self, client):
  127. max_attempts = client.meta.config.retries.get('total_max_attempts')
  128. kwargs = {'client': client}
  129. if max_attempts is not None:
  130. kwargs['max_attempts'] = max_attempts
  131. standard.register_retry_handler(**kwargs)
  132. def _register_v2_adaptive_retries(self, client):
  133. adaptive.register_retry_handler(client)
  134. def _register_legacy_retries(self, client):
  135. endpoint_prefix = client.meta.service_model.endpoint_prefix
  136. service_id = client.meta.service_model.service_id
  137. service_event_name = service_id.hyphenize()
  138. # First, we load the entire retry config for all services,
  139. # then pull out just the information we need.
  140. original_config = self._loader.load_data('_retry')
  141. if not original_config:
  142. return
  143. retries = self._transform_legacy_retries(client.meta.config.retries)
  144. retry_config = self._retry_config_translator.build_retry_config(
  145. endpoint_prefix, original_config.get('retry', {}),
  146. original_config.get('definitions', {}),
  147. retries
  148. )
  149. logger.debug("Registering retry handlers for service: %s",
  150. client.meta.service_model.service_name)
  151. handler = self._retry_handler_factory.create_retry_handler(
  152. retry_config, endpoint_prefix)
  153. unique_id = 'retry-config-%s' % service_event_name
  154. client.meta.events.register(
  155. 'needs-retry.%s' % service_event_name, handler,
  156. unique_id=unique_id
  157. )
  158. def _transform_legacy_retries(self, retries):
  159. if retries is None:
  160. return
  161. copied_args = retries.copy()
  162. if 'total_max_attempts' in retries:
  163. copied_args = retries.copy()
  164. copied_args['max_attempts'] = (
  165. copied_args.pop('total_max_attempts') - 1)
  166. return copied_args
  167. def _get_retry_mode(self, client, config_store):
  168. client_retries = client.meta.config.retries
  169. if client_retries is not None and \
  170. client_retries.get('mode') is not None:
  171. return client_retries['mode']
  172. return config_store.get_config_variable('retry_mode') or 'legacy'
  173. def _register_endpoint_discovery(self, client, endpoint_url, config):
  174. if endpoint_url is not None:
  175. # Don't register any handlers in the case of a custom endpoint url
  176. return
  177. # Only attach handlers if the service supports discovery
  178. if client.meta.service_model.endpoint_discovery_operation is None:
  179. return
  180. events = client.meta.events
  181. service_id = client.meta.service_model.service_id.hyphenize()
  182. enabled = False
  183. if config and config.endpoint_discovery_enabled is not None:
  184. enabled = config.endpoint_discovery_enabled
  185. elif self._config_store:
  186. enabled = self._config_store.get_config_variable(
  187. 'endpoint_discovery_enabled')
  188. enabled = self._normalize_endpoint_discovery_config(enabled)
  189. if enabled and self._requires_endpoint_discovery(client, enabled):
  190. discover = enabled is True
  191. manager = EndpointDiscoveryManager(client, always_discover=discover)
  192. handler = EndpointDiscoveryHandler(manager)
  193. handler.register(events, service_id)
  194. else:
  195. events.register('before-parameter-build',
  196. block_endpoint_discovery_required_operations)
  197. def _normalize_endpoint_discovery_config(self, enabled):
  198. """Config must either be a boolean-string or string-literal 'auto'"""
  199. if isinstance(enabled, str):
  200. enabled = enabled.lower().strip()
  201. if enabled == 'auto':
  202. return enabled
  203. elif enabled in ('true', 'false'):
  204. return ensure_boolean(enabled)
  205. elif isinstance(enabled, bool):
  206. return enabled
  207. raise InvalidEndpointDiscoveryConfigurationError(config_value=enabled)
  208. def _requires_endpoint_discovery(self, client, enabled):
  209. if enabled == "auto":
  210. return client.meta.service_model.endpoint_discovery_required
  211. return enabled
  212. def _register_lazy_block_unknown_fips_pseudo_regions(self, client):
  213. # This function blocks usage of FIPS pseudo-regions when the endpoint
  214. # is not explicitly known to exist to the client to prevent accidental
  215. # usage of incorrect or non-FIPS endpoints. This is done lazily by
  216. # registering an exception on the before-sign event to allow for
  217. # general client usage such as can_paginate, exceptions, etc.
  218. region_name = client.meta.region_name
  219. if not region_name or 'fips' not in region_name.lower():
  220. return
  221. partition = client.meta.partition
  222. endpoint_prefix = client.meta.service_model.endpoint_prefix
  223. known_regions = self._endpoint_resolver.get_available_endpoints(
  224. endpoint_prefix,
  225. partition,
  226. allow_non_regional=True,
  227. )
  228. if region_name not in known_regions:
  229. def _lazy_fips_exception(**kwargs):
  230. service_name = client.meta.service_model.service_name
  231. raise UnknownFIPSEndpointError(
  232. region_name=region_name,
  233. service_name=service_name,
  234. )
  235. client.meta.events.register('before-sign', _lazy_fips_exception)
  236. def _register_s3_events(self, client, endpoint_bridge, endpoint_url,
  237. client_config, scoped_config):
  238. if client.meta.service_model.service_name != 's3':
  239. return
  240. S3RegionRedirector(endpoint_bridge, client).register()
  241. S3ArnParamHandler().register(client.meta.events)
  242. S3EndpointSetter(
  243. endpoint_resolver=self._endpoint_resolver,
  244. region=client.meta.region_name,
  245. s3_config=client.meta.config.s3,
  246. endpoint_url=endpoint_url,
  247. partition=client.meta.partition
  248. ).register(client.meta.events)
  249. self._set_s3_presign_signature_version(
  250. client.meta, client_config, scoped_config)
  251. def _register_s3_control_events(
  252. self, client, endpoint_bridge,
  253. endpoint_url, client_config, scoped_config
  254. ):
  255. if client.meta.service_model.service_name != 's3control':
  256. return
  257. S3ControlArnParamHandler().register(client.meta.events)
  258. S3ControlEndpointSetter(
  259. endpoint_resolver=self._endpoint_resolver,
  260. region=client.meta.region_name,
  261. s3_config=client.meta.config.s3,
  262. endpoint_url=endpoint_url,
  263. partition=client.meta.partition
  264. ).register(client.meta.events)
  265. def _set_s3_presign_signature_version(self, client_meta,
  266. client_config, scoped_config):
  267. # This will return the manually configured signature version, or None
  268. # if none was manually set. If a customer manually sets the signature
  269. # version, we always want to use what they set.
  270. provided_signature_version = _get_configured_signature_version(
  271. 's3', client_config, scoped_config)
  272. if provided_signature_version is not None:
  273. return
  274. # Check to see if the region is a region that we know about. If we
  275. # don't know about a region, then we can safely assume it's a new
  276. # region that is sigv4 only, since all new S3 regions only allow sigv4.
  277. # The only exception is aws-global. This is a pseudo-region for the
  278. # global endpoint, we should respect the signature versions it
  279. # supports, which includes v2.
  280. regions = self._endpoint_resolver.get_available_endpoints(
  281. 's3', client_meta.partition)
  282. if client_meta.region_name != 'aws-global' and \
  283. client_meta.region_name not in regions:
  284. return
  285. # If it is a region we know about, we want to default to sigv2, so here
  286. # we check to see if it is available.
  287. endpoint = self._endpoint_resolver.construct_endpoint(
  288. 's3', client_meta.region_name)
  289. signature_versions = endpoint['signatureVersions']
  290. if 's3' not in signature_versions:
  291. return
  292. # We now know that we're in a known region that supports sigv2 and
  293. # the customer hasn't set a signature version so we default the
  294. # signature version to sigv2.
  295. client_meta.events.register(
  296. 'choose-signer.s3', self._default_s3_presign_to_sigv2)
  297. def _default_s3_presign_to_sigv2(self, signature_version, **kwargs):
  298. """
  299. Returns the 's3' (sigv2) signer if presigning an s3 request. This is
  300. intended to be used to set the default signature version for the signer
  301. to sigv2.
  302. :type signature_version: str
  303. :param signature_version: The current client signature version.
  304. :type signing_name: str
  305. :param signing_name: The signing name of the service.
  306. :return: 's3' if the request is an s3 presign request, None otherwise
  307. """
  308. for suffix in ['-query', '-presign-post']:
  309. if signature_version.endswith(suffix):
  310. return 's3' + suffix
  311. def _get_client_args(self, service_model, region_name, is_secure,
  312. endpoint_url, verify, credentials,
  313. scoped_config, client_config, endpoint_bridge):
  314. args_creator = ClientArgsCreator(
  315. self._event_emitter, self._user_agent,
  316. self._response_parser_factory, self._loader,
  317. self._exceptions_factory, config_store=self._config_store)
  318. return args_creator.get_client_args(
  319. service_model, region_name, is_secure, endpoint_url,
  320. verify, credentials, scoped_config, client_config, endpoint_bridge)
  321. def _create_methods(self, service_model):
  322. op_dict = {}
  323. for operation_name in service_model.operation_names:
  324. py_operation_name = xform_name(operation_name)
  325. op_dict[py_operation_name] = self._create_api_method(
  326. py_operation_name, operation_name, service_model)
  327. return op_dict
  328. def _create_name_mapping(self, service_model):
  329. # py_name -> OperationName, for every operation available
  330. # for a service.
  331. mapping = {}
  332. for operation_name in service_model.operation_names:
  333. py_operation_name = xform_name(operation_name)
  334. mapping[py_operation_name] = operation_name
  335. return mapping
  336. def _create_api_method(self, py_operation_name, operation_name,
  337. service_model):
  338. def _api_call(self, *args, **kwargs):
  339. # We're accepting *args so that we can give a more helpful
  340. # error message than TypeError: _api_call takes exactly
  341. # 1 argument.
  342. if args:
  343. raise TypeError(
  344. "%s() only accepts keyword arguments." % py_operation_name)
  345. # The "self" in this scope is referring to the BaseClient.
  346. return self._make_api_call(operation_name, kwargs)
  347. _api_call.__name__ = str(py_operation_name)
  348. # Add the docstring to the client method
  349. operation_model = service_model.operation_model(operation_name)
  350. docstring = ClientMethodDocstring(
  351. operation_model=operation_model,
  352. method_name=operation_name,
  353. event_emitter=self._event_emitter,
  354. method_description=operation_model.documentation,
  355. example_prefix='response = client.%s' % py_operation_name,
  356. include_signature=False
  357. )
  358. _api_call.__doc__ = docstring
  359. return _api_call
  360. class ClientEndpointBridge(object):
  361. """Bridges endpoint data and client creation
  362. This class handles taking out the relevant arguments from the endpoint
  363. resolver and determining which values to use, taking into account any
  364. client configuration options and scope configuration options.
  365. This class also handles determining what, if any, region to use if no
  366. explicit region setting is provided. For example, Amazon S3 client will
  367. utilize "us-east-1" by default if no region can be resolved."""
  368. DEFAULT_ENDPOINT = '{service}.{region}.amazonaws.com'
  369. _DUALSTACK_ENABLED_SERVICES = ['s3', 's3-control']
  370. def __init__(self, endpoint_resolver, scoped_config=None,
  371. client_config=None, default_endpoint=None,
  372. service_signing_name=None):
  373. self.service_signing_name = service_signing_name
  374. self.endpoint_resolver = endpoint_resolver
  375. self.scoped_config = scoped_config
  376. self.client_config = client_config
  377. self.default_endpoint = default_endpoint or self.DEFAULT_ENDPOINT
  378. def resolve(self, service_name, region_name=None, endpoint_url=None,
  379. is_secure=True):
  380. region_name = self._check_default_region(service_name, region_name)
  381. resolved = self.endpoint_resolver.construct_endpoint(
  382. service_name, region_name)
  383. # If we can't resolve the region, we'll attempt to get a global
  384. # endpoint for non-regionalized services (iam, route53, etc)
  385. if not resolved:
  386. # TODO: fallback partition_name should be configurable in the
  387. # future for users to define as needed.
  388. resolved = self.endpoint_resolver.construct_endpoint(
  389. service_name, region_name, partition_name='aws')
  390. if resolved:
  391. return self._create_endpoint(
  392. resolved, service_name, region_name, endpoint_url, is_secure)
  393. else:
  394. return self._assume_endpoint(service_name, region_name,
  395. endpoint_url, is_secure)
  396. def _check_default_region(self, service_name, region_name):
  397. if region_name is not None:
  398. return region_name
  399. # Use the client_config region if no explicit region was provided.
  400. if self.client_config and self.client_config.region_name is not None:
  401. return self.client_config.region_name
  402. def _create_endpoint(self, resolved, service_name, region_name,
  403. endpoint_url, is_secure):
  404. explicit_region = region_name is not None
  405. region_name, signing_region = self._pick_region_values(
  406. resolved, region_name, endpoint_url)
  407. if endpoint_url is None:
  408. if self._is_s3_dualstack_mode(service_name):
  409. endpoint_url = self._create_dualstack_endpoint(
  410. service_name, region_name,
  411. resolved['dnsSuffix'], is_secure, explicit_region)
  412. else:
  413. # Use the sslCommonName over the hostname for Python 2.6 compat.
  414. hostname = resolved.get('sslCommonName', resolved.get('hostname'))
  415. endpoint_url = self._make_url(hostname, is_secure,
  416. resolved.get('protocols', []))
  417. signature_version = self._resolve_signature_version(
  418. service_name, resolved)
  419. signing_name = self._resolve_signing_name(service_name, resolved)
  420. return self._create_result(
  421. service_name=service_name, region_name=region_name,
  422. signing_region=signing_region, signing_name=signing_name,
  423. endpoint_url=endpoint_url, metadata=resolved,
  424. signature_version=signature_version)
  425. def _is_s3_dualstack_mode(self, service_name):
  426. if service_name not in self._DUALSTACK_ENABLED_SERVICES:
  427. return False
  428. # TODO: This normalization logic is duplicated from the
  429. # ClientArgsCreator class. Consolidate everything to
  430. # ClientArgsCreator. _resolve_signature_version also has similarly
  431. # duplicated logic.
  432. client_config = self.client_config
  433. if client_config is not None and client_config.s3 is not None and \
  434. 'use_dualstack_endpoint' in client_config.s3:
  435. # Client config trumps scoped config.
  436. return client_config.s3['use_dualstack_endpoint']
  437. if self.scoped_config is None:
  438. return False
  439. enabled = self.scoped_config.get('s3', {}).get(
  440. 'use_dualstack_endpoint', False)
  441. if enabled in [True, 'True', 'true']:
  442. return True
  443. return False
  444. def _create_dualstack_endpoint(self, service_name, region_name,
  445. dns_suffix, is_secure, explicit_region):
  446. if not explicit_region and region_name == 'aws-global':
  447. # If the region_name passed was not explicitly set, default to
  448. # us-east-1 instead of the modeled default aws-global. Dualstack
  449. # does not support aws-global
  450. region_name = 'us-east-1'
  451. hostname = '{service}.dualstack.{region}.{dns_suffix}'.format(
  452. service=service_name, region=region_name,
  453. dns_suffix=dns_suffix)
  454. # Dualstack supports http and https so were hardcoding this value for
  455. # now. This can potentially move into the endpoints.json file.
  456. return self._make_url(hostname, is_secure, ['http', 'https'])
  457. def _assume_endpoint(self, service_name, region_name, endpoint_url,
  458. is_secure):
  459. if endpoint_url is None:
  460. # Expand the default hostname URI template.
  461. hostname = self.default_endpoint.format(
  462. service=service_name, region=region_name)
  463. endpoint_url = self._make_url(hostname, is_secure,
  464. ['http', 'https'])
  465. logger.debug('Assuming an endpoint for %s, %s: %s',
  466. service_name, region_name, endpoint_url)
  467. # We still want to allow the user to provide an explicit version.
  468. signature_version = self._resolve_signature_version(
  469. service_name, {'signatureVersions': ['v4']})
  470. signing_name = self._resolve_signing_name(service_name, resolved={})
  471. return self._create_result(
  472. service_name=service_name, region_name=region_name,
  473. signing_region=region_name, signing_name=signing_name,
  474. signature_version=signature_version, endpoint_url=endpoint_url,
  475. metadata={})
  476. def _create_result(self, service_name, region_name, signing_region,
  477. signing_name, endpoint_url, signature_version,
  478. metadata):
  479. return {
  480. 'service_name': service_name,
  481. 'region_name': region_name,
  482. 'signing_region': signing_region,
  483. 'signing_name': signing_name,
  484. 'endpoint_url': endpoint_url,
  485. 'signature_version': signature_version,
  486. 'metadata': metadata
  487. }
  488. def _make_url(self, hostname, is_secure, supported_protocols):
  489. if is_secure and 'https' in supported_protocols:
  490. scheme = 'https'
  491. else:
  492. scheme = 'http'
  493. return '%s://%s' % (scheme, hostname)
  494. def _resolve_signing_name(self, service_name, resolved):
  495. # CredentialScope overrides everything else.
  496. if 'credentialScope' in resolved \
  497. and 'service' in resolved['credentialScope']:
  498. return resolved['credentialScope']['service']
  499. # Use the signingName from the model if present.
  500. if self.service_signing_name:
  501. return self.service_signing_name
  502. # Just assume is the same as the service name.
  503. return service_name
  504. def _pick_region_values(self, resolved, region_name, endpoint_url):
  505. signing_region = region_name
  506. if endpoint_url is None:
  507. # Do not use the region name or signing name from the resolved
  508. # endpoint if the user explicitly provides an endpoint_url. This
  509. # would happen if we resolve to an endpoint where the service has
  510. # a "defaults" section that overrides all endpoint with a single
  511. # hostname and credentialScope. This has been the case historically
  512. # for how STS has worked. The only way to resolve an STS endpoint
  513. # was to provide a region_name and an endpoint_url. In that case,
  514. # we would still resolve an endpoint, but we would not use the
  515. # resolved endpointName or signingRegion because we want to allow
  516. # custom endpoints.
  517. region_name = resolved['endpointName']
  518. signing_region = region_name
  519. if 'credentialScope' in resolved \
  520. and 'region' in resolved['credentialScope']:
  521. signing_region = resolved['credentialScope']['region']
  522. return region_name, signing_region
  523. def _resolve_signature_version(self, service_name, resolved):
  524. configured_version = _get_configured_signature_version(
  525. service_name, self.client_config, self.scoped_config)
  526. if configured_version is not None:
  527. return configured_version
  528. # Pick a signature version from the endpoint metadata if present.
  529. if 'signatureVersions' in resolved:
  530. potential_versions = resolved['signatureVersions']
  531. if service_name == 's3':
  532. return 's3v4'
  533. if 'v4' in potential_versions:
  534. return 'v4'
  535. # Now just iterate over the signature versions in order until we
  536. # find the first one that is known to Botocore.
  537. for known in potential_versions:
  538. if known in AUTH_TYPE_MAPS:
  539. return known
  540. raise UnknownSignatureVersionError(
  541. signature_version=resolved.get('signatureVersions'))
  542. class BaseClient(object):
  543. # This is actually reassigned with the py->op_name mapping
  544. # when the client creator creates the subclass. This value is used
  545. # because calls such as client.get_paginator('list_objects') use the
  546. # snake_case name, but we need to know the ListObjects form.
  547. # xform_name() does the ListObjects->list_objects conversion, but
  548. # we need the reverse mapping here.
  549. _PY_TO_OP_NAME = {}
  550. def __init__(self, serializer, endpoint, response_parser,
  551. event_emitter, request_signer, service_model, loader,
  552. client_config, partition, exceptions_factory):
  553. self._serializer = serializer
  554. self._endpoint = endpoint
  555. self._response_parser = response_parser
  556. self._request_signer = request_signer
  557. self._cache = {}
  558. self._loader = loader
  559. self._client_config = client_config
  560. self.meta = ClientMeta(event_emitter, self._client_config,
  561. endpoint.host, service_model,
  562. self._PY_TO_OP_NAME, partition)
  563. self._exceptions_factory = exceptions_factory
  564. self._exceptions = None
  565. self._register_handlers()
  566. def __getattr__(self, item):
  567. event_name = 'getattr.%s.%s' % (
  568. self._service_model.service_id.hyphenize(), item
  569. )
  570. handler, event_response = self.meta.events.emit_until_response(
  571. event_name, client=self)
  572. if event_response is not None:
  573. return event_response
  574. raise AttributeError(
  575. "'%s' object has no attribute '%s'" % (
  576. self.__class__.__name__, item)
  577. )
  578. def _register_handlers(self):
  579. # Register the handler required to sign requests.
  580. service_id = self.meta.service_model.service_id.hyphenize()
  581. self.meta.events.register(
  582. 'request-created.%s' % service_id,
  583. self._request_signer.handler
  584. )
  585. @property
  586. def _service_model(self):
  587. return self.meta.service_model
  588. def _make_api_call(self, operation_name, api_params):
  589. operation_model = self._service_model.operation_model(operation_name)
  590. service_name = self._service_model.service_name
  591. history_recorder.record('API_CALL', {
  592. 'service': service_name,
  593. 'operation': operation_name,
  594. 'params': api_params,
  595. })
  596. if operation_model.deprecated:
  597. logger.debug('Warning: %s.%s() is deprecated',
  598. service_name, operation_name)
  599. request_context = {
  600. 'client_region': self.meta.region_name,
  601. 'client_config': self.meta.config,
  602. 'has_streaming_input': operation_model.has_streaming_input,
  603. 'auth_type': operation_model.auth_type,
  604. }
  605. request_dict = self._convert_to_request_dict(
  606. api_params, operation_model, context=request_context)
  607. service_id = self._service_model.service_id.hyphenize()
  608. handler, event_response = self.meta.events.emit_until_response(
  609. 'before-call.{service_id}.{operation_name}'.format(
  610. service_id=service_id,
  611. operation_name=operation_name),
  612. model=operation_model, params=request_dict,
  613. request_signer=self._request_signer, context=request_context)
  614. if event_response is not None:
  615. http, parsed_response = event_response
  616. else:
  617. http, parsed_response = self._make_request(
  618. operation_model, request_dict, request_context)
  619. self.meta.events.emit(
  620. 'after-call.{service_id}.{operation_name}'.format(
  621. service_id=service_id,
  622. operation_name=operation_name),
  623. http_response=http, parsed=parsed_response,
  624. model=operation_model, context=request_context
  625. )
  626. if http.status_code >= 300:
  627. error_code = parsed_response.get("Error", {}).get("Code")
  628. error_class = self.exceptions.from_code(error_code)
  629. raise error_class(parsed_response, operation_name)
  630. else:
  631. return parsed_response
  632. def _make_request(self, operation_model, request_dict, request_context):
  633. try:
  634. return self._endpoint.make_request(operation_model, request_dict)
  635. except Exception as e:
  636. self.meta.events.emit(
  637. 'after-call-error.{service_id}.{operation_name}'.format(
  638. service_id=self._service_model.service_id.hyphenize(),
  639. operation_name=operation_model.name),
  640. exception=e, context=request_context
  641. )
  642. raise
  643. def _convert_to_request_dict(self, api_params, operation_model,
  644. context=None):
  645. api_params = self._emit_api_params(
  646. api_params, operation_model, context)
  647. request_dict = self._serializer.serialize_to_request(
  648. api_params, operation_model)
  649. if not self._client_config.inject_host_prefix:
  650. request_dict.pop('host_prefix', None)
  651. prepare_request_dict(request_dict, endpoint_url=self._endpoint.host,
  652. user_agent=self._client_config.user_agent,
  653. context=context)
  654. return request_dict
  655. def _emit_api_params(self, api_params, operation_model, context):
  656. # Given the API params provided by the user and the operation_model
  657. # we can serialize the request to a request_dict.
  658. operation_name = operation_model.name
  659. # Emit an event that allows users to modify the parameters at the
  660. # beginning of the method. It allows handlers to modify existing
  661. # parameters or return a new set of parameters to use.
  662. service_id = self._service_model.service_id.hyphenize()
  663. responses = self.meta.events.emit(
  664. 'provide-client-params.{service_id}.{operation_name}'.format(
  665. service_id=service_id,
  666. operation_name=operation_name),
  667. params=api_params, model=operation_model, context=context)
  668. api_params = first_non_none_response(responses, default=api_params)
  669. event_name = (
  670. 'before-parameter-build.{service_id}.{operation_name}')
  671. self.meta.events.emit(
  672. event_name.format(
  673. service_id=service_id,
  674. operation_name=operation_name),
  675. params=api_params, model=operation_model, context=context)
  676. return api_params
  677. def get_paginator(self, operation_name):
  678. """Create a paginator for an operation.
  679. :type operation_name: string
  680. :param operation_name: The operation name. This is the same name
  681. as the method name on the client. For example, if the
  682. method name is ``create_foo``, and you'd normally invoke the
  683. operation as ``client.create_foo(**kwargs)``, if the
  684. ``create_foo`` operation can be paginated, you can use the
  685. call ``client.get_paginator("create_foo")``.
  686. :raise OperationNotPageableError: Raised if the operation is not
  687. pageable. You can use the ``client.can_paginate`` method to
  688. check if an operation is pageable.
  689. :rtype: L{botocore.paginate.Paginator}
  690. :return: A paginator object.
  691. """
  692. if not self.can_paginate(operation_name):
  693. raise OperationNotPageableError(operation_name=operation_name)
  694. else:
  695. actual_operation_name = self._PY_TO_OP_NAME[operation_name]
  696. # Create a new paginate method that will serve as a proxy to
  697. # the underlying Paginator.paginate method. This is needed to
  698. # attach a docstring to the method.
  699. def paginate(self, **kwargs):
  700. return Paginator.paginate(self, **kwargs)
  701. paginator_config = self._cache['page_config'][
  702. actual_operation_name]
  703. # Add the docstring for the paginate method.
  704. paginate.__doc__ = PaginatorDocstring(
  705. paginator_name=actual_operation_name,
  706. event_emitter=self.meta.events,
  707. service_model=self.meta.service_model,
  708. paginator_config=paginator_config,
  709. include_signature=False
  710. )
  711. # Rename the paginator class based on the type of paginator.
  712. paginator_class_name = str('%s.Paginator.%s' % (
  713. get_service_module_name(self.meta.service_model),
  714. actual_operation_name))
  715. # Create the new paginator class
  716. documented_paginator_cls = type(
  717. paginator_class_name, (Paginator,), {'paginate': paginate})
  718. operation_model = self._service_model.operation_model(actual_operation_name)
  719. paginator = documented_paginator_cls(
  720. getattr(self, operation_name),
  721. paginator_config,
  722. operation_model)
  723. return paginator
  724. def can_paginate(self, operation_name):
  725. """Check if an operation can be paginated.
  726. :type operation_name: string
  727. :param operation_name: The operation name. This is the same name
  728. as the method name on the client. For example, if the
  729. method name is ``create_foo``, and you'd normally invoke the
  730. operation as ``client.create_foo(**kwargs)``, if the
  731. ``create_foo`` operation can be paginated, you can use the
  732. call ``client.get_paginator("create_foo")``.
  733. :return: ``True`` if the operation can be paginated,
  734. ``False`` otherwise.
  735. """
  736. if 'page_config' not in self._cache:
  737. try:
  738. page_config = self._loader.load_service_model(
  739. self._service_model.service_name,
  740. 'paginators-1',
  741. self._service_model.api_version)['pagination']
  742. self._cache['page_config'] = page_config
  743. except DataNotFoundError:
  744. self._cache['page_config'] = {}
  745. actual_operation_name = self._PY_TO_OP_NAME[operation_name]
  746. return actual_operation_name in self._cache['page_config']
  747. def _get_waiter_config(self):
  748. if 'waiter_config' not in self._cache:
  749. try:
  750. waiter_config = self._loader.load_service_model(
  751. self._service_model.service_name,
  752. 'waiters-2',
  753. self._service_model.api_version)
  754. self._cache['waiter_config'] = waiter_config
  755. except DataNotFoundError:
  756. self._cache['waiter_config'] = {}
  757. return self._cache['waiter_config']
  758. def get_waiter(self, waiter_name):
  759. """Returns an object that can wait for some condition.
  760. :type waiter_name: str
  761. :param waiter_name: The name of the waiter to get. See the waiters
  762. section of the service docs for a list of available waiters.
  763. :returns: The specified waiter object.
  764. :rtype: botocore.waiter.Waiter
  765. """
  766. config = self._get_waiter_config()
  767. if not config:
  768. raise ValueError("Waiter does not exist: %s" % waiter_name)
  769. model = waiter.WaiterModel(config)
  770. mapping = {}
  771. for name in model.waiter_names:
  772. mapping[xform_name(name)] = name
  773. if waiter_name not in mapping:
  774. raise ValueError("Waiter does not exist: %s" % waiter_name)
  775. return waiter.create_waiter_with_client(
  776. mapping[waiter_name], model, self)
  777. @CachedProperty
  778. def waiter_names(self):
  779. """Returns a list of all available waiters."""
  780. config = self._get_waiter_config()
  781. if not config:
  782. return []
  783. model = waiter.WaiterModel(config)
  784. # Waiter configs is a dict, we just want the waiter names
  785. # which are the keys in the dict.
  786. return [xform_name(name) for name in model.waiter_names]
  787. @property
  788. def exceptions(self):
  789. if self._exceptions is None:
  790. self._exceptions = self._load_exceptions()
  791. return self._exceptions
  792. def _load_exceptions(self):
  793. return self._exceptions_factory.create_client_exceptions(
  794. self._service_model)
  795. class ClientMeta(object):
  796. """Holds additional client methods.
  797. This class holds additional information for clients. It exists for
  798. two reasons:
  799. * To give advanced functionality to clients
  800. * To namespace additional client attributes from the operation
  801. names which are mapped to methods at runtime. This avoids
  802. ever running into collisions with operation names.
  803. """
  804. def __init__(self, events, client_config, endpoint_url, service_model,
  805. method_to_api_mapping, partition):
  806. self.events = events
  807. self._client_config = client_config
  808. self._endpoint_url = endpoint_url
  809. self._service_model = service_model
  810. self._method_to_api_mapping = method_to_api_mapping
  811. self._partition = partition
  812. @property
  813. def service_model(self):
  814. return self._service_model
  815. @property
  816. def region_name(self):
  817. return self._client_config.region_name
  818. @property
  819. def endpoint_url(self):
  820. return self._endpoint_url
  821. @property
  822. def config(self):
  823. return self._client_config
  824. @property
  825. def method_to_api_mapping(self):
  826. return self._method_to_api_mapping
  827. @property
  828. def partition(self):
  829. return self._partition
  830. def _get_configured_signature_version(service_name, client_config,
  831. scoped_config):
  832. """
  833. Gets the manually configured signature version.
  834. :returns: the customer configured signature version, or None if no
  835. signature version was configured.
  836. """
  837. # Client config overrides everything.
  838. if client_config and client_config.signature_version is not None:
  839. return client_config.signature_version
  840. # Scoped config overrides picking from the endpoint metadata.
  841. if scoped_config is not None:
  842. # A given service may have service specific configuration in the
  843. # config file, so we need to check there as well.
  844. service_config = scoped_config.get(service_name)
  845. if service_config is not None and isinstance(service_config, dict):
  846. version = service_config.get('signature_version')
  847. if version:
  848. logger.debug(
  849. "Switching signature version for service %s "
  850. "to version %s based on config file override.",
  851. service_name, version)
  852. return version
  853. return None