config.py 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290
  1. # Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved.
  2. #
  3. # Licensed under the Apache License, Version 2.0 (the "License"). You
  4. # may not use this file except in compliance with the License. A copy of
  5. # the License is located at
  6. #
  7. # http://aws.amazon.com/apache2.0/
  8. #
  9. # or in the "license" file accompanying this file. This file is
  10. # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
  11. # ANY KIND, either express or implied. See the License for the specific
  12. # language governing permissions and limitations under the License.
  13. import copy
  14. from botocore.compat import OrderedDict
  15. from botocore.endpoint import DEFAULT_TIMEOUT, MAX_POOL_CONNECTIONS
  16. from botocore.exceptions import InvalidS3AddressingStyleError
  17. from botocore.exceptions import InvalidRetryConfigurationError
  18. from botocore.exceptions import InvalidMaxRetryAttemptsError
  19. from botocore.exceptions import InvalidRetryModeError
  20. class Config(object):
  21. """Advanced configuration for Botocore clients.
  22. :type region_name: str
  23. :param region_name: The region to use in instantiating the client
  24. :type signature_version: str
  25. :param signature_version: The signature version when signing requests.
  26. :type user_agent: str
  27. :param user_agent: The value to use in the User-Agent header.
  28. :type user_agent_extra: str
  29. :param user_agent_extra: The value to append to the current User-Agent
  30. header value.
  31. :type connect_timeout: float or int
  32. :param connect_timeout: The time in seconds till a timeout exception is
  33. thrown when attempting to make a connection. The default is 60
  34. seconds.
  35. :type read_timeout: float or int
  36. :param read_timeout: The time in seconds till a timeout exception is
  37. thrown when attempting to read from a connection. The default is
  38. 60 seconds.
  39. :type parameter_validation: bool
  40. :param parameter_validation: Whether parameter validation should occur
  41. when serializing requests. The default is True. You can disable
  42. parameter validation for performance reasons. Otherwise, it's
  43. recommended to leave parameter validation enabled.
  44. :type max_pool_connections: int
  45. :param max_pool_connections: The maximum number of connections to
  46. keep in a connection pool. If this value is not set, the default
  47. value of 10 is used.
  48. :type proxies: dict
  49. :param proxies: A dictionary of proxy servers to use by protocol or
  50. endpoint, e.g.:
  51. {'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}.
  52. The proxies are used on each request.
  53. :type proxies_config: dict
  54. :param proxies_config: A dictionary of additional proxy configurations.
  55. Valid keys are:
  56. * 'proxy_ca_bundle' -- The path to a custom certificate bundle to use
  57. when establishing SSL/TLS connections with proxy.
  58. * 'proxy_client_cert' -- The path to a certificate for proxy
  59. TLS client authentication.
  60. When a str is provided it is treated as a path to a proxy client
  61. certificate. When a two element tuple is provided, it will be
  62. interpreted as the path to the client certificate, and the path
  63. to the certificate key.
  64. * 'proxy_use_forwarding_for_https' -- For HTTPS proxies,
  65. forward your requests to HTTPS destinations with an absolute
  66. URI. We strongly recommend you only use this option with
  67. trusted or corporate proxies. Value must be boolean.
  68. :type s3: dict
  69. :param s3: A dictionary of s3 specific configurations.
  70. Valid keys are:
  71. * 'use_accelerate_endpoint' -- Refers to whether to use the S3
  72. Accelerate endpoint. The value must be a boolean. If True, the
  73. client will use the S3 Accelerate endpoint. If the S3 Accelerate
  74. endpoint is being used then the addressing style will always
  75. be virtual.
  76. * 'payload_signing_enabled' -- Refers to whether or not to SHA256
  77. sign sigv4 payloads. By default, this is disabled for streaming
  78. uploads (UploadPart and PutObject).
  79. * 'addressing_style' -- Refers to the style in which to address
  80. s3 endpoints. Values must be a string that equals:
  81. * auto -- Addressing style is chosen for user. Depending
  82. on the configuration of client, the endpoint may be addressed in
  83. the virtual or the path style. Note that this is the default
  84. behavior if no style is specified.
  85. * virtual -- Addressing style is always virtual. The name of the
  86. bucket must be DNS compatible or an exception will be thrown.
  87. Endpoints will be addressed as such: mybucket.s3.amazonaws.com
  88. * path -- Addressing style is always by path. Endpoints will be
  89. addressed as such: s3.amazonaws.com/mybucket
  90. * 'us_east_1_regional_endpoint' - Refers to what S3 endpoint to use
  91. when the region is configured to be us-east-1. Values must be a
  92. string that equals:
  93. * regional -- Use the us-east-1.amazonaws.com endpoint if the
  94. client is configured to use the us-east-1 region.
  95. * legacy -- Use the s3.amazonaws.com endpoint if the client is
  96. configured to use the us-east-1 region. This is the default if
  97. the configuration option is not specified.
  98. :type retries: dict
  99. :param retries: A dictionary for retry specific configurations.
  100. Valid keys are:
  101. * 'total_max_attempts' -- An integer representing the maximum number of
  102. total attempts that will be made on a single request. This includes
  103. the initial request, so a value of 1 indicates that no requests
  104. will be retried. If ``total_max_attempts`` and ``max_attempts``
  105. are both provided, ``total_max_attempts`` takes precedence.
  106. ``total_max_attempts`` is preferred over ``max_attempts`` because
  107. it maps to the ``AWS_MAX_ATTEMPTS`` environment variable and
  108. the ``max_attempts`` config file value.
  109. * 'max_attempts' -- An integer representing the maximum number of
  110. retry attempts that will be made on a single request. For
  111. example, setting this value to 2 will result in the request
  112. being retried at most two times after the initial request. Setting
  113. this value to 0 will result in no retries ever being attempted on
  114. the initial request. If not provided, the number of retries will
  115. default to whatever is modeled, which is typically four retries.
  116. * 'mode' -- A string representing the type of retry mode botocore
  117. should use. Valid values are:
  118. * ``legacy`` - The pre-existing retry behavior.
  119. * ``standard`` - The standardized set of retry rules. This
  120. will also default to 3 max attempts unless overridden.
  121. * ``adaptive`` - Retries with additional client side throttling.
  122. :type client_cert: str, (str, str)
  123. :param client_cert: The path to a certificate for TLS client authentication.
  124. When a str is provided it is treated as a path to a client certificate
  125. to be used when creating a TLS connection.
  126. If a client key is to be provided alongside the client certificate the
  127. client_cert should be set to a tuple of length two where the first
  128. element is the path to the client certificate and the second element is
  129. the path to the certificate key.
  130. :type inject_host_prefix: bool
  131. :param inject_host_prefix: Whether host prefix injection should occur.
  132. Defaults to True.
  133. Setting this to False disables the injection of operation parameters
  134. into the prefix of the hostname. This is useful for clients providing
  135. custom endpoints that should not have their host prefix modified.
  136. """
  137. OPTION_DEFAULTS = OrderedDict([
  138. ('region_name', None),
  139. ('signature_version', None),
  140. ('user_agent', None),
  141. ('user_agent_extra', None),
  142. ('connect_timeout', DEFAULT_TIMEOUT),
  143. ('read_timeout', DEFAULT_TIMEOUT),
  144. ('parameter_validation', True),
  145. ('max_pool_connections', MAX_POOL_CONNECTIONS),
  146. ('proxies', None),
  147. ('proxies_config', None),
  148. ('s3', None),
  149. ('retries', None),
  150. ('client_cert', None),
  151. ('inject_host_prefix', True),
  152. ('endpoint_discovery_enabled', None),
  153. ])
  154. def __init__(self, *args, **kwargs):
  155. self._user_provided_options = self._record_user_provided_options(
  156. args, kwargs)
  157. # Merge the user_provided options onto the default options
  158. config_vars = copy.copy(self.OPTION_DEFAULTS)
  159. config_vars.update(self._user_provided_options)
  160. # Set the attributes based on the config_vars
  161. for key, value in config_vars.items():
  162. setattr(self, key, value)
  163. # Validate the s3 options
  164. self._validate_s3_configuration(self.s3)
  165. self._validate_retry_configuration(self.retries)
  166. def _record_user_provided_options(self, args, kwargs):
  167. option_order = list(self.OPTION_DEFAULTS)
  168. user_provided_options = {}
  169. # Iterate through the kwargs passed through to the constructor and
  170. # map valid keys to the dictionary
  171. for key, value in kwargs.items():
  172. if key in self.OPTION_DEFAULTS:
  173. user_provided_options[key] = value
  174. # The key must exist in the available options
  175. else:
  176. raise TypeError(
  177. 'Got unexpected keyword argument \'%s\'' % key)
  178. # The number of args should not be longer than the allowed
  179. # options
  180. if len(args) > len(option_order):
  181. raise TypeError(
  182. 'Takes at most %s arguments (%s given)' % (
  183. len(option_order), len(args)))
  184. # Iterate through the args passed through to the constructor and map
  185. # them to appropriate keys.
  186. for i, arg in enumerate(args):
  187. # If it a kwarg was specified for the arg, then error out
  188. if option_order[i] in user_provided_options:
  189. raise TypeError(
  190. 'Got multiple values for keyword argument \'%s\'' % (
  191. option_order[i]))
  192. user_provided_options[option_order[i]] = arg
  193. return user_provided_options
  194. def _validate_s3_configuration(self, s3):
  195. if s3 is not None:
  196. addressing_style = s3.get('addressing_style')
  197. if addressing_style not in ['virtual', 'auto', 'path', None]:
  198. raise InvalidS3AddressingStyleError(
  199. s3_addressing_style=addressing_style)
  200. def _validate_retry_configuration(self, retries):
  201. if retries is not None:
  202. for key, value in retries.items():
  203. if key not in ['max_attempts', 'mode', 'total_max_attempts']:
  204. raise InvalidRetryConfigurationError(
  205. retry_config_option=key)
  206. if key == 'max_attempts' and value < 0:
  207. raise InvalidMaxRetryAttemptsError(
  208. provided_max_attempts=value,
  209. min_value=0,
  210. )
  211. if key == 'total_max_attempts' and value < 1:
  212. raise InvalidMaxRetryAttemptsError(
  213. provided_max_attempts=value,
  214. min_value=1,
  215. )
  216. if key == 'mode' and value not in ['legacy', 'standard',
  217. 'adaptive']:
  218. raise InvalidRetryModeError(
  219. provided_retry_mode=value
  220. )
  221. def merge(self, other_config):
  222. """Merges the config object with another config object
  223. This will merge in all non-default values from the provided config
  224. and return a new config object
  225. :type other_config: botocore.config.Config
  226. :param other config: Another config object to merge with. The values
  227. in the provided config object will take precedence in the merging
  228. :returns: A config object built from the merged values of both
  229. config objects.
  230. """
  231. # Make a copy of the current attributes in the config object.
  232. config_options = copy.copy(self._user_provided_options)
  233. # Merge in the user provided options from the other config
  234. config_options.update(other_config._user_provided_options)
  235. # Return a new config object with the merged properties.
  236. return Config(**config_options)