loaders.py 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461
  1. # Copyright 2012-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.
  2. #
  3. # Licensed under the Apache License, Version 2.0 (the "License"). You
  4. # may not use this file except in compliance with the License. A copy of
  5. # the License is located at
  6. #
  7. # http://aws.amazon.com/apache2.0/
  8. #
  9. # or in the "license" file accompanying this file. This file is
  10. # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
  11. # ANY KIND, either express or implied. See the License for the specific
  12. # language governing permissions and limitations under the License.
  13. """Module for loading various model files.
  14. This module provides the classes that are used to load models used
  15. by botocore. This can include:
  16. * Service models (e.g. the model for EC2, S3, DynamoDB, etc.)
  17. * Service model extras which customize the service models
  18. * Other models associated with a service (pagination, waiters)
  19. * Non service-specific config (Endpoint data, retry config)
  20. Loading a module is broken down into several steps:
  21. * Determining the path to load
  22. * Search the data_path for files to load
  23. * The mechanics of loading the file
  24. * Searching for extras and applying them to the loaded file
  25. The last item is used so that other faster loading mechanism
  26. besides the default JSON loader can be used.
  27. The Search Path
  28. ===============
  29. Similar to how the PATH environment variable is to finding executables
  30. and the PYTHONPATH environment variable is to finding python modules
  31. to import, the botocore loaders have the concept of a data path exposed
  32. through AWS_DATA_PATH.
  33. This enables end users to provide additional search paths where we
  34. will attempt to load models outside of the models we ship with
  35. botocore. When you create a ``Loader``, there are two paths
  36. automatically added to the model search path:
  37. * <botocore root>/data/
  38. * ~/.aws/models
  39. The first value is the path where all the model files shipped with
  40. botocore are located.
  41. The second path is so that users can just drop new model files in
  42. ``~/.aws/models`` without having to mess around with the AWS_DATA_PATH.
  43. The AWS_DATA_PATH using the platform specific path separator to
  44. separate entries (typically ``:`` on linux and ``;`` on windows).
  45. Directory Layout
  46. ================
  47. The Loader expects a particular directory layout. In order for any
  48. directory specified in AWS_DATA_PATH to be considered, it must have
  49. this structure for service models::
  50. <root>
  51. |
  52. |-- servicename1
  53. | |-- 2012-10-25
  54. | |-- service-2.json
  55. |-- ec2
  56. | |-- 2014-01-01
  57. | | |-- paginators-1.json
  58. | | |-- service-2.json
  59. | | |-- waiters-2.json
  60. | |-- 2015-03-01
  61. | |-- paginators-1.json
  62. | |-- service-2.json
  63. | |-- waiters-2.json
  64. | |-- service-2.sdk-extras.json
  65. That is:
  66. * The root directory contains sub directories that are the name
  67. of the services.
  68. * Within each service directory, there's a sub directory for each
  69. available API version.
  70. * Within each API version, there are model specific files, including
  71. (but not limited to): service-2.json, waiters-2.json, paginators-1.json
  72. The ``-1`` and ``-2`` suffix at the end of the model files denote which version
  73. schema is used within the model. Even though this information is available in
  74. the ``version`` key within the model, this version is also part of the filename
  75. so that code does not need to load the JSON model in order to determine which
  76. version to use.
  77. The ``sdk-extras`` and similar files represent extra data that needs to be
  78. applied to the model after it is loaded. Data in these files might represent
  79. information that doesn't quite fit in the original models, but is still needed
  80. for the sdk. For instance, additional operation parameters might be added here
  81. which don't represent the actual service api.
  82. """
  83. import os
  84. import logging
  85. from botocore import BOTOCORE_ROOT
  86. from botocore.compat import json
  87. from botocore.compat import OrderedDict
  88. from botocore.exceptions import DataNotFoundError, UnknownServiceError
  89. from botocore.utils import deep_merge
  90. logger = logging.getLogger(__name__)
  91. def instance_cache(func):
  92. """Cache the result of a method on a per instance basis.
  93. This is not a general purpose caching decorator. In order
  94. for this to be used, it must be used on methods on an
  95. instance, and that instance *must* provide a
  96. ``self._cache`` dictionary.
  97. """
  98. def _wrapper(self, *args, **kwargs):
  99. key = (func.__name__,) + args
  100. for pair in sorted(kwargs.items()):
  101. key += pair
  102. if key in self._cache:
  103. return self._cache[key]
  104. data = func(self, *args, **kwargs)
  105. self._cache[key] = data
  106. return data
  107. return _wrapper
  108. class JSONFileLoader(object):
  109. """Loader JSON files.
  110. This class can load the default format of models, which is a JSON file.
  111. """
  112. def exists(self, file_path):
  113. """Checks if the file exists.
  114. :type file_path: str
  115. :param file_path: The full path to the file to load without
  116. the '.json' extension.
  117. :return: True if file path exists, False otherwise.
  118. """
  119. return os.path.isfile(file_path + '.json')
  120. def load_file(self, file_path):
  121. """Attempt to load the file path.
  122. :type file_path: str
  123. :param file_path: The full path to the file to load without
  124. the '.json' extension.
  125. :return: The loaded data if it exists, otherwise None.
  126. """
  127. full_path = file_path + '.json'
  128. if not os.path.isfile(full_path):
  129. return
  130. # By default the file will be opened with locale encoding on Python 3.
  131. # We specify "utf8" here to ensure the correct behavior.
  132. with open(full_path, 'rb') as fp:
  133. payload = fp.read().decode('utf-8')
  134. logger.debug("Loading JSON file: %s", full_path)
  135. return json.loads(payload, object_pairs_hook=OrderedDict)
  136. def create_loader(search_path_string=None):
  137. """Create a Loader class.
  138. This factory function creates a loader given a search string path.
  139. :type search_string_path: str
  140. :param search_string_path: The AWS_DATA_PATH value. A string
  141. of data path values separated by the ``os.path.pathsep`` value,
  142. which is typically ``:`` on POSIX platforms and ``;`` on
  143. windows.
  144. :return: A ``Loader`` instance.
  145. """
  146. if search_path_string is None:
  147. return Loader()
  148. paths = []
  149. extra_paths = search_path_string.split(os.pathsep)
  150. for path in extra_paths:
  151. path = os.path.expanduser(os.path.expandvars(path))
  152. paths.append(path)
  153. return Loader(extra_search_paths=paths)
  154. class Loader(object):
  155. """Find and load data models.
  156. This class will handle searching for and loading data models.
  157. The main method used here is ``load_service_model``, which is a
  158. convenience method over ``load_data`` and ``determine_latest_version``.
  159. """
  160. FILE_LOADER_CLASS = JSONFileLoader
  161. # The included models in botocore/data/ that we ship with botocore.
  162. BUILTIN_DATA_PATH = os.path.join(BOTOCORE_ROOT, 'data')
  163. # For convenience we automatically add ~/.aws/models to the data path.
  164. CUSTOMER_DATA_PATH = os.path.join(os.path.expanduser('~'),
  165. '.aws', 'models')
  166. BUILTIN_EXTRAS_TYPES = ['sdk']
  167. def __init__(self, extra_search_paths=None, file_loader=None,
  168. cache=None, include_default_search_paths=True,
  169. include_default_extras=True):
  170. self._cache = {}
  171. if file_loader is None:
  172. file_loader = self.FILE_LOADER_CLASS()
  173. self.file_loader = file_loader
  174. if extra_search_paths is not None:
  175. self._search_paths = extra_search_paths
  176. else:
  177. self._search_paths = []
  178. if include_default_search_paths:
  179. self._search_paths.extend([self.CUSTOMER_DATA_PATH,
  180. self.BUILTIN_DATA_PATH])
  181. self._extras_types = []
  182. if include_default_extras:
  183. self._extras_types.extend(self.BUILTIN_EXTRAS_TYPES)
  184. self._extras_processor = ExtrasProcessor()
  185. @property
  186. def search_paths(self):
  187. return self._search_paths
  188. @property
  189. def extras_types(self):
  190. return self._extras_types
  191. @instance_cache
  192. def list_available_services(self, type_name):
  193. """List all known services.
  194. This will traverse the search path and look for all known
  195. services.
  196. :type type_name: str
  197. :param type_name: The type of the service (service-2,
  198. paginators-1, waiters-2, etc). This is needed because
  199. the list of available services depends on the service
  200. type. For example, the latest API version available for
  201. a resource-1.json file may not be the latest API version
  202. available for a services-2.json file.
  203. :return: A list of all services. The list of services will
  204. be sorted.
  205. """
  206. services = set()
  207. for possible_path in self._potential_locations():
  208. # Any directory in the search path is potentially a service.
  209. # We'll collect any initial list of potential services,
  210. # but we'll then need to further process these directories
  211. # by searching for the corresponding type_name in each
  212. # potential directory.
  213. possible_services = [
  214. d for d in os.listdir(possible_path)
  215. if os.path.isdir(os.path.join(possible_path, d))]
  216. for service_name in possible_services:
  217. full_dirname = os.path.join(possible_path, service_name)
  218. api_versions = os.listdir(full_dirname)
  219. for api_version in api_versions:
  220. full_load_path = os.path.join(full_dirname,
  221. api_version,
  222. type_name)
  223. if self.file_loader.exists(full_load_path):
  224. services.add(service_name)
  225. break
  226. return sorted(services)
  227. @instance_cache
  228. def determine_latest_version(self, service_name, type_name):
  229. """Find the latest API version available for a service.
  230. :type service_name: str
  231. :param service_name: The name of the service.
  232. :type type_name: str
  233. :param type_name: The type of the service (service-2,
  234. paginators-1, waiters-2, etc). This is needed because
  235. the latest API version available can depend on the service
  236. type. For example, the latest API version available for
  237. a resource-1.json file may not be the latest API version
  238. available for a services-2.json file.
  239. :rtype: str
  240. :return: The latest API version. If the service does not exist
  241. or does not have any available API data, then a
  242. ``DataNotFoundError`` exception will be raised.
  243. """
  244. return max(self.list_api_versions(service_name, type_name))
  245. @instance_cache
  246. def list_api_versions(self, service_name, type_name):
  247. """List all API versions available for a particular service type
  248. :type service_name: str
  249. :param service_name: The name of the service
  250. :type type_name: str
  251. :param type_name: The type name for the service (i.e service-2,
  252. paginators-1, etc.)
  253. :rtype: list
  254. :return: A list of API version strings in sorted order.
  255. """
  256. known_api_versions = set()
  257. for possible_path in self._potential_locations(service_name,
  258. must_exist=True,
  259. is_dir=True):
  260. for dirname in os.listdir(possible_path):
  261. full_path = os.path.join(possible_path, dirname, type_name)
  262. # Only add to the known_api_versions if the directory
  263. # contains a service-2, paginators-1, etc. file corresponding
  264. # to the type_name passed in.
  265. if self.file_loader.exists(full_path):
  266. known_api_versions.add(dirname)
  267. if not known_api_versions:
  268. raise DataNotFoundError(data_path=service_name)
  269. return sorted(known_api_versions)
  270. @instance_cache
  271. def load_service_model(self, service_name, type_name, api_version=None):
  272. """Load a botocore service model
  273. This is the main method for loading botocore models (e.g. a service
  274. model, pagination configs, waiter configs, etc.).
  275. :type service_name: str
  276. :param service_name: The name of the service (e.g ``ec2``, ``s3``).
  277. :type type_name: str
  278. :param type_name: The model type. Valid types include, but are not
  279. limited to: ``service-2``, ``paginators-1``, ``waiters-2``.
  280. :type api_version: str
  281. :param api_version: The API version to load. If this is not
  282. provided, then the latest API version will be used.
  283. :type load_extras: bool
  284. :param load_extras: Whether or not to load the tool extras which
  285. contain additional data to be added to the model.
  286. :raises: UnknownServiceError if there is no known service with
  287. the provided service_name.
  288. :raises: DataNotFoundError if no data could be found for the
  289. service_name/type_name/api_version.
  290. :return: The loaded data, as a python type (e.g. dict, list, etc).
  291. """
  292. # Wrapper around the load_data. This will calculate the path
  293. # to call load_data with.
  294. known_services = self.list_available_services(type_name)
  295. if service_name not in known_services:
  296. raise UnknownServiceError(
  297. service_name=service_name,
  298. known_service_names=', '.join(sorted(known_services)))
  299. if api_version is None:
  300. api_version = self.determine_latest_version(
  301. service_name, type_name)
  302. full_path = os.path.join(service_name, api_version, type_name)
  303. model = self.load_data(full_path)
  304. # Load in all the extras
  305. extras_data = self._find_extras(service_name, type_name, api_version)
  306. self._extras_processor.process(model, extras_data)
  307. return model
  308. def _find_extras(self, service_name, type_name, api_version):
  309. """Creates an iterator over all the extras data."""
  310. for extras_type in self.extras_types:
  311. extras_name = '%s.%s-extras' % (type_name, extras_type)
  312. full_path = os.path.join(service_name, api_version, extras_name)
  313. try:
  314. yield self.load_data(full_path)
  315. except DataNotFoundError:
  316. pass
  317. @instance_cache
  318. def load_data(self, name):
  319. """Load data given a data path.
  320. This is a low level method that will search through the various
  321. search paths until it's able to load a value. This is typically
  322. only needed to load *non* model files (such as _endpoints and
  323. _retry). If you need to load model files, you should prefer
  324. ``load_service_model``.
  325. :type name: str
  326. :param name: The data path, i.e ``ec2/2015-03-01/service-2``.
  327. :return: The loaded data. If no data could be found then
  328. a DataNotFoundError is raised.
  329. """
  330. for possible_path in self._potential_locations(name):
  331. found = self.file_loader.load_file(possible_path)
  332. if found is not None:
  333. return found
  334. # We didn't find anything that matched on any path.
  335. raise DataNotFoundError(data_path=name)
  336. def _potential_locations(self, name=None, must_exist=False,
  337. is_dir=False):
  338. # Will give an iterator over the full path of potential locations
  339. # according to the search path.
  340. for path in self.search_paths:
  341. if os.path.isdir(path):
  342. full_path = path
  343. if name is not None:
  344. full_path = os.path.join(path, name)
  345. if not must_exist:
  346. yield full_path
  347. else:
  348. if is_dir and os.path.isdir(full_path):
  349. yield full_path
  350. elif os.path.exists(full_path):
  351. yield full_path
  352. class ExtrasProcessor(object):
  353. """Processes data from extras files into service models."""
  354. def process(self, original_model, extra_models):
  355. """Processes data from a list of loaded extras files into a model
  356. :type original_model: dict
  357. :param original_model: The service model to load all the extras into.
  358. :type extra_models: iterable of dict
  359. :param extra_models: A list of loaded extras models.
  360. """
  361. for extras in extra_models:
  362. self._process(original_model, extras)
  363. def _process(self, model, extra_model):
  364. """Process a single extras model into a service model."""
  365. if 'merge' in extra_model:
  366. deep_merge(model, extra_model['merge'])