configloader.py 9.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272
  1. # Copyright (c) 2012-2013 Mitch Garnaat http://garnaat.org/
  2. # Copyright 2012-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved.
  3. #
  4. # Licensed under the Apache License, Version 2.0 (the "License"). You
  5. # may not use this file except in compliance with the License. A copy of
  6. # the License is located at
  7. #
  8. # http://aws.amazon.com/apache2.0/
  9. #
  10. # or in the "license" file accompanying this file. This file is
  11. # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
  12. # ANY KIND, either express or implied. See the License for the specific
  13. # language governing permissions and limitations under the License.
  14. import os
  15. import shlex
  16. import copy
  17. import sys
  18. from botocore.compat import six
  19. import botocore.exceptions
  20. def multi_file_load_config(*filenames):
  21. """Load and combine multiple INI configs with profiles.
  22. This function will take a list of filesnames and return
  23. a single dictionary that represents the merging of the loaded
  24. config files.
  25. If any of the provided filenames does not exist, then that file
  26. is ignored. It is therefore ok to provide a list of filenames,
  27. some of which may not exist.
  28. Configuration files are **not** deep merged, only the top level
  29. keys are merged. The filenames should be passed in order of
  30. precedence. The first config file has precedence over the
  31. second config file, which has precedence over the third config file,
  32. etc. The only exception to this is that the "profiles" key is
  33. merged to combine profiles from multiple config files into a
  34. single profiles mapping. However, if a profile is defined in
  35. multiple config files, then the config file with the highest
  36. precedence is used. Profile values themselves are not merged.
  37. For example::
  38. FileA FileB FileC
  39. [foo] [foo] [bar]
  40. a=1 a=2 a=3
  41. b=2
  42. [bar] [baz] [profile a]
  43. a=2 a=3 region=e
  44. [profile a] [profile b] [profile c]
  45. region=c region=d region=f
  46. The final result of ``multi_file_load_config(FileA, FileB, FileC)``
  47. would be::
  48. {"foo": {"a": 1}, "bar": {"a": 2}, "baz": {"a": 3},
  49. "profiles": {"a": {"region": "c"}}, {"b": {"region": d"}},
  50. {"c": {"region": "f"}}}
  51. Note that the "foo" key comes from A, even though it's defined in both
  52. FileA and FileB. Because "foo" was defined in FileA first, then the values
  53. for "foo" from FileA are used and the values for "foo" from FileB are
  54. ignored. Also note where the profiles originate from. Profile "a"
  55. comes FileA, profile "b" comes from FileB, and profile "c" comes
  56. from FileC.
  57. """
  58. configs = []
  59. profiles = []
  60. for filename in filenames:
  61. try:
  62. loaded = load_config(filename)
  63. except botocore.exceptions.ConfigNotFound:
  64. continue
  65. profiles.append(loaded.pop('profiles'))
  66. configs.append(loaded)
  67. merged_config = _merge_list_of_dicts(configs)
  68. merged_profiles = _merge_list_of_dicts(profiles)
  69. merged_config['profiles'] = merged_profiles
  70. return merged_config
  71. def _merge_list_of_dicts(list_of_dicts):
  72. merged_dicts = {}
  73. for single_dict in list_of_dicts:
  74. for key, value in single_dict.items():
  75. if key not in merged_dicts:
  76. merged_dicts[key] = value
  77. return merged_dicts
  78. def load_config(config_filename):
  79. """Parse a INI config with profiles.
  80. This will parse an INI config file and map top level profiles
  81. into a top level "profile" key.
  82. If you want to parse an INI file and map all section names to
  83. top level keys, use ``raw_config_parse`` instead.
  84. """
  85. parsed = raw_config_parse(config_filename)
  86. return build_profile_map(parsed)
  87. def raw_config_parse(config_filename, parse_subsections=True):
  88. """Returns the parsed INI config contents.
  89. Each section name is a top level key.
  90. :param config_filename: The name of the INI file to parse
  91. :param parse_subsections: If True, parse indented blocks as
  92. subsections that represent their own configuration dictionary.
  93. For example, if the config file had the contents::
  94. s3 =
  95. signature_version = s3v4
  96. addressing_style = path
  97. The resulting ``raw_config_parse`` would be::
  98. {'s3': {'signature_version': 's3v4', 'addressing_style': 'path'}}
  99. If False, do not try to parse subsections and return the indented
  100. block as its literal value::
  101. {'s3': '\nsignature_version = s3v4\naddressing_style = path'}
  102. :returns: A dict with keys for each profile found in the config
  103. file and the value of each key being a dict containing name
  104. value pairs found in that profile.
  105. :raises: ConfigNotFound, ConfigParseError
  106. """
  107. config = {}
  108. path = config_filename
  109. if path is not None:
  110. path = os.path.expandvars(path)
  111. path = os.path.expanduser(path)
  112. if not os.path.isfile(path):
  113. raise botocore.exceptions.ConfigNotFound(path=_unicode_path(path))
  114. cp = six.moves.configparser.RawConfigParser()
  115. try:
  116. cp.read([path])
  117. except (six.moves.configparser.Error, UnicodeDecodeError):
  118. raise botocore.exceptions.ConfigParseError(
  119. path=_unicode_path(path))
  120. else:
  121. for section in cp.sections():
  122. config[section] = {}
  123. for option in cp.options(section):
  124. config_value = cp.get(section, option)
  125. if parse_subsections and config_value.startswith('\n'):
  126. # Then we need to parse the inner contents as
  127. # hierarchical. We support a single level
  128. # of nesting for now.
  129. try:
  130. config_value = _parse_nested(config_value)
  131. except ValueError:
  132. raise botocore.exceptions.ConfigParseError(
  133. path=_unicode_path(path))
  134. config[section][option] = config_value
  135. return config
  136. def _unicode_path(path):
  137. if isinstance(path, six.text_type):
  138. return path
  139. # According to the documentation getfilesystemencoding can return None
  140. # on unix in which case the default encoding is used instead.
  141. filesystem_encoding = sys.getfilesystemencoding()
  142. if filesystem_encoding is None:
  143. filesystem_encoding = sys.getdefaultencoding()
  144. return path.decode(filesystem_encoding, 'replace')
  145. def _parse_nested(config_value):
  146. # Given a value like this:
  147. # \n
  148. # foo = bar
  149. # bar = baz
  150. # We need to parse this into
  151. # {'foo': 'bar', 'bar': 'baz}
  152. parsed = {}
  153. for line in config_value.splitlines():
  154. line = line.strip()
  155. if not line:
  156. continue
  157. # The caller will catch ValueError
  158. # and raise an appropriate error
  159. # if this fails.
  160. key, value = line.split('=', 1)
  161. parsed[key.strip()] = value.strip()
  162. return parsed
  163. def build_profile_map(parsed_ini_config):
  164. """Convert the parsed INI config into a profile map.
  165. The config file format requires that every profile except the
  166. default to be prepended with "profile", e.g.::
  167. [profile test]
  168. aws_... = foo
  169. aws_... = bar
  170. [profile bar]
  171. aws_... = foo
  172. aws_... = bar
  173. # This is *not* a profile
  174. [preview]
  175. otherstuff = 1
  176. # Neither is this
  177. [foobar]
  178. morestuff = 2
  179. The build_profile_map will take a parsed INI config file where each top
  180. level key represents a section name, and convert into a format where all
  181. the profiles are under a single top level "profiles" key, and each key in
  182. the sub dictionary is a profile name. For example, the above config file
  183. would be converted from::
  184. {"profile test": {"aws_...": "foo", "aws...": "bar"},
  185. "profile bar": {"aws...": "foo", "aws...": "bar"},
  186. "preview": {"otherstuff": ...},
  187. "foobar": {"morestuff": ...},
  188. }
  189. into::
  190. {"profiles": {"test": {"aws_...": "foo", "aws...": "bar"},
  191. "bar": {"aws...": "foo", "aws...": "bar"},
  192. "preview": {"otherstuff": ...},
  193. "foobar": {"morestuff": ...},
  194. }
  195. If there are no profiles in the provided parsed INI contents, then
  196. an empty dict will be the value associated with the ``profiles`` key.
  197. .. note::
  198. This will not mutate the passed in parsed_ini_config. Instead it will
  199. make a deepcopy and return that value.
  200. """
  201. parsed_config = copy.deepcopy(parsed_ini_config)
  202. profiles = {}
  203. final_config = {}
  204. for key, values in parsed_config.items():
  205. if key.startswith("profile"):
  206. try:
  207. parts = shlex.split(key)
  208. except ValueError:
  209. continue
  210. if len(parts) == 2:
  211. profiles[parts[1]] = values
  212. elif key == 'default':
  213. # default section is special and is considered a profile
  214. # name but we don't require you use 'profile "default"'
  215. # as a section.
  216. profiles[key] = values
  217. else:
  218. final_config[key] = values
  219. final_config['profiles'] = profiles
  220. return final_config