build_clib.py 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401
  1. """ Modified version of build_clib that handles fortran source files.
  2. """
  3. import os
  4. from glob import glob
  5. import shutil
  6. from distutils.command.build_clib import build_clib as old_build_clib
  7. from distutils.errors import DistutilsSetupError, DistutilsError, \
  8. DistutilsFileError
  9. from numpy.distutils import log
  10. from distutils.dep_util import newer_group
  11. from numpy.distutils.misc_util import (
  12. filter_sources, get_lib_source_files, get_numpy_include_dirs,
  13. has_cxx_sources, has_f_sources, is_sequence
  14. )
  15. from numpy.distutils.ccompiler_opt import new_ccompiler_opt
  16. # Fix Python distutils bug sf #1718574:
  17. _l = old_build_clib.user_options
  18. for _i in range(len(_l)):
  19. if _l[_i][0] in ['build-clib', 'build-temp']:
  20. _l[_i] = (_l[_i][0] + '=',) + _l[_i][1:]
  21. #
  22. class build_clib(old_build_clib):
  23. description = "build C/C++/F libraries used by Python extensions"
  24. user_options = old_build_clib.user_options + [
  25. ('fcompiler=', None,
  26. "specify the Fortran compiler type"),
  27. ('inplace', 'i', 'Build in-place'),
  28. ('parallel=', 'j',
  29. "number of parallel jobs"),
  30. ('warn-error', None,
  31. "turn all warnings into errors (-Werror)"),
  32. ('cpu-baseline=', None,
  33. "specify a list of enabled baseline CPU optimizations"),
  34. ('cpu-dispatch=', None,
  35. "specify a list of dispatched CPU optimizations"),
  36. ('disable-optimization', None,
  37. "disable CPU optimized code(dispatch,simd,fast...)"),
  38. ]
  39. boolean_options = old_build_clib.boolean_options + \
  40. ['inplace', 'warn-error', 'disable-optimization']
  41. def initialize_options(self):
  42. old_build_clib.initialize_options(self)
  43. self.fcompiler = None
  44. self.inplace = 0
  45. self.parallel = None
  46. self.warn_error = None
  47. self.cpu_baseline = None
  48. self.cpu_dispatch = None
  49. self.disable_optimization = None
  50. def finalize_options(self):
  51. if self.parallel:
  52. try:
  53. self.parallel = int(self.parallel)
  54. except ValueError as e:
  55. raise ValueError("--parallel/-j argument must be an integer") from e
  56. old_build_clib.finalize_options(self)
  57. self.set_undefined_options('build',
  58. ('parallel', 'parallel'),
  59. ('warn_error', 'warn_error'),
  60. ('cpu_baseline', 'cpu_baseline'),
  61. ('cpu_dispatch', 'cpu_dispatch'),
  62. ('disable_optimization', 'disable_optimization')
  63. )
  64. def have_f_sources(self):
  65. for (lib_name, build_info) in self.libraries:
  66. if has_f_sources(build_info.get('sources', [])):
  67. return True
  68. return False
  69. def have_cxx_sources(self):
  70. for (lib_name, build_info) in self.libraries:
  71. if has_cxx_sources(build_info.get('sources', [])):
  72. return True
  73. return False
  74. def run(self):
  75. if not self.libraries:
  76. return
  77. # Make sure that library sources are complete.
  78. languages = []
  79. # Make sure that extension sources are complete.
  80. self.run_command('build_src')
  81. for (lib_name, build_info) in self.libraries:
  82. l = build_info.get('language', None)
  83. if l and l not in languages:
  84. languages.append(l)
  85. from distutils.ccompiler import new_compiler
  86. self.compiler = new_compiler(compiler=self.compiler,
  87. dry_run=self.dry_run,
  88. force=self.force)
  89. self.compiler.customize(self.distribution,
  90. need_cxx=self.have_cxx_sources())
  91. if self.warn_error:
  92. self.compiler.compiler.append('-Werror')
  93. self.compiler.compiler_so.append('-Werror')
  94. libraries = self.libraries
  95. self.libraries = None
  96. self.compiler.customize_cmd(self)
  97. self.libraries = libraries
  98. self.compiler.show_customization()
  99. if not self.disable_optimization:
  100. dispatch_hpath = os.path.join("numpy", "distutils", "include", "npy_cpu_dispatch_config.h")
  101. dispatch_hpath = os.path.join(self.get_finalized_command("build_src").build_src, dispatch_hpath)
  102. opt_cache_path = os.path.abspath(
  103. os.path.join(self.build_temp, 'ccompiler_opt_cache_clib.py')
  104. )
  105. self.compiler_opt = new_ccompiler_opt(
  106. compiler=self.compiler, dispatch_hpath=dispatch_hpath,
  107. cpu_baseline=self.cpu_baseline, cpu_dispatch=self.cpu_dispatch,
  108. cache_path=opt_cache_path
  109. )
  110. if not self.compiler_opt.is_cached():
  111. log.info("Detected changes on compiler optimizations, force rebuilding")
  112. self.force = True
  113. import atexit
  114. def report():
  115. log.info("\n########### CLIB COMPILER OPTIMIZATION ###########")
  116. log.info(self.compiler_opt.report(full=True))
  117. atexit.register(report)
  118. if self.have_f_sources():
  119. from numpy.distutils.fcompiler import new_fcompiler
  120. self._f_compiler = new_fcompiler(compiler=self.fcompiler,
  121. verbose=self.verbose,
  122. dry_run=self.dry_run,
  123. force=self.force,
  124. requiref90='f90' in languages,
  125. c_compiler=self.compiler)
  126. if self._f_compiler is not None:
  127. self._f_compiler.customize(self.distribution)
  128. libraries = self.libraries
  129. self.libraries = None
  130. self._f_compiler.customize_cmd(self)
  131. self.libraries = libraries
  132. self._f_compiler.show_customization()
  133. else:
  134. self._f_compiler = None
  135. self.build_libraries(self.libraries)
  136. if self.inplace:
  137. for l in self.distribution.installed_libraries:
  138. libname = self.compiler.library_filename(l.name)
  139. source = os.path.join(self.build_clib, libname)
  140. target = os.path.join(l.target_dir, libname)
  141. self.mkpath(l.target_dir)
  142. shutil.copy(source, target)
  143. def get_source_files(self):
  144. self.check_library_list(self.libraries)
  145. filenames = []
  146. for lib in self.libraries:
  147. filenames.extend(get_lib_source_files(lib))
  148. return filenames
  149. def build_libraries(self, libraries):
  150. for (lib_name, build_info) in libraries:
  151. self.build_a_library(build_info, lib_name, libraries)
  152. def build_a_library(self, build_info, lib_name, libraries):
  153. # default compilers
  154. compiler = self.compiler
  155. fcompiler = self._f_compiler
  156. sources = build_info.get('sources')
  157. if sources is None or not is_sequence(sources):
  158. raise DistutilsSetupError(("in 'libraries' option (library '%s'), " +
  159. "'sources' must be present and must be " +
  160. "a list of source filenames") % lib_name)
  161. sources = list(sources)
  162. c_sources, cxx_sources, f_sources, fmodule_sources \
  163. = filter_sources(sources)
  164. requiref90 = not not fmodule_sources or \
  165. build_info.get('language', 'c') == 'f90'
  166. # save source type information so that build_ext can use it.
  167. source_languages = []
  168. if c_sources:
  169. source_languages.append('c')
  170. if cxx_sources:
  171. source_languages.append('c++')
  172. if requiref90:
  173. source_languages.append('f90')
  174. elif f_sources:
  175. source_languages.append('f77')
  176. build_info['source_languages'] = source_languages
  177. lib_file = compiler.library_filename(lib_name,
  178. output_dir=self.build_clib)
  179. depends = sources + build_info.get('depends', [])
  180. if not (self.force or newer_group(depends, lib_file, 'newer')):
  181. log.debug("skipping '%s' library (up-to-date)", lib_name)
  182. return
  183. else:
  184. log.info("building '%s' library", lib_name)
  185. config_fc = build_info.get('config_fc', {})
  186. if fcompiler is not None and config_fc:
  187. log.info('using additional config_fc from setup script '
  188. 'for fortran compiler: %s'
  189. % (config_fc,))
  190. from numpy.distutils.fcompiler import new_fcompiler
  191. fcompiler = new_fcompiler(compiler=fcompiler.compiler_type,
  192. verbose=self.verbose,
  193. dry_run=self.dry_run,
  194. force=self.force,
  195. requiref90=requiref90,
  196. c_compiler=self.compiler)
  197. if fcompiler is not None:
  198. dist = self.distribution
  199. base_config_fc = dist.get_option_dict('config_fc').copy()
  200. base_config_fc.update(config_fc)
  201. fcompiler.customize(base_config_fc)
  202. # check availability of Fortran compilers
  203. if (f_sources or fmodule_sources) and fcompiler is None:
  204. raise DistutilsError("library %s has Fortran sources"
  205. " but no Fortran compiler found" % (lib_name))
  206. if fcompiler is not None:
  207. fcompiler.extra_f77_compile_args = build_info.get(
  208. 'extra_f77_compile_args') or []
  209. fcompiler.extra_f90_compile_args = build_info.get(
  210. 'extra_f90_compile_args') or []
  211. macros = build_info.get('macros')
  212. if macros is None:
  213. macros = []
  214. include_dirs = build_info.get('include_dirs')
  215. if include_dirs is None:
  216. include_dirs = []
  217. extra_postargs = build_info.get('extra_compiler_args') or []
  218. include_dirs.extend(get_numpy_include_dirs())
  219. # where compiled F90 module files are:
  220. module_dirs = build_info.get('module_dirs') or []
  221. module_build_dir = os.path.dirname(lib_file)
  222. if requiref90:
  223. self.mkpath(module_build_dir)
  224. if compiler.compiler_type == 'msvc':
  225. # this hack works around the msvc compiler attributes
  226. # problem, msvc uses its own convention :(
  227. c_sources += cxx_sources
  228. cxx_sources = []
  229. # filtering C dispatch-table sources when optimization is not disabled,
  230. # otherwise treated as normal sources.
  231. copt_c_sources = []
  232. copt_baseline_flags = []
  233. copt_macros = []
  234. if not self.disable_optimization:
  235. bsrc_dir = self.get_finalized_command("build_src").build_src
  236. dispatch_hpath = os.path.join("numpy", "distutils", "include")
  237. dispatch_hpath = os.path.join(bsrc_dir, dispatch_hpath)
  238. include_dirs.append(dispatch_hpath)
  239. copt_build_src = None if self.inplace else bsrc_dir
  240. copt_c_sources = [
  241. c_sources.pop(c_sources.index(src))
  242. for src in c_sources[:] if src.endswith(".dispatch.c")
  243. ]
  244. copt_baseline_flags = self.compiler_opt.cpu_baseline_flags()
  245. else:
  246. copt_macros.append(("NPY_DISABLE_OPTIMIZATION", 1))
  247. objects = []
  248. if copt_c_sources:
  249. log.info("compiling C dispatch-able sources")
  250. objects += self.compiler_opt.try_dispatch(copt_c_sources,
  251. output_dir=self.build_temp,
  252. src_dir=copt_build_src,
  253. macros=macros + copt_macros,
  254. include_dirs=include_dirs,
  255. debug=self.debug,
  256. extra_postargs=extra_postargs)
  257. if c_sources:
  258. log.info("compiling C sources")
  259. objects += compiler.compile(c_sources,
  260. output_dir=self.build_temp,
  261. macros=macros + copt_macros,
  262. include_dirs=include_dirs,
  263. debug=self.debug,
  264. extra_postargs=extra_postargs + copt_baseline_flags)
  265. if cxx_sources:
  266. log.info("compiling C++ sources")
  267. cxx_compiler = compiler.cxx_compiler()
  268. cxx_objects = cxx_compiler.compile(cxx_sources,
  269. output_dir=self.build_temp,
  270. macros=macros + copt_macros,
  271. include_dirs=include_dirs,
  272. debug=self.debug,
  273. extra_postargs=extra_postargs + copt_baseline_flags)
  274. objects.extend(cxx_objects)
  275. if f_sources or fmodule_sources:
  276. extra_postargs = []
  277. f_objects = []
  278. if requiref90:
  279. if fcompiler.module_dir_switch is None:
  280. existing_modules = glob('*.mod')
  281. extra_postargs += fcompiler.module_options(
  282. module_dirs, module_build_dir)
  283. if fmodule_sources:
  284. log.info("compiling Fortran 90 module sources")
  285. f_objects += fcompiler.compile(fmodule_sources,
  286. output_dir=self.build_temp,
  287. macros=macros,
  288. include_dirs=include_dirs,
  289. debug=self.debug,
  290. extra_postargs=extra_postargs)
  291. if requiref90 and self._f_compiler.module_dir_switch is None:
  292. # move new compiled F90 module files to module_build_dir
  293. for f in glob('*.mod'):
  294. if f in existing_modules:
  295. continue
  296. t = os.path.join(module_build_dir, f)
  297. if os.path.abspath(f) == os.path.abspath(t):
  298. continue
  299. if os.path.isfile(t):
  300. os.remove(t)
  301. try:
  302. self.move_file(f, module_build_dir)
  303. except DistutilsFileError:
  304. log.warn('failed to move %r to %r'
  305. % (f, module_build_dir))
  306. if f_sources:
  307. log.info("compiling Fortran sources")
  308. f_objects += fcompiler.compile(f_sources,
  309. output_dir=self.build_temp,
  310. macros=macros,
  311. include_dirs=include_dirs,
  312. debug=self.debug,
  313. extra_postargs=extra_postargs)
  314. else:
  315. f_objects = []
  316. if f_objects and not fcompiler.can_ccompiler_link(compiler):
  317. # Default linker cannot link Fortran object files, and results
  318. # need to be wrapped later. Instead of creating a real static
  319. # library, just keep track of the object files.
  320. listfn = os.path.join(self.build_clib,
  321. lib_name + '.fobjects')
  322. with open(listfn, 'w') as f:
  323. f.write("\n".join(os.path.abspath(obj) for obj in f_objects))
  324. listfn = os.path.join(self.build_clib,
  325. lib_name + '.cobjects')
  326. with open(listfn, 'w') as f:
  327. f.write("\n".join(os.path.abspath(obj) for obj in objects))
  328. # create empty "library" file for dependency tracking
  329. lib_fname = os.path.join(self.build_clib,
  330. lib_name + compiler.static_lib_extension)
  331. with open(lib_fname, 'wb') as f:
  332. pass
  333. else:
  334. # assume that default linker is suitable for
  335. # linking Fortran object files
  336. objects.extend(f_objects)
  337. compiler.create_static_lib(objects, lib_name,
  338. output_dir=self.build_clib,
  339. debug=self.debug)
  340. # fix library dependencies
  341. clib_libraries = build_info.get('libraries', [])
  342. for lname, binfo in libraries:
  343. if lname in clib_libraries:
  344. clib_libraries.extend(binfo.get('libraries', []))
  345. if clib_libraries:
  346. build_info['libraries'] = clib_libraries