ccompiler_opt.py 93 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535
  1. """Provides the `CCompilerOpt` class, used for handling the CPU/hardware
  2. optimization, starting from parsing the command arguments, to managing the
  3. relation between the CPU baseline and dispatch-able features,
  4. also generating the required C headers and ending with compiling
  5. the sources with proper compiler's flags.
  6. `CCompilerOpt` doesn't provide runtime detection for the CPU features,
  7. instead only focuses on the compiler side, but it creates abstract C headers
  8. that can be used later for the final runtime dispatching process."""
  9. import sys, io, os, re, textwrap, pprint, inspect, atexit, subprocess
  10. class _Config:
  11. """An abstract class holds all configurable attributes of `CCompilerOpt`,
  12. these class attributes can be used to change the default behavior
  13. of `CCompilerOpt` in order to fit other requirements.
  14. Attributes
  15. ----------
  16. conf_nocache : bool
  17. Set True to disable memory and file cache.
  18. Default is False.
  19. conf_noopt : bool
  20. Set True to forces the optimization to be disabled,
  21. in this case `CCompilerOpt` tends to generate all
  22. expected headers in order to 'not' break the build.
  23. Default is False.
  24. conf_cache_factors : list
  25. Add extra factors to the primary caching factors. The caching factors
  26. are utilized to determine if there are changes had happened that
  27. requires to discard the cache and re-updating it. The primary factors
  28. are the arguments of `CCompilerOpt` and `CCompiler`'s properties(type, flags, etc).
  29. Default is list of two items, containing the time of last modification
  30. of `ccompiler_opt` and value of attribute "conf_noopt"
  31. conf_tmp_path : str,
  32. The path of temporary directory. Default is auto-created
  33. temporary directory via ``tempfile.mkdtemp()``.
  34. conf_check_path : str
  35. The path of testing files. Each added CPU feature must have a
  36. **C** source file contains at least one intrinsic or instruction that
  37. related to this feature, so it can be tested against the compiler.
  38. Default is ``./distutils/checks``.
  39. conf_target_groups : dict
  40. Extra tokens that can be reached from dispatch-able sources through
  41. the special mark ``@targets``. Default is an empty dictionary.
  42. **Notes**:
  43. - case-insensitive for tokens and group names
  44. - sign '#' must stick in the begin of group name and only within ``@targets``
  45. **Example**:
  46. .. code-block:: console
  47. $ "@targets #avx_group other_tokens" > group_inside.c
  48. >>> CCompilerOpt.conf_target_groups["avx_group"] = \\
  49. "$werror $maxopt avx2 avx512f avx512_skx"
  50. >>> cco = CCompilerOpt(cc_instance)
  51. >>> cco.try_dispatch(["group_inside.c"])
  52. conf_c_prefix : str
  53. The prefix of public C definitions. Default is ``"NPY_"``.
  54. conf_c_prefix_ : str
  55. The prefix of internal C definitions. Default is ``"NPY__"``.
  56. conf_cc_flags : dict
  57. Nested dictionaries defining several compiler flags
  58. that linked to some major functions, the main key
  59. represent the compiler name and sub-keys represent
  60. flags names. Default is already covers all supported
  61. **C** compilers.
  62. Sub-keys explained as follows:
  63. "native": str or None
  64. used by argument option `native`, to detect the current
  65. machine support via the compiler.
  66. "werror": str or None
  67. utilized to treat warning as errors during testing CPU features
  68. against the compiler and also for target's policy `$werror`
  69. via dispatch-able sources.
  70. "maxopt": str or None
  71. utilized for target's policy '$maxopt' and the value should
  72. contains the maximum acceptable optimization by the compiler.
  73. e.g. in gcc `'-O3'`
  74. **Notes**:
  75. * case-sensitive for compiler names and flags
  76. * use space to separate multiple flags
  77. * any flag will tested against the compiler and it will skipped
  78. if it's not applicable.
  79. conf_min_features : dict
  80. A dictionary defines the used CPU features for
  81. argument option `'min'`, the key represent the CPU architecture
  82. name e.g. `'x86'`. Default values provide the best effort
  83. on wide range of users platforms.
  84. **Note**: case-sensitive for architecture names.
  85. conf_features : dict
  86. Nested dictionaries used for identifying the CPU features.
  87. the primary key is represented as a feature name or group name
  88. that gathers several features. Default values covers all
  89. supported features but without the major options like "flags",
  90. these undefined options handle it by method `conf_features_partial()`.
  91. Default value is covers almost all CPU features for *X86*, *IBM/Power64*
  92. and *ARM 7/8*.
  93. Sub-keys explained as follows:
  94. "implies" : str or list, optional,
  95. List of CPU feature names to be implied by it,
  96. the feature name must be defined within `conf_features`.
  97. Default is None.
  98. "flags": str or list, optional
  99. List of compiler flags. Default is None.
  100. "detect": str or list, optional
  101. List of CPU feature names that required to be detected
  102. in runtime. By default, its the feature name or features
  103. in "group" if its specified.
  104. "implies_detect": bool, optional
  105. If True, all "detect" of implied features will be combined.
  106. Default is True. see `feature_detect()`.
  107. "group": str or list, optional
  108. Same as "implies" but doesn't require the feature name to be
  109. defined within `conf_features`.
  110. "interest": int, required
  111. a key for sorting CPU features
  112. "headers": str or list, optional
  113. intrinsics C header file
  114. "disable": str, optional
  115. force disable feature, the string value should contains the
  116. reason of disabling.
  117. "autovec": bool or None, optional
  118. True or False to declare that CPU feature can be auto-vectorized
  119. by the compiler.
  120. By default(None), treated as True if the feature contains at
  121. least one applicable flag. see `feature_can_autovec()`
  122. "extra_checks": str or list, optional
  123. Extra test case names for the CPU feature that need to be tested
  124. against the compiler.
  125. Each test case must have a C file named ``extra_xxxx.c``, where
  126. ``xxxx`` is the case name in lower case, under 'conf_check_path'.
  127. It should contain at least one intrinsic or function related to the test case.
  128. If the compiler able to successfully compile the C file then `CCompilerOpt`
  129. will add a C ``#define`` for it into the main dispatch header, e.g.
  130. ```#define {conf_c_prefix}_XXXX`` where ``XXXX`` is the case name in upper case.
  131. **NOTES**:
  132. * space can be used as separator with options that supports "str or list"
  133. * case-sensitive for all values and feature name must be in upper-case.
  134. * if flags aren't applicable, its will skipped rather than disable the
  135. CPU feature
  136. * the CPU feature will disabled if the compiler fail to compile
  137. the test file
  138. """
  139. conf_nocache = False
  140. conf_noopt = False
  141. conf_cache_factors = None
  142. conf_tmp_path = None
  143. conf_check_path = os.path.join(
  144. os.path.dirname(os.path.realpath(__file__)), "checks"
  145. )
  146. conf_target_groups = {}
  147. conf_c_prefix = 'NPY_'
  148. conf_c_prefix_ = 'NPY__'
  149. conf_cc_flags = dict(
  150. gcc = dict(
  151. # native should always fail on arm and ppc64,
  152. # native usually works only with x86
  153. native = '-march=native',
  154. opt = '-O3',
  155. werror = '-Werror'
  156. ),
  157. clang = dict(
  158. native = '-march=native',
  159. opt = "-O3",
  160. werror = '-Werror'
  161. ),
  162. icc = dict(
  163. native = '-xHost',
  164. opt = '-O3',
  165. werror = '-Werror'
  166. ),
  167. iccw = dict(
  168. native = '/QxHost',
  169. opt = '/O3',
  170. werror = '/Werror'
  171. ),
  172. msvc = dict(
  173. native = None,
  174. opt = '/O2',
  175. werror = '/WX'
  176. )
  177. )
  178. conf_min_features = dict(
  179. x86 = "SSE SSE2",
  180. x64 = "SSE SSE2 SSE3",
  181. ppc64 = '', # play it safe
  182. ppc64le = "VSX VSX2",
  183. armhf = '', # play it safe
  184. aarch64 = "NEON NEON_FP16 NEON_VFPV4 ASIMD"
  185. )
  186. conf_features = dict(
  187. # X86
  188. SSE = dict(
  189. interest=1, headers="xmmintrin.h",
  190. # enabling SSE without SSE2 is useless also
  191. # it's non-optional for x86_64
  192. implies="SSE2"
  193. ),
  194. SSE2 = dict(interest=2, implies="SSE", headers="emmintrin.h"),
  195. SSE3 = dict(interest=3, implies="SSE2", headers="pmmintrin.h"),
  196. SSSE3 = dict(interest=4, implies="SSE3", headers="tmmintrin.h"),
  197. SSE41 = dict(interest=5, implies="SSSE3", headers="smmintrin.h"),
  198. POPCNT = dict(interest=6, implies="SSE41", headers="popcntintrin.h"),
  199. SSE42 = dict(interest=7, implies="POPCNT"),
  200. AVX = dict(
  201. interest=8, implies="SSE42", headers="immintrin.h",
  202. implies_detect=False
  203. ),
  204. XOP = dict(interest=9, implies="AVX", headers="x86intrin.h"),
  205. FMA4 = dict(interest=10, implies="AVX", headers="x86intrin.h"),
  206. F16C = dict(interest=11, implies="AVX"),
  207. FMA3 = dict(interest=12, implies="F16C"),
  208. AVX2 = dict(interest=13, implies="F16C"),
  209. AVX512F = dict(
  210. interest=20, implies="FMA3 AVX2", implies_detect=False,
  211. extra_checks="AVX512F_REDUCE"
  212. ),
  213. AVX512CD = dict(interest=21, implies="AVX512F"),
  214. AVX512_KNL = dict(
  215. interest=40, implies="AVX512CD", group="AVX512ER AVX512PF",
  216. detect="AVX512_KNL", implies_detect=False
  217. ),
  218. AVX512_KNM = dict(
  219. interest=41, implies="AVX512_KNL",
  220. group="AVX5124FMAPS AVX5124VNNIW AVX512VPOPCNTDQ",
  221. detect="AVX512_KNM", implies_detect=False
  222. ),
  223. AVX512_SKX = dict(
  224. interest=42, implies="AVX512CD", group="AVX512VL AVX512BW AVX512DQ",
  225. detect="AVX512_SKX", implies_detect=False,
  226. extra_checks="AVX512BW_MASK AVX512DQ_MASK"
  227. ),
  228. AVX512_CLX = dict(
  229. interest=43, implies="AVX512_SKX", group="AVX512VNNI",
  230. detect="AVX512_CLX"
  231. ),
  232. AVX512_CNL = dict(
  233. interest=44, implies="AVX512_SKX", group="AVX512IFMA AVX512VBMI",
  234. detect="AVX512_CNL", implies_detect=False
  235. ),
  236. AVX512_ICL = dict(
  237. interest=45, implies="AVX512_CLX AVX512_CNL",
  238. group="AVX512VBMI2 AVX512BITALG AVX512VPOPCNTDQ",
  239. detect="AVX512_ICL", implies_detect=False
  240. ),
  241. # IBM/Power
  242. ## Power7/ISA 2.06
  243. VSX = dict(interest=1, headers="altivec.h"),
  244. ## Power8/ISA 2.07
  245. VSX2 = dict(interest=2, implies="VSX", implies_detect=False),
  246. ## Power9/ISA 3.00
  247. VSX3 = dict(interest=3, implies="VSX2", implies_detect=False),
  248. # ARM
  249. NEON = dict(interest=1, headers="arm_neon.h"),
  250. NEON_FP16 = dict(interest=2, implies="NEON"),
  251. ## FMA
  252. NEON_VFPV4 = dict(interest=3, implies="NEON_FP16"),
  253. ## Advanced SIMD
  254. ASIMD = dict(interest=4, implies="NEON_FP16 NEON_VFPV4", implies_detect=False),
  255. ## ARMv8.2 half-precision & vector arithm
  256. ASIMDHP = dict(interest=5, implies="ASIMD"),
  257. ## ARMv8.2 dot product
  258. ASIMDDP = dict(interest=6, implies="ASIMD"),
  259. ## ARMv8.2 Single & half-precision Multiply
  260. ASIMDFHM = dict(interest=7, implies="ASIMDHP"),
  261. )
  262. def conf_features_partial(self):
  263. """Return a dictionary of supported CPU features by the platform,
  264. and accumulate the rest of undefined options in `conf_features`,
  265. the returned dict has same rules and notes in
  266. class attribute `conf_features`, also its override
  267. any options that been set in 'conf_features'.
  268. """
  269. if self.cc_noopt:
  270. # optimization is disabled
  271. return {}
  272. on_x86 = self.cc_on_x86 or self.cc_on_x64
  273. is_unix = self.cc_is_gcc or self.cc_is_clang
  274. if on_x86 and is_unix: return dict(
  275. SSE = dict(flags="-msse"),
  276. SSE2 = dict(flags="-msse2"),
  277. SSE3 = dict(flags="-msse3"),
  278. SSSE3 = dict(flags="-mssse3"),
  279. SSE41 = dict(flags="-msse4.1"),
  280. POPCNT = dict(flags="-mpopcnt"),
  281. SSE42 = dict(flags="-msse4.2"),
  282. AVX = dict(flags="-mavx"),
  283. F16C = dict(flags="-mf16c"),
  284. XOP = dict(flags="-mxop"),
  285. FMA4 = dict(flags="-mfma4"),
  286. FMA3 = dict(flags="-mfma"),
  287. AVX2 = dict(flags="-mavx2"),
  288. AVX512F = dict(flags="-mavx512f"),
  289. AVX512CD = dict(flags="-mavx512cd"),
  290. AVX512_KNL = dict(flags="-mavx512er -mavx512pf"),
  291. AVX512_KNM = dict(
  292. flags="-mavx5124fmaps -mavx5124vnniw -mavx512vpopcntdq"
  293. ),
  294. AVX512_SKX = dict(flags="-mavx512vl -mavx512bw -mavx512dq"),
  295. AVX512_CLX = dict(flags="-mavx512vnni"),
  296. AVX512_CNL = dict(flags="-mavx512ifma -mavx512vbmi"),
  297. AVX512_ICL = dict(
  298. flags="-mavx512vbmi2 -mavx512bitalg -mavx512vpopcntdq"
  299. )
  300. )
  301. if on_x86 and self.cc_is_icc: return dict(
  302. SSE = dict(flags="-msse"),
  303. SSE2 = dict(flags="-msse2"),
  304. SSE3 = dict(flags="-msse3"),
  305. SSSE3 = dict(flags="-mssse3"),
  306. SSE41 = dict(flags="-msse4.1"),
  307. POPCNT = {},
  308. SSE42 = dict(flags="-msse4.2"),
  309. AVX = dict(flags="-mavx"),
  310. F16C = {},
  311. XOP = dict(disable="Intel Compiler doesn't support it"),
  312. FMA4 = dict(disable="Intel Compiler doesn't support it"),
  313. # Intel Compiler doesn't support AVX2 or FMA3 independently
  314. FMA3 = dict(
  315. implies="F16C AVX2", flags="-march=core-avx2"
  316. ),
  317. AVX2 = dict(implies="FMA3", flags="-march=core-avx2"),
  318. # Intel Compiler doesn't support AVX512F or AVX512CD independently
  319. AVX512F = dict(
  320. implies="AVX2 AVX512CD", flags="-march=common-avx512"
  321. ),
  322. AVX512CD = dict(
  323. implies="AVX2 AVX512F", flags="-march=common-avx512"
  324. ),
  325. AVX512_KNL = dict(flags="-xKNL"),
  326. AVX512_KNM = dict(flags="-xKNM"),
  327. AVX512_SKX = dict(flags="-xSKYLAKE-AVX512"),
  328. AVX512_CLX = dict(flags="-xCASCADELAKE"),
  329. AVX512_CNL = dict(flags="-xCANNONLAKE"),
  330. AVX512_ICL = dict(flags="-xICELAKE-CLIENT"),
  331. )
  332. if on_x86 and self.cc_is_iccw: return dict(
  333. SSE = dict(flags="/arch:SSE"),
  334. SSE2 = dict(flags="/arch:SSE2"),
  335. SSE3 = dict(flags="/arch:SSE3"),
  336. SSSE3 = dict(flags="/arch:SSSE3"),
  337. SSE41 = dict(flags="/arch:SSE4.1"),
  338. POPCNT = {},
  339. SSE42 = dict(flags="/arch:SSE4.2"),
  340. AVX = dict(flags="/arch:AVX"),
  341. F16C = {},
  342. XOP = dict(disable="Intel Compiler doesn't support it"),
  343. FMA4 = dict(disable="Intel Compiler doesn't support it"),
  344. # Intel Compiler doesn't support FMA3 or AVX2 independently
  345. FMA3 = dict(
  346. implies="F16C AVX2", flags="/arch:CORE-AVX2"
  347. ),
  348. AVX2 = dict(
  349. implies="FMA3", flags="/arch:CORE-AVX2"
  350. ),
  351. # Intel Compiler doesn't support AVX512F or AVX512CD independently
  352. AVX512F = dict(
  353. implies="AVX2 AVX512CD", flags="/Qx:COMMON-AVX512"
  354. ),
  355. AVX512CD = dict(
  356. implies="AVX2 AVX512F", flags="/Qx:COMMON-AVX512"
  357. ),
  358. AVX512_KNL = dict(flags="/Qx:KNL"),
  359. AVX512_KNM = dict(flags="/Qx:KNM"),
  360. AVX512_SKX = dict(flags="/Qx:SKYLAKE-AVX512"),
  361. AVX512_CLX = dict(flags="/Qx:CASCADELAKE"),
  362. AVX512_CNL = dict(flags="/Qx:CANNONLAKE"),
  363. AVX512_ICL = dict(flags="/Qx:ICELAKE-CLIENT")
  364. )
  365. if on_x86 and self.cc_is_msvc: return dict(
  366. SSE = dict(flags="/arch:SSE"),
  367. SSE2 = dict(flags="/arch:SSE2"),
  368. SSE3 = {},
  369. SSSE3 = {},
  370. SSE41 = {},
  371. POPCNT = dict(headers="nmmintrin.h"),
  372. SSE42 = {},
  373. AVX = dict(flags="/arch:AVX"),
  374. F16C = {},
  375. XOP = dict(headers="ammintrin.h"),
  376. FMA4 = dict(headers="ammintrin.h"),
  377. # MSVC doesn't support FMA3 or AVX2 independently
  378. FMA3 = dict(
  379. implies="F16C AVX2", flags="/arch:AVX2"
  380. ),
  381. AVX2 = dict(
  382. implies="F16C FMA3", flags="/arch:AVX2"
  383. ),
  384. # MSVC doesn't support AVX512F or AVX512CD independently,
  385. # always generate instructions belong to (VL/VW/DQ)
  386. AVX512F = dict(
  387. implies="AVX2 AVX512CD AVX512_SKX", flags="/arch:AVX512"
  388. ),
  389. AVX512CD = dict(
  390. implies="AVX512F AVX512_SKX", flags="/arch:AVX512"
  391. ),
  392. AVX512_KNL = dict(
  393. disable="MSVC compiler doesn't support it"
  394. ),
  395. AVX512_KNM = dict(
  396. disable="MSVC compiler doesn't support it"
  397. ),
  398. AVX512_SKX = dict(flags="/arch:AVX512"),
  399. AVX512_CLX = {},
  400. AVX512_CNL = {},
  401. AVX512_ICL = {}
  402. )
  403. on_power = self.cc_on_ppc64le or self.cc_on_ppc64
  404. if on_power:
  405. partial = dict(
  406. VSX = dict(
  407. implies=("VSX2" if self.cc_on_ppc64le else ""),
  408. flags="-mvsx"
  409. ),
  410. VSX2 = dict(
  411. flags="-mcpu=power8", implies_detect=False
  412. ),
  413. VSX3 = dict(
  414. flags="-mcpu=power9 -mtune=power9", implies_detect=False
  415. )
  416. )
  417. if self.cc_is_clang:
  418. partial["VSX"]["flags"] = "-maltivec -mvsx"
  419. partial["VSX2"]["flags"] = "-mpower8-vector"
  420. partial["VSX3"]["flags"] = "-mpower9-vector"
  421. return partial
  422. if self.cc_on_aarch64 and is_unix: return dict(
  423. NEON = dict(
  424. implies="NEON_FP16 NEON_VFPV4 ASIMD", autovec=True
  425. ),
  426. NEON_FP16 = dict(
  427. implies="NEON NEON_VFPV4 ASIMD", autovec=True
  428. ),
  429. NEON_VFPV4 = dict(
  430. implies="NEON NEON_FP16 ASIMD", autovec=True
  431. ),
  432. ASIMD = dict(
  433. implies="NEON NEON_FP16 NEON_VFPV4", autovec=True
  434. ),
  435. ASIMDHP = dict(
  436. flags="-march=armv8.2-a+fp16"
  437. ),
  438. ASIMDDP = dict(
  439. flags="-march=armv8.2-a+dotprod"
  440. ),
  441. ASIMDFHM = dict(
  442. flags="-march=armv8.2-a+fp16fml"
  443. ),
  444. )
  445. if self.cc_on_armhf and is_unix: return dict(
  446. NEON = dict(
  447. flags="-mfpu=neon"
  448. ),
  449. NEON_FP16 = dict(
  450. flags="-mfpu=neon-fp16 -mfp16-format=ieee"
  451. ),
  452. NEON_VFPV4 = dict(
  453. flags="-mfpu=neon-vfpv4",
  454. ),
  455. ASIMD = dict(
  456. flags="-mfpu=neon-fp-armv8 -march=armv8-a+simd",
  457. ),
  458. ASIMDHP = dict(
  459. flags="-march=armv8.2-a+fp16"
  460. ),
  461. ASIMDDP = dict(
  462. flags="-march=armv8.2-a+dotprod",
  463. ),
  464. ASIMDFHM = dict(
  465. flags="-march=armv8.2-a+fp16fml"
  466. )
  467. )
  468. # TODO: ARM MSVC
  469. return {}
  470. def __init__(self):
  471. if self.conf_tmp_path is None:
  472. import tempfile, shutil
  473. tmp = tempfile.mkdtemp()
  474. def rm_temp():
  475. try:
  476. shutil.rmtree(tmp)
  477. except IOError:
  478. pass
  479. atexit.register(rm_temp)
  480. self.conf_tmp_path = tmp
  481. if self.conf_cache_factors is None:
  482. self.conf_cache_factors = [
  483. os.path.getmtime(__file__),
  484. self.conf_nocache
  485. ]
  486. class _Distutils:
  487. """A helper class that provides a collection of fundamental methods
  488. implemented in a top of Python and NumPy Distutils.
  489. The idea behind this class is to gather all methods that it may
  490. need to override in case of reuse 'CCompilerOpt' in environment
  491. different than of what NumPy has.
  492. Parameters
  493. ----------
  494. ccompiler : `CCompiler`
  495. The generate instance that returned from `distutils.ccompiler.new_compiler()`.
  496. """
  497. def __init__(self, ccompiler):
  498. self._ccompiler = ccompiler
  499. def dist_compile(self, sources, flags, **kwargs):
  500. """Wrap CCompiler.compile()"""
  501. assert(isinstance(sources, list))
  502. assert(isinstance(flags, list))
  503. flags = kwargs.pop("extra_postargs", []) + flags
  504. return self._ccompiler.compile(
  505. sources, extra_postargs=flags, **kwargs
  506. )
  507. def dist_test(self, source, flags):
  508. """Return True if 'CCompiler.compile()' able to compile
  509. a source file with certain flags.
  510. """
  511. assert(isinstance(source, str))
  512. from distutils.errors import CompileError
  513. cc = self._ccompiler;
  514. bk_spawn = getattr(cc, 'spawn', None)
  515. if bk_spawn:
  516. cc_type = getattr(self._ccompiler, "compiler_type", "")
  517. if cc_type in ("msvc",):
  518. setattr(cc, 'spawn', self._dist_test_spawn_paths)
  519. else:
  520. setattr(cc, 'spawn', self._dist_test_spawn)
  521. test = False
  522. try:
  523. self.dist_compile(
  524. [source], flags, output_dir=self.conf_tmp_path
  525. )
  526. test = True
  527. except CompileError as e:
  528. self.dist_log(str(e), stderr=True)
  529. if bk_spawn:
  530. setattr(cc, 'spawn', bk_spawn)
  531. return test
  532. def dist_info(self):
  533. """
  534. Return a tuple containing info about (platform, compiler, extra_args),
  535. required by the abstract class '_CCompiler' for discovering the
  536. platform environment. This is also used as a cache factor in order
  537. to detect any changes happening from outside.
  538. """
  539. if hasattr(self, "_dist_info"):
  540. return self._dist_info
  541. cc_type = getattr(self._ccompiler, "compiler_type", '')
  542. if cc_type in ("intelem", "intelemw"):
  543. platform = "x86_64"
  544. elif cc_type in ("intel", "intelw", "intele"):
  545. platform = "x86"
  546. else:
  547. from distutils.util import get_platform
  548. platform = get_platform()
  549. cc_info = getattr(self._ccompiler, "compiler", getattr(self._ccompiler, "compiler_so", ''))
  550. if not cc_type or cc_type == "unix":
  551. if hasattr(cc_info, "__iter__"):
  552. compiler = cc_info[0]
  553. else:
  554. compiler = str(cc_info)
  555. else:
  556. compiler = cc_type
  557. if hasattr(cc_info, "__iter__") and len(cc_info) > 1:
  558. extra_args = ' '.join(cc_info[1:])
  559. else:
  560. extra_args = os.environ.get("CFLAGS", "")
  561. extra_args += os.environ.get("CPPFLAGS", "")
  562. self._dist_info = (platform, compiler, extra_args)
  563. return self._dist_info
  564. @staticmethod
  565. def dist_error(*args):
  566. """Raise a compiler error"""
  567. from distutils.errors import CompileError
  568. raise CompileError(_Distutils._dist_str(*args))
  569. @staticmethod
  570. def dist_fatal(*args):
  571. """Raise a distutils error"""
  572. from distutils.errors import DistutilsError
  573. raise DistutilsError(_Distutils._dist_str(*args))
  574. @staticmethod
  575. def dist_log(*args, stderr=False):
  576. """Print a console message"""
  577. from numpy.distutils import log
  578. out = _Distutils._dist_str(*args)
  579. if stderr:
  580. log.warn(out)
  581. else:
  582. log.info(out)
  583. @staticmethod
  584. def dist_load_module(name, path):
  585. """Load a module from file, required by the abstract class '_Cache'."""
  586. from numpy.compat import npy_load_module
  587. try:
  588. return npy_load_module(name, path)
  589. except Exception as e:
  590. _Distutils.dist_log(e, stderr=True)
  591. return None
  592. @staticmethod
  593. def _dist_str(*args):
  594. """Return a string to print by log and errors."""
  595. def to_str(arg):
  596. if not isinstance(arg, str) and hasattr(arg, '__iter__'):
  597. ret = []
  598. for a in arg:
  599. ret.append(to_str(a))
  600. return '('+ ' '.join(ret) + ')'
  601. return str(arg)
  602. stack = inspect.stack()[2]
  603. start = "CCompilerOpt.%s[%d] : " % (stack.function, stack.lineno)
  604. out = ' '.join([
  605. to_str(a)
  606. for a in (*args,)
  607. ])
  608. return start + out
  609. def _dist_test_spawn_paths(self, cmd, display=None):
  610. """
  611. Fix msvc SDK ENV path same as distutils do
  612. without it we get c1: fatal error C1356: unable to find mspdbcore.dll
  613. """
  614. if not hasattr(self._ccompiler, "_paths"):
  615. self._dist_test_spawn(cmd)
  616. return
  617. old_path = os.getenv("path")
  618. try:
  619. os.environ["path"] = self._ccompiler._paths
  620. self._dist_test_spawn(cmd)
  621. finally:
  622. os.environ["path"] = old_path
  623. _dist_warn_regex = re.compile(
  624. # intel and msvc compilers don't raise
  625. # fatal errors when flags are wrong or unsupported
  626. ".*("
  627. "warning D9002|" # msvc, it should be work with any language.
  628. "invalid argument for option" # intel
  629. ").*"
  630. )
  631. @staticmethod
  632. def _dist_test_spawn(cmd, display=None):
  633. from distutils.errors import CompileError
  634. try:
  635. o = subprocess.check_output(cmd, stderr=subprocess.STDOUT,
  636. universal_newlines=True)
  637. if o and re.match(_Distutils._dist_warn_regex, o):
  638. _Distutils.dist_error(
  639. "Flags in command", cmd ,"aren't supported by the compiler"
  640. ", output -> \n%s" % o
  641. )
  642. except subprocess.CalledProcessError as exc:
  643. o = exc.output
  644. s = exc.returncode
  645. except OSError:
  646. o = b''
  647. s = 127
  648. else:
  649. return None
  650. _Distutils.dist_error(
  651. "Command", cmd, "failed with exit status %d output -> \n%s" % (
  652. s, o
  653. ))
  654. _share_cache = {}
  655. class _Cache:
  656. """An abstract class handles caching functionality, provides two
  657. levels of caching, in-memory by share instances attributes among
  658. each other and by store attributes into files.
  659. **Note**:
  660. any attributes that start with ``_`` or ``conf_`` will be ignored.
  661. Parameters
  662. ----------
  663. cache_path: str or None
  664. The path of cache file, if None then cache in file will disabled.
  665. *factors:
  666. The caching factors that need to utilize next to `conf_cache_factors`.
  667. Attributes
  668. ----------
  669. cache_private: set
  670. Hold the attributes that need be skipped from "in-memory cache".
  671. cache_infile: bool
  672. Utilized during initializing this class, to determine if the cache was able
  673. to loaded from the specified cache path in 'cache_path'.
  674. """
  675. # skip attributes from cache
  676. _cache_ignore = re.compile("^(_|conf_)")
  677. def __init__(self, cache_path=None, *factors):
  678. self.cache_me = {}
  679. self.cache_private = set()
  680. self.cache_infile = False
  681. if self.conf_nocache:
  682. self.dist_log("cache is disabled by `Config`")
  683. return
  684. chash = self.cache_hash(*factors, *self.conf_cache_factors)
  685. if cache_path:
  686. if os.path.exists(cache_path):
  687. self.dist_log("load cache from file ->", cache_path)
  688. cache_mod = self.dist_load_module("cache", cache_path)
  689. if not cache_mod:
  690. self.dist_log(
  691. "unable to load the cache file as a module",
  692. stderr=True
  693. )
  694. elif not hasattr(cache_mod, "hash") or \
  695. not hasattr(cache_mod, "data"):
  696. self.dist_log("invalid cache file", stderr=True)
  697. elif chash == cache_mod.hash:
  698. self.dist_log("hit the file cache")
  699. for attr, val in cache_mod.data.items():
  700. setattr(self, attr, val)
  701. self.cache_infile = True
  702. else:
  703. self.dist_log("miss the file cache")
  704. atexit.register(self._cache_write, cache_path, chash)
  705. if not self.cache_infile:
  706. other_cache = _share_cache.get(chash)
  707. if other_cache:
  708. self.dist_log("hit the memory cache")
  709. for attr, val in other_cache.__dict__.items():
  710. if attr in other_cache.cache_private or \
  711. re.match(self._cache_ignore, attr):
  712. continue
  713. setattr(self, attr, val)
  714. _share_cache[chash] = self
  715. def __del__(self):
  716. # TODO: remove the cache form share on del
  717. pass
  718. def _cache_write(self, cache_path, cache_hash):
  719. # TODO: don't write if the cache doesn't change
  720. self.dist_log("write cache to path ->", cache_path)
  721. for attr in list(self.__dict__.keys()):
  722. if re.match(self._cache_ignore, attr):
  723. self.__dict__.pop(attr)
  724. d = os.path.dirname(cache_path)
  725. if not os.path.exists(d):
  726. os.makedirs(d)
  727. repr_dict = pprint.pformat(self.__dict__, compact=True)
  728. with open(cache_path, "w") as f:
  729. f.write(textwrap.dedent("""\
  730. # AUTOGENERATED DON'T EDIT
  731. # Please make changes to the code generator \
  732. (distutils/ccompiler_opt.py)
  733. hash = {}
  734. data = \\
  735. """).format(cache_hash))
  736. f.write(repr_dict)
  737. def cache_hash(self, *factors):
  738. # is there a built-in non-crypto hash?
  739. # sdbm
  740. chash = 0
  741. for f in factors:
  742. for char in str(f):
  743. chash = ord(char) + (chash << 6) + (chash << 16) - chash
  744. chash &= 0xFFFFFFFF
  745. return chash
  746. @staticmethod
  747. def me(cb):
  748. """
  749. A static method that can be treated as a decorator to
  750. dynamically cache certain methods.
  751. """
  752. def cache_wrap_me(self, *args, **kwargs):
  753. # good for normal args
  754. cache_key = str((
  755. cb.__name__, *args, *kwargs.keys(), *kwargs.values()
  756. ))
  757. if cache_key in self.cache_me:
  758. return self.cache_me[cache_key]
  759. ccb = cb(self, *args, **kwargs)
  760. self.cache_me[cache_key] = ccb
  761. return ccb
  762. return cache_wrap_me
  763. class _CCompiler(object):
  764. """A helper class for `CCompilerOpt` containing all utilities that
  765. related to the fundamental compiler's functions.
  766. Attributes
  767. ----------
  768. cc_on_x86 : bool
  769. True when the target architecture is 32-bit x86
  770. cc_on_x64 : bool
  771. True when the target architecture is 64-bit x86
  772. cc_on_ppc64 : bool
  773. True when the target architecture is 64-bit big-endian PowerPC
  774. cc_on_armhf : bool
  775. True when the target architecture is 32-bit ARMv7+
  776. cc_on_aarch64 : bool
  777. True when the target architecture is 64-bit Armv8-a+
  778. cc_on_noarch : bool
  779. True when the target architecture is unknown or not supported
  780. cc_is_gcc : bool
  781. True if the compiler is GNU or
  782. if the compiler is unknown
  783. cc_is_clang : bool
  784. True if the compiler is Clang
  785. cc_is_icc : bool
  786. True if the compiler is Intel compiler (unix like)
  787. cc_is_iccw : bool
  788. True if the compiler is Intel compiler (msvc like)
  789. cc_is_nocc : bool
  790. True if the compiler isn't supported directly,
  791. Note: that cause a fail-back to gcc
  792. cc_has_debug : bool
  793. True if the compiler has debug flags
  794. cc_has_native : bool
  795. True if the compiler has native flags
  796. cc_noopt : bool
  797. True if the compiler has definition 'DISABLE_OPT*',
  798. or 'cc_on_noarch' is True
  799. cc_march : str
  800. The target architecture name, or "unknown" if
  801. the architecture isn't supported
  802. cc_name : str
  803. The compiler name, or "unknown" if the compiler isn't supported
  804. cc_flags : dict
  805. Dictionary containing the initialized flags of `_Config.conf_cc_flags`
  806. """
  807. def __init__(self):
  808. if hasattr(self, "cc_is_cached"):
  809. return
  810. # attr regex
  811. detect_arch = (
  812. ("cc_on_x64", ".*(x|x86_|amd)64.*"),
  813. ("cc_on_x86", ".*(win32|x86|i386|i686).*"),
  814. ("cc_on_ppc64le", ".*(powerpc|ppc)64(el|le).*"),
  815. ("cc_on_ppc64", ".*(powerpc|ppc)64.*"),
  816. ("cc_on_aarch64", ".*(aarch64|arm64).*"),
  817. ("cc_on_armhf", ".*arm.*"),
  818. # undefined platform
  819. ("cc_on_noarch", ""),
  820. )
  821. detect_compiler = (
  822. ("cc_is_gcc", r".*(gcc|gnu\-g).*"),
  823. ("cc_is_clang", ".*clang.*"),
  824. ("cc_is_iccw", ".*(intelw|intelemw|iccw).*"), # intel msvc like
  825. ("cc_is_icc", ".*(intel|icc).*"), # intel unix like
  826. ("cc_is_msvc", ".*msvc.*"),
  827. # undefined compiler will be treat it as gcc
  828. ("cc_is_nocc", ""),
  829. )
  830. detect_args = (
  831. ("cc_has_debug", ".*(O0|Od|ggdb|coverage|debug:full).*"),
  832. ("cc_has_native", ".*(-march=native|-xHost|/QxHost).*"),
  833. # in case if the class run with -DNPY_DISABLE_OPTIMIZATION
  834. ("cc_noopt", ".*DISABLE_OPT.*"),
  835. )
  836. dist_info = self.dist_info()
  837. platform, compiler_info, extra_args = dist_info
  838. # set False to all attrs
  839. for section in (detect_arch, detect_compiler, detect_args):
  840. for attr, rgex in section:
  841. setattr(self, attr, False)
  842. for detect, searchin in ((detect_arch, platform), (detect_compiler, compiler_info)):
  843. for attr, rgex in detect:
  844. if rgex and not re.match(rgex, searchin, re.IGNORECASE):
  845. continue
  846. setattr(self, attr, True)
  847. break
  848. for attr, rgex in detect_args:
  849. if rgex and not re.match(rgex, extra_args, re.IGNORECASE):
  850. continue
  851. setattr(self, attr, True)
  852. if self.cc_on_noarch:
  853. self.dist_log(
  854. "unable to detect CPU architecture which lead to disable the optimization. "
  855. f"check dist_info:<<\n{dist_info}\n>>",
  856. stderr=True
  857. )
  858. self.cc_noopt = True
  859. if self.conf_noopt:
  860. self.dist_log("Optimization is disabled by the Config", stderr=True)
  861. self.cc_noopt = True
  862. if self.cc_is_nocc:
  863. """
  864. mingw can be treated as a gcc, and also xlc even if it based on clang,
  865. but still has the same gcc optimization flags.
  866. """
  867. self.dist_log(
  868. "unable to detect compiler type which leads to treating it as GCC. "
  869. "this is a normal behavior if you're using gcc-like compiler such as MinGW or IBM/XLC."
  870. f"check dist_info:<<\n{dist_info}\n>>",
  871. stderr=True
  872. )
  873. self.cc_is_gcc = True
  874. self.cc_march = "unknown"
  875. for arch in ("x86", "x64", "ppc64", "ppc64le", "armhf", "aarch64"):
  876. if getattr(self, "cc_on_" + arch):
  877. self.cc_march = arch
  878. break
  879. self.cc_name = "unknown"
  880. for name in ("gcc", "clang", "iccw", "icc", "msvc"):
  881. if getattr(self, "cc_is_" + name):
  882. self.cc_name = name
  883. break
  884. self.cc_flags = {}
  885. compiler_flags = self.conf_cc_flags.get(self.cc_name)
  886. if compiler_flags is None:
  887. self.dist_fatal(
  888. "undefined flag for compiler '%s', "
  889. "leave an empty dict instead" % self.cc_name
  890. )
  891. for name, flags in compiler_flags.items():
  892. self.cc_flags[name] = nflags = []
  893. if flags:
  894. assert(isinstance(flags, str))
  895. flags = flags.split()
  896. for f in flags:
  897. if self.cc_test_flags([f]):
  898. nflags.append(f)
  899. self.cc_is_cached = True
  900. @_Cache.me
  901. def cc_test_flags(self, flags):
  902. """
  903. Returns True if the compiler supports 'flags'.
  904. """
  905. assert(isinstance(flags, list))
  906. self.dist_log("testing flags", flags)
  907. test_path = os.path.join(self.conf_check_path, "test_flags.c")
  908. test = self.dist_test(test_path, flags)
  909. if not test:
  910. self.dist_log("testing failed", stderr=True)
  911. return test
  912. def cc_normalize_flags(self, flags):
  913. """
  914. Remove the conflicts that caused due gathering implied features flags.
  915. Parameters
  916. ----------
  917. 'flags' list, compiler flags
  918. flags should be sorted from the lowest to the highest interest.
  919. Returns
  920. -------
  921. list, filtered from any conflicts.
  922. Examples
  923. --------
  924. >>> self.cc_normalize_flags(['-march=armv8.2-a+fp16', '-march=armv8.2-a+dotprod'])
  925. ['armv8.2-a+fp16+dotprod']
  926. >>> self.cc_normalize_flags(
  927. ['-msse', '-msse2', '-msse3', '-mssse3', '-msse4.1', '-msse4.2', '-mavx', '-march=core-avx2']
  928. )
  929. ['-march=core-avx2']
  930. """
  931. assert(isinstance(flags, list))
  932. if self.cc_is_gcc or self.cc_is_clang or self.cc_is_icc:
  933. return self._cc_normalize_unix(flags)
  934. if self.cc_is_msvc or self.cc_is_iccw:
  935. return self._cc_normalize_win(flags)
  936. return flags
  937. _cc_normalize_unix_mrgx = re.compile(
  938. # 1- to check the highest of
  939. r"^(-mcpu=|-march=|-x[A-Z0-9\-])"
  940. )
  941. _cc_normalize_unix_frgx = re.compile(
  942. # 2- to remove any flags starts with
  943. # -march, -mcpu, -x(INTEL) and '-m' without '='
  944. r"^(?!(-mcpu=|-march=|-x[A-Z0-9\-]))(?!-m[a-z0-9\-\.]*.$)"
  945. )
  946. _cc_normalize_unix_krgx = re.compile(
  947. # 3- keep only the highest of
  948. r"^(-mfpu|-mtune)"
  949. )
  950. _cc_normalize_arch_ver = re.compile(
  951. r"[0-9.]"
  952. )
  953. def _cc_normalize_unix(self, flags):
  954. def ver_flags(f):
  955. # arch ver subflag
  956. # -march=armv8.2-a+fp16fml
  957. tokens = f.split('+')
  958. ver = float('0' + ''.join(
  959. re.findall(self._cc_normalize_arch_ver, tokens[0])
  960. ))
  961. return ver, tokens[0], tokens[1:]
  962. if len(flags) <= 1:
  963. return flags
  964. # get the highest matched flag
  965. for i, cur_flag in enumerate(reversed(flags)):
  966. if not re.match(self._cc_normalize_unix_mrgx, cur_flag):
  967. continue
  968. lower_flags = flags[:-(i+1)]
  969. upper_flags = flags[-i:]
  970. filterd = list(filter(
  971. self._cc_normalize_unix_frgx.search, lower_flags
  972. ))
  973. # gather subflags
  974. ver, arch, subflags = ver_flags(cur_flag)
  975. if ver > 0 and len(subflags) > 0:
  976. for xflag in lower_flags:
  977. xver, _, xsubflags = ver_flags(xflag)
  978. if ver == xver:
  979. subflags = xsubflags + subflags
  980. cur_flag = arch + '+' + '+'.join(subflags)
  981. flags = filterd + [cur_flag]
  982. if i > 0:
  983. flags += upper_flags
  984. break
  985. # to remove overridable flags
  986. final_flags = []
  987. matched = set()
  988. for f in reversed(flags):
  989. match = re.match(self._cc_normalize_unix_krgx, f)
  990. if not match:
  991. pass
  992. elif match[0] in matched:
  993. continue
  994. else:
  995. matched.add(match[0])
  996. final_flags.insert(0, f)
  997. return final_flags
  998. _cc_normalize_win_frgx = re.compile(
  999. r"^(?!(/arch\:|/Qx\:))"
  1000. )
  1001. _cc_normalize_win_mrgx = re.compile(
  1002. r"^(/arch|/Qx:)"
  1003. )
  1004. def _cc_normalize_win(self, flags):
  1005. for i, f in enumerate(reversed(flags)):
  1006. if not re.match(self._cc_normalize_win_mrgx, f):
  1007. continue
  1008. i += 1
  1009. return list(filter(
  1010. self._cc_normalize_win_frgx.search, flags[:-i]
  1011. )) + flags[-i:]
  1012. return flags
  1013. class _Feature:
  1014. """A helper class for `CCompilerOpt` that managing CPU features.
  1015. Attributes
  1016. ----------
  1017. feature_supported : dict
  1018. Dictionary containing all CPU features that supported
  1019. by the platform, according to the specified values in attribute
  1020. `_Config.conf_features` and `_Config.conf_features_partial()`
  1021. feature_min : set
  1022. The minimum support of CPU features, according to
  1023. the specified values in attribute `_Config.conf_min_features`.
  1024. """
  1025. def __init__(self):
  1026. if hasattr(self, "feature_is_cached"):
  1027. return
  1028. self.feature_supported = pfeatures = self.conf_features_partial()
  1029. for feature_name in list(pfeatures.keys()):
  1030. feature = pfeatures[feature_name]
  1031. cfeature = self.conf_features[feature_name]
  1032. feature.update({
  1033. k:v for k,v in cfeature.items() if k not in feature
  1034. })
  1035. disabled = feature.get("disable")
  1036. if disabled is not None:
  1037. pfeatures.pop(feature_name)
  1038. self.dist_log(
  1039. "feature '%s' is disabled," % feature_name,
  1040. disabled, stderr=True
  1041. )
  1042. continue
  1043. # list is used internally for these options
  1044. for option in (
  1045. "implies", "group", "detect", "headers", "flags", "extra_checks"
  1046. ) :
  1047. oval = feature.get(option)
  1048. if isinstance(oval, str):
  1049. feature[option] = oval.split()
  1050. self.feature_min = set()
  1051. min_f = self.conf_min_features.get(self.cc_march, "")
  1052. for F in min_f.upper().split():
  1053. if F in self.feature_supported:
  1054. self.feature_min.add(F)
  1055. self.feature_is_cached = True
  1056. def feature_names(self, names=None, force_flags=None):
  1057. """
  1058. Returns a set of CPU feature names that supported by platform and the **C** compiler.
  1059. Parameters
  1060. ----------
  1061. 'names': sequence or None, optional
  1062. Specify certain CPU features to test it against the **C** compiler.
  1063. if None(default), it will test all current supported features.
  1064. **Note**: feature names must be in upper-case.
  1065. 'force_flags': list or None, optional
  1066. If None(default), default compiler flags for every CPU feature will be used
  1067. during the test.
  1068. """
  1069. assert(
  1070. names is None or (
  1071. not isinstance(names, str) and
  1072. hasattr(names, "__iter__")
  1073. )
  1074. )
  1075. assert(force_flags is None or isinstance(force_flags, list))
  1076. if names is None:
  1077. names = self.feature_supported.keys()
  1078. supported_names = set()
  1079. for f in names:
  1080. if self.feature_is_supported(f, force_flags=force_flags):
  1081. supported_names.add(f)
  1082. return supported_names
  1083. def feature_is_exist(self, name):
  1084. """
  1085. Returns True if a certain feature is exist and covered within
  1086. `_Config.conf_features`.
  1087. Parameters
  1088. ----------
  1089. 'name': str
  1090. feature name in uppercase.
  1091. """
  1092. assert(name.isupper())
  1093. return name in self.conf_features
  1094. def feature_sorted(self, names, reverse=False):
  1095. """
  1096. Sort a list of CPU features ordered by the lowest interest.
  1097. Parameters
  1098. ----------
  1099. 'names': sequence
  1100. sequence of supported feature names in uppercase.
  1101. 'reverse': bool, optional
  1102. If true, the sorted features is reversed. (highest interest)
  1103. Returns
  1104. -------
  1105. list, sorted CPU features
  1106. """
  1107. def sort_cb(k):
  1108. if isinstance(k, str):
  1109. return self.feature_supported[k]["interest"]
  1110. # multiple features
  1111. rank = max([self.feature_supported[f]["interest"] for f in k])
  1112. # FIXME: that's not a safe way to increase the rank for
  1113. # multi targets
  1114. rank += len(k) -1
  1115. return rank
  1116. return sorted(names, reverse=reverse, key=sort_cb)
  1117. def feature_implies(self, names, keep_origins=False):
  1118. """
  1119. Return a set of CPU features that implied by 'names'
  1120. Parameters
  1121. ----------
  1122. names: str or sequence of str
  1123. CPU feature name(s) in uppercase.
  1124. keep_origins: bool
  1125. if False(default) then the returned set will not contain any
  1126. features from 'names'. This case happens only when two features
  1127. imply each other.
  1128. Examples
  1129. --------
  1130. >>> self.feature_implies("SSE3")
  1131. {'SSE', 'SSE2'}
  1132. >>> self.feature_implies("SSE2")
  1133. {'SSE'}
  1134. >>> self.feature_implies("SSE2", keep_origins=True)
  1135. # 'SSE2' found here since 'SSE' and 'SSE2' imply each other
  1136. {'SSE', 'SSE2'}
  1137. """
  1138. def get_implies(name, _caller=set()):
  1139. implies = set()
  1140. d = self.feature_supported[name]
  1141. for i in d.get("implies", []):
  1142. implies.add(i)
  1143. if i in _caller:
  1144. # infinity recursive guard since
  1145. # features can imply each other
  1146. continue
  1147. _caller.add(name)
  1148. implies = implies.union(get_implies(i, _caller))
  1149. return implies
  1150. if isinstance(names, str):
  1151. implies = get_implies(names)
  1152. names = [names]
  1153. else:
  1154. assert(hasattr(names, "__iter__"))
  1155. implies = set()
  1156. for n in names:
  1157. implies = implies.union(get_implies(n))
  1158. if not keep_origins:
  1159. implies.difference_update(names)
  1160. return implies
  1161. def feature_implies_c(self, names):
  1162. """same as feature_implies() but combining 'names'"""
  1163. if isinstance(names, str):
  1164. names = set((names,))
  1165. else:
  1166. names = set(names)
  1167. return names.union(self.feature_implies(names))
  1168. def feature_ahead(self, names):
  1169. """
  1170. Return list of features in 'names' after remove any
  1171. implied features and keep the origins.
  1172. Parameters
  1173. ----------
  1174. 'names': sequence
  1175. sequence of CPU feature names in uppercase.
  1176. Returns
  1177. -------
  1178. list of CPU features sorted as-is 'names'
  1179. Examples
  1180. --------
  1181. >>> self.feature_ahead(["SSE2", "SSE3", "SSE41"])
  1182. ["SSE41"]
  1183. # assume AVX2 and FMA3 implies each other and AVX2
  1184. # is the highest interest
  1185. >>> self.feature_ahead(["SSE2", "SSE3", "SSE41", "AVX2", "FMA3"])
  1186. ["AVX2"]
  1187. # assume AVX2 and FMA3 don't implies each other
  1188. >>> self.feature_ahead(["SSE2", "SSE3", "SSE41", "AVX2", "FMA3"])
  1189. ["AVX2", "FMA3"]
  1190. """
  1191. assert(
  1192. not isinstance(names, str)
  1193. and hasattr(names, '__iter__')
  1194. )
  1195. implies = self.feature_implies(names, keep_origins=True)
  1196. ahead = [n for n in names if n not in implies]
  1197. if len(ahead) == 0:
  1198. # return the highest interested feature
  1199. # if all features imply each other
  1200. ahead = self.feature_sorted(names, reverse=True)[:1]
  1201. return ahead
  1202. def feature_untied(self, names):
  1203. """
  1204. same as 'feature_ahead()' but if both features implied each other
  1205. and keep the highest interest.
  1206. Parameters
  1207. ----------
  1208. 'names': sequence
  1209. sequence of CPU feature names in uppercase.
  1210. Returns
  1211. -------
  1212. list of CPU features sorted as-is 'names'
  1213. Examples
  1214. --------
  1215. >>> self.feature_untied(["SSE2", "SSE3", "SSE41"])
  1216. ["SSE2", "SSE3", "SSE41"]
  1217. # assume AVX2 and FMA3 implies each other
  1218. >>> self.feature_untied(["SSE2", "SSE3", "SSE41", "FMA3", "AVX2"])
  1219. ["SSE2", "SSE3", "SSE41", "AVX2"]
  1220. """
  1221. assert(
  1222. not isinstance(names, str)
  1223. and hasattr(names, '__iter__')
  1224. )
  1225. final = []
  1226. for n in names:
  1227. implies = self.feature_implies(n)
  1228. tied = [
  1229. nn for nn in final
  1230. if nn in implies and n in self.feature_implies(nn)
  1231. ]
  1232. if tied:
  1233. tied = self.feature_sorted(tied + [n])
  1234. if n not in tied[1:]:
  1235. continue
  1236. final.remove(tied[:1][0])
  1237. final.append(n)
  1238. return final
  1239. def feature_get_til(self, names, keyisfalse):
  1240. """
  1241. same as `feature_implies_c()` but stop collecting implied
  1242. features when feature's option that provided through
  1243. parameter 'keyisfalse' is False, also sorting the returned
  1244. features.
  1245. """
  1246. def til(tnames):
  1247. # sort from highest to lowest interest then cut if "key" is False
  1248. tnames = self.feature_implies_c(tnames)
  1249. tnames = self.feature_sorted(tnames, reverse=True)
  1250. for i, n in enumerate(tnames):
  1251. if not self.feature_supported[n].get(keyisfalse, True):
  1252. tnames = tnames[:i+1]
  1253. break
  1254. return tnames
  1255. if isinstance(names, str) or len(names) <= 1:
  1256. names = til(names)
  1257. # normalize the sort
  1258. names.reverse()
  1259. return names
  1260. names = self.feature_ahead(names)
  1261. names = {t for n in names for t in til(n)}
  1262. return self.feature_sorted(names)
  1263. def feature_detect(self, names):
  1264. """
  1265. Return a list of CPU features that required to be detected
  1266. sorted from the lowest to highest interest.
  1267. """
  1268. names = self.feature_get_til(names, "implies_detect")
  1269. detect = []
  1270. for n in names:
  1271. d = self.feature_supported[n]
  1272. detect += d.get("detect", d.get("group", [n]))
  1273. return detect
  1274. @_Cache.me
  1275. def feature_flags(self, names):
  1276. """
  1277. Return a list of CPU features flags sorted from the lowest
  1278. to highest interest.
  1279. """
  1280. names = self.feature_sorted(self.feature_implies_c(names))
  1281. flags = []
  1282. for n in names:
  1283. d = self.feature_supported[n]
  1284. f = d.get("flags", [])
  1285. if not f or not self.cc_test_flags(f):
  1286. continue
  1287. flags += f
  1288. return self.cc_normalize_flags(flags)
  1289. @_Cache.me
  1290. def feature_test(self, name, force_flags=None):
  1291. """
  1292. Test a certain CPU feature against the compiler through its own
  1293. check file.
  1294. Parameters
  1295. ----------
  1296. 'name': str
  1297. Supported CPU feature name.
  1298. 'force_flags': list or None, optional
  1299. If None(default), the returned flags from `feature_flags()`
  1300. will be used.
  1301. """
  1302. if force_flags is None:
  1303. force_flags = self.feature_flags(name)
  1304. self.dist_log(
  1305. "testing feature '%s' with flags (%s)" % (
  1306. name, ' '.join(force_flags)
  1307. ))
  1308. # Each CPU feature must have C source code contains at
  1309. # least one intrinsic or instruction related to this feature.
  1310. test_path = os.path.join(
  1311. self.conf_check_path, "cpu_%s.c" % name.lower()
  1312. )
  1313. if not os.path.exists(test_path):
  1314. self.dist_fatal("feature test file is not exist", test_path)
  1315. test = self.dist_test(test_path, force_flags + self.cc_flags["werror"])
  1316. if not test:
  1317. self.dist_log("testing failed", stderr=True)
  1318. return test
  1319. @_Cache.me
  1320. def feature_is_supported(self, name, force_flags=None):
  1321. """
  1322. Check if a certain CPU feature is supported by the platform and compiler.
  1323. Parameters
  1324. ----------
  1325. 'name': str
  1326. CPU feature name in uppercase.
  1327. 'force_flags': list or None, optional
  1328. If None(default), default compiler flags for every CPU feature will be used
  1329. during test.
  1330. """
  1331. assert(name.isupper())
  1332. assert(force_flags is None or isinstance(force_flags, list))
  1333. supported = name in self.feature_supported
  1334. if supported:
  1335. for impl in self.feature_implies(name):
  1336. if not self.feature_test(impl, force_flags):
  1337. return False
  1338. if not self.feature_test(name, force_flags):
  1339. return False
  1340. return supported
  1341. @_Cache.me
  1342. def feature_can_autovec(self, name):
  1343. """
  1344. check if the feature can be auto-vectorized by the compiler
  1345. """
  1346. assert(isinstance(name, str))
  1347. d = self.feature_supported[name]
  1348. can = d.get("autovec", None)
  1349. if can is None:
  1350. valid_flags = [
  1351. self.cc_test_flags([f]) for f in d.get("flags", [])
  1352. ]
  1353. can = valid_flags and any(valid_flags)
  1354. return can
  1355. @_Cache.me
  1356. def feature_extra_checks(self, name):
  1357. """
  1358. Return a list of supported extra checks after testing them against
  1359. the compiler.
  1360. Parameters
  1361. ----------
  1362. names: str
  1363. CPU feature name in uppercase.
  1364. """
  1365. assert isinstance(name, str)
  1366. d = self.feature_supported[name]
  1367. extra_checks = d.get("extra_checks", [])
  1368. if not extra_checks:
  1369. return []
  1370. self.dist_log("Testing extra checks for feature '%s'" % name, extra_checks)
  1371. flags = self.feature_flags(name)
  1372. available = []
  1373. not_available = []
  1374. for chk in extra_checks:
  1375. test_path = os.path.join(
  1376. self.conf_check_path, "extra_%s.c" % chk.lower()
  1377. )
  1378. if not os.path.exists(test_path):
  1379. self.dist_fatal("extra check file does not exist", test_path)
  1380. is_supported = self.dist_test(test_path, flags + self.cc_flags["werror"])
  1381. if is_supported:
  1382. available.append(chk)
  1383. else:
  1384. not_available.append(chk)
  1385. if not_available:
  1386. self.dist_log("testing failed for checks", not_available, stderr=True)
  1387. return available
  1388. def feature_c_preprocessor(self, feature_name, tabs=0):
  1389. """
  1390. Generate C preprocessor definitions and include headers of a CPU feature.
  1391. Parameters
  1392. ----------
  1393. 'feature_name': str
  1394. CPU feature name in uppercase.
  1395. 'tabs': int
  1396. if > 0, align the generated strings to the right depend on number of tabs.
  1397. Returns
  1398. -------
  1399. str, generated C preprocessor
  1400. Examples
  1401. --------
  1402. >>> self.feature_c_preprocessor("SSE3")
  1403. /** SSE3 **/
  1404. #define NPY_HAVE_SSE3 1
  1405. #include <pmmintrin.h>
  1406. """
  1407. assert(feature_name.isupper())
  1408. feature = self.feature_supported.get(feature_name)
  1409. assert(feature is not None)
  1410. prepr = [
  1411. "/** %s **/" % feature_name,
  1412. "#define %sHAVE_%s 1" % (self.conf_c_prefix, feature_name)
  1413. ]
  1414. prepr += [
  1415. "#include <%s>" % h for h in feature.get("headers", [])
  1416. ]
  1417. extra_defs = feature.get("group", [])
  1418. extra_defs += self.feature_extra_checks(feature_name)
  1419. for edef in extra_defs:
  1420. # Guard extra definitions in case of duplicate with
  1421. # another feature
  1422. prepr += [
  1423. "#ifndef %sHAVE_%s" % (self.conf_c_prefix, edef),
  1424. "\t#define %sHAVE_%s 1" % (self.conf_c_prefix, edef),
  1425. "#endif",
  1426. ]
  1427. if tabs > 0:
  1428. prepr = [('\t'*tabs) + l for l in prepr]
  1429. return '\n'.join(prepr)
  1430. class _Parse:
  1431. """A helper class that parsing main arguments of `CCompilerOpt`,
  1432. also parsing configuration statements in dispatch-able sources.
  1433. Parameters
  1434. ----------
  1435. cpu_baseline: str or None
  1436. minimal set of required CPU features or special options.
  1437. cpu_dispatch: str or None
  1438. dispatched set of additional CPU features or special options.
  1439. Special options can be:
  1440. - **MIN**: Enables the minimum CPU features that utilized via `_Config.conf_min_features`
  1441. - **MAX**: Enables all supported CPU features by the Compiler and platform.
  1442. - **NATIVE**: Enables all CPU features that supported by the current machine.
  1443. - **NONE**: Enables nothing
  1444. - **Operand +/-**: remove or add features, useful with options **MAX**, **MIN** and **NATIVE**.
  1445. NOTE: operand + is only added for nominal reason.
  1446. NOTES:
  1447. - Case-insensitive among all CPU features and special options.
  1448. - Comma or space can be used as a separator.
  1449. - If the CPU feature is not supported by the user platform or compiler,
  1450. it will be skipped rather than raising a fatal error.
  1451. - Any specified CPU features to 'cpu_dispatch' will be skipped if its part of CPU baseline features
  1452. - 'cpu_baseline' force enables implied features.
  1453. Attributes
  1454. ----------
  1455. parse_baseline_names : list
  1456. Final CPU baseline's feature names(sorted from low to high)
  1457. parse_baseline_flags : list
  1458. Compiler flags of baseline features
  1459. parse_dispatch_names : list
  1460. Final CPU dispatch-able feature names(sorted from low to high)
  1461. parse_target_groups : dict
  1462. Dictionary containing initialized target groups that configured
  1463. through class attribute `conf_target_groups`.
  1464. The key is represent the group name and value is a tuple
  1465. contains three items :
  1466. - bool, True if group has the 'baseline' option.
  1467. - list, list of CPU features.
  1468. - list, list of extra compiler flags.
  1469. """
  1470. def __init__(self, cpu_baseline, cpu_dispatch):
  1471. self._parse_policies = dict(
  1472. # POLICY NAME, (HAVE, NOT HAVE, [DEB])
  1473. KEEP_BASELINE = (
  1474. None, self._parse_policy_not_keepbase,
  1475. []
  1476. ),
  1477. KEEP_SORT = (
  1478. self._parse_policy_keepsort,
  1479. self._parse_policy_not_keepsort,
  1480. []
  1481. ),
  1482. MAXOPT = (
  1483. self._parse_policy_maxopt, None,
  1484. []
  1485. ),
  1486. WERROR = (
  1487. self._parse_policy_werror, None,
  1488. []
  1489. ),
  1490. AUTOVEC = (
  1491. self._parse_policy_autovec, None,
  1492. ["MAXOPT"]
  1493. )
  1494. )
  1495. if hasattr(self, "parse_is_cached"):
  1496. return
  1497. self.parse_baseline_names = []
  1498. self.parse_baseline_flags = []
  1499. self.parse_dispatch_names = []
  1500. self.parse_target_groups = {}
  1501. if self.cc_noopt:
  1502. # skip parsing baseline and dispatch args and keep parsing target groups
  1503. cpu_baseline = cpu_dispatch = None
  1504. self.dist_log("check requested baseline")
  1505. if cpu_baseline is not None:
  1506. cpu_baseline = self._parse_arg_features("cpu_baseline", cpu_baseline)
  1507. baseline_names = self.feature_names(cpu_baseline)
  1508. self.parse_baseline_flags = self.feature_flags(baseline_names)
  1509. self.parse_baseline_names = self.feature_sorted(
  1510. self.feature_implies_c(baseline_names)
  1511. )
  1512. self.dist_log("check requested dispatch-able features")
  1513. if cpu_dispatch is not None:
  1514. cpu_dispatch_ = self._parse_arg_features("cpu_dispatch", cpu_dispatch)
  1515. cpu_dispatch = {
  1516. f for f in cpu_dispatch_
  1517. if f not in self.parse_baseline_names
  1518. }
  1519. conflict_baseline = cpu_dispatch_.difference(cpu_dispatch)
  1520. self.parse_dispatch_names = self.feature_sorted(
  1521. self.feature_names(cpu_dispatch)
  1522. )
  1523. if len(conflict_baseline) > 0:
  1524. self.dist_log(
  1525. "skip features", conflict_baseline, "since its part of baseline"
  1526. )
  1527. self.dist_log("initialize targets groups")
  1528. for group_name, tokens in self.conf_target_groups.items():
  1529. self.dist_log("parse target group", group_name)
  1530. GROUP_NAME = group_name.upper()
  1531. if not tokens or not tokens.strip():
  1532. # allow empty groups, useful in case if there's a need
  1533. # to disable certain group since '_parse_target_tokens()'
  1534. # requires at least one valid target
  1535. self.parse_target_groups[GROUP_NAME] = (
  1536. False, [], []
  1537. )
  1538. continue
  1539. has_baseline, features, extra_flags = \
  1540. self._parse_target_tokens(tokens)
  1541. self.parse_target_groups[GROUP_NAME] = (
  1542. has_baseline, features, extra_flags
  1543. )
  1544. self.parse_is_cached = True
  1545. def parse_targets(self, source):
  1546. """
  1547. Fetch and parse configuration statements that required for
  1548. defining the targeted CPU features, statements should be declared
  1549. in the top of source in between **C** comment and start
  1550. with a special mark **@targets**.
  1551. Configuration statements are sort of keywords representing
  1552. CPU features names, group of statements and policies, combined
  1553. together to determine the required optimization.
  1554. Parameters
  1555. ----------
  1556. source: str
  1557. the path of **C** source file.
  1558. Returns
  1559. -------
  1560. - bool, True if group has the 'baseline' option
  1561. - list, list of CPU features
  1562. - list, list of extra compiler flags
  1563. """
  1564. self.dist_log("looking for '@targets' inside -> ", source)
  1565. # get lines between /*@targets and */
  1566. with open(source) as fd:
  1567. tokens = ""
  1568. max_to_reach = 1000 # good enough, isn't?
  1569. start_with = "@targets"
  1570. start_pos = -1
  1571. end_with = "*/"
  1572. end_pos = -1
  1573. for current_line, line in enumerate(fd):
  1574. if current_line == max_to_reach:
  1575. self.dist_fatal("reached the max of lines")
  1576. break
  1577. if start_pos == -1:
  1578. start_pos = line.find(start_with)
  1579. if start_pos == -1:
  1580. continue
  1581. start_pos += len(start_with)
  1582. tokens += line
  1583. end_pos = line.find(end_with)
  1584. if end_pos != -1:
  1585. end_pos += len(tokens) - len(line)
  1586. break
  1587. if start_pos == -1:
  1588. self.dist_fatal("expected to find '%s' within a C comment" % start_with)
  1589. if end_pos == -1:
  1590. self.dist_fatal("expected to end with '%s'" % end_with)
  1591. tokens = tokens[start_pos:end_pos]
  1592. return self._parse_target_tokens(tokens)
  1593. _parse_regex_arg = re.compile(r'\s|[,]|([+-])')
  1594. def _parse_arg_features(self, arg_name, req_features):
  1595. if not isinstance(req_features, str):
  1596. self.dist_fatal("expected a string in '%s'" % arg_name)
  1597. final_features = set()
  1598. # space and comma can be used as a separator
  1599. tokens = list(filter(None, re.split(self._parse_regex_arg, req_features)))
  1600. append = True # append is the default
  1601. for tok in tokens:
  1602. if tok[0] in ("#", "$"):
  1603. self.dist_fatal(
  1604. arg_name, "target groups and policies "
  1605. "aren't allowed from arguments, "
  1606. "only from dispatch-able sources"
  1607. )
  1608. if tok == '+':
  1609. append = True
  1610. continue
  1611. if tok == '-':
  1612. append = False
  1613. continue
  1614. TOK = tok.upper() # we use upper-case internally
  1615. features_to = set()
  1616. if TOK == "NONE":
  1617. pass
  1618. elif TOK == "NATIVE":
  1619. native = self.cc_flags["native"]
  1620. if not native:
  1621. self.dist_fatal(arg_name,
  1622. "native option isn't supported by the compiler"
  1623. )
  1624. features_to = self.feature_names(force_flags=native)
  1625. elif TOK == "MAX":
  1626. features_to = self.feature_supported.keys()
  1627. elif TOK == "MIN":
  1628. features_to = self.feature_min
  1629. else:
  1630. if TOK in self.feature_supported:
  1631. features_to.add(TOK)
  1632. else:
  1633. if not self.feature_is_exist(TOK):
  1634. self.dist_fatal(arg_name,
  1635. ", '%s' isn't a known feature or option" % tok
  1636. )
  1637. if append:
  1638. final_features = final_features.union(features_to)
  1639. else:
  1640. final_features = final_features.difference(features_to)
  1641. append = True # back to default
  1642. return final_features
  1643. _parse_regex_target = re.compile(r'\s|[*,/]|([()])')
  1644. def _parse_target_tokens(self, tokens):
  1645. assert(isinstance(tokens, str))
  1646. final_targets = [] # to keep it sorted as specified
  1647. extra_flags = []
  1648. has_baseline = False
  1649. skipped = set()
  1650. policies = set()
  1651. multi_target = None
  1652. tokens = list(filter(None, re.split(self._parse_regex_target, tokens)))
  1653. if not tokens:
  1654. self.dist_fatal("expected one token at least")
  1655. for tok in tokens:
  1656. TOK = tok.upper()
  1657. ch = tok[0]
  1658. if ch in ('+', '-'):
  1659. self.dist_fatal(
  1660. "+/- are 'not' allowed from target's groups or @targets, "
  1661. "only from cpu_baseline and cpu_dispatch parms"
  1662. )
  1663. elif ch == '$':
  1664. if multi_target is not None:
  1665. self.dist_fatal(
  1666. "policies aren't allowed inside multi-target '()'"
  1667. ", only CPU features"
  1668. )
  1669. policies.add(self._parse_token_policy(TOK))
  1670. elif ch == '#':
  1671. if multi_target is not None:
  1672. self.dist_fatal(
  1673. "target groups aren't allowed inside multi-target '()'"
  1674. ", only CPU features"
  1675. )
  1676. has_baseline, final_targets, extra_flags = \
  1677. self._parse_token_group(TOK, has_baseline, final_targets, extra_flags)
  1678. elif ch == '(':
  1679. if multi_target is not None:
  1680. self.dist_fatal("unclosed multi-target, missing ')'")
  1681. multi_target = set()
  1682. elif ch == ')':
  1683. if multi_target is None:
  1684. self.dist_fatal("multi-target opener '(' wasn't found")
  1685. targets = self._parse_multi_target(multi_target)
  1686. if targets is None:
  1687. skipped.add(tuple(multi_target))
  1688. else:
  1689. if len(targets) == 1:
  1690. targets = targets[0]
  1691. if targets and targets not in final_targets:
  1692. final_targets.append(targets)
  1693. multi_target = None # back to default
  1694. else:
  1695. if TOK == "BASELINE":
  1696. if multi_target is not None:
  1697. self.dist_fatal("baseline isn't allowed inside multi-target '()'")
  1698. has_baseline = True
  1699. continue
  1700. if multi_target is not None:
  1701. multi_target.add(TOK)
  1702. continue
  1703. if not self.feature_is_exist(TOK):
  1704. self.dist_fatal("invalid target name '%s'" % TOK)
  1705. is_enabled = (
  1706. TOK in self.parse_baseline_names or
  1707. TOK in self.parse_dispatch_names
  1708. )
  1709. if is_enabled:
  1710. if TOK not in final_targets:
  1711. final_targets.append(TOK)
  1712. continue
  1713. skipped.add(TOK)
  1714. if multi_target is not None:
  1715. self.dist_fatal("unclosed multi-target, missing ')'")
  1716. if skipped:
  1717. self.dist_log(
  1718. "skip targets", skipped,
  1719. "not part of baseline or dispatch-able features"
  1720. )
  1721. final_targets = self.feature_untied(final_targets)
  1722. # add polices dependencies
  1723. for p in list(policies):
  1724. _, _, deps = self._parse_policies[p]
  1725. for d in deps:
  1726. if d in policies:
  1727. continue
  1728. self.dist_log(
  1729. "policy '%s' force enables '%s'" % (
  1730. p, d
  1731. ))
  1732. policies.add(d)
  1733. # release policies filtrations
  1734. for p, (have, nhave, _) in self._parse_policies.items():
  1735. func = None
  1736. if p in policies:
  1737. func = have
  1738. self.dist_log("policy '%s' is ON" % p)
  1739. else:
  1740. func = nhave
  1741. if not func:
  1742. continue
  1743. has_baseline, final_targets, extra_flags = func(
  1744. has_baseline, final_targets, extra_flags
  1745. )
  1746. return has_baseline, final_targets, extra_flags
  1747. def _parse_token_policy(self, token):
  1748. """validate policy token"""
  1749. if len(token) <= 1 or token[-1:] == token[0]:
  1750. self.dist_fatal("'$' must stuck in the begin of policy name")
  1751. token = token[1:]
  1752. if token not in self._parse_policies:
  1753. self.dist_fatal(
  1754. "'%s' is an invalid policy name, available policies are" % token,
  1755. self._parse_policies.keys()
  1756. )
  1757. return token
  1758. def _parse_token_group(self, token, has_baseline, final_targets, extra_flags):
  1759. """validate group token"""
  1760. if len(token) <= 1 or token[-1:] == token[0]:
  1761. self.dist_fatal("'#' must stuck in the begin of group name")
  1762. token = token[1:]
  1763. ghas_baseline, gtargets, gextra_flags = self.parse_target_groups.get(
  1764. token, (False, None, [])
  1765. )
  1766. if gtargets is None:
  1767. self.dist_fatal(
  1768. "'%s' is an invalid target group name, " % token + \
  1769. "available target groups are",
  1770. self.parse_target_groups.keys()
  1771. )
  1772. if ghas_baseline:
  1773. has_baseline = True
  1774. # always keep sorting as specified
  1775. final_targets += [f for f in gtargets if f not in final_targets]
  1776. extra_flags += [f for f in gextra_flags if f not in extra_flags]
  1777. return has_baseline, final_targets, extra_flags
  1778. def _parse_multi_target(self, targets):
  1779. """validate multi targets that defined between parentheses()"""
  1780. # remove any implied features and keep the origins
  1781. if not targets:
  1782. self.dist_fatal("empty multi-target '()'")
  1783. if not all([
  1784. self.feature_is_exist(tar) for tar in targets
  1785. ]) :
  1786. self.dist_fatal("invalid target name in multi-target", targets)
  1787. if not all([
  1788. (
  1789. tar in self.parse_baseline_names or
  1790. tar in self.parse_dispatch_names
  1791. )
  1792. for tar in targets
  1793. ]) :
  1794. return None
  1795. targets = self.feature_ahead(targets)
  1796. if not targets:
  1797. return None
  1798. # force sort multi targets, so it can be comparable
  1799. targets = self.feature_sorted(targets)
  1800. targets = tuple(targets) # hashable
  1801. return targets
  1802. def _parse_policy_not_keepbase(self, has_baseline, final_targets, extra_flags):
  1803. """skip all baseline features"""
  1804. skipped = []
  1805. for tar in final_targets[:]:
  1806. is_base = False
  1807. if isinstance(tar, str):
  1808. is_base = tar in self.parse_baseline_names
  1809. else:
  1810. # multi targets
  1811. is_base = all([
  1812. f in self.parse_baseline_names
  1813. for f in tar
  1814. ])
  1815. if is_base:
  1816. skipped.append(tar)
  1817. final_targets.remove(tar)
  1818. if skipped:
  1819. self.dist_log("skip baseline features", skipped)
  1820. return has_baseline, final_targets, extra_flags
  1821. def _parse_policy_keepsort(self, has_baseline, final_targets, extra_flags):
  1822. """leave a notice that $keep_sort is on"""
  1823. self.dist_log(
  1824. "policy 'keep_sort' is on, dispatch-able targets", final_targets, "\n"
  1825. "are 'not' sorted depend on the highest interest but"
  1826. "as specified in the dispatch-able source or the extra group"
  1827. )
  1828. return has_baseline, final_targets, extra_flags
  1829. def _parse_policy_not_keepsort(self, has_baseline, final_targets, extra_flags):
  1830. """sorted depend on the highest interest"""
  1831. final_targets = self.feature_sorted(final_targets, reverse=True)
  1832. return has_baseline, final_targets, extra_flags
  1833. def _parse_policy_maxopt(self, has_baseline, final_targets, extra_flags):
  1834. """append the compiler optimization flags"""
  1835. if self.cc_has_debug:
  1836. self.dist_log("debug mode is detected, policy 'maxopt' is skipped.")
  1837. elif self.cc_noopt:
  1838. self.dist_log("optimization is disabled, policy 'maxopt' is skipped.")
  1839. else:
  1840. flags = self.cc_flags["opt"]
  1841. if not flags:
  1842. self.dist_log(
  1843. "current compiler doesn't support optimization flags, "
  1844. "policy 'maxopt' is skipped", stderr=True
  1845. )
  1846. else:
  1847. extra_flags += flags
  1848. return has_baseline, final_targets, extra_flags
  1849. def _parse_policy_werror(self, has_baseline, final_targets, extra_flags):
  1850. """force warnings to treated as errors"""
  1851. flags = self.cc_flags["werror"]
  1852. if not flags:
  1853. self.dist_log(
  1854. "current compiler doesn't support werror flags, "
  1855. "warnings will 'not' treated as errors", stderr=True
  1856. )
  1857. else:
  1858. self.dist_log("compiler warnings are treated as errors")
  1859. extra_flags += flags
  1860. return has_baseline, final_targets, extra_flags
  1861. def _parse_policy_autovec(self, has_baseline, final_targets, extra_flags):
  1862. """skip features that has no auto-vectorized support by compiler"""
  1863. skipped = []
  1864. for tar in final_targets[:]:
  1865. if isinstance(tar, str):
  1866. can = self.feature_can_autovec(tar)
  1867. else: # multiple target
  1868. can = all([
  1869. self.feature_can_autovec(t)
  1870. for t in tar
  1871. ])
  1872. if not can:
  1873. final_targets.remove(tar)
  1874. skipped.append(tar)
  1875. if skipped:
  1876. self.dist_log("skip non auto-vectorized features", skipped)
  1877. return has_baseline, final_targets, extra_flags
  1878. class CCompilerOpt(_Config, _Distutils, _Cache, _CCompiler, _Feature, _Parse):
  1879. """
  1880. A helper class for `CCompiler` aims to provide extra build options
  1881. to effectively control of compiler optimizations that are directly
  1882. related to CPU features.
  1883. """
  1884. def __init__(self, ccompiler, cpu_baseline="min", cpu_dispatch="max", cache_path=None):
  1885. _Config.__init__(self)
  1886. _Distutils.__init__(self, ccompiler)
  1887. _Cache.__init__(self, cache_path, self.dist_info(), cpu_baseline, cpu_dispatch)
  1888. _CCompiler.__init__(self)
  1889. _Feature.__init__(self)
  1890. if not self.cc_noopt and self.cc_has_native:
  1891. self.dist_log(
  1892. "native flag is specified through environment variables. "
  1893. "force cpu-baseline='native'"
  1894. )
  1895. cpu_baseline = "native"
  1896. _Parse.__init__(self, cpu_baseline, cpu_dispatch)
  1897. # keep the requested features untouched, need it later for report
  1898. # and trace purposes
  1899. self._requested_baseline = cpu_baseline
  1900. self._requested_dispatch = cpu_dispatch
  1901. # key is the dispatch-able source and value is a tuple
  1902. # contains two items (has_baseline[boolean], dispatched-features[list])
  1903. self.sources_status = getattr(self, "sources_status", {})
  1904. # every instance should has a separate one
  1905. self.cache_private.add("sources_status")
  1906. # set it at the end to make sure the cache writing was done after init
  1907. # this class
  1908. self.hit_cache = hasattr(self, "hit_cache")
  1909. def is_cached(self):
  1910. """
  1911. Returns True if the class loaded from the cache file
  1912. """
  1913. return self.cache_infile and self.hit_cache
  1914. def cpu_baseline_flags(self):
  1915. """
  1916. Returns a list of final CPU baseline compiler flags
  1917. """
  1918. return self.parse_baseline_flags
  1919. def cpu_baseline_names(self):
  1920. """
  1921. return a list of final CPU baseline feature names
  1922. """
  1923. return self.parse_baseline_names
  1924. def cpu_dispatch_names(self):
  1925. """
  1926. return a list of final CPU dispatch feature names
  1927. """
  1928. return self.parse_dispatch_names
  1929. def try_dispatch(self, sources, src_dir=None, **kwargs):
  1930. """
  1931. Compile one or more dispatch-able sources and generates object files,
  1932. also generates abstract C config headers and macros that
  1933. used later for the final runtime dispatching process.
  1934. The mechanism behind it is to takes each source file that specified
  1935. in 'sources' and branching it into several files depend on
  1936. special configuration statements that must be declared in the
  1937. top of each source which contains targeted CPU features,
  1938. then it compiles every branched source with the proper compiler flags.
  1939. Parameters
  1940. ----------
  1941. sources : list
  1942. Must be a list of dispatch-able sources file paths,
  1943. and configuration statements must be declared inside
  1944. each file.
  1945. src_dir : str
  1946. Path of parent directory for the generated headers and wrapped sources.
  1947. If None(default) the files will generated in-place.
  1948. **kwargs : any
  1949. Arguments to pass on to the `CCompiler.compile()`
  1950. Returns
  1951. -------
  1952. list : generated object files
  1953. Raises
  1954. ------
  1955. CompileError
  1956. Raises by `CCompiler.compile()` on compiling failure.
  1957. DistutilsError
  1958. Some errors during checking the sanity of configuration statements.
  1959. See Also
  1960. --------
  1961. parse_targets :
  1962. Parsing the configuration statements of dispatch-able sources.
  1963. """
  1964. to_compile = {}
  1965. baseline_flags = self.cpu_baseline_flags()
  1966. include_dirs = kwargs.setdefault("include_dirs", [])
  1967. for src in sources:
  1968. output_dir = os.path.dirname(src)
  1969. if src_dir:
  1970. if not output_dir.startswith(src_dir):
  1971. output_dir = os.path.join(src_dir, output_dir)
  1972. if output_dir not in include_dirs:
  1973. # To allow including the generated config header(*.dispatch.h)
  1974. # by the dispatch-able sources
  1975. include_dirs.append(output_dir)
  1976. has_baseline, targets, extra_flags = self.parse_targets(src)
  1977. nochange = self._generate_config(output_dir, src, targets, has_baseline)
  1978. for tar in targets:
  1979. tar_src = self._wrap_target(output_dir, src, tar, nochange=nochange)
  1980. flags = tuple(extra_flags + self.feature_flags(tar))
  1981. to_compile.setdefault(flags, []).append(tar_src)
  1982. if has_baseline:
  1983. flags = tuple(extra_flags + baseline_flags)
  1984. to_compile.setdefault(flags, []).append(src)
  1985. self.sources_status[src] = (has_baseline, targets)
  1986. # For these reasons, the sources are compiled in a separate loop:
  1987. # - Gathering all sources with the same flags to benefit from
  1988. # the parallel compiling as much as possible.
  1989. # - To generate all config headers of the dispatchable sources,
  1990. # before the compilation in case if there are dependency relationships
  1991. # among them.
  1992. objects = []
  1993. for flags, srcs in to_compile.items():
  1994. objects += self.dist_compile(srcs, list(flags), **kwargs)
  1995. return objects
  1996. def generate_dispatch_header(self, header_path):
  1997. """
  1998. Generate the dispatch header which contains the #definitions and headers
  1999. for platform-specific instruction-sets for the enabled CPU baseline and
  2000. dispatch-able features.
  2001. Its highly recommended to take a look at the generated header
  2002. also the generated source files via `try_dispatch()`
  2003. in order to get the full picture.
  2004. """
  2005. self.dist_log("generate CPU dispatch header: (%s)" % header_path)
  2006. baseline_names = self.cpu_baseline_names()
  2007. dispatch_names = self.cpu_dispatch_names()
  2008. baseline_len = len(baseline_names)
  2009. dispatch_len = len(dispatch_names)
  2010. header_dir = os.path.dirname(header_path)
  2011. if not os.path.exists(header_dir):
  2012. self.dist_log(
  2013. f"dispatch header dir {header_dir} does not exist, creating it",
  2014. stderr=True
  2015. )
  2016. os.makedirs(header_dir)
  2017. with open(header_path, 'w') as f:
  2018. baseline_calls = ' \\\n'.join([
  2019. (
  2020. "\t%sWITH_CPU_EXPAND_(MACRO_TO_CALL(%s, __VA_ARGS__))"
  2021. ) % (self.conf_c_prefix, f)
  2022. for f in baseline_names
  2023. ])
  2024. dispatch_calls = ' \\\n'.join([
  2025. (
  2026. "\t%sWITH_CPU_EXPAND_(MACRO_TO_CALL(%s, __VA_ARGS__))"
  2027. ) % (self.conf_c_prefix, f)
  2028. for f in dispatch_names
  2029. ])
  2030. f.write(textwrap.dedent("""\
  2031. /*
  2032. * AUTOGENERATED DON'T EDIT
  2033. * Please make changes to the code generator (distutils/ccompiler_opt.py)
  2034. */
  2035. #define {pfx}WITH_CPU_BASELINE "{baseline_str}"
  2036. #define {pfx}WITH_CPU_DISPATCH "{dispatch_str}"
  2037. #define {pfx}WITH_CPU_BASELINE_N {baseline_len}
  2038. #define {pfx}WITH_CPU_DISPATCH_N {dispatch_len}
  2039. #define {pfx}WITH_CPU_EXPAND_(X) X
  2040. #define {pfx}WITH_CPU_BASELINE_CALL(MACRO_TO_CALL, ...) \\
  2041. {baseline_calls}
  2042. #define {pfx}WITH_CPU_DISPATCH_CALL(MACRO_TO_CALL, ...) \\
  2043. {dispatch_calls}
  2044. """).format(
  2045. pfx=self.conf_c_prefix, baseline_str=" ".join(baseline_names),
  2046. dispatch_str=" ".join(dispatch_names), baseline_len=baseline_len,
  2047. dispatch_len=dispatch_len, baseline_calls=baseline_calls,
  2048. dispatch_calls=dispatch_calls
  2049. ))
  2050. baseline_pre = ''
  2051. for name in baseline_names:
  2052. baseline_pre += self.feature_c_preprocessor(name, tabs=1) + '\n'
  2053. dispatch_pre = ''
  2054. for name in dispatch_names:
  2055. dispatch_pre += textwrap.dedent("""\
  2056. #ifdef {pfx}CPU_TARGET_{name}
  2057. {pre}
  2058. #endif /*{pfx}CPU_TARGET_{name}*/
  2059. """).format(
  2060. pfx=self.conf_c_prefix_, name=name, pre=self.feature_c_preprocessor(
  2061. name, tabs=1
  2062. ))
  2063. f.write(textwrap.dedent("""\
  2064. /******* baseline features *******/
  2065. {baseline_pre}
  2066. /******* dispatch features *******/
  2067. {dispatch_pre}
  2068. """).format(
  2069. pfx=self.conf_c_prefix_, baseline_pre=baseline_pre,
  2070. dispatch_pre=dispatch_pre
  2071. ))
  2072. def report(self, full=False):
  2073. report = []
  2074. platform_rows = []
  2075. baseline_rows = []
  2076. dispatch_rows = []
  2077. report.append(("Platform", platform_rows))
  2078. report.append(("", ""))
  2079. report.append(("CPU baseline", baseline_rows))
  2080. report.append(("", ""))
  2081. report.append(("CPU dispatch", dispatch_rows))
  2082. ########## platform ##########
  2083. platform_rows.append(("Architecture", (
  2084. "unsupported" if self.cc_on_noarch else self.cc_march)
  2085. ))
  2086. platform_rows.append(("Compiler", (
  2087. "unix-like" if self.cc_is_nocc else self.cc_name)
  2088. ))
  2089. ########## baseline ##########
  2090. if self.cc_noopt:
  2091. baseline_rows.append(("Requested", "optimization disabled"))
  2092. else:
  2093. baseline_rows.append(("Requested", repr(self._requested_baseline)))
  2094. baseline_names = self.cpu_baseline_names()
  2095. baseline_rows.append((
  2096. "Enabled", (' '.join(baseline_names) if baseline_names else "none")
  2097. ))
  2098. baseline_flags = self.cpu_baseline_flags()
  2099. baseline_rows.append((
  2100. "Flags", (' '.join(baseline_flags) if baseline_flags else "none")
  2101. ))
  2102. extra_checks = []
  2103. for name in baseline_names:
  2104. extra_checks += self.feature_extra_checks(name)
  2105. baseline_rows.append((
  2106. "Extra checks", (' '.join(extra_checks) if extra_checks else "none")
  2107. ))
  2108. ########## dispatch ##########
  2109. if self.cc_noopt:
  2110. baseline_rows.append(("Requested", "optimization disabled"))
  2111. else:
  2112. dispatch_rows.append(("Requested", repr(self._requested_dispatch)))
  2113. dispatch_names = self.cpu_dispatch_names()
  2114. dispatch_rows.append((
  2115. "Enabled", (' '.join(dispatch_names) if dispatch_names else "none")
  2116. ))
  2117. ########## Generated ##########
  2118. # TODO:
  2119. # - collect object names from 'try_dispatch()'
  2120. # then get size of each object and printed
  2121. # - give more details about the features that not
  2122. # generated due compiler support
  2123. # - find a better output's design.
  2124. #
  2125. target_sources = {}
  2126. for source, (_, targets) in self.sources_status.items():
  2127. for tar in targets:
  2128. target_sources.setdefault(tar, []).append(source)
  2129. if not full or not target_sources:
  2130. generated = ""
  2131. for tar in self.feature_sorted(target_sources):
  2132. sources = target_sources[tar]
  2133. name = tar if isinstance(tar, str) else '(%s)' % ' '.join(tar)
  2134. generated += name + "[%d] " % len(sources)
  2135. dispatch_rows.append(("Generated", generated[:-1] if generated else "none"))
  2136. else:
  2137. dispatch_rows.append(("Generated", ''))
  2138. for tar in self.feature_sorted(target_sources):
  2139. sources = target_sources[tar]
  2140. pretty_name = tar if isinstance(tar, str) else '(%s)' % ' '.join(tar)
  2141. flags = ' '.join(self.feature_flags(tar))
  2142. implies = ' '.join(self.feature_sorted(self.feature_implies(tar)))
  2143. detect = ' '.join(self.feature_detect(tar))
  2144. extra_checks = []
  2145. for name in ((tar,) if isinstance(tar, str) else tar):
  2146. extra_checks += self.feature_extra_checks(name)
  2147. extra_checks = (' '.join(extra_checks) if extra_checks else "none")
  2148. dispatch_rows.append(('', ''))
  2149. dispatch_rows.append((pretty_name, implies))
  2150. dispatch_rows.append(("Flags", flags))
  2151. dispatch_rows.append(("Extra checks", extra_checks))
  2152. dispatch_rows.append(("Detect", detect))
  2153. for src in sources:
  2154. dispatch_rows.append(("", src))
  2155. ###############################
  2156. # TODO: add support for 'markdown' format
  2157. text = []
  2158. secs_len = [len(secs) for secs, _ in report]
  2159. cols_len = [len(col) for _, rows in report for col, _ in rows]
  2160. tab = ' ' * 2
  2161. pad = max(max(secs_len), max(cols_len))
  2162. for sec, rows in report:
  2163. if not sec:
  2164. text.append("") # empty line
  2165. continue
  2166. sec += ' ' * (pad - len(sec))
  2167. text.append(sec + tab + ': ')
  2168. for col, val in rows:
  2169. col += ' ' * (pad - len(col))
  2170. text.append(tab + col + ': ' + val)
  2171. return '\n'.join(text)
  2172. def _wrap_target(self, output_dir, dispatch_src, target, nochange=False):
  2173. assert(isinstance(target, (str, tuple)))
  2174. if isinstance(target, str):
  2175. ext_name = target_name = target
  2176. else:
  2177. # multi-target
  2178. ext_name = '.'.join(target)
  2179. target_name = '__'.join(target)
  2180. wrap_path = os.path.join(output_dir, os.path.basename(dispatch_src))
  2181. wrap_path = "{0}.{2}{1}".format(*os.path.splitext(wrap_path), ext_name.lower())
  2182. if nochange and os.path.exists(wrap_path):
  2183. return wrap_path
  2184. self.dist_log("wrap dispatch-able target -> ", wrap_path)
  2185. # sorting for readability
  2186. features = self.feature_sorted(self.feature_implies_c(target))
  2187. target_join = "#define %sCPU_TARGET_" % self.conf_c_prefix_
  2188. target_defs = [target_join + f for f in features]
  2189. target_defs = '\n'.join(target_defs)
  2190. with open(wrap_path, "w") as fd:
  2191. fd.write(textwrap.dedent("""\
  2192. /**
  2193. * AUTOGENERATED DON'T EDIT
  2194. * Please make changes to the code generator \
  2195. (distutils/ccompiler_opt.py)
  2196. */
  2197. #define {pfx}CPU_TARGET_MODE
  2198. #define {pfx}CPU_TARGET_CURRENT {target_name}
  2199. {target_defs}
  2200. #include "{path}"
  2201. """).format(
  2202. pfx=self.conf_c_prefix_, target_name=target_name,
  2203. path=os.path.abspath(dispatch_src), target_defs=target_defs
  2204. ))
  2205. return wrap_path
  2206. def _generate_config(self, output_dir, dispatch_src, targets, has_baseline=False):
  2207. config_path = os.path.basename(dispatch_src).replace(".c", ".h")
  2208. config_path = os.path.join(output_dir, config_path)
  2209. # check if targets didn't change to avoid recompiling
  2210. cache_hash = self.cache_hash(targets, has_baseline)
  2211. try:
  2212. with open(config_path) as f:
  2213. last_hash = f.readline().split("cache_hash:")
  2214. if len(last_hash) == 2 and int(last_hash[1]) == cache_hash:
  2215. return True
  2216. except IOError:
  2217. pass
  2218. self.dist_log("generate dispatched config -> ", config_path)
  2219. dispatch_calls = []
  2220. for tar in targets:
  2221. if isinstance(tar, str):
  2222. target_name = tar
  2223. else: # multi target
  2224. target_name = '__'.join([t for t in tar])
  2225. req_detect = self.feature_detect(tar)
  2226. req_detect = '&&'.join([
  2227. "CHK(%s)" % f for f in req_detect
  2228. ])
  2229. dispatch_calls.append(
  2230. "\t%sCPU_DISPATCH_EXPAND_(CB((%s), %s, __VA_ARGS__))" % (
  2231. self.conf_c_prefix_, req_detect, target_name
  2232. ))
  2233. dispatch_calls = ' \\\n'.join(dispatch_calls)
  2234. if has_baseline:
  2235. baseline_calls = (
  2236. "\t%sCPU_DISPATCH_EXPAND_(CB(__VA_ARGS__))"
  2237. ) % self.conf_c_prefix_
  2238. else:
  2239. baseline_calls = ''
  2240. with open(config_path, "w") as fd:
  2241. fd.write(textwrap.dedent("""\
  2242. // cache_hash:{cache_hash}
  2243. /**
  2244. * AUTOGENERATED DON'T EDIT
  2245. * Please make changes to the code generator (distutils/ccompiler_opt.py)
  2246. */
  2247. #ifndef {pfx}CPU_DISPATCH_EXPAND_
  2248. #define {pfx}CPU_DISPATCH_EXPAND_(X) X
  2249. #endif
  2250. #undef {pfx}CPU_DISPATCH_BASELINE_CALL
  2251. #undef {pfx}CPU_DISPATCH_CALL
  2252. #define {pfx}CPU_DISPATCH_BASELINE_CALL(CB, ...) \\
  2253. {baseline_calls}
  2254. #define {pfx}CPU_DISPATCH_CALL(CHK, CB, ...) \\
  2255. {dispatch_calls}
  2256. """).format(
  2257. pfx=self.conf_c_prefix_, baseline_calls=baseline_calls,
  2258. dispatch_calls=dispatch_calls, cache_hash=cache_hash
  2259. ))
  2260. return False
  2261. def new_ccompiler_opt(compiler, dispatch_hpath, **kwargs):
  2262. """
  2263. Create a new instance of 'CCompilerOpt' and generate the dispatch header
  2264. which contains the #definitions and headers of platform-specific instruction-sets for
  2265. the enabled CPU baseline and dispatch-able features.
  2266. Parameters
  2267. ----------
  2268. compiler : CCompiler instance
  2269. dispatch_hpath : str
  2270. path of the dispatch header
  2271. **kwargs: passed as-is to `CCompilerOpt(...)`
  2272. Returns
  2273. -------
  2274. new instance of CCompilerOpt
  2275. """
  2276. opt = CCompilerOpt(compiler, **kwargs)
  2277. if not os.path.exists(dispatch_hpath) or not opt.is_cached():
  2278. opt.generate_dispatch_header(dispatch_hpath)
  2279. return opt