setup_common.py 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449
  1. # Code common to build tools
  2. import sys
  3. import warnings
  4. import copy
  5. import textwrap
  6. from numpy.distutils.misc_util import mingw32
  7. #-------------------
  8. # Versioning support
  9. #-------------------
  10. # How to change C_API_VERSION ?
  11. # - increase C_API_VERSION value
  12. # - record the hash for the new C API with the cversions.py script
  13. # and add the hash to cversions.txt
  14. # The hash values are used to remind developers when the C API number was not
  15. # updated - generates a MismatchCAPIWarning warning which is turned into an
  16. # exception for released version.
  17. # Binary compatibility version number. This number is increased whenever the
  18. # C-API is changed such that binary compatibility is broken, i.e. whenever a
  19. # recompile of extension modules is needed.
  20. C_ABI_VERSION = 0x01000009
  21. # Minor API version. This number is increased whenever a change is made to the
  22. # C-API -- whether it breaks binary compatibility or not. Some changes, such
  23. # as adding a function pointer to the end of the function table, can be made
  24. # without breaking binary compatibility. In this case, only the C_API_VERSION
  25. # (*not* C_ABI_VERSION) would be increased. Whenever binary compatibility is
  26. # broken, both C_API_VERSION and C_ABI_VERSION should be increased.
  27. #
  28. # 0x00000008 - 1.7.x
  29. # 0x00000009 - 1.8.x
  30. # 0x00000009 - 1.9.x
  31. # 0x0000000a - 1.10.x
  32. # 0x0000000a - 1.11.x
  33. # 0x0000000a - 1.12.x
  34. # 0x0000000b - 1.13.x
  35. # 0x0000000c - 1.14.x
  36. # 0x0000000c - 1.15.x
  37. # 0x0000000d - 1.16.x
  38. # 0x0000000d - 1.19.x
  39. # 0x0000000e - 1.20.x
  40. C_API_VERSION = 0x0000000e
  41. class MismatchCAPIWarning(Warning):
  42. pass
  43. def is_released(config):
  44. """Return True if a released version of numpy is detected."""
  45. from distutils.version import LooseVersion
  46. v = config.get_version('../version.py')
  47. if v is None:
  48. raise ValueError("Could not get version")
  49. pv = LooseVersion(vstring=v).version
  50. if len(pv) > 3:
  51. return False
  52. return True
  53. def get_api_versions(apiversion, codegen_dir):
  54. """
  55. Return current C API checksum and the recorded checksum.
  56. Return current C API checksum and the recorded checksum for the given
  57. version of the C API version.
  58. """
  59. # Compute the hash of the current API as defined in the .txt files in
  60. # code_generators
  61. sys.path.insert(0, codegen_dir)
  62. try:
  63. m = __import__('genapi')
  64. numpy_api = __import__('numpy_api')
  65. curapi_hash = m.fullapi_hash(numpy_api.full_api)
  66. apis_hash = m.get_versions_hash()
  67. finally:
  68. del sys.path[0]
  69. return curapi_hash, apis_hash[apiversion]
  70. def check_api_version(apiversion, codegen_dir):
  71. """Emits a MismatchCAPIWarning if the C API version needs updating."""
  72. curapi_hash, api_hash = get_api_versions(apiversion, codegen_dir)
  73. # If different hash, it means that the api .txt files in
  74. # codegen_dir have been updated without the API version being
  75. # updated. Any modification in those .txt files should be reflected
  76. # in the api and eventually abi versions.
  77. # To compute the checksum of the current API, use numpy/core/cversions.py
  78. if not curapi_hash == api_hash:
  79. msg = ("API mismatch detected, the C API version "
  80. "numbers have to be updated. Current C api version is %d, "
  81. "with checksum %s, but recorded checksum for C API version %d "
  82. "in core/codegen_dir/cversions.txt is %s. If functions were "
  83. "added in the C API, you have to update C_API_VERSION in %s."
  84. )
  85. warnings.warn(msg % (apiversion, curapi_hash, apiversion, api_hash,
  86. __file__),
  87. MismatchCAPIWarning, stacklevel=2)
  88. # Mandatory functions: if not found, fail the build
  89. MANDATORY_FUNCS = ["sin", "cos", "tan", "sinh", "cosh", "tanh", "fabs",
  90. "floor", "ceil", "sqrt", "log10", "log", "exp", "asin",
  91. "acos", "atan", "fmod", 'modf', 'frexp', 'ldexp']
  92. # Standard functions which may not be available and for which we have a
  93. # replacement implementation. Note that some of these are C99 functions.
  94. OPTIONAL_STDFUNCS = ["expm1", "log1p", "acosh", "asinh", "atanh",
  95. "rint", "trunc", "exp2", "log2", "hypot", "atan2", "pow",
  96. "copysign", "nextafter", "ftello", "fseeko",
  97. "strtoll", "strtoull", "cbrt", "strtold_l", "fallocate",
  98. "backtrace", "madvise"]
  99. OPTIONAL_HEADERS = [
  100. # sse headers only enabled automatically on amd64/x32 builds
  101. "xmmintrin.h", # SSE
  102. "emmintrin.h", # SSE2
  103. "immintrin.h", # AVX
  104. "features.h", # for glibc version linux
  105. "xlocale.h", # see GH#8367
  106. "dlfcn.h", # dladdr
  107. "sys/mman.h", #madvise
  108. ]
  109. # optional gcc compiler builtins and their call arguments and optional a
  110. # required header and definition name (HAVE_ prepended)
  111. # call arguments are required as the compiler will do strict signature checking
  112. OPTIONAL_INTRINSICS = [("__builtin_isnan", '5.'),
  113. ("__builtin_isinf", '5.'),
  114. ("__builtin_isfinite", '5.'),
  115. ("__builtin_bswap32", '5u'),
  116. ("__builtin_bswap64", '5u'),
  117. ("__builtin_expect", '5, 0'),
  118. ("__builtin_mul_overflow", '5, 5, (int*)5'),
  119. # MMX only needed for icc, but some clangs don't have it
  120. ("_m_from_int64", '0', "emmintrin.h"),
  121. ("_mm_load_ps", '(float*)0', "xmmintrin.h"), # SSE
  122. ("_mm_prefetch", '(float*)0, _MM_HINT_NTA',
  123. "xmmintrin.h"), # SSE
  124. ("_mm_load_pd", '(double*)0', "emmintrin.h"), # SSE2
  125. ("__builtin_prefetch", "(float*)0, 0, 3"),
  126. # check that the linker can handle avx
  127. ("__asm__ volatile", '"vpand %xmm1, %xmm2, %xmm3"',
  128. "stdio.h", "LINK_AVX"),
  129. ("__asm__ volatile", '"vpand %ymm1, %ymm2, %ymm3"',
  130. "stdio.h", "LINK_AVX2"),
  131. ("__asm__ volatile", '"vpaddd %zmm1, %zmm2, %zmm3"',
  132. "stdio.h", "LINK_AVX512F"),
  133. ("__asm__ volatile", '"vfpclasspd $0x40, %zmm15, %k6\\n"\
  134. "vmovdqu8 %xmm0, %xmm1\\n"\
  135. "vpbroadcastmb2q %k0, %xmm0\\n"',
  136. "stdio.h", "LINK_AVX512_SKX"),
  137. ("__asm__ volatile", '"xgetbv"', "stdio.h", "XGETBV"),
  138. ]
  139. # function attributes
  140. # tested via "int %s %s(void *);" % (attribute, name)
  141. # function name will be converted to HAVE_<upper-case-name> preprocessor macro
  142. OPTIONAL_FUNCTION_ATTRIBUTES = [('__attribute__((optimize("unroll-loops")))',
  143. 'attribute_optimize_unroll_loops'),
  144. ('__attribute__((optimize("O3")))',
  145. 'attribute_optimize_opt_3'),
  146. ('__attribute__((nonnull (1)))',
  147. 'attribute_nonnull'),
  148. ('__attribute__((target ("avx")))',
  149. 'attribute_target_avx'),
  150. ('__attribute__((target ("avx2")))',
  151. 'attribute_target_avx2'),
  152. ('__attribute__((target ("avx512f")))',
  153. 'attribute_target_avx512f'),
  154. ('__attribute__((target ("avx512f,avx512dq,avx512bw,avx512vl,avx512cd")))',
  155. 'attribute_target_avx512_skx'),
  156. ]
  157. # function attributes with intrinsics
  158. # To ensure your compiler can compile avx intrinsics with just the attributes
  159. # gcc 4.8.4 support attributes but not with intrisics
  160. # tested via "#include<%s> int %s %s(void *){code; return 0;};" % (header, attribute, name, code)
  161. # function name will be converted to HAVE_<upper-case-name> preprocessor macro
  162. # The _mm512_castps_si512 instruction is specific check for AVX-512F support
  163. # in gcc-4.9 which is missing a subset of intrinsics. See
  164. # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=61878
  165. OPTIONAL_FUNCTION_ATTRIBUTES_WITH_INTRINSICS = [('__attribute__((target("avx2,fma")))',
  166. 'attribute_target_avx2_with_intrinsics',
  167. '__m256 temp = _mm256_set1_ps(1.0); temp = \
  168. _mm256_fmadd_ps(temp, temp, temp)',
  169. 'immintrin.h'),
  170. ('__attribute__((target("avx512f")))',
  171. 'attribute_target_avx512f_with_intrinsics',
  172. '__m512i temp = _mm512_castps_si512(_mm512_set1_ps(1.0))',
  173. 'immintrin.h'),
  174. ('__attribute__((target ("avx512f,avx512dq,avx512bw,avx512vl,avx512cd")))',
  175. 'attribute_target_avx512_skx_with_intrinsics',
  176. '__mmask8 temp = _mm512_fpclass_pd_mask(_mm512_set1_pd(1.0), 0x01);\
  177. __m512i temp = _mm512_castps_si512(_mm512_set1_ps(1.0));\
  178. _mm_mask_storeu_epi8(NULL, 0xFF, _mm_broadcastmb_epi64(temp))',
  179. 'immintrin.h'),
  180. ]
  181. # variable attributes tested via "int %s a" % attribute
  182. OPTIONAL_VARIABLE_ATTRIBUTES = ["__thread", "__declspec(thread)"]
  183. # Subset of OPTIONAL_STDFUNCS which may already have HAVE_* defined by Python.h
  184. OPTIONAL_STDFUNCS_MAYBE = [
  185. "expm1", "log1p", "acosh", "atanh", "asinh", "hypot", "copysign",
  186. "ftello", "fseeko"
  187. ]
  188. # C99 functions: float and long double versions
  189. C99_FUNCS = [
  190. "sin", "cos", "tan", "sinh", "cosh", "tanh", "fabs", "floor", "ceil",
  191. "rint", "trunc", "sqrt", "log10", "log", "log1p", "exp", "expm1",
  192. "asin", "acos", "atan", "asinh", "acosh", "atanh", "hypot", "atan2",
  193. "pow", "fmod", "modf", 'frexp', 'ldexp', "exp2", "log2", "copysign",
  194. "nextafter", "cbrt"
  195. ]
  196. C99_FUNCS_SINGLE = [f + 'f' for f in C99_FUNCS]
  197. C99_FUNCS_EXTENDED = [f + 'l' for f in C99_FUNCS]
  198. C99_COMPLEX_TYPES = [
  199. 'complex double', 'complex float', 'complex long double'
  200. ]
  201. C99_COMPLEX_FUNCS = [
  202. "cabs", "cacos", "cacosh", "carg", "casin", "casinh", "catan",
  203. "catanh", "ccos", "ccosh", "cexp", "cimag", "clog", "conj", "cpow",
  204. "cproj", "creal", "csin", "csinh", "csqrt", "ctan", "ctanh"
  205. ]
  206. def fname2def(name):
  207. return "HAVE_%s" % name.upper()
  208. def sym2def(symbol):
  209. define = symbol.replace(' ', '')
  210. return define.upper()
  211. def type2def(symbol):
  212. define = symbol.replace(' ', '_')
  213. return define.upper()
  214. # Code to detect long double representation taken from MPFR m4 macro
  215. def check_long_double_representation(cmd):
  216. cmd._check_compiler()
  217. body = LONG_DOUBLE_REPRESENTATION_SRC % {'type': 'long double'}
  218. # Disable whole program optimization (the default on vs2015, with python 3.5+)
  219. # which generates intermediary object files and prevents checking the
  220. # float representation.
  221. if sys.platform == "win32" and not mingw32():
  222. try:
  223. cmd.compiler.compile_options.remove("/GL")
  224. except (AttributeError, ValueError):
  225. pass
  226. # Disable multi-file interprocedural optimization in the Intel compiler on Linux
  227. # which generates intermediary object files and prevents checking the
  228. # float representation.
  229. elif (sys.platform != "win32"
  230. and cmd.compiler.compiler_type.startswith('intel')
  231. and '-ipo' in cmd.compiler.cc_exe):
  232. newcompiler = cmd.compiler.cc_exe.replace(' -ipo', '')
  233. cmd.compiler.set_executables(
  234. compiler=newcompiler,
  235. compiler_so=newcompiler,
  236. compiler_cxx=newcompiler,
  237. linker_exe=newcompiler,
  238. linker_so=newcompiler + ' -shared'
  239. )
  240. # We need to use _compile because we need the object filename
  241. src, obj = cmd._compile(body, None, None, 'c')
  242. try:
  243. ltype = long_double_representation(pyod(obj))
  244. return ltype
  245. except ValueError:
  246. # try linking to support CC="gcc -flto" or icc -ipo
  247. # struct needs to be volatile so it isn't optimized away
  248. # additionally "clang -flto" requires the foo struct to be used
  249. body = body.replace('struct', 'volatile struct')
  250. body += "int main(void) { return foo.before[0]; }\n"
  251. src, obj = cmd._compile(body, None, None, 'c')
  252. cmd.temp_files.append("_configtest")
  253. cmd.compiler.link_executable([obj], "_configtest")
  254. ltype = long_double_representation(pyod("_configtest"))
  255. return ltype
  256. finally:
  257. cmd._clean()
  258. LONG_DOUBLE_REPRESENTATION_SRC = r"""
  259. /* "before" is 16 bytes to ensure there's no padding between it and "x".
  260. * We're not expecting any "long double" bigger than 16 bytes or with
  261. * alignment requirements stricter than 16 bytes. */
  262. typedef %(type)s test_type;
  263. struct {
  264. char before[16];
  265. test_type x;
  266. char after[8];
  267. } foo = {
  268. { '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0',
  269. '\001', '\043', '\105', '\147', '\211', '\253', '\315', '\357' },
  270. -123456789.0,
  271. { '\376', '\334', '\272', '\230', '\166', '\124', '\062', '\020' }
  272. };
  273. """
  274. def pyod(filename):
  275. """Python implementation of the od UNIX utility (od -b, more exactly).
  276. Parameters
  277. ----------
  278. filename : str
  279. name of the file to get the dump from.
  280. Returns
  281. -------
  282. out : seq
  283. list of lines of od output
  284. Note
  285. ----
  286. We only implement enough to get the necessary information for long double
  287. representation, this is not intended as a compatible replacement for od.
  288. """
  289. out = []
  290. with open(filename, 'rb') as fid:
  291. yo2 = [oct(o)[2:] for o in fid.read()]
  292. for i in range(0, len(yo2), 16):
  293. line = ['%07d' % int(oct(i)[2:])]
  294. line.extend(['%03d' % int(c) for c in yo2[i:i+16]])
  295. out.append(" ".join(line))
  296. return out
  297. _BEFORE_SEQ = ['000', '000', '000', '000', '000', '000', '000', '000',
  298. '001', '043', '105', '147', '211', '253', '315', '357']
  299. _AFTER_SEQ = ['376', '334', '272', '230', '166', '124', '062', '020']
  300. _IEEE_DOUBLE_BE = ['301', '235', '157', '064', '124', '000', '000', '000']
  301. _IEEE_DOUBLE_LE = _IEEE_DOUBLE_BE[::-1]
  302. _INTEL_EXTENDED_12B = ['000', '000', '000', '000', '240', '242', '171', '353',
  303. '031', '300', '000', '000']
  304. _INTEL_EXTENDED_16B = ['000', '000', '000', '000', '240', '242', '171', '353',
  305. '031', '300', '000', '000', '000', '000', '000', '000']
  306. _MOTOROLA_EXTENDED_12B = ['300', '031', '000', '000', '353', '171',
  307. '242', '240', '000', '000', '000', '000']
  308. _IEEE_QUAD_PREC_BE = ['300', '031', '326', '363', '105', '100', '000', '000',
  309. '000', '000', '000', '000', '000', '000', '000', '000']
  310. _IEEE_QUAD_PREC_LE = _IEEE_QUAD_PREC_BE[::-1]
  311. _IBM_DOUBLE_DOUBLE_BE = (['301', '235', '157', '064', '124', '000', '000', '000'] +
  312. ['000'] * 8)
  313. _IBM_DOUBLE_DOUBLE_LE = (['000', '000', '000', '124', '064', '157', '235', '301'] +
  314. ['000'] * 8)
  315. def long_double_representation(lines):
  316. """Given a binary dump as given by GNU od -b, look for long double
  317. representation."""
  318. # Read contains a list of 32 items, each item is a byte (in octal
  319. # representation, as a string). We 'slide' over the output until read is of
  320. # the form before_seq + content + after_sequence, where content is the long double
  321. # representation:
  322. # - content is 12 bytes: 80 bits Intel representation
  323. # - content is 16 bytes: 80 bits Intel representation (64 bits) or quad precision
  324. # - content is 8 bytes: same as double (not implemented yet)
  325. read = [''] * 32
  326. saw = None
  327. for line in lines:
  328. # we skip the first word, as od -b output an index at the beginning of
  329. # each line
  330. for w in line.split()[1:]:
  331. read.pop(0)
  332. read.append(w)
  333. # If the end of read is equal to the after_sequence, read contains
  334. # the long double
  335. if read[-8:] == _AFTER_SEQ:
  336. saw = copy.copy(read)
  337. # if the content was 12 bytes, we only have 32 - 8 - 12 = 12
  338. # "before" bytes. In other words the first 4 "before" bytes went
  339. # past the sliding window.
  340. if read[:12] == _BEFORE_SEQ[4:]:
  341. if read[12:-8] == _INTEL_EXTENDED_12B:
  342. return 'INTEL_EXTENDED_12_BYTES_LE'
  343. if read[12:-8] == _MOTOROLA_EXTENDED_12B:
  344. return 'MOTOROLA_EXTENDED_12_BYTES_BE'
  345. # if the content was 16 bytes, we are left with 32-8-16 = 16
  346. # "before" bytes, so 8 went past the sliding window.
  347. elif read[:8] == _BEFORE_SEQ[8:]:
  348. if read[8:-8] == _INTEL_EXTENDED_16B:
  349. return 'INTEL_EXTENDED_16_BYTES_LE'
  350. elif read[8:-8] == _IEEE_QUAD_PREC_BE:
  351. return 'IEEE_QUAD_BE'
  352. elif read[8:-8] == _IEEE_QUAD_PREC_LE:
  353. return 'IEEE_QUAD_LE'
  354. elif read[8:-8] == _IBM_DOUBLE_DOUBLE_LE:
  355. return 'IBM_DOUBLE_DOUBLE_LE'
  356. elif read[8:-8] == _IBM_DOUBLE_DOUBLE_BE:
  357. return 'IBM_DOUBLE_DOUBLE_BE'
  358. # if the content was 8 bytes, left with 32-8-8 = 16 bytes
  359. elif read[:16] == _BEFORE_SEQ:
  360. if read[16:-8] == _IEEE_DOUBLE_LE:
  361. return 'IEEE_DOUBLE_LE'
  362. elif read[16:-8] == _IEEE_DOUBLE_BE:
  363. return 'IEEE_DOUBLE_BE'
  364. if saw is not None:
  365. raise ValueError("Unrecognized format (%s)" % saw)
  366. else:
  367. # We never detected the after_sequence
  368. raise ValueError("Could not lock sequences (%s)" % saw)
  369. def check_for_right_shift_internal_compiler_error(cmd):
  370. """
  371. On our arm CI, this fails with an internal compilation error
  372. The failure looks like the following, and can be reproduced on ARM64 GCC 5.4:
  373. <source>: In function 'right_shift':
  374. <source>:4:20: internal compiler error: in expand_shift_1, at expmed.c:2349
  375. ip1[i] = ip1[i] >> in2;
  376. ^
  377. Please submit a full bug report,
  378. with preprocessed source if appropriate.
  379. See <http://gcc.gnu.org/bugs.html> for instructions.
  380. Compiler returned: 1
  381. This function returns True if this compiler bug is present, and we need to
  382. turn off optimization for the function
  383. """
  384. cmd._check_compiler()
  385. has_optimize = cmd.try_compile(textwrap.dedent("""\
  386. __attribute__((optimize("O3"))) void right_shift() {}
  387. """), None, None)
  388. if not has_optimize:
  389. return False
  390. no_err = cmd.try_compile(textwrap.dedent("""\
  391. typedef long the_type; /* fails also for unsigned and long long */
  392. __attribute__((optimize("O3"))) void right_shift(the_type in2, the_type *ip1, int n) {
  393. for (int i = 0; i < n; i++) {
  394. if (in2 < (the_type)sizeof(the_type) * 8) {
  395. ip1[i] = ip1[i] >> in2;
  396. }
  397. }
  398. }
  399. """), None, None)
  400. return not no_err