test_format.py 37 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962
  1. # doctest
  2. r''' Test the .npy file format.
  3. Set up:
  4. >>> import sys
  5. >>> from io import BytesIO
  6. >>> from numpy.lib import format
  7. >>>
  8. >>> scalars = [
  9. ... np.uint8,
  10. ... np.int8,
  11. ... np.uint16,
  12. ... np.int16,
  13. ... np.uint32,
  14. ... np.int32,
  15. ... np.uint64,
  16. ... np.int64,
  17. ... np.float32,
  18. ... np.float64,
  19. ... np.complex64,
  20. ... np.complex128,
  21. ... object,
  22. ... ]
  23. >>>
  24. >>> basic_arrays = []
  25. >>>
  26. >>> for scalar in scalars:
  27. ... for endian in '<>':
  28. ... dtype = np.dtype(scalar).newbyteorder(endian)
  29. ... basic = np.arange(15).astype(dtype)
  30. ... basic_arrays.extend([
  31. ... np.array([], dtype=dtype),
  32. ... np.array(10, dtype=dtype),
  33. ... basic,
  34. ... basic.reshape((3,5)),
  35. ... basic.reshape((3,5)).T,
  36. ... basic.reshape((3,5))[::-1,::2],
  37. ... ])
  38. ...
  39. >>>
  40. >>> Pdescr = [
  41. ... ('x', 'i4', (2,)),
  42. ... ('y', 'f8', (2, 2)),
  43. ... ('z', 'u1')]
  44. >>>
  45. >>>
  46. >>> PbufferT = [
  47. ... ([3,2], [[6.,4.],[6.,4.]], 8),
  48. ... ([4,3], [[7.,5.],[7.,5.]], 9),
  49. ... ]
  50. >>>
  51. >>>
  52. >>> Ndescr = [
  53. ... ('x', 'i4', (2,)),
  54. ... ('Info', [
  55. ... ('value', 'c16'),
  56. ... ('y2', 'f8'),
  57. ... ('Info2', [
  58. ... ('name', 'S2'),
  59. ... ('value', 'c16', (2,)),
  60. ... ('y3', 'f8', (2,)),
  61. ... ('z3', 'u4', (2,))]),
  62. ... ('name', 'S2'),
  63. ... ('z2', 'b1')]),
  64. ... ('color', 'S2'),
  65. ... ('info', [
  66. ... ('Name', 'U8'),
  67. ... ('Value', 'c16')]),
  68. ... ('y', 'f8', (2, 2)),
  69. ... ('z', 'u1')]
  70. >>>
  71. >>>
  72. >>> NbufferT = [
  73. ... ([3,2], (6j, 6., ('nn', [6j,4j], [6.,4.], [1,2]), 'NN', True), 'cc', ('NN', 6j), [[6.,4.],[6.,4.]], 8),
  74. ... ([4,3], (7j, 7., ('oo', [7j,5j], [7.,5.], [2,1]), 'OO', False), 'dd', ('OO', 7j), [[7.,5.],[7.,5.]], 9),
  75. ... ]
  76. >>>
  77. >>>
  78. >>> record_arrays = [
  79. ... np.array(PbufferT, dtype=np.dtype(Pdescr).newbyteorder('<')),
  80. ... np.array(NbufferT, dtype=np.dtype(Ndescr).newbyteorder('<')),
  81. ... np.array(PbufferT, dtype=np.dtype(Pdescr).newbyteorder('>')),
  82. ... np.array(NbufferT, dtype=np.dtype(Ndescr).newbyteorder('>')),
  83. ... ]
  84. Test the magic string writing.
  85. >>> format.magic(1, 0)
  86. '\x93NUMPY\x01\x00'
  87. >>> format.magic(0, 0)
  88. '\x93NUMPY\x00\x00'
  89. >>> format.magic(255, 255)
  90. '\x93NUMPY\xff\xff'
  91. >>> format.magic(2, 5)
  92. '\x93NUMPY\x02\x05'
  93. Test the magic string reading.
  94. >>> format.read_magic(BytesIO(format.magic(1, 0)))
  95. (1, 0)
  96. >>> format.read_magic(BytesIO(format.magic(0, 0)))
  97. (0, 0)
  98. >>> format.read_magic(BytesIO(format.magic(255, 255)))
  99. (255, 255)
  100. >>> format.read_magic(BytesIO(format.magic(2, 5)))
  101. (2, 5)
  102. Test the header writing.
  103. >>> for arr in basic_arrays + record_arrays:
  104. ... f = BytesIO()
  105. ... format.write_array_header_1_0(f, arr) # XXX: arr is not a dict, items gets called on it
  106. ... print(repr(f.getvalue()))
  107. ...
  108. "F\x00{'descr': '|u1', 'fortran_order': False, 'shape': (0,)} \n"
  109. "F\x00{'descr': '|u1', 'fortran_order': False, 'shape': ()} \n"
  110. "F\x00{'descr': '|u1', 'fortran_order': False, 'shape': (15,)} \n"
  111. "F\x00{'descr': '|u1', 'fortran_order': False, 'shape': (3, 5)} \n"
  112. "F\x00{'descr': '|u1', 'fortran_order': True, 'shape': (5, 3)} \n"
  113. "F\x00{'descr': '|u1', 'fortran_order': False, 'shape': (3, 3)} \n"
  114. "F\x00{'descr': '|u1', 'fortran_order': False, 'shape': (0,)} \n"
  115. "F\x00{'descr': '|u1', 'fortran_order': False, 'shape': ()} \n"
  116. "F\x00{'descr': '|u1', 'fortran_order': False, 'shape': (15,)} \n"
  117. "F\x00{'descr': '|u1', 'fortran_order': False, 'shape': (3, 5)} \n"
  118. "F\x00{'descr': '|u1', 'fortran_order': True, 'shape': (5, 3)} \n"
  119. "F\x00{'descr': '|u1', 'fortran_order': False, 'shape': (3, 3)} \n"
  120. "F\x00{'descr': '|i1', 'fortran_order': False, 'shape': (0,)} \n"
  121. "F\x00{'descr': '|i1', 'fortran_order': False, 'shape': ()} \n"
  122. "F\x00{'descr': '|i1', 'fortran_order': False, 'shape': (15,)} \n"
  123. "F\x00{'descr': '|i1', 'fortran_order': False, 'shape': (3, 5)} \n"
  124. "F\x00{'descr': '|i1', 'fortran_order': True, 'shape': (5, 3)} \n"
  125. "F\x00{'descr': '|i1', 'fortran_order': False, 'shape': (3, 3)} \n"
  126. "F\x00{'descr': '|i1', 'fortran_order': False, 'shape': (0,)} \n"
  127. "F\x00{'descr': '|i1', 'fortran_order': False, 'shape': ()} \n"
  128. "F\x00{'descr': '|i1', 'fortran_order': False, 'shape': (15,)} \n"
  129. "F\x00{'descr': '|i1', 'fortran_order': False, 'shape': (3, 5)} \n"
  130. "F\x00{'descr': '|i1', 'fortran_order': True, 'shape': (5, 3)} \n"
  131. "F\x00{'descr': '|i1', 'fortran_order': False, 'shape': (3, 3)} \n"
  132. "F\x00{'descr': '<u2', 'fortran_order': False, 'shape': (0,)} \n"
  133. "F\x00{'descr': '<u2', 'fortran_order': False, 'shape': ()} \n"
  134. "F\x00{'descr': '<u2', 'fortran_order': False, 'shape': (15,)} \n"
  135. "F\x00{'descr': '<u2', 'fortran_order': False, 'shape': (3, 5)} \n"
  136. "F\x00{'descr': '<u2', 'fortran_order': True, 'shape': (5, 3)} \n"
  137. "F\x00{'descr': '<u2', 'fortran_order': False, 'shape': (3, 3)} \n"
  138. "F\x00{'descr': '>u2', 'fortran_order': False, 'shape': (0,)} \n"
  139. "F\x00{'descr': '>u2', 'fortran_order': False, 'shape': ()} \n"
  140. "F\x00{'descr': '>u2', 'fortran_order': False, 'shape': (15,)} \n"
  141. "F\x00{'descr': '>u2', 'fortran_order': False, 'shape': (3, 5)} \n"
  142. "F\x00{'descr': '>u2', 'fortran_order': True, 'shape': (5, 3)} \n"
  143. "F\x00{'descr': '>u2', 'fortran_order': False, 'shape': (3, 3)} \n"
  144. "F\x00{'descr': '<i2', 'fortran_order': False, 'shape': (0,)} \n"
  145. "F\x00{'descr': '<i2', 'fortran_order': False, 'shape': ()} \n"
  146. "F\x00{'descr': '<i2', 'fortran_order': False, 'shape': (15,)} \n"
  147. "F\x00{'descr': '<i2', 'fortran_order': False, 'shape': (3, 5)} \n"
  148. "F\x00{'descr': '<i2', 'fortran_order': True, 'shape': (5, 3)} \n"
  149. "F\x00{'descr': '<i2', 'fortran_order': False, 'shape': (3, 3)} \n"
  150. "F\x00{'descr': '>i2', 'fortran_order': False, 'shape': (0,)} \n"
  151. "F\x00{'descr': '>i2', 'fortran_order': False, 'shape': ()} \n"
  152. "F\x00{'descr': '>i2', 'fortran_order': False, 'shape': (15,)} \n"
  153. "F\x00{'descr': '>i2', 'fortran_order': False, 'shape': (3, 5)} \n"
  154. "F\x00{'descr': '>i2', 'fortran_order': True, 'shape': (5, 3)} \n"
  155. "F\x00{'descr': '>i2', 'fortran_order': False, 'shape': (3, 3)} \n"
  156. "F\x00{'descr': '<u4', 'fortran_order': False, 'shape': (0,)} \n"
  157. "F\x00{'descr': '<u4', 'fortran_order': False, 'shape': ()} \n"
  158. "F\x00{'descr': '<u4', 'fortran_order': False, 'shape': (15,)} \n"
  159. "F\x00{'descr': '<u4', 'fortran_order': False, 'shape': (3, 5)} \n"
  160. "F\x00{'descr': '<u4', 'fortran_order': True, 'shape': (5, 3)} \n"
  161. "F\x00{'descr': '<u4', 'fortran_order': False, 'shape': (3, 3)} \n"
  162. "F\x00{'descr': '>u4', 'fortran_order': False, 'shape': (0,)} \n"
  163. "F\x00{'descr': '>u4', 'fortran_order': False, 'shape': ()} \n"
  164. "F\x00{'descr': '>u4', 'fortran_order': False, 'shape': (15,)} \n"
  165. "F\x00{'descr': '>u4', 'fortran_order': False, 'shape': (3, 5)} \n"
  166. "F\x00{'descr': '>u4', 'fortran_order': True, 'shape': (5, 3)} \n"
  167. "F\x00{'descr': '>u4', 'fortran_order': False, 'shape': (3, 3)} \n"
  168. "F\x00{'descr': '<i4', 'fortran_order': False, 'shape': (0,)} \n"
  169. "F\x00{'descr': '<i4', 'fortran_order': False, 'shape': ()} \n"
  170. "F\x00{'descr': '<i4', 'fortran_order': False, 'shape': (15,)} \n"
  171. "F\x00{'descr': '<i4', 'fortran_order': False, 'shape': (3, 5)} \n"
  172. "F\x00{'descr': '<i4', 'fortran_order': True, 'shape': (5, 3)} \n"
  173. "F\x00{'descr': '<i4', 'fortran_order': False, 'shape': (3, 3)} \n"
  174. "F\x00{'descr': '>i4', 'fortran_order': False, 'shape': (0,)} \n"
  175. "F\x00{'descr': '>i4', 'fortran_order': False, 'shape': ()} \n"
  176. "F\x00{'descr': '>i4', 'fortran_order': False, 'shape': (15,)} \n"
  177. "F\x00{'descr': '>i4', 'fortran_order': False, 'shape': (3, 5)} \n"
  178. "F\x00{'descr': '>i4', 'fortran_order': True, 'shape': (5, 3)} \n"
  179. "F\x00{'descr': '>i4', 'fortran_order': False, 'shape': (3, 3)} \n"
  180. "F\x00{'descr': '<u8', 'fortran_order': False, 'shape': (0,)} \n"
  181. "F\x00{'descr': '<u8', 'fortran_order': False, 'shape': ()} \n"
  182. "F\x00{'descr': '<u8', 'fortran_order': False, 'shape': (15,)} \n"
  183. "F\x00{'descr': '<u8', 'fortran_order': False, 'shape': (3, 5)} \n"
  184. "F\x00{'descr': '<u8', 'fortran_order': True, 'shape': (5, 3)} \n"
  185. "F\x00{'descr': '<u8', 'fortran_order': False, 'shape': (3, 3)} \n"
  186. "F\x00{'descr': '>u8', 'fortran_order': False, 'shape': (0,)} \n"
  187. "F\x00{'descr': '>u8', 'fortran_order': False, 'shape': ()} \n"
  188. "F\x00{'descr': '>u8', 'fortran_order': False, 'shape': (15,)} \n"
  189. "F\x00{'descr': '>u8', 'fortran_order': False, 'shape': (3, 5)} \n"
  190. "F\x00{'descr': '>u8', 'fortran_order': True, 'shape': (5, 3)} \n"
  191. "F\x00{'descr': '>u8', 'fortran_order': False, 'shape': (3, 3)} \n"
  192. "F\x00{'descr': '<i8', 'fortran_order': False, 'shape': (0,)} \n"
  193. "F\x00{'descr': '<i8', 'fortran_order': False, 'shape': ()} \n"
  194. "F\x00{'descr': '<i8', 'fortran_order': False, 'shape': (15,)} \n"
  195. "F\x00{'descr': '<i8', 'fortran_order': False, 'shape': (3, 5)} \n"
  196. "F\x00{'descr': '<i8', 'fortran_order': True, 'shape': (5, 3)} \n"
  197. "F\x00{'descr': '<i8', 'fortran_order': False, 'shape': (3, 3)} \n"
  198. "F\x00{'descr': '>i8', 'fortran_order': False, 'shape': (0,)} \n"
  199. "F\x00{'descr': '>i8', 'fortran_order': False, 'shape': ()} \n"
  200. "F\x00{'descr': '>i8', 'fortran_order': False, 'shape': (15,)} \n"
  201. "F\x00{'descr': '>i8', 'fortran_order': False, 'shape': (3, 5)} \n"
  202. "F\x00{'descr': '>i8', 'fortran_order': True, 'shape': (5, 3)} \n"
  203. "F\x00{'descr': '>i8', 'fortran_order': False, 'shape': (3, 3)} \n"
  204. "F\x00{'descr': '<f4', 'fortran_order': False, 'shape': (0,)} \n"
  205. "F\x00{'descr': '<f4', 'fortran_order': False, 'shape': ()} \n"
  206. "F\x00{'descr': '<f4', 'fortran_order': False, 'shape': (15,)} \n"
  207. "F\x00{'descr': '<f4', 'fortran_order': False, 'shape': (3, 5)} \n"
  208. "F\x00{'descr': '<f4', 'fortran_order': True, 'shape': (5, 3)} \n"
  209. "F\x00{'descr': '<f4', 'fortran_order': False, 'shape': (3, 3)} \n"
  210. "F\x00{'descr': '>f4', 'fortran_order': False, 'shape': (0,)} \n"
  211. "F\x00{'descr': '>f4', 'fortran_order': False, 'shape': ()} \n"
  212. "F\x00{'descr': '>f4', 'fortran_order': False, 'shape': (15,)} \n"
  213. "F\x00{'descr': '>f4', 'fortran_order': False, 'shape': (3, 5)} \n"
  214. "F\x00{'descr': '>f4', 'fortran_order': True, 'shape': (5, 3)} \n"
  215. "F\x00{'descr': '>f4', 'fortran_order': False, 'shape': (3, 3)} \n"
  216. "F\x00{'descr': '<f8', 'fortran_order': False, 'shape': (0,)} \n"
  217. "F\x00{'descr': '<f8', 'fortran_order': False, 'shape': ()} \n"
  218. "F\x00{'descr': '<f8', 'fortran_order': False, 'shape': (15,)} \n"
  219. "F\x00{'descr': '<f8', 'fortran_order': False, 'shape': (3, 5)} \n"
  220. "F\x00{'descr': '<f8', 'fortran_order': True, 'shape': (5, 3)} \n"
  221. "F\x00{'descr': '<f8', 'fortran_order': False, 'shape': (3, 3)} \n"
  222. "F\x00{'descr': '>f8', 'fortran_order': False, 'shape': (0,)} \n"
  223. "F\x00{'descr': '>f8', 'fortran_order': False, 'shape': ()} \n"
  224. "F\x00{'descr': '>f8', 'fortran_order': False, 'shape': (15,)} \n"
  225. "F\x00{'descr': '>f8', 'fortran_order': False, 'shape': (3, 5)} \n"
  226. "F\x00{'descr': '>f8', 'fortran_order': True, 'shape': (5, 3)} \n"
  227. "F\x00{'descr': '>f8', 'fortran_order': False, 'shape': (3, 3)} \n"
  228. "F\x00{'descr': '<c8', 'fortran_order': False, 'shape': (0,)} \n"
  229. "F\x00{'descr': '<c8', 'fortran_order': False, 'shape': ()} \n"
  230. "F\x00{'descr': '<c8', 'fortran_order': False, 'shape': (15,)} \n"
  231. "F\x00{'descr': '<c8', 'fortran_order': False, 'shape': (3, 5)} \n"
  232. "F\x00{'descr': '<c8', 'fortran_order': True, 'shape': (5, 3)} \n"
  233. "F\x00{'descr': '<c8', 'fortran_order': False, 'shape': (3, 3)} \n"
  234. "F\x00{'descr': '>c8', 'fortran_order': False, 'shape': (0,)} \n"
  235. "F\x00{'descr': '>c8', 'fortran_order': False, 'shape': ()} \n"
  236. "F\x00{'descr': '>c8', 'fortran_order': False, 'shape': (15,)} \n"
  237. "F\x00{'descr': '>c8', 'fortran_order': False, 'shape': (3, 5)} \n"
  238. "F\x00{'descr': '>c8', 'fortran_order': True, 'shape': (5, 3)} \n"
  239. "F\x00{'descr': '>c8', 'fortran_order': False, 'shape': (3, 3)} \n"
  240. "F\x00{'descr': '<c16', 'fortran_order': False, 'shape': (0,)} \n"
  241. "F\x00{'descr': '<c16', 'fortran_order': False, 'shape': ()} \n"
  242. "F\x00{'descr': '<c16', 'fortran_order': False, 'shape': (15,)} \n"
  243. "F\x00{'descr': '<c16', 'fortran_order': False, 'shape': (3, 5)} \n"
  244. "F\x00{'descr': '<c16', 'fortran_order': True, 'shape': (5, 3)} \n"
  245. "F\x00{'descr': '<c16', 'fortran_order': False, 'shape': (3, 3)} \n"
  246. "F\x00{'descr': '>c16', 'fortran_order': False, 'shape': (0,)} \n"
  247. "F\x00{'descr': '>c16', 'fortran_order': False, 'shape': ()} \n"
  248. "F\x00{'descr': '>c16', 'fortran_order': False, 'shape': (15,)} \n"
  249. "F\x00{'descr': '>c16', 'fortran_order': False, 'shape': (3, 5)} \n"
  250. "F\x00{'descr': '>c16', 'fortran_order': True, 'shape': (5, 3)} \n"
  251. "F\x00{'descr': '>c16', 'fortran_order': False, 'shape': (3, 3)} \n"
  252. "F\x00{'descr': 'O', 'fortran_order': False, 'shape': (0,)} \n"
  253. "F\x00{'descr': 'O', 'fortran_order': False, 'shape': ()} \n"
  254. "F\x00{'descr': 'O', 'fortran_order': False, 'shape': (15,)} \n"
  255. "F\x00{'descr': 'O', 'fortran_order': False, 'shape': (3, 5)} \n"
  256. "F\x00{'descr': 'O', 'fortran_order': True, 'shape': (5, 3)} \n"
  257. "F\x00{'descr': 'O', 'fortran_order': False, 'shape': (3, 3)} \n"
  258. "F\x00{'descr': 'O', 'fortran_order': False, 'shape': (0,)} \n"
  259. "F\x00{'descr': 'O', 'fortran_order': False, 'shape': ()} \n"
  260. "F\x00{'descr': 'O', 'fortran_order': False, 'shape': (15,)} \n"
  261. "F\x00{'descr': 'O', 'fortran_order': False, 'shape': (3, 5)} \n"
  262. "F\x00{'descr': 'O', 'fortran_order': True, 'shape': (5, 3)} \n"
  263. "F\x00{'descr': 'O', 'fortran_order': False, 'shape': (3, 3)} \n"
  264. "v\x00{'descr': [('x', '<i4', (2,)), ('y', '<f8', (2, 2)), ('z', '|u1')],\n 'fortran_order': False,\n 'shape': (2,)} \n"
  265. "\x16\x02{'descr': [('x', '<i4', (2,)),\n ('Info',\n [('value', '<c16'),\n ('y2', '<f8'),\n ('Info2',\n [('name', '|S2'),\n ('value', '<c16', (2,)),\n ('y3', '<f8', (2,)),\n ('z3', '<u4', (2,))]),\n ('name', '|S2'),\n ('z2', '|b1')]),\n ('color', '|S2'),\n ('info', [('Name', '<U8'), ('Value', '<c16')]),\n ('y', '<f8', (2, 2)),\n ('z', '|u1')],\n 'fortran_order': False,\n 'shape': (2,)} \n"
  266. "v\x00{'descr': [('x', '>i4', (2,)), ('y', '>f8', (2, 2)), ('z', '|u1')],\n 'fortran_order': False,\n 'shape': (2,)} \n"
  267. "\x16\x02{'descr': [('x', '>i4', (2,)),\n ('Info',\n [('value', '>c16'),\n ('y2', '>f8'),\n ('Info2',\n [('name', '|S2'),\n ('value', '>c16', (2,)),\n ('y3', '>f8', (2,)),\n ('z3', '>u4', (2,))]),\n ('name', '|S2'),\n ('z2', '|b1')]),\n ('color', '|S2'),\n ('info', [('Name', '>U8'), ('Value', '>c16')]),\n ('y', '>f8', (2, 2)),\n ('z', '|u1')],\n 'fortran_order': False,\n 'shape': (2,)} \n"
  268. '''
  269. import sys
  270. import os
  271. import shutil
  272. import tempfile
  273. import warnings
  274. import pytest
  275. from io import BytesIO
  276. import numpy as np
  277. from numpy.testing import (
  278. assert_, assert_array_equal, assert_raises, assert_raises_regex,
  279. assert_warns,
  280. )
  281. from numpy.lib import format
  282. # Generate some basic arrays to test with.
  283. scalars = [
  284. np.uint8,
  285. np.int8,
  286. np.uint16,
  287. np.int16,
  288. np.uint32,
  289. np.int32,
  290. np.uint64,
  291. np.int64,
  292. np.float32,
  293. np.float64,
  294. np.complex64,
  295. np.complex128,
  296. object,
  297. ]
  298. basic_arrays = []
  299. for scalar in scalars:
  300. for endian in '<>':
  301. dtype = np.dtype(scalar).newbyteorder(endian)
  302. basic = np.arange(1500).astype(dtype)
  303. basic_arrays.extend([
  304. # Empty
  305. np.array([], dtype=dtype),
  306. # Rank-0
  307. np.array(10, dtype=dtype),
  308. # 1-D
  309. basic,
  310. # 2-D C-contiguous
  311. basic.reshape((30, 50)),
  312. # 2-D F-contiguous
  313. basic.reshape((30, 50)).T,
  314. # 2-D non-contiguous
  315. basic.reshape((30, 50))[::-1, ::2],
  316. ])
  317. # More complicated record arrays.
  318. # This is the structure of the table used for plain objects:
  319. #
  320. # +-+-+-+
  321. # |x|y|z|
  322. # +-+-+-+
  323. # Structure of a plain array description:
  324. Pdescr = [
  325. ('x', 'i4', (2,)),
  326. ('y', 'f8', (2, 2)),
  327. ('z', 'u1')]
  328. # A plain list of tuples with values for testing:
  329. PbufferT = [
  330. # x y z
  331. ([3, 2], [[6., 4.], [6., 4.]], 8),
  332. ([4, 3], [[7., 5.], [7., 5.]], 9),
  333. ]
  334. # This is the structure of the table used for nested objects (DON'T PANIC!):
  335. #
  336. # +-+---------------------------------+-----+----------+-+-+
  337. # |x|Info |color|info |y|z|
  338. # | +-----+--+----------------+----+--+ +----+-----+ | |
  339. # | |value|y2|Info2 |name|z2| |Name|Value| | |
  340. # | | | +----+-----+--+--+ | | | | | | |
  341. # | | | |name|value|y3|z3| | | | | | | |
  342. # +-+-----+--+----+-----+--+--+----+--+-----+----+-----+-+-+
  343. #
  344. # The corresponding nested array description:
  345. Ndescr = [
  346. ('x', 'i4', (2,)),
  347. ('Info', [
  348. ('value', 'c16'),
  349. ('y2', 'f8'),
  350. ('Info2', [
  351. ('name', 'S2'),
  352. ('value', 'c16', (2,)),
  353. ('y3', 'f8', (2,)),
  354. ('z3', 'u4', (2,))]),
  355. ('name', 'S2'),
  356. ('z2', 'b1')]),
  357. ('color', 'S2'),
  358. ('info', [
  359. ('Name', 'U8'),
  360. ('Value', 'c16')]),
  361. ('y', 'f8', (2, 2)),
  362. ('z', 'u1')]
  363. NbufferT = [
  364. # x Info color info y z
  365. # value y2 Info2 name z2 Name Value
  366. # name value y3 z3
  367. ([3, 2], (6j, 6., ('nn', [6j, 4j], [6., 4.], [1, 2]), 'NN', True),
  368. 'cc', ('NN', 6j), [[6., 4.], [6., 4.]], 8),
  369. ([4, 3], (7j, 7., ('oo', [7j, 5j], [7., 5.], [2, 1]), 'OO', False),
  370. 'dd', ('OO', 7j), [[7., 5.], [7., 5.]], 9),
  371. ]
  372. record_arrays = [
  373. np.array(PbufferT, dtype=np.dtype(Pdescr).newbyteorder('<')),
  374. np.array(NbufferT, dtype=np.dtype(Ndescr).newbyteorder('<')),
  375. np.array(PbufferT, dtype=np.dtype(Pdescr).newbyteorder('>')),
  376. np.array(NbufferT, dtype=np.dtype(Ndescr).newbyteorder('>')),
  377. np.zeros(1, dtype=[('c', ('<f8', (5,)), (2,))])
  378. ]
  379. #BytesIO that reads a random number of bytes at a time
  380. class BytesIOSRandomSize(BytesIO):
  381. def read(self, size=None):
  382. import random
  383. size = random.randint(1, size)
  384. return super(BytesIOSRandomSize, self).read(size)
  385. def roundtrip(arr):
  386. f = BytesIO()
  387. format.write_array(f, arr)
  388. f2 = BytesIO(f.getvalue())
  389. arr2 = format.read_array(f2, allow_pickle=True)
  390. return arr2
  391. def roundtrip_randsize(arr):
  392. f = BytesIO()
  393. format.write_array(f, arr)
  394. f2 = BytesIOSRandomSize(f.getvalue())
  395. arr2 = format.read_array(f2)
  396. return arr2
  397. def roundtrip_truncated(arr):
  398. f = BytesIO()
  399. format.write_array(f, arr)
  400. #BytesIO is one byte short
  401. f2 = BytesIO(f.getvalue()[0:-1])
  402. arr2 = format.read_array(f2)
  403. return arr2
  404. def assert_equal_(o1, o2):
  405. assert_(o1 == o2)
  406. def test_roundtrip():
  407. for arr in basic_arrays + record_arrays:
  408. arr2 = roundtrip(arr)
  409. assert_array_equal(arr, arr2)
  410. def test_roundtrip_randsize():
  411. for arr in basic_arrays + record_arrays:
  412. if arr.dtype != object:
  413. arr2 = roundtrip_randsize(arr)
  414. assert_array_equal(arr, arr2)
  415. def test_roundtrip_truncated():
  416. for arr in basic_arrays:
  417. if arr.dtype != object:
  418. assert_raises(ValueError, roundtrip_truncated, arr)
  419. def test_long_str():
  420. # check items larger than internal buffer size, gh-4027
  421. long_str_arr = np.ones(1, dtype=np.dtype((str, format.BUFFER_SIZE + 1)))
  422. long_str_arr2 = roundtrip(long_str_arr)
  423. assert_array_equal(long_str_arr, long_str_arr2)
  424. def test_memmap_roundtrip(tmpdir):
  425. for i, arr in enumerate(basic_arrays + record_arrays):
  426. if arr.dtype.hasobject:
  427. # Skip these since they can't be mmap'ed.
  428. continue
  429. # Write it out normally and through mmap.
  430. nfn = os.path.join(tmpdir, f'normal{i}.npy')
  431. mfn = os.path.join(tmpdir, f'memmap{i}.npy')
  432. with open(nfn, 'wb') as fp:
  433. format.write_array(fp, arr)
  434. fortran_order = (
  435. arr.flags.f_contiguous and not arr.flags.c_contiguous)
  436. ma = format.open_memmap(mfn, mode='w+', dtype=arr.dtype,
  437. shape=arr.shape, fortran_order=fortran_order)
  438. ma[...] = arr
  439. ma.flush()
  440. # Check that both of these files' contents are the same.
  441. with open(nfn, 'rb') as fp:
  442. normal_bytes = fp.read()
  443. with open(mfn, 'rb') as fp:
  444. memmap_bytes = fp.read()
  445. assert_equal_(normal_bytes, memmap_bytes)
  446. # Check that reading the file using memmap works.
  447. ma = format.open_memmap(nfn, mode='r')
  448. ma.flush()
  449. def test_compressed_roundtrip(tmpdir):
  450. arr = np.random.rand(200, 200)
  451. npz_file = os.path.join(tmpdir, 'compressed.npz')
  452. np.savez_compressed(npz_file, arr=arr)
  453. with np.load(npz_file) as npz:
  454. arr1 = npz['arr']
  455. assert_array_equal(arr, arr1)
  456. # aligned
  457. dt1 = np.dtype('i1, i4, i1', align=True)
  458. # non-aligned, explicit offsets
  459. dt2 = np.dtype({'names': ['a', 'b'], 'formats': ['i4', 'i4'],
  460. 'offsets': [1, 6]})
  461. # nested struct-in-struct
  462. dt3 = np.dtype({'names': ['c', 'd'], 'formats': ['i4', dt2]})
  463. # field with '' name
  464. dt4 = np.dtype({'names': ['a', '', 'b'], 'formats': ['i4']*3})
  465. # titles
  466. dt5 = np.dtype({'names': ['a', 'b'], 'formats': ['i4', 'i4'],
  467. 'offsets': [1, 6], 'titles': ['aa', 'bb']})
  468. # empty
  469. dt6 = np.dtype({'names': [], 'formats': [], 'itemsize': 8})
  470. @pytest.mark.parametrize("dt", [dt1, dt2, dt3, dt4, dt5, dt6])
  471. def test_load_padded_dtype(tmpdir, dt):
  472. arr = np.zeros(3, dt)
  473. for i in range(3):
  474. arr[i] = i + 5
  475. npz_file = os.path.join(tmpdir, 'aligned.npz')
  476. np.savez(npz_file, arr=arr)
  477. with np.load(npz_file) as npz:
  478. arr1 = npz['arr']
  479. assert_array_equal(arr, arr1)
  480. def test_python2_python3_interoperability():
  481. fname = 'win64python2.npy'
  482. path = os.path.join(os.path.dirname(__file__), 'data', fname)
  483. data = np.load(path)
  484. assert_array_equal(data, np.ones(2))
  485. def test_pickle_python2_python3():
  486. # Test that loading object arrays saved on Python 2 works both on
  487. # Python 2 and Python 3 and vice versa
  488. data_dir = os.path.join(os.path.dirname(__file__), 'data')
  489. expected = np.array([None, range, u'\u512a\u826f',
  490. b'\xe4\xb8\x8d\xe8\x89\xaf'],
  491. dtype=object)
  492. for fname in ['py2-objarr.npy', 'py2-objarr.npz',
  493. 'py3-objarr.npy', 'py3-objarr.npz']:
  494. path = os.path.join(data_dir, fname)
  495. for encoding in ['bytes', 'latin1']:
  496. data_f = np.load(path, allow_pickle=True, encoding=encoding)
  497. if fname.endswith('.npz'):
  498. data = data_f['x']
  499. data_f.close()
  500. else:
  501. data = data_f
  502. if encoding == 'latin1' and fname.startswith('py2'):
  503. assert_(isinstance(data[3], str))
  504. assert_array_equal(data[:-1], expected[:-1])
  505. # mojibake occurs
  506. assert_array_equal(data[-1].encode(encoding), expected[-1])
  507. else:
  508. assert_(isinstance(data[3], bytes))
  509. assert_array_equal(data, expected)
  510. if fname.startswith('py2'):
  511. if fname.endswith('.npz'):
  512. data = np.load(path, allow_pickle=True)
  513. assert_raises(UnicodeError, data.__getitem__, 'x')
  514. data.close()
  515. data = np.load(path, allow_pickle=True, fix_imports=False,
  516. encoding='latin1')
  517. assert_raises(ImportError, data.__getitem__, 'x')
  518. data.close()
  519. else:
  520. assert_raises(UnicodeError, np.load, path,
  521. allow_pickle=True)
  522. assert_raises(ImportError, np.load, path,
  523. allow_pickle=True, fix_imports=False,
  524. encoding='latin1')
  525. def test_pickle_disallow(tmpdir):
  526. data_dir = os.path.join(os.path.dirname(__file__), 'data')
  527. path = os.path.join(data_dir, 'py2-objarr.npy')
  528. assert_raises(ValueError, np.load, path,
  529. allow_pickle=False, encoding='latin1')
  530. path = os.path.join(data_dir, 'py2-objarr.npz')
  531. with np.load(path, allow_pickle=False, encoding='latin1') as f:
  532. assert_raises(ValueError, f.__getitem__, 'x')
  533. path = os.path.join(tmpdir, 'pickle-disabled.npy')
  534. assert_raises(ValueError, np.save, path, np.array([None], dtype=object),
  535. allow_pickle=False)
  536. @pytest.mark.parametrize('dt', [
  537. np.dtype(np.dtype([('a', np.int8),
  538. ('b', np.int16),
  539. ('c', np.int32),
  540. ], align=True),
  541. (3,)),
  542. np.dtype([('x', np.dtype({'names':['a','b'],
  543. 'formats':['i1','i1'],
  544. 'offsets':[0,4],
  545. 'itemsize':8,
  546. },
  547. (3,)),
  548. (4,),
  549. )]),
  550. np.dtype([('x',
  551. ('<f8', (5,)),
  552. (2,),
  553. )]),
  554. np.dtype([('x', np.dtype((
  555. np.dtype((
  556. np.dtype({'names':['a','b'],
  557. 'formats':['i1','i1'],
  558. 'offsets':[0,4],
  559. 'itemsize':8}),
  560. (3,)
  561. )),
  562. (4,)
  563. )))
  564. ]),
  565. np.dtype([
  566. ('a', np.dtype((
  567. np.dtype((
  568. np.dtype((
  569. np.dtype([
  570. ('a', int),
  571. ('b', np.dtype({'names':['a','b'],
  572. 'formats':['i1','i1'],
  573. 'offsets':[0,4],
  574. 'itemsize':8})),
  575. ]),
  576. (3,),
  577. )),
  578. (4,),
  579. )),
  580. (5,),
  581. )))
  582. ]),
  583. ])
  584. def test_descr_to_dtype(dt):
  585. dt1 = format.descr_to_dtype(dt.descr)
  586. assert_equal_(dt1, dt)
  587. arr1 = np.zeros(3, dt)
  588. arr2 = roundtrip(arr1)
  589. assert_array_equal(arr1, arr2)
  590. def test_version_2_0():
  591. f = BytesIO()
  592. # requires more than 2 byte for header
  593. dt = [(("%d" % i) * 100, float) for i in range(500)]
  594. d = np.ones(1000, dtype=dt)
  595. format.write_array(f, d, version=(2, 0))
  596. with warnings.catch_warnings(record=True) as w:
  597. warnings.filterwarnings('always', '', UserWarning)
  598. format.write_array(f, d)
  599. assert_(w[0].category is UserWarning)
  600. # check alignment of data portion
  601. f.seek(0)
  602. header = f.readline()
  603. assert_(len(header) % format.ARRAY_ALIGN == 0)
  604. f.seek(0)
  605. n = format.read_array(f)
  606. assert_array_equal(d, n)
  607. # 1.0 requested but data cannot be saved this way
  608. assert_raises(ValueError, format.write_array, f, d, (1, 0))
  609. def test_version_2_0_memmap(tmpdir):
  610. # requires more than 2 byte for header
  611. dt = [(("%d" % i) * 100, float) for i in range(500)]
  612. d = np.ones(1000, dtype=dt)
  613. tf1 = os.path.join(tmpdir, f'version2_01.npy')
  614. tf2 = os.path.join(tmpdir, f'version2_02.npy')
  615. # 1.0 requested but data cannot be saved this way
  616. assert_raises(ValueError, format.open_memmap, tf1, mode='w+', dtype=d.dtype,
  617. shape=d.shape, version=(1, 0))
  618. ma = format.open_memmap(tf1, mode='w+', dtype=d.dtype,
  619. shape=d.shape, version=(2, 0))
  620. ma[...] = d
  621. ma.flush()
  622. ma = format.open_memmap(tf1, mode='r')
  623. assert_array_equal(ma, d)
  624. with warnings.catch_warnings(record=True) as w:
  625. warnings.filterwarnings('always', '', UserWarning)
  626. ma = format.open_memmap(tf2, mode='w+', dtype=d.dtype,
  627. shape=d.shape, version=None)
  628. assert_(w[0].category is UserWarning)
  629. ma[...] = d
  630. ma.flush()
  631. ma = format.open_memmap(tf2, mode='r')
  632. assert_array_equal(ma, d)
  633. def test_write_version():
  634. f = BytesIO()
  635. arr = np.arange(1)
  636. # These should pass.
  637. format.write_array(f, arr, version=(1, 0))
  638. format.write_array(f, arr)
  639. format.write_array(f, arr, version=None)
  640. format.write_array(f, arr)
  641. format.write_array(f, arr, version=(2, 0))
  642. format.write_array(f, arr)
  643. # These should all fail.
  644. bad_versions = [
  645. (1, 1),
  646. (0, 0),
  647. (0, 1),
  648. (2, 2),
  649. (255, 255),
  650. ]
  651. for version in bad_versions:
  652. with assert_raises_regex(ValueError,
  653. 'we only support format version.*'):
  654. format.write_array(f, arr, version=version)
  655. bad_version_magic = [
  656. b'\x93NUMPY\x01\x01',
  657. b'\x93NUMPY\x00\x00',
  658. b'\x93NUMPY\x00\x01',
  659. b'\x93NUMPY\x02\x00',
  660. b'\x93NUMPY\x02\x02',
  661. b'\x93NUMPY\xff\xff',
  662. ]
  663. malformed_magic = [
  664. b'\x92NUMPY\x01\x00',
  665. b'\x00NUMPY\x01\x00',
  666. b'\x93numpy\x01\x00',
  667. b'\x93MATLB\x01\x00',
  668. b'\x93NUMPY\x01',
  669. b'\x93NUMPY',
  670. b'',
  671. ]
  672. def test_read_magic():
  673. s1 = BytesIO()
  674. s2 = BytesIO()
  675. arr = np.ones((3, 6), dtype=float)
  676. format.write_array(s1, arr, version=(1, 0))
  677. format.write_array(s2, arr, version=(2, 0))
  678. s1.seek(0)
  679. s2.seek(0)
  680. version1 = format.read_magic(s1)
  681. version2 = format.read_magic(s2)
  682. assert_(version1 == (1, 0))
  683. assert_(version2 == (2, 0))
  684. assert_(s1.tell() == format.MAGIC_LEN)
  685. assert_(s2.tell() == format.MAGIC_LEN)
  686. def test_read_magic_bad_magic():
  687. for magic in malformed_magic:
  688. f = BytesIO(magic)
  689. assert_raises(ValueError, format.read_array, f)
  690. def test_read_version_1_0_bad_magic():
  691. for magic in bad_version_magic + malformed_magic:
  692. f = BytesIO(magic)
  693. assert_raises(ValueError, format.read_array, f)
  694. def test_bad_magic_args():
  695. assert_raises(ValueError, format.magic, -1, 1)
  696. assert_raises(ValueError, format.magic, 256, 1)
  697. assert_raises(ValueError, format.magic, 1, -1)
  698. assert_raises(ValueError, format.magic, 1, 256)
  699. def test_large_header():
  700. s = BytesIO()
  701. d = {'a': 1, 'b': 2}
  702. format.write_array_header_1_0(s, d)
  703. s = BytesIO()
  704. d = {'a': 1, 'b': 2, 'c': 'x'*256*256}
  705. assert_raises(ValueError, format.write_array_header_1_0, s, d)
  706. def test_read_array_header_1_0():
  707. s = BytesIO()
  708. arr = np.ones((3, 6), dtype=float)
  709. format.write_array(s, arr, version=(1, 0))
  710. s.seek(format.MAGIC_LEN)
  711. shape, fortran, dtype = format.read_array_header_1_0(s)
  712. assert_(s.tell() % format.ARRAY_ALIGN == 0)
  713. assert_((shape, fortran, dtype) == ((3, 6), False, float))
  714. def test_read_array_header_2_0():
  715. s = BytesIO()
  716. arr = np.ones((3, 6), dtype=float)
  717. format.write_array(s, arr, version=(2, 0))
  718. s.seek(format.MAGIC_LEN)
  719. shape, fortran, dtype = format.read_array_header_2_0(s)
  720. assert_(s.tell() % format.ARRAY_ALIGN == 0)
  721. assert_((shape, fortran, dtype) == ((3, 6), False, float))
  722. def test_bad_header():
  723. # header of length less than 2 should fail
  724. s = BytesIO()
  725. assert_raises(ValueError, format.read_array_header_1_0, s)
  726. s = BytesIO(b'1')
  727. assert_raises(ValueError, format.read_array_header_1_0, s)
  728. # header shorter than indicated size should fail
  729. s = BytesIO(b'\x01\x00')
  730. assert_raises(ValueError, format.read_array_header_1_0, s)
  731. # headers without the exact keys required should fail
  732. d = {"shape": (1, 2),
  733. "descr": "x"}
  734. s = BytesIO()
  735. format.write_array_header_1_0(s, d)
  736. assert_raises(ValueError, format.read_array_header_1_0, s)
  737. d = {"shape": (1, 2),
  738. "fortran_order": False,
  739. "descr": "x",
  740. "extrakey": -1}
  741. s = BytesIO()
  742. format.write_array_header_1_0(s, d)
  743. assert_raises(ValueError, format.read_array_header_1_0, s)
  744. def test_large_file_support(tmpdir):
  745. if (sys.platform == 'win32' or sys.platform == 'cygwin'):
  746. pytest.skip("Unknown if Windows has sparse filesystems")
  747. # try creating a large sparse file
  748. tf_name = os.path.join(tmpdir, 'sparse_file')
  749. try:
  750. # seek past end would work too, but linux truncate somewhat
  751. # increases the chances that we have a sparse filesystem and can
  752. # avoid actually writing 5GB
  753. import subprocess as sp
  754. sp.check_call(["truncate", "-s", "5368709120", tf_name])
  755. except Exception:
  756. pytest.skip("Could not create 5GB large file")
  757. # write a small array to the end
  758. with open(tf_name, "wb") as f:
  759. f.seek(5368709120)
  760. d = np.arange(5)
  761. np.save(f, d)
  762. # read it back
  763. with open(tf_name, "rb") as f:
  764. f.seek(5368709120)
  765. r = np.load(f)
  766. assert_array_equal(r, d)
  767. @pytest.mark.skipif(np.dtype(np.intp).itemsize < 8,
  768. reason="test requires 64-bit system")
  769. @pytest.mark.slow
  770. def test_large_archive(tmpdir):
  771. # Regression test for product of saving arrays with dimensions of array
  772. # having a product that doesn't fit in int32. See gh-7598 for details.
  773. try:
  774. a = np.empty((2**30, 2), dtype=np.uint8)
  775. except MemoryError:
  776. pytest.skip("Could not create large file")
  777. fname = os.path.join(tmpdir, "large_archive")
  778. with open(fname, "wb") as f:
  779. np.savez(f, arr=a)
  780. with open(fname, "rb") as f:
  781. new_a = np.load(f)["arr"]
  782. assert_(a.shape == new_a.shape)
  783. def test_empty_npz(tmpdir):
  784. # Test for gh-9989
  785. fname = os.path.join(tmpdir, "nothing.npz")
  786. np.savez(fname)
  787. with np.load(fname) as nps:
  788. pass
  789. def test_unicode_field_names(tmpdir):
  790. # gh-7391
  791. arr = np.array([
  792. (1, 3),
  793. (1, 2),
  794. (1, 3),
  795. (1, 2)
  796. ], dtype=[
  797. ('int', int),
  798. (u'\N{CJK UNIFIED IDEOGRAPH-6574}\N{CJK UNIFIED IDEOGRAPH-5F62}', int)
  799. ])
  800. fname = os.path.join(tmpdir, "unicode.npy")
  801. with open(fname, 'wb') as f:
  802. format.write_array(f, arr, version=(3, 0))
  803. with open(fname, 'rb') as f:
  804. arr2 = format.read_array(f)
  805. assert_array_equal(arr, arr2)
  806. # notifies the user that 3.0 is selected
  807. with open(fname, 'wb') as f:
  808. with assert_warns(UserWarning):
  809. format.write_array(f, arr, version=None)
  810. @pytest.mark.parametrize('dt, fail', [
  811. (np.dtype({'names': ['a', 'b'], 'formats': [float, np.dtype('S3',
  812. metadata={'some': 'stuff'})]}), True),
  813. (np.dtype(int, metadata={'some': 'stuff'}), False),
  814. (np.dtype([('subarray', (int, (2,)))], metadata={'some': 'stuff'}), False),
  815. # recursive: metadata on the field of a dtype
  816. (np.dtype({'names': ['a', 'b'], 'formats': [
  817. float, np.dtype({'names': ['c'], 'formats': [np.dtype(int, metadata={})]})
  818. ]}), False)
  819. ])
  820. def test_metadata_dtype(dt, fail):
  821. # gh-14142
  822. arr = np.ones(10, dtype=dt)
  823. buf = BytesIO()
  824. with assert_warns(UserWarning):
  825. np.save(buf, arr)
  826. buf.seek(0)
  827. if fail:
  828. with assert_raises(ValueError):
  829. np.load(buf)
  830. else:
  831. arr2 = np.load(buf)
  832. # BUG: assert_array_equal does not check metadata
  833. from numpy.lib.format import _has_metadata
  834. assert_array_equal(arr, arr2)
  835. assert _has_metadata(arr.dtype)
  836. assert not _has_metadata(arr2.dtype)