test_recfunctions.py 40 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979
  1. import pytest
  2. import numpy as np
  3. import numpy.ma as ma
  4. from numpy.ma.mrecords import MaskedRecords
  5. from numpy.ma.testutils import assert_equal
  6. from numpy.testing import assert_, assert_raises
  7. from numpy.lib.recfunctions import (
  8. drop_fields, rename_fields, get_fieldstructure, recursive_fill_fields,
  9. find_duplicates, merge_arrays, append_fields, stack_arrays, join_by,
  10. repack_fields, unstructured_to_structured, structured_to_unstructured,
  11. apply_along_fields, require_fields, assign_fields_by_name)
  12. get_fieldspec = np.lib.recfunctions._get_fieldspec
  13. get_names = np.lib.recfunctions.get_names
  14. get_names_flat = np.lib.recfunctions.get_names_flat
  15. zip_descr = np.lib.recfunctions._zip_descr
  16. zip_dtype = np.lib.recfunctions._zip_dtype
  17. class TestRecFunctions:
  18. # Misc tests
  19. def setup(self):
  20. x = np.array([1, 2, ])
  21. y = np.array([10, 20, 30])
  22. z = np.array([('A', 1.), ('B', 2.)],
  23. dtype=[('A', '|S3'), ('B', float)])
  24. w = np.array([(1, (2, 3.0)), (4, (5, 6.0))],
  25. dtype=[('a', int), ('b', [('ba', float), ('bb', int)])])
  26. self.data = (w, x, y, z)
  27. def test_zip_descr(self):
  28. # Test zip_descr
  29. (w, x, y, z) = self.data
  30. # Std array
  31. test = zip_descr((x, x), flatten=True)
  32. assert_equal(test,
  33. np.dtype([('', int), ('', int)]))
  34. test = zip_descr((x, x), flatten=False)
  35. assert_equal(test,
  36. np.dtype([('', int), ('', int)]))
  37. # Std & flexible-dtype
  38. test = zip_descr((x, z), flatten=True)
  39. assert_equal(test,
  40. np.dtype([('', int), ('A', '|S3'), ('B', float)]))
  41. test = zip_descr((x, z), flatten=False)
  42. assert_equal(test,
  43. np.dtype([('', int),
  44. ('', [('A', '|S3'), ('B', float)])]))
  45. # Standard & nested dtype
  46. test = zip_descr((x, w), flatten=True)
  47. assert_equal(test,
  48. np.dtype([('', int),
  49. ('a', int),
  50. ('ba', float), ('bb', int)]))
  51. test = zip_descr((x, w), flatten=False)
  52. assert_equal(test,
  53. np.dtype([('', int),
  54. ('', [('a', int),
  55. ('b', [('ba', float), ('bb', int)])])]))
  56. def test_drop_fields(self):
  57. # Test drop_fields
  58. a = np.array([(1, (2, 3.0)), (4, (5, 6.0))],
  59. dtype=[('a', int), ('b', [('ba', float), ('bb', int)])])
  60. # A basic field
  61. test = drop_fields(a, 'a')
  62. control = np.array([((2, 3.0),), ((5, 6.0),)],
  63. dtype=[('b', [('ba', float), ('bb', int)])])
  64. assert_equal(test, control)
  65. # Another basic field (but nesting two fields)
  66. test = drop_fields(a, 'b')
  67. control = np.array([(1,), (4,)], dtype=[('a', int)])
  68. assert_equal(test, control)
  69. # A nested sub-field
  70. test = drop_fields(a, ['ba', ])
  71. control = np.array([(1, (3.0,)), (4, (6.0,))],
  72. dtype=[('a', int), ('b', [('bb', int)])])
  73. assert_equal(test, control)
  74. # All the nested sub-field from a field: zap that field
  75. test = drop_fields(a, ['ba', 'bb'])
  76. control = np.array([(1,), (4,)], dtype=[('a', int)])
  77. assert_equal(test, control)
  78. # dropping all fields results in an array with no fields
  79. test = drop_fields(a, ['a', 'b'])
  80. control = np.array([(), ()], dtype=[])
  81. assert_equal(test, control)
  82. def test_rename_fields(self):
  83. # Test rename fields
  84. a = np.array([(1, (2, [3.0, 30.])), (4, (5, [6.0, 60.]))],
  85. dtype=[('a', int),
  86. ('b', [('ba', float), ('bb', (float, 2))])])
  87. test = rename_fields(a, {'a': 'A', 'bb': 'BB'})
  88. newdtype = [('A', int), ('b', [('ba', float), ('BB', (float, 2))])]
  89. control = a.view(newdtype)
  90. assert_equal(test.dtype, newdtype)
  91. assert_equal(test, control)
  92. def test_get_names(self):
  93. # Test get_names
  94. ndtype = np.dtype([('A', '|S3'), ('B', float)])
  95. test = get_names(ndtype)
  96. assert_equal(test, ('A', 'B'))
  97. ndtype = np.dtype([('a', int), ('b', [('ba', float), ('bb', int)])])
  98. test = get_names(ndtype)
  99. assert_equal(test, ('a', ('b', ('ba', 'bb'))))
  100. ndtype = np.dtype([('a', int), ('b', [])])
  101. test = get_names(ndtype)
  102. assert_equal(test, ('a', ('b', ())))
  103. ndtype = np.dtype([])
  104. test = get_names(ndtype)
  105. assert_equal(test, ())
  106. def test_get_names_flat(self):
  107. # Test get_names_flat
  108. ndtype = np.dtype([('A', '|S3'), ('B', float)])
  109. test = get_names_flat(ndtype)
  110. assert_equal(test, ('A', 'B'))
  111. ndtype = np.dtype([('a', int), ('b', [('ba', float), ('bb', int)])])
  112. test = get_names_flat(ndtype)
  113. assert_equal(test, ('a', 'b', 'ba', 'bb'))
  114. ndtype = np.dtype([('a', int), ('b', [])])
  115. test = get_names_flat(ndtype)
  116. assert_equal(test, ('a', 'b'))
  117. ndtype = np.dtype([])
  118. test = get_names_flat(ndtype)
  119. assert_equal(test, ())
  120. def test_get_fieldstructure(self):
  121. # Test get_fieldstructure
  122. # No nested fields
  123. ndtype = np.dtype([('A', '|S3'), ('B', float)])
  124. test = get_fieldstructure(ndtype)
  125. assert_equal(test, {'A': [], 'B': []})
  126. # One 1-nested field
  127. ndtype = np.dtype([('A', int), ('B', [('BA', float), ('BB', '|S1')])])
  128. test = get_fieldstructure(ndtype)
  129. assert_equal(test, {'A': [], 'B': [], 'BA': ['B', ], 'BB': ['B']})
  130. # One 2-nested fields
  131. ndtype = np.dtype([('A', int),
  132. ('B', [('BA', int),
  133. ('BB', [('BBA', int), ('BBB', int)])])])
  134. test = get_fieldstructure(ndtype)
  135. control = {'A': [], 'B': [], 'BA': ['B'], 'BB': ['B'],
  136. 'BBA': ['B', 'BB'], 'BBB': ['B', 'BB']}
  137. assert_equal(test, control)
  138. # 0 fields
  139. ndtype = np.dtype([])
  140. test = get_fieldstructure(ndtype)
  141. assert_equal(test, {})
  142. def test_find_duplicates(self):
  143. # Test find_duplicates
  144. a = ma.array([(2, (2., 'B')), (1, (2., 'B')), (2, (2., 'B')),
  145. (1, (1., 'B')), (2, (2., 'B')), (2, (2., 'C'))],
  146. mask=[(0, (0, 0)), (0, (0, 0)), (0, (0, 0)),
  147. (0, (0, 0)), (1, (0, 0)), (0, (1, 0))],
  148. dtype=[('A', int), ('B', [('BA', float), ('BB', '|S1')])])
  149. test = find_duplicates(a, ignoremask=False, return_index=True)
  150. control = [0, 2]
  151. assert_equal(sorted(test[-1]), control)
  152. assert_equal(test[0], a[test[-1]])
  153. test = find_duplicates(a, key='A', return_index=True)
  154. control = [0, 1, 2, 3, 5]
  155. assert_equal(sorted(test[-1]), control)
  156. assert_equal(test[0], a[test[-1]])
  157. test = find_duplicates(a, key='B', return_index=True)
  158. control = [0, 1, 2, 4]
  159. assert_equal(sorted(test[-1]), control)
  160. assert_equal(test[0], a[test[-1]])
  161. test = find_duplicates(a, key='BA', return_index=True)
  162. control = [0, 1, 2, 4]
  163. assert_equal(sorted(test[-1]), control)
  164. assert_equal(test[0], a[test[-1]])
  165. test = find_duplicates(a, key='BB', return_index=True)
  166. control = [0, 1, 2, 3, 4]
  167. assert_equal(sorted(test[-1]), control)
  168. assert_equal(test[0], a[test[-1]])
  169. def test_find_duplicates_ignoremask(self):
  170. # Test the ignoremask option of find_duplicates
  171. ndtype = [('a', int)]
  172. a = ma.array([1, 1, 1, 2, 2, 3, 3],
  173. mask=[0, 0, 1, 0, 0, 0, 1]).view(ndtype)
  174. test = find_duplicates(a, ignoremask=True, return_index=True)
  175. control = [0, 1, 3, 4]
  176. assert_equal(sorted(test[-1]), control)
  177. assert_equal(test[0], a[test[-1]])
  178. test = find_duplicates(a, ignoremask=False, return_index=True)
  179. control = [0, 1, 2, 3, 4, 6]
  180. assert_equal(sorted(test[-1]), control)
  181. assert_equal(test[0], a[test[-1]])
  182. def test_repack_fields(self):
  183. dt = np.dtype('u1,f4,i8', align=True)
  184. a = np.zeros(2, dtype=dt)
  185. assert_equal(repack_fields(dt), np.dtype('u1,f4,i8'))
  186. assert_equal(repack_fields(a).itemsize, 13)
  187. assert_equal(repack_fields(repack_fields(dt), align=True), dt)
  188. # make sure type is preserved
  189. dt = np.dtype((np.record, dt))
  190. assert_(repack_fields(dt).type is np.record)
  191. def test_structured_to_unstructured(self):
  192. a = np.zeros(4, dtype=[('a', 'i4'), ('b', 'f4,u2'), ('c', 'f4', 2)])
  193. out = structured_to_unstructured(a)
  194. assert_equal(out, np.zeros((4,5), dtype='f8'))
  195. b = np.array([(1, 2, 5), (4, 5, 7), (7, 8 ,11), (10, 11, 12)],
  196. dtype=[('x', 'i4'), ('y', 'f4'), ('z', 'f8')])
  197. out = np.mean(structured_to_unstructured(b[['x', 'z']]), axis=-1)
  198. assert_equal(out, np.array([ 3. , 5.5, 9. , 11. ]))
  199. out = np.mean(structured_to_unstructured(b[['x']]), axis=-1)
  200. assert_equal(out, np.array([ 1. , 4. , 7. , 10. ]))
  201. c = np.arange(20).reshape((4,5))
  202. out = unstructured_to_structured(c, a.dtype)
  203. want = np.array([( 0, ( 1., 2), [ 3., 4.]),
  204. ( 5, ( 6., 7), [ 8., 9.]),
  205. (10, (11., 12), [13., 14.]),
  206. (15, (16., 17), [18., 19.])],
  207. dtype=[('a', 'i4'),
  208. ('b', [('f0', 'f4'), ('f1', 'u2')]),
  209. ('c', 'f4', (2,))])
  210. assert_equal(out, want)
  211. d = np.array([(1, 2, 5), (4, 5, 7), (7, 8 ,11), (10, 11, 12)],
  212. dtype=[('x', 'i4'), ('y', 'f4'), ('z', 'f8')])
  213. assert_equal(apply_along_fields(np.mean, d),
  214. np.array([ 8.0/3, 16.0/3, 26.0/3, 11. ]))
  215. assert_equal(apply_along_fields(np.mean, d[['x', 'z']]),
  216. np.array([ 3. , 5.5, 9. , 11. ]))
  217. # check that for uniform field dtypes we get a view, not a copy:
  218. d = np.array([(1, 2, 5), (4, 5, 7), (7, 8 ,11), (10, 11, 12)],
  219. dtype=[('x', 'i4'), ('y', 'i4'), ('z', 'i4')])
  220. dd = structured_to_unstructured(d)
  221. ddd = unstructured_to_structured(dd, d.dtype)
  222. assert_(dd.base is d)
  223. assert_(ddd.base is d)
  224. # including uniform fields with subarrays unpacked
  225. d = np.array([(1, [2, 3], [[ 4, 5], [ 6, 7]]),
  226. (8, [9, 10], [[11, 12], [13, 14]])],
  227. dtype=[('x0', 'i4'), ('x1', ('i4', 2)),
  228. ('x2', ('i4', (2, 2)))])
  229. dd = structured_to_unstructured(d)
  230. ddd = unstructured_to_structured(dd, d.dtype)
  231. assert_(dd.base is d)
  232. assert_(ddd.base is d)
  233. # test that nested fields with identical names don't break anything
  234. point = np.dtype([('x', int), ('y', int)])
  235. triangle = np.dtype([('a', point), ('b', point), ('c', point)])
  236. arr = np.zeros(10, triangle)
  237. res = structured_to_unstructured(arr, dtype=int)
  238. assert_equal(res, np.zeros((10, 6), dtype=int))
  239. # test nested combinations of subarrays and structured arrays, gh-13333
  240. def subarray(dt, shape):
  241. return np.dtype((dt, shape))
  242. def structured(*dts):
  243. return np.dtype([('x{}'.format(i), dt) for i, dt in enumerate(dts)])
  244. def inspect(dt, dtype=None):
  245. arr = np.zeros((), dt)
  246. ret = structured_to_unstructured(arr, dtype=dtype)
  247. backarr = unstructured_to_structured(ret, dt)
  248. return ret.shape, ret.dtype, backarr.dtype
  249. dt = structured(subarray(structured(np.int32, np.int32), 3))
  250. assert_equal(inspect(dt), ((6,), np.int32, dt))
  251. dt = structured(subarray(subarray(np.int32, 2), 2))
  252. assert_equal(inspect(dt), ((4,), np.int32, dt))
  253. dt = structured(np.int32)
  254. assert_equal(inspect(dt), ((1,), np.int32, dt))
  255. dt = structured(np.int32, subarray(subarray(np.int32, 2), 2))
  256. assert_equal(inspect(dt), ((5,), np.int32, dt))
  257. dt = structured()
  258. assert_raises(ValueError, structured_to_unstructured, np.zeros(3, dt))
  259. # these currently don't work, but we may make it work in the future
  260. assert_raises(NotImplementedError, structured_to_unstructured,
  261. np.zeros(3, dt), dtype=np.int32)
  262. assert_raises(NotImplementedError, unstructured_to_structured,
  263. np.zeros((3,0), dtype=np.int32))
  264. def test_field_assignment_by_name(self):
  265. a = np.ones(2, dtype=[('a', 'i4'), ('b', 'f8'), ('c', 'u1')])
  266. newdt = [('b', 'f4'), ('c', 'u1')]
  267. assert_equal(require_fields(a, newdt), np.ones(2, newdt))
  268. b = np.array([(1,2), (3,4)], dtype=newdt)
  269. assign_fields_by_name(a, b, zero_unassigned=False)
  270. assert_equal(a, np.array([(1,1,2),(1,3,4)], dtype=a.dtype))
  271. assign_fields_by_name(a, b)
  272. assert_equal(a, np.array([(0,1,2),(0,3,4)], dtype=a.dtype))
  273. # test nested fields
  274. a = np.ones(2, dtype=[('a', [('b', 'f8'), ('c', 'u1')])])
  275. newdt = [('a', [('c', 'u1')])]
  276. assert_equal(require_fields(a, newdt), np.ones(2, newdt))
  277. b = np.array([((2,),), ((3,),)], dtype=newdt)
  278. assign_fields_by_name(a, b, zero_unassigned=False)
  279. assert_equal(a, np.array([((1,2),), ((1,3),)], dtype=a.dtype))
  280. assign_fields_by_name(a, b)
  281. assert_equal(a, np.array([((0,2),), ((0,3),)], dtype=a.dtype))
  282. # test unstructured code path for 0d arrays
  283. a, b = np.array(3), np.array(0)
  284. assign_fields_by_name(b, a)
  285. assert_equal(b[()], 3)
  286. class TestRecursiveFillFields:
  287. # Test recursive_fill_fields.
  288. def test_simple_flexible(self):
  289. # Test recursive_fill_fields on flexible-array
  290. a = np.array([(1, 10.), (2, 20.)], dtype=[('A', int), ('B', float)])
  291. b = np.zeros((3,), dtype=a.dtype)
  292. test = recursive_fill_fields(a, b)
  293. control = np.array([(1, 10.), (2, 20.), (0, 0.)],
  294. dtype=[('A', int), ('B', float)])
  295. assert_equal(test, control)
  296. def test_masked_flexible(self):
  297. # Test recursive_fill_fields on masked flexible-array
  298. a = ma.array([(1, 10.), (2, 20.)], mask=[(0, 1), (1, 0)],
  299. dtype=[('A', int), ('B', float)])
  300. b = ma.zeros((3,), dtype=a.dtype)
  301. test = recursive_fill_fields(a, b)
  302. control = ma.array([(1, 10.), (2, 20.), (0, 0.)],
  303. mask=[(0, 1), (1, 0), (0, 0)],
  304. dtype=[('A', int), ('B', float)])
  305. assert_equal(test, control)
  306. class TestMergeArrays:
  307. # Test merge_arrays
  308. def setup(self):
  309. x = np.array([1, 2, ])
  310. y = np.array([10, 20, 30])
  311. z = np.array(
  312. [('A', 1.), ('B', 2.)], dtype=[('A', '|S3'), ('B', float)])
  313. w = np.array(
  314. [(1, (2, 3.0, ())), (4, (5, 6.0, ()))],
  315. dtype=[('a', int), ('b', [('ba', float), ('bb', int), ('bc', [])])])
  316. self.data = (w, x, y, z)
  317. def test_solo(self):
  318. # Test merge_arrays on a single array.
  319. (_, x, _, z) = self.data
  320. test = merge_arrays(x)
  321. control = np.array([(1,), (2,)], dtype=[('f0', int)])
  322. assert_equal(test, control)
  323. test = merge_arrays((x,))
  324. assert_equal(test, control)
  325. test = merge_arrays(z, flatten=False)
  326. assert_equal(test, z)
  327. test = merge_arrays(z, flatten=True)
  328. assert_equal(test, z)
  329. def test_solo_w_flatten(self):
  330. # Test merge_arrays on a single array w & w/o flattening
  331. w = self.data[0]
  332. test = merge_arrays(w, flatten=False)
  333. assert_equal(test, w)
  334. test = merge_arrays(w, flatten=True)
  335. control = np.array([(1, 2, 3.0), (4, 5, 6.0)],
  336. dtype=[('a', int), ('ba', float), ('bb', int)])
  337. assert_equal(test, control)
  338. def test_standard(self):
  339. # Test standard & standard
  340. # Test merge arrays
  341. (_, x, y, _) = self.data
  342. test = merge_arrays((x, y), usemask=False)
  343. control = np.array([(1, 10), (2, 20), (-1, 30)],
  344. dtype=[('f0', int), ('f1', int)])
  345. assert_equal(test, control)
  346. test = merge_arrays((x, y), usemask=True)
  347. control = ma.array([(1, 10), (2, 20), (-1, 30)],
  348. mask=[(0, 0), (0, 0), (1, 0)],
  349. dtype=[('f0', int), ('f1', int)])
  350. assert_equal(test, control)
  351. assert_equal(test.mask, control.mask)
  352. def test_flatten(self):
  353. # Test standard & flexible
  354. (_, x, _, z) = self.data
  355. test = merge_arrays((x, z), flatten=True)
  356. control = np.array([(1, 'A', 1.), (2, 'B', 2.)],
  357. dtype=[('f0', int), ('A', '|S3'), ('B', float)])
  358. assert_equal(test, control)
  359. test = merge_arrays((x, z), flatten=False)
  360. control = np.array([(1, ('A', 1.)), (2, ('B', 2.))],
  361. dtype=[('f0', int),
  362. ('f1', [('A', '|S3'), ('B', float)])])
  363. assert_equal(test, control)
  364. def test_flatten_wflexible(self):
  365. # Test flatten standard & nested
  366. (w, x, _, _) = self.data
  367. test = merge_arrays((x, w), flatten=True)
  368. control = np.array([(1, 1, 2, 3.0), (2, 4, 5, 6.0)],
  369. dtype=[('f0', int),
  370. ('a', int), ('ba', float), ('bb', int)])
  371. assert_equal(test, control)
  372. test = merge_arrays((x, w), flatten=False)
  373. controldtype = [('f0', int),
  374. ('f1', [('a', int),
  375. ('b', [('ba', float), ('bb', int), ('bc', [])])])]
  376. control = np.array([(1., (1, (2, 3.0, ()))), (2, (4, (5, 6.0, ())))],
  377. dtype=controldtype)
  378. assert_equal(test, control)
  379. def test_wmasked_arrays(self):
  380. # Test merge_arrays masked arrays
  381. (_, x, _, _) = self.data
  382. mx = ma.array([1, 2, 3], mask=[1, 0, 0])
  383. test = merge_arrays((x, mx), usemask=True)
  384. control = ma.array([(1, 1), (2, 2), (-1, 3)],
  385. mask=[(0, 1), (0, 0), (1, 0)],
  386. dtype=[('f0', int), ('f1', int)])
  387. assert_equal(test, control)
  388. test = merge_arrays((x, mx), usemask=True, asrecarray=True)
  389. assert_equal(test, control)
  390. assert_(isinstance(test, MaskedRecords))
  391. def test_w_singlefield(self):
  392. # Test single field
  393. test = merge_arrays((np.array([1, 2]).view([('a', int)]),
  394. np.array([10., 20., 30.])),)
  395. control = ma.array([(1, 10.), (2, 20.), (-1, 30.)],
  396. mask=[(0, 0), (0, 0), (1, 0)],
  397. dtype=[('a', int), ('f1', float)])
  398. assert_equal(test, control)
  399. def test_w_shorter_flex(self):
  400. # Test merge_arrays w/ a shorter flexndarray.
  401. z = self.data[-1]
  402. # Fixme, this test looks incomplete and broken
  403. #test = merge_arrays((z, np.array([10, 20, 30]).view([('C', int)])))
  404. #control = np.array([('A', 1., 10), ('B', 2., 20), ('-1', -1, 20)],
  405. # dtype=[('A', '|S3'), ('B', float), ('C', int)])
  406. #assert_equal(test, control)
  407. # Hack to avoid pyflakes warnings about unused variables
  408. merge_arrays((z, np.array([10, 20, 30]).view([('C', int)])))
  409. np.array([('A', 1., 10), ('B', 2., 20), ('-1', -1, 20)],
  410. dtype=[('A', '|S3'), ('B', float), ('C', int)])
  411. def test_singlerecord(self):
  412. (_, x, y, z) = self.data
  413. test = merge_arrays((x[0], y[0], z[0]), usemask=False)
  414. control = np.array([(1, 10, ('A', 1))],
  415. dtype=[('f0', int),
  416. ('f1', int),
  417. ('f2', [('A', '|S3'), ('B', float)])])
  418. assert_equal(test, control)
  419. class TestAppendFields:
  420. # Test append_fields
  421. def setup(self):
  422. x = np.array([1, 2, ])
  423. y = np.array([10, 20, 30])
  424. z = np.array(
  425. [('A', 1.), ('B', 2.)], dtype=[('A', '|S3'), ('B', float)])
  426. w = np.array([(1, (2, 3.0)), (4, (5, 6.0))],
  427. dtype=[('a', int), ('b', [('ba', float), ('bb', int)])])
  428. self.data = (w, x, y, z)
  429. def test_append_single(self):
  430. # Test simple case
  431. (_, x, _, _) = self.data
  432. test = append_fields(x, 'A', data=[10, 20, 30])
  433. control = ma.array([(1, 10), (2, 20), (-1, 30)],
  434. mask=[(0, 0), (0, 0), (1, 0)],
  435. dtype=[('f0', int), ('A', int)],)
  436. assert_equal(test, control)
  437. def test_append_double(self):
  438. # Test simple case
  439. (_, x, _, _) = self.data
  440. test = append_fields(x, ('A', 'B'), data=[[10, 20, 30], [100, 200]])
  441. control = ma.array([(1, 10, 100), (2, 20, 200), (-1, 30, -1)],
  442. mask=[(0, 0, 0), (0, 0, 0), (1, 0, 1)],
  443. dtype=[('f0', int), ('A', int), ('B', int)],)
  444. assert_equal(test, control)
  445. def test_append_on_flex(self):
  446. # Test append_fields on flexible type arrays
  447. z = self.data[-1]
  448. test = append_fields(z, 'C', data=[10, 20, 30])
  449. control = ma.array([('A', 1., 10), ('B', 2., 20), (-1, -1., 30)],
  450. mask=[(0, 0, 0), (0, 0, 0), (1, 1, 0)],
  451. dtype=[('A', '|S3'), ('B', float), ('C', int)],)
  452. assert_equal(test, control)
  453. def test_append_on_nested(self):
  454. # Test append_fields on nested fields
  455. w = self.data[0]
  456. test = append_fields(w, 'C', data=[10, 20, 30])
  457. control = ma.array([(1, (2, 3.0), 10),
  458. (4, (5, 6.0), 20),
  459. (-1, (-1, -1.), 30)],
  460. mask=[(
  461. 0, (0, 0), 0), (0, (0, 0), 0), (1, (1, 1), 0)],
  462. dtype=[('a', int),
  463. ('b', [('ba', float), ('bb', int)]),
  464. ('C', int)],)
  465. assert_equal(test, control)
  466. class TestStackArrays:
  467. # Test stack_arrays
  468. def setup(self):
  469. x = np.array([1, 2, ])
  470. y = np.array([10, 20, 30])
  471. z = np.array(
  472. [('A', 1.), ('B', 2.)], dtype=[('A', '|S3'), ('B', float)])
  473. w = np.array([(1, (2, 3.0)), (4, (5, 6.0))],
  474. dtype=[('a', int), ('b', [('ba', float), ('bb', int)])])
  475. self.data = (w, x, y, z)
  476. def test_solo(self):
  477. # Test stack_arrays on single arrays
  478. (_, x, _, _) = self.data
  479. test = stack_arrays((x,))
  480. assert_equal(test, x)
  481. assert_(test is x)
  482. test = stack_arrays(x)
  483. assert_equal(test, x)
  484. assert_(test is x)
  485. def test_unnamed_fields(self):
  486. # Tests combinations of arrays w/o named fields
  487. (_, x, y, _) = self.data
  488. test = stack_arrays((x, x), usemask=False)
  489. control = np.array([1, 2, 1, 2])
  490. assert_equal(test, control)
  491. test = stack_arrays((x, y), usemask=False)
  492. control = np.array([1, 2, 10, 20, 30])
  493. assert_equal(test, control)
  494. test = stack_arrays((y, x), usemask=False)
  495. control = np.array([10, 20, 30, 1, 2])
  496. assert_equal(test, control)
  497. def test_unnamed_and_named_fields(self):
  498. # Test combination of arrays w/ & w/o named fields
  499. (_, x, _, z) = self.data
  500. test = stack_arrays((x, z))
  501. control = ma.array([(1, -1, -1), (2, -1, -1),
  502. (-1, 'A', 1), (-1, 'B', 2)],
  503. mask=[(0, 1, 1), (0, 1, 1),
  504. (1, 0, 0), (1, 0, 0)],
  505. dtype=[('f0', int), ('A', '|S3'), ('B', float)])
  506. assert_equal(test, control)
  507. assert_equal(test.mask, control.mask)
  508. test = stack_arrays((z, x))
  509. control = ma.array([('A', 1, -1), ('B', 2, -1),
  510. (-1, -1, 1), (-1, -1, 2), ],
  511. mask=[(0, 0, 1), (0, 0, 1),
  512. (1, 1, 0), (1, 1, 0)],
  513. dtype=[('A', '|S3'), ('B', float), ('f2', int)])
  514. assert_equal(test, control)
  515. assert_equal(test.mask, control.mask)
  516. test = stack_arrays((z, z, x))
  517. control = ma.array([('A', 1, -1), ('B', 2, -1),
  518. ('A', 1, -1), ('B', 2, -1),
  519. (-1, -1, 1), (-1, -1, 2), ],
  520. mask=[(0, 0, 1), (0, 0, 1),
  521. (0, 0, 1), (0, 0, 1),
  522. (1, 1, 0), (1, 1, 0)],
  523. dtype=[('A', '|S3'), ('B', float), ('f2', int)])
  524. assert_equal(test, control)
  525. def test_matching_named_fields(self):
  526. # Test combination of arrays w/ matching field names
  527. (_, x, _, z) = self.data
  528. zz = np.array([('a', 10., 100.), ('b', 20., 200.), ('c', 30., 300.)],
  529. dtype=[('A', '|S3'), ('B', float), ('C', float)])
  530. test = stack_arrays((z, zz))
  531. control = ma.array([('A', 1, -1), ('B', 2, -1),
  532. (
  533. 'a', 10., 100.), ('b', 20., 200.), ('c', 30., 300.)],
  534. dtype=[('A', '|S3'), ('B', float), ('C', float)],
  535. mask=[(0, 0, 1), (0, 0, 1),
  536. (0, 0, 0), (0, 0, 0), (0, 0, 0)])
  537. assert_equal(test, control)
  538. assert_equal(test.mask, control.mask)
  539. test = stack_arrays((z, zz, x))
  540. ndtype = [('A', '|S3'), ('B', float), ('C', float), ('f3', int)]
  541. control = ma.array([('A', 1, -1, -1), ('B', 2, -1, -1),
  542. ('a', 10., 100., -1), ('b', 20., 200., -1),
  543. ('c', 30., 300., -1),
  544. (-1, -1, -1, 1), (-1, -1, -1, 2)],
  545. dtype=ndtype,
  546. mask=[(0, 0, 1, 1), (0, 0, 1, 1),
  547. (0, 0, 0, 1), (0, 0, 0, 1), (0, 0, 0, 1),
  548. (1, 1, 1, 0), (1, 1, 1, 0)])
  549. assert_equal(test, control)
  550. assert_equal(test.mask, control.mask)
  551. def test_defaults(self):
  552. # Test defaults: no exception raised if keys of defaults are not fields.
  553. (_, _, _, z) = self.data
  554. zz = np.array([('a', 10., 100.), ('b', 20., 200.), ('c', 30., 300.)],
  555. dtype=[('A', '|S3'), ('B', float), ('C', float)])
  556. defaults = {'A': '???', 'B': -999., 'C': -9999., 'D': -99999.}
  557. test = stack_arrays((z, zz), defaults=defaults)
  558. control = ma.array([('A', 1, -9999.), ('B', 2, -9999.),
  559. (
  560. 'a', 10., 100.), ('b', 20., 200.), ('c', 30., 300.)],
  561. dtype=[('A', '|S3'), ('B', float), ('C', float)],
  562. mask=[(0, 0, 1), (0, 0, 1),
  563. (0, 0, 0), (0, 0, 0), (0, 0, 0)])
  564. assert_equal(test, control)
  565. assert_equal(test.data, control.data)
  566. assert_equal(test.mask, control.mask)
  567. def test_autoconversion(self):
  568. # Tests autoconversion
  569. adtype = [('A', int), ('B', bool), ('C', float)]
  570. a = ma.array([(1, 2, 3)], mask=[(0, 1, 0)], dtype=adtype)
  571. bdtype = [('A', int), ('B', float), ('C', float)]
  572. b = ma.array([(4, 5, 6)], dtype=bdtype)
  573. control = ma.array([(1, 2, 3), (4, 5, 6)], mask=[(0, 1, 0), (0, 0, 0)],
  574. dtype=bdtype)
  575. test = stack_arrays((a, b), autoconvert=True)
  576. assert_equal(test, control)
  577. assert_equal(test.mask, control.mask)
  578. with assert_raises(TypeError):
  579. stack_arrays((a, b), autoconvert=False)
  580. def test_checktitles(self):
  581. # Test using titles in the field names
  582. adtype = [(('a', 'A'), int), (('b', 'B'), bool), (('c', 'C'), float)]
  583. a = ma.array([(1, 2, 3)], mask=[(0, 1, 0)], dtype=adtype)
  584. bdtype = [(('a', 'A'), int), (('b', 'B'), bool), (('c', 'C'), float)]
  585. b = ma.array([(4, 5, 6)], dtype=bdtype)
  586. test = stack_arrays((a, b))
  587. control = ma.array([(1, 2, 3), (4, 5, 6)], mask=[(0, 1, 0), (0, 0, 0)],
  588. dtype=bdtype)
  589. assert_equal(test, control)
  590. assert_equal(test.mask, control.mask)
  591. def test_subdtype(self):
  592. z = np.array([
  593. ('A', 1), ('B', 2)
  594. ], dtype=[('A', '|S3'), ('B', float, (1,))])
  595. zz = np.array([
  596. ('a', [10.], 100.), ('b', [20.], 200.), ('c', [30.], 300.)
  597. ], dtype=[('A', '|S3'), ('B', float, (1,)), ('C', float)])
  598. res = stack_arrays((z, zz))
  599. expected = ma.array(
  600. data=[
  601. (b'A', [1.0], 0),
  602. (b'B', [2.0], 0),
  603. (b'a', [10.0], 100.0),
  604. (b'b', [20.0], 200.0),
  605. (b'c', [30.0], 300.0)],
  606. mask=[
  607. (False, [False], True),
  608. (False, [False], True),
  609. (False, [False], False),
  610. (False, [False], False),
  611. (False, [False], False)
  612. ],
  613. dtype=zz.dtype
  614. )
  615. assert_equal(res.dtype, expected.dtype)
  616. assert_equal(res, expected)
  617. assert_equal(res.mask, expected.mask)
  618. class TestJoinBy:
  619. def setup(self):
  620. self.a = np.array(list(zip(np.arange(10), np.arange(50, 60),
  621. np.arange(100, 110))),
  622. dtype=[('a', int), ('b', int), ('c', int)])
  623. self.b = np.array(list(zip(np.arange(5, 15), np.arange(65, 75),
  624. np.arange(100, 110))),
  625. dtype=[('a', int), ('b', int), ('d', int)])
  626. def test_inner_join(self):
  627. # Basic test of join_by
  628. a, b = self.a, self.b
  629. test = join_by('a', a, b, jointype='inner')
  630. control = np.array([(5, 55, 65, 105, 100), (6, 56, 66, 106, 101),
  631. (7, 57, 67, 107, 102), (8, 58, 68, 108, 103),
  632. (9, 59, 69, 109, 104)],
  633. dtype=[('a', int), ('b1', int), ('b2', int),
  634. ('c', int), ('d', int)])
  635. assert_equal(test, control)
  636. def test_join(self):
  637. a, b = self.a, self.b
  638. # Fixme, this test is broken
  639. #test = join_by(('a', 'b'), a, b)
  640. #control = np.array([(5, 55, 105, 100), (6, 56, 106, 101),
  641. # (7, 57, 107, 102), (8, 58, 108, 103),
  642. # (9, 59, 109, 104)],
  643. # dtype=[('a', int), ('b', int),
  644. # ('c', int), ('d', int)])
  645. #assert_equal(test, control)
  646. # Hack to avoid pyflakes unused variable warnings
  647. join_by(('a', 'b'), a, b)
  648. np.array([(5, 55, 105, 100), (6, 56, 106, 101),
  649. (7, 57, 107, 102), (8, 58, 108, 103),
  650. (9, 59, 109, 104)],
  651. dtype=[('a', int), ('b', int),
  652. ('c', int), ('d', int)])
  653. def test_join_subdtype(self):
  654. # tests the bug in https://stackoverflow.com/q/44769632/102441
  655. foo = np.array([(1,)],
  656. dtype=[('key', int)])
  657. bar = np.array([(1, np.array([1,2,3]))],
  658. dtype=[('key', int), ('value', 'uint16', 3)])
  659. res = join_by('key', foo, bar)
  660. assert_equal(res, bar.view(ma.MaskedArray))
  661. def test_outer_join(self):
  662. a, b = self.a, self.b
  663. test = join_by(('a', 'b'), a, b, 'outer')
  664. control = ma.array([(0, 50, 100, -1), (1, 51, 101, -1),
  665. (2, 52, 102, -1), (3, 53, 103, -1),
  666. (4, 54, 104, -1), (5, 55, 105, -1),
  667. (5, 65, -1, 100), (6, 56, 106, -1),
  668. (6, 66, -1, 101), (7, 57, 107, -1),
  669. (7, 67, -1, 102), (8, 58, 108, -1),
  670. (8, 68, -1, 103), (9, 59, 109, -1),
  671. (9, 69, -1, 104), (10, 70, -1, 105),
  672. (11, 71, -1, 106), (12, 72, -1, 107),
  673. (13, 73, -1, 108), (14, 74, -1, 109)],
  674. mask=[(0, 0, 0, 1), (0, 0, 0, 1),
  675. (0, 0, 0, 1), (0, 0, 0, 1),
  676. (0, 0, 0, 1), (0, 0, 0, 1),
  677. (0, 0, 1, 0), (0, 0, 0, 1),
  678. (0, 0, 1, 0), (0, 0, 0, 1),
  679. (0, 0, 1, 0), (0, 0, 0, 1),
  680. (0, 0, 1, 0), (0, 0, 0, 1),
  681. (0, 0, 1, 0), (0, 0, 1, 0),
  682. (0, 0, 1, 0), (0, 0, 1, 0),
  683. (0, 0, 1, 0), (0, 0, 1, 0)],
  684. dtype=[('a', int), ('b', int),
  685. ('c', int), ('d', int)])
  686. assert_equal(test, control)
  687. def test_leftouter_join(self):
  688. a, b = self.a, self.b
  689. test = join_by(('a', 'b'), a, b, 'leftouter')
  690. control = ma.array([(0, 50, 100, -1), (1, 51, 101, -1),
  691. (2, 52, 102, -1), (3, 53, 103, -1),
  692. (4, 54, 104, -1), (5, 55, 105, -1),
  693. (6, 56, 106, -1), (7, 57, 107, -1),
  694. (8, 58, 108, -1), (9, 59, 109, -1)],
  695. mask=[(0, 0, 0, 1), (0, 0, 0, 1),
  696. (0, 0, 0, 1), (0, 0, 0, 1),
  697. (0, 0, 0, 1), (0, 0, 0, 1),
  698. (0, 0, 0, 1), (0, 0, 0, 1),
  699. (0, 0, 0, 1), (0, 0, 0, 1)],
  700. dtype=[('a', int), ('b', int), ('c', int), ('d', int)])
  701. assert_equal(test, control)
  702. def test_different_field_order(self):
  703. # gh-8940
  704. a = np.zeros(3, dtype=[('a', 'i4'), ('b', 'f4'), ('c', 'u1')])
  705. b = np.ones(3, dtype=[('c', 'u1'), ('b', 'f4'), ('a', 'i4')])
  706. # this should not give a FutureWarning:
  707. j = join_by(['c', 'b'], a, b, jointype='inner', usemask=False)
  708. assert_equal(j.dtype.names, ['b', 'c', 'a1', 'a2'])
  709. def test_duplicate_keys(self):
  710. a = np.zeros(3, dtype=[('a', 'i4'), ('b', 'f4'), ('c', 'u1')])
  711. b = np.ones(3, dtype=[('c', 'u1'), ('b', 'f4'), ('a', 'i4')])
  712. assert_raises(ValueError, join_by, ['a', 'b', 'b'], a, b)
  713. @pytest.mark.xfail(reason="See comment at gh-9343")
  714. def test_same_name_different_dtypes_key(self):
  715. a_dtype = np.dtype([('key', 'S5'), ('value', '<f4')])
  716. b_dtype = np.dtype([('key', 'S10'), ('value', '<f4')])
  717. expected_dtype = np.dtype([
  718. ('key', 'S10'), ('value1', '<f4'), ('value2', '<f4')])
  719. a = np.array([('Sarah', 8.0), ('John', 6.0)], dtype=a_dtype)
  720. b = np.array([('Sarah', 10.0), ('John', 7.0)], dtype=b_dtype)
  721. res = join_by('key', a, b)
  722. assert_equal(res.dtype, expected_dtype)
  723. def test_same_name_different_dtypes(self):
  724. # gh-9338
  725. a_dtype = np.dtype([('key', 'S10'), ('value', '<f4')])
  726. b_dtype = np.dtype([('key', 'S10'), ('value', '<f8')])
  727. expected_dtype = np.dtype([
  728. ('key', '|S10'), ('value1', '<f4'), ('value2', '<f8')])
  729. a = np.array([('Sarah', 8.0), ('John', 6.0)], dtype=a_dtype)
  730. b = np.array([('Sarah', 10.0), ('John', 7.0)], dtype=b_dtype)
  731. res = join_by('key', a, b)
  732. assert_equal(res.dtype, expected_dtype)
  733. def test_subarray_key(self):
  734. a_dtype = np.dtype([('pos', int, 3), ('f', '<f4')])
  735. a = np.array([([1, 1, 1], np.pi), ([1, 2, 3], 0.0)], dtype=a_dtype)
  736. b_dtype = np.dtype([('pos', int, 3), ('g', '<f4')])
  737. b = np.array([([1, 1, 1], 3), ([3, 2, 1], 0.0)], dtype=b_dtype)
  738. expected_dtype = np.dtype([('pos', int, 3), ('f', '<f4'), ('g', '<f4')])
  739. expected = np.array([([1, 1, 1], np.pi, 3)], dtype=expected_dtype)
  740. res = join_by('pos', a, b)
  741. assert_equal(res.dtype, expected_dtype)
  742. assert_equal(res, expected)
  743. def test_padded_dtype(self):
  744. dt = np.dtype('i1,f4', align=True)
  745. dt.names = ('k', 'v')
  746. assert_(len(dt.descr), 3) # padding field is inserted
  747. a = np.array([(1, 3), (3, 2)], dt)
  748. b = np.array([(1, 1), (2, 2)], dt)
  749. res = join_by('k', a, b)
  750. # no padding fields remain
  751. expected_dtype = np.dtype([
  752. ('k', 'i1'), ('v1', 'f4'), ('v2', 'f4')
  753. ])
  754. assert_equal(res.dtype, expected_dtype)
  755. class TestJoinBy2:
  756. @classmethod
  757. def setup(cls):
  758. cls.a = np.array(list(zip(np.arange(10), np.arange(50, 60),
  759. np.arange(100, 110))),
  760. dtype=[('a', int), ('b', int), ('c', int)])
  761. cls.b = np.array(list(zip(np.arange(10), np.arange(65, 75),
  762. np.arange(100, 110))),
  763. dtype=[('a', int), ('b', int), ('d', int)])
  764. def test_no_r1postfix(self):
  765. # Basic test of join_by no_r1postfix
  766. a, b = self.a, self.b
  767. test = join_by(
  768. 'a', a, b, r1postfix='', r2postfix='2', jointype='inner')
  769. control = np.array([(0, 50, 65, 100, 100), (1, 51, 66, 101, 101),
  770. (2, 52, 67, 102, 102), (3, 53, 68, 103, 103),
  771. (4, 54, 69, 104, 104), (5, 55, 70, 105, 105),
  772. (6, 56, 71, 106, 106), (7, 57, 72, 107, 107),
  773. (8, 58, 73, 108, 108), (9, 59, 74, 109, 109)],
  774. dtype=[('a', int), ('b', int), ('b2', int),
  775. ('c', int), ('d', int)])
  776. assert_equal(test, control)
  777. def test_no_postfix(self):
  778. assert_raises(ValueError, join_by, 'a', self.a, self.b,
  779. r1postfix='', r2postfix='')
  780. def test_no_r2postfix(self):
  781. # Basic test of join_by no_r2postfix
  782. a, b = self.a, self.b
  783. test = join_by(
  784. 'a', a, b, r1postfix='1', r2postfix='', jointype='inner')
  785. control = np.array([(0, 50, 65, 100, 100), (1, 51, 66, 101, 101),
  786. (2, 52, 67, 102, 102), (3, 53, 68, 103, 103),
  787. (4, 54, 69, 104, 104), (5, 55, 70, 105, 105),
  788. (6, 56, 71, 106, 106), (7, 57, 72, 107, 107),
  789. (8, 58, 73, 108, 108), (9, 59, 74, 109, 109)],
  790. dtype=[('a', int), ('b1', int), ('b', int),
  791. ('c', int), ('d', int)])
  792. assert_equal(test, control)
  793. def test_two_keys_two_vars(self):
  794. a = np.array(list(zip(np.tile([10, 11], 5), np.repeat(np.arange(5), 2),
  795. np.arange(50, 60), np.arange(10, 20))),
  796. dtype=[('k', int), ('a', int), ('b', int), ('c', int)])
  797. b = np.array(list(zip(np.tile([10, 11], 5), np.repeat(np.arange(5), 2),
  798. np.arange(65, 75), np.arange(0, 10))),
  799. dtype=[('k', int), ('a', int), ('b', int), ('c', int)])
  800. control = np.array([(10, 0, 50, 65, 10, 0), (11, 0, 51, 66, 11, 1),
  801. (10, 1, 52, 67, 12, 2), (11, 1, 53, 68, 13, 3),
  802. (10, 2, 54, 69, 14, 4), (11, 2, 55, 70, 15, 5),
  803. (10, 3, 56, 71, 16, 6), (11, 3, 57, 72, 17, 7),
  804. (10, 4, 58, 73, 18, 8), (11, 4, 59, 74, 19, 9)],
  805. dtype=[('k', int), ('a', int), ('b1', int),
  806. ('b2', int), ('c1', int), ('c2', int)])
  807. test = join_by(
  808. ['a', 'k'], a, b, r1postfix='1', r2postfix='2', jointype='inner')
  809. assert_equal(test.dtype, control.dtype)
  810. assert_equal(test, control)
  811. class TestAppendFieldsObj:
  812. """
  813. Test append_fields with arrays containing objects
  814. """
  815. # https://github.com/numpy/numpy/issues/2346
  816. def setup(self):
  817. from datetime import date
  818. self.data = dict(obj=date(2000, 1, 1))
  819. def test_append_to_objects(self):
  820. "Test append_fields when the base array contains objects"
  821. obj = self.data['obj']
  822. x = np.array([(obj, 1.), (obj, 2.)],
  823. dtype=[('A', object), ('B', float)])
  824. y = np.array([10, 20], dtype=int)
  825. test = append_fields(x, 'C', data=y, usemask=False)
  826. control = np.array([(obj, 1.0, 10), (obj, 2.0, 20)],
  827. dtype=[('A', object), ('B', float), ('C', int)])
  828. assert_equal(test, control)