records.py 37 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101
  1. """
  2. Record Arrays
  3. =============
  4. Record arrays expose the fields of structured arrays as properties.
  5. Most commonly, ndarrays contain elements of a single type, e.g. floats,
  6. integers, bools etc. However, it is possible for elements to be combinations
  7. of these using structured types, such as::
  8. >>> a = np.array([(1, 2.0), (1, 2.0)], dtype=[('x', np.int64), ('y', np.float64)])
  9. >>> a
  10. array([(1, 2.), (1, 2.)], dtype=[('x', '<i8'), ('y', '<f8')])
  11. Here, each element consists of two fields: x (and int), and y (a float).
  12. This is known as a structured array. The different fields are analogous
  13. to columns in a spread-sheet. The different fields can be accessed as
  14. one would a dictionary::
  15. >>> a['x']
  16. array([1, 1])
  17. >>> a['y']
  18. array([2., 2.])
  19. Record arrays allow us to access fields as properties::
  20. >>> ar = np.rec.array(a)
  21. >>> ar.x
  22. array([1, 1])
  23. >>> ar.y
  24. array([2., 2.])
  25. """
  26. import os
  27. import warnings
  28. from collections import Counter, OrderedDict
  29. from . import numeric as sb
  30. from . import numerictypes as nt
  31. from numpy.compat import (
  32. os_fspath, contextlib_nullcontext
  33. )
  34. from numpy.core.overrides import set_module
  35. from .arrayprint import get_printoptions
  36. # All of the functions allow formats to be a dtype
  37. __all__ = ['record', 'recarray', 'format_parser']
  38. ndarray = sb.ndarray
  39. _byteorderconv = {'b':'>',
  40. 'l':'<',
  41. 'n':'=',
  42. 'B':'>',
  43. 'L':'<',
  44. 'N':'=',
  45. 'S':'s',
  46. 's':'s',
  47. '>':'>',
  48. '<':'<',
  49. '=':'=',
  50. '|':'|',
  51. 'I':'|',
  52. 'i':'|'}
  53. # formats regular expression
  54. # allows multidimension spec with a tuple syntax in front
  55. # of the letter code '(2,3)f4' and ' ( 2 , 3 ) f4 '
  56. # are equally allowed
  57. numfmt = nt.typeDict
  58. # taken from OrderedDict recipes in the Python documentation
  59. # https://docs.python.org/3.3/library/collections.html#ordereddict-examples-and-recipes
  60. class _OrderedCounter(Counter, OrderedDict):
  61. """Counter that remembers the order elements are first encountered"""
  62. def __repr__(self):
  63. return '%s(%r)' % (self.__class__.__name__, OrderedDict(self))
  64. def __reduce__(self):
  65. return self.__class__, (OrderedDict(self),)
  66. def find_duplicate(list):
  67. """Find duplication in a list, return a list of duplicated elements"""
  68. return [
  69. item
  70. for item, counts in _OrderedCounter(list).items()
  71. if counts > 1
  72. ]
  73. @set_module('numpy')
  74. class format_parser:
  75. """
  76. Class to convert formats, names, titles description to a dtype.
  77. After constructing the format_parser object, the dtype attribute is
  78. the converted data-type:
  79. ``dtype = format_parser(formats, names, titles).dtype``
  80. Attributes
  81. ----------
  82. dtype : dtype
  83. The converted data-type.
  84. Parameters
  85. ----------
  86. formats : str or list of str
  87. The format description, either specified as a string with
  88. comma-separated format descriptions in the form ``'f8, i4, a5'``, or
  89. a list of format description strings in the form
  90. ``['f8', 'i4', 'a5']``.
  91. names : str or list/tuple of str
  92. The field names, either specified as a comma-separated string in the
  93. form ``'col1, col2, col3'``, or as a list or tuple of strings in the
  94. form ``['col1', 'col2', 'col3']``.
  95. An empty list can be used, in that case default field names
  96. ('f0', 'f1', ...) are used.
  97. titles : sequence
  98. Sequence of title strings. An empty list can be used to leave titles
  99. out.
  100. aligned : bool, optional
  101. If True, align the fields by padding as the C-compiler would.
  102. Default is False.
  103. byteorder : str, optional
  104. If specified, all the fields will be changed to the
  105. provided byte-order. Otherwise, the default byte-order is
  106. used. For all available string specifiers, see `dtype.newbyteorder`.
  107. See Also
  108. --------
  109. dtype, typename, sctype2char
  110. Examples
  111. --------
  112. >>> np.format_parser(['<f8', '<i4', '<a5'], ['col1', 'col2', 'col3'],
  113. ... ['T1', 'T2', 'T3']).dtype
  114. dtype([(('T1', 'col1'), '<f8'), (('T2', 'col2'), '<i4'), (('T3', 'col3'), 'S5')])
  115. `names` and/or `titles` can be empty lists. If `titles` is an empty list,
  116. titles will simply not appear. If `names` is empty, default field names
  117. will be used.
  118. >>> np.format_parser(['f8', 'i4', 'a5'], ['col1', 'col2', 'col3'],
  119. ... []).dtype
  120. dtype([('col1', '<f8'), ('col2', '<i4'), ('col3', '<S5')])
  121. >>> np.format_parser(['<f8', '<i4', '<a5'], [], []).dtype
  122. dtype([('f0', '<f8'), ('f1', '<i4'), ('f2', 'S5')])
  123. """
  124. def __init__(self, formats, names, titles, aligned=False, byteorder=None):
  125. self._parseFormats(formats, aligned)
  126. self._setfieldnames(names, titles)
  127. self._createdtype(byteorder)
  128. def _parseFormats(self, formats, aligned=False):
  129. """ Parse the field formats """
  130. if formats is None:
  131. raise ValueError("Need formats argument")
  132. if isinstance(formats, list):
  133. dtype = sb.dtype(
  134. [('f{}'.format(i), format_) for i, format_ in enumerate(formats)],
  135. aligned,
  136. )
  137. else:
  138. dtype = sb.dtype(formats, aligned)
  139. fields = dtype.fields
  140. if fields is None:
  141. dtype = sb.dtype([('f1', dtype)], aligned)
  142. fields = dtype.fields
  143. keys = dtype.names
  144. self._f_formats = [fields[key][0] for key in keys]
  145. self._offsets = [fields[key][1] for key in keys]
  146. self._nfields = len(keys)
  147. def _setfieldnames(self, names, titles):
  148. """convert input field names into a list and assign to the _names
  149. attribute """
  150. if names:
  151. if type(names) in [list, tuple]:
  152. pass
  153. elif isinstance(names, str):
  154. names = names.split(',')
  155. else:
  156. raise NameError("illegal input names %s" % repr(names))
  157. self._names = [n.strip() for n in names[:self._nfields]]
  158. else:
  159. self._names = []
  160. # if the names are not specified, they will be assigned as
  161. # "f0, f1, f2,..."
  162. # if not enough names are specified, they will be assigned as "f[n],
  163. # f[n+1],..." etc. where n is the number of specified names..."
  164. self._names += ['f%d' % i for i in range(len(self._names),
  165. self._nfields)]
  166. # check for redundant names
  167. _dup = find_duplicate(self._names)
  168. if _dup:
  169. raise ValueError("Duplicate field names: %s" % _dup)
  170. if titles:
  171. self._titles = [n.strip() for n in titles[:self._nfields]]
  172. else:
  173. self._titles = []
  174. titles = []
  175. if self._nfields > len(titles):
  176. self._titles += [None] * (self._nfields - len(titles))
  177. def _createdtype(self, byteorder):
  178. dtype = sb.dtype({
  179. 'names': self._names,
  180. 'formats': self._f_formats,
  181. 'offsets': self._offsets,
  182. 'titles': self._titles,
  183. })
  184. if byteorder is not None:
  185. byteorder = _byteorderconv[byteorder[0]]
  186. dtype = dtype.newbyteorder(byteorder)
  187. self.dtype = dtype
  188. class record(nt.void):
  189. """A data-type scalar that allows field access as attribute lookup.
  190. """
  191. # manually set name and module so that this class's type shows up
  192. # as numpy.record when printed
  193. __name__ = 'record'
  194. __module__ = 'numpy'
  195. def __repr__(self):
  196. if get_printoptions()['legacy'] == '1.13':
  197. return self.__str__()
  198. return super(record, self).__repr__()
  199. def __str__(self):
  200. if get_printoptions()['legacy'] == '1.13':
  201. return str(self.item())
  202. return super(record, self).__str__()
  203. def __getattribute__(self, attr):
  204. if attr in ('setfield', 'getfield', 'dtype'):
  205. return nt.void.__getattribute__(self, attr)
  206. try:
  207. return nt.void.__getattribute__(self, attr)
  208. except AttributeError:
  209. pass
  210. fielddict = nt.void.__getattribute__(self, 'dtype').fields
  211. res = fielddict.get(attr, None)
  212. if res:
  213. obj = self.getfield(*res[:2])
  214. # if it has fields return a record,
  215. # otherwise return the object
  216. try:
  217. dt = obj.dtype
  218. except AttributeError:
  219. #happens if field is Object type
  220. return obj
  221. if dt.names is not None:
  222. return obj.view((self.__class__, obj.dtype))
  223. return obj
  224. else:
  225. raise AttributeError("'record' object has no "
  226. "attribute '%s'" % attr)
  227. def __setattr__(self, attr, val):
  228. if attr in ('setfield', 'getfield', 'dtype'):
  229. raise AttributeError("Cannot set '%s' attribute" % attr)
  230. fielddict = nt.void.__getattribute__(self, 'dtype').fields
  231. res = fielddict.get(attr, None)
  232. if res:
  233. return self.setfield(val, *res[:2])
  234. else:
  235. if getattr(self, attr, None):
  236. return nt.void.__setattr__(self, attr, val)
  237. else:
  238. raise AttributeError("'record' object has no "
  239. "attribute '%s'" % attr)
  240. def __getitem__(self, indx):
  241. obj = nt.void.__getitem__(self, indx)
  242. # copy behavior of record.__getattribute__,
  243. if isinstance(obj, nt.void) and obj.dtype.names is not None:
  244. return obj.view((self.__class__, obj.dtype))
  245. else:
  246. # return a single element
  247. return obj
  248. def pprint(self):
  249. """Pretty-print all fields."""
  250. # pretty-print all fields
  251. names = self.dtype.names
  252. maxlen = max(len(name) for name in names)
  253. fmt = '%% %ds: %%s' % maxlen
  254. rows = [fmt % (name, getattr(self, name)) for name in names]
  255. return "\n".join(rows)
  256. # The recarray is almost identical to a standard array (which supports
  257. # named fields already) The biggest difference is that it can use
  258. # attribute-lookup to find the fields and it is constructed using
  259. # a record.
  260. # If byteorder is given it forces a particular byteorder on all
  261. # the fields (and any subfields)
  262. class recarray(ndarray):
  263. """Construct an ndarray that allows field access using attributes.
  264. Arrays may have a data-types containing fields, analogous
  265. to columns in a spread sheet. An example is ``[(x, int), (y, float)]``,
  266. where each entry in the array is a pair of ``(int, float)``. Normally,
  267. these attributes are accessed using dictionary lookups such as ``arr['x']``
  268. and ``arr['y']``. Record arrays allow the fields to be accessed as members
  269. of the array, using ``arr.x`` and ``arr.y``.
  270. Parameters
  271. ----------
  272. shape : tuple
  273. Shape of output array.
  274. dtype : data-type, optional
  275. The desired data-type. By default, the data-type is determined
  276. from `formats`, `names`, `titles`, `aligned` and `byteorder`.
  277. formats : list of data-types, optional
  278. A list containing the data-types for the different columns, e.g.
  279. ``['i4', 'f8', 'i4']``. `formats` does *not* support the new
  280. convention of using types directly, i.e. ``(int, float, int)``.
  281. Note that `formats` must be a list, not a tuple.
  282. Given that `formats` is somewhat limited, we recommend specifying
  283. `dtype` instead.
  284. names : tuple of str, optional
  285. The name of each column, e.g. ``('x', 'y', 'z')``.
  286. buf : buffer, optional
  287. By default, a new array is created of the given shape and data-type.
  288. If `buf` is specified and is an object exposing the buffer interface,
  289. the array will use the memory from the existing buffer. In this case,
  290. the `offset` and `strides` keywords are available.
  291. Other Parameters
  292. ----------------
  293. titles : tuple of str, optional
  294. Aliases for column names. For example, if `names` were
  295. ``('x', 'y', 'z')`` and `titles` is
  296. ``('x_coordinate', 'y_coordinate', 'z_coordinate')``, then
  297. ``arr['x']`` is equivalent to both ``arr.x`` and ``arr.x_coordinate``.
  298. byteorder : {'<', '>', '='}, optional
  299. Byte-order for all fields.
  300. aligned : bool, optional
  301. Align the fields in memory as the C-compiler would.
  302. strides : tuple of ints, optional
  303. Buffer (`buf`) is interpreted according to these strides (strides
  304. define how many bytes each array element, row, column, etc.
  305. occupy in memory).
  306. offset : int, optional
  307. Start reading buffer (`buf`) from this offset onwards.
  308. order : {'C', 'F'}, optional
  309. Row-major (C-style) or column-major (Fortran-style) order.
  310. Returns
  311. -------
  312. rec : recarray
  313. Empty array of the given shape and type.
  314. See Also
  315. --------
  316. core.records.fromrecords : Construct a record array from data.
  317. record : fundamental data-type for `recarray`.
  318. format_parser : determine a data-type from formats, names, titles.
  319. Notes
  320. -----
  321. This constructor can be compared to ``empty``: it creates a new record
  322. array but does not fill it with data. To create a record array from data,
  323. use one of the following methods:
  324. 1. Create a standard ndarray and convert it to a record array,
  325. using ``arr.view(np.recarray)``
  326. 2. Use the `buf` keyword.
  327. 3. Use `np.rec.fromrecords`.
  328. Examples
  329. --------
  330. Create an array with two fields, ``x`` and ``y``:
  331. >>> x = np.array([(1.0, 2), (3.0, 4)], dtype=[('x', '<f8'), ('y', '<i8')])
  332. >>> x
  333. array([(1., 2), (3., 4)], dtype=[('x', '<f8'), ('y', '<i8')])
  334. >>> x['x']
  335. array([1., 3.])
  336. View the array as a record array:
  337. >>> x = x.view(np.recarray)
  338. >>> x.x
  339. array([1., 3.])
  340. >>> x.y
  341. array([2, 4])
  342. Create a new, empty record array:
  343. >>> np.recarray((2,),
  344. ... dtype=[('x', int), ('y', float), ('z', int)]) #doctest: +SKIP
  345. rec.array([(-1073741821, 1.2249118382103472e-301, 24547520),
  346. (3471280, 1.2134086255804012e-316, 0)],
  347. dtype=[('x', '<i4'), ('y', '<f8'), ('z', '<i4')])
  348. """
  349. # manually set name and module so that this class's type shows
  350. # up as "numpy.recarray" when printed
  351. __name__ = 'recarray'
  352. __module__ = 'numpy'
  353. def __new__(subtype, shape, dtype=None, buf=None, offset=0, strides=None,
  354. formats=None, names=None, titles=None,
  355. byteorder=None, aligned=False, order='C'):
  356. if dtype is not None:
  357. descr = sb.dtype(dtype)
  358. else:
  359. descr = format_parser(formats, names, titles, aligned, byteorder).dtype
  360. if buf is None:
  361. self = ndarray.__new__(subtype, shape, (record, descr), order=order)
  362. else:
  363. self = ndarray.__new__(subtype, shape, (record, descr),
  364. buffer=buf, offset=offset,
  365. strides=strides, order=order)
  366. return self
  367. def __array_finalize__(self, obj):
  368. if self.dtype.type is not record and self.dtype.names is not None:
  369. # if self.dtype is not np.record, invoke __setattr__ which will
  370. # convert it to a record if it is a void dtype.
  371. self.dtype = self.dtype
  372. def __getattribute__(self, attr):
  373. # See if ndarray has this attr, and return it if so. (note that this
  374. # means a field with the same name as an ndarray attr cannot be
  375. # accessed by attribute).
  376. try:
  377. return object.__getattribute__(self, attr)
  378. except AttributeError: # attr must be a fieldname
  379. pass
  380. # look for a field with this name
  381. fielddict = ndarray.__getattribute__(self, 'dtype').fields
  382. try:
  383. res = fielddict[attr][:2]
  384. except (TypeError, KeyError) as e:
  385. raise AttributeError("recarray has no attribute %s" % attr) from e
  386. obj = self.getfield(*res)
  387. # At this point obj will always be a recarray, since (see
  388. # PyArray_GetField) the type of obj is inherited. Next, if obj.dtype is
  389. # non-structured, convert it to an ndarray. Then if obj is structured
  390. # with void type convert it to the same dtype.type (eg to preserve
  391. # numpy.record type if present), since nested structured fields do not
  392. # inherit type. Don't do this for non-void structures though.
  393. if obj.dtype.names is not None:
  394. if issubclass(obj.dtype.type, nt.void):
  395. return obj.view(dtype=(self.dtype.type, obj.dtype))
  396. return obj
  397. else:
  398. return obj.view(ndarray)
  399. # Save the dictionary.
  400. # If the attr is a field name and not in the saved dictionary
  401. # Undo any "setting" of the attribute and do a setfield
  402. # Thus, you can't create attributes on-the-fly that are field names.
  403. def __setattr__(self, attr, val):
  404. # Automatically convert (void) structured types to records
  405. # (but not non-void structures, subarrays, or non-structured voids)
  406. if attr == 'dtype' and issubclass(val.type, nt.void) and val.names is not None:
  407. val = sb.dtype((record, val))
  408. newattr = attr not in self.__dict__
  409. try:
  410. ret = object.__setattr__(self, attr, val)
  411. except Exception:
  412. fielddict = ndarray.__getattribute__(self, 'dtype').fields or {}
  413. if attr not in fielddict:
  414. raise
  415. else:
  416. fielddict = ndarray.__getattribute__(self, 'dtype').fields or {}
  417. if attr not in fielddict:
  418. return ret
  419. if newattr:
  420. # We just added this one or this setattr worked on an
  421. # internal attribute.
  422. try:
  423. object.__delattr__(self, attr)
  424. except Exception:
  425. return ret
  426. try:
  427. res = fielddict[attr][:2]
  428. except (TypeError, KeyError) as e:
  429. raise AttributeError(
  430. "record array has no attribute %s" % attr
  431. ) from e
  432. return self.setfield(val, *res)
  433. def __getitem__(self, indx):
  434. obj = super(recarray, self).__getitem__(indx)
  435. # copy behavior of getattr, except that here
  436. # we might also be returning a single element
  437. if isinstance(obj, ndarray):
  438. if obj.dtype.names is not None:
  439. obj = obj.view(type(self))
  440. if issubclass(obj.dtype.type, nt.void):
  441. return obj.view(dtype=(self.dtype.type, obj.dtype))
  442. return obj
  443. else:
  444. return obj.view(type=ndarray)
  445. else:
  446. # return a single element
  447. return obj
  448. def __repr__(self):
  449. repr_dtype = self.dtype
  450. if self.dtype.type is record or not issubclass(self.dtype.type, nt.void):
  451. # If this is a full record array (has numpy.record dtype),
  452. # or if it has a scalar (non-void) dtype with no records,
  453. # represent it using the rec.array function. Since rec.array
  454. # converts dtype to a numpy.record for us, convert back
  455. # to non-record before printing
  456. if repr_dtype.type is record:
  457. repr_dtype = sb.dtype((nt.void, repr_dtype))
  458. prefix = "rec.array("
  459. fmt = 'rec.array(%s,%sdtype=%s)'
  460. else:
  461. # otherwise represent it using np.array plus a view
  462. # This should only happen if the user is playing
  463. # strange games with dtypes.
  464. prefix = "array("
  465. fmt = 'array(%s,%sdtype=%s).view(numpy.recarray)'
  466. # get data/shape string. logic taken from numeric.array_repr
  467. if self.size > 0 or self.shape == (0,):
  468. lst = sb.array2string(
  469. self, separator=', ', prefix=prefix, suffix=',')
  470. else:
  471. # show zero-length shape unless it is (0,)
  472. lst = "[], shape=%s" % (repr(self.shape),)
  473. lf = '\n'+' '*len(prefix)
  474. if get_printoptions()['legacy'] == '1.13':
  475. lf = ' ' + lf # trailing space
  476. return fmt % (lst, lf, repr_dtype)
  477. def field(self, attr, val=None):
  478. if isinstance(attr, int):
  479. names = ndarray.__getattribute__(self, 'dtype').names
  480. attr = names[attr]
  481. fielddict = ndarray.__getattribute__(self, 'dtype').fields
  482. res = fielddict[attr][:2]
  483. if val is None:
  484. obj = self.getfield(*res)
  485. if obj.dtype.names is not None:
  486. return obj
  487. return obj.view(ndarray)
  488. else:
  489. return self.setfield(val, *res)
  490. def _deprecate_shape_0_as_None(shape):
  491. if shape == 0:
  492. warnings.warn(
  493. "Passing `shape=0` to have the shape be inferred is deprecated, "
  494. "and in future will be equivalent to `shape=(0,)`. To infer "
  495. "the shape and suppress this warning, pass `shape=None` instead.",
  496. FutureWarning, stacklevel=3)
  497. return None
  498. else:
  499. return shape
  500. def fromarrays(arrayList, dtype=None, shape=None, formats=None,
  501. names=None, titles=None, aligned=False, byteorder=None):
  502. """Create a record array from a (flat) list of arrays
  503. Parameters
  504. ----------
  505. arrayList : list or tuple
  506. List of array-like objects (such as lists, tuples,
  507. and ndarrays).
  508. dtype : data-type, optional
  509. valid dtype for all arrays
  510. shape : int or tuple of ints, optional
  511. Shape of the resulting array. If not provided, inferred from
  512. ``arrayList[0]``.
  513. formats, names, titles, aligned, byteorder :
  514. If `dtype` is ``None``, these arguments are passed to
  515. `numpy.format_parser` to construct a dtype. See that function for
  516. detailed documentation.
  517. Returns
  518. -------
  519. np.recarray
  520. Record array consisting of given arrayList columns.
  521. Examples
  522. --------
  523. >>> x1=np.array([1,2,3,4])
  524. >>> x2=np.array(['a','dd','xyz','12'])
  525. >>> x3=np.array([1.1,2,3,4])
  526. >>> r = np.core.records.fromarrays([x1,x2,x3],names='a,b,c')
  527. >>> print(r[1])
  528. (2, 'dd', 2.0) # may vary
  529. >>> x1[1]=34
  530. >>> r.a
  531. array([1, 2, 3, 4])
  532. >>> x1 = np.array([1, 2, 3, 4])
  533. >>> x2 = np.array(['a', 'dd', 'xyz', '12'])
  534. >>> x3 = np.array([1.1, 2, 3,4])
  535. >>> r = np.core.records.fromarrays(
  536. ... [x1, x2, x3],
  537. ... dtype=np.dtype([('a', np.int32), ('b', 'S3'), ('c', np.float32)]))
  538. >>> r
  539. rec.array([(1, b'a', 1.1), (2, b'dd', 2. ), (3, b'xyz', 3. ),
  540. (4, b'12', 4. )],
  541. dtype=[('a', '<i4'), ('b', 'S3'), ('c', '<f4')])
  542. """
  543. arrayList = [sb.asarray(x) for x in arrayList]
  544. # NumPy 1.19.0, 2020-01-01
  545. shape = _deprecate_shape_0_as_None(shape)
  546. if shape is None:
  547. shape = arrayList[0].shape
  548. elif isinstance(shape, int):
  549. shape = (shape,)
  550. if formats is None and dtype is None:
  551. # go through each object in the list to see if it is an ndarray
  552. # and determine the formats.
  553. formats = [obj.dtype for obj in arrayList]
  554. if dtype is not None:
  555. descr = sb.dtype(dtype)
  556. else:
  557. descr = format_parser(formats, names, titles, aligned, byteorder).dtype
  558. _names = descr.names
  559. # Determine shape from data-type.
  560. if len(descr) != len(arrayList):
  561. raise ValueError("mismatch between the number of fields "
  562. "and the number of arrays")
  563. d0 = descr[0].shape
  564. nn = len(d0)
  565. if nn > 0:
  566. shape = shape[:-nn]
  567. for k, obj in enumerate(arrayList):
  568. nn = descr[k].ndim
  569. testshape = obj.shape[:obj.ndim - nn]
  570. if testshape != shape:
  571. raise ValueError("array-shape mismatch in array %d" % k)
  572. _array = recarray(shape, descr)
  573. # populate the record array (makes a copy)
  574. for i in range(len(arrayList)):
  575. _array[_names[i]] = arrayList[i]
  576. return _array
  577. def fromrecords(recList, dtype=None, shape=None, formats=None, names=None,
  578. titles=None, aligned=False, byteorder=None):
  579. """Create a recarray from a list of records in text form.
  580. Parameters
  581. ----------
  582. recList : sequence
  583. data in the same field may be heterogeneous - they will be promoted
  584. to the highest data type.
  585. dtype : data-type, optional
  586. valid dtype for all arrays
  587. shape : int or tuple of ints, optional
  588. shape of each array.
  589. formats, names, titles, aligned, byteorder :
  590. If `dtype` is ``None``, these arguments are passed to
  591. `numpy.format_parser` to construct a dtype. See that function for
  592. detailed documentation.
  593. If both `formats` and `dtype` are None, then this will auto-detect
  594. formats. Use list of tuples rather than list of lists for faster
  595. processing.
  596. Returns
  597. -------
  598. np.recarray
  599. record array consisting of given recList rows.
  600. Examples
  601. --------
  602. >>> r=np.core.records.fromrecords([(456,'dbe',1.2),(2,'de',1.3)],
  603. ... names='col1,col2,col3')
  604. >>> print(r[0])
  605. (456, 'dbe', 1.2)
  606. >>> r.col1
  607. array([456, 2])
  608. >>> r.col2
  609. array(['dbe', 'de'], dtype='<U3')
  610. >>> import pickle
  611. >>> pickle.loads(pickle.dumps(r))
  612. rec.array([(456, 'dbe', 1.2), ( 2, 'de', 1.3)],
  613. dtype=[('col1', '<i8'), ('col2', '<U3'), ('col3', '<f8')])
  614. """
  615. if formats is None and dtype is None: # slower
  616. obj = sb.array(recList, dtype=object)
  617. arrlist = [sb.array(obj[..., i].tolist()) for i in range(obj.shape[-1])]
  618. return fromarrays(arrlist, formats=formats, shape=shape, names=names,
  619. titles=titles, aligned=aligned, byteorder=byteorder)
  620. if dtype is not None:
  621. descr = sb.dtype((record, dtype))
  622. else:
  623. descr = format_parser(formats, names, titles, aligned, byteorder).dtype
  624. try:
  625. retval = sb.array(recList, dtype=descr)
  626. except (TypeError, ValueError):
  627. # NumPy 1.19.0, 2020-01-01
  628. shape = _deprecate_shape_0_as_None(shape)
  629. if shape is None:
  630. shape = len(recList)
  631. if isinstance(shape, int):
  632. shape = (shape,)
  633. if len(shape) > 1:
  634. raise ValueError("Can only deal with 1-d array.")
  635. _array = recarray(shape, descr)
  636. for k in range(_array.size):
  637. _array[k] = tuple(recList[k])
  638. # list of lists instead of list of tuples ?
  639. # 2018-02-07, 1.14.1
  640. warnings.warn(
  641. "fromrecords expected a list of tuples, may have received a list "
  642. "of lists instead. In the future that will raise an error",
  643. FutureWarning, stacklevel=2)
  644. return _array
  645. else:
  646. if shape is not None and retval.shape != shape:
  647. retval.shape = shape
  648. res = retval.view(recarray)
  649. return res
  650. def fromstring(datastring, dtype=None, shape=None, offset=0, formats=None,
  651. names=None, titles=None, aligned=False, byteorder=None):
  652. r"""Create a record array from binary data
  653. Note that despite the name of this function it does not accept `str`
  654. instances.
  655. Parameters
  656. ----------
  657. datastring : bytes-like
  658. Buffer of binary data
  659. dtype : data-type, optional
  660. Valid dtype for all arrays
  661. shape : int or tuple of ints, optional
  662. Shape of each array.
  663. offset : int, optional
  664. Position in the buffer to start reading from.
  665. formats, names, titles, aligned, byteorder :
  666. If `dtype` is ``None``, these arguments are passed to
  667. `numpy.format_parser` to construct a dtype. See that function for
  668. detailed documentation.
  669. Returns
  670. -------
  671. np.recarray
  672. Record array view into the data in datastring. This will be readonly
  673. if `datastring` is readonly.
  674. See Also
  675. --------
  676. numpy.frombuffer
  677. Examples
  678. --------
  679. >>> a = b'\x01\x02\x03abc'
  680. >>> np.core.records.fromstring(a, dtype='u1,u1,u1,S3')
  681. rec.array([(1, 2, 3, b'abc')],
  682. dtype=[('f0', 'u1'), ('f1', 'u1'), ('f2', 'u1'), ('f3', 'S3')])
  683. >>> grades_dtype = [('Name', (np.str_, 10)), ('Marks', np.float64),
  684. ... ('GradeLevel', np.int32)]
  685. >>> grades_array = np.array([('Sam', 33.3, 3), ('Mike', 44.4, 5),
  686. ... ('Aadi', 66.6, 6)], dtype=grades_dtype)
  687. >>> np.core.records.fromstring(grades_array.tobytes(), dtype=grades_dtype)
  688. rec.array([('Sam', 33.3, 3), ('Mike', 44.4, 5), ('Aadi', 66.6, 6)],
  689. dtype=[('Name', '<U10'), ('Marks', '<f8'), ('GradeLevel', '<i4')])
  690. >>> s = '\x01\x02\x03abc'
  691. >>> np.core.records.fromstring(s, dtype='u1,u1,u1,S3')
  692. Traceback (most recent call last)
  693. ...
  694. TypeError: a bytes-like object is required, not 'str'
  695. """
  696. if dtype is None and formats is None:
  697. raise TypeError("fromstring() needs a 'dtype' or 'formats' argument")
  698. if dtype is not None:
  699. descr = sb.dtype(dtype)
  700. else:
  701. descr = format_parser(formats, names, titles, aligned, byteorder).dtype
  702. itemsize = descr.itemsize
  703. # NumPy 1.19.0, 2020-01-01
  704. shape = _deprecate_shape_0_as_None(shape)
  705. if shape in (None, -1):
  706. shape = (len(datastring) - offset) // itemsize
  707. _array = recarray(shape, descr, buf=datastring, offset=offset)
  708. return _array
  709. def get_remaining_size(fd):
  710. pos = fd.tell()
  711. try:
  712. fd.seek(0, 2)
  713. return fd.tell() - pos
  714. finally:
  715. fd.seek(pos, 0)
  716. def fromfile(fd, dtype=None, shape=None, offset=0, formats=None,
  717. names=None, titles=None, aligned=False, byteorder=None):
  718. """Create an array from binary file data
  719. Parameters
  720. ----------
  721. fd : str or file type
  722. If file is a string or a path-like object then that file is opened,
  723. else it is assumed to be a file object. The file object must
  724. support random access (i.e. it must have tell and seek methods).
  725. dtype : data-type, optional
  726. valid dtype for all arrays
  727. shape : int or tuple of ints, optional
  728. shape of each array.
  729. offset : int, optional
  730. Position in the file to start reading from.
  731. formats, names, titles, aligned, byteorder :
  732. If `dtype` is ``None``, these arguments are passed to
  733. `numpy.format_parser` to construct a dtype. See that function for
  734. detailed documentation
  735. Returns
  736. -------
  737. np.recarray
  738. record array consisting of data enclosed in file.
  739. Examples
  740. --------
  741. >>> from tempfile import TemporaryFile
  742. >>> a = np.empty(10,dtype='f8,i4,a5')
  743. >>> a[5] = (0.5,10,'abcde')
  744. >>>
  745. >>> fd=TemporaryFile()
  746. >>> a = a.newbyteorder('<')
  747. >>> a.tofile(fd)
  748. >>>
  749. >>> _ = fd.seek(0)
  750. >>> r=np.core.records.fromfile(fd, formats='f8,i4,a5', shape=10,
  751. ... byteorder='<')
  752. >>> print(r[5])
  753. (0.5, 10, 'abcde')
  754. >>> r.shape
  755. (10,)
  756. """
  757. if dtype is None and formats is None:
  758. raise TypeError("fromfile() needs a 'dtype' or 'formats' argument")
  759. # NumPy 1.19.0, 2020-01-01
  760. shape = _deprecate_shape_0_as_None(shape)
  761. if shape is None:
  762. shape = (-1,)
  763. elif isinstance(shape, int):
  764. shape = (shape,)
  765. if hasattr(fd, 'readinto'):
  766. # GH issue 2504. fd supports io.RawIOBase or io.BufferedIOBase interface.
  767. # Example of fd: gzip, BytesIO, BufferedReader
  768. # file already opened
  769. ctx = contextlib_nullcontext(fd)
  770. else:
  771. # open file
  772. ctx = open(os_fspath(fd), 'rb')
  773. with ctx as fd:
  774. if offset > 0:
  775. fd.seek(offset, 1)
  776. size = get_remaining_size(fd)
  777. if dtype is not None:
  778. descr = sb.dtype(dtype)
  779. else:
  780. descr = format_parser(formats, names, titles, aligned, byteorder).dtype
  781. itemsize = descr.itemsize
  782. shapeprod = sb.array(shape).prod(dtype=nt.intp)
  783. shapesize = shapeprod * itemsize
  784. if shapesize < 0:
  785. shape = list(shape)
  786. shape[shape.index(-1)] = size // -shapesize
  787. shape = tuple(shape)
  788. shapeprod = sb.array(shape).prod(dtype=nt.intp)
  789. nbytes = shapeprod * itemsize
  790. if nbytes > size:
  791. raise ValueError(
  792. "Not enough bytes left in file for specified shape and type")
  793. # create the array
  794. _array = recarray(shape, descr)
  795. nbytesread = fd.readinto(_array.data)
  796. if nbytesread != nbytes:
  797. raise IOError("Didn't read as many bytes as expected")
  798. return _array
  799. def array(obj, dtype=None, shape=None, offset=0, strides=None, formats=None,
  800. names=None, titles=None, aligned=False, byteorder=None, copy=True):
  801. """
  802. Construct a record array from a wide-variety of objects.
  803. A general-purpose record array constructor that dispatches to the
  804. appropriate `recarray` creation function based on the inputs (see Notes).
  805. Parameters
  806. ----------
  807. obj: any
  808. Input object. See Notes for details on how various input types are
  809. treated.
  810. dtype: data-type, optional
  811. Valid dtype for array.
  812. shape: int or tuple of ints, optional
  813. Shape of each array.
  814. offset: int, optional
  815. Position in the file or buffer to start reading from.
  816. strides: tuple of ints, optional
  817. Buffer (`buf`) is interpreted according to these strides (strides
  818. define how many bytes each array element, row, column, etc.
  819. occupy in memory).
  820. formats, names, titles, aligned, byteorder :
  821. If `dtype` is ``None``, these arguments are passed to
  822. `numpy.format_parser` to construct a dtype. See that function for
  823. detailed documentation.
  824. copy: bool, optional
  825. Whether to copy the input object (True), or to use a reference instead.
  826. This option only applies when the input is an ndarray or recarray.
  827. Defaults to True.
  828. Returns
  829. -------
  830. np.recarray
  831. Record array created from the specified object.
  832. Notes
  833. -----
  834. If `obj` is ``None``, then call the `~numpy.recarray` constructor. If
  835. `obj` is a string, then call the `fromstring` constructor. If `obj` is a
  836. list or a tuple, then if the first object is an `~numpy.ndarray`, call
  837. `fromarrays`, otherwise call `fromrecords`. If `obj` is a
  838. `~numpy.recarray`, then make a copy of the data in the recarray
  839. (if ``copy=True``) and use the new formats, names, and titles. If `obj`
  840. is a file, then call `fromfile`. Finally, if obj is an `ndarray`, then
  841. return ``obj.view(recarray)``, making a copy of the data if ``copy=True``.
  842. Examples
  843. --------
  844. >>> a = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]])
  845. array([[1, 2, 3],
  846. [4, 5, 6],
  847. [7, 8, 9]])
  848. >>> np.core.records.array(a)
  849. rec.array([[1, 2, 3],
  850. [4, 5, 6],
  851. [7, 8, 9]],
  852. dtype=int32)
  853. >>> b = [(1, 1), (2, 4), (3, 9)]
  854. >>> c = np.core.records.array(b, formats = ['i2', 'f2'], names = ('x', 'y'))
  855. >>> c
  856. rec.array([(1, 1.0), (2, 4.0), (3, 9.0)],
  857. dtype=[('x', '<i2'), ('y', '<f2')])
  858. >>> c.x
  859. rec.array([1, 2, 3], dtype=int16)
  860. >>> c.y
  861. rec.array([ 1.0, 4.0, 9.0], dtype=float16)
  862. >>> r = np.rec.array(['abc','def'], names=['col1','col2'])
  863. >>> print(r.col1)
  864. abc
  865. >>> r.col1
  866. array('abc', dtype='<U3')
  867. >>> r.col2
  868. array('def', dtype='<U3')
  869. """
  870. if ((isinstance(obj, (type(None), str)) or hasattr(obj, 'readinto')) and
  871. formats is None and dtype is None):
  872. raise ValueError("Must define formats (or dtype) if object is "
  873. "None, string, or an open file")
  874. kwds = {}
  875. if dtype is not None:
  876. dtype = sb.dtype(dtype)
  877. elif formats is not None:
  878. dtype = format_parser(formats, names, titles,
  879. aligned, byteorder).dtype
  880. else:
  881. kwds = {'formats': formats,
  882. 'names': names,
  883. 'titles': titles,
  884. 'aligned': aligned,
  885. 'byteorder': byteorder
  886. }
  887. if obj is None:
  888. if shape is None:
  889. raise ValueError("Must define a shape if obj is None")
  890. return recarray(shape, dtype, buf=obj, offset=offset, strides=strides)
  891. elif isinstance(obj, bytes):
  892. return fromstring(obj, dtype, shape=shape, offset=offset, **kwds)
  893. elif isinstance(obj, (list, tuple)):
  894. if isinstance(obj[0], (tuple, list)):
  895. return fromrecords(obj, dtype=dtype, shape=shape, **kwds)
  896. else:
  897. return fromarrays(obj, dtype=dtype, shape=shape, **kwds)
  898. elif isinstance(obj, recarray):
  899. if dtype is not None and (obj.dtype != dtype):
  900. new = obj.view(dtype)
  901. else:
  902. new = obj
  903. if copy:
  904. new = new.copy()
  905. return new
  906. elif hasattr(obj, 'readinto'):
  907. return fromfile(obj, dtype=dtype, shape=shape, offset=offset)
  908. elif isinstance(obj, ndarray):
  909. if dtype is not None and (obj.dtype != dtype):
  910. new = obj.view(dtype)
  911. else:
  912. new = obj
  913. if copy:
  914. new = new.copy()
  915. return new.view(recarray)
  916. else:
  917. interface = getattr(obj, "__array_interface__", None)
  918. if interface is None or not isinstance(interface, dict):
  919. raise ValueError("Unknown input type")
  920. obj = sb.array(obj)
  921. if dtype is not None and (obj.dtype != dtype):
  922. obj = obj.view(dtype)
  923. return obj.view(recarray)