npyio.py 87 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413
  1. import sys
  2. import os
  3. import re
  4. import functools
  5. import itertools
  6. import warnings
  7. import weakref
  8. import contextlib
  9. from operator import itemgetter, index as opindex
  10. from collections.abc import Mapping
  11. import numpy as np
  12. from . import format
  13. from ._datasource import DataSource
  14. from numpy.core import overrides
  15. from numpy.core.multiarray import packbits, unpackbits
  16. from numpy.core.overrides import set_array_function_like_doc, set_module
  17. from numpy.core._internal import recursive
  18. from ._iotools import (
  19. LineSplitter, NameValidator, StringConverter, ConverterError,
  20. ConverterLockError, ConversionWarning, _is_string_like,
  21. has_nested_fields, flatten_dtype, easy_dtype, _decode_line
  22. )
  23. from numpy.compat import (
  24. asbytes, asstr, asunicode, os_fspath, os_PathLike,
  25. pickle, contextlib_nullcontext
  26. )
  27. @set_module('numpy')
  28. def loads(*args, **kwargs):
  29. # NumPy 1.15.0, 2017-12-10
  30. warnings.warn(
  31. "np.loads is deprecated, use pickle.loads instead",
  32. DeprecationWarning, stacklevel=2)
  33. return pickle.loads(*args, **kwargs)
  34. __all__ = [
  35. 'savetxt', 'loadtxt', 'genfromtxt', 'ndfromtxt', 'mafromtxt',
  36. 'recfromtxt', 'recfromcsv', 'load', 'loads', 'save', 'savez',
  37. 'savez_compressed', 'packbits', 'unpackbits', 'fromregex', 'DataSource'
  38. ]
  39. array_function_dispatch = functools.partial(
  40. overrides.array_function_dispatch, module='numpy')
  41. class BagObj:
  42. """
  43. BagObj(obj)
  44. Convert attribute look-ups to getitems on the object passed in.
  45. Parameters
  46. ----------
  47. obj : class instance
  48. Object on which attribute look-up is performed.
  49. Examples
  50. --------
  51. >>> from numpy.lib.npyio import BagObj as BO
  52. >>> class BagDemo:
  53. ... def __getitem__(self, key): # An instance of BagObj(BagDemo)
  54. ... # will call this method when any
  55. ... # attribute look-up is required
  56. ... result = "Doesn't matter what you want, "
  57. ... return result + "you're gonna get this"
  58. ...
  59. >>> demo_obj = BagDemo()
  60. >>> bagobj = BO(demo_obj)
  61. >>> bagobj.hello_there
  62. "Doesn't matter what you want, you're gonna get this"
  63. >>> bagobj.I_can_be_anything
  64. "Doesn't matter what you want, you're gonna get this"
  65. """
  66. def __init__(self, obj):
  67. # Use weakref to make NpzFile objects collectable by refcount
  68. self._obj = weakref.proxy(obj)
  69. def __getattribute__(self, key):
  70. try:
  71. return object.__getattribute__(self, '_obj')[key]
  72. except KeyError:
  73. raise AttributeError(key) from None
  74. def __dir__(self):
  75. """
  76. Enables dir(bagobj) to list the files in an NpzFile.
  77. This also enables tab-completion in an interpreter or IPython.
  78. """
  79. return list(object.__getattribute__(self, '_obj').keys())
  80. def zipfile_factory(file, *args, **kwargs):
  81. """
  82. Create a ZipFile.
  83. Allows for Zip64, and the `file` argument can accept file, str, or
  84. pathlib.Path objects. `args` and `kwargs` are passed to the zipfile.ZipFile
  85. constructor.
  86. """
  87. if not hasattr(file, 'read'):
  88. file = os_fspath(file)
  89. import zipfile
  90. kwargs['allowZip64'] = True
  91. return zipfile.ZipFile(file, *args, **kwargs)
  92. class NpzFile(Mapping):
  93. """
  94. NpzFile(fid)
  95. A dictionary-like object with lazy-loading of files in the zipped
  96. archive provided on construction.
  97. `NpzFile` is used to load files in the NumPy ``.npz`` data archive
  98. format. It assumes that files in the archive have a ``.npy`` extension,
  99. other files are ignored.
  100. The arrays and file strings are lazily loaded on either
  101. getitem access using ``obj['key']`` or attribute lookup using
  102. ``obj.f.key``. A list of all files (without ``.npy`` extensions) can
  103. be obtained with ``obj.files`` and the ZipFile object itself using
  104. ``obj.zip``.
  105. Attributes
  106. ----------
  107. files : list of str
  108. List of all files in the archive with a ``.npy`` extension.
  109. zip : ZipFile instance
  110. The ZipFile object initialized with the zipped archive.
  111. f : BagObj instance
  112. An object on which attribute can be performed as an alternative
  113. to getitem access on the `NpzFile` instance itself.
  114. allow_pickle : bool, optional
  115. Allow loading pickled data. Default: False
  116. .. versionchanged:: 1.16.3
  117. Made default False in response to CVE-2019-6446.
  118. pickle_kwargs : dict, optional
  119. Additional keyword arguments to pass on to pickle.load.
  120. These are only useful when loading object arrays saved on
  121. Python 2 when using Python 3.
  122. Parameters
  123. ----------
  124. fid : file or str
  125. The zipped archive to open. This is either a file-like object
  126. or a string containing the path to the archive.
  127. own_fid : bool, optional
  128. Whether NpzFile should close the file handle.
  129. Requires that `fid` is a file-like object.
  130. Examples
  131. --------
  132. >>> from tempfile import TemporaryFile
  133. >>> outfile = TemporaryFile()
  134. >>> x = np.arange(10)
  135. >>> y = np.sin(x)
  136. >>> np.savez(outfile, x=x, y=y)
  137. >>> _ = outfile.seek(0)
  138. >>> npz = np.load(outfile)
  139. >>> isinstance(npz, np.lib.io.NpzFile)
  140. True
  141. >>> sorted(npz.files)
  142. ['x', 'y']
  143. >>> npz['x'] # getitem access
  144. array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])
  145. >>> npz.f.x # attribute lookup
  146. array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])
  147. """
  148. # Make __exit__ safe if zipfile_factory raises an exception
  149. zip = None
  150. fid = None
  151. def __init__(self, fid, own_fid=False, allow_pickle=False,
  152. pickle_kwargs=None):
  153. # Import is postponed to here since zipfile depends on gzip, an
  154. # optional component of the so-called standard library.
  155. _zip = zipfile_factory(fid)
  156. self._files = _zip.namelist()
  157. self.files = []
  158. self.allow_pickle = allow_pickle
  159. self.pickle_kwargs = pickle_kwargs
  160. for x in self._files:
  161. if x.endswith('.npy'):
  162. self.files.append(x[:-4])
  163. else:
  164. self.files.append(x)
  165. self.zip = _zip
  166. self.f = BagObj(self)
  167. if own_fid:
  168. self.fid = fid
  169. def __enter__(self):
  170. return self
  171. def __exit__(self, exc_type, exc_value, traceback):
  172. self.close()
  173. def close(self):
  174. """
  175. Close the file.
  176. """
  177. if self.zip is not None:
  178. self.zip.close()
  179. self.zip = None
  180. if self.fid is not None:
  181. self.fid.close()
  182. self.fid = None
  183. self.f = None # break reference cycle
  184. def __del__(self):
  185. self.close()
  186. # Implement the Mapping ABC
  187. def __iter__(self):
  188. return iter(self.files)
  189. def __len__(self):
  190. return len(self.files)
  191. def __getitem__(self, key):
  192. # FIXME: This seems like it will copy strings around
  193. # more than is strictly necessary. The zipfile
  194. # will read the string and then
  195. # the format.read_array will copy the string
  196. # to another place in memory.
  197. # It would be better if the zipfile could read
  198. # (or at least uncompress) the data
  199. # directly into the array memory.
  200. member = False
  201. if key in self._files:
  202. member = True
  203. elif key in self.files:
  204. member = True
  205. key += '.npy'
  206. if member:
  207. bytes = self.zip.open(key)
  208. magic = bytes.read(len(format.MAGIC_PREFIX))
  209. bytes.close()
  210. if magic == format.MAGIC_PREFIX:
  211. bytes = self.zip.open(key)
  212. return format.read_array(bytes,
  213. allow_pickle=self.allow_pickle,
  214. pickle_kwargs=self.pickle_kwargs)
  215. else:
  216. return self.zip.read(key)
  217. else:
  218. raise KeyError("%s is not a file in the archive" % key)
  219. # deprecate the python 2 dict apis that we supported by accident in
  220. # python 3. We forgot to implement itervalues() at all in earlier
  221. # versions of numpy, so no need to deprecated it here.
  222. def iteritems(self):
  223. # Numpy 1.15, 2018-02-20
  224. warnings.warn(
  225. "NpzFile.iteritems is deprecated in python 3, to match the "
  226. "removal of dict.itertems. Use .items() instead.",
  227. DeprecationWarning, stacklevel=2)
  228. return self.items()
  229. def iterkeys(self):
  230. # Numpy 1.15, 2018-02-20
  231. warnings.warn(
  232. "NpzFile.iterkeys is deprecated in python 3, to match the "
  233. "removal of dict.iterkeys. Use .keys() instead.",
  234. DeprecationWarning, stacklevel=2)
  235. return self.keys()
  236. @set_module('numpy')
  237. def load(file, mmap_mode=None, allow_pickle=False, fix_imports=True,
  238. encoding='ASCII'):
  239. """
  240. Load arrays or pickled objects from ``.npy``, ``.npz`` or pickled files.
  241. .. warning:: Loading files that contain object arrays uses the ``pickle``
  242. module, which is not secure against erroneous or maliciously
  243. constructed data. Consider passing ``allow_pickle=False`` to
  244. load data that is known not to contain object arrays for the
  245. safer handling of untrusted sources.
  246. Parameters
  247. ----------
  248. file : file-like object, string, or pathlib.Path
  249. The file to read. File-like objects must support the
  250. ``seek()`` and ``read()`` methods. Pickled files require that the
  251. file-like object support the ``readline()`` method as well.
  252. mmap_mode : {None, 'r+', 'r', 'w+', 'c'}, optional
  253. If not None, then memory-map the file, using the given mode (see
  254. `numpy.memmap` for a detailed description of the modes). A
  255. memory-mapped array is kept on disk. However, it can be accessed
  256. and sliced like any ndarray. Memory mapping is especially useful
  257. for accessing small fragments of large files without reading the
  258. entire file into memory.
  259. allow_pickle : bool, optional
  260. Allow loading pickled object arrays stored in npy files. Reasons for
  261. disallowing pickles include security, as loading pickled data can
  262. execute arbitrary code. If pickles are disallowed, loading object
  263. arrays will fail. Default: False
  264. .. versionchanged:: 1.16.3
  265. Made default False in response to CVE-2019-6446.
  266. fix_imports : bool, optional
  267. Only useful when loading Python 2 generated pickled files on Python 3,
  268. which includes npy/npz files containing object arrays. If `fix_imports`
  269. is True, pickle will try to map the old Python 2 names to the new names
  270. used in Python 3.
  271. encoding : str, optional
  272. What encoding to use when reading Python 2 strings. Only useful when
  273. loading Python 2 generated pickled files in Python 3, which includes
  274. npy/npz files containing object arrays. Values other than 'latin1',
  275. 'ASCII', and 'bytes' are not allowed, as they can corrupt numerical
  276. data. Default: 'ASCII'
  277. Returns
  278. -------
  279. result : array, tuple, dict, etc.
  280. Data stored in the file. For ``.npz`` files, the returned instance
  281. of NpzFile class must be closed to avoid leaking file descriptors.
  282. Raises
  283. ------
  284. IOError
  285. If the input file does not exist or cannot be read.
  286. ValueError
  287. The file contains an object array, but allow_pickle=False given.
  288. See Also
  289. --------
  290. save, savez, savez_compressed, loadtxt
  291. memmap : Create a memory-map to an array stored in a file on disk.
  292. lib.format.open_memmap : Create or load a memory-mapped ``.npy`` file.
  293. Notes
  294. -----
  295. - If the file contains pickle data, then whatever object is stored
  296. in the pickle is returned.
  297. - If the file is a ``.npy`` file, then a single array is returned.
  298. - If the file is a ``.npz`` file, then a dictionary-like object is
  299. returned, containing ``{filename: array}`` key-value pairs, one for
  300. each file in the archive.
  301. - If the file is a ``.npz`` file, the returned value supports the
  302. context manager protocol in a similar fashion to the open function::
  303. with load('foo.npz') as data:
  304. a = data['a']
  305. The underlying file descriptor is closed when exiting the 'with'
  306. block.
  307. Examples
  308. --------
  309. Store data to disk, and load it again:
  310. >>> np.save('/tmp/123', np.array([[1, 2, 3], [4, 5, 6]]))
  311. >>> np.load('/tmp/123.npy')
  312. array([[1, 2, 3],
  313. [4, 5, 6]])
  314. Store compressed data to disk, and load it again:
  315. >>> a=np.array([[1, 2, 3], [4, 5, 6]])
  316. >>> b=np.array([1, 2])
  317. >>> np.savez('/tmp/123.npz', a=a, b=b)
  318. >>> data = np.load('/tmp/123.npz')
  319. >>> data['a']
  320. array([[1, 2, 3],
  321. [4, 5, 6]])
  322. >>> data['b']
  323. array([1, 2])
  324. >>> data.close()
  325. Mem-map the stored array, and then access the second row
  326. directly from disk:
  327. >>> X = np.load('/tmp/123.npy', mmap_mode='r')
  328. >>> X[1, :]
  329. memmap([4, 5, 6])
  330. """
  331. if encoding not in ('ASCII', 'latin1', 'bytes'):
  332. # The 'encoding' value for pickle also affects what encoding
  333. # the serialized binary data of NumPy arrays is loaded
  334. # in. Pickle does not pass on the encoding information to
  335. # NumPy. The unpickling code in numpy.core.multiarray is
  336. # written to assume that unicode data appearing where binary
  337. # should be is in 'latin1'. 'bytes' is also safe, as is 'ASCII'.
  338. #
  339. # Other encoding values can corrupt binary data, and we
  340. # purposefully disallow them. For the same reason, the errors=
  341. # argument is not exposed, as values other than 'strict'
  342. # result can similarly silently corrupt numerical data.
  343. raise ValueError("encoding must be 'ASCII', 'latin1', or 'bytes'")
  344. pickle_kwargs = dict(encoding=encoding, fix_imports=fix_imports)
  345. with contextlib.ExitStack() as stack:
  346. if hasattr(file, 'read'):
  347. fid = file
  348. own_fid = False
  349. else:
  350. fid = stack.enter_context(open(os_fspath(file), "rb"))
  351. own_fid = True
  352. # Code to distinguish from NumPy binary files and pickles.
  353. _ZIP_PREFIX = b'PK\x03\x04'
  354. _ZIP_SUFFIX = b'PK\x05\x06' # empty zip files start with this
  355. N = len(format.MAGIC_PREFIX)
  356. magic = fid.read(N)
  357. # If the file size is less than N, we need to make sure not
  358. # to seek past the beginning of the file
  359. fid.seek(-min(N, len(magic)), 1) # back-up
  360. if magic.startswith(_ZIP_PREFIX) or magic.startswith(_ZIP_SUFFIX):
  361. # zip-file (assume .npz)
  362. # Potentially transfer file ownership to NpzFile
  363. stack.pop_all()
  364. ret = NpzFile(fid, own_fid=own_fid, allow_pickle=allow_pickle,
  365. pickle_kwargs=pickle_kwargs)
  366. return ret
  367. elif magic == format.MAGIC_PREFIX:
  368. # .npy file
  369. if mmap_mode:
  370. return format.open_memmap(file, mode=mmap_mode)
  371. else:
  372. return format.read_array(fid, allow_pickle=allow_pickle,
  373. pickle_kwargs=pickle_kwargs)
  374. else:
  375. # Try a pickle
  376. if not allow_pickle:
  377. raise ValueError("Cannot load file containing pickled data "
  378. "when allow_pickle=False")
  379. try:
  380. return pickle.load(fid, **pickle_kwargs)
  381. except Exception as e:
  382. raise IOError(
  383. "Failed to interpret file %s as a pickle" % repr(file)) from e
  384. def _save_dispatcher(file, arr, allow_pickle=None, fix_imports=None):
  385. return (arr,)
  386. @array_function_dispatch(_save_dispatcher)
  387. def save(file, arr, allow_pickle=True, fix_imports=True):
  388. """
  389. Save an array to a binary file in NumPy ``.npy`` format.
  390. Parameters
  391. ----------
  392. file : file, str, or pathlib.Path
  393. File or filename to which the data is saved. If file is a file-object,
  394. then the filename is unchanged. If file is a string or Path, a ``.npy``
  395. extension will be appended to the filename if it does not already
  396. have one.
  397. arr : array_like
  398. Array data to be saved.
  399. allow_pickle : bool, optional
  400. Allow saving object arrays using Python pickles. Reasons for disallowing
  401. pickles include security (loading pickled data can execute arbitrary
  402. code) and portability (pickled objects may not be loadable on different
  403. Python installations, for example if the stored objects require libraries
  404. that are not available, and not all pickled data is compatible between
  405. Python 2 and Python 3).
  406. Default: True
  407. fix_imports : bool, optional
  408. Only useful in forcing objects in object arrays on Python 3 to be
  409. pickled in a Python 2 compatible way. If `fix_imports` is True, pickle
  410. will try to map the new Python 3 names to the old module names used in
  411. Python 2, so that the pickle data stream is readable with Python 2.
  412. See Also
  413. --------
  414. savez : Save several arrays into a ``.npz`` archive
  415. savetxt, load
  416. Notes
  417. -----
  418. For a description of the ``.npy`` format, see :py:mod:`numpy.lib.format`.
  419. Any data saved to the file is appended to the end of the file.
  420. Examples
  421. --------
  422. >>> from tempfile import TemporaryFile
  423. >>> outfile = TemporaryFile()
  424. >>> x = np.arange(10)
  425. >>> np.save(outfile, x)
  426. >>> _ = outfile.seek(0) # Only needed here to simulate closing & reopening file
  427. >>> np.load(outfile)
  428. array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])
  429. >>> with open('test.npy', 'wb') as f:
  430. ... np.save(f, np.array([1, 2]))
  431. ... np.save(f, np.array([1, 3]))
  432. >>> with open('test.npy', 'rb') as f:
  433. ... a = np.load(f)
  434. ... b = np.load(f)
  435. >>> print(a, b)
  436. # [1 2] [1 3]
  437. """
  438. if hasattr(file, 'write'):
  439. file_ctx = contextlib_nullcontext(file)
  440. else:
  441. file = os_fspath(file)
  442. if not file.endswith('.npy'):
  443. file = file + '.npy'
  444. file_ctx = open(file, "wb")
  445. with file_ctx as fid:
  446. arr = np.asanyarray(arr)
  447. format.write_array(fid, arr, allow_pickle=allow_pickle,
  448. pickle_kwargs=dict(fix_imports=fix_imports))
  449. def _savez_dispatcher(file, *args, **kwds):
  450. yield from args
  451. yield from kwds.values()
  452. @array_function_dispatch(_savez_dispatcher)
  453. def savez(file, *args, **kwds):
  454. """Save several arrays into a single file in uncompressed ``.npz`` format.
  455. If arguments are passed in with no keywords, the corresponding variable
  456. names, in the ``.npz`` file, are 'arr_0', 'arr_1', etc. If keyword
  457. arguments are given, the corresponding variable names, in the ``.npz``
  458. file will match the keyword names.
  459. Parameters
  460. ----------
  461. file : str or file
  462. Either the filename (string) or an open file (file-like object)
  463. where the data will be saved. If file is a string or a Path, the
  464. ``.npz`` extension will be appended to the filename if it is not
  465. already there.
  466. args : Arguments, optional
  467. Arrays to save to the file. Since it is not possible for Python to
  468. know the names of the arrays outside `savez`, the arrays will be saved
  469. with names "arr_0", "arr_1", and so on. These arguments can be any
  470. expression.
  471. kwds : Keyword arguments, optional
  472. Arrays to save to the file. Arrays will be saved in the file with the
  473. keyword names.
  474. Returns
  475. -------
  476. None
  477. See Also
  478. --------
  479. save : Save a single array to a binary file in NumPy format.
  480. savetxt : Save an array to a file as plain text.
  481. savez_compressed : Save several arrays into a compressed ``.npz`` archive
  482. Notes
  483. -----
  484. The ``.npz`` file format is a zipped archive of files named after the
  485. variables they contain. The archive is not compressed and each file
  486. in the archive contains one variable in ``.npy`` format. For a
  487. description of the ``.npy`` format, see :py:mod:`numpy.lib.format`.
  488. When opening the saved ``.npz`` file with `load` a `NpzFile` object is
  489. returned. This is a dictionary-like object which can be queried for
  490. its list of arrays (with the ``.files`` attribute), and for the arrays
  491. themselves.
  492. When saving dictionaries, the dictionary keys become filenames
  493. inside the ZIP archive. Therefore, keys should be valid filenames.
  494. E.g., avoid keys that begin with ``/`` or contain ``.``.
  495. Examples
  496. --------
  497. >>> from tempfile import TemporaryFile
  498. >>> outfile = TemporaryFile()
  499. >>> x = np.arange(10)
  500. >>> y = np.sin(x)
  501. Using `savez` with \\*args, the arrays are saved with default names.
  502. >>> np.savez(outfile, x, y)
  503. >>> _ = outfile.seek(0) # Only needed here to simulate closing & reopening file
  504. >>> npzfile = np.load(outfile)
  505. >>> npzfile.files
  506. ['arr_0', 'arr_1']
  507. >>> npzfile['arr_0']
  508. array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])
  509. Using `savez` with \\**kwds, the arrays are saved with the keyword names.
  510. >>> outfile = TemporaryFile()
  511. >>> np.savez(outfile, x=x, y=y)
  512. >>> _ = outfile.seek(0)
  513. >>> npzfile = np.load(outfile)
  514. >>> sorted(npzfile.files)
  515. ['x', 'y']
  516. >>> npzfile['x']
  517. array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])
  518. """
  519. _savez(file, args, kwds, False)
  520. def _savez_compressed_dispatcher(file, *args, **kwds):
  521. yield from args
  522. yield from kwds.values()
  523. @array_function_dispatch(_savez_compressed_dispatcher)
  524. def savez_compressed(file, *args, **kwds):
  525. """
  526. Save several arrays into a single file in compressed ``.npz`` format.
  527. If keyword arguments are given, then filenames are taken from the keywords.
  528. If arguments are passed in with no keywords, then stored filenames are
  529. arr_0, arr_1, etc.
  530. Parameters
  531. ----------
  532. file : str or file
  533. Either the filename (string) or an open file (file-like object)
  534. where the data will be saved. If file is a string or a Path, the
  535. ``.npz`` extension will be appended to the filename if it is not
  536. already there.
  537. args : Arguments, optional
  538. Arrays to save to the file. Since it is not possible for Python to
  539. know the names of the arrays outside `savez`, the arrays will be saved
  540. with names "arr_0", "arr_1", and so on. These arguments can be any
  541. expression.
  542. kwds : Keyword arguments, optional
  543. Arrays to save to the file. Arrays will be saved in the file with the
  544. keyword names.
  545. Returns
  546. -------
  547. None
  548. See Also
  549. --------
  550. numpy.save : Save a single array to a binary file in NumPy format.
  551. numpy.savetxt : Save an array to a file as plain text.
  552. numpy.savez : Save several arrays into an uncompressed ``.npz`` file format
  553. numpy.load : Load the files created by savez_compressed.
  554. Notes
  555. -----
  556. The ``.npz`` file format is a zipped archive of files named after the
  557. variables they contain. The archive is compressed with
  558. ``zipfile.ZIP_DEFLATED`` and each file in the archive contains one variable
  559. in ``.npy`` format. For a description of the ``.npy`` format, see
  560. :py:mod:`numpy.lib.format`.
  561. When opening the saved ``.npz`` file with `load` a `NpzFile` object is
  562. returned. This is a dictionary-like object which can be queried for
  563. its list of arrays (with the ``.files`` attribute), and for the arrays
  564. themselves.
  565. Examples
  566. --------
  567. >>> test_array = np.random.rand(3, 2)
  568. >>> test_vector = np.random.rand(4)
  569. >>> np.savez_compressed('/tmp/123', a=test_array, b=test_vector)
  570. >>> loaded = np.load('/tmp/123.npz')
  571. >>> print(np.array_equal(test_array, loaded['a']))
  572. True
  573. >>> print(np.array_equal(test_vector, loaded['b']))
  574. True
  575. """
  576. _savez(file, args, kwds, True)
  577. def _savez(file, args, kwds, compress, allow_pickle=True, pickle_kwargs=None):
  578. # Import is postponed to here since zipfile depends on gzip, an optional
  579. # component of the so-called standard library.
  580. import zipfile
  581. if not hasattr(file, 'write'):
  582. file = os_fspath(file)
  583. if not file.endswith('.npz'):
  584. file = file + '.npz'
  585. namedict = kwds
  586. for i, val in enumerate(args):
  587. key = 'arr_%d' % i
  588. if key in namedict.keys():
  589. raise ValueError(
  590. "Cannot use un-named variables and keyword %s" % key)
  591. namedict[key] = val
  592. if compress:
  593. compression = zipfile.ZIP_DEFLATED
  594. else:
  595. compression = zipfile.ZIP_STORED
  596. zipf = zipfile_factory(file, mode="w", compression=compression)
  597. for key, val in namedict.items():
  598. fname = key + '.npy'
  599. val = np.asanyarray(val)
  600. # always force zip64, gh-10776
  601. with zipf.open(fname, 'w', force_zip64=True) as fid:
  602. format.write_array(fid, val,
  603. allow_pickle=allow_pickle,
  604. pickle_kwargs=pickle_kwargs)
  605. zipf.close()
  606. def _getconv(dtype):
  607. """ Find the correct dtype converter. Adapted from matplotlib """
  608. def floatconv(x):
  609. x.lower()
  610. if '0x' in x:
  611. return float.fromhex(x)
  612. return float(x)
  613. typ = dtype.type
  614. if issubclass(typ, np.bool_):
  615. return lambda x: bool(int(x))
  616. if issubclass(typ, np.uint64):
  617. return np.uint64
  618. if issubclass(typ, np.int64):
  619. return np.int64
  620. if issubclass(typ, np.integer):
  621. return lambda x: int(float(x))
  622. elif issubclass(typ, np.longdouble):
  623. return np.longdouble
  624. elif issubclass(typ, np.floating):
  625. return floatconv
  626. elif issubclass(typ, complex):
  627. return lambda x: complex(asstr(x).replace('+-', '-'))
  628. elif issubclass(typ, np.bytes_):
  629. return asbytes
  630. elif issubclass(typ, np.unicode_):
  631. return asunicode
  632. else:
  633. return asstr
  634. # amount of lines loadtxt reads in one chunk, can be overridden for testing
  635. _loadtxt_chunksize = 50000
  636. def _loadtxt_dispatcher(fname, dtype=None, comments=None, delimiter=None,
  637. converters=None, skiprows=None, usecols=None, unpack=None,
  638. ndmin=None, encoding=None, max_rows=None, *, like=None):
  639. return (like,)
  640. @set_array_function_like_doc
  641. @set_module('numpy')
  642. def loadtxt(fname, dtype=float, comments='#', delimiter=None,
  643. converters=None, skiprows=0, usecols=None, unpack=False,
  644. ndmin=0, encoding='bytes', max_rows=None, *, like=None):
  645. r"""
  646. Load data from a text file.
  647. Each row in the text file must have the same number of values.
  648. Parameters
  649. ----------
  650. fname : file, str, or pathlib.Path
  651. File, filename, or generator to read. If the filename extension is
  652. ``.gz`` or ``.bz2``, the file is first decompressed. Note that
  653. generators should return byte strings.
  654. dtype : data-type, optional
  655. Data-type of the resulting array; default: float. If this is a
  656. structured data-type, the resulting array will be 1-dimensional, and
  657. each row will be interpreted as an element of the array. In this
  658. case, the number of columns used must match the number of fields in
  659. the data-type.
  660. comments : str or sequence of str, optional
  661. The characters or list of characters used to indicate the start of a
  662. comment. None implies no comments. For backwards compatibility, byte
  663. strings will be decoded as 'latin1'. The default is '#'.
  664. delimiter : str, optional
  665. The string used to separate values. For backwards compatibility, byte
  666. strings will be decoded as 'latin1'. The default is whitespace.
  667. converters : dict, optional
  668. A dictionary mapping column number to a function that will parse the
  669. column string into the desired value. E.g., if column 0 is a date
  670. string: ``converters = {0: datestr2num}``. Converters can also be
  671. used to provide a default value for missing data (but see also
  672. `genfromtxt`): ``converters = {3: lambda s: float(s.strip() or 0)}``.
  673. Default: None.
  674. skiprows : int, optional
  675. Skip the first `skiprows` lines, including comments; default: 0.
  676. usecols : int or sequence, optional
  677. Which columns to read, with 0 being the first. For example,
  678. ``usecols = (1,4,5)`` will extract the 2nd, 5th and 6th columns.
  679. The default, None, results in all columns being read.
  680. .. versionchanged:: 1.11.0
  681. When a single column has to be read it is possible to use
  682. an integer instead of a tuple. E.g ``usecols = 3`` reads the
  683. fourth column the same way as ``usecols = (3,)`` would.
  684. unpack : bool, optional
  685. If True, the returned array is transposed, so that arguments may be
  686. unpacked using ``x, y, z = loadtxt(...)``. When used with a
  687. structured data-type, arrays are returned for each field.
  688. Default is False.
  689. ndmin : int, optional
  690. The returned array will have at least `ndmin` dimensions.
  691. Otherwise mono-dimensional axes will be squeezed.
  692. Legal values: 0 (default), 1 or 2.
  693. .. versionadded:: 1.6.0
  694. encoding : str, optional
  695. Encoding used to decode the inputfile. Does not apply to input streams.
  696. The special value 'bytes' enables backward compatibility workarounds
  697. that ensures you receive byte arrays as results if possible and passes
  698. 'latin1' encoded strings to converters. Override this value to receive
  699. unicode arrays and pass strings as input to converters. If set to None
  700. the system default is used. The default value is 'bytes'.
  701. .. versionadded:: 1.14.0
  702. max_rows : int, optional
  703. Read `max_rows` lines of content after `skiprows` lines. The default
  704. is to read all the lines.
  705. .. versionadded:: 1.16.0
  706. ${ARRAY_FUNCTION_LIKE}
  707. .. versionadded:: 1.20.0
  708. Returns
  709. -------
  710. out : ndarray
  711. Data read from the text file.
  712. See Also
  713. --------
  714. load, fromstring, fromregex
  715. genfromtxt : Load data with missing values handled as specified.
  716. scipy.io.loadmat : reads MATLAB data files
  717. Notes
  718. -----
  719. This function aims to be a fast reader for simply formatted files. The
  720. `genfromtxt` function provides more sophisticated handling of, e.g.,
  721. lines with missing values.
  722. .. versionadded:: 1.10.0
  723. The strings produced by the Python float.hex method can be used as
  724. input for floats.
  725. Examples
  726. --------
  727. >>> from io import StringIO # StringIO behaves like a file object
  728. >>> c = StringIO("0 1\n2 3")
  729. >>> np.loadtxt(c)
  730. array([[0., 1.],
  731. [2., 3.]])
  732. >>> d = StringIO("M 21 72\nF 35 58")
  733. >>> np.loadtxt(d, dtype={'names': ('gender', 'age', 'weight'),
  734. ... 'formats': ('S1', 'i4', 'f4')})
  735. array([(b'M', 21, 72.), (b'F', 35, 58.)],
  736. dtype=[('gender', 'S1'), ('age', '<i4'), ('weight', '<f4')])
  737. >>> c = StringIO("1,0,2\n3,0,4")
  738. >>> x, y = np.loadtxt(c, delimiter=',', usecols=(0, 2), unpack=True)
  739. >>> x
  740. array([1., 3.])
  741. >>> y
  742. array([2., 4.])
  743. This example shows how `converters` can be used to convert a field
  744. with a trailing minus sign into a negative number.
  745. >>> s = StringIO('10.01 31.25-\n19.22 64.31\n17.57- 63.94')
  746. >>> def conv(fld):
  747. ... return -float(fld[:-1]) if fld.endswith(b'-') else float(fld)
  748. ...
  749. >>> np.loadtxt(s, converters={0: conv, 1: conv})
  750. array([[ 10.01, -31.25],
  751. [ 19.22, 64.31],
  752. [-17.57, 63.94]])
  753. """
  754. if like is not None:
  755. return _loadtxt_with_like(
  756. fname, dtype=dtype, comments=comments, delimiter=delimiter,
  757. converters=converters, skiprows=skiprows, usecols=usecols,
  758. unpack=unpack, ndmin=ndmin, encoding=encoding,
  759. max_rows=max_rows, like=like
  760. )
  761. # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
  762. # Nested functions used by loadtxt.
  763. # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
  764. # not to be confused with the flatten_dtype we import...
  765. @recursive
  766. def flatten_dtype_internal(self, dt):
  767. """Unpack a structured data-type, and produce re-packing info."""
  768. if dt.names is None:
  769. # If the dtype is flattened, return.
  770. # If the dtype has a shape, the dtype occurs
  771. # in the list more than once.
  772. shape = dt.shape
  773. if len(shape) == 0:
  774. return ([dt.base], None)
  775. else:
  776. packing = [(shape[-1], list)]
  777. if len(shape) > 1:
  778. for dim in dt.shape[-2::-1]:
  779. packing = [(dim*packing[0][0], packing*dim)]
  780. return ([dt.base] * int(np.prod(dt.shape)), packing)
  781. else:
  782. types = []
  783. packing = []
  784. for field in dt.names:
  785. tp, bytes = dt.fields[field]
  786. flat_dt, flat_packing = self(tp)
  787. types.extend(flat_dt)
  788. # Avoid extra nesting for subarrays
  789. if tp.ndim > 0:
  790. packing.extend(flat_packing)
  791. else:
  792. packing.append((len(flat_dt), flat_packing))
  793. return (types, packing)
  794. @recursive
  795. def pack_items(self, items, packing):
  796. """Pack items into nested lists based on re-packing info."""
  797. if packing is None:
  798. return items[0]
  799. elif packing is tuple:
  800. return tuple(items)
  801. elif packing is list:
  802. return list(items)
  803. else:
  804. start = 0
  805. ret = []
  806. for length, subpacking in packing:
  807. ret.append(self(items[start:start+length], subpacking))
  808. start += length
  809. return tuple(ret)
  810. def split_line(line):
  811. """Chop off comments, strip, and split at delimiter. """
  812. line = _decode_line(line, encoding=encoding)
  813. if comments is not None:
  814. line = regex_comments.split(line, maxsplit=1)[0]
  815. line = line.strip('\r\n')
  816. return line.split(delimiter) if line else []
  817. def read_data(chunk_size):
  818. """Parse each line, including the first.
  819. The file read, `fh`, is a global defined above.
  820. Parameters
  821. ----------
  822. chunk_size : int
  823. At most `chunk_size` lines are read at a time, with iteration
  824. until all lines are read.
  825. """
  826. X = []
  827. line_iter = itertools.chain([first_line], fh)
  828. line_iter = itertools.islice(line_iter, max_rows)
  829. for i, line in enumerate(line_iter):
  830. vals = split_line(line)
  831. if len(vals) == 0:
  832. continue
  833. if usecols:
  834. vals = [vals[j] for j in usecols]
  835. if len(vals) != N:
  836. line_num = i + skiprows + 1
  837. raise ValueError("Wrong number of columns at line %d"
  838. % line_num)
  839. # Convert each value according to its column and store
  840. items = [conv(val) for (conv, val) in zip(converters, vals)]
  841. # Then pack it according to the dtype's nesting
  842. items = pack_items(items, packing)
  843. X.append(items)
  844. if len(X) > chunk_size:
  845. yield X
  846. X = []
  847. if X:
  848. yield X
  849. # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
  850. # Main body of loadtxt.
  851. # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
  852. # Check correctness of the values of `ndmin`
  853. if ndmin not in [0, 1, 2]:
  854. raise ValueError('Illegal value of ndmin keyword: %s' % ndmin)
  855. # Type conversions for Py3 convenience
  856. if comments is not None:
  857. if isinstance(comments, (str, bytes)):
  858. comments = [comments]
  859. comments = [_decode_line(x) for x in comments]
  860. # Compile regex for comments beforehand
  861. comments = (re.escape(comment) for comment in comments)
  862. regex_comments = re.compile('|'.join(comments))
  863. if delimiter is not None:
  864. delimiter = _decode_line(delimiter)
  865. user_converters = converters
  866. byte_converters = False
  867. if encoding == 'bytes':
  868. encoding = None
  869. byte_converters = True
  870. if usecols is not None:
  871. # Allow usecols to be a single int or a sequence of ints
  872. try:
  873. usecols_as_list = list(usecols)
  874. except TypeError:
  875. usecols_as_list = [usecols]
  876. for col_idx in usecols_as_list:
  877. try:
  878. opindex(col_idx)
  879. except TypeError as e:
  880. e.args = (
  881. "usecols must be an int or a sequence of ints but "
  882. "it contains at least one element of type %s" %
  883. type(col_idx),
  884. )
  885. raise
  886. # Fall back to existing code
  887. usecols = usecols_as_list
  888. # Make sure we're dealing with a proper dtype
  889. dtype = np.dtype(dtype)
  890. defconv = _getconv(dtype)
  891. dtype_types, packing = flatten_dtype_internal(dtype)
  892. fown = False
  893. try:
  894. if isinstance(fname, os_PathLike):
  895. fname = os_fspath(fname)
  896. if _is_string_like(fname):
  897. fh = np.lib._datasource.open(fname, 'rt', encoding=encoding)
  898. fencoding = getattr(fh, 'encoding', 'latin1')
  899. fh = iter(fh)
  900. fown = True
  901. else:
  902. fh = iter(fname)
  903. fencoding = getattr(fname, 'encoding', 'latin1')
  904. except TypeError as e:
  905. raise ValueError(
  906. 'fname must be a string, file handle, or generator'
  907. ) from e
  908. # input may be a python2 io stream
  909. if encoding is not None:
  910. fencoding = encoding
  911. # we must assume local encoding
  912. # TODO emit portability warning?
  913. elif fencoding is None:
  914. import locale
  915. fencoding = locale.getpreferredencoding()
  916. try:
  917. # Skip the first `skiprows` lines
  918. for i in range(skiprows):
  919. next(fh)
  920. # Read until we find a line with some values, and use
  921. # it to estimate the number of columns, N.
  922. first_vals = None
  923. try:
  924. while not first_vals:
  925. first_line = next(fh)
  926. first_vals = split_line(first_line)
  927. except StopIteration:
  928. # End of lines reached
  929. first_line = ''
  930. first_vals = []
  931. warnings.warn('loadtxt: Empty input file: "%s"' % fname,
  932. stacklevel=2)
  933. N = len(usecols or first_vals)
  934. # Now that we know N, create the default converters list, and
  935. # set packing, if necessary.
  936. if len(dtype_types) > 1:
  937. # We're dealing with a structured array, each field of
  938. # the dtype matches a column
  939. converters = [_getconv(dt) for dt in dtype_types]
  940. else:
  941. # All fields have the same dtype
  942. converters = [defconv for i in range(N)]
  943. if N > 1:
  944. packing = [(N, tuple)]
  945. # By preference, use the converters specified by the user
  946. for i, conv in (user_converters or {}).items():
  947. if usecols:
  948. try:
  949. i = usecols.index(i)
  950. except ValueError:
  951. # Unused converter specified
  952. continue
  953. if byte_converters:
  954. # converters may use decode to workaround numpy's old
  955. # behaviour, so encode the string again before passing to
  956. # the user converter
  957. def tobytes_first(x, conv):
  958. if type(x) is bytes:
  959. return conv(x)
  960. return conv(x.encode("latin1"))
  961. converters[i] = functools.partial(tobytes_first, conv=conv)
  962. else:
  963. converters[i] = conv
  964. converters = [conv if conv is not bytes else
  965. lambda x: x.encode(fencoding) for conv in converters]
  966. # read data in chunks and fill it into an array via resize
  967. # over-allocating and shrinking the array later may be faster but is
  968. # probably not relevant compared to the cost of actually reading and
  969. # converting the data
  970. X = None
  971. for x in read_data(_loadtxt_chunksize):
  972. if X is None:
  973. X = np.array(x, dtype)
  974. else:
  975. nshape = list(X.shape)
  976. pos = nshape[0]
  977. nshape[0] += len(x)
  978. X.resize(nshape, refcheck=False)
  979. X[pos:, ...] = x
  980. finally:
  981. if fown:
  982. fh.close()
  983. if X is None:
  984. X = np.array([], dtype)
  985. # Multicolumn data are returned with shape (1, N, M), i.e.
  986. # (1, 1, M) for a single row - remove the singleton dimension there
  987. if X.ndim == 3 and X.shape[:2] == (1, 1):
  988. X.shape = (1, -1)
  989. # Verify that the array has at least dimensions `ndmin`.
  990. # Tweak the size and shape of the arrays - remove extraneous dimensions
  991. if X.ndim > ndmin:
  992. X = np.squeeze(X)
  993. # and ensure we have the minimum number of dimensions asked for
  994. # - has to be in this order for the odd case ndmin=1, X.squeeze().ndim=0
  995. if X.ndim < ndmin:
  996. if ndmin == 1:
  997. X = np.atleast_1d(X)
  998. elif ndmin == 2:
  999. X = np.atleast_2d(X).T
  1000. if unpack:
  1001. if len(dtype_types) > 1:
  1002. # For structured arrays, return an array for each field.
  1003. return [X[field] for field in dtype.names]
  1004. else:
  1005. return X.T
  1006. else:
  1007. return X
  1008. _loadtxt_with_like = array_function_dispatch(
  1009. _loadtxt_dispatcher
  1010. )(loadtxt)
  1011. def _savetxt_dispatcher(fname, X, fmt=None, delimiter=None, newline=None,
  1012. header=None, footer=None, comments=None,
  1013. encoding=None):
  1014. return (X,)
  1015. @array_function_dispatch(_savetxt_dispatcher)
  1016. def savetxt(fname, X, fmt='%.18e', delimiter=' ', newline='\n', header='',
  1017. footer='', comments='# ', encoding=None):
  1018. """
  1019. Save an array to a text file.
  1020. Parameters
  1021. ----------
  1022. fname : filename or file handle
  1023. If the filename ends in ``.gz``, the file is automatically saved in
  1024. compressed gzip format. `loadtxt` understands gzipped files
  1025. transparently.
  1026. X : 1D or 2D array_like
  1027. Data to be saved to a text file.
  1028. fmt : str or sequence of strs, optional
  1029. A single format (%10.5f), a sequence of formats, or a
  1030. multi-format string, e.g. 'Iteration %d -- %10.5f', in which
  1031. case `delimiter` is ignored. For complex `X`, the legal options
  1032. for `fmt` are:
  1033. * a single specifier, `fmt='%.4e'`, resulting in numbers formatted
  1034. like `' (%s+%sj)' % (fmt, fmt)`
  1035. * a full string specifying every real and imaginary part, e.g.
  1036. `' %.4e %+.4ej %.4e %+.4ej %.4e %+.4ej'` for 3 columns
  1037. * a list of specifiers, one per column - in this case, the real
  1038. and imaginary part must have separate specifiers,
  1039. e.g. `['%.3e + %.3ej', '(%.15e%+.15ej)']` for 2 columns
  1040. delimiter : str, optional
  1041. String or character separating columns.
  1042. newline : str, optional
  1043. String or character separating lines.
  1044. .. versionadded:: 1.5.0
  1045. header : str, optional
  1046. String that will be written at the beginning of the file.
  1047. .. versionadded:: 1.7.0
  1048. footer : str, optional
  1049. String that will be written at the end of the file.
  1050. .. versionadded:: 1.7.0
  1051. comments : str, optional
  1052. String that will be prepended to the ``header`` and ``footer`` strings,
  1053. to mark them as comments. Default: '# ', as expected by e.g.
  1054. ``numpy.loadtxt``.
  1055. .. versionadded:: 1.7.0
  1056. encoding : {None, str}, optional
  1057. Encoding used to encode the outputfile. Does not apply to output
  1058. streams. If the encoding is something other than 'bytes' or 'latin1'
  1059. you will not be able to load the file in NumPy versions < 1.14. Default
  1060. is 'latin1'.
  1061. .. versionadded:: 1.14.0
  1062. See Also
  1063. --------
  1064. save : Save an array to a binary file in NumPy ``.npy`` format
  1065. savez : Save several arrays into an uncompressed ``.npz`` archive
  1066. savez_compressed : Save several arrays into a compressed ``.npz`` archive
  1067. Notes
  1068. -----
  1069. Further explanation of the `fmt` parameter
  1070. (``%[flag]width[.precision]specifier``):
  1071. flags:
  1072. ``-`` : left justify
  1073. ``+`` : Forces to precede result with + or -.
  1074. ``0`` : Left pad the number with zeros instead of space (see width).
  1075. width:
  1076. Minimum number of characters to be printed. The value is not truncated
  1077. if it has more characters.
  1078. precision:
  1079. - For integer specifiers (eg. ``d,i,o,x``), the minimum number of
  1080. digits.
  1081. - For ``e, E`` and ``f`` specifiers, the number of digits to print
  1082. after the decimal point.
  1083. - For ``g`` and ``G``, the maximum number of significant digits.
  1084. - For ``s``, the maximum number of characters.
  1085. specifiers:
  1086. ``c`` : character
  1087. ``d`` or ``i`` : signed decimal integer
  1088. ``e`` or ``E`` : scientific notation with ``e`` or ``E``.
  1089. ``f`` : decimal floating point
  1090. ``g,G`` : use the shorter of ``e,E`` or ``f``
  1091. ``o`` : signed octal
  1092. ``s`` : string of characters
  1093. ``u`` : unsigned decimal integer
  1094. ``x,X`` : unsigned hexadecimal integer
  1095. This explanation of ``fmt`` is not complete, for an exhaustive
  1096. specification see [1]_.
  1097. References
  1098. ----------
  1099. .. [1] `Format Specification Mini-Language
  1100. <https://docs.python.org/library/string.html#format-specification-mini-language>`_,
  1101. Python Documentation.
  1102. Examples
  1103. --------
  1104. >>> x = y = z = np.arange(0.0,5.0,1.0)
  1105. >>> np.savetxt('test.out', x, delimiter=',') # X is an array
  1106. >>> np.savetxt('test.out', (x,y,z)) # x,y,z equal sized 1D arrays
  1107. >>> np.savetxt('test.out', x, fmt='%1.4e') # use exponential notation
  1108. """
  1109. # Py3 conversions first
  1110. if isinstance(fmt, bytes):
  1111. fmt = asstr(fmt)
  1112. delimiter = asstr(delimiter)
  1113. class WriteWrap:
  1114. """Convert to bytes on bytestream inputs.
  1115. """
  1116. def __init__(self, fh, encoding):
  1117. self.fh = fh
  1118. self.encoding = encoding
  1119. self.do_write = self.first_write
  1120. def close(self):
  1121. self.fh.close()
  1122. def write(self, v):
  1123. self.do_write(v)
  1124. def write_bytes(self, v):
  1125. if isinstance(v, bytes):
  1126. self.fh.write(v)
  1127. else:
  1128. self.fh.write(v.encode(self.encoding))
  1129. def write_normal(self, v):
  1130. self.fh.write(asunicode(v))
  1131. def first_write(self, v):
  1132. try:
  1133. self.write_normal(v)
  1134. self.write = self.write_normal
  1135. except TypeError:
  1136. # input is probably a bytestream
  1137. self.write_bytes(v)
  1138. self.write = self.write_bytes
  1139. own_fh = False
  1140. if isinstance(fname, os_PathLike):
  1141. fname = os_fspath(fname)
  1142. if _is_string_like(fname):
  1143. # datasource doesn't support creating a new file ...
  1144. open(fname, 'wt').close()
  1145. fh = np.lib._datasource.open(fname, 'wt', encoding=encoding)
  1146. own_fh = True
  1147. elif hasattr(fname, 'write'):
  1148. # wrap to handle byte output streams
  1149. fh = WriteWrap(fname, encoding or 'latin1')
  1150. else:
  1151. raise ValueError('fname must be a string or file handle')
  1152. try:
  1153. X = np.asarray(X)
  1154. # Handle 1-dimensional arrays
  1155. if X.ndim == 0 or X.ndim > 2:
  1156. raise ValueError(
  1157. "Expected 1D or 2D array, got %dD array instead" % X.ndim)
  1158. elif X.ndim == 1:
  1159. # Common case -- 1d array of numbers
  1160. if X.dtype.names is None:
  1161. X = np.atleast_2d(X).T
  1162. ncol = 1
  1163. # Complex dtype -- each field indicates a separate column
  1164. else:
  1165. ncol = len(X.dtype.names)
  1166. else:
  1167. ncol = X.shape[1]
  1168. iscomplex_X = np.iscomplexobj(X)
  1169. # `fmt` can be a string with multiple insertion points or a
  1170. # list of formats. E.g. '%10.5f\t%10d' or ('%10.5f', '$10d')
  1171. if type(fmt) in (list, tuple):
  1172. if len(fmt) != ncol:
  1173. raise AttributeError('fmt has wrong shape. %s' % str(fmt))
  1174. format = asstr(delimiter).join(map(asstr, fmt))
  1175. elif isinstance(fmt, str):
  1176. n_fmt_chars = fmt.count('%')
  1177. error = ValueError('fmt has wrong number of %% formats: %s' % fmt)
  1178. if n_fmt_chars == 1:
  1179. if iscomplex_X:
  1180. fmt = [' (%s+%sj)' % (fmt, fmt), ] * ncol
  1181. else:
  1182. fmt = [fmt, ] * ncol
  1183. format = delimiter.join(fmt)
  1184. elif iscomplex_X and n_fmt_chars != (2 * ncol):
  1185. raise error
  1186. elif ((not iscomplex_X) and n_fmt_chars != ncol):
  1187. raise error
  1188. else:
  1189. format = fmt
  1190. else:
  1191. raise ValueError('invalid fmt: %r' % (fmt,))
  1192. if len(header) > 0:
  1193. header = header.replace('\n', '\n' + comments)
  1194. fh.write(comments + header + newline)
  1195. if iscomplex_X:
  1196. for row in X:
  1197. row2 = []
  1198. for number in row:
  1199. row2.append(number.real)
  1200. row2.append(number.imag)
  1201. s = format % tuple(row2) + newline
  1202. fh.write(s.replace('+-', '-'))
  1203. else:
  1204. for row in X:
  1205. try:
  1206. v = format % tuple(row) + newline
  1207. except TypeError as e:
  1208. raise TypeError("Mismatch between array dtype ('%s') and "
  1209. "format specifier ('%s')"
  1210. % (str(X.dtype), format)) from e
  1211. fh.write(v)
  1212. if len(footer) > 0:
  1213. footer = footer.replace('\n', '\n' + comments)
  1214. fh.write(comments + footer + newline)
  1215. finally:
  1216. if own_fh:
  1217. fh.close()
  1218. @set_module('numpy')
  1219. def fromregex(file, regexp, dtype, encoding=None):
  1220. """
  1221. Construct an array from a text file, using regular expression parsing.
  1222. The returned array is always a structured array, and is constructed from
  1223. all matches of the regular expression in the file. Groups in the regular
  1224. expression are converted to fields of the structured array.
  1225. Parameters
  1226. ----------
  1227. file : str or file
  1228. Filename or file object to read.
  1229. regexp : str or regexp
  1230. Regular expression used to parse the file.
  1231. Groups in the regular expression correspond to fields in the dtype.
  1232. dtype : dtype or list of dtypes
  1233. Dtype for the structured array.
  1234. encoding : str, optional
  1235. Encoding used to decode the inputfile. Does not apply to input streams.
  1236. .. versionadded:: 1.14.0
  1237. Returns
  1238. -------
  1239. output : ndarray
  1240. The output array, containing the part of the content of `file` that
  1241. was matched by `regexp`. `output` is always a structured array.
  1242. Raises
  1243. ------
  1244. TypeError
  1245. When `dtype` is not a valid dtype for a structured array.
  1246. See Also
  1247. --------
  1248. fromstring, loadtxt
  1249. Notes
  1250. -----
  1251. Dtypes for structured arrays can be specified in several forms, but all
  1252. forms specify at least the data type and field name. For details see
  1253. `basics.rec`.
  1254. Examples
  1255. --------
  1256. >>> f = open('test.dat', 'w')
  1257. >>> _ = f.write("1312 foo\\n1534 bar\\n444 qux")
  1258. >>> f.close()
  1259. >>> regexp = r"(\\d+)\\s+(...)" # match [digits, whitespace, anything]
  1260. >>> output = np.fromregex('test.dat', regexp,
  1261. ... [('num', np.int64), ('key', 'S3')])
  1262. >>> output
  1263. array([(1312, b'foo'), (1534, b'bar'), ( 444, b'qux')],
  1264. dtype=[('num', '<i8'), ('key', 'S3')])
  1265. >>> output['num']
  1266. array([1312, 1534, 444])
  1267. """
  1268. own_fh = False
  1269. if not hasattr(file, "read"):
  1270. file = np.lib._datasource.open(file, 'rt', encoding=encoding)
  1271. own_fh = True
  1272. try:
  1273. if not isinstance(dtype, np.dtype):
  1274. dtype = np.dtype(dtype)
  1275. content = file.read()
  1276. if isinstance(content, bytes) and isinstance(regexp, np.compat.unicode):
  1277. regexp = asbytes(regexp)
  1278. elif isinstance(content, np.compat.unicode) and isinstance(regexp, bytes):
  1279. regexp = asstr(regexp)
  1280. if not hasattr(regexp, 'match'):
  1281. regexp = re.compile(regexp)
  1282. seq = regexp.findall(content)
  1283. if seq and not isinstance(seq[0], tuple):
  1284. # Only one group is in the regexp.
  1285. # Create the new array as a single data-type and then
  1286. # re-interpret as a single-field structured array.
  1287. newdtype = np.dtype(dtype[dtype.names[0]])
  1288. output = np.array(seq, dtype=newdtype)
  1289. output.dtype = dtype
  1290. else:
  1291. output = np.array(seq, dtype=dtype)
  1292. return output
  1293. finally:
  1294. if own_fh:
  1295. file.close()
  1296. #####--------------------------------------------------------------------------
  1297. #---- --- ASCII functions ---
  1298. #####--------------------------------------------------------------------------
  1299. def _genfromtxt_dispatcher(fname, dtype=None, comments=None, delimiter=None,
  1300. skip_header=None, skip_footer=None, converters=None,
  1301. missing_values=None, filling_values=None, usecols=None,
  1302. names=None, excludelist=None, deletechars=None,
  1303. replace_space=None, autostrip=None, case_sensitive=None,
  1304. defaultfmt=None, unpack=None, usemask=None, loose=None,
  1305. invalid_raise=None, max_rows=None, encoding=None, *,
  1306. like=None):
  1307. return (like,)
  1308. @set_array_function_like_doc
  1309. @set_module('numpy')
  1310. def genfromtxt(fname, dtype=float, comments='#', delimiter=None,
  1311. skip_header=0, skip_footer=0, converters=None,
  1312. missing_values=None, filling_values=None, usecols=None,
  1313. names=None, excludelist=None,
  1314. deletechars=''.join(sorted(NameValidator.defaultdeletechars)),
  1315. replace_space='_', autostrip=False, case_sensitive=True,
  1316. defaultfmt="f%i", unpack=None, usemask=False, loose=True,
  1317. invalid_raise=True, max_rows=None, encoding='bytes', *,
  1318. like=None):
  1319. """
  1320. Load data from a text file, with missing values handled as specified.
  1321. Each line past the first `skip_header` lines is split at the `delimiter`
  1322. character, and characters following the `comments` character are discarded.
  1323. Parameters
  1324. ----------
  1325. fname : file, str, pathlib.Path, list of str, generator
  1326. File, filename, list, or generator to read. If the filename
  1327. extension is `.gz` or `.bz2`, the file is first decompressed. Note
  1328. that generators must return byte strings. The strings
  1329. in a list or produced by a generator are treated as lines.
  1330. dtype : dtype, optional
  1331. Data type of the resulting array.
  1332. If None, the dtypes will be determined by the contents of each
  1333. column, individually.
  1334. comments : str, optional
  1335. The character used to indicate the start of a comment.
  1336. All the characters occurring on a line after a comment are discarded
  1337. delimiter : str, int, or sequence, optional
  1338. The string used to separate values. By default, any consecutive
  1339. whitespaces act as delimiter. An integer or sequence of integers
  1340. can also be provided as width(s) of each field.
  1341. skiprows : int, optional
  1342. `skiprows` was removed in numpy 1.10. Please use `skip_header` instead.
  1343. skip_header : int, optional
  1344. The number of lines to skip at the beginning of the file.
  1345. skip_footer : int, optional
  1346. The number of lines to skip at the end of the file.
  1347. converters : variable, optional
  1348. The set of functions that convert the data of a column to a value.
  1349. The converters can also be used to provide a default value
  1350. for missing data: ``converters = {3: lambda s: float(s or 0)}``.
  1351. missing : variable, optional
  1352. `missing` was removed in numpy 1.10. Please use `missing_values`
  1353. instead.
  1354. missing_values : variable, optional
  1355. The set of strings corresponding to missing data.
  1356. filling_values : variable, optional
  1357. The set of values to be used as default when the data are missing.
  1358. usecols : sequence, optional
  1359. Which columns to read, with 0 being the first. For example,
  1360. ``usecols = (1, 4, 5)`` will extract the 2nd, 5th and 6th columns.
  1361. names : {None, True, str, sequence}, optional
  1362. If `names` is True, the field names are read from the first line after
  1363. the first `skip_header` lines. This line can optionally be proceeded
  1364. by a comment delimiter. If `names` is a sequence or a single-string of
  1365. comma-separated names, the names will be used to define the field names
  1366. in a structured dtype. If `names` is None, the names of the dtype
  1367. fields will be used, if any.
  1368. excludelist : sequence, optional
  1369. A list of names to exclude. This list is appended to the default list
  1370. ['return','file','print']. Excluded names are appended an underscore:
  1371. for example, `file` would become `file_`.
  1372. deletechars : str, optional
  1373. A string combining invalid characters that must be deleted from the
  1374. names.
  1375. defaultfmt : str, optional
  1376. A format used to define default field names, such as "f%i" or "f_%02i".
  1377. autostrip : bool, optional
  1378. Whether to automatically strip white spaces from the variables.
  1379. replace_space : char, optional
  1380. Character(s) used in replacement of white spaces in the variables
  1381. names. By default, use a '_'.
  1382. case_sensitive : {True, False, 'upper', 'lower'}, optional
  1383. If True, field names are case sensitive.
  1384. If False or 'upper', field names are converted to upper case.
  1385. If 'lower', field names are converted to lower case.
  1386. unpack : bool, optional
  1387. If True, the returned array is transposed, so that arguments may be
  1388. unpacked using ``x, y, z = genfromtxt(...)``. When used with a
  1389. structured data-type, arrays are returned for each field.
  1390. Default is False.
  1391. usemask : bool, optional
  1392. If True, return a masked array.
  1393. If False, return a regular array.
  1394. loose : bool, optional
  1395. If True, do not raise errors for invalid values.
  1396. invalid_raise : bool, optional
  1397. If True, an exception is raised if an inconsistency is detected in the
  1398. number of columns.
  1399. If False, a warning is emitted and the offending lines are skipped.
  1400. max_rows : int, optional
  1401. The maximum number of rows to read. Must not be used with skip_footer
  1402. at the same time. If given, the value must be at least 1. Default is
  1403. to read the entire file.
  1404. .. versionadded:: 1.10.0
  1405. encoding : str, optional
  1406. Encoding used to decode the inputfile. Does not apply when `fname` is
  1407. a file object. The special value 'bytes' enables backward compatibility
  1408. workarounds that ensure that you receive byte arrays when possible
  1409. and passes latin1 encoded strings to converters. Override this value to
  1410. receive unicode arrays and pass strings as input to converters. If set
  1411. to None the system default is used. The default value is 'bytes'.
  1412. .. versionadded:: 1.14.0
  1413. ${ARRAY_FUNCTION_LIKE}
  1414. .. versionadded:: 1.20.0
  1415. Returns
  1416. -------
  1417. out : ndarray
  1418. Data read from the text file. If `usemask` is True, this is a
  1419. masked array.
  1420. See Also
  1421. --------
  1422. numpy.loadtxt : equivalent function when no data is missing.
  1423. Notes
  1424. -----
  1425. * When spaces are used as delimiters, or when no delimiter has been given
  1426. as input, there should not be any missing data between two fields.
  1427. * When the variables are named (either by a flexible dtype or with `names`),
  1428. there must not be any header in the file (else a ValueError
  1429. exception is raised).
  1430. * Individual values are not stripped of spaces by default.
  1431. When using a custom converter, make sure the function does remove spaces.
  1432. References
  1433. ----------
  1434. .. [1] NumPy User Guide, section `I/O with NumPy
  1435. <https://docs.scipy.org/doc/numpy/user/basics.io.genfromtxt.html>`_.
  1436. Examples
  1437. --------
  1438. >>> from io import StringIO
  1439. >>> import numpy as np
  1440. Comma delimited file with mixed dtype
  1441. >>> s = StringIO(u"1,1.3,abcde")
  1442. >>> data = np.genfromtxt(s, dtype=[('myint','i8'),('myfloat','f8'),
  1443. ... ('mystring','S5')], delimiter=",")
  1444. >>> data
  1445. array((1, 1.3, b'abcde'),
  1446. dtype=[('myint', '<i8'), ('myfloat', '<f8'), ('mystring', 'S5')])
  1447. Using dtype = None
  1448. >>> _ = s.seek(0) # needed for StringIO example only
  1449. >>> data = np.genfromtxt(s, dtype=None,
  1450. ... names = ['myint','myfloat','mystring'], delimiter=",")
  1451. >>> data
  1452. array((1, 1.3, b'abcde'),
  1453. dtype=[('myint', '<i8'), ('myfloat', '<f8'), ('mystring', 'S5')])
  1454. Specifying dtype and names
  1455. >>> _ = s.seek(0)
  1456. >>> data = np.genfromtxt(s, dtype="i8,f8,S5",
  1457. ... names=['myint','myfloat','mystring'], delimiter=",")
  1458. >>> data
  1459. array((1, 1.3, b'abcde'),
  1460. dtype=[('myint', '<i8'), ('myfloat', '<f8'), ('mystring', 'S5')])
  1461. An example with fixed-width columns
  1462. >>> s = StringIO(u"11.3abcde")
  1463. >>> data = np.genfromtxt(s, dtype=None, names=['intvar','fltvar','strvar'],
  1464. ... delimiter=[1,3,5])
  1465. >>> data
  1466. array((1, 1.3, b'abcde'),
  1467. dtype=[('intvar', '<i8'), ('fltvar', '<f8'), ('strvar', 'S5')])
  1468. An example to show comments
  1469. >>> f = StringIO('''
  1470. ... text,# of chars
  1471. ... hello world,11
  1472. ... numpy,5''')
  1473. >>> np.genfromtxt(f, dtype='S12,S12', delimiter=',')
  1474. array([(b'text', b''), (b'hello world', b'11'), (b'numpy', b'5')],
  1475. dtype=[('f0', 'S12'), ('f1', 'S12')])
  1476. """
  1477. if like is not None:
  1478. return _genfromtxt_with_like(
  1479. fname, dtype=dtype, comments=comments, delimiter=delimiter,
  1480. skip_header=skip_header, skip_footer=skip_footer,
  1481. converters=converters, missing_values=missing_values,
  1482. filling_values=filling_values, usecols=usecols, names=names,
  1483. excludelist=excludelist, deletechars=deletechars,
  1484. replace_space=replace_space, autostrip=autostrip,
  1485. case_sensitive=case_sensitive, defaultfmt=defaultfmt,
  1486. unpack=unpack, usemask=usemask, loose=loose,
  1487. invalid_raise=invalid_raise, max_rows=max_rows, encoding=encoding,
  1488. like=like
  1489. )
  1490. if max_rows is not None:
  1491. if skip_footer:
  1492. raise ValueError(
  1493. "The keywords 'skip_footer' and 'max_rows' can not be "
  1494. "specified at the same time.")
  1495. if max_rows < 1:
  1496. raise ValueError("'max_rows' must be at least 1.")
  1497. if usemask:
  1498. from numpy.ma import MaskedArray, make_mask_descr
  1499. # Check the input dictionary of converters
  1500. user_converters = converters or {}
  1501. if not isinstance(user_converters, dict):
  1502. raise TypeError(
  1503. "The input argument 'converter' should be a valid dictionary "
  1504. "(got '%s' instead)" % type(user_converters))
  1505. if encoding == 'bytes':
  1506. encoding = None
  1507. byte_converters = True
  1508. else:
  1509. byte_converters = False
  1510. # Initialize the filehandle, the LineSplitter and the NameValidator
  1511. try:
  1512. if isinstance(fname, os_PathLike):
  1513. fname = os_fspath(fname)
  1514. if isinstance(fname, str):
  1515. fid = np.lib._datasource.open(fname, 'rt', encoding=encoding)
  1516. fid_ctx = contextlib.closing(fid)
  1517. else:
  1518. fid = fname
  1519. fid_ctx = contextlib_nullcontext(fid)
  1520. fhd = iter(fid)
  1521. except TypeError as e:
  1522. raise TypeError(
  1523. "fname must be a string, filehandle, list of strings, "
  1524. "or generator. Got %s instead." % type(fname)) from e
  1525. with fid_ctx:
  1526. split_line = LineSplitter(delimiter=delimiter, comments=comments,
  1527. autostrip=autostrip, encoding=encoding)
  1528. validate_names = NameValidator(excludelist=excludelist,
  1529. deletechars=deletechars,
  1530. case_sensitive=case_sensitive,
  1531. replace_space=replace_space)
  1532. # Skip the first `skip_header` rows
  1533. try:
  1534. for i in range(skip_header):
  1535. next(fhd)
  1536. # Keep on until we find the first valid values
  1537. first_values = None
  1538. while not first_values:
  1539. first_line = _decode_line(next(fhd), encoding)
  1540. if (names is True) and (comments is not None):
  1541. if comments in first_line:
  1542. first_line = (
  1543. ''.join(first_line.split(comments)[1:]))
  1544. first_values = split_line(first_line)
  1545. except StopIteration:
  1546. # return an empty array if the datafile is empty
  1547. first_line = ''
  1548. first_values = []
  1549. warnings.warn('genfromtxt: Empty input file: "%s"' % fname, stacklevel=2)
  1550. # Should we take the first values as names ?
  1551. if names is True:
  1552. fval = first_values[0].strip()
  1553. if comments is not None:
  1554. if fval in comments:
  1555. del first_values[0]
  1556. # Check the columns to use: make sure `usecols` is a list
  1557. if usecols is not None:
  1558. try:
  1559. usecols = [_.strip() for _ in usecols.split(",")]
  1560. except AttributeError:
  1561. try:
  1562. usecols = list(usecols)
  1563. except TypeError:
  1564. usecols = [usecols, ]
  1565. nbcols = len(usecols or first_values)
  1566. # Check the names and overwrite the dtype.names if needed
  1567. if names is True:
  1568. names = validate_names([str(_.strip()) for _ in first_values])
  1569. first_line = ''
  1570. elif _is_string_like(names):
  1571. names = validate_names([_.strip() for _ in names.split(',')])
  1572. elif names:
  1573. names = validate_names(names)
  1574. # Get the dtype
  1575. if dtype is not None:
  1576. dtype = easy_dtype(dtype, defaultfmt=defaultfmt, names=names,
  1577. excludelist=excludelist,
  1578. deletechars=deletechars,
  1579. case_sensitive=case_sensitive,
  1580. replace_space=replace_space)
  1581. # Make sure the names is a list (for 2.5)
  1582. if names is not None:
  1583. names = list(names)
  1584. if usecols:
  1585. for (i, current) in enumerate(usecols):
  1586. # if usecols is a list of names, convert to a list of indices
  1587. if _is_string_like(current):
  1588. usecols[i] = names.index(current)
  1589. elif current < 0:
  1590. usecols[i] = current + len(first_values)
  1591. # If the dtype is not None, make sure we update it
  1592. if (dtype is not None) and (len(dtype) > nbcols):
  1593. descr = dtype.descr
  1594. dtype = np.dtype([descr[_] for _ in usecols])
  1595. names = list(dtype.names)
  1596. # If `names` is not None, update the names
  1597. elif (names is not None) and (len(names) > nbcols):
  1598. names = [names[_] for _ in usecols]
  1599. elif (names is not None) and (dtype is not None):
  1600. names = list(dtype.names)
  1601. # Process the missing values ...............................
  1602. # Rename missing_values for convenience
  1603. user_missing_values = missing_values or ()
  1604. if isinstance(user_missing_values, bytes):
  1605. user_missing_values = user_missing_values.decode('latin1')
  1606. # Define the list of missing_values (one column: one list)
  1607. missing_values = [list(['']) for _ in range(nbcols)]
  1608. # We have a dictionary: process it field by field
  1609. if isinstance(user_missing_values, dict):
  1610. # Loop on the items
  1611. for (key, val) in user_missing_values.items():
  1612. # Is the key a string ?
  1613. if _is_string_like(key):
  1614. try:
  1615. # Transform it into an integer
  1616. key = names.index(key)
  1617. except ValueError:
  1618. # We couldn't find it: the name must have been dropped
  1619. continue
  1620. # Redefine the key as needed if it's a column number
  1621. if usecols:
  1622. try:
  1623. key = usecols.index(key)
  1624. except ValueError:
  1625. pass
  1626. # Transform the value as a list of string
  1627. if isinstance(val, (list, tuple)):
  1628. val = [str(_) for _ in val]
  1629. else:
  1630. val = [str(val), ]
  1631. # Add the value(s) to the current list of missing
  1632. if key is None:
  1633. # None acts as default
  1634. for miss in missing_values:
  1635. miss.extend(val)
  1636. else:
  1637. missing_values[key].extend(val)
  1638. # We have a sequence : each item matches a column
  1639. elif isinstance(user_missing_values, (list, tuple)):
  1640. for (value, entry) in zip(user_missing_values, missing_values):
  1641. value = str(value)
  1642. if value not in entry:
  1643. entry.append(value)
  1644. # We have a string : apply it to all entries
  1645. elif isinstance(user_missing_values, str):
  1646. user_value = user_missing_values.split(",")
  1647. for entry in missing_values:
  1648. entry.extend(user_value)
  1649. # We have something else: apply it to all entries
  1650. else:
  1651. for entry in missing_values:
  1652. entry.extend([str(user_missing_values)])
  1653. # Process the filling_values ...............................
  1654. # Rename the input for convenience
  1655. user_filling_values = filling_values
  1656. if user_filling_values is None:
  1657. user_filling_values = []
  1658. # Define the default
  1659. filling_values = [None] * nbcols
  1660. # We have a dictionary : update each entry individually
  1661. if isinstance(user_filling_values, dict):
  1662. for (key, val) in user_filling_values.items():
  1663. if _is_string_like(key):
  1664. try:
  1665. # Transform it into an integer
  1666. key = names.index(key)
  1667. except ValueError:
  1668. # We couldn't find it: the name must have been dropped,
  1669. continue
  1670. # Redefine the key if it's a column number and usecols is defined
  1671. if usecols:
  1672. try:
  1673. key = usecols.index(key)
  1674. except ValueError:
  1675. pass
  1676. # Add the value to the list
  1677. filling_values[key] = val
  1678. # We have a sequence : update on a one-to-one basis
  1679. elif isinstance(user_filling_values, (list, tuple)):
  1680. n = len(user_filling_values)
  1681. if (n <= nbcols):
  1682. filling_values[:n] = user_filling_values
  1683. else:
  1684. filling_values = user_filling_values[:nbcols]
  1685. # We have something else : use it for all entries
  1686. else:
  1687. filling_values = [user_filling_values] * nbcols
  1688. # Initialize the converters ................................
  1689. if dtype is None:
  1690. # Note: we can't use a [...]*nbcols, as we would have 3 times the same
  1691. # ... converter, instead of 3 different converters.
  1692. converters = [StringConverter(None, missing_values=miss, default=fill)
  1693. for (miss, fill) in zip(missing_values, filling_values)]
  1694. else:
  1695. dtype_flat = flatten_dtype(dtype, flatten_base=True)
  1696. # Initialize the converters
  1697. if len(dtype_flat) > 1:
  1698. # Flexible type : get a converter from each dtype
  1699. zipit = zip(dtype_flat, missing_values, filling_values)
  1700. converters = [StringConverter(dt, locked=True,
  1701. missing_values=miss, default=fill)
  1702. for (dt, miss, fill) in zipit]
  1703. else:
  1704. # Set to a default converter (but w/ different missing values)
  1705. zipit = zip(missing_values, filling_values)
  1706. converters = [StringConverter(dtype, locked=True,
  1707. missing_values=miss, default=fill)
  1708. for (miss, fill) in zipit]
  1709. # Update the converters to use the user-defined ones
  1710. uc_update = []
  1711. for (j, conv) in user_converters.items():
  1712. # If the converter is specified by column names, use the index instead
  1713. if _is_string_like(j):
  1714. try:
  1715. j = names.index(j)
  1716. i = j
  1717. except ValueError:
  1718. continue
  1719. elif usecols:
  1720. try:
  1721. i = usecols.index(j)
  1722. except ValueError:
  1723. # Unused converter specified
  1724. continue
  1725. else:
  1726. i = j
  1727. # Find the value to test - first_line is not filtered by usecols:
  1728. if len(first_line):
  1729. testing_value = first_values[j]
  1730. else:
  1731. testing_value = None
  1732. if conv is bytes:
  1733. user_conv = asbytes
  1734. elif byte_converters:
  1735. # converters may use decode to workaround numpy's old behaviour,
  1736. # so encode the string again before passing to the user converter
  1737. def tobytes_first(x, conv):
  1738. if type(x) is bytes:
  1739. return conv(x)
  1740. return conv(x.encode("latin1"))
  1741. user_conv = functools.partial(tobytes_first, conv=conv)
  1742. else:
  1743. user_conv = conv
  1744. converters[i].update(user_conv, locked=True,
  1745. testing_value=testing_value,
  1746. default=filling_values[i],
  1747. missing_values=missing_values[i],)
  1748. uc_update.append((i, user_conv))
  1749. # Make sure we have the corrected keys in user_converters...
  1750. user_converters.update(uc_update)
  1751. # Fixme: possible error as following variable never used.
  1752. # miss_chars = [_.missing_values for _ in converters]
  1753. # Initialize the output lists ...
  1754. # ... rows
  1755. rows = []
  1756. append_to_rows = rows.append
  1757. # ... masks
  1758. if usemask:
  1759. masks = []
  1760. append_to_masks = masks.append
  1761. # ... invalid
  1762. invalid = []
  1763. append_to_invalid = invalid.append
  1764. # Parse each line
  1765. for (i, line) in enumerate(itertools.chain([first_line, ], fhd)):
  1766. values = split_line(line)
  1767. nbvalues = len(values)
  1768. # Skip an empty line
  1769. if nbvalues == 0:
  1770. continue
  1771. if usecols:
  1772. # Select only the columns we need
  1773. try:
  1774. values = [values[_] for _ in usecols]
  1775. except IndexError:
  1776. append_to_invalid((i + skip_header + 1, nbvalues))
  1777. continue
  1778. elif nbvalues != nbcols:
  1779. append_to_invalid((i + skip_header + 1, nbvalues))
  1780. continue
  1781. # Store the values
  1782. append_to_rows(tuple(values))
  1783. if usemask:
  1784. append_to_masks(tuple([v.strip() in m
  1785. for (v, m) in zip(values,
  1786. missing_values)]))
  1787. if len(rows) == max_rows:
  1788. break
  1789. # Upgrade the converters (if needed)
  1790. if dtype is None:
  1791. for (i, converter) in enumerate(converters):
  1792. current_column = [itemgetter(i)(_m) for _m in rows]
  1793. try:
  1794. converter.iterupgrade(current_column)
  1795. except ConverterLockError:
  1796. errmsg = "Converter #%i is locked and cannot be upgraded: " % i
  1797. current_column = map(itemgetter(i), rows)
  1798. for (j, value) in enumerate(current_column):
  1799. try:
  1800. converter.upgrade(value)
  1801. except (ConverterError, ValueError):
  1802. errmsg += "(occurred line #%i for value '%s')"
  1803. errmsg %= (j + 1 + skip_header, value)
  1804. raise ConverterError(errmsg)
  1805. # Check that we don't have invalid values
  1806. nbinvalid = len(invalid)
  1807. if nbinvalid > 0:
  1808. nbrows = len(rows) + nbinvalid - skip_footer
  1809. # Construct the error message
  1810. template = " Line #%%i (got %%i columns instead of %i)" % nbcols
  1811. if skip_footer > 0:
  1812. nbinvalid_skipped = len([_ for _ in invalid
  1813. if _[0] > nbrows + skip_header])
  1814. invalid = invalid[:nbinvalid - nbinvalid_skipped]
  1815. skip_footer -= nbinvalid_skipped
  1816. #
  1817. # nbrows -= skip_footer
  1818. # errmsg = [template % (i, nb)
  1819. # for (i, nb) in invalid if i < nbrows]
  1820. # else:
  1821. errmsg = [template % (i, nb)
  1822. for (i, nb) in invalid]
  1823. if len(errmsg):
  1824. errmsg.insert(0, "Some errors were detected !")
  1825. errmsg = "\n".join(errmsg)
  1826. # Raise an exception ?
  1827. if invalid_raise:
  1828. raise ValueError(errmsg)
  1829. # Issue a warning ?
  1830. else:
  1831. warnings.warn(errmsg, ConversionWarning, stacklevel=2)
  1832. # Strip the last skip_footer data
  1833. if skip_footer > 0:
  1834. rows = rows[:-skip_footer]
  1835. if usemask:
  1836. masks = masks[:-skip_footer]
  1837. # Convert each value according to the converter:
  1838. # We want to modify the list in place to avoid creating a new one...
  1839. if loose:
  1840. rows = list(
  1841. zip(*[[conv._loose_call(_r) for _r in map(itemgetter(i), rows)]
  1842. for (i, conv) in enumerate(converters)]))
  1843. else:
  1844. rows = list(
  1845. zip(*[[conv._strict_call(_r) for _r in map(itemgetter(i), rows)]
  1846. for (i, conv) in enumerate(converters)]))
  1847. # Reset the dtype
  1848. data = rows
  1849. if dtype is None:
  1850. # Get the dtypes from the types of the converters
  1851. column_types = [conv.type for conv in converters]
  1852. # Find the columns with strings...
  1853. strcolidx = [i for (i, v) in enumerate(column_types)
  1854. if v == np.unicode_]
  1855. if byte_converters and strcolidx:
  1856. # convert strings back to bytes for backward compatibility
  1857. warnings.warn(
  1858. "Reading unicode strings without specifying the encoding "
  1859. "argument is deprecated. Set the encoding, use None for the "
  1860. "system default.",
  1861. np.VisibleDeprecationWarning, stacklevel=2)
  1862. def encode_unicode_cols(row_tup):
  1863. row = list(row_tup)
  1864. for i in strcolidx:
  1865. row[i] = row[i].encode('latin1')
  1866. return tuple(row)
  1867. try:
  1868. data = [encode_unicode_cols(r) for r in data]
  1869. except UnicodeEncodeError:
  1870. pass
  1871. else:
  1872. for i in strcolidx:
  1873. column_types[i] = np.bytes_
  1874. # Update string types to be the right length
  1875. sized_column_types = column_types[:]
  1876. for i, col_type in enumerate(column_types):
  1877. if np.issubdtype(col_type, np.character):
  1878. n_chars = max(len(row[i]) for row in data)
  1879. sized_column_types[i] = (col_type, n_chars)
  1880. if names is None:
  1881. # If the dtype is uniform (before sizing strings)
  1882. base = {
  1883. c_type
  1884. for c, c_type in zip(converters, column_types)
  1885. if c._checked}
  1886. if len(base) == 1:
  1887. uniform_type, = base
  1888. (ddtype, mdtype) = (uniform_type, bool)
  1889. else:
  1890. ddtype = [(defaultfmt % i, dt)
  1891. for (i, dt) in enumerate(sized_column_types)]
  1892. if usemask:
  1893. mdtype = [(defaultfmt % i, bool)
  1894. for (i, dt) in enumerate(sized_column_types)]
  1895. else:
  1896. ddtype = list(zip(names, sized_column_types))
  1897. mdtype = list(zip(names, [bool] * len(sized_column_types)))
  1898. output = np.array(data, dtype=ddtype)
  1899. if usemask:
  1900. outputmask = np.array(masks, dtype=mdtype)
  1901. else:
  1902. # Overwrite the initial dtype names if needed
  1903. if names and dtype.names is not None:
  1904. dtype.names = names
  1905. # Case 1. We have a structured type
  1906. if len(dtype_flat) > 1:
  1907. # Nested dtype, eg [('a', int), ('b', [('b0', int), ('b1', 'f4')])]
  1908. # First, create the array using a flattened dtype:
  1909. # [('a', int), ('b1', int), ('b2', float)]
  1910. # Then, view the array using the specified dtype.
  1911. if 'O' in (_.char for _ in dtype_flat):
  1912. if has_nested_fields(dtype):
  1913. raise NotImplementedError(
  1914. "Nested fields involving objects are not supported...")
  1915. else:
  1916. output = np.array(data, dtype=dtype)
  1917. else:
  1918. rows = np.array(data, dtype=[('', _) for _ in dtype_flat])
  1919. output = rows.view(dtype)
  1920. # Now, process the rowmasks the same way
  1921. if usemask:
  1922. rowmasks = np.array(
  1923. masks, dtype=np.dtype([('', bool) for t in dtype_flat]))
  1924. # Construct the new dtype
  1925. mdtype = make_mask_descr(dtype)
  1926. outputmask = rowmasks.view(mdtype)
  1927. # Case #2. We have a basic dtype
  1928. else:
  1929. # We used some user-defined converters
  1930. if user_converters:
  1931. ishomogeneous = True
  1932. descr = []
  1933. for i, ttype in enumerate([conv.type for conv in converters]):
  1934. # Keep the dtype of the current converter
  1935. if i in user_converters:
  1936. ishomogeneous &= (ttype == dtype.type)
  1937. if np.issubdtype(ttype, np.character):
  1938. ttype = (ttype, max(len(row[i]) for row in data))
  1939. descr.append(('', ttype))
  1940. else:
  1941. descr.append(('', dtype))
  1942. # So we changed the dtype ?
  1943. if not ishomogeneous:
  1944. # We have more than one field
  1945. if len(descr) > 1:
  1946. dtype = np.dtype(descr)
  1947. # We have only one field: drop the name if not needed.
  1948. else:
  1949. dtype = np.dtype(ttype)
  1950. #
  1951. output = np.array(data, dtype)
  1952. if usemask:
  1953. if dtype.names is not None:
  1954. mdtype = [(_, bool) for _ in dtype.names]
  1955. else:
  1956. mdtype = bool
  1957. outputmask = np.array(masks, dtype=mdtype)
  1958. # Try to take care of the missing data we missed
  1959. names = output.dtype.names
  1960. if usemask and names:
  1961. for (name, conv) in zip(names, converters):
  1962. missing_values = [conv(_) for _ in conv.missing_values
  1963. if _ != '']
  1964. for mval in missing_values:
  1965. outputmask[name] |= (output[name] == mval)
  1966. # Construct the final array
  1967. if usemask:
  1968. output = output.view(MaskedArray)
  1969. output._mask = outputmask
  1970. output = np.squeeze(output)
  1971. if unpack:
  1972. if names is None:
  1973. return output.T
  1974. elif len(names) == 1:
  1975. # squeeze single-name dtypes too
  1976. return output[names[0]]
  1977. else:
  1978. # For structured arrays with multiple fields,
  1979. # return an array for each field.
  1980. return [output[field] for field in names]
  1981. return output
  1982. _genfromtxt_with_like = array_function_dispatch(
  1983. _genfromtxt_dispatcher
  1984. )(genfromtxt)
  1985. def ndfromtxt(fname, **kwargs):
  1986. """
  1987. Load ASCII data stored in a file and return it as a single array.
  1988. .. deprecated:: 1.17
  1989. ndfromtxt` is a deprecated alias of `genfromtxt` which
  1990. overwrites the ``usemask`` argument with `False` even when
  1991. explicitly called as ``ndfromtxt(..., usemask=True)``.
  1992. Use `genfromtxt` instead.
  1993. Parameters
  1994. ----------
  1995. fname, kwargs : For a description of input parameters, see `genfromtxt`.
  1996. See Also
  1997. --------
  1998. numpy.genfromtxt : generic function.
  1999. """
  2000. kwargs['usemask'] = False
  2001. # Numpy 1.17
  2002. warnings.warn(
  2003. "np.ndfromtxt is a deprecated alias of np.genfromtxt, "
  2004. "prefer the latter.",
  2005. DeprecationWarning, stacklevel=2)
  2006. return genfromtxt(fname, **kwargs)
  2007. def mafromtxt(fname, **kwargs):
  2008. """
  2009. Load ASCII data stored in a text file and return a masked array.
  2010. .. deprecated:: 1.17
  2011. np.mafromtxt is a deprecated alias of `genfromtxt` which
  2012. overwrites the ``usemask`` argument with `True` even when
  2013. explicitly called as ``mafromtxt(..., usemask=False)``.
  2014. Use `genfromtxt` instead.
  2015. Parameters
  2016. ----------
  2017. fname, kwargs : For a description of input parameters, see `genfromtxt`.
  2018. See Also
  2019. --------
  2020. numpy.genfromtxt : generic function to load ASCII data.
  2021. """
  2022. kwargs['usemask'] = True
  2023. # Numpy 1.17
  2024. warnings.warn(
  2025. "np.mafromtxt is a deprecated alias of np.genfromtxt, "
  2026. "prefer the latter.",
  2027. DeprecationWarning, stacklevel=2)
  2028. return genfromtxt(fname, **kwargs)
  2029. def recfromtxt(fname, **kwargs):
  2030. """
  2031. Load ASCII data from a file and return it in a record array.
  2032. If ``usemask=False`` a standard `recarray` is returned,
  2033. if ``usemask=True`` a MaskedRecords array is returned.
  2034. Parameters
  2035. ----------
  2036. fname, kwargs : For a description of input parameters, see `genfromtxt`.
  2037. See Also
  2038. --------
  2039. numpy.genfromtxt : generic function
  2040. Notes
  2041. -----
  2042. By default, `dtype` is None, which means that the data-type of the output
  2043. array will be determined from the data.
  2044. """
  2045. kwargs.setdefault("dtype", None)
  2046. usemask = kwargs.get('usemask', False)
  2047. output = genfromtxt(fname, **kwargs)
  2048. if usemask:
  2049. from numpy.ma.mrecords import MaskedRecords
  2050. output = output.view(MaskedRecords)
  2051. else:
  2052. output = output.view(np.recarray)
  2053. return output
  2054. def recfromcsv(fname, **kwargs):
  2055. """
  2056. Load ASCII data stored in a comma-separated file.
  2057. The returned array is a record array (if ``usemask=False``, see
  2058. `recarray`) or a masked record array (if ``usemask=True``,
  2059. see `ma.mrecords.MaskedRecords`).
  2060. Parameters
  2061. ----------
  2062. fname, kwargs : For a description of input parameters, see `genfromtxt`.
  2063. See Also
  2064. --------
  2065. numpy.genfromtxt : generic function to load ASCII data.
  2066. Notes
  2067. -----
  2068. By default, `dtype` is None, which means that the data-type of the output
  2069. array will be determined from the data.
  2070. """
  2071. # Set default kwargs for genfromtxt as relevant to csv import.
  2072. kwargs.setdefault("case_sensitive", "lower")
  2073. kwargs.setdefault("names", True)
  2074. kwargs.setdefault("delimiter", ",")
  2075. kwargs.setdefault("dtype", None)
  2076. output = genfromtxt(fname, **kwargs)
  2077. usemask = kwargs.get("usemask", False)
  2078. if usemask:
  2079. from numpy.ma.mrecords import MaskedRecords
  2080. output = output.view(MaskedRecords)
  2081. else:
  2082. output = output.view(np.recarray)
  2083. return output