test_histograms.py 33 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838
  1. import numpy as np
  2. from numpy.lib.histograms import histogram, histogramdd, histogram_bin_edges
  3. from numpy.testing import (
  4. assert_, assert_equal, assert_array_equal, assert_almost_equal,
  5. assert_array_almost_equal, assert_raises, assert_allclose,
  6. assert_array_max_ulp, assert_raises_regex, suppress_warnings,
  7. )
  8. import pytest
  9. class TestHistogram:
  10. def setup(self):
  11. pass
  12. def teardown(self):
  13. pass
  14. def test_simple(self):
  15. n = 100
  16. v = np.random.rand(n)
  17. (a, b) = histogram(v)
  18. # check if the sum of the bins equals the number of samples
  19. assert_equal(np.sum(a, axis=0), n)
  20. # check that the bin counts are evenly spaced when the data is from
  21. # a linear function
  22. (a, b) = histogram(np.linspace(0, 10, 100))
  23. assert_array_equal(a, 10)
  24. def test_one_bin(self):
  25. # Ticket 632
  26. hist, edges = histogram([1, 2, 3, 4], [1, 2])
  27. assert_array_equal(hist, [2, ])
  28. assert_array_equal(edges, [1, 2])
  29. assert_raises(ValueError, histogram, [1, 2], bins=0)
  30. h, e = histogram([1, 2], bins=1)
  31. assert_equal(h, np.array([2]))
  32. assert_allclose(e, np.array([1., 2.]))
  33. def test_normed(self):
  34. sup = suppress_warnings()
  35. with sup:
  36. rec = sup.record(np.VisibleDeprecationWarning, '.*normed.*')
  37. # Check that the integral of the density equals 1.
  38. n = 100
  39. v = np.random.rand(n)
  40. a, b = histogram(v, normed=True)
  41. area = np.sum(a * np.diff(b))
  42. assert_almost_equal(area, 1)
  43. assert_equal(len(rec), 1)
  44. sup = suppress_warnings()
  45. with sup:
  46. rec = sup.record(np.VisibleDeprecationWarning, '.*normed.*')
  47. # Check with non-constant bin widths (buggy but backwards
  48. # compatible)
  49. v = np.arange(10)
  50. bins = [0, 1, 5, 9, 10]
  51. a, b = histogram(v, bins, normed=True)
  52. area = np.sum(a * np.diff(b))
  53. assert_almost_equal(area, 1)
  54. assert_equal(len(rec), 1)
  55. def test_density(self):
  56. # Check that the integral of the density equals 1.
  57. n = 100
  58. v = np.random.rand(n)
  59. a, b = histogram(v, density=True)
  60. area = np.sum(a * np.diff(b))
  61. assert_almost_equal(area, 1)
  62. # Check with non-constant bin widths
  63. v = np.arange(10)
  64. bins = [0, 1, 3, 6, 10]
  65. a, b = histogram(v, bins, density=True)
  66. assert_array_equal(a, .1)
  67. assert_equal(np.sum(a * np.diff(b)), 1)
  68. # Test that passing False works too
  69. a, b = histogram(v, bins, density=False)
  70. assert_array_equal(a, [1, 2, 3, 4])
  71. # Variable bin widths are especially useful to deal with
  72. # infinities.
  73. v = np.arange(10)
  74. bins = [0, 1, 3, 6, np.inf]
  75. a, b = histogram(v, bins, density=True)
  76. assert_array_equal(a, [.1, .1, .1, 0.])
  77. # Taken from a bug report from N. Becker on the numpy-discussion
  78. # mailing list Aug. 6, 2010.
  79. counts, dmy = np.histogram(
  80. [1, 2, 3, 4], [0.5, 1.5, np.inf], density=True)
  81. assert_equal(counts, [.25, 0])
  82. def test_outliers(self):
  83. # Check that outliers are not tallied
  84. a = np.arange(10) + .5
  85. # Lower outliers
  86. h, b = histogram(a, range=[0, 9])
  87. assert_equal(h.sum(), 9)
  88. # Upper outliers
  89. h, b = histogram(a, range=[1, 10])
  90. assert_equal(h.sum(), 9)
  91. # Normalization
  92. h, b = histogram(a, range=[1, 9], density=True)
  93. assert_almost_equal((h * np.diff(b)).sum(), 1, decimal=15)
  94. # Weights
  95. w = np.arange(10) + .5
  96. h, b = histogram(a, range=[1, 9], weights=w, density=True)
  97. assert_equal((h * np.diff(b)).sum(), 1)
  98. h, b = histogram(a, bins=8, range=[1, 9], weights=w)
  99. assert_equal(h, w[1:-1])
  100. def test_arr_weights_mismatch(self):
  101. a = np.arange(10) + .5
  102. w = np.arange(11) + .5
  103. with assert_raises_regex(ValueError, "same shape as"):
  104. h, b = histogram(a, range=[1, 9], weights=w, density=True)
  105. def test_type(self):
  106. # Check the type of the returned histogram
  107. a = np.arange(10) + .5
  108. h, b = histogram(a)
  109. assert_(np.issubdtype(h.dtype, np.integer))
  110. h, b = histogram(a, density=True)
  111. assert_(np.issubdtype(h.dtype, np.floating))
  112. h, b = histogram(a, weights=np.ones(10, int))
  113. assert_(np.issubdtype(h.dtype, np.integer))
  114. h, b = histogram(a, weights=np.ones(10, float))
  115. assert_(np.issubdtype(h.dtype, np.floating))
  116. def test_f32_rounding(self):
  117. # gh-4799, check that the rounding of the edges works with float32
  118. x = np.array([276.318359, -69.593948, 21.329449], dtype=np.float32)
  119. y = np.array([5005.689453, 4481.327637, 6010.369629], dtype=np.float32)
  120. counts_hist, xedges, yedges = np.histogram2d(x, y, bins=100)
  121. assert_equal(counts_hist.sum(), 3.)
  122. def test_bool_conversion(self):
  123. # gh-12107
  124. # Reference integer histogram
  125. a = np.array([1, 1, 0], dtype=np.uint8)
  126. int_hist, int_edges = np.histogram(a)
  127. # Should raise an warning on booleans
  128. # Ensure that the histograms are equivalent, need to suppress
  129. # the warnings to get the actual outputs
  130. with suppress_warnings() as sup:
  131. rec = sup.record(RuntimeWarning, 'Converting input from .*')
  132. hist, edges = np.histogram([True, True, False])
  133. # A warning should be issued
  134. assert_equal(len(rec), 1)
  135. assert_array_equal(hist, int_hist)
  136. assert_array_equal(edges, int_edges)
  137. def test_weights(self):
  138. v = np.random.rand(100)
  139. w = np.ones(100) * 5
  140. a, b = histogram(v)
  141. na, nb = histogram(v, density=True)
  142. wa, wb = histogram(v, weights=w)
  143. nwa, nwb = histogram(v, weights=w, density=True)
  144. assert_array_almost_equal(a * 5, wa)
  145. assert_array_almost_equal(na, nwa)
  146. # Check weights are properly applied.
  147. v = np.linspace(0, 10, 10)
  148. w = np.concatenate((np.zeros(5), np.ones(5)))
  149. wa, wb = histogram(v, bins=np.arange(11), weights=w)
  150. assert_array_almost_equal(wa, w)
  151. # Check with integer weights
  152. wa, wb = histogram([1, 2, 2, 4], bins=4, weights=[4, 3, 2, 1])
  153. assert_array_equal(wa, [4, 5, 0, 1])
  154. wa, wb = histogram(
  155. [1, 2, 2, 4], bins=4, weights=[4, 3, 2, 1], density=True)
  156. assert_array_almost_equal(wa, np.array([4, 5, 0, 1]) / 10. / 3. * 4)
  157. # Check weights with non-uniform bin widths
  158. a, b = histogram(
  159. np.arange(9), [0, 1, 3, 6, 10],
  160. weights=[2, 1, 1, 1, 1, 1, 1, 1, 1], density=True)
  161. assert_almost_equal(a, [.2, .1, .1, .075])
  162. def test_exotic_weights(self):
  163. # Test the use of weights that are not integer or floats, but e.g.
  164. # complex numbers or object types.
  165. # Complex weights
  166. values = np.array([1.3, 2.5, 2.3])
  167. weights = np.array([1, -1, 2]) + 1j * np.array([2, 1, 2])
  168. # Check with custom bins
  169. wa, wb = histogram(values, bins=[0, 2, 3], weights=weights)
  170. assert_array_almost_equal(wa, np.array([1, 1]) + 1j * np.array([2, 3]))
  171. # Check with even bins
  172. wa, wb = histogram(values, bins=2, range=[1, 3], weights=weights)
  173. assert_array_almost_equal(wa, np.array([1, 1]) + 1j * np.array([2, 3]))
  174. # Decimal weights
  175. from decimal import Decimal
  176. values = np.array([1.3, 2.5, 2.3])
  177. weights = np.array([Decimal(1), Decimal(2), Decimal(3)])
  178. # Check with custom bins
  179. wa, wb = histogram(values, bins=[0, 2, 3], weights=weights)
  180. assert_array_almost_equal(wa, [Decimal(1), Decimal(5)])
  181. # Check with even bins
  182. wa, wb = histogram(values, bins=2, range=[1, 3], weights=weights)
  183. assert_array_almost_equal(wa, [Decimal(1), Decimal(5)])
  184. def test_no_side_effects(self):
  185. # This is a regression test that ensures that values passed to
  186. # ``histogram`` are unchanged.
  187. values = np.array([1.3, 2.5, 2.3])
  188. np.histogram(values, range=[-10, 10], bins=100)
  189. assert_array_almost_equal(values, [1.3, 2.5, 2.3])
  190. def test_empty(self):
  191. a, b = histogram([], bins=([0, 1]))
  192. assert_array_equal(a, np.array([0]))
  193. assert_array_equal(b, np.array([0, 1]))
  194. def test_error_binnum_type (self):
  195. # Tests if right Error is raised if bins argument is float
  196. vals = np.linspace(0.0, 1.0, num=100)
  197. histogram(vals, 5)
  198. assert_raises(TypeError, histogram, vals, 2.4)
  199. def test_finite_range(self):
  200. # Normal ranges should be fine
  201. vals = np.linspace(0.0, 1.0, num=100)
  202. histogram(vals, range=[0.25,0.75])
  203. assert_raises(ValueError, histogram, vals, range=[np.nan,0.75])
  204. assert_raises(ValueError, histogram, vals, range=[0.25,np.inf])
  205. def test_invalid_range(self):
  206. # start of range must be < end of range
  207. vals = np.linspace(0.0, 1.0, num=100)
  208. with assert_raises_regex(ValueError, "max must be larger than"):
  209. np.histogram(vals, range=[0.1, 0.01])
  210. def test_bin_edge_cases(self):
  211. # Ensure that floating-point computations correctly place edge cases.
  212. arr = np.array([337, 404, 739, 806, 1007, 1811, 2012])
  213. hist, edges = np.histogram(arr, bins=8296, range=(2, 2280))
  214. mask = hist > 0
  215. left_edges = edges[:-1][mask]
  216. right_edges = edges[1:][mask]
  217. for x, left, right in zip(arr, left_edges, right_edges):
  218. assert_(x >= left)
  219. assert_(x < right)
  220. def test_last_bin_inclusive_range(self):
  221. arr = np.array([0., 0., 0., 1., 2., 3., 3., 4., 5.])
  222. hist, edges = np.histogram(arr, bins=30, range=(-0.5, 5))
  223. assert_equal(hist[-1], 1)
  224. def test_bin_array_dims(self):
  225. # gracefully handle bins object > 1 dimension
  226. vals = np.linspace(0.0, 1.0, num=100)
  227. bins = np.array([[0, 0.5], [0.6, 1.0]])
  228. with assert_raises_regex(ValueError, "must be 1d"):
  229. np.histogram(vals, bins=bins)
  230. def test_unsigned_monotonicity_check(self):
  231. # Ensures ValueError is raised if bins not increasing monotonically
  232. # when bins contain unsigned values (see #9222)
  233. arr = np.array([2])
  234. bins = np.array([1, 3, 1], dtype='uint64')
  235. with assert_raises(ValueError):
  236. hist, edges = np.histogram(arr, bins=bins)
  237. def test_object_array_of_0d(self):
  238. # gh-7864
  239. assert_raises(ValueError,
  240. histogram, [np.array(0.4) for i in range(10)] + [-np.inf])
  241. assert_raises(ValueError,
  242. histogram, [np.array(0.4) for i in range(10)] + [np.inf])
  243. # these should not crash
  244. np.histogram([np.array(0.5) for i in range(10)] + [.500000000000001])
  245. np.histogram([np.array(0.5) for i in range(10)] + [.5])
  246. def test_some_nan_values(self):
  247. # gh-7503
  248. one_nan = np.array([0, 1, np.nan])
  249. all_nan = np.array([np.nan, np.nan])
  250. # the internal comparisons with NaN give warnings
  251. sup = suppress_warnings()
  252. sup.filter(RuntimeWarning)
  253. with sup:
  254. # can't infer range with nan
  255. assert_raises(ValueError, histogram, one_nan, bins='auto')
  256. assert_raises(ValueError, histogram, all_nan, bins='auto')
  257. # explicit range solves the problem
  258. h, b = histogram(one_nan, bins='auto', range=(0, 1))
  259. assert_equal(h.sum(), 2) # nan is not counted
  260. h, b = histogram(all_nan, bins='auto', range=(0, 1))
  261. assert_equal(h.sum(), 0) # nan is not counted
  262. # as does an explicit set of bins
  263. h, b = histogram(one_nan, bins=[0, 1])
  264. assert_equal(h.sum(), 2) # nan is not counted
  265. h, b = histogram(all_nan, bins=[0, 1])
  266. assert_equal(h.sum(), 0) # nan is not counted
  267. def test_datetime(self):
  268. begin = np.datetime64('2000-01-01', 'D')
  269. offsets = np.array([0, 0, 1, 1, 2, 3, 5, 10, 20])
  270. bins = np.array([0, 2, 7, 20])
  271. dates = begin + offsets
  272. date_bins = begin + bins
  273. td = np.dtype('timedelta64[D]')
  274. # Results should be the same for integer offsets or datetime values.
  275. # For now, only explicit bins are supported, since linspace does not
  276. # work on datetimes or timedeltas
  277. d_count, d_edge = histogram(dates, bins=date_bins)
  278. t_count, t_edge = histogram(offsets.astype(td), bins=bins.astype(td))
  279. i_count, i_edge = histogram(offsets, bins=bins)
  280. assert_equal(d_count, i_count)
  281. assert_equal(t_count, i_count)
  282. assert_equal((d_edge - begin).astype(int), i_edge)
  283. assert_equal(t_edge.astype(int), i_edge)
  284. assert_equal(d_edge.dtype, dates.dtype)
  285. assert_equal(t_edge.dtype, td)
  286. def do_signed_overflow_bounds(self, dtype):
  287. exponent = 8 * np.dtype(dtype).itemsize - 1
  288. arr = np.array([-2**exponent + 4, 2**exponent - 4], dtype=dtype)
  289. hist, e = histogram(arr, bins=2)
  290. assert_equal(e, [-2**exponent + 4, 0, 2**exponent - 4])
  291. assert_equal(hist, [1, 1])
  292. def test_signed_overflow_bounds(self):
  293. self.do_signed_overflow_bounds(np.byte)
  294. self.do_signed_overflow_bounds(np.short)
  295. self.do_signed_overflow_bounds(np.intc)
  296. self.do_signed_overflow_bounds(np.int_)
  297. self.do_signed_overflow_bounds(np.longlong)
  298. def do_precision_lower_bound(self, float_small, float_large):
  299. eps = np.finfo(float_large).eps
  300. arr = np.array([1.0], float_small)
  301. range = np.array([1.0 + eps, 2.0], float_large)
  302. # test is looking for behavior when the bounds change between dtypes
  303. if range.astype(float_small)[0] != 1:
  304. return
  305. # previously crashed
  306. count, x_loc = np.histogram(arr, bins=1, range=range)
  307. assert_equal(count, [1])
  308. # gh-10322 means that the type comes from arr - this may change
  309. assert_equal(x_loc.dtype, float_small)
  310. def do_precision_upper_bound(self, float_small, float_large):
  311. eps = np.finfo(float_large).eps
  312. arr = np.array([1.0], float_small)
  313. range = np.array([0.0, 1.0 - eps], float_large)
  314. # test is looking for behavior when the bounds change between dtypes
  315. if range.astype(float_small)[-1] != 1:
  316. return
  317. # previously crashed
  318. count, x_loc = np.histogram(arr, bins=1, range=range)
  319. assert_equal(count, [1])
  320. # gh-10322 means that the type comes from arr - this may change
  321. assert_equal(x_loc.dtype, float_small)
  322. def do_precision(self, float_small, float_large):
  323. self.do_precision_lower_bound(float_small, float_large)
  324. self.do_precision_upper_bound(float_small, float_large)
  325. def test_precision(self):
  326. # not looping results in a useful stack trace upon failure
  327. self.do_precision(np.half, np.single)
  328. self.do_precision(np.half, np.double)
  329. self.do_precision(np.half, np.longdouble)
  330. self.do_precision(np.single, np.double)
  331. self.do_precision(np.single, np.longdouble)
  332. self.do_precision(np.double, np.longdouble)
  333. def test_histogram_bin_edges(self):
  334. hist, e = histogram([1, 2, 3, 4], [1, 2])
  335. edges = histogram_bin_edges([1, 2, 3, 4], [1, 2])
  336. assert_array_equal(edges, e)
  337. arr = np.array([0., 0., 0., 1., 2., 3., 3., 4., 5.])
  338. hist, e = histogram(arr, bins=30, range=(-0.5, 5))
  339. edges = histogram_bin_edges(arr, bins=30, range=(-0.5, 5))
  340. assert_array_equal(edges, e)
  341. hist, e = histogram(arr, bins='auto', range=(0, 1))
  342. edges = histogram_bin_edges(arr, bins='auto', range=(0, 1))
  343. assert_array_equal(edges, e)
  344. class TestHistogramOptimBinNums:
  345. """
  346. Provide test coverage when using provided estimators for optimal number of
  347. bins
  348. """
  349. def test_empty(self):
  350. estimator_list = ['fd', 'scott', 'rice', 'sturges',
  351. 'doane', 'sqrt', 'auto', 'stone']
  352. # check it can deal with empty data
  353. for estimator in estimator_list:
  354. a, b = histogram([], bins=estimator)
  355. assert_array_equal(a, np.array([0]))
  356. assert_array_equal(b, np.array([0, 1]))
  357. def test_simple(self):
  358. """
  359. Straightforward testing with a mixture of linspace data (for
  360. consistency). All test values have been precomputed and the values
  361. shouldn't change
  362. """
  363. # Some basic sanity checking, with some fixed data.
  364. # Checking for the correct number of bins
  365. basic_test = {50: {'fd': 4, 'scott': 4, 'rice': 8, 'sturges': 7,
  366. 'doane': 8, 'sqrt': 8, 'auto': 7, 'stone': 2},
  367. 500: {'fd': 8, 'scott': 8, 'rice': 16, 'sturges': 10,
  368. 'doane': 12, 'sqrt': 23, 'auto': 10, 'stone': 9},
  369. 5000: {'fd': 17, 'scott': 17, 'rice': 35, 'sturges': 14,
  370. 'doane': 17, 'sqrt': 71, 'auto': 17, 'stone': 20}}
  371. for testlen, expectedResults in basic_test.items():
  372. # Create some sort of non uniform data to test with
  373. # (2 peak uniform mixture)
  374. x1 = np.linspace(-10, -1, testlen // 5 * 2)
  375. x2 = np.linspace(1, 10, testlen // 5 * 3)
  376. x = np.concatenate((x1, x2))
  377. for estimator, numbins in expectedResults.items():
  378. a, b = np.histogram(x, estimator)
  379. assert_equal(len(a), numbins, err_msg="For the {0} estimator "
  380. "with datasize of {1}".format(estimator, testlen))
  381. def test_small(self):
  382. """
  383. Smaller datasets have the potential to cause issues with the data
  384. adaptive methods, especially the FD method. All bin numbers have been
  385. precalculated.
  386. """
  387. small_dat = {1: {'fd': 1, 'scott': 1, 'rice': 1, 'sturges': 1,
  388. 'doane': 1, 'sqrt': 1, 'stone': 1},
  389. 2: {'fd': 2, 'scott': 1, 'rice': 3, 'sturges': 2,
  390. 'doane': 1, 'sqrt': 2, 'stone': 1},
  391. 3: {'fd': 2, 'scott': 2, 'rice': 3, 'sturges': 3,
  392. 'doane': 3, 'sqrt': 2, 'stone': 1}}
  393. for testlen, expectedResults in small_dat.items():
  394. testdat = np.arange(testlen)
  395. for estimator, expbins in expectedResults.items():
  396. a, b = np.histogram(testdat, estimator)
  397. assert_equal(len(a), expbins, err_msg="For the {0} estimator "
  398. "with datasize of {1}".format(estimator, testlen))
  399. def test_incorrect_methods(self):
  400. """
  401. Check a Value Error is thrown when an unknown string is passed in
  402. """
  403. check_list = ['mad', 'freeman', 'histograms', 'IQR']
  404. for estimator in check_list:
  405. assert_raises(ValueError, histogram, [1, 2, 3], estimator)
  406. def test_novariance(self):
  407. """
  408. Check that methods handle no variance in data
  409. Primarily for Scott and FD as the SD and IQR are both 0 in this case
  410. """
  411. novar_dataset = np.ones(100)
  412. novar_resultdict = {'fd': 1, 'scott': 1, 'rice': 1, 'sturges': 1,
  413. 'doane': 1, 'sqrt': 1, 'auto': 1, 'stone': 1}
  414. for estimator, numbins in novar_resultdict.items():
  415. a, b = np.histogram(novar_dataset, estimator)
  416. assert_equal(len(a), numbins, err_msg="{0} estimator, "
  417. "No Variance test".format(estimator))
  418. def test_limited_variance(self):
  419. """
  420. Check when IQR is 0, but variance exists, we return the sturges value
  421. and not the fd value.
  422. """
  423. lim_var_data = np.ones(1000)
  424. lim_var_data[:3] = 0
  425. lim_var_data[-4:] = 100
  426. edges_auto = histogram_bin_edges(lim_var_data, 'auto')
  427. assert_equal(edges_auto, np.linspace(0, 100, 12))
  428. edges_fd = histogram_bin_edges(lim_var_data, 'fd')
  429. assert_equal(edges_fd, np.array([0, 100]))
  430. edges_sturges = histogram_bin_edges(lim_var_data, 'sturges')
  431. assert_equal(edges_sturges, np.linspace(0, 100, 12))
  432. def test_outlier(self):
  433. """
  434. Check the FD, Scott and Doane with outliers.
  435. The FD estimates a smaller binwidth since it's less affected by
  436. outliers. Since the range is so (artificially) large, this means more
  437. bins, most of which will be empty, but the data of interest usually is
  438. unaffected. The Scott estimator is more affected and returns fewer bins,
  439. despite most of the variance being in one area of the data. The Doane
  440. estimator lies somewhere between the other two.
  441. """
  442. xcenter = np.linspace(-10, 10, 50)
  443. outlier_dataset = np.hstack((np.linspace(-110, -100, 5), xcenter))
  444. outlier_resultdict = {'fd': 21, 'scott': 5, 'doane': 11, 'stone': 6}
  445. for estimator, numbins in outlier_resultdict.items():
  446. a, b = np.histogram(outlier_dataset, estimator)
  447. assert_equal(len(a), numbins)
  448. def test_scott_vs_stone(self):
  449. """Verify that Scott's rule and Stone's rule converges for normally distributed data"""
  450. def nbins_ratio(seed, size):
  451. rng = np.random.RandomState(seed)
  452. x = rng.normal(loc=0, scale=2, size=size)
  453. a, b = len(np.histogram(x, 'stone')[0]), len(np.histogram(x, 'scott')[0])
  454. return a / (a + b)
  455. ll = [[nbins_ratio(seed, size) for size in np.geomspace(start=10, stop=100, num=4).round().astype(int)]
  456. for seed in range(10)]
  457. # the average difference between the two methods decreases as the dataset size increases.
  458. avg = abs(np.mean(ll, axis=0) - 0.5)
  459. assert_almost_equal(avg, [0.15, 0.09, 0.08, 0.03], decimal=2)
  460. def test_simple_range(self):
  461. """
  462. Straightforward testing with a mixture of linspace data (for
  463. consistency). Adding in a 3rd mixture that will then be
  464. completely ignored. All test values have been precomputed and
  465. the shouldn't change.
  466. """
  467. # some basic sanity checking, with some fixed data.
  468. # Checking for the correct number of bins
  469. basic_test = {
  470. 50: {'fd': 8, 'scott': 8, 'rice': 15,
  471. 'sturges': 14, 'auto': 14, 'stone': 8},
  472. 500: {'fd': 15, 'scott': 16, 'rice': 32,
  473. 'sturges': 20, 'auto': 20, 'stone': 80},
  474. 5000: {'fd': 33, 'scott': 33, 'rice': 69,
  475. 'sturges': 27, 'auto': 33, 'stone': 80}
  476. }
  477. for testlen, expectedResults in basic_test.items():
  478. # create some sort of non uniform data to test with
  479. # (3 peak uniform mixture)
  480. x1 = np.linspace(-10, -1, testlen // 5 * 2)
  481. x2 = np.linspace(1, 10, testlen // 5 * 3)
  482. x3 = np.linspace(-100, -50, testlen)
  483. x = np.hstack((x1, x2, x3))
  484. for estimator, numbins in expectedResults.items():
  485. a, b = np.histogram(x, estimator, range = (-20, 20))
  486. msg = "For the {0} estimator".format(estimator)
  487. msg += " with datasize of {0}".format(testlen)
  488. assert_equal(len(a), numbins, err_msg=msg)
  489. @pytest.mark.parametrize("bins", ['auto', 'fd', 'doane', 'scott',
  490. 'stone', 'rice', 'sturges'])
  491. def test_signed_integer_data(self, bins):
  492. # Regression test for gh-14379.
  493. a = np.array([-2, 0, 127], dtype=np.int8)
  494. hist, edges = np.histogram(a, bins=bins)
  495. hist32, edges32 = np.histogram(a.astype(np.int32), bins=bins)
  496. assert_array_equal(hist, hist32)
  497. assert_array_equal(edges, edges32)
  498. def test_simple_weighted(self):
  499. """
  500. Check that weighted data raises a TypeError
  501. """
  502. estimator_list = ['fd', 'scott', 'rice', 'sturges', 'auto']
  503. for estimator in estimator_list:
  504. assert_raises(TypeError, histogram, [1, 2, 3],
  505. estimator, weights=[1, 2, 3])
  506. class TestHistogramdd:
  507. def test_simple(self):
  508. x = np.array([[-.5, .5, 1.5], [-.5, 1.5, 2.5], [-.5, 2.5, .5],
  509. [.5, .5, 1.5], [.5, 1.5, 2.5], [.5, 2.5, 2.5]])
  510. H, edges = histogramdd(x, (2, 3, 3),
  511. range=[[-1, 1], [0, 3], [0, 3]])
  512. answer = np.array([[[0, 1, 0], [0, 0, 1], [1, 0, 0]],
  513. [[0, 1, 0], [0, 0, 1], [0, 0, 1]]])
  514. assert_array_equal(H, answer)
  515. # Check normalization
  516. ed = [[-2, 0, 2], [0, 1, 2, 3], [0, 1, 2, 3]]
  517. H, edges = histogramdd(x, bins=ed, density=True)
  518. assert_(np.all(H == answer / 12.))
  519. # Check that H has the correct shape.
  520. H, edges = histogramdd(x, (2, 3, 4),
  521. range=[[-1, 1], [0, 3], [0, 4]],
  522. density=True)
  523. answer = np.array([[[0, 1, 0, 0], [0, 0, 1, 0], [1, 0, 0, 0]],
  524. [[0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 1, 0]]])
  525. assert_array_almost_equal(H, answer / 6., 4)
  526. # Check that a sequence of arrays is accepted and H has the correct
  527. # shape.
  528. z = [np.squeeze(y) for y in np.split(x, 3, axis=1)]
  529. H, edges = histogramdd(
  530. z, bins=(4, 3, 2), range=[[-2, 2], [0, 3], [0, 2]])
  531. answer = np.array([[[0, 0], [0, 0], [0, 0]],
  532. [[0, 1], [0, 0], [1, 0]],
  533. [[0, 1], [0, 0], [0, 0]],
  534. [[0, 0], [0, 0], [0, 0]]])
  535. assert_array_equal(H, answer)
  536. Z = np.zeros((5, 5, 5))
  537. Z[list(range(5)), list(range(5)), list(range(5))] = 1.
  538. H, edges = histogramdd([np.arange(5), np.arange(5), np.arange(5)], 5)
  539. assert_array_equal(H, Z)
  540. def test_shape_3d(self):
  541. # All possible permutations for bins of different lengths in 3D.
  542. bins = ((5, 4, 6), (6, 4, 5), (5, 6, 4), (4, 6, 5), (6, 5, 4),
  543. (4, 5, 6))
  544. r = np.random.rand(10, 3)
  545. for b in bins:
  546. H, edges = histogramdd(r, b)
  547. assert_(H.shape == b)
  548. def test_shape_4d(self):
  549. # All possible permutations for bins of different lengths in 4D.
  550. bins = ((7, 4, 5, 6), (4, 5, 7, 6), (5, 6, 4, 7), (7, 6, 5, 4),
  551. (5, 7, 6, 4), (4, 6, 7, 5), (6, 5, 7, 4), (7, 5, 4, 6),
  552. (7, 4, 6, 5), (6, 4, 7, 5), (6, 7, 5, 4), (4, 6, 5, 7),
  553. (4, 7, 5, 6), (5, 4, 6, 7), (5, 7, 4, 6), (6, 7, 4, 5),
  554. (6, 5, 4, 7), (4, 7, 6, 5), (4, 5, 6, 7), (7, 6, 4, 5),
  555. (5, 4, 7, 6), (5, 6, 7, 4), (6, 4, 5, 7), (7, 5, 6, 4))
  556. r = np.random.rand(10, 4)
  557. for b in bins:
  558. H, edges = histogramdd(r, b)
  559. assert_(H.shape == b)
  560. def test_weights(self):
  561. v = np.random.rand(100, 2)
  562. hist, edges = histogramdd(v)
  563. n_hist, edges = histogramdd(v, density=True)
  564. w_hist, edges = histogramdd(v, weights=np.ones(100))
  565. assert_array_equal(w_hist, hist)
  566. w_hist, edges = histogramdd(v, weights=np.ones(100) * 2, density=True)
  567. assert_array_equal(w_hist, n_hist)
  568. w_hist, edges = histogramdd(v, weights=np.ones(100, int) * 2)
  569. assert_array_equal(w_hist, 2 * hist)
  570. def test_identical_samples(self):
  571. x = np.zeros((10, 2), int)
  572. hist, edges = histogramdd(x, bins=2)
  573. assert_array_equal(edges[0], np.array([-0.5, 0., 0.5]))
  574. def test_empty(self):
  575. a, b = histogramdd([[], []], bins=([0, 1], [0, 1]))
  576. assert_array_max_ulp(a, np.array([[0.]]))
  577. a, b = np.histogramdd([[], [], []], bins=2)
  578. assert_array_max_ulp(a, np.zeros((2, 2, 2)))
  579. def test_bins_errors(self):
  580. # There are two ways to specify bins. Check for the right errors
  581. # when mixing those.
  582. x = np.arange(8).reshape(2, 4)
  583. assert_raises(ValueError, np.histogramdd, x, bins=[-1, 2, 4, 5])
  584. assert_raises(ValueError, np.histogramdd, x, bins=[1, 0.99, 1, 1])
  585. assert_raises(
  586. ValueError, np.histogramdd, x, bins=[1, 1, 1, [1, 2, 3, -3]])
  587. assert_(np.histogramdd(x, bins=[1, 1, 1, [1, 2, 3, 4]]))
  588. def test_inf_edges(self):
  589. # Test using +/-inf bin edges works. See #1788.
  590. with np.errstate(invalid='ignore'):
  591. x = np.arange(6).reshape(3, 2)
  592. expected = np.array([[1, 0], [0, 1], [0, 1]])
  593. h, e = np.histogramdd(x, bins=[3, [-np.inf, 2, 10]])
  594. assert_allclose(h, expected)
  595. h, e = np.histogramdd(x, bins=[3, np.array([-1, 2, np.inf])])
  596. assert_allclose(h, expected)
  597. h, e = np.histogramdd(x, bins=[3, [-np.inf, 3, np.inf]])
  598. assert_allclose(h, expected)
  599. def test_rightmost_binedge(self):
  600. # Test event very close to rightmost binedge. See Github issue #4266
  601. x = [0.9999999995]
  602. bins = [[0., 0.5, 1.0]]
  603. hist, _ = histogramdd(x, bins=bins)
  604. assert_(hist[0] == 0.0)
  605. assert_(hist[1] == 1.)
  606. x = [1.0]
  607. bins = [[0., 0.5, 1.0]]
  608. hist, _ = histogramdd(x, bins=bins)
  609. assert_(hist[0] == 0.0)
  610. assert_(hist[1] == 1.)
  611. x = [1.0000000001]
  612. bins = [[0., 0.5, 1.0]]
  613. hist, _ = histogramdd(x, bins=bins)
  614. assert_(hist[0] == 0.0)
  615. assert_(hist[1] == 0.0)
  616. x = [1.0001]
  617. bins = [[0., 0.5, 1.0]]
  618. hist, _ = histogramdd(x, bins=bins)
  619. assert_(hist[0] == 0.0)
  620. assert_(hist[1] == 0.0)
  621. def test_finite_range(self):
  622. vals = np.random.random((100, 3))
  623. histogramdd(vals, range=[[0.0, 1.0], [0.25, 0.75], [0.25, 0.5]])
  624. assert_raises(ValueError, histogramdd, vals,
  625. range=[[0.0, 1.0], [0.25, 0.75], [0.25, np.inf]])
  626. assert_raises(ValueError, histogramdd, vals,
  627. range=[[0.0, 1.0], [np.nan, 0.75], [0.25, 0.5]])
  628. def test_equal_edges(self):
  629. """ Test that adjacent entries in an edge array can be equal """
  630. x = np.array([0, 1, 2])
  631. y = np.array([0, 1, 2])
  632. x_edges = np.array([0, 2, 2])
  633. y_edges = 1
  634. hist, edges = histogramdd((x, y), bins=(x_edges, y_edges))
  635. hist_expected = np.array([
  636. [2.],
  637. [1.], # x == 2 falls in the final bin
  638. ])
  639. assert_equal(hist, hist_expected)
  640. def test_edge_dtype(self):
  641. """ Test that if an edge array is input, its type is preserved """
  642. x = np.array([0, 10, 20])
  643. y = x / 10
  644. x_edges = np.array([0, 5, 15, 20])
  645. y_edges = x_edges / 10
  646. hist, edges = histogramdd((x, y), bins=(x_edges, y_edges))
  647. assert_equal(edges[0].dtype, x_edges.dtype)
  648. assert_equal(edges[1].dtype, y_edges.dtype)
  649. def test_large_integers(self):
  650. big = 2**60 # Too large to represent with a full precision float
  651. x = np.array([0], np.int64)
  652. x_edges = np.array([-1, +1], np.int64)
  653. y = big + x
  654. y_edges = big + x_edges
  655. hist, edges = histogramdd((x, y), bins=(x_edges, y_edges))
  656. assert_equal(hist[0, 0], 1)
  657. def test_density_non_uniform_2d(self):
  658. # Defines the following grid:
  659. #
  660. # 0 2 8
  661. # 0+-+-----+
  662. # + | +
  663. # + | +
  664. # 6+-+-----+
  665. # 8+-+-----+
  666. x_edges = np.array([0, 2, 8])
  667. y_edges = np.array([0, 6, 8])
  668. relative_areas = np.array([
  669. [3, 9],
  670. [1, 3]])
  671. # ensure the number of points in each region is proportional to its area
  672. x = np.array([1] + [1]*3 + [7]*3 + [7]*9)
  673. y = np.array([7] + [1]*3 + [7]*3 + [1]*9)
  674. # sanity check that the above worked as intended
  675. hist, edges = histogramdd((y, x), bins=(y_edges, x_edges))
  676. assert_equal(hist, relative_areas)
  677. # resulting histogram should be uniform, since counts and areas are proportional
  678. hist, edges = histogramdd((y, x), bins=(y_edges, x_edges), density=True)
  679. assert_equal(hist, 1 / (8*8))
  680. def test_density_non_uniform_1d(self):
  681. # compare to histogram to show the results are the same
  682. v = np.arange(10)
  683. bins = np.array([0, 1, 3, 6, 10])
  684. hist, edges = histogram(v, bins, density=True)
  685. hist_dd, edges_dd = histogramdd((v,), (bins,), density=True)
  686. assert_equal(hist, hist_dd)
  687. assert_equal(edges, edges_dd[0])
  688. def test_density_via_normed(self):
  689. # normed should simply alias to density argument
  690. v = np.arange(10)
  691. bins = np.array([0, 1, 3, 6, 10])
  692. hist, edges = histogram(v, bins, density=True)
  693. hist_dd, edges_dd = histogramdd((v,), (bins,), normed=True)
  694. assert_equal(hist, hist_dd)
  695. assert_equal(edges, edges_dd[0])
  696. def test_density_normed_redundancy(self):
  697. v = np.arange(10)
  698. bins = np.array([0, 1, 3, 6, 10])
  699. with assert_raises_regex(TypeError, "Cannot specify both"):
  700. hist_dd, edges_dd = histogramdd((v,), (bins,),
  701. density=True,
  702. normed=True)