| FazBrowse GitHub Viewer | Trending | | Home |
| Tools: [Download Repo ZIP] [Original HTTPS Page] |
1 parent b4b8f23 commit f9d0b12
9 files changed
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -393,6 +393,20 @@ always available. | |||
| 393 | 393 | .. versionadded:: 3.1 | |
| 394 | 394 | ||
| 395 | 395 | ||
| 396 | + .. function:: getallocatedblocks() | ||
| 397 | + | ||
| 398 | + Return the number of memory blocks currently allocated by the interpreter, | ||
| 399 | + regardless of their size. This function is mainly useful for debugging | ||
| 400 | + small memory leaks. Because of the interpreter's internal caches, the | ||
| 401 | + result can vary from call to call; you may have to call | ||
| 402 | + :func:`_clear_type_cache()` to get more predictable results. | ||
| 403 | + | ||
| 404 | + .. versionadded:: 3.4 | ||
| 405 | + | ||
| 406 | + .. impl-detail:: | ||
| 407 | + Not all Python implementations may be able to return this information. | ||
| 408 | + | ||
| 409 | + | ||
| 396 | 410 | .. function:: getcheckinterval() | |
| 397 | 411 | ||
| 398 | 412 | Return the interpreter's "check interval"; see :func:`setcheckinterval`. | |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -98,6 +98,8 @@ PyAPI_FUNC(void *) PyObject_Malloc(size_t); | |||
| 98 | 98 | PyAPI_FUNC(void *) PyObject_Realloc(void *, size_t); | |
| 99 | 99 | PyAPI_FUNC(void) PyObject_Free(void *); | |
| 100 | 100 | ||
| 101 | + /* This function returns the number of allocated memory blocks, regardless of size */ | ||
| 102 | + PyAPI_FUNC(Py_ssize_t) _Py_GetAllocatedBlocks(void); | ||
| 101 | 103 | ||
| 102 | 104 | /* Macros */ | |
| 103 | 105 | #ifdef WITH_PYMALLOC | |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -615,7 +615,7 @@ def test_forever(tests=list(selected)): | |||
| 615 | 615 | sys.exit(2) | |
| 616 | 616 | from queue import Queue | |
| 617 | 617 | from subprocess import Popen, PIPE | |
| 618 | - debug_output_pat = re.compile(r"\[\d+ refs\]$") | ||
| 618 | + debug_output_pat = re.compile(r"\[\d+ refs, \d+ blocks\]$") | ||
| 619 | 619 | output = Queue() | |
| 620 | 620 | pending = MultiprocessTests(tests) | |
| 621 | 621 | opt_args = support.args_from_interpreter_flags() | |
@@ -1320,33 +1320,50 @@ def run_the_test(): | |||
| 1320 | 1320 | del sys.modules[the_module.__name__] | |
| 1321 | 1321 | exec('import ' + the_module.__name__) | |
| 1322 | 1322 | ||
| 1323 | - deltas = [] | ||
| 1324 | 1323 | nwarmup, ntracked, fname = huntrleaks | |
| 1325 | 1324 | fname = os.path.join(support.SAVEDCWD, fname) | |
| 1326 | 1325 | repcount = nwarmup + ntracked | |
| 1326 | + rc_deltas = [0] * repcount | ||
| 1327 | + alloc_deltas = [0] * repcount | ||
| 1328 | + | ||
| 1327 | 1329 | print("beginning", repcount, "repetitions", file=sys.stderr) | |
| 1328 | 1330 | print(("1234567890"*(repcount//10 + 1))[:repcount], file=sys.stderr) | |
| 1329 | 1331 | sys.stderr.flush() | |
| 1330 | - dash_R_cleanup(fs, ps, pic, zdc, abcs) | ||
| 1331 | 1332 | for i in range(repcount): | |
| 1332 | - rc_before = sys.gettotalrefcount() | ||
| 1333 | 1333 | run_the_test() | |
| 1334 | + alloc_after, rc_after = dash_R_cleanup(fs, ps, pic, zdc, abcs) | ||
| 1334 | 1335 | sys.stderr.write('.') | |
| 1335 | 1336 | sys.stderr.flush() | |
| 1336 | - dash_R_cleanup(fs, ps, pic, zdc, abcs) | ||
| 1337 | - rc_after = sys.gettotalrefcount() | ||
| 1338 | 1337 | if i >= nwarmup: | |
| 1339 | - deltas.append(rc_after - rc_before) | ||
| 1338 | + rc_deltas[i] = rc_after - rc_before | ||
| 1339 | + alloc_deltas[i] = alloc_after - alloc_before | ||
| 1340 | + alloc_before, rc_before = alloc_after, rc_after | ||
| 1340 | 1341 | print(file=sys.stderr) | |
| 1341 | - if any(deltas): | ||
| 1342 | - msg = '%s leaked %s references, sum=%s' % (test, deltas, sum(deltas)) | ||
| 1343 | - print(msg, file=sys.stderr) | ||
| 1344 | - sys.stderr.flush() | ||
| 1345 | - with open(fname, "a") as refrep: | ||
| 1346 | - print(msg, file=refrep) | ||
| 1347 | - refrep.flush() | ||
| 1348 | - return True | ||
| 1349 | - return False | ||
| 1342 | + # These checkers return False on success, True on failure | ||
| 1343 | + def check_rc_deltas(deltas): | ||
| 1344 | + return any(deltas) | ||
| 1345 | + def check_alloc_deltas(deltas): | ||
| 1346 | + # At least 1/3rd of 0s | ||
| 1347 | + if 3 * deltas.count(0) < len(deltas): | ||
| 1348 | + return True | ||
| 1349 | + # Nothing else than 1s, 0s and -1s | ||
| 1350 | + if not set(deltas) <= {1,0,-1}: | ||
| 1351 | + return True | ||
| 1352 | + return False | ||
| 1353 | + failed = False | ||
| 1354 | + for deltas, item_name, checker in [ | ||
| 1355 | + (rc_deltas, 'references', check_rc_deltas), | ||
| 1356 | + (alloc_deltas, 'memory blocks', check_alloc_deltas)]: | ||
| 1357 | + if checker(deltas): | ||
| 1358 | + msg = '%s leaked %s %s, sum=%s' % ( | ||
| 1359 | + test, deltas[nwarmup:], item_name, sum(deltas)) | ||
| 1360 | + print(msg, file=sys.stderr) | ||
| 1361 | + sys.stderr.flush() | ||
| 1362 | + with open(fname, "a") as refrep: | ||
| 1363 | + print(msg, file=refrep) | ||
| 1364 | + refrep.flush() | ||
| 1365 | + failed = True | ||
| 1366 | + return failed | ||
| 1350 | 1367 | ||
| 1351 | 1368 | def dash_R_cleanup(fs, ps, pic, zdc, abcs): | |
| 1352 | 1369 | import gc, copyreg | |
@@ -1412,8 +1429,11 @@ def dash_R_cleanup(fs, ps, pic, zdc, abcs): | |||
| 1412 | 1429 | else: | |
| 1413 | 1430 | ctypes._reset_cache() | |
| 1414 | 1431 | ||
| 1415 | - # Collect cyclic trash. | ||
| 1432 | + # Collect cyclic trash and read memory statistics immediately after. | ||
| 1433 | + func1 = sys.getallocatedblocks | ||
| 1434 | + func2 = sys.gettotalrefcount | ||
| 1416 | 1435 | gc.collect() | |
| 1436 | + return func1(), func2() | ||
| 1417 | 1437 | ||
| 1418 | 1438 | def warm_caches(): | |
| 1419 | 1439 | # char cache | |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -1772,7 +1772,7 @@ def strip_python_stderr(stderr): | |||
| 1772 | 1772 | This will typically be run on the result of the communicate() method | |
| 1773 | 1773 | of a subprocess.Popen object. | |
| 1774 | 1774 | """ | |
| 1775 | - stderr = re.sub(br"\[\d+ refs\]\r?\n?", b"", stderr).strip() | ||
| 1775 | + stderr = re.sub(br"\[\d+ refs, \d+ blocks\]\r?\n?", b"", stderr).strip() | ||
| 1776 | 1776 | return stderr | |
| 1777 | 1777 | ||
| 1778 | 1778 | def args_from_interpreter_flags(): | |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -6,6 +6,7 @@ | |||
| 6 | 6 | import warnings | |
| 7 | 7 | import operator | |
| 8 | 8 | import codecs | |
| 9 | + import gc | ||
| 9 | 10 | ||
| 10 | 11 | # count the number of test runs, used to create unique | |
| 11 | 12 | # strings to intern in test_intern() | |
@@ -611,6 +612,29 @@ def test_debugmallocstats(self): | |||
| 611 | 612 | ret, out, err = assert_python_ok(*args) | |
| 612 | 613 | self.assertIn(b"free PyDictObjects", err) | |
| 613 | 614 | ||
| 615 | + @unittest.skipUnless(hasattr(sys, "getallocatedblocks"), | ||
| 616 | + "sys.getallocatedblocks unavailable on this build") | ||
| 617 | + def test_getallocatedblocks(self): | ||
| 618 | + # Some sanity checks | ||
| 619 | + a = sys.getallocatedblocks() | ||
| 620 | + self.assertIs(type(a), int) | ||
| 621 | + self.assertGreater(a, 0) | ||
| 622 | + try: | ||
| 623 | + # While we could imagine a Python session where the number of | ||
| 624 | + # multiple buffer objects would exceed the sharing of references, | ||
| 625 | + # it is unlikely to happen in a normal test run. | ||
| 626 | + self.assertLess(a, sys.gettotalrefcount()) | ||
| 627 | + except AttributeError: | ||
| 628 | + # gettotalrefcount() not available | ||
| 629 | + pass | ||
| 630 | + gc.collect() | ||
| 631 | + b = sys.getallocatedblocks() | ||
| 632 | + self.assertLessEqual(b, a) | ||
| 633 | + gc.collect() | ||
| 634 | + c = sys.getallocatedblocks() | ||
| 635 | + self.assertIn(c, range(b - 50, b + 50)) | ||
| 636 | + | ||
| 637 | + | ||
| 614 | 638 | class SizeofTest(unittest.TestCase): | |
| 615 | 639 | ||
| 616 | 640 | def setUp(self): | |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -163,6 +163,9 @@ Core and Builtins | |||
| 163 | 163 | Library | |
| 164 | 164 | ------- | |
| 165 | 165 | ||
| 166 | + - Issue #13390: New function :func:`sys.getallocatedblocks()` returns the | ||
| 167 | + number of memory blocks currently allocated. | ||
| 168 | + | ||
| 166 | 169 | - Issue #16628: Fix a memory leak in ctypes.resize(). | |
| 167 | 170 | ||
| 168 | 171 | - Issue #13614: Fix setup.py register failure with invalid rst in description. | |
@@ -433,6 +436,9 @@ Extension Modules | |||
| 433 | 436 | Tests | |
| 434 | 437 | ----- | |
| 435 | 438 | ||
| 439 | + - Issue #13390: The ``-R`` option to regrtest now also checks for memory | ||
| 440 | + allocation leaks, using :func:`sys.getallocatedblocks()`. | ||
| 441 | + | ||
| 436 | 442 | - Issue #16559: Add more tests for the json module, including some from the | |
| 437 | 443 | official test suite at json.org. Patch by Serhiy Storchaka. | |
| 438 | 444 | ||
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -525,6 +525,15 @@ static size_t ntimes_arena_allocated = 0; | |||
| 525 | 525 | /* High water mark (max value ever seen) for narenas_currently_allocated. */ | |
| 526 | 526 | static size_t narenas_highwater = 0; | |
| 527 | 527 | ||
| 528 | + static Py_ssize_t _Py_AllocatedBlocks = 0; | ||
| 529 | + | ||
| 530 | + Py_ssize_t | ||
| 531 | + _Py_GetAllocatedBlocks(void) | ||
| 532 | + { | ||
| 533 | + return _Py_AllocatedBlocks; | ||
| 534 | + } | ||
| 535 | + | ||
| 536 | + | ||
| 528 | 537 | /* Allocate a new arena. If we run out of memory, return NULL. Else | |
| 529 | 538 | * allocate a new arena, and return the address of an arena_object | |
| 530 | 539 | * describing the new arena. It's expected that the caller will set | |
@@ -785,6 +794,8 @@ PyObject_Malloc(size_t nbytes) | |||
| 785 | 794 | if (nbytes > PY_SSIZE_T_MAX) | |
| 786 | 795 | return NULL; | |
| 787 | 796 | ||
| 797 | + _Py_AllocatedBlocks++; | ||
| 798 | + | ||
| 788 | 799 | /* | |
| 789 | 800 | * This implicitly redirects malloc(0). | |
| 790 | 801 | */ | |
@@ -901,6 +912,7 @@ PyObject_Malloc(size_t nbytes) | |||
| 901 | 912 | * and free list are already initialized. | |
| 902 | 913 | */ | |
| 903 | 914 | bp = pool->freeblock; | |
| 915 | + assert(bp != NULL); | ||
| 904 | 916 | pool->freeblock = *(block **)bp; | |
| 905 | 917 | UNLOCK(); | |
| 906 | 918 | return (void *)bp; | |
@@ -958,7 +970,12 @@ PyObject_Malloc(size_t nbytes) | |||
| 958 | 970 | */ | |
| 959 | 971 | if (nbytes == 0) | |
| 960 | 972 | nbytes = 1; | |
| 961 | - return (void *)malloc(nbytes); | ||
| 973 | + { | ||
| 974 | + void *result = malloc(nbytes); | ||
| 975 | + if (!result) | ||
| 976 | + _Py_AllocatedBlocks--; | ||
| 977 | + return result; | ||
| 978 | + } | ||
| 962 | 979 | } | |
| 963 | 980 | ||
| 964 | 981 | /* free */ | |
@@ -978,6 +995,8 @@ PyObject_Free(void *p) | |||
| 978 | 995 | if (p == NULL) /* free(NULL) has no effect */ | |
| 979 | 996 | return; | |
| 980 | 997 | ||
| 998 | + _Py_AllocatedBlocks--; | ||
| 999 | + | ||
| 981 | 1000 | #ifdef WITH_VALGRIND | |
| 982 | 1001 | if (UNLIKELY(running_on_valgrind > 0)) | |
| 983 | 1002 | goto redirect; | |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -38,9 +38,10 @@ | |||
| 38 | 38 | #ifndef Py_REF_DEBUG | |
| 39 | 39 | #define PRINT_TOTAL_REFS() | |
| 40 | 40 | #else /* Py_REF_DEBUG */ | |
| 41 | - #define PRINT_TOTAL_REFS() fprintf(stderr, \ | ||
| 42 | - "[%" PY_FORMAT_SIZE_T "d refs]\n", \ | ||
| 43 | - _Py_GetRefTotal()) | ||
| 41 | + #define PRINT_TOTAL_REFS() fprintf(stderr, \ | ||
| 42 | + "[%" PY_FORMAT_SIZE_T "d refs, " \ | ||
| 43 | + "%" PY_FORMAT_SIZE_T "d blocks]\n", \ | ||
| 44 | + _Py_GetRefTotal(), _Py_GetAllocatedBlocks()) | ||
| 44 | 45 | #endif | |
| 45 | 46 | ||
| 46 | 47 | #ifdef __cplusplus | |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -894,6 +894,19 @@ one higher than you might expect, because it includes the (temporary)\n\ | |||
| 894 | 894 | reference as an argument to getrefcount()." | |
| 895 | 895 | ); | |
| 896 | 896 | ||
| 897 | + static PyObject * | ||
| 898 | + sys_getallocatedblocks(PyObject *self) | ||
| 899 | + { | ||
| 900 | + return PyLong_FromSsize_t(_Py_GetAllocatedBlocks()); | ||
| 901 | + } | ||
| 902 | + | ||
| 903 | + PyDoc_STRVAR(getallocatedblocks_doc, | ||
| 904 | + "getallocatedblocks() -> integer\n\ | ||
| 905 | + \n\ | ||
| 906 | + Return the number of memory blocks currently allocated, regardless of their\n\ | ||
| 907 | + size." | ||
| 908 | + ); | ||
| 909 | + | ||
| 897 | 910 | #ifdef COUNT_ALLOCS | |
| 898 | 911 | static PyObject * | |
| 899 | 912 | sys_getcounts(PyObject *self) | |
@@ -1062,6 +1075,8 @@ static PyMethodDef sys_methods[] = { | |||
| 1062 | 1075 | {"getdlopenflags", (PyCFunction)sys_getdlopenflags, METH_NOARGS, | |
| 1063 | 1076 | getdlopenflags_doc}, | |
| 1064 | 1077 | #endif | |
| 1078 | + {"getallocatedblocks", (PyCFunction)sys_getallocatedblocks, METH_NOARGS, | ||
| 1079 | + getallocatedblocks_doc}, | ||
| 1065 | 1080 | #ifdef COUNT_ALLOCS | |
| 1066 | 1081 | {"getcounts", (PyCFunction)sys_getcounts, METH_NOARGS}, | |
| 1067 | 1082 | #endif | |
| Back | FazBrowse Home | New Git URL |
0 commit comments