FazBrowse GitHub Viewer | Trending |
URL:
| Home
Tools: [Download Repo ZIP]   [Original HTTPS Page]

gh-75876: Report the peak memory use of a bigmem test by serhiy-storchaka · Pull Request #156024 · python/cpython · GitHub

Repository navigation

Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension .py  (2) .rst  (1) All 2 file types selected
Viewed files
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Unified
Split
Hide whitespace
Diff view
Unified
Split
Hide whitespace
25 changes: 23 additions & 2 deletions Lib/test/support/__init__.py
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Original file line number Diff line number Diff line change
Expand Up @@ -1271,15 +1271,20 @@ def set_memlimit(limit: str) -> None:


def _memory_watchdog(pid):
"""Return a function printing the memory usage of process *pid*."""
"""Return a function printing the memory usage of process *pid*.

The largest value it saw is kept in its ``peak`` attribute.
"""
# Imported here: test.support does not depend on test.libregrtest.
from test.libregrtest.utils import get_process_memory_usage

def watch():
mem = get_process_memory_usage(pid)
if mem is not None:
watch.peak = max(watch.peak, mem)
print(f" ... process data size: {mem / (1024 ** 3):.1f} GiB",
flush=True)
watch.peak = 0
return watch


Expand Down Expand Up @@ -1333,7 +1338,23 @@ def wrapper(self):
qualname = f'{cls.__qualname__}.{f.__name__}'
proc = isolation._start_test(cls.__module__, qualname)
watchdog = _memory_watchdog(proc.pid) if verbose else None
isolation._replay_test(self, *proc.wait(tick=watchdog))
payload, output, returncode = proc.wait(tick=watchdog)
if watchdog:
# The subprocess measures its own peak exactly. What the
# parent sampled is only a lower bound.
maxrss = payload and payload.get('maxrss')
peak = maxrss or watchdog.peak
if peak:
print(f" ... peak memory use: "
f"{peak / (1024 ** 3):.1f} GiB"
f"{'' if maxrss else ' or more'}", flush=True)
majflt = payload and payload.get('majflt')
if majflt:
# The test did not fit in memory, so its timing means
# little.
print(f" ... {majflt} major page faults: the test "
f"waited for the disk", flush=True)
isolation._replay_test(self, payload, output, returncode)
return

return f(self, maxsize)
Expand Down
40 changes: 39 additions & 1 deletion Lib/test/support/subprocess_runner.py
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,45 @@ def _outcome(kind, test, detail):
for t, tb in result.expectedFailures]
outcomes += [_outcome('skipped', t, reason) for t, reason in result.skipped]

payload = {'outcomes': outcomes, 'durations': result.id_durations}
def _usage():
"""What this process used: peak resident set size in bytes, and the
number of major page faults it took, either of which can be None.

A major page fault is served from disk, so a non-zero count means swapping.

The modules are imported here, after the test has run, so that the test
does not see them.
"""
try:
import resource
except ImportError:
pass
else:
usage = resource.getrusage(resource.RUSAGE_SELF)
# Solaris and illumos leave these fields at 0, which no live process
# has, so treat it as "not supported".
if not usage.ru_maxrss:
return {'maxrss': None, 'majflt': None}
# ru_maxrss is in bytes on macOS, in kilobytes on Linux and the BSDs.
maxrss = usage.ru_maxrss
return {'maxrss': maxrss if sys.platform == 'darwin' else maxrss * 1024,
'majflt': usage.ru_majflt}
try:
import os
import _winapi
handle = _winapi.OpenProcess(
_winapi.PROCESS_QUERY_LIMITED_INFORMATION, False, os.getpid())
except (ImportError, OSError):
return {'maxrss': None, 'majflt': None}
try:
info = _winapi.GetProcessMemoryInfo(handle)
finally:
_winapi.CloseHandle(handle)
# PageFaultCount counts all faults, not only the ones served from disk.
return {'maxrss': info['PeakWorkingSetSize'], 'majflt': None}


payload = {'outcomes': outcomes, 'durations': result.id_durations, **_usage()}
with open(outfile, 'wb') as f:
marshal.dump(payload, f)

Expand Down
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
In verbose mode, a test decorated with :func:`~test.support.bigmemtest` now
reports how much memory it really used, and the number of major page faults
it took, if any.
Loading

Back | FazBrowse Home | New Git URL