FazBrowse GitHub Viewer | Trending |
URL:
| Home
Tools: [Download Repo ZIP]   [Original HTTPS Page]

gh-159620: Fix remote asyncio stacks stopping at async generators (GH… · python/cpython@2188f9b · GitHub

Repository navigation

Commit 2188f9b

Browse files
gh-159620: Fix remote asyncio stacks stopping at async generators (GH-159623)
Follow asynchronous-generator send/throw wrappers when unwinding suspended and running asyncio tasks. Co-authored-by: Pablo Galindo Salgado <pablogsal@gmail.com>
1 parent 0fedf78 commit 2188f9b

3 files changed

Lines changed: 175 additions & 33 deletions

File tree

‎Lib/test/test_external_inspection.py‎

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -475,6 +475,17 @@ def _extract_coroutine_stacks_lineno_only(self, stack_trace):
475475

476476
@requires_remote_subprocess_debugging()
477477
class TestSelfStackTrace(RemoteInspectionTestBase):
478+
def _extract_coroutine_funcnames(self, trace):
479+
return {
480+
task.task_name: [
481+
frame.funcname.rpartition(".")[2]
482+
for coro in task.coroutine_stack
483+
for frame in coro.call_stack
484+
]
485+
for info in trace
486+
for task in info.awaited_by
487+
}
488+
478489
@skip_if_not_supported
479490
def test_long_task_name_is_truncated(self):
480491
# gh-157788
@@ -508,6 +519,101 @@ async def rec(n):
508519

509520
self.assertEqual(asyncio.run(rec(3)), ["rec"] * 4)
510521

522+
@skip_if_not_supported
523+
def test_async_generator_stack_is_not_truncated(self):
524+
# gh-159620
525+
async def agen():
526+
await asyncio.sleep(10_000)
527+
yield
528+
529+
async def agen_with_cleanup():
530+
try:
531+
yield
532+
finally:
533+
await asyncio.sleep(10_000)
534+
535+
async def outer_agen():
536+
async for _ in agen():
537+
yield
538+
539+
async def consumer(gen):
540+
async for _ in gen:
541+
pass
542+
543+
async def main():
544+
g = agen_with_cleanup()
545+
await anext(g)
546+
async_for_task = asyncio.create_task(consumer(agen()), name="async for")
547+
nested_task = asyncio.create_task(
548+
consumer(outer_agen()), name="nested async for")
549+
anext_task = asyncio.create_task(anext(agen()), name="anext")
550+
aclose_task = asyncio.create_task(g.aclose(), name="aclose")
551+
await asyncio.sleep(0)
552+
return RemoteUnwinder(os.getpid()).get_all_awaited_by()
553+
554+
stacks = self._extract_coroutine_funcnames(asyncio.run(main()))
555+
self.assertEqual(stacks["async for"], ["sleep", "agen", "consumer"])
556+
self.assertEqual(stacks["nested async for"],
557+
["sleep", "agen", "outer_agen", "consumer"])
558+
self.assertEqual(stacks["anext"], ["sleep", "agen"])
559+
self.assertEqual(stacks["aclose"], ["sleep", "agen_with_cleanup"])
560+
561+
@skip_if_not_supported
562+
def test_async_generator_waiter_in_async_stack_trace(self):
563+
# gh-159620
564+
async def agen(task):
565+
await task
566+
yield
567+
568+
async def main():
569+
waiter_task = asyncio.create_task(
570+
anext(agen(asyncio.current_task())), name="anext")
571+
await asyncio.sleep(0)
572+
return RemoteUnwinder(os.getpid()).get_async_stack_trace()
573+
574+
stacks = self._extract_coroutine_funcnames(asyncio.run(main()))
575+
self.assertEqual(stacks["anext"], ["agen"])
576+
577+
@skip_if_not_supported
578+
def test_async_generator_running_task_stack(self):
579+
# gh-159620
580+
async def agen():
581+
yield RemoteUnwinder(os.getpid()).get_async_stack_trace()
582+
583+
async def main():
584+
return await asyncio.create_task(anext(agen()), name="anext")
585+
586+
stacks = self._extract_coroutine_funcnames(asyncio.run(main()))
587+
self.assertEqual(stacks["anext"], ["agen"])
588+
589+
@skip_if_not_supported
590+
def test_class_named_like_async_generator_wrapper_is_not_walked(self):
591+
# gh-159620
592+
async def decoy():
593+
await asyncio.sleep(10_000)
594+
595+
class async_generator_asend:
596+
__slots__ = ("gen",)
597+
def __init__(self, gen):
598+
self.gen = gen
599+
def __await__(self):
600+
return self
601+
def __next__(self):
602+
return None
603+
604+
async def consumer(aw):
605+
await aw
606+
607+
async def main():
608+
with contextlib.closing(decoy()) as gen:
609+
consumer_task = asyncio.create_task(
610+
consumer(async_generator_asend(gen)), name="consumer")
611+
await asyncio.sleep(0)
612+
return RemoteUnwinder(os.getpid()).get_all_awaited_by()
613+
614+
stacks = self._extract_coroutine_funcnames(asyncio.run(main()))
615+
self.assertEqual(stacks["consumer"], ["consumer"])
616+
511617
@skip_if_not_supported
512618
@unittest.skipIf(
513619
sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED,
Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
Fix ``python -m asyncio ps``, ``pstree`` and async-aware
2+
:mod:`profiling.sampling` stacks stopping at asynchronous generators.
3+
Patch by Timofei Ivankov.

‎Modules/_remote_debugging/asyncio.c‎

Lines changed: 66 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -240,11 +240,63 @@ parse_task_name(
240240
* COROUTINE CHAIN PARSING
241241
* ============================================================================ */
242242

243+
// gh-159620
244+
/* Return in *coro_addr the coroutine or async generator to walk for the
245+
awaitable at addr, or 0 if it is neither (for example, a future). */
246+
static int
247+
get_coro_like_address(
248+
RemoteUnwinderObject *unwinder,
249+
uintptr_t addr,
250+
uintptr_t *coro_addr
251+
) {
252+
*coro_addr = 0;
253+
254+
// Skip user classes that share a name with the generator types below
255+
uintptr_t type_addr;
256+
unsigned long tp_flags;
257+
if (read_ptr(unwinder, addr + (uintptr_t)unwinder->debug_offsets.pyobject.ob_type,
258+
&type_addr) < 0
259+
|| _Py_RemoteDebug_PagedReadRemoteMemory(&unwinder->handle,
260+
type_addr + (uintptr_t)unwinder->debug_offsets.type_object.tp_flags,
261+
sizeof(tp_flags), &tp_flags) < 0)
262+
{
263+
set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read awaitable type");
264+
return -1;
265+
}
266+
267+
if (tp_flags & Py_TPFLAGS_HEAPTYPE) {
268+
return 0;
269+
}
270+
271+
uintptr_t name_addr;
272+
char name[32] = {0};
273+
if (read_ptr(unwinder, type_addr + (uintptr_t)unwinder->debug_offsets.type_object.tp_name,
274+
&name_addr) < 0
275+
|| _Py_RemoteDebug_PagedReadRemoteMemory(&unwinder->handle, name_addr,
276+
sizeof(name) - 1, name) < 0)
277+
{
278+
set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read awaitable type name");
279+
return -1;
280+
}
281+
282+
if (strcmp(name, "coroutine") == 0) {
283+
*coro_addr = addr;
284+
}
285+
else if (strcmp(name, "async_generator_asend") == 0
286+
|| strcmp(name, "async_generator_athrow") == 0)
287+
{
288+
// asend and athrow are separate wrapper objects: read the pointer
289+
// to their parent async generator, stored right after PyObject_HEAD
290+
return read_ptr(unwinder, addr + (uintptr_t)unwinder->debug_offsets.pyobject.size,
291+
coro_addr);
292+
}
293+
return 0;
294+
}
295+
243296
static int
244297
get_awaited_coro_address(
245298
RemoteUnwinderObject *unwinder,
246299
uintptr_t gi_iframe_addr,
247-
uintptr_t gen_type_addr,
248300
uintptr_t *next_coro
249301
) {
250302
*next_coro = 0;
@@ -272,41 +324,14 @@ get_awaited_coro_address(
272324
uintptr_t stackpointer_addr = GET_MEMBER_NO_TAG(uintptr_t, iframe, unwinder->debug_offsets.interpreter_frame.stackpointer);
273325

274326
if ((void*)stackpointer_addr != NULL) {
275-
uintptr_t gi_await_addr;
276327
err = read_py_ptr(
277328
unwinder,
278329
stackpointer_addr - sizeof(void*) * 2,
279-
&gi_await_addr);
330+
next_coro);
280331
if (err) {
281332
set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read gi_await address");
282333
return -1;
283334
}
284-
285-
if ((void*)gi_await_addr != NULL) {
286-
uintptr_t gi_await_addr_type_addr;
287-
err = read_ptr(
288-
unwinder,
289-
gi_await_addr + (uintptr_t)unwinder->debug_offsets.pyobject.ob_type,
290-
&gi_await_addr_type_addr);
291-
if (err) {
292-
set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read gi_await type address");
293-
return -1;
294-
}
295-
296-
if (gen_type_addr == gi_await_addr_type_addr) {
297-
/* This needs an explanation. We always start with parsing
298-
native coroutine / generator frames. Ultimately they
299-
are awaiting on something. That something can be
300-
a native coroutine frame or... an iterator.
301-
If it's the latter -- we can't continue building
302-
our chain. So the condition to bail out of this is
303-
to do that when the type of the current coroutine
304-
doesn't match the type of whatever it points to
305-
in its cr_await.
306-
*/
307-
*next_coro = gi_await_addr;
308-
}
309-
}
310335
}
311336

312337
return 0;
@@ -329,6 +354,13 @@ parse_coro_chain(
329354
return -1;
330355
}
331356

357+
if (get_coro_like_address(unwinder, coro_address, &coro_address) < 0) {
358+
return -1;
359+
}
360+
if ((void*)coro_address == NULL) {
361+
return 0;
362+
}
363+
332364
// Read the entire generator object at once
333365
char gen_object[SIZEOF_GEN_OBJ];
334366
int err = _Py_RemoteDebug_PagedReadRemoteMemory(
@@ -346,8 +378,6 @@ parse_coro_chain(
346378
return 0;
347379
}
348380

349-
uintptr_t gen_type_addr = GET_MEMBER(uintptr_t, gen_object, unwinder->debug_offsets.pyobject.ob_type);
350-
351381
PyObject* name = NULL;
352382

353383
// Parse the previous frame using the gi_iframe from local copy
@@ -374,8 +404,7 @@ parse_coro_chain(
374404
return 0;
375405
}
376406

377-
if (get_awaited_coro_address(unwinder, gi_iframe_addr, gen_type_addr,
378-
&coro_address) < 0) {
407+
if (get_awaited_coro_address(unwinder, gi_iframe_addr, &coro_address) < 0) {
379408
return -1;
380409
}
381410
}
@@ -733,6 +762,10 @@ get_task_frame(RemoteUnwinderObject *unwinder, uintptr_t task_addr, uintptr_t *f
733762
return -1;
734763
}
735764

765+
if (get_coro_like_address(unwinder, running_coro_addr, &running_coro_addr) < 0) {
766+
return -1;
767+
}
768+
736769
*frame_addr = running_coro_addr + (uintptr_t)unwinder->debug_offsets.gen_object.gi_iframe;
737770
return 0;
738771
}

0 commit comments

Comments
 (0)

Back | FazBrowse Home | New Git URL