| FazBrowse GitHub Viewer | Trending | | Home |
| Tools: [Download Repo ZIP] [Original HTTPS Page] |
1 parent 2261a3e commit 2224fff
3 files changed
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -1,6 +1,7 @@ | |||
| 1 | 1 | import unittest | |
| 2 | 2 | from contextlib import contextmanager | |
| 3 | 3 | import asyncio | |
| 4 | + import contextlib | ||
| 4 | 5 | import os | |
| 5 | 6 | import textwrap | |
| 6 | 7 | import importlib | |
@@ -68,6 +69,112 @@ def get_all_awaited_by(pid): | |||
| 68 | 69 | class TestGetStackTrace(unittest.TestCase): | |
| 69 | 70 | maxDiff = None | |
| 70 | 71 | ||
| 72 | + def _extract_coroutine_funcnames(self, trace): | ||
| 73 | + return { | ||
| 74 | + task.task_name: [ | ||
| 75 | + frame.funcname.rpartition(".")[2] | ||
| 76 | + for coro in task.coroutine_stack | ||
| 77 | + for frame in coro.call_stack | ||
| 78 | + ] | ||
| 79 | + for info in trace | ||
| 80 | + for task in info.awaited_by | ||
| 81 | + } | ||
| 82 | + | ||
| 83 | + @skip_if_not_supported | ||
| 84 | + def test_async_generator_stack_is_not_truncated(self): | ||
| 85 | + # gh-159620 | ||
| 86 | + async def agen(): | ||
| 87 | + await asyncio.sleep(10_000) | ||
| 88 | + yield | ||
| 89 | + | ||
| 90 | + async def agen_with_cleanup(): | ||
| 91 | + try: | ||
| 92 | + yield | ||
| 93 | + finally: | ||
| 94 | + await asyncio.sleep(10_000) | ||
| 95 | + | ||
| 96 | + async def outer_agen(): | ||
| 97 | + async for _ in agen(): | ||
| 98 | + yield | ||
| 99 | + | ||
| 100 | + async def consumer(gen): | ||
| 101 | + async for _ in gen: | ||
| 102 | + pass | ||
| 103 | + | ||
| 104 | + async def main(): | ||
| 105 | + g = agen_with_cleanup() | ||
| 106 | + await anext(g) | ||
| 107 | + async_for_task = asyncio.create_task(consumer(agen()), name="async for") | ||
| 108 | + nested_task = asyncio.create_task( | ||
| 109 | + consumer(outer_agen()), name="nested async for") | ||
| 110 | + anext_task = asyncio.create_task(anext(agen()), name="anext") | ||
| 111 | + aclose_task = asyncio.create_task(g.aclose(), name="aclose") | ||
| 112 | + await asyncio.sleep(0) | ||
| 113 | + return RemoteUnwinder(os.getpid()).get_all_awaited_by() | ||
| 114 | + | ||
| 115 | + stacks = self._extract_coroutine_funcnames(asyncio.run(main())) | ||
| 116 | + self.assertEqual(stacks["async for"], ["sleep", "agen", "consumer"]) | ||
| 117 | + self.assertEqual(stacks["nested async for"], | ||
| 118 | + ["sleep", "agen", "outer_agen", "consumer"]) | ||
| 119 | + self.assertEqual(stacks["anext"], ["sleep", "agen"]) | ||
| 120 | + self.assertEqual(stacks["aclose"], ["sleep", "agen_with_cleanup"]) | ||
| 121 | + | ||
| 122 | + @skip_if_not_supported | ||
| 123 | + def test_async_generator_waiter_in_async_stack_trace(self): | ||
| 124 | + # gh-159620 | ||
| 125 | + async def agen(task): | ||
| 126 | + await task | ||
| 127 | + yield | ||
| 128 | + | ||
| 129 | + async def main(): | ||
| 130 | + waiter_task = asyncio.create_task( | ||
| 131 | + anext(agen(asyncio.current_task())), name="anext") | ||
| 132 | + await asyncio.sleep(0) | ||
| 133 | + return RemoteUnwinder(os.getpid()).get_async_stack_trace() | ||
| 134 | + | ||
| 135 | + stacks = self._extract_coroutine_funcnames(asyncio.run(main())) | ||
| 136 | + self.assertEqual(stacks["anext"], ["agen"]) | ||
| 137 | + | ||
| 138 | + @skip_if_not_supported | ||
| 139 | + def test_async_generator_running_task_stack(self): | ||
| 140 | + # gh-159620 | ||
| 141 | + async def agen(): | ||
| 142 | + yield RemoteUnwinder(os.getpid()).get_async_stack_trace() | ||
| 143 | + | ||
| 144 | + async def main(): | ||
| 145 | + return await asyncio.create_task(anext(agen()), name="anext") | ||
| 146 | + | ||
| 147 | + stacks = self._extract_coroutine_funcnames(asyncio.run(main())) | ||
| 148 | + self.assertEqual(stacks["anext"], ["agen"]) | ||
| 149 | + | ||
| 150 | + @skip_if_not_supported | ||
| 151 | + def test_class_named_like_async_generator_wrapper_is_not_walked(self): | ||
| 152 | + # gh-159620 | ||
| 153 | + async def decoy(): | ||
| 154 | + await asyncio.sleep(10_000) | ||
| 155 | + | ||
| 156 | + class async_generator_asend: | ||
| 157 | + __slots__ = ("gen",) | ||
| 158 | + def __init__(self, gen): | ||
| 159 | + self.gen = gen | ||
| 160 | + def __await__(self): | ||
| 161 | + return self | ||
| 162 | + def __next__(self): | ||
| 163 | + return None | ||
| 164 | + | ||
| 165 | + async def consumer(aw): | ||
| 166 | + await aw | ||
| 167 | + | ||
| 168 | + async def main(): | ||
| 169 | + with contextlib.closing(decoy()) as gen: | ||
| 170 | + consumer_task = asyncio.create_task( | ||
| 171 | + consumer(async_generator_asend(gen)), name="consumer") | ||
| 172 | + await asyncio.sleep(0) | ||
| 173 | + return RemoteUnwinder(os.getpid()).get_all_awaited_by() | ||
| 174 | + | ||
| 175 | + stacks = self._extract_coroutine_funcnames(asyncio.run(main())) | ||
| 176 | + self.assertEqual(stacks["consumer"], ["consumer"]) | ||
| 177 | + | ||
| 71 | 178 | @skip_if_not_supported | |
| 72 | 179 | def test_long_task_name_is_truncated(self): | |
| 73 | 180 | # gh-157788 | |
@@ -935,22 +1042,10 @@ async def main(): | |||
| 935 | 1042 | for _ in busy_retry(SHORT_TIMEOUT): | |
| 936 | 1043 | try: | |
| 937 | 1044 | all_awaited_by = get_all_awaited_by(p.pid) | |
| 938 | - except RuntimeError as re: | ||
| 939 | - # This call reads a linked list in another process with | ||
| 940 | - # no synchronization. That occasionally leads to invalid | ||
| 941 | - # reads. Here we avoid making the test flaky. | ||
| 942 | - msg = str(re) | ||
| 943 | - if msg.startswith("Task list appears corrupted"): | ||
| 944 | - continue | ||
| 945 | - elif msg.startswith( | ||
| 946 | - "Invalid linked list structure reading remote memory" | ||
| 947 | - ): | ||
| 948 | - continue | ||
| 949 | - elif msg.startswith("Unknown error reading memory"): | ||
| 950 | - continue | ||
| 951 | - elif msg.startswith("Unhandled frame owner"): | ||
| 952 | - continue | ||
| 953 | - raise # Unrecognized exception, safest not to ignore it | ||
| 1045 | + except RuntimeError: | ||
| 1046 | + # Tasks and awaited objects can change while we read | ||
| 1047 | + # the running process. Retry with a fresh sample. | ||
| 1048 | + continue | ||
| 954 | 1049 | else: | |
| 955 | 1050 | break | |
| 956 | 1051 | # expected: a list of two elements: 1 thread, 1 interp | |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -0,0 +1,2 @@ | |||
| 1 | + Fix ``python -m asyncio ps`` and ``pstree`` stacks stopping at | ||
| 2 | + asynchronous generators. Patch by Timofei Ivankov. | ||
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -1426,11 +1426,63 @@ parse_task_name( | |||
| 1426 | 1426 | } | |
| 1427 | 1427 | ||
| 1428 | 1428 | ||
| 1429 | + // gh-159620 | ||
| 1430 | + /* Return in *coro_addr the coroutine or async generator to walk for the | ||
| 1431 | + awaitable at addr, or 0 if it is neither (for example, a future). */ | ||
| 1432 | + static int | ||
| 1433 | + get_coro_like_address( | ||
| 1434 | + RemoteUnwinderObject *unwinder, | ||
| 1435 | + uintptr_t addr, | ||
| 1436 | + uintptr_t *coro_addr | ||
| 1437 | + ) { | ||
| 1438 | + *coro_addr = 0; | ||
| 1439 | + | ||
| 1440 | + // Skip user classes that share a name with the generator types below | ||
| 1441 | + uintptr_t type_addr; | ||
| 1442 | + unsigned long tp_flags; | ||
| 1443 | + if (read_ptr(unwinder, addr + (uintptr_t)unwinder->debug_offsets.pyobject.ob_type, | ||
| 1444 | + &type_addr) < 0 | ||
| 1445 | + || _Py_RemoteDebug_PagedReadRemoteMemory(&unwinder->handle, | ||
| 1446 | + type_addr + (uintptr_t)unwinder->debug_offsets.type_object.tp_flags, | ||
| 1447 | + sizeof(tp_flags), &tp_flags) < 0) | ||
| 1448 | + { | ||
| 1449 | + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read awaitable type"); | ||
| 1450 | + return -1; | ||
| 1451 | + } | ||
| 1452 | + | ||
| 1453 | + if (tp_flags & Py_TPFLAGS_HEAPTYPE) { | ||
| 1454 | + return 0; | ||
| 1455 | + } | ||
| 1456 | + | ||
| 1457 | + uintptr_t name_addr; | ||
| 1458 | + char name[32] = {0}; | ||
| 1459 | + if (read_ptr(unwinder, type_addr + (uintptr_t)unwinder->debug_offsets.type_object.tp_name, | ||
| 1460 | + &name_addr) < 0 | ||
| 1461 | + || _Py_RemoteDebug_PagedReadRemoteMemory(&unwinder->handle, name_addr, | ||
| 1462 | + sizeof(name) - 1, name) < 0) | ||
| 1463 | + { | ||
| 1464 | + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read awaitable type name"); | ||
| 1465 | + return -1; | ||
| 1466 | + } | ||
| 1467 | + | ||
| 1468 | + if (strcmp(name, "coroutine") == 0) { | ||
| 1469 | + *coro_addr = addr; | ||
| 1470 | + } | ||
| 1471 | + else if (strcmp(name, "async_generator_asend") == 0 | ||
| 1472 | + || strcmp(name, "async_generator_athrow") == 0) | ||
| 1473 | + { | ||
| 1474 | + // asend and athrow are separate wrapper objects: read the pointer | ||
| 1475 | + // to their parent async generator, stored right after PyObject_HEAD | ||
| 1476 | + return read_ptr(unwinder, addr + (uintptr_t)unwinder->debug_offsets.pyobject.size, | ||
| 1477 | + coro_addr); | ||
| 1478 | + } | ||
| 1479 | + return 0; | ||
| 1480 | + } | ||
| 1481 | + | ||
| 1429 | 1482 | static int | |
| 1430 | 1483 | get_awaited_coro_address( | |
| 1431 | 1484 | RemoteUnwinderObject *unwinder, | |
| 1432 | 1485 | uintptr_t gi_iframe_addr, | |
| 1433 | - uintptr_t gen_type_addr, | ||
| 1434 | 1486 | uintptr_t *next_coro | |
| 1435 | 1487 | ) { | |
| 1436 | 1488 | *next_coro = 0; | |
@@ -1458,41 +1510,14 @@ get_awaited_coro_address( | |||
| 1458 | 1510 | uintptr_t stackpointer_addr = GET_MEMBER_NO_TAG(uintptr_t, iframe, unwinder->debug_offsets.interpreter_frame.stackpointer); | |
| 1459 | 1511 | ||
| 1460 | 1512 | if ((void*)stackpointer_addr != NULL) { | |
| 1461 | - uintptr_t gi_await_addr; | ||
| 1462 | 1513 | err = read_py_ptr( | |
| 1463 | 1514 | unwinder, | |
| 1464 | 1515 | stackpointer_addr - sizeof(void*), | |
| 1465 | - &gi_await_addr); | ||
| 1516 | + next_coro); | ||
| 1466 | 1517 | if (err) { | |
| 1467 | 1518 | set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read gi_await address"); | |
| 1468 | 1519 | return -1; | |
| 1469 | 1520 | } | |
| 1470 | - | ||
| 1471 | - if ((void*)gi_await_addr != NULL) { | ||
| 1472 | - uintptr_t gi_await_addr_type_addr; | ||
| 1473 | - err = read_ptr( | ||
| 1474 | - unwinder, | ||
| 1475 | - gi_await_addr + (uintptr_t)unwinder->debug_offsets.pyobject.ob_type, | ||
| 1476 | - &gi_await_addr_type_addr); | ||
| 1477 | - if (err) { | ||
| 1478 | - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read gi_await type address"); | ||
| 1479 | - return -1; | ||
| 1480 | - } | ||
| 1481 | - | ||
| 1482 | - if (gen_type_addr == gi_await_addr_type_addr) { | ||
| 1483 | - /* This needs an explanation. We always start with parsing | ||
| 1484 | - native coroutine / generator frames. Ultimately they | ||
| 1485 | - are awaiting on something. That something can be | ||
| 1486 | - a native coroutine frame or... an iterator. | ||
| 1487 | - If it's the latter -- we can't continue building | ||
| 1488 | - our chain. So the condition to bail out of this is | ||
| 1489 | - to do that when the type of the current coroutine | ||
| 1490 | - doesn't match the type of whatever it points to | ||
| 1491 | - in its cr_await. | ||
| 1492 | - */ | ||
| 1493 | - *next_coro = gi_await_addr; | ||
| 1494 | - } | ||
| 1495 | - } | ||
| 1496 | 1521 | } | |
| 1497 | 1522 | ||
| 1498 | 1523 | return 0; | |
@@ -1515,6 +1540,13 @@ parse_coro_chain( | |||
| 1515 | 1540 | return -1; | |
| 1516 | 1541 | } | |
| 1517 | 1542 | ||
| 1543 | + if (get_coro_like_address(unwinder, coro_address, &coro_address) < 0) { | ||
| 1544 | + return -1; | ||
| 1545 | + } | ||
| 1546 | + if ((void*)coro_address == NULL) { | ||
| 1547 | + return 0; | ||
| 1548 | + } | ||
| 1549 | + | ||
| 1518 | 1550 | // Read the entire generator object at once | |
| 1519 | 1551 | char gen_object[SIZEOF_GEN_OBJ]; | |
| 1520 | 1552 | int err = _Py_RemoteDebug_PagedReadRemoteMemory( | |
@@ -1532,8 +1564,6 @@ parse_coro_chain( | |||
| 1532 | 1564 | return 0; | |
| 1533 | 1565 | } | |
| 1534 | 1566 | ||
| 1535 | - uintptr_t gen_type_addr = GET_MEMBER(uintptr_t, gen_object, unwinder->debug_offsets.pyobject.ob_type); | ||
| 1536 | - | ||
| 1537 | 1567 | PyObject* name = NULL; | |
| 1538 | 1568 | ||
| 1539 | 1569 | // Parse the previous frame using the gi_iframe from local copy | |
@@ -1560,8 +1590,7 @@ parse_coro_chain( | |||
| 1560 | 1590 | return 0; | |
| 1561 | 1591 | } | |
| 1562 | 1592 | ||
| 1563 | - if (get_awaited_coro_address(unwinder, gi_iframe_addr, gen_type_addr, | ||
| 1564 | - &coro_address) < 0) { | ||
| 1593 | + if (get_awaited_coro_address(unwinder, gi_iframe_addr, &coro_address) < 0) { | ||
| 1565 | 1594 | return -1; | |
| 1566 | 1595 | } | |
| 1567 | 1596 | } | |
@@ -2034,6 +2063,10 @@ get_task_code_object(RemoteUnwinderObject *unwinder, uintptr_t task_addr, uintpt | |||
| 2034 | 2063 | return -1; | |
| 2035 | 2064 | } | |
| 2036 | 2065 | ||
| 2066 | + if (get_coro_like_address(unwinder, running_coro_addr, &running_coro_addr) < 0) { | ||
| 2067 | + return -1; | ||
| 2068 | + } | ||
| 2069 | + | ||
| 2037 | 2070 | // note: genobject's gi_iframe is an embedded struct so the address to | |
| 2038 | 2071 | // the offset leads directly to its first field: f_executable | |
| 2039 | 2072 | if (read_py_ptr( | |
| Back | FazBrowse Home | New Git URL |
0 commit comments