coveragepy
diff --git a/‎CHANGES.rst‎
Lines changed: 2 additions & 1 deletion b/‎CHANGES.rst‎
Lines changed: 2 additions & 1 deletion
diff --git a/‎coverage/parser.py‎
Lines changed: 45 additions & 8 deletions b/‎coverage/parser.py‎
Lines changed: 45 additions & 8 deletions
diff --git a/‎coverage/sysmon.py‎
Lines changed: 24 additions & 13 deletions b/‎coverage/sysmon.py‎
Lines changed: 24 additions & 13 deletions
diff --git a/‎tests/test_parser.py‎
Lines changed: 170 additions & 1 deletion b/‎tests/test_parser.py‎
Lines changed: 170 additions & 1 deletion
@@ -30,13 +30,14 @@ Unreleased
 
 - A number of performance improvements thanks to Paul Kehrer, in pull requests
   `2213 <pull 2213_>`_, `2214 <pull 2214_>`_, `2215 <pull 2215_>`_, `2216
-  <pull 2216_>`_, and `2218 <pull 2218_>`_.
+  <pull 2216_>`_, `2218 <pull 2218_>`_, and `2220 <pull 2220_>`_.
 
 .. _pull 2213: https://github.com/coveragepy/coveragepy/pull/2213
 .. _pull 2214: https://github.com/coveragepy/coveragepy/pull/2214
 .. _pull 2215: https://github.com/coveragepy/coveragepy/pull/2215
 .. _pull 2216: https://github.com/coveragepy/coveragepy/pull/2216
 .. _pull 2218: https://github.com/coveragepy/coveragepy/pull/2218
+.. _pull 2220: https://github.com/coveragepy/coveragepy/pull/2220
 .. _pull 2224: https://github.com/coveragepy/coveragepy/pull/2224
 
 .. start-releases
 
@@ -26,6 +26,45 @@
 os = isolate_module(os)
 
 
+def multiline_map_from_tokens(tokens: Iterable[tokenize.TokenInfo]) -> dict[TLineNo, TLineNo]:
+    """Compute the multiline map from a stream of tokens.
+
+    The result maps line numbers in multi-line statements to the first line
+    number of their statement.  This is the only place the map is computed:
+    `PythonParser._raw_parse` uses it for parsing and reporting, and the
+    sys.monitoring core uses `multiline_map_from_text` to get the map without
+    paying for a full parse during measurement.
+
+    """
+    multiline_map: dict[TLineNo, TLineNo] = {}
+    # The line number of the first line in a multi-line statement.
+    first_line = 0
+    for toktype, ttext, (slineno, _), (elineno, _), _ in tokens:
+        if toktype == token.NEWLINE:
+            if first_line and elineno != first_line:
+                # We're at the end of a line, and we've ended on a
+                # different line than the first line of the statement,
+                # so record a multi-line range.
+                for l in range(first_line, elineno + 1):
+                    multiline_map[l] = first_line
+            first_line = 0
+        if ttext.strip() and toktype != tokenize.COMMENT:
+            # A non-white-space token, the first in a statement.
+            if not first_line:
+                first_line = slineno
+    return multiline_map
+
+
+def multiline_map_from_text(text: str) -> dict[TLineNo, TLineNo]:
+    """Compute just the multiline map for `text`, without a full parse.
+
+    Can raise tokenize.TokenError, IndentationError, or SyntaxError if the
+    text isn't parsable as Python.
+
+    """
+    return multiline_map_from_tokens(generate_tokens(text))
+
+
 class PythonParser:
     """Parse code to find executable lines, excluded lines, etc.
 
@@ -148,8 +187,9 @@ def _raw_parse(self) -> None:
         nesting: int = 0
 
         assert self.text is not None
-        tokgen = generate_tokens(self.text)
-        for toktype, ttext, (slineno, _), (elineno, _), ltext in tokgen:
+        tokens = list(generate_tokens(self.text))
+        self.multiline_map = multiline_map_from_tokens(tokens)
+        for toktype, ttext, (slineno, _), (elineno, _), ltext in tokens:
             if self.show_tokens:  # pragma: debugging
                 print(
                     "%10s %5s %-20r %r"
@@ -179,12 +219,9 @@ def _raw_parse(self) -> None:
                 elif ttext in ")]}":
                     nesting -= 1
             elif toktype == token.NEWLINE:
-                if first_line and elineno != first_line:
-                    # We're at the end of a line, and we've ended on a
-                    # different line than the first line of the statement,
-                    # so record a multi-line range.
-                    for l in range(first_line, elineno + 1):
-                        self.multiline_map[l] = first_line
+                # multiline_map_from_tokens() has already recorded this
+                # statement's lines; we only track first_line here for the
+                # exclusion logic.
                 first_line = 0
 
             if ttext.strip() and toktype != tokenize.COMMENT:
 
@@ -12,6 +12,7 @@
 import os.path
 import sys
 import threading
+import tokenize
 import traceback
 from collections.abc import Callable
 from dataclasses import dataclass
@@ -21,9 +22,10 @@
 from coverage import env
 from coverage.bytecode import TBranchTrails, always_jumps, branch_trails, bytes_to_lines
 from coverage.debug import short_filename, short_stack
-from coverage.exceptions import NoSource, NotPython
+from coverage.exceptions import NoSource
 from coverage.misc import isolate_module
-from coverage.parser import PythonParser
+from coverage.parser import multiline_map_from_text
+from coverage.python import get_python_source
 from coverage.types import (
     AnyCallable,
     TFileDisposition,
@@ -223,6 +225,9 @@ def __init__(self) -> None:
         # Map filename:__name__ -> set(id(code_object))
         self.filename_code_ids: dict[str, set[int]] = collections.defaultdict(set)
 
+        # Map filename -> multiline map, so each file is parsed at most once.
+        self.multiline_maps: dict[str, dict[TLineNo, TLineNo]] = {}
+
         self.sysmon_on = False
         self.lock = threading.Lock()
 
@@ -458,7 +463,7 @@ def sysmon_branch_either(
         if not code_info.branch_trails:
             if self.stats is not None:
                 self.stats["branch_trails"] += 1
-            multiline_map = get_multiline_map(code.co_filename)
+            multiline_map = self.get_multiline_map(code.co_filename)
             code_info.branch_trails = branch_trails(code, multiline_map=multiline_map)
             code_info.always_jumps = always_jumps(code)
             # log(f"branch_trails for {code}:\n{ppformat(code_info.branch_trails)}")
@@ -495,20 +500,26 @@ def sysmon_branch_either(
 
         return DISABLE
 
+    def get_multiline_map(self, filename: str) -> dict[TLineNo, TLineNo]:
+        """Get the multiline map for `filename`, computing it at most once."""
+        multiline_map = self.multiline_maps.get(filename)
+        if multiline_map is None:
+            multiline_map = self.multiline_maps[filename] = compute_multiline_map(filename)
+        return multiline_map
+
 
-@functools.lru_cache(maxsize=20)
-def get_multiline_map(filename: str) -> dict[TLineNo, TLineNo]:
-    """Get a PythonParser for the given filename, cached."""
+def compute_multiline_map(filename: str) -> dict[TLineNo, TLineNo]:
+    """Tokenize `filename` and return its multiline map."""
     try:
-        parser = PythonParser(filename=filename)
-        parser.parse_source()
-    except NotPython:
+        text = get_python_source(filename)
+    except (OSError, NoSource):
+        # This can happen if open() in python.py fails.
+        return {}
+    try:
+        return multiline_map_from_text(text)
+    except (tokenize.TokenError, IndentationError, SyntaxError):
         # The file was not Python. This can happen when the code object refers
         # to an original non-Python source file, like a Jinja template.
         # In that case, just return an empty map, which might lead to slightly
         # wrong branch coverage, but we don't have any better option.
         return {}
-    except NoSource:
-        # This can happen if open() in python.py fails.
-        return {}
-    return parser.multiline_map
 
@@ -8,13 +8,15 @@
 import ast
 import re
 import textwrap
+import tokenize
 from unittest import mock
 
 import pytest
 
 from coverage import env
 from coverage.exceptions import NoSource, NotPython
-from coverage.parser import PythonParser, is_constant_test_expr
+from coverage.parser import PythonParser, is_constant_test_expr, multiline_map_from_text
+from coverage.types import TLineNo
 
 from tests.coveragetest import CoverageTest
 from tests.helpers import arcz_to_arcs
@@ -1320,3 +1322,170 @@ def test_is_constant_test_expr(expr: str, ret: tuple[bool, bool]) -> None:
     node = ast.parse(expr, mode="eval").body
     print(ast.dump(node, indent=4))
     assert is_constant_test_expr(node) == ret
+
+
+class MultilineMapTest(CoverageTest):
+    """Tests of multiline_map_from_text."""
+
+    run_in_temp_dir = False
+
+    def multiline_map(self, text: str) -> dict[TLineNo, TLineNo]:
+        """Compute the multiline map of dedented `text`."""
+        return multiline_map_from_text(textwrap.dedent(text))
+
+    def test_single_line_statements_have_no_entries(self) -> None:
+        assert (
+            self.multiline_map("""\
+                a = 1
+                b = 2; c = 3
+                # a comment
+
+                def f(x):
+                    return x
+                """)
+            == {}
+        )
+
+    def test_parenthesized_expression(self) -> None:
+        assert self.multiline_map("""\
+            x = (
+                1 +
+                2
+            )
+            y = 5
+            """) == {1: 1, 2: 1, 3: 1, 4: 1}
+
+    def test_backslash_continuation(self) -> None:
+        assert self.multiline_map("""\
+            total = 1 + \\
+                2
+            """) == {1: 1, 2: 1}
+
+    def test_triple_quoted_string(self) -> None:
+        assert self.multiline_map("""\
+            s = '''one
+            two
+            three'''
+            """) == {1: 1, 2: 1, 3: 1}
+
+    def test_multiline_signature(self) -> None:
+        assert self.multiline_map("""\
+            def f(
+                a,
+                b,
+            ):
+                return a + b
+            """) == {1: 1, 2: 1, 3: 1, 4: 1}
+
+    def test_multiline_if_header(self) -> None:
+        assert self.multiline_map("""\
+            if (a and
+                    b):
+                c = 1
+            """) == {1: 1, 2: 1}
+
+    def test_blank_and_comment_lines_inside_statement(self) -> None:
+        # Lines inside a multi-line statement belong to it, even blank lines
+        # and comment lines.
+        assert self.multiline_map("""\
+            x = [
+                1,
+
+                # two comes next
+                2,
+            ]
+            """) == {lineno: 1 for lineno in range(1, 7)}
+
+    def test_statements_map_to_their_own_starts(self) -> None:
+        assert self.multiline_map("""\
+            a = (1 +
+                2)
+            b = 3
+            c = (4 +
+                5)
+            """) == {1: 1, 2: 1, 4: 4, 5: 4}
+
+    def test_unparsable_text_raises(self) -> None:
+        with pytest.raises(tokenize.TokenError):
+            multiline_map_from_text("x = (\n")
+
+    @pytest.mark.parametrize(
+        "text",
+        [
+            """\
+                a = 1
+                b = 2
+                """,
+            """\
+                x = (
+                    1 +
+                    2
+                )
+                y = 5
+                """,
+            """\
+                def f(
+                    a,
+                    b=[1,
+                        2],
+                ):
+                    return (a +
+                        b)
+                """,
+            """\
+                @decorator(
+                    arg,
+                )
+                def f():
+                    pass
+                """,
+            """\
+                if (a and
+                        b):
+                    c = (1,
+                        2)
+                """,
+            """\
+                with (open('a') as fa,
+                        open('b') as fb):
+                    pass
+                """,
+            """\
+                s = f'''one {x
+                    + 1} two
+                three'''
+                t = 4
+                """,
+            """\
+                class C(
+                    Base,
+                ):
+                    attr = 1
+                """,
+            """\
+                result = [x
+                    for x in items
+                    if x > 0
+                ]
+                """,
+            """\
+                match (command,
+                        arg):
+                    case (1, 2):
+                        pass
+                """,
+            """\
+                total = 1 + \\
+                    2 + \\
+                    3
+                """,
+        ],
+    )
+    def test_agrees_with_python_parser(self, text: str) -> None:
+        # PythonParser._raw_parse gets its multiline map from the same code,
+        # but through a different path (tokenizing to a list first).  The two
+        # must never drift apart: the sys.monitoring core uses this map to
+        # attribute branch arcs to the lines that reports are keyed by.
+        parser = PythonParser(text=textwrap.dedent(text))
+        parser.parse_source()
+        assert multiline_map_from_text(text) == parser.multiline_map