FazBrowse GitHub Viewer | Trending |
URL:
| Home
Tools: [Download Repo ZIP]   [Original HTTPS Page]

compute multiline maps cheaply in the sysmon core by reaperhulk · Pull Request #2220 · coveragepy/coveragepy · GitHub

Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension .py  (4) .rst  (1) All 2 file types selected
Viewed files
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Unified
Split
Hide whitespace
Diff view
Unified
Split
Hide whitespace
3 changes: 2 additions & 1 deletion CHANGES.rst
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Original file line number Diff line number Diff line change
Expand Up @@ -30,13 +30,14 @@ Unreleased

- A number of performance improvements thanks to Paul Kehrer, in pull requests
`2213 <pull 2213_>`_, `2214 <pull 2214_>`_, `2215 <pull 2215_>`_, `2216
<pull 2216_>`_, and `2218 <pull 2218_>`_.
<pull 2216_>`_, `2218 <pull 2218_>`_, and `2220 <pull 2220_>`_.

.. _pull 2213: https://github.com/coveragepy/coveragepy/pull/2213
.. _pull 2214: https://github.com/coveragepy/coveragepy/pull/2214
.. _pull 2215: https://github.com/coveragepy/coveragepy/pull/2215
.. _pull 2216: https://github.com/coveragepy/coveragepy/pull/2216
.. _pull 2218: https://github.com/coveragepy/coveragepy/pull/2218
.. _pull 2220: https://github.com/coveragepy/coveragepy/pull/2220
.. _pull 2224: https://github.com/coveragepy/coveragepy/pull/2224

.. start-releases
Expand Down
53 changes: 45 additions & 8 deletions coverage/parser.py
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,45 @@
os = isolate_module(os)


def multiline_map_from_tokens(tokens: Iterable[tokenize.TokenInfo]) -> dict[TLineNo, TLineNo]:
"""Compute the multiline map from a stream of tokens.

The result maps line numbers in multi-line statements to the first line
number of their statement. This is the only place the map is computed:
`PythonParser._raw_parse` uses it for parsing and reporting, and the
sys.monitoring core uses `multiline_map_from_text` to get the map without
paying for a full parse during measurement.

"""
multiline_map: dict[TLineNo, TLineNo] = {}
# The line number of the first line in a multi-line statement.
first_line = 0
for toktype, ttext, (slineno, _), (elineno, _), _ in tokens:
if toktype == token.NEWLINE:
if first_line and elineno != first_line:
# We're at the end of a line, and we've ended on a
# different line than the first line of the statement,
# so record a multi-line range.
for l in range(first_line, elineno + 1):
multiline_map[l] = first_line
first_line = 0
if ttext.strip() and toktype != tokenize.COMMENT:
# A non-white-space token, the first in a statement.
if not first_line:
first_line = slineno
return multiline_map


def multiline_map_from_text(text: str) -> dict[TLineNo, TLineNo]:
"""Compute just the multiline map for `text`, without a full parse.

Can raise tokenize.TokenError, IndentationError, or SyntaxError if the
text isn't parsable as Python.

"""
return multiline_map_from_tokens(generate_tokens(text))


class PythonParser:
"""Parse code to find executable lines, excluded lines, etc.

Expand Down Expand Up @@ -148,8 +187,9 @@ def _raw_parse(self) -> None:
nesting: int = 0

assert self.text is not None
tokgen = generate_tokens(self.text)
for toktype, ttext, (slineno, _), (elineno, _), ltext in tokgen:
tokens = list(generate_tokens(self.text))
self.multiline_map = multiline_map_from_tokens(tokens)
for toktype, ttext, (slineno, _), (elineno, _), ltext in tokens:
if self.show_tokens: # pragma: debugging
print(
"%10s %5s %-20r %r"
Expand Down Expand Up @@ -179,12 +219,9 @@ def _raw_parse(self) -> None:
elif ttext in ")]}":
nesting -= 1
elif toktype == token.NEWLINE:
if first_line and elineno != first_line:
# We're at the end of a line, and we've ended on a
# different line than the first line of the statement,
# so record a multi-line range.
for l in range(first_line, elineno + 1):
self.multiline_map[l] = first_line
# multiline_map_from_tokens() has already recorded this
# statement's lines; we only track first_line here for the
# exclusion logic.
first_line = 0

if ttext.strip() and toktype != tokenize.COMMENT:
Expand Down
37 changes: 24 additions & 13 deletions coverage/sysmon.py
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
import os.path
import sys
import threading
import tokenize
import traceback
from collections.abc import Callable
from dataclasses import dataclass
Expand All @@ -21,9 +22,10 @@
from coverage import env
from coverage.bytecode import TBranchTrails, always_jumps, branch_trails, bytes_to_lines
from coverage.debug import short_filename, short_stack
from coverage.exceptions import NoSource, NotPython
from coverage.exceptions import NoSource
from coverage.misc import isolate_module
from coverage.parser import PythonParser
from coverage.parser import multiline_map_from_text
from coverage.python import get_python_source
from coverage.types import (
AnyCallable,
TFileDisposition,
Expand Down Expand Up @@ -223,6 +225,9 @@ def __init__(self) -> None:
# Map filename:__name__ -> set(id(code_object))
self.filename_code_ids: dict[str, set[int]] = collections.defaultdict(set)

# Map filename -> multiline map, so each file is parsed at most once.
self.multiline_maps: dict[str, dict[TLineNo, TLineNo]] = {}

self.sysmon_on = False
self.lock = threading.Lock()

Expand Down Expand Up @@ -458,7 +463,7 @@ def sysmon_branch_either(
if not code_info.branch_trails:
if self.stats is not None:
self.stats["branch_trails"] += 1
multiline_map = get_multiline_map(code.co_filename)
multiline_map = self.get_multiline_map(code.co_filename)
code_info.branch_trails = branch_trails(code, multiline_map=multiline_map)
code_info.always_jumps = always_jumps(code)
# log(f"branch_trails for {code}:\n{ppformat(code_info.branch_trails)}")
Expand Down Expand Up @@ -495,20 +500,26 @@ def sysmon_branch_either(

return DISABLE

def get_multiline_map(self, filename: str) -> dict[TLineNo, TLineNo]:
"""Get the multiline map for `filename`, computing it at most once."""
multiline_map = self.multiline_maps.get(filename)
if multiline_map is None:
multiline_map = self.multiline_maps[filename] = compute_multiline_map(filename)
return multiline_map


@functools.lru_cache(maxsize=20)
def get_multiline_map(filename: str) -> dict[TLineNo, TLineNo]:
"""Get a PythonParser for the given filename, cached."""
def compute_multiline_map(filename: str) -> dict[TLineNo, TLineNo]:
"""Tokenize `filename` and return its multiline map."""
try:
parser = PythonParser(filename=filename)
parser.parse_source()
except NotPython:
text = get_python_source(filename)
except (OSError, NoSource):
# This can happen if open() in python.py fails.
return {}
try:
return multiline_map_from_text(text)
except (tokenize.TokenError, IndentationError, SyntaxError):
# The file was not Python. This can happen when the code object refers
# to an original non-Python source file, like a Jinja template.
# In that case, just return an empty map, which might lead to slightly
# wrong branch coverage, but we don't have any better option.
return {}
except NoSource:
# This can happen if open() in python.py fails.
return {}
return parser.multiline_map
171 changes: 170 additions & 1 deletion tests/test_parser.py
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Original file line number Diff line number Diff line change
Expand Up @@ -8,13 +8,15 @@
import ast
import re
import textwrap
import tokenize
from unittest import mock

import pytest

from coverage import env
from coverage.exceptions import NoSource, NotPython
from coverage.parser import PythonParser, is_constant_test_expr
from coverage.parser import PythonParser, is_constant_test_expr, multiline_map_from_text
from coverage.types import TLineNo

from tests.coveragetest import CoverageTest
from tests.helpers import arcz_to_arcs
Expand Down Expand Up @@ -1320,3 +1322,170 @@ def test_is_constant_test_expr(expr: str, ret: tuple[bool, bool]) -> None:
node = ast.parse(expr, mode="eval").body
print(ast.dump(node, indent=4))
assert is_constant_test_expr(node) == ret


class MultilineMapTest(CoverageTest):
"""Tests of multiline_map_from_text."""

run_in_temp_dir = False

def multiline_map(self, text: str) -> dict[TLineNo, TLineNo]:
"""Compute the multiline map of dedented `text`."""
return multiline_map_from_text(textwrap.dedent(text))

def test_single_line_statements_have_no_entries(self) -> None:
assert (
self.multiline_map("""\
a = 1
b = 2; c = 3
# a comment

def f(x):
return x
""")
== {}
)

def test_parenthesized_expression(self) -> None:
assert self.multiline_map("""\
x = (
1 +
2
)
y = 5
""") == {1: 1, 2: 1, 3: 1, 4: 1}

def test_backslash_continuation(self) -> None:
assert self.multiline_map("""\
total = 1 + \\
2
""") == {1: 1, 2: 1}

def test_triple_quoted_string(self) -> None:
assert self.multiline_map("""\
s = '''one
two
three'''
""") == {1: 1, 2: 1, 3: 1}

def test_multiline_signature(self) -> None:
assert self.multiline_map("""\
def f(
a,
b,
):
return a + b
""") == {1: 1, 2: 1, 3: 1, 4: 1}

def test_multiline_if_header(self) -> None:
assert self.multiline_map("""\
if (a and
b):
c = 1
""") == {1: 1, 2: 1}

def test_blank_and_comment_lines_inside_statement(self) -> None:
# Lines inside a multi-line statement belong to it, even blank lines
# and comment lines.
assert self.multiline_map("""\
x = [
1,

# two comes next
2,
]
""") == {lineno: 1 for lineno in range(1, 7)}

def test_statements_map_to_their_own_starts(self) -> None:
assert self.multiline_map("""\
a = (1 +
2)
b = 3
c = (4 +
5)
""") == {1: 1, 2: 1, 4: 4, 5: 4}

def test_unparsable_text_raises(self) -> None:
with pytest.raises(tokenize.TokenError):
multiline_map_from_text("x = (\n")

@pytest.mark.parametrize(
"text",
[
"""\
a = 1
b = 2
""",
"""\
x = (
1 +
2
)
y = 5
""",
"""\
def f(
a,
b=[1,
2],
):
return (a +
b)
""",
"""\
@decorator(
arg,
)
def f():
pass
""",
"""\
if (a and
b):
c = (1,
2)
""",
"""\
with (open('a') as fa,
open('b') as fb):
pass
""",
"""\
s = f'''one {x
+ 1} two
three'''
t = 4
""",
"""\
class C(
Base,
):
attr = 1
""",
"""\
result = [x
for x in items
if x > 0
]
""",
"""\
match (command,
arg):
case (1, 2):
pass
""",
"""\
total = 1 + \\
2 + \\
3
""",
],
)
def test_agrees_with_python_parser(self, text: str) -> None:
# PythonParser._raw_parse gets its multiline map from the same code,
# but through a different path (tokenizing to a list first). The two
# must never drift apart: the sys.monitoring core uses this map to
# attribute branch arcs to the lines that reports are keyed by.
parser = PythonParser(text=textwrap.dedent(text))
parser.parse_source()
assert multiline_map_from_text(text) == parser.multiline_map
Loading
Loading

Back | FazBrowse Home | New Git URL