Commit c5728b2
authored
fix(perf): avoid repeated scan of entire venv via packages_distributions() at import time (#16579)
packages_distributions() scans every installed package in the
environment to build a complete module-to-distribution mapping. In large
venvs (500+ packages, common with many google-cloud-* libs), this causes
multi-second import delays for google.api_core and every library that
depends on it.
This PR contains 2 changes:
- Wrap `packages_distributions()` with functools.cache so the expensive
O(n) scan happens at most once per process.
- Defer the package label resolution in `check_python_version()` so it
only runs when a warning is actually emitted, not on the common happy
path of a supported Python version.
Thank you for opening a Pull Request! Before submitting your PR, there
are a few things you can do to make sure it goes smoothly:
- [x] Make sure to open an issue as a
[bug/issue](https://github.com/googleapis/google-cloud-python/issues)
before writing your code! That way we can discuss the change, evaluate
designs, and agree on the general idea
- [ ] Ensure the tests and linter pass
- [x] Code coverage does not decrease (if any source code was changed)
- [x] Appropriate docs were updated (if necessary)
Fixes #15015 and #16552.1 parent f86bdfd commit c5728b2
1 file changed
Lines changed: 15 additions & 5 deletions
Lines changed: 15 additions & 5 deletions
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -16,6 +16,7 @@ | |||
| 16 | 16 | ||
| 17 | 17 | import datetime | |
| 18 | 18 | import enum | |
| 19 | + import functools | ||
| 19 | 20 | import logging | |
| 20 | 21 | import warnings | |
| 21 | 22 | import sys | |
@@ -152,15 +153,16 @@ def _get_pypi_package_name(module_name): # pragma: NO COVER | |||
| 152 | 153 | else: | |
| 153 | 154 | from importlib import metadata | |
| 154 | 155 | ||
| 156 | + @functools.cache | ||
| 157 | + def _cached_packages_distributions(): | ||
| 158 | + return metadata.packages_distributions() | ||
| 159 | + | ||
| 155 | 160 | def _get_pypi_package_name(module_name): | |
| 156 | 161 | """Determine the PyPI package name for a given module name.""" | |
| 157 | 162 | try: | |
| 158 | - # Get the mapping of modules to distributions | ||
| 159 | - module_to_distributions = metadata.packages_distributions() | ||
| 163 | + module_to_distributions = _cached_packages_distributions() | ||
| 160 | 164 | ||
| 161 | - # Check if the module is found in the mapping | ||
| 162 | 165 | if module_name in module_to_distributions: # pragma: NO COVER | |
| 163 | - # The value is a list of distribution names, take the first one | ||
| 164 | 166 | return module_to_distributions[module_name][0] | |
| 165 | 167 | except Exception as e: # pragma: NO COVER | |
| 166 | 168 | _LOGGER.info( | |
@@ -195,7 +197,6 @@ def check_python_version( | |||
| 195 | 197 | The support status of the current Python version. | |
| 196 | 198 | """ | |
| 197 | 199 | today = today or datetime.date.today() | |
| 198 | - package_label, _ = _get_distribution_and_import_packages(package) | ||
| 199 | 200 | ||
| 200 | 201 | python_version = sys.version_info | |
| 201 | 202 | version_tuple = (python_version.major, python_version.minor) | |
@@ -221,7 +222,14 @@ def min_python(date: datetime.date) -> str: | |||
| 221 | 222 | return f"{version[0]}.{version[1]}" | |
| 222 | 223 | return "at a currently supported version [https://devguide.python.org/versions]" | |
| 223 | 224 | ||
| 225 | + # Resolve the pretty package label lazily so we avoid any work on | ||
| 226 | + # the happy path (supported Python version, no warning needed). | ||
| 227 | + def get_package_label(): | ||
| 228 | + label, _ = _get_distribution_and_import_packages(package) | ||
| 229 | + return label | ||
| 230 | + | ||
| 224 | 231 | if gapic_end < today: | |
| 232 | + package_label = get_package_label() | ||
| 225 | 233 | message = _flatten_message( | |
| 226 | 234 | f""" | |
| 227 | 235 | You are using a non-supported Python version ({py_version_str}). | |
@@ -236,6 +244,7 @@ def min_python(date: datetime.date) -> str: | |||
| 236 | 244 | ||
| 237 | 245 | eol_date = version_info.python_eol + EOL_GRACE_PERIOD | |
| 238 | 246 | if eol_date <= today <= gapic_end: | |
| 247 | + package_label = get_package_label() | ||
| 239 | 248 | message = _flatten_message( | |
| 240 | 249 | f""" | |
| 241 | 250 | You are using a Python version ({py_version_str}) | |
@@ -250,6 +259,7 @@ def min_python(date: datetime.date) -> str: | |||
| 250 | 259 | return PythonVersionStatus.PYTHON_VERSION_EOL | |
| 251 | 260 | ||
| 252 | 261 | if gapic_deprecation <= today <= gapic_end: | |
| 262 | + package_label = get_package_label() | ||
| 253 | 263 | message = _flatten_message( | |
| 254 | 264 | f""" | |
| 255 | 265 | You are using a Python version ({py_version_str}) which Google will | |
0 commit comments