#!/usr/bin/env python
"""
Show dependency information for a module.
Usage:
python scripts/update_lib deps dis
python scripts/update_lib deps dataclasses
python scripts/update_lib deps dis --depth 2
python scripts/update_lib deps all # Show all modules' dependencies
"""
import argparse
import pathlib
import sys
sys.path.insert(0, str(pathlib.Path(__file__).parent.parent))
def get_all_modules(cpython_prefix: str) -> list[str]:
"""Get all top-level module names from cpython/Lib/.
Includes private modules (_*) that are not hard_deps of other modules.
Returns:
Sorted list of module names (without .py extension)
"""
from update_lib.deps import resolve_hard_dep_parent
lib_dir = pathlib.Path(cpython_prefix) / "Lib"
if not lib_dir.exists():
return []
modules = set()
for entry in lib_dir.iterdir():
# Skip hidden files
if entry.name.startswith("."):
continue
# Skip test directory
if entry.name == "test":
continue
if entry.is_file() and entry.suffix == ".py":
name = entry.stem
elif entry.is_dir() and (entry / "__init__.py").exists():
name = entry.name
else:
continue
# Skip modules that are hard_deps of other modules
# e.g., _pydatetime is a hard_dep of datetime, pydoc_data is a hard_dep of pydoc
if resolve_hard_dep_parent(name, cpython_prefix) is not None:
continue
modules.add(name)
return sorted(modules)
def format_deps_tree(
cpython_prefix: str,
lib_prefix: str,
max_depth: int,
*,
name: str | None = None,
soft_deps: set[str] | None = None,
hard_deps: set[str] | None = None,
_depth: int = 0,
_visited: set[str] | None = None,
_indent: str = "",
) -> list[str]:
"""Format soft dependencies as a tree with up-to-date status.
Args:
cpython_prefix: CPython directory prefix
lib_prefix: Local Lib directory prefix
max_depth: Maximum recursion depth
name: Module name (used to compute deps if soft_deps not provided)
soft_deps: Pre-computed soft dependencies (optional)
hard_deps: Hard dependencies to show under the module (root level only)
_depth: Current depth (internal)
_visited: Already visited modules (internal)
_indent: Current indentation (internal)
Returns:
List of formatted lines
"""
from update_lib.deps import (
get_lib_paths,
get_rust_deps,
get_soft_deps,
is_up_to_date,
)
lines = []
if _visited is None:
_visited = set()
# Compute deps from name if not provided
if soft_deps is None:
soft_deps = get_soft_deps(name, cpython_prefix) if name else set()
soft_deps = sorted(soft_deps)
if not soft_deps and not hard_deps:
return lines
# Separate up-to-date and outdated modules
up_to_date_deps = []
outdated_deps = []
dup_deps = []
for dep in soft_deps:
# Skip if library doesn't exist in cpython
lib_paths = get_lib_paths(dep, cpython_prefix)
if not any(p.exists() for p in lib_paths):
continue
up_to_date = is_up_to_date(dep, cpython_prefix, lib_prefix)
if up_to_date:
# Up-to-date modules collected compactly, no dup tracking needed
up_to_date_deps.append(dep)
elif dep in _visited:
# Only track dup for outdated modules
dup_deps.append(dep)
else:
outdated_deps.append(dep)
# Show outdated modules with expansion
for dep in outdated_deps:
dep_native = get_rust_deps(dep, cpython_prefix)
native_suffix = (
f" (native: {', '.join(sorted(dep_native))})" if dep_native else ""
)
lines.append(f"{_indent}- [ ] {dep}{native_suffix}")
_visited.add(dep)
# Show hard_deps under this module (only at root level, i.e., when hard_deps is provided)
if hard_deps and dep in soft_deps:
for hd in sorted(hard_deps):
hd_up_to_date = is_up_to_date(hd, cpython_prefix, lib_prefix)
hd_marker = "[x]" if hd_up_to_date else "[ ]"
lines.append(f"{_indent} - {hd_marker} {hd}")
hard_deps = None # Only show once
# Recurse if within depth limit
if _depth < max_depth - 1:
lines.extend(
format_deps_tree(
cpython_prefix,
lib_prefix,
max_depth,
name=dep,
_depth=_depth + 1,
_visited=_visited,
_indent=_indent + " ",
)
)
# Show duplicates compactly (only for outdated)
if dup_deps:
lines.append(f"{_indent}- [ ] {', '.join(dup_deps)}")
# Show up-to-date modules compactly on one line
if up_to_date_deps:
lines.append(f"{_indent}- [x] {', '.join(up_to_date_deps)}")
return lines
def format_deps(
name: str,
cpython_prefix: str,
lib_prefix: str,
max_depth: int = 10,
_visited: set[str] | None = None,
) -> list[str]:
"""Format all dependency information for a module.
Args:
name: Module name
cpython_prefix: CPython directory prefix
lib_prefix: Local Lib directory prefix
max_depth: Maximum recursion depth
_visited: Shared visited set for deduplication across modules
Returns:
List of formatted lines
"""
from update_lib.deps import (
DEPENDENCIES,
count_test_todos,
find_dependent_tests_tree,
get_lib_paths,
get_test_paths,
is_path_synced,
is_test_up_to_date,
resolve_hard_dep_parent,
)
if _visited is None:
_visited = set()
lines = []
# Resolve test_ prefix to module (e.g., test_pydoc -> pydoc)
if name.startswith("test_"):
module_name = name[5:] # strip "test_"
lines.append(f"(redirecting {name} -> {module_name})")
name = module_name
# Resolve hard_dep to parent module (e.g., pydoc_data -> pydoc)
parent = resolve_hard_dep_parent(name, cpython_prefix)
if parent:
lines.append(f"(redirecting {name} -> {parent})")
name = parent
# lib paths (only show existing)
lib_paths = get_lib_paths(name, cpython_prefix)
existing_lib_paths = [p for p in lib_paths if p.exists()]
for p in existing_lib_paths:
synced = is_path_synced(p, cpython_prefix, lib_prefix)
marker = "[x]" if synced else "[ ]"
lines.append(f"{marker} lib: {p}")
# test paths (only show existing)
test_paths = get_test_paths(name, cpython_prefix)
existing_test_paths = [p for p in test_paths if p.exists()]
for p in existing_test_paths:
test_name = p.stem if p.is_file() else p.name
synced = is_test_up_to_date(test_name, cpython_prefix, lib_prefix)
marker = "[x]" if synced else "[ ]"
todo_count = count_test_todos(test_name, lib_prefix)
todo_suffix = f" (TODO: {todo_count})" if todo_count > 0 else ""
lines.append(f"{marker} test: {p}{todo_suffix}")
# If no lib or test paths exist, module doesn't exist
if not existing_lib_paths and not existing_test_paths:
lines.append(f"(module '{name}' not found)")
return lines
# Collect all hard_deps (explicit from DEPENDENCIES + implicit from lib_paths)
dep_info = DEPENDENCIES.get(name, {})
explicit_hard_deps = dep_info.get("hard_deps", [])
# Get implicit hard_deps from lib_paths (e.g., _pydecimal.py for decimal)
all_hard_deps = set()
for hd in explicit_hard_deps:
# Remove .py extension if present
all_hard_deps.add(hd[:-3] if hd.endswith(".py") else hd)
for p in existing_lib_paths:
dep_name = p.stem if p.is_file() else p.name
if dep_name != name: # Skip the main module itself
all_hard_deps.add(dep_name)
lines.append("\ndependencies:")
lines.extend(
format_deps_tree(
cpython_prefix,
lib_prefix,
max_depth,
soft_deps={name},
_visited=_visited,
hard_deps=all_hard_deps,
)
)
# Show dependent tests as tree (depth 2: module + direct importers + their importers)
tree = find_dependent_tests_tree(name, lib_prefix=lib_prefix, max_depth=2)
lines.extend(_format_dependent_tests_tree(tree, cpython_prefix, lib_prefix))
return lines
def _format_dependent_tests_tree(
tree: dict,
cpython_prefix: str,
lib_prefix: str,
indent: str = "",
) -> list[str]:
"""Format dependent tests tree for display."""
from update_lib.deps import is_up_to_date
lines = []
module = tree["module"]
tests = tree["tests"]
children = tree["children"]
if indent == "":
# Root level
# Count total tests in tree
def count_tests(t: dict) -> int:
total = len(t.get("tests", []))
for c in t.get("children", []):
total += count_tests(c)
return total
total = count_tests(tree)
if total == 0 and not children:
lines.append(f"\ndependent tests: (no tests depend on {module})")
return lines
lines.append(f"\ndependent tests: ({total} tests)")
# Check if module is up-to-date
synced = is_up_to_date(module.split(".")[0], cpython_prefix, lib_prefix)
marker = "[x]" if synced else "[ ]"
# Format this node
if tests:
test_str = " ".join(tests)
if indent == "":
lines.append(f"- {marker} {module}: {test_str}")
else:
lines.append(f"{indent}- {marker} {module}: {test_str}")
elif indent != "" and children:
# Has children but no direct tests
lines.append(f"{indent}- {marker} {module}:")
# Format children
child_indent = indent + " " if indent else " "
for child in children:
lines.extend(
_format_dependent_tests_tree(
child, cpython_prefix, lib_prefix, child_indent
)
)
return lines
def _resolve_module_name(
name: str,
cpython_prefix: str,
lib_prefix: str,
) -> list[str]:
"""Resolve module name through redirects.
Returns a list of module names (usually 1, but test support files may expand to multiple).
"""
import pathlib
from update_lib.deps import (
_build_test_import_graph,
get_lib_paths,
get_test_paths,
resolve_hard_dep_parent,
resolve_test_to_lib,
)
# Resolve test to library group (e.g., test_urllib2 -> urllib)
if name.startswith("test_"):
lib_group = resolve_test_to_lib(name)
if lib_group:
return [lib_group]
name = name[5:]
# Resolve hard_dep to parent
parent = resolve_hard_dep_parent(name, cpython_prefix)
if parent:
return [parent]
# Check if it's a valid module
lib_paths = get_lib_paths(name, cpython_prefix)
test_paths = get_test_paths(name, cpython_prefix)
if any(p.exists() for p in lib_paths) or any(p.exists() for p in test_paths):
return [name]
# Check for test support files (e.g., string_tests -> bytes, str, userstring)
test_support_path = pathlib.Path(cpython_prefix) / "Lib" / "test" / f"{name}.py"
if test_support_path.exists():
test_dir = pathlib.Path(lib_prefix) / "test"
if test_dir.exists():
import_graph, _ = _build_test_import_graph(test_dir)
importing_tests = []
for file_key, imports in import_graph.items():
if name in imports and file_key.startswith("test_"):
importing_tests.append(file_key)
if importing_tests:
# Resolve test names to module names (test_bytes -> bytes)
return sorted(set(t[5:] for t in importing_tests))
return [name]
def show_deps(
names: list[str],
cpython_prefix: str,
lib_prefix: str,
max_depth: int = 10,
) -> None:
"""Show all dependency information for modules."""
# Expand "all" to all module names
expanded_names = []
for name in set(names):
if name == "all":
expanded_names.extend(get_all_modules(cpython_prefix))
else:
expanded_names.append(name)
# Resolve and deduplicate names (preserving order)
seen: set[str] = set()
resolved_names: list[str] = []
for name in expanded_names:
for resolved in _resolve_module_name(name, cpython_prefix, lib_prefix):
if resolved not in seen:
seen.add(resolved)
resolved_names.append(resolved)
# Shared visited set across all modules
visited: set[str] = set()
for i, name in enumerate(resolved_names):
if i > 0:
print() # blank line between modules
for line in format_deps(name, cpython_prefix, lib_prefix, max_depth, visited):
print(line)
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(
description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter,
)
parser.add_argument(
"names",
nargs="+",
help="Module names (e.g., dis, dataclasses) or 'all' for all modules",
)
parser.add_argument(
"--cpython",
default="cpython",
help="CPython directory prefix (default: cpython)",
)
parser.add_argument(
"--lib",
default="Lib",
help="Local Lib directory prefix (default: Lib)",
)
parser.add_argument(
"--depth",
type=int,
default=10,
help="Maximum recursion depth for soft_deps tree (default: 10)",
)
args = parser.parse_args(argv)
# When user does `./update_lib/ deps "foo bar" "baz"`
# We still want to get a list of
# ["foo", "bar", "baz"], not ["foo bar", "baz"]
args.names = [name for names in args.names for name in names.split()]
try:
show_deps(args.names, args.cpython, args.lib, args.depth)
return 0
except Exception as e:
print(f"Error: {e}", file=sys.stderr)
return 1
if __name__ == "__main__":
sys.exit(main())