#!/usr/bin/env python
"""
Quick update for test files from CPython.
Usage:
# Library + test: copy lib, then patch + auto-mark test + commit
python scripts/update_lib quick cpython/Lib/dataclasses.py
# Shortcut: just the module name
python scripts/update_lib quick dataclasses
# Test file: patch + auto-mark
python scripts/update_lib quick cpython/Lib/test/test_foo.py
# Test file: migrate only
python scripts/update_lib quick cpython/Lib/test/test_foo.py --no-auto-mark
# Test file: auto-mark only (Lib/ path implies --no-migrate)
python scripts/update_lib quick Lib/test/test_foo.py
# Directory: patch all + auto-mark all
python scripts/update_lib quick cpython/Lib/test/test_dataclasses/
# Skip git commit
python scripts/update_lib quick dataclasses --no-commit
"""
import argparse
import pathlib
import sys
sys.path.insert(0, str(pathlib.Path(__file__).parent.parent))
from update_lib.deps import DEPENDENCIES, get_test_paths
from update_lib.file_utils import (
construct_lib_path,
get_cpython_dir,
get_module_name,
get_test_files,
is_lib_path,
is_test_path,
lib_to_test_path,
parse_lib_path,
resolve_module_path,
safe_read_text,
)
def collect_original_methods(
lib_path: pathlib.Path,
) -> set[tuple[str, str]] | dict[pathlib.Path, set[tuple[str, str]]] | None:
"""
Collect original test methods from lib path before patching.
Returns:
- For file: set of (class_name, method_name) or None if file doesn't exist
- For directory: dict mapping file path to set of methods, or None if dir doesn't exist
"""
from update_lib.cmd_auto_mark import extract_test_methods
if not lib_path.exists():
return None
if lib_path.is_file():
content = safe_read_text(lib_path)
return extract_test_methods(content) if content else set()
else:
result = {}
for lib_file in get_test_files(lib_path):
content = safe_read_text(lib_file)
if content:
result[lib_file.resolve()] = extract_test_methods(content)
return result
def quick(
src_path: pathlib.Path,
no_migrate: bool = False,
no_auto_mark: bool = False,
mark_failure: bool = False,
verbose: bool = True,
skip_build: bool = False,
) -> list[pathlib.Path]:
"""
Process a file or directory: migrate + auto-mark.
Args:
src_path: Source path (file or directory)
no_migrate: Skip migration step
no_auto_mark: Skip auto-mark step
mark_failure: Add @expectedFailure to ALL failing tests
verbose: Print progress messages
skip_build: Skip cargo build, use pre-built binary
Returns:
List of extra paths (data dirs, hard deps) that were copied/migrated.
"""
from update_lib.cmd_auto_mark import auto_mark_directory, auto_mark_file
from update_lib.cmd_migrate import patch_directory, patch_file
extra_paths: list[pathlib.Path] = []
# Determine lib_path and whether to migrate
if is_lib_path(src_path):
no_migrate = True
lib_path = src_path
else:
lib_path = parse_lib_path(src_path)
is_dir = src_path.is_dir()
# Capture original test methods before migration (for smart auto-mark)
original_methods = collect_original_methods(lib_path)
# Step 1: Migrate
if not no_migrate:
if is_dir:
patch_directory(src_path, lib_path, verbose=verbose)
else:
patch_file(src_path, lib_path, verbose=verbose)
# Step 1.5: Handle test dependencies
from update_lib.deps import get_test_dependencies
test_deps = get_test_dependencies(src_path)
# Migrate dependency files
for dep_src in test_deps["hard_deps"]:
dep_lib = parse_lib_path(dep_src)
if verbose:
print(f"Migrating dependency: {dep_src.name}")
if dep_src.is_dir():
patch_directory(dep_src, dep_lib, verbose=False)
else:
patch_file(dep_src, dep_lib, verbose=False)
extra_paths.append(dep_lib)
# Copy data directories (no migration)
import shutil
for data_src in test_deps["data"]:
data_lib = parse_lib_path(data_src)
if verbose:
print(f"Copying data: {data_src.name}")
if data_lib.exists():
if data_lib.is_dir():
shutil.rmtree(data_lib)
else:
data_lib.unlink()
if data_src.is_dir():
shutil.copytree(data_src, data_lib)
else:
data_lib.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(data_src, data_lib)
extra_paths.append(data_lib)
# Step 2: Auto-mark
if not no_auto_mark:
if not lib_path.exists():
raise FileNotFoundError(f"Path not found: {lib_path}")
if is_dir:
num_added, num_removed, _ = auto_mark_directory(
lib_path,
mark_failure=mark_failure,
verbose=verbose,
original_methods_per_file=original_methods,
skip_build=skip_build,
)
else:
num_added, num_removed, _ = auto_mark_file(
lib_path,
mark_failure=mark_failure,
verbose=verbose,
original_methods=original_methods,
skip_build=skip_build,
)
if verbose:
if num_added:
print(f"Added expectedFailure to {num_added} tests")
print(f"Removed expectedFailure from {num_removed} tests")
return extra_paths
def get_cpython_version(cpython_dir: pathlib.Path) -> str:
"""Get CPython version from git tag."""
import subprocess
result = subprocess.run(
["git", "describe", "--tags"],
cwd=cpython_dir,
capture_output=True,
text=True,
check=True,
)
return result.stdout.strip()
def git_commit(
name: str,
lib_path: pathlib.Path | None,
test_paths: list[pathlib.Path] | pathlib.Path | None,
cpython_dir: pathlib.Path,
hard_deps: list[pathlib.Path] | None = None,
verbose: bool = True,
) -> bool:
"""Commit changes with CPython author.
Args:
name: Module name (e.g., "dataclasses")
lib_path: Path to library file/directory (or None)
test_paths: Path(s) to test file/directory (or None)
cpython_dir: Path to cpython directory
hard_deps: Path(s) to hard dependency files (or None)
verbose: Print progress messages
Returns:
True if commit was created, False otherwise
"""
import subprocess
# Normalize test_paths to list
if test_paths is None:
test_paths = []
elif isinstance(test_paths, pathlib.Path):
test_paths = [test_paths]
# Normalize hard_deps to list
if hard_deps is None:
hard_deps = []
# Stage changes
paths_to_add = []
if lib_path and lib_path.exists():
paths_to_add.append(str(lib_path))
for test_path in test_paths:
if test_path and test_path.exists():
paths_to_add.append(str(test_path))
for dep_path in hard_deps:
if dep_path and dep_path.exists():
paths_to_add.append(str(dep_path))
if not paths_to_add:
return False
version = get_cpython_version(cpython_dir)
subprocess.run(["git", "add"] + paths_to_add, check=True)
# Check if there are staged changes
result = subprocess.run(
["git", "diff", "--cached", "--quiet"],
capture_output=True,
)
if result.returncode == 0:
if verbose:
print("No changes to commit")
return False
# Commit with CPython author
message = f"Update {name} from {version}"
subprocess.run(
[
"git",
"commit",
"--author",
"CPython Developers ",
"-m",
message,
],
check=True,
)
if verbose:
print(f"Committed: {message}")
return True
def _expand_shortcut(path: pathlib.Path) -> pathlib.Path:
"""Expand simple name to cpython/Lib path if it exists.
Examples:
dataclasses -> cpython/Lib/dataclasses.py (if exists)
json -> cpython/Lib/json/ (if exists)
test_types -> cpython/Lib/test/test_types.py (if exists)
regrtest -> cpython/Lib/test/libregrtest (from DEPENDENCIES)
"""
# Only expand if it's a simple name (no path separators) and doesn't exist
if "/" in str(path) or path.exists():
return path
name = str(path)
# Check DEPENDENCIES table for path overrides (e.g., regrtest)
from update_lib.deps import DEPENDENCIES
if name in DEPENDENCIES and "lib" in DEPENDENCIES[name]:
lib_paths = DEPENDENCIES[name]["lib"]
if lib_paths:
override_path = construct_lib_path("cpython", lib_paths[0])
if override_path.exists():
return override_path
# Test shortcut: test_foo -> cpython/Lib/test/test_foo
if name.startswith("test_"):
resolved = resolve_module_path(f"test/{name}", "cpython", prefer="dir")
if resolved.exists():
return resolved
# Library shortcut: foo -> cpython/Lib/foo
resolved = resolve_module_path(name, "cpython", prefer="file")
if resolved.exists():
return resolved
# Extension module shortcut: winreg -> cpython/Lib/test/test_winreg
# For C/Rust extension modules that have no Python source but have tests
resolved = resolve_module_path(f"test/test_{name}", "cpython", prefer="dir")
if resolved.exists():
return resolved
# Return original (will likely fail later with a clear error)
return path
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(
description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter,
)
parser.add_argument(
"path",
type=pathlib.Path,
help="Source path (file or directory)",
)
parser.add_argument(
"--copy",
action=argparse.BooleanOptionalAction,
default=True,
help="Copy library file (default: enabled, implied disabled if test path)",
)
parser.add_argument(
"--migrate",
action=argparse.BooleanOptionalAction,
default=True,
help="Migrate test file (default: enabled, implied disabled if Lib/ path)",
)
parser.add_argument(
"--auto-mark",
action=argparse.BooleanOptionalAction,
default=True,
help="Auto-mark test failures (default: enabled)",
)
parser.add_argument(
"--mark-failure",
action="store_true",
help="Add @expectedFailure to failing tests",
)
parser.add_argument(
"--commit",
action=argparse.BooleanOptionalAction,
default=True,
help="Create git commit (default: enabled)",
)
parser.add_argument(
"--build",
action=argparse.BooleanOptionalAction,
default=True,
help="Build with cargo (default: enabled)",
)
args = parser.parse_args(argv)
try:
src_path = args.path
# Shortcut: expand simple name to cpython/Lib path
src_path = _expand_shortcut(src_path)
original_src = src_path # Keep for commit
# Track library path for commit
lib_file_path = None
test_path = None
hard_deps_for_commit = []
# If it's a library path (not test path), do copy_lib first
if not is_test_path(src_path):
# Get library destination path for commit
lib_file_path = parse_lib_path(src_path)
if args.copy:
from update_lib.cmd_copy_lib import copy_lib
copy_lib(src_path)
# Get all test paths from DEPENDENCIES (or fall back to default)
module_name = get_module_name(original_src)
cpython_dir = get_cpython_dir(original_src)
test_src_paths = get_test_paths(module_name, str(cpython_dir))
# Fall back to default test path if DEPENDENCIES has no entry
if not test_src_paths:
default_test = lib_to_test_path(original_src)
if default_test.exists():
test_src_paths = (default_test,)
# Collect hard dependencies for commit
lib_deps = DEPENDENCIES.get(module_name, {})
for dep_name in lib_deps.get("hard_deps", []):
dep_lib_path = pathlib.Path("Lib") / dep_name
if dep_lib_path.exists():
hard_deps_for_commit.append(dep_lib_path)
# Process all test paths
test_paths_for_commit = []
for test_src in test_src_paths:
if not test_src.exists():
print(f"Warning: Test path does not exist: {test_src}")
continue
test_lib_path = parse_lib_path(test_src)
test_paths_for_commit.append(test_lib_path)
extra = quick(
test_src,
no_migrate=not args.migrate,
no_auto_mark=not args.auto_mark,
mark_failure=args.mark_failure,
skip_build=not args.build,
)
hard_deps_for_commit.extend(extra)
test_paths = test_paths_for_commit
else:
# It's a test path - process single test
test_path = (
parse_lib_path(src_path) if not is_lib_path(src_path) else src_path
)
extra = quick(
src_path,
no_migrate=not args.migrate,
no_auto_mark=not args.auto_mark,
mark_failure=args.mark_failure,
skip_build=not args.build,
)
hard_deps_for_commit.extend(extra)
test_paths = [test_path]
# Step 3: Git commit
if args.commit:
cpython_dir = get_cpython_dir(original_src)
git_commit(
get_module_name(original_src),
lib_file_path,
test_paths,
cpython_dir,
hard_deps=hard_deps_for_commit,
)
return 0
except ValueError as e:
print(f"Error: {e}", file=sys.stderr)
return 1
except FileNotFoundError as e:
print(f"Error: {e}", file=sys.stderr)
return 1
except Exception as e:
# Handle TestRunError with a clean message
from update_lib.cmd_auto_mark import TestRunError
if isinstance(e, TestRunError):
print(f"Error: {e}", file=sys.stderr)
return 1
raise
if __name__ == "__main__":
sys.exit(main())