RustPython/scripts/update_lib/cmd_quick.py at unify-python-version · ShaharNaveh/RustPython · GitHub

History

478 lines (399 loc) · 14.7 KB

Raw

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

187

188

189

190

191

192

193

194

195

196

197

198

199

200

201

202

203

204

205

206

207

208

209

210

211

212

213

214

215

216

217

218

219

220

221

222

223

224

225

226

227

228

229

230

231

232

233

234

235

236

237

238

239

240

241

242

243

244

245

246

247

248

249

250

251

252

253

254

255

256

257

258

259

260

261

262

263

264

265

266

267

268

269

270

271

272

273

274

275

276

277

278

279

280

281

282

283

284

285

286

287

288

289

290

291

292

293

294

295

296

297

298

299

300

301

302

303

304

305

306

307

308

309

310

311

312

313

314

315

316

317

318

319

320

321

322

323

324

325

326

327

328

329

330

331

332

333

334

335

336

337

338

339

340

341

342

343

344

345

346

347

348

349

350

351

352

353

354

355

356

357

358

359

360

361

362

363

364

365

366

367

368

369

370

371

372

373

374

375

376

377

378

379

380

381

382

383

384

385

386

387

388

389

390

391

392

393

394

395

396

397

398

399

400

401

402

403

404

405

406

407

408

409

410

411

412

413

414

415

416

417

418

419

420

421

422

423

424

425

426

427

428

429

430

431

432

433

434

435

436

437

438

439

440

441

442

443

444

445

446

447

448

449

450

451

452

453

454

455

456

457

458

459

460

461

462

463

464

465

466

467

468

469

470

471

472

473

474

475

476

477

478

#!/usr/bin/env python

"""

Quick update for test files from CPython.

Usage:

# Library + test: copy lib, then patch + auto-mark test + commit

python scripts/update_lib quick cpython/Lib/dataclasses.py

# Shortcut: just the module name

python scripts/update_lib quick dataclasses

# Test file: patch + auto-mark

python scripts/update_lib quick cpython/Lib/test/test_foo.py

# Test file: migrate only

python scripts/update_lib quick cpython/Lib/test/test_foo.py --no-auto-mark

# Test file: auto-mark only (Lib/ path implies --no-migrate)

python scripts/update_lib quick Lib/test/test_foo.py

# Directory: patch all + auto-mark all

python scripts/update_lib quick cpython/Lib/test/test_dataclasses/

# Skip git commit

python scripts/update_lib quick dataclasses --no-commit

"""

import argparse

import pathlib

import sys

sys.path.insert(0, str(pathlib.Path(__file__).parent.parent))

from update_lib.deps import DEPENDENCIES, get_test_paths

from update_lib.file_utils import (

construct_lib_path,

get_cpython_dir,

get_module_name,

get_test_files,

is_lib_path,

is_test_path,

lib_to_test_path,

parse_lib_path,

resolve_module_path,

safe_read_text,

)

def collect_original_methods(

lib_path: pathlib.Path,

) -> set[tuple[str, str]] | dict[pathlib.Path, set[tuple[str, str]]] | None:

"""

Collect original test methods from lib path before patching.

Returns:

- For file: set of (class_name, method_name) or None if file doesn't exist

- For directory: dict mapping file path to set of methods, or None if dir doesn't exist

"""

from update_lib.cmd_auto_mark import extract_test_methods

if not lib_path.exists():

return None

if lib_path.is_file():

content = safe_read_text(lib_path)

return extract_test_methods(content) if content else set()

else:

result = {}

for lib_file in get_test_files(lib_path):

content = safe_read_text(lib_file)

if content:

result[lib_file.resolve()] = extract_test_methods(content)

return result

def quick(

src_path: pathlib.Path,

no_migrate: bool = False,

no_auto_mark: bool = False,

mark_failure: bool = False,

verbose: bool = True,

skip_build: bool = False,

) -> list[pathlib.Path]:

"""

Process a file or directory: migrate + auto-mark.

Args:

src_path: Source path (file or directory)

no_migrate: Skip migration step

no_auto_mark: Skip auto-mark step

mark_failure: Add @expectedFailure to ALL failing tests

verbose: Print progress messages

skip_build: Skip cargo build, use pre-built binary

Returns:

List of extra paths (data dirs, hard deps) that were copied/migrated.

"""

from update_lib.cmd_auto_mark import auto_mark_directory, auto_mark_file

from update_lib.cmd_migrate import patch_directory, patch_file

extra_paths: list[pathlib.Path] = []

# Determine lib_path and whether to migrate

if is_lib_path(src_path):

no_migrate = True

lib_path = src_path

else:

lib_path = parse_lib_path(src_path)

is_dir = src_path.is_dir()

# Capture original test methods before migration (for smart auto-mark)

original_methods = collect_original_methods(lib_path)

# Step 1: Migrate

if not no_migrate:

if is_dir:

patch_directory(src_path, lib_path, verbose=verbose)

else:

patch_file(src_path, lib_path, verbose=verbose)

# Step 1.5: Handle test dependencies

from update_lib.deps import get_test_dependencies

test_deps = get_test_dependencies(src_path)

# Migrate dependency files

for dep_src in test_deps["hard_deps"]:

dep_lib = parse_lib_path(dep_src)

if verbose:

print(f"Migrating dependency: {dep_src.name}")

if dep_src.is_dir():

patch_directory(dep_src, dep_lib, verbose=False)

else:

patch_file(dep_src, dep_lib, verbose=False)

extra_paths.append(dep_lib)

# Copy data directories (no migration)

import shutil

for data_src in test_deps["data"]:

data_lib = parse_lib_path(data_src)

if verbose:

print(f"Copying data: {data_src.name}")

if data_lib.exists():

if data_lib.is_dir():

shutil.rmtree(data_lib)

else:

data_lib.unlink()

if data_src.is_dir():

shutil.copytree(data_src, data_lib)

else:

data_lib.parent.mkdir(parents=True, exist_ok=True)

shutil.copy2(data_src, data_lib)

extra_paths.append(data_lib)

# Step 2: Auto-mark

if not no_auto_mark:

if not lib_path.exists():

raise FileNotFoundError(f"Path not found: {lib_path}")

if is_dir:

num_added, num_removed, _ = auto_mark_directory(

lib_path,

mark_failure=mark_failure,

verbose=verbose,

original_methods_per_file=original_methods,

skip_build=skip_build,

)

else:

num_added, num_removed, _ = auto_mark_file(

lib_path,

mark_failure=mark_failure,

verbose=verbose,

original_methods=original_methods,

skip_build=skip_build,

)

if verbose:

if num_added:

print(f"Added expectedFailure to {num_added} tests")

print(f"Removed expectedFailure from {num_removed} tests")

return extra_paths

def get_cpython_version(cpython_dir: pathlib.Path) -> str:

"""Get CPython version from git tag."""

import subprocess

result = subprocess.run(

["git", "describe", "--tags"],

cwd=cpython_dir,

capture_output=True,

text=True,

check=True,

)

return result.stdout.strip()

def git_commit(

name: str,

lib_path: pathlib.Path | None,

test_paths: list[pathlib.Path] | pathlib.Path | None,

cpython_dir: pathlib.Path,

hard_deps: list[pathlib.Path] | None = None,

verbose: bool = True,

) -> bool:

"""Commit changes with CPython author.

Args:

name: Module name (e.g., "dataclasses")

lib_path: Path to library file/directory (or None)

test_paths: Path(s) to test file/directory (or None)

cpython_dir: Path to cpython directory

hard_deps: Path(s) to hard dependency files (or None)

verbose: Print progress messages

Returns:

True if commit was created, False otherwise

"""

import subprocess

# Normalize test_paths to list

if test_paths is None:

test_paths = []

elif isinstance(test_paths, pathlib.Path):

test_paths = [test_paths]

# Normalize hard_deps to list

if hard_deps is None:

hard_deps = []

# Stage changes

paths_to_add = []

if lib_path and lib_path.exists():

paths_to_add.append(str(lib_path))

for test_path in test_paths:

if test_path and test_path.exists():

paths_to_add.append(str(test_path))

for dep_path in hard_deps:

if dep_path and dep_path.exists():

paths_to_add.append(str(dep_path))

if not paths_to_add:

return False

version = get_cpython_version(cpython_dir)

subprocess.run(["git", "add"] + paths_to_add, check=True)

# Check if there are staged changes

result = subprocess.run(

["git", "diff", "--cached", "--quiet"],

capture_output=True,

)

if result.returncode == 0:

if verbose:

print("No changes to commit")

return False

# Commit with CPython author

message = f"Update {name} from {version}"

subprocess.run(

[

"git",

"commit",

"--author",

"CPython Developers <>",

"-m",

message,

check=True,

)

if verbose:

print(f"Committed: {message}")

return True

def _expand_shortcut(path: pathlib.Path) -> pathlib.Path:

"""Expand simple name to cpython/Lib path if it exists.

Examples:

dataclasses -> cpython/Lib/dataclasses.py (if exists)

json -> cpython/Lib/json/ (if exists)

test_types -> cpython/Lib/test/test_types.py (if exists)

regrtest -> cpython/Lib/test/libregrtest (from DEPENDENCIES)

"""

# Only expand if it's a simple name (no path separators) and doesn't exist

if "/" in str(path) or path.exists():

return path

name = str(path)

# Check DEPENDENCIES table for path overrides (e.g., regrtest)

from update_lib.deps import DEPENDENCIES

if name in DEPENDENCIES and "lib" in DEPENDENCIES[name]:

lib_paths = DEPENDENCIES[name]["lib"]

if lib_paths:

override_path = construct_lib_path("cpython", lib_paths[0])

if override_path.exists():

return override_path

# Test shortcut: test_foo -> cpython/Lib/test/test_foo

if name.startswith("test_"):

resolved = resolve_module_path(f"test/{name}", "cpython", prefer="dir")

if resolved.exists():

return resolved

# Library shortcut: foo -> cpython/Lib/foo

resolved = resolve_module_path(name, "cpython", prefer="file")

if resolved.exists():

return resolved

# Extension module shortcut: winreg -> cpython/Lib/test/test_winreg

# For C/Rust extension modules that have no Python source but have tests

resolved = resolve_module_path(f"test/test_{name}", "cpython", prefer="dir")

if resolved.exists():

return resolved

# Return original (will likely fail later with a clear error)

return path

def main(argv: list[str] | None = None) -> int:

parser = argparse.ArgumentParser(

description=__doc__,

formatter_class=argparse.RawDescriptionHelpFormatter,

)

parser.add_argument(

"path",

type=pathlib.Path,

help="Source path (file or directory)",

)

parser.add_argument(

"--copy",

action=argparse.BooleanOptionalAction,

default=True,

help="Copy library file (default: enabled, implied disabled if test path)",

)

parser.add_argument(

"--migrate",

action=argparse.BooleanOptionalAction,

default=True,

help="Migrate test file (default: enabled, implied disabled if Lib/ path)",

)

parser.add_argument(

"--auto-mark",

action=argparse.BooleanOptionalAction,

default=True,

help="Auto-mark test failures (default: enabled)",

)

parser.add_argument(

"--mark-failure",

action="store_true",

help="Add @expectedFailure to failing tests",

)

parser.add_argument(

"--commit",

action=argparse.BooleanOptionalAction,

default=True,

help="Create git commit (default: enabled)",

)

parser.add_argument(

"--build",

action=argparse.BooleanOptionalAction,

default=True,

help="Build with cargo (default: enabled)",

)

args = parser.parse_args(argv)

try:

src_path = args.path

# Shortcut: expand simple name to cpython/Lib path

src_path = _expand_shortcut(src_path)

original_src = src_path # Keep for commit

# Track library path for commit

lib_file_path = None

test_path = None

hard_deps_for_commit = []

# If it's a library path (not test path), do copy_lib first

if not is_test_path(src_path):

# Get library destination path for commit

lib_file_path = parse_lib_path(src_path)

if args.copy:

from update_lib.cmd_copy_lib import copy_lib

copy_lib(src_path)

# Get all test paths from DEPENDENCIES (or fall back to default)

module_name = get_module_name(original_src)

cpython_dir = get_cpython_dir(original_src)

test_src_paths = get_test_paths(module_name, str(cpython_dir))

# Fall back to default test path if DEPENDENCIES has no entry

if not test_src_paths:

default_test = lib_to_test_path(original_src)

if default_test.exists():

test_src_paths = (default_test,)

# Collect hard dependencies for commit

lib_deps = DEPENDENCIES.get(module_name, {})

for dep_name in lib_deps.get("hard_deps", []):

dep_lib_path = pathlib.Path("Lib") / dep_name

if dep_lib_path.exists():

hard_deps_for_commit.append(dep_lib_path)

# Process all test paths

test_paths_for_commit = []

for test_src in test_src_paths:

if not test_src.exists():

print(f"Warning: Test path does not exist: {test_src}")

continue

test_lib_path = parse_lib_path(test_src)

test_paths_for_commit.append(test_lib_path)

extra = quick(

test_src,

no_migrate=not args.migrate,

no_auto_mark=not args.auto_mark,

mark_failure=args.mark_failure,

skip_build=not args.build,

)

hard_deps_for_commit.extend(extra)

test_paths = test_paths_for_commit

else:

# It's a test path - process single test

test_path = (

parse_lib_path(src_path) if not is_lib_path(src_path) else src_path

)

extra = quick(

src_path,

no_migrate=not args.migrate,

no_auto_mark=not args.auto_mark,

mark_failure=args.mark_failure,

skip_build=not args.build,

)

hard_deps_for_commit.extend(extra)

test_paths = [test_path]

# Step 3: Git commit

if args.commit:

cpython_dir = get_cpython_dir(original_src)

git_commit(

get_module_name(original_src),

lib_file_path,

test_paths,

cpython_dir,

hard_deps=hard_deps_for_commit,

)

return 0

except ValueError as e:

print(f"Error: {e}", file=sys.stderr)

return 1

except FileNotFoundError as e:

print(f"Error: {e}", file=sys.stderr)

return 1

except Exception as e:

# Handle TestRunError with a clean message

from update_lib.cmd_auto_mark import TestRunError

if isinstance(e, TestRunError):

print(f"Error: {e}", file=sys.stderr)

return 1

raise

if __name__ == "__main__":

sys.exit(main())

RustPython/scripts/update_lib/cmd_quick.py at unify-python-version · ShaharNaveh/RustPython · GitHub

Expand file tree

Breadcrumbs

cmd_quick.py

Latest commit

History

Breadcrumbs

cmd_quick.py

File metadata and controls