RustPython/scripts/update_lib/cmd_deps.py at main · RustPython/RustPython · GitHub

Repository navigation

History

463 lines (385 loc) · 14.1 KB

Raw

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

187

188

189

190

191

192

193

194

195

196

197

198

199

200

201

202

203

204

205

206

207

208

209

210

211

212

213

214

215

216

217

218

219

220

221

222

223

224

225

226

227

228

229

230

231

232

233

234

235

236

237

238

239

240

241

242

243

244

245

246

247

248

249

250

251

252

253

254

255

256

257

258

259

260

261

262

263

264

265

266

267

268

269

270

271

272

273

274

275

276

277

278

279

280

281

282

283

284

285

286

287

288

289

290

291

292

293

294

295

296

297

298

299

300

301

302

303

304

305

306

307

308

309

310

311

312

313

314

315

316

317

318

319

320

321

322

323

324

325

326

327

328

329

330

331

332

333

334

335

336

337

338

339

340

341

342

343

344

345

346

347

348

349

350

351

352

353

354

355

356

357

358

359

360

361

362

363

364

365

366

367

368

369

370

371

372

373

374

375

376

377

378

379

380

381

382

383

384

385

386

387

388

389

390

391

392

393

394

395

396

397

398

399

400

401

402

403

404

405

406

407

408

409

410

411

412

413

414

415

416

417

418

419

420

421

422

423

424

425

426

427

428

429

430

431

432

433

434

435

436

437

438

439

440

441

442

443

444

445

446

447

448

449

450

451

452

453

454

455

456

457

458

459

460

461

462

463

#!/usr/bin/env python

"""

Show dependency information for a module.

Usage:

python scripts/update_lib deps dis

python scripts/update_lib deps dataclasses

python scripts/update_lib deps dis --depth 2

python scripts/update_lib deps all # Show all modules' dependencies

"""

import argparse

import pathlib

import sys

sys.path.insert(0, str(pathlib.Path(__file__).parent.parent))

def get_all_modules(cpython_prefix: str) -> list[str]:

"""Get all top-level module names from cpython/Lib/.

Includes private modules (_*) that are not hard_deps of other modules.

Returns:

Sorted list of module names (without .py extension)

"""

from update_lib.deps import resolve_hard_dep_parent

lib_dir = pathlib.Path(cpython_prefix) / "Lib"

if not lib_dir.exists():

return []

modules = set()

for entry in lib_dir.iterdir():

# Skip hidden files

if entry.name.startswith("."):

continue

# Skip test directory

if entry.name == "test":

continue

if entry.is_file() and entry.suffix == ".py":

name = entry.stem

elif entry.is_dir() and (entry / "__init__.py").exists():

name = entry.name

else:

continue

# Skip modules that are hard_deps of other modules

# e.g., _pydatetime is a hard_dep of datetime, pydoc_data is a hard_dep of pydoc

if resolve_hard_dep_parent(name, cpython_prefix) is not None:

continue

modules.add(name)

return sorted(modules)

def format_deps_tree(

cpython_prefix: str,

lib_prefix: str,

max_depth: int,

soft_deps: set[str] | None = None,

hard_deps: set[str] | None = None,

_depth: int = 0,

_visited: set[str] | None = None,

_indent: str = "",

) -> list[str]:

"""Format soft dependencies as a tree with up-to-date status.

Args:

cpython_prefix: CPython directory prefix

lib_prefix: Local Lib directory prefix

max_depth: Maximum recursion depth

soft_deps: Pre-computed soft dependencies (optional)

hard_deps: Hard dependencies to show under the module (root level only)

_depth: Current depth (internal)

_visited: Already visited modules (internal)

_indent: Current indentation (internal)

Returns:

List of formatted lines

"""

from update_lib.deps import (

get_lib_paths,

get_rust_deps,

get_soft_deps,

is_up_to_date,

)

lines = []

if _visited is None:

_visited = set()

# Compute deps from name if not provided

if soft_deps is None:

soft_deps = get_soft_deps(name, cpython_prefix) if name else set()

soft_deps = sorted(soft_deps)

if not soft_deps and not hard_deps:

return lines

# Separate up-to-date and outdated modules

up_to_date_deps = []

outdated_deps = []

dup_deps = []

for dep in soft_deps:

# Skip if library doesn't exist in cpython

lib_paths = get_lib_paths(dep, cpython_prefix)

if not any(p.exists() for p in lib_paths):

continue

up_to_date = is_up_to_date(dep, cpython_prefix, lib_prefix)

if up_to_date:

# Up-to-date modules collected compactly, no dup tracking needed

up_to_date_deps.append(dep)

elif dep in _visited:

# Only track dup for outdated modules

dup_deps.append(dep)

else:

outdated_deps.append(dep)

# Show outdated modules with expansion

for dep in outdated_deps:

dep_native = get_rust_deps(dep, cpython_prefix)

native_suffix = (

f" (native: {', '.join(sorted(dep_native))})" if dep_native else ""

)

lines.append(f"{_indent}- [ ] {dep}{native_suffix}")

_visited.add(dep)

# Show hard_deps under this module (only at root level, i.e., when hard_deps is provided)

if hard_deps and dep in soft_deps:

for hd in sorted(hard_deps):

hd_up_to_date = is_up_to_date(hd, cpython_prefix, lib_prefix)

hd_marker = "[x]" if hd_up_to_date else "[ ]"

lines.append(f"{_indent} - {hd_marker} {hd}")

hard_deps = None # Only show once

# Recurse if within depth limit

if _depth < max_depth - 1:

lines.extend(

format_deps_tree(

cpython_prefix,

lib_prefix,

max_depth,

name=dep,

_depth=_depth + 1,

_visited=_visited,

_indent=_indent + " ",

)

# Show duplicates compactly (only for outdated)

if dup_deps:

lines.append(f"{_indent}- [ ] {', '.join(dup_deps)}")

# Show up-to-date modules compactly on one line

if up_to_date_deps:

lines.append(f"{_indent}- [x] {', '.join(up_to_date_deps)}")

return lines

def format_deps(

cpython_prefix: str,

lib_prefix: str,

max_depth: int = 10,

_visited: set[str] | None = None,

) -> list[str]:

"""Format all dependency information for a module.

Args:

cpython_prefix: CPython directory prefix

lib_prefix: Local Lib directory prefix

max_depth: Maximum recursion depth

_visited: Shared visited set for deduplication across modules

Returns:

List of formatted lines

"""

from update_lib.deps import (

DEPENDENCIES,

count_test_todos,

find_dependent_tests_tree,

get_lib_paths,

get_test_paths,

is_path_synced,

is_test_up_to_date,

resolve_hard_dep_parent,

)

if _visited is None:

_visited = set()

lines = []

# Resolve test_ prefix to module (e.g., test_pydoc -> pydoc)

if name.startswith("test_"):

module_name = name[5:] # strip "test_"

lines.append(f"(redirecting {name} -> {module_name})")

name = module_name

# Resolve hard_dep to parent module (e.g., pydoc_data -> pydoc)

parent = resolve_hard_dep_parent(name, cpython_prefix)

if parent:

lines.append(f"(redirecting {name} -> {parent})")

name = parent

# lib paths (only show existing)

lib_paths = get_lib_paths(name, cpython_prefix)

existing_lib_paths = [p for p in lib_paths if p.exists()]

for p in existing_lib_paths:

synced = is_path_synced(p, cpython_prefix, lib_prefix)

marker = "[x]" if synced else "[ ]"

lines.append(f"{marker} lib: {p}")

# test paths (only show existing)

test_paths = get_test_paths(name, cpython_prefix)

existing_test_paths = [p for p in test_paths if p.exists()]

for p in existing_test_paths:

test_name = p.stem if p.is_file() else p.name

synced = is_test_up_to_date(test_name, cpython_prefix, lib_prefix)

marker = "[x]" if synced else "[ ]"

todo_count = count_test_todos(test_name, lib_prefix)

todo_suffix = f" (TODO: {todo_count})" if todo_count > 0 else ""

lines.append(f"{marker} test: {p}{todo_suffix}")

# If no lib or test paths exist, module doesn't exist

if not existing_lib_paths and not existing_test_paths:

lines.append(f"(module '{name}' not found)")

return lines

# Collect all hard_deps (explicit from DEPENDENCIES + implicit from lib_paths)

dep_info = DEPENDENCIES.get(name, {})

explicit_hard_deps = dep_info.get("hard_deps", [])

# Get implicit hard_deps from lib_paths (e.g., _pydecimal.py for decimal)

all_hard_deps = set()

for hd in explicit_hard_deps:

# Remove .py extension if present

all_hard_deps.add(hd[:-3] if hd.endswith(".py") else hd)

for p in existing_lib_paths:

dep_name = p.stem if p.is_file() else p.name

if dep_name != name: # Skip the main module itself

all_hard_deps.add(dep_name)

lines.append("\ndependencies:")

lines.extend(

format_deps_tree(

cpython_prefix,

lib_prefix,

max_depth,

soft_deps={name},

_visited=_visited,

hard_deps=all_hard_deps,

)

# Show dependent tests as tree (depth 2: module + direct importers + their importers)

tree = find_dependent_tests_tree(name, lib_prefix=lib_prefix, max_depth=2)

lines.extend(_format_dependent_tests_tree(tree, cpython_prefix, lib_prefix))

return lines

def _format_dependent_tests_tree(

tree: dict,

cpython_prefix: str,

lib_prefix: str,

indent: str = "",

) -> list[str]:

"""Format dependent tests tree for display."""

from update_lib.deps import is_up_to_date

lines = []

module = tree["module"]

tests = tree["tests"]

children = tree["children"]

if indent == "":

# Root level

# Count total tests in tree

def count_tests(t: dict) -> int:

total = len(t.get("tests", []))

for c in t.get("children", []):

total += count_tests(c)

return total

total = count_tests(tree)

if total == 0 and not children:

lines.append(f"\ndependent tests: (no tests depend on {module})")

return lines

lines.append(f"\ndependent tests: ({total} tests)")

# Check if module is up-to-date

synced = is_up_to_date(module.split(".")[0], cpython_prefix, lib_prefix)

marker = "[x]" if synced else "[ ]"

# Format this node

if tests:

test_str = " ".join(tests)

if indent == "":

lines.append(f"- {marker} {module}: {test_str}")

else:

lines.append(f"{indent}- {marker} {module}: {test_str}")

elif indent != "" and children:

# Has children but no direct tests

lines.append(f"{indent}- {marker} {module}:")

# Format children

child_indent = indent + " " if indent else " "

for child in children:

lines.extend(

_format_dependent_tests_tree(

child, cpython_prefix, lib_prefix, child_indent

)

return lines

def _resolve_module_name(

cpython_prefix: str,

lib_prefix: str,

) -> list[str]:

"""Resolve module name through redirects.

Returns a list of module names (usually 1, but test support files may expand to multiple).

"""

import pathlib

from update_lib.deps import (

_build_test_import_graph,

get_lib_paths,

get_test_paths,

resolve_hard_dep_parent,

resolve_test_to_lib,

)

# Resolve test to library group (e.g., test_urllib2 -> urllib)

if name.startswith("test_"):

lib_group = resolve_test_to_lib(name)

if lib_group:

return [lib_group]

name = name[5:]

# Resolve hard_dep to parent

parent = resolve_hard_dep_parent(name, cpython_prefix)

if parent:

return [parent]

# Check if it's a valid module

lib_paths = get_lib_paths(name, cpython_prefix)

test_paths = get_test_paths(name, cpython_prefix)

if any(p.exists() for p in lib_paths) or any(p.exists() for p in test_paths):

return [name]

# Check for test support files (e.g., string_tests -> bytes, str, userstring)

test_support_path = pathlib.Path(cpython_prefix) / "Lib" / "test" / f"{name}.py"

if test_support_path.exists():

test_dir = pathlib.Path(lib_prefix) / "test"

if test_dir.exists():

import_graph, _ = _build_test_import_graph(test_dir)

importing_tests = []

for file_key, imports in import_graph.items():

if name in imports and file_key.startswith("test_"):

importing_tests.append(file_key)

if importing_tests:

# Resolve test names to module names (test_bytes -> bytes)

return sorted(set(t[5:] for t in importing_tests))

return [name]

def show_deps(

names: list[str],

cpython_prefix: str,

lib_prefix: str,

max_depth: int = 10,

) -> None:

"""Show all dependency information for modules."""

# Expand "all" to all module names

expanded_names = []

for name in set(names):

if name == "all":

expanded_names.extend(get_all_modules(cpython_prefix))

else:

expanded_names.append(name)

# Resolve and deduplicate names (preserving order)

seen: set[str] = set()

resolved_names: list[str] = []

for name in expanded_names:

for resolved in _resolve_module_name(name, cpython_prefix, lib_prefix):

if resolved not in seen:

seen.add(resolved)

resolved_names.append(resolved)

# Shared visited set across all modules

visited: set[str] = set()

for i, name in enumerate(resolved_names):

if i > 0:

print() # blank line between modules

for line in format_deps(name, cpython_prefix, lib_prefix, max_depth, visited):

print(line)

def main(argv: list[str] | None = None) -> int:

parser = argparse.ArgumentParser(

description=__doc__,

formatter_class=argparse.RawDescriptionHelpFormatter,

)

parser.add_argument(

"names",

nargs="+",

help="Module names (e.g., dis, dataclasses) or 'all' for all modules",

)

parser.add_argument(

"--cpython",

default="cpython",

help="CPython directory prefix (default: cpython)",

)

parser.add_argument(

"--lib",

default="Lib",

help="Local Lib directory prefix (default: Lib)",

)

parser.add_argument(

"--depth",

type=int,

default=10,

help="Maximum recursion depth for soft_deps tree (default: 10)",

)

args = parser.parse_args(argv)

# When user does `./update_lib/ deps "foo bar" "baz"`

# We still want to get a list of

# ["foo", "bar", "baz"], not ["foo bar", "baz"]

args.names = [name for names in args.names for name in names.split()]

try:

show_deps(args.names, args.cpython, args.lib, args.depth)

return 0

except Exception as e:

print(f"Error: {e}", file=sys.stderr)

return 1

if __name__ == "__main__":

sys.exit(main())

RustPython/scripts/update_lib/cmd_deps.py at main · RustPython/RustPython · GitHub

Repository navigation

Expand file tree

Breadcrumbs

cmd_deps.py

Latest commit

History

Breadcrumbs

cmd_deps.py

File metadata and controls