FazBrowse GitHub Viewer | Trending |
URL:
| Home
Tools: [Download Repo ZIP]   [Original HTTPS Page]

autoupdate: speedup & show progress by casperdcl · Pull Request #3736 · pre-commit/pre-commit · GitHub

Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension .cfg  (1) .py  (3) All 2 file types selected
Viewed files
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Unified
Split
Hide whitespace
Diff view
Unified
Split
Hide whitespace
96 changes: 34 additions & 62 deletions pre_commit/commands/autoupdate.py
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Original file line number Diff line number Diff line change
@@ -1,17 +1,16 @@
from __future__ import annotations

import concurrent.futures
import os.path
import re
import tempfile
from collections.abc import Sequence
from typing import Any
from typing import NamedTuple

from tqdm import tqdm
from tqdm.contrib.concurrent import thread_map

import pre_commit.constants as C
from pre_commit import git
from pre_commit import output
from pre_commit import xargs
from pre_commit.clientlib import InvalidManifestError
from pre_commit.clientlib import load_config
from pre_commit.clientlib import load_manifest
Expand Down Expand Up @@ -100,19 +99,6 @@ def _check_hooks_still_exist_at_rev(
)


def _update_one(
i: int,
repo: dict[str, Any],
*,
tags_only: bool,
freeze: bool,
) -> tuple[int, RevInfo, RevInfo]:
old = RevInfo.from_config(repo)
new = old.update(tags_only=tags_only, freeze=freeze)
_check_hooks_still_exist_at_rev(repo, new)
return i, old, new


REV_LINE_RE = re.compile(r'^(\s+)rev:(\s*)([\'"]?)([^\s#]+)(.*)(\r?\n)$')


Expand Down Expand Up @@ -163,60 +149,46 @@ def autoupdate(
config_file: str,
tags_only: bool,
freeze: bool,
repos: Sequence[str] = (),
jobs: int = 1,
jobs: int | None = None,
) -> int:
"""Auto-update the pre-commit config to the latest versions of repos."""
migrate_config(config_file, quiet=True)
changed = False
retv = 0
changed_retv = [False, 0]

config_repos = [
repo for repo in load_config(config_file)['repos']
if repo['repo'] not in {LOCAL, META}
]
missing_repos = set(repos) - {r['repo'] for r in config_repos}
if missing_repos:
output.write_line(
f'repos ({", ".join(sorted(missing_repos))}) were '
f'not found in {config_file}',
)
return 1

rev_infos: list[RevInfo | None] = [None] * len(config_repos)
jobs = jobs or xargs.cpu_count() # 0 => number of cpus
jobs = min(jobs, len(repos) or len(config_repos)) # max 1-per-thread
jobs = max(jobs, 1) # at least one thread
with concurrent.futures.ThreadPoolExecutor(jobs) as exe:
futures = [
exe.submit(
_update_one,
i, repo, tags_only=tags_only, freeze=freeze,
)
for i, repo in enumerate(config_repos)
if not repos or repo['repo'] in repos
]
for future in concurrent.futures.as_completed(futures):
try:
i, old, new = future.result()
except RepositoryCannotBeUpdatedError as e:
output.write_line(str(e))
retv = 1
else:
if new.rev != old.rev:
changed = True
if new.frozen:
new_s = f'{new.frozen} (frozen)'
else:
new_s = new.rev
msg = f'updating {old.rev} -> {new_s}'
rev_infos[i] = new
else:
msg = 'already up to date!'

output.write_line(f'[{old.repo}] {msg}')

if changed:
def _update_one(i: int, repo: dict[str, Any]) -> None:
try:
old = RevInfo.from_config(repo)
new = old.update(tags_only=tags_only, freeze=freeze)
_check_hooks_still_exist_at_rev(repo, new)
except RepositoryCannotBeUpdatedError as e:
tqdm.write(str(e))
changed_retv[1] = 1
else:
if new.rev != old.rev:
changed_retv[0] = True
if new.frozen:
new_s = f'{new.frozen} (frozen)'
else:
new_s = new.rev
msg = f'updating {old.rev} -> {new_s}'
rev_infos[i] = new
else:
msg = 'already up to date!'
tqdm.write(f'[{old.repo}] {msg}')

list(
thread_map(
_update_one, range(len(config_repos)), config_repos,
unit='repo', desc='Updating', leave=False, max_workers=jobs,
),
)
if changed_retv[0]:
_write_new_config(config_file, rev_infos)

return retv
return changed_retv[1]
5 changes: 2 additions & 3 deletions pre_commit/main.py
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Original file line number Diff line number Diff line change
Expand Up @@ -238,8 +238,8 @@ def _add_cmd(name: str, *, help: str) -> argparse.ArgumentParser:
help='Only update this repository -- may be specified multiple times.',
)
autoupdate_parser.add_argument(
'-j', '--jobs', type=int, default=1,
help='Number of threads to use. (default %(default)s).',
'-j', '--jobs', type=int,
help='Number of threads to use. (default: automatic).',
)

_add_cmd('clean', help='Clean out pre-commit files.')
Expand Down Expand Up @@ -388,7 +388,6 @@ def _add_cmd(name: str, *, help: str) -> argparse.ArgumentParser:
args.config,
tags_only=not args.bleeding_edge,
freeze=args.freeze,
repos=args.repos,
jobs=args.jobs,
)
elif args.command == 'clean':
Expand Down
1 change: 1 addition & 0 deletions setup.cfg
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ install_requires =
identify>=1.0.0
nodeenv>=0.11.1
pyyaml>=5.1
tqdm>=4.42
virtualenv>=20.10.0
python_requires = >=3.10

Expand Down
11 changes: 2 additions & 9 deletions tests/commands/autoupdate_test.py
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Original file line number Diff line number Diff line change
Expand Up @@ -247,11 +247,7 @@ def test_autoupdate_out_of_date_repo_with_correct_repo_name(

with open(C.CONFIG_FILE) as f:
before = f.read()
repo_name = f'file://{out_of_date.path}'
ret = autoupdate(
C.CONFIG_FILE, freeze=False, tags_only=False,
repos=(repo_name,),
)
ret = autoupdate(C.CONFIG_FILE, freeze=False, tags_only=False)
with open(C.CONFIG_FILE) as f:
after = f.read()
assert ret == 0
Expand All @@ -270,10 +266,7 @@ def test_autoupdate_missing_repo_name(

with open(C.CONFIG_FILE) as f:
before = f.read()
ret = autoupdate(
C.CONFIG_FILE, freeze=False, tags_only=False,
repos=('dne', 'foo'),
)
ret = autoupdate(C.CONFIG_FILE, freeze=False, tags_only=False)
with open(C.CONFIG_FILE) as f:
after = f.read()
assert ret == 1
Expand Down
Loading

Back | FazBrowse Home | New Git URL