FazBrowse GitHub Viewer | Trending |
URL:
| Home
Tools: [Download Repo ZIP]   [Original HTTPS Page]

add zipapp support by asottile · Pull Request #1616 · pre-commit/pre-commit · GitHub

Repository navigation

Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension .txt  (1) No extension  (4) All 2 file types selected
Only manifest files
Viewed files
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Unified
Split
Hide whitespace
Diff view
Unified
Split
Hide whitespace
1 change: 1 addition & 0 deletions requirements-dev.txt
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
covdefaults
coverage
distlib
pytest
pytest-env
re-assert
14 changes: 14 additions & 0 deletions testing/zipapp/Dockerfile
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
FROM ubuntu:bionic
RUN : \
&& apt-get update \
&& DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
python3 \
python3-distutils \
python3-venv \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*

ENV LANG=C.UTF-8 PATH=/venv/bin:$PATH
RUN : \
&& python3.6 -mvenv /venv \
&& pip install --no-cache-dir pip setuptools wheel no-manylinux --upgrade
71 changes: 71 additions & 0 deletions testing/zipapp/entry
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
#!/usr/bin/env python3
import os.path
import shutil
import stat
import sys
import tempfile
import zipfile

from pre_commit.file_lock import lock

CACHE_DIR = os.path.expanduser('~/.cache/pre-commit-zipapp')


def _make_executable(filename: str) -> None:
os.chmod(filename, os.stat(filename).st_mode | stat.S_IXUSR)


def _ensure_cache(zipf: zipfile.ZipFile, cache_key: str) -> str:
os.makedirs(CACHE_DIR, exist_ok=True)

cache_dest = os.path.join(CACHE_DIR, cache_key)
lock_filename = os.path.join(CACHE_DIR, f'{cache_key}.lock')

if os.path.exists(cache_dest):
return cache_dest

with lock(lock_filename, blocked_cb=lambda: None):
# another process may have completed this work
if os.path.exists(cache_dest):
return cache_dest

tmpdir = tempfile.mkdtemp(prefix=os.path.join(CACHE_DIR, ''))
try:
zipf.extractall(tmpdir)
# zip doesn't maintain permissions
_make_executable(os.path.join(tmpdir, 'python'))
_make_executable(os.path.join(tmpdir, 'python.exe'))
os.rename(tmpdir, cache_dest)
except BaseException:
shutil.rmtree(tmpdir)
raise

return cache_dest


def main() -> int:
with zipfile.ZipFile(os.path.dirname(__file__)) as zipf:
with zipf.open('CACHE_KEY') as f:
cache_key = f.read().decode().strip()

cache_dest = _ensure_cache(zipf, cache_key)

if sys.platform != 'win32':
exe = os.path.join(cache_dest, 'python')
else:
exe = os.path.join(cache_dest, 'python.exe')

cmd = (exe, '-mpre_commit', *sys.argv[1:])
if sys.platform == 'win32': # https://bugs.python.org/issue19124
import subprocess

if sys.version_info < (3, 7): # https://bugs.python.org/issue25942
return subprocess.Popen(cmd).wait()
else:
return subprocess.call(cmd)
else:
os.execvp(cmd[0], cmd)


if __name__ == '__main__':
exit(main())
106 changes: 106 additions & 0 deletions testing/zipapp/make
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Original file line number Diff line number Diff line change
@@ -0,0 +1,106 @@
#!/usr/bin/env python3
import argparse
import base64
import hashlib
import importlib.resources
import io
import os.path
import shutil
import subprocess
import tempfile
import zipapp
import zipfile

HERE = os.path.dirname(os.path.realpath(__file__))
IMG = 'make-pre-commit-zipapp'


def _msg(s: str) -> None:
print(f'\033[7m{s}\033[m')


def _exit_if_retv(*cmd: str) -> None:
if subprocess.call(cmd):
raise SystemExit(1)


def _check_no_shared_objects(wheeldir: str) -> None:
for zip_filename in os.listdir(wheeldir):
with zipfile.ZipFile(os.path.join(wheeldir, zip_filename)) as zipf:
for filename in zipf.namelist():
if filename.endswith('.so') or '.so.' in filename:
raise AssertionError(zip_filename, filename)


def _add_shim(dest: str) -> None:
shim = os.path.join(HERE, 'python')
shutil.copy(shim, dest)

bio = io.BytesIO()
with zipfile.ZipFile(bio, 'w') as zipf:
zipf.write(shim, arcname='__main__.py')

with open(os.path.join(dest, 'python.exe'), 'wb') as f:
f.write(importlib.resources.read_binary('distlib', 't32.exe'))
f.write(b'#!py.exe -3\n')
f.write(bio.getvalue())


def _write_cache_key(version: str, wheeldir: str, dest: str) -> None:
cache_hash = hashlib.sha256(f'{version}\n'.encode())
for filename in sorted(os.listdir(wheeldir)):
cache_hash.update(f'{filename}\n'.encode())
with open(os.path.join(HERE, 'python'), 'rb') as f:
cache_hash.update(f.read())
with open(os.path.join(dest, 'CACHE_KEY'), 'wb') as f:
f.write(base64.urlsafe_b64encode(cache_hash.digest()).rstrip(b'='))


def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument('version')
args = parser.parse_args()

with tempfile.TemporaryDirectory() as tmpdir:
wheeldir = os.path.join(tmpdir, 'wheels')
os.mkdir(wheeldir)

_msg('building podman image...')
_exit_if_retv('podman', 'build', '-q', '-t', IMG, HERE)

_msg('populating wheels...')
_exit_if_retv(
'podman', 'run', '--rm', '--volume', f'{wheeldir}:/wheels:rw', IMG,
'pip', 'wheel', f'pre_commit=={args.version}',
'--wheel-dir', '/wheels',
)

_msg('validating wheels...')
_check_no_shared_objects(wheeldir)

_msg('adding __main__.py...')
mainfile = os.path.join(tmpdir, '__main__.py')
shutil.copy(os.path.join(HERE, 'entry'), mainfile)

_msg('adding shim...')
_add_shim(tmpdir)

_msg('copying file_lock.py...')
file_lock_py = os.path.join(HERE, '../../pre_commit/file_lock.py')
file_lock_py_dest = os.path.join(tmpdir, 'pre_commit/file_lock.py')
os.makedirs(os.path.dirname(file_lock_py_dest))
shutil.copy(file_lock_py, file_lock_py_dest)

_msg('writing CACHE_KEY...')
_write_cache_key(args.version, wheeldir, tmpdir)

filename = f'pre-commit-{args.version}.pyz'
_msg(f'writing {filename}...')
shebang = '/usr/bin/env python3'
zipapp.create_archive(tmpdir, filename, interpreter=shebang)

return 0


if __name__ == '__main__':
exit(main())
48 changes: 48 additions & 0 deletions testing/zipapp/python
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
#!/usr/bin/env python3
"""A shim executable to put dependencies on sys.path"""
import argparse
import os.path
import runpy
import sys

# an exe-zipapp will have a __file__ of shim.exe/__main__.py
EXE = __file__ if os.path.isfile(__file__) else os.path.dirname(__file__)
EXE = os.path.realpath(EXE)
HERE = os.path.dirname(EXE)
WHEELDIR = os.path.join(HERE, 'wheels')
SITE_DIRS = frozenset(('dist-packages', 'site-packages'))


def main() -> int:
parser = argparse.ArgumentParser(add_help=False)
parser.add_argument('-m')
args, rest = parser.parse_known_args()

if args.m:
# try and remove site-packages from sys.path so our packages win
sys.path[:] = [
p for p in sys.path
if os.path.split(p)[1] not in SITE_DIRS
]
for wheel in sorted(os.listdir(WHEELDIR)):
sys.path.append(os.path.join(WHEELDIR, wheel))
if args.m == 'pre_commit' or args.m.startswith('pre_commit.'):
sys.executable = EXE
sys.argv[1:] = rest
runpy.run_module(args.m, run_name='__main__', alter_sys=True)
return 0
else:
cmd = (sys.executable, *sys.argv[1:])
if sys.platform == 'win32': # https://bugs.python.org/issue19124
import subprocess

if sys.version_info < (3, 7): # https://bugs.python.org/issue25942
return subprocess.Popen(cmd).wait()
else:
return subprocess.call(cmd)
else:
os.execvp(cmd[0], cmd)


if __name__ == '__main__':
exit(main())

Back | FazBrowse Home | New Git URL