FazBrowse GitHub Viewer | Trending |
URL:
| Home
Tools: [Download Repo ZIP]   [Original HTTPS Page]

[2.7] bpo-30450: Pull Windows dependencies from GitHub rather than SV… · python/cpython@986b7ff · GitHub

/ cpython Public

Commit 986b7ff

Browse files
zware
authored
[2.7] bpo-30450: Pull Windows dependencies from GitHub rather than SVN (GH-1783) (GH-3306)
The Windows build now depends on Python 3.6 to fetch externals, but it will be downloaded via NuGet (which is downloaded via PowerShell) if it is not available via `py -3.6`. This means the only thing that must be installed on a modern Windows box to do a full build of CPython with all extensions is Visual Studio. Cherry-picked from 51599e2, parts of 40a23e8, parts of 68d663c, d5cd21d, and possibly others that I've missed. Also: * Rename db -> bsddb for disambiguity * Update sqlite3 to 3.14.2.0 since it's the version we use on 3.x, and it's simpler to just use it than to also upload the old version to cpython-source-deps * Add PCbuild/*.ilk to .gitignore
1 parent 57d963b commit 986b7ff

14 files changed

Lines changed: 310 additions & 102 deletions

File tree

‎.github/appveyor.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,8 @@ build_script:
1010
- cmd: PCbuild\python.exe -m test.pythoninfo
1111
test_script:
1212
- cmd: PCbuild\rt.bat -q -uall -u-cpu -rwW --slowest -j2
13+
environment:
14+
HOST_PYTHON: C:\Python36\python.exe
1315

1416
# Only trigger AppVeyor if actual code or its configuration changes
1517
only_commits:

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ PC/*/*.suo
4444
PC/*/Win32-temp-*
4545
PC/*/x64-temp-*
4646
PC/*/amd64
47+
PCbuild/*.ilk
4748
PCbuild/*.user
4849
PCbuild/*.suo
4950
PCbuild/*.*sdf

‎Doc/make.bat‎

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,19 @@ pushd %~dp0
55

66
set this=%~n0
77

8-
if "%SPHINXBUILD%" EQU "" set SPHINXBUILD=sphinx-build
9-
if "%PYTHON%" EQU "" set PYTHON=py
8+
call ..\PCBuild\find_python.bat %PYTHON%
9+
if not defined SPHINXBUILD if defined PYTHON (
10+
%PYTHON% -c "import sphinx" > nul 2> nul
11+
if errorlevel 1 (
12+
echo Installing sphinx with %PYTHON%
13+
%PYTHON% -m pip install sphinx
14+
if errorlevel 1 exit /B
15+
)
16+
set SPHINXBUILD=%PYTHON% -c "import sphinx, sys; sys.argv[0] = 'sphinx-build'; sphinx.main()"
17+
)
18+
19+
if not defined PYTHON set PYTHON=py
20+
if not defined SPHINXBUILD set SPHINXBUILD=sphinx-build
1021

1122
if DEFINED ProgramFiles(x86) set _PRGMFLS=%ProgramFiles(x86)%
1223
if NOT DEFINED ProgramFiles(x86) set _PRGMFLS=%ProgramFiles%
@@ -73,7 +84,7 @@ goto end
7384
if NOT "%PAPER%" == "" (
7485
set SPHINXOPTS=-D latex_elements.papersize=%PAPER% %SPHINXOPTS%
7586
)
76-
cmd /C %SPHINXBUILD% %SPHINXOPTS% -b%1 -dbuild\doctrees . %BUILDDIR%\%*
87+
cmd /C "%SPHINXBUILD% %SPHINXOPTS% -b%1 -dbuild\doctrees . %BUILDDIR%\%*"
7788

7889
if "%1" EQU "htmlhelp" (
7990
if not exist "%HTMLHELP%" (

‎Misc/NEWS.d/next/Windows/2017-09-04-14-00-37.bpo-30450.YwitaJ.rst‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Pull build dependencies from GitHub rather than svn.python.org.

‎PC/VS9.0/pyproject.vsprops‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@
5050
/>
5151
<UserMacro
5252
Name="bsddb47Dir"
53-
Value="$(externalsDir)\db-4.7.25.0\build_windows"
53+
Value="$(externalsDir)\bsddb-4.7.25.0\build_windows"
5454
/>
5555
<UserMacro
5656
Name="bsddb47DepLibs"
@@ -74,7 +74,7 @@
7474
/>
7575
<UserMacro
7676
Name="sqlite3Dir"
77-
Value="$(externalsDir)\sqlite-3.8.11.0"
77+
Value="$(externalsDir)\sqlite-3.14.2.0"
7878
/>
7979
<UserMacro
8080
Name="bz2Dir"

‎PCbuild/build.bat‎

Lines changed: 18 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -5,12 +5,15 @@ echo.%~nx0 [flags and arguments] [quoted MSBuild options]
55
echo.
66
echo.Build CPython from the command line. Requires the appropriate
77
echo.version(s) of Microsoft Visual Studio to be installed (see readme.txt).
8-
echo.Also requires Subversion (svn.exe) to be on PATH if the '-e' flag is
9-
echo.given.
108
echo.
119
echo.After the flags recognized by this script, up to 9 arguments to be passed
1210
echo.directly to MSBuild may be passed. If the argument contains an '=', the
13-
echo.entire argument must be quoted (e.g. `%~nx0 "/p:PlatformToolset=v100"`)
11+
echo.entire argument must be quoted (e.g. `%~nx0 "/p:PlatformToolset=v100"`).
12+
echo.Alternatively you can put extra flags for MSBuild in a file named
13+
echo.`msbuild.rsp` in the `PCbuild` directory, one flag per line. This file
14+
echo.will be picked automatically by MSBuild. Flags put in this file does not
15+
echo.need to be quoted. You can still use environment variables inside the
16+
echo.response file.
1417
echo.
1518
echo.Available flags:
1619
echo. -h Display this help message
@@ -47,7 +50,6 @@ exit /b 127
4750
:Run
4851
setlocal
4952
set platf=Win32
50-
set vs_platf=x86
5153
set conf=Release
5254
set target=Build
5355
set dir=%~dp0
@@ -56,10 +58,6 @@ set verbose=/nologo /v:m
5658
set kill=
5759
set do_pgo=
5860
set pgo_job=-m test.regrtest --pgo
59-
set on_64_bit=true
60-
61-
rem This may not be 100% accurate, but close enough.
62-
if "%ProgramFiles(x86)%"=="" (set on_64_bit=false)
6361

6462
:CheckOpts
6563
if "%~1"=="-h" goto Usage
@@ -89,18 +87,12 @@ if "%IncludeBsddb%"=="" set IncludeBsddb=true
8987

9088
if "%IncludeExternals%"=="true" call "%dir%get_externals.bat"
9189

92-
if "%platf%"=="x64" (
93-
if "%on_64_bit%"=="true" (
94-
rem This ought to always be correct these days...
95-
set vs_platf=amd64
96-
) else (
97-
if "%do_pgo%"=="true" (
98-
echo.ERROR: Cannot cross-compile with PGO
99-
echo. 32bit operating system detected, if this is incorrect,
100-
echo. make sure the ProgramFiles(x86^) environment variable is set
101-
exit /b 1
102-
)
103-
set vs_platf=x86_amd64
90+
if "%do_pgo%" EQU "true" if "%platf%" EQU "x64" (
91+
if "%PROCESSOR_ARCHITEW6432%" NEQ "AMD64" if "%PROCESSOR_ARCHITECTURE%" NEQ "AMD64" (
92+
echo.ERROR: Cannot cross-compile with PGO
93+
echo. 32bit operating system detected. Ensure your PROCESSOR_ARCHITECTURE
94+
echo. and PROCESSOR_ARCHITEW6432 environment variables are correct.
95+
exit /b 1
10496
)
10597
)
10698

@@ -109,26 +101,28 @@ if exist "%GIT%" set GITProperty=/p:GIT="%GIT%"
109101
if not exist "%GIT%" echo Cannot find Git on PATH & set GITProperty=
110102

111103
rem Setup the environment
112-
call "%dir%env.bat" %vs_platf% >nul
104+
call "%dir%find_msbuild.bat" %MSBUILD%
105+
if ERRORLEVEL 1 (echo Cannot locate MSBuild.exe on PATH or as MSBUILD variable & exit /b 2)
113106

114107
if "%kill%"=="true" call :Kill
115108

116109
if "%do_pgo%"=="true" (
117110
set conf=PGInstrument
118-
call :Build
111+
call :Build %1 %2 %3 %4 %5 %6 %7 %8 %9
119112
del /s "%dir%\*.pgc"
120113
del /s "%dir%\..\Lib\*.pyc"
121114
echo on
122115
call "%dir%\..\python.bat" %pgo_job%
123116
@echo off
124117
call :Kill
125118
set conf=PGUpdate
119+
set target=Build
126120
)
127121
goto Build
128122

129123
:Kill
130124
echo on
131-
msbuild "%dir%\pythoncore.vcxproj" /t:KillPython %verbose%^
125+
%MSBUILD% "%dir%\pythoncore.vcxproj" /t:KillPython %verbose%^
132126
/p:Configuration=%conf% /p:Platform=%platf%^
133127
/p:KillPython=true
134128

@@ -140,7 +134,7 @@ rem Call on MSBuild to do the work, echo the command.
140134
rem Passing %1-9 is not the preferred option, but argument parsing in
141135
rem batch is, shall we say, "lackluster"
142136
echo on
143-
msbuild "%dir%pcbuild.proj" /t:%target% %parallel% %verbose%^
137+
%MSBUILD% "%dir%pcbuild.proj" /t:%target% %parallel% %verbose%^
144138
/p:Configuration=%conf% /p:Platform=%platf%^
145139
/p:IncludeExternals=%IncludeExternals%^
146140
/p:IncludeSSL=%IncludeSSL% /p:IncludeTkinter=%IncludeTkinter%^

‎PCbuild/find_msbuild.bat‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
@rem
2+
@rem Searches for MSBuild.exe. This is the only tool we need to initiate
3+
@rem a build, so we no longer search for the full VC toolset.
4+
@rem
5+
@rem This file is supposed to modify the state of the caller (specifically
6+
@rem the MSBUILD variable), so we do not use setlocal or echo, and avoid
7+
@rem changing any other persistent state.
8+
@rem
9+
10+
@rem No arguments provided means do full search
11+
@if '%1' EQU '' goto :begin_search
12+
13+
@rem One argument may be the full path. Use a goto so we don't try to
14+
@rem parse the next if statement - incorrect quoting in the multi-arg
15+
@rem case can cause us to break immediately.
16+
@if '%2' EQU '' goto :one_arg
17+
18+
@rem Entire command line may represent the full path if quoting failed.
19+
@if exist "%*" (set MSBUILD="%*") & (set _Py_MSBuild_Source=environment) & goto :found
20+
@goto :begin_search
21+
22+
:one_arg
23+
@if exist "%~1" (set MSBUILD="%~1") & (set _Py_MSBuild_Source=environment) & goto :found
24+
25+
:begin_search
26+
@set MSBUILD=
27+
28+
@rem If msbuild.exe is on the PATH, assume that the user wants that one.
29+
@where msbuild > "%TEMP%\msbuild.loc" 2> nul && set /P MSBUILD= < "%TEMP%\msbuild.loc" & del "%TEMP%\msbuild.loc"
30+
@if exist "%MSBUILD%" set MSBUILD="%MSBUILD%" & (set _Py_MSBuild_Source=PATH) & goto :found
31+
32+
@rem VS 2015 and earlier register MSBuild separately, so we can find it.
33+
@rem Prefer MSBuild 14.0 over MSBuild 15.0, since the latter may not be able to find a VC14 install.
34+
@reg query "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\MSBuild\ToolsVersions\14.0" /v MSBuildToolsPath /reg:32 >nul 2>nul
35+
@if NOT ERRORLEVEL 1 @for /F "tokens=1,2*" %%i in ('reg query "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\MSBuild\ToolsVersions\14.0" /v MSBuildToolsPath /reg:32') DO @(
36+
@if "%%i"=="MSBuildToolsPath" @if exist "%%k\msbuild.exe" @(set MSBUILD="%%k\msbuild.exe")
37+
)
38+
@if exist %MSBUILD% (set _Py_MSBuild_Source=registry) & goto :found
39+
40+
@rem VS 2017 sets exactly one install as the "main" install, so we may find MSBuild in there.
41+
@reg query "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\VisualStudio\SxS\VS7" /v 15.0 /reg:32 >nul 2>nul
42+
@if NOT ERRORLEVEL 1 @for /F "tokens=1,2*" %%i in ('reg query "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\VisualStudio\SxS\VS7" /v 15.0 /reg:32') DO @(
43+
@if "%%i"=="15.0" @if exist "%%k\MSBuild\15.0\Bin\msbuild.exe" @(set MSBUILD="%%k\MSBuild\15.0\Bin\msbuild.exe")
44+
)
45+
@if exist %MSBUILD% (set _Py_MSBuild_Source=Visual Studio 2017 registry) & goto :found
46+
47+
48+
@exit /b 1
49+
50+
:found
51+
@echo Using %MSBUILD% (found in the %_Py_MSBuild_Source%)
52+
@set _Py_MSBuild_Source=

‎PCbuild/find_python.bat‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
@rem
2+
@rem Searches for python.exe and may download a private copy from nuget.
3+
@rem
4+
@rem This file is supposed to modify the state of the caller (specifically
5+
@rem the MSBUILD variable), so we do not use setlocal or echo, and avoid
6+
@rem changing any other persistent state.
7+
@rem
8+
9+
@rem No arguments provided means do full search
10+
@if '%1' EQU '' goto :begin_search
11+
12+
@rem One argument may be the full path. Use a goto so we don't try to
13+
@rem parse the next if statement - incorrect quoting in the multi-arg
14+
@rem case can cause us to break immediately.
15+
@if '%2' EQU '' goto :one_arg
16+
17+
@rem Entire command line may represent the full path if quoting failed.
18+
@if exist "%*" (set PYTHON="%*") & (set _Py_Python_Source=from environment) & goto :found
19+
@goto :begin_search
20+
21+
:one_arg
22+
@if exist "%~1" (set PYTHON="%~1") & (set _Py_Python_Source=from environment) & goto :found
23+
24+
:begin_search
25+
@set PYTHON=
26+
27+
@set _Py_EXTERNALS_DIR=%EXTERNAL_DIR%
28+
@if "%_Py_EXTERNALS_DIR%"=="" (set _Py_EXTERNALS_DIR=%~dp0\..\externals)
29+
30+
@rem If we have Python in externals, use that one
31+
@if exist "%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe" (set PYTHON="%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe") & (set _Py_Python_Source=found in externals directory) & goto :found
32+
33+
@rem If HOST_PYTHON is recent enough, use that
34+
@if NOT "%HOST_PYTHON%"=="" @%HOST_PYTHON% -c "import sys; assert sys.version_info[:2] >= (3, 6)" >nul 2>nul && (set PYTHON="%HOST_PYTHON%") && (set _Py_Python_Source=found as HOST_PYTHON) && goto :found
35+
36+
@rem If py.exe finds a recent enough version, use that one
37+
@py -3.6 -V >nul 2>&1 && (set PYTHON=py -3.6) && (set _Py_Python_Source=found with py.exe) && goto :found
38+
39+
@if NOT exist "%_Py_EXTERNALS_DIR%" mkdir "%_Py_EXTERNALS_DIR%"
40+
@set _Py_NUGET=%NUGET%
41+
@set _Py_NUGET_URL=%NUGET_URL%
42+
@set _Py_HOST_PYTHON=%HOST_PYTHON%
43+
@if "%_Py_HOST_PYTHON%"=="" set _Py_HOST_PYTHON=py
44+
@if "%_Py_NUGET%"=="" (set _Py_NUGET=%_Py_EXTERNALS_DIR%\nuget.exe)
45+
@if "%_Py_NUGET_URL%"=="" (set _Py_NUGET_URL=https://aka.ms/nugetclidl)
46+
@if NOT exist "%_Py_NUGET%" (
47+
@echo Downloading nuget...
48+
@rem NB: Must use single quotes around NUGET here, NOT double!
49+
@rem Otherwise, a space in the path would break things
50+
@rem If it fails, retry with any available copy of Python
51+
@powershell.exe -Command Invoke-WebRequest %_Py_NUGET_URL% -OutFile '%_Py_NUGET%'
52+
@if errorlevel 1 (
53+
@%_Py_HOST_PYTHON% "%~dp0\urlretrieve.py" "%_Py_NUGET_URL%" "%_Py_NUGET%"
54+
)
55+
)
56+
@echo Installing Python via nuget...
57+
@"%_Py_NUGET%" install pythonx86 -ExcludeVersion -OutputDirectory "%_Py_EXTERNALS_DIR%"
58+
@rem Quote it here; it's not quoted later because "py -3.6" wouldn't work
59+
@if not errorlevel 1 (set PYTHON="%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe") & (set _Py_Python_Source=found on nuget.org) & goto :found
60+
61+
62+
@set _Py_Python_Source=
63+
@set _Py_EXTERNALS_DIR=
64+
@set _Py_NUGET=
65+
@set _Py_NUGET_URL=
66+
@set _Py_HOST_PYTHON=
67+
@exit /b 1
68+
69+
:found
70+
@echo Using %PYTHON% (%_Py_Python_Source%)
71+
@set _Py_Python_Source=
72+
@set _Py_EXTERNALS_DIR=
73+
@set _Py_NUGET=
74+
@set _Py_NUGET_URL=
75+
@set _Py_HOST_PYTHON=

‎PCbuild/get_external.py‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
#!/usr/bin/env python3
2+
3+
import argparse
4+
import os
5+
import pathlib
6+
import zipfile
7+
from urllib.request import urlretrieve
8+
9+
10+
def fetch_zip(commit_hash, zip_dir, *, org='python', binary=False, verbose):
11+
repo = f'cpython-{"bin" if binary else "source"}-deps'
12+
url = f'https://github.com/{org}/{repo}/archive/{commit_hash}.zip'
13+
reporthook = None
14+
if verbose:
15+
reporthook = print
16+
zip_dir.mkdir(parents=True, exist_ok=True)
17+
filename, headers = urlretrieve(
18+
url,
19+
zip_dir / f'{commit_hash}.zip',
20+
reporthook=reporthook,
21+
)
22+
return filename
23+
24+
25+
def extract_zip(externals_dir, zip_path):
26+
with zipfile.ZipFile(os.fspath(zip_path)) as zf:
27+
zf.extractall(os.fspath(externals_dir))
28+
return externals_dir / zf.namelist()[0].split('/')[0]
29+
30+
31+
def parse_args():
32+
p = argparse.ArgumentParser()
33+
p.add_argument('-v', '--verbose', action='store_true')
34+
p.add_argument('-b', '--binary', action='store_true',
35+
help='Is the dependency in the binary repo?')
36+
p.add_argument('-O', '--organization',
37+
help='Organization owning the deps repos', default='python')
38+
p.add_argument('-e', '--externals-dir', type=pathlib.Path,
39+
help='Directory in which to store dependencies',
40+
default=pathlib.Path(__file__).parent.parent / 'externals')
41+
p.add_argument('tag',
42+
help='tag of the dependency')
43+
return p.parse_args()
44+
45+
46+
def main():
47+
args = parse_args()
48+
zip_path = fetch_zip(
49+
args.tag,
50+
args.externals_dir / 'zips',
51+
org=args.organization,
52+
binary=args.binary,
53+
verbose=args.verbose,
54+
)
55+
final_name = args.externals_dir / args.tag
56+
extract_zip(args.externals_dir, zip_path).replace(final_name)
57+
58+
59+
if __name__ == '__main__':
60+
main()

0 commit comments

Comments
 (0)
Back | FazBrowse Home | New Git URL