# Copyright (c) 2016 CEF Python, see the Authors file. All rights reserved.
"""
Prepares CEF binaries and libraries for work with the build.py tool.
Option 1 is to build CEF from sources with the CEF Python patches applied
using the --build-cef flag.
Option 2 is to use CEF binaries from Spotify Automated Builds using
the --prebuilt-cef flag. In such case check the cefpython/src/version/
directory to know which version of CEF to download from Spotify:
http://opensource.spotify.com/cefbuilds/index.html
Download and extract it so that for example you have such a directory:
cefpython/build/cef_binary_3.2883.1553.g80bd606_windows32/ .
This tool generates CEF binaries and libraries that are ready for work
with cefpython, with the build.py script. When automate.py tool completes
job you should see a new subdirectory in the build/ directory, for example:
cefpython/build/cef55_3.2883.1553.g80bd606_win32/ .
Usage:
automate.py (--prebuilt-cef | --build-cef)
[--fast-build FAST_BUILD]
[--force-chromium-update FORCE_CHROMIUM_UPDATE]
[--no-cef-update NO_CEF_UPDATE]
[--cef-branch BRANCH] [--cef-commit COMMIT]
[--build-dir BUILD_DIR] [--cef-build-dir CEF_BUIL_DDIR]
[--ninja-jobs JOBS] [--gyp-generators GENERATORS]
[--gyp-msvs-version MSVS]
automate.py (-h | --help) [type -h to show full description for options]
Options:
-h --help Show this help message.
--prebuilt-cef Whether to use prebuilt CEF binaries. Prebuilt
binaries for Linux are built on Ubuntu.
--build-cef Whether to build CEF from sources with the
cefpython patches applied.
--fast-build Fast build with is_official_build=False
--force-chromium-update Force Chromium update (gclient sync etc).
--no-cef-update Do not update CEF sources (by default both cef/
directories are deleted on every run).
--cef-branch= CEF branch. Defaults to CHROME_VERSION_BUILD from
"src/version/cef_version_{platform}.h".
--cef-commit= CEF revision. Defaults to CEF_COMMIT_HASH from
"src/version/cef_version_{platform}.h".
--build-dir= Build directory.
--cef-build-dir= CEF build directory. By default same
as --build-dir.
--ninja-jobs= How many CEF jobs to run in parallel. To speed up
building set it to number of cores in your CPU.
By default set to cpu_count / 2.
--gyp-generators= Set GYP_GENERATORS [default: ninja].
--gyp-msvs-version= Set GYP_MSVS_VERSION.
"""
from common import *
import os
import sys
import shlex
import subprocess
import platform
import docopt
import stat
import glob
import shutil
import multiprocessing
CEF_GIT_URL = "https://bitbucket.org/chromiumembedded/cef.git"
class Options(object):
"""Options from command-line and internal options."""
# From command-line
prebuilt_cef = False
build_cef = False
fast_build = False
force_chromium_update = False
no_cef_update = False
cef_branch = ""
cef_commit = ""
cef_version = ""
build_dir = ""
cef_build_dir = ""
ninja_jobs = None
gyp_generators = "ninja" # Even though CEF uses now GN, still some GYP
gyp_msvs_version = "" # env variables are being used.
# Internal options
depot_tools_dir = ""
tools_dir = ""
cefpython_dir = ""
binary_distrib = ""
release_build = True
build_type = "" # Will be set according to "release_build" value
cef_binary = ""
build_cefclient_dir = ""
build_wrapper_mt_dir = ""
build_wrapper_md_dir = ""
def main():
"""Main entry point."""
if not ((2, 7) > warning C4275: non dll-interface class 'stdext::exception'
# >> used as base for dll-interface class 'std::bad_cast'
# This warning occurs only in VS2008, in VS2013 not.
# This replacements must be unique for the undo operation
# to be reliable.
mt_find = r"/MT "
mt_replace = r"/MD /wd4275 "
mtd_find = r"/MTd "
mtd_replace = r"/MDd /wd4275 "
cmake_variables = os.path.join(Options.cef_binary, "cmake",
"cef_variables.cmake")
with open(cmake_variables, "rb") as fp:
contents = fp.read()
if try_undo:
matches1 = re.findall(re.escape(mt_replace), contents)
matches2 = re.findall(re.escape(mtd_replace), contents)
if len(matches1) or len(matches2):
undo = True
else:
return
if undo:
(contents, count) = re.subn(re.escape(mt_replace), mt_find,
contents)
assert count == 2
(contents, count) = re.subn(re.escape(mtd_replace), mtd_find,
contents)
assert count == 1
else:
(contents, count) = re.subn(re.escape(mt_find), mt_replace,
contents)
assert count == 2
(contents, count) = re.subn(re.escape(mtd_find), mtd_replace,
contents)
assert count == 1
with open(cmake_variables, "wb") as fp:
fp.write(contents)
def build_cefclient_succeeded():
"""Whether building cefclient/cefsimple/ceftests succeeded."""
assert Options.build_cefclient_dir
cefclient_exe = "cefclient.exe" if WINDOWS else "cefclient"
return os.path.exists(os.path.join(Options.build_cefclient_dir,
"tests",
"cefclient",
Options.build_type,
cefclient_exe))
def build_wrapper_mt_succeeded():
"""Whether building /MT library succeeded (Windows-only)."""
assert Options.build_wrapper_mt_dir
return os.path.exists(os.path.join(Options.build_wrapper_mt_dir,
"libcef_dll_wrapper",
"libcef_dll_wrapper.lib"))
def build_wrapper_md_succeeded():
"""Whether building /MD library succeeded (Windows-only)."""
assert Options.build_wrapper_md_dir
return os.path.exists(os.path.join(Options.build_wrapper_md_dir,
"libcef_dll_wrapper",
"libcef_dll_wrapper.lib"))
def prepare_build_command(build_lib=False):
"""On Windows VS env variables must be set up by calling vcvarsall.bat"""
command = list()
if platform.system() == "Windows":
if build_lib:
msvs = get_msvs_for_python()
command.append(globals()["VS"+msvs+"_VCVARS"])
else:
if int(Options.cef_branch) >= 2704:
command.append(VS2015_VCVARS)
else:
command.append(VS2013_VCVARS)
command.append("&&")
return command
def fix_cef_include_files():
"""Fixes to CEF include header files for eg. VS2008 on Windows."""
if platform.system() == "Windows" and get_msvs_for_python() == "2008":
print("[automate.py] Fixing CEF include/ files")
# cef_types_wrappers.h
cef_types_wrappers = os.path.join(Options.cef_binary, "include",
"internal", "cef_types_wrappers.h")
with open(cef_types_wrappers, "rb") as fp:
contents = fp.read()
# error C2059: syntax error : '{'
contents = contents.replace("s->range = {0, 0};",
"s->range.from = 0; s->range.to = 0;")
with open(cef_types_wrappers, "wb") as fp:
fp.write(contents)
def create_prebuilt_binaries():
"""After building copy binaries/libs to build/cef_xxxx/. """
# Directories
src = Options.cef_binary
version_header = os.path.join(src, "include", "cef_version.h")
dst = get_prebuilt_name(version_header)
dst = os.path.join(Options.build_dir, dst)
rmdir(dst)
os.makedirs(dst)
bindir = os.path.join(dst, "bin")
libdir = os.path.join(dst, "lib")
os.makedirs(bindir)
os.makedirs(libdir)
# Copy Release/Debug and Resources
cpdir(os.path.join(src, Options.build_type), bindir)
cpdir(os.path.join(src, "Resources"), bindir)
# Copy cefclient, cefsimple, ceftests
# cefclient
cefclient = os.path.join(
src,
"build_cefclient", "tests", "cefclient",
Options.build_type,
"cefclient")
if platform.system() != "Windows":
# On Windows resources/*.html files are embedded inside exe
cefclient_files = os.path.join(
src,
"build_cefclient", "tests", "cefclient",
Options.build_type,
"cefclient_files")
cpdir(cefclient_files, os.path.join(bindir, "cefclient_files"))
# cefsimple
cefsimple = os.path.join(
src,
"build_cefclient", "tests", "cefsimple",
Options.build_type,
"cefsimple")
# ceftests
ceftests = os.path.join(
src,
"build_cefclient", "tests", "ceftests",
Options.build_type,
"ceftests")
if platform.system() != "Windows":
# On Windows resources/*.html files are embedded inside exe
ceftests_files = os.path.join(
src,
"build_cefclient", "tests", "ceftests",
Options.build_type,
"ceftests_files")
cpdir(ceftests_files, os.path.join(bindir, "ceftests_files"))
if platform.system() == "Windows":
cefclient += ".exe"
cefsimple += ".exe"
ceftests += ".exe"
shutil.copy(cefclient, bindir)
shutil.copy(cefsimple, bindir)
shutil.copy(ceftests, bindir)
# END: Copy cefclient, cefsimple, ceftests
# Copy libraries
if platform.system() == "Windows":
# libcef.lib and cef_sandbox.lib
mvfiles(bindir, libdir, ".lib")
# MT lib
libsrc = os.path.join(src, "build_wrapper_mt", "libcef_dll_wrapper",
"libcef_dll_wrapper.lib")
libdst = os.path.join(libdir, "libcef_dll_wrapper_mt.lib")
shutil.copy(libsrc, libdst)
# MD lib
libsrc = os.path.join(src, "build_wrapper_md", "libcef_dll_wrapper",
"libcef_dll_wrapper.lib")
libdst = os.path.join(libdir, "libcef_dll_wrapper_md.lib")
shutil.copy(libsrc, libdst)
elif platform.system() == "Linux":
cpdir(os.path.join(src, "build_cefclient", "libcef_dll_wrapper"),
libdir)
# Remove .lib files from bin/ only after libraries were copied
libs = glob.glob(os.path.join(bindir, "*.lib"))
for lib in libs:
os.remove(lib)
# Remove cef_sandbox.lib (huge file)
cef_sandbox = os.path.join(libdir, "cef_sandbox.lib")
if os.path.exists(cef_sandbox):
os.remove(cef_sandbox)
# Copy README.txt and LICENSE.txt
shutil.copy(os.path.join(src, "README.txt"), dst)
shutil.copy(os.path.join(src, "LICENSE.txt"), dst)
print("[automate.py] OK prebuilt binaries created in '%s/'" % dst)
def get_msvs_for_python():
"""Get MSVS version (eg 2008) for current python running."""
if sys.version_info[:2] == (2, 7):
return "2008"
elif sys.version_info[:2] == (3, 4):
return "2010"
elif sys.version_info[:2] == (3, 5):
return "2015"
else:
print("ERROR: This python version is not yet supported")
sys.exit(1)
def getenv():
"""Env variables passed to shell when running commands."""
env = os.environ
env["PATH"] = Options.depot_tools_dir + os.pathsep + env["PATH"]
env["GYP_GENERATORS"] = Options.gyp_generators
if platform.system() == "Windows":
env["GYP_MSVS_VERSION"] = Options.gyp_msvs_version
# See cef/AutomatedBuildSetup.md for reference.
# Issue73 patch applied with "use_allocator=none"
# TODO: 32-bit gyp defines: host_arch=x86_64 target_arch=ia32
env["GN_DEFINES"] = "use_sysroot=true use_allocator=none symbol_level=1"
# To perform an official build set GYP_DEFINES=buildtype=Official.
# This will disable debugging code and enable additional link-time
# optimizations in Release builds.
if Options.release_build and not Options.fast_build:
env["GN_DEFINES"] += " is_official_build=true"
# Modifications to automate-git.py
env["CEFPYTHON_NINJA_JOBS"] = str(Options.ninja_jobs)
return env
def run_command(command, working_dir):
"""Run command in a given directory with env variables set.
On Linux multiple commands on one line with the use of && are not allowed.
"""
print("[automate.py] Running '"+" ".join(command)+"' in '" +
working_dir+"'...")
if isinstance(command, str):
args = shlex.split(command.replace("\\", "\\\\"))
else:
args = command
return subprocess.check_call(args, cwd=working_dir, env=getenv(),
shell=(platform.system() == "Windows"))
def run_python(command_line, working_dir):
"""Run python script using depot_tools."""
python = "python"
return run_command("%s %s" % (python, command_line), working_dir)
def run_git(command_line, working_dir):
"""Run git command using depot_tools."""
return run_command("git %s" % command_line, working_dir)
def run_automate_git():
"""Run CEF automate-git.py."""
script = os.path.join(Options.cefpython_dir, "tools", "automate-git.py")
"""
Example automate-git.py command:
C:\chromium>call python automate-git.py --download-dir=./test/
--branch=2526 --no-debug-build --verbose-build
Run ninja build manually:
cd chromium/src
ninja -v -j2 -Cout\Release cefclient
"""
args = []
if ARCH64:
args.append("--x64-build")
args.append("--download-dir=" + Options.cef_build_dir)
args.append("--branch=" + Options.cef_branch)
if Options.release_build:
args.append("--no-debug-build")
args.append("--verbose-build")
# --force-build sets --force-distrib by default
# ninja will only recompile files that changed
args.append("--force-build")
# We clone cef repository ourselves and update cef patches with ours,
# so don't fetch/update CEF repo.
args.append("--no-cef-update")
# Force Chromium update so that gclient sync is called. It may fail
# sometimes with files missing and must re-run to fix.
if Options.force_chromium_update:
args.append("--force-update")
args.append("--no-distrib-archive")
if platform.system() == "Linux":
# Building cefclient target isn't supported on Linux when
# using sysroot (cef/#1916). However building cefclient
# later in cef_binary/ with cmake/ninja do works fine.
args.append("--build-target=cefsimple")
args = " ".join(args)
return run_python(script+" "+args, Options.cef_build_dir)
def rmdir(path):
"""Delete directory recursively."""
if os.path.exists(path):
print("[automate.py] Removing directory %s" % path)
shutil.rmtree(path, onerror=onerror)
def cpdir(src, dst):
"""An equivalent of linux 'cp -r src/* dst/'. """
names = os.listdir(src)
if not os.path.exists(dst):
os.makedirs(dst)
for name in names:
path = os.path.join(src, name)
if os.path.isdir(path):
dst_subdir = os.path.join(dst, name)
shutil.copytree(path, dst_subdir)
else:
shutil.copy(path, dst)
def mvfiles(src, dst, ext):
"""An equivalent of linux 'mv src/*.ext dst/'. """
names = os.listdir(src)
if not os.path.exists(dst):
os.makedirs(dst)
for name in names:
path = os.path.join(src, name)
if os.path.isfile(path) and name.endswith(ext):
shutil.copy(path, dst)
def onerror(func, path, _):
"""Fix file permission on error and retry operation."""
if not os.access(path, os.W_OK):
os.chmod(path, stat.S_IWUSR)
func(path)
else:
raise Exception("Not a file permission error, dunno what to do")
def get_prebuilt_name(header_file=""):
if header_file:
version = get_version_from_file(header_file)
else:
version = get_cefpython_version()
name = "cef%s_%s_%s" % (
version["CHROME_VERSION_MAJOR"],
version["CEF_VERSION"],
OS_POSTFIX2,
)
return name
if __name__ == "__main__":
main()