livecode/buildbot.py at develop · livecodeali/livecode · GitHub

History

executable file

269 lines (225 loc) · 9.07 KB

Raw

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

187

188

189

190

191

192

193

194

195

196

197

198

199

200

201

202

203

204

205

206

207

208

209

210

211

212

213

214

215

216

217

218

219

220

221

222

223

224

225

226

227

228

229

230

231

232

233

234

235

236

237

238

239

240

241

242

243

244

245

246

247

248

249

250

251

252

253

254

255

256

257

258

259

260

261

262

263

264

265

266

267

268

269

#!/usr/bin/env python

# This file is part of LiveCode.

# LiveCode is free software; you can redistribute it and/or modify it under

# the terms of the GNU General Public License v3 as published by the Free

# Software Foundation.

# LiveCode is distributed in the hope that it will be useful, but WITHOUT ANY

# WARRANTY; without even the implied warranty of MERCHANTABILITY or

# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License

# for more details.

# You should have received a copy of the GNU General Public License

# along with LiveCode. If not see <http://www.gnu.org/licenses/>.

# This file contains rules used by the LiveCode Buildbot installation

# at <https://vulcan.livecode.com/>

# Tasks that may be run on Windows workers must be implemented in this

# file. Other tasks may be implemented in buildbot.mk, and this

# script redirects to that Makefile. All buildbot operations should

# occur via this script, so that buildbot does not invoke

# platform-specific tools directly.

import sys

import subprocess

import os

import platform as _platform

import shutil

import tarfile

import uuid

# LiveCode build configuration script

import config

import fetch

# The set of build tasks that this branch supports

BUILDBOT_TARGETS = ('fetch', 'config', 'compile', 'bin-archive', 'bin-extract',

'dist-notes', 'dist-docs', 'dist-server', 'dist-tools', 'dist-upload',

'distmac-archive', 'distmac-extract', 'distmac-disk')

SKIP_EXIT_STATUS = 88

def usage(exit_status):

print(

"""Perform continuous integration and release build steps.

Usage:

buildbot.py TARGET

Environment variables:

BUILD_EDITION LiveCode edition name ("commercial" or "community")

BUILD_PLATFORM The target platform for build (e.g. "ios")

BUILD_SUBPLATFORM The target subplatform (e.g. "iphoneos10.2")

Many tasks are deferred to the "buildbot.mk" Makefile and the

"config.py" configuration script. Refer to these for other

environment variables that may affect the build.

""")

sys.exit(exit_status)

def error(message):

print("ERROR: " + message)

sys.exit(1)

def get_target_triple():

return os.environ.get('BUILD_TARGET_TRIPLE')

def get_build_platform():

platform = (os.environ.get('BUILD_PLATFORM'),

os.environ.get('BUILD_SUBPLATFORM'))

if platform[0] is None:

error('You must set $BUILD_PLATFORM')

return platform

def get_buildtype():

return os.environ.get('BUILDTYPE', 'Debug')

def get_build_edition():

return os.environ.get('BUILD_EDITION', 'community')

def check_target_triple():

# Check that this branch can actually be built for the specified platform

triple = get_target_triple()

if not triple in config.BUILDBOT_PLATFORM_TRIPLES:

print('Buildbot build for "{}" platform is not supported'.format(triple))

sys.exit(SKIP_EXIT_STATUS)

################################################################

# Defer to buildbot.mk

################################################################

def exec_buildbot_make(target):

args = ["make", "-f", "buildbot.mk", target]

print(' '.join(args))

sys.exit(subprocess.call(args))

################################################################

# Fetch prebuilts

################################################################

def exec_fetch(args):

print('fetch.py ' + ' '.join(args))

sys.exit(fetch.fetch(args))

def do_fetch():

check_target_triple()

exec_fetch(['--target', get_target_triple()])

################################################################

# Configure with gyp

################################################################

def exec_configure(args):

print('config.py ' + ' '.join(args))

sys.exit(config.configure(args))

def do_configure():

check_target_triple()

platform, subplatform = get_build_platform()

if platform == 'ios':

if subplatform is None:

error('You must set $BUILD_SUBPLATFORM for iOS builds')

exec_configure(['--platform', 'ios',

'--generator-output',

'build-{}-{}/livecode'.format(platform, subplatform),

'-Dtarget_sdk=' + subplatform])

else:

exec_configure(['--platform', platform])

return 0

################################################################

# Compile

################################################################

def exec_make(target):

args = ['make', target]

print(' '.join(args))

sys.exit(subprocess.call(args))

# mspdbsrv is the service used by Visual Studio to collect debug

# data during compilation. One instance is shared by all C++

# compiler instances and threads. It poses a unique challenge in

# several ways:

# - If not running when the build job starts, the build job will

# automatically spawn it as soon as it needs to emit debug symbols.

# There's no way to prevent this from happening.

# - The build job _doesn't_ automatically clean it up when it finishes

# - By default, mspdbsrv inherits its parent process' file handles,

# including (unfortunately) some log handles owned by Buildbot. This

# can prevent Buildbot from detecting that the compile job is finished

# - If a compile job starts and detects an instance of mspdbsrv already

# running, by default it will reuse it. So, if you have a compile

# job A running, and start a second job B, job B will use job A's

# instance of mspdbsrv. If you kill mspdbsrv when job A finishes,

# job B will die horribly. To make matters worse, the version of

# mspdbsrv should match the version of Visual Studio being used.

# This class works around these problems:

# - It sets the _MSPDBSRV_ENDPOINT_ to a value that's probably unique to

# the build, to prevent other builds on the same machine from sharing

# the same mspdbsrv endpoint

# - It launches mspdbsrv with _all_ file handles closed, so that it

# can't block the build from being detected as finished.

# - It explicitly kills mspdbsrv after the build job has finished.

# - It wraps all of this into a context manager, so mspdbsrv gets killed

# even if a Python exception causes a non-local exit.

class UniqueMspdbsrv(object):

def __enter__(self):

os.environ['_MSPDBSRV_ENDPOINT_'] = str(uuid.uuid4())

mspdbsrv_exe = os.path.join(config.get_program_files_x86(),

'Microsoft Visual Studio\\2017\\BuildTools\\VC\\Tools\\MSVC\\14.10.25017\\bin\\HostX86\\x86\\mspdbsrv.exe')

args = [mspdbsrv_exe, '-start', '-shutdowntime', '-1']

print(' '.join(args))

self.proc = subprocess.Popen(args, close_fds=True)

return self

def __exit__(self, type, value, traceback):

self.proc.terminate()

return False

def exec_msbuild(platform):

# Run the make.cmd batch script; it's run using Wine if this is

# not actually a Windows system.

cwd = 'build-' + platform

if _platform.system() == 'Windows':

with UniqueMspdbsrv() as mspdbsrv:

args = ['cmd', '/C', '..\\make.cmd']

print(' '.join(args))

result = subprocess.call(args, cwd=cwd)

sys.exit(result)

else:

args = ['wine', 'cmd', '/K', '..\\make.cmd']

print(' '.join(args))

exit_status = sys.exit(subprocess.call(args, cwd=cwd))

# Clean up any Wine processes that are still hanging around.

# This is important in case the build fails.

args = ['wineserver', '-k', '-w']

subprocess.call(args, cwd=cwd)

sys.exit(exit_status)

def do_compile():

check_target_triple()

platform, subplatform = get_build_platform()

if platform.startswith('win-'):

return exec_msbuild(platform)

else:

# Just defer to the top level Makefile

if platform == 'ios':

if subplatform is None:

error('You must set $BUILD_SUBPLATFORM for iOS builds')

target = 'compile-{}-{}'.format(platform, subplatform)

else:

target = 'compile-' + platform

return exec_make(target)

################################################################

# Archive / extract built binaries

################################################################

def do_bin_archive():

platform, subplatform = get_build_platform()

bindir = platform + '-bin'

shutil.make_archive(bindir, 'bztar', '.', bindir)

################################################################

# Main entry point

################################################################

def buildbot_task(target):

# Check that this branch supports performing the requested buildbot task

if not target in BUILDBOT_TARGETS:

print('Buildbot build step "{}" is not supported'.format(target))

sys.exit(SKIP_EXIT_STATUS)

if target == 'fetch':

return do_fetch()

elif target == 'config':

return do_configure()

elif target == 'compile':

return do_compile()

elif target == 'bin-archive':

return do_bin_archive()

else:

return exec_buildbot_make(target)

if __name__ == '__main__':

if len(sys.argv) < 2:

error("You must specify a buildbot target stage")

sys.exit(buildbot_task(sys.argv[1]))

livecode/buildbot.py at develop · livecodeali/livecode · GitHub

Expand file tree

Breadcrumbs

buildbot.py

Latest commit

History

Breadcrumbs

buildbot.py

File metadata and controls