jsoncpp/devtools/batchbuild.py at master · cloria/jsoncpp · GitHub

History

278 lines (256 loc) · 11.2 KB

Raw

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

187

188

189

190

191

192

193

194

195

196

197

198

199

200

201

202

203

204

205

206

207

208

209

210

211

212

213

214

215

216

217

218

219

220

221

222

223

224

225

226

227

228

229

230

231

232

233

234

235

236

237

238

239

240

241

242

243

244

245

246

247

248

249

250

251

252

253

254

255

256

257

258

259

260

261

262

263

264

265

266

267

268

269

270

271

272

273

274

275

276

277

from __future__ import print_function

import collections

import itertools

import json

import os

import os.path

import re

import shutil

import string

import subprocess

import sys

import cgi

class BuildDesc:

def __init__(self, prepend_envs=None, variables=None, build_type=None, generator=None):

self.prepend_envs = prepend_envs or [] # [ { "var": "value" } ]

self.variables = variables or []

self.build_type = build_type

self.generator = generator

def merged_with(self, build_desc):

"""Returns a new BuildDesc by merging field content.

Prefer build_desc fields to self fields for single valued field.

"""

return BuildDesc(self.prepend_envs + build_desc.prepend_envs,

self.variables + build_desc.variables,

build_desc.build_type or self.build_type,

build_desc.generator or self.generator)

def env(self):

environ = os.environ.copy()

for values_by_name in self.prepend_envs:

for var, value in list(values_by_name.items()):

var = var.upper()

if type(value) is unicode:

value = value.encode(sys.getdefaultencoding())

if var in environ:

environ[var] = value + os.pathsep + environ[var]

else:

environ[var] = value

return environ

def cmake_args(self):

args = ["-D%s" % var for var in self.variables]

# skip build type for Visual Studio solution as it cause warning

if self.build_type and 'Visual' not in self.generator:

args.append("-DCMAKE_BUILD_TYPE=%s" % self.build_type)

if self.generator:

args.extend(['-G', self.generator])

return args

def __repr__(self):

return "BuildDesc(%s, build_type=%s)" % (" ".join(self.cmake_args()), self.build_type)

class BuildData:

def __init__(self, desc, work_dir, source_dir):

self.desc = desc

self.work_dir = work_dir

self.source_dir = source_dir

self.cmake_log_path = os.path.join(work_dir, 'batchbuild_cmake.log')

self.build_log_path = os.path.join(work_dir, 'batchbuild_build.log')

self.cmake_succeeded = False

self.build_succeeded = False

def execute_build(self):

print('Build %s' % self.desc)

self._make_new_work_dir()

self.cmake_succeeded = self._generate_makefiles()

if self.cmake_succeeded:

self.build_succeeded = self._build_using_makefiles()

return self.build_succeeded

def _generate_makefiles(self):

print(' Generating makefiles: ', end=' ')

cmd = ['cmake'] + self.desc.cmake_args() + [os.path.abspath(self.source_dir)]

succeeded = self._execute_build_subprocess(cmd, self.desc.env(), self.cmake_log_path)

print('done' if succeeded else 'FAILED')

return succeeded

def _build_using_makefiles(self):

print(' Building:', end=' ')

cmd = ['cmake', '--build', self.work_dir]

if self.desc.build_type:

cmd += ['--config', self.desc.build_type]

succeeded = self._execute_build_subprocess(cmd, self.desc.env(), self.build_log_path)

print('done' if succeeded else 'FAILED')

return succeeded

def _execute_build_subprocess(self, cmd, env, log_path):

process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=self.work_dir,

env=env)

stdout, _ = process.communicate()

succeeded = (process.returncode == 0)

with open(log_path, 'wb') as flog:

log = ' '.join(cmd) + '\n' + stdout + '\nExit code: %r\n' % process.returncode

flog.write(fix_eol(log))

return succeeded

def _make_new_work_dir(self):

if os.path.isdir(self.work_dir):

print(' Removing work directory', self.work_dir)

shutil.rmtree(self.work_dir, ignore_errors=True)

if not os.path.isdir(self.work_dir):

os.makedirs(self.work_dir)

def fix_eol(stdout):

"""Fixes wrong EOL produced by cmake --build on Windows (\r\r\n instead of \r\n).

"""

return re.sub('\r*\n', os.linesep, stdout)

def load_build_variants_from_config(config_path):

with open(config_path, 'rb') as fconfig:

data = json.load(fconfig)

variants = data[ 'cmake_variants' ]

build_descs_by_axis = collections.defaultdict(list)

for axis in variants:

axis_name = axis["name"]

build_descs = []

if "generators" in axis:

for generator_data in axis["generators"]:

for generator in generator_data["generator"]:

build_desc = BuildDesc(generator=generator,

prepend_envs=generator_data.get("env_prepend"))

build_descs.append(build_desc)

elif "variables" in axis:

for variables in axis["variables"]:

build_desc = BuildDesc(variables=variables)

build_descs.append(build_desc)

elif "build_types" in axis:

for build_type in axis["build_types"]:

build_desc = BuildDesc(build_type=build_type)

build_descs.append(build_desc)

build_descs_by_axis[axis_name].extend(build_descs)

return build_descs_by_axis

def generate_build_variants(build_descs_by_axis):

"""Returns a list of BuildDesc generated for the partial BuildDesc for each axis."""

axis_names = list(build_descs_by_axis.keys())

build_descs = []

for axis_name, axis_build_descs in list(build_descs_by_axis.items()):

if len(build_descs):

# for each existing build_desc and each axis build desc, create a new build_desc

new_build_descs = []

for prototype_build_desc, axis_build_desc in itertools.product(build_descs, axis_build_descs):

new_build_descs.append(prototype_build_desc.merged_with(axis_build_desc))

build_descs = new_build_descs

else:

build_descs = axis_build_descs

return build_descs

HTML_TEMPLATE = string.Template('''<html>

<head>

<title>$title</title>

td.failed {background-color:#f08080;}

td.ok {background-color:#c0eec0;}

</style>

</head>

<body>

<thead>

<tr>

<th>Variables</th>

$th_vars

</tr>

<tr>

<th>Build type</th>

$th_build_types

</tr>

</thead>

<tbody>

$tr_builds

</tbody>

</table>

</body></html>''')

def generate_html_report(html_report_path, builds):

report_dir = os.path.dirname(html_report_path)

# Vertical axis: generator

# Horizontal: variables, then build_type

builds_by_generator = collections.defaultdict(list)

variables = set()

build_types_by_variable = collections.defaultdict(set)

build_by_pos_key = {} # { (generator, var_key, build_type): build }

for build in builds:

builds_by_generator[build.desc.generator].append(build)

var_key = tuple(sorted(build.desc.variables))

variables.add(var_key)

build_types_by_variable[var_key].add(build.desc.build_type)

pos_key = (build.desc.generator, var_key, build.desc.build_type)

build_by_pos_key[pos_key] = build

variables = sorted(variables)

th_vars = []

th_build_types = []

for variable in variables:

build_types = sorted(build_types_by_variable[variable])

nb_build_type = len(build_types_by_variable[variable])

th_vars.append('<th colspan="%d">%s</th>' % (nb_build_type, cgi.escape(' '.join(variable))))

for build_type in build_types:

th_build_types.append('<th>%s</th>' % cgi.escape(build_type))

tr_builds = []

for generator in sorted(builds_by_generator):

tds = [ '<td>%s</td>\n' % cgi.escape(generator) ]

for variable in variables:

build_types = sorted(build_types_by_variable[variable])

for build_type in build_types:

pos_key = (generator, variable, build_type)

build = build_by_pos_key.get(pos_key)

if build:

cmake_status = 'ok' if build.cmake_succeeded else 'FAILED'

build_status = 'ok' if build.build_succeeded else 'FAILED'

cmake_log_url = os.path.relpath(build.cmake_log_path, report_dir)

build_log_url = os.path.relpath(build.build_log_path, report_dir)

td = '<td class="%s"><a href="%s" class="%s">CMake: %s</a>' % ( build_status.lower(), cmake_log_url, cmake_status.lower(), cmake_status)

if build.cmake_succeeded:

td += '<br><a href="%s" class="%s">Build: %s</a>' % ( build_log_url, build_status.lower(), build_status)

td += '</td>'

else:

td = '<td></td>'

tds.append(td)

tr_builds.append('<tr>%s</tr>' % '\n'.join(tds))

html = HTML_TEMPLATE.substitute( title='Batch build report',

th_vars=' '.join(th_vars),

th_build_types=' '.join(th_build_types),

tr_builds='\n'.join(tr_builds))

with open(html_report_path, 'wt') as fhtml:

fhtml.write(html)

print('HTML report generated in:', html_report_path)

def main():

usage = r"""%prog WORK_DIR SOURCE_DIR CONFIG_JSON_PATH [CONFIG2_JSON_PATH...]

Build a given CMake based project located in SOURCE_DIR with multiple generators/options.dry_run

as described in CONFIG_JSON_PATH building in WORK_DIR.

Example of call:

python devtools\batchbuild.py e:\buildbots\jsoncpp\build . devtools\agent_vmw7.json

"""

from optparse import OptionParser

parser = OptionParser(usage=usage)

parser.allow_interspersed_args = True

# parser.add_option('-v', '--verbose', dest="verbose", action='store_true',

# help="""Be verbose.""")

parser.enable_interspersed_args()

options, args = parser.parse_args()

if len(args) < 3:

parser.error("Missing one of WORK_DIR SOURCE_DIR CONFIG_JSON_PATH.")

work_dir = args[0]

source_dir = args[1].rstrip('/\\')

config_paths = args[2:]

for config_path in config_paths:

if not os.path.isfile(config_path):

parser.error("Can not read: %r" % config_path)

# generate build variants

build_descs = []

for config_path in config_paths:

build_descs_by_axis = load_build_variants_from_config(config_path)

build_descs.extend(generate_build_variants(build_descs_by_axis))

print('Build variants (%d):' % len(build_descs))

# assign build directory for each variant

if not os.path.isdir(work_dir):

os.makedirs(work_dir)

builds = []

with open(os.path.join(work_dir, 'matrix-dir-map.txt'), 'wt') as fmatrixmap:

for index, build_desc in enumerate(build_descs):

build_desc_work_dir = os.path.join(work_dir, '%03d' % (index+1))

builds.append(BuildData(build_desc, build_desc_work_dir, source_dir))

fmatrixmap.write('%s: %s\n' % (build_desc_work_dir, build_desc))

for build in builds:

build.execute_build()

html_report_path = os.path.join(work_dir, 'batchbuild-report.html')

generate_html_report(html_report_path, builds)

print('Done')

if __name__ == '__main__':

main()

jsoncpp/devtools/batchbuild.py at master · cloria/jsoncpp · GitHub

Expand file tree

Breadcrumbs

batchbuild.py

Latest commit

History

Breadcrumbs

batchbuild.py

File metadata and controls