modularity-utils/generate-modulemd.py at master · fedora-java/modularity-utils · GitHub

History

executable file

378 lines (323 loc) · 13.4 KB

Raw

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

187

188

189

190

191

192

193

194

195

196

197

198

199

200

201

202

203

204

205

206

207

208

209

210

211

212

213

214

215

216

217

218

219

220

221

222

223

224

225

226

227

228

229

230

231

232

233

234

235

236

237

238

239

240

241

242

243

244

245

246

247

248

249

250

251

252

253

254

255

256

257

258

259

260

261

262

263

264

265

266

267

268

269

270

271

272

273

274

275

276

277

278

279

280

281

282

283

284

285

286

287

288

289

290

291

292

293

294

295

296

297

298

299

300

301

302

303

304

305

306

307

308

309

310

311

312

313

314

315

316

317

318

319

320

321

322

323

324

325

326

327

328

329

330

331

332

333

334

335

336

337

338

339

340

341

342

343

344

345

346

347

348

349

350

351

352

353

354

355

356

357

358

359

360

361

362

363

364

365

366

367

368

369

370

371

372

373

374

375

376

377

378

#!/usr/bin/python3

# Permission is hereby granted, free of charge, to any person obtaining

# a copy of this software and associated documentation files (the

# "Software"), to deal in the Software without restriction, including

# without limitation the rights to use, copy, modify, merge, publish,

# distribute, sublicense, and/or sell copies of the Software, and to

# permit persons to whom the Software is furnished to do so, subject to

# the following conditions:

# The above copyright notice and this permission notice shall be included

# in all copies or substantial portions of the Software.

# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,

# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF

# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.

# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY

# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,

# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE

# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

# Author: Mikolaj Izdebski <mizdebsk@redhat.com>

# Author: Michael Simacek <msimacek@redhat.com>

import os

import re

import logging

import koji

import hawkey

import jinja2

from textwrap import dedent, fill

from koschei import config

try:

from koschei.backend.koji_util import KojiRepoDescriptor

except ImportError:

from koschei.backend.repo_util import KojiRepoDescriptor

from koschei.backend import koji_util, depsolve, repo_cache

log = logging.getLogger('')

# Use git repo name and branch for module/stream.

module = os.path.basename(os.getcwd())

with open('.git/HEAD') as f:

stream = re.match(r'ref: refs/heads/(.*)', f.readlines()[0]).group(1)

config.load_config(['koschei.cfg'], ignore_env=True)

full_refs = config.get_config('full_refs')

default_ref = config.get_config('default_ref')

include_build_deps = config.get_config('include_build_deps')

api = config.get_config('api')

profiles = config.get_config('profiles')

includes = config.get_config('includes')

excludes = config.get_config('excludes')

frozen_refs = config.get_config('frozen_refs')

stream_override = config.get_config('stream_override')

macros = config.get_config('macros', None)

ks = koji_util.KojiSession()

tag_name = config.get_koji_config('primary', 'tag_name')

repo_id = ks.getRepo(tag_name, state=koji.REPO_READY)['id']

repo_descriptor = KojiRepoDescriptor('primary', tag_name, repo_id)

# Parse RPM name - split it into NVRA dict

def parse_nvra(str):

m = re.match(r'^(.+)-([^-]+)-([^-]+)\.([^-.]+).rpm$', str)

return {'name': m.group(1), 'version': m.group(2),

'release': m.group(3), 'arch': m.group(4)}

# Get RPM name

def name(rpm):

return parse_nvra(rpm)['name']

# Try to heuristically guess whether given hawkey package is Java package and

# should be part of maven module, or not. Explicit includes and excludes always

# have preference over heuristic.

def is_maven_pkg(pkg):

if name(pkg.sourcerpm) in includes:

return True

if name(pkg.sourcerpm) in excludes:

return False

for file in pkg.files:

if file.startswith('/usr/share/maven-metadata/'):

return True

if file.startswith('/usr/share/java/'):

return True

if file.startswith('/usr/lib/java/'):

return True

return False

# Simulate installation of given dependencies.

def resolve_deps(sack, deps):

log.info('Resolving deps...')

resolved, problems, installs = depsolve.run_goal(sack, deps, [])

if not resolved:

installs = set()

for pkg in deps:

resolved, problems, pkg_installs = \

depsolve.run_goal(sack, [pkg], [])

if resolved:

installs.update(pkg_installs)

else:

log.warning('Dependency problems for {}:\n{}'

.format(pkg, '\n'.join(problems)))

java = {pkg.sourcerpm for pkg in installs if is_maven_pkg(pkg)}

return java, set(installs)

# Input: BuildRequires string

# Output: matched hawkey packages

def resolve_builddep(sack, br):

return depsolve._get_builddep_selector(sack, br).matches()

# Input: hawkey reldep

# Output: matched hawkey packages

def resolve_reldep(sack, reldep):

return hawkey.Query(sack).filter(provides=reldep)

# Get build-requires of given SRPMs from Koji.

def get_build_requires(srpms):

log.info('Getting build-requires from Koji...')

return koji_util.get_rpm_requires(ks, [parse_nvra(srpm) for srpm in srpms])

# Get names of binary RPMs corresponding to given list of source RPMs.

def get_binary_rpms(srpms):

rpm_names = set()

builds = koji_util.itercall(ks, list(srpms), lambda ks, srpm: ks.getBuild(parse_nvra(srpm)))

rpms_gen = koji_util.itercall(ks, list(builds), lambda ks, build: ks.listRPMs(build['id'], arches=('noarch', 'x86_64')))

for rpms in rpms_gen:

rpm_names.update([rpm['name'] for rpm in rpms if not rpm['name'].endswith('-debuginfo') and not rpm['name'].endswith('-debugsource')])

return rpm_names

def topo_sort(V, E):

W = {}

i = 1

while V:

U = set()

for v in V:

for u in V:

if v in E.get(u, ()):

break

else:

W[v] = i

U.add(v)

if not U:

log.error('There are dependency cycles, topological sort is not possible')

return None

V = V - U

i += 1

return W

# For each SRPM, figure out from which git commit it was built.

def resolve_refs(srpms):

def get_ref(build):

scm_url = build['extra']['source']['original_url']

match = re.search(r'#([0-9a-f]{7})[0-9a-f]*$', scm_url)

assert match

return match.group(1)

builds = koji_util.itercall(ks, list(srpms), lambda ks, srpm: ks.getBuild(parse_nvra(srpm)))

return {srpm: get_ref(build) for srpm, build in zip(srpms, builds)}

# Do the main work. Recursively resolve requires and build-requires, starting

# from initial list of seed packages.

def work(sack):

filtered = set(config.get_config('filter'))

srpms_done = set()

br_map = {}

srpms_todo, pkgs = resolve_deps(sack, api)

def add(map, key, val):

_set = map.get(key, set())

map[key] = _set

_set.add(val)

while srpms_todo:

log.info('Round: {} packages'.format(len(srpms_todo)))

srpms_todo = {srpm for srpm in srpms_todo

if name(srpm) not in excludes}

srpms_done |= srpms_todo

combined_br = set()

if include_build_deps:

for srpm, build_requires in zip(srpms_todo, get_build_requires(srpms_todo)):

combined_br |= set(build_requires)

for br in build_requires:

add(br_map, br, srpm)

if config.get_config('closure'):

combined_br.update(get_binary_rpms(srpms_todo) - filtered)

java, all = resolve_deps(sack, combined_br)

srpms_todo |= java

srpms_todo -= srpms_done

pkgs |= all

def our(pkgs):

return (pkg for pkg in pkgs if pkg.sourcerpm in srpms_done)

api_srpms = {pkg.sourcerpm for pkg in pkgs if pkg.name in api}

if filtered:

for pkg in filtered - get_binary_rpms(srpms_done):

log.warning('Filtered RPM {} was not found'.format(pkg))

if config.get_config('filter_unused'):

filtered.update(get_binary_rpms(srpms_done) - {pkg.name for pkg in pkgs})

def pretty_rpm_name(rpm):

if rpm.name == name(rpm.sourcerpm):

return rpm.name

else:

return '{} (subpackage of {})'.format(rpm.name, name(rpm.sourcerpm))

build_deps = {}

for br, srpms in br_map.items():

for dep in our(resolve_builddep(sack, br)):

for srpm in srpms:

if dep.name in filtered:

log.warning('Build dependency broken by filter: component {} BuildRequires "{}", which pulls in filtered RPM {}.'

.format(name(srpm), br, pretty_rpm_name(dep)))

add(build_deps, dep.sourcerpm, srpm)

runtime_deps = {}

for pkg in our(pkgs):

for reldep in pkg.requires:

for dep in resolve_reldep(sack, reldep):

if dep.name in filtered and pkg.name not in filtered:

log.warning('Runtime dependency broken by filter: package {} Requires "{}", which pulls in filtered RPM {}.'

.format(pretty_rpm_name(pkg), reldep, pretty_rpm_name(dep)))

if dep.sourcerpm != pkg.sourcerpm:

add(runtime_deps, dep.sourcerpm, pkg.sourcerpm)

if config.get_config('topo_sort', False):

buildorder = topo_sort(srpms_done, {**runtime_deps, **build_deps})

log.info('Resolving git refs...')

if full_refs:

refs = resolve_refs(srpms_done)

else:

refs = resolve_refs([srpm for srpm in srpms_done if name(srpm) in frozen_refs])

def get_stream(dep):

return stream_override.get(dep, {}).get(stream, stream)

yaml = dedent("""\

{% macro format_rationale(type, srpms, cont_indent) %}

{% filter wrap(width=(80 - cont_indent)) | indent(cont_indent) %}

{{ type }} of {{ sorted(srpms) | map('name') | join(', ') }}.

{% endfilter %}

{%- endmacro %}

---

document: modulemd

version: 2

data:

summary: {{ config.get_config('summary') }}

description: >-

license:

module:

- MIT

dependencies:

- buildrequires:

{% for dep in config.get_config('buildrequires') %}

{{ dep }}: {{ get_stream(dep) }}

{% endfor %}

requires:

{% for dep in config.get_config('requires') %}

{{ dep }}: {{ get_stream(dep) }}

{% endfor %}

profiles:

{% for profile, content in profiles.items() %}

{{ profile }}:

rpms:

{% for package in sorted(content) %}

- {{ package }}

{% endfor %}

api:

rpms:

{% for package in sorted(api) %}

- {{ package }}

{% endfor %}

{% if filtered %}

filter:

rpms:

{% for rpm in sorted(filtered) %}

- {{ rpm }}

{% endfor %}

{% endif %}

{% if macros %}

buildopts:

rpms:

macros: |

{% for macro, value in macros.items() %}

%{{ macro }} {{ value }}

{% endfor %}

{% endif %}

components:

rpms:

{% for srpm in sorted(srpms_done) %}

{{ srpm | name }}:

{% if buildorder is defined %}

buildorder: {{ buildorder[srpm] }}0

{% endif %}

{% set ref = refs.get(srpm, default_ref) %}

{% if ref %}

ref: {{ ref }}

{% endif %}

rationale: >

{% if srpm in api_srpms %}

Module API.

{% endif %}

{% if srpm in runtime_deps %}

{{ format_rationale('Runtime dependency', runtime_deps[srpm], 25) }}

{% endif %}

{% if srpm in build_deps %}

{% endif %}

{% endfor %}

""")

env = jinja2.Environment(trim_blocks=True, lstrip_blocks=True)

env.globals.update(vars(__builtins__))

env.globals.update(globals())

env.filters['name'] = name

def wrap(text, width):

return fill(text, width, break_on_hyphens=False, break_long_words=False)

env.filters['wrap'] = wrap

content = env.from_string(yaml).render(**locals())

with open('{}.yaml'.format(module), 'w') as f:

f.write(content)

# Get set of components contained withing given module/stream in Koji.

def get_module_components(module, stream=None, version=None):

log.info('Querying Koji for content of module {}, stream {}...'.format(module, stream))

module_builds = ks.listBuilds(ks.getPackage(module)['id'],

state=koji.BUILD_STATES['COMPLETE'],

type='module',

queryOpts={'order': '-creation_event_id'})

for build in module_builds:

if ((not stream or build['version'] == stream) and

(not version or build['release'] == version)):

tag = build['extra']['typeinfo']['module']['content_koji_tag']

(_, module_srpms) = ks.listTaggedRPMS(tag, arch='src')

return set(srpm['name'] for srpm in module_srpms)

log.warning('Module module {}, stream {} was not found in Koji'.format(module, stream))

return set()

def main():

if not os.path.exists('/tmp/maven-modulemd-gen/repodata'):

os.makedirs('/tmp/maven-modulemd-gen/repodata')

for module_coords in config.get_config('module_excludes', []):

excludes.extend(get_module_components(*module_coords))

log.info('Loading sack...')

with repo_cache.RepoCache().get_sack(repo_descriptor) as sack:

work(sack)

if __name__ == '__main__':

main()

modularity-utils/generate-modulemd.py at master · fedora-java/modularity-utils · GitHub

Uh oh!

Expand file tree

Breadcrumbs

generate-modulemd.py

Latest commit

History

Breadcrumbs

generate-modulemd.py

File metadata and controls