cpython/Tools/ssl/make_ssl_data.py at main · python/cpython · GitHub

Repository navigation

History

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

187

188

189

190

191

192

193

194

195

196

197

198

199

200

201

202

203

204

205

206

207

208

209

210

211

212

213

214

215

216

217

218

219

220

221

222

223

224

225

226

227

228

229

230

231

232

233

234

235

236

237

238

239

240

241

242

243

#!/usr/bin/env python3

"""

This script should be called *manually* when we want to upgrade SSLError

`library` and `reason` mnemonics to a more recent OpenSSL version. Note

that error codes are version specific.

It takes two arguments:

- the path to the OpenSSL folder with the correct git checkout (see below)

- the path to the header file to be generated, usually

Modules/_ssl_data_<MAJOR><MINOR><PATCH>.h

The OpenSSL git checkout should be at a specific tag, using commands like:

git tag --list 'openssl-*'

git switch --detach openssl-3.4.1

After generating the definitions, compare the result with the newest

pre-existing file. You can use a command like:

git diff --no-index Modules/_ssl_data_340.h Modules/_ssl_data_341.h

- If the new version *only* adds new definitions, remove the pre-existing file

and adjust the #include in _ssl.c to point to the new version.

- If the new version removes or renumbers some definitions, keep both files and

add a new #include in _ssl.c.

By convention, the latest OpenSSL mnemonics are gathered in the following file:

Modules/_ssl_data_<MAJOR><MINOR>.h

If those mnemonics are renumbered or removed in a subsequent OpenSSL version,

the file is renamed to "Modules/_ssl_data_<MAJOR><MINOR><PATCH>.h" and the

latest mnemonics are stored in the patchless file (see below for an example).

A newly supported OpenSSL version should also be added to:

- Tools/ssl/multissltests.py

- .github/workflows/build.yml

Example: new mnemonics are added

--------------------------------

Assume that "Modules/_ssl_data_32x.h" contains the latest mnemonics for

CPython and was generated from OpenSSL 3.2.1. If only new mnemonics are

added in OpenSSL 3.2.2, the following commands should be executed:

# in the OpenSSL git directory

git switch --detach openssl-3.2.2

# in the CPython git directory

python make_ssl_data.py PATH_TO_OPENSSL_GIT_CLONE Modules/_ssl_data_322.h

mv Modules/_ssl_data_322.h Modules/_ssl_data_32.h

Example: mnemonics are renamed/removed

--------------------------------------

Assume that the existing file is Modules/_ssl_data_34x.h and is based

on OpenSSL 3.4.0. Since some mnemonics were renamed in OpenSSL 3.4.1,

the following commands should be executed:

# in the OpenSSL git directory

git switch --detach openssl-3.4.1

# in the CPython git directory

mv Modules/_ssl_data_34.h Modules/_ssl_data_340.h

python make_ssl_data.py PATH_TO_OPENSSL_GIT_CLONE Modules/_ssl_data_341.h

mv Modules/_ssl_data_341.h Modules/_ssl_data_34.h

"""

import argparse

import datetime

import logging

import operator

import os

import re

import subprocess

logger = logging.getLogger(__name__)

parser = argparse.ArgumentParser(

formatter_class=argparse.RawTextHelpFormatter,

description="Generate SSL data headers from OpenSSL sources"

)

parser.add_argument("srcdir", help="OpenSSL source directory")

parser.add_argument(

"output", nargs="?", default=None,

help="output file (default: standard output)",

)

def error(format_string, *format_args, **kwargs):

# do not use parser.error() to avoid printing short help

logger.error(format_string, *format_args, **kwargs)

raise SystemExit(1)

def _file_search(fname, pat):

with open(fname, encoding="utf-8") as f:

for line in f:

match = pat.search(line)

if match is not None:

yield match

def parse_err_h(args):

"""Parse error codes from include/openssl/err.h.in.

Detected lines match (up to spaces) "#define ERR_LIB_<LIBNAME> <ERRCODE>",

e.g., "# define ERR_LIB_NONE 1".

"""

pat = re.compile(r"#\s*define\W+(ERR_LIB_(\w+))\s+(\d+)")

lib2errnum = {}

for match in _file_search(args.err_h, pat):

macroname, libname, num = match.groups()

if macroname in ['ERR_LIB_OFFSET', 'ERR_LIB_MASK']:

# ignore: "# define ERR_LIB_OFFSET 23L"

# ignore: "# define ERR_LIB_MASK 0xFF"

continue

actual = int(num)

expect = lib2errnum.setdefault(libname, actual)

if actual != expect:

logger.warning("OpenSSL inconsistency for ERR_LIB_%s (%d != %d)",

libname, actual, expect)

return lib2errnum

def parse_openssl_error_text(args):

"""Parse error reasons from crypto/err/openssl.txt.

Detected lines match "<LIBNAME>_R_<ERRNAME>:<ERRCODE>:<MESSAGE>",

e.g., "ASN1_R_ADDING_OBJECT:171:adding object". The <MESSAGE> part

is not stored as it will be recovered at runtime when needed.

"""

# ignore backslash line continuation (placed before <MESSAGE> if present)

pat = re.compile(r"^((\w+?)_R_(\w+)):(\d+):")

seen = {}

for match in _file_search(args.errtxt, pat):

reason, libname, errname, num = match.groups()

if "_F_" in reason: # ignore function codes

# FEAT(picnixz): in the future, we may want to also check

# the consistency of the OpenSSL files with an external tool.

# See https://github.com/python/cpython/issues/132745.

continue

yield reason, libname, errname, int(num)

def parse_extra_reasons(args):

"""Parse extra reasons from crypto/err/openssl.ec.

Detected lines are matched against "R <LIBNAME>_R_<ERRNAME> <ERRCODE>",

e.g., "R SSL_R_SSLV3_ALERT_UNEXPECTED_MESSAGE 1010".

"""

pat = re.compile(r"^R\s+((\w+)_R_(\w+))\s+(\d+)")

for match in _file_search(args.errcodes, pat):

reason, libname, errname, num = match.groups()

yield reason, libname, errname, int(num)

def gen_library_codes(args):

"""Generate table short libname to numeric code."""

yield "/* generated from args.lib2errnum */"

yield "static struct py_ssl_library_code library_codes[] = {"

for libname in sorted(args.lib2errnum):

yield f"#ifdef ERR_LIB_{libname}"

yield f' {{"{libname}", ERR_LIB_{libname}}},'

yield "#endif"

yield " {NULL, 0} /* sentinel */"

yield "};"

def gen_error_codes(args):

"""Generate error code table for error reasons."""

yield "/* generated from args.reasons */"

yield "static struct py_ssl_error_code error_codes[] = {"

for reason, libname, errname, num in args.reasons:

yield f" #ifdef {reason}"

yield f' {{"{errname}", ERR_LIB_{libname}, {reason}}},'

yield " #else"

yield f' {{"{errname}", {args.lib2errnum[libname]}, {num}}},'

yield " #endif"

yield " {NULL, 0, 0} /* sentinel */"

yield "};"

def get_openssl_git_commit(args):

git_describe = subprocess.run(

['git', 'describe', '--long', '--dirty'],

cwd=args.srcdir,

capture_output=True,

encoding='utf-8',

check=True,

)

return git_describe.stdout.strip()

def main(args=None):

args = parser.parse_args(args)

if not os.path.isdir(args.srcdir):

error(f"OpenSSL directory not found: {args.srcdir}")

args.err_h = os.path.join(args.srcdir, "include", "openssl", "err.h")

if not os.path.isfile(args.err_h):

# Fall back to infile for OpenSSL 3.0.0 and later.

args.err_h += ".in"

args.errcodes = os.path.join(args.srcdir, "crypto", "err", "openssl.ec")

if not os.path.isfile(args.errcodes):

error(f"file {args.errcodes} not found in {args.srcdir}")

args.errtxt = os.path.join(args.srcdir, "crypto", "err", "openssl.txt")

if not os.path.isfile(args.errtxt):

error(f"file {args.errtxt} not found in {args.srcdir}")

# [("ERR_LIB_X509", "X509", 11), ...]

args.lib2errnum = parse_err_h(args)

# [('X509_R_AKID_MISMATCH', 'X509', 'AKID_MISMATCH', 110), ...]

reasons = []

reasons.extend(parse_openssl_error_text(args))

reasons.extend(parse_extra_reasons(args))

# sort by macro name and numeric error code

args.reasons = sorted(reasons, key=operator.itemgetter(0, 3))

commit = get_openssl_git_commit(args)

lines = [

"/* File generated by Tools/ssl/make_ssl_data.py */",

f"/* Generated on {datetime.datetime.now(datetime.UTC).isoformat()} */",

f"/* Generated from Git commit {commit} */",

"",

]

lines.extend(gen_library_codes(args))

lines.append("")

lines.extend(gen_error_codes(args))

if args.output is None:

for line in lines:

print(line)

else:

with open(args.output, 'w') as output:

for line in lines:

print(line, file=output)

if __name__ == "__main__":

main()

cpython/Tools/ssl/make_ssl_data.py at main · python/cpython · GitHub

Repository navigation

Expand file tree

Breadcrumbs

make_ssl_data.py

Latest commit

History

Breadcrumbs

make_ssl_data.py

File metadata and controls