Source.Python/addons/source-python/Python3/email/charset.py at master · srpg/Source.Python · GitHub

Repository navigation

History

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

187

188

189

190

191

192

193

194

195

196

197

198

199

200

201

202

203

204

205

206

207

208

209

210

211

212

213

214

215

216

217

218

219

220

221

222

223

224

225

226

227

228

229

230

231

232

233

234

235

236

237

238

239

240

241

242

243

244

245

246

247

248

249

250

251

252

253

254

255

256

257

258

259

260

261

262

263

264

265

266

267

268

269

270

271

272

273

274

275

276

277

278

279

280

281

282

283

284

285

286

287

288

289

290

291

292

293

294

295

296

297

298

299

300

301

302

303

304

305

306

307

308

309

310

311

312

313

314

315

316

317

318

319

320

321

322

323

324

325

326

327

328

329

330

331

332

333

334

335

336

337

338

339

340

341

342

343

344

345

346

347

348

349

350

351

352

353

354

355

356

357

358

359

360

361

362

363

364

365

366

367

368

369

370

371

372

373

374

375

376

377

378

379

380

381

382

383

384

385

386

387

388

389

390

391

392

393

394

395

396

397

398

399

400

401

402

403

404

405

406

# Author: Ben Gertzfield, Barry Warsaw

# Contact: email-sig@python.org

__all__ = [

'Charset',

'add_alias',

'add_charset',

'add_codec',

]

from functools import partial

import email.base64mime

import email.quoprimime

from email import errors

from email.encoders import encode_7or8bit

# Flags for types of header encodings

QP = 1 # Quoted-Printable

BASE64 = 2 # Base64

SHORTEST = 3 # the shorter of QP and base64, but only for headers

# In "=?charset?q?hello_world?=", the =?, ?q?, and ?= add up to 7

RFC2047_CHROME_LEN = 7

DEFAULT_CHARSET = 'us-ascii'

UNKNOWN8BIT = 'unknown-8bit'

EMPTYSTRING = ''

# Defaults

CHARSETS = {

# input header enc body enc output conv

'iso-8859-1': (QP, QP, None),

'iso-8859-2': (QP, QP, None),

'iso-8859-3': (QP, QP, None),

'iso-8859-4': (QP, QP, None),

# iso-8859-5 is Cyrillic, and not especially used

# iso-8859-6 is Arabic, also not particularly used

# iso-8859-7 is Greek, QP will not make it readable

# iso-8859-8 is Hebrew, QP will not make it readable

'iso-8859-9': (QP, QP, None),

'iso-8859-10': (QP, QP, None),

# iso-8859-11 is Thai, QP will not make it readable

'iso-8859-13': (QP, QP, None),

'iso-8859-14': (QP, QP, None),

'iso-8859-15': (QP, QP, None),

'iso-8859-16': (QP, QP, None),

'windows-1252':(QP, QP, None),

'viscii': (QP, QP, None),

'us-ascii': (None, None, None),

'big5': (BASE64, BASE64, None),

'gb2312': (BASE64, BASE64, None),

'euc-jp': (BASE64, None, 'iso-2022-jp'),

'shift_jis': (BASE64, None, 'iso-2022-jp'),

'iso-2022-jp': (BASE64, None, None),

'koi8-r': (BASE64, BASE64, None),

'utf-8': (SHORTEST, BASE64, 'utf-8'),

}

# Aliases for other commonly-used names for character sets. Map

# them to the real ones used in email.

ALIASES = {

'latin_1': 'iso-8859-1',

'latin-1': 'iso-8859-1',

'latin_2': 'iso-8859-2',

'latin-2': 'iso-8859-2',

'latin_3': 'iso-8859-3',

'latin-3': 'iso-8859-3',

'latin_4': 'iso-8859-4',

'latin-4': 'iso-8859-4',

'latin_5': 'iso-8859-9',

'latin-5': 'iso-8859-9',

'latin_6': 'iso-8859-10',

'latin-6': 'iso-8859-10',

'latin_7': 'iso-8859-13',

'latin-7': 'iso-8859-13',

'latin_8': 'iso-8859-14',

'latin-8': 'iso-8859-14',

'latin_9': 'iso-8859-15',

'latin-9': 'iso-8859-15',

'latin_10':'iso-8859-16',

'latin-10':'iso-8859-16',

'cp949': 'ks_c_5601-1987',

'euc_jp': 'euc-jp',

'euc_kr': 'euc-kr',

'ascii': 'us-ascii',

}

# Map charsets to their Unicode codec strings.

CODEC_MAP = {

'gb2312': 'eucgb2312_cn',

'big5': 'big5_tw',

# Hack: We don't want *any* conversion for stuff marked us-ascii, as all

# sorts of garbage might be sent to us in the guise of 7-bit us-ascii.

# Let that stuff pass through without conversion to/from Unicode.

'us-ascii': None,

}

# Convenience functions for extending the above mappings

def add_charset(charset, header_enc=None, body_enc=None, output_charset=None):

"""Add character set properties to the global registry.

charset is the input character set, and must be the canonical name of a

character set.

Optional header_enc and body_enc is either Charset.QP for

quoted-printable, Charset.BASE64 for base64 encoding, Charset.SHORTEST for

the shortest of qp or base64 encoding, or None for no encoding. SHORTEST

is only valid for header_enc. It describes how message headers and

message bodies in the input charset are to be encoded. Default is no

encoding.

Optional output_charset is the character set that the output should be

in. Conversions will proceed from input charset, to Unicode, to the

output charset when the method Charset.convert() is called. The default

is to output in the same character set as the input.

Both input_charset and output_charset must have Unicode codec entries in

the module's charset-to-codec mapping; use add_codec(charset, codecname)

to add codecs the module does not know about. See the codecs module's

documentation for more information.

"""

if body_enc == SHORTEST:

raise ValueError('SHORTEST not allowed for body_enc')

CHARSETS[charset] = (header_enc, body_enc, output_charset)

def add_alias(alias, canonical):

"""Add a character set alias.

alias is the alias name, e.g. latin-1

canonical is the character set's canonical name, e.g. iso-8859-1

"""

ALIASES[alias] = canonical

def add_codec(charset, codecname):

"""Add a codec that map characters in the given charset to/from Unicode.

charset is the canonical name of a character set. codecname is the name

of a Python codec, as appropriate for the second argument to the unicode()

built-in, or to the encode() method of a Unicode string.

"""

CODEC_MAP[charset] = codecname

# Convenience function for encoding strings, taking into account

# that they might be unknown-8bit (ie: have surrogate-escaped bytes)

def _encode(string, codec):

if codec == UNKNOWN8BIT:

return string.encode('ascii', 'surrogateescape')

else:

return string.encode(codec)

class Charset:

"""Map character sets to their email properties.

This class provides information about the requirements imposed on email

for a specific character set. It also provides convenience routines for

converting between character sets, given the availability of the

applicable codecs. Given a character set, it will do its best to provide

information on how to use that character set in an email in an

RFC-compliant way.

Certain character sets must be encoded with quoted-printable or base64

when used in email headers or bodies. Certain character sets must be

converted outright, and are not allowed in email. Instances of this

module expose the following information about a character set:

input_charset: The initial character set specified. Common aliases

are converted to their `official' email names (e.g. latin_1

is converted to iso-8859-1). Defaults to 7-bit us-ascii.

header_encoding: If the character set must be encoded before it can be

used in an email header, this attribute will be set to

Charset.QP (for quoted-printable), Charset.BASE64 (for

base64 encoding), or Charset.SHORTEST for the shortest of

QP or BASE64 encoding. Otherwise, it will be None.

body_encoding: Same as header_encoding, but describes the encoding for the

mail message's body, which indeed may be different than the

header encoding. Charset.SHORTEST is not allowed for

body_encoding.

output_charset: Some character sets must be converted before they can be

used in email headers or bodies. If the input_charset is

one of them, this attribute will contain the name of the

charset output will be converted to. Otherwise, it will

be None.

input_codec: The name of the Python codec used to convert the

input_charset to Unicode. If no conversion codec is

necessary, this attribute will be None.

output_codec: The name of the Python codec used to convert Unicode

to the output_charset. If no conversion codec is necessary,

this attribute will have the same value as the input_codec.

"""

def __init__(self, input_charset=DEFAULT_CHARSET):

# RFC 2046, $4.1.2 says charsets are not case sensitive. We coerce to

# unicode because its .lower() is locale insensitive. If the argument

# is already a unicode, we leave it at that, but ensure that the

# charset is ASCII, as the standard (RFC XXX) requires.

try:

if isinstance(input_charset, str):

input_charset.encode('ascii')

else:

input_charset = str(input_charset, 'ascii')

except UnicodeError:

raise errors.CharsetError(input_charset)

input_charset = input_charset.lower()

# Set the input charset after filtering through the aliases

self.input_charset = ALIASES.get(input_charset, input_charset)

# We can try to guess which encoding and conversion to use by the

# charset_map dictionary. Try that first, but let the user override

# it.

henc, benc, conv = CHARSETS.get(self.input_charset,

(SHORTEST, BASE64, None))

if not conv:

conv = self.input_charset

# Set the attributes, allowing the arguments to override the default.

self.header_encoding = henc

self.body_encoding = benc

self.output_charset = ALIASES.get(conv, conv)

# Now set the codecs. If one isn't defined for input_charset,

# guess and try a Unicode codec with the same name as input_codec.

self.input_codec = CODEC_MAP.get(self.input_charset,

self.input_charset)

self.output_codec = CODEC_MAP.get(self.output_charset,

self.output_charset)

def __str__(self):

return self.input_charset.lower()

__repr__ = __str__

def __eq__(self, other):

return str(self) == str(other).lower()

def get_body_encoding(self):

"""Return the content-transfer-encoding used for body encoding.

This is either the string `quoted-printable' or `base64' depending on

the encoding used, or it is a function in which case you should call

the function with a single argument, the Message object being

encoded. The function should then set the Content-Transfer-Encoding

header itself to whatever is appropriate.

Returns "quoted-printable" if self.body_encoding is QP.

Returns "base64" if self.body_encoding is BASE64.

Returns conversion function otherwise.

"""

assert self.body_encoding != SHORTEST

if self.body_encoding == QP:

return 'quoted-printable'

elif self.body_encoding == BASE64:

return 'base64'

else:

return encode_7or8bit

def get_output_charset(self):

"""Return the output character set.

This is self.output_charset if that is not None, otherwise it is

self.input_charset.

"""

return self.output_charset or self.input_charset

def header_encode(self, string):

"""Header-encode a string by converting it first to bytes.

The type of encoding (base64 or quoted-printable) will be based on

this charset's `header_encoding`.

:param string: A unicode string for the header. It must be possible

to encode this string to bytes using the character set's

output codec.

:return: The encoded string, with RFC 2047 chrome.

"""

codec = self.output_codec or 'us-ascii'

header_bytes = _encode(string, codec)

# 7bit/8bit encodings return the string unchanged (modulo conversions)

encoder_module = self._get_encoder(header_bytes)

if encoder_module is None:

return string

return encoder_module.header_encode(header_bytes, codec)

def header_encode_lines(self, string, maxlengths):

"""Header-encode a string by converting it first to bytes.

This is similar to `header_encode()` except that the string is fit

into maximum line lengths as given by the argument.

:param string: A unicode string for the header. It must be possible

to encode this string to bytes using the character set's

output codec.

:param maxlengths: Maximum line length iterator. Each element

returned from this iterator will provide the next maximum line

length. This parameter is used as an argument to built-in next()

and should never be exhausted. The maximum line lengths should

not count the RFC 2047 chrome. These line lengths are only a

hint; the splitter does the best it can.

:return: Lines of encoded strings, each with RFC 2047 chrome.

"""

# See which encoding we should use.

codec = self.output_codec or 'us-ascii'

header_bytes = _encode(string, codec)

encoder_module = self._get_encoder(header_bytes)

encoder = partial(encoder_module.header_encode, charset=codec)

# Calculate the number of characters that the RFC 2047 chrome will

# contribute to each line.

charset = self.get_output_charset()

extra = len(charset) + RFC2047_CHROME_LEN

# Now comes the hard part. We must encode bytes but we can't split on

# bytes because some character sets are variable length and each

# encoded word must stand on its own. So the problem is you have to

# encode to bytes to figure out this word's length, but you must split

# on characters. This causes two problems: first, we don't know how

# many octets a specific substring of unicode characters will get

# encoded to, and second, we don't know how many ASCII characters

# those octets will get encoded to. Unless we try it. Which seems

# inefficient. In the interest of being correct rather than fast (and

# in the hope that there will be few encoded headers in any such

# message), brute force it. :(

lines = []

current_line = []

maxlen = next(maxlengths) - extra

for character in string:

current_line.append(character)

this_line = EMPTYSTRING.join(current_line)

length = encoder_module.header_length(_encode(this_line, charset))

if length > maxlen:

# This last character doesn't fit so pop it off.

current_line.pop()

# Does nothing fit on the first line?

if not lines and not current_line:

lines.append(None)

else:

separator = (' ' if lines else '')

joined_line = EMPTYSTRING.join(current_line)

header_bytes = _encode(joined_line, codec)

lines.append(encoder(header_bytes))

current_line = [character]

maxlen = next(maxlengths) - extra

joined_line = EMPTYSTRING.join(current_line)

header_bytes = _encode(joined_line, codec)

lines.append(encoder(header_bytes))

return lines

def _get_encoder(self, header_bytes):

if self.header_encoding == BASE64:

return email.base64mime

elif self.header_encoding == QP:

return email.quoprimime

elif self.header_encoding == SHORTEST:

len64 = email.base64mime.header_length(header_bytes)

lenqp = email.quoprimime.header_length(header_bytes)

if len64 < lenqp:

return email.base64mime

else:

return email.quoprimime

else:

return None

def body_encode(self, string):

"""Body-encode a string by converting it first to bytes.

The type of encoding (base64 or quoted-printable) will be based on

self.body_encoding. If body_encoding is None, we assume the

output charset is a 7bit encoding, so re-encoding the decoded

string using the ascii codec produces the correct string version

of the content.

"""

if not string:

return string

if self.body_encoding is BASE64:

if isinstance(string, str):

string = string.encode(self.output_charset)

return email.base64mime.body_encode(string)

elif self.body_encoding is QP:

# quopromime.body_encode takes a string, but operates on it as if

# it were a list of byte codes. For a (minimal) history on why

# this is so, see changeset 0cf700464177. To correctly encode a

# character set, then, we must turn it into pseudo bytes via the

# latin1 charset, which will encode any byte as a single code point

# between 0 and 255, which is what body_encode is expecting.

if isinstance(string, str):

string = string.encode(self.output_charset)

string = string.decode('latin1')

return email.quoprimime.body_encode(string)

else:

if isinstance(string, str):

string = string.encode(self.output_charset).decode('ascii')

return string

Source.Python/addons/source-python/Python3/email/charset.py at master · srpg/Source.Python · GitHub

Repository navigation

Expand file tree

Breadcrumbs

charset.py

Latest commit

History

Breadcrumbs

charset.py

File metadata and controls