| FazBrowse GitHub Viewer | Trending | | Home |
| Tools: [Download Repo ZIP] [Original HTTPS Page] |
1 parent 9668d26 commit d96ffc1
7 files changed
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -403,11 +403,26 @@ added matters. To illustrate:: | |||
| 403 | 403 | .. attribute:: utf8 | |
| 404 | 404 | ||
| 405 | 405 | If ``False``, follow :rfc:`5322`, supporting non-ASCII characters in | |
| 406 | - headers by encoding them as "encoded words". If ``True``, follow | ||
| 407 | - :rfc:`6532` and use ``utf-8`` encoding for headers. Messages | ||
| 406 | + headers by encoding them as :rfc:`2047` "encoded words". If ``True``, | ||
| 407 | + follow :rfc:`6532` and use ``utf-8`` encoding for headers. Messages | ||
| 408 | 408 | formatted in this way may be passed to SMTP servers that support | |
| 409 | 409 | the ``SMTPUTF8`` extension (:rfc:`6531`). | |
| 410 | 410 | ||
| 411 | + When ``False``, the generator will raise | ||
| 412 | + :exc:`~email.errors.HeaderWriteError` if any header includes non-ASCII | ||
| 413 | + characters in a context where :rfc:`2047` does not permit encoded words. | ||
| 414 | + This particularly applies to mailboxes ("addr-spec") with non-ASCII | ||
| 415 | + characters, which can be created via | ||
| 416 | + :class:`~email.headerregistry.Address`. To use a mailbox with a non-ASCII | ||
| 417 | + domain name with ``utf8=False``, first encode the domain using the | ||
| 418 | + third-party :pypi:`idna` or :pypi:`uts46` module or with | ||
| 419 | + :mod:`encodings.idna`. It is not possible to use a non-ASCII username | ||
| 420 | + ("local-part") in a mailbox when ``utf8=False``. | ||
| 421 | + | ||
| 422 | + .. versionchanged:: 3.15 | ||
| 423 | + Can trigger the raising of :exc:`~email.errors.HeaderWriteError`. | ||
| 424 | + (Earlier versions incorrectly applied :rfc:`2047` in certain contexts, | ||
| 425 | + mostly notably in addr-specs.) | ||
| 411 | 426 | ||
| 412 | 427 | .. attribute:: refold_source | |
| 413 | 428 | ||
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -914,6 +914,16 @@ faulthandler | |||
| 914 | 914 | (Contributed by Eric Froemling in :gh:`149085`.) | |
| 915 | 915 | ||
| 916 | 916 | ||
| 917 | + email | ||
| 918 | + ----- | ||
| 919 | + | ||
| 920 | + * Email generators now raise an error when an :class:`.EmailMessage` cannot be | ||
| 921 | + accurately flattened due to a non-ASCII email address (mailbox) in an address | ||
| 922 | + header. Options for supporting Email Address Internationalization (EAI) are | ||
| 923 | + discussed in :attr:`.EmailPolicy.utf8`. | ||
| 924 | + (Contributed by R David Murray and Mike Edmunds in :gh:`122540`.) | ||
| 925 | + | ||
| 926 | + | ||
| 917 | 927 | functools | |
| 918 | 928 | --------- | |
| 919 | 929 | ||
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -157,10 +157,7 @@ def all_defects(self): | |||
| 157 | 157 | def startswith_fws(self): | |
| 158 | 158 | return self[0].startswith_fws() | |
| 159 | 159 | ||
| 160 | - @property | ||
| 161 | - def as_ew_allowed(self): | ||
| 162 | - """True if all top level tokens of this part may be RFC2047 encoded.""" | ||
| 163 | - return all(part.as_ew_allowed for part in self) | ||
| 160 | + as_ew_allowed = True | ||
| 164 | 161 | ||
| 165 | 162 | @property | |
| 166 | 163 | def comments(self): | |
@@ -429,6 +426,7 @@ def addr_spec(self): | |||
| 429 | 426 | class AngleAddr(TokenList): | |
| 430 | 427 | ||
| 431 | 428 | token_type = 'angle-addr' | |
| 429 | + as_ew_allowed = False | ||
| 432 | 430 | ||
| 433 | 431 | @property | |
| 434 | 432 | def local_part(self): | |
@@ -847,26 +845,22 @@ def params(self): | |||
| 847 | 845 | ||
| 848 | 846 | class ContentType(ParameterizedHeaderValue): | |
| 849 | 847 | token_type = 'content-type' | |
| 850 | - as_ew_allowed = False | ||
| 851 | 848 | maintype = 'text' | |
| 852 | 849 | subtype = 'plain' | |
| 853 | 850 | ||
| 854 | 851 | ||
| 855 | 852 | class ContentDisposition(ParameterizedHeaderValue): | |
| 856 | 853 | token_type = 'content-disposition' | |
| 857 | - as_ew_allowed = False | ||
| 858 | 854 | content_disposition = None | |
| 859 | 855 | ||
| 860 | 856 | ||
| 861 | 857 | class ContentTransferEncoding(TokenList): | |
| 862 | 858 | token_type = 'content-transfer-encoding' | |
| 863 | - as_ew_allowed = False | ||
| 864 | 859 | cte = '7bit' | |
| 865 | 860 | ||
| 866 | 861 | ||
| 867 | 862 | class HeaderLabel(TokenList): | |
| 868 | 863 | token_type = 'header-label' | |
| 869 | - as_ew_allowed = False | ||
| 870 | 864 | ||
| 871 | 865 | ||
| 872 | 866 | class MsgID(TokenList): | |
@@ -2835,13 +2829,68 @@ def _steal_trailing_WSP_if_exists(lines): | |||
| 2835 | 2829 | ||
| 2836 | 2830 | ||
| 2837 | 2831 | def _refold_parse_tree(parse_tree, *, policy): | |
| 2838 | - """Return string of contents of parse_tree folded according to RFC rules. | ||
| 2839 | - | ||
| 2840 | - """ | ||
| 2841 | 2832 | # max_line_length 0/None means no limit, ie: infinitely long. | |
| 2842 | 2833 | maxlen = policy.max_line_length or sys.maxsize | |
| 2843 | 2834 | encoding = 'utf-8' if policy.utf8 else 'us-ascii' | |
| 2844 | 2835 | lines = [''] # Folded lines to be output | |
| 2836 | + if parse_tree.as_ew_allowed: | ||
| 2837 | + _refold_with_ew(parse_tree, lines, maxlen, encoding, policy=policy) | ||
| 2838 | + else: | ||
| 2839 | + _refold_without_ew(parse_tree, lines, maxlen, encoding, policy=policy) | ||
| 2840 | + return policy.linesep.join(lines) + policy.linesep | ||
| 2841 | + | ||
| 2842 | + def _refold_without_ew(parse_tree, lines, maxlen, encoding, *, policy): | ||
| 2843 | + parts = list(parse_tree) | ||
| 2844 | + while parts: | ||
| 2845 | + part = parts.pop(0) | ||
| 2846 | + tstr = str(part) | ||
| 2847 | + try: | ||
| 2848 | + tstr.encode(encoding) | ||
| 2849 | + except UnicodeEncodeError: | ||
| 2850 | + if any(isinstance(x, errors.UndecodableBytesDefect) | ||
| 2851 | + for x in part.all_defects): | ||
| 2852 | + # There is garbage data from parsing a message in binary mode, | ||
| 2853 | + # just pass it through. Not good, but the best we can do. | ||
| 2854 | + pass | ||
| 2855 | + elif policy.utf8: | ||
| 2856 | + # If this happens, it's a programmer error. | ||
| 2857 | + raise | ||
| 2858 | + else: | ||
| 2859 | + raise errors.HeaderWriteError( | ||
| 2860 | + f"Non-ASCII {part.token_type} '{part}' is invalid" | ||
| 2861 | + " under current policy setting (utf8=False)" | ||
| 2862 | + ) | ||
| 2863 | + if len(tstr) <= maxlen - len(lines[-1]): | ||
| 2864 | + lines[-1] += tstr | ||
| 2865 | + continue | ||
| 2866 | + # This part is too long to fit. The RFC wants us to break at | ||
| 2867 | + # "major syntactic breaks", so unless we don't consider this | ||
| 2868 | + # to be one, check if it will fit on the next line by itself. | ||
| 2869 | + if (part.syntactic_break and | ||
| 2870 | + len(tstr) + 1 <= maxlen): | ||
| 2871 | + newline = _steal_trailing_WSP_if_exists(lines) | ||
| 2872 | + if newline or part.startswith_fws(): | ||
| 2873 | + lines.append(newline + tstr) | ||
| 2874 | + continue | ||
| 2875 | + if not hasattr(part, 'encode'): | ||
| 2876 | + # It's not a terminal, try folding the subparts. | ||
| 2877 | + newparts = list(part) | ||
| 2878 | + parts = newparts + parts | ||
| 2879 | + continue | ||
| 2880 | + # We can't figure out how to wrap, it, so give up. | ||
| 2881 | + newline = _steal_trailing_WSP_if_exists(lines) | ||
| 2882 | + if newline or part.startswith_fws(): | ||
| 2883 | + lines.append(newline + tstr) | ||
| 2884 | + else: | ||
| 2885 | + # We can't fold it onto the next line either... | ||
| 2886 | + lines[-1] += tstr | ||
| 2887 | + return | ||
| 2888 | + | ||
| 2889 | + | ||
| 2890 | + def _refold_with_ew(parse_tree, lines, maxlen, encoding, *, policy): | ||
| 2891 | + """Return string of contents of parse_tree folded according to RFC rules. | ||
| 2892 | + | ||
| 2893 | + """ | ||
| 2845 | 2894 | last_word_is_ew = False | |
| 2846 | 2895 | last_ew = None # if there is an encoded word in the last line of lines, | |
| 2847 | 2896 | # points to the encoded word's first character | |
@@ -2855,6 +2904,11 @@ def _refold_parse_tree(parse_tree, *, policy): | |||
| 2855 | 2904 | if part is end_ew_not_allowed: | |
| 2856 | 2905 | wrap_as_ew_blocked -= 1 | |
| 2857 | 2906 | continue | |
| 2907 | + if part.token_type == 'mime-parameters': | ||
| 2908 | + # Mime parameter folding (using RFC2231) is extra special. | ||
| 2909 | + _fold_mime_parameters(part, lines, maxlen, encoding) | ||
| 2910 | + last_word_is_ew = False | ||
| 2911 | + continue | ||
| 2858 | 2912 | tstr = str(part) | |
| 2859 | 2913 | if not want_encoding: | |
| 2860 | 2914 | if part.token_type in ('ptext', 'vtext'): | |
@@ -2876,14 +2930,11 @@ def _refold_parse_tree(parse_tree, *, policy): | |||
| 2876 | 2930 | charset = 'utf-8' | |
| 2877 | 2931 | want_encoding = True | |
| 2878 | 2932 | ||
| 2879 | - if part.token_type == 'mime-parameters': | ||
| 2880 | - # Mime parameter folding (using RFC2231) is extra special. | ||
| 2881 | - _fold_mime_parameters(part, lines, maxlen, encoding) | ||
| 2882 | - last_word_is_ew = False | ||
| 2883 | - continue | ||
| 2884 | - | ||
| 2885 | 2933 | if want_encoding and not wrap_as_ew_blocked: | |
| 2886 | - if not part.as_ew_allowed: | ||
| 2934 | + if any( | ||
| 2935 | + not x.as_ew_allowed for x in part | ||
| 2936 | + if hasattr(x, 'as_ew_allowed') | ||
| 2937 | + ): | ||
| 2887 | 2938 | want_encoding = False | |
| 2888 | 2939 | last_ew = None | |
| 2889 | 2940 | if part.syntactic_break: | |
@@ -2964,6 +3015,8 @@ def _refold_parse_tree(parse_tree, *, policy): | |||
| 2964 | 3015 | [ValueTerminal(make_quoted_pairs(p), 'ptext') | |
| 2965 | 3016 | for p in newparts] + | |
| 2966 | 3017 | [ValueTerminal('"', 'ptext')]) | |
| 3018 | + _refold_without_ew(newparts, lines, maxlen, encoding, policy=policy) | ||
| 3019 | + continue | ||
| 2967 | 3020 | if part.token_type == 'comment': | |
| 2968 | 3021 | newparts = ( | |
| 2969 | 3022 | [ValueTerminal('(', 'ptext')] + | |
@@ -2991,7 +3044,7 @@ def _refold_parse_tree(parse_tree, *, policy): | |||
| 2991 | 3044 | lines[-1] += tstr | |
| 2992 | 3045 | last_word_is_ew = last_word_is_ew and not bool(tstr.strip(_WSP)) | |
| 2993 | 3046 | ||
| 2994 | - return policy.linesep.join(lines) + policy.linesep | ||
| 3047 | + return | ||
| 2995 | 3048 | ||
| 2996 | 3049 | def _fold_as_ew(to_encode, lines, maxlen, last_ew, ew_combine_allowed, charset, last_word_is_ew): | |
| 2997 | 3050 | """Fold string to_encode into lines as encoded word, combining if allowed. | |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -3374,10 +3374,12 @@ def test_fold_unfoldable_element_stealing_whitespace(self): | |||
| 3374 | 3374 | self._test(token, expected, policy=policy) | |
| 3375 | 3375 | ||
| 3376 | 3376 | def test_encoded_word_with_undecodable_bytes(self): | |
| 3377 | - self._test(parser.get_address_list( | ||
| 3378 | - ' =?utf-8?Q?=E5=AE=A2=E6=88=B6=E6=AD=A3=E8=A6=8F=E4=BA=A4=E7?=' | ||
| 3377 | + self._test( | ||
| 3378 | + parser.get_address_list( | ||
| 3379 | + ' =?utf-8?Q?=E5=AE=A2=E6=88=B6=E6=AD=A3=E8=A6=8F=E4=BA=A4=E7?=' | ||
| 3380 | + ' <xyz@abc.com>' | ||
| 3379 | 3381 | )[0], | |
| 3380 | - ' =?unknown-8bit?b?5a6i5oi25q2j6KaP5Lqk5w==?=\n', | ||
| 3382 | + ' =?unknown-8bit?b?5a6i5oi25q2j6KaP5Lqk5w==?= <xyz@abc.com>\n', | ||
| 3381 | 3383 | ) | |
| 3382 | 3384 | ||
| 3383 | 3385 | ||
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -1,4 +1,5 @@ | |||
| 1 | 1 | import io | |
| 2 | + import re | ||
| 2 | 3 | import textwrap | |
| 3 | 4 | import unittest | |
| 4 | 5 | import random | |
@@ -295,6 +296,69 @@ def test_keep_long_encoded_newlines(self): | |||
| 295 | 296 | g.flatten(msg) | |
| 296 | 297 | self.assertEqual(s.getvalue(), self.typ(expected)) | |
| 297 | 298 | ||
| 299 | + def test_non_ascii_addr_spec_raises(self): | ||
| 300 | + # non-ascii is not permitted in any part of an addr-spec. If the | ||
| 301 | + # programmer generated it, it's an error. (See also | ||
| 302 | + # test_non_ascii_addr_spec_preserved below.) | ||
| 303 | + p = self.policy.clone(utf8=False, max_line_length=20) | ||
| 304 | + g = self.genclass(self.ioclass(), policy=p) | ||
| 305 | + # XXX The particular part detected here isn't part of a behavioral | ||
| 306 | + # spec and may change in the future. | ||
| 307 | + cases = [ | ||
| 308 | + ('wők@example.com', 'wők', 'local-part'), | ||
| 309 | + ('wok@exàmple.com', 'exàmple.com', 'domain'), | ||
| 310 | + ('wők@exàmple.com', 'wők', 'local-part'), | ||
| 311 | + ( | ||
| 312 | + '"Name, for display" <wők@example.com>', | ||
| 313 | + 'wők@example.com', | ||
| 314 | + 'addr-spec', | ||
| 315 | + ), | ||
| 316 | + ( | ||
| 317 | + 'Näyttönimi <wők@example.com>', | ||
| 318 | + 'wők@example.com', | ||
| 319 | + 'addr-spec', | ||
| 320 | + ), | ||
| 321 | + ( | ||
| 322 | + '"a lőng quoted string as the local part"@example.com', | ||
| 323 | + 'a lőng quoted string as the local part', | ||
| 324 | + 'local-part', | ||
| 325 | + ), | ||
| 326 | + | ||
| 327 | + ] | ||
| 328 | + for address, badtoken, partname in cases: | ||
| 329 | + with self.subTest(address=address): | ||
| 330 | + msg = EmailMessage() | ||
| 331 | + msg['To'] = address | ||
| 332 | + expected_error = ( | ||
| 333 | + fr"(?i)(?=.*non-ascii)" | ||
| 334 | + fr"(?=.*{re.escape(badtoken)})" | ||
| 335 | + fr"(?=.*{partname})" | ||
| 336 | + fr"(?=.*policy.*utf8)" | ||
| 337 | + ) | ||
| 338 | + with self.assertRaisesRegex( | ||
| 339 | + email.errors.HeaderWriteError, expected_error | ||
| 340 | + ): | ||
| 341 | + g.flatten(msg) | ||
| 342 | + | ||
| 343 | + def test_local_part_quoted_string_wrapped_correctly(self): | ||
| 344 | + msg = self.msgmaker(self.typ(textwrap.dedent("""\ | ||
| 345 | + To: <"a long local part in a quoted string"@example.com> | ||
| 346 | + Subject: test | ||
| 347 | + | ||
| 348 | + None | ||
| 349 | + """)), policy=self.policy.clone(max_line_length=20)) | ||
| 350 | + expected = textwrap.dedent("""\ | ||
| 351 | + To: <"a long local part in a | ||
| 352 | + quoted string"@example.com> | ||
| 353 | + Subject: test | ||
| 354 | + | ||
| 355 | + None | ||
| 356 | + """) | ||
| 357 | + s = self.ioclass() | ||
| 358 | + g = self.genclass(s, policy=self.policy.clone(max_line_length=30)) | ||
| 359 | + g.flatten(msg) | ||
| 360 | + self.assertEqual(s.getvalue(), self.typ(expected)) | ||
| 361 | + | ||
| 298 | 362 | def _test_boundary_detection(self, linesep): | |
| 299 | 363 | # Generate a boundary token in the same way as _make_boundary | |
| 300 | 364 | token = random.randrange(sys.maxsize) | |
@@ -515,12 +579,12 @@ def test_cte_type_7bit_transforms_8bit_cte(self): | |||
| 515 | 579 | ||
| 516 | 580 | def test_smtputf8_policy(self): | |
| 517 | 581 | msg = EmailMessage() | |
| 518 | - msg['From'] = "Páolo <főo@bar.com>" | ||
| 582 | + msg['From'] = "Páolo <főo@bàr.com>" | ||
| 519 | 583 | msg['To'] = 'Dinsdale' | |
| 520 | 584 | msg['Subject'] = 'Nudge nudge, wink, wink \u1F609' | |
| 521 | 585 | msg.set_content("oh là là, know what I mean, know what I mean?") | |
| 522 | 586 | expected = textwrap.dedent("""\ | |
| 523 | - From: Páolo <főo@bar.com> | ||
| 587 | + From: Páolo <főo@bàr.com> | ||
| 524 | 588 | To: Dinsdale | |
| 525 | 589 | Subject: Nudge nudge, wink, wink \u1F609 | |
| 526 | 590 | Content-Type: text/plain; charset="utf-8" | |
@@ -555,6 +619,37 @@ def test_smtp_policy(self): | |||
| 555 | 619 | g.flatten(msg) | |
| 556 | 620 | self.assertEqual(s.getvalue(), expected) | |
| 557 | 621 | ||
| 622 | + def test_non_ascii_addr_spec_preserved(self): | ||
| 623 | + # A defective non-ASCII addr-spec parsed from the original | ||
| 624 | + # message is left unchanged when flattening. | ||
| 625 | + # (See also test_non_ascii_addr_spec_raises above.) | ||
| 626 | + source = ( | ||
| 627 | + 'To: jörg@example.com, "But a long name still works with refold_source" <jörg@example.com>' | ||
| 628 | + ).encode() | ||
| 629 | + expected = ( | ||
| 630 | + b'To: j\xc3\xb6rg@example.com,\n' | ||
| 631 | + b' "But a long name still works with refold_source" <j\xc3\xb6rg@example.com>\n' | ||
| 632 | + b'\n' | ||
| 633 | + ) | ||
| 634 | + msg = message_from_bytes(source, policy=policy.default) | ||
| 635 | + s = io.BytesIO() | ||
| 636 | + g = BytesGenerator(s, policy=policy.default) | ||
| 637 | + g.flatten(msg) | ||
| 638 | + self.assertEqual(s.getvalue(), expected) | ||
| 639 | + | ||
| 640 | + def test_idna_encoding_preserved(self): | ||
| 641 | + # Nothing tries to decode a pre-encoded IDNA domain. | ||
| 642 | + msg = EmailMessage() | ||
| 643 | + msg["To"] = Address( | ||
| 644 | + username='jörg', | ||
| 645 | + domain='☕.example'.encode('idna').decode() # IDNA 2003 | ||
| 646 | + ) | ||
| 647 | + expected = 'To: jörg@xn--53h.example\n\n'.encode() | ||
| 648 | + s = io.BytesIO() | ||
| 649 | + g = BytesGenerator(s, policy=policy.default.clone(utf8=True)) | ||
| 650 | + g.flatten(msg) | ||
| 651 | + self.assertEqual(s.getvalue(), expected) | ||
| 652 | + | ||
| 558 | 653 | ||
| 559 | 654 | if __name__ == '__main__': | |
| 560 | 655 | unittest.main() | |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -0,0 +1,8 @@ | |||
| 1 | + The :mod:`email` module no longer incorrectly uses :rfc:`2047` encoding for | ||
| 2 | + a mailbox with non-ASCII characters in its domain. Under a policy with | ||
| 3 | + :attr:`~email.policy.EmailPolicy.utf8` set ``False``, attempting to serialize | ||
| 4 | + such a message will now raise an :exc:`~email.errors.HeaderWriteError`. | ||
| 5 | + Either apply an appropriate IDNA encoding to convert the domain to ASCII before | ||
| 6 | + serialization, or use :data:`email.policy.SMTPUTF8` (or another policy with | ||
| 7 | + ``utf8=True``) to correctly pass through the internationalized domain name | ||
| 8 | + as Unicode characters. | ||
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -0,0 +1,7 @@ | |||
| 1 | + The :mod:`email` module no longer incorrectly uses :rfc:`2047` encoding for | ||
| 2 | + a mailbox with non-ASCII characters in its local-part. Under a policy with | ||
| 3 | + :attr:`~email.policy.EmailPolicy.utf8` set ``False``, attempting to serialize | ||
| 4 | + such a message will now raise an :exc:`~email.errors.HeaderWriteError`. | ||
| 5 | + There is no valid 7-bit encoding for an internationalized local-part. Use | ||
| 6 | + :data:`email.policy.SMTPUTF8` (or another policy with ``utf8=True``) to | ||
| 7 | + correctly pass through the local-part as Unicode characters. | ||
| Back | FazBrowse Home | New Git URL |
0 commit comments