FazBrowse GitHub Viewer | Trending |
URL:
| Home
Tools: [Download Repo ZIP]   [Original HTTPS Page]

[3.10] gh-155292: Don't consider Unicode codepoint attributes outside… · python/cpython@d397a49 · GitHub

  1. python
  2. cpython

Repository navigation

Commit d397a49

Browse files
authored
[3.10] gh-155292: Don't consider Unicode codepoint attributes outside RFC 3454 (GH-155293) (GH-156020) (#156932)
* [3.10] gh-155292: Don't consider Unicode codepoint attributes outside RFC 3454 (GH-155293) (GH-156020) Due to a bug, some Unicode codepoint attributes were considered for characters not yet defined in Unicode 3.2.0 or attributes which changed in later Unicode versions. RFC 3454 (StringPrep) requires using Unicode 3.2.0 strictly. (cherry picked from commit 7e109d0) The cherry-pick needed reworking as GH-144815 wasn't backported to 3.14 and below, so unassigned characters don't have bidi values. (cherry picked from commit 1e54caa) Co-authored-by: Petr Viktorin <encukou@gmail.com> Co-authored-by: Seth Larson <seth@python.org> Co-authored-by: Stan Ulbrych <89152624+stanfromireland@users.noreply.github.com> Co-authored-by: Petr Viktorin <encukou@gmail.com> * Add Unicode_3_2_0_FunctionsTest * Regenerate Unicode data --------- Co-authored-by: Seth Larson <seth@python.org> Co-authored-by: Stan Ulbrych <89152624+stanfromireland@users.noreply.github.com>
1 parent 4fda068 commit d397a49

6 files changed

Lines changed: 433 additions & 185 deletions

File tree

‎Lib/stringprep.py‎

Lines changed: 315 additions & 148 deletions
Large diffs are not rendered by default.

‎Lib/test/test_codecs.py‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1547,6 +1547,15 @@ def test_builtin_encode(self):
15471547
self.assertEqual("pyth\xf6n.org".encode("idna"), b"xn--pythn-mua.org")
15481548
self.assertEqual("pyth\xf6n.org.".encode("idna"), b"xn--pythn-mua.org.")
15491549

1550+
@support.subTests(['unicode', 'encoded'], [
1551+
('\N{CHEROKEE LETTER A}\N{CHEROKEE LETTER A}', b"xn--58da"),
1552+
('\N{GEORGIAN CAPITAL LETTER AN}.', b"xn--7md."),
1553+
('\N{CYRILLIC LETTER PALOCHKA}.example', b"xn--d5a.example"),
1554+
('\N{ROMAN NUMERAL REVERSED ONE HUNDRED}.example.', b"xn--q5g.example."),
1555+
])
1556+
def test_new_unicode_case_folding(self, unicode, encoded):
1557+
self.assertEqual(unicode.encode("idna"), encoded)
1558+
15501559
def test_builtin_decode_length_limit(self):
15511560
with self.assertRaisesRegex(UnicodeError, "too long"):
15521561
(b"xn--016c"+b"a"*1100).decode("idna")

‎Lib/test/test_unicodedata.py‎

Lines changed: 28 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,7 @@ class UnicodeDatabaseTest(unittest.TestCase):
6969
db = unicodedata
7070

7171
class UnicodeFunctionsTest(UnicodeDatabaseTest):
72+
old = False
7273

7374
# Update this if the database changes. Make sure to do a full rebuild
7475
# (e.g. 'make distclean && make') to get the correct checksum.
@@ -94,7 +95,8 @@ def test_function_checksum(self):
9495
]
9596
h.update(''.join(data).encode("ascii"))
9697
result = h.hexdigest()
97-
self.assertEqual(result, self.expectedchecksum)
98+
if not self.old:
99+
self.assertEqual(result, self.expectedchecksum)
98100

99101
@requires_resource('cpu')
100102
def test_name_inverse_lookup(self):
@@ -120,9 +122,13 @@ def test_numeric(self):
120122
self.assertEqual(self.db.numeric('9'), 9)
121123
self.assertEqual(self.db.numeric('\u215b'), 0.125)
122124
self.assertEqual(self.db.numeric('\u2468'), 9.0)
123-
self.assertEqual(self.db.numeric('\ua627'), 7.0)
125+
# New in 5.1.0
126+
self.assertEqual(self.db.numeric('\ua627', None),
127+
None if self.old else 7.0)
124128
self.assertEqual(self.db.numeric('\U00020000', None), None)
125-
self.assertEqual(self.db.numeric('\U0001012A'), 9000)
129+
# New in 4.1.0
130+
self.assertEqual(self.db.numeric('\U0001012A', None),
131+
None if self.old else 9000)
126132

127133
self.assertRaises(TypeError, self.db.numeric)
128134
self.assertRaises(TypeError, self.db.numeric, 'xx')
@@ -145,7 +151,8 @@ def test_category(self):
145151
self.assertEqual(self.db.category('a'), 'Ll')
146152
self.assertEqual(self.db.category('A'), 'Lu')
147153
self.assertEqual(self.db.category('\U00020000'), 'Lo')
148-
self.assertEqual(self.db.category('\U0001012A'), 'No')
154+
self.assertEqual(self.db.category('\U0001012A'),
155+
'Cn' if self.old else 'No')
149156

150157
self.assertRaises(TypeError, self.db.category)
151158
self.assertRaises(TypeError, self.db.category, 'xx')
@@ -159,6 +166,15 @@ def test_bidirectional(self):
159166
self.assertRaises(TypeError, self.db.bidirectional)
160167
self.assertRaises(TypeError, self.db.bidirectional, 'xx')
161168

169+
def test_bidirectional_unassigned(self):
170+
self.assertEqual(self.db.bidirectional('\u0378'), '')
171+
self.assertEqual(self.db.bidirectional('\u077F'), '' if self.old else 'AL')
172+
self.assertEqual(self.db.bidirectional('\u20CF'), '')
173+
self.assertEqual(self.db.bidirectional('\u0590'), '')
174+
self.assertEqual(self.db.bidirectional('\uFFFF'), '')
175+
self.assertEqual(self.db.bidirectional('\U0001FFFE'), '')
176+
self.assertEqual(self.db.bidirectional('\U00010D01'), '' if self.old else 'AL')
177+
162178
def test_decomposition(self):
163179
self.assertEqual(self.db.decomposition('\uFFFE'),'')
164180
self.assertEqual(self.db.decomposition('\u00bc'), '<fraction> 0031 2044 0034')
@@ -257,8 +273,14 @@ def test_east_asian_width(self):
257273
self.assertEqual(eaw('\U00020000'), 'W')
258274

259275
def test_east_asian_width_9_0_changes(self):
260-
self.assertEqual(self.db.ucd_3_2_0.east_asian_width('\u231a'), 'N')
261-
self.assertEqual(self.db.east_asian_width('\u231a'), 'W')
276+
self.assertEqual(self.db.east_asian_width('\u231a'),
277+
'N' if self.old else 'W')
278+
279+
280+
class Unicode_3_2_0_FunctionsTest(UnicodeFunctionsTest):
281+
db = unicodedata.ucd_3_2_0
282+
old = True
283+
262284

263285
class UnicodeMiscTest(UnicodeDatabaseTest):
264286

‎Misc/NEWS.d/next/Security/2026-08-06-11-43-20.gh-issue-155292.j4pHBO.rst‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
Change the :mod:`stringprep` module and :mod:`encodings.idna` codec to not
2+
consider Unicode codepoint attributes beyond those defined in :rfc:`3454`.

‎Tools/unicode/makeunicodedata.py‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,14 +28,15 @@
2828

2929
import dataclasses
3030
import os
31+
import subprocess
3132
import sys
3233
import zipfile
3334

3435
from functools import partial
3536
from textwrap import dedent
3637
from typing import Iterator, List, Optional, Set, Tuple
3738

38-
SCRIPT = sys.argv[0]
39+
SCRIPT = os.path.normpath(sys.argv[0])
3940
VERSION = "3.3"
4041

4142
# The Unicode Database
@@ -128,6 +129,7 @@ def maketables(trace=0):
128129
makeunicodename(unicode, trace)
129130
makeunicodedata(unicode, trace)
130131
makeunicodetype(unicode, trace)
132+
makestringprep()
131133

132134

133135
# --------------------------------------------------------------------
@@ -792,6 +794,19 @@ def word_key(a):
792794
fprint('};')
793795

794796

797+
798+
def makestringprep():
799+
FILE = "Lib/stringprep.py"
800+
801+
print("--- Preparing", FILE, "...")
802+
803+
MKSTRINGPREP = "Tools/unicode/mkstringprep.py"
804+
805+
with open(FILE, "w") as f:
806+
f.truncate()
807+
subprocess.check_call([sys.executable, MKSTRINGPREP], stdout=f)
808+
809+
795810
def merge_old_version(version, new, old):
796811
# Changes to exclusion file not implemented yet
797812
if old.exclusions != new.exclusions:

‎Tools/unicode/mkstringprep.py‎

Lines changed: 63 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,20 @@
11
import re
2-
from unicodedata import ucd_3_2_0 as unicodedata
2+
import os
3+
import unicodedata as unicodedata_current
4+
from unicodedata import ucd_3_2_0 as unicodedata_320
5+
6+
FILENAME = "Tools/unicode/data/rfc3454.txt"
7+
URL = "https://www.rfc-editor.org/rfc/rfc3454.txt"
38

49
def gen_category(cats):
510
for i in range(0, 0x110000):
6-
if unicodedata.category(chr(i)) in cats:
7-
yield(i)
11+
if unicodedata_320.category(chr(i)) in cats:
12+
yield i
813

914
def gen_bidirectional(cats):
1015
for i in range(0, 0x110000):
11-
if unicodedata.bidirectional(chr(i)) in cats:
12-
yield(i)
16+
if unicodedata_320.bidirectional(chr(i)) in cats:
17+
yield i
1318

1419
def compact_set(l):
1520
single = []
@@ -47,8 +52,16 @@ def compact_set(l):
4752

4853
############## Read the tables in the RFC #######################
4954

50-
with open("rfc3454.txt") as f:
51-
data = f.readlines()
55+
try:
56+
data_file = open(FILENAME, encoding='utf-8')
57+
except FileNotFoundError:
58+
import urllib.request
59+
os.makedirs(os.path.dirname(FILENAME), exist_ok=True)
60+
urllib.request.urlretrieve(URL, filename=FILENAME)
61+
data_file = open(FILENAME, encoding='utf-8')
62+
63+
with data_file:
64+
data = data_file.readlines()
5265

5366
tables = []
5467
curname = None
@@ -116,10 +129,18 @@ def compact_set(l):
116129
and mappings, for which a mapping function is provided.
117130
\"\"\"
118131
119-
from unicodedata import ucd_3_2_0 as unicodedata
132+
# This check asserts that mkstringprep.py has been run
133+
# when unicodedata is modified to ensure conformant behavior.
134+
import unicodedata
135+
""")
136+
137+
print("assert unicodedata.unidata_version == %r" % (unicodedata_current.unidata_version,))
138+
139+
print("""
140+
from unicodedata import ucd_3_2_0 as unicodedata_320
120141
""")
121142

122-
print("assert unicodedata.unidata_version == %r" % (unicodedata.unidata_version,))
143+
print("assert unicodedata_320.unidata_version == %r" % (unicodedata_320.unidata_version,))
123144

124145
# A.1 is the table of unassigned characters
125146
# XXX Plane 15 PUA is listed as unassigned in Python.
@@ -139,7 +160,7 @@ def compact_set(l):
139160

140161
print("""
141162
def in_table_a1(code):
142-
if unicodedata.category(code) != 'Cn': return False
163+
if unicodedata_320.category(code) != 'Cn': return False
143164
c = ord(code)
144165
if 0xFDD0 <= c < 0xFDF0: return False
145166
return (c & 0xFFFF) not in (0xFFFE, 0xFFFF)
@@ -172,21 +193,33 @@ def in_table_b1(code):
172193

173194
# B.3 is mostly Python's .lower, except for a number
174195
# of special cases, e.g. considering canonical forms.
196+
# To enforce Unicode 3.2.0 behavior of .lower instead of
197+
# whatever Unicode version is included with Python we
198+
# add unassigned or newly case-folding codepoints to
199+
# the exception map, too.
175200

176201
b3_exceptions = {}
177202

178203
for k,v in table_b2.items():
179204
if list(map(ord, chr(k).lower())) != v:
180205
b3_exceptions[k] = "".join(map(chr,v))
206+
for cp in range(0x110000):
207+
ch = chr(cp)
208+
# Assigned in current Unicode version
209+
# and supports case folding, but not
210+
# explicitly in B.2 or B.3 tables.
211+
if (unicodedata_current.category(ch) != "Cn"
212+
and ch.lower() != ch
213+
and cp not in table_b2
214+
and cp not in table_b3):
215+
b3_exceptions[cp] = ch # Identity.
181216

182217
b3 = sorted(b3_exceptions.items())
183218

184219
print("""
185220
b3_exceptions = {""")
186221
for i, kv in enumerate(b3):
187-
print("0x%x:%a," % kv, end=' ')
188-
if i % 4 == 3:
189-
print()
222+
print("0x%x:%a," % kv, end='\n' if i % 4 == 3 else ' ')
190223
print("}")
191224

192225
print("""
@@ -207,9 +240,9 @@ def map_table_b3(code):
207240

208241
def map_table_b2(a):
209242
al = map_table_b3(a)
210-
b = unicodedata.normalize("NFKC", al)
243+
b = unicodedata_320.normalize("NFKC", al)
211244
bl = "".join([map_table_b3(ch) for ch in b])
212-
c = unicodedata.normalize("NFKC", bl)
245+
c = unicodedata_320.normalize("NFKC", bl)
213246
if b != c:
214247
return c
215248
else:
@@ -226,9 +259,9 @@ def map_table_b2(a):
226259
print("""
227260
def map_table_b2(a):
228261
al = map_table_b3(a)
229-
b = unicodedata.normalize("NFKC", al)
262+
b = unicodedata_320.normalize("NFKC", al)
230263
bl = "".join([map_table_b3(ch) for ch in b])
231-
c = unicodedata.normalize("NFKC", bl)
264+
c = unicodedata_320.normalize("NFKC", bl)
232265
if b != c:
233266
return c
234267
else:
@@ -251,16 +284,16 @@ def in_table_c11(code):
251284
del tables[0]
252285
assert name == "C.1.2"
253286

254-
# table = set(table.keys())
255-
# Zs = set(gen_category(["Zs"])) - {0x20}
256-
# assert Zs == table
287+
table = set(table.keys())
288+
Zs = set(gen_category(["Zs"])) - {0x20}
289+
assert Zs == table
257290

258291
print("""
259292
def in_table_c12(code):
260-
return unicodedata.category(code) == "Zs" and code != " "
293+
return unicodedata_320.category(code) == "Zs" and code != " "
261294
262295
def in_table_c11_c12(code):
263-
return unicodedata.category(code) == "Zs"
296+
return unicodedata_320.category(code) == "Zs"
264297
""")
265298

266299
# C.2.1 ASCII control characters
@@ -275,7 +308,7 @@ def in_table_c11_c12(code):
275308

276309
print("""
277310
def in_table_c21(code):
278-
return ord(code) < 128 and unicodedata.category(code) == "Cc"
311+
return ord(code) < 128 and unicodedata_320.category(code) == "Cc"
279312
""")
280313

281314
# C.2.2 Non-ASCII control characters. It also includes
@@ -295,11 +328,11 @@ def in_table_c21(code):
295328
def in_table_c22(code):
296329
c = ord(code)
297330
if c < 128: return False
298-
if unicodedata.category(code) == "Cc": return True
331+
if unicodedata_320.category(code) == "Cc": return True
299332
return c in c22_specials
300333
301334
def in_table_c21_c22(code):
302-
return unicodedata.category(code) == "Cc" or \\
335+
return unicodedata_320.category(code) == "Cc" or \\
303336
ord(code) in c22_specials
304337
""")
305338

@@ -313,7 +346,7 @@ def in_table_c21_c22(code):
313346

314347
print("""
315348
def in_table_c3(code):
316-
return unicodedata.category(code) == "Co"
349+
return unicodedata_320.category(code) == "Co"
317350
""")
318351

319352
# C.4 Non-character code points, xFFFE, xFFFF
@@ -346,7 +379,7 @@ def in_table_c4(code):
346379

347380
print("""
348381
def in_table_c5(code):
349-
return unicodedata.category(code) == "Cs"
382+
return unicodedata_320.category(code) == "Cs"
350383
""")
351384

352385
# C.6 Inappropriate for plain text
@@ -411,7 +444,7 @@ def in_table_c9(code):
411444

412445
print("""
413446
def in_table_d1(code):
414-
return unicodedata.bidirectional(code) in ("R","AL")
447+
return unicodedata_320.bidirectional(code) in ("R","AL")
415448
""")
416449

417450
# D.2 Characters with bidirectional property "L"
@@ -424,5 +457,5 @@ def in_table_d1(code):
424457

425458
print("""
426459
def in_table_d2(code):
427-
return unicodedata.bidirectional(code) == "L"
428-
""")
460+
return unicodedata_320.bidirectional(code) == "L"
461+
""", end="")

0 commit comments

Comments
 (0)
Back | FazBrowse Home | New Git URL