| FazBrowse GitHub Viewer | Trending | | Home |
| Tools: [Download Repo ZIP] [Original HTTPS Page] |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -60,9 +60,21 @@ Requirements | |||
| 60 | 60 | ||
| 61 | 61 | Nothing except Python itself. | |
| 62 | 62 | ||
| 63 | - You will need a Python build with "wide" Unicode characters in order for | ||
| 64 | - unidecode to work correctly with characters outside of Basic Multilingual | ||
| 65 | - Plane. Surrogate pair encoding of "narrow" builds is not supported. | ||
| 63 | + You need a Python build with "wide" Unicode characters (also called "UCS-4 | ||
| 64 | + build") in order for unidecode to work correctly with characters outside of | ||
| 65 | + Basic Multilingual Plane (BMP). Common characters outside BMP are bold, italic, | ||
| 66 | + script, etc. variants of the Latin alphabet intended for mathematical notation. | ||
| 67 | + Surrogate pair encoding of "narrow" builds is not supported in unidecode. | ||
| 68 | + | ||
| 69 | + If your Python build supports "wide" Unicode the following expression will | ||
| 70 | + return True:: | ||
| 71 | + | ||
| 72 | + >>> import sys | ||
| 73 | + >>> sys.maxunicode > 0xffff | ||
| 74 | + True | ||
| 75 | + | ||
| 76 | + See PEP 261 for details regarding support for "wide" Unicode characters in | ||
| 77 | + Python. | ||
| 66 | 78 | ||
| 67 | 79 | ||
| 68 | 80 | Installation | |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -51,10 +51,51 @@ def showwarning_new(message, category, *args): | |||
| 51 | 51 | ||
| 52 | 52 | def test_bmp(self): | |
| 53 | 53 | for n in xrange(0,0x10000): | |
| 54 | + # skip over surrogate pairs, which throw a warning | ||
| 55 | + if 0xd800 <= n <= 0xdfff: | ||
| 56 | + continue | ||
| 57 | + | ||
| 54 | 58 | # Just check that it doesn't throw an exception | |
| 55 | 59 | t = unichr(n) | |
| 56 | 60 | unidecode(t) | |
| 57 | 61 | ||
| 62 | + @unittest.skipIf(sys.maxunicode < 0x10000, "narrow build") | ||
| 63 | + def test_surrogate_pairs(self): | ||
| 64 | + # same character, written as a non-BMP character and a | ||
| 65 | + # surrogate pair | ||
| 66 | + s = u'\U0001d4e3' | ||
| 67 | + | ||
| 68 | + # Note: this needs to be constructed at run-time, otherwise | ||
| 69 | + # a "wide" Python seems to optimize it automatically into a | ||
| 70 | + # single character. | ||
| 71 | + s_sp_1 = u'\ud835' | ||
| 72 | + s_sp_2 = u'\udce3' | ||
| 73 | + s_sp = s_sp_1 + s_sp_2 | ||
| 74 | + | ||
| 75 | + self.assertEqual(s.encode('utf16'), s_sp.encode('utf16')) | ||
| 76 | + | ||
| 77 | + log = [] | ||
| 78 | + def showwarning_new(message, category, *args): | ||
| 79 | + if ("Surrogate character" in str(message)) and \ | ||
| 80 | + (category is RuntimeWarning): | ||
| 81 | + log.append((message, category)) | ||
| 82 | + else: | ||
| 83 | + showwarning_old(message, category, *args) | ||
| 84 | + | ||
| 85 | + showwarning_old = warnings.showwarning | ||
| 86 | + warnings.showwarning = showwarning_new | ||
| 87 | + warnings.filterwarnings("always") | ||
| 88 | + | ||
| 89 | + a = unidecode(s) | ||
| 90 | + a_sp = unidecode(s_sp) | ||
| 91 | + | ||
| 92 | + self.assertEqual('T', a) | ||
| 93 | + | ||
| 94 | + # Two warnings should have been logged | ||
| 95 | + self.assertEqual(2, len(log)) | ||
| 96 | + | ||
| 97 | + warnings.showwarning = showwarning_old | ||
| 98 | + | ||
| 58 | 99 | def test_circled_latin(self): | |
| 59 | 100 | # 1 sequence of a-z | |
| 60 | 101 | for n in xrange(0, 26): | |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -11,6 +11,10 @@ def test_ascii(self): | |||
| 11 | 11 | ||
| 12 | 12 | def test_bmp(self): | |
| 13 | 13 | for n in range(0,0x10000): | |
| 14 | + # skip over surrogate pairs, which throw a warning | ||
| 15 | + if 0xd800 <= n <= 0xdfff: | ||
| 16 | + continue | ||
| 17 | + | ||
| 14 | 18 | # Just check that it doesn't throw an exception | |
| 15 | 19 | t = chr(n) | |
| 16 | 20 | unidecode(t) | |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -44,6 +44,11 @@ def unidecode(string): | |||
| 44 | 44 | if codepoint > 0xeffff: | |
| 45 | 45 | continue # Characters in Private Use Area and above are ignored | |
| 46 | 46 | ||
| 47 | + if 0xd800 <= codepoint <= 0xdfff: | ||
| 48 | + warnings.warn( "Surrogate character %r will be ignored. " | ||
| 49 | + "You might be using a narrow Python build." % (char,), | ||
| 50 | + RuntimeWarning, 2) | ||
| 51 | + | ||
| 47 | 52 | section = codepoint >> 8 # Chop off the last two hex digits | |
| 48 | 53 | position = codepoint % 256 # Last two hex digits | |
| 49 | 54 | ||
| Back | FazBrowse Home | New Git URL |
0 commit comments