FazBrowse GitHub Viewer | Trending |
URL:
| Home
Tools: [Download Repo ZIP]   [Original HTTPS Page]

convert UNICODE whitespace to the ordinary ASCII 32 space · avian2/unidecode@dc5e3f4 · GitHub

Repository navigation

Commit dc5e3f4

Browse files
committed
convert UNICODE whitespace to the ordinary ASCII 32 space
1 parent e3c730e commit dc5e3f4

2 files changed

Lines changed: 11 additions & 1 deletion

File tree

‎tests/test_unidecode.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,11 @@ def test_ascii(self):
7777

7878
wlog.stop()
7979

80+
def test_whitespace(self):
81+
self.assertEqual(" ", self.unidecode(_chr(0xa0)))
82+
self.assertEqual(" ", self.unidecode(_chr(0x2000)))
83+
self.assertEqual(" ", self.unidecode(_chr(0x3000)))
84+
8085
def test_bmp(self):
8186
for n in range(0,0x10000):
8287
# skip over surrogate pairs, which throw a warning

‎unidecode/__init__.py‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
"""
1717
import warnings
1818
from sys import version_info
19+
import re
1920

2021
Cache = {}
2122

@@ -74,10 +75,14 @@ def _unidecode(string):
7475
if codepoint < 0x80: # Basic ASCII
7576
retval.append(str(char))
7677
continue
77-
78+
7879
if codepoint > 0xeffff:
7980
continue # Characters in Private Use Area and above are ignored
8081

82+
if re.match(r"\s",char,re.UNICODE):
83+
retval.append(" ")
84+
continue
85+
8186
if 0xd800 <= codepoint <= 0xdfff:
8287
warnings.warn( "Surrogate character %r will be ignored. "
8388
"You might be using a narrow Python build." % (char,),

0 commit comments

Comments
 (0)

Back | FazBrowse Home | New Git URL