FazBrowse GitHub Viewer
|
Trending
|
URL:
|
Home
Tools:
[Download Repo ZIP]
[View Raw Code]
[Original HTTPS Page]
unidecode/unidecode/__init__.py at unidecode-1.1.2 · feenes/unidecode · GitHub
feenes
unidecode
Repository navigation
Code
Pull requests
Actions
Projects
Security and quality
Insights
Expand file tree
Breadcrumbs
unidecode
/
unidecode
/
__init__.py
Copy path
More file actions
More file actions
Latest commit
History
History
History
103 lines (77 loc) · 3.02 KB
Breadcrumbs
unidecode
/
unidecode
/
__init__.py
Copy path
File metadata and controls
103 lines (77 loc) · 3.02 KB
Raw
Copy raw file
Download raw file
Open symbols panel
Edit and raw actions
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
# -*- coding: utf-8 -*-
# vi:tabstop=4:expandtab:sw=4
"""Transliterate Unicode text into plain 7-bit ASCII.
Example usage:
>>> from unidecode import unidecode
>>> unidecode(u"
\u5317
\u4EB0
")
"Bei Jing "
The transliteration uses a straightforward map, and doesn't have alternatives
for the same character based on language, position, or anything else.
In Python 3, a standard string object will be returned. If you need bytes, use:
>>> unidecode("Κνωσός").encode("ascii")
b'Knosos'
"""
import
warnings
from
sys
import
version_info
Cache
=
{}
def
_warn_if_not_unicode
(
string
):
if
version_info
[
0
]
<
3
and
not
isinstance
(
string
,
unicode
):
warnings
.
warn
(
"Argument %r is not an unicode object. "
"Passing an encoded string will likely have "
"unexpected results."
%
(
type
(
string
),),
RuntimeWarning
,
2
)
def
unidecode_expect_ascii
(
string
):
"""Transliterate an Unicode object into an ASCII string
>>> unidecode(u"
\u5317
\u4EB0
")
"Bei Jing "
This function first tries to convert the string using ASCII codec.
If it fails (because of non-ASCII characters), it falls back to
transliteration using the character tables.
This is approx. five times faster if the string only contains ASCII
characters, but slightly slower than using unidecode directly if non-ASCII
chars are present.
"""
_warn_if_not_unicode
(
string
)
try
:
bytestring
=
string
.
encode
(
'ASCII'
)
except
UnicodeEncodeError
:
return
_unidecode
(
string
)
if
version_info
[
0
]
>=
3
:
return
string
else
:
return
bytestring
def
unidecode_expect_nonascii
(
string
):
"""Transliterate an Unicode object into an ASCII string
>>> unidecode(u"
\u5317
\u4EB0
")
"Bei Jing "
"""
_warn_if_not_unicode
(
string
)
return
_unidecode
(
string
)
unidecode
=
unidecode_expect_ascii
def
_unidecode
(
string
):
retval
=
[]
for
char
in
string
:
codepoint
=
ord
(
char
)
if
codepoint
<
0x80
:
# Basic ASCII
retval
.
append
(
str
(
char
))
continue
if
codepoint
>
0xeffff
:
continue
# Characters in Private Use Area and above are ignored
if
0xd800
<=
codepoint
<=
0xdfff
:
warnings
.
warn
(
"Surrogate character %r will be ignored. "
"You might be using a narrow Python build."
%
(
char
,),
RuntimeWarning
,
2
)
section
=
codepoint
>>
8
# Chop off the last two hex digits
position
=
codepoint
%
256
# Last two hex digits
try
:
table
=
Cache
[
section
]
except
KeyError
:
try
:
mod
=
__import__
(
'unidecode.x%03x'
%
(
section
),
globals
(),
locals
(), [
'data'
])
except
ImportError
:
Cache
[
section
]
=
None
continue
# No match: ignore this character and carry on.
Cache
[
section
]
=
table
=
mod
.
data
if
table
and
len
(
table
)
>
position
:
retval
.
append
(
table
[
position
] )
return
''
.
join
(
retval
)
Back
|
FazBrowse Home
|
New Git URL