unidecode/unidecode/__init__.py at master · AnkiHubSoftware/unidecode · GitHub

Repository navigation

History

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

# -*- coding: utf-8 -*-

# vi:tabstop=4:expandtab:sw=4

"""Transliterate Unicode text into plain 7-bit ASCII.

Example usage:

>>> from unidecode import unidecode

>>> unidecode("\u5317\u4EB0")

"Bei Jing "

The transliteration uses a straightforward map, and doesn't have alternatives

for the same character based on language, position, or anything else.

A standard string object will be returned. If you need bytes, use:

>>> unidecode("Κνωσός").encode("ascii")

b'Knosos'

"""

import warnings

from typing import Dict, Optional, Sequence

Cache: Dict[int, Optional[Sequence[Optional[str]]]] = {}

class UnidecodeError(ValueError):

def __init__(self, message: str, index: Optional[int] = None) -> None:

"""Raised for Unidecode-related errors.

The index attribute contains the index of the character that caused

the error.

"""

super(UnidecodeError, self).__init__(message)

self.index = index

def unidecode_expect_ascii(string: str, errors: str = 'ignore', replace_str: str = '?') -> str:

"""Transliterate an Unicode object into an ASCII string

>>> unidecode("\u5317\u4EB0")

"Bei Jing "

This function first tries to convert the string using ASCII codec.

If it fails (because of non-ASCII characters), it falls back to

transliteration using the character tables.

This is approx. five times faster if the string only contains ASCII

characters, but slightly slower than unicode_expect_nonascii if

non-ASCII characters are present.

errors specifies what to do with characters that have not been

found in replacement tables. The default is 'ignore' which ignores

the character. 'strict' raises an UnidecodeError. 'replace'

substitutes the character with replace_str (default is '?').

'preserve' keeps the original character.

Note that if 'preserve' is used the returned string might not be

ASCII!

"""

try:

bytestring = string.encode('ASCII')

except UnicodeEncodeError:

pass

else:

return string

return _unidecode(string, errors, replace_str)

def unidecode_expect_nonascii(string: str, errors: str = 'ignore', replace_str: str = '?') -> str:

"""Transliterate an Unicode object into an ASCII string

>>> unidecode("\u5317\u4EB0")

"Bei Jing "

See unidecode_expect_ascii.

"""

return _unidecode(string, errors, replace_str)

unidecode = unidecode_expect_ascii

def _get_repl_str(char: str) -> Optional[str]:

codepoint = ord(char)

if codepoint < 0x80:

# Already ASCII

return str(char)

if codepoint > 0xeffff:

# No data on characters in Private Use Area and above.

return None

if 0xd800 <= codepoint <= 0xdfff:

warnings.warn( "Surrogate character %r will be ignored. "

"You might be using a narrow Python build." % (char,),

RuntimeWarning, 2)

section = codepoint >> 8 # Chop off the last two hex digits

position = codepoint % 256 # Last two hex digits

try:

table = Cache[section]

except KeyError:

try:

mod = __import__('unidecode.x%03x'%(section), globals(), locals(), ['data'])

except ImportError:

# No data on this character

Cache[section] = None

return None

Cache[section] = table = mod.data

if table and len(table) > position:

return table[position]

else:

return None

def _unidecode(string: str, errors: str, replace_str:str) -> str:

retval = []

for index, char in enumerate(string):

repl = _get_repl_str(char)

if repl is None:

if errors == 'ignore':

repl = ''

elif errors == 'strict':

raise UnidecodeError('no replacement found for character %r '

'in position %d' % (char, index), index)

elif errors == 'replace':

repl = replace_str

elif errors == 'preserve':

repl = char

else:

raise UnidecodeError('invalid value for errors parameter %r' % (errors,))

retval.append(repl)

return ''.join(retval)

unidecode/unidecode/init.py at master · AnkiHubSoftware/unidecode · GitHub

Repository navigation

init.py

init.py

unidecode/unidecode/__init__.py at master · AnkiHubSoftware/unidecode · GitHub

Repository navigation

Expand file tree

Breadcrumbs

__init__.py

Latest commit

History

Breadcrumbs

__init__.py

File metadata and controls

unidecode/unidecode/init.py at master · AnkiHubSoftware/unidecode · GitHub

init.py

init.py