| FazBrowse GitHub Viewer | Trending | | Home |
| Tools: [Download Repo ZIP] [Original HTTPS Page] |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -42,9 +42,9 @@ Sean M. Burke <sburke@cpan.org>. | |||
| 42 | 42 | Module content | |
| 43 | 43 | -------------- | |
| 44 | 44 | ||
| 45 | - The module exports a single function that takes an Unicode object (Python | ||
| 46 | - 2.x) or string (Python 3.x) and returns a string (that can be encoded to | ||
| 47 | - ASCII bytes in Python 3.x):: | ||
| 45 | + The module exports a function that takes an Unicode object (Python 2.x) or | ||
| 46 | + string (Python 3.x) and returns a string (that can be encoded to ASCII bytes in | ||
| 47 | + Python 3.x):: | ||
| 48 | 48 | ||
| 49 | 49 | >>> from unidecode import unidecode | |
| 50 | 50 | >>> unidecode(u'ko\u017eu\u0161\u010dek') | |
@@ -54,6 +54,17 @@ ASCII bytes in Python 3.x):: | |||
| 54 | 54 | >>> unidecode(u"\u5317\u4EB0") | |
| 55 | 55 | 'Bei Jing ' | |
| 56 | 56 | ||
| 57 | + For use cases where most strings passed are ASCII and only some occassional | ||
| 58 | + non-ASCII ones, use the `unidecode_fast` function:: | ||
| 59 | + | ||
| 60 | + >>> from unidecode import unidecode_fast | ||
| 61 | + >>> unidecode_fast(u'Hello world!') | ||
| 62 | + 'Hello world!' | ||
| 63 | + | ||
| 64 | + This function about five times faster if the string only contains ASCII | ||
| 65 | + characters, but sligthly slower than using `unidecode` directly non-ASCII chars | ||
| 66 | + are present. | ||
| 67 | + | ||
| 57 | 68 | A utility is also included that allows you to transliterate text from the | |
| 58 | 69 | command line in several ways. Reading from standard input:: | |
| 59 | 70 | ||
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -2,7 +2,7 @@ | |||
| 2 | 2 | # vim:ts=4 sw=4 expandtab softtabstop=4 | |
| 3 | 3 | import unittest | |
| 4 | 4 | import sys | |
| 5 | - from unidecode import unidecode | ||
| 5 | + from unidecode import unidecode, unidecode_fast | ||
| 6 | 6 | import warnings | |
| 7 | 7 | ||
| 8 | 8 | # workaround for Python < 2.7 | |
@@ -502,5 +502,16 @@ def test_unicode_text_converter(self): | |||
| 502 | 502 | ||
| 503 | 503 | self.assertEqual('THE QUICK BROWN FOX JUMPS OVER THE LAZY DOG 1234567890', o) | |
| 504 | 504 | ||
| 505 | + | ||
| 506 | + class TestUnidecodeFast(unittest.TestCase): | ||
| 507 | + def test_ascii_fast(self): | ||
| 508 | + out = unidecode_fast(_u('Hello, World!')) | ||
| 509 | + self.assertEqual(out, 'Hello, World!') | ||
| 510 | + | ||
| 511 | + def test_nonascii_fast(self): | ||
| 512 | + out = unidecode_fast(_u('příliš žluťoučký kůň pěl ďábelské ódy')) | ||
| 513 | + self.assertEqual(out, 'prilis zlutoucky kun pel dabelske ody') | ||
| 514 | + | ||
| 515 | + | ||
| 505 | 516 | if __name__ == "__main__": | |
| 506 | 517 | unittest.main() | |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -19,18 +19,42 @@ | |||
| 19 | 19 | ||
| 20 | 20 | Cache = {} | |
| 21 | 21 | ||
| 22 | + | ||
| 23 | + def _warn_if_not_unicode(string): | ||
| 24 | + if version_info[0] < 3 and not isinstance(string, unicode): | ||
| 25 | + warnings.warn( "Argument %r is not an unicode object. " | ||
| 26 | + "Passing an encoded string will likely have " | ||
| 27 | + "unexpected results." % (type(string),), | ||
| 28 | + RuntimeWarning, 2) | ||
| 29 | + | ||
| 30 | + | ||
| 31 | + def unidecode_fast(string): | ||
| 32 | + """ | ||
| 33 | + Try to transliterate using ASCII codec. If it fails, fall back to | ||
| 34 | + transliteration using the character tables. | ||
| 35 | + | ||
| 36 | + This is approx. five times faster if the string only contains ASCII | ||
| 37 | + characters, but sligthly slower than using unidecode directly non-ASCII | ||
| 38 | + chars are present. | ||
| 39 | + """ | ||
| 40 | + _warn_if_not_unicode(string) | ||
| 41 | + try: | ||
| 42 | + bytestring = string.encode('ASCII') | ||
| 43 | + except UnicodeEncodeError: | ||
| 44 | + return unidecode(string) | ||
| 45 | + if version_info[0] >= 3: | ||
| 46 | + return string | ||
| 47 | + else: | ||
| 48 | + return bytestring | ||
| 49 | + | ||
| 50 | + | ||
| 22 | 51 | def unidecode(string): | |
| 23 | 52 | """Transliterate an Unicode object into an ASCII string | |
| 24 | 53 | ||
| 25 | 54 | >>> unidecode(u"\u5317\u4EB0") | |
| 26 | 55 | "Bei Jing " | |
| 27 | 56 | """ | |
| 28 | - | ||
| 29 | - if version_info[0] < 3 and not isinstance(string, unicode): | ||
| 30 | - warnings.warn( "Argument %r is not an unicode object. " | ||
| 31 | - "Passing an encoded string will likely have " | ||
| 32 | - "unexpected results." % (type(string),), | ||
| 33 | - RuntimeWarning, 2) | ||
| 57 | + _warn_if_not_unicode(string) | ||
| 34 | 58 | ||
| 35 | 59 | retval = [] | |
| 36 | 60 | ||
| Back | FazBrowse Home | New Git URL |
0 commit comments