# Note:
# This *is* now explicitly RPython.
# Please make sure not to break this.
# XXX RUSTPYTHON: this was originally from PyPy and has been updated to run on
# Python 3. It's currently in the process of being rewritten
# to a native Rust module in vm/src/stdlib/codecs.rs
"""
_codecs -- Provides access to the codec registry and the builtin
codecs.
This module should never be imported directly. The standard library
module "codecs" wraps this builtin module for use within Python.
The codec registry is accessible via:
register(search_function) -> None
lookup(encoding) -> (encoder, decoder, stream_reader, stream_writer)
The builtin Unicode codecs use the following interface:
_encode(Unicode_object[,errors='strict']) ->
(string object, bytes consumed)
_decode(char_buffer_obj[,errors='strict']) ->
(Unicode object, bytes consumed)
_encode() interfaces also accept non-Unicode object as
input. The objects are then converted to Unicode using
PyUnicode_FromObject() prior to applying the conversion.
These s are available: utf_8, unicode_escape,
raw_unicode_escape, unicode_internal, latin_1, ascii (7-bit),
mbcs (on win32).
Written by Marc-Andre Lemburg (mal@lemburg.com).
Copyright (c) Corporation for National Research Initiatives.
From PyPy v1.0.0
"""
#from unicodecodec import *
__all__ = ['register', 'lookup', 'lookup_error', 'register_error', 'encode', 'decode',
'latin_1_encode', 'mbcs_decode', 'readbuffer_encode', 'escape_encode',
'utf_8_decode', 'raw_unicode_escape_decode', 'utf_7_decode',
'unicode_escape_encode', 'latin_1_decode', 'utf_16_decode',
'unicode_escape_decode', 'ascii_decode', 'charmap_encode', 'charmap_build',
'unicode_internal_encode', 'unicode_internal_decode', 'utf_16_ex_decode',
'escape_decode', 'charmap_decode', 'utf_7_encode', 'mbcs_encode',
'ascii_encode', 'utf_16_encode', 'raw_unicode_escape_encode', 'utf_8_encode',
'utf_16_le_encode', 'utf_16_be_encode', 'utf_16_le_decode', 'utf_16_be_decode',]
import sys
import warnings
from _codecs import *
def latin_1_encode( obj, errors='strict'):
"""None
"""
res = PyUnicode_EncodeLatin1(obj, len(obj), errors)
res = bytes(res)
return res, len(obj)
# XXX MBCS codec might involve ctypes ?
def mbcs_decode():
"""None
"""
pass
def readbuffer_encode( obj, errors='strict'):
"""None
"""
if isinstance(obj, str):
res = obj.encode()
else:
res = bytes(obj)
return res, len(obj)
def escape_encode( obj, errors='strict'):
"""None
"""
if not isinstance(obj, bytes):
raise TypeError("must be bytes")
s = repr(obj).encode()
v = s[2:-1]
if s[1] == ord('"'):
v = v.replace(b"'", b"\\'").replace(b'\\"', b'"')
return v, len(obj)
def raw_unicode_escape_decode( data, errors='strict', final=False):
"""None
"""
res = PyUnicode_DecodeRawUnicodeEscape(data, len(data), errors, final)
res = ''.join(res)
return res, len(data)
def utf_7_decode( data, errors='strict'):
"""None
"""
res = PyUnicode_DecodeUTF7(data, len(data), errors)
res = ''.join(res)
return res, len(data)
def unicode_escape_encode( obj, errors='strict'):
"""None
"""
res = unicodeescape_string(obj, len(obj), 0)
res = b''.join(res)
return res, len(obj)
def latin_1_decode( data, errors='strict'):
"""None
"""
res = PyUnicode_DecodeLatin1(data, len(data), errors)
res = ''.join(res)
return res, len(data)
def utf_16_decode( data, errors='strict', final=False):
"""None
"""
consumed = len(data)
if final:
consumed = 0
res, consumed, byteorder = PyUnicode_DecodeUTF16Stateful(data, len(data), errors, 'native', final)
res = ''.join(res)
return res, consumed
def unicode_escape_decode( data, errors='strict', final=False):
"""None
"""
res = PyUnicode_DecodeUnicodeEscape(data, len(data), errors, final)
res = ''.join(res)
return res, len(data)
def ascii_decode( data, errors='strict'):
"""None
"""
res = PyUnicode_DecodeASCII(data, len(data), errors)
res = ''.join(res)
return res, len(data)
def charmap_encode(obj, errors='strict', mapping='latin-1'):
"""None
"""
res = PyUnicode_EncodeCharmap(obj, len(obj), mapping, errors)
res = bytes(res)
return res, len(obj)
def charmap_build(s):
return {ord(c): i for i, c in enumerate(s)}
if sys.maxunicode == 65535:
unicode_bytes = 2
else:
unicode_bytes = 4
def unicode_internal_encode( obj, errors='strict'):
"""None
"""
if type(obj) == str:
p = bytearray()
t = [ord(x) for x in obj]
for i in t:
b = bytearray()
for j in range(unicode_bytes):
b.append(i%256)
i >>= 8
if sys.byteorder == "big":
b.reverse()
p += b
res = bytes(p)
return res, len(res)
else:
res = "You can do better than this" # XXX make this right
return res, len(res)
def unicode_internal_decode( unistr, errors='strict'):
"""None
"""
if type(unistr) == str:
return unistr, len(unistr)
else:
p = []
i = 0
if sys.byteorder == "big":
start = unicode_bytes - 1
stop = -1
step = -1
else:
start = 0
stop = unicode_bytes
step = 1
while i < len(unistr)-unicode_bytes+1:
t = 0
h = 0
for j in range(start, stop, step):
t += ord(unistr[i+j])= b'A':
return ord(c) - 65
else:
return ord(c) + 4
def ENCODE( ch, bits) :
out = []
while (bits >= 6):
out += B64(ch >> (bits-6))
bits -= 6
return out, bits
def PyUnicode_DecodeUTF7(s, size, errors):
starts = s
errmsg = ""
inShift = 0
bitsleft = 0
charsleft = 0
surrogate = 0
p = []
errorHandler = None
exc = None
if (size == 0):
return ''
i = 0
while i < size:
ch = bytes([s[i]])
if (inShift):
if ((ch == b'-') or not B64CHAR(ch)):
inShift = 0
i += 1
while (bitsleft >= 16):
outCh = ((charsleft) >> (bitsleft-16)) & 0xffff
bitsleft -= 16
if (surrogate):
## We have already generated an error for the high surrogate
## so let's not bother seeing if the low surrogate is correct or not
surrogate = 0
elif (0xDC00 = chr(0x7F)) :
p.append(b'\\x%02x' % ord(ch))
#/* Copy everything else as-is */
else:
p.append(bytes([ord(ch)]))
pos += 1
if (quotes):
p.append(p[0])
return p
def PyUnicode_DecodeASCII(s, size, errors):
# /* ASCII is equivalent to the first 128 ordinals in Unicode. */
if (size == 1 and ord(s) < 128) :
return [chr(ord(s))]
if (size == 0):
return [''] #unicode('')
p = []
pos = 0
while pos < len(s):
c = s[pos]
if c < 128:
p += chr(c)
pos += 1
else:
res = unicode_call_errorhandler(
errors, "ascii", "ordinal not in range(128)",
s, pos, pos+1)
p += res[0]
pos = res[1]
return p
def PyUnicode_EncodeASCII(p, size, errors):
return unicode_encode_ucs1(p, size, errors, 128)
def PyUnicode_AsASCIIString(unistr):
if not type(unistr) == str:
raise TypeError
return PyUnicode_EncodeASCII(str(unistr),
len(str),
None)
def PyUnicode_DecodeUTF16Stateful(s, size, errors, byteorder='native', final=True):
bo = 0 #/* assume native ordering by default */
consumed = 0
errmsg = ""
if sys.byteorder == 'little':
ihi = 1
ilo = 0
else:
ihi = 0
ilo = 1
#/* Unpack UTF-16 encoded data */
## /* Check for BOM marks (U+FEFF) in the input and adjust current
## byte order setting accordingly. In native mode, the leading BOM
## mark is skipped, in all other modes, it is copied to the output
## stream as-is (giving a ZWNBSP character). */
q = 0
p = []
if byteorder == 'native':
if (size >= 2):
bom = (s[ihi]