/*
Unicode implementation based on original code by Fredrik Lundh,
modified by Marc-Andre Lemburg .
Major speed upgrades to the method implementations at the Reykjavik
NeedForSpeed sprint, by Fredrik Lundh and Andrew Dalke.
Copyright (c) Corporation for National Research Initiatives.
--------------------------------------------------------------------
The original string type implementation is:
Copyright (c) 1999 by Secret Labs AB
Copyright (c) 1999 by Fredrik Lundh
By obtaining, using, and/or copying this software and/or its
associated documentation, you agree that you have read, understood,
and will comply with the following terms and conditions:
Permission to use, copy, modify, and distribute this software and its
associated documentation for any purpose and without fee is hereby
granted, provided that the above copyright notice appears in all
copies, and that both that copyright notice and this permission notice
appear in supporting documentation, and that the name of Secret Labs
AB or the author not be used in advertising or publicity pertaining to
distribution of the software without specific, written prior
permission.
SECRET LABS AB AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO
THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
FITNESS. IN NO EVENT SHALL SECRET LABS AB OR THE AUTHOR BE LIABLE FOR
ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT
OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
--------------------------------------------------------------------
*/
#define PY_SSIZE_T_CLEAN
#include "Python.h"
#include "pycore_abstract.h" // _PyIndex_Check()
#include "pycore_bytes_methods.h"
#include "pycore_fileutils.h"
#include "pycore_initconfig.h"
#include "pycore_interp.h" // PyInterpreterState.fs_codec
#include "pycore_object.h"
#include "pycore_pathconfig.h"
#include "pycore_pylifecycle.h"
#include "pycore_pystate.h" // _PyInterpreterState_GET()
#include "ucnhash.h"
#include "stringlib/eq.h"
#ifdef MS_WINDOWS
#include
#endif
#ifdef HAVE_NON_UNICODE_WCHAR_T_REPRESENTATION
#include "pycore_fileutils.h" // _Py_LocaleUsesNonUnicodeWchar()
#endif
/* Uncomment to display statistics on interned strings at exit when
using Valgrind or Insecure++. */
/* #define INTERNED_STATS 1 */
/*[clinic input]
class str "PyObject *" "&PyUnicode_Type"
[clinic start generated code]*/
/*[clinic end generated code: output=da39a3ee5e6b4b0d input=4884c934de622cf6]*/
/*[python input]
class Py_UCS4_converter(CConverter):
type = 'Py_UCS4'
converter = 'convert_uc'
def converter_init(self):
if self.default is not unspecified:
self.c_default = ascii(self.default)
if len(self.c_default) > 4 or self.c_default[0] != "'":
self.c_default = hex(ord(self.default))
[python start generated code]*/
/*[python end generated code: output=da39a3ee5e6b4b0d input=88f5dd06cd8e7a61]*/
/* --- Globals ------------------------------------------------------------
NOTE: In the interpreter's initialization phase, some globals are currently
initialized dynamically as needed. In the process Unicode objects may
be created before the Unicode type is ready.
*/
#ifdef __cplusplus
extern "C" {
#endif
// Maximum code point of Unicode 6.0: 0x10ffff (1,114,111).
// The value must be the same in fileutils.c.
#define MAX_UNICODE 0x10ffff
#ifdef Py_DEBUG
# define _PyUnicode_CHECK(op) _PyUnicode_CheckConsistency(op, 0)
#else
# define _PyUnicode_CHECK(op) PyUnicode_Check(op)
#endif
#define _PyUnicode_UTF8(op) \
(((PyCompactUnicodeObject*)(op))->utf8)
#define PyUnicode_UTF8(op) \
(assert(_PyUnicode_CHECK(op)), \
assert(PyUnicode_IS_READY(op)), \
PyUnicode_IS_COMPACT_ASCII(op) ? \
((char*)((PyASCIIObject*)(op) + 1)) : \
_PyUnicode_UTF8(op))
#define _PyUnicode_UTF8_LENGTH(op) \
(((PyCompactUnicodeObject*)(op))->utf8_length)
#define PyUnicode_UTF8_LENGTH(op) \
(assert(_PyUnicode_CHECK(op)), \
assert(PyUnicode_IS_READY(op)), \
PyUnicode_IS_COMPACT_ASCII(op) ? \
((PyASCIIObject*)(op))->length : \
_PyUnicode_UTF8_LENGTH(op))
#define _PyUnicode_WSTR(op) \
(((PyASCIIObject*)(op))->wstr)
/* Don't use deprecated macro of unicodeobject.h */
#undef PyUnicode_WSTR_LENGTH
#define PyUnicode_WSTR_LENGTH(op) \
(PyUnicode_IS_COMPACT_ASCII(op) ? \
((PyASCIIObject*)op)->length : \
((PyCompactUnicodeObject*)op)->wstr_length)
#define _PyUnicode_WSTR_LENGTH(op) \
(((PyCompactUnicodeObject*)(op))->wstr_length)
#define _PyUnicode_LENGTH(op) \
(((PyASCIIObject *)(op))->length)
#define _PyUnicode_STATE(op) \
(((PyASCIIObject *)(op))->state)
#define _PyUnicode_HASH(op) \
(((PyASCIIObject *)(op))->hash)
#define _PyUnicode_KIND(op) \
(assert(_PyUnicode_CHECK(op)), \
((PyASCIIObject *)(op))->state.kind)
#define _PyUnicode_GET_LENGTH(op) \
(assert(_PyUnicode_CHECK(op)), \
((PyASCIIObject *)(op))->length)
#define _PyUnicode_DATA_ANY(op) \
(((PyUnicodeObject*)(op))->data.any)
#undef PyUnicode_READY
#define PyUnicode_READY(op) \
(assert(_PyUnicode_CHECK(op)), \
(PyUnicode_IS_READY(op) ? \
0 : \
_PyUnicode_Ready(op)))
#define _PyUnicode_SHARE_UTF8(op) \
(assert(_PyUnicode_CHECK(op)), \
assert(!PyUnicode_IS_COMPACT_ASCII(op)), \
(_PyUnicode_UTF8(op) == PyUnicode_DATA(op)))
#define _PyUnicode_SHARE_WSTR(op) \
(assert(_PyUnicode_CHECK(op)), \
(_PyUnicode_WSTR(unicode) == PyUnicode_DATA(op)))
/* true if the Unicode object has an allocated UTF-8 memory block
(not shared with other data) */
#define _PyUnicode_HAS_UTF8_MEMORY(op) \
((!PyUnicode_IS_COMPACT_ASCII(op) \
&& _PyUnicode_UTF8(op) \
&& _PyUnicode_UTF8(op) != PyUnicode_DATA(op)))
/* true if the Unicode object has an allocated wstr memory block
(not shared with other data) */
#define _PyUnicode_HAS_WSTR_MEMORY(op) \
((_PyUnicode_WSTR(op) && \
(!PyUnicode_IS_READY(op) || \
_PyUnicode_WSTR(op) != PyUnicode_DATA(op))))
/* Generic helper macro to convert characters of different types.
from_type and to_type have to be valid type names, begin and end
are pointers to the source characters which should be of type
"from_type *". to is a pointer of type "to_type *" and points to the
buffer where the result characters are written to. */
#define _PyUnicode_CONVERT_BYTES(from_type, to_type, begin, end, to) \
do { \
to_type *_to = (to_type *)(to); \
const from_type *_iter = (const from_type *)(begin);\
const from_type *_end = (const from_type *)(end);\
Py_ssize_t n = (_end) - (_iter); \
const from_type *_unrolled_end = \
_iter + _Py_SIZE_ROUND_DOWN(n, 4); \
while (_iter < (_unrolled_end)) { \
_to[0] = (to_type) _iter[0]; \
_to[1] = (to_type) _iter[1]; \
_to[2] = (to_type) _iter[2]; \
_to[3] = (to_type) _iter[3]; \
_iter += 4; _to += 4; \
} \
while (_iter < (_end)) \
*_to++ = (to_type) *_iter++; \
} while (0)
#ifdef MS_WINDOWS
/* On Windows, overallocate by 50% is the best factor */
# define OVERALLOCATE_FACTOR 2
#else
/* On Linux, overallocate by 25% is the best factor */
# define OVERALLOCATE_FACTOR 4
#endif
#define INTERNED_STRINGS
/* This dictionary holds all interned unicode strings. Note that references
to strings in this dictionary are *not* counted in the string's ob_refcnt.
When the interned string reaches a refcnt of 0 the string deallocation
function will delete the reference from this dictionary.
Another way to look at this is that to say that the actual reference
count of a string is: s->ob_refcnt + (s->state ? 2 : 0)
*/
#ifdef INTERNED_STRINGS
static PyObject *interned = NULL;
#endif
/* The empty Unicode object is shared to improve performance. */
static PyObject *unicode_empty = NULL;
#define _Py_INCREF_UNICODE_EMPTY() \
do { \
if (unicode_empty != NULL) \
Py_INCREF(unicode_empty); \
else { \
unicode_empty = PyUnicode_New(0, 0); \
if (unicode_empty != NULL) { \
Py_INCREF(unicode_empty); \
assert(_PyUnicode_CheckConsistency(unicode_empty, 1)); \
} \
} \
} while (0)
#define _Py_RETURN_UNICODE_EMPTY() \
do { \
_Py_INCREF_UNICODE_EMPTY(); \
return unicode_empty; \
} while (0)
static inline void
unicode_fill(enum PyUnicode_Kind kind, void *data, Py_UCS4 value,
Py_ssize_t start, Py_ssize_t length)
{
assert(0 state.kind;
if (ascii->state.ascii == 1 && ascii->state.compact == 1) {
CHECK(kind == PyUnicode_1BYTE_KIND);
CHECK(ascii->state.ready == 1);
}
else {
PyCompactUnicodeObject *compact = (PyCompactUnicodeObject *)op;
void *data;
if (ascii->state.compact == 1) {
data = compact + 1;
CHECK(kind == PyUnicode_1BYTE_KIND
|| kind == PyUnicode_2BYTE_KIND
|| kind == PyUnicode_4BYTE_KIND);
CHECK(ascii->state.ascii == 0);
CHECK(ascii->state.ready == 1);
CHECK(compact->utf8 != data);
}
else {
PyUnicodeObject *unicode = (PyUnicodeObject *)op;
data = unicode->data.any;
if (kind == PyUnicode_WCHAR_KIND) {
CHECK(ascii->length == 0);
CHECK(ascii->hash == -1);
CHECK(ascii->state.compact == 0);
CHECK(ascii->state.ascii == 0);
CHECK(ascii->state.ready == 0);
CHECK(ascii->state.interned == SSTATE_NOT_INTERNED);
CHECK(ascii->wstr != NULL);
CHECK(data == NULL);
CHECK(compact->utf8 == NULL);
}
else {
CHECK(kind == PyUnicode_1BYTE_KIND
|| kind == PyUnicode_2BYTE_KIND
|| kind == PyUnicode_4BYTE_KIND);
CHECK(ascii->state.compact == 0);
CHECK(ascii->state.ready == 1);
CHECK(data != NULL);
if (ascii->state.ascii) {
CHECK(compact->utf8 == data);
CHECK(compact->utf8_length == ascii->length);
}
else
CHECK(compact->utf8 != data);
}
}
if (kind != PyUnicode_WCHAR_KIND) {
if (
#if SIZEOF_WCHAR_T == 2
kind == PyUnicode_2BYTE_KIND
#else
kind == PyUnicode_4BYTE_KIND
#endif
)
{
CHECK(ascii->wstr == data);
CHECK(compact->wstr_length == ascii->length);
} else
CHECK(ascii->wstr != data);
}
if (compact->utf8 == NULL)
CHECK(compact->utf8_length == 0);
if (ascii->wstr == NULL)
CHECK(compact->wstr_length == 0);
}
/* check that the best kind is used: O(n) operation */
if (check_content && kind != PyUnicode_WCHAR_KIND) {
Py_ssize_t i;
Py_UCS4 maxchar = 0;
const void *data;
Py_UCS4 ch;
data = PyUnicode_DATA(ascii);
for (i=0; i < ascii->length; i++)
{
ch = PyUnicode_READ(kind, data, i);
if (ch > maxchar)
maxchar = ch;
}
if (kind == PyUnicode_1BYTE_KIND) {
if (ascii->state.ascii == 0) {
CHECK(maxchar >= 128);
CHECK(maxchar = 0x100);
CHECK(maxchar = 0x10000);
CHECK(maxchar length) == 0);
}
return 1;
#undef CHECK
}
static PyObject*
unicode_result_wchar(PyObject *unicode)
{
#ifndef Py_DEBUG
Py_ssize_t len;
len = _PyUnicode_WSTR_LENGTH(unicode);
if (len == 0) {
Py_DECREF(unicode);
_Py_RETURN_UNICODE_EMPTY();
}
if (len == 1) {
wchar_t ch = _PyUnicode_WSTR(unicode)[0];
if ((Py_UCS4)ch < 256) {
PyObject *latin1_char = get_latin1_char((unsigned char)ch);
Py_DECREF(unicode);
return latin1_char;
}
}
if (_PyUnicode_Ready(unicode) < 0) {
Py_DECREF(unicode);
return NULL;
}
#else
assert(Py_REFCNT(unicode) == 1);
/* don't make the result ready in debug mode to ensure that the caller
makes the string ready before using it */
assert(_PyUnicode_CheckConsistency(unicode, 1));
#endif
return unicode;
}
static PyObject*
unicode_result_ready(PyObject *unicode)
{
Py_ssize_t length;
length = PyUnicode_GET_LENGTH(unicode);
if (length == 0) {
if (unicode != unicode_empty) {
Py_DECREF(unicode);
_Py_RETURN_UNICODE_EMPTY();
}
return unicode_empty;
}
#ifdef LATIN1_SINGLETONS
if (length == 1) {
const void *data = PyUnicode_DATA(unicode);
int kind = PyUnicode_KIND(unicode);
Py_UCS4 ch = PyUnicode_READ(kind, data, 0);
if (ch < 256) {
PyObject *latin1_char = unicode_latin1[ch];
if (latin1_char != NULL) {
if (unicode != latin1_char) {
Py_INCREF(latin1_char);
Py_DECREF(unicode);
}
return latin1_char;
}
else {
assert(_PyUnicode_CheckConsistency(unicode, 1));
Py_INCREF(unicode);
unicode_latin1[ch] = unicode;
return unicode;
}
}
}
#endif
assert(_PyUnicode_CheckConsistency(unicode, 1));
return unicode;
}
static PyObject*
unicode_result(PyObject *unicode)
{
assert(_PyUnicode_CHECK(unicode));
if (PyUnicode_IS_READY(unicode))
return unicode_result_ready(unicode);
else
return unicode_result_wchar(unicode);
}
static PyObject*
unicode_result_unchanged(PyObject *unicode)
{
if (PyUnicode_CheckExact(unicode)) {
if (PyUnicode_READY(unicode) == -1)
return NULL;
Py_INCREF(unicode);
return unicode;
}
else
/* Subtype -- return genuine unicode string with the same value. */
return _PyUnicode_Copy(unicode);
}
/* Implementation of the "backslashreplace" error handler for 8-bit encodings:
ASCII, Latin1, UTF-8, etc. */
static char*
backslashreplace(_PyBytesWriter *writer, char *str,
PyObject *unicode, Py_ssize_t collstart, Py_ssize_t collend)
{
Py_ssize_t size, i;
Py_UCS4 ch;
enum PyUnicode_Kind kind;
const void *data;
assert(PyUnicode_IS_READY(unicode));
kind = PyUnicode_KIND(unicode);
data = PyUnicode_DATA(unicode);
size = 0;
/* determine replacement size */
for (i = collstart; i < collend; ++i) {
Py_ssize_t incr;
ch = PyUnicode_READ(kind, data, i);
if (ch < 0x100)
incr = 2+2;
else if (ch < 0x10000)
incr = 2+4;
else {
assert(ch PY_SSIZE_T_MAX - incr) {
PyErr_SetString(PyExc_OverflowError,
"encoded result is too long for a Python string");
return NULL;
}
size += incr;
}
str = _PyBytesWriter_Prepare(writer, str, size);
if (str == NULL)
return NULL;
/* generate replacement */
for (i = collstart; i < collend; ++i) {
ch = PyUnicode_READ(kind, data, i);
*str++ = '\\';
if (ch >= 0x00010000) {
*str++ = 'U';
*str++ = Py_hexdigits[(ch>>28)&0xf];
*str++ = Py_hexdigits[(ch>>24)&0xf];
*str++ = Py_hexdigits[(ch>>20)&0xf];
*str++ = Py_hexdigits[(ch>>16)&0xf];
*str++ = Py_hexdigits[(ch>>12)&0xf];
*str++ = Py_hexdigits[(ch>>8)&0xf];
}
else if (ch >= 0x100) {
*str++ = 'u';
*str++ = Py_hexdigits[(ch>>12)&0xf];
*str++ = Py_hexdigits[(ch>>8)&0xf];
}
else
*str++ = 'x';
*str++ = Py_hexdigits[(ch>>4)&0xf];
*str++ = Py_hexdigits[ch&0xf];
}
return str;
}
/* Implementation of the "xmlcharrefreplace" error handler for 8-bit encodings:
ASCII, Latin1, UTF-8, etc. */
static char*
xmlcharrefreplace(_PyBytesWriter *writer, char *str,
PyObject *unicode, Py_ssize_t collstart, Py_ssize_t collend)
{
Py_ssize_t size, i;
Py_UCS4 ch;
enum PyUnicode_Kind kind;
const void *data;
assert(PyUnicode_IS_READY(unicode));
kind = PyUnicode_KIND(unicode);
data = PyUnicode_DATA(unicode);
size = 0;
/* determine replacement size */
for (i = collstart; i < collend; ++i) {
Py_ssize_t incr;
ch = PyUnicode_READ(kind, data, i);
if (ch < 10)
incr = 2+1+1;
else if (ch < 100)
incr = 2+2+1;
else if (ch < 1000)
incr = 2+3+1;
else if (ch < 10000)
incr = 2+4+1;
else if (ch < 100000)
incr = 2+5+1;
else if (ch < 1000000)
incr = 2+6+1;
else {
assert(ch PY_SSIZE_T_MAX - incr) {
PyErr_SetString(PyExc_OverflowError,
"encoded result is too long for a Python string");
return NULL;
}
size += incr;
}
str = _PyBytesWriter_Prepare(writer, str, size);
if (str == NULL)
return NULL;
/* generate replacement */
for (i = collstart; i < collend; ++i) {
size = sprintf(str, "&#%d;", PyUnicode_READ(kind, data, i));
if (size < 0) {
return NULL;
}
str += size;
}
return str;
}
/* --- Bloom Filters ----------------------------------------------------- */
/* stuff to implement simple "bloom filters" for Unicode characters.
to keep things simple, we use a single bitmask, using the least 5
bits from each unicode characters as the bit index. */
/* the linebreak mask is set up by Unicode_Init below */
#if LONG_BIT >= 128
#define BLOOM_WIDTH 128
#elif LONG_BIT >= 64
#define BLOOM_WIDTH 64
#elif LONG_BIT >= 32
#define BLOOM_WIDTH 32
#else
#error "LONG_BIT is smaller than 32"
#endif
#define BLOOM_MASK unsigned long
static BLOOM_MASK bloom_linebreak = ~(BLOOM_MASK)0;
#define BLOOM(mask, ch) ((mask & (1UL 0)
return ucs1lib_find_char((const Py_UCS1 *) s, size, (Py_UCS1) ch);
else
return ucs1lib_rfind_char((const Py_UCS1 *) s, size, (Py_UCS1) ch);
case PyUnicode_2BYTE_KIND:
if ((Py_UCS2) ch != ch)
return -1;
if (direction > 0)
return ucs2lib_find_char((const Py_UCS2 *) s, size, (Py_UCS2) ch);
else
return ucs2lib_rfind_char((const Py_UCS2 *) s, size, (Py_UCS2) ch);
case PyUnicode_4BYTE_KIND:
if (direction > 0)
return ucs4lib_find_char((const Py_UCS4 *) s, size, ch);
else
return ucs4lib_rfind_char((const Py_UCS4 *) s, size, ch);
default:
Py_UNREACHABLE();
}
}
#ifdef Py_DEBUG
/* Fill the data of a Unicode string with invalid characters to detect bugs
earlier.
_PyUnicode_CheckConsistency(str, 1) detects invalid characters, at least for
ASCII and UCS-4 strings. U+00FF is invalid in ASCII and U+FFFFFFFF is an
invalid character in Unicode 6.0. */
static void
unicode_fill_invalid(PyObject *unicode, Py_ssize_t old_length)
{
int kind = PyUnicode_KIND(unicode);
Py_UCS1 *data = PyUnicode_1BYTE_DATA(unicode);
Py_ssize_t length = _PyUnicode_LENGTH(unicode);
if (length ((PY_SSIZE_T_MAX - struct_size) / char_size - 1)) {
PyErr_NoMemory();
return NULL;
}
new_size = (struct_size + (length + 1) * char_size);
if (_PyUnicode_HAS_UTF8_MEMORY(unicode)) {
PyObject_DEL(_PyUnicode_UTF8(unicode));
_PyUnicode_UTF8(unicode) = NULL;
_PyUnicode_UTF8_LENGTH(unicode) = 0;
}
#ifdef Py_REF_DEBUG
_Py_RefTotal--;
#endif
#ifdef Py_TRACE_REFS
_Py_ForgetReference(unicode);
#endif
new_unicode = (PyObject *)PyObject_REALLOC(unicode, new_size);
if (new_unicode == NULL) {
_Py_NewReference(unicode);
PyErr_NoMemory();
return NULL;
}
unicode = new_unicode;
_Py_NewReference(unicode);
_PyUnicode_LENGTH(unicode) = length;
if (share_wstr) {
_PyUnicode_WSTR(unicode) = PyUnicode_DATA(unicode);
if (!PyUnicode_IS_ASCII(unicode))
_PyUnicode_WSTR_LENGTH(unicode) = length;
}
else if (_PyUnicode_HAS_WSTR_MEMORY(unicode)) {
PyObject_DEL(_PyUnicode_WSTR(unicode));
_PyUnicode_WSTR(unicode) = NULL;
if (!PyUnicode_IS_ASCII(unicode))
_PyUnicode_WSTR_LENGTH(unicode) = 0;
}
#ifdef Py_DEBUG
unicode_fill_invalid(unicode, old_length);
#endif
PyUnicode_WRITE(PyUnicode_KIND(unicode), PyUnicode_DATA(unicode),
length, 0);
assert(_PyUnicode_CheckConsistency(unicode, 0));
return unicode;
}
static int
resize_inplace(PyObject *unicode, Py_ssize_t length)
{
wchar_t *wstr;
Py_ssize_t new_size;
assert(!PyUnicode_IS_COMPACT(unicode));
assert(Py_REFCNT(unicode) == 1);
if (PyUnicode_IS_READY(unicode)) {
Py_ssize_t char_size;
int share_wstr, share_utf8;
void *data;
#ifdef Py_DEBUG
Py_ssize_t old_length = _PyUnicode_LENGTH(unicode);
#endif
data = _PyUnicode_DATA_ANY(unicode);
char_size = PyUnicode_KIND(unicode);
share_wstr = _PyUnicode_SHARE_WSTR(unicode);
share_utf8 = _PyUnicode_SHARE_UTF8(unicode);
if (length > (PY_SSIZE_T_MAX / char_size - 1)) {
PyErr_NoMemory();
return -1;
}
new_size = (length + 1) * char_size;
if (!share_utf8 && _PyUnicode_HAS_UTF8_MEMORY(unicode))
{
PyObject_DEL(_PyUnicode_UTF8(unicode));
_PyUnicode_UTF8(unicode) = NULL;
_PyUnicode_UTF8_LENGTH(unicode) = 0;
}
data = (PyObject *)PyObject_REALLOC(data, new_size);
if (data == NULL) {
PyErr_NoMemory();
return -1;
}
_PyUnicode_DATA_ANY(unicode) = data;
if (share_wstr) {
_PyUnicode_WSTR(unicode) = data;
_PyUnicode_WSTR_LENGTH(unicode) = length;
}
if (share_utf8) {
_PyUnicode_UTF8(unicode) = data;
_PyUnicode_UTF8_LENGTH(unicode) = length;
}
_PyUnicode_LENGTH(unicode) = length;
PyUnicode_WRITE(PyUnicode_KIND(unicode), data, length, 0);
#ifdef Py_DEBUG
unicode_fill_invalid(unicode, old_length);
#endif
if (share_wstr || _PyUnicode_WSTR(unicode) == NULL) {
assert(_PyUnicode_CheckConsistency(unicode, 0));
return 0;
}
}
assert(_PyUnicode_WSTR(unicode) != NULL);
/* check for integer overflow */
if (length > PY_SSIZE_T_MAX / (Py_ssize_t)sizeof(wchar_t) - 1) {
PyErr_NoMemory();
return -1;
}
new_size = sizeof(wchar_t) * (length + 1);
wstr = _PyUnicode_WSTR(unicode);
wstr = PyObject_REALLOC(wstr, new_size);
if (!wstr) {
PyErr_NoMemory();
return -1;
}
_PyUnicode_WSTR(unicode) = wstr;
_PyUnicode_WSTR(unicode)[length] = 0;
_PyUnicode_WSTR_LENGTH(unicode) = length;
assert(_PyUnicode_CheckConsistency(unicode, 0));
return 0;
}
static PyObject*
resize_copy(PyObject *unicode, Py_ssize_t length)
{
Py_ssize_t copy_length;
if (_PyUnicode_KIND(unicode) != PyUnicode_WCHAR_KIND) {
PyObject *copy;
assert(PyUnicode_IS_READY(unicode));
copy = PyUnicode_New(length, PyUnicode_MAX_CHAR_VALUE(unicode));
if (copy == NULL)
return NULL;
copy_length = Py_MIN(length, PyUnicode_GET_LENGTH(unicode));
_PyUnicode_FastCopyCharacters(copy, 0, unicode, 0, copy_length);
return copy;
}
else {
PyObject *w;
w = (PyObject*)_PyUnicode_New(length);
if (w == NULL)
return NULL;
copy_length = _PyUnicode_WSTR_LENGTH(unicode);
copy_length = Py_MIN(copy_length, length);
memcpy(_PyUnicode_WSTR(w), _PyUnicode_WSTR(unicode),
copy_length * sizeof(wchar_t));
return w;
}
}
/* We allocate one more byte to make sure the string is
Ux0000 terminated; some code (e.g. new_identifier)
relies on that.
XXX This allocator could further be enhanced by assuring that the
free list never reduces its size below 1.
*/
static PyUnicodeObject *
_PyUnicode_New(Py_ssize_t length)
{
PyUnicodeObject *unicode;
size_t new_size;
/* Optimization for empty strings */
if (length == 0 && unicode_empty != NULL) {
Py_INCREF(unicode_empty);
return (PyUnicodeObject*)unicode_empty;
}
/* Ensure we won't overflow the size. */
if (length > ((PY_SSIZE_T_MAX / (Py_ssize_t)sizeof(Py_UNICODE)) - 1)) {
return (PyUnicodeObject *)PyErr_NoMemory();
}
if (length < 0) {
PyErr_SetString(PyExc_SystemError,
"Negative size passed to _PyUnicode_New");
return NULL;
}
unicode = PyObject_New(PyUnicodeObject, &PyUnicode_Type);
if (unicode == NULL)
return NULL;
new_size = sizeof(Py_UNICODE) * ((size_t)length + 1);
_PyUnicode_WSTR_LENGTH(unicode) = length;
_PyUnicode_HASH(unicode) = -1;
_PyUnicode_STATE(unicode).interned = 0;
_PyUnicode_STATE(unicode).kind = 0;
_PyUnicode_STATE(unicode).compact = 0;
_PyUnicode_STATE(unicode).ready = 0;
_PyUnicode_STATE(unicode).ascii = 0;
_PyUnicode_DATA_ANY(unicode) = NULL;
_PyUnicode_LENGTH(unicode) = 0;
_PyUnicode_UTF8(unicode) = NULL;
_PyUnicode_UTF8_LENGTH(unicode) = 0;
_PyUnicode_WSTR(unicode) = (Py_UNICODE*) PyObject_MALLOC(new_size);
if (!_PyUnicode_WSTR(unicode)) {
Py_DECREF(unicode);
PyErr_NoMemory();
return NULL;
}
/* Initialize the first element to guard against cases where
* the caller fails before initializing str -- unicode_resize()
* reads str[0], and the Keep-Alive optimization can keep memory
* allocated for str alive across a call to unicode_dealloc(unicode).
* We don't want unicode_resize to read uninitialized memory in
* that case.
*/
_PyUnicode_WSTR(unicode)[0] = 0;
_PyUnicode_WSTR(unicode)[length] = 0;
assert(_PyUnicode_CheckConsistency((PyObject *)unicode, 0));
return unicode;
}
static const char*
unicode_kind_name(PyObject *unicode)
{
/* don't check consistency: unicode_kind_name() is called from
_PyUnicode_Dump() */
if (!PyUnicode_IS_COMPACT(unicode))
{
if (!PyUnicode_IS_READY(unicode))
return "wstr";
switch (PyUnicode_KIND(unicode))
{
case PyUnicode_1BYTE_KIND:
if (PyUnicode_IS_ASCII(unicode))
return "legacy ascii";
else
return "legacy latin1";
case PyUnicode_2BYTE_KIND:
return "legacy UCS2";
case PyUnicode_4BYTE_KIND:
return "legacy UCS4";
default:
return "";
}
}
assert(PyUnicode_IS_READY(unicode));
switch (PyUnicode_KIND(unicode)) {
case PyUnicode_1BYTE_KIND:
if (PyUnicode_IS_ASCII(unicode))
return "ascii";
else
return "latin1";
case PyUnicode_2BYTE_KIND:
return "UCS2";
case PyUnicode_4BYTE_KIND:
return "UCS4";
default:
return "";
}
}
#ifdef Py_DEBUG
/* Functions wrapping macros for use in debugger */
const char *_PyUnicode_utf8(void *unicode_raw){
PyObject *unicode = _PyObject_CAST(unicode_raw);
return PyUnicode_UTF8(unicode);
}
const void *_PyUnicode_compact_data(void *unicode_raw) {
PyObject *unicode = _PyObject_CAST(unicode_raw);
return _PyUnicode_COMPACT_DATA(unicode);
}
const void *_PyUnicode_data(void *unicode_raw) {
PyObject *unicode = _PyObject_CAST(unicode_raw);
printf("obj %p\n", (void*)unicode);
printf("compact %d\n", PyUnicode_IS_COMPACT(unicode));
printf("compact ascii %d\n", PyUnicode_IS_COMPACT_ASCII(unicode));
printf("ascii op %p\n", ((void*)((PyASCIIObject*)(unicode) + 1)));
printf("compact op %p\n", ((void*)((PyCompactUnicodeObject*)(unicode) + 1)));
printf("compact data %p\n", _PyUnicode_COMPACT_DATA(unicode));
return PyUnicode_DATA(unicode);
}
void
_PyUnicode_Dump(PyObject *op)
{
PyASCIIObject *ascii = (PyASCIIObject *)op;
PyCompactUnicodeObject *compact = (PyCompactUnicodeObject *)op;
PyUnicodeObject *unicode = (PyUnicodeObject *)op;
const void *data;
if (ascii->state.compact)
{
if (ascii->state.ascii)
data = (ascii + 1);
else
data = (compact + 1);
}
else
data = unicode->data.any;
printf("%s: len=%" PY_FORMAT_SIZE_T "u, ",
unicode_kind_name(op), ascii->length);
if (ascii->wstr == data)
printf("shared ");
printf("wstr=%p", (void *)ascii->wstr);
if (!(ascii->state.ascii == 1 && ascii->state.compact == 1)) {
printf(" (%" PY_FORMAT_SIZE_T "u), ", compact->wstr_length);
if (!ascii->state.compact && compact->utf8 == unicode->data.any)
printf("shared ");
printf("utf8=%p (%" PY_FORMAT_SIZE_T "u)",
(void *)compact->utf8, compact->utf8_length);
}
printf(", data=%p\n", data);
}
#endif
PyObject *
PyUnicode_New(Py_ssize_t size, Py_UCS4 maxchar)
{
PyObject *obj;
PyCompactUnicodeObject *unicode;
void *data;
enum PyUnicode_Kind kind;
int is_sharing, is_ascii;
Py_ssize_t char_size;
Py_ssize_t struct_size;
/* Optimization for empty strings */
if (size == 0 && unicode_empty != NULL) {
Py_INCREF(unicode_empty);
return unicode_empty;
}
is_ascii = 0;
is_sharing = 0;
struct_size = sizeof(PyCompactUnicodeObject);
if (maxchar < 128) {
kind = PyUnicode_1BYTE_KIND;
char_size = 1;
is_ascii = 1;
struct_size = sizeof(PyASCIIObject);
}
else if (maxchar < 256) {
kind = PyUnicode_1BYTE_KIND;
char_size = 1;
}
else if (maxchar < 65536) {
kind = PyUnicode_2BYTE_KIND;
char_size = 2;
if (sizeof(wchar_t) == 2)
is_sharing = 1;
}
else {
if (maxchar > MAX_UNICODE) {
PyErr_SetString(PyExc_SystemError,
"invalid maximum character passed to PyUnicode_New");
return NULL;
}
kind = PyUnicode_4BYTE_KIND;
char_size = 4;
if (sizeof(wchar_t) == 4)
is_sharing = 1;
}
/* Ensure we won't overflow the size. */
if (size < 0) {
PyErr_SetString(PyExc_SystemError,
"Negative size passed to PyUnicode_New");
return NULL;
}
if (size > ((PY_SSIZE_T_MAX - struct_size) / char_size - 1))
return PyErr_NoMemory();
/* Duplicated allocation code from _PyObject_New() instead of a call to
* PyObject_New() so we are able to allocate space for the object and
* it's data buffer.
*/
obj = (PyObject *) PyObject_MALLOC(struct_size + (size + 1) * char_size);
if (obj == NULL)
return PyErr_NoMemory();
obj = PyObject_INIT(obj, &PyUnicode_Type);
if (obj == NULL)
return NULL;
unicode = (PyCompactUnicodeObject *)obj;
if (is_ascii)
data = ((PyASCIIObject*)obj) + 1;
else
data = unicode + 1;
_PyUnicode_LENGTH(unicode) = size;
_PyUnicode_HASH(unicode) = -1;
_PyUnicode_STATE(unicode).interned = 0;
_PyUnicode_STATE(unicode).kind = kind;
_PyUnicode_STATE(unicode).compact = 1;
_PyUnicode_STATE(unicode).ready = 1;
_PyUnicode_STATE(unicode).ascii = is_ascii;
if (is_ascii) {
((char*)data)[size] = 0;
_PyUnicode_WSTR(unicode) = NULL;
}
else if (kind == PyUnicode_1BYTE_KIND) {
((char*)data)[size] = 0;
_PyUnicode_WSTR(unicode) = NULL;
_PyUnicode_WSTR_LENGTH(unicode) = 0;
unicode->utf8 = NULL;
unicode->utf8_length = 0;
}
else {
unicode->utf8 = NULL;
unicode->utf8_length = 0;
if (kind == PyUnicode_2BYTE_KIND)
((Py_UCS2*)data)[size] = 0;
else /* kind == PyUnicode_4BYTE_KIND */
((Py_UCS4*)data)[size] = 0;
if (is_sharing) {
_PyUnicode_WSTR_LENGTH(unicode) = size;
_PyUnicode_WSTR(unicode) = (wchar_t *)data;
}
else {
_PyUnicode_WSTR_LENGTH(unicode) = 0;
_PyUnicode_WSTR(unicode) = NULL;
}
}
#ifdef Py_DEBUG
unicode_fill_invalid((PyObject*)unicode, 0);
#endif
assert(_PyUnicode_CheckConsistency((PyObject*)unicode, 0));
return obj;
}
#if SIZEOF_WCHAR_T == 2
/* Helper function to convert a 16-bits wchar_t representation to UCS4, this
will decode surrogate pairs, the other conversions are implemented as macros
for efficiency.
This function assumes that unicode can hold one more code point than wstr
characters for a terminating null character. */
static void
unicode_convert_wchar_to_ucs4(const wchar_t *begin, const wchar_t *end,
PyObject *unicode)
{
const wchar_t *iter;
Py_UCS4 *ucs4_out;
assert(unicode != NULL);
assert(_PyUnicode_CHECK(unicode));
assert(_PyUnicode_KIND(unicode) == PyUnicode_4BYTE_KIND);
ucs4_out = PyUnicode_4BYTE_DATA(unicode);
for (iter = begin; iter < end; ) {
assert(ucs4_out < (PyUnicode_4BYTE_DATA(unicode) +
_PyUnicode_GET_LENGTH(unicode)));
if (Py_UNICODE_IS_HIGH_SURROGATE(iter[0])
&& (iter+1) < end
&& Py_UNICODE_IS_LOW_SURROGATE(iter[1]))
{
*ucs4_out++ = Py_UNICODE_JOIN_SURROGATES(iter[0], iter[1]);
iter += 2;
}
else {
*ucs4_out++ = *iter;
iter++;
}
}
assert(ucs4_out == (PyUnicode_4BYTE_DATA(unicode) +
_PyUnicode_GET_LENGTH(unicode)));
}
#endif
static int
unicode_check_modifiable(PyObject *unicode)
{
if (!unicode_modifiable(unicode)) {
PyErr_SetString(PyExc_SystemError,
"Cannot modify a string currently used");
return -1;
}
return 0;
}
static int
_copy_characters(PyObject *to, Py_ssize_t to_start,
PyObject *from, Py_ssize_t from_start,
Py_ssize_t how_many, int check_maxchar)
{
unsigned int from_kind, to_kind;
const void *from_data;
void *to_data;
assert(0 (size_t)PyUnicode_GET_LENGTH(to)) {
PyErr_SetString(PyExc_IndexError, "string index out of range");
return -1;
}
if (how_many < 0) {
PyErr_SetString(PyExc_SystemError, "how_many cannot be negative");
return -1;
}
how_many = Py_MIN(PyUnicode_GET_LENGTH(from)-from_start, how_many);
if (to_start + how_many > PyUnicode_GET_LENGTH(to)) {
PyErr_Format(PyExc_SystemError,
"Cannot write %zi characters at %zi "
"in a string of %zi characters",
how_many, to_start, PyUnicode_GET_LENGTH(to));
return -1;
}
if (how_many == 0)
return 0;
if (unicode_check_modifiable(to))
return -1;
err = _copy_characters(to, to_start, from, from_start, how_many, 1);
if (err) {
PyErr_Format(PyExc_SystemError,
"Cannot copy %s characters "
"into a string of %s characters",
unicode_kind_name(from),
unicode_kind_name(to));
return -1;
}
return how_many;
}
/* Find the maximum code point and count the number of surrogate pairs so a
correct string length can be computed before converting a string to UCS4.
This function counts single surrogates as a character and not as a pair.
Return 0 on success, or -1 on error. */
static int
find_maxchar_surrogates(const wchar_t *begin, const wchar_t *end,
Py_UCS4 *maxchar, Py_ssize_t *num_surrogates)
{
const wchar_t *iter;
Py_UCS4 ch;
assert(num_surrogates != NULL && maxchar != NULL);
*num_surrogates = 0;
*maxchar = 0;
for (iter = begin; iter < end; ) {
#if SIZEOF_WCHAR_T == 2
if (Py_UNICODE_IS_HIGH_SURROGATE(iter[0])
&& (iter+1) < end
&& Py_UNICODE_IS_LOW_SURROGATE(iter[1]))
{
ch = Py_UNICODE_JOIN_SURROGATES(iter[0], iter[1]);
++(*num_surrogates);
iter += 2;
}
else
#endif
{
ch = *iter;
iter++;
}
if (ch > *maxchar) {
*maxchar = ch;
if (*maxchar > MAX_UNICODE) {
PyErr_Format(PyExc_ValueError,
"character U+%x is not in range [U+0000; U+%x]",
ch, MAX_UNICODE);
return -1;
}
}
}
return 0;
}
int
_PyUnicode_Ready(PyObject *unicode)
{
wchar_t *end;
Py_UCS4 maxchar = 0;
Py_ssize_t num_surrogates;
#if SIZEOF_WCHAR_T == 2
Py_ssize_t length_wo_surrogates;
#endif
/* _PyUnicode_Ready() is only intended for old-style API usage where
strings were created using _PyObject_New() and where no canonical
representation (the str field) has been set yet aka strings
which are not yet ready. */
assert(_PyUnicode_CHECK(unicode));
assert(_PyUnicode_KIND(unicode) == PyUnicode_WCHAR_KIND);
assert(_PyUnicode_WSTR(unicode) != NULL);
assert(_PyUnicode_DATA_ANY(unicode) == NULL);
assert(_PyUnicode_UTF8(unicode) == NULL);
/* Actually, it should neither be interned nor be anything else: */
assert(_PyUnicode_STATE(unicode).interned == SSTATE_NOT_INTERNED);
end = _PyUnicode_WSTR(unicode) + _PyUnicode_WSTR_LENGTH(unicode);
if (find_maxchar_surrogates(_PyUnicode_WSTR(unicode), end,
&maxchar, &num_surrogates) == -1)
return -1;
if (maxchar < 256) {
_PyUnicode_DATA_ANY(unicode) = PyObject_MALLOC(_PyUnicode_WSTR_LENGTH(unicode) + 1);
if (!_PyUnicode_DATA_ANY(unicode)) {
PyErr_NoMemory();
return -1;
}
_PyUnicode_CONVERT_BYTES(wchar_t, unsigned char,
_PyUnicode_WSTR(unicode), end,
PyUnicode_1BYTE_DATA(unicode));
PyUnicode_1BYTE_DATA(unicode)[_PyUnicode_WSTR_LENGTH(unicode)] = '\0';
_PyUnicode_LENGTH(unicode) = _PyUnicode_WSTR_LENGTH(unicode);
_PyUnicode_STATE(unicode).kind = PyUnicode_1BYTE_KIND;
if (maxchar < 128) {
_PyUnicode_STATE(unicode).ascii = 1;
_PyUnicode_UTF8(unicode) = _PyUnicode_DATA_ANY(unicode);
_PyUnicode_UTF8_LENGTH(unicode) = _PyUnicode_WSTR_LENGTH(unicode);
}
else {
_PyUnicode_STATE(unicode).ascii = 0;
_PyUnicode_UTF8(unicode) = NULL;
_PyUnicode_UTF8_LENGTH(unicode) = 0;
}
PyObject_FREE(_PyUnicode_WSTR(unicode));
_PyUnicode_WSTR(unicode) = NULL;
_PyUnicode_WSTR_LENGTH(unicode) = 0;
}
/* In this case we might have to convert down from 4-byte native
wchar_t to 2-byte unicode. */
else if (maxchar < 65536) {
assert(num_surrogates == 0 &&
"FindMaxCharAndNumSurrogatePairs() messed up");
#if SIZEOF_WCHAR_T == 2
/* We can share representations and are done. */
_PyUnicode_DATA_ANY(unicode) = _PyUnicode_WSTR(unicode);
PyUnicode_2BYTE_DATA(unicode)[_PyUnicode_WSTR_LENGTH(unicode)] = '\0';
_PyUnicode_LENGTH(unicode) = _PyUnicode_WSTR_LENGTH(unicode);
_PyUnicode_STATE(unicode).kind = PyUnicode_2BYTE_KIND;
_PyUnicode_UTF8(unicode) = NULL;
_PyUnicode_UTF8_LENGTH(unicode) = 0;
#else
/* sizeof(wchar_t) == 4 */
_PyUnicode_DATA_ANY(unicode) = PyObject_MALLOC(
2 * (_PyUnicode_WSTR_LENGTH(unicode) + 1));
if (!_PyUnicode_DATA_ANY(unicode)) {
PyErr_NoMemory();
return -1;
}
_PyUnicode_CONVERT_BYTES(wchar_t, Py_UCS2,
_PyUnicode_WSTR(unicode), end,
PyUnicode_2BYTE_DATA(unicode));
PyUnicode_2BYTE_DATA(unicode)[_PyUnicode_WSTR_LENGTH(unicode)] = '\0';
_PyUnicode_LENGTH(unicode) = _PyUnicode_WSTR_LENGTH(unicode);
_PyUnicode_STATE(unicode).kind = PyUnicode_2BYTE_KIND;
_PyUnicode_UTF8(unicode) = NULL;
_PyUnicode_UTF8_LENGTH(unicode) = 0;
PyObject_FREE(_PyUnicode_WSTR(unicode));
_PyUnicode_WSTR(unicode) = NULL;
_PyUnicode_WSTR_LENGTH(unicode) = 0;
#endif
}
/* maxchar exceeds 16 bit, wee need 4 bytes for unicode characters */
else {
#if SIZEOF_WCHAR_T == 2
/* in case the native representation is 2-bytes, we need to allocate a
new normalized 4-byte version. */
length_wo_surrogates = _PyUnicode_WSTR_LENGTH(unicode) - num_surrogates;
if (length_wo_surrogates > PY_SSIZE_T_MAX / 4 - 1) {
PyErr_NoMemory();
return -1;
}
_PyUnicode_DATA_ANY(unicode) = PyObject_MALLOC(4 * (length_wo_surrogates + 1));
if (!_PyUnicode_DATA_ANY(unicode)) {
PyErr_NoMemory();
return -1;
}
_PyUnicode_LENGTH(unicode) = length_wo_surrogates;
_PyUnicode_STATE(unicode).kind = PyUnicode_4BYTE_KIND;
_PyUnicode_UTF8(unicode) = NULL;
_PyUnicode_UTF8_LENGTH(unicode) = 0;
/* unicode_convert_wchar_to_ucs4() requires a ready string */
_PyUnicode_STATE(unicode).ready = 1;
unicode_convert_wchar_to_ucs4(_PyUnicode_WSTR(unicode), end, unicode);
PyObject_FREE(_PyUnicode_WSTR(unicode));
_PyUnicode_WSTR(unicode) = NULL;
_PyUnicode_WSTR_LENGTH(unicode) = 0;
#else
assert(num_surrogates == 0);
_PyUnicode_DATA_ANY(unicode) = _PyUnicode_WSTR(unicode);
_PyUnicode_LENGTH(unicode) = _PyUnicode_WSTR_LENGTH(unicode);
_PyUnicode_UTF8(unicode) = NULL;
_PyUnicode_UTF8_LENGTH(unicode) = 0;
_PyUnicode_STATE(unicode).kind = PyUnicode_4BYTE_KIND;
#endif
PyUnicode_4BYTE_DATA(unicode)[_PyUnicode_LENGTH(unicode)] = '\0';
}
_PyUnicode_STATE(unicode).ready = 1;
assert(_PyUnicode_CheckConsistency(unicode, 1));
return 0;
}
static void
unicode_dealloc(PyObject *unicode)
{
switch (PyUnicode_CHECK_INTERNED(unicode)) {
case SSTATE_NOT_INTERNED:
break;
case SSTATE_INTERNED_MORTAL:
/* revive dead object temporarily for DelItem */
Py_SET_REFCNT(unicode, 3);
#ifdef INTERNED_STRINGS
if (PyDict_DelItem(interned, unicode) != 0) {
_PyErr_WriteUnraisableMsg("deletion of interned string failed",
NULL);
}
#endif
break;
case SSTATE_INTERNED_IMMORTAL:
_PyObject_ASSERT_FAILED_MSG(unicode, "Immortal interned string died");
break;
default:
Py_UNREACHABLE();
}
if (_PyUnicode_HAS_WSTR_MEMORY(unicode)) {
PyObject_DEL(_PyUnicode_WSTR(unicode));
}
if (_PyUnicode_HAS_UTF8_MEMORY(unicode)) {
PyObject_DEL(_PyUnicode_UTF8(unicode));
}
if (!PyUnicode_IS_COMPACT(unicode) && _PyUnicode_DATA_ANY(unicode)) {
PyObject_DEL(_PyUnicode_DATA_ANY(unicode));
}
Py_TYPE(unicode)->tp_free(unicode);
}
#ifdef Py_DEBUG
static int
unicode_is_singleton(PyObject *unicode)
{
if (unicode == unicode_empty) {
return 1;
}
#ifdef LATIN1_SINGLETONS
PyASCIIObject *ascii = (PyASCIIObject *)unicode;
if (ascii->state.kind != PyUnicode_WCHAR_KIND && ascii->length == 1)
{
Py_UCS4 ch = PyUnicode_READ_CHAR(unicode, 0);
if (ch < 256 && unicode_latin1[ch] == unicode)
return 1;
}
#endif
return 0;
}
#endif
static int
unicode_modifiable(PyObject *unicode)
{
assert(_PyUnicode_CHECK(unicode));
if (Py_REFCNT(unicode) != 1)
return 0;
if (_PyUnicode_HASH(unicode) != -1)
return 0;
if (PyUnicode_CHECK_INTERNED(unicode))
return 0;
if (!PyUnicode_CheckExact(unicode))
return 0;
#ifdef Py_DEBUG
/* singleton refcount is greater than 1 */
assert(!unicode_is_singleton(unicode));
#endif
return 1;
}
static int
unicode_resize(PyObject **p_unicode, Py_ssize_t length)
{
PyObject *unicode;
Py_ssize_t old_length;
assert(p_unicode != NULL);
unicode = *p_unicode;
assert(unicode != NULL);
assert(PyUnicode_Check(unicode));
assert(0 string),
NULL, NULL);
if (!id->object)
return NULL;
PyUnicode_InternInPlace(&id->object);
assert(!id->next);
id->next = static_strings;
static_strings = id;
}
return id->object;
}
static void
unicode_clear_static_strings(void)
{
_Py_Identifier *tmp, *s = static_strings;
while (s) {
Py_CLEAR(s->object);
tmp = s->next;
s->next = NULL;
s = tmp;
}
static_strings = NULL;
}
/* Internal function, doesn't check maximum character */
PyObject*
_PyUnicode_FromASCII(const char *buffer, Py_ssize_t size)
{
const unsigned char *s = (const unsigned char *)buffer;
PyObject *unicode;
if (size == 1) {
#ifdef Py_DEBUG
assert((unsigned char)s[0] < 128);
#endif
return get_latin1_char(s[0]);
}
unicode = PyUnicode_New(size, 127);
if (!unicode)
return NULL;
memcpy(PyUnicode_1BYTE_DATA(unicode), s, size);
assert(_PyUnicode_CheckConsistency(unicode, 1));
return unicode;
}
static Py_UCS4
kind_maxchar_limit(unsigned int kind)
{
switch (kind) {
case PyUnicode_1BYTE_KIND:
return 0x80;
case PyUnicode_2BYTE_KIND:
return 0x100;
case PyUnicode_4BYTE_KIND:
return 0x10000;
default:
Py_UNREACHABLE();
}
}
static PyObject*
_PyUnicode_FromUCS1(const Py_UCS1* u, Py_ssize_t size)
{
PyObject *res;
unsigned char max_char;
if (size == 0)
_Py_RETURN_UNICODE_EMPTY();
assert(size > 0);
if (size == 1)
return get_latin1_char(u[0]);
max_char = ucs1lib_find_max_char(u, u + size);
res = PyUnicode_New(size, max_char);
if (!res)
return NULL;
memcpy(PyUnicode_1BYTE_DATA(res), u, size);
assert(_PyUnicode_CheckConsistency(res, 1));
return res;
}
static PyObject*
_PyUnicode_FromUCS2(const Py_UCS2 *u, Py_ssize_t size)
{
PyObject *res;
Py_UCS2 max_char;
if (size == 0)
_Py_RETURN_UNICODE_EMPTY();
assert(size > 0);
if (size == 1)
return unicode_char(u[0]);
max_char = ucs2lib_find_max_char(u, u + size);
res = PyUnicode_New(size, max_char);
if (!res)
return NULL;
if (max_char >= 256)
memcpy(PyUnicode_2BYTE_DATA(res), u, sizeof(Py_UCS2)*size);
else {
_PyUnicode_CONVERT_BYTES(
Py_UCS2, Py_UCS1, u, u + size, PyUnicode_1BYTE_DATA(res));
}
assert(_PyUnicode_CheckConsistency(res, 1));
return res;
}
static PyObject*
_PyUnicode_FromUCS4(const Py_UCS4 *u, Py_ssize_t size)
{
PyObject *res;
Py_UCS4 max_char;
if (size == 0)
_Py_RETURN_UNICODE_EMPTY();
assert(size > 0);
if (size == 1)
return unicode_char(u[0]);
max_char = ucs4lib_find_max_char(u, u + size);
res = PyUnicode_New(size, max_char);
if (!res)
return NULL;
if (max_char < 256)
_PyUnicode_CONVERT_BYTES(Py_UCS4, Py_UCS1, u, u + size,
PyUnicode_1BYTE_DATA(res));
else if (max_char < 0x10000)
_PyUnicode_CONVERT_BYTES(Py_UCS4, Py_UCS2, u, u + size,
PyUnicode_2BYTE_DATA(res));
else
memcpy(PyUnicode_4BYTE_DATA(res), u, sizeof(Py_UCS4)*size);
assert(_PyUnicode_CheckConsistency(res, 1));
return res;
}
PyObject*
PyUnicode_FromKindAndData(int kind, const void *buffer, Py_ssize_t size)
{
if (size < 0) {
PyErr_SetString(PyExc_ValueError, "size must be positive");
return NULL;
}
switch (kind) {
case PyUnicode_1BYTE_KIND:
return _PyUnicode_FromUCS1(buffer, size);
case PyUnicode_2BYTE_KIND:
return _PyUnicode_FromUCS2(buffer, size);
case PyUnicode_4BYTE_KIND:
return _PyUnicode_FromUCS4(buffer, size);
default:
PyErr_SetString(PyExc_SystemError, "invalid kind");
return NULL;
}
}
Py_UCS4
_PyUnicode_FindMaxChar(PyObject *unicode, Py_ssize_t start, Py_ssize_t end)
{
enum PyUnicode_Kind kind;
const void *startptr, *endptr;
assert(PyUnicode_IS_READY(unicode));
assert(0 = 0x10000)
return;
}
else
Py_UNREACHABLE();
copy = PyUnicode_New(len, max_char);
if (copy != NULL)
_PyUnicode_FastCopyCharacters(copy, 0, unicode, 0, len);
Py_DECREF(unicode);
*p_unicode = copy;
}
PyObject*
_PyUnicode_Copy(PyObject *unicode)
{
Py_ssize_t length;
PyObject *copy;
if (!PyUnicode_Check(unicode)) {
PyErr_BadInternalCall();
return NULL;
}
if (PyUnicode_READY(unicode) == -1)
return NULL;
length = PyUnicode_GET_LENGTH(unicode);
copy = PyUnicode_New(length, PyUnicode_MAX_CHAR_VALUE(unicode));
if (!copy)
return NULL;
assert(PyUnicode_KIND(copy) == PyUnicode_KIND(unicode));
memcpy(PyUnicode_DATA(copy), PyUnicode_DATA(unicode),
length * PyUnicode_KIND(unicode));
assert(_PyUnicode_CheckConsistency(copy, 1));
return copy;
}
/* Widen Unicode objects to larger buffers. Don't write terminating null
character. Return NULL on error. */
static void*
unicode_askind(unsigned int skind, void const *data, Py_ssize_t len, unsigned int kind)
{
void *result;
assert(skind < kind);
switch (kind) {
case PyUnicode_2BYTE_KIND:
result = PyMem_New(Py_UCS2, len);
if (!result)
return PyErr_NoMemory();
assert(skind == PyUnicode_1BYTE_KIND);
_PyUnicode_CONVERT_BYTES(
Py_UCS1, Py_UCS2,
(const Py_UCS1 *)data,
((const Py_UCS1 *)data) + len,
result);
return result;
case PyUnicode_4BYTE_KIND:
result = PyMem_New(Py_UCS4, len);
if (!result)
return PyErr_NoMemory();
if (skind == PyUnicode_2BYTE_KIND) {
_PyUnicode_CONVERT_BYTES(
Py_UCS2, Py_UCS4,
(const Py_UCS2 *)data,
((const Py_UCS2 *)data) + len,
result);
}
else {
assert(skind == PyUnicode_1BYTE_KIND);
_PyUnicode_CONVERT_BYTES(
Py_UCS1, Py_UCS4,
(const Py_UCS1 *)data,
((const Py_UCS1 *)data) + len,
result);
}
return result;
default:
Py_UNREACHABLE();
return NULL;
}
}
static Py_UCS4*
as_ucs4(PyObject *string, Py_UCS4 *target, Py_ssize_t targetsize,
int copy_null)
{
int kind;
const void *data;
Py_ssize_t len, targetlen;
if (PyUnicode_READY(string) == -1)
return NULL;
kind = PyUnicode_KIND(string);
data = PyUnicode_DATA(string);
len = PyUnicode_GET_LENGTH(string);
targetlen = len;
if (copy_null)
targetlen++;
if (!target) {
target = PyMem_New(Py_UCS4, targetlen);
if (!target) {
PyErr_NoMemory();
return NULL;
}
}
else {
if (targetsize < targetlen) {
PyErr_Format(PyExc_SystemError,
"string is longer than the buffer");
if (copy_null && 0 < targetsize)
target[0] = 0;
return NULL;
}
}
if (kind == PyUnicode_1BYTE_KIND) {
const Py_UCS1 *start = (const Py_UCS1 *) data;
_PyUnicode_CONVERT_BYTES(Py_UCS1, Py_UCS4, start, start + len, target);
}
else if (kind == PyUnicode_2BYTE_KIND) {
const Py_UCS2 *start = (const Py_UCS2 *) data;
_PyUnicode_CONVERT_BYTES(Py_UCS2, Py_UCS4, start, start + len, target);
}
else if (kind == PyUnicode_4BYTE_KIND) {
memcpy(target, data, len * sizeof(Py_UCS4));
}
else {
Py_UNREACHABLE();
}
if (copy_null)
target[len] = 0;
return target;
}
Py_UCS4*
PyUnicode_AsUCS4(PyObject *string, Py_UCS4 *target, Py_ssize_t targetsize,
int copy_null)
{
if (target == NULL || targetsize < 0) {
PyErr_BadInternalCall();
return NULL;
}
return as_ucs4(string, target, targetsize, copy_null);
}
Py_UCS4*
PyUnicode_AsUCS4Copy(PyObject *string)
{
return as_ucs4(string, NULL, 0, 1);
}
/* maximum number of characters required for output of %lld or %p.
We need at most ceil(log10(256)*SIZEOF_LONG_LONG) digits,
plus 1 for the sign. 53/22 is an upper bound for log10(256). */
#define MAX_LONG_LONG_CHARS (2 + (SIZEOF_LONG_LONG*53-1) / 22)
static int
unicode_fromformat_write_str(_PyUnicodeWriter *writer, PyObject *str,
Py_ssize_t width, Py_ssize_t precision)
{
Py_ssize_t length, fill, arglen;
Py_UCS4 maxchar;
if (PyUnicode_READY(str) == -1)
return -1;
length = PyUnicode_GET_LENGTH(str);
if ((precision == -1 || precision >= length)
&& width writer->maxchar)
maxchar = _PyUnicode_FindMaxChar(str, 0, length);
else
maxchar = writer->maxchar;
if (_PyUnicodeWriter_Prepare(writer, arglen, maxchar) == -1)
return -1;
if (width > length) {
fill = width - length;
if (PyUnicode_Fill(writer->buffer, writer->pos, fill, ' ') == -1)
return -1;
writer->pos += fill;
}
_PyUnicode_FastCopyCharacters(writer->buffer, writer->pos,
str, 0, length);
writer->pos += length;
return 0;
}
static int
unicode_fromformat_write_cstr(_PyUnicodeWriter *writer, const char *str,
Py_ssize_t width, Py_ssize_t precision)
{
/* UTF-8 */
Py_ssize_t length;
PyObject *unicode;
int res;
if (precision == -1) {
length = strlen(str);
}
else {
length = 0;
while (length < precision && str[length]) {
length++;
}
}
unicode = PyUnicode_DecodeUTF8Stateful(str, length, "replace", NULL);
if (unicode == NULL)
return -1;
res = unicode_fromformat_write_str(writer, unicode, width, -1);
Py_DECREF(unicode);
return res;
}
static const char*
unicode_fromformat_arg(_PyUnicodeWriter *writer,
const char *f, va_list *vargs)
{
const char *p;
Py_ssize_t len;
int zeropad;
Py_ssize_t width;
Py_ssize_t precision;
int longflag;
int longlongflag;
int size_tflag;
Py_ssize_t fill;
p = f;
f++;
zeropad = 0;
if (*f == '0') {
zeropad = 1;
f++;
}
/* parse the width.precision part, e.g. "%2.5s" => width=2, precision=5 */
width = -1;
if (Py_ISDIGIT((unsigned)*f)) {
width = *f - '0';
f++;
while (Py_ISDIGIT((unsigned)*f)) {
if (width > (PY_SSIZE_T_MAX - ((int)*f - '0')) / 10) {
PyErr_SetString(PyExc_ValueError,
"width too big");
return NULL;
}
width = (width * 10) + (*f - '0');
f++;
}
}
precision = -1;
if (*f == '.') {
f++;
if (Py_ISDIGIT((unsigned)*f)) {
precision = (*f - '0');
f++;
while (Py_ISDIGIT((unsigned)*f)) {
if (precision > (PY_SSIZE_T_MAX - ((int)*f - '0')) / 10) {
PyErr_SetString(PyExc_ValueError,
"precision too big");
return NULL;
}
precision = (precision * 10) + (*f - '0');
f++;
}
}
if (*f == '%') {
/* "%.3%s" => f points to "3" */
f--;
}
}
if (*f == '\0') {
/* bogus format "%.123" => go backward, f points to "3" */
f--;
}
/* Handle %ld, %lu, %lld and %llu. */
longflag = 0;
longlongflag = 0;
size_tflag = 0;
if (*f == 'l') {
if (f[1] == 'd' || f[1] == 'u' || f[1] == 'i') {
longflag = 1;
++f;
}
else if (f[1] == 'l' &&
(f[2] == 'd' || f[2] == 'u' || f[2] == 'i')) {
longlongflag = 1;
f += 2;
}
}
/* handle the size_t flag. */
else if (*f == 'z' && (f[1] == 'd' || f[1] == 'u' || f[1] == 'i')) {
size_tflag = 1;
++f;
}
if (f[1] == '\0')
writer->overallocate = 0;
switch (*f) {
case 'c':
{
int ordinal = va_arg(*vargs, int);
if (ordinal < 0 || ordinal > MAX_UNICODE) {
PyErr_SetString(PyExc_OverflowError,
"character argument not in range(0x110000)");
return NULL;
}
if (_PyUnicodeWriter_WriteCharInline(writer, ordinal) < 0)
return NULL;
break;
}
case 'i':
case 'd':
case 'u':
case 'x':
{
/* used by sprintf */
char buffer[MAX_LONG_LONG_CHARS];
Py_ssize_t arglen;
if (*f == 'u') {
if (longflag)
len = sprintf(buffer, "%lu",
va_arg(*vargs, unsigned long));
else if (longlongflag)
len = sprintf(buffer, "%llu",
va_arg(*vargs, unsigned long long));
else if (size_tflag)
len = sprintf(buffer, "%" PY_FORMAT_SIZE_T "u",
va_arg(*vargs, size_t));
else
len = sprintf(buffer, "%u",
va_arg(*vargs, unsigned int));
}
else if (*f == 'x') {
len = sprintf(buffer, "%x", va_arg(*vargs, int));
}
else {
if (longflag)
len = sprintf(buffer, "%li",
va_arg(*vargs, long));
else if (longlongflag)
len = sprintf(buffer, "%lli",
va_arg(*vargs, long long));
else if (size_tflag)
len = sprintf(buffer, "%" PY_FORMAT_SIZE_T "i",
va_arg(*vargs, Py_ssize_t));
else
len = sprintf(buffer, "%i",
va_arg(*vargs, int));
}
assert(len >= 0);
if (precision < len)
precision = len;
arglen = Py_MAX(precision, width);
if (_PyUnicodeWriter_Prepare(writer, arglen, 127) == -1)
return NULL;
if (width > precision) {
Py_UCS4 fillchar;
fill = width - precision;
fillchar = zeropad?'0':' ';
if (PyUnicode_Fill(writer->buffer, writer->pos, fill, fillchar) == -1)
return NULL;
writer->pos += fill;
}
if (precision > len) {
fill = precision - len;
if (PyUnicode_Fill(writer->buffer, writer->pos, fill, '0') == -1)
return NULL;
writer->pos += fill;
}
if (_PyUnicodeWriter_WriteASCIIString(writer, buffer, len) < 0)
return NULL;
break;
}
case 'p':
{
char number[MAX_LONG_LONG_CHARS];
len = sprintf(number, "%p", va_arg(*vargs, void*));
assert(len >= 0);
/* %p is ill-defined: ensure leading 0x. */
if (number[1] == 'X')
number[1] = 'x';
else if (number[1] != 'x') {
memmove(number + 2, number,
strlen(number) + 1);
number[0] = '0';
number[1] = 'x';
len += 2;
}
if (_PyUnicodeWriter_WriteASCIIString(writer, number, len) < 0)
return NULL;
break;
}
case 's':
{
/* UTF-8 */
const char *s = va_arg(*vargs, const char*);
if (unicode_fromformat_write_cstr(writer, s, width, precision) < 0)
return NULL;
break;
}
case 'U':
{
PyObject *obj = va_arg(*vargs, PyObject *);
assert(obj && _PyUnicode_CHECK(obj));
if (unicode_fromformat_write_str(writer, obj, width, precision) == -1)
return NULL;
break;
}
case 'V':
{
PyObject *obj = va_arg(*vargs, PyObject *);
const char *str = va_arg(*vargs, const char *);
if (obj) {
assert(_PyUnicode_CHECK(obj));
if (unicode_fromformat_write_str(writer, obj, width, precision) == -1)
return NULL;
}
else {
assert(str != NULL);
if (unicode_fromformat_write_cstr(writer, str, width, precision) < 0)
return NULL;
}
break;
}
case 'S':
{
PyObject *obj = va_arg(*vargs, PyObject *);
PyObject *str;
assert(obj);
str = PyObject_Str(obj);
if (!str)
return NULL;
if (unicode_fromformat_write_str(writer, str, width, precision) == -1) {
Py_DECREF(str);
return NULL;
}
Py_DECREF(str);
break;
}
case 'R':
{
PyObject *obj = va_arg(*vargs, PyObject *);
PyObject *repr;
assert(obj);
repr = PyObject_Repr(obj);
if (!repr)
return NULL;
if (unicode_fromformat_write_str(writer, repr, width, precision) == -1) {
Py_DECREF(repr);
return NULL;
}
Py_DECREF(repr);
break;
}
case 'A':
{
PyObject *obj = va_arg(*vargs, PyObject *);
PyObject *ascii;
assert(obj);
ascii = PyObject_ASCII(obj);
if (!ascii)
return NULL;
if (unicode_fromformat_write_str(writer, ascii, width, precision) == -1) {
Py_DECREF(ascii);
return NULL;
}
Py_DECREF(ascii);
break;
}
case '%':
if (_PyUnicodeWriter_WriteCharInline(writer, '%') < 0)
return NULL;
break;
default:
/* if we stumble upon an unknown formatting code, copy the rest
of the format string to the output string. (we cannot just
skip the code, since there's no way to know what's in the
argument list) */
len = strlen(p);
if (_PyUnicodeWriter_WriteLatin1String(writer, p, len) == -1)
return NULL;
f = p+len;
return f;
}
f++;
return f;
}
PyObject *
PyUnicode_FromFormatV(const char *format, va_list vargs)
{
va_list vargs2;
const char *f;
_PyUnicodeWriter writer;
_PyUnicodeWriter_Init(&writer);
writer.min_length = strlen(format) + 100;
writer.overallocate = 1;
// Copy varags to be able to pass a reference to a subfunction.
va_copy(vargs2, vargs);
for (f = format; *f; ) {
if (*f == '%') {
f = unicode_fromformat_arg(&writer, f, &vargs2);
if (f == NULL)
goto fail;
}
else {
const char *p;
Py_ssize_t len;
p = f;
do
{
if ((unsigned char)*p > 127) {
PyErr_Format(PyExc_ValueError,
"PyUnicode_FromFormatV() expects an ASCII-encoded format "
"string, got a non-ASCII byte: 0x%02x",
(unsigned char)*p);
goto fail;
}
p++;
}
while (*p != '\0' && *p != '%');
len = p - f;
if (*p == '\0')
writer.overallocate = 0;
if (_PyUnicodeWriter_WriteASCIIString(&writer, f, len) < 0)
goto fail;
f = p;
}
}
va_end(vargs2);
return _PyUnicodeWriter_Finish(&writer);
fail:
va_end(vargs2);
_PyUnicodeWriter_Dealloc(&writer);
return NULL;
}
PyObject *
PyUnicode_FromFormat(const char *format, ...)
{
PyObject* ret;
va_list vargs;
#ifdef HAVE_STDARG_PROTOTYPES
va_start(vargs, format);
#else
va_start(vargs);
#endif
ret = PyUnicode_FromFormatV(format, vargs);
va_end(vargs);
return ret;
}
static Py_ssize_t
unicode_get_widechar_size(PyObject *unicode)
{
Py_ssize_t res;
assert(unicode != NULL);
assert(_PyUnicode_CHECK(unicode));
if (_PyUnicode_WSTR(unicode) != NULL) {
return PyUnicode_WSTR_LENGTH(unicode);
}
assert(PyUnicode_IS_READY(unicode));
res = _PyUnicode_LENGTH(unicode);
#if SIZEOF_WCHAR_T == 2
if (PyUnicode_KIND(unicode) == PyUnicode_4BYTE_KIND) {
const Py_UCS4 *s = PyUnicode_4BYTE_DATA(unicode);
const Py_UCS4 *end = s + res;
for (; s < end; ++s) {
if (*s > 0xFFFF) {
++res;
}
}
}
#endif
return res;
}
static void
unicode_copy_as_widechar(PyObject *unicode, wchar_t *w, Py_ssize_t size)
{
const wchar_t *wstr;
assert(unicode != NULL);
assert(_PyUnicode_CHECK(unicode));
wstr = _PyUnicode_WSTR(unicode);
if (wstr != NULL) {
memcpy(w, wstr, size * sizeof(wchar_t));
return;
}
assert(PyUnicode_IS_READY(unicode));
if (PyUnicode_KIND(unicode) == PyUnicode_1BYTE_KIND) {
const Py_UCS1 *s = PyUnicode_1BYTE_DATA(unicode);
for (; size--; ++s, ++w) {
*w = *s;
}
}
else {
#if SIZEOF_WCHAR_T == 4
assert(PyUnicode_KIND(unicode) == PyUnicode_2BYTE_KIND);
const Py_UCS2 *s = PyUnicode_2BYTE_DATA(unicode);
for (; size--; ++s, ++w) {
*w = *s;
}
#else
assert(PyUnicode_KIND(unicode) == PyUnicode_4BYTE_KIND);
const Py_UCS4 *s = PyUnicode_4BYTE_DATA(unicode);
for (; size--; ++s, ++w) {
Py_UCS4 ch = *s;
if (ch > 0xFFFF) {
assert(ch res) {
size = res + 1;
}
else {
res = size;
}
unicode_copy_as_widechar(unicode, w, size);
#if HAVE_NON_UNICODE_WCHAR_T_REPRESENTATION
/* Oracle Solaris uses non-Unicode internal wchar_t form for
non-Unicode locales and hence needs conversion first. */
if (_Py_LocaleUsesNonUnicodeWchar()) {
if (_Py_EncodeNonUnicodeWchar_InPlace(w, size) < 0) {
return -1;
}
}
#endif
return res;
}
wchar_t*
PyUnicode_AsWideCharString(PyObject *unicode,
Py_ssize_t *size)
{
wchar_t *buffer;
Py_ssize_t buflen;
if (unicode == NULL) {
PyErr_BadInternalCall();
return NULL;
}
if (!PyUnicode_Check(unicode)) {
PyErr_BadArgument();
return NULL;
}
buflen = unicode_get_widechar_size(unicode);
buffer = (wchar_t *) PyMem_NEW(wchar_t, (buflen + 1));
if (buffer == NULL) {
PyErr_NoMemory();
return NULL;
}
unicode_copy_as_widechar(unicode, buffer, buflen + 1);
#if HAVE_NON_UNICODE_WCHAR_T_REPRESENTATION
/* Oracle Solaris uses non-Unicode internal wchar_t form for
non-Unicode locales and hence needs conversion first. */
if (_Py_LocaleUsesNonUnicodeWchar()) {
if (_Py_EncodeNonUnicodeWchar_InPlace(buffer, (buflen + 1)) < 0) {
return NULL;
}
}
#endif
if (size != NULL) {
*size = buflen;
}
else if (wcslen(buffer) != (size_t)buflen) {
PyMem_FREE(buffer);
PyErr_SetString(PyExc_ValueError,
"embedded null character");
return NULL;
}
return buffer;
}
#endif /* HAVE_WCHAR_H */
PyObject *
PyUnicode_FromOrdinal(int ordinal)
{
if (ordinal < 0 || ordinal > MAX_UNICODE) {
PyErr_SetString(PyExc_ValueError,
"chr() arg not in range(0x110000)");
return NULL;
}
return unicode_char((Py_UCS4)ordinal);
}
PyObject *
PyUnicode_FromObject(PyObject *obj)
{
/* XXX Perhaps we should make this API an alias of
PyObject_Str() instead ?! */
if (PyUnicode_CheckExact(obj)) {
if (PyUnicode_READY(obj) == -1)
return NULL;
Py_INCREF(obj);
return obj;
}
if (PyUnicode_Check(obj)) {
/* For a Unicode subtype that's not a Unicode object,
return a true Unicode object with the same data. */
return _PyUnicode_Copy(obj);
}
PyErr_Format(PyExc_TypeError,
"Can't convert '%.100s' object to str implicitly",
Py_TYPE(obj)->tp_name);
return NULL;
}
PyObject *
PyUnicode_FromEncodedObject(PyObject *obj,
const char *encoding,
const char *errors)
{
Py_buffer buffer;
PyObject *v;
if (obj == NULL) {
PyErr_BadInternalCall();
return NULL;
}
/* Decoding bytes objects is the most common case and should be fast */
if (PyBytes_Check(obj)) {
if (PyBytes_GET_SIZE(obj) == 0) {
if (unicode_check_encoding_errors(encoding, errors) < 0) {
return NULL;
}
_Py_RETURN_UNICODE_EMPTY();
}
return PyUnicode_Decode(
PyBytes_AS_STRING(obj), PyBytes_GET_SIZE(obj),
encoding, errors);
}
if (PyUnicode_Check(obj)) {
PyErr_SetString(PyExc_TypeError,
"decoding str is not supported");
return NULL;
}
/* Retrieve a bytes buffer view through the PEP 3118 buffer interface */
if (PyObject_GetBuffer(obj, &buffer, PyBUF_SIMPLE) < 0) {
PyErr_Format(PyExc_TypeError,
"decoding to str: need a bytes-like object, %.80s found",
Py_TYPE(obj)->tp_name);
return NULL;
}
if (buffer.len == 0) {
PyBuffer_Release(&buffer);
if (unicode_check_encoding_errors(encoding, errors) < 0) {
return NULL;
}
_Py_RETURN_UNICODE_EMPTY();
}
v = PyUnicode_Decode((char*) buffer.buf, buffer.len, encoding, errors);
PyBuffer_Release(&buffer);
return v;
}
/* Normalize an encoding name: similar to encodings.normalize_encoding(), but
also convert to lowercase. Return 1 on success, or 0 on error (encoding is
longer than lower_len-1). */
int
_Py_normalize_encoding(const char *encoding,
char *lower,
size_t lower_len)
{
const char *e;
char *l;
char *l_end;
int punct;
assert(encoding != NULL);
e = encoding;
l = lower;
l_end = &lower[lower_len - 1];
punct = 0;
while (1) {
char c = *e;
if (c == 0) {
break;
}
if (Py_ISALNUM(c) || c == '.') {
if (punct && l != lower) {
if (l == l_end) {
return 0;
}
*l++ = '_';
}
punct = 0;
if (l == l_end) {
return 0;
}
*l++ = Py_TOLOWER(c);
}
else {
punct = 1;
}
e++;
}
*l = '\0';
return 1;
}
PyObject *
PyUnicode_Decode(const char *s,
Py_ssize_t size,
const char *encoding,
const char *errors)
{
PyObject *buffer = NULL, *unicode;
Py_buffer info;
char buflower[11]; /* strlen("iso-8859-1\0") == 11, longest shortcut */
if (unicode_check_encoding_errors(encoding, errors) < 0) {
return NULL;
}
if (size == 0) {
_Py_RETURN_UNICODE_EMPTY();
}
if (encoding == NULL) {
return PyUnicode_DecodeUTF8Stateful(s, size, errors, NULL);
}
/* Shortcuts for common default encodings */
if (_Py_normalize_encoding(encoding, buflower, sizeof(buflower))) {
char *lower = buflower;
/* Fast paths */
if (lower[0] == 'u' && lower[1] == 't' && lower[2] == 'f') {
lower += 3;
if (*lower == '_') {
/* Match "utf8" and "utf_8" */
lower++;
}
if (lower[0] == '8' && lower[1] == 0) {
return PyUnicode_DecodeUTF8Stateful(s, size, errors, NULL);
}
else if (lower[0] == '1' && lower[1] == '6' && lower[2] == 0) {
return PyUnicode_DecodeUTF16(s, size, errors, 0);
}
else if (lower[0] == '3' && lower[1] == '2' && lower[2] == 0) {
return PyUnicode_DecodeUTF32(s, size, errors, 0);
}
}
else {
if (strcmp(lower, "ascii") == 0
|| strcmp(lower, "us_ascii") == 0) {
return PyUnicode_DecodeASCII(s, size, errors);
}
#ifdef MS_WINDOWS
else if (strcmp(lower, "mbcs") == 0) {
return PyUnicode_DecodeMBCS(s, size, errors);
}
#endif
else if (strcmp(lower, "latin1") == 0
|| strcmp(lower, "latin_1") == 0
|| strcmp(lower, "iso_8859_1") == 0
|| strcmp(lower, "iso8859_1") == 0) {
return PyUnicode_DecodeLatin1(s, size, errors);
}
}
}
/* Decode via the codec registry */
buffer = NULL;
if (PyBuffer_FillInfo(&info, NULL, (void *)s, size, 1, PyBUF_FULL_RO) < 0)
goto onError;
buffer = PyMemoryView_FromBuffer(&info);
if (buffer == NULL)
goto onError;
unicode = _PyCodec_DecodeText(buffer, encoding, errors);
if (unicode == NULL)
goto onError;
if (!PyUnicode_Check(unicode)) {
PyErr_Format(PyExc_TypeError,
"'%.400s' decoder returned '%.400s' instead of 'str'; "
"use codecs.decode() to decode to arbitrary types",
encoding,
Py_TYPE(unicode)->tp_name);
Py_DECREF(unicode);
goto onError;
}
Py_DECREF(buffer);
return unicode_result(unicode);
onError:
Py_XDECREF(buffer);
return NULL;
}
PyObject *
PyUnicode_AsDecodedObject(PyObject *unicode,
const char *encoding,
const char *errors)
{
if (!PyUnicode_Check(unicode)) {
PyErr_BadArgument();
return NULL;
}
if (PyErr_WarnEx(PyExc_DeprecationWarning,
"PyUnicode_AsDecodedObject() is deprecated; "
"use PyCodec_Decode() to decode from str", 1) < 0)
return NULL;
if (encoding == NULL)
encoding = PyUnicode_GetDefaultEncoding();
/* Decode via the codec registry */
return PyCodec_Decode(unicode, encoding, errors);
}
PyObject *
PyUnicode_AsDecodedUnicode(PyObject *unicode,
const char *encoding,
const char *errors)
{
PyObject *v;
if (!PyUnicode_Check(unicode)) {
PyErr_BadArgument();
goto onError;
}
if (PyErr_WarnEx(PyExc_DeprecationWarning,
"PyUnicode_AsDecodedUnicode() is deprecated; "
"use PyCodec_Decode() to decode from str to str", 1) < 0)
return NULL;
if (encoding == NULL)
encoding = PyUnicode_GetDefaultEncoding();
/* Decode via the codec registry */
v = PyCodec_Decode(unicode, encoding, errors);
if (v == NULL)
goto onError;
if (!PyUnicode_Check(v)) {
PyErr_Format(PyExc_TypeError,
"'%.400s' decoder returned '%.400s' instead of 'str'; "
"use codecs.decode() to decode to arbitrary types",
encoding,
Py_TYPE(unicode)->tp_name);
Py_DECREF(v);
goto onError;
}
return unicode_result(v);
onError:
return NULL;
}
PyObject *
PyUnicode_Encode(const Py_UNICODE *s,
Py_ssize_t size,
const char *encoding,
const char *errors)
{
PyObject *v, *unicode;
unicode = PyUnicode_FromWideChar(s, size);
if (unicode == NULL)
return NULL;
v = PyUnicode_AsEncodedString(unicode, encoding, errors);
Py_DECREF(unicode);
return v;
}
PyObject *
PyUnicode_AsEncodedObject(PyObject *unicode,
const char *encoding,
const char *errors)
{
PyObject *v;
if (!PyUnicode_Check(unicode)) {
PyErr_BadArgument();
goto onError;
}
if (PyErr_WarnEx(PyExc_DeprecationWarning,
"PyUnicode_AsEncodedObject() is deprecated; "
"use PyUnicode_AsEncodedString() to encode from str to bytes "
"or PyCodec_Encode() for generic encoding", 1) < 0)
return NULL;
if (encoding == NULL)
encoding = PyUnicode_GetDefaultEncoding();
/* Encode via the codec registry */
v = PyCodec_Encode(unicode, encoding, errors);
if (v == NULL)
goto onError;
return v;
onError:
return NULL;
}
static PyObject *
unicode_encode_locale(PyObject *unicode, _Py_error_handler error_handler,
int current_locale)
{
Py_ssize_t wlen;
wchar_t *wstr = PyUnicode_AsWideCharString(unicode, &wlen);
if (wstr == NULL) {
return NULL;
}
if ((size_t)wlen != wcslen(wstr)) {
PyErr_SetString(PyExc_ValueError, "embedded null character");
PyMem_Free(wstr);
return NULL;
}
char *str;
size_t error_pos;
const char *reason;
int res = _Py_EncodeLocaleEx(wstr, &str, &error_pos, &reason,
current_locale, error_handler);
PyMem_Free(wstr);
if (res != 0) {
if (res == -2) {
PyObject *exc;
exc = PyObject_CallFunction(PyExc_UnicodeEncodeError, "sOnns",
"locale", unicode,
(Py_ssize_t)error_pos,
(Py_ssize_t)(error_pos+1),
reason);
if (exc != NULL) {
PyCodec_StrictErrors(exc);
Py_DECREF(exc);
}
}
else if (res == -3) {
PyErr_SetString(PyExc_ValueError, "unsupported error handler");
}
else {
PyErr_NoMemory();
}
return NULL;
}
PyObject *bytes = PyBytes_FromString(str);
PyMem_RawFree(str);
return bytes;
}
PyObject *
PyUnicode_EncodeLocale(PyObject *unicode, const char *errors)
{
_Py_error_handler error_handler = _Py_GetErrorHandler(errors);
return unicode_encode_locale(unicode, error_handler, 1);
}
PyObject *
PyUnicode_EncodeFSDefault(PyObject *unicode)
{
PyInterpreterState *interp = _PyInterpreterState_GET();
struct _Py_unicode_fs_codec *fs_codec = &interp->unicode.fs_codec;
if (fs_codec->utf8) {
return unicode_encode_utf8(unicode,
fs_codec->error_handler,
fs_codec->errors);
}
#ifndef _Py_FORCE_UTF8_FS_ENCODING
else if (fs_codec->encoding) {
return PyUnicode_AsEncodedString(unicode,
fs_codec->encoding,
fs_codec->errors);
}
#endif
else {
/* Before _PyUnicode_InitEncodings() is called, the Python codec
machinery is not ready and so cannot be used:
use wcstombs() in this case. */
const PyConfig *config = _PyInterpreterState_GetConfig(interp);
const wchar_t *filesystem_errors = config->filesystem_errors;
assert(filesystem_errors != NULL);
_Py_error_handler errors = get_error_handler_wide(filesystem_errors);
assert(errors != _Py_ERROR_UNKNOWN);
#ifdef _Py_FORCE_UTF8_FS_ENCODING
return unicode_encode_utf8(unicode, errors, NULL);
#else
return unicode_encode_locale(unicode, errors, 0);
#endif
}
}
PyObject *
PyUnicode_AsEncodedString(PyObject *unicode,
const char *encoding,
const char *errors)
{
PyObject *v;
char buflower[11]; /* strlen("iso_8859_1\0") == 11, longest shortcut */
if (!PyUnicode_Check(unicode)) {
PyErr_BadArgument();
return NULL;
}
if (unicode_check_encoding_errors(encoding, errors) < 0) {
return NULL;
}
if (encoding == NULL) {
return _PyUnicode_AsUTF8String(unicode, errors);
}
/* Shortcuts for common default encodings */
if (_Py_normalize_encoding(encoding, buflower, sizeof(buflower))) {
char *lower = buflower;
/* Fast paths */
if (lower[0] == 'u' && lower[1] == 't' && lower[2] == 'f') {
lower += 3;
if (*lower == '_') {
/* Match "utf8" and "utf_8" */
lower++;
}
if (lower[0] == '8' && lower[1] == 0) {
return _PyUnicode_AsUTF8String(unicode, errors);
}
else if (lower[0] == '1' && lower[1] == '6' && lower[2] == 0) {
return _PyUnicode_EncodeUTF16(unicode, errors, 0);
}
else if (lower[0] == '3' && lower[1] == '2' && lower[2] == 0) {
return _PyUnicode_EncodeUTF32(unicode, errors, 0);
}
}
else {
if (strcmp(lower, "ascii") == 0
|| strcmp(lower, "us_ascii") == 0) {
return _PyUnicode_AsASCIIString(unicode, errors);
}
#ifdef MS_WINDOWS
else if (strcmp(lower, "mbcs") == 0) {
return PyUnicode_EncodeCodePage(CP_ACP, unicode, errors);
}
#endif
else if (strcmp(lower, "latin1") == 0 ||
strcmp(lower, "latin_1") == 0 ||
strcmp(lower, "iso_8859_1") == 0 ||
strcmp(lower, "iso8859_1") == 0) {
return _PyUnicode_AsLatin1String(unicode, errors);
}
}
}
/* Encode via the codec registry */
v = _PyCodec_EncodeText(unicode, encoding, errors);
if (v == NULL)
return NULL;
/* The normal path */
if (PyBytes_Check(v))
return v;
/* If the codec returns a buffer, raise a warning and convert to bytes */
if (PyByteArray_Check(v)) {
int error;
PyObject *b;
error = PyErr_WarnFormat(PyExc_RuntimeWarning, 1,
"encoder %s returned bytearray instead of bytes; "
"use codecs.encode() to encode to arbitrary types",
encoding);
if (error) {
Py_DECREF(v);
return NULL;
}
b = PyBytes_FromStringAndSize(PyByteArray_AS_STRING(v),
PyByteArray_GET_SIZE(v));
Py_DECREF(v);
return b;
}
PyErr_Format(PyExc_TypeError,
"'%.400s' encoder returned '%.400s' instead of 'bytes'; "
"use codecs.encode() to encode to arbitrary types",
encoding,
Py_TYPE(v)->tp_name);
Py_DECREF(v);
return NULL;
}
PyObject *
PyUnicode_AsEncodedUnicode(PyObject *unicode,
const char *encoding,
const char *errors)
{
PyObject *v;
if (!PyUnicode_Check(unicode)) {
PyErr_BadArgument();
goto onError;
}
if (PyErr_WarnEx(PyExc_DeprecationWarning,
"PyUnicode_AsEncodedUnicode() is deprecated; "
"use PyCodec_Encode() to encode from str to str", 1) < 0)
return NULL;
if (encoding == NULL)
encoding = PyUnicode_GetDefaultEncoding();
/* Encode via the codec registry */
v = PyCodec_Encode(unicode, encoding, errors);
if (v == NULL)
goto onError;
if (!PyUnicode_Check(v)) {
PyErr_Format(PyExc_TypeError,
"'%.400s' encoder returned '%.400s' instead of 'str'; "
"use codecs.encode() to encode to arbitrary types",
encoding,
Py_TYPE(v)->tp_name);
Py_DECREF(v);
goto onError;
}
return v;
onError:
return NULL;
}
static PyObject*
unicode_decode_locale(const char *str, Py_ssize_t len,
_Py_error_handler errors, int current_locale)
{
if (str[len] != '\0' || (size_t)len != strlen(str)) {
PyErr_SetString(PyExc_ValueError, "embedded null byte");
return NULL;
}
wchar_t *wstr;
size_t wlen;
const char *reason;
int res = _Py_DecodeLocaleEx(str, &wstr, &wlen, &reason,
current_locale, errors);
if (res != 0) {
if (res == -2) {
PyObject *exc;
exc = PyObject_CallFunction(PyExc_UnicodeDecodeError, "sy#nns",
"locale", str, len,
(Py_ssize_t)wlen,
(Py_ssize_t)(wlen + 1),
reason);
if (exc != NULL) {
PyCodec_StrictErrors(exc);
Py_DECREF(exc);
}
}
else if (res == -3) {
PyErr_SetString(PyExc_ValueError, "unsupported error handler");
}
else {
PyErr_NoMemory();
}
return NULL;
}
PyObject *unicode = PyUnicode_FromWideChar(wstr, wlen);
PyMem_RawFree(wstr);
return unicode;
}
PyObject*
PyUnicode_DecodeLocaleAndSize(const char *str, Py_ssize_t len,
const char *errors)
{
_Py_error_handler error_handler = _Py_GetErrorHandler(errors);
return unicode_decode_locale(str, len, error_handler, 1);
}
PyObject*
PyUnicode_DecodeLocale(const char *str, const char *errors)
{
Py_ssize_t size = (Py_ssize_t)strlen(str);
_Py_error_handler error_handler = _Py_GetErrorHandler(errors);
return unicode_decode_locale(str, size, error_handler, 1);
}
PyObject*
PyUnicode_DecodeFSDefault(const char *s) {
Py_ssize_t size = (Py_ssize_t)strlen(s);
return PyUnicode_DecodeFSDefaultAndSize(s, size);
}
PyObject*
PyUnicode_DecodeFSDefaultAndSize(const char *s, Py_ssize_t size)
{
PyInterpreterState *interp = _PyInterpreterState_GET();
struct _Py_unicode_fs_codec *fs_codec = &interp->unicode.fs_codec;
if (fs_codec->utf8) {
return unicode_decode_utf8(s, size,
fs_codec->error_handler,
fs_codec->errors,
NULL);
}
#ifndef _Py_FORCE_UTF8_FS_ENCODING
else if (fs_codec->encoding) {
return PyUnicode_Decode(s, size,
fs_codec->encoding,
fs_codec->errors);
}
#endif
else {
/* Before _PyUnicode_InitEncodings() is called, the Python codec
machinery is not ready and so cannot be used:
use mbstowcs() in this case. */
const PyConfig *config = _PyInterpreterState_GetConfig(interp);
const wchar_t *filesystem_errors = config->filesystem_errors;
assert(filesystem_errors != NULL);
_Py_error_handler errors = get_error_handler_wide(filesystem_errors);
assert(errors != _Py_ERROR_UNKNOWN);
#ifdef _Py_FORCE_UTF8_FS_ENCODING
return unicode_decode_utf8(s, size, errors, NULL, NULL);
#else
return unicode_decode_locale(s, size, errors, 0);
#endif
}
}
int
PyUnicode_FSConverter(PyObject* arg, void* addr)
{
PyObject *path = NULL;
PyObject *output = NULL;
Py_ssize_t size;
const char *data;
if (arg == NULL) {
Py_DECREF(*(PyObject**)addr);
*(PyObject**)addr = NULL;
return 1;
}
path = PyOS_FSPath(arg);
if (path == NULL) {
return 0;
}
if (PyBytes_Check(path)) {
output = path;
}
else { // PyOS_FSPath() guarantees its returned value is bytes or str.
output = PyUnicode_EncodeFSDefault(path);
Py_DECREF(path);
if (!output) {
return 0;
}
assert(PyBytes_Check(output));
}
size = PyBytes_GET_SIZE(output);
data = PyBytes_AS_STRING(output);
if ((size_t)size != strlen(data)) {
PyErr_SetString(PyExc_ValueError, "embedded null byte");
Py_DECREF(output);
return 0;
}
*(PyObject**)addr = output;
return Py_CLEANUP_SUPPORTED;
}
int
PyUnicode_FSDecoder(PyObject* arg, void* addr)
{
int is_buffer = 0;
PyObject *path = NULL;
PyObject *output = NULL;
if (arg == NULL) {
Py_DECREF(*(PyObject**)addr);
*(PyObject**)addr = NULL;
return 1;
}
is_buffer = PyObject_CheckBuffer(arg);
if (!is_buffer) {
path = PyOS_FSPath(arg);
if (path == NULL) {
return 0;
}
}
else {
path = arg;
Py_INCREF(arg);
}
if (PyUnicode_Check(path)) {
output = path;
}
else if (PyBytes_Check(path) || is_buffer) {
PyObject *path_bytes = NULL;
if (!PyBytes_Check(path) &&
PyErr_WarnFormat(PyExc_DeprecationWarning, 1,
"path should be string, bytes, or os.PathLike, not %.200s",
Py_TYPE(arg)->tp_name)) {
Py_DECREF(path);
return 0;
}
path_bytes = PyBytes_FromObject(path);
Py_DECREF(path);
if (!path_bytes) {
return 0;
}
output = PyUnicode_DecodeFSDefaultAndSize(PyBytes_AS_STRING(path_bytes),
PyBytes_GET_SIZE(path_bytes));
Py_DECREF(path_bytes);
if (!output) {
return 0;
}
}
else {
PyErr_Format(PyExc_TypeError,
"path should be string, bytes, or os.PathLike, not %.200s",
Py_TYPE(arg)->tp_name);
Py_DECREF(path);
return 0;
}
if (PyUnicode_READY(output) == -1) {
Py_DECREF(output);
return 0;
}
if (findchar(PyUnicode_DATA(output), PyUnicode_KIND(output),
PyUnicode_GET_LENGTH(output), 0, 1) >= 0) {
PyErr_SetString(PyExc_ValueError, "embedded null character");
Py_DECREF(output);
return 0;
}
*(PyObject**)addr = output;
return Py_CLEANUP_SUPPORTED;
}
static int unicode_fill_utf8(PyObject *unicode);
const char *
PyUnicode_AsUTF8AndSize(PyObject *unicode, Py_ssize_t *psize)
{
if (!PyUnicode_Check(unicode)) {
PyErr_BadArgument();
return NULL;
}
if (PyUnicode_READY(unicode) == -1)
return NULL;
if (PyUnicode_UTF8(unicode) == NULL) {
if (unicode_fill_utf8(unicode) == -1) {
return NULL;
}
}
if (psize)
*psize = PyUnicode_UTF8_LENGTH(unicode);
return PyUnicode_UTF8(unicode);
}
const char *
PyUnicode_AsUTF8(PyObject *unicode)
{
return PyUnicode_AsUTF8AndSize(unicode, NULL);
}
Py_UNICODE *
PyUnicode_AsUnicodeAndSize(PyObject *unicode, Py_ssize_t *size)
{
if (!PyUnicode_Check(unicode)) {
PyErr_BadArgument();
return NULL;
}
Py_UNICODE *w = _PyUnicode_WSTR(unicode);
if (w == NULL) {
/* Non-ASCII compact unicode object */
assert(_PyUnicode_KIND(unicode) != PyUnicode_WCHAR_KIND);
assert(PyUnicode_IS_READY(unicode));
Py_ssize_t wlen = unicode_get_widechar_size(unicode);
if ((size_t)wlen > PY_SSIZE_T_MAX / sizeof(wchar_t) - 1) {
PyErr_NoMemory();
return NULL;
}
w = (wchar_t *) PyObject_MALLOC(sizeof(wchar_t) * (wlen + 1));
if (w == NULL) {
PyErr_NoMemory();
return NULL;
}
unicode_copy_as_widechar(unicode, w, wlen + 1);
_PyUnicode_WSTR(unicode) = w;
if (!PyUnicode_IS_COMPACT_ASCII(unicode)) {
_PyUnicode_WSTR_LENGTH(unicode) = wlen;
}
}
if (size != NULL)
*size = PyUnicode_WSTR_LENGTH(unicode);
return w;
}
/* Deprecated APIs */
_Py_COMP_DIAG_PUSH
_Py_COMP_DIAG_IGNORE_DEPR_DECLS
Py_UNICODE *
PyUnicode_AsUnicode(PyObject *unicode)
{
return PyUnicode_AsUnicodeAndSize(unicode, NULL);
}
const Py_UNICODE *
_PyUnicode_AsUnicode(PyObject *unicode)
{
Py_ssize_t size;
const Py_UNICODE *wstr;
wstr = PyUnicode_AsUnicodeAndSize(unicode, &size);
if (wstr && wcslen(wstr) != (size_t)size) {
PyErr_SetString(PyExc_ValueError, "embedded null character");
return NULL;
}
return wstr;
}
Py_ssize_t
PyUnicode_GetSize(PyObject *unicode)
{
if (!PyUnicode_Check(unicode)) {
PyErr_BadArgument();
goto onError;
}
if (_PyUnicode_WSTR(unicode) == NULL) {
if (PyUnicode_AsUnicode(unicode) == NULL)
goto onError;
}
return PyUnicode_WSTR_LENGTH(unicode);
onError:
return -1;
}
_Py_COMP_DIAG_POP
Py_ssize_t
PyUnicode_GetLength(PyObject *unicode)
{
if (!PyUnicode_Check(unicode)) {
PyErr_BadArgument();
return -1;
}
if (PyUnicode_READY(unicode) == -1)
return -1;
return PyUnicode_GET_LENGTH(unicode);
}
Py_UCS4
PyUnicode_ReadChar(PyObject *unicode, Py_ssize_t index)
{
const void *data;
int kind;
if (!PyUnicode_Check(unicode)) {
PyErr_BadArgument();
return (Py_UCS4)-1;
}
if (PyUnicode_READY(unicode) == -1) {
return (Py_UCS4)-1;
}
if (index < 0 || index >= PyUnicode_GET_LENGTH(unicode)) {
PyErr_SetString(PyExc_IndexError, "string index out of range");
return (Py_UCS4)-1;
}
data = PyUnicode_DATA(unicode);
kind = PyUnicode_KIND(unicode);
return PyUnicode_READ(kind, data, index);
}
int
PyUnicode_WriteChar(PyObject *unicode, Py_ssize_t index, Py_UCS4 ch)
{
if (!PyUnicode_Check(unicode) || !PyUnicode_IS_COMPACT(unicode)) {
PyErr_BadArgument();
return -1;
}
assert(PyUnicode_IS_READY(unicode));
if (index < 0 || index >= PyUnicode_GET_LENGTH(unicode)) {
PyErr_SetString(PyExc_IndexError, "string index out of range");
return -1;
}
if (unicode_check_modifiable(unicode))
return -1;
if (ch > PyUnicode_MAX_CHAR_VALUE(unicode)) {
PyErr_SetString(PyExc_ValueError, "character out of range");
return -1;
}
PyUnicode_WRITE(PyUnicode_KIND(unicode), PyUnicode_DATA(unicode),
index, ch);
return 0;
}
const char *
PyUnicode_GetDefaultEncoding(void)
{
return "utf-8";
}
/* create or adjust a UnicodeDecodeError */
static void
make_decode_exception(PyObject **exceptionObject,
const char *encoding,
const char *input, Py_ssize_t length,
Py_ssize_t startpos, Py_ssize_t endpos,
const char *reason)
{
if (*exceptionObject == NULL) {
*exceptionObject = PyUnicodeDecodeError_Create(
encoding, input, length, startpos, endpos, reason);
}
else {
if (PyUnicodeDecodeError_SetStart(*exceptionObject, startpos))
goto onError;
if (PyUnicodeDecodeError_SetEnd(*exceptionObject, endpos))
goto onError;
if (PyUnicodeDecodeError_SetReason(*exceptionObject, reason))
goto onError;
}
return;
onError:
Py_CLEAR(*exceptionObject);
}
#ifdef MS_WINDOWS
static int
widechar_resize(wchar_t **buf, Py_ssize_t *size, Py_ssize_t newsize)
{
if (newsize > *size) {
wchar_t *newbuf = *buf;
if (PyMem_Resize(newbuf, wchar_t, newsize) == NULL) {
PyErr_NoMemory();
return -1;
}
*buf = newbuf;
}
*size = newsize;
return 0;
}
/* error handling callback helper:
build arguments, call the callback and check the arguments,
if no exception occurred, copy the replacement to the output
and adjust various state variables.
return 0 on success, -1 on error
*/
static int
unicode_decode_call_errorhandler_wchar(
const char *errors, PyObject **errorHandler,
const char *encoding, const char *reason,
const char **input, const char **inend, Py_ssize_t *startinpos,
Py_ssize_t *endinpos, PyObject **exceptionObject, const char **inptr,
wchar_t **buf, Py_ssize_t *bufsize, Py_ssize_t *outpos)
{
static const char *argparse = "Un;decoding error handler must return (str, int) tuple";
PyObject *restuple = NULL;
PyObject *repunicode = NULL;
Py_ssize_t outsize;
Py_ssize_t insize;
Py_ssize_t requiredsize;
Py_ssize_t newpos;
PyObject *inputobj = NULL;
wchar_t *repwstr;
Py_ssize_t repwlen;
if (*errorHandler == NULL) {
*errorHandler = PyCodec_LookupError(errors);
if (*errorHandler == NULL)
goto onError;
}
make_decode_exception(exceptionObject,
encoding,
*input, *inend - *input,
*startinpos, *endinpos,
reason);
if (*exceptionObject == NULL)
goto onError;
restuple = PyObject_CallOneArg(*errorHandler, *exceptionObject);
if (restuple == NULL)
goto onError;
if (!PyTuple_Check(restuple)) {
PyErr_SetString(PyExc_TypeError, &argparse[3]);
goto onError;
}
if (!PyArg_ParseTuple(restuple, argparse, &repunicode, &newpos))
goto onError;
/* Copy back the bytes variables, which might have been modified by the
callback */
inputobj = PyUnicodeDecodeError_GetObject(*exceptionObject);
if (!inputobj)
goto onError;
*input = PyBytes_AS_STRING(inputobj);
insize = PyBytes_GET_SIZE(inputobj);
*inend = *input + insize;
/* we can DECREF safely, as the exception has another reference,
so the object won't go away. */
Py_DECREF(inputobj);
if (newposmin_length += replen - 1;
need_to_grow = 1;
}
new_inptr = *input + newpos;
if (*inend - new_inptr > remain) {
/* We don't know the decoding algorithm here so we make the worst
assumption that one byte decodes to one unicode character.
If unfortunately one byte could decode to more unicode characters,
the decoder may write out-of-bound then. Is it possible for the
algorithms using this function? */
writer->min_length += *inend - new_inptr - remain;
need_to_grow = 1;
}
if (need_to_grow) {
writer->overallocate = 1;
if (_PyUnicodeWriter_Prepare(writer, writer->min_length - writer->pos,
PyUnicode_MAX_CHAR_VALUE(repunicode)) == -1)
goto onError;
}
if (_PyUnicodeWriter_WriteStr(writer, repunicode) == -1)
goto onError;
*endinpos = newpos;
*inptr = new_inptr;
/* we made it! */
Py_DECREF(restuple);
return 0;
onError:
Py_XDECREF(restuple);
return -1;
}
/* --- UTF-7 Codec -------------------------------------------------------- */
/* See RFC2152 for details. We encode conservatively and decode liberally. */
/* Three simple macros defining base-64. */
/* Is c a base-64 character? */
#define IS_BASE64(c) \
(((c) >= 'A' && (c) = 'a' && (c) = '0' && (c) = 'A' && (c) = 'a' && (c) = '0' && (c) ? */
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0,
/* @ A B C D E F G H I J K L M N O */
1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
/* P Q R S T U V W X Y Z [ \ ] ^ _ */
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 1, 1, 1,
/* ` a b c d e f g h i j k l m n o */
1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
/* p q r s t u v w x y z { | } ~ del */
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 3, 3,
};
/* ENCODE_DIRECT: this character should be encoded as itself. The
* answer depends on whether we are encoding set O as itself, and also
* on whether we are encoding whitespace as itself. RFC2152 makes it
* clear that the answers to these questions vary between
* applications, so this code needs to be flexible. */
#define ENCODE_DIRECT(c, directO, directWS) \
((c) < 128 && (c) > 0 && \
((utf7_category[(c)] == 0) || \
(directWS && (utf7_category[(c)] == 2)) || \
(directO && (utf7_category[(c)] == 1))))
PyObject *
PyUnicode_DecodeUTF7(const char *s,
Py_ssize_t size,
const char *errors)
{
return PyUnicode_DecodeUTF7Stateful(s, size, errors, NULL);
}
/* The decoder. The only state we preserve is our read position,
* i.e. how many characters we have consumed. So if we end in the
* middle of a shift sequence we have to back off the read position
* and the output to the beginning of the sequence, otherwise we lose
* all the shift state (seen bits, number of bits seen, high
* surrogate). */
PyObject *
PyUnicode_DecodeUTF7Stateful(const char *s,
Py_ssize_t size,
const char *errors,
Py_ssize_t *consumed)
{
const char *starts = s;
Py_ssize_t startinpos;
Py_ssize_t endinpos;
const char *e;
_PyUnicodeWriter writer;
const char *errmsg = "";
int inShift = 0;
Py_ssize_t shiftOutStart;
unsigned int base64bits = 0;
unsigned long base64buffer = 0;
Py_UCS4 surrogate = 0;
PyObject *errorHandler = NULL;
PyObject *exc = NULL;
if (size == 0) {
if (consumed)
*consumed = 0;
_Py_RETURN_UNICODE_EMPTY();
}
/* Start off assuming it's all ASCII. Widen later as necessary. */
_PyUnicodeWriter_Init(&writer);
writer.min_length = size;
shiftOutStart = 0;
e = s + size;
while (s < e) {
Py_UCS4 ch;
restart:
ch = (unsigned char) *s;
if (inShift) { /* in a base-64 section */
if (IS_BASE64(ch)) { /* consume a base-64 character */
base64buffer = (base64buffer = 16) {
/* we have enough bits for a UTF-16 value */
Py_UCS4 outCh = (Py_UCS4)(base64buffer >> (base64bits-16));
base64bits -= 16;
base64buffer &= (1 = 6) {
/* We've seen at least one base-64 character */
s++;
errmsg = "partial character in shift sequence";
goto utf7Error;
}
else {
/* Some bits remain; they should be zero */
if (base64buffer != 0) {
s++;
errmsg = "non-zero padding bits in shift sequence";
goto utf7Error;
}
}
}
if (surrogate && DECODE_DIRECT(ch)) {
if (_PyUnicodeWriter_WriteCharInline(&writer, surrogate) < 0)
goto onError;
}
surrogate = 0;
if (ch == '-') {
/* '-' is absorbed; other terminating
characters are preserved */
s++;
}
}
}
else if ( ch == '+' ) {
startinpos = s-starts;
s++; /* consume '+' */
if (s < e && *s == '-') { /* '+-' encodes '+' */
s++;
if (_PyUnicodeWriter_WriteCharInline(&writer, '+') < 0)
goto onError;
}
else if (s < e && !IS_BASE64(*s)) {
s++;
errmsg = "ill-formed sequence";
goto utf7Error;
}
else { /* begin base64-encoded section */
inShift = 1;
surrogate = 0;
shiftOutStart = writer.pos;
base64bits = 0;
base64buffer = 0;
}
}
else if (DECODE_DIRECT(ch)) { /* character decodes as itself */
s++;
if (_PyUnicodeWriter_WriteCharInline(&writer, ch) < 0)
goto onError;
}
else {
startinpos = s-starts;
s++;
errmsg = "unexpected special character";
goto utf7Error;
}
continue;
utf7Error:
endinpos = s-starts;
if (unicode_decode_call_errorhandler_writer(
errors, &errorHandler,
"utf7", errmsg,
&starts, &e, &startinpos, &endinpos, &exc, &s,
&writer))
goto onError;
}
/* end of string */
if (inShift && !consumed) { /* in shift sequence, no more to follow */
/* if we're in an inconsistent state, that's an error */
inShift = 0;
if (surrogate ||
(base64bits >= 6) ||
(base64bits > 0 && base64buffer != 0)) {
endinpos = size;
if (unicode_decode_call_errorhandler_writer(
errors, &errorHandler,
"utf7", "unterminated shift sequence",
&starts, &e, &startinpos, &endinpos, &exc, &s,
&writer))
goto onError;
if (s < e)
goto restart;
}
}
/* return state */
if (consumed) {
if (inShift) {
*consumed = startinpos;
if (writer.pos != shiftOutStart && writer.maxchar > 127) {
PyObject *result = PyUnicode_FromKindAndData(
writer.kind, writer.data, shiftOutStart);
Py_XDECREF(errorHandler);
Py_XDECREF(exc);
_PyUnicodeWriter_Dealloc(&writer);
return result;
}
writer.pos = shiftOutStart; /* back off output */
}
else {
*consumed = s-starts;
}
}
Py_XDECREF(errorHandler);
Py_XDECREF(exc);
return _PyUnicodeWriter_Finish(&writer);
onError:
Py_XDECREF(errorHandler);
Py_XDECREF(exc);
_PyUnicodeWriter_Dealloc(&writer);
return NULL;
}
PyObject *
_PyUnicode_EncodeUTF7(PyObject *str,
int base64SetO,
int base64WhiteSpace,
const char *errors)
{
int kind;
const void *data;
Py_ssize_t len;
PyObject *v;
int inShift = 0;
Py_ssize_t i;
unsigned int base64bits = 0;
unsigned long base64buffer = 0;
char * out;
const char * start;
if (PyUnicode_READY(str) == -1)
return NULL;
kind = PyUnicode_KIND(str);
data = PyUnicode_DATA(str);
len = PyUnicode_GET_LENGTH(str);
if (len == 0)
return PyBytes_FromStringAndSize(NULL, 0);
/* It might be possible to tighten this worst case */
if (len > PY_SSIZE_T_MAX / 8)
return PyErr_NoMemory();
v = PyBytes_FromStringAndSize(NULL, len * 8);
if (v == NULL)
return NULL;
start = out = PyBytes_AS_STRING(v);
for (i = 0; i < len; ++i) {
Py_UCS4 ch = PyUnicode_READ(kind, data, i);
if (inShift) {
if (ENCODE_DIRECT(ch, !base64SetO, !base64WhiteSpace)) {
/* shifting out */
if (base64bits) { /* output remaining bits */
*out++ = TO_BASE64(base64buffer = 0x10000) {
assert(ch > (base64bits-6));
base64bits -= 6;
}
/* prepare second surrogate */
ch = Py_UNICODE_LOW_SURROGATE(ch);
}
base64bits += 16;
base64buffer = (base64buffer = 6) {
*out++ = TO_BASE64(base64buffer >> (base64bits-6));
base64bits -= 6;
}
}
if (base64bits)
*out++= TO_BASE64(base64buffer 0) {
incr = tabsize - (line_pos % tabsize); /* cannot overflow */
if (j > PY_SSIZE_T_MAX - incr)
goto overflow;
line_pos += incr;
j += incr;
}
}
else {
if (j > PY_SSIZE_T_MAX - 1)
goto overflow;
line_pos++;
j++;
if (ch == '\n' || ch == '\r')
line_pos = 0;
}
}
if (!found)
return unicode_result_unchanged(self);
/* Second pass: create output string and fill it */
u = PyUnicode_New(j, PyUnicode_MAX_CHAR_VALUE(self));
if (!u)
return NULL;
dest_data = PyUnicode_DATA(u);
i = j = line_pos = 0;
for (; i < src_len; i++) {
ch = PyUnicode_READ(kind, src_data, i);
if (ch == '\t') {
if (tabsize > 0) {
incr = tabsize - (line_pos % tabsize);
line_pos += incr;
unicode_fill(kind, dest_data, ' ', j, incr);
j += incr;
}
}
else {
line_pos++;
PyUnicode_WRITE(kind, dest_data, j, ch);
j++;
if (ch == '\n' || ch == '\r')
line_pos = 0;
}
}
assert (j == PyUnicode_GET_LENGTH(u));
return unicode_result(u);
overflow:
PyErr_SetString(PyExc_OverflowError, "new string is too long");
return NULL;
}
PyDoc_STRVAR(find__doc__,
"S.find(sub[, start[, end]]) -> int\n\
\n\
Return the lowest index in S where substring sub is found,\n\
such that sub is contained within S[start:end]. Optional\n\
arguments start and end are interpreted as in slice notation.\n\
\n\
Return -1 on failure.");
static PyObject *
unicode_find(PyObject *self, PyObject *args)
{
/* initialize variables to prevent gcc warning */
PyObject *substring = NULL;
Py_ssize_t start = 0;
Py_ssize_t end = 0;
Py_ssize_t result;
if (!parse_args_finds_unicode("find", args, &substring, &start, &end))
return NULL;
if (PyUnicode_READY(self) == -1)
return NULL;
result = any_find_slice(self, substring, start, end, 1);
if (result == -2)
return NULL;
return PyLong_FromSsize_t(result);
}
static PyObject *
unicode_getitem(PyObject *self, Py_ssize_t index)
{
const void *data;
enum PyUnicode_Kind kind;
Py_UCS4 ch;
if (!PyUnicode_Check(self)) {
PyErr_BadArgument();
return NULL;
}
if (PyUnicode_READY(self) == -1) {
return NULL;
}
if (index < 0 || index >= PyUnicode_GET_LENGTH(self)) {
PyErr_SetString(PyExc_IndexError, "string index out of range");
return NULL;
}
kind = PyUnicode_KIND(self);
data = PyUnicode_DATA(self);
ch = PyUnicode_READ(kind, data, index);
return unicode_char(ch);
}
/* Believe it or not, this produces the same value for ASCII strings
as bytes_hash(). */
static Py_hash_t
unicode_hash(PyObject *self)
{
Py_uhash_t x; /* Unsigned for defined overflow behavior. */
#ifdef Py_DEBUG
assert(_Py_HashSecret_Initialized);
#endif
if (_PyUnicode_HASH(self) != -1)
return _PyUnicode_HASH(self);
if (PyUnicode_READY(self) == -1)
return -1;
x = _Py_HashBytes(PyUnicode_DATA(self),
PyUnicode_GET_LENGTH(self) * PyUnicode_KIND(self));
_PyUnicode_HASH(self) = x;
return x;
}
PyDoc_STRVAR(index__doc__,
"S.index(sub[, start[, end]]) -> int\n\
\n\
Return the lowest index in S where substring sub is found,\n\
such that sub is contained within S[start:end]. Optional\n\
arguments start and end are interpreted as in slice notation.\n\
\n\
Raises ValueError when the substring is not found.");
static PyObject *
unicode_index(PyObject *self, PyObject *args)
{
/* initialize variables to prevent gcc warning */
Py_ssize_t result;
PyObject *substring = NULL;
Py_ssize_t start = 0;
Py_ssize_t end = 0;
if (!parse_args_finds_unicode("index", args, &substring, &start, &end))
return NULL;
if (PyUnicode_READY(self) == -1)
return NULL;
result = any_find_slice(self, substring, start, end, 1);
if (result == -2)
return NULL;
if (result < 0) {
PyErr_SetString(PyExc_ValueError, "substring not found");
return NULL;
}
return PyLong_FromSsize_t(result);
}
/*[clinic input]
str.isascii as unicode_isascii
Return True if all characters in the string are ASCII, False otherwise.
ASCII characters have code points in the range U+0000-U+007F.
Empty string is ASCII too.
[clinic start generated code]*/
static PyObject *
unicode_isascii_impl(PyObject *self)
/*[clinic end generated code: output=c5910d64b5a8003f input=5a43cbc6399621d5]*/
{
if (PyUnicode_READY(self) == -1) {
return NULL;
}
return PyBool_FromLong(PyUnicode_IS_ASCII(self));
}
/*[clinic input]
str.islower as unicode_islower
Return True if the string is a lowercase string, False otherwise.
A string is lowercase if all cased characters in the string are lowercase and
there is at least one cased character in the string.
[clinic start generated code]*/
static PyObject *
unicode_islower_impl(PyObject *self)
/*[clinic end generated code: output=dbd41995bd005b81 input=acec65ac6821ae47]*/
{
Py_ssize_t i, length;
int kind;
const void *data;
int cased;
if (PyUnicode_READY(self) == -1)
return NULL;
length = PyUnicode_GET_LENGTH(self);
kind = PyUnicode_KIND(self);
data = PyUnicode_DATA(self);
/* Shortcut for single character strings */
if (length == 1)
return PyBool_FromLong(
Py_UNICODE_ISLOWER(PyUnicode_READ(kind, data, 0)));
/* Special case for empty strings */
if (length == 0)
Py_RETURN_FALSE;
cased = 0;
for (i = 0; i < length; i++) {
const Py_UCS4 ch = PyUnicode_READ(kind, data, i);
if (Py_UNICODE_ISUPPER(ch) || Py_UNICODE_ISTITLE(ch))
Py_RETURN_FALSE;
else if (!cased && Py_UNICODE_ISLOWER(ch))
cased = 1;
}
return PyBool_FromLong(cased);
}
/*[clinic input]
str.isupper as unicode_isupper
Return True if the string is an uppercase string, False otherwise.
A string is uppercase if all cased characters in the string are uppercase and
there is at least one cased character in the string.
[clinic start generated code]*/
static PyObject *
unicode_isupper_impl(PyObject *self)
/*[clinic end generated code: output=049209c8e7f15f59 input=e9b1feda5d17f2d3]*/
{
Py_ssize_t i, length;
int kind;
const void *data;
int cased;
if (PyUnicode_READY(self) == -1)
return NULL;
length = PyUnicode_GET_LENGTH(self);
kind = PyUnicode_KIND(self);
data = PyUnicode_DATA(self);
/* Shortcut for single character strings */
if (length == 1)
return PyBool_FromLong(
Py_UNICODE_ISUPPER(PyUnicode_READ(kind, data, 0)) != 0);
/* Special case for empty strings */
if (length == 0)
Py_RETURN_FALSE;
cased = 0;
for (i = 0; i < length; i++) {
const Py_UCS4 ch = PyUnicode_READ(kind, data, i);
if (Py_UNICODE_ISLOWER(ch) || Py_UNICODE_ISTITLE(ch))
Py_RETURN_FALSE;
else if (!cased && Py_UNICODE_ISUPPER(ch))
cased = 1;
}
return PyBool_FromLong(cased);
}
/*[clinic input]
str.istitle as unicode_istitle
Return True if the string is a title-cased string, False otherwise.
In a title-cased string, upper- and title-case characters may only
follow uncased characters and lowercase characters only cased ones.
[clinic start generated code]*/
static PyObject *
unicode_istitle_impl(PyObject *self)
/*[clinic end generated code: output=e9bf6eb91f5d3f0e input=98d32bd2e1f06f8c]*/
{
Py_ssize_t i, length;
int kind;
const void *data;
int cased, previous_is_cased;
if (PyUnicode_READY(self) == -1)
return NULL;
length = PyUnicode_GET_LENGTH(self);
kind = PyUnicode_KIND(self);
data = PyUnicode_DATA(self);
/* Shortcut for single character strings */
if (length == 1) {
Py_UCS4 ch = PyUnicode_READ(kind, data, 0);
return PyBool_FromLong((Py_UNICODE_ISTITLE(ch) != 0) ||
(Py_UNICODE_ISUPPER(ch) != 0));
}
/* Special case for empty strings */
if (length == 0)
Py_RETURN_FALSE;
cased = 0;
previous_is_cased = 0;
for (i = 0; i < length; i++) {
const Py_UCS4 ch = PyUnicode_READ(kind, data, i);
if (Py_UNICODE_ISUPPER(ch) || Py_UNICODE_ISTITLE(ch)) {
if (previous_is_cased)
Py_RETURN_FALSE;
previous_is_cased = 1;
cased = 1;
}
else if (Py_UNICODE_ISLOWER(ch)) {
if (!previous_is_cased)
Py_RETURN_FALSE;
previous_is_cased = 1;
cased = 1;
}
else
previous_is_cased = 0;
}
return PyBool_FromLong(cased);
}
/*[clinic input]
str.isspace as unicode_isspace
Return True if the string is a whitespace string, False otherwise.
A string is whitespace if all characters in the string are whitespace and there
is at least one character in the string.
[clinic start generated code]*/
static PyObject *
unicode_isspace_impl(PyObject *self)
/*[clinic end generated code: output=163a63bfa08ac2b9 input=fe462cb74f8437d8]*/
{
Py_ssize_t i, length;
int kind;
const void *data;
if (PyUnicode_READY(self) == -1)
return NULL;
length = PyUnicode_GET_LENGTH(self);
kind = PyUnicode_KIND(self);
data = PyUnicode_DATA(self);
/* Shortcut for single character strings */
if (length == 1)
return PyBool_FromLong(
Py_UNICODE_ISSPACE(PyUnicode_READ(kind, data, 0)));
/* Special case for empty strings */
if (length == 0)
Py_RETURN_FALSE;
for (i = 0; i < length; i++) {
const Py_UCS4 ch = PyUnicode_READ(kind, data, i);
if (!Py_UNICODE_ISSPACE(ch))
Py_RETURN_FALSE;
}
Py_RETURN_TRUE;
}
/*[clinic input]
str.isalpha as unicode_isalpha
Return True if the string is an alphabetic string, False otherwise.
A string is alphabetic if all characters in the string are alphabetic and there
is at least one character in the string.
[clinic start generated code]*/
static PyObject *
unicode_isalpha_impl(PyObject *self)
/*[clinic end generated code: output=cc81b9ac3883ec4f input=d0fd18a96cbca5eb]*/
{
Py_ssize_t i, length;
int kind;
const void *data;
if (PyUnicode_READY(self) == -1)
return NULL;
length = PyUnicode_GET_LENGTH(self);
kind = PyUnicode_KIND(self);
data = PyUnicode_DATA(self);
/* Shortcut for single character strings */
if (length == 1)
return PyBool_FromLong(
Py_UNICODE_ISALPHA(PyUnicode_READ(kind, data, 0)));
/* Special case for empty strings */
if (length == 0)
Py_RETURN_FALSE;
for (i = 0; i < length; i++) {
if (!Py_UNICODE_ISALPHA(PyUnicode_READ(kind, data, i)))
Py_RETURN_FALSE;
}
Py_RETURN_TRUE;
}
/*[clinic input]
str.isalnum as unicode_isalnum
Return True if the string is an alpha-numeric string, False otherwise.
A string is alpha-numeric if all characters in the string are alpha-numeric and
there is at least one character in the string.
[clinic start generated code]*/
static PyObject *
unicode_isalnum_impl(PyObject *self)
/*[clinic end generated code: output=a5a23490ffc3660c input=5c6579bf2e04758c]*/
{
int kind;
const void *data;
Py_ssize_t len, i;
if (PyUnicode_READY(self) == -1)
return NULL;
kind = PyUnicode_KIND(self);
data = PyUnicode_DATA(self);
len = PyUnicode_GET_LENGTH(self);
/* Shortcut for single character strings */
if (len == 1) {
const Py_UCS4 ch = PyUnicode_READ(kind, data, 0);
return PyBool_FromLong(Py_UNICODE_ISALNUM(ch));
}
/* Special case for empty strings */
if (len == 0)
Py_RETURN_FALSE;
for (i = 0; i < len; i++) {
const Py_UCS4 ch = PyUnicode_READ(kind, data, i);
if (!Py_UNICODE_ISALNUM(ch))
Py_RETURN_FALSE;
}
Py_RETURN_TRUE;
}
/*[clinic input]
str.isdecimal as unicode_isdecimal
Return True if the string is a decimal string, False otherwise.
A string is a decimal string if all characters in the string are decimal and
there is at least one character in the string.
[clinic start generated code]*/
static PyObject *
unicode_isdecimal_impl(PyObject *self)
/*[clinic end generated code: output=fb2dcdb62d3fc548 input=336bc97ab4c8268f]*/
{
Py_ssize_t i, length;
int kind;
const void *data;
if (PyUnicode_READY(self) == -1)
return NULL;
length = PyUnicode_GET_LENGTH(self);
kind = PyUnicode_KIND(self);
data = PyUnicode_DATA(self);
/* Shortcut for single character strings */
if (length == 1)
return PyBool_FromLong(
Py_UNICODE_ISDECIMAL(PyUnicode_READ(kind, data, 0)));
/* Special case for empty strings */
if (length == 0)
Py_RETURN_FALSE;
for (i = 0; i < length; i++) {
if (!Py_UNICODE_ISDECIMAL(PyUnicode_READ(kind, data, i)))
Py_RETURN_FALSE;
}
Py_RETURN_TRUE;
}
/*[clinic input]
str.isdigit as unicode_isdigit
Return True if the string is a digit string, False otherwise.
A string is a digit string if all characters in the string are digits and there
is at least one character in the string.
[clinic start generated code]*/
static PyObject *
unicode_isdigit_impl(PyObject *self)
/*[clinic end generated code: output=10a6985311da6858 input=901116c31deeea4c]*/
{
Py_ssize_t i, length;
int kind;
const void *data;
if (PyUnicode_READY(self) == -1)
return NULL;
length = PyUnicode_GET_LENGTH(self);
kind = PyUnicode_KIND(self);
data = PyUnicode_DATA(self);
/* Shortcut for single character strings */
if (length == 1) {
const Py_UCS4 ch = PyUnicode_READ(kind, data, 0);
return PyBool_FromLong(Py_UNICODE_ISDIGIT(ch));
}
/* Special case for empty strings */
if (length == 0)
Py_RETURN_FALSE;
for (i = 0; i < length; i++) {
if (!Py_UNICODE_ISDIGIT(PyUnicode_READ(kind, data, i)))
Py_RETURN_FALSE;
}
Py_RETURN_TRUE;
}
/*[clinic input]
str.isnumeric as unicode_isnumeric
Return True if the string is a numeric string, False otherwise.
A string is numeric if all characters in the string are numeric and there is at
least one character in the string.
[clinic start generated code]*/
static PyObject *
unicode_isnumeric_impl(PyObject *self)
/*[clinic end generated code: output=9172a32d9013051a input=722507db976f826c]*/
{
Py_ssize_t i, length;
int kind;
const void *data;
if (PyUnicode_READY(self) == -1)
return NULL;
length = PyUnicode_GET_LENGTH(self);
kind = PyUnicode_KIND(self);
data = PyUnicode_DATA(self);
/* Shortcut for single character strings */
if (length == 1)
return PyBool_FromLong(
Py_UNICODE_ISNUMERIC(PyUnicode_READ(kind, data, 0)));
/* Special case for empty strings */
if (length == 0)
Py_RETURN_FALSE;
for (i = 0; i < length; i++) {
if (!Py_UNICODE_ISNUMERIC(PyUnicode_READ(kind, data, i)))
Py_RETURN_FALSE;
}
Py_RETURN_TRUE;
}
Py_ssize_t
_PyUnicode_ScanIdentifier(PyObject *self)
{
Py_ssize_t i;
if (PyUnicode_READY(self) == -1)
return -1;
Py_ssize_t len = PyUnicode_GET_LENGTH(self);
if (len == 0) {
/* an empty string is not a valid identifier */
return 0;
}
int kind = PyUnicode_KIND(self);
const void *data = PyUnicode_DATA(self);
Py_UCS4 ch = PyUnicode_READ(kind, data, 0);
/* PEP 3131 says that the first character must be in
XID_Start and subsequent characters in XID_Continue,
and for the ASCII range, the 2.x rules apply (i.e
start with letters and underscore, continue with
letters, digits, underscore). However, given the current
definition of XID_Start and XID_Continue, it is sufficient
to check just for these, except that _ must be allowed
as starting an identifier. */
if (!_PyUnicode_IsXidStart(ch) && ch != 0x5F /* LOW LINE */) {
return 0;
}
for (i = 1; i < len; i++) {
ch = PyUnicode_READ(kind, data, i);
if (!_PyUnicode_IsXidContinue(ch)) {
return i;
}
}
return i;
}
int
PyUnicode_IsIdentifier(PyObject *self)
{
if (PyUnicode_IS_READY(self)) {
Py_ssize_t i = _PyUnicode_ScanIdentifier(self);
Py_ssize_t len = PyUnicode_GET_LENGTH(self);
/* an empty string is not a valid identifier */
return len && i == len;
}
else {
_Py_COMP_DIAG_PUSH
_Py_COMP_DIAG_IGNORE_DEPR_DECLS
Py_ssize_t i = 0, len = PyUnicode_GET_SIZE(self);
if (len == 0) {
/* an empty string is not a valid identifier */
return 0;
}
const wchar_t *wstr = _PyUnicode_WSTR(self);
Py_UCS4 ch = wstr[i++];
#if SIZEOF_WCHAR_T == 2
if (Py_UNICODE_IS_HIGH_SURROGATE(ch)
&& i < len
&& Py_UNICODE_IS_LOW_SURROGATE(wstr[i]))
{
ch = Py_UNICODE_JOIN_SURROGATES(ch, wstr[i]);
i++;
}
#endif
if (!_PyUnicode_IsXidStart(ch) && ch != 0x5F /* LOW LINE */) {
return 0;
}
while (i < len) {
ch = wstr[i++];
#if SIZEOF_WCHAR_T == 2
if (Py_UNICODE_IS_HIGH_SURROGATE(ch)
&& i < len
&& Py_UNICODE_IS_LOW_SURROGATE(wstr[i]))
{
ch = Py_UNICODE_JOIN_SURROGATES(ch, wstr[i]);
i++;
}
#endif
if (!_PyUnicode_IsXidContinue(ch)) {
return 0;
}
}
return 1;
_Py_COMP_DIAG_POP
}
}
/*[clinic input]
str.isidentifier as unicode_isidentifier
Return True if the string is a valid Python identifier, False otherwise.
Call keyword.iskeyword(s) to test whether string s is a reserved identifier,
such as "def" or "class".
[clinic start generated code]*/
static PyObject *
unicode_isidentifier_impl(PyObject *self)
/*[clinic end generated code: output=fe585a9666572905 input=2d807a104f21c0c5]*/
{
return PyBool_FromLong(PyUnicode_IsIdentifier(self));
}
/*[clinic input]
str.isprintable as unicode_isprintable
Return True if the string is printable, False otherwise.
A string is printable if all of its characters are considered printable in
repr() or if it is empty.
[clinic start generated code]*/
static PyObject *
unicode_isprintable_impl(PyObject *self)
/*[clinic end generated code: output=3ab9626cd32dd1a0 input=98a0e1c2c1813209]*/
{
Py_ssize_t i, length;
int kind;
const void *data;
if (PyUnicode_READY(self) == -1)
return NULL;
length = PyUnicode_GET_LENGTH(self);
kind = PyUnicode_KIND(self);
data = PyUnicode_DATA(self);
/* Shortcut for single character strings */
if (length == 1)
return PyBool_FromLong(
Py_UNICODE_ISPRINTABLE(PyUnicode_READ(kind, data, 0)));
for (i = 0; i < length; i++) {
if (!Py_UNICODE_ISPRINTABLE(PyUnicode_READ(kind, data, i))) {
Py_RETURN_FALSE;
}
}
Py_RETURN_TRUE;
}
/*[clinic input]
str.join as unicode_join
iterable: object
/
Concatenate any number of strings.
The string whose method is called is inserted in between each given string.
The result is returned as a new string.
Example: '.'.join(['ab', 'pq', 'rs']) -> 'ab.pq.rs'
[clinic start generated code]*/
static PyObject *
unicode_join(PyObject *self, PyObject *iterable)
/*[clinic end generated code: output=6857e7cecfe7bf98 input=2f70422bfb8fa189]*/
{
return PyUnicode_Join(self, iterable);
}
static Py_ssize_t
unicode_length(PyObject *self)
{
if (PyUnicode_READY(self) == -1)
return -1;
return PyUnicode_GET_LENGTH(self);
}
/*[clinic input]
str.ljust as unicode_ljust
width: Py_ssize_t
fillchar: Py_UCS4 = ' '
/
Return a left-justified string of length width.
Padding is done using the specified fill character (default is a space).
[clinic start generated code]*/
static PyObject *
unicode_ljust_impl(PyObject *self, Py_ssize_t width, Py_UCS4 fillchar)
/*[clinic end generated code: output=1cce0e0e0a0b84b3 input=3ab599e335e60a32]*/
{
if (PyUnicode_READY(self) == -1)
return NULL;
if (PyUnicode_GET_LENGTH(self) >= width)
return unicode_result_unchanged(self);
return pad(self, 0, width - PyUnicode_GET_LENGTH(self), fillchar);
}
/*[clinic input]
str.lower as unicode_lower
Return a copy of the string converted to lowercase.
[clinic start generated code]*/
static PyObject *
unicode_lower_impl(PyObject *self)
/*[clinic end generated code: output=84ef9ed42efad663 input=60a2984b8beff23a]*/
{
if (PyUnicode_READY(self) == -1)
return NULL;
if (PyUnicode_IS_ASCII(self))
return ascii_upper_or_lower(self, 1);
return case_operation(self, do_lower);
}
#define LEFTSTRIP 0
#define RIGHTSTRIP 1
#define BOTHSTRIP 2
/* Arrays indexed by above */
static const char *stripfuncnames[] = {"lstrip", "rstrip", "strip"};
#define STRIPNAME(i) (stripfuncnames[i])
/* externally visible for str.strip(unicode) */
PyObject *
_PyUnicode_XStrip(PyObject *self, int striptype, PyObject *sepobj)
{
const void *data;
int kind;
Py_ssize_t i, j, len;
BLOOM_MASK sepmask;
Py_ssize_t seplen;
if (PyUnicode_READY(self) == -1 || PyUnicode_READY(sepobj) == -1)
return NULL;
kind = PyUnicode_KIND(self);
data = PyUnicode_DATA(self);
len = PyUnicode_GET_LENGTH(self);
seplen = PyUnicode_GET_LENGTH(sepobj);
sepmask = make_bloom_mask(PyUnicode_KIND(sepobj),
PyUnicode_DATA(sepobj),
seplen);
i = 0;
if (striptype != RIGHTSTRIP) {
while (i < len) {
Py_UCS4 ch = PyUnicode_READ(kind, data, i);
if (!BLOOM(sepmask, ch))
break;
if (PyUnicode_FindChar(sepobj, ch, 0, seplen, 1) < 0)
break;
i++;
}
}
j = len;
if (striptype != LEFTSTRIP) {
j--;
while (j >= i) {
Py_UCS4 ch = PyUnicode_READ(kind, data, j);
if (!BLOOM(sepmask, ch))
break;
if (PyUnicode_FindChar(sepobj, ch, 0, seplen, 1) < 0)
break;
j--;
}
j++;
}
return PyUnicode_Substring(self, i, j);
}
PyObject*
PyUnicode_Substring(PyObject *self, Py_ssize_t start, Py_ssize_t end)
{
const unsigned char *data;
int kind;
Py_ssize_t length;
if (PyUnicode_READY(self) == -1)
return NULL;
length = PyUnicode_GET_LENGTH(self);
end = Py_MIN(end, length);
if (start == 0 && end == length)
return unicode_result_unchanged(self);
if (start < 0 || end < 0) {
PyErr_SetString(PyExc_IndexError, "string index out of range");
return NULL;
}
if (start >= length || end < start)
_Py_RETURN_UNICODE_EMPTY();
length = end - start;
if (PyUnicode_IS_ASCII(self)) {
data = PyUnicode_1BYTE_DATA(self);
return _PyUnicode_FromASCII((const char*)(data + start), length);
}
else {
kind = PyUnicode_KIND(self);
data = PyUnicode_1BYTE_DATA(self);
return PyUnicode_FromKindAndData(kind,
data + kind * start,
length);
}
}
static PyObject *
do_strip(PyObject *self, int striptype)
{
Py_ssize_t len, i, j;
if (PyUnicode_READY(self) == -1)
return NULL;
len = PyUnicode_GET_LENGTH(self);
if (PyUnicode_IS_ASCII(self)) {
const Py_UCS1 *data = PyUnicode_1BYTE_DATA(self);
i = 0;
if (striptype != RIGHTSTRIP) {
while (i < len) {
Py_UCS1 ch = data[i];
if (!_Py_ascii_whitespace[ch])
break;
i++;
}
}
j = len;
if (striptype != LEFTSTRIP) {
j--;
while (j >= i) {
Py_UCS1 ch = data[j];
if (!_Py_ascii_whitespace[ch])
break;
j--;
}
j++;
}
}
else {
int kind = PyUnicode_KIND(self);
const void *data = PyUnicode_DATA(self);
i = 0;
if (striptype != RIGHTSTRIP) {
while (i < len) {
Py_UCS4 ch = PyUnicode_READ(kind, data, i);
if (!Py_UNICODE_ISSPACE(ch))
break;
i++;
}
}
j = len;
if (striptype != LEFTSTRIP) {
j--;
while (j >= i) {
Py_UCS4 ch = PyUnicode_READ(kind, data, j);
if (!Py_UNICODE_ISSPACE(ch))
break;
j--;
}
j++;
}
}
return PyUnicode_Substring(self, i, j);
}
static PyObject *
do_argstrip(PyObject *self, int striptype, PyObject *sep)
{
if (sep != Py_None) {
if (PyUnicode_Check(sep))
return _PyUnicode_XStrip(self, striptype, sep);
else {
PyErr_Format(PyExc_TypeError,
"%s arg must be None or str",
STRIPNAME(striptype));
return NULL;
}
}
return do_strip(self, striptype);
}
/*[clinic input]
str.strip as unicode_strip
chars: object = None
/
Return a copy of the string with leading and trailing whitespace removed.
If chars is given and not None, remove characters in chars instead.
[clinic start generated code]*/
static PyObject *
unicode_strip_impl(PyObject *self, PyObject *chars)
/*[clinic end generated code: output=ca19018454345d57 input=385289c6f423b954]*/
{
return do_argstrip(self, BOTHSTRIP, chars);
}
/*[clinic input]
str.lstrip as unicode_lstrip
chars: object = None
/
Return a copy of the string with leading whitespace removed.
If chars is given and not None, remove characters in chars instead.
[clinic start generated code]*/
static PyObject *
unicode_lstrip_impl(PyObject *self, PyObject *chars)
/*[clinic end generated code: output=3b43683251f79ca7 input=529f9f3834448671]*/
{
return do_argstrip(self, LEFTSTRIP, chars);
}
/*[clinic input]
str.rstrip as unicode_rstrip
chars: object = None
/
Return a copy of the string with trailing whitespace removed.
If chars is given and not None, remove characters in chars instead.
[clinic start generated code]*/
static PyObject *
unicode_rstrip_impl(PyObject *self, PyObject *chars)
/*[clinic end generated code: output=4a59230017cc3b7a input=62566c627916557f]*/
{
return do_argstrip(self, RIGHTSTRIP, chars);
}
static PyObject*
unicode_repeat(PyObject *str, Py_ssize_t len)
{
PyObject *u;
Py_ssize_t nchars, n;
if (len < 1)
_Py_RETURN_UNICODE_EMPTY();
/* no repeat, return original string */
if (len == 1)
return unicode_result_unchanged(str);
if (PyUnicode_READY(str) == -1)
return NULL;
if (PyUnicode_GET_LENGTH(str) > PY_SSIZE_T_MAX / len) {
PyErr_SetString(PyExc_OverflowError,
"repeated string is too long");
return NULL;
}
nchars = len * PyUnicode_GET_LENGTH(str);
u = PyUnicode_New(nchars, PyUnicode_MAX_CHAR_VALUE(str));
if (!u)
return NULL;
assert(PyUnicode_KIND(u) == PyUnicode_KIND(str));
if (PyUnicode_GET_LENGTH(str) == 1) {
int kind = PyUnicode_KIND(str);
Py_UCS4 fill_char = PyUnicode_READ(kind, PyUnicode_DATA(str), 0);
if (kind == PyUnicode_1BYTE_KIND) {
void *to = PyUnicode_DATA(u);
memset(to, (unsigned char)fill_char, len);
}
else if (kind == PyUnicode_2BYTE_KIND) {
Py_UCS2 *ucs2 = PyUnicode_2BYTE_DATA(u);
for (n = 0; n < len; ++n)
ucs2[n] = fill_char;
} else {
Py_UCS4 *ucs4 = PyUnicode_4BYTE_DATA(u);
assert(kind == PyUnicode_4BYTE_KIND);
for (n = 0; n < len; ++n)
ucs4[n] = fill_char;
}
}
else {
/* number of characters copied this far */
Py_ssize_t done = PyUnicode_GET_LENGTH(str);
Py_ssize_t char_size = PyUnicode_KIND(str);
char *to = (char *) PyUnicode_DATA(u);
memcpy(to, PyUnicode_DATA(str),
PyUnicode_GET_LENGTH(str) * char_size);
while (done < nchars) {
n = (done > 4) & 0x000F]);
PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[ch & 0x000F]);
}
/* Copy ASCII characters as-is */
else if (ch < 0x7F) {
PyUnicode_WRITE(okind, odata, o++, ch);
}
/* Non-ASCII characters */
else {
/* Map Unicode whitespace and control characters
(categories Z* and C* except ASCII space)
*/
if (!Py_UNICODE_ISPRINTABLE(ch)) {
PyUnicode_WRITE(okind, odata, o++, '\\');
/* Map 8-bit characters to '\xhh' */
if (ch > 4) & 0x000F]);
PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[ch & 0x000F]);
}
/* Map 16-bit characters to '\uxxxx' */
else if (ch > 12) & 0xF]);
PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[(ch >> 8) & 0xF]);
PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[(ch >> 4) & 0xF]);
PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[ch & 0xF]);
}
/* Map 21-bit characters to '\U00xxxxxx' */
else {
PyUnicode_WRITE(okind, odata, o++, 'U');
PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[(ch >> 28) & 0xF]);
PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[(ch >> 24) & 0xF]);
PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[(ch >> 20) & 0xF]);
PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[(ch >> 16) & 0xF]);
PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[(ch >> 12) & 0xF]);
PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[(ch >> 8) & 0xF]);
PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[(ch >> 4) & 0xF]);
PyUnicode_WRITE(okind, odata, o++, Py_hexdigits[ch & 0xF]);
}
}
/* Copy characters as-is */
else {
PyUnicode_WRITE(okind, odata, o++, ch);
}
}
}
}
/* Closing quote already added at the beginning */
assert(_PyUnicode_CheckConsistency(repr, 1));
return repr;
}
PyDoc_STRVAR(rfind__doc__,
"S.rfind(sub[, start[, end]]) -> int\n\
\n\
Return the highest index in S where substring sub is found,\n\
such that sub is contained within S[start:end]. Optional\n\
arguments start and end are interpreted as in slice notation.\n\
\n\
Return -1 on failure.");
static PyObject *
unicode_rfind(PyObject *self, PyObject *args)
{
/* initialize variables to prevent gcc warning */
PyObject *substring = NULL;
Py_ssize_t start = 0;
Py_ssize_t end = 0;
Py_ssize_t result;
if (!parse_args_finds_unicode("rfind", args, &substring, &start, &end))
return NULL;
if (PyUnicode_READY(self) == -1)
return NULL;
result = any_find_slice(self, substring, start, end, -1);
if (result == -2)
return NULL;
return PyLong_FromSsize_t(result);
}
PyDoc_STRVAR(rindex__doc__,
"S.rindex(sub[, start[, end]]) -> int\n\
\n\
Return the highest index in S where substring sub is found,\n\
such that sub is contained within S[start:end]. Optional\n\
arguments start and end are interpreted as in slice notation.\n\
\n\
Raises ValueError when the substring is not found.");
static PyObject *
unicode_rindex(PyObject *self, PyObject *args)
{
/* initialize variables to prevent gcc warning */
PyObject *substring = NULL;
Py_ssize_t start = 0;
Py_ssize_t end = 0;
Py_ssize_t result;
if (!parse_args_finds_unicode("rindex", args, &substring, &start, &end))
return NULL;
if (PyUnicode_READY(self) == -1)
return NULL;
result = any_find_slice(self, substring, start, end, -1);
if (result == -2)
return NULL;
if (result < 0) {
PyErr_SetString(PyExc_ValueError, "substring not found");
return NULL;
}
return PyLong_FromSsize_t(result);
}
/*[clinic input]
str.rjust as unicode_rjust
width: Py_ssize_t
fillchar: Py_UCS4 = ' '
/
Return a right-justified string of length width.
Padding is done using the specified fill character (default is a space).
[clinic start generated code]*/
static PyObject *
unicode_rjust_impl(PyObject *self, Py_ssize_t width, Py_UCS4 fillchar)
/*[clinic end generated code: output=804a1a57fbe8d5cf input=d05f550b5beb1f72]*/
{
if (PyUnicode_READY(self) == -1)
return NULL;
if (PyUnicode_GET_LENGTH(self) >= width)
return unicode_result_unchanged(self);
return pad(self, width - PyUnicode_GET_LENGTH(self), 0, fillchar);
}
PyObject *
PyUnicode_Split(PyObject *s, PyObject *sep, Py_ssize_t maxsplit)
{
if (ensure_unicode(s) < 0 || (sep != NULL && ensure_unicode(sep) < 0))
return NULL;
return split(s, sep, maxsplit);
}
/*[clinic input]
str.split as unicode_split
sep: object = None
The delimiter according which to split the string.
None (the default value) means split according to any whitespace,
and discard empty strings from the result.
maxsplit: Py_ssize_t = -1
Maximum number of splits to do.
-1 (the default value) means no limit.
Return a list of the words in the string, using sep as the delimiter string.
[clinic start generated code]*/
static PyObject *
unicode_split_impl(PyObject *self, PyObject *sep, Py_ssize_t maxsplit)
/*[clinic end generated code: output=3a65b1db356948dc input=606e750488a82359]*/
{
if (sep == Py_None)
return split(self, NULL, maxsplit);
if (PyUnicode_Check(sep))
return split(self, sep, maxsplit);
PyErr_Format(PyExc_TypeError,
"must be str or None, not %.100s",
Py_TYPE(sep)->tp_name);
return NULL;
}
PyObject *
PyUnicode_Partition(PyObject *str_obj, PyObject *sep_obj)
{
PyObject* out;
int kind1, kind2;
const void *buf1, *buf2;
Py_ssize_t len1, len2;
if (ensure_unicode(str_obj) < 0 || ensure_unicode(sep_obj) < 0)
return NULL;
kind1 = PyUnicode_KIND(str_obj);
kind2 = PyUnicode_KIND(sep_obj);
len1 = PyUnicode_GET_LENGTH(str_obj);
len2 = PyUnicode_GET_LENGTH(sep_obj);
if (kind1 < kind2 || len1 < len2) {
_Py_INCREF_UNICODE_EMPTY();
if (!unicode_empty)
out = NULL;
else {
out = PyTuple_Pack(3, str_obj, unicode_empty, unicode_empty);
Py_DECREF(unicode_empty);
}
return out;
}
buf1 = PyUnicode_DATA(str_obj);
buf2 = PyUnicode_DATA(sep_obj);
if (kind2 != kind1) {
buf2 = unicode_askind(kind2, buf2, len2, kind1);
if (!buf2)
return NULL;
}
switch (kind1) {
case PyUnicode_1BYTE_KIND:
if (PyUnicode_IS_ASCII(str_obj) && PyUnicode_IS_ASCII(sep_obj))
out = asciilib_partition(str_obj, buf1, len1, sep_obj, buf2, len2);
else
out = ucs1lib_partition(str_obj, buf1, len1, sep_obj, buf2, len2);
break;
case PyUnicode_2BYTE_KIND:
out = ucs2lib_partition(str_obj, buf1, len1, sep_obj, buf2, len2);
break;
case PyUnicode_4BYTE_KIND:
out = ucs4lib_partition(str_obj, buf1, len1, sep_obj, buf2, len2);
break;
default:
Py_UNREACHABLE();
}
assert((kind2 == kind1) == (buf2 == PyUnicode_DATA(sep_obj)));
if (kind2 != kind1)
PyMem_Free((void *)buf2);
return out;
}
PyObject *
PyUnicode_RPartition(PyObject *str_obj, PyObject *sep_obj)
{
PyObject* out;
int kind1, kind2;
const void *buf1, *buf2;
Py_ssize_t len1, len2;
if (ensure_unicode(str_obj) < 0 || ensure_unicode(sep_obj) < 0)
return NULL;
kind1 = PyUnicode_KIND(str_obj);
kind2 = PyUnicode_KIND(sep_obj);
len1 = PyUnicode_GET_LENGTH(str_obj);
len2 = PyUnicode_GET_LENGTH(sep_obj);
if (kind1 < kind2 || len1 < len2) {
_Py_INCREF_UNICODE_EMPTY();
if (!unicode_empty)
out = NULL;
else {
out = PyTuple_Pack(3, unicode_empty, unicode_empty, str_obj);
Py_DECREF(unicode_empty);
}
return out;
}
buf1 = PyUnicode_DATA(str_obj);
buf2 = PyUnicode_DATA(sep_obj);
if (kind2 != kind1) {
buf2 = unicode_askind(kind2, buf2, len2, kind1);
if (!buf2)
return NULL;
}
switch (kind1) {
case PyUnicode_1BYTE_KIND:
if (PyUnicode_IS_ASCII(str_obj) && PyUnicode_IS_ASCII(sep_obj))
out = asciilib_rpartition(str_obj, buf1, len1, sep_obj, buf2, len2);
else
out = ucs1lib_rpartition(str_obj, buf1, len1, sep_obj, buf2, len2);
break;
case PyUnicode_2BYTE_KIND:
out = ucs2lib_rpartition(str_obj, buf1, len1, sep_obj, buf2, len2);
break;
case PyUnicode_4BYTE_KIND:
out = ucs4lib_rpartition(str_obj, buf1, len1, sep_obj, buf2, len2);
break;
default:
Py_UNREACHABLE();
}
assert((kind2 == kind1) == (buf2 == PyUnicode_DATA(sep_obj)));
if (kind2 != kind1)
PyMem_Free((void *)buf2);
return out;
}
/*[clinic input]
str.partition as unicode_partition
sep: object
/
Partition the string into three parts using the given separator.
This will search for the separator in the string. If the separator is found,
returns a 3-tuple containing the part before the separator, the separator
itself, and the part after it.
If the separator is not found, returns a 3-tuple containing the original string
and two empty strings.
[clinic start generated code]*/
static PyObject *
unicode_partition(PyObject *self, PyObject *sep)
/*[clinic end generated code: output=e4ced7bd253ca3c4 input=f29b8d06c63e50be]*/
{
return PyUnicode_Partition(self, sep);
}
/*[clinic input]
str.rpartition as unicode_rpartition = str.partition
Partition the string into three parts using the given separator.
This will search for the separator in the string, starting at the end. If
the separator is found, returns a 3-tuple containing the part before the
separator, the separator itself, and the part after it.
If the separator is not found, returns a 3-tuple containing two empty strings
and the original string.
[clinic start generated code]*/
static PyObject *
unicode_rpartition(PyObject *self, PyObject *sep)
/*[clinic end generated code: output=1aa13cf1156572aa input=c4b7db3ef5cf336a]*/
{
return PyUnicode_RPartition(self, sep);
}
PyObject *
PyUnicode_RSplit(PyObject *s, PyObject *sep, Py_ssize_t maxsplit)
{
if (ensure_unicode(s) < 0 || (sep != NULL && ensure_unicode(sep) < 0))
return NULL;
return rsplit(s, sep, maxsplit);
}
/*[clinic input]
str.rsplit as unicode_rsplit = str.split
Return a list of the words in the string, using sep as the delimiter string.
Splits are done starting at the end of the string and working to the front.
[clinic start generated code]*/
static PyObject *
unicode_rsplit_impl(PyObject *self, PyObject *sep, Py_ssize_t maxsplit)
/*[clinic end generated code: output=c2b815c63bcabffc input=12ad4bf57dd35f15]*/
{
if (sep == Py_None)
return rsplit(self, NULL, maxsplit);
if (PyUnicode_Check(sep))
return rsplit(self, sep, maxsplit);
PyErr_Format(PyExc_TypeError,
"must be str or None, not %.100s",
Py_TYPE(sep)->tp_name);
return NULL;
}
/*[clinic input]
str.splitlines as unicode_splitlines
keepends: bool(accept={int}) = False
Return a list of the lines in the string, breaking at line boundaries.
Line breaks are not included in the resulting list unless keepends is given and
true.
[clinic start generated code]*/
static PyObject *
unicode_splitlines_impl(PyObject *self, int keepends)
/*[clinic end generated code: output=f664dcdad153ec40 input=b508e180459bdd8b]*/
{
return PyUnicode_Splitlines(self, keepends);
}
static
PyObject *unicode_str(PyObject *self)
{
return unicode_result_unchanged(self);
}
/*[clinic input]
str.swapcase as unicode_swapcase
Convert uppercase characters to lowercase and lowercase characters to uppercase.
[clinic start generated code]*/
static PyObject *
unicode_swapcase_impl(PyObject *self)
/*[clinic end generated code: output=5d28966bf6d7b2af input=3f3ef96d5798a7bb]*/
{
if (PyUnicode_READY(self) == -1)
return NULL;
return case_operation(self, do_swapcase);
}
/*[clinic input]
@staticmethod
str.maketrans as unicode_maketrans
x: object
y: unicode=NULL
z: unicode=NULL
/
Return a translation table usable for str.translate().
If there is only one argument, it must be a dictionary mapping Unicode
ordinals (integers) or characters to Unicode ordinals, strings or None.
Character keys will be then converted to ordinals.
If there are two arguments, they must be strings of equal length, and
in the resulting dictionary, each character in x will be mapped to the
character at the same position in y. If there is a third argument, it
must be a string, whose characters will be mapped to None in the result.
[clinic start generated code]*/
static PyObject *
unicode_maketrans_impl(PyObject *x, PyObject *y, PyObject *z)
/*[clinic end generated code: output=a925c89452bd5881 input=7bfbf529a293c6c5]*/
{
PyObject *new = NULL, *key, *value;
Py_ssize_t i = 0;
int res;
new = PyDict_New();
if (!new)
return NULL;
if (y != NULL) {
int x_kind, y_kind, z_kind;
const void *x_data, *y_data, *z_data;
/* x must be a string too, of equal length */
if (!PyUnicode_Check(x)) {
PyErr_SetString(PyExc_TypeError, "first maketrans argument must "
"be a string if there is a second argument");
goto err;
}
if (PyUnicode_GET_LENGTH(x) != PyUnicode_GET_LENGTH(y)) {
PyErr_SetString(PyExc_ValueError, "the first two maketrans "
"arguments must have equal length");
goto err;
}
/* create entries for translating chars in x to those in y */
x_kind = PyUnicode_KIND(x);
y_kind = PyUnicode_KIND(y);
x_data = PyUnicode_DATA(x);
y_data = PyUnicode_DATA(y);
for (i = 0; i < PyUnicode_GET_LENGTH(x); i++) {
key = PyLong_FromLong(PyUnicode_READ(x_kind, x_data, i));
if (!key)
goto err;
value = PyLong_FromLong(PyUnicode_READ(y_kind, y_data, i));
if (!value) {
Py_DECREF(key);
goto err;
}
res = PyDict_SetItem(new, key, value);
Py_DECREF(key);
Py_DECREF(value);
if (res < 0)
goto err;
}
/* create entries for deleting chars in z */
if (z != NULL) {
z_kind = PyUnicode_KIND(z);
z_data = PyUnicode_DATA(z);
for (i = 0; i < PyUnicode_GET_LENGTH(z); i++) {
key = PyLong_FromLong(PyUnicode_READ(z_kind, z_data, i));
if (!key)
goto err;
res = PyDict_SetItem(new, key, Py_None);
Py_DECREF(key);
if (res < 0)
goto err;
}
}
} else {
int kind;
const void *data;
/* x must be a dict */
if (!PyDict_CheckExact(x)) {
PyErr_SetString(PyExc_TypeError, "if you give only one argument "
"to maketrans it must be a dict");
goto err;
}
/* copy entries into the new dict, converting string keys to int keys */
while (PyDict_Next(x, &i, &key, &value)) {
if (PyUnicode_Check(key)) {
/* convert string keys to integer keys */
PyObject *newkey;
if (PyUnicode_GET_LENGTH(key) != 1) {
PyErr_SetString(PyExc_ValueError, "string keys in translate "
"table must be of length 1");
goto err;
}
kind = PyUnicode_KIND(key);
data = PyUnicode_DATA(key);
newkey = PyLong_FromLong(PyUnicode_READ(kind, data, 0));
if (!newkey)
goto err;
res = PyDict_SetItem(new, newkey, value);
Py_DECREF(newkey);
if (res < 0)
goto err;
} else if (PyLong_Check(key)) {
/* just keep integer keys */
if (PyDict_SetItem(new, key, value) < 0)
goto err;
} else {
PyErr_SetString(PyExc_TypeError, "keys in translate table must "
"be strings or integers");
goto err;
}
}
}
return new;
err:
Py_DECREF(new);
return NULL;
}
/*[clinic input]
str.translate as unicode_translate
table: object
Translation table, which must be a mapping of Unicode ordinals to
Unicode ordinals, strings, or None.
/
Replace each character in the string using the given translation table.
The table must implement lookup/indexing via __getitem__, for instance a
dictionary or list. If this operation raises LookupError, the character is
left untouched. Characters mapped to None are deleted.
[clinic start generated code]*/
static PyObject *
unicode_translate(PyObject *self, PyObject *table)
/*[clinic end generated code: output=3cb448ff2fd96bf3 input=6d38343db63d8eb0]*/
{
return _PyUnicode_TranslateCharmap(self, table, "ignore");
}
/*[clinic input]
str.upper as unicode_upper
Return a copy of the string converted to uppercase.
[clinic start generated code]*/
static PyObject *
unicode_upper_impl(PyObject *self)
/*[clinic end generated code: output=1b7ddd16bbcdc092 input=db3d55682dfe2e6c]*/
{
if (PyUnicode_READY(self) == -1)
return NULL;
if (PyUnicode_IS_ASCII(self))
return ascii_upper_or_lower(self, 0);
return case_operation(self, do_upper);
}
/*[clinic input]
str.zfill as unicode_zfill
width: Py_ssize_t
/
Pad a numeric string with zeros on the left, to fill a field of the given width.
The string is never truncated.
[clinic start generated code]*/
static PyObject *
unicode_zfill_impl(PyObject *self, Py_ssize_t width)
/*[clinic end generated code: output=e13fb6bdf8e3b9df input=c6b2f772c6f27799]*/
{
Py_ssize_t fill;
PyObject *u;
int kind;
const void *data;
Py_UCS4 chr;
if (PyUnicode_READY(self) == -1)
return NULL;
if (PyUnicode_GET_LENGTH(self) >= width)
return unicode_result_unchanged(self);
fill = width - PyUnicode_GET_LENGTH(self);
u = pad(self, fill, 0, '0');
if (u == NULL)
return NULL;
kind = PyUnicode_KIND(u);
data = PyUnicode_DATA(u);
chr = PyUnicode_READ(kind, data, fill);
if (chr == '+' || chr == '-') {
/* move sign to beginning of string */
PyUnicode_WRITE(kind, data, 0, chr);
PyUnicode_WRITE(kind, data, fill, '0');
}
assert(_PyUnicode_CheckConsistency(u, 1));
return u;
}
#if 0
static PyObject *
unicode__decimal2ascii(PyObject *self)
{
return PyUnicode_TransformDecimalAndSpaceToASCII(self);
}
#endif
PyDoc_STRVAR(startswith__doc__,
"S.startswith(prefix[, start[, end]]) -> bool\n\
\n\
Return True if S starts with the specified prefix, False otherwise.\n\
With optional start, test S beginning at that position.\n\
With optional end, stop comparing S at that position.\n\
prefix can also be a tuple of strings to try.");
static PyObject *
unicode_startswith(PyObject *self,
PyObject *args)
{
PyObject *subobj;
PyObject *substring;
Py_ssize_t start = 0;
Py_ssize_t end = PY_SSIZE_T_MAX;
int result;
if (!stringlib_parse_args_finds("startswith", args, &subobj, &start, &end))
return NULL;
if (PyTuple_Check(subobj)) {
Py_ssize_t i;
for (i = 0; i < PyTuple_GET_SIZE(subobj); i++) {
substring = PyTuple_GET_ITEM(subobj, i);
if (!PyUnicode_Check(substring)) {
PyErr_Format(PyExc_TypeError,
"tuple for startswith must only contain str, "
"not %.100s",
Py_TYPE(substring)->tp_name);
return NULL;
}
result = tailmatch(self, substring, start, end, -1);
if (result == -1)
return NULL;
if (result) {
Py_RETURN_TRUE;
}
}
/* nothing matched */
Py_RETURN_FALSE;
}
if (!PyUnicode_Check(subobj)) {
PyErr_Format(PyExc_TypeError,
"startswith first arg must be str or "
"a tuple of str, not %.100s", Py_TYPE(subobj)->tp_name);
return NULL;
}
result = tailmatch(self, subobj, start, end, -1);
if (result == -1)
return NULL;
return PyBool_FromLong(result);
}
PyDoc_STRVAR(endswith__doc__,
"S.endswith(suffix[, start[, end]]) -> bool\n\
\n\
Return True if S ends with the specified suffix, False otherwise.\n\
With optional start, test S beginning at that position.\n\
With optional end, stop comparing S at that position.\n\
suffix can also be a tuple of strings to try.");
static PyObject *
unicode_endswith(PyObject *self,
PyObject *args)
{
PyObject *subobj;
PyObject *substring;
Py_ssize_t start = 0;
Py_ssize_t end = PY_SSIZE_T_MAX;
int result;
if (!stringlib_parse_args_finds("endswith", args, &subobj, &start, &end))
return NULL;
if (PyTuple_Check(subobj)) {
Py_ssize_t i;
for (i = 0; i < PyTuple_GET_SIZE(subobj); i++) {
substring = PyTuple_GET_ITEM(subobj, i);
if (!PyUnicode_Check(substring)) {
PyErr_Format(PyExc_TypeError,
"tuple for endswith must only contain str, "
"not %.100s",
Py_TYPE(substring)->tp_name);
return NULL;
}
result = tailmatch(self, substring, start, end, +1);
if (result == -1)
return NULL;
if (result) {
Py_RETURN_TRUE;
}
}
Py_RETURN_FALSE;
}
if (!PyUnicode_Check(subobj)) {
PyErr_Format(PyExc_TypeError,
"endswith first arg must be str or "
"a tuple of str, not %.100s", Py_TYPE(subobj)->tp_name);
return NULL;
}
result = tailmatch(self, subobj, start, end, +1);
if (result == -1)
return NULL;
return PyBool_FromLong(result);
}
static inline void
_PyUnicodeWriter_Update(_PyUnicodeWriter *writer)
{
writer->maxchar = PyUnicode_MAX_CHAR_VALUE(writer->buffer);
writer->data = PyUnicode_DATA(writer->buffer);
if (!writer->readonly) {
writer->kind = PyUnicode_KIND(writer->buffer);
writer->size = PyUnicode_GET_LENGTH(writer->buffer);
}
else {
/* use a value smaller than PyUnicode_1BYTE_KIND() so
_PyUnicodeWriter_PrepareKind() will copy the buffer. */
writer->kind = PyUnicode_WCHAR_KIND;
assert(writer->kind size = 0;
}
}
void
_PyUnicodeWriter_Init(_PyUnicodeWriter *writer)
{
memset(writer, 0, sizeof(*writer));
/* ASCII is the bare minimum */
writer->min_char = 127;
/* use a value smaller than PyUnicode_1BYTE_KIND() so
_PyUnicodeWriter_PrepareKind() will copy the buffer. */
writer->kind = PyUnicode_WCHAR_KIND;
assert(writer->kind buffer = buffer;
_PyUnicodeWriter_Update(writer);
writer->min_length = writer->size;
}
int
_PyUnicodeWriter_PrepareInternal(_PyUnicodeWriter *writer,
Py_ssize_t length, Py_UCS4 maxchar)
{
Py_ssize_t newlen;
PyObject *newbuffer;
assert(maxchar writer->maxchar && length >= 0)
|| length > 0);
if (length > PY_SSIZE_T_MAX - writer->pos) {
PyErr_NoMemory();
return -1;
}
newlen = writer->pos + length;
maxchar = Py_MAX(maxchar, writer->min_char);
if (writer->buffer == NULL) {
assert(!writer->readonly);
if (writer->overallocate
&& newlen min_length)
newlen = writer->min_length;
writer->buffer = PyUnicode_New(newlen, maxchar);
if (writer->buffer == NULL)
return -1;
}
else if (newlen > writer->size) {
if (writer->overallocate
&& newlen min_length)
newlen = writer->min_length;
if (maxchar > writer->maxchar || writer->readonly) {
/* resize + widen */
maxchar = Py_MAX(maxchar, writer->maxchar);
newbuffer = PyUnicode_New(newlen, maxchar);
if (newbuffer == NULL)
return -1;
_PyUnicode_FastCopyCharacters(newbuffer, 0,
writer->buffer, 0, writer->pos);
Py_DECREF(writer->buffer);
writer->readonly = 0;
}
else {
newbuffer = resize_compact(writer->buffer, newlen);
if (newbuffer == NULL)
return -1;
}
writer->buffer = newbuffer;
}
else if (maxchar > writer->maxchar) {
assert(!writer->readonly);
newbuffer = PyUnicode_New(writer->size, maxchar);
if (newbuffer == NULL)
return -1;
_PyUnicode_FastCopyCharacters(newbuffer, 0,
writer->buffer, 0, writer->pos);
Py_SETREF(writer->buffer, newbuffer);
}
_PyUnicodeWriter_Update(writer);
return 0;
#undef OVERALLOCATE_FACTOR
}
int
_PyUnicodeWriter_PrepareKindInternal(_PyUnicodeWriter *writer,
enum PyUnicode_Kind kind)
{
Py_UCS4 maxchar;
/* ensure that the _PyUnicodeWriter_PrepareKind macro was used */
assert(writer->kind < kind);
switch (kind)
{
case PyUnicode_1BYTE_KIND: maxchar = 0xff; break;
case PyUnicode_2BYTE_KIND: maxchar = 0xffff; break;
case PyUnicode_4BYTE_KIND: maxchar = MAX_UNICODE; break;
default:
Py_UNREACHABLE();
}
return _PyUnicodeWriter_PrepareInternal(writer, 0, maxchar);
}
static inline int
_PyUnicodeWriter_WriteCharInline(_PyUnicodeWriter *writer, Py_UCS4 ch)
{
assert(ch kind, writer->data, writer->pos, ch);
writer->pos++;
return 0;
}
int
_PyUnicodeWriter_WriteChar(_PyUnicodeWriter *writer, Py_UCS4 ch)
{
return _PyUnicodeWriter_WriteCharInline(writer, ch);
}
int
_PyUnicodeWriter_WriteStr(_PyUnicodeWriter *writer, PyObject *str)
{
Py_UCS4 maxchar;
Py_ssize_t len;
if (PyUnicode_READY(str) == -1)
return -1;
len = PyUnicode_GET_LENGTH(str);
if (len == 0)
return 0;
maxchar = PyUnicode_MAX_CHAR_VALUE(str);
if (maxchar > writer->maxchar || len > writer->size - writer->pos) {
if (writer->buffer == NULL && !writer->overallocate) {
assert(_PyUnicode_CheckConsistency(str, 1));
writer->readonly = 1;
Py_INCREF(str);
writer->buffer = str;
_PyUnicodeWriter_Update(writer);
writer->pos += len;
return 0;
}
if (_PyUnicodeWriter_PrepareInternal(writer, len, maxchar) == -1)
return -1;
}
_PyUnicode_FastCopyCharacters(writer->buffer, writer->pos,
str, 0, len);
writer->pos += len;
return 0;
}
int
_PyUnicodeWriter_WriteSubstring(_PyUnicodeWriter *writer, PyObject *str,
Py_ssize_t start, Py_ssize_t end)
{
Py_UCS4 maxchar;
Py_ssize_t len;
if (PyUnicode_READY(str) == -1)
return -1;
assert(0 maxchar;
len = end - start;
if (_PyUnicodeWriter_Prepare(writer, len, maxchar) < 0)
return -1;
_PyUnicode_FastCopyCharacters(writer->buffer, writer->pos,
str, start, len);
writer->pos += len;
return 0;
}
int
_PyUnicodeWriter_WriteASCIIString(_PyUnicodeWriter *writer,
const char *ascii, Py_ssize_t len)
{
if (len == -1)
len = strlen(ascii);
assert(ucs1lib_find_max_char((const Py_UCS1*)ascii, (const Py_UCS1*)ascii + len) < 128);
if (writer->buffer == NULL && !writer->overallocate) {
PyObject *str;
str = _PyUnicode_FromASCII(ascii, len);
if (str == NULL)
return -1;
writer->readonly = 1;
writer->buffer = str;
_PyUnicodeWriter_Update(writer);
writer->pos += len;
return 0;
}
if (_PyUnicodeWriter_Prepare(writer, len, 127) == -1)
return -1;
switch (writer->kind)
{
case PyUnicode_1BYTE_KIND:
{
const Py_UCS1 *str = (const Py_UCS1 *)ascii;
Py_UCS1 *data = writer->data;
memcpy(data + writer->pos, str, len);
break;
}
case PyUnicode_2BYTE_KIND:
{
_PyUnicode_CONVERT_BYTES(
Py_UCS1, Py_UCS2,
ascii, ascii + len,
(Py_UCS2 *)writer->data + writer->pos);
break;
}
case PyUnicode_4BYTE_KIND:
{
_PyUnicode_CONVERT_BYTES(
Py_UCS1, Py_UCS4,
ascii, ascii + len,
(Py_UCS4 *)writer->data + writer->pos);
break;
}
default:
Py_UNREACHABLE();
}
writer->pos += len;
return 0;
}
int
_PyUnicodeWriter_WriteLatin1String(_PyUnicodeWriter *writer,
const char *str, Py_ssize_t len)
{
Py_UCS4 maxchar;
maxchar = ucs1lib_find_max_char((const Py_UCS1*)str, (const Py_UCS1*)str + len);
if (_PyUnicodeWriter_Prepare(writer, len, maxchar) == -1)
return -1;
unicode_write_cstr(writer->buffer, writer->pos, str, len);
writer->pos += len;
return 0;
}
PyObject *
_PyUnicodeWriter_Finish(_PyUnicodeWriter *writer)
{
PyObject *str;
if (writer->pos == 0) {
Py_CLEAR(writer->buffer);
_Py_RETURN_UNICODE_EMPTY();
}
str = writer->buffer;
writer->buffer = NULL;
if (writer->readonly) {
assert(PyUnicode_GET_LENGTH(str) == writer->pos);
return str;
}
if (PyUnicode_GET_LENGTH(str) != writer->pos) {
PyObject *str2;
str2 = resize_compact(str, writer->pos);
if (str2 == NULL) {
Py_DECREF(str);
return NULL;
}
str = str2;
}
assert(_PyUnicode_CheckConsistency(str, 1));
return unicode_result_ready(str);
}
void
_PyUnicodeWriter_Dealloc(_PyUnicodeWriter *writer)
{
Py_CLEAR(writer->buffer);
}
#include "stringlib/unicode_format.h"
PyDoc_STRVAR(format__doc__,
"S.format(*args, **kwargs) -> str\n\
\n\
Return a formatted version of S, using substitutions from args and kwargs.\n\
The substitutions are identified by braces ('{' and '}').");
PyDoc_STRVAR(format_map__doc__,
"S.format_map(mapping) -> str\n\
\n\
Return a formatted version of S, using substitutions from mapping.\n\
The substitutions are identified by braces ('{' and '}').");
/*[clinic input]
str.__format__ as unicode___format__
format_spec: unicode
/
Return a formatted version of the string as described by format_spec.
[clinic start generated code]*/
static PyObject *
unicode___format___impl(PyObject *self, PyObject *format_spec)
/*[clinic end generated code: output=45fceaca6d2ba4c8 input=5e135645d167a214]*/
{
_PyUnicodeWriter writer;
int ret;
if (PyUnicode_READY(self) == -1)
return NULL;
_PyUnicodeWriter_Init(&writer);
ret = _PyUnicode_FormatAdvancedWriter(&writer,
self, format_spec, 0,
PyUnicode_GET_LENGTH(format_spec));
if (ret == -1) {
_PyUnicodeWriter_Dealloc(&writer);
return NULL;
}
return _PyUnicodeWriter_Finish(&writer);
}
/*[clinic input]
str.__sizeof__ as unicode_sizeof
Return the size of the string in memory, in bytes.
[clinic start generated code]*/
static PyObject *
unicode_sizeof_impl(PyObject *self)
/*[clinic end generated code: output=6dbc2f5a408b6d4f input=6dd011c108e33fb0]*/
{
Py_ssize_t size;
/* If it's a compact object, account for base structure +
character data. */
if (PyUnicode_IS_COMPACT_ASCII(self))
size = sizeof(PyASCIIObject) + PyUnicode_GET_LENGTH(self) + 1;
else if (PyUnicode_IS_COMPACT(self))
size = sizeof(PyCompactUnicodeObject) +
(PyUnicode_GET_LENGTH(self) + 1) * PyUnicode_KIND(self);
else {
/* If it is a two-block object, account for base object, and
for character block if present. */
size = sizeof(PyUnicodeObject);
if (_PyUnicode_DATA_ANY(self))
size += (PyUnicode_GET_LENGTH(self) + 1) *
PyUnicode_KIND(self);
}
/* If the wstr pointer is present, account for it unless it is shared
with the data pointer. Check if the data is not shared. */
if (_PyUnicode_HAS_WSTR_MEMORY(self))
size += (PyUnicode_WSTR_LENGTH(self) + 1) * sizeof(wchar_t);
if (_PyUnicode_HAS_UTF8_MEMORY(self))
size += PyUnicode_UTF8_LENGTH(self) + 1;
return PyLong_FromSsize_t(size);
}
static PyObject *
unicode_getnewargs(PyObject *v, PyObject *Py_UNUSED(ignored))
{
PyObject *copy = _PyUnicode_Copy(v);
if (!copy)
return NULL;
return Py_BuildValue("(N)", copy);
}
static PyMethodDef unicode_methods[] = {
UNICODE_ENCODE_METHODDEF
UNICODE_REPLACE_METHODDEF
UNICODE_SPLIT_METHODDEF
UNICODE_RSPLIT_METHODDEF
UNICODE_JOIN_METHODDEF
UNICODE_CAPITALIZE_METHODDEF
UNICODE_CASEFOLD_METHODDEF
UNICODE_TITLE_METHODDEF
UNICODE_CENTER_METHODDEF
{"count", (PyCFunction) unicode_count, METH_VARARGS, count__doc__},
UNICODE_EXPANDTABS_METHODDEF
{"find", (PyCFunction) unicode_find, METH_VARARGS, find__doc__},
UNICODE_PARTITION_METHODDEF
{"index", (PyCFunction) unicode_index, METH_VARARGS, index__doc__},
UNICODE_LJUST_METHODDEF
UNICODE_LOWER_METHODDEF
UNICODE_LSTRIP_METHODDEF
{"rfind", (PyCFunction) unicode_rfind, METH_VARARGS, rfind__doc__},
{"rindex", (PyCFunction) unicode_rindex, METH_VARARGS, rindex__doc__},
UNICODE_RJUST_METHODDEF
UNICODE_RSTRIP_METHODDEF
UNICODE_RPARTITION_METHODDEF
UNICODE_SPLITLINES_METHODDEF
UNICODE_STRIP_METHODDEF
UNICODE_SWAPCASE_METHODDEF
UNICODE_TRANSLATE_METHODDEF
UNICODE_UPPER_METHODDEF
{"startswith", (PyCFunction) unicode_startswith, METH_VARARGS, startswith__doc__},
{"endswith", (PyCFunction) unicode_endswith, METH_VARARGS, endswith__doc__},
UNICODE_REMOVEPREFIX_METHODDEF
UNICODE_REMOVESUFFIX_METHODDEF
UNICODE_ISASCII_METHODDEF
UNICODE_ISLOWER_METHODDEF
UNICODE_ISUPPER_METHODDEF
UNICODE_ISTITLE_METHODDEF
UNICODE_ISSPACE_METHODDEF
UNICODE_ISDECIMAL_METHODDEF
UNICODE_ISDIGIT_METHODDEF
UNICODE_ISNUMERIC_METHODDEF
UNICODE_ISALPHA_METHODDEF
UNICODE_ISALNUM_METHODDEF
UNICODE_ISIDENTIFIER_METHODDEF
UNICODE_ISPRINTABLE_METHODDEF
UNICODE_ZFILL_METHODDEF
{"format", (PyCFunction)(void(*)(void)) do_string_format, METH_VARARGS | METH_KEYWORDS, format__doc__},
{"format_map", (PyCFunction) do_string_format_map, METH_O, format_map__doc__},
UNICODE___FORMAT___METHODDEF
UNICODE_MAKETRANS_METHODDEF
UNICODE_SIZEOF_METHODDEF
#if 0
/* These methods are just used for debugging the implementation. */
{"_decimal2ascii", (PyCFunction) unicode__decimal2ascii, METH_NOARGS},
#endif
{"__getnewargs__", unicode_getnewargs, METH_NOARGS},
{NULL, NULL}
};
static PyObject *
unicode_mod(PyObject *v, PyObject *w)
{
if (!PyUnicode_Check(v))
Py_RETURN_NOTIMPLEMENTED;
return PyUnicode_Format(v, w);
}
static PyNumberMethods unicode_as_number = {
0, /*nb_add*/
0, /*nb_subtract*/
0, /*nb_multiply*/
unicode_mod, /*nb_remainder*/
};
static PySequenceMethods unicode_as_sequence = {
(lenfunc) unicode_length, /* sq_length */
PyUnicode_Concat, /* sq_concat */
(ssizeargfunc) unicode_repeat, /* sq_repeat */
(ssizeargfunc) unicode_getitem, /* sq_item */
0, /* sq_slice */
0, /* sq_ass_item */
0, /* sq_ass_slice */
PyUnicode_Contains, /* sq_contains */
};
static PyObject*
unicode_subscript(PyObject* self, PyObject* item)
{
if (PyUnicode_READY(self) == -1)
return NULL;
if (_PyIndex_Check(item)) {
Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError);
if (i == -1 && PyErr_Occurred())
return NULL;
if (i < 0)
i += PyUnicode_GET_LENGTH(self);
return unicode_getitem(self, i);
} else if (PySlice_Check(item)) {
Py_ssize_t start, stop, step, slicelength, i;
size_t cur;
PyObject *result;
const void *src_data;
void *dest_data;
int src_kind, dest_kind;
Py_UCS4 ch, max_char, kind_limit;
if (PySlice_Unpack(item, &start, &stop, &step) < 0) {
return NULL;
}
slicelength = PySlice_AdjustIndices(PyUnicode_GET_LENGTH(self),
&start, &stop, step);
if (slicelength max_char) {
max_char = ch;
if (max_char >= kind_limit)
break;
}
}
}
else
max_char = 127;
result = PyUnicode_New(slicelength, max_char);
if (result == NULL)
return NULL;
dest_kind = PyUnicode_KIND(result);
dest_data = PyUnicode_DATA(result);
for (cur = start, i = 0; i < slicelength; cur += step, i++) {
Py_UCS4 ch = PyUnicode_READ(src_kind, src_data, cur);
PyUnicode_WRITE(dest_kind, dest_data, i, ch);
}
assert(_PyUnicode_CheckConsistency(result, 1));
return result;
} else {
PyErr_SetString(PyExc_TypeError, "string indices must be integers");
return NULL;
}
}
static PyMappingMethods unicode_as_mapping = {
(lenfunc)unicode_length, /* mp_length */
(binaryfunc)unicode_subscript, /* mp_subscript */
(objobjargproc)0, /* mp_ass_subscript */
};
/* Helpers for PyUnicode_Format() */
struct unicode_formatter_t {
PyObject *args;
int args_owned;
Py_ssize_t arglen, argidx;
PyObject *dict;
enum PyUnicode_Kind fmtkind;
Py_ssize_t fmtcnt, fmtpos;
const void *fmtdata;
PyObject *fmtstr;
_PyUnicodeWriter writer;
};
struct unicode_format_arg_t {
Py_UCS4 ch;
int flags;
Py_ssize_t width;
int prec;
int sign;
};
static PyObject *
unicode_format_getnextarg(struct unicode_formatter_t *ctx)
{
Py_ssize_t argidx = ctx->argidx;
if (argidx < ctx->arglen) {
ctx->argidx++;
if (ctx->arglen < 0)
return ctx->args;
else
return PyTuple_GetItem(ctx->args, argidx);
}
PyErr_SetString(PyExc_TypeError,
"not enough arguments for format string");
return NULL;
}
/* Returns a new reference to a PyUnicode object, or NULL on failure. */
/* Format a float into the writer if the writer is not NULL, or into *p_output
otherwise.
Return 0 on success, raise an exception and return -1 on error. */
static int
formatfloat(PyObject *v, struct unicode_format_arg_t *arg,
PyObject **p_output,
_PyUnicodeWriter *writer)
{
char *p;
double x;
Py_ssize_t len;
int prec;
int dtoa_flags;
x = PyFloat_AsDouble(v);
if (x == -1.0 && PyErr_Occurred())
return -1;
prec = arg->prec;
if (prec < 0)
prec = 6;
if (arg->flags & F_ALT)
dtoa_flags = Py_DTSF_ALT;
else
dtoa_flags = 0;
p = PyOS_double_to_string(x, arg->ch, prec, dtoa_flags, NULL);
if (p == NULL)
return -1;
len = strlen(p);
if (writer) {
if (_PyUnicodeWriter_WriteASCIIString(writer, p, len) < 0) {
PyMem_Free(p);
return -1;
}
}
else
*p_output = _PyUnicode_FromASCII(p, len);
PyMem_Free(p);
return 0;
}
/* formatlong() emulates the format codes d, u, o, x and X, and
* the F_ALT flag, for Python's long (unbounded) ints. It's not used for
* Python's regular ints.
* Return value: a new PyUnicodeObject*, or NULL if error.
* The output string is of the form
* "-"? ("0x" | "0X")? digit+
* "0x"/"0X" are present only for x and X conversions, with F_ALT
* set in flags. The case of hex digits will be correct,
* There will be at least prec digits, zero-filled on the left if
* necessary to get that many.
* val object to be converted
* flags bitmask of format flags; only F_ALT is looked at
* prec minimum number of digits; 0-fill on left if needed
* type a character in [duoxX]; u acts the same as d
*
* CAUTION: o, x and X conversions on regular ints can never
* produce a '-' sign, but can for Python's unbounded ints.
*/
PyObject *
_PyUnicode_FormatLong(PyObject *val, int alt, int prec, int type)
{
PyObject *result = NULL;
char *buf;
Py_ssize_t i;
int sign; /* 1 if '-', else 0 */
int len; /* number of characters */
Py_ssize_t llen;
int numdigits; /* len == numnondigits + numdigits */
int numnondigits = 0;
/* Avoid exceeding SSIZE_T_MAX */
if (prec > INT_MAX-3) {
PyErr_SetString(PyExc_OverflowError,
"precision too large");
return NULL;
}
assert(PyLong_Check(val));
switch (type) {
default:
Py_UNREACHABLE();
case 'd':
case 'i':
case 'u':
/* int and int subclasses should print numerically when a numeric */
/* format code is used (see issue18780) */
result = PyNumber_ToBase(val, 10);
break;
case 'o':
numnondigits = 2;
result = PyNumber_ToBase(val, 8);
break;
case 'x':
case 'X':
numnondigits = 2;
result = PyNumber_ToBase(val, 16);
break;
}
if (!result)
return NULL;
assert(unicode_modifiable(result));
assert(PyUnicode_IS_READY(result));
assert(PyUnicode_IS_ASCII(result));
/* To modify the string in-place, there can only be one reference. */
if (Py_REFCNT(result) != 1) {
Py_DECREF(result);
PyErr_BadInternalCall();
return NULL;
}
buf = PyUnicode_DATA(result);
llen = PyUnicode_GET_LENGTH(result);
if (llen > INT_MAX) {
Py_DECREF(result);
PyErr_SetString(PyExc_ValueError,
"string too large in _PyUnicode_FormatLong");
return NULL;
}
len = (int)llen;
sign = buf[0] == '-';
numnondigits += sign;
numdigits = len - numnondigits;
assert(numdigits > 0);
/* Get rid of base marker unless F_ALT */
if (((alt) == 0 &&
(type == 'o' || type == 'x' || type == 'X'))) {
assert(buf[sign] == '0');
assert(buf[sign+1] == 'x' || buf[sign+1] == 'X' ||
buf[sign+1] == 'o');
numnondigits -= 2;
buf += 2;
len -= 2;
if (sign)
buf[0] = '-';
assert(len == numnondigits + numdigits);
assert(numdigits > 0);
}
/* Fill with leading zeroes to meet minimum width. */
if (prec > numdigits) {
PyObject *r1 = PyBytes_FromStringAndSize(NULL,
numnondigits + prec);
char *b1;
if (!r1) {
Py_DECREF(result);
return NULL;
}
b1 = PyBytes_AS_STRING(r1);
for (i = 0; i < numnondigits; ++i)
*b1++ = *buf++;
for (i = 0; i < prec - numdigits; i++)
*b1++ = '0';
for (i = 0; i < numdigits; i++)
*b1++ = *buf++;
*b1 = '\0';
Py_DECREF(result);
result = r1;
buf = PyBytes_AS_STRING(result);
len = numnondigits + prec;
}
/* Fix up case for hex conversions. */
if (type == 'X') {
/* Need to convert all lower case letters to upper case.
and need to convert 0x to 0X (and -0x to -0X). */
for (i = 0; i < len; i++)
if (buf[i] >= 'a' && buf[i] ch;
if (!PyNumber_Check(v))
goto wrongtype;
/* make sure number is a type of integer for o, x, and X */
if (!PyLong_Check(v)) {
if (type == 'o' || type == 'x' || type == 'X') {
iobj = PyNumber_Index(v);
if (iobj == NULL) {
if (PyErr_ExceptionMatches(PyExc_TypeError))
goto wrongtype;
return -1;
}
}
else {
iobj = PyNumber_Long(v);
if (iobj == NULL ) {
if (PyErr_ExceptionMatches(PyExc_TypeError))
goto wrongtype;
return -1;
}
}
assert(PyLong_Check(iobj));
}
else {
iobj = v;
Py_INCREF(iobj);
}
if (PyLong_CheckExact(v)
&& arg->width == -1 && arg->prec == -1
&& !(arg->flags & (F_SIGN | F_BLANK))
&& type != 'X')
{
/* Fast path */
int alternate = arg->flags & F_ALT;
int base;
switch(type)
{
default:
Py_UNREACHABLE();
case 'd':
case 'i':
case 'u':
base = 10;
break;
case 'o':
base = 8;
break;
case 'x':
case 'X':
base = 16;
break;
}
if (_PyLong_FormatWriter(writer, v, base, alternate) == -1) {
Py_DECREF(iobj);
return -1;
}
Py_DECREF(iobj);
return 1;
}
res = _PyUnicode_FormatLong(iobj, arg->flags & F_ALT, arg->prec, type);
Py_DECREF(iobj);
if (res == NULL)
return -1;
*p_output = res;
return 0;
wrongtype:
switch(type)
{
case 'o':
case 'x':
case 'X':
PyErr_Format(PyExc_TypeError,
"%%%c format: an integer is required, "
"not %.200s",
type, Py_TYPE(v)->tp_name);
break;
default:
PyErr_Format(PyExc_TypeError,
"%%%c format: a number is required, "
"not %.200s",
type, Py_TYPE(v)->tp_name);
break;
}
return -1;
}
static Py_UCS4
formatchar(PyObject *v)
{
/* presume that the buffer is at least 3 characters long */
if (PyUnicode_Check(v)) {
if (PyUnicode_GET_LENGTH(v) == 1) {
return PyUnicode_READ_CHAR(v, 0);
}
goto onError;
}
else {
PyObject *iobj;
long x;
/* make sure number is a type of integer */
if (!PyLong_Check(v)) {
iobj = PyNumber_Index(v);
if (iobj == NULL) {
goto onError;
}
x = PyLong_AsLong(iobj);
Py_DECREF(iobj);
}
else {
x = PyLong_AsLong(v);
}
if (x == -1 && PyErr_Occurred())
goto onError;
if (x < 0 || x > MAX_UNICODE) {
PyErr_SetString(PyExc_OverflowError,
"%c arg not in range(0x110000)");
return (Py_UCS4) -1;
}
return (Py_UCS4) x;
}
onError:
PyErr_SetString(PyExc_TypeError,
"%c requires int or char");
return (Py_UCS4) -1;
}
/* Parse options of an argument: flags, width, precision.
Handle also "%(name)" syntax.
Return 0 if the argument has been formatted into arg->str.
Return 1 if the argument has been written into ctx->writer,
Raise an exception and return -1 on error. */
static int
unicode_format_arg_parse(struct unicode_formatter_t *ctx,
struct unicode_format_arg_t *arg)
{
#define FORMAT_READ(ctx) \
PyUnicode_READ((ctx)->fmtkind, (ctx)->fmtdata, (ctx)->fmtpos)
PyObject *v;
if (arg->ch == '(') {
/* Get argument value from a dictionary. Example: "%(name)s". */
Py_ssize_t keystart;
Py_ssize_t keylen;
PyObject *key;
int pcount = 1;
if (ctx->dict == NULL) {
PyErr_SetString(PyExc_TypeError,
"format requires a mapping");
return -1;
}
++ctx->fmtpos;
--ctx->fmtcnt;
keystart = ctx->fmtpos;
/* Skip over balanced parentheses */
while (pcount > 0 && --ctx->fmtcnt >= 0) {
arg->ch = FORMAT_READ(ctx);
if (arg->ch == ')')
--pcount;
else if (arg->ch == '(')
++pcount;
ctx->fmtpos++;
}
keylen = ctx->fmtpos - keystart - 1;
if (ctx->fmtcnt < 0 || pcount > 0) {
PyErr_SetString(PyExc_ValueError,
"incomplete format key");
return -1;
}
key = PyUnicode_Substring(ctx->fmtstr,
keystart, keystart + keylen);
if (key == NULL)
return -1;
if (ctx->args_owned) {
ctx->args_owned = 0;
Py_DECREF(ctx->args);
}
ctx->args = PyObject_GetItem(ctx->dict, key);
Py_DECREF(key);
if (ctx->args == NULL)
return -1;
ctx->args_owned = 1;
ctx->arglen = -1;
ctx->argidx = -2;
}
/* Parse flags. Example: "%+i" => flags=F_SIGN. */
while (--ctx->fmtcnt >= 0) {
arg->ch = FORMAT_READ(ctx);
ctx->fmtpos++;
switch (arg->ch) {
case '-': arg->flags |= F_LJUST; continue;
case '+': arg->flags |= F_SIGN; continue;
case ' ': arg->flags |= F_BLANK; continue;
case '#': arg->flags |= F_ALT; continue;
case '0': arg->flags |= F_ZERO; continue;
}
break;
}
/* Parse width. Example: "%10s" => width=10 */
if (arg->ch == '*') {
v = unicode_format_getnextarg(ctx);
if (v == NULL)
return -1;
if (!PyLong_Check(v)) {
PyErr_SetString(PyExc_TypeError,
"* wants int");
return -1;
}
arg->width = PyLong_AsSsize_t(v);
if (arg->width == -1 && PyErr_Occurred())
return -1;
if (arg->width < 0) {
arg->flags |= F_LJUST;
arg->width = -arg->width;
}
if (--ctx->fmtcnt >= 0) {
arg->ch = FORMAT_READ(ctx);
ctx->fmtpos++;
}
}
else if (arg->ch >= '0' && arg->ch width = arg->ch - '0';
while (--ctx->fmtcnt >= 0) {
arg->ch = FORMAT_READ(ctx);
ctx->fmtpos++;
if (arg->ch < '0' || arg->ch > '9')
break;
/* Since arg->ch is unsigned, the RHS would end up as unsigned,
mixing signed and unsigned comparison. Since arg->ch is between
'0' and '9', casting to int is safe. */
if (arg->width > (PY_SSIZE_T_MAX - ((int)arg->ch - '0')) / 10) {
PyErr_SetString(PyExc_ValueError,
"width too big");
return -1;
}
arg->width = arg->width*10 + (arg->ch - '0');
}
}
/* Parse precision. Example: "%.3f" => prec=3 */
if (arg->ch == '.') {
arg->prec = 0;
if (--ctx->fmtcnt >= 0) {
arg->ch = FORMAT_READ(ctx);
ctx->fmtpos++;
}
if (arg->ch == '*') {
v = unicode_format_getnextarg(ctx);
if (v == NULL)
return -1;
if (!PyLong_Check(v)) {
PyErr_SetString(PyExc_TypeError,
"* wants int");
return -1;
}
arg->prec = _PyLong_AsInt(v);
if (arg->prec == -1 && PyErr_Occurred())
return -1;
if (arg->prec < 0)
arg->prec = 0;
if (--ctx->fmtcnt >= 0) {
arg->ch = FORMAT_READ(ctx);
ctx->fmtpos++;
}
}
else if (arg->ch >= '0' && arg->ch prec = arg->ch - '0';
while (--ctx->fmtcnt >= 0) {
arg->ch = FORMAT_READ(ctx);
ctx->fmtpos++;
if (arg->ch < '0' || arg->ch > '9')
break;
if (arg->prec > (INT_MAX - ((int)arg->ch - '0')) / 10) {
PyErr_SetString(PyExc_ValueError,
"precision too big");
return -1;
}
arg->prec = arg->prec*10 + (arg->ch - '0');
}
}
}
/* Ignore "h", "l" and "L" format prefix (ex: "%hi" or "%ls") */
if (ctx->fmtcnt >= 0) {
if (arg->ch == 'h' || arg->ch == 'l' || arg->ch == 'L') {
if (--ctx->fmtcnt >= 0) {
arg->ch = FORMAT_READ(ctx);
ctx->fmtpos++;
}
}
}
if (ctx->fmtcnt < 0) {
PyErr_SetString(PyExc_ValueError,
"incomplete format");
return -1;
}
return 0;
#undef FORMAT_READ
}
/* Format one argument. Supported conversion specifiers:
- "s", "r", "a": any type
- "i", "d", "u": int or float
- "o", "x", "X": int
- "e", "E", "f", "F", "g", "G": float
- "c": int or str (1 character)
When possible, the output is written directly into the Unicode writer
(ctx->writer). A string is created when padding is required.
Return 0 if the argument has been formatted into *p_str,
1 if the argument has been written into ctx->writer,
-1 on error. */
static int
unicode_format_arg_format(struct unicode_formatter_t *ctx,
struct unicode_format_arg_t *arg,
PyObject **p_str)
{
PyObject *v;
_PyUnicodeWriter *writer = &ctx->writer;
if (ctx->fmtcnt == 0)
ctx->writer.overallocate = 0;
v = unicode_format_getnextarg(ctx);
if (v == NULL)
return -1;
switch (arg->ch) {
case 's':
case 'r':
case 'a':
if (PyLong_CheckExact(v) && arg->width == -1 && arg->prec == -1) {
/* Fast path */
if (_PyLong_FormatWriter(writer, v, 10, arg->flags & F_ALT) == -1)
return -1;
return 1;
}
if (PyUnicode_CheckExact(v) && arg->ch == 's') {
*p_str = v;
Py_INCREF(*p_str);
}
else {
if (arg->ch == 's')
*p_str = PyObject_Str(v);
else if (arg->ch == 'r')
*p_str = PyObject_Repr(v);
else
*p_str = PyObject_ASCII(v);
}
break;
case 'i':
case 'd':
case 'u':
case 'o':
case 'x':
case 'X':
{
int ret = mainformatlong(v, arg, p_str, writer);
if (ret != 0)
return ret;
arg->sign = 1;
break;
}
case 'e':
case 'E':
case 'f':
case 'F':
case 'g':
case 'G':
if (arg->width == -1 && arg->prec == -1
&& !(arg->flags & (F_SIGN | F_BLANK)))
{
/* Fast path */
if (formatfloat(v, arg, NULL, writer) == -1)
return -1;
return 1;
}
arg->sign = 1;
if (formatfloat(v, arg, p_str, NULL) == -1)
return -1;
break;
case 'c':
{
Py_UCS4 ch = formatchar(v);
if (ch == (Py_UCS4) -1)
return -1;
if (arg->width == -1 && arg->prec == -1) {
/* Fast path */
if (_PyUnicodeWriter_WriteCharInline(writer, ch) < 0)
return -1;
return 1;
}
*p_str = PyUnicode_FromOrdinal(ch);
break;
}
default:
PyErr_Format(PyExc_ValueError,
"unsupported format character '%c' (0x%x) "
"at index %zd",
(31ch && arg->chch : '?',
(int)arg->ch,
ctx->fmtpos - 1);
return -1;
}
if (*p_str == NULL)
return -1;
assert (PyUnicode_Check(*p_str));
return 0;
}
static int
unicode_format_arg_output(struct unicode_formatter_t *ctx,
struct unicode_format_arg_t *arg,
PyObject *str)
{
Py_ssize_t len;
enum PyUnicode_Kind kind;
const void *pbuf;
Py_ssize_t pindex;
Py_UCS4 signchar;
Py_ssize_t buflen;
Py_UCS4 maxchar;
Py_ssize_t sublen;
_PyUnicodeWriter *writer = &ctx->writer;
Py_UCS4 fill;
fill = ' ';
if (arg->sign && arg->flags & F_ZERO)
fill = '0';
if (PyUnicode_READY(str) == -1)
return -1;
len = PyUnicode_GET_LENGTH(str);
if ((arg->width == -1 || arg->width prec == -1 || arg->prec >= len)
&& !(arg->flags & (F_SIGN | F_BLANK)))
{
/* Fast path */
if (_PyUnicodeWriter_WriteStr(writer, str) == -1)
return -1;
return 0;
}
/* Truncate the string for "s", "r" and "a" formats
if the precision is set */
if (arg->ch == 's' || arg->ch == 'r' || arg->ch == 'a') {
if (arg->prec >= 0 && len > arg->prec)
len = arg->prec;
}
/* Adjust sign and width */
kind = PyUnicode_KIND(str);
pbuf = PyUnicode_DATA(str);
pindex = 0;
signchar = '\0';
if (arg->sign) {
Py_UCS4 ch = PyUnicode_READ(kind, pbuf, pindex);
if (ch == '-' || ch == '+') {
signchar = ch;
len--;
pindex++;
}
else if (arg->flags & F_SIGN)
signchar = '+';
else if (arg->flags & F_BLANK)
signchar = ' ';
else
arg->sign = 0;
}
if (arg->width < len)
arg->width = len;
/* Prepare the writer */
maxchar = writer->maxchar;
if (!(arg->flags & F_LJUST)) {
if (arg->sign) {
if ((arg->width-1) > len)
maxchar = Py_MAX(maxchar, fill);
}
else {
if (arg->width > len)
maxchar = Py_MAX(maxchar, fill);
}
}
if (PyUnicode_MAX_CHAR_VALUE(str) > maxchar) {
Py_UCS4 strmaxchar = _PyUnicode_FindMaxChar(str, 0, pindex+len);
maxchar = Py_MAX(maxchar, strmaxchar);
}
buflen = arg->width;
if (arg->sign && len == arg->width)
buflen++;
if (_PyUnicodeWriter_Prepare(writer, buflen, maxchar) == -1)
return -1;
/* Write the sign if needed */
if (arg->sign) {
if (fill != ' ') {
PyUnicode_WRITE(writer->kind, writer->data, writer->pos, signchar);
writer->pos += 1;
}
if (arg->width > len)
arg->width--;
}
/* Write the numeric prefix for "x", "X" and "o" formats
if the alternate form is used.
For example, write "0x" for the "%#x" format. */
if ((arg->flags & F_ALT) && (arg->ch == 'x' || arg->ch == 'X' || arg->ch == 'o')) {
assert(PyUnicode_READ(kind, pbuf, pindex) == '0');
assert(PyUnicode_READ(kind, pbuf, pindex + 1) == arg->ch);
if (fill != ' ') {
PyUnicode_WRITE(writer->kind, writer->data, writer->pos, '0');
PyUnicode_WRITE(writer->kind, writer->data, writer->pos+1, arg->ch);
writer->pos += 2;
pindex += 2;
}
arg->width -= 2;
if (arg->width < 0)
arg->width = 0;
len -= 2;
}
/* Pad left with the fill character if needed */
if (arg->width > len && !(arg->flags & F_LJUST)) {
sublen = arg->width - len;
unicode_fill(writer->kind, writer->data, fill, writer->pos, sublen);
writer->pos += sublen;
arg->width = len;
}
/* If padding with spaces: write sign if needed and/or numeric prefix if
the alternate form is used */
if (fill == ' ') {
if (arg->sign) {
PyUnicode_WRITE(writer->kind, writer->data, writer->pos, signchar);
writer->pos += 1;
}
if ((arg->flags & F_ALT) && (arg->ch == 'x' || arg->ch == 'X' || arg->ch == 'o')) {
assert(PyUnicode_READ(kind, pbuf, pindex) == '0');
assert(PyUnicode_READ(kind, pbuf, pindex+1) == arg->ch);
PyUnicode_WRITE(writer->kind, writer->data, writer->pos, '0');
PyUnicode_WRITE(writer->kind, writer->data, writer->pos+1, arg->ch);
writer->pos += 2;
pindex += 2;
}
}
/* Write characters */
if (len) {
_PyUnicode_FastCopyCharacters(writer->buffer, writer->pos,
str, pindex, len);
writer->pos += len;
}
/* Pad right with the fill character if needed */
if (arg->width > len) {
sublen = arg->width - len;
unicode_fill(writer->kind, writer->data, ' ', writer->pos, sublen);
writer->pos += sublen;
}
return 0;
}
/* Helper of PyUnicode_Format(): format one arg.
Return 0 on success, raise an exception and return -1 on error. */
static int
unicode_format_arg(struct unicode_formatter_t *ctx)
{
struct unicode_format_arg_t arg;
PyObject *str;
int ret;
arg.ch = PyUnicode_READ(ctx->fmtkind, ctx->fmtdata, ctx->fmtpos);
if (arg.ch == '%') {
ctx->fmtpos++;
ctx->fmtcnt--;
if (_PyUnicodeWriter_WriteCharInline(&ctx->writer, '%') < 0)
return -1;
return 0;
}
arg.flags = 0;
arg.width = -1;
arg.prec = -1;
arg.sign = 0;
str = NULL;
ret = unicode_format_arg_parse(ctx, &arg);
if (ret == -1)
return -1;
ret = unicode_format_arg_format(ctx, &arg, &str);
if (ret == -1)
return -1;
if (ret != 1) {
ret = unicode_format_arg_output(ctx, &arg, str);
Py_DECREF(str);
if (ret == -1)
return -1;
}
if (ctx->dict && (ctx->argidx < ctx->arglen)) {
PyErr_SetString(PyExc_TypeError,
"not all arguments converted during string formatting");
return -1;
}
return 0;
}
PyObject *
PyUnicode_Format(PyObject *format, PyObject *args)
{
struct unicode_formatter_t ctx;
if (format == NULL || args == NULL) {
PyErr_BadInternalCall();
return NULL;
}
if (ensure_unicode(format) < 0)
return NULL;
ctx.fmtstr = format;
ctx.fmtdata = PyUnicode_DATA(ctx.fmtstr);
ctx.fmtkind = PyUnicode_KIND(ctx.fmtstr);
ctx.fmtcnt = PyUnicode_GET_LENGTH(ctx.fmtstr);
ctx.fmtpos = 0;
_PyUnicodeWriter_Init(&ctx.writer);
ctx.writer.min_length = ctx.fmtcnt + 100;
ctx.writer.overallocate = 1;
if (PyTuple_Check(args)) {
ctx.arglen = PyTuple_Size(args);
ctx.argidx = 0;
}
else {
ctx.arglen = -1;
ctx.argidx = -2;
}
ctx.args_owned = 0;
if (PyMapping_Check(args) && !PyTuple_Check(args) && !PyUnicode_Check(args))
ctx.dict = args;
else
ctx.dict = NULL;
ctx.args = args;
while (--ctx.fmtcnt >= 0) {
if (PyUnicode_READ(ctx.fmtkind, ctx.fmtdata, ctx.fmtpos) != '%') {
Py_ssize_t nonfmtpos;
nonfmtpos = ctx.fmtpos++;
while (ctx.fmtcnt >= 0 &&
PyUnicode_READ(ctx.fmtkind, ctx.fmtdata, ctx.fmtpos) != '%') {
ctx.fmtpos++;
ctx.fmtcnt--;
}
if (ctx.fmtcnt < 0) {
ctx.fmtpos--;
ctx.writer.overallocate = 0;
}
if (_PyUnicodeWriter_WriteSubstring(&ctx.writer, ctx.fmtstr,
nonfmtpos, ctx.fmtpos) < 0)
goto onError;
}
else {
ctx.fmtpos++;
if (unicode_format_arg(&ctx) == -1)
goto onError;
}
}
if (ctx.argidx < ctx.arglen && !ctx.dict) {
PyErr_SetString(PyExc_TypeError,
"not all arguments converted during string formatting");
goto onError;
}
if (ctx.args_owned) {
Py_DECREF(ctx.args);
}
return _PyUnicodeWriter_Finish(&ctx.writer);
onError:
_PyUnicodeWriter_Dealloc(&ctx.writer);
if (ctx.args_owned) {
Py_DECREF(ctx.args);
}
return NULL;
}
static PyObject *
unicode_subtype_new(PyTypeObject *type, PyObject *args, PyObject *kwds);
static PyObject *
unicode_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
{
PyObject *x = NULL;
static char *kwlist[] = {"object", "encoding", "errors", 0};
char *encoding = NULL;
char *errors = NULL;
if (type != &PyUnicode_Type)
return unicode_subtype_new(type, args, kwds);
if (!PyArg_ParseTupleAndKeywords(args, kwds, "|Oss:str",
kwlist, &x, &encoding, &errors))
return NULL;
if (x == NULL)
_Py_RETURN_UNICODE_EMPTY();
if (encoding == NULL && errors == NULL)
return PyObject_Str(x);
else
return PyUnicode_FromEncodedObject(x, encoding, errors);
}
static PyObject *
unicode_subtype_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
{
PyObject *unicode, *self;
Py_ssize_t length, char_size;
int share_wstr, share_utf8;
unsigned int kind;
void *data;
assert(PyType_IsSubtype(type, &PyUnicode_Type));
unicode = unicode_new(&PyUnicode_Type, args, kwds);
if (unicode == NULL)
return NULL;
assert(_PyUnicode_CHECK(unicode));
if (PyUnicode_READY(unicode) == -1) {
Py_DECREF(unicode);
return NULL;
}
self = type->tp_alloc(type, 0);
if (self == NULL) {
Py_DECREF(unicode);
return NULL;
}
kind = PyUnicode_KIND(unicode);
length = PyUnicode_GET_LENGTH(unicode);
_PyUnicode_LENGTH(self) = length;
#ifdef Py_DEBUG
_PyUnicode_HASH(self) = -1;
#else
_PyUnicode_HASH(self) = _PyUnicode_HASH(unicode);
#endif
_PyUnicode_STATE(self).interned = 0;
_PyUnicode_STATE(self).kind = kind;
_PyUnicode_STATE(self).compact = 0;
_PyUnicode_STATE(self).ascii = _PyUnicode_STATE(unicode).ascii;
_PyUnicode_STATE(self).ready = 1;
_PyUnicode_WSTR(self) = NULL;
_PyUnicode_UTF8_LENGTH(self) = 0;
_PyUnicode_UTF8(self) = NULL;
_PyUnicode_WSTR_LENGTH(self) = 0;
_PyUnicode_DATA_ANY(self) = NULL;
share_utf8 = 0;
share_wstr = 0;
if (kind == PyUnicode_1BYTE_KIND) {
char_size = 1;
if (PyUnicode_MAX_CHAR_VALUE(unicode) < 128)
share_utf8 = 1;
}
else if (kind == PyUnicode_2BYTE_KIND) {
char_size = 2;
if (sizeof(wchar_t) == 2)
share_wstr = 1;
}
else {
assert(kind == PyUnicode_4BYTE_KIND);
char_size = 4;
if (sizeof(wchar_t) == 4)
share_wstr = 1;
}
/* Ensure we won't overflow the length. */
if (length > (PY_SSIZE_T_MAX / char_size - 1)) {
PyErr_NoMemory();
goto onError;
}
data = PyObject_MALLOC((length + 1) * char_size);
if (data == NULL) {
PyErr_NoMemory();
goto onError;
}
_PyUnicode_DATA_ANY(self) = data;
if (share_utf8) {
_PyUnicode_UTF8_LENGTH(self) = length;
_PyUnicode_UTF8(self) = data;
}
if (share_wstr) {
_PyUnicode_WSTR_LENGTH(self) = length;
_PyUnicode_WSTR(self) = (wchar_t *)data;
}
memcpy(data, PyUnicode_DATA(unicode),
kind * (length + 1));
assert(_PyUnicode_CheckConsistency(self, 1));
#ifdef Py_DEBUG
_PyUnicode_HASH(self) = _PyUnicode_HASH(unicode);
#endif
Py_DECREF(unicode);
return self;
onError:
Py_DECREF(unicode);
Py_DECREF(self);
return NULL;
}
PyDoc_STRVAR(unicode_doc,
"str(object='') -> str\n\
str(bytes_or_buffer[, encoding[, errors]]) -> str\n\
\n\
Create a new string object from the given object. If encoding or\n\
errors is specified, then the object must expose a data buffer\n\
that will be decoded using the given encoding and error handler.\n\
Otherwise, returns the result of object.__str__() (if defined)\n\
or repr(object).\n\
encoding defaults to sys.getdefaultencoding().\n\
errors defaults to 'strict'.");
static PyObject *unicode_iter(PyObject *seq);
PyTypeObject PyUnicode_Type = {
PyVarObject_HEAD_INIT(&PyType_Type, 0)
"str", /* tp_name */
sizeof(PyUnicodeObject), /* tp_basicsize */
0, /* tp_itemsize */
/* Slots */
(destructor)unicode_dealloc, /* tp_dealloc */
0, /* tp_vectorcall_offset */
0, /* tp_getattr */
0, /* tp_setattr */
0, /* tp_as_async */
unicode_repr, /* tp_repr */
&unicode_as_number, /* tp_as_number */
&unicode_as_sequence, /* tp_as_sequence */
&unicode_as_mapping, /* tp_as_mapping */
(hashfunc) unicode_hash, /* tp_hash*/
0, /* tp_call*/
(reprfunc) unicode_str, /* tp_str */
PyObject_GenericGetAttr, /* tp_getattro */
0, /* tp_setattro */
0, /* tp_as_buffer */
Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE |
Py_TPFLAGS_UNICODE_SUBCLASS, /* tp_flags */
unicode_doc, /* tp_doc */
0, /* tp_traverse */
0, /* tp_clear */
PyUnicode_RichCompare, /* tp_richcompare */
0, /* tp_weaklistoffset */
unicode_iter, /* tp_iter */
0, /* tp_iternext */
unicode_methods, /* tp_methods */
0, /* tp_members */
0, /* tp_getset */
&PyBaseObject_Type, /* tp_base */
0, /* tp_dict */
0, /* tp_descr_get */
0, /* tp_descr_set */
0, /* tp_dictoffset */
0, /* tp_init */
0, /* tp_alloc */
unicode_new, /* tp_new */
PyObject_Del, /* tp_free */
};
/* Initialize the Unicode implementation */
PyStatus
_PyUnicode_Init(void)
{
/* XXX - move this array to unicodectype.c ? */
Py_UCS2 linebreak[] = {
0x000A, /* LINE FEED */
0x000D, /* CARRIAGE RETURN */
0x001C, /* FILE SEPARATOR */
0x001D, /* GROUP SEPARATOR */
0x001E, /* RECORD SEPARATOR */
0x0085, /* NEXT LINE */
0x2028, /* LINE SEPARATOR */
0x2029, /* PARAGRAPH SEPARATOR */
};
/* Init the implementation */
_Py_INCREF_UNICODE_EMPTY();
if (!unicode_empty) {
return _PyStatus_ERR("Can't create empty string");
}
Py_DECREF(unicode_empty);
if (PyType_Ready(&PyUnicode_Type) < 0) {
return _PyStatus_ERR("Can't initialize unicode type");
}
/* initialize the linebreak bloom filter */
bloom_linebreak = make_bloom_mask(
PyUnicode_2BYTE_KIND, linebreak,
Py_ARRAY_LENGTH(linebreak));
if (PyType_Ready(&EncodingMapType) < 0) {
return _PyStatus_ERR("Can't initialize encoding map type");
}
if (PyType_Ready(&PyFieldNameIter_Type) < 0) {
return _PyStatus_ERR("Can't initialize field name iterator type");
}
if (PyType_Ready(&PyFormatterIter_Type) < 0) {
return _PyStatus_ERR("Can't initialize formatter iter type");
}
return _PyStatus_OK();
}
void
PyUnicode_InternInPlace(PyObject **p)
{
PyObject *s = *p;
#ifdef Py_DEBUG
assert(s != NULL);
assert(_PyUnicode_CHECK(s));
#else
if (s == NULL || !PyUnicode_Check(s)) {
return;
}
#endif
/* If it's a subclass, we don't really know what putting
it in the interned dict might do. */
if (!PyUnicode_CheckExact(s)) {
return;
}
if (PyUnicode_CHECK_INTERNED(s)) {
return;
}
#ifdef INTERNED_STRINGS
if (interned == NULL) {
interned = PyDict_New();
if (interned == NULL) {
PyErr_Clear(); /* Don't leave an exception */
return;
}
}
PyObject *t;
t = PyDict_SetDefault(interned, s, s);
if (t == NULL) {
PyErr_Clear();
return;
}
if (t != s) {
Py_INCREF(t);
Py_SETREF(*p, t);
return;
}
/* The two references in interned are not counted by refcnt.
The deallocator will take care of this */
Py_SET_REFCNT(s, Py_REFCNT(s) - 2);
_PyUnicode_STATE(s).interned = SSTATE_INTERNED_MORTAL;
#endif
}
void
PyUnicode_InternImmortal(PyObject **p)
{
PyUnicode_InternInPlace(p);
if (PyUnicode_CHECK_INTERNED(*p) != SSTATE_INTERNED_IMMORTAL) {
_PyUnicode_STATE(*p).interned = SSTATE_INTERNED_IMMORTAL;
Py_INCREF(*p);
}
}
PyObject *
PyUnicode_InternFromString(const char *cp)
{
PyObject *s = PyUnicode_FromString(cp);
if (s == NULL)
return NULL;
PyUnicode_InternInPlace(&s);
return s;
}
#if defined(WITH_VALGRIND) || defined(__INSURE__)
static void
unicode_release_interned(void)
{
if (interned == NULL || !PyDict_Check(interned)) {
return;
}
PyObject *keys = PyDict_Keys(interned);
if (keys == NULL || !PyList_Check(keys)) {
PyErr_Clear();
return;
}
/* Since unicode_release_interned() is intended to help a leak
detector, interned unicode strings are not forcibly deallocated;
rather, we give them their stolen references back, and then clear
and DECREF the interned dict. */
Py_ssize_t n = PyList_GET_SIZE(keys);
#ifdef INTERNED_STATS
fprintf(stderr, "releasing %" PY_FORMAT_SIZE_T "d interned strings\n",
n);
Py_ssize_t immortal_size = 0, mortal_size = 0;
#endif
for (Py_ssize_t i = 0; i < n; i++) {
PyObject *s = PyList_GET_ITEM(keys, i);
if (PyUnicode_READY(s) == -1) {
Py_UNREACHABLE();
}
switch (PyUnicode_CHECK_INTERNED(s)) {
case SSTATE_INTERNED_IMMORTAL:
Py_REFCNT(s) += 1;
#ifdef INTERNED_STATS
immortal_size += PyUnicode_GET_LENGTH(s);
#endif
break;
case SSTATE_INTERNED_MORTAL:
Py_REFCNT(s) += 2;
#ifdef INTERNED_STATS
mortal_size += PyUnicode_GET_LENGTH(s);
#endif
break;
case SSTATE_NOT_INTERNED:
/* fall through */
default:
Py_UNREACHABLE();
}
_PyUnicode_STATE(s).interned = SSTATE_NOT_INTERNED;
}
#ifdef INTERNED_STATS
fprintf(stderr, "total size of all interned strings: "
"%" PY_FORMAT_SIZE_T "d/%" PY_FORMAT_SIZE_T "d "
"mortal/immortal\n", mortal_size, immortal_size);
#endif
Py_DECREF(keys);
PyDict_Clear(interned);
Py_CLEAR(interned);
}
#endif
/********************* Unicode Iterator **************************/
typedef struct {
PyObject_HEAD
Py_ssize_t it_index;
PyObject *it_seq; /* Set to NULL when iterator is exhausted */
} unicodeiterobject;
static void
unicodeiter_dealloc(unicodeiterobject *it)
{
_PyObject_GC_UNTRACK(it);
Py_XDECREF(it->it_seq);
PyObject_GC_Del(it);
}
static int
unicodeiter_traverse(unicodeiterobject *it, visitproc visit, void *arg)
{
Py_VISIT(it->it_seq);
return 0;
}
static PyObject *
unicodeiter_next(unicodeiterobject *it)
{
PyObject *seq, *item;
assert(it != NULL);
seq = it->it_seq;
if (seq == NULL)
return NULL;
assert(_PyUnicode_CHECK(seq));
if (it->it_index < PyUnicode_GET_LENGTH(seq)) {
int kind = PyUnicode_KIND(seq);
const void *data = PyUnicode_DATA(seq);
Py_UCS4 chr = PyUnicode_READ(kind, data, it->it_index);
item = PyUnicode_FromOrdinal(chr);
if (item != NULL)
++it->it_index;
return item;
}
it->it_seq = NULL;
Py_DECREF(seq);
return NULL;
}
static PyObject *
unicodeiter_len(unicodeiterobject *it, PyObject *Py_UNUSED(ignored))
{
Py_ssize_t len = 0;
if (it->it_seq)
len = PyUnicode_GET_LENGTH(it->it_seq) - it->it_index;
return PyLong_FromSsize_t(len);
}
PyDoc_STRVAR(length_hint_doc, "Private method returning an estimate of len(list(it)).");
static PyObject *
unicodeiter_reduce(unicodeiterobject *it, PyObject *Py_UNUSED(ignored))
{
_Py_IDENTIFIER(iter);
if (it->it_seq != NULL) {
return Py_BuildValue("N(O)n", _PyEval_GetBuiltinId(&PyId_iter),
it->it_seq, it->it_index);
} else {
PyObject *u = (PyObject *)_PyUnicode_New(0);
if (u == NULL)
return NULL;
return Py_BuildValue("N(N)", _PyEval_GetBuiltinId(&PyId_iter), u);
}
}
PyDoc_STRVAR(reduce_doc, "Return state information for pickling.");
static PyObject *
unicodeiter_setstate(unicodeiterobject *it, PyObject *state)
{
Py_ssize_t index = PyLong_AsSsize_t(state);
if (index == -1 && PyErr_Occurred())
return NULL;
if (it->it_seq != NULL) {
if (index < 0)
index = 0;
else if (index > PyUnicode_GET_LENGTH(it->it_seq))
index = PyUnicode_GET_LENGTH(it->it_seq); /* iterator truncated */
it->it_index = index;
}
Py_RETURN_NONE;
}
PyDoc_STRVAR(setstate_doc, "Set state information for unpickling.");
static PyMethodDef unicodeiter_methods[] = {
{"__length_hint__", (PyCFunction)unicodeiter_len, METH_NOARGS,
length_hint_doc},
{"__reduce__", (PyCFunction)unicodeiter_reduce, METH_NOARGS,
reduce_doc},
{"__setstate__", (PyCFunction)unicodeiter_setstate, METH_O,
setstate_doc},
{NULL, NULL} /* sentinel */
};
PyTypeObject PyUnicodeIter_Type = {
PyVarObject_HEAD_INIT(&PyType_Type, 0)
"str_iterator", /* tp_name */
sizeof(unicodeiterobject), /* tp_basicsize */
0, /* tp_itemsize */
/* methods */
(destructor)unicodeiter_dealloc, /* tp_dealloc */
0, /* tp_vectorcall_offset */
0, /* tp_getattr */
0, /* tp_setattr */
0, /* tp_as_async */
0, /* tp_repr */
0, /* tp_as_number */
0, /* tp_as_sequence */
0, /* tp_as_mapping */
0, /* tp_hash */
0, /* tp_call */
0, /* tp_str */
PyObject_GenericGetAttr, /* tp_getattro */
0, /* tp_setattro */
0, /* tp_as_buffer */
Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC,/* tp_flags */
0, /* tp_doc */
(traverseproc)unicodeiter_traverse, /* tp_traverse */
0, /* tp_clear */
0, /* tp_richcompare */
0, /* tp_weaklistoffset */
PyObject_SelfIter, /* tp_iter */
(iternextfunc)unicodeiter_next, /* tp_iternext */
unicodeiter_methods, /* tp_methods */
0,
};
static PyObject *
unicode_iter(PyObject *seq)
{
unicodeiterobject *it;
if (!PyUnicode_Check(seq)) {
PyErr_BadInternalCall();
return NULL;
}
if (PyUnicode_READY(seq) == -1)
return NULL;
it = PyObject_GC_New(unicodeiterobject, &PyUnicodeIter_Type);
if (it == NULL)
return NULL;
it->it_index = 0;
Py_INCREF(seq);
it->it_seq = seq;
_PyObject_GC_TRACK(it);
return (PyObject *)it;
}
size_t
Py_UNICODE_strlen(const Py_UNICODE *u)
{
return wcslen(u);
}
Py_UNICODE*
Py_UNICODE_strcpy(Py_UNICODE *s1, const Py_UNICODE *s2)
{
Py_UNICODE *u = s1;
while ((*u++ = *s2++));
return s1;
}
Py_UNICODE*
Py_UNICODE_strncpy(Py_UNICODE *s1, const Py_UNICODE *s2, size_t n)
{
Py_UNICODE *u = s1;
while ((*u++ = *s2++))
if (n-- == 0)
break;
return s1;
}
Py_UNICODE*
Py_UNICODE_strcat(Py_UNICODE *s1, const Py_UNICODE *s2)
{
Py_UNICODE *u1 = s1;
u1 += wcslen(u1);
while ((*u1++ = *s2++));
return s1;
}
int
Py_UNICODE_strcmp(const Py_UNICODE *s1, const Py_UNICODE *s2)
{
while (*s1 && *s2 && *s1 == *s2)
s1++, s2++;
if (*s1 && *s2)
return (*s1 < *s2) ? -1 : +1;
if (*s1)
return 1;
if (*s2)
return -1;
return 0;
}
int
Py_UNICODE_strncmp(const Py_UNICODE *s1, const Py_UNICODE *s2, size_t n)
{
Py_UNICODE u1, u2;
for (; n != 0; n--) {
u1 = *s1;
u2 = *s2;
if (u1 != u2)
return (u1 < u2) ? -1 : +1;
if (u1 == '\0')
return 0;
s1++;
s2++;
}
return 0;
}
Py_UNICODE*
Py_UNICODE_strchr(const Py_UNICODE *s, Py_UNICODE c)
{
const Py_UNICODE *p;
for (p = s; *p; p++)
if (*p == c)
return (Py_UNICODE*)p;
return NULL;
}
Py_UNICODE*
Py_UNICODE_strrchr(const Py_UNICODE *s, Py_UNICODE c)
{
const Py_UNICODE *p;
p = s + wcslen(s);
while (p != s) {
p--;
if (*p == c)
return (Py_UNICODE*)p;
}
return NULL;
}
Py_UNICODE*
PyUnicode_AsUnicodeCopy(PyObject *unicode)
{
Py_UNICODE *u, *copy;
Py_ssize_t len, size;
if (!PyUnicode_Check(unicode)) {
PyErr_BadArgument();
return NULL;
}
_Py_COMP_DIAG_PUSH
_Py_COMP_DIAG_IGNORE_DEPR_DECLS
u = PyUnicode_AsUnicodeAndSize(unicode, &len);
_Py_COMP_DIAG_POP
if (u == NULL)
return NULL;
/* Ensure we won't overflow the size. */
if (len > ((PY_SSIZE_T_MAX / (Py_ssize_t)sizeof(Py_UNICODE)) - 1)) {
PyErr_NoMemory();
return NULL;
}
size = len + 1; /* copy the null character */
size *= sizeof(Py_UNICODE);
copy = PyMem_Malloc(size);
if (copy == NULL) {
PyErr_NoMemory();
return NULL;
}
memcpy(copy, u, size);
return copy;
}
static int
encode_wstr_utf8(wchar_t *wstr, char **str, const char *name)
{
int res;
res = _Py_EncodeUTF8Ex(wstr, str, NULL, NULL, 1, _Py_ERROR_STRICT);
if (res == -2) {
PyErr_Format(PyExc_RuntimeWarning, "cannot decode %s", name);
return -1;
}
if (res < 0) {
PyErr_NoMemory();
return -1;
}
return 0;
}
static int
config_get_codec_name(wchar_t **config_encoding)
{
char *encoding;
if (encode_wstr_utf8(*config_encoding, &encoding, "stdio_encoding") < 0) {
return -1;
}
PyObject *name_obj = NULL;
PyObject *codec = _PyCodec_Lookup(encoding);
PyMem_RawFree(encoding);
if (!codec)
goto error;
name_obj = PyObject_GetAttrString(codec, "name");
Py_CLEAR(codec);
if (!name_obj) {
goto error;
}
wchar_t *wname = PyUnicode_AsWideCharString(name_obj, NULL);
Py_DECREF(name_obj);
if (wname == NULL) {
goto error;
}
wchar_t *raw_wname = _PyMem_RawWcsdup(wname);
if (raw_wname == NULL) {
PyMem_Free(wname);
PyErr_NoMemory();
goto error;
}
PyMem_RawFree(*config_encoding);
*config_encoding = raw_wname;
PyMem_Free(wname);
return 0;
error:
Py_XDECREF(codec);
Py_XDECREF(name_obj);
return -1;
}
static PyStatus
init_stdio_encoding(PyThreadState *tstate)
{
/* Update the stdio encoding to the normalized Python codec name. */
PyConfig *config = (PyConfig*)_PyInterpreterState_GetConfig(tstate->interp);
if (config_get_codec_name(&config->stdio_encoding) < 0) {
return _PyStatus_ERR("failed to get the Python codec name "
"of the stdio encoding");
}
return _PyStatus_OK();
}
static int
init_fs_codec(PyInterpreterState *interp)
{
const PyConfig *config = _PyInterpreterState_GetConfig(interp);
_Py_error_handler error_handler;
error_handler = get_error_handler_wide(config->filesystem_errors);
if (error_handler == _Py_ERROR_UNKNOWN) {
PyErr_SetString(PyExc_RuntimeError, "unknown filesystem error handler");
return -1;
}
char *encoding, *errors;
if (encode_wstr_utf8(config->filesystem_encoding,
&encoding,
"filesystem_encoding") < 0) {
return -1;
}
if (encode_wstr_utf8(config->filesystem_errors,
&errors,
"filesystem_errors") < 0) {
PyMem_RawFree(encoding);
return -1;
}
struct _Py_unicode_fs_codec *fs_codec = &interp->unicode.fs_codec;
PyMem_RawFree(fs_codec->encoding);
fs_codec->encoding = encoding;
/* encoding has been normalized by init_fs_encoding() */
fs_codec->utf8 = (strcmp(encoding, "utf-8") == 0);
PyMem_RawFree(fs_codec->errors);
fs_codec->errors = errors;
fs_codec->error_handler = error_handler;
#ifdef _Py_FORCE_UTF8_FS_ENCODING
assert(fs_codec->utf8 == 1);
#endif
/* At this point, PyUnicode_EncodeFSDefault() and
PyUnicode_DecodeFSDefault() can now use the Python codec rather than
the C implementation of the filesystem encoding. */
/* Set Py_FileSystemDefaultEncoding and Py_FileSystemDefaultEncodeErrors
global configuration variables. */
if (_Py_SetFileSystemEncoding(fs_codec->encoding,
fs_codec->errors) < 0) {
PyErr_NoMemory();
return -1;
}
return 0;
}
static PyStatus
init_fs_encoding(PyThreadState *tstate)
{
PyInterpreterState *interp = tstate->interp;
/* Update the filesystem encoding to the normalized Python codec name.
For example, replace "ANSI_X3.4-1968" (locale encoding) with "ascii"
(Python codec name). */
PyConfig *config = (PyConfig*)_PyInterpreterState_GetConfig(interp);
if (config_get_codec_name(&config->filesystem_encoding) < 0) {
_Py_DumpPathConfig(tstate);
return _PyStatus_ERR("failed to get the Python codec "
"of the filesystem encoding");
}
if (init_fs_codec(interp) < 0) {
return _PyStatus_ERR("cannot initialize filesystem codec");
}
return _PyStatus_OK();
}
PyStatus
_PyUnicode_InitEncodings(PyThreadState *tstate)
{
PyStatus status = init_fs_encoding(tstate);
if (_PyStatus_EXCEPTION(status)) {
return status;
}
return init_stdio_encoding(tstate);
}
static void
_PyUnicode_FiniEncodings(struct _Py_unicode_fs_codec *fs_codec)
{
PyMem_RawFree(fs_codec->encoding);
fs_codec->encoding = NULL;
fs_codec->utf8 = 0;
PyMem_RawFree(fs_codec->errors);
fs_codec->errors = NULL;
fs_codec->error_handler = _Py_ERROR_UNKNOWN;
}
#ifdef MS_WINDOWS
int
_PyUnicode_EnableLegacyWindowsFSEncoding(void)
{
PyInterpreterState *interp = _PyInterpreterState_GET();
PyConfig *config = (PyConfig *)_PyInterpreterState_GetConfig(interp);
/* Set the filesystem encoding to mbcs/replace (PEP 529) */
wchar_t *encoding = _PyMem_RawWcsdup(L"mbcs");
wchar_t *errors = _PyMem_RawWcsdup(L"replace");
if (encoding == NULL || errors == NULL) {
PyMem_RawFree(encoding);
PyMem_RawFree(errors);
PyErr_NoMemory();
return -1;
}
PyMem_RawFree(config->filesystem_encoding);
config->filesystem_encoding = encoding;
PyMem_RawFree(config->filesystem_errors);
config->filesystem_errors = errors;
return init_fs_codec(interp);
}
#endif
void
_PyUnicode_Fini(PyThreadState *tstate)
{
if (_Py_IsMainInterpreter(tstate)) {
#if defined(WITH_VALGRIND) || defined(__INSURE__)
/* Insure++ is a memory analysis tool that aids in discovering
* memory leaks and other memory problems. On Python exit, the
* interned string dictionaries are flagged as being in use at exit
* (which it is). Under normal circumstances, this is fine because
* the memory will be automatically reclaimed by the system. Under
* memory debugging, it's a huge source of useless noise, so we
* trade off slower shutdown for less distraction in the memory
* reports. -baw
*/
unicode_release_interned();
#endif /* __INSURE__ */
Py_CLEAR(unicode_empty);
#ifdef LATIN1_SINGLETONS
for (Py_ssize_t i = 0; i < 256; i++) {
Py_CLEAR(unicode_latin1[i]);
}
#endif
unicode_clear_static_strings();
}
_PyUnicode_FiniEncodings(&tstate->interp->unicode.fs_codec);
}
/* A _string module, to export formatter_parser and formatter_field_name_split
to the string.Formatter class implemented in Python. */
static PyMethodDef _string_methods[] = {
{"formatter_field_name_split", (PyCFunction) formatter_field_name_split,
METH_O, PyDoc_STR("split the argument as a field name")},
{"formatter_parser", (PyCFunction) formatter_parser,
METH_O, PyDoc_STR("parse the argument as a format string")},
{NULL, NULL}
};
static struct PyModuleDef _string_module = {
PyModuleDef_HEAD_INIT,
"_string",
PyDoc_STR("string helper module"),
0,
_string_methods,
NULL,
NULL,
NULL,
NULL
};
PyMODINIT_FUNC
PyInit__string(void)
{
return PyModule_Create(&_string_module);
}
#ifdef __cplusplus
}
#endif