| FazBrowse GitHub Viewer | Trending | | Home |
| Tools: [Download Repo ZIP] [Original HTTPS Page] |
1 parent df715ba commit 0507bf5
8 files changed
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -84,15 +84,65 @@ the C library allocator as shown in the previous example, the allocated memory | |||
| 84 | 84 | for the I/O buffer escapes completely the Python memory manager. | |
| 85 | 85 | ||
| 86 | 86 | ||
| 87 | + Raw Memory Interface | ||
| 88 | + ==================== | ||
| 89 | + | ||
| 90 | + The following function sets are wrappers to the system allocator. These | ||
| 91 | + functions are thread-safe, the :term:`GIL <global interpreter lock>` does not | ||
| 92 | + need to be held. | ||
| 93 | + | ||
| 94 | + The default raw memory block allocator uses the following functions: | ||
| 95 | + :c:func:`malloc`, :c:func:`realloc` and :c:func:`free`; call ``malloc(1)`` when | ||
| 96 | + requesting zero bytes. | ||
| 97 | + | ||
| 98 | + .. versionadded:: 3.4 | ||
| 99 | + | ||
| 100 | + .. c:function:: void* PyMem_RawMalloc(size_t n) | ||
| 101 | + | ||
| 102 | + Allocates *n* bytes and returns a pointer of type :c:type:`void\*` to the | ||
| 103 | + allocated memory, or *NULL* if the request fails. Requesting zero bytes | ||
| 104 | + returns a distinct non-*NULL* pointer if possible, as if | ||
| 105 | + ``PyMem_RawMalloc(1)`` had been called instead. The memory will not have | ||
| 106 | + been initialized in any way. | ||
| 107 | + | ||
| 108 | + | ||
| 109 | + .. c:function:: void* PyMem_RawRealloc(void *p, size_t n) | ||
| 110 | + | ||
| 111 | + Resizes the memory block pointed to by *p* to *n* bytes. The contents will | ||
| 112 | + be unchanged to the minimum of the old and the new sizes. If *p* is *NULL*, | ||
| 113 | + the call is equivalent to ``PyMem_RawMalloc(n)``; else if *n* is equal to | ||
| 114 | + zero, the memory block is resized but is not freed, and the returned pointer | ||
| 115 | + is non-*NULL*. Unless *p* is *NULL*, it must have been returned by a | ||
| 116 | + previous call to :c:func:`PyMem_RawMalloc` or :c:func:`PyMem_RawRealloc`. If | ||
| 117 | + the request fails, :c:func:`PyMem_RawRealloc` returns *NULL* and *p* remains | ||
| 118 | + a valid pointer to the previous memory area. | ||
| 119 | + | ||
| 120 | + | ||
| 121 | + .. c:function:: void PyMem_RawFree(void *p) | ||
| 122 | + | ||
| 123 | + Frees the memory block pointed to by *p*, which must have been returned by a | ||
| 124 | + previous call to :c:func:`PyMem_RawMalloc` or :c:func:`PyMem_RawRealloc`. | ||
| 125 | + Otherwise, or if ``PyMem_Free(p)`` has been called before, undefined | ||
| 126 | + behavior occurs. If *p* is *NULL*, no operation is performed. | ||
| 127 | + | ||
| 128 | + | ||
| 87 | 129 | .. _memoryinterface: | |
| 88 | 130 | ||
| 89 | 131 | Memory Interface | |
| 90 | 132 | ================ | |
| 91 | 133 | ||
| 92 | 134 | The following function sets, modeled after the ANSI C standard, but specifying | |
| 93 | 135 | behavior when requesting zero bytes, are available for allocating and releasing | |
| 94 | - memory from the Python heap: | ||
| 136 | + memory from the Python heap. | ||
| 137 | + | ||
| 138 | + The default memory block allocator uses the following functions: | ||
| 139 | + :c:func:`malloc`, :c:func:`realloc` and :c:func:`free`; call ``malloc(1)`` when | ||
| 140 | + requesting zero bytes. | ||
| 141 | + | ||
| 142 | + .. warning:: | ||
| 95 | 143 | ||
| 144 | + The :term:`GIL <global interpreter lock>` must be held when using these | ||
| 145 | + functions. | ||
| 96 | 146 | ||
| 97 | 147 | .. c:function:: void* PyMem_Malloc(size_t n) | |
| 98 | 148 | ||
@@ -155,6 +205,125 @@ versions and is therefore deprecated in extension modules. | |||
| 155 | 205 | :c:func:`PyMem_NEW`, :c:func:`PyMem_RESIZE`, :c:func:`PyMem_DEL`. | |
| 156 | 206 | ||
| 157 | 207 | ||
| 208 | + Customize Memory Allocators | ||
| 209 | + =========================== | ||
| 210 | + | ||
| 211 | + .. versionadded:: 3.4 | ||
| 212 | + | ||
| 213 | + .. c:type:: PyMemAllocator | ||
| 214 | + | ||
| 215 | + Structure used to describe a memory block allocator. The structure has | ||
| 216 | + four fields: | ||
| 217 | + | ||
| 218 | + +----------------------------------------------------------+---------------------------------------+ | ||
| 219 | + | Field | Meaning | | ||
| 220 | + +==========================================================+=======================================+ | ||
| 221 | + | ``void *ctx`` | user context passed as first argument | | ||
| 222 | + +----------------------------------------------------------+---------------------------------------+ | ||
| 223 | + | ``void* malloc(void *ctx, size_t size)`` | allocate a memory block | | ||
| 224 | + +----------------------------------------------------------+---------------------------------------+ | ||
| 225 | + | ``void* realloc(void *ctx, void *ptr, size_t new_size)`` | allocate or resize a memory block | | ||
| 226 | + +----------------------------------------------------------+---------------------------------------+ | ||
| 227 | + | ``void free(void *ctx, void *ptr)`` | free a memory block | | ||
| 228 | + +----------------------------------------------------------+---------------------------------------+ | ||
| 229 | + | ||
| 230 | + .. c:type:: PyMemAllocatorDomain | ||
| 231 | + | ||
| 232 | + Enum used to identify an allocator domain. Domains: | ||
| 233 | + | ||
| 234 | + * :c:data:`PYMEM_DOMAIN_RAW`: functions :c:func:`PyMem_RawMalloc`, | ||
| 235 | + :c:func:`PyMem_RawRealloc` and :c:func:`PyMem_RawFree` | ||
| 236 | + * :c:data:`PYMEM_DOMAIN_MEM`: functions :c:func:`PyMem_Malloc`, | ||
| 237 | + :c:func:`PyMem_Realloc` and :c:func:`PyMem_Free` | ||
| 238 | + * :c:data:`PYMEM_DOMAIN_OBJ`: functions :c:func:`PyObject_Malloc`, | ||
| 239 | + :c:func:`PyObject_Realloc` and :c:func:`PyObject_Free` | ||
| 240 | + | ||
| 241 | + | ||
| 242 | + .. c:function:: void PyMem_GetAllocator(PyMemAllocatorDomain domain, PyMemAllocator *allocator) | ||
| 243 | + | ||
| 244 | + Get the memory block allocator of the specified domain. | ||
| 245 | + | ||
| 246 | + | ||
| 247 | + .. c:function:: void PyMem_SetAllocator(PyMemAllocatorDomain domain, PyMemAllocator *allocator) | ||
| 248 | + | ||
| 249 | + Set the memory block allocator of the specified domain. | ||
| 250 | + | ||
| 251 | + The new allocator must return a distinct non-NULL pointer when requesting | ||
| 252 | + zero bytes. | ||
| 253 | + | ||
| 254 | + For the :c:data:`PYMEM_DOMAIN_RAW` domain, the allocator must be | ||
| 255 | + thread-safe: the :term:`GIL <global interpreter lock>` is not held when the | ||
| 256 | + allocator is called. | ||
| 257 | + | ||
| 258 | + If the new allocator is not a hook (does not call the previous allocator), | ||
| 259 | + the :c:func:`PyMem_SetupDebugHooks` function must be called to reinstall the | ||
| 260 | + debug hooks on top on the new allocator. | ||
| 261 | + | ||
| 262 | + | ||
| 263 | + .. c:function:: void PyMem_SetupDebugHooks(void) | ||
| 264 | + | ||
| 265 | + Setup hooks to detect bugs in the following Python memory allocator | ||
| 266 | + functions: | ||
| 267 | + | ||
| 268 | + - :c:func:`PyMem_RawMalloc`, :c:func:`PyMem_RawRealloc`, | ||
| 269 | + :c:func:`PyMem_RawFree` | ||
| 270 | + - :c:func:`PyMem_Malloc`, :c:func:`PyMem_Realloc`, :c:func:`PyMem_Free` | ||
| 271 | + - :c:func:`PyObject_Malloc`, :c:func:`PyObject_Realloc`, | ||
| 272 | + :c:func:`PyObject_Free` | ||
| 273 | + | ||
| 274 | + Newly allocated memory is filled with the byte ``0xCB``, freed memory is | ||
| 275 | + filled with the byte ``0xDB``. Additionnal checks: | ||
| 276 | + | ||
| 277 | + - detect API violations, ex: :c:func:`PyObject_Free` called on a buffer | ||
| 278 | + allocated by :c:func:`PyMem_Malloc` | ||
| 279 | + - detect write before the start of the buffer (buffer underflow) | ||
| 280 | + - detect write after the end of the buffer (buffer overflow) | ||
| 281 | + | ||
| 282 | + The function does nothing if Python is not compiled is debug mode. | ||
| 283 | + | ||
| 284 | + | ||
| 285 | + Customize PyObject Arena Allocator | ||
| 286 | + ================================== | ||
| 287 | + | ||
| 288 | + Python has a *pymalloc* allocator for allocations smaller than 512 bytes. This | ||
| 289 | + allocator is optimized for small objects with a short lifetime. It uses memory | ||
| 290 | + mappings called "arenas" with a fixed size of 256 KB. It falls back to | ||
| 291 | + :c:func:`PyMem_Malloc` and :c:func:`PyMem_Realloc` for allocations larger than | ||
| 292 | + 512 bytes. *pymalloc* is the default allocator used by | ||
| 293 | + :c:func:`PyObject_Malloc`. | ||
| 294 | + | ||
| 295 | + The default arena allocator uses the following functions: | ||
| 296 | + | ||
| 297 | + * :c:func:`VirtualAlloc` and :c:func:`VirtualFree` on Windows, | ||
| 298 | + * :c:func:`mmap` and :c:func:`munmap` if available, | ||
| 299 | + * :c:func:`malloc` and :c:func:`free` otherwise. | ||
| 300 | + | ||
| 301 | + .. versionadded:: 3.4 | ||
| 302 | + | ||
| 303 | + .. c:type:: PyObjectArenaAllocator | ||
| 304 | + | ||
| 305 | + Structure used to describe an arena allocator. The structure has | ||
| 306 | + three fields: | ||
| 307 | + | ||
| 308 | + +--------------------------------------------------+---------------------------------------+ | ||
| 309 | + | Field | Meaning | | ||
| 310 | + +==================================================+=======================================+ | ||
| 311 | + | ``void *ctx`` | user context passed as first argument | | ||
| 312 | + +--------------------------------------------------+---------------------------------------+ | ||
| 313 | + | ``void* alloc(void *ctx, size_t size)`` | allocate an arena of size bytes | | ||
| 314 | + +--------------------------------------------------+---------------------------------------+ | ||
| 315 | + | ``void free(void *ctx, size_t size, void *ptr)`` | free an arena | | ||
| 316 | + +--------------------------------------------------+---------------------------------------+ | ||
| 317 | + | ||
| 318 | + .. c:function:: PyObject_GetArenaAllocator(PyObjectArenaAllocator *allocator) | ||
| 319 | + | ||
| 320 | + Get the arena allocator. | ||
| 321 | + | ||
| 322 | + .. c:function:: PyObject_SetArenaAllocator(PyObjectArenaAllocator *allocator) | ||
| 323 | + | ||
| 324 | + Set the arena allocator. | ||
| 325 | + | ||
| 326 | + | ||
| 158 | 327 | .. _memoryexamples: | |
| 159 | 328 | ||
| 160 | 329 | Examples | |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -112,21 +112,11 @@ Security improvements: | |||
| 112 | 112 | Please read on for a comprehensive list of user-facing changes. | |
| 113 | 113 | ||
| 114 | 114 | ||
| 115 | - .. PEP-sized items next. | ||
| 116 | - | ||
| 117 | - .. _pep-4XX: | ||
| 118 | - | ||
| 119 | - .. PEP 4XX: Example PEP | ||
| 120 | - .. ==================== | ||
| 121 | - | ||
| 122 | - | ||
| 123 | - .. (Implemented by Foo Bar.) | ||
| 124 | - | ||
| 125 | - .. .. seealso:: | ||
| 126 | - | ||
| 127 | - :pep:`4XX` - Example PEP | ||
| 128 | - PEP written by Example Author | ||
| 115 | + PEP 445: Add new APIs to customize Python memory allocators | ||
| 116 | + =========================================================== | ||
| 129 | 117 | ||
| 118 | + The :pep:`445` adds new Application Programming Interfaces (API) to customize | ||
| 119 | + Python memory allocators. | ||
| 130 | 120 | ||
| 131 | 121 | ||
| 132 | 122 | ||
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -94,9 +94,9 @@ PyObject_{New, NewVar, Del}. | |||
| 94 | 94 | the object gets initialized via PyObject_{Init, InitVar} after obtaining | |
| 95 | 95 | the raw memory. | |
| 96 | 96 | */ | |
| 97 | - PyAPI_FUNC(void *) PyObject_Malloc(size_t); | ||
| 98 | - PyAPI_FUNC(void *) PyObject_Realloc(void *, size_t); | ||
| 99 | - PyAPI_FUNC(void) PyObject_Free(void *); | ||
| 97 | + PyAPI_FUNC(void *) PyObject_Malloc(size_t size); | ||
| 98 | + PyAPI_FUNC(void *) PyObject_Realloc(void *ptr, size_t new_size); | ||
| 99 | + PyAPI_FUNC(void) PyObject_Free(void *ptr); | ||
| 100 | 100 | ||
| 101 | 101 | /* This function returns the number of allocated memory blocks, regardless of size */ | |
| 102 | 102 | PyAPI_FUNC(Py_ssize_t) _Py_GetAllocatedBlocks(void); | |
@@ -106,41 +106,15 @@ PyAPI_FUNC(Py_ssize_t) _Py_GetAllocatedBlocks(void); | |||
| 106 | 106 | #ifndef Py_LIMITED_API | |
| 107 | 107 | PyAPI_FUNC(void) _PyObject_DebugMallocStats(FILE *out); | |
| 108 | 108 | #endif /* #ifndef Py_LIMITED_API */ | |
| 109 | - #ifdef PYMALLOC_DEBUG /* WITH_PYMALLOC && PYMALLOC_DEBUG */ | ||
| 110 | - PyAPI_FUNC(void *) _PyObject_DebugMalloc(size_t nbytes); | ||
| 111 | - PyAPI_FUNC(void *) _PyObject_DebugRealloc(void *p, size_t nbytes); | ||
| 112 | - PyAPI_FUNC(void) _PyObject_DebugFree(void *p); | ||
| 113 | - PyAPI_FUNC(void) _PyObject_DebugDumpAddress(const void *p); | ||
| 114 | - PyAPI_FUNC(void) _PyObject_DebugCheckAddress(const void *p); | ||
| 115 | - PyAPI_FUNC(void *) _PyObject_DebugMallocApi(char api, size_t nbytes); | ||
| 116 | - PyAPI_FUNC(void *) _PyObject_DebugReallocApi(char api, void *p, size_t nbytes); | ||
| 117 | - PyAPI_FUNC(void) _PyObject_DebugFreeApi(char api, void *p); | ||
| 118 | - PyAPI_FUNC(void) _PyObject_DebugCheckAddressApi(char api, const void *p); | ||
| 119 | - PyAPI_FUNC(void *) _PyMem_DebugMalloc(size_t nbytes); | ||
| 120 | - PyAPI_FUNC(void *) _PyMem_DebugRealloc(void *p, size_t nbytes); | ||
| 121 | - PyAPI_FUNC(void) _PyMem_DebugFree(void *p); | ||
| 122 | - #define PyObject_MALLOC _PyObject_DebugMalloc | ||
| 123 | - #define PyObject_Malloc _PyObject_DebugMalloc | ||
| 124 | - #define PyObject_REALLOC _PyObject_DebugRealloc | ||
| 125 | - #define PyObject_Realloc _PyObject_DebugRealloc | ||
| 126 | - #define PyObject_FREE _PyObject_DebugFree | ||
| 127 | - #define PyObject_Free _PyObject_DebugFree | ||
| 128 | - | ||
| 129 | - #else /* WITH_PYMALLOC && ! PYMALLOC_DEBUG */ | ||
| 109 | + #endif | ||
| 110 | + | ||
| 111 | + /* Macros */ | ||
| 130 | 112 | #define PyObject_MALLOC PyObject_Malloc | |
| 131 | 113 | #define PyObject_REALLOC PyObject_Realloc | |
| 132 | 114 | #define PyObject_FREE PyObject_Free | |
| 133 | - #endif | ||
| 134 | - | ||
| 135 | - #else /* ! WITH_PYMALLOC */ | ||
| 136 | - #define PyObject_MALLOC PyMem_MALLOC | ||
| 137 | - #define PyObject_REALLOC PyMem_REALLOC | ||
| 138 | - #define PyObject_FREE PyMem_FREE | ||
| 139 | - | ||
| 140 | - #endif /* WITH_PYMALLOC */ | ||
| 141 | - | ||
| 142 | 115 | #define PyObject_Del PyObject_Free | |
| 143 | - #define PyObject_DEL PyObject_FREE | ||
| 116 | + #define PyObject_DEL PyObject_Free | ||
| 117 | + | ||
| 144 | 118 | ||
| 145 | 119 | /* | |
| 146 | 120 | * Generic object allocator interface | |
@@ -224,6 +198,26 @@ PyAPI_FUNC(PyVarObject *) _PyObject_NewVar(PyTypeObject *, Py_ssize_t); | |||
| 224 | 198 | constructor you would start directly with PyObject_Init/InitVar | |
| 225 | 199 | */ | |
| 226 | 200 | ||
| 201 | + #ifndef Py_LIMITED_API | ||
| 202 | + typedef struct { | ||
| 203 | + /* user context passed as the first argument to the 2 functions */ | ||
| 204 | + void *ctx; | ||
| 205 | + | ||
| 206 | + /* allocate an arena of size bytes */ | ||
| 207 | + void* (*alloc) (void *ctx, size_t size); | ||
| 208 | + | ||
| 209 | + /* free an arena */ | ||
| 210 | + void (*free) (void *ctx, void *ptr, size_t size); | ||
| 211 | + } PyObjectArenaAllocator; | ||
| 212 | + | ||
| 213 | + /* Get the arena allocator. */ | ||
| 214 | + PyAPI_FUNC(void) PyObject_GetArenaAllocator(PyObjectArenaAllocator *allocator); | ||
| 215 | + | ||
| 216 | + /* Set the arena allocator. */ | ||
| 217 | + PyAPI_FUNC(void) PyObject_SetArenaAllocator(PyObjectArenaAllocator *allocator); | ||
| 218 | + #endif | ||
| 219 | + | ||
| 220 | + | ||
| 227 | 221 | /* | |
| 228 | 222 | * Garbage Collection Support | |
| 229 | 223 | * ========================== | |
| Back | FazBrowse Home | New Git URL |
0 commit comments