diff --git a/pep-0445.txt b/pep-0445.txt index ea1ec0209..292627abd 100644 --- a/pep-0445.txt +++ b/pep-0445.txt @@ -80,114 +80,178 @@ API changes Examples ======== -Use case 1: replace memory allocators, keeping pymalloc -------------------------------------------------------- +Use case 1: Replace Memory Allocators, keep pymalloc +---------------------------------------------------- Setup your custom memory allocators, keeping pymalloc:: - /* global variable, don't use a variable allocated on the stack! */ - int magic = 42; + #include - int my_malloc(void *ctx, size_t size); - int my_realloc(void *ctx, void *ptr, size_t new_size); - void my_free(void *ctx, void *ptr); + int alloc_padding = 2; + int arena_padding = 10; - int my_alloc_arena(void *ctx, size_t size); - int my_free_arena(void *ctx, void *ptr, size_t size); + void* my_malloc(void *ctx, size_t size) + { + int padding = *(int *)ctx; + return malloc(size + padding); + } + + void* my_realloc(void *ctx, void *ptr, size_t new_size) + { + int padding = *(int *)ctx; + return realloc(ptr, new_size + padding); + } + + void my_free(void *ctx, void *ptr) + { + free(ptr); + } + + void* my_alloc_arena(void *ctx, size_t size) + { + int padding = *(int *)ctx; + return malloc(size + padding); + } + + void my_free_arena(void *ctx, void *ptr, size_t size) + { + free(ptr); + } void setup_custom_allocators(void) { PyMemAllocators alloc; - alloc.ctx = &magic; + + alloc.ctx = &alloc_padding; alloc.malloc = my_malloc; alloc.realloc = my_realloc; alloc.free = my_free; PyMem_SetRawAllocators(&alloc); PyMem_SetAllocators(&alloc); - _PyObject_SetArenaAllocators(&magic, my_alloc_arena, my_free_arena); + + _PyObject_SetArenaAllocators(&arena_padding, + my_alloc_arena, my_free_arena); PyMem_SetupDebugHooks(); } .. warning:: - Remove call ``PyMem_SetRawAllocators(&alloc);`` if the new allocators are - not thread-safe. - -Full example: -`replace_allocs.c `_. + Remove the call ``PyMem_SetRawAllocators(&alloc);`` if the new allocators + are not thread-safe. -Use case 2: replace memory allocators, overriding pymalloc +Use case 2: Replace Memory Allocators, overriding pymalloc ---------------------------------------------------------- If your allocator is optimized for allocation of small objects (less than 512 bytes) with a short liftime, you can replace override pymalloc (replace ``PyObject_Malloc()``). Example:: - /* global variable, don't use a variable allocated on the stack! */ - int magic = 42; + #include - int my_malloc(void *ctx, size_t size); - int my_realloc(void *ctx, void *ptr, size_t new_size); - void my_free(void *ctx, void *ptr); + int padding = 2; + + void* my_malloc(void *ctx, size_t size) + { + int padding = *(int *)ctx; + return malloc(size + padding); + } + + void* my_realloc(void *ctx, void *ptr, size_t new_size) + { + int padding = *(int *)ctx; + return realloc(ptr, new_size + padding); + } + + void my_free(void *ctx, void *ptr) + { + free(ptr); + } void setup_custom_allocators(void) { PyMemAllocators alloc; - alloc.ctx = &magic; + alloc.ctx = &padding; alloc.malloc = my_malloc; alloc.realloc = my_realloc; alloc.free = my_free; PyMem_SetRawAllocators(&alloc); PyMem_SetAllocators(&alloc); - PyObject_SetAllocators(&areana); + PyObject_SetAllocators(&alloc); + PyMem_SetupDebugHooks(); } -Full example: -`replace_pymalloc.c `_. +.. warning:: + Remove the call ``PyMem_SetRawAllocators(&alloc);`` if the new allocators + are not thread-safe. -Use case 3: hook allocators ---------------------------- + +Use case 3: Setup Allocator Hooks +--------------------------------- Setup hooks on memory allocators:: - /* global variable, don't use a variable allocated on the stack! */ struct { PyMemAllocators pymem; PyMemAllocators pymem_raw; PyMemAllocators pyobj; - int magic; + /* ... */ } hook; - int hook_malloc(void *ctx, size_t size); - int hook_realloc(void *ctx, void *ptr, size_t new_size); - void hook_free(void *ctx, void *ptr); + static void* hook_malloc(void *ctx, size_t size) + { + PyMemAllocators *alloc = (PyMemAllocators *)ctx; + /* ... */ + ptr = alloc->malloc(alloc->ctx, size); + /* ... */ + return ptr; + } - /* Must be called before the first allocation, or hook_realloc() and - hook_free() will crash */ - void setup_custom_allocators(void) + static void* hook_realloc(void *ctx, void *ptr, size_t new_size) + { + PyMemAllocators *alloc = (PyMemAllocators *)ctx; + void *ptr2; + /* ... */ + ptr2 = alloc->realloc(alloc->ctx, ptr, new_size); + /* ... */ + return ptr2; + } + + static void hook_free(void *ctx, void *ptr) + { + PyMemAllocators *alloc = (PyMemAllocators *)ctx; + /* ... */ + alloc->free(alloc->ctx, ptr); + /* ... */ + } + + void setup_hooks(void) { PyMemAllocators alloc; + static int registered = 0; + + if (registered) + return; + registered = 1; - alloc.ctx = &magic; alloc.malloc = hook_malloc; alloc.realloc = hook_realloc; alloc.free = hook_free; - PyMem_GetRawAllocators(&alloc.pymem_raw); - alloc.ctx = &alloc.pymem_raw; + PyMem_GetRawAllocators(&hook.pymem_raw); + alloc.ctx = &hook.pymem_raw; PyMem_SetRawAllocators(&alloc); - PyMem_GetAllocators(&alloc.pymem); - alloc.ctx = &alloc.pymem; + PyMem_GetAllocators(&hook.pymem); + alloc.ctx = &hook.pymem; PyMem_SetAllocators(&alloc); - PyObject_GetAllocators(&alloc.pyobj); - alloc.ctx = &alloc.pyobj; + PyObject_GetAllocators(&hook.pyobj); + alloc.ctx = &hook.pyobj; PyObject_SetAllocators(&alloc); } @@ -195,10 +259,6 @@ Setup hooks on memory allocators:: No need to call ``PyMem_SetupDebugHooks()``: it is already installed by default. -Full example tracking memory usage: -`alloc_hooks.c `_. - - Performances ============ diff --git a/pep-0445/alloc_hooks.c b/pep-0445/alloc_hooks.c deleted file mode 100644 index 2e69bedbd..000000000 --- a/pep-0445/alloc_hooks.c +++ /dev/null @@ -1,114 +0,0 @@ -/* global variable, don't use a variable allocated on the stack! */ -struct { - PyMemAllocators pymem; - PyMemAllocators pymem_raw; - PyMemAllocators pyobj; - size_t allocated; -} hook; - -/* read_size_t() and write_size_t() are not needed if malloc() and realloc() - always return a pointer aligned to sizeof(size_t) bytes */ -static size_t read_size_t(const void *p) -{ - const unsigned char *q = (const unsigned char *)p; - size_t result = *q++; - int i; - - for (i = SST; --i > 0; ++q) - result = (result << 8) | *q; - return result; -} - -static void write_size_t(void *p, size_t n) -{ - unsigned char *q = (unsigned char *)p + SST - 1; - int i; - - for (i = SST; --i >= 0; --q) { - *q = (unsigned char)(n & 0xff); - n >>= 8; - } -} - -static int hook_malloc(void *ctx, size_t size) -{ - PyMemAllocators *alloc; - char *ptr; - - size += sizeof(size_t); - ptr = alloc->malloc(size); - if (ptr != NULL) { - write_size_t(ptr, size); - ptr += sizeof(size_t); - allocated += size; - } - return ptr; -} - -static int hook_realloc(void *ctx, void *void_ptr, size_t new_size) -{ - PyMemAllocators *alloc; - char *ptr, *ptr2; - size_t old_size; - - ptr = void_ptr; - if (ptr) { - ptr -= sizeof(size_t); - old_size = read_size_t(ptr); - } - else { - old_size = 0; - } - - ptr2 = alloc->realloc(ptr, size); - if (ptr2 != NULL) { - write_size_t(ptr2, size); - ptr2 += sizeof(size_t); - allocated -= old_size; - allocated += new_size; - } - return ptr2; -} - -static void hook_free(void *ctx, void *void_ptr) -{ - PyMemAllocators *alloc; - char *ptr; - size_t size; - - ptr = void_ptr; - if (!ptr) - return; - - ptr -= sizeof(size_t); - size = read_size_t(ptr); - - alloc->free(ptr); - allocated -= size; -} - -/* Must be called before the first allocation, or hook_realloc() and - hook_free() will crash */ -void setup_custom_allocators(void) -{ - PyMemAllocators alloc; - - alloc.malloc = my_malloc; - alloc.realloc = my_realloc; - alloc.free = my_free; - - /* disabled: the hook is not thread-safe */ -#if 0 - PyMem_GetRawAllocators(&alloc.pymem_raw); - alloc.ctx = &alloc.pymem_raw; - PyMem_SetRawAllocators(&alloc); -#endif - - PyMem_GetAllocators(&alloc.pymem); - alloc.ctx = &alloc.pymem; - PyMem_SetAllocators(&alloc); - - PyObject_GetAllocators(&alloc.pyobj); - alloc.ctx = &alloc.pyobj; - PyObject_SetAllocators(&alloc); -} diff --git a/pep-0445/replace_allocs.c b/pep-0445/replace_allocs.c deleted file mode 100644 index abe55c1de..000000000 --- a/pep-0445/replace_allocs.c +++ /dev/null @@ -1,44 +0,0 @@ -#include - -/* global variable, don't use a variable allocated on the stack! */ -int magic = 42; - -int my_malloc(void *ctx, size_t size) -{ - return malloc(size); -} - -int my_realloc(void *ctx, void *ptr, size_t new_size) -{ - return realloc(ptr, new_size); -} - -void my_free(void *ctx, void *ptr) -{ - free(ptr); -} - -int my_alloc_arena(void *ctx, size_t size) -{ - return malloc(size); -} - -int my_free_arena(void *ctx, void *ptr, size_t size) -{ - free(ptr); -} - -void setup_custom_allocators(void) -{ - PyMemAllocators alloc; - alloc.ctx = &magic; - alloc.malloc = my_malloc; - alloc.realloc = my_realloc; - alloc.free = my_free; - - PyMem_SetRawAllocators(&alloc); - PyMem_SetAllocators(&alloc); - _PyObject_SetArenaAllocators(&magic, my_alloc_arena, my_free_arena); - PyMem_SetupDebugHooks(); -} - diff --git a/pep-0445/replace_pymalloc.c b/pep-0445/replace_pymalloc.c deleted file mode 100644 index efe9f810a..000000000 --- a/pep-0445/replace_pymalloc.c +++ /dev/null @@ -1,34 +0,0 @@ -#include - -/* global variable, don't use a variable allocated on the stack! */ -int magic = 42; - -int my_malloc(void *ctx, size_t size) -{ - return malloc(size); -} - -int my_realloc(void *ctx, void *ptr, size_t new_size) -{ - return realloc(ptr, new_size); -} - -void my_free(void *ctx, void *ptr) -{ - free(ptr); -} - -void setup_custom_allocators(void) -{ - PyMemAllocators alloc; - alloc.ctx = &magic; - alloc.malloc = my_malloc; - alloc.realloc = my_realloc; - alloc.free = my_free; - - PyMem_SetRawAllocators(&alloc); - PyMem_SetAllocators(&alloc); - PyObject_SetAllocators(&areana); - PyMem_SetupDebugHooks(); -} -