| FazBrowse GitHub Viewer | Trending | | Home |
| Tools: [Download Repo ZIP] [Original HTTPS Page] |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -342,6 +342,7 @@ typedef enum mi_option_e { | |||
| 342 | 342 | mi_option_arena_reserve, // initial memory size in KiB for arena reservation (1GiB on 64-bit) | |
| 343 | 343 | mi_option_arena_purge_mult, | |
| 344 | 344 | mi_option_purge_extend_delay, | |
| 345 | + mi_option_abandoned_reclaim_on_free, // reclaim abandoned segments on a free | ||
| 345 | 346 | _mi_option_last, | |
| 346 | 347 | // legacy option names | |
| 347 | 348 | mi_option_large_os_pages = mi_option_allow_large_os_pages, | |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -147,6 +147,7 @@ void _mi_segment_huge_page_reset(mi_segment_t* segment, mi_page_t* page, m | |||
| 147 | 147 | void _mi_segment_thread_collect(mi_segments_tld_t* tld); | |
| 148 | 148 | void _mi_abandoned_reclaim_all(mi_heap_t* heap, mi_segments_tld_t* tld); | |
| 149 | 149 | void _mi_abandoned_await_readers(void); | |
| 150 | + bool _mi_segment_attempt_reclaim(mi_heap_t* heap, mi_segment_t* segment); | ||
| 150 | 151 | ||
| 151 | 152 | // "page.c" | |
| 152 | 153 | void* _mi_malloc_generic(mi_heap_t* heap, size_t size, bool zero, size_t huge_alignment) mi_attr_noexcept mi_attr_malloc; | |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -406,12 +406,24 @@ static void mi_stat_huge_free(const mi_page_t* page) { | |||
| 406 | 406 | // multi-threaded free (or free in huge block if compiled with MI_HUGE_PAGE_ABANDON) | |
| 407 | 407 | static mi_decl_noinline void _mi_free_block_mt(mi_page_t* page, mi_block_t* block) | |
| 408 | 408 | { | |
| 409 | + // first see if the segment was abandoned and we can reclaim it | ||
| 410 | + mi_segment_t* const segment = _mi_page_segment(page); | ||
| 411 | + if (mi_option_is_enabled(mi_option_abandoned_reclaim_on_free) && | ||
| 412 | + mi_atomic_load_relaxed(&segment->thread_id) == 0) | ||
| 413 | + { | ||
| 414 | + // the segment is abandoned, try to reclaim it into our heap | ||
| 415 | + if (_mi_segment_attempt_reclaim(mi_prim_get_default_heap(), segment)) { | ||
| 416 | + mi_assert_internal(_mi_prim_thread_id() == mi_atomic_load_relaxed(&segment->thread_id)); | ||
| 417 | + mi_free(block); // recursively free as now it will be a local free in our heap | ||
| 418 | + return; | ||
| 419 | + } | ||
| 420 | + } | ||
| 421 | + | ||
| 409 | 422 | // The padding check may access the non-thread-owned page for the key values. | |
| 410 | 423 | // that is safe as these are constant and the page won't be freed (as the block is not freed yet). | |
| 411 | 424 | mi_check_padding(page, block); | |
| 412 | 425 | _mi_padding_shrink(page, block, sizeof(mi_block_t)); // for small size, ensure we can fit the delayed thread pointers without triggering overflow detection | |
| 413 | 426 | ||
| 414 | - mi_segment_t* const segment = _mi_page_segment(page); | ||
| 415 | 427 | if (segment->page_kind == MI_PAGE_HUGE) { | |
| 416 | 428 | #if MI_HUGE_PAGE_ABANDON | |
| 417 | 429 | // huge page segments are always abandoned and can be freed immediately | |
@@ -424,9 +436,8 @@ static mi_decl_noinline void _mi_free_block_mt(mi_page_t* page, mi_block_t* bloc | |||
| 424 | 436 | // (as the owning thread needs to actually free the memory later). | |
| 425 | 437 | _mi_segment_huge_page_reset(segment, page, block); | |
| 426 | 438 | #endif | |
| 427 | - } | ||
| 428 | - | ||
| 429 | - | ||
| 439 | + } | ||
| 440 | + | ||
| 430 | 441 | #if (MI_DEBUG>0) && !MI_TRACK_ENABLED && !MI_TSAN // note: when tracking, cannot use mi_usable_size with multi-threading | |
| 431 | 442 | memset(block, MI_DEBUG_FREED, mi_usable_size(block)); | |
| 432 | 443 | #endif | |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -749,9 +749,9 @@ bool _mi_arena_segment_clear_abandoned(mi_memid_t memid ) | |||
| 749 | 749 | mi_arena_t* arena = mi_atomic_load_ptr_acquire(mi_arena_t, &mi_arenas[arena_idx]); | |
| 750 | 750 | mi_assert_internal(arena != NULL); | |
| 751 | 751 | bool was_abandoned = _mi_bitmap_unclaim(arena->blocks_abandoned, arena->field_count, 1, bitmap_idx); | |
| 752 | - mi_assert_internal(was_abandoned); | ||
| 753 | - mi_assert_internal(_mi_bitmap_is_claimed(arena->blocks_inuse, arena->field_count, 1, bitmap_idx)); | ||
| 754 | - mi_assert_internal(arena->blocks_committed == NULL || _mi_bitmap_is_claimed(arena->blocks_committed, arena->field_count, 1, bitmap_idx)); | ||
| 752 | + // mi_assert_internal(was_abandoned); | ||
| 753 | + mi_assert_internal(!was_abandoned || _mi_bitmap_is_claimed(arena->blocks_inuse, arena->field_count, 1, bitmap_idx)); | ||
| 754 | + //mi_assert_internal(arena->blocks_committed == NULL || _mi_bitmap_is_claimed(arena->blocks_committed, arena->field_count, 1, bitmap_idx)); | ||
| 755 | 755 | return was_abandoned; | |
| 756 | 756 | } | |
| 757 | 757 | ||
@@ -766,6 +766,7 @@ void _mi_arena_segment_mark_abandoned(mi_memid_t memid) | |||
| 766 | 766 | mi_arena_t* arena = mi_atomic_load_ptr_acquire(mi_arena_t, &mi_arenas[arena_idx]); | |
| 767 | 767 | mi_assert_internal(arena != NULL); | |
| 768 | 768 | const bool was_unset = _mi_bitmap_claim(arena->blocks_abandoned, arena->field_count, 1, bitmap_idx, NULL); | |
| 769 | + MI_UNUSED_RELEASE(was_unset); | ||
| 769 | 770 | mi_assert_internal(was_unset); | |
| 770 | 771 | mi_assert_internal(_mi_bitmap_is_claimed(arena->blocks_inuse, arena->field_count, 1, bitmap_idx)); | |
| 771 | 772 | } | |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -81,16 +81,17 @@ static mi_option_desc_t options[_mi_option_last] = | |||
| 81 | 81 | { 100, UNINIT, MI_OPTION(os_tag) }, // only apple specific for now but might serve more or less related purpose | |
| 82 | 82 | { 16, UNINIT, MI_OPTION(max_errors) }, // maximum errors that are output | |
| 83 | 83 | { 16, UNINIT, MI_OPTION(max_warnings) }, // maximum warnings that are output | |
| 84 | - { 8, UNINIT, MI_OPTION(max_segment_reclaim)}, // max. number of segment reclaims from the abandoned segments per try. | ||
| 84 | + { 16, UNINIT, MI_OPTION(max_segment_reclaim)}, // max. number of segment reclaims from the abandoned segments per try. | ||
| 85 | 85 | { 0, UNINIT, MI_OPTION(destroy_on_exit)}, // release all OS memory on process exit; careful with dangling pointer or after-exit frees! | |
| 86 | 86 | #if (MI_INTPTR_SIZE>4) | |
| 87 | 87 | { 1024L * 1024L, UNINIT, MI_OPTION(arena_reserve) }, // reserve memory N KiB at a time | |
| 88 | 88 | #else | |
| 89 | 89 | { 128L * 1024L, UNINIT, MI_OPTION(arena_reserve) }, | |
| 90 | 90 | #endif | |
| 91 | 91 | ||
| 92 | - { 10, UNINIT, MI_OPTION(arena_purge_mult) }, // purge delay multiplier for arena's | ||
| 92 | + { 10, UNINIT, MI_OPTION(arena_purge_mult) }, // purge delay multiplier for arena's | ||
| 93 | 93 | { 1, UNINIT, MI_OPTION_LEGACY(purge_extend_delay, decommit_extend_delay) }, | |
| 94 | + { 1, UNINIT, MI_OPTION(abandoned_reclaim_on_free) }, // reclaim an abandoned segment on a free | ||
| 94 | 95 | }; | |
| 95 | 96 | ||
| 96 | 97 | static void mi_option_init(mi_option_desc_t* desc); | |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -336,7 +336,7 @@ static void mi_segment_remove_all_purges(mi_segment_t* segment, bool force_purge | |||
| 336 | 336 | mi_page_t* page = &segment->pages[i]; | |
| 337 | 337 | if (!page->segment_in_use) { | |
| 338 | 338 | mi_page_purge_remove(page, tld); | |
| 339 | - if (force_purge) { | ||
| 339 | + if (force_purge && page->is_committed) { | ||
| 340 | 340 | mi_page_purge(segment, page, tld); | |
| 341 | 341 | } | |
| 342 | 342 | } | |
@@ -902,6 +902,16 @@ static mi_segment_t* mi_segment_reclaim(mi_segment_t* segment, mi_heap_t* heap, | |||
| 902 | 902 | } | |
| 903 | 903 | } | |
| 904 | 904 | ||
| 905 | + // attempt to reclaim a particular segment (called from multi threaded free `alloc.c:mi_free_block_mt`) | ||
| 906 | + bool _mi_segment_attempt_reclaim(mi_heap_t* heap, mi_segment_t* segment) { | ||
| 907 | + if (mi_atomic_load_relaxed(&segment->thread_id) != 0) return false; // it is not abandoned | ||
| 908 | + if (_mi_arena_segment_clear_abandoned(segment->memid)) { // atomically unabandon | ||
| 909 | + mi_segment_t* res = mi_segment_reclaim(segment, heap, 0, NULL, &heap->tld->segments); | ||
| 910 | + mi_assert_internal(res != NULL); | ||
| 911 | + return (res != NULL); | ||
| 912 | + } | ||
| 913 | + return false; | ||
| 914 | + } | ||
| 905 | 915 | ||
| 906 | 916 | void _mi_abandoned_reclaim_all(mi_heap_t* heap, mi_segments_tld_t* tld) { | |
| 907 | 917 | mi_segment_t* segment; | |
| Back | FazBrowse Home | New Git URL |
0 commit comments