@@ -493,8 +493,8 @@ typedef enum mi_option_e {
   mi_option_deprecated_visit_abandoned, // allow visiting theap blocks from abandoned threads (=0)
   mi_option_guarded_min,                // only used when building with MI_GUARDED: minimal rounded object size for guarded objects (=0)
   mi_option_guarded_max,                // only used when building with MI_GUARDED: maximal rounded object size for guarded objects (=0)
-  mi_option_guarded_precise,            // disregard minimal alignment requirement to always place guarded blocks exactly in front of a guard page (=0)
-  mi_option_guarded_sample_rate,        // 1 out of N allocations in the min/max range will be guarded (=1000)
+  mi_option_unsafe_guarded_precise,     // disregard minimal alignment requirement to always place guarded blocks exactly in front of a guard page (=0)
+  mi_option_guarded_sample_rate,        // guard an object in the min/max range about every N requested bytes (=0)
   mi_option_guarded_sample_seed,        // can be set to allow for a (more) deterministic re-execution when a guard page is triggered (=0)
   mi_option_generic_collect,            // collect theaps every N (=10000) generic allocation calls
   mi_option_page_reclaim_on_free,       // heap-creation default for reclaim on free: -1 disallow, 0 originating theap only, 1 allow across theaps
@@ -510,10 +510,10 @@ typedef enum mi_option_e {
   mi_option_arena_max_object_size,      // set maximal object size that can be allocated in an arena (in KiB) (=2GiB on 64-bit).
   mi_option_arena_is_numa_local,        // experimental: associate local numa node with an initial arena allocation
   mi_option_stats_merge_on_collect,     // on each theap collection, the theap stats are merged automatically with the parent heap
-  mi_option_profile_alloc_interval,     // number of allocated bytes between automatic dumps of the `MIMALLOC_PROFILE` heap profiler (internally, this value is in KiB; use `mi_option_get_size`) (=0, disabled)
-  mi_option_profile_inuse_interval,     // number of in-use bytes grown between automatic dumps of the `MIMALLOC_PROFILE` heap profiler (internally, this value is in KiB; use `mi_option_get_size`) (=0, disabled)
-  mi_option_profile_time_interval,      // number of seconds between automatic dumps of the `MIMALLOC_PROFILE` heap profiler (=0, disabled)
-  mi_option_profile_sample_rate,        // sample rate (in bytes) for the `MIMALLOC_PROFILE` heap profiler (internally, this value is in KiB; use `mi_option_get_size`) (=16 KiB)
+  mi_option_profile_alloc_interval,     // number of allocated bytes between automatic snapshots of the heap profiler (internally, this value is in KiB; use `mi_option_get_size`) (=1GiB, disabled)
+  mi_option_profile_inuse_interval,     // number of in-use bytes grown between automatic snapshots of the heap profiler (internally, this value is in KiB; use `mi_option_get_size`) (=0, disabled)
+  mi_option_profile_time_interval,      // number of seconds between automatic snapshots of the  heap profiler (=0, disabled)
+  mi_option_profile_sample_rate,        // sample rate (in bytes) for heap profiler (internally, this value is in KiB; use `mi_option_get_size`) (=512 KiB)
   mi_option_stats_merge_threshold,      // on a page stats update, merge theap stats into the parent heap if N bytes were freed/allocated (=0, disabled)
   mi_option_arena_purge_immediate_size, // ranges above this size are purged immediately
   _mi_option_last,
 
@@ -28,7 +28,7 @@ mi_decl_nodiscard bool _mi_page_init(mi_theap_t* theap, mi_page_t* page);
 void          _mi_page_update_stats(mi_page_t* page);
 
 void          _mi_theap_collect_retired(mi_theap_t* theap, bool force);
-void          _mi_theap_page_reclaim(mi_theap_t* theap, mi_page_t* page);
+bool          _mi_theap_page_reclaim(mi_theap_t* theap, mi_page_t* page);  // true if an empty page was freed
 #if MI_DEBUG>1
 bool          _mi_page_is_valid(mi_page_t* page);
 #endif
 
@@ -73,14 +73,22 @@ terms of the MIT license. A copy of the license can be found in the file
 #endif
 
 #if defined(__GNUC__) || defined(__clang__)
-#define mi_unlikely(x)     (__builtin_expect(!!(x),false))
-#define mi_likely(x)       (__builtin_expect(!!(x),true))
+#define mi_unlikely(x)          (__builtin_expect(!!(x),false))
+#define mi_likely(x)            (__builtin_expect(!!(x),true))
 #elif (defined(__cplusplus) && (__cplusplus >= 202002L)) || (defined(_MSVC_LANG) && _MSVC_LANG >= 202002L)
-#define mi_unlikely(x)     (x) [[unlikely]]
-#define mi_likely(x)       (x) [[likely]]
+#define mi_unlikely(x)          (x) [[unlikely]]
+#define mi_likely(x)            (x) [[likely]]
 #else
-#define mi_unlikely(x)     (x)
-#define mi_likely(x)       (x)
+#define mi_unlikely(x)          (x)
+#define mi_likely(x)            (x)
+#endif
+
+#if (defined(__GNUC__) && (__GNUC__ >= 4)) || defined(__clang__)
+#define mi_prefetch_write(p)    __builtin_prefetch(p, 1, 3)
+#define mi_prefetch_read(p)     __builtin_prefetch(p, 0, 3)
+#else
+#define mi_prefetch_write(p)
+#define mi_prefetch_read(p)
 #endif
 
 #if (defined(__GNUC__) && (__GNUC__ >= 7)) || defined(__clang__)
 
@@ -21,7 +21,7 @@ size_t        _mi_os_guard_page_size(void);
 size_t        _mi_os_good_alloc_size(size_t size);
 bool          _mi_os_has_overcommit(void);
 bool          _mi_os_has_virtual_reserve(void);
-bool          _mi_os_canuse_thp(void);
+bool          _mi_os_has_thp(void);
 size_t        _mi_os_virtual_address_bits(void);
 size_t        _mi_os_minimal_purge_size(void);
 
 
@@ -25,6 +25,7 @@ char*         _mi_strnstr(char* s, size_t max_len, const char* pat);
 const char*   _mi_strchr(const char* s, char c);
 const char*   _mi_strrchr(const char* s, char c);
 bool          _mi_streq(const char* s, const char* t);
+bool          _mi_strntol(const char* s, size_t max_len, const char** end, long* result); // parses at most max_len bytes; no terminator required
 int           _mi_getenv(const char* name, char* result, size_t result_size);
 void          _mi_detect_cpu_features(void);
 
 
@@ -17,6 +17,9 @@
 // CPython never uses mimalloc's C++ operator new APIs, so don't define a weak
 // std::get_new_handler() (_ZSt15get_new_handlerv) in libpython.
 #  define MI_NO_CXX_NEW_HANDLER 1
+// The aligned OS allocation fallback is benign, but its warning is printed to
+// stderr and breaks tests that check output (e.g. test_cmd_line under ASan).
+#  define MI_NO_ALIGN_FALLBACK_WARNING 1
 #  ifdef MS_WINDOWS
 // Initialize from a CRT initializer and use FLS for thread cleanup instead
 // of TLS callbacks, which run during DLL attach before the CRT is set up.
 
@@ -107,7 +107,13 @@ void mi_free_block_mt(mi_page_t* page, mi_block_t* block, bool was_guarded, bool
   do {
     mi_block_set_next(page, block, mi_tf_block(tf_old));
     tf_new = mi_tf_create(block, true);
-  } while (!mi_atomic_cas_weak_acq_rel(&page->xthread_free, &tf_old, tf_new));
+    if (mi_tf_is_owned(tf_old)) {
+      if (mi_atomic_cas_weak_release(&page->xthread_free, &tf_old, tf_new)) break; // release is enough if we don't change ownership
+    }
+    else {
+      if (mi_atomic_cas_weak_acq_rel(&page->xthread_free, &tf_old, tf_new)) break; // otherwise we need acquire-release
+    }
+  } while(true);
 
   // and atomically try to collect the page if it was abandoned
   const bool is_newly_owned = mi_tf_is_owned(tf_new) && !mi_tf_is_owned(tf_old);
@@ -216,7 +222,7 @@ static mi_decl_forceinline bool mi_ptr_page_is_valid_ex(const void* p, const cha
 {
   MI_UNUSED_RELEASE(msg); MI_UNUSED(free_small);
   #if MI_DEBUG
-  if mi_unlikely(!mi_is_aligned(p,MI_SIZE_SIZE) && !mi_option_is_enabled(mi_option_guarded_precise)) {
+  if mi_unlikely(!mi_is_aligned(p,MI_SIZE_SIZE) && !mi_option_is_enabled(mi_option_unsafe_guarded_precise)) {
     _mi_error_message(EINVAL, "%s: invalid (unaligned) pointer: %p\n", msg, p);
     return false;
   }
@@ -466,19 +472,29 @@ bool mi_cfree(void* p) mi_attr_noexcept {
 // 3. if it went to < 7/8th used, re-abandon to be mapped (so it can be found by theaps looking for free pages)
 // --------------------------------------------------------------------------------------------
 
-// Consume an empty page, either freeing it or leaving it abandoned until its grace period ends.
+// Consume an empty page, reclaiming delayed pages if this thread has a theap for their heap.
 static bool mi_abandoned_page_collect_empty(mi_page_t* page)
 {
   if (!mi_page_all_free(page)) return false;
   const bool should_delay = _mi_page_qsbr_should_delay(page);
   // first remove it from the abandoned pages in the arena (if mapped, this might wait for any readers to finish)
   _mi_arenas_page_unabandon(page,NULL);
-  if (!should_delay) {
+  if mi_likely(!should_delay) {
     _mi_page_qsbr_clear(page);
     _mi_arenas_page_free(page,NULL);
   }
   else {
-    _mi_arenas_page_abandon(page,NULL);
+    mi_heap_t* const heap = mi_page_heap(page);
+    mi_theap_t* const theap = mi_heap_theap(heap);
+    if (theap != NULL) {
+      // reclaim such that it gets part of the theap's qsbr list.
+      _mi_theap_page_reclaim(theap, page);
+    }
+    else {
+      mi_assert_internal(false);
+      // abandon as empty; it can only be collected by _mi_heap_collect_abandoned at this point
+      _mi_arenas_page_abandon(page,NULL);
+    }
   }
   return true;
 }
 
@@ -450,7 +450,7 @@ mi_block_t* _mi_page_free_collect_partly(mi_page_t* page, mi_block_t* head) {
 ----------------------------------------------------------- */
 
 // called from `mi_free` on a reclaim, and fresh_alloc if we get an abandoned page
-void _mi_theap_page_reclaim(mi_theap_t* theap, mi_page_t* page)
+bool _mi_theap_page_reclaim(mi_theap_t* theap, mi_page_t* page)
 {
   mi_assert_internal(mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN));
   mi_assert_internal(mi_ptr_page(mi_page_start(page))==page);
@@ -463,8 +463,8 @@ void _mi_theap_page_reclaim(mi_theap_t* theap, mi_page_t* page)
   mi_page_queue_t* pq = mi_theap_page_queue_of(theap, page);
   mi_page_queue_push_at_end(theap, pq, page);
   if (!mi_page_all_free(page)) { _mi_page_qsbr_clear(page); }
-  else if (!_mi_page_qsbr_is_clear(page)) { mi_page_qsbr_enqueue(page, theap); }
   mi_assert_expensive(_mi_page_is_valid(page));
+  return (mi_page_all_free(page) && _mi_page_free(page, pq));
 }
 
 void _mi_page_abandon(mi_page_t* page, mi_page_queue_t* pq) {
@@ -488,13 +488,17 @@ mi_page_t* _mi_page_fresh_alloc(mi_theap_t* theap, mi_page_queue_t* pq, size_t b
   mi_assert_internal(mi_theap_contains_queue(theap, pq));
   mi_assert_internal(page_alignment > 0 || block_size > MI_LARGE_MAX_OBJ_SIZE || block_size == pq->block_size);
   #endif
-  mi_page_t* page = _mi_arenas_page_alloc(theap, block_size, page_alignment);
-  if (page == NULL) {
-    // out-of-memory
-    return NULL;
-  }
-  if (mi_page_is_abandoned(page)) {
-    _mi_theap_page_reclaim(theap, page);
+  mi_page_t* page;
+  bool reclaimed;
+  do {
+    page = _mi_arenas_page_alloc(theap, block_size, page_alignment);
+    if (page == NULL) {
+      // out-of-memory
+      return NULL;
+    }
+    reclaimed = mi_page_is_abandoned(page);
+  } while (reclaimed && _mi_theap_page_reclaim(theap, page));
+  if (reclaimed) {
     if (!mi_page_immediate_available(page)) {
       if (mi_page_is_expandable(page)) {
         if (!_mi_page_extend_free(theap, page)) {
 
@@ -8,7 +8,7 @@ terms of the MIT license. A copy of the license can be found in the file
 #ifndef MI_PAGE_H
 #define MI_PAGE_H
 
-#include <mimalloc.h>
+#include "mimalloc.h"
 #include "page-queue.h"
 
 bool       _mi_page_extend_free(mi_theap_t* theap, mi_page_t* page);
 
@@ -220,7 +220,7 @@ mi_theap_t* mi_theap_set_default(mi_theap_t* theap) {
 
 static void mi_theap_options_init(mi_theap_t* theap, mi_heap_t* heap) {
   theap->allow_page_reclaim = (mi_heap_reclaim_on_free(heap) >= 0);
-  theap->allow_page_abandon = (mi_option_get(mi_option_page_full_retain) >= 0 && !mi_heap_qsbr_is_enabled(heap));
+  theap->allow_page_abandon = (mi_option_get(mi_option_page_full_retain) >= 0); // nowadays qsbr pages can be abandoned: && !mi_heap_qsbr_is_enabled(heap));
   theap->page_full_retain = mi_option_get_clamp(mi_option_page_full_retain, -1, 32);
   theap->is_detached = (theap->tld->thread_id == MI_THREADID_DETACHED);
 }
Original file line number	Diff line number	Diff line change
@@ -220,7 +220,7 @@ mi_theap_t* mi_theap_set_default(mi_theap_t* theap) {
220	220
221	221	static void mi_theap_options_init(mi_theap_t* theap, mi_heap_t* heap) {
222	222	theap->allow_page_reclaim = (mi_heap_reclaim_on_free(heap) >= 0);
223		- theap->allow_page_abandon = (mi_option_get(mi_option_page_full_retain) >= 0 && !mi_heap_qsbr_is_enabled(heap));
	223	+ theap->allow_page_abandon = (mi_option_get(mi_option_page_full_retain) >= 0); // nowadays qsbr pages can be abandoned: && !mi_heap_qsbr_is_enabled(heap));
224	224	theap->page_full_retain = mi_option_get_clamp(mi_option_page_full_retain, -1, 32);
225	225	theap->is_detached = (theap->tld->thread_id == MI_THREADID_DETACHED);
226	226	}