1 #ifndef JEMALLOC_INTERNAL_PROF_INLINES_A_H 2 #define JEMALLOC_INTERNAL_PROF_INLINES_A_H 3 4 #include "jemalloc/internal/mutex.h" 5 6 static inline bool 7 prof_accum_add(tsdn_t *tsdn, prof_accum_t *prof_accum, uint64_t accumbytes) { 8 cassert(config_prof); 9 10 bool overflow; 11 uint64_t a0, a1; 12 13 /* 14 * If the application allocates fast enough (and/or if idump is slow 15 * enough), extreme overflow here (a1 >= prof_interval * 2) can cause 16 * idump trigger coalescing. This is an intentional mechanism that 17 * avoids rate-limiting allocation. 18 */ 19 #ifdef JEMALLOC_ATOMIC_U64 20 a0 = atomic_load_u64(&prof_accum->accumbytes, ATOMIC_RELAXED); 21 do { 22 a1 = a0 + accumbytes; 23 assert(a1 >= a0); 24 overflow = (a1 >= prof_interval); 25 if (overflow) { 26 a1 %= prof_interval; 27 } 28 } while (!atomic_compare_exchange_weak_u64(&prof_accum->accumbytes, &a0, 29 a1, ATOMIC_RELAXED, ATOMIC_RELAXED)); 30 #else 31 malloc_mutex_lock(tsdn, &prof_accum->mtx); 32 a0 = prof_accum->accumbytes; 33 a1 = a0 + accumbytes; 34 overflow = (a1 >= prof_interval); 35 if (overflow) { 36 a1 %= prof_interval; 37 } 38 prof_accum->accumbytes = a1; 39 malloc_mutex_unlock(tsdn, &prof_accum->mtx); 40 #endif 41 return overflow; 42 } 43 44 static inline void 45 prof_accum_cancel(tsdn_t *tsdn, prof_accum_t *prof_accum, size_t usize) { 46 cassert(config_prof); 47 48 /* 49 * Cancel out as much of the excessive prof_accumbytes increase as 50 * possible without underflowing. Interval-triggered dumps occur 51 * slightly more often than intended as a result of incomplete 52 * canceling. 53 */ 54 uint64_t a0, a1; 55 #ifdef JEMALLOC_ATOMIC_U64 56 a0 = atomic_load_u64(&prof_accum->accumbytes, ATOMIC_RELAXED); 57 do { 58 a1 = (a0 >= LARGE_MINCLASS - usize) ? a0 - (LARGE_MINCLASS - 59 usize) : 0; 60 } while (!atomic_compare_exchange_weak_u64(&prof_accum->accumbytes, &a0, 61 a1, ATOMIC_RELAXED, ATOMIC_RELAXED)); 62 #else 63 malloc_mutex_lock(tsdn, &prof_accum->mtx); 64 a0 = prof_accum->accumbytes; 65 a1 = (a0 >= LARGE_MINCLASS - usize) ? a0 - (LARGE_MINCLASS - usize) : 66 0; 67 prof_accum->accumbytes = a1; 68 malloc_mutex_unlock(tsdn, &prof_accum->mtx); 69 #endif 70 } 71 72 #endif /* JEMALLOC_INTERNAL_PROF_INLINES_A_H */ 73