1 #ifndef JEMALLOC_INTERNAL_PROF_INLINES_A_H 2 #define JEMALLOC_INTERNAL_PROF_INLINES_A_H 3 4 #include "jemalloc/internal/mutex.h" 5 6 static inline bool 7 prof_accum_add(tsdn_t *tsdn, prof_accum_t *prof_accum, uint64_t accumbytes) { 8 cassert(config_prof); 9 10 bool overflow; 11 uint64_t a0, a1; 12 13 /* 14 * If the application allocates fast enough (and/or if idump is slow 15 * enough), extreme overflow here (a1 >= prof_interval * 2) can cause 16 * idump trigger coalescing. This is an intentional mechanism that 17 * avoids rate-limiting allocation. 18 */ 19 #ifdef JEMALLOC_ATOMIC_U64 20 a0 = atomic_load_u64(&prof_accum->accumbytes, ATOMIC_RELAXED); 21 do { 22 a1 = a0 + accumbytes; 23 assert(a1 >= a0); 24 overflow = (a1 >= prof_interval); 25 if (overflow) { 26 a1 %= prof_interval; 27 } 28 } while (!atomic_compare_exchange_weak_u64(&prof_accum->accumbytes, &a0, 29 a1, ATOMIC_RELAXED, ATOMIC_RELAXED)); 30 #else 31 malloc_mutex_lock(tsdn, &prof_accum->mtx); 32 a0 = prof_accum->accumbytes; 33 a1 = a0 + accumbytes; 34 overflow = (a1 >= prof_interval); 35 if (overflow) { 36 a1 %= prof_interval; 37 } 38 prof_accum->accumbytes = a1; 39 malloc_mutex_unlock(tsdn, &prof_accum->mtx); 40 #endif 41 return overflow; 42 } 43 44 static inline void 45 prof_accum_cancel(tsdn_t *tsdn, prof_accum_t *prof_accum, size_t usize) { 46 cassert(config_prof); 47 48 /* 49 * Cancel out as much of the excessive prof_accumbytes increase as 50 * possible without underflowing. Interval-triggered dumps occur 51 * slightly more often than intended as a result of incomplete 52 * canceling. 53 */ 54 uint64_t a0, a1; 55 #ifdef JEMALLOC_ATOMIC_U64 56 a0 = atomic_load_u64(&prof_accum->accumbytes, ATOMIC_RELAXED); 57 do { 58 a1 = (a0 >= LARGE_MINCLASS - usize) ? a0 - (LARGE_MINCLASS - 59 usize) : 0; 60 } while (!atomic_compare_exchange_weak_u64(&prof_accum->accumbytes, &a0, 61 a1, ATOMIC_RELAXED, ATOMIC_RELAXED)); 62 #else 63 malloc_mutex_lock(tsdn, &prof_accum->mtx); 64 a0 = prof_accum->accumbytes; 65 a1 = (a0 >= LARGE_MINCLASS - usize) ? a0 - (LARGE_MINCLASS - usize) : 66 0; 67 prof_accum->accumbytes = a1; 68 malloc_mutex_unlock(tsdn, &prof_accum->mtx); 69 #endif 70 } 71 72 JEMALLOC_ALWAYS_INLINE bool 73 prof_active_get_unlocked(void) { 74 /* 75 * Even if opt_prof is true, sampling can be temporarily disabled by 76 * setting prof_active to false. No locking is used when reading 77 * prof_active in the fast path, so there are no guarantees regarding 78 * how long it will take for all threads to notice state changes. 79 */ 80 return prof_active; 81 } 82 83 #endif /* JEMALLOC_INTERNAL_PROF_INLINES_A_H */ 84