1 #ifndef JEMALLOC_INTERNAL_SEC_H 2 #define JEMALLOC_INTERNAL_SEC_H 3 4 #include "jemalloc/internal/atomic.h" 5 #include "jemalloc/internal/pai.h" 6 7 /* 8 * Small extent cache. 9 * 10 * This includes some utilities to cache small extents. We have a per-pszind 11 * bin with its own list of extents of that size. We don't try to do any 12 * coalescing of extents (since it would in general require cross-shard locks or 13 * knowledge of the underlying PAI implementation). 14 */ 15 16 /* 17 * For now, this is just one field; eventually, we'll probably want to get more 18 * fine-grained data out (like per-size class statistics). 19 */ 20 typedef struct sec_stats_s sec_stats_t; 21 struct sec_stats_s { 22 /* Sum of bytes_cur across all shards. */ 23 size_t bytes; 24 }; 25 26 static inline void 27 sec_stats_accum(sec_stats_t *dst, sec_stats_t *src) { 28 dst->bytes += src->bytes; 29 } 30 31 /* A collections of free extents, all of the same size. */ 32 typedef struct sec_bin_s sec_bin_t; 33 struct sec_bin_s { 34 /* 35 * When we fail to fulfill an allocation, we do a batch-alloc on the 36 * underlying allocator to fill extra items, as well. We drop the SEC 37 * lock while doing so, to allow operations on other bins to succeed. 38 * That introduces the possibility of other threads also trying to 39 * allocate out of this bin, failing, and also going to the backing 40 * allocator. To avoid a thundering herd problem in which lots of 41 * threads do batch allocs and overfill this bin as a result, we only 42 * allow one batch allocation at a time for a bin. This bool tracks 43 * whether or not some thread is already batch allocating. 44 * 45 * Eventually, the right answer may be a smarter sharding policy for the 46 * bins (e.g. a mutex per bin, which would also be more scalable 47 * generally; the batch-allocating thread could hold it while 48 * batch-allocating). 49 */ 50 bool being_batch_filled; 51 52 /* 53 * Number of bytes in this particular bin (as opposed to the 54 * sec_shard_t's bytes_cur. This isn't user visible or reported in 55 * stats; rather, it allows us to quickly determine the change in the 56 * centralized counter when flushing. 57 */ 58 size_t bytes_cur; 59 edata_list_active_t freelist; 60 }; 61 62 typedef struct sec_shard_s sec_shard_t; 63 struct sec_shard_s { 64 /* 65 * We don't keep per-bin mutexes, even though that would allow more 66 * sharding; this allows global cache-eviction, which in turn allows for 67 * better balancing across free lists. 68 */ 69 malloc_mutex_t mtx; 70 /* 71 * A SEC may need to be shut down (i.e. flushed of its contents and 72 * prevented from further caching). To avoid tricky synchronization 73 * issues, we just track enabled-status in each shard, guarded by a 74 * mutex. In practice, this is only ever checked during brief races, 75 * since the arena-level atomic boolean tracking HPA enabled-ness means 76 * that we won't go down these pathways very often after custom extent 77 * hooks are installed. 78 */ 79 bool enabled; 80 sec_bin_t *bins; 81 /* Number of bytes in all bins in the shard. */ 82 size_t bytes_cur; 83 /* The next pszind to flush in the flush-some pathways. */ 84 pszind_t to_flush_next; 85 }; 86 87 typedef struct sec_s sec_t; 88 struct sec_s { 89 pai_t pai; 90 pai_t *fallback; 91 92 sec_opts_t opts; 93 sec_shard_t *shards; 94 pszind_t npsizes; 95 }; 96 97 bool sec_init(tsdn_t *tsdn, sec_t *sec, base_t *base, pai_t *fallback, 98 const sec_opts_t *opts); 99 void sec_flush(tsdn_t *tsdn, sec_t *sec); 100 void sec_disable(tsdn_t *tsdn, sec_t *sec); 101 102 /* 103 * Morally, these two stats methods probably ought to be a single one (and the 104 * mutex_prof_data ought to live in the sec_stats_t. But splitting them apart 105 * lets them fit easily into the pa_shard stats framework (which also has this 106 * split), which simplifies the stats management. 107 */ 108 void sec_stats_merge(tsdn_t *tsdn, sec_t *sec, sec_stats_t *stats); 109 void sec_mutex_stats_read(tsdn_t *tsdn, sec_t *sec, 110 mutex_prof_data_t *mutex_prof_data); 111 112 /* 113 * We use the arena lock ordering; these are acquired in phase 2 of forking, but 114 * should be acquired before the underlying allocator mutexes. 115 */ 116 void sec_prefork2(tsdn_t *tsdn, sec_t *sec); 117 void sec_postfork_parent(tsdn_t *tsdn, sec_t *sec); 118 void sec_postfork_child(tsdn_t *tsdn, sec_t *sec); 119 120 #endif /* JEMALLOC_INTERNAL_SEC_H */ 121