1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _MM_PERCPU_INTERNAL_H 3 #define _MM_PERCPU_INTERNAL_H 4 5 #include <linux/types.h> 6 #include <linux/percpu.h> 7 8 /* 9 * pcpu_block_md is the metadata block struct. 10 * Each chunk's bitmap is split into a number of full blocks. 11 * All units are in terms of bits. 12 * 13 * The scan hint is the largest known contiguous area before the contig hint. 14 * It is not necessarily the actual largest contig hint though. There is an 15 * invariant that the scan_hint_start > contig_hint_start iff 16 * scan_hint == contig_hint. This is necessary because when scanning forward, 17 * we don't know if a new contig hint would be better than the current one. 18 */ 19 struct pcpu_block_md { 20 int scan_hint; /* scan hint for block */ 21 int scan_hint_start; /* block relative starting 22 position of the scan hint */ 23 int contig_hint; /* contig hint for block */ 24 int contig_hint_start; /* block relative starting 25 position of the contig hint */ 26 int left_free; /* size of free space along 27 the left side of the block */ 28 int right_free; /* size of free space along 29 the right side of the block */ 30 int first_free; /* block position of first free */ 31 int nr_bits; /* total bits responsible for */ 32 }; 33 34 struct pcpu_chunk { 35 #ifdef CONFIG_PERCPU_STATS 36 int nr_alloc; /* # of allocations */ 37 size_t max_alloc_size; /* largest allocation size */ 38 #endif 39 40 struct list_head list; /* linked to pcpu_slot lists */ 41 int free_bytes; /* free bytes in the chunk */ 42 struct pcpu_block_md chunk_md; 43 void *base_addr; /* base address of this chunk */ 44 45 unsigned long *alloc_map; /* allocation map */ 46 unsigned long *bound_map; /* boundary map */ 47 struct pcpu_block_md *md_blocks; /* metadata blocks */ 48 49 void *data; /* chunk data */ 50 bool immutable; /* no [de]population allowed */ 51 bool isolated; /* isolated from active chunk 52 slots */ 53 int start_offset; /* the overlap with the previous 54 region to have a page aligned 55 base_addr */ 56 int end_offset; /* additional area required to 57 have the region end page 58 aligned */ 59 #ifdef CONFIG_MEMCG_KMEM 60 struct obj_cgroup **obj_cgroups; /* vector of object cgroups */ 61 #endif 62 63 int nr_pages; /* # of pages served by this chunk */ 64 int nr_populated; /* # of populated pages */ 65 int nr_empty_pop_pages; /* # of empty populated pages */ 66 unsigned long populated[]; /* populated bitmap */ 67 }; 68 69 extern spinlock_t pcpu_lock; 70 71 extern struct list_head *pcpu_chunk_lists; 72 extern int pcpu_nr_slots; 73 extern int pcpu_sidelined_slot; 74 extern int pcpu_to_depopulate_slot; 75 extern int pcpu_nr_empty_pop_pages; 76 77 extern struct pcpu_chunk *pcpu_first_chunk; 78 extern struct pcpu_chunk *pcpu_reserved_chunk; 79 80 /** 81 * pcpu_chunk_nr_blocks - converts nr_pages to # of md_blocks 82 * @chunk: chunk of interest 83 * 84 * This conversion is from the number of physical pages that the chunk 85 * serves to the number of bitmap blocks used. 86 */ 87 static inline int pcpu_chunk_nr_blocks(struct pcpu_chunk *chunk) 88 { 89 return chunk->nr_pages * PAGE_SIZE / PCPU_BITMAP_BLOCK_SIZE; 90 } 91 92 /** 93 * pcpu_nr_pages_to_map_bits - converts the pages to size of bitmap 94 * @pages: number of physical pages 95 * 96 * This conversion is from physical pages to the number of bits 97 * required in the bitmap. 98 */ 99 static inline int pcpu_nr_pages_to_map_bits(int pages) 100 { 101 return pages * PAGE_SIZE / PCPU_MIN_ALLOC_SIZE; 102 } 103 104 /** 105 * pcpu_chunk_map_bits - helper to convert nr_pages to size of bitmap 106 * @chunk: chunk of interest 107 * 108 * This conversion is from the number of physical pages that the chunk 109 * serves to the number of bits in the bitmap. 110 */ 111 static inline int pcpu_chunk_map_bits(struct pcpu_chunk *chunk) 112 { 113 return pcpu_nr_pages_to_map_bits(chunk->nr_pages); 114 } 115 116 /** 117 * pcpu_obj_full_size - helper to calculate size of each accounted object 118 * @size: size of area to allocate in bytes 119 * 120 * For each accounted object there is an extra space which is used to store 121 * obj_cgroup membership. Charge it too. 122 */ 123 static inline size_t pcpu_obj_full_size(size_t size) 124 { 125 size_t extra_size = 0; 126 127 #ifdef CONFIG_MEMCG_KMEM 128 extra_size += size / PCPU_MIN_ALLOC_SIZE * sizeof(struct obj_cgroup *); 129 #endif 130 131 return size * num_possible_cpus() + extra_size; 132 } 133 134 #ifdef CONFIG_PERCPU_STATS 135 136 #include <linux/spinlock.h> 137 138 struct percpu_stats { 139 u64 nr_alloc; /* lifetime # of allocations */ 140 u64 nr_dealloc; /* lifetime # of deallocations */ 141 u64 nr_cur_alloc; /* current # of allocations */ 142 u64 nr_max_alloc; /* max # of live allocations */ 143 u32 nr_chunks; /* current # of live chunks */ 144 u32 nr_max_chunks; /* max # of live chunks */ 145 size_t min_alloc_size; /* min allocation size */ 146 size_t max_alloc_size; /* max allocation size */ 147 }; 148 149 extern struct percpu_stats pcpu_stats; 150 extern struct pcpu_alloc_info pcpu_stats_ai; 151 152 /* 153 * For debug purposes. We don't care about the flexible array. 154 */ 155 static inline void pcpu_stats_save_ai(const struct pcpu_alloc_info *ai) 156 { 157 memcpy(&pcpu_stats_ai, ai, sizeof(struct pcpu_alloc_info)); 158 159 /* initialize min_alloc_size to unit_size */ 160 pcpu_stats.min_alloc_size = pcpu_stats_ai.unit_size; 161 } 162 163 /* 164 * pcpu_stats_area_alloc - increment area allocation stats 165 * @chunk: the location of the area being allocated 166 * @size: size of area to allocate in bytes 167 * 168 * CONTEXT: 169 * pcpu_lock. 170 */ 171 static inline void pcpu_stats_area_alloc(struct pcpu_chunk *chunk, size_t size) 172 { 173 lockdep_assert_held(&pcpu_lock); 174 175 pcpu_stats.nr_alloc++; 176 pcpu_stats.nr_cur_alloc++; 177 pcpu_stats.nr_max_alloc = 178 max(pcpu_stats.nr_max_alloc, pcpu_stats.nr_cur_alloc); 179 pcpu_stats.min_alloc_size = 180 min(pcpu_stats.min_alloc_size, size); 181 pcpu_stats.max_alloc_size = 182 max(pcpu_stats.max_alloc_size, size); 183 184 chunk->nr_alloc++; 185 chunk->max_alloc_size = max(chunk->max_alloc_size, size); 186 } 187 188 /* 189 * pcpu_stats_area_dealloc - decrement allocation stats 190 * @chunk: the location of the area being deallocated 191 * 192 * CONTEXT: 193 * pcpu_lock. 194 */ 195 static inline void pcpu_stats_area_dealloc(struct pcpu_chunk *chunk) 196 { 197 lockdep_assert_held(&pcpu_lock); 198 199 pcpu_stats.nr_dealloc++; 200 pcpu_stats.nr_cur_alloc--; 201 202 chunk->nr_alloc--; 203 } 204 205 /* 206 * pcpu_stats_chunk_alloc - increment chunk stats 207 */ 208 static inline void pcpu_stats_chunk_alloc(void) 209 { 210 unsigned long flags; 211 spin_lock_irqsave(&pcpu_lock, flags); 212 213 pcpu_stats.nr_chunks++; 214 pcpu_stats.nr_max_chunks = 215 max(pcpu_stats.nr_max_chunks, pcpu_stats.nr_chunks); 216 217 spin_unlock_irqrestore(&pcpu_lock, flags); 218 } 219 220 /* 221 * pcpu_stats_chunk_dealloc - decrement chunk stats 222 */ 223 static inline void pcpu_stats_chunk_dealloc(void) 224 { 225 unsigned long flags; 226 spin_lock_irqsave(&pcpu_lock, flags); 227 228 pcpu_stats.nr_chunks--; 229 230 spin_unlock_irqrestore(&pcpu_lock, flags); 231 } 232 233 #else 234 235 static inline void pcpu_stats_save_ai(const struct pcpu_alloc_info *ai) 236 { 237 } 238 239 static inline void pcpu_stats_area_alloc(struct pcpu_chunk *chunk, size_t size) 240 { 241 } 242 243 static inline void pcpu_stats_area_dealloc(struct pcpu_chunk *chunk) 244 { 245 } 246 247 static inline void pcpu_stats_chunk_alloc(void) 248 { 249 } 250 251 static inline void pcpu_stats_chunk_dealloc(void) 252 { 253 } 254 255 #endif /* !CONFIG_PERCPU_STATS */ 256 257 #endif 258