1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _MM_PERCPU_INTERNAL_H 3 #define _MM_PERCPU_INTERNAL_H 4 5 #include <linux/types.h> 6 #include <linux/percpu.h> 7 8 /* 9 * pcpu_block_md is the metadata block struct. 10 * Each chunk's bitmap is split into a number of full blocks. 11 * All units are in terms of bits. 12 * 13 * The scan hint is the largest known contiguous area before the contig hint. 14 * It is not necessarily the actual largest contig hint though. There is an 15 * invariant that the scan_hint_start > contig_hint_start iff 16 * scan_hint == contig_hint. This is necessary because when scanning forward, 17 * we don't know if a new contig hint would be better than the current one. 18 */ 19 struct pcpu_block_md { 20 int scan_hint; /* scan hint for block */ 21 int scan_hint_start; /* block relative starting 22 position of the scan hint */ 23 int contig_hint; /* contig hint for block */ 24 int contig_hint_start; /* block relative starting 25 position of the contig hint */ 26 int left_free; /* size of free space along 27 the left side of the block */ 28 int right_free; /* size of free space along 29 the right side of the block */ 30 int first_free; /* block position of first free */ 31 int nr_bits; /* total bits responsible for */ 32 }; 33 34 struct pcpu_chunk { 35 #ifdef CONFIG_PERCPU_STATS 36 int nr_alloc; /* # of allocations */ 37 size_t max_alloc_size; /* largest allocation size */ 38 #endif 39 40 struct list_head list; /* linked to pcpu_slot lists */ 41 int free_bytes; /* free bytes in the chunk */ 42 struct pcpu_block_md chunk_md; 43 void *base_addr; /* base address of this chunk */ 44 45 unsigned long *alloc_map; /* allocation map */ 46 unsigned long *bound_map; /* boundary map */ 47 struct pcpu_block_md *md_blocks; /* metadata blocks */ 48 49 void *data; /* chunk data */ 50 bool immutable; /* no [de]population allowed */ 51 int start_offset; /* the overlap with the previous 52 region to have a page aligned 53 base_addr */ 54 int end_offset; /* additional area required to 55 have the region end page 56 aligned */ 57 58 int nr_pages; /* # of pages served by this chunk */ 59 int nr_populated; /* # of populated pages */ 60 int nr_empty_pop_pages; /* # of empty populated pages */ 61 unsigned long populated[]; /* populated bitmap */ 62 }; 63 64 extern spinlock_t pcpu_lock; 65 66 extern struct list_head *pcpu_slot; 67 extern int pcpu_nr_slots; 68 extern int pcpu_nr_empty_pop_pages; 69 70 extern struct pcpu_chunk *pcpu_first_chunk; 71 extern struct pcpu_chunk *pcpu_reserved_chunk; 72 73 /** 74 * pcpu_chunk_nr_blocks - converts nr_pages to # of md_blocks 75 * @chunk: chunk of interest 76 * 77 * This conversion is from the number of physical pages that the chunk 78 * serves to the number of bitmap blocks used. 79 */ 80 static inline int pcpu_chunk_nr_blocks(struct pcpu_chunk *chunk) 81 { 82 return chunk->nr_pages * PAGE_SIZE / PCPU_BITMAP_BLOCK_SIZE; 83 } 84 85 /** 86 * pcpu_nr_pages_to_map_bits - converts the pages to size of bitmap 87 * @pages: number of physical pages 88 * 89 * This conversion is from physical pages to the number of bits 90 * required in the bitmap. 91 */ 92 static inline int pcpu_nr_pages_to_map_bits(int pages) 93 { 94 return pages * PAGE_SIZE / PCPU_MIN_ALLOC_SIZE; 95 } 96 97 /** 98 * pcpu_chunk_map_bits - helper to convert nr_pages to size of bitmap 99 * @chunk: chunk of interest 100 * 101 * This conversion is from the number of physical pages that the chunk 102 * serves to the number of bits in the bitmap. 103 */ 104 static inline int pcpu_chunk_map_bits(struct pcpu_chunk *chunk) 105 { 106 return pcpu_nr_pages_to_map_bits(chunk->nr_pages); 107 } 108 109 #ifdef CONFIG_PERCPU_STATS 110 111 #include <linux/spinlock.h> 112 113 struct percpu_stats { 114 u64 nr_alloc; /* lifetime # of allocations */ 115 u64 nr_dealloc; /* lifetime # of deallocations */ 116 u64 nr_cur_alloc; /* current # of allocations */ 117 u64 nr_max_alloc; /* max # of live allocations */ 118 u32 nr_chunks; /* current # of live chunks */ 119 u32 nr_max_chunks; /* max # of live chunks */ 120 size_t min_alloc_size; /* min allocaiton size */ 121 size_t max_alloc_size; /* max allocation size */ 122 }; 123 124 extern struct percpu_stats pcpu_stats; 125 extern struct pcpu_alloc_info pcpu_stats_ai; 126 127 /* 128 * For debug purposes. We don't care about the flexible array. 129 */ 130 static inline void pcpu_stats_save_ai(const struct pcpu_alloc_info *ai) 131 { 132 memcpy(&pcpu_stats_ai, ai, sizeof(struct pcpu_alloc_info)); 133 134 /* initialize min_alloc_size to unit_size */ 135 pcpu_stats.min_alloc_size = pcpu_stats_ai.unit_size; 136 } 137 138 /* 139 * pcpu_stats_area_alloc - increment area allocation stats 140 * @chunk: the location of the area being allocated 141 * @size: size of area to allocate in bytes 142 * 143 * CONTEXT: 144 * pcpu_lock. 145 */ 146 static inline void pcpu_stats_area_alloc(struct pcpu_chunk *chunk, size_t size) 147 { 148 lockdep_assert_held(&pcpu_lock); 149 150 pcpu_stats.nr_alloc++; 151 pcpu_stats.nr_cur_alloc++; 152 pcpu_stats.nr_max_alloc = 153 max(pcpu_stats.nr_max_alloc, pcpu_stats.nr_cur_alloc); 154 pcpu_stats.min_alloc_size = 155 min(pcpu_stats.min_alloc_size, size); 156 pcpu_stats.max_alloc_size = 157 max(pcpu_stats.max_alloc_size, size); 158 159 chunk->nr_alloc++; 160 chunk->max_alloc_size = max(chunk->max_alloc_size, size); 161 } 162 163 /* 164 * pcpu_stats_area_dealloc - decrement allocation stats 165 * @chunk: the location of the area being deallocated 166 * 167 * CONTEXT: 168 * pcpu_lock. 169 */ 170 static inline void pcpu_stats_area_dealloc(struct pcpu_chunk *chunk) 171 { 172 lockdep_assert_held(&pcpu_lock); 173 174 pcpu_stats.nr_dealloc++; 175 pcpu_stats.nr_cur_alloc--; 176 177 chunk->nr_alloc--; 178 } 179 180 /* 181 * pcpu_stats_chunk_alloc - increment chunk stats 182 */ 183 static inline void pcpu_stats_chunk_alloc(void) 184 { 185 unsigned long flags; 186 spin_lock_irqsave(&pcpu_lock, flags); 187 188 pcpu_stats.nr_chunks++; 189 pcpu_stats.nr_max_chunks = 190 max(pcpu_stats.nr_max_chunks, pcpu_stats.nr_chunks); 191 192 spin_unlock_irqrestore(&pcpu_lock, flags); 193 } 194 195 /* 196 * pcpu_stats_chunk_dealloc - decrement chunk stats 197 */ 198 static inline void pcpu_stats_chunk_dealloc(void) 199 { 200 unsigned long flags; 201 spin_lock_irqsave(&pcpu_lock, flags); 202 203 pcpu_stats.nr_chunks--; 204 205 spin_unlock_irqrestore(&pcpu_lock, flags); 206 } 207 208 #else 209 210 static inline void pcpu_stats_save_ai(const struct pcpu_alloc_info *ai) 211 { 212 } 213 214 static inline void pcpu_stats_area_alloc(struct pcpu_chunk *chunk, size_t size) 215 { 216 } 217 218 static inline void pcpu_stats_area_dealloc(struct pcpu_chunk *chunk) 219 { 220 } 221 222 static inline void pcpu_stats_chunk_alloc(void) 223 { 224 } 225 226 static inline void pcpu_stats_chunk_dealloc(void) 227 { 228 } 229 230 #endif /* !CONFIG_PERCPU_STATS */ 231 232 #endif 233