1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef __LINUX_PERCPU_H 3 #define __LINUX_PERCPU_H 4 5 #include <linux/alloc_tag.h> 6 #include <linux/mmdebug.h> 7 #include <linux/preempt.h> 8 #include <linux/smp.h> 9 #include <linux/pfn.h> 10 #include <linux/init.h> 11 #include <linux/cleanup.h> 12 #include <linux/sched.h> 13 14 #include <asm/percpu.h> 15 16 /* enough to cover all DEFINE_PER_CPUs in modules */ 17 #ifdef CONFIG_MODULES 18 #ifdef CONFIG_MEM_ALLOC_PROFILING 19 #define PERCPU_MODULE_RESERVE (8 << 13) 20 #else 21 #define PERCPU_MODULE_RESERVE (8 << 10) 22 #endif 23 #else 24 #define PERCPU_MODULE_RESERVE 0 25 #endif 26 27 /* minimum unit size, also is the maximum supported allocation size */ 28 #define PCPU_MIN_UNIT_SIZE PFN_ALIGN(32 << 10) 29 30 /* minimum allocation size and shift in bytes */ 31 #define PCPU_MIN_ALLOC_SHIFT 2 32 #define PCPU_MIN_ALLOC_SIZE (1 << PCPU_MIN_ALLOC_SHIFT) 33 34 /* 35 * The PCPU_BITMAP_BLOCK_SIZE must be the same size as PAGE_SIZE as the 36 * updating of hints is used to manage the nr_empty_pop_pages in both 37 * the chunk and globally. 38 */ 39 #define PCPU_BITMAP_BLOCK_SIZE PAGE_SIZE 40 #define PCPU_BITMAP_BLOCK_BITS (PCPU_BITMAP_BLOCK_SIZE >> \ 41 PCPU_MIN_ALLOC_SHIFT) 42 43 #ifdef CONFIG_RANDOM_KMALLOC_CACHES 44 #define PERCPU_DYNAMIC_SIZE_SHIFT 12 45 #else 46 #define PERCPU_DYNAMIC_SIZE_SHIFT 10 47 #endif 48 49 /* 50 * Percpu allocator can serve percpu allocations before slab is 51 * initialized which allows slab to depend on the percpu allocator. 52 * The following parameter decide how much resource to preallocate 53 * for this. Keep PERCPU_DYNAMIC_RESERVE equal to or larger than 54 * PERCPU_DYNAMIC_EARLY_SIZE. 55 */ 56 #define PERCPU_DYNAMIC_EARLY_SIZE (20 << PERCPU_DYNAMIC_SIZE_SHIFT) 57 58 /* 59 * PERCPU_DYNAMIC_RESERVE indicates the amount of free area to piggy 60 * back on the first chunk for dynamic percpu allocation if arch is 61 * manually allocating and mapping it for faster access (as a part of 62 * large page mapping for example). 63 * 64 * The following values give between one and two pages of free space 65 * after typical minimal boot (2-way SMP, single disk and NIC) with 66 * both defconfig and a distro config on x86_64 and 32. More 67 * intelligent way to determine this would be nice. 68 */ 69 #if BITS_PER_LONG > 32 70 #define PERCPU_DYNAMIC_RESERVE (28 << PERCPU_DYNAMIC_SIZE_SHIFT) 71 #else 72 #define PERCPU_DYNAMIC_RESERVE (20 << PERCPU_DYNAMIC_SIZE_SHIFT) 73 #endif 74 75 extern void *pcpu_base_addr; 76 extern const unsigned long *pcpu_unit_offsets; 77 78 struct pcpu_group_info { 79 int nr_units; /* aligned # of units */ 80 unsigned long base_offset; /* base address offset */ 81 unsigned int *cpu_map; /* unit->cpu map, empty 82 * entries contain NR_CPUS */ 83 }; 84 85 struct pcpu_alloc_info { 86 size_t static_size; 87 size_t reserved_size; 88 size_t dyn_size; 89 size_t unit_size; 90 size_t atom_size; 91 size_t alloc_size; 92 size_t __ai_size; /* internal, don't use */ 93 int nr_groups; /* 0 if grouping unnecessary */ 94 struct pcpu_group_info groups[]; 95 }; 96 97 enum pcpu_fc { 98 PCPU_FC_AUTO, 99 PCPU_FC_EMBED, 100 PCPU_FC_PAGE, 101 102 PCPU_FC_NR, 103 }; 104 extern const char * const pcpu_fc_names[PCPU_FC_NR]; 105 106 extern enum pcpu_fc pcpu_chosen_fc; 107 108 typedef int (pcpu_fc_cpu_to_node_fn_t)(int cpu); 109 typedef int (pcpu_fc_cpu_distance_fn_t)(unsigned int from, unsigned int to); 110 111 extern struct pcpu_alloc_info * __init pcpu_alloc_alloc_info(int nr_groups, 112 int nr_units); 113 extern void __init pcpu_free_alloc_info(struct pcpu_alloc_info *ai); 114 115 extern void __init pcpu_setup_first_chunk(const struct pcpu_alloc_info *ai, 116 void *base_addr); 117 118 extern int __init pcpu_embed_first_chunk(size_t reserved_size, size_t dyn_size, 119 size_t atom_size, 120 pcpu_fc_cpu_distance_fn_t cpu_distance_fn, 121 pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn); 122 123 #ifdef CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK 124 void __init pcpu_populate_pte(unsigned long addr); 125 extern int __init pcpu_page_first_chunk(size_t reserved_size, 126 pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn); 127 #endif 128 129 extern bool __is_kernel_percpu_address(unsigned long addr, unsigned long *can_addr); 130 extern bool is_kernel_percpu_address(unsigned long addr); 131 132 #if !defined(CONFIG_SMP) || !defined(CONFIG_HAVE_SETUP_PER_CPU_AREA) 133 extern void __init setup_per_cpu_areas(void); 134 #endif 135 136 extern void __percpu *pcpu_alloc_noprof(size_t size, size_t align, bool reserved, 137 gfp_t gfp) __alloc_size(1); 138 extern size_t pcpu_alloc_size(void __percpu *__pdata); 139 140 #define __alloc_percpu_gfp(_size, _align, _gfp) \ 141 alloc_hooks(pcpu_alloc_noprof(_size, _align, false, _gfp)) 142 #define __alloc_percpu(_size, _align) \ 143 alloc_hooks(pcpu_alloc_noprof(_size, _align, false, GFP_KERNEL)) 144 #define __alloc_reserved_percpu(_size, _align) \ 145 alloc_hooks(pcpu_alloc_noprof(_size, _align, true, GFP_KERNEL)) 146 147 #define alloc_percpu_gfp(type, gfp) \ 148 (typeof(type) __percpu *)__alloc_percpu_gfp(sizeof(type), \ 149 __alignof__(type), gfp) 150 #define alloc_percpu(type) \ 151 (typeof(type) __percpu *)__alloc_percpu(sizeof(type), \ 152 __alignof__(type)) 153 #define alloc_percpu_noprof(type) \ 154 ((typeof(type) __percpu *)pcpu_alloc_noprof(sizeof(type), \ 155 __alignof__(type), false, GFP_KERNEL)) 156 157 extern void free_percpu(void __percpu *__pdata); 158 159 DEFINE_FREE(free_percpu, void __percpu *, free_percpu(_T)) 160 161 extern phys_addr_t per_cpu_ptr_to_phys(void *addr); 162 163 extern unsigned long pcpu_nr_pages(void); 164 165 #endif /* __LINUX_PERCPU_H */ 166