1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef __LINUX_PERCPU_H 3 #define __LINUX_PERCPU_H 4 5 #include <linux/alloc_tag.h> 6 #include <linux/mmdebug.h> 7 #include <linux/preempt.h> 8 #include <linux/smp.h> 9 #include <linux/pfn.h> 10 #include <linux/init.h> 11 #include <linux/cleanup.h> 12 #include <linux/sched.h> 13 14 #include <asm/percpu.h> 15 16 /* enough to cover all DEFINE_PER_CPUs in modules */ 17 #ifdef CONFIG_MODULES 18 #define PERCPU_MODULE_RESERVE (8 << 10) 19 #else 20 #define PERCPU_MODULE_RESERVE 0 21 #endif 22 23 /* minimum unit size, also is the maximum supported allocation size */ 24 #define PCPU_MIN_UNIT_SIZE PFN_ALIGN(32 << 10) 25 26 /* minimum allocation size and shift in bytes */ 27 #define PCPU_MIN_ALLOC_SHIFT 2 28 #define PCPU_MIN_ALLOC_SIZE (1 << PCPU_MIN_ALLOC_SHIFT) 29 30 /* 31 * The PCPU_BITMAP_BLOCK_SIZE must be the same size as PAGE_SIZE as the 32 * updating of hints is used to manage the nr_empty_pop_pages in both 33 * the chunk and globally. 34 */ 35 #define PCPU_BITMAP_BLOCK_SIZE PAGE_SIZE 36 #define PCPU_BITMAP_BLOCK_BITS (PCPU_BITMAP_BLOCK_SIZE >> \ 37 PCPU_MIN_ALLOC_SHIFT) 38 39 #ifdef CONFIG_RANDOM_KMALLOC_CACHES 40 # if defined(CONFIG_LOCKDEP) && !defined(CONFIG_PAGE_SIZE_4KB) 41 # define PERCPU_DYNAMIC_SIZE_SHIFT 13 42 # else 43 # define PERCPU_DYNAMIC_SIZE_SHIFT 12 44 #endif /* LOCKDEP and PAGE_SIZE > 4KiB */ 45 #else 46 #define PERCPU_DYNAMIC_SIZE_SHIFT 10 47 #endif 48 49 /* 50 * Percpu allocator can serve percpu allocations before slab is 51 * initialized which allows slab to depend on the percpu allocator. 52 * The following parameter decide how much resource to preallocate 53 * for this. Keep PERCPU_DYNAMIC_RESERVE equal to or larger than 54 * PERCPU_DYNAMIC_EARLY_SIZE. 55 */ 56 #define PERCPU_DYNAMIC_EARLY_SIZE (20 << PERCPU_DYNAMIC_SIZE_SHIFT) 57 58 /* 59 * PERCPU_DYNAMIC_RESERVE indicates the amount of free area to piggy 60 * back on the first chunk for dynamic percpu allocation if arch is 61 * manually allocating and mapping it for faster access (as a part of 62 * large page mapping for example). 63 * 64 * The following values give between one and two pages of free space 65 * after typical minimal boot (2-way SMP, single disk and NIC) with 66 * both defconfig and a distro config on x86_64 and 32. More 67 * intelligent way to determine this would be nice. 68 */ 69 #if BITS_PER_LONG > 32 70 #define PERCPU_DYNAMIC_RESERVE (28 << PERCPU_DYNAMIC_SIZE_SHIFT) 71 #else 72 #define PERCPU_DYNAMIC_RESERVE (20 << PERCPU_DYNAMIC_SIZE_SHIFT) 73 #endif 74 75 extern void *pcpu_base_addr; 76 extern const unsigned long *pcpu_unit_offsets; 77 78 struct pcpu_group_info { 79 int nr_units; /* aligned # of units */ 80 unsigned long base_offset; /* base address offset */ 81 unsigned int *cpu_map; /* unit->cpu map, empty 82 * entries contain NR_CPUS */ 83 }; 84 85 struct pcpu_alloc_info { 86 size_t static_size; 87 size_t reserved_size; 88 size_t dyn_size; 89 size_t unit_size; 90 size_t atom_size; 91 size_t alloc_size; 92 size_t __ai_size; /* internal, don't use */ 93 int nr_groups; /* 0 if grouping unnecessary */ 94 struct pcpu_group_info groups[]; 95 }; 96 97 enum pcpu_fc { 98 PCPU_FC_AUTO, 99 PCPU_FC_EMBED, 100 PCPU_FC_PAGE, 101 102 PCPU_FC_NR, 103 }; 104 extern const char * const pcpu_fc_names[PCPU_FC_NR]; 105 106 extern enum pcpu_fc pcpu_chosen_fc; 107 108 typedef int (pcpu_fc_cpu_to_node_fn_t)(int cpu); 109 typedef int (pcpu_fc_cpu_distance_fn_t)(unsigned int from, unsigned int to); 110 111 extern struct pcpu_alloc_info * __init pcpu_alloc_alloc_info(int nr_groups, 112 int nr_units); 113 extern void __init pcpu_free_alloc_info(struct pcpu_alloc_info *ai); 114 115 extern void __init pcpu_setup_first_chunk(const struct pcpu_alloc_info *ai, 116 void *base_addr); 117 118 extern int __init pcpu_embed_first_chunk(size_t reserved_size, size_t dyn_size, 119 size_t atom_size, 120 pcpu_fc_cpu_distance_fn_t cpu_distance_fn, 121 pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn); 122 123 #ifdef CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK 124 void __init pcpu_populate_pte(unsigned long addr); 125 extern int __init pcpu_page_first_chunk(size_t reserved_size, 126 pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn); 127 #endif 128 129 extern bool __is_kernel_percpu_address(unsigned long addr, unsigned long *can_addr); 130 extern bool is_kernel_percpu_address(unsigned long addr); 131 132 #if !defined(CONFIG_SMP) || !defined(CONFIG_HAVE_SETUP_PER_CPU_AREA) 133 extern void __init setup_per_cpu_areas(void); 134 #endif 135 136 extern void __percpu *pcpu_alloc_noprof(size_t size, size_t align, bool reserved, 137 gfp_t gfp) __alloc_size(1); 138 139 #define __alloc_percpu_gfp(_size, _align, _gfp) \ 140 alloc_hooks(pcpu_alloc_noprof(_size, _align, false, _gfp)) 141 #define __alloc_percpu(_size, _align) \ 142 alloc_hooks(pcpu_alloc_noprof(_size, _align, false, GFP_KERNEL)) 143 #define __alloc_reserved_percpu(_size, _align) \ 144 alloc_hooks(pcpu_alloc_noprof(_size, _align, true, GFP_KERNEL)) 145 146 #define alloc_percpu_gfp(type, gfp) \ 147 (typeof(type) __percpu *)__alloc_percpu_gfp(sizeof(type), \ 148 __alignof__(type), gfp) 149 #define alloc_percpu(type) \ 150 (typeof(type) __percpu *)__alloc_percpu(sizeof(type), \ 151 __alignof__(type)) 152 #define alloc_percpu_noprof(type) \ 153 ((typeof(type) __percpu *)pcpu_alloc_noprof(sizeof(type), \ 154 __alignof__(type), false, GFP_KERNEL)) 155 156 extern void free_percpu(void __percpu *__pdata); 157 158 DEFINE_FREE(free_percpu, void __percpu *, free_percpu(_T)) 159 160 extern phys_addr_t per_cpu_ptr_to_phys(void *addr); 161 162 extern unsigned long pcpu_nr_pages(void); 163 164 #endif /* __LINUX_PERCPU_H */ 165