1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * sparse memory mappings. 4 */ 5 #include <linux/mm.h> 6 #include <linux/slab.h> 7 #include <linux/mmzone.h> 8 #include <linux/memblock.h> 9 #include <linux/compiler.h> 10 #include <linux/highmem.h> 11 #include <linux/export.h> 12 #include <linux/spinlock.h> 13 #include <linux/vmalloc.h> 14 #include <linux/swap.h> 15 #include <linux/swapops.h> 16 #include <linux/vmstat.h> 17 #include "internal.h" 18 #include "mm_init.h" 19 #include "sparse.h" 20 #include <asm/dma.h> 21 22 /* 23 * Permanent SPARSEMEM data: 24 * 25 * 1) mem_section - memory sections, mem_map's for valid memory 26 */ 27 #ifdef CONFIG_SPARSEMEM_EXTREME 28 struct mem_section **mem_section; 29 #else 30 struct mem_section mem_section[NR_SECTION_ROOTS][SECTIONS_PER_ROOT] 31 ____cacheline_internodealigned_in_smp; 32 #endif 33 EXPORT_SYMBOL(mem_section); 34 35 #ifdef NODE_NOT_IN_PAGE_FLAGS 36 /* 37 * If we did not store the node number in the page then we have to 38 * do a lookup in the section_to_node_table in order to find which 39 * node the page belongs to. 40 */ 41 #if MAX_NUMNODES <= 256 42 static u8 section_to_node_table[NR_MEM_SECTIONS] __cacheline_aligned; 43 #else 44 static u16 section_to_node_table[NR_MEM_SECTIONS] __cacheline_aligned; 45 #endif 46 47 int memdesc_nid(const memdesc_flags_t *mdf) 48 { 49 return section_to_node_table[memdesc_section(mdf)]; 50 } 51 EXPORT_SYMBOL(memdesc_nid); 52 53 static void set_section_nid(unsigned long section_nr, int nid) 54 { 55 section_to_node_table[section_nr] = nid; 56 } 57 #else /* !NODE_NOT_IN_PAGE_FLAGS */ 58 static inline void set_section_nid(unsigned long section_nr, int nid) 59 { 60 } 61 #endif 62 63 #ifdef CONFIG_SPARSEMEM_EXTREME 64 static noinline struct mem_section __ref *sparse_index_alloc(int nid) 65 { 66 struct mem_section *section = NULL; 67 unsigned long array_size = SECTIONS_PER_ROOT * 68 sizeof(struct mem_section); 69 70 if (slab_is_available()) { 71 section = kzalloc_node(array_size, GFP_KERNEL, nid); 72 } else { 73 section = memblock_alloc_node(array_size, SMP_CACHE_BYTES, 74 nid); 75 if (!section) 76 panic("%s: Failed to allocate %lu bytes nid=%d\n", 77 __func__, array_size, nid); 78 } 79 80 return section; 81 } 82 83 int __meminit sparse_index_init(unsigned long section_nr, int nid) 84 { 85 unsigned long root = SECTION_NR_TO_ROOT(section_nr); 86 struct mem_section *section; 87 88 /* 89 * An existing section is possible in the sub-section hotplug 90 * case. First hot-add instantiates, follow-on hot-add reuses 91 * the existing section. 92 * 93 * The mem_hotplug_lock resolves the apparent race below. 94 */ 95 if (mem_section[root]) 96 return 0; 97 98 section = sparse_index_alloc(nid); 99 if (!section) 100 return -ENOMEM; 101 102 mem_section[root] = section; 103 104 return 0; 105 } 106 #else /* !SPARSEMEM_EXTREME */ 107 int __meminit sparse_index_init(unsigned long section_nr, int nid) 108 { 109 return 0; 110 } 111 #endif 112 113 /* 114 * During early boot, before section_mem_map is used for an actual 115 * mem_map, we use section_mem_map to store the section's NUMA 116 * node. This keeps us from having to use another data structure. The 117 * node information is cleared just before we store the real mem_map. 118 */ 119 static inline unsigned long sparse_encode_early_nid(int nid) 120 { 121 return ((unsigned long)nid << SECTION_NID_SHIFT); 122 } 123 124 static inline int sparse_early_nid(struct mem_section *section) 125 { 126 return (section->section_mem_map >> SECTION_NID_SHIFT); 127 } 128 129 /* Validate the physical addressing limitations of the model */ 130 static void __init mminit_validate_memmodel_limits(unsigned long *start_pfn, 131 unsigned long *end_pfn) 132 { 133 unsigned long max_sparsemem_pfn = (DIRECT_MAP_PHYSMEM_END + 1) >> PAGE_SHIFT; 134 135 /* 136 * Sanity checks - do not allow an architecture to pass 137 * in larger pfns than the maximum scope of sparsemem: 138 */ 139 if (*start_pfn > max_sparsemem_pfn) { 140 mminit_dprintk(MMINIT_WARNING, "pfnvalidation", 141 "Start of range %lu -> %lu exceeds SPARSEMEM max %lu\n", 142 *start_pfn, *end_pfn, max_sparsemem_pfn); 143 WARN_ON_ONCE(1); 144 *start_pfn = max_sparsemem_pfn; 145 *end_pfn = max_sparsemem_pfn; 146 } else if (*end_pfn > max_sparsemem_pfn) { 147 mminit_dprintk(MMINIT_WARNING, "pfnvalidation", 148 "End of range %lu -> %lu exceeds SPARSEMEM max %lu\n", 149 *start_pfn, *end_pfn, max_sparsemem_pfn); 150 WARN_ON_ONCE(1); 151 *end_pfn = max_sparsemem_pfn; 152 } 153 } 154 155 /* 156 * There are a number of times that we loop over NR_MEM_SECTIONS, 157 * looking for section_present() on each. But, when we have very 158 * large physical address spaces, NR_MEM_SECTIONS can also be 159 * very large which makes the loops quite long. 160 * 161 * Keeping track of this gives us an easy way to break out of 162 * those loops early. 163 */ 164 unsigned long __highest_present_section_nr; 165 166 static inline unsigned long first_present_section_nr(void) 167 { 168 return next_present_section_nr(-1); 169 } 170 171 /* Record a memory area against a node. */ 172 static void __init memory_present(int nid, unsigned long start, unsigned long end) 173 { 174 unsigned long pfn; 175 176 start &= PAGE_SECTION_MASK; 177 mminit_validate_memmodel_limits(&start, &end); 178 for (pfn = start; pfn < end; pfn += PAGES_PER_SECTION) { 179 unsigned long section_nr = pfn_to_section_nr(pfn); 180 struct mem_section *ms; 181 182 sparse_index_init(section_nr, nid); 183 set_section_nid(section_nr, nid); 184 185 ms = __nr_to_section(section_nr); 186 if (!ms->section_mem_map) { 187 ms->section_mem_map = sparse_encode_early_nid(nid) | 188 SECTION_IS_ONLINE; 189 __section_mark_present(ms, section_nr); 190 } 191 } 192 } 193 194 void __init sparse_sections_init(void) 195 { 196 unsigned long start, end; 197 int i, nid; 198 199 #ifdef CONFIG_SPARSEMEM_EXTREME 200 unsigned long size, align; 201 202 size = sizeof(struct mem_section *) * NR_SECTION_ROOTS; 203 align = 1 << (INTERNODE_CACHE_SHIFT); 204 mem_section = memblock_alloc_or_panic(size, align); 205 #endif 206 207 for_each_mem_pfn_range(i, MAX_NUMNODES, &start, &end, &nid) 208 memory_present(nid, start, end); 209 } 210 211 #ifndef CONFIG_SPARSEMEM_VMEMMAP 212 struct page __init *__populate_section_memmap(unsigned long pfn, 213 unsigned long nr_pages, int nid, struct vmem_altmap *altmap, 214 struct dev_pagemap *pgmap) 215 { 216 const unsigned long size = PAGE_ALIGN(sizeof(struct page) * PAGES_PER_SECTION); 217 218 return memmap_alloc(size, size, __pa(MAX_DMA_ADDRESS), nid, false); 219 } 220 #endif /* !CONFIG_SPARSEMEM_VMEMMAP */ 221 222 void __weak __meminit vmemmap_populate_print_last(void) 223 { 224 } 225 226 /* 227 * Initialize sparse on a specific node. The node spans [pnum_begin, pnum_end) 228 * And number of present sections in this node is map_count. 229 */ 230 static void __init sparse_init_nid(int nid, unsigned long pnum_begin, 231 unsigned long pnum_end, 232 unsigned long map_count) 233 { 234 unsigned long pnum; 235 struct mem_section_usage *usage; 236 237 usage = memblock_alloc_node(map_count * mem_section_usage_size(), 238 SMP_CACHE_BYTES, nid); 239 if (!usage) 240 panic("Failed to allocate usemap for node %d\n", nid); 241 242 for_each_present_section_nr(pnum_begin, pnum) { 243 unsigned long pfn = section_nr_to_pfn(pnum); 244 struct page *map; 245 246 if (pnum >= pnum_end) 247 break; 248 249 map = __populate_section_memmap(pfn, PAGES_PER_SECTION, 250 nid, NULL, NULL); 251 if (!map) 252 panic("Failed to allocate memmap for section %lu\n", pnum); 253 memmap_boot_pages_add(section_nr_vmemmap_pages(pfn, PAGES_PER_SECTION)); 254 sparse_init_one_section(__nr_to_section(pnum), pnum, map, usage, 255 SECTION_IS_EARLY); 256 usage = (void *)usage + mem_section_usage_size(); 257 } 258 } 259 260 /* 261 * Allocate the accumulated non-linear sections, allocate a mem_map 262 * for each and record the physical to section mapping. 263 */ 264 void __init sparse_init(void) 265 { 266 unsigned long pnum_end, pnum_begin, map_count = 1; 267 int nid_begin; 268 269 if (compound_info_has_mask()) { 270 VM_WARN_ON_ONCE(!IS_ALIGNED((unsigned long) pfn_to_page(0), 271 MAX_FOLIO_VMEMMAP_ALIGN)); 272 } 273 274 pnum_begin = first_present_section_nr(); 275 nid_begin = sparse_early_nid(__nr_to_section(pnum_begin)); 276 277 for_each_present_section_nr(pnum_begin + 1, pnum_end) { 278 int nid = sparse_early_nid(__nr_to_section(pnum_end)); 279 280 if (nid == nid_begin) { 281 map_count++; 282 continue; 283 } 284 /* Init node with sections in range [pnum_begin, pnum_end) */ 285 sparse_init_nid(nid_begin, pnum_begin, pnum_end, map_count); 286 nid_begin = nid; 287 pnum_begin = pnum_end; 288 map_count = 1; 289 } 290 /* cover the last node */ 291 sparse_init_nid(nid_begin, pnum_begin, pnum_end, map_count); 292 sparse_init_subsection_map(); 293 vmemmap_populate_print_last(); 294 } 295