1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * sparse memory mappings. 4 */ 5 #include <linux/mm.h> 6 #include <linux/slab.h> 7 #include <linux/mmzone.h> 8 #include <linux/memblock.h> 9 #include <linux/compiler.h> 10 #include <linux/highmem.h> 11 #include <linux/export.h> 12 #include <linux/spinlock.h> 13 #include <linux/vmalloc.h> 14 #include <linux/swap.h> 15 #include <linux/swapops.h> 16 #include <linux/bootmem_info.h> 17 #include <linux/vmstat.h> 18 #include "internal.h" 19 #include <asm/dma.h> 20 21 /* 22 * Permanent SPARSEMEM data: 23 * 24 * 1) mem_section - memory sections, mem_map's for valid memory 25 */ 26 #ifdef CONFIG_SPARSEMEM_EXTREME 27 struct mem_section **mem_section; 28 #else 29 struct mem_section mem_section[NR_SECTION_ROOTS][SECTIONS_PER_ROOT] 30 ____cacheline_internodealigned_in_smp; 31 #endif 32 EXPORT_SYMBOL(mem_section); 33 34 #ifdef NODE_NOT_IN_PAGE_FLAGS 35 /* 36 * If we did not store the node number in the page then we have to 37 * do a lookup in the section_to_node_table in order to find which 38 * node the page belongs to. 39 */ 40 #if MAX_NUMNODES <= 256 41 static u8 section_to_node_table[NR_MEM_SECTIONS] __cacheline_aligned; 42 #else 43 static u16 section_to_node_table[NR_MEM_SECTIONS] __cacheline_aligned; 44 #endif 45 46 int memdesc_nid(memdesc_flags_t mdf) 47 { 48 return section_to_node_table[memdesc_section(mdf)]; 49 } 50 EXPORT_SYMBOL(memdesc_nid); 51 52 static void set_section_nid(unsigned long section_nr, int nid) 53 { 54 section_to_node_table[section_nr] = nid; 55 } 56 #else /* !NODE_NOT_IN_PAGE_FLAGS */ 57 static inline void set_section_nid(unsigned long section_nr, int nid) 58 { 59 } 60 #endif 61 62 #ifdef CONFIG_SPARSEMEM_EXTREME 63 static noinline struct mem_section __ref *sparse_index_alloc(int nid) 64 { 65 struct mem_section *section = NULL; 66 unsigned long array_size = SECTIONS_PER_ROOT * 67 sizeof(struct mem_section); 68 69 if (slab_is_available()) { 70 section = kzalloc_node(array_size, GFP_KERNEL, nid); 71 } else { 72 section = memblock_alloc_node(array_size, SMP_CACHE_BYTES, 73 nid); 74 if (!section) 75 panic("%s: Failed to allocate %lu bytes nid=%d\n", 76 __func__, array_size, nid); 77 } 78 79 return section; 80 } 81 82 int __meminit sparse_index_init(unsigned long section_nr, int nid) 83 { 84 unsigned long root = SECTION_NR_TO_ROOT(section_nr); 85 struct mem_section *section; 86 87 /* 88 * An existing section is possible in the sub-section hotplug 89 * case. First hot-add instantiates, follow-on hot-add reuses 90 * the existing section. 91 * 92 * The mem_hotplug_lock resolves the apparent race below. 93 */ 94 if (mem_section[root]) 95 return 0; 96 97 section = sparse_index_alloc(nid); 98 if (!section) 99 return -ENOMEM; 100 101 mem_section[root] = section; 102 103 return 0; 104 } 105 #else /* !SPARSEMEM_EXTREME */ 106 int sparse_index_init(unsigned long section_nr, int nid) 107 { 108 return 0; 109 } 110 #endif 111 112 /* 113 * During early boot, before section_mem_map is used for an actual 114 * mem_map, we use section_mem_map to store the section's NUMA 115 * node. This keeps us from having to use another data structure. The 116 * node information is cleared just before we store the real mem_map. 117 */ 118 static inline unsigned long sparse_encode_early_nid(int nid) 119 { 120 return ((unsigned long)nid << SECTION_NID_SHIFT); 121 } 122 123 static inline int sparse_early_nid(struct mem_section *section) 124 { 125 return (section->section_mem_map >> SECTION_NID_SHIFT); 126 } 127 128 /* Validate the physical addressing limitations of the model */ 129 static void __meminit mminit_validate_memmodel_limits(unsigned long *start_pfn, 130 unsigned long *end_pfn) 131 { 132 unsigned long max_sparsemem_pfn = (DIRECT_MAP_PHYSMEM_END + 1) >> PAGE_SHIFT; 133 134 /* 135 * Sanity checks - do not allow an architecture to pass 136 * in larger pfns than the maximum scope of sparsemem: 137 */ 138 if (*start_pfn > max_sparsemem_pfn) { 139 mminit_dprintk(MMINIT_WARNING, "pfnvalidation", 140 "Start of range %lu -> %lu exceeds SPARSEMEM max %lu\n", 141 *start_pfn, *end_pfn, max_sparsemem_pfn); 142 WARN_ON_ONCE(1); 143 *start_pfn = max_sparsemem_pfn; 144 *end_pfn = max_sparsemem_pfn; 145 } else if (*end_pfn > max_sparsemem_pfn) { 146 mminit_dprintk(MMINIT_WARNING, "pfnvalidation", 147 "End of range %lu -> %lu exceeds SPARSEMEM max %lu\n", 148 *start_pfn, *end_pfn, max_sparsemem_pfn); 149 WARN_ON_ONCE(1); 150 *end_pfn = max_sparsemem_pfn; 151 } 152 } 153 154 /* 155 * There are a number of times that we loop over NR_MEM_SECTIONS, 156 * looking for section_present() on each. But, when we have very 157 * large physical address spaces, NR_MEM_SECTIONS can also be 158 * very large which makes the loops quite long. 159 * 160 * Keeping track of this gives us an easy way to break out of 161 * those loops early. 162 */ 163 unsigned long __highest_present_section_nr; 164 165 static inline unsigned long first_present_section_nr(void) 166 { 167 return next_present_section_nr(-1); 168 } 169 170 /* Record a memory area against a node. */ 171 static void __init memory_present(int nid, unsigned long start, unsigned long end) 172 { 173 unsigned long pfn; 174 175 start &= PAGE_SECTION_MASK; 176 mminit_validate_memmodel_limits(&start, &end); 177 for (pfn = start; pfn < end; pfn += PAGES_PER_SECTION) { 178 unsigned long section_nr = pfn_to_section_nr(pfn); 179 struct mem_section *ms; 180 181 sparse_index_init(section_nr, nid); 182 set_section_nid(section_nr, nid); 183 184 ms = __nr_to_section(section_nr); 185 if (!ms->section_mem_map) { 186 ms->section_mem_map = sparse_encode_early_nid(nid) | 187 SECTION_IS_ONLINE; 188 __section_mark_present(ms, section_nr); 189 } 190 } 191 } 192 193 /* 194 * Mark all memblocks as present using memory_present(). 195 * This is a convenience function that is useful to mark all of the systems 196 * memory as present during initialization. 197 */ 198 static void __init memblocks_present(void) 199 { 200 unsigned long start, end; 201 int i, nid; 202 203 #ifdef CONFIG_SPARSEMEM_EXTREME 204 unsigned long size, align; 205 206 size = sizeof(struct mem_section *) * NR_SECTION_ROOTS; 207 align = 1 << (INTERNODE_CACHE_SHIFT); 208 mem_section = memblock_alloc_or_panic(size, align); 209 #endif 210 211 for_each_mem_pfn_range(i, MAX_NUMNODES, &start, &end, &nid) 212 memory_present(nid, start, end); 213 } 214 215 static unsigned long usemap_size(void) 216 { 217 return BITS_TO_LONGS(SECTION_BLOCKFLAGS_BITS) * sizeof(unsigned long); 218 } 219 220 size_t mem_section_usage_size(void) 221 { 222 return sizeof(struct mem_section_usage) + usemap_size(); 223 } 224 225 #ifdef CONFIG_SPARSEMEM_VMEMMAP 226 unsigned long __init section_map_size(void) 227 { 228 return ALIGN(sizeof(struct page) * PAGES_PER_SECTION, PMD_SIZE); 229 } 230 231 #else 232 unsigned long __init section_map_size(void) 233 { 234 return PAGE_ALIGN(sizeof(struct page) * PAGES_PER_SECTION); 235 } 236 237 struct page __init *__populate_section_memmap(unsigned long pfn, 238 unsigned long nr_pages, int nid, struct vmem_altmap *altmap, 239 struct dev_pagemap *pgmap) 240 { 241 unsigned long size = section_map_size(); 242 struct page *map; 243 phys_addr_t addr = __pa(MAX_DMA_ADDRESS); 244 245 map = memmap_alloc(size, size, addr, nid, false); 246 if (!map) 247 panic("%s: Failed to allocate %lu bytes align=0x%lx nid=%d from=%pa\n", 248 __func__, size, PAGE_SIZE, nid, &addr); 249 250 return map; 251 } 252 #endif /* !CONFIG_SPARSEMEM_VMEMMAP */ 253 254 void __weak __meminit vmemmap_populate_print_last(void) 255 { 256 } 257 258 static void *sparse_usagebuf __meminitdata; 259 static void *sparse_usagebuf_end __meminitdata; 260 261 /* 262 * Helper function that is used for generic section initialization, and 263 * can also be used by any hooks added above. 264 */ 265 void __init sparse_init_early_section(int nid, struct page *map, 266 unsigned long pnum, unsigned long flags) 267 { 268 BUG_ON(!sparse_usagebuf || sparse_usagebuf >= sparse_usagebuf_end); 269 sparse_init_one_section(__nr_to_section(pnum), pnum, map, 270 sparse_usagebuf, SECTION_IS_EARLY | flags); 271 sparse_usagebuf = (void *)sparse_usagebuf + mem_section_usage_size(); 272 } 273 274 static int __init sparse_usage_init(int nid, unsigned long map_count) 275 { 276 unsigned long size; 277 278 size = mem_section_usage_size() * map_count; 279 sparse_usagebuf = memblock_alloc_node(size, SMP_CACHE_BYTES, nid); 280 if (!sparse_usagebuf) { 281 sparse_usagebuf_end = NULL; 282 return -ENOMEM; 283 } 284 285 sparse_usagebuf_end = sparse_usagebuf + size; 286 return 0; 287 } 288 289 static void __init sparse_usage_fini(void) 290 { 291 sparse_usagebuf = sparse_usagebuf_end = NULL; 292 } 293 294 /* 295 * Initialize sparse on a specific node. The node spans [pnum_begin, pnum_end) 296 * And number of present sections in this node is map_count. 297 */ 298 static void __init sparse_init_nid(int nid, unsigned long pnum_begin, 299 unsigned long pnum_end, 300 unsigned long map_count) 301 { 302 unsigned long pnum; 303 struct page *map; 304 struct mem_section *ms; 305 306 if (sparse_usage_init(nid, map_count)) { 307 pr_err("%s: node[%d] usemap allocation failed", __func__, nid); 308 goto failed; 309 } 310 311 sparse_vmemmap_init_nid_early(nid); 312 313 for_each_present_section_nr(pnum_begin, pnum) { 314 unsigned long pfn = section_nr_to_pfn(pnum); 315 316 if (pnum >= pnum_end) 317 break; 318 319 ms = __nr_to_section(pnum); 320 if (!preinited_vmemmap_section(ms)) { 321 map = __populate_section_memmap(pfn, PAGES_PER_SECTION, 322 nid, NULL, NULL); 323 if (!map) { 324 pr_err("%s: node[%d] memory map backing failed. Some memory will not be available.", 325 __func__, nid); 326 pnum_begin = pnum; 327 sparse_usage_fini(); 328 goto failed; 329 } 330 memmap_boot_pages_add(DIV_ROUND_UP(PAGES_PER_SECTION * sizeof(struct page), 331 PAGE_SIZE)); 332 sparse_init_early_section(nid, map, pnum, 0); 333 } 334 } 335 sparse_usage_fini(); 336 return; 337 failed: 338 /* 339 * We failed to allocate, mark all the following pnums as not present, 340 * except the ones already initialized earlier. 341 */ 342 for_each_present_section_nr(pnum_begin, pnum) { 343 if (pnum >= pnum_end) 344 break; 345 ms = __nr_to_section(pnum); 346 if (!preinited_vmemmap_section(ms)) 347 ms->section_mem_map = 0; 348 } 349 } 350 351 /* 352 * Allocate the accumulated non-linear sections, allocate a mem_map 353 * for each and record the physical to section mapping. 354 */ 355 void __init sparse_init(void) 356 { 357 unsigned long pnum_end, pnum_begin, map_count = 1; 358 int nid_begin; 359 360 /* see include/linux/mmzone.h 'struct mem_section' definition */ 361 BUILD_BUG_ON(!is_power_of_2(sizeof(struct mem_section))); 362 memblocks_present(); 363 364 if (compound_info_has_mask()) { 365 VM_WARN_ON_ONCE(!IS_ALIGNED((unsigned long) pfn_to_page(0), 366 MAX_FOLIO_VMEMMAP_ALIGN)); 367 } 368 369 pnum_begin = first_present_section_nr(); 370 nid_begin = sparse_early_nid(__nr_to_section(pnum_begin)); 371 372 /* Setup pageblock_order for HUGETLB_PAGE_SIZE_VARIABLE */ 373 set_pageblock_order(); 374 375 for_each_present_section_nr(pnum_begin + 1, pnum_end) { 376 int nid = sparse_early_nid(__nr_to_section(pnum_end)); 377 378 if (nid == nid_begin) { 379 map_count++; 380 continue; 381 } 382 /* Init node with sections in range [pnum_begin, pnum_end) */ 383 sparse_init_nid(nid_begin, pnum_begin, pnum_end, map_count); 384 nid_begin = nid; 385 pnum_begin = pnum_end; 386 map_count = 1; 387 } 388 /* cover the last node */ 389 sparse_init_nid(nid_begin, pnum_begin, pnum_end, map_count); 390 vmemmap_populate_print_last(); 391 } 392