1 // SPDX-License-Identifier: GPL-2.0 2 #include <linux/mm.h> 3 #include <linux/mmzone.h> 4 #include <linux/memblock.h> 5 #include <linux/page_ext.h> 6 #include <linux/memory.h> 7 #include <linux/vmalloc.h> 8 #include <linux/kmemleak.h> 9 #include <linux/page_owner.h> 10 #include <linux/page_idle.h> 11 #include <linux/page_table_check.h> 12 #include <linux/rcupdate.h> 13 #include <linux/pgalloc_tag.h> 14 #include <linux/iommu-debug-pagealloc.h> 15 16 /* 17 * struct page extension 18 * 19 * This is the feature to manage memory for extended data per page. 20 * 21 * Until now, we must modify struct page itself to store extra data per page. 22 * This requires rebuilding the kernel and it is really time consuming process. 23 * And, sometimes, rebuild is impossible due to third party module dependency. 24 * At last, enlarging struct page could cause un-wanted system behaviour change. 25 * 26 * This feature is intended to overcome above mentioned problems. This feature 27 * allocates memory for extended data per page in certain place rather than 28 * the struct page itself. This memory can be accessed by the accessor 29 * functions provided by this code. During the boot process, it checks whether 30 * allocation of huge chunk of memory is needed or not. If not, it avoids 31 * allocating memory at all. With this advantage, we can include this feature 32 * into the kernel in default and can avoid rebuild and solve related problems. 33 * 34 * To help these things to work well, there are two callbacks for clients. One 35 * is the need callback which is mandatory if user wants to avoid useless 36 * memory allocation at boot-time. The other is optional, init callback, which 37 * is used to do proper initialization after memory is allocated. 38 * 39 * The need callback is used to decide whether extended memory allocation is 40 * needed or not. Sometimes users want to deactivate some features in this 41 * boot and extra memory would be unnecessary. In this case, to avoid 42 * allocating huge chunk of memory, each clients represent their need of 43 * extra memory through the need callback. If one of the need callbacks 44 * returns true, it means that someone needs extra memory so that 45 * page extension core should allocates memory for page extension. If 46 * none of need callbacks return true, memory isn't needed at all in this boot 47 * and page extension core can skip to allocate memory. As result, 48 * none of memory is wasted. 49 * 50 * When need callback returns true, page_ext checks if there is a request for 51 * extra memory through size in struct page_ext_operations. If it is non-zero, 52 * extra space is allocated for each page_ext entry and offset is returned to 53 * user through offset in struct page_ext_operations. 54 * 55 * The init callback is used to do proper initialization after page extension 56 * is completely initialized. In sparse memory system, extra memory is 57 * allocated some time later than memmap is allocated. In other words, lifetime 58 * of memory for page extension isn't same with memmap for struct page. 59 * Therefore, clients can't store extra data until page extension is 60 * initialized, even if pages are allocated and used freely. This could 61 * cause inadequate state of extra data per page, so, to prevent it, client 62 * can utilize this callback to initialize the state of it correctly. 63 */ 64 65 #ifdef CONFIG_SPARSEMEM 66 #define PAGE_EXT_INVALID (0x1) 67 #endif 68 69 #if defined(CONFIG_PAGE_IDLE_FLAG) && !defined(CONFIG_64BIT) 70 static bool need_page_idle(void) 71 { 72 return true; 73 } 74 static struct page_ext_operations page_idle_ops __initdata = { 75 .need = need_page_idle, 76 .need_shared_flags = true, 77 }; 78 #endif 79 80 static struct page_ext_operations *page_ext_ops[] __initdata = { 81 #ifdef CONFIG_PAGE_OWNER 82 &page_owner_ops, 83 #endif 84 #if defined(CONFIG_PAGE_IDLE_FLAG) && !defined(CONFIG_64BIT) 85 &page_idle_ops, 86 #endif 87 #ifdef CONFIG_MEM_ALLOC_PROFILING 88 &page_alloc_tagging_ops, 89 #endif 90 #ifdef CONFIG_PAGE_TABLE_CHECK 91 &page_table_check_ops, 92 #endif 93 #ifdef CONFIG_IOMMU_DEBUG_PAGEALLOC 94 &page_iommu_debug_ops, 95 #endif 96 }; 97 98 unsigned long page_ext_size; 99 100 static unsigned long total_usage; 101 102 #ifdef CONFIG_MEM_ALLOC_PROFILING_DEBUG 103 /* 104 * To ensure correct allocation tagging for pages, page_ext should be available 105 * before the first page allocation. Otherwise early task stacks will be 106 * allocated before page_ext initialization and missing tags will be flagged. 107 */ 108 bool early_page_ext __meminitdata = true; 109 #else 110 bool early_page_ext __meminitdata; 111 #endif 112 static int __init setup_early_page_ext(char *str) 113 { 114 early_page_ext = true; 115 return 0; 116 } 117 early_param("early_page_ext", setup_early_page_ext); 118 119 static bool __init invoke_need_callbacks(void) 120 { 121 int i; 122 int entries = ARRAY_SIZE(page_ext_ops); 123 bool need = false; 124 125 for (i = 0; i < entries; i++) { 126 if (page_ext_ops[i]->need()) { 127 if (page_ext_ops[i]->need_shared_flags) { 128 page_ext_size = sizeof(struct page_ext); 129 break; 130 } 131 } 132 } 133 134 for (i = 0; i < entries; i++) { 135 if (page_ext_ops[i]->need()) { 136 page_ext_ops[i]->offset = page_ext_size; 137 page_ext_size += page_ext_ops[i]->size; 138 need = true; 139 } 140 } 141 142 return need; 143 } 144 145 static void __init invoke_init_callbacks(void) 146 { 147 int i; 148 int entries = ARRAY_SIZE(page_ext_ops); 149 150 for (i = 0; i < entries; i++) { 151 if (page_ext_ops[i]->init) 152 page_ext_ops[i]->init(); 153 } 154 } 155 156 static inline struct page_ext *get_entry(void *base, unsigned long index) 157 { 158 return base + page_ext_size * index; 159 } 160 161 #ifndef CONFIG_SPARSEMEM 162 void __init page_ext_init_flatmem_late(void) 163 { 164 invoke_init_callbacks(); 165 } 166 167 static struct page_ext *lookup_page_ext(const struct page *page) 168 { 169 unsigned long pfn = page_to_pfn(page); 170 unsigned long index; 171 struct page_ext *base; 172 173 WARN_ON_ONCE(!rcu_read_lock_held()); 174 base = NODE_DATA(page_to_nid(page))->node_page_ext; 175 /* 176 * The sanity checks the page allocator does upon freeing a 177 * page can reach here before the page_ext arrays are 178 * allocated when feeding a range of pages to the allocator 179 * for the first time during bootup or memory hotplug. 180 */ 181 if (unlikely(!base)) 182 return NULL; 183 index = pfn - round_down(node_start_pfn(page_to_nid(page)), 184 MAX_ORDER_NR_PAGES); 185 return get_entry(base, index); 186 } 187 188 static int __init alloc_node_page_ext(int nid) 189 { 190 struct page_ext *base; 191 unsigned long table_size; 192 unsigned long nr_pages; 193 194 nr_pages = NODE_DATA(nid)->node_spanned_pages; 195 if (!nr_pages) 196 return 0; 197 198 /* 199 * Need extra space if node range is not aligned with 200 * MAX_ORDER_NR_PAGES. When page allocator's buddy algorithm 201 * checks buddy's status, range could be out of exact node range. 202 */ 203 if (!IS_ALIGNED(node_start_pfn(nid), MAX_ORDER_NR_PAGES) || 204 !IS_ALIGNED(node_end_pfn(nid), MAX_ORDER_NR_PAGES)) 205 nr_pages += MAX_ORDER_NR_PAGES; 206 207 table_size = page_ext_size * nr_pages; 208 209 base = memblock_alloc_try_nid( 210 table_size, PAGE_SIZE, __pa(MAX_DMA_ADDRESS), 211 MEMBLOCK_ALLOC_ACCESSIBLE, nid); 212 if (!base) 213 return -ENOMEM; 214 NODE_DATA(nid)->node_page_ext = base; 215 total_usage += table_size; 216 memmap_boot_pages_add(DIV_ROUND_UP(table_size, PAGE_SIZE)); 217 return 0; 218 } 219 220 void __init page_ext_init_flatmem(void) 221 { 222 223 int nid, fail; 224 225 if (!invoke_need_callbacks()) 226 return; 227 228 for_each_online_node(nid) { 229 fail = alloc_node_page_ext(nid); 230 if (fail) 231 goto fail; 232 } 233 pr_info("allocated %ld bytes of page_ext\n", total_usage); 234 return; 235 236 fail: 237 pr_crit("allocation of page_ext failed.\n"); 238 panic("Out of memory"); 239 } 240 241 #else /* CONFIG_SPARSEMEM */ 242 static bool page_ext_invalid(struct page_ext *page_ext) 243 { 244 return !page_ext || (((unsigned long)page_ext & PAGE_EXT_INVALID) == PAGE_EXT_INVALID); 245 } 246 247 static struct page_ext *lookup_page_ext(const struct page *page) 248 { 249 unsigned long pfn = page_to_pfn(page); 250 struct mem_section *section = __pfn_to_section(pfn); 251 struct page_ext *page_ext = READ_ONCE(section->page_ext); 252 253 WARN_ON_ONCE(!rcu_read_lock_held()); 254 /* 255 * The sanity checks the page allocator does upon freeing a 256 * page can reach here before the page_ext arrays are 257 * allocated when feeding a range of pages to the allocator 258 * for the first time during bootup or memory hotplug. 259 */ 260 if (page_ext_invalid(page_ext)) 261 return NULL; 262 return get_entry(page_ext, pfn); 263 } 264 265 static void *__meminit alloc_page_ext(size_t size, int nid) 266 { 267 gfp_t flags = GFP_KERNEL | __GFP_ZERO | __GFP_NOWARN; 268 void *addr = NULL; 269 270 addr = alloc_pages_exact_nid(nid, size, flags); 271 if (addr) 272 kmemleak_alloc(addr, size, 1, flags); 273 else 274 addr = vzalloc_node(size, nid); 275 276 if (addr) 277 memmap_pages_add(DIV_ROUND_UP(size, PAGE_SIZE)); 278 279 return addr; 280 } 281 282 static int __meminit init_section_page_ext(unsigned long pfn, int nid) 283 { 284 struct mem_section *section; 285 struct page_ext *base; 286 unsigned long table_size; 287 288 section = __pfn_to_section(pfn); 289 290 if (section->page_ext) 291 return 0; 292 293 table_size = page_ext_size * PAGES_PER_SECTION; 294 base = alloc_page_ext(table_size, nid); 295 296 /* 297 * The value stored in section->page_ext is (base - pfn) 298 * and it does not point to the memory block allocated above, 299 * causing kmemleak false positives. 300 */ 301 kmemleak_not_leak(base); 302 303 if (!base) { 304 pr_err("page ext allocation failure\n"); 305 return -ENOMEM; 306 } 307 308 /* 309 * The passed "pfn" may not be aligned to SECTION. For the calculation 310 * we need to apply a mask. 311 */ 312 pfn &= PAGE_SECTION_MASK; 313 section->page_ext = (void *)base - page_ext_size * pfn; 314 total_usage += table_size; 315 return 0; 316 } 317 318 static void free_page_ext(void *addr) 319 { 320 size_t table_size; 321 struct page *page; 322 323 table_size = page_ext_size * PAGES_PER_SECTION; 324 memmap_pages_add(-1L * (DIV_ROUND_UP(table_size, PAGE_SIZE))); 325 326 if (is_vmalloc_addr(addr)) { 327 vfree(addr); 328 } else { 329 page = virt_to_page(addr); 330 BUG_ON(PageReserved(page)); 331 kmemleak_free(addr); 332 free_pages_exact(addr, table_size); 333 } 334 } 335 336 static void __free_page_ext(unsigned long pfn) 337 { 338 struct mem_section *ms; 339 struct page_ext *base; 340 341 ms = __pfn_to_section(pfn); 342 if (!ms || !ms->page_ext) 343 return; 344 345 base = READ_ONCE(ms->page_ext); 346 /* 347 * page_ext here can be valid while doing the roll back 348 * operation in online_page_ext(). 349 */ 350 if (page_ext_invalid(base)) 351 base = (void *)base - PAGE_EXT_INVALID; 352 WRITE_ONCE(ms->page_ext, NULL); 353 354 base = get_entry(base, pfn); 355 free_page_ext(base); 356 } 357 358 static void __invalidate_page_ext(unsigned long pfn) 359 { 360 struct mem_section *ms; 361 void *val; 362 363 ms = __pfn_to_section(pfn); 364 if (!ms || !ms->page_ext) 365 return; 366 val = (void *)ms->page_ext + PAGE_EXT_INVALID; 367 WRITE_ONCE(ms->page_ext, val); 368 } 369 370 static int __meminit online_page_ext(unsigned long start_pfn, 371 unsigned long nr_pages) 372 { 373 int nid = pfn_to_nid(start_pfn); 374 unsigned long start, end, pfn; 375 int fail = 0; 376 377 start = SECTION_ALIGN_DOWN(start_pfn); 378 end = SECTION_ALIGN_UP(start_pfn + nr_pages); 379 380 for (pfn = start; !fail && pfn < end; pfn += PAGES_PER_SECTION) 381 fail = init_section_page_ext(pfn, nid); 382 if (!fail) 383 return 0; 384 385 /* rollback */ 386 end = pfn - PAGES_PER_SECTION; 387 for (pfn = start; pfn < end; pfn += PAGES_PER_SECTION) 388 __free_page_ext(pfn); 389 390 return -ENOMEM; 391 } 392 393 static void __meminit offline_page_ext(unsigned long start_pfn, 394 unsigned long nr_pages) 395 { 396 unsigned long start, end, pfn; 397 398 start = SECTION_ALIGN_DOWN(start_pfn); 399 end = SECTION_ALIGN_UP(start_pfn + nr_pages); 400 401 /* 402 * Freeing of page_ext is done in 3 steps to avoid 403 * use-after-free of it: 404 * 1) Traverse all the sections and mark their page_ext 405 * as invalid. 406 * 2) Wait for all the existing users of page_ext who 407 * started before invalidation to finish. 408 * 3) Free the page_ext. 409 */ 410 for (pfn = start; pfn < end; pfn += PAGES_PER_SECTION) 411 __invalidate_page_ext(pfn); 412 413 synchronize_rcu(); 414 415 for (pfn = start; pfn < end; pfn += PAGES_PER_SECTION) 416 __free_page_ext(pfn); 417 } 418 419 static int __meminit page_ext_callback(struct notifier_block *self, 420 unsigned long action, void *arg) 421 { 422 struct memory_notify *mn = arg; 423 int ret = 0; 424 425 switch (action) { 426 case MEM_GOING_ONLINE: 427 ret = online_page_ext(mn->start_pfn, mn->nr_pages); 428 break; 429 case MEM_OFFLINE: 430 offline_page_ext(mn->start_pfn, 431 mn->nr_pages); 432 break; 433 case MEM_CANCEL_ONLINE: 434 offline_page_ext(mn->start_pfn, 435 mn->nr_pages); 436 break; 437 case MEM_GOING_OFFLINE: 438 break; 439 case MEM_ONLINE: 440 case MEM_CANCEL_OFFLINE: 441 break; 442 } 443 444 return notifier_from_errno(ret); 445 } 446 447 void __init page_ext_init(void) 448 { 449 unsigned long pfn; 450 int nid; 451 452 if (!invoke_need_callbacks()) 453 return; 454 455 for_each_node_state(nid, N_MEMORY) { 456 unsigned long start_pfn, end_pfn; 457 458 start_pfn = node_start_pfn(nid); 459 end_pfn = node_end_pfn(nid); 460 /* 461 * start_pfn and end_pfn may not be aligned to SECTION and the 462 * page->flags of out of node pages are not initialized. So we 463 * scan [start_pfn, the biggest section's pfn < end_pfn) here. 464 */ 465 for (pfn = start_pfn; pfn < end_pfn; 466 pfn = ALIGN(pfn + 1, PAGES_PER_SECTION)) { 467 468 if (!pfn_valid(pfn)) 469 continue; 470 /* 471 * Nodes's pfns can be overlapping. 472 * We know some arch can have a nodes layout such as 473 * -------------pfn--------------> 474 * N0 | N1 | N2 | N0 | N1 | N2|.... 475 */ 476 if (pfn_to_nid(pfn) != nid) 477 continue; 478 if (init_section_page_ext(pfn, nid)) 479 goto oom; 480 cond_resched(); 481 } 482 } 483 hotplug_memory_notifier(page_ext_callback, DEFAULT_CALLBACK_PRI); 484 pr_info("allocated %ld bytes of page_ext\n", total_usage); 485 invoke_init_callbacks(); 486 return; 487 488 oom: 489 panic("Out of memory"); 490 } 491 492 #endif 493 494 /** 495 * page_ext_lookup() - Lookup a page extension for a PFN. 496 * @pfn: PFN of the page we're interested in. 497 * 498 * Must be called with RCU read lock taken and @pfn must be valid. 499 * 500 * Return: NULL if no page_ext exists for this page. 501 */ 502 struct page_ext *page_ext_lookup(unsigned long pfn) 503 { 504 return lookup_page_ext(pfn_to_page(pfn)); 505 } 506 507 /** 508 * page_ext_get() - Get the extended information for a page. 509 * @page: The page we're interested in. 510 * 511 * Ensures that the page_ext will remain valid until page_ext_put() 512 * is called. 513 * 514 * Return: NULL if no page_ext exists for this page. 515 * Context: Any context. Caller may not sleep until they have called 516 * page_ext_put(). 517 */ 518 struct page_ext *page_ext_get(const struct page *page) 519 { 520 struct page_ext *page_ext; 521 522 rcu_read_lock(); 523 page_ext = lookup_page_ext(page); 524 if (!page_ext) { 525 rcu_read_unlock(); 526 return NULL; 527 } 528 529 return page_ext; 530 } 531 532 /** 533 * page_ext_from_phys() - Get the page_ext structure for a physical address. 534 * @phys: The physical address to query. 535 * 536 * This function safely gets the `struct page_ext` associated with a given 537 * physical address. It performs validation to ensure the address corresponds 538 * to a valid, online struct page before attempting to access it. 539 * It returns NULL for MMIO, ZONE_DEVICE, holes and offline memory. 540 * 541 * Return: NULL if no page_ext exists for this physical address. 542 * Context: Any context. Caller may not sleep until they have called 543 * page_ext_put(). 544 */ 545 struct page_ext *page_ext_from_phys(phys_addr_t phys) 546 { 547 struct page *page = pfn_to_online_page(__phys_to_pfn(phys)); 548 549 if (!page) 550 return NULL; 551 552 return page_ext_get(page); 553 } 554 555 /** 556 * page_ext_put() - Working with page extended information is done. 557 * @page_ext: Page extended information received from page_ext_get(). 558 * 559 * The page extended information of the page may not be valid after this 560 * function is called. 561 * 562 * Return: None. 563 * Context: Any context with corresponding page_ext_get() is called. 564 */ 565 void page_ext_put(struct page_ext *page_ext) 566 { 567 if (unlikely(!page_ext)) 568 return; 569 570 rcu_read_unlock(); 571 } 572