1426e5c42SMuchun Song // SPDX-License-Identifier: GPL-2.0 2426e5c42SMuchun Song /* 3426e5c42SMuchun Song * Bootmem core functions. 4426e5c42SMuchun Song * 5426e5c42SMuchun Song * Copyright (c) 2020, Bytedance. 6426e5c42SMuchun Song * 7426e5c42SMuchun Song * Author: Muchun Song <songmuchun@bytedance.com> 8426e5c42SMuchun Song * 9426e5c42SMuchun Song */ 10426e5c42SMuchun Song #include <linux/mm.h> 11426e5c42SMuchun Song #include <linux/compiler.h> 12426e5c42SMuchun Song #include <linux/memblock.h> 13426e5c42SMuchun Song #include <linux/bootmem_info.h> 14426e5c42SMuchun Song #include <linux/memory_hotplug.h> 15dd0ff4d1SLiu Shixin #include <linux/kmemleak.h> 16426e5c42SMuchun Song 17426e5c42SMuchun Song void get_page_bootmem(unsigned long info, struct page *page, unsigned long type) 18426e5c42SMuchun Song { 19c5e97ed1SMatthew Wilcox (Oracle) page->index = type; 20426e5c42SMuchun Song SetPagePrivate(page); 21426e5c42SMuchun Song set_page_private(page, info); 22426e5c42SMuchun Song page_ref_inc(page); 23426e5c42SMuchun Song } 24426e5c42SMuchun Song 25426e5c42SMuchun Song void put_page_bootmem(struct page *page) 26426e5c42SMuchun Song { 27c5e97ed1SMatthew Wilcox (Oracle) unsigned long type = page->index; 28426e5c42SMuchun Song 29426e5c42SMuchun Song BUG_ON(type < MEMORY_HOTPLUG_MIN_BOOTMEM_TYPE || 30426e5c42SMuchun Song type > MEMORY_HOTPLUG_MAX_BOOTMEM_TYPE); 31426e5c42SMuchun Song 32426e5c42SMuchun Song if (page_ref_dec_return(page) == 1) { 33c5e97ed1SMatthew Wilcox (Oracle) page->index = 0; 34426e5c42SMuchun Song ClearPagePrivate(page); 35426e5c42SMuchun Song set_page_private(page, 0); 36426e5c42SMuchun Song INIT_LIST_HEAD(&page->lru); 37*6d4e2cdaSLiu Shixin kmemleak_free_part_phys(PFN_PHYS(page_to_pfn(page)), PAGE_SIZE); 38426e5c42SMuchun Song free_reserved_page(page); 39426e5c42SMuchun Song } 40426e5c42SMuchun Song } 41426e5c42SMuchun Song 42426e5c42SMuchun Song #ifndef CONFIG_SPARSEMEM_VMEMMAP 43cdcfc631SMuchun Song static void __init register_page_bootmem_info_section(unsigned long start_pfn) 44426e5c42SMuchun Song { 45426e5c42SMuchun Song unsigned long mapsize, section_nr, i; 46426e5c42SMuchun Song struct mem_section *ms; 47426e5c42SMuchun Song struct page *page, *memmap; 48426e5c42SMuchun Song struct mem_section_usage *usage; 49426e5c42SMuchun Song 50426e5c42SMuchun Song section_nr = pfn_to_section_nr(start_pfn); 51426e5c42SMuchun Song ms = __nr_to_section(section_nr); 52426e5c42SMuchun Song 53426e5c42SMuchun Song /* Get section's memmap address */ 54426e5c42SMuchun Song memmap = sparse_decode_mem_map(ms->section_mem_map, section_nr); 55426e5c42SMuchun Song 56426e5c42SMuchun Song /* 57426e5c42SMuchun Song * Get page for the memmap's phys address 58426e5c42SMuchun Song * XXX: need more consideration for sparse_vmemmap... 59426e5c42SMuchun Song */ 60426e5c42SMuchun Song page = virt_to_page(memmap); 61426e5c42SMuchun Song mapsize = sizeof(struct page) * PAGES_PER_SECTION; 62426e5c42SMuchun Song mapsize = PAGE_ALIGN(mapsize) >> PAGE_SHIFT; 63426e5c42SMuchun Song 64426e5c42SMuchun Song /* remember memmap's page */ 65426e5c42SMuchun Song for (i = 0; i < mapsize; i++, page++) 66426e5c42SMuchun Song get_page_bootmem(section_nr, page, SECTION_INFO); 67426e5c42SMuchun Song 68426e5c42SMuchun Song usage = ms->usage; 69426e5c42SMuchun Song page = virt_to_page(usage); 70426e5c42SMuchun Song 71426e5c42SMuchun Song mapsize = PAGE_ALIGN(mem_section_usage_size()) >> PAGE_SHIFT; 72426e5c42SMuchun Song 73426e5c42SMuchun Song for (i = 0; i < mapsize; i++, page++) 74426e5c42SMuchun Song get_page_bootmem(section_nr, page, MIX_SECTION_INFO); 75426e5c42SMuchun Song 76426e5c42SMuchun Song } 77426e5c42SMuchun Song #else /* CONFIG_SPARSEMEM_VMEMMAP */ 78cdcfc631SMuchun Song static void __init register_page_bootmem_info_section(unsigned long start_pfn) 79426e5c42SMuchun Song { 80426e5c42SMuchun Song unsigned long mapsize, section_nr, i; 81426e5c42SMuchun Song struct mem_section *ms; 82426e5c42SMuchun Song struct page *page, *memmap; 83426e5c42SMuchun Song struct mem_section_usage *usage; 84426e5c42SMuchun Song 85426e5c42SMuchun Song section_nr = pfn_to_section_nr(start_pfn); 86426e5c42SMuchun Song ms = __nr_to_section(section_nr); 87426e5c42SMuchun Song 88426e5c42SMuchun Song memmap = sparse_decode_mem_map(ms->section_mem_map, section_nr); 89426e5c42SMuchun Song 90426e5c42SMuchun Song register_page_bootmem_memmap(section_nr, memmap, PAGES_PER_SECTION); 91426e5c42SMuchun Song 92426e5c42SMuchun Song usage = ms->usage; 93426e5c42SMuchun Song page = virt_to_page(usage); 94426e5c42SMuchun Song 95426e5c42SMuchun Song mapsize = PAGE_ALIGN(mem_section_usage_size()) >> PAGE_SHIFT; 96426e5c42SMuchun Song 97426e5c42SMuchun Song for (i = 0; i < mapsize; i++, page++) 98426e5c42SMuchun Song get_page_bootmem(section_nr, page, MIX_SECTION_INFO); 99426e5c42SMuchun Song } 100426e5c42SMuchun Song #endif /* !CONFIG_SPARSEMEM_VMEMMAP */ 101426e5c42SMuchun Song 102426e5c42SMuchun Song void __init register_page_bootmem_info_node(struct pglist_data *pgdat) 103426e5c42SMuchun Song { 104426e5c42SMuchun Song unsigned long i, pfn, end_pfn, nr_pages; 105426e5c42SMuchun Song int node = pgdat->node_id; 106426e5c42SMuchun Song struct page *page; 107426e5c42SMuchun Song 108426e5c42SMuchun Song nr_pages = PAGE_ALIGN(sizeof(struct pglist_data)) >> PAGE_SHIFT; 109426e5c42SMuchun Song page = virt_to_page(pgdat); 110426e5c42SMuchun Song 111426e5c42SMuchun Song for (i = 0; i < nr_pages; i++, page++) 112426e5c42SMuchun Song get_page_bootmem(node, page, NODE_INFO); 113426e5c42SMuchun Song 114426e5c42SMuchun Song pfn = pgdat->node_start_pfn; 115426e5c42SMuchun Song end_pfn = pgdat_end_pfn(pgdat); 116426e5c42SMuchun Song 117426e5c42SMuchun Song /* register section info */ 118426e5c42SMuchun Song for (; pfn < end_pfn; pfn += PAGES_PER_SECTION) { 119426e5c42SMuchun Song /* 120426e5c42SMuchun Song * Some platforms can assign the same pfn to multiple nodes - on 121426e5c42SMuchun Song * node0 as well as nodeN. To avoid registering a pfn against 122426e5c42SMuchun Song * multiple nodes we check that this pfn does not already 123426e5c42SMuchun Song * reside in some other nodes. 124426e5c42SMuchun Song */ 125426e5c42SMuchun Song if (pfn_valid(pfn) && (early_pfn_to_nid(pfn) == node)) 126426e5c42SMuchun Song register_page_bootmem_info_section(pfn); 127426e5c42SMuchun Song } 128426e5c42SMuchun Song } 129