1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Bootmem core functions. 4 * 5 * Copyright (c) 2020, Bytedance. 6 * 7 * Author: Muchun Song <songmuchun@bytedance.com> 8 * 9 */ 10 #include <linux/mm.h> 11 #include <linux/compiler.h> 12 #include <linux/memblock.h> 13 #include <linux/bootmem_info.h> 14 #include <linux/memory_hotplug.h> 15 #include <linux/kmemleak.h> 16 17 void get_page_bootmem(unsigned long info, struct page *page, unsigned long type) 18 { 19 page->index = type; 20 SetPagePrivate(page); 21 set_page_private(page, info); 22 page_ref_inc(page); 23 } 24 25 void put_page_bootmem(struct page *page) 26 { 27 unsigned long type = page->index; 28 29 BUG_ON(type < MEMORY_HOTPLUG_MIN_BOOTMEM_TYPE || 30 type > MEMORY_HOTPLUG_MAX_BOOTMEM_TYPE); 31 32 if (page_ref_dec_return(page) == 1) { 33 page->index = 0; 34 ClearPagePrivate(page); 35 set_page_private(page, 0); 36 INIT_LIST_HEAD(&page->lru); 37 kmemleak_free_part_phys(PFN_PHYS(page_to_pfn(page)), PAGE_SIZE); 38 free_reserved_page(page); 39 } 40 } 41 42 #ifndef CONFIG_SPARSEMEM_VMEMMAP 43 static void __init register_page_bootmem_info_section(unsigned long start_pfn) 44 { 45 unsigned long mapsize, section_nr, i; 46 struct mem_section *ms; 47 struct page *page, *memmap; 48 struct mem_section_usage *usage; 49 50 section_nr = pfn_to_section_nr(start_pfn); 51 ms = __nr_to_section(section_nr); 52 53 /* Get section's memmap address */ 54 memmap = sparse_decode_mem_map(ms->section_mem_map, section_nr); 55 56 /* 57 * Get page for the memmap's phys address 58 * XXX: need more consideration for sparse_vmemmap... 59 */ 60 page = virt_to_page(memmap); 61 mapsize = sizeof(struct page) * PAGES_PER_SECTION; 62 mapsize = PAGE_ALIGN(mapsize) >> PAGE_SHIFT; 63 64 /* remember memmap's page */ 65 for (i = 0; i < mapsize; i++, page++) 66 get_page_bootmem(section_nr, page, SECTION_INFO); 67 68 usage = ms->usage; 69 page = virt_to_page(usage); 70 71 mapsize = PAGE_ALIGN(mem_section_usage_size()) >> PAGE_SHIFT; 72 73 for (i = 0; i < mapsize; i++, page++) 74 get_page_bootmem(section_nr, page, MIX_SECTION_INFO); 75 76 } 77 #else /* CONFIG_SPARSEMEM_VMEMMAP */ 78 static void __init register_page_bootmem_info_section(unsigned long start_pfn) 79 { 80 unsigned long mapsize, section_nr, i; 81 struct mem_section *ms; 82 struct page *page, *memmap; 83 struct mem_section_usage *usage; 84 85 section_nr = pfn_to_section_nr(start_pfn); 86 ms = __nr_to_section(section_nr); 87 88 memmap = sparse_decode_mem_map(ms->section_mem_map, section_nr); 89 90 register_page_bootmem_memmap(section_nr, memmap, PAGES_PER_SECTION); 91 92 usage = ms->usage; 93 page = virt_to_page(usage); 94 95 mapsize = PAGE_ALIGN(mem_section_usage_size()) >> PAGE_SHIFT; 96 97 for (i = 0; i < mapsize; i++, page++) 98 get_page_bootmem(section_nr, page, MIX_SECTION_INFO); 99 } 100 #endif /* !CONFIG_SPARSEMEM_VMEMMAP */ 101 102 void __init register_page_bootmem_info_node(struct pglist_data *pgdat) 103 { 104 unsigned long i, pfn, end_pfn, nr_pages; 105 int node = pgdat->node_id; 106 struct page *page; 107 108 nr_pages = PAGE_ALIGN(sizeof(struct pglist_data)) >> PAGE_SHIFT; 109 page = virt_to_page(pgdat); 110 111 for (i = 0; i < nr_pages; i++, page++) 112 get_page_bootmem(node, page, NODE_INFO); 113 114 pfn = pgdat->node_start_pfn; 115 end_pfn = pgdat_end_pfn(pgdat); 116 117 /* register section info */ 118 for (; pfn < end_pfn; pfn += PAGES_PER_SECTION) { 119 /* 120 * Some platforms can assign the same pfn to multiple nodes - on 121 * node0 as well as nodeN. To avoid registering a pfn against 122 * multiple nodes we check that this pfn does not already 123 * reside in some other nodes. 124 */ 125 if (pfn_valid(pfn) && (early_pfn_to_nid(pfn) == node)) 126 register_page_bootmem_info_section(pfn); 127 } 128 } 129