1 // SPDX-License-Identifier: GPL-2.0
23  * At last, enlarging struct page could cause un-wanted system behaviour change.
28  * functions provided by this code. During the boot process, it checks whether
35  * memory allocation at boot-time. The other is optional, init callback, which
40  * boot and extra memory would be unnecessary. In this case, to avoid
45  * none of need callbacks return true, memory isn't needed at all in this boot
50  * extra memory through size in struct page_ext_operations. If it is non-zero,
59  * initialized, even if pages are allocated and used freely. This could
100  * To ensure correct allocation tagging for pages, page_ext should be available
122 		if (page_ext_ops[i]->need()) {  in invoke_need_callbacks()
123 			if (page_ext_ops[i]->need_shared_flags) {  in invoke_need_callbacks()
131 		if (page_ext_ops[i]->need()) {  in invoke_need_callbacks()
132 			page_ext_ops[i]->offset = page_ext_size;  in invoke_need_callbacks()
133 			page_ext_size += page_ext_ops[i]->size;  in invoke_need_callbacks()
147 		if (page_ext_ops[i]->init)  in invoke_init_callbacks()
148 			page_ext_ops[i]->init();  in invoke_init_callbacks()
165 	pgdat->node_page_ext = NULL;  in pgdat_page_ext_init()
175 	base = NODE_DATA(page_to_nid(page))->node_page_ext;  in lookup_page_ext()
179 	 * allocated when feeding a range of pages to the allocator  in lookup_page_ext()
184 	index = pfn - round_down(node_start_pfn(page_to_nid(page)),  in lookup_page_ext()
195 	nr_pages = NODE_DATA(nid)->node_spanned_pages;  in alloc_node_page_ext()
214 		return -ENOMEM;  in alloc_node_page_ext()
215 	NODE_DATA(nid)->node_page_ext = base;  in alloc_node_page_ext()
252 	struct page_ext *page_ext = READ_ONCE(section->page_ext);  in lookup_page_ext()
258 	 * allocated when feeding a range of pages to the allocator  in lookup_page_ext()
291 	if (section->page_ext)  in init_section_page_ext()
298 	 * The value stored in section->page_ext is (base - pfn)  in init_section_page_ext()
306 		return -ENOMEM;  in init_section_page_ext()
314 	section->page_ext = (void *)base - page_ext_size * pfn;  in init_section_page_ext()
325 	memmap_pages_add(-1L * (DIV_ROUND_UP(table_size, PAGE_SIZE)));  in free_page_ext()
343 	if (!ms || !ms->page_ext)  in __free_page_ext()
346 	base = READ_ONCE(ms->page_ext);  in __free_page_ext()
352 		base = (void *)base - PAGE_EXT_INVALID;  in __free_page_ext()
353 	WRITE_ONCE(ms->page_ext, NULL);  in __free_page_ext()
365 	if (!ms || !ms->page_ext)  in __invalidate_page_ext()
367 	val = (void *)ms->page_ext + PAGE_EXT_INVALID;  in __invalidate_page_ext()
368 	WRITE_ONCE(ms->page_ext, val);  in __invalidate_page_ext()
397 	end = pfn - PAGES_PER_SECTION;  in online_page_ext()
401 	return -ENOMEM;  in online_page_ext()
414 	 * use-after-free of it:  in offline_page_ext()
438 		ret = online_page_ext(mn->start_pfn,  in page_ext_callback()
439 				   mn->nr_pages, mn->status_change_nid);  in page_ext_callback()
442 		offline_page_ext(mn->start_pfn,  in page_ext_callback()
443 				mn->nr_pages);  in page_ext_callback()
446 		offline_page_ext(mn->start_pfn,  in page_ext_callback()
447 				mn->nr_pages);  in page_ext_callback()
474 		 * page->flags of out of node pages are not initialized.  So we  in page_ext_init()
485 			 * -------------pfn-------------->  in page_ext_init()
511  * page_ext_get() - Get the extended information for a page.
536  * page_ext_put() - Working with page extended information is done.