1d41dee36SAndy Whitcroft /* 2d41dee36SAndy Whitcroft * sparse memory mappings. 3d41dee36SAndy Whitcroft */ 4d41dee36SAndy Whitcroft #include <linux/mm.h> 5*5a0e3ad6STejun Heo #include <linux/slab.h> 6d41dee36SAndy Whitcroft #include <linux/mmzone.h> 7d41dee36SAndy Whitcroft #include <linux/bootmem.h> 80b0acbecSDave Hansen #include <linux/highmem.h> 9d41dee36SAndy Whitcroft #include <linux/module.h> 1028ae55c9SDave Hansen #include <linux/spinlock.h> 110b0acbecSDave Hansen #include <linux/vmalloc.h> 120c0a4a51SYasunori Goto #include "internal.h" 13d41dee36SAndy Whitcroft #include <asm/dma.h> 148f6aac41SChristoph Lameter #include <asm/pgalloc.h> 158f6aac41SChristoph Lameter #include <asm/pgtable.h> 16d41dee36SAndy Whitcroft 17d41dee36SAndy Whitcroft /* 18d41dee36SAndy Whitcroft * Permanent SPARSEMEM data: 19d41dee36SAndy Whitcroft * 20d41dee36SAndy Whitcroft * 1) mem_section - memory sections, mem_map's for valid memory 21d41dee36SAndy Whitcroft */ 223e347261SBob Picco #ifdef CONFIG_SPARSEMEM_EXTREME 23802f192eSBob Picco struct mem_section *mem_section[NR_SECTION_ROOTS] 2422fc6eccSRavikiran G Thirumalai ____cacheline_internodealigned_in_smp; 253e347261SBob Picco #else 263e347261SBob Picco struct mem_section mem_section[NR_SECTION_ROOTS][SECTIONS_PER_ROOT] 2722fc6eccSRavikiran G Thirumalai ____cacheline_internodealigned_in_smp; 283e347261SBob Picco #endif 293e347261SBob Picco EXPORT_SYMBOL(mem_section); 303e347261SBob Picco 3189689ae7SChristoph Lameter #ifdef NODE_NOT_IN_PAGE_FLAGS 3289689ae7SChristoph Lameter /* 3389689ae7SChristoph Lameter * If we did not store the node number in the page then we have to 3489689ae7SChristoph Lameter * do a lookup in the section_to_node_table in order to find which 3589689ae7SChristoph Lameter * node the page belongs to. 3689689ae7SChristoph Lameter */ 3789689ae7SChristoph Lameter #if MAX_NUMNODES <= 256 3889689ae7SChristoph Lameter static u8 section_to_node_table[NR_MEM_SECTIONS] __cacheline_aligned; 3989689ae7SChristoph Lameter #else 4089689ae7SChristoph Lameter static u16 section_to_node_table[NR_MEM_SECTIONS] __cacheline_aligned; 4189689ae7SChristoph Lameter #endif 4289689ae7SChristoph Lameter 4325ba77c1SAndy Whitcroft int page_to_nid(struct page *page) 4489689ae7SChristoph Lameter { 4589689ae7SChristoph Lameter return section_to_node_table[page_to_section(page)]; 4689689ae7SChristoph Lameter } 4789689ae7SChristoph Lameter EXPORT_SYMBOL(page_to_nid); 4885770ffeSAndy Whitcroft 4985770ffeSAndy Whitcroft static void set_section_nid(unsigned long section_nr, int nid) 5085770ffeSAndy Whitcroft { 5185770ffeSAndy Whitcroft section_to_node_table[section_nr] = nid; 5285770ffeSAndy Whitcroft } 5385770ffeSAndy Whitcroft #else /* !NODE_NOT_IN_PAGE_FLAGS */ 5485770ffeSAndy Whitcroft static inline void set_section_nid(unsigned long section_nr, int nid) 5585770ffeSAndy Whitcroft { 5685770ffeSAndy Whitcroft } 5789689ae7SChristoph Lameter #endif 5889689ae7SChristoph Lameter 593e347261SBob Picco #ifdef CONFIG_SPARSEMEM_EXTREME 60577a32f6SSam Ravnborg static struct mem_section noinline __init_refok *sparse_index_alloc(int nid) 61802f192eSBob Picco { 6228ae55c9SDave Hansen struct mem_section *section = NULL; 6328ae55c9SDave Hansen unsigned long array_size = SECTIONS_PER_ROOT * 6428ae55c9SDave Hansen sizeof(struct mem_section); 65802f192eSBob Picco 66f52407ceSShaohua Li if (slab_is_available()) { 67f52407ceSShaohua Li if (node_state(nid, N_HIGH_MEMORY)) 6846a66eecSMike Kravetz section = kmalloc_node(array_size, GFP_KERNEL, nid); 6946a66eecSMike Kravetz else 70f52407ceSShaohua Li section = kmalloc(array_size, GFP_KERNEL); 71f52407ceSShaohua Li } else 7228ae55c9SDave Hansen section = alloc_bootmem_node(NODE_DATA(nid), array_size); 733e347261SBob Picco 7428ae55c9SDave Hansen if (section) 7528ae55c9SDave Hansen memset(section, 0, array_size); 763e347261SBob Picco 7728ae55c9SDave Hansen return section; 78802f192eSBob Picco } 7928ae55c9SDave Hansen 80a3142c8eSYasunori Goto static int __meminit sparse_index_init(unsigned long section_nr, int nid) 8128ae55c9SDave Hansen { 8234af946aSIngo Molnar static DEFINE_SPINLOCK(index_init_lock); 8328ae55c9SDave Hansen unsigned long root = SECTION_NR_TO_ROOT(section_nr); 8428ae55c9SDave Hansen struct mem_section *section; 8528ae55c9SDave Hansen int ret = 0; 8628ae55c9SDave Hansen 8728ae55c9SDave Hansen if (mem_section[root]) 8828ae55c9SDave Hansen return -EEXIST; 8928ae55c9SDave Hansen 9028ae55c9SDave Hansen section = sparse_index_alloc(nid); 91af0cd5a7SWANG Cong if (!section) 92af0cd5a7SWANG Cong return -ENOMEM; 9328ae55c9SDave Hansen /* 9428ae55c9SDave Hansen * This lock keeps two different sections from 9528ae55c9SDave Hansen * reallocating for the same index 9628ae55c9SDave Hansen */ 9728ae55c9SDave Hansen spin_lock(&index_init_lock); 9828ae55c9SDave Hansen 9928ae55c9SDave Hansen if (mem_section[root]) { 10028ae55c9SDave Hansen ret = -EEXIST; 10128ae55c9SDave Hansen goto out; 10228ae55c9SDave Hansen } 10328ae55c9SDave Hansen 10428ae55c9SDave Hansen mem_section[root] = section; 10528ae55c9SDave Hansen out: 10628ae55c9SDave Hansen spin_unlock(&index_init_lock); 10728ae55c9SDave Hansen return ret; 10828ae55c9SDave Hansen } 10928ae55c9SDave Hansen #else /* !SPARSEMEM_EXTREME */ 11028ae55c9SDave Hansen static inline int sparse_index_init(unsigned long section_nr, int nid) 11128ae55c9SDave Hansen { 11228ae55c9SDave Hansen return 0; 11328ae55c9SDave Hansen } 11428ae55c9SDave Hansen #endif 11528ae55c9SDave Hansen 1164ca644d9SDave Hansen /* 1174ca644d9SDave Hansen * Although written for the SPARSEMEM_EXTREME case, this happens 118cd881a6bSAndy Whitcroft * to also work for the flat array case because 1194ca644d9SDave Hansen * NR_SECTION_ROOTS==NR_MEM_SECTIONS. 1204ca644d9SDave Hansen */ 1214ca644d9SDave Hansen int __section_nr(struct mem_section* ms) 1224ca644d9SDave Hansen { 1234ca644d9SDave Hansen unsigned long root_nr; 1244ca644d9SDave Hansen struct mem_section* root; 1254ca644d9SDave Hansen 12612783b00SMike Kravetz for (root_nr = 0; root_nr < NR_SECTION_ROOTS; root_nr++) { 12712783b00SMike Kravetz root = __nr_to_section(root_nr * SECTIONS_PER_ROOT); 1284ca644d9SDave Hansen if (!root) 1294ca644d9SDave Hansen continue; 1304ca644d9SDave Hansen 1314ca644d9SDave Hansen if ((ms >= root) && (ms < (root + SECTIONS_PER_ROOT))) 1324ca644d9SDave Hansen break; 1334ca644d9SDave Hansen } 1344ca644d9SDave Hansen 1354ca644d9SDave Hansen return (root_nr * SECTIONS_PER_ROOT) + (ms - root); 1364ca644d9SDave Hansen } 1374ca644d9SDave Hansen 13830c253e6SAndy Whitcroft /* 13930c253e6SAndy Whitcroft * During early boot, before section_mem_map is used for an actual 14030c253e6SAndy Whitcroft * mem_map, we use section_mem_map to store the section's NUMA 14130c253e6SAndy Whitcroft * node. This keeps us from having to use another data structure. The 14230c253e6SAndy Whitcroft * node information is cleared just before we store the real mem_map. 14330c253e6SAndy Whitcroft */ 14430c253e6SAndy Whitcroft static inline unsigned long sparse_encode_early_nid(int nid) 14530c253e6SAndy Whitcroft { 14630c253e6SAndy Whitcroft return (nid << SECTION_NID_SHIFT); 14730c253e6SAndy Whitcroft } 14830c253e6SAndy Whitcroft 14930c253e6SAndy Whitcroft static inline int sparse_early_nid(struct mem_section *section) 15030c253e6SAndy Whitcroft { 15130c253e6SAndy Whitcroft return (section->section_mem_map >> SECTION_NID_SHIFT); 15230c253e6SAndy Whitcroft } 15330c253e6SAndy Whitcroft 1542dbb51c4SMel Gorman /* Validate the physical addressing limitations of the model */ 1552dbb51c4SMel Gorman void __meminit mminit_validate_memmodel_limits(unsigned long *start_pfn, 1562dbb51c4SMel Gorman unsigned long *end_pfn) 157d41dee36SAndy Whitcroft { 1582dbb51c4SMel Gorman unsigned long max_sparsemem_pfn = 1UL << (MAX_PHYSMEM_BITS-PAGE_SHIFT); 159d41dee36SAndy Whitcroft 160bead9a3aSIngo Molnar /* 161bead9a3aSIngo Molnar * Sanity checks - do not allow an architecture to pass 162bead9a3aSIngo Molnar * in larger pfns than the maximum scope of sparsemem: 163bead9a3aSIngo Molnar */ 1642dbb51c4SMel Gorman if (*start_pfn > max_sparsemem_pfn) { 1652dbb51c4SMel Gorman mminit_dprintk(MMINIT_WARNING, "pfnvalidation", 1662dbb51c4SMel Gorman "Start of range %lu -> %lu exceeds SPARSEMEM max %lu\n", 1672dbb51c4SMel Gorman *start_pfn, *end_pfn, max_sparsemem_pfn); 1682dbb51c4SMel Gorman WARN_ON_ONCE(1); 1692dbb51c4SMel Gorman *start_pfn = max_sparsemem_pfn; 1702dbb51c4SMel Gorman *end_pfn = max_sparsemem_pfn; 171ef161a98SCyrill Gorcunov } else if (*end_pfn > max_sparsemem_pfn) { 1722dbb51c4SMel Gorman mminit_dprintk(MMINIT_WARNING, "pfnvalidation", 1732dbb51c4SMel Gorman "End of range %lu -> %lu exceeds SPARSEMEM max %lu\n", 1742dbb51c4SMel Gorman *start_pfn, *end_pfn, max_sparsemem_pfn); 1752dbb51c4SMel Gorman WARN_ON_ONCE(1); 1762dbb51c4SMel Gorman *end_pfn = max_sparsemem_pfn; 1772dbb51c4SMel Gorman } 1782dbb51c4SMel Gorman } 1792dbb51c4SMel Gorman 1802dbb51c4SMel Gorman /* Record a memory area against a node. */ 1812dbb51c4SMel Gorman void __init memory_present(int nid, unsigned long start, unsigned long end) 1822dbb51c4SMel Gorman { 1832dbb51c4SMel Gorman unsigned long pfn; 184bead9a3aSIngo Molnar 185d41dee36SAndy Whitcroft start &= PAGE_SECTION_MASK; 1862dbb51c4SMel Gorman mminit_validate_memmodel_limits(&start, &end); 187d41dee36SAndy Whitcroft for (pfn = start; pfn < end; pfn += PAGES_PER_SECTION) { 188d41dee36SAndy Whitcroft unsigned long section = pfn_to_section_nr(pfn); 189802f192eSBob Picco struct mem_section *ms; 190802f192eSBob Picco 191802f192eSBob Picco sparse_index_init(section, nid); 19285770ffeSAndy Whitcroft set_section_nid(section, nid); 193802f192eSBob Picco 194802f192eSBob Picco ms = __nr_to_section(section); 195802f192eSBob Picco if (!ms->section_mem_map) 19630c253e6SAndy Whitcroft ms->section_mem_map = sparse_encode_early_nid(nid) | 19730c253e6SAndy Whitcroft SECTION_MARKED_PRESENT; 198d41dee36SAndy Whitcroft } 199d41dee36SAndy Whitcroft } 200d41dee36SAndy Whitcroft 201d41dee36SAndy Whitcroft /* 202d41dee36SAndy Whitcroft * Only used by the i386 NUMA architecures, but relatively 203d41dee36SAndy Whitcroft * generic code. 204d41dee36SAndy Whitcroft */ 205d41dee36SAndy Whitcroft unsigned long __init node_memmap_size_bytes(int nid, unsigned long start_pfn, 206d41dee36SAndy Whitcroft unsigned long end_pfn) 207d41dee36SAndy Whitcroft { 208d41dee36SAndy Whitcroft unsigned long pfn; 209d41dee36SAndy Whitcroft unsigned long nr_pages = 0; 210d41dee36SAndy Whitcroft 2112dbb51c4SMel Gorman mminit_validate_memmodel_limits(&start_pfn, &end_pfn); 212d41dee36SAndy Whitcroft for (pfn = start_pfn; pfn < end_pfn; pfn += PAGES_PER_SECTION) { 213d41dee36SAndy Whitcroft if (nid != early_pfn_to_nid(pfn)) 214d41dee36SAndy Whitcroft continue; 215d41dee36SAndy Whitcroft 216540557b9SAndy Whitcroft if (pfn_present(pfn)) 217d41dee36SAndy Whitcroft nr_pages += PAGES_PER_SECTION; 218d41dee36SAndy Whitcroft } 219d41dee36SAndy Whitcroft 220d41dee36SAndy Whitcroft return nr_pages * sizeof(struct page); 221d41dee36SAndy Whitcroft } 222d41dee36SAndy Whitcroft 223d41dee36SAndy Whitcroft /* 22429751f69SAndy Whitcroft * Subtle, we encode the real pfn into the mem_map such that 22529751f69SAndy Whitcroft * the identity pfn - section_mem_map will return the actual 22629751f69SAndy Whitcroft * physical page frame number. 22729751f69SAndy Whitcroft */ 22829751f69SAndy Whitcroft static unsigned long sparse_encode_mem_map(struct page *mem_map, unsigned long pnum) 22929751f69SAndy Whitcroft { 23029751f69SAndy Whitcroft return (unsigned long)(mem_map - (section_nr_to_pfn(pnum))); 23129751f69SAndy Whitcroft } 23229751f69SAndy Whitcroft 23329751f69SAndy Whitcroft /* 234ea01ea93SBadari Pulavarty * Decode mem_map from the coded memmap 23529751f69SAndy Whitcroft */ 23629751f69SAndy Whitcroft struct page *sparse_decode_mem_map(unsigned long coded_mem_map, unsigned long pnum) 23729751f69SAndy Whitcroft { 238ea01ea93SBadari Pulavarty /* mask off the extra low bits of information */ 239ea01ea93SBadari Pulavarty coded_mem_map &= SECTION_MAP_MASK; 24029751f69SAndy Whitcroft return ((struct page *)coded_mem_map) + section_nr_to_pfn(pnum); 24129751f69SAndy Whitcroft } 24229751f69SAndy Whitcroft 243a3142c8eSYasunori Goto static int __meminit sparse_init_one_section(struct mem_section *ms, 2445c0e3066SMel Gorman unsigned long pnum, struct page *mem_map, 2455c0e3066SMel Gorman unsigned long *pageblock_bitmap) 24629751f69SAndy Whitcroft { 247540557b9SAndy Whitcroft if (!present_section(ms)) 24829751f69SAndy Whitcroft return -EINVAL; 24929751f69SAndy Whitcroft 25030c253e6SAndy Whitcroft ms->section_mem_map &= ~SECTION_MAP_MASK; 251540557b9SAndy Whitcroft ms->section_mem_map |= sparse_encode_mem_map(mem_map, pnum) | 252540557b9SAndy Whitcroft SECTION_HAS_MEM_MAP; 2535c0e3066SMel Gorman ms->pageblock_flags = pageblock_bitmap; 25429751f69SAndy Whitcroft 25529751f69SAndy Whitcroft return 1; 25629751f69SAndy Whitcroft } 25729751f69SAndy Whitcroft 25804753278SYasunori Goto unsigned long usemap_size(void) 2595c0e3066SMel Gorman { 2605c0e3066SMel Gorman unsigned long size_bytes; 2615c0e3066SMel Gorman size_bytes = roundup(SECTION_BLOCKFLAGS_BITS, 8) / 8; 2625c0e3066SMel Gorman size_bytes = roundup(size_bytes, sizeof(unsigned long)); 2635c0e3066SMel Gorman return size_bytes; 2645c0e3066SMel Gorman } 2655c0e3066SMel Gorman 2665c0e3066SMel Gorman #ifdef CONFIG_MEMORY_HOTPLUG 2675c0e3066SMel Gorman static unsigned long *__kmalloc_section_usemap(void) 2685c0e3066SMel Gorman { 2695c0e3066SMel Gorman return kmalloc(usemap_size(), GFP_KERNEL); 2705c0e3066SMel Gorman } 2715c0e3066SMel Gorman #endif /* CONFIG_MEMORY_HOTPLUG */ 2725c0e3066SMel Gorman 27348c90682SYasunori Goto #ifdef CONFIG_MEMORY_HOTREMOVE 27448c90682SYasunori Goto static unsigned long * __init 275a4322e1bSYinghai Lu sparse_early_usemaps_alloc_pgdat_section(struct pglist_data *pgdat, 276a4322e1bSYinghai Lu unsigned long count) 27748c90682SYasunori Goto { 27848c90682SYasunori Goto unsigned long section_nr; 27948c90682SYasunori Goto 28048c90682SYasunori Goto /* 28148c90682SYasunori Goto * A page may contain usemaps for other sections preventing the 28248c90682SYasunori Goto * page being freed and making a section unremovable while 28348c90682SYasunori Goto * other sections referencing the usemap retmain active. Similarly, 28448c90682SYasunori Goto * a pgdat can prevent a section being removed. If section A 28548c90682SYasunori Goto * contains a pgdat and section B contains the usemap, both 28648c90682SYasunori Goto * sections become inter-dependent. This allocates usemaps 28748c90682SYasunori Goto * from the same section as the pgdat where possible to avoid 28848c90682SYasunori Goto * this problem. 28948c90682SYasunori Goto */ 29048c90682SYasunori Goto section_nr = pfn_to_section_nr(__pa(pgdat) >> PAGE_SHIFT); 291a4322e1bSYinghai Lu return alloc_bootmem_section(usemap_size() * count, section_nr); 29248c90682SYasunori Goto } 29348c90682SYasunori Goto 29448c90682SYasunori Goto static void __init check_usemap_section_nr(int nid, unsigned long *usemap) 29548c90682SYasunori Goto { 29648c90682SYasunori Goto unsigned long usemap_snr, pgdat_snr; 29748c90682SYasunori Goto static unsigned long old_usemap_snr = NR_MEM_SECTIONS; 29848c90682SYasunori Goto static unsigned long old_pgdat_snr = NR_MEM_SECTIONS; 29948c90682SYasunori Goto struct pglist_data *pgdat = NODE_DATA(nid); 30048c90682SYasunori Goto int usemap_nid; 30148c90682SYasunori Goto 30248c90682SYasunori Goto usemap_snr = pfn_to_section_nr(__pa(usemap) >> PAGE_SHIFT); 30348c90682SYasunori Goto pgdat_snr = pfn_to_section_nr(__pa(pgdat) >> PAGE_SHIFT); 30448c90682SYasunori Goto if (usemap_snr == pgdat_snr) 30548c90682SYasunori Goto return; 30648c90682SYasunori Goto 30748c90682SYasunori Goto if (old_usemap_snr == usemap_snr && old_pgdat_snr == pgdat_snr) 30848c90682SYasunori Goto /* skip redundant message */ 30948c90682SYasunori Goto return; 31048c90682SYasunori Goto 31148c90682SYasunori Goto old_usemap_snr = usemap_snr; 31248c90682SYasunori Goto old_pgdat_snr = pgdat_snr; 31348c90682SYasunori Goto 31448c90682SYasunori Goto usemap_nid = sparse_early_nid(__nr_to_section(usemap_snr)); 31548c90682SYasunori Goto if (usemap_nid != nid) { 31648c90682SYasunori Goto printk(KERN_INFO 31748c90682SYasunori Goto "node %d must be removed before remove section %ld\n", 31848c90682SYasunori Goto nid, usemap_snr); 31948c90682SYasunori Goto return; 32048c90682SYasunori Goto } 32148c90682SYasunori Goto /* 32248c90682SYasunori Goto * There is a circular dependency. 32348c90682SYasunori Goto * Some platforms allow un-removable section because they will just 32448c90682SYasunori Goto * gather other removable sections for dynamic partitioning. 32548c90682SYasunori Goto * Just notify un-removable section's number here. 32648c90682SYasunori Goto */ 32748c90682SYasunori Goto printk(KERN_INFO "Section %ld and %ld (node %d)", usemap_snr, 32848c90682SYasunori Goto pgdat_snr, nid); 32948c90682SYasunori Goto printk(KERN_CONT 33048c90682SYasunori Goto " have a circular dependency on usemap and pgdat allocations\n"); 33148c90682SYasunori Goto } 33248c90682SYasunori Goto #else 33348c90682SYasunori Goto static unsigned long * __init 334a4322e1bSYinghai Lu sparse_early_usemaps_alloc_pgdat_section(struct pglist_data *pgdat, 335a4322e1bSYinghai Lu unsigned long count) 33648c90682SYasunori Goto { 33748c90682SYasunori Goto return NULL; 33848c90682SYasunori Goto } 33948c90682SYasunori Goto 34048c90682SYasunori Goto static void __init check_usemap_section_nr(int nid, unsigned long *usemap) 34148c90682SYasunori Goto { 34248c90682SYasunori Goto } 34348c90682SYasunori Goto #endif /* CONFIG_MEMORY_HOTREMOVE */ 34448c90682SYasunori Goto 345a4322e1bSYinghai Lu static void __init sparse_early_usemaps_alloc_node(unsigned long**usemap_map, 346a4322e1bSYinghai Lu unsigned long pnum_begin, 347a4322e1bSYinghai Lu unsigned long pnum_end, 348a4322e1bSYinghai Lu unsigned long usemap_count, int nodeid) 3495c0e3066SMel Gorman { 350a4322e1bSYinghai Lu void *usemap; 351a4322e1bSYinghai Lu unsigned long pnum; 352a4322e1bSYinghai Lu int size = usemap_size(); 3535c0e3066SMel Gorman 354a4322e1bSYinghai Lu usemap = sparse_early_usemaps_alloc_pgdat_section(NODE_DATA(nodeid), 355a4322e1bSYinghai Lu usemap_count); 35648c90682SYasunori Goto if (usemap) { 357a4322e1bSYinghai Lu for (pnum = pnum_begin; pnum < pnum_end; pnum++) { 358a4322e1bSYinghai Lu if (!present_section_nr(pnum)) 359a4322e1bSYinghai Lu continue; 360a4322e1bSYinghai Lu usemap_map[pnum] = usemap; 361a4322e1bSYinghai Lu usemap += size; 362a4322e1bSYinghai Lu } 363a4322e1bSYinghai Lu return; 36448c90682SYasunori Goto } 36548c90682SYasunori Goto 366a4322e1bSYinghai Lu usemap = alloc_bootmem_node(NODE_DATA(nodeid), size * usemap_count); 367a4322e1bSYinghai Lu if (usemap) { 368a4322e1bSYinghai Lu for (pnum = pnum_begin; pnum < pnum_end; pnum++) { 369a4322e1bSYinghai Lu if (!present_section_nr(pnum)) 370a4322e1bSYinghai Lu continue; 371a4322e1bSYinghai Lu usemap_map[pnum] = usemap; 372a4322e1bSYinghai Lu usemap += size; 373a4322e1bSYinghai Lu check_usemap_section_nr(nodeid, usemap_map[pnum]); 374a4322e1bSYinghai Lu } 375a4322e1bSYinghai Lu return; 376a4322e1bSYinghai Lu } 3775c0e3066SMel Gorman 378d40cee24SHarvey Harrison printk(KERN_WARNING "%s: allocation failed\n", __func__); 3795c0e3066SMel Gorman } 3805c0e3066SMel Gorman 3818f6aac41SChristoph Lameter #ifndef CONFIG_SPARSEMEM_VMEMMAP 38298f3cfc1SYasunori Goto struct page __init *sparse_mem_map_populate(unsigned long pnum, int nid) 38329751f69SAndy Whitcroft { 38429751f69SAndy Whitcroft struct page *map; 38529751f69SAndy Whitcroft 38629751f69SAndy Whitcroft map = alloc_remap(nid, sizeof(struct page) * PAGES_PER_SECTION); 38729751f69SAndy Whitcroft if (map) 38829751f69SAndy Whitcroft return map; 38929751f69SAndy Whitcroft 3909d99217aSYasunori Goto map = alloc_bootmem_pages_node(NODE_DATA(nid), 3919d99217aSYasunori Goto PAGE_ALIGN(sizeof(struct page) * PAGES_PER_SECTION)); 3928f6aac41SChristoph Lameter return map; 3938f6aac41SChristoph Lameter } 3949bdac914SYinghai Lu void __init sparse_mem_maps_populate_node(struct page **map_map, 3959bdac914SYinghai Lu unsigned long pnum_begin, 3969bdac914SYinghai Lu unsigned long pnum_end, 3979bdac914SYinghai Lu unsigned long map_count, int nodeid) 3989bdac914SYinghai Lu { 3999bdac914SYinghai Lu void *map; 4009bdac914SYinghai Lu unsigned long pnum; 4019bdac914SYinghai Lu unsigned long size = sizeof(struct page) * PAGES_PER_SECTION; 4029bdac914SYinghai Lu 4039bdac914SYinghai Lu map = alloc_remap(nodeid, size * map_count); 4049bdac914SYinghai Lu if (map) { 4059bdac914SYinghai Lu for (pnum = pnum_begin; pnum < pnum_end; pnum++) { 4069bdac914SYinghai Lu if (!present_section_nr(pnum)) 4079bdac914SYinghai Lu continue; 4089bdac914SYinghai Lu map_map[pnum] = map; 4099bdac914SYinghai Lu map += size; 4109bdac914SYinghai Lu } 4119bdac914SYinghai Lu return; 4129bdac914SYinghai Lu } 4139bdac914SYinghai Lu 4149bdac914SYinghai Lu size = PAGE_ALIGN(size); 4159bdac914SYinghai Lu map = alloc_bootmem_pages_node(NODE_DATA(nodeid), size * map_count); 4169bdac914SYinghai Lu if (map) { 4179bdac914SYinghai Lu for (pnum = pnum_begin; pnum < pnum_end; pnum++) { 4189bdac914SYinghai Lu if (!present_section_nr(pnum)) 4199bdac914SYinghai Lu continue; 4209bdac914SYinghai Lu map_map[pnum] = map; 4219bdac914SYinghai Lu map += size; 4229bdac914SYinghai Lu } 4239bdac914SYinghai Lu return; 4249bdac914SYinghai Lu } 4259bdac914SYinghai Lu 4269bdac914SYinghai Lu /* fallback */ 4279bdac914SYinghai Lu for (pnum = pnum_begin; pnum < pnum_end; pnum++) { 4289bdac914SYinghai Lu struct mem_section *ms; 4299bdac914SYinghai Lu 4309bdac914SYinghai Lu if (!present_section_nr(pnum)) 4319bdac914SYinghai Lu continue; 4329bdac914SYinghai Lu map_map[pnum] = sparse_mem_map_populate(pnum, nodeid); 4339bdac914SYinghai Lu if (map_map[pnum]) 4349bdac914SYinghai Lu continue; 4359bdac914SYinghai Lu ms = __nr_to_section(pnum); 4369bdac914SYinghai Lu printk(KERN_ERR "%s: sparsemem memory map backing failed " 4379bdac914SYinghai Lu "some memory will not be available.\n", __func__); 4389bdac914SYinghai Lu ms->section_mem_map = 0; 4399bdac914SYinghai Lu } 4409bdac914SYinghai Lu } 4418f6aac41SChristoph Lameter #endif /* !CONFIG_SPARSEMEM_VMEMMAP */ 4428f6aac41SChristoph Lameter 44381d0d950SYinghai Lu #ifdef CONFIG_SPARSEMEM_ALLOC_MEM_MAP_TOGETHER 4449bdac914SYinghai Lu static void __init sparse_early_mem_maps_alloc_node(struct page **map_map, 4459bdac914SYinghai Lu unsigned long pnum_begin, 4469bdac914SYinghai Lu unsigned long pnum_end, 4479bdac914SYinghai Lu unsigned long map_count, int nodeid) 4489bdac914SYinghai Lu { 4499bdac914SYinghai Lu sparse_mem_maps_populate_node(map_map, pnum_begin, pnum_end, 4509bdac914SYinghai Lu map_count, nodeid); 4519bdac914SYinghai Lu } 45281d0d950SYinghai Lu #else 4539e5c6da7SAdrian Bunk static struct page __init *sparse_early_mem_map_alloc(unsigned long pnum) 4548f6aac41SChristoph Lameter { 4558f6aac41SChristoph Lameter struct page *map; 4568f6aac41SChristoph Lameter struct mem_section *ms = __nr_to_section(pnum); 4578f6aac41SChristoph Lameter int nid = sparse_early_nid(ms); 4588f6aac41SChristoph Lameter 45998f3cfc1SYasunori Goto map = sparse_mem_map_populate(pnum, nid); 46029751f69SAndy Whitcroft if (map) 46129751f69SAndy Whitcroft return map; 46229751f69SAndy Whitcroft 4638f6aac41SChristoph Lameter printk(KERN_ERR "%s: sparsemem memory map backing failed " 464d40cee24SHarvey Harrison "some memory will not be available.\n", __func__); 465802f192eSBob Picco ms->section_mem_map = 0; 46629751f69SAndy Whitcroft return NULL; 46729751f69SAndy Whitcroft } 4689bdac914SYinghai Lu #endif 46929751f69SAndy Whitcroft 470c2b91e2eSYinghai Lu void __attribute__((weak)) __meminit vmemmap_populate_print_last(void) 471c2b91e2eSYinghai Lu { 472c2b91e2eSYinghai Lu } 473a4322e1bSYinghai Lu 474193faea9SStephen Rothwell /* 475193faea9SStephen Rothwell * Allocate the accumulated non-linear sections, allocate a mem_map 476193faea9SStephen Rothwell * for each and record the physical to section mapping. 477193faea9SStephen Rothwell */ 478193faea9SStephen Rothwell void __init sparse_init(void) 479193faea9SStephen Rothwell { 480193faea9SStephen Rothwell unsigned long pnum; 481193faea9SStephen Rothwell struct page *map; 4825c0e3066SMel Gorman unsigned long *usemap; 483e123dd3fSYinghai Lu unsigned long **usemap_map; 48481d0d950SYinghai Lu int size; 485a4322e1bSYinghai Lu int nodeid_begin = 0; 486a4322e1bSYinghai Lu unsigned long pnum_begin = 0; 487a4322e1bSYinghai Lu unsigned long usemap_count; 48881d0d950SYinghai Lu #ifdef CONFIG_SPARSEMEM_ALLOC_MEM_MAP_TOGETHER 4899bdac914SYinghai Lu unsigned long map_count; 49081d0d950SYinghai Lu int size2; 49181d0d950SYinghai Lu struct page **map_map; 49281d0d950SYinghai Lu #endif 493e123dd3fSYinghai Lu 494e123dd3fSYinghai Lu /* 495e123dd3fSYinghai Lu * map is using big page (aka 2M in x86 64 bit) 496e123dd3fSYinghai Lu * usemap is less one page (aka 24 bytes) 497e123dd3fSYinghai Lu * so alloc 2M (with 2M align) and 24 bytes in turn will 498e123dd3fSYinghai Lu * make next 2M slip to one more 2M later. 499e123dd3fSYinghai Lu * then in big system, the memory will have a lot of holes... 500e123dd3fSYinghai Lu * here try to allocate 2M pages continously. 501e123dd3fSYinghai Lu * 502e123dd3fSYinghai Lu * powerpc need to call sparse_init_one_section right after each 503e123dd3fSYinghai Lu * sparse_early_mem_map_alloc, so allocate usemap_map at first. 504e123dd3fSYinghai Lu */ 505e123dd3fSYinghai Lu size = sizeof(unsigned long *) * NR_MEM_SECTIONS; 506e123dd3fSYinghai Lu usemap_map = alloc_bootmem(size); 507e123dd3fSYinghai Lu if (!usemap_map) 508e123dd3fSYinghai Lu panic("can not allocate usemap_map\n"); 509193faea9SStephen Rothwell 510193faea9SStephen Rothwell for (pnum = 0; pnum < NR_MEM_SECTIONS; pnum++) { 511a4322e1bSYinghai Lu struct mem_section *ms; 512a4322e1bSYinghai Lu 513540557b9SAndy Whitcroft if (!present_section_nr(pnum)) 514193faea9SStephen Rothwell continue; 515a4322e1bSYinghai Lu ms = __nr_to_section(pnum); 516a4322e1bSYinghai Lu nodeid_begin = sparse_early_nid(ms); 517a4322e1bSYinghai Lu pnum_begin = pnum; 518a4322e1bSYinghai Lu break; 519e123dd3fSYinghai Lu } 520a4322e1bSYinghai Lu usemap_count = 1; 521a4322e1bSYinghai Lu for (pnum = pnum_begin + 1; pnum < NR_MEM_SECTIONS; pnum++) { 522a4322e1bSYinghai Lu struct mem_section *ms; 523a4322e1bSYinghai Lu int nodeid; 524a4322e1bSYinghai Lu 525a4322e1bSYinghai Lu if (!present_section_nr(pnum)) 526a4322e1bSYinghai Lu continue; 527a4322e1bSYinghai Lu ms = __nr_to_section(pnum); 528a4322e1bSYinghai Lu nodeid = sparse_early_nid(ms); 529a4322e1bSYinghai Lu if (nodeid == nodeid_begin) { 530a4322e1bSYinghai Lu usemap_count++; 531a4322e1bSYinghai Lu continue; 532a4322e1bSYinghai Lu } 533a4322e1bSYinghai Lu /* ok, we need to take cake of from pnum_begin to pnum - 1*/ 534a4322e1bSYinghai Lu sparse_early_usemaps_alloc_node(usemap_map, pnum_begin, pnum, 535a4322e1bSYinghai Lu usemap_count, nodeid_begin); 536a4322e1bSYinghai Lu /* new start, update count etc*/ 537a4322e1bSYinghai Lu nodeid_begin = nodeid; 538a4322e1bSYinghai Lu pnum_begin = pnum; 539a4322e1bSYinghai Lu usemap_count = 1; 540a4322e1bSYinghai Lu } 541a4322e1bSYinghai Lu /* ok, last chunk */ 542a4322e1bSYinghai Lu sparse_early_usemaps_alloc_node(usemap_map, pnum_begin, NR_MEM_SECTIONS, 543a4322e1bSYinghai Lu usemap_count, nodeid_begin); 544e123dd3fSYinghai Lu 5459bdac914SYinghai Lu #ifdef CONFIG_SPARSEMEM_ALLOC_MEM_MAP_TOGETHER 5469bdac914SYinghai Lu size2 = sizeof(struct page *) * NR_MEM_SECTIONS; 5479bdac914SYinghai Lu map_map = alloc_bootmem(size2); 5489bdac914SYinghai Lu if (!map_map) 5499bdac914SYinghai Lu panic("can not allocate map_map\n"); 5509bdac914SYinghai Lu 5519bdac914SYinghai Lu for (pnum = 0; pnum < NR_MEM_SECTIONS; pnum++) { 5529bdac914SYinghai Lu struct mem_section *ms; 5539bdac914SYinghai Lu 5549bdac914SYinghai Lu if (!present_section_nr(pnum)) 5559bdac914SYinghai Lu continue; 5569bdac914SYinghai Lu ms = __nr_to_section(pnum); 5579bdac914SYinghai Lu nodeid_begin = sparse_early_nid(ms); 5589bdac914SYinghai Lu pnum_begin = pnum; 5599bdac914SYinghai Lu break; 5609bdac914SYinghai Lu } 5619bdac914SYinghai Lu map_count = 1; 5629bdac914SYinghai Lu for (pnum = pnum_begin + 1; pnum < NR_MEM_SECTIONS; pnum++) { 5639bdac914SYinghai Lu struct mem_section *ms; 5649bdac914SYinghai Lu int nodeid; 5659bdac914SYinghai Lu 5669bdac914SYinghai Lu if (!present_section_nr(pnum)) 5679bdac914SYinghai Lu continue; 5689bdac914SYinghai Lu ms = __nr_to_section(pnum); 5699bdac914SYinghai Lu nodeid = sparse_early_nid(ms); 5709bdac914SYinghai Lu if (nodeid == nodeid_begin) { 5719bdac914SYinghai Lu map_count++; 5729bdac914SYinghai Lu continue; 5739bdac914SYinghai Lu } 5749bdac914SYinghai Lu /* ok, we need to take cake of from pnum_begin to pnum - 1*/ 5759bdac914SYinghai Lu sparse_early_mem_maps_alloc_node(map_map, pnum_begin, pnum, 5769bdac914SYinghai Lu map_count, nodeid_begin); 5779bdac914SYinghai Lu /* new start, update count etc*/ 5789bdac914SYinghai Lu nodeid_begin = nodeid; 5799bdac914SYinghai Lu pnum_begin = pnum; 5809bdac914SYinghai Lu map_count = 1; 5819bdac914SYinghai Lu } 5829bdac914SYinghai Lu /* ok, last chunk */ 5839bdac914SYinghai Lu sparse_early_mem_maps_alloc_node(map_map, pnum_begin, NR_MEM_SECTIONS, 5849bdac914SYinghai Lu map_count, nodeid_begin); 5859bdac914SYinghai Lu #endif 5869bdac914SYinghai Lu 587e123dd3fSYinghai Lu for (pnum = 0; pnum < NR_MEM_SECTIONS; pnum++) { 588e123dd3fSYinghai Lu if (!present_section_nr(pnum)) 589e123dd3fSYinghai Lu continue; 590e123dd3fSYinghai Lu 591e123dd3fSYinghai Lu usemap = usemap_map[pnum]; 592e123dd3fSYinghai Lu if (!usemap) 593e123dd3fSYinghai Lu continue; 594193faea9SStephen Rothwell 5959bdac914SYinghai Lu #ifdef CONFIG_SPARSEMEM_ALLOC_MEM_MAP_TOGETHER 5969bdac914SYinghai Lu map = map_map[pnum]; 5979bdac914SYinghai Lu #else 598193faea9SStephen Rothwell map = sparse_early_mem_map_alloc(pnum); 5999bdac914SYinghai Lu #endif 600193faea9SStephen Rothwell if (!map) 601193faea9SStephen Rothwell continue; 6025c0e3066SMel Gorman 6035c0e3066SMel Gorman sparse_init_one_section(__nr_to_section(pnum), pnum, map, 6045c0e3066SMel Gorman usemap); 605193faea9SStephen Rothwell } 606e123dd3fSYinghai Lu 607c2b91e2eSYinghai Lu vmemmap_populate_print_last(); 608c2b91e2eSYinghai Lu 6099bdac914SYinghai Lu #ifdef CONFIG_SPARSEMEM_ALLOC_MEM_MAP_TOGETHER 6109bdac914SYinghai Lu free_bootmem(__pa(map_map), size2); 6119bdac914SYinghai Lu #endif 612e123dd3fSYinghai Lu free_bootmem(__pa(usemap_map), size); 613193faea9SStephen Rothwell } 614193faea9SStephen Rothwell 615193faea9SStephen Rothwell #ifdef CONFIG_MEMORY_HOTPLUG 61698f3cfc1SYasunori Goto #ifdef CONFIG_SPARSEMEM_VMEMMAP 61798f3cfc1SYasunori Goto static inline struct page *kmalloc_section_memmap(unsigned long pnum, int nid, 61898f3cfc1SYasunori Goto unsigned long nr_pages) 61998f3cfc1SYasunori Goto { 62098f3cfc1SYasunori Goto /* This will make the necessary allocations eventually. */ 62198f3cfc1SYasunori Goto return sparse_mem_map_populate(pnum, nid); 62298f3cfc1SYasunori Goto } 62398f3cfc1SYasunori Goto static void __kfree_section_memmap(struct page *memmap, unsigned long nr_pages) 62498f3cfc1SYasunori Goto { 62598f3cfc1SYasunori Goto return; /* XXX: Not implemented yet */ 62698f3cfc1SYasunori Goto } 6270c0a4a51SYasunori Goto static void free_map_bootmem(struct page *page, unsigned long nr_pages) 6280c0a4a51SYasunori Goto { 6290c0a4a51SYasunori Goto } 63098f3cfc1SYasunori Goto #else 6310b0acbecSDave Hansen static struct page *__kmalloc_section_memmap(unsigned long nr_pages) 6320b0acbecSDave Hansen { 6330b0acbecSDave Hansen struct page *page, *ret; 6340b0acbecSDave Hansen unsigned long memmap_size = sizeof(struct page) * nr_pages; 6350b0acbecSDave Hansen 636f2d0aa5bSYasunori Goto page = alloc_pages(GFP_KERNEL|__GFP_NOWARN, get_order(memmap_size)); 6370b0acbecSDave Hansen if (page) 6380b0acbecSDave Hansen goto got_map_page; 6390b0acbecSDave Hansen 6400b0acbecSDave Hansen ret = vmalloc(memmap_size); 6410b0acbecSDave Hansen if (ret) 6420b0acbecSDave Hansen goto got_map_ptr; 6430b0acbecSDave Hansen 6440b0acbecSDave Hansen return NULL; 6450b0acbecSDave Hansen got_map_page: 6460b0acbecSDave Hansen ret = (struct page *)pfn_to_kaddr(page_to_pfn(page)); 6470b0acbecSDave Hansen got_map_ptr: 6480b0acbecSDave Hansen memset(ret, 0, memmap_size); 6490b0acbecSDave Hansen 6500b0acbecSDave Hansen return ret; 6510b0acbecSDave Hansen } 6520b0acbecSDave Hansen 65398f3cfc1SYasunori Goto static inline struct page *kmalloc_section_memmap(unsigned long pnum, int nid, 65498f3cfc1SYasunori Goto unsigned long nr_pages) 65598f3cfc1SYasunori Goto { 65698f3cfc1SYasunori Goto return __kmalloc_section_memmap(nr_pages); 65798f3cfc1SYasunori Goto } 65898f3cfc1SYasunori Goto 6590b0acbecSDave Hansen static void __kfree_section_memmap(struct page *memmap, unsigned long nr_pages) 6600b0acbecSDave Hansen { 6619e2779faSChristoph Lameter if (is_vmalloc_addr(memmap)) 6620b0acbecSDave Hansen vfree(memmap); 6630b0acbecSDave Hansen else 6640b0acbecSDave Hansen free_pages((unsigned long)memmap, 6650b0acbecSDave Hansen get_order(sizeof(struct page) * nr_pages)); 6660b0acbecSDave Hansen } 6670c0a4a51SYasunori Goto 6680c0a4a51SYasunori Goto static void free_map_bootmem(struct page *page, unsigned long nr_pages) 6690c0a4a51SYasunori Goto { 6700c0a4a51SYasunori Goto unsigned long maps_section_nr, removing_section_nr, i; 6710c0a4a51SYasunori Goto int magic; 6720c0a4a51SYasunori Goto 6730c0a4a51SYasunori Goto for (i = 0; i < nr_pages; i++, page++) { 6740c0a4a51SYasunori Goto magic = atomic_read(&page->_mapcount); 6750c0a4a51SYasunori Goto 6760c0a4a51SYasunori Goto BUG_ON(magic == NODE_INFO); 6770c0a4a51SYasunori Goto 6780c0a4a51SYasunori Goto maps_section_nr = pfn_to_section_nr(page_to_pfn(page)); 6790c0a4a51SYasunori Goto removing_section_nr = page->private; 6800c0a4a51SYasunori Goto 6810c0a4a51SYasunori Goto /* 6820c0a4a51SYasunori Goto * When this function is called, the removing section is 6830c0a4a51SYasunori Goto * logical offlined state. This means all pages are isolated 6840c0a4a51SYasunori Goto * from page allocator. If removing section's memmap is placed 6850c0a4a51SYasunori Goto * on the same section, it must not be freed. 6860c0a4a51SYasunori Goto * If it is freed, page allocator may allocate it which will 6870c0a4a51SYasunori Goto * be removed physically soon. 6880c0a4a51SYasunori Goto */ 6890c0a4a51SYasunori Goto if (maps_section_nr != removing_section_nr) 6900c0a4a51SYasunori Goto put_page_bootmem(page); 6910c0a4a51SYasunori Goto } 6920c0a4a51SYasunori Goto } 69398f3cfc1SYasunori Goto #endif /* CONFIG_SPARSEMEM_VMEMMAP */ 6940b0acbecSDave Hansen 695ea01ea93SBadari Pulavarty static void free_section_usemap(struct page *memmap, unsigned long *usemap) 696ea01ea93SBadari Pulavarty { 6970c0a4a51SYasunori Goto struct page *usemap_page; 6980c0a4a51SYasunori Goto unsigned long nr_pages; 6990c0a4a51SYasunori Goto 700ea01ea93SBadari Pulavarty if (!usemap) 701ea01ea93SBadari Pulavarty return; 702ea01ea93SBadari Pulavarty 7030c0a4a51SYasunori Goto usemap_page = virt_to_page(usemap); 704ea01ea93SBadari Pulavarty /* 705ea01ea93SBadari Pulavarty * Check to see if allocation came from hot-plug-add 706ea01ea93SBadari Pulavarty */ 7070c0a4a51SYasunori Goto if (PageSlab(usemap_page)) { 708ea01ea93SBadari Pulavarty kfree(usemap); 709ea01ea93SBadari Pulavarty if (memmap) 710ea01ea93SBadari Pulavarty __kfree_section_memmap(memmap, PAGES_PER_SECTION); 711ea01ea93SBadari Pulavarty return; 712ea01ea93SBadari Pulavarty } 713ea01ea93SBadari Pulavarty 714ea01ea93SBadari Pulavarty /* 7150c0a4a51SYasunori Goto * The usemap came from bootmem. This is packed with other usemaps 7160c0a4a51SYasunori Goto * on the section which has pgdat at boot time. Just keep it as is now. 717ea01ea93SBadari Pulavarty */ 7180c0a4a51SYasunori Goto 7190c0a4a51SYasunori Goto if (memmap) { 7200c0a4a51SYasunori Goto struct page *memmap_page; 7210c0a4a51SYasunori Goto memmap_page = virt_to_page(memmap); 7220c0a4a51SYasunori Goto 7230c0a4a51SYasunori Goto nr_pages = PAGE_ALIGN(PAGES_PER_SECTION * sizeof(struct page)) 7240c0a4a51SYasunori Goto >> PAGE_SHIFT; 7250c0a4a51SYasunori Goto 7260c0a4a51SYasunori Goto free_map_bootmem(memmap_page, nr_pages); 7270c0a4a51SYasunori Goto } 728ea01ea93SBadari Pulavarty } 729ea01ea93SBadari Pulavarty 73029751f69SAndy Whitcroft /* 73129751f69SAndy Whitcroft * returns the number of sections whose mem_maps were properly 73229751f69SAndy Whitcroft * set. If this is <=0, then that means that the passed-in 73329751f69SAndy Whitcroft * map was not consumed and must be freed. 734d41dee36SAndy Whitcroft */ 73531168481SAl Viro int __meminit sparse_add_one_section(struct zone *zone, unsigned long start_pfn, 7360b0acbecSDave Hansen int nr_pages) 73729751f69SAndy Whitcroft { 7380b0acbecSDave Hansen unsigned long section_nr = pfn_to_section_nr(start_pfn); 7390b0acbecSDave Hansen struct pglist_data *pgdat = zone->zone_pgdat; 7400b0acbecSDave Hansen struct mem_section *ms; 7410b0acbecSDave Hansen struct page *memmap; 7425c0e3066SMel Gorman unsigned long *usemap; 7430b0acbecSDave Hansen unsigned long flags; 7440b0acbecSDave Hansen int ret; 74529751f69SAndy Whitcroft 7460b0acbecSDave Hansen /* 7470b0acbecSDave Hansen * no locking for this, because it does its own 7480b0acbecSDave Hansen * plus, it does a kmalloc 7490b0acbecSDave Hansen */ 750bbd06825SWANG Cong ret = sparse_index_init(section_nr, pgdat->node_id); 751bbd06825SWANG Cong if (ret < 0 && ret != -EEXIST) 752bbd06825SWANG Cong return ret; 75398f3cfc1SYasunori Goto memmap = kmalloc_section_memmap(section_nr, pgdat->node_id, nr_pages); 754bbd06825SWANG Cong if (!memmap) 755bbd06825SWANG Cong return -ENOMEM; 7565c0e3066SMel Gorman usemap = __kmalloc_section_usemap(); 757bbd06825SWANG Cong if (!usemap) { 758bbd06825SWANG Cong __kfree_section_memmap(memmap, nr_pages); 759bbd06825SWANG Cong return -ENOMEM; 760bbd06825SWANG Cong } 76129751f69SAndy Whitcroft 7620b0acbecSDave Hansen pgdat_resize_lock(pgdat, &flags); 7630b0acbecSDave Hansen 7640b0acbecSDave Hansen ms = __pfn_to_section(start_pfn); 7650b0acbecSDave Hansen if (ms->section_mem_map & SECTION_MARKED_PRESENT) { 7660b0acbecSDave Hansen ret = -EEXIST; 7670b0acbecSDave Hansen goto out; 7680b0acbecSDave Hansen } 7695c0e3066SMel Gorman 77029751f69SAndy Whitcroft ms->section_mem_map |= SECTION_MARKED_PRESENT; 77129751f69SAndy Whitcroft 7725c0e3066SMel Gorman ret = sparse_init_one_section(ms, section_nr, memmap, usemap); 7730b0acbecSDave Hansen 7740b0acbecSDave Hansen out: 7750b0acbecSDave Hansen pgdat_resize_unlock(pgdat, &flags); 776bbd06825SWANG Cong if (ret <= 0) { 777bbd06825SWANG Cong kfree(usemap); 77846a66eecSMike Kravetz __kfree_section_memmap(memmap, nr_pages); 779bbd06825SWANG Cong } 7800b0acbecSDave Hansen return ret; 781d41dee36SAndy Whitcroft } 782ea01ea93SBadari Pulavarty 783ea01ea93SBadari Pulavarty void sparse_remove_one_section(struct zone *zone, struct mem_section *ms) 784ea01ea93SBadari Pulavarty { 785ea01ea93SBadari Pulavarty struct page *memmap = NULL; 786ea01ea93SBadari Pulavarty unsigned long *usemap = NULL; 787ea01ea93SBadari Pulavarty 788ea01ea93SBadari Pulavarty if (ms->section_mem_map) { 789ea01ea93SBadari Pulavarty usemap = ms->pageblock_flags; 790ea01ea93SBadari Pulavarty memmap = sparse_decode_mem_map(ms->section_mem_map, 791ea01ea93SBadari Pulavarty __section_nr(ms)); 792ea01ea93SBadari Pulavarty ms->section_mem_map = 0; 793ea01ea93SBadari Pulavarty ms->pageblock_flags = NULL; 794ea01ea93SBadari Pulavarty } 795ea01ea93SBadari Pulavarty 796ea01ea93SBadari Pulavarty free_section_usemap(memmap, usemap); 797ea01ea93SBadari Pulavarty } 798a3142c8eSYasunori Goto #endif 799