1 /* 2 * sparse memory mappings. 3 */ 4 #include <linux/mm.h> 5 #include <linux/slab.h> 6 #include <linux/mmzone.h> 7 #include <linux/bootmem.h> 8 #include <linux/highmem.h> 9 #include <linux/export.h> 10 #include <linux/spinlock.h> 11 #include <linux/vmalloc.h> 12 #include "internal.h" 13 #include <asm/dma.h> 14 #include <asm/pgalloc.h> 15 #include <asm/pgtable.h> 16 17 /* 18 * Permanent SPARSEMEM data: 19 * 20 * 1) mem_section - memory sections, mem_map's for valid memory 21 */ 22 #ifdef CONFIG_SPARSEMEM_EXTREME 23 struct mem_section *mem_section[NR_SECTION_ROOTS] 24 ____cacheline_internodealigned_in_smp; 25 #else 26 struct mem_section mem_section[NR_SECTION_ROOTS][SECTIONS_PER_ROOT] 27 ____cacheline_internodealigned_in_smp; 28 #endif 29 EXPORT_SYMBOL(mem_section); 30 31 #ifdef NODE_NOT_IN_PAGE_FLAGS 32 /* 33 * If we did not store the node number in the page then we have to 34 * do a lookup in the section_to_node_table in order to find which 35 * node the page belongs to. 36 */ 37 #if MAX_NUMNODES <= 256 38 static u8 section_to_node_table[NR_MEM_SECTIONS] __cacheline_aligned; 39 #else 40 static u16 section_to_node_table[NR_MEM_SECTIONS] __cacheline_aligned; 41 #endif 42 43 int page_to_nid(const struct page *page) 44 { 45 return section_to_node_table[page_to_section(page)]; 46 } 47 EXPORT_SYMBOL(page_to_nid); 48 49 static void set_section_nid(unsigned long section_nr, int nid) 50 { 51 section_to_node_table[section_nr] = nid; 52 } 53 #else /* !NODE_NOT_IN_PAGE_FLAGS */ 54 static inline void set_section_nid(unsigned long section_nr, int nid) 55 { 56 } 57 #endif 58 59 #ifdef CONFIG_SPARSEMEM_EXTREME 60 static struct mem_section noinline __init_refok *sparse_index_alloc(int nid) 61 { 62 struct mem_section *section = NULL; 63 unsigned long array_size = SECTIONS_PER_ROOT * 64 sizeof(struct mem_section); 65 66 if (slab_is_available()) { 67 if (node_state(nid, N_HIGH_MEMORY)) 68 section = kmalloc_node(array_size, GFP_KERNEL, nid); 69 else 70 section = kmalloc(array_size, GFP_KERNEL); 71 } else 72 section = alloc_bootmem_node(NODE_DATA(nid), array_size); 73 74 if (section) 75 memset(section, 0, array_size); 76 77 return section; 78 } 79 80 static int __meminit sparse_index_init(unsigned long section_nr, int nid) 81 { 82 static DEFINE_SPINLOCK(index_init_lock); 83 unsigned long root = SECTION_NR_TO_ROOT(section_nr); 84 struct mem_section *section; 85 int ret = 0; 86 87 if (mem_section[root]) 88 return -EEXIST; 89 90 section = sparse_index_alloc(nid); 91 if (!section) 92 return -ENOMEM; 93 /* 94 * This lock keeps two different sections from 95 * reallocating for the same index 96 */ 97 spin_lock(&index_init_lock); 98 99 if (mem_section[root]) { 100 ret = -EEXIST; 101 goto out; 102 } 103 104 mem_section[root] = section; 105 out: 106 spin_unlock(&index_init_lock); 107 return ret; 108 } 109 #else /* !SPARSEMEM_EXTREME */ 110 static inline int sparse_index_init(unsigned long section_nr, int nid) 111 { 112 return 0; 113 } 114 #endif 115 116 /* 117 * Although written for the SPARSEMEM_EXTREME case, this happens 118 * to also work for the flat array case because 119 * NR_SECTION_ROOTS==NR_MEM_SECTIONS. 120 */ 121 int __section_nr(struct mem_section* ms) 122 { 123 unsigned long root_nr; 124 struct mem_section* root; 125 126 for (root_nr = 0; root_nr < NR_SECTION_ROOTS; root_nr++) { 127 root = __nr_to_section(root_nr * SECTIONS_PER_ROOT); 128 if (!root) 129 continue; 130 131 if ((ms >= root) && (ms < (root + SECTIONS_PER_ROOT))) 132 break; 133 } 134 135 return (root_nr * SECTIONS_PER_ROOT) + (ms - root); 136 } 137 138 /* 139 * During early boot, before section_mem_map is used for an actual 140 * mem_map, we use section_mem_map to store the section's NUMA 141 * node. This keeps us from having to use another data structure. The 142 * node information is cleared just before we store the real mem_map. 143 */ 144 static inline unsigned long sparse_encode_early_nid(int nid) 145 { 146 return (nid << SECTION_NID_SHIFT); 147 } 148 149 static inline int sparse_early_nid(struct mem_section *section) 150 { 151 return (section->section_mem_map >> SECTION_NID_SHIFT); 152 } 153 154 /* Validate the physical addressing limitations of the model */ 155 void __meminit mminit_validate_memmodel_limits(unsigned long *start_pfn, 156 unsigned long *end_pfn) 157 { 158 unsigned long max_sparsemem_pfn = 1UL << (MAX_PHYSMEM_BITS-PAGE_SHIFT); 159 160 /* 161 * Sanity checks - do not allow an architecture to pass 162 * in larger pfns than the maximum scope of sparsemem: 163 */ 164 if (*start_pfn > max_sparsemem_pfn) { 165 mminit_dprintk(MMINIT_WARNING, "pfnvalidation", 166 "Start of range %lu -> %lu exceeds SPARSEMEM max %lu\n", 167 *start_pfn, *end_pfn, max_sparsemem_pfn); 168 WARN_ON_ONCE(1); 169 *start_pfn = max_sparsemem_pfn; 170 *end_pfn = max_sparsemem_pfn; 171 } else if (*end_pfn > max_sparsemem_pfn) { 172 mminit_dprintk(MMINIT_WARNING, "pfnvalidation", 173 "End of range %lu -> %lu exceeds SPARSEMEM max %lu\n", 174 *start_pfn, *end_pfn, max_sparsemem_pfn); 175 WARN_ON_ONCE(1); 176 *end_pfn = max_sparsemem_pfn; 177 } 178 } 179 180 /* Record a memory area against a node. */ 181 void __init memory_present(int nid, unsigned long start, unsigned long end) 182 { 183 unsigned long pfn; 184 185 start &= PAGE_SECTION_MASK; 186 mminit_validate_memmodel_limits(&start, &end); 187 for (pfn = start; pfn < end; pfn += PAGES_PER_SECTION) { 188 unsigned long section = pfn_to_section_nr(pfn); 189 struct mem_section *ms; 190 191 sparse_index_init(section, nid); 192 set_section_nid(section, nid); 193 194 ms = __nr_to_section(section); 195 if (!ms->section_mem_map) 196 ms->section_mem_map = sparse_encode_early_nid(nid) | 197 SECTION_MARKED_PRESENT; 198 } 199 } 200 201 /* 202 * Only used by the i386 NUMA architecures, but relatively 203 * generic code. 204 */ 205 unsigned long __init node_memmap_size_bytes(int nid, unsigned long start_pfn, 206 unsigned long end_pfn) 207 { 208 unsigned long pfn; 209 unsigned long nr_pages = 0; 210 211 mminit_validate_memmodel_limits(&start_pfn, &end_pfn); 212 for (pfn = start_pfn; pfn < end_pfn; pfn += PAGES_PER_SECTION) { 213 if (nid != early_pfn_to_nid(pfn)) 214 continue; 215 216 if (pfn_present(pfn)) 217 nr_pages += PAGES_PER_SECTION; 218 } 219 220 return nr_pages * sizeof(struct page); 221 } 222 223 /* 224 * Subtle, we encode the real pfn into the mem_map such that 225 * the identity pfn - section_mem_map will return the actual 226 * physical page frame number. 227 */ 228 static unsigned long sparse_encode_mem_map(struct page *mem_map, unsigned long pnum) 229 { 230 return (unsigned long)(mem_map - (section_nr_to_pfn(pnum))); 231 } 232 233 /* 234 * Decode mem_map from the coded memmap 235 */ 236 struct page *sparse_decode_mem_map(unsigned long coded_mem_map, unsigned long pnum) 237 { 238 /* mask off the extra low bits of information */ 239 coded_mem_map &= SECTION_MAP_MASK; 240 return ((struct page *)coded_mem_map) + section_nr_to_pfn(pnum); 241 } 242 243 static int __meminit sparse_init_one_section(struct mem_section *ms, 244 unsigned long pnum, struct page *mem_map, 245 unsigned long *pageblock_bitmap) 246 { 247 if (!present_section(ms)) 248 return -EINVAL; 249 250 ms->section_mem_map &= ~SECTION_MAP_MASK; 251 ms->section_mem_map |= sparse_encode_mem_map(mem_map, pnum) | 252 SECTION_HAS_MEM_MAP; 253 ms->pageblock_flags = pageblock_bitmap; 254 255 return 1; 256 } 257 258 unsigned long usemap_size(void) 259 { 260 unsigned long size_bytes; 261 size_bytes = roundup(SECTION_BLOCKFLAGS_BITS, 8) / 8; 262 size_bytes = roundup(size_bytes, sizeof(unsigned long)); 263 return size_bytes; 264 } 265 266 #ifdef CONFIG_MEMORY_HOTPLUG 267 static unsigned long *__kmalloc_section_usemap(void) 268 { 269 return kmalloc(usemap_size(), GFP_KERNEL); 270 } 271 #endif /* CONFIG_MEMORY_HOTPLUG */ 272 273 #ifdef CONFIG_MEMORY_HOTREMOVE 274 static unsigned long * __init 275 sparse_early_usemaps_alloc_pgdat_section(struct pglist_data *pgdat, 276 unsigned long size) 277 { 278 pg_data_t *host_pgdat; 279 unsigned long goal; 280 /* 281 * A page may contain usemaps for other sections preventing the 282 * page being freed and making a section unremovable while 283 * other sections referencing the usemap retmain active. Similarly, 284 * a pgdat can prevent a section being removed. If section A 285 * contains a pgdat and section B contains the usemap, both 286 * sections become inter-dependent. This allocates usemaps 287 * from the same section as the pgdat where possible to avoid 288 * this problem. 289 */ 290 goal = __pa(pgdat) & PAGE_SECTION_MASK; 291 host_pgdat = NODE_DATA(early_pfn_to_nid(goal >> PAGE_SHIFT)); 292 return __alloc_bootmem_node_nopanic(host_pgdat, size, 293 SMP_CACHE_BYTES, goal); 294 } 295 296 static void __init check_usemap_section_nr(int nid, unsigned long *usemap) 297 { 298 unsigned long usemap_snr, pgdat_snr; 299 static unsigned long old_usemap_snr = NR_MEM_SECTIONS; 300 static unsigned long old_pgdat_snr = NR_MEM_SECTIONS; 301 struct pglist_data *pgdat = NODE_DATA(nid); 302 int usemap_nid; 303 304 usemap_snr = pfn_to_section_nr(__pa(usemap) >> PAGE_SHIFT); 305 pgdat_snr = pfn_to_section_nr(__pa(pgdat) >> PAGE_SHIFT); 306 if (usemap_snr == pgdat_snr) 307 return; 308 309 if (old_usemap_snr == usemap_snr && old_pgdat_snr == pgdat_snr) 310 /* skip redundant message */ 311 return; 312 313 old_usemap_snr = usemap_snr; 314 old_pgdat_snr = pgdat_snr; 315 316 usemap_nid = sparse_early_nid(__nr_to_section(usemap_snr)); 317 if (usemap_nid != nid) { 318 printk(KERN_INFO 319 "node %d must be removed before remove section %ld\n", 320 nid, usemap_snr); 321 return; 322 } 323 /* 324 * There is a circular dependency. 325 * Some platforms allow un-removable section because they will just 326 * gather other removable sections for dynamic partitioning. 327 * Just notify un-removable section's number here. 328 */ 329 printk(KERN_INFO "Section %ld and %ld (node %d)", usemap_snr, 330 pgdat_snr, nid); 331 printk(KERN_CONT 332 " have a circular dependency on usemap and pgdat allocations\n"); 333 } 334 #else 335 static unsigned long * __init 336 sparse_early_usemaps_alloc_pgdat_section(struct pglist_data *pgdat, 337 unsigned long size) 338 { 339 return alloc_bootmem_node_nopanic(pgdat, size); 340 } 341 342 static void __init check_usemap_section_nr(int nid, unsigned long *usemap) 343 { 344 } 345 #endif /* CONFIG_MEMORY_HOTREMOVE */ 346 347 static void __init sparse_early_usemaps_alloc_node(unsigned long**usemap_map, 348 unsigned long pnum_begin, 349 unsigned long pnum_end, 350 unsigned long usemap_count, int nodeid) 351 { 352 void *usemap; 353 unsigned long pnum; 354 int size = usemap_size(); 355 356 usemap = sparse_early_usemaps_alloc_pgdat_section(NODE_DATA(nodeid), 357 size * usemap_count); 358 if (!usemap) { 359 printk(KERN_WARNING "%s: allocation failed\n", __func__); 360 return; 361 } 362 363 for (pnum = pnum_begin; pnum < pnum_end; pnum++) { 364 if (!present_section_nr(pnum)) 365 continue; 366 usemap_map[pnum] = usemap; 367 usemap += size; 368 check_usemap_section_nr(nodeid, usemap_map[pnum]); 369 } 370 } 371 372 #ifndef CONFIG_SPARSEMEM_VMEMMAP 373 struct page __init *sparse_mem_map_populate(unsigned long pnum, int nid) 374 { 375 struct page *map; 376 unsigned long size; 377 378 map = alloc_remap(nid, sizeof(struct page) * PAGES_PER_SECTION); 379 if (map) 380 return map; 381 382 size = PAGE_ALIGN(sizeof(struct page) * PAGES_PER_SECTION); 383 map = __alloc_bootmem_node_high(NODE_DATA(nid), size, 384 PAGE_SIZE, __pa(MAX_DMA_ADDRESS)); 385 return map; 386 } 387 void __init sparse_mem_maps_populate_node(struct page **map_map, 388 unsigned long pnum_begin, 389 unsigned long pnum_end, 390 unsigned long map_count, int nodeid) 391 { 392 void *map; 393 unsigned long pnum; 394 unsigned long size = sizeof(struct page) * PAGES_PER_SECTION; 395 396 map = alloc_remap(nodeid, size * map_count); 397 if (map) { 398 for (pnum = pnum_begin; pnum < pnum_end; pnum++) { 399 if (!present_section_nr(pnum)) 400 continue; 401 map_map[pnum] = map; 402 map += size; 403 } 404 return; 405 } 406 407 size = PAGE_ALIGN(size); 408 map = __alloc_bootmem_node_high(NODE_DATA(nodeid), size * map_count, 409 PAGE_SIZE, __pa(MAX_DMA_ADDRESS)); 410 if (map) { 411 for (pnum = pnum_begin; pnum < pnum_end; pnum++) { 412 if (!present_section_nr(pnum)) 413 continue; 414 map_map[pnum] = map; 415 map += size; 416 } 417 return; 418 } 419 420 /* fallback */ 421 for (pnum = pnum_begin; pnum < pnum_end; pnum++) { 422 struct mem_section *ms; 423 424 if (!present_section_nr(pnum)) 425 continue; 426 map_map[pnum] = sparse_mem_map_populate(pnum, nodeid); 427 if (map_map[pnum]) 428 continue; 429 ms = __nr_to_section(pnum); 430 printk(KERN_ERR "%s: sparsemem memory map backing failed " 431 "some memory will not be available.\n", __func__); 432 ms->section_mem_map = 0; 433 } 434 } 435 #endif /* !CONFIG_SPARSEMEM_VMEMMAP */ 436 437 #ifdef CONFIG_SPARSEMEM_ALLOC_MEM_MAP_TOGETHER 438 static void __init sparse_early_mem_maps_alloc_node(struct page **map_map, 439 unsigned long pnum_begin, 440 unsigned long pnum_end, 441 unsigned long map_count, int nodeid) 442 { 443 sparse_mem_maps_populate_node(map_map, pnum_begin, pnum_end, 444 map_count, nodeid); 445 } 446 #else 447 static struct page __init *sparse_early_mem_map_alloc(unsigned long pnum) 448 { 449 struct page *map; 450 struct mem_section *ms = __nr_to_section(pnum); 451 int nid = sparse_early_nid(ms); 452 453 map = sparse_mem_map_populate(pnum, nid); 454 if (map) 455 return map; 456 457 printk(KERN_ERR "%s: sparsemem memory map backing failed " 458 "some memory will not be available.\n", __func__); 459 ms->section_mem_map = 0; 460 return NULL; 461 } 462 #endif 463 464 void __attribute__((weak)) __meminit vmemmap_populate_print_last(void) 465 { 466 } 467 468 /* 469 * Allocate the accumulated non-linear sections, allocate a mem_map 470 * for each and record the physical to section mapping. 471 */ 472 void __init sparse_init(void) 473 { 474 unsigned long pnum; 475 struct page *map; 476 unsigned long *usemap; 477 unsigned long **usemap_map; 478 int size; 479 int nodeid_begin = 0; 480 unsigned long pnum_begin = 0; 481 unsigned long usemap_count; 482 #ifdef CONFIG_SPARSEMEM_ALLOC_MEM_MAP_TOGETHER 483 unsigned long map_count; 484 int size2; 485 struct page **map_map; 486 #endif 487 488 /* 489 * map is using big page (aka 2M in x86 64 bit) 490 * usemap is less one page (aka 24 bytes) 491 * so alloc 2M (with 2M align) and 24 bytes in turn will 492 * make next 2M slip to one more 2M later. 493 * then in big system, the memory will have a lot of holes... 494 * here try to allocate 2M pages continuously. 495 * 496 * powerpc need to call sparse_init_one_section right after each 497 * sparse_early_mem_map_alloc, so allocate usemap_map at first. 498 */ 499 size = sizeof(unsigned long *) * NR_MEM_SECTIONS; 500 usemap_map = alloc_bootmem(size); 501 if (!usemap_map) 502 panic("can not allocate usemap_map\n"); 503 504 for (pnum = 0; pnum < NR_MEM_SECTIONS; pnum++) { 505 struct mem_section *ms; 506 507 if (!present_section_nr(pnum)) 508 continue; 509 ms = __nr_to_section(pnum); 510 nodeid_begin = sparse_early_nid(ms); 511 pnum_begin = pnum; 512 break; 513 } 514 usemap_count = 1; 515 for (pnum = pnum_begin + 1; pnum < NR_MEM_SECTIONS; pnum++) { 516 struct mem_section *ms; 517 int nodeid; 518 519 if (!present_section_nr(pnum)) 520 continue; 521 ms = __nr_to_section(pnum); 522 nodeid = sparse_early_nid(ms); 523 if (nodeid == nodeid_begin) { 524 usemap_count++; 525 continue; 526 } 527 /* ok, we need to take cake of from pnum_begin to pnum - 1*/ 528 sparse_early_usemaps_alloc_node(usemap_map, pnum_begin, pnum, 529 usemap_count, nodeid_begin); 530 /* new start, update count etc*/ 531 nodeid_begin = nodeid; 532 pnum_begin = pnum; 533 usemap_count = 1; 534 } 535 /* ok, last chunk */ 536 sparse_early_usemaps_alloc_node(usemap_map, pnum_begin, NR_MEM_SECTIONS, 537 usemap_count, nodeid_begin); 538 539 #ifdef CONFIG_SPARSEMEM_ALLOC_MEM_MAP_TOGETHER 540 size2 = sizeof(struct page *) * NR_MEM_SECTIONS; 541 map_map = alloc_bootmem(size2); 542 if (!map_map) 543 panic("can not allocate map_map\n"); 544 545 for (pnum = 0; pnum < NR_MEM_SECTIONS; pnum++) { 546 struct mem_section *ms; 547 548 if (!present_section_nr(pnum)) 549 continue; 550 ms = __nr_to_section(pnum); 551 nodeid_begin = sparse_early_nid(ms); 552 pnum_begin = pnum; 553 break; 554 } 555 map_count = 1; 556 for (pnum = pnum_begin + 1; pnum < NR_MEM_SECTIONS; pnum++) { 557 struct mem_section *ms; 558 int nodeid; 559 560 if (!present_section_nr(pnum)) 561 continue; 562 ms = __nr_to_section(pnum); 563 nodeid = sparse_early_nid(ms); 564 if (nodeid == nodeid_begin) { 565 map_count++; 566 continue; 567 } 568 /* ok, we need to take cake of from pnum_begin to pnum - 1*/ 569 sparse_early_mem_maps_alloc_node(map_map, pnum_begin, pnum, 570 map_count, nodeid_begin); 571 /* new start, update count etc*/ 572 nodeid_begin = nodeid; 573 pnum_begin = pnum; 574 map_count = 1; 575 } 576 /* ok, last chunk */ 577 sparse_early_mem_maps_alloc_node(map_map, pnum_begin, NR_MEM_SECTIONS, 578 map_count, nodeid_begin); 579 #endif 580 581 for (pnum = 0; pnum < NR_MEM_SECTIONS; pnum++) { 582 if (!present_section_nr(pnum)) 583 continue; 584 585 usemap = usemap_map[pnum]; 586 if (!usemap) 587 continue; 588 589 #ifdef CONFIG_SPARSEMEM_ALLOC_MEM_MAP_TOGETHER 590 map = map_map[pnum]; 591 #else 592 map = sparse_early_mem_map_alloc(pnum); 593 #endif 594 if (!map) 595 continue; 596 597 sparse_init_one_section(__nr_to_section(pnum), pnum, map, 598 usemap); 599 } 600 601 vmemmap_populate_print_last(); 602 603 #ifdef CONFIG_SPARSEMEM_ALLOC_MEM_MAP_TOGETHER 604 free_bootmem(__pa(map_map), size2); 605 #endif 606 free_bootmem(__pa(usemap_map), size); 607 } 608 609 #ifdef CONFIG_MEMORY_HOTPLUG 610 #ifdef CONFIG_SPARSEMEM_VMEMMAP 611 static inline struct page *kmalloc_section_memmap(unsigned long pnum, int nid, 612 unsigned long nr_pages) 613 { 614 /* This will make the necessary allocations eventually. */ 615 return sparse_mem_map_populate(pnum, nid); 616 } 617 static void __kfree_section_memmap(struct page *memmap, unsigned long nr_pages) 618 { 619 return; /* XXX: Not implemented yet */ 620 } 621 static void free_map_bootmem(struct page *page, unsigned long nr_pages) 622 { 623 } 624 #else 625 static struct page *__kmalloc_section_memmap(unsigned long nr_pages) 626 { 627 struct page *page, *ret; 628 unsigned long memmap_size = sizeof(struct page) * nr_pages; 629 630 page = alloc_pages(GFP_KERNEL|__GFP_NOWARN, get_order(memmap_size)); 631 if (page) 632 goto got_map_page; 633 634 ret = vmalloc(memmap_size); 635 if (ret) 636 goto got_map_ptr; 637 638 return NULL; 639 got_map_page: 640 ret = (struct page *)pfn_to_kaddr(page_to_pfn(page)); 641 got_map_ptr: 642 memset(ret, 0, memmap_size); 643 644 return ret; 645 } 646 647 static inline struct page *kmalloc_section_memmap(unsigned long pnum, int nid, 648 unsigned long nr_pages) 649 { 650 return __kmalloc_section_memmap(nr_pages); 651 } 652 653 static void __kfree_section_memmap(struct page *memmap, unsigned long nr_pages) 654 { 655 if (is_vmalloc_addr(memmap)) 656 vfree(memmap); 657 else 658 free_pages((unsigned long)memmap, 659 get_order(sizeof(struct page) * nr_pages)); 660 } 661 662 static void free_map_bootmem(struct page *page, unsigned long nr_pages) 663 { 664 unsigned long maps_section_nr, removing_section_nr, i; 665 unsigned long magic; 666 667 for (i = 0; i < nr_pages; i++, page++) { 668 magic = (unsigned long) page->lru.next; 669 670 BUG_ON(magic == NODE_INFO); 671 672 maps_section_nr = pfn_to_section_nr(page_to_pfn(page)); 673 removing_section_nr = page->private; 674 675 /* 676 * When this function is called, the removing section is 677 * logical offlined state. This means all pages are isolated 678 * from page allocator. If removing section's memmap is placed 679 * on the same section, it must not be freed. 680 * If it is freed, page allocator may allocate it which will 681 * be removed physically soon. 682 */ 683 if (maps_section_nr != removing_section_nr) 684 put_page_bootmem(page); 685 } 686 } 687 #endif /* CONFIG_SPARSEMEM_VMEMMAP */ 688 689 static void free_section_usemap(struct page *memmap, unsigned long *usemap) 690 { 691 struct page *usemap_page; 692 unsigned long nr_pages; 693 694 if (!usemap) 695 return; 696 697 usemap_page = virt_to_page(usemap); 698 /* 699 * Check to see if allocation came from hot-plug-add 700 */ 701 if (PageSlab(usemap_page)) { 702 kfree(usemap); 703 if (memmap) 704 __kfree_section_memmap(memmap, PAGES_PER_SECTION); 705 return; 706 } 707 708 /* 709 * The usemap came from bootmem. This is packed with other usemaps 710 * on the section which has pgdat at boot time. Just keep it as is now. 711 */ 712 713 if (memmap) { 714 struct page *memmap_page; 715 memmap_page = virt_to_page(memmap); 716 717 nr_pages = PAGE_ALIGN(PAGES_PER_SECTION * sizeof(struct page)) 718 >> PAGE_SHIFT; 719 720 free_map_bootmem(memmap_page, nr_pages); 721 } 722 } 723 724 /* 725 * returns the number of sections whose mem_maps were properly 726 * set. If this is <=0, then that means that the passed-in 727 * map was not consumed and must be freed. 728 */ 729 int __meminit sparse_add_one_section(struct zone *zone, unsigned long start_pfn, 730 int nr_pages) 731 { 732 unsigned long section_nr = pfn_to_section_nr(start_pfn); 733 struct pglist_data *pgdat = zone->zone_pgdat; 734 struct mem_section *ms; 735 struct page *memmap; 736 unsigned long *usemap; 737 unsigned long flags; 738 int ret; 739 740 /* 741 * no locking for this, because it does its own 742 * plus, it does a kmalloc 743 */ 744 ret = sparse_index_init(section_nr, pgdat->node_id); 745 if (ret < 0 && ret != -EEXIST) 746 return ret; 747 memmap = kmalloc_section_memmap(section_nr, pgdat->node_id, nr_pages); 748 if (!memmap) 749 return -ENOMEM; 750 usemap = __kmalloc_section_usemap(); 751 if (!usemap) { 752 __kfree_section_memmap(memmap, nr_pages); 753 return -ENOMEM; 754 } 755 756 pgdat_resize_lock(pgdat, &flags); 757 758 ms = __pfn_to_section(start_pfn); 759 if (ms->section_mem_map & SECTION_MARKED_PRESENT) { 760 ret = -EEXIST; 761 goto out; 762 } 763 764 ms->section_mem_map |= SECTION_MARKED_PRESENT; 765 766 ret = sparse_init_one_section(ms, section_nr, memmap, usemap); 767 768 out: 769 pgdat_resize_unlock(pgdat, &flags); 770 if (ret <= 0) { 771 kfree(usemap); 772 __kfree_section_memmap(memmap, nr_pages); 773 } 774 return ret; 775 } 776 777 void sparse_remove_one_section(struct zone *zone, struct mem_section *ms) 778 { 779 struct page *memmap = NULL; 780 unsigned long *usemap = NULL; 781 782 if (ms->section_mem_map) { 783 usemap = ms->pageblock_flags; 784 memmap = sparse_decode_mem_map(ms->section_mem_map, 785 __section_nr(ms)); 786 ms->section_mem_map = 0; 787 ms->pageblock_flags = NULL; 788 } 789 790 free_section_usemap(memmap, usemap); 791 } 792 #endif 793