1 /* Common code for 32 and 64-bit NUMA */ 2 #include <linux/kernel.h> 3 #include <linux/mm.h> 4 #include <linux/string.h> 5 #include <linux/init.h> 6 #include <linux/bootmem.h> 7 #include <linux/memblock.h> 8 #include <linux/mmzone.h> 9 #include <linux/ctype.h> 10 #include <linux/module.h> 11 #include <linux/nodemask.h> 12 #include <linux/sched.h> 13 #include <linux/topology.h> 14 15 #include <asm/e820.h> 16 #include <asm/proto.h> 17 #include <asm/dma.h> 18 #include <asm/acpi.h> 19 #include <asm/amd_nb.h> 20 21 #include "numa_internal.h" 22 23 int __initdata numa_off; 24 nodemask_t numa_nodes_parsed __initdata; 25 26 struct pglist_data *node_data[MAX_NUMNODES] __read_mostly; 27 EXPORT_SYMBOL(node_data); 28 29 static struct numa_meminfo numa_meminfo 30 #ifndef CONFIG_MEMORY_HOTPLUG 31 __initdata 32 #endif 33 ; 34 35 static int numa_distance_cnt; 36 static u8 *numa_distance; 37 38 static __init int numa_setup(char *opt) 39 { 40 if (!opt) 41 return -EINVAL; 42 if (!strncmp(opt, "off", 3)) 43 numa_off = 1; 44 #ifdef CONFIG_NUMA_EMU 45 if (!strncmp(opt, "fake=", 5)) 46 numa_emu_cmdline(opt + 5); 47 #endif 48 #ifdef CONFIG_ACPI_NUMA 49 if (!strncmp(opt, "noacpi", 6)) 50 acpi_numa = -1; 51 #endif 52 return 0; 53 } 54 early_param("numa", numa_setup); 55 56 /* 57 * apicid, cpu, node mappings 58 */ 59 s16 __apicid_to_node[MAX_LOCAL_APIC] = { 60 [0 ... MAX_LOCAL_APIC-1] = NUMA_NO_NODE 61 }; 62 63 int numa_cpu_node(int cpu) 64 { 65 int apicid = early_per_cpu(x86_cpu_to_apicid, cpu); 66 67 if (apicid != BAD_APICID) 68 return __apicid_to_node[apicid]; 69 return NUMA_NO_NODE; 70 } 71 72 cpumask_var_t node_to_cpumask_map[MAX_NUMNODES]; 73 EXPORT_SYMBOL(node_to_cpumask_map); 74 75 /* 76 * Map cpu index to node index 77 */ 78 DEFINE_EARLY_PER_CPU(int, x86_cpu_to_node_map, NUMA_NO_NODE); 79 EXPORT_EARLY_PER_CPU_SYMBOL(x86_cpu_to_node_map); 80 81 void numa_set_node(int cpu, int node) 82 { 83 int *cpu_to_node_map = early_per_cpu_ptr(x86_cpu_to_node_map); 84 85 /* early setting, no percpu area yet */ 86 if (cpu_to_node_map) { 87 cpu_to_node_map[cpu] = node; 88 return; 89 } 90 91 #ifdef CONFIG_DEBUG_PER_CPU_MAPS 92 if (cpu >= nr_cpu_ids || !cpu_possible(cpu)) { 93 printk(KERN_ERR "numa_set_node: invalid cpu# (%d)\n", cpu); 94 dump_stack(); 95 return; 96 } 97 #endif 98 per_cpu(x86_cpu_to_node_map, cpu) = node; 99 100 set_cpu_numa_node(cpu, node); 101 } 102 103 void numa_clear_node(int cpu) 104 { 105 numa_set_node(cpu, NUMA_NO_NODE); 106 } 107 108 /* 109 * Allocate node_to_cpumask_map based on number of available nodes 110 * Requires node_possible_map to be valid. 111 * 112 * Note: cpumask_of_node() is not valid until after this is done. 113 * (Use CONFIG_DEBUG_PER_CPU_MAPS to check this.) 114 */ 115 void __init setup_node_to_cpumask_map(void) 116 { 117 unsigned int node; 118 119 /* setup nr_node_ids if not done yet */ 120 if (nr_node_ids == MAX_NUMNODES) 121 setup_nr_node_ids(); 122 123 /* allocate the map */ 124 for (node = 0; node < nr_node_ids; node++) 125 alloc_bootmem_cpumask_var(&node_to_cpumask_map[node]); 126 127 /* cpumask_of_node() will now work */ 128 pr_debug("Node to cpumask map for %d nodes\n", nr_node_ids); 129 } 130 131 static int __init numa_add_memblk_to(int nid, u64 start, u64 end, 132 struct numa_meminfo *mi) 133 { 134 /* ignore zero length blks */ 135 if (start == end) 136 return 0; 137 138 /* whine about and ignore invalid blks */ 139 if (start > end || nid < 0 || nid >= MAX_NUMNODES) { 140 pr_warning("NUMA: Warning: invalid memblk node %d [mem %#010Lx-%#010Lx]\n", 141 nid, start, end - 1); 142 return 0; 143 } 144 145 if (mi->nr_blks >= NR_NODE_MEMBLKS) { 146 pr_err("NUMA: too many memblk ranges\n"); 147 return -EINVAL; 148 } 149 150 mi->blk[mi->nr_blks].start = start; 151 mi->blk[mi->nr_blks].end = end; 152 mi->blk[mi->nr_blks].nid = nid; 153 mi->nr_blks++; 154 return 0; 155 } 156 157 /** 158 * numa_remove_memblk_from - Remove one numa_memblk from a numa_meminfo 159 * @idx: Index of memblk to remove 160 * @mi: numa_meminfo to remove memblk from 161 * 162 * Remove @idx'th numa_memblk from @mi by shifting @mi->blk[] and 163 * decrementing @mi->nr_blks. 164 */ 165 void __init numa_remove_memblk_from(int idx, struct numa_meminfo *mi) 166 { 167 mi->nr_blks--; 168 memmove(&mi->blk[idx], &mi->blk[idx + 1], 169 (mi->nr_blks - idx) * sizeof(mi->blk[0])); 170 } 171 172 /** 173 * numa_add_memblk - Add one numa_memblk to numa_meminfo 174 * @nid: NUMA node ID of the new memblk 175 * @start: Start address of the new memblk 176 * @end: End address of the new memblk 177 * 178 * Add a new memblk to the default numa_meminfo. 179 * 180 * RETURNS: 181 * 0 on success, -errno on failure. 182 */ 183 int __init numa_add_memblk(int nid, u64 start, u64 end) 184 { 185 return numa_add_memblk_to(nid, start, end, &numa_meminfo); 186 } 187 188 /* Initialize NODE_DATA for a node on the local memory */ 189 static void __init setup_node_data(int nid, u64 start, u64 end) 190 { 191 const size_t nd_size = roundup(sizeof(pg_data_t), PAGE_SIZE); 192 u64 nd_pa; 193 void *nd; 194 int tnid; 195 196 /* 197 * Don't confuse VM with a node that doesn't have the 198 * minimum amount of memory: 199 */ 200 if (end && (end - start) < NODE_MIN_SIZE) 201 return; 202 203 start = roundup(start, ZONE_ALIGN); 204 205 printk(KERN_INFO "Initmem setup node %d [mem %#010Lx-%#010Lx]\n", 206 nid, start, end - 1); 207 208 /* 209 * Allocate node data. Try node-local memory and then any node. 210 * Never allocate in DMA zone. 211 */ 212 nd_pa = memblock_alloc_nid(nd_size, SMP_CACHE_BYTES, nid); 213 if (!nd_pa) { 214 nd_pa = __memblock_alloc_base(nd_size, SMP_CACHE_BYTES, 215 MEMBLOCK_ALLOC_ACCESSIBLE); 216 if (!nd_pa) { 217 pr_err("Cannot find %zu bytes in node %d\n", 218 nd_size, nid); 219 return; 220 } 221 } 222 nd = __va(nd_pa); 223 224 /* report and initialize */ 225 printk(KERN_INFO " NODE_DATA [mem %#010Lx-%#010Lx]\n", 226 nd_pa, nd_pa + nd_size - 1); 227 tnid = early_pfn_to_nid(nd_pa >> PAGE_SHIFT); 228 if (tnid != nid) 229 printk(KERN_INFO " NODE_DATA(%d) on node %d\n", nid, tnid); 230 231 node_data[nid] = nd; 232 memset(NODE_DATA(nid), 0, sizeof(pg_data_t)); 233 NODE_DATA(nid)->node_id = nid; 234 NODE_DATA(nid)->node_start_pfn = start >> PAGE_SHIFT; 235 NODE_DATA(nid)->node_spanned_pages = (end - start) >> PAGE_SHIFT; 236 237 node_set_online(nid); 238 } 239 240 /** 241 * numa_cleanup_meminfo - Cleanup a numa_meminfo 242 * @mi: numa_meminfo to clean up 243 * 244 * Sanitize @mi by merging and removing unncessary memblks. Also check for 245 * conflicts and clear unused memblks. 246 * 247 * RETURNS: 248 * 0 on success, -errno on failure. 249 */ 250 int __init numa_cleanup_meminfo(struct numa_meminfo *mi) 251 { 252 const u64 low = 0; 253 const u64 high = PFN_PHYS(max_pfn); 254 int i, j, k; 255 256 /* first, trim all entries */ 257 for (i = 0; i < mi->nr_blks; i++) { 258 struct numa_memblk *bi = &mi->blk[i]; 259 260 /* make sure all blocks are inside the limits */ 261 bi->start = max(bi->start, low); 262 bi->end = min(bi->end, high); 263 264 /* and there's no empty block */ 265 if (bi->start >= bi->end) 266 numa_remove_memblk_from(i--, mi); 267 } 268 269 /* merge neighboring / overlapping entries */ 270 for (i = 0; i < mi->nr_blks; i++) { 271 struct numa_memblk *bi = &mi->blk[i]; 272 273 for (j = i + 1; j < mi->nr_blks; j++) { 274 struct numa_memblk *bj = &mi->blk[j]; 275 u64 start, end; 276 277 /* 278 * See whether there are overlapping blocks. Whine 279 * about but allow overlaps of the same nid. They 280 * will be merged below. 281 */ 282 if (bi->end > bj->start && bi->start < bj->end) { 283 if (bi->nid != bj->nid) { 284 pr_err("NUMA: node %d [mem %#010Lx-%#010Lx] overlaps with node %d [mem %#010Lx-%#010Lx]\n", 285 bi->nid, bi->start, bi->end - 1, 286 bj->nid, bj->start, bj->end - 1); 287 return -EINVAL; 288 } 289 pr_warning("NUMA: Warning: node %d [mem %#010Lx-%#010Lx] overlaps with itself [mem %#010Lx-%#010Lx]\n", 290 bi->nid, bi->start, bi->end - 1, 291 bj->start, bj->end - 1); 292 } 293 294 /* 295 * Join together blocks on the same node, holes 296 * between which don't overlap with memory on other 297 * nodes. 298 */ 299 if (bi->nid != bj->nid) 300 continue; 301 start = min(bi->start, bj->start); 302 end = max(bi->end, bj->end); 303 for (k = 0; k < mi->nr_blks; k++) { 304 struct numa_memblk *bk = &mi->blk[k]; 305 306 if (bi->nid == bk->nid) 307 continue; 308 if (start < bk->end && end > bk->start) 309 break; 310 } 311 if (k < mi->nr_blks) 312 continue; 313 printk(KERN_INFO "NUMA: Node %d [mem %#010Lx-%#010Lx] + [mem %#010Lx-%#010Lx] -> [mem %#010Lx-%#010Lx]\n", 314 bi->nid, bi->start, bi->end - 1, bj->start, 315 bj->end - 1, start, end - 1); 316 bi->start = start; 317 bi->end = end; 318 numa_remove_memblk_from(j--, mi); 319 } 320 } 321 322 /* clear unused ones */ 323 for (i = mi->nr_blks; i < ARRAY_SIZE(mi->blk); i++) { 324 mi->blk[i].start = mi->blk[i].end = 0; 325 mi->blk[i].nid = NUMA_NO_NODE; 326 } 327 328 return 0; 329 } 330 331 /* 332 * Set nodes, which have memory in @mi, in *@nodemask. 333 */ 334 static void __init numa_nodemask_from_meminfo(nodemask_t *nodemask, 335 const struct numa_meminfo *mi) 336 { 337 int i; 338 339 for (i = 0; i < ARRAY_SIZE(mi->blk); i++) 340 if (mi->blk[i].start != mi->blk[i].end && 341 mi->blk[i].nid != NUMA_NO_NODE) 342 node_set(mi->blk[i].nid, *nodemask); 343 } 344 345 /** 346 * numa_reset_distance - Reset NUMA distance table 347 * 348 * The current table is freed. The next numa_set_distance() call will 349 * create a new one. 350 */ 351 void __init numa_reset_distance(void) 352 { 353 size_t size = numa_distance_cnt * numa_distance_cnt * sizeof(numa_distance[0]); 354 355 /* numa_distance could be 1LU marking allocation failure, test cnt */ 356 if (numa_distance_cnt) 357 memblock_free(__pa(numa_distance), size); 358 numa_distance_cnt = 0; 359 numa_distance = NULL; /* enable table creation */ 360 } 361 362 static int __init numa_alloc_distance(void) 363 { 364 nodemask_t nodes_parsed; 365 size_t size; 366 int i, j, cnt = 0; 367 u64 phys; 368 369 /* size the new table and allocate it */ 370 nodes_parsed = numa_nodes_parsed; 371 numa_nodemask_from_meminfo(&nodes_parsed, &numa_meminfo); 372 373 for_each_node_mask(i, nodes_parsed) 374 cnt = i; 375 cnt++; 376 size = cnt * cnt * sizeof(numa_distance[0]); 377 378 phys = memblock_find_in_range(0, PFN_PHYS(max_pfn_mapped), 379 size, PAGE_SIZE); 380 if (!phys) { 381 pr_warning("NUMA: Warning: can't allocate distance table!\n"); 382 /* don't retry until explicitly reset */ 383 numa_distance = (void *)1LU; 384 return -ENOMEM; 385 } 386 memblock_reserve(phys, size); 387 388 numa_distance = __va(phys); 389 numa_distance_cnt = cnt; 390 391 /* fill with the default distances */ 392 for (i = 0; i < cnt; i++) 393 for (j = 0; j < cnt; j++) 394 numa_distance[i * cnt + j] = i == j ? 395 LOCAL_DISTANCE : REMOTE_DISTANCE; 396 printk(KERN_DEBUG "NUMA: Initialized distance table, cnt=%d\n", cnt); 397 398 return 0; 399 } 400 401 /** 402 * numa_set_distance - Set NUMA distance from one NUMA to another 403 * @from: the 'from' node to set distance 404 * @to: the 'to' node to set distance 405 * @distance: NUMA distance 406 * 407 * Set the distance from node @from to @to to @distance. If distance table 408 * doesn't exist, one which is large enough to accommodate all the currently 409 * known nodes will be created. 410 * 411 * If such table cannot be allocated, a warning is printed and further 412 * calls are ignored until the distance table is reset with 413 * numa_reset_distance(). 414 * 415 * If @from or @to is higher than the highest known node or lower than zero 416 * at the time of table creation or @distance doesn't make sense, the call 417 * is ignored. 418 * This is to allow simplification of specific NUMA config implementations. 419 */ 420 void __init numa_set_distance(int from, int to, int distance) 421 { 422 if (!numa_distance && numa_alloc_distance() < 0) 423 return; 424 425 if (from >= numa_distance_cnt || to >= numa_distance_cnt || 426 from < 0 || to < 0) { 427 pr_warn_once("NUMA: Warning: node ids are out of bound, from=%d to=%d distance=%d\n", 428 from, to, distance); 429 return; 430 } 431 432 if ((u8)distance != distance || 433 (from == to && distance != LOCAL_DISTANCE)) { 434 pr_warn_once("NUMA: Warning: invalid distance parameter, from=%d to=%d distance=%d\n", 435 from, to, distance); 436 return; 437 } 438 439 numa_distance[from * numa_distance_cnt + to] = distance; 440 } 441 442 int __node_distance(int from, int to) 443 { 444 if (from >= numa_distance_cnt || to >= numa_distance_cnt) 445 return from == to ? LOCAL_DISTANCE : REMOTE_DISTANCE; 446 return numa_distance[from * numa_distance_cnt + to]; 447 } 448 EXPORT_SYMBOL(__node_distance); 449 450 /* 451 * Sanity check to catch more bad NUMA configurations (they are amazingly 452 * common). Make sure the nodes cover all memory. 453 */ 454 static bool __init numa_meminfo_cover_memory(const struct numa_meminfo *mi) 455 { 456 u64 numaram, e820ram; 457 int i; 458 459 numaram = 0; 460 for (i = 0; i < mi->nr_blks; i++) { 461 u64 s = mi->blk[i].start >> PAGE_SHIFT; 462 u64 e = mi->blk[i].end >> PAGE_SHIFT; 463 numaram += e - s; 464 numaram -= __absent_pages_in_range(mi->blk[i].nid, s, e); 465 if ((s64)numaram < 0) 466 numaram = 0; 467 } 468 469 e820ram = max_pfn - absent_pages_in_range(0, max_pfn); 470 471 /* We seem to lose 3 pages somewhere. Allow 1M of slack. */ 472 if ((s64)(e820ram - numaram) >= (1 << (20 - PAGE_SHIFT))) { 473 printk(KERN_ERR "NUMA: nodes only cover %LuMB of your %LuMB e820 RAM. Not used.\n", 474 (numaram << PAGE_SHIFT) >> 20, 475 (e820ram << PAGE_SHIFT) >> 20); 476 return false; 477 } 478 return true; 479 } 480 481 static int __init numa_register_memblks(struct numa_meminfo *mi) 482 { 483 unsigned long uninitialized_var(pfn_align); 484 int i, nid; 485 486 /* Account for nodes with cpus and no memory */ 487 node_possible_map = numa_nodes_parsed; 488 numa_nodemask_from_meminfo(&node_possible_map, mi); 489 if (WARN_ON(nodes_empty(node_possible_map))) 490 return -EINVAL; 491 492 for (i = 0; i < mi->nr_blks; i++) { 493 struct numa_memblk *mb = &mi->blk[i]; 494 memblock_set_node(mb->start, mb->end - mb->start, 495 &memblock.memory, mb->nid); 496 497 /* 498 * At this time, all memory regions reserved by memblock are 499 * used by the kernel. Set the nid in memblock.reserved will 500 * mark out all the nodes the kernel resides in. 501 */ 502 memblock_set_node(mb->start, mb->end - mb->start, 503 &memblock.reserved, mb->nid); 504 } 505 506 /* 507 * If sections array is gonna be used for pfn -> nid mapping, check 508 * whether its granularity is fine enough. 509 */ 510 #ifdef NODE_NOT_IN_PAGE_FLAGS 511 pfn_align = node_map_pfn_alignment(); 512 if (pfn_align && pfn_align < PAGES_PER_SECTION) { 513 printk(KERN_WARNING "Node alignment %LuMB < min %LuMB, rejecting NUMA config\n", 514 PFN_PHYS(pfn_align) >> 20, 515 PFN_PHYS(PAGES_PER_SECTION) >> 20); 516 return -EINVAL; 517 } 518 #endif 519 if (!numa_meminfo_cover_memory(mi)) 520 return -EINVAL; 521 522 /* Finally register nodes. */ 523 for_each_node_mask(nid, node_possible_map) { 524 u64 start = PFN_PHYS(max_pfn); 525 u64 end = 0; 526 527 for (i = 0; i < mi->nr_blks; i++) { 528 if (nid != mi->blk[i].nid) 529 continue; 530 start = min(mi->blk[i].start, start); 531 end = max(mi->blk[i].end, end); 532 } 533 534 if (start < end) 535 setup_node_data(nid, start, end); 536 } 537 538 /* Dump memblock with node info and return. */ 539 memblock_dump_all(); 540 return 0; 541 } 542 543 /* 544 * There are unfortunately some poorly designed mainboards around that 545 * only connect memory to a single CPU. This breaks the 1:1 cpu->node 546 * mapping. To avoid this fill in the mapping for all possible CPUs, 547 * as the number of CPUs is not known yet. We round robin the existing 548 * nodes. 549 */ 550 static void __init numa_init_array(void) 551 { 552 int rr, i; 553 554 rr = first_node(node_online_map); 555 for (i = 0; i < nr_cpu_ids; i++) { 556 if (early_cpu_to_node(i) != NUMA_NO_NODE) 557 continue; 558 numa_set_node(i, rr); 559 rr = next_node(rr, node_online_map); 560 if (rr == MAX_NUMNODES) 561 rr = first_node(node_online_map); 562 } 563 } 564 565 static void __init numa_clear_kernel_node_hotplug(void) 566 { 567 int i, nid; 568 nodemask_t numa_kernel_nodes; 569 unsigned long start, end; 570 struct memblock_type *type = &memblock.reserved; 571 572 /* Mark all kernel nodes. */ 573 for (i = 0; i < type->cnt; i++) 574 node_set(type->regions[i].nid, numa_kernel_nodes); 575 576 /* Clear MEMBLOCK_HOTPLUG flag for memory in kernel nodes. */ 577 for (i = 0; i < numa_meminfo.nr_blks; i++) { 578 nid = numa_meminfo.blk[i].nid; 579 if (!node_isset(nid, numa_kernel_nodes)) 580 continue; 581 582 start = numa_meminfo.blk[i].start; 583 end = numa_meminfo.blk[i].end; 584 585 memblock_clear_hotplug(start, end - start); 586 } 587 } 588 589 static int __init numa_init(int (*init_func)(void)) 590 { 591 int i; 592 int ret; 593 594 for (i = 0; i < MAX_LOCAL_APIC; i++) 595 set_apicid_to_node(i, NUMA_NO_NODE); 596 597 nodes_clear(numa_nodes_parsed); 598 nodes_clear(node_possible_map); 599 nodes_clear(node_online_map); 600 memset(&numa_meminfo, 0, sizeof(numa_meminfo)); 601 WARN_ON(memblock_set_node(0, ULLONG_MAX, &memblock.memory, 602 MAX_NUMNODES)); 603 WARN_ON(memblock_set_node(0, ULLONG_MAX, &memblock.reserved, 604 MAX_NUMNODES)); 605 /* In case that parsing SRAT failed. */ 606 WARN_ON(memblock_clear_hotplug(0, ULLONG_MAX)); 607 numa_reset_distance(); 608 609 ret = init_func(); 610 if (ret < 0) 611 return ret; 612 613 /* 614 * We reset memblock back to the top-down direction 615 * here because if we configured ACPI_NUMA, we have 616 * parsed SRAT in init_func(). It is ok to have the 617 * reset here even if we did't configure ACPI_NUMA 618 * or acpi numa init fails and fallbacks to dummy 619 * numa init. 620 */ 621 memblock_set_bottom_up(false); 622 623 ret = numa_cleanup_meminfo(&numa_meminfo); 624 if (ret < 0) 625 return ret; 626 627 numa_emulation(&numa_meminfo, numa_distance_cnt); 628 629 ret = numa_register_memblks(&numa_meminfo); 630 if (ret < 0) 631 return ret; 632 633 for (i = 0; i < nr_cpu_ids; i++) { 634 int nid = early_cpu_to_node(i); 635 636 if (nid == NUMA_NO_NODE) 637 continue; 638 if (!node_online(nid)) 639 numa_clear_node(i); 640 } 641 numa_init_array(); 642 643 /* 644 * At very early time, the kernel have to use some memory such as 645 * loading the kernel image. We cannot prevent this anyway. So any 646 * node the kernel resides in should be un-hotpluggable. 647 * 648 * And when we come here, numa_init() won't fail. 649 */ 650 numa_clear_kernel_node_hotplug(); 651 652 return 0; 653 } 654 655 /** 656 * dummy_numa_init - Fallback dummy NUMA init 657 * 658 * Used if there's no underlying NUMA architecture, NUMA initialization 659 * fails, or NUMA is disabled on the command line. 660 * 661 * Must online at least one node and add memory blocks that cover all 662 * allowed memory. This function must not fail. 663 */ 664 static int __init dummy_numa_init(void) 665 { 666 printk(KERN_INFO "%s\n", 667 numa_off ? "NUMA turned off" : "No NUMA configuration found"); 668 printk(KERN_INFO "Faking a node at [mem %#018Lx-%#018Lx]\n", 669 0LLU, PFN_PHYS(max_pfn) - 1); 670 671 node_set(0, numa_nodes_parsed); 672 numa_add_memblk(0, 0, PFN_PHYS(max_pfn)); 673 674 return 0; 675 } 676 677 /** 678 * x86_numa_init - Initialize NUMA 679 * 680 * Try each configured NUMA initialization method until one succeeds. The 681 * last fallback is dummy single node config encomapssing whole memory and 682 * never fails. 683 */ 684 void __init x86_numa_init(void) 685 { 686 if (!numa_off) { 687 #ifdef CONFIG_X86_NUMAQ 688 if (!numa_init(numaq_numa_init)) 689 return; 690 #endif 691 #ifdef CONFIG_ACPI_NUMA 692 if (!numa_init(x86_acpi_numa_init)) 693 return; 694 #endif 695 #ifdef CONFIG_AMD_NUMA 696 if (!numa_init(amd_numa_init)) 697 return; 698 #endif 699 } 700 701 numa_init(dummy_numa_init); 702 } 703 704 static __init int find_near_online_node(int node) 705 { 706 int n, val; 707 int min_val = INT_MAX; 708 int best_node = -1; 709 710 for_each_online_node(n) { 711 val = node_distance(node, n); 712 713 if (val < min_val) { 714 min_val = val; 715 best_node = n; 716 } 717 } 718 719 return best_node; 720 } 721 722 /* 723 * Setup early cpu_to_node. 724 * 725 * Populate cpu_to_node[] only if x86_cpu_to_apicid[], 726 * and apicid_to_node[] tables have valid entries for a CPU. 727 * This means we skip cpu_to_node[] initialisation for NUMA 728 * emulation and faking node case (when running a kernel compiled 729 * for NUMA on a non NUMA box), which is OK as cpu_to_node[] 730 * is already initialized in a round robin manner at numa_init_array, 731 * prior to this call, and this initialization is good enough 732 * for the fake NUMA cases. 733 * 734 * Called before the per_cpu areas are setup. 735 */ 736 void __init init_cpu_to_node(void) 737 { 738 int cpu; 739 u16 *cpu_to_apicid = early_per_cpu_ptr(x86_cpu_to_apicid); 740 741 BUG_ON(cpu_to_apicid == NULL); 742 743 for_each_possible_cpu(cpu) { 744 int node = numa_cpu_node(cpu); 745 746 if (node == NUMA_NO_NODE) 747 continue; 748 if (!node_online(node)) 749 node = find_near_online_node(node); 750 numa_set_node(cpu, node); 751 } 752 } 753 754 #ifndef CONFIG_DEBUG_PER_CPU_MAPS 755 756 # ifndef CONFIG_NUMA_EMU 757 void numa_add_cpu(int cpu) 758 { 759 cpumask_set_cpu(cpu, node_to_cpumask_map[early_cpu_to_node(cpu)]); 760 } 761 762 void numa_remove_cpu(int cpu) 763 { 764 cpumask_clear_cpu(cpu, node_to_cpumask_map[early_cpu_to_node(cpu)]); 765 } 766 # endif /* !CONFIG_NUMA_EMU */ 767 768 #else /* !CONFIG_DEBUG_PER_CPU_MAPS */ 769 770 int __cpu_to_node(int cpu) 771 { 772 if (early_per_cpu_ptr(x86_cpu_to_node_map)) { 773 printk(KERN_WARNING 774 "cpu_to_node(%d): usage too early!\n", cpu); 775 dump_stack(); 776 return early_per_cpu_ptr(x86_cpu_to_node_map)[cpu]; 777 } 778 return per_cpu(x86_cpu_to_node_map, cpu); 779 } 780 EXPORT_SYMBOL(__cpu_to_node); 781 782 /* 783 * Same function as cpu_to_node() but used if called before the 784 * per_cpu areas are setup. 785 */ 786 int early_cpu_to_node(int cpu) 787 { 788 if (early_per_cpu_ptr(x86_cpu_to_node_map)) 789 return early_per_cpu_ptr(x86_cpu_to_node_map)[cpu]; 790 791 if (!cpu_possible(cpu)) { 792 printk(KERN_WARNING 793 "early_cpu_to_node(%d): no per_cpu area!\n", cpu); 794 dump_stack(); 795 return NUMA_NO_NODE; 796 } 797 return per_cpu(x86_cpu_to_node_map, cpu); 798 } 799 800 void debug_cpumask_set_cpu(int cpu, int node, bool enable) 801 { 802 struct cpumask *mask; 803 char buf[64]; 804 805 if (node == NUMA_NO_NODE) { 806 /* early_cpu_to_node() already emits a warning and trace */ 807 return; 808 } 809 mask = node_to_cpumask_map[node]; 810 if (!mask) { 811 pr_err("node_to_cpumask_map[%i] NULL\n", node); 812 dump_stack(); 813 return; 814 } 815 816 if (enable) 817 cpumask_set_cpu(cpu, mask); 818 else 819 cpumask_clear_cpu(cpu, mask); 820 821 cpulist_scnprintf(buf, sizeof(buf), mask); 822 printk(KERN_DEBUG "%s cpu %d node %d: mask now %s\n", 823 enable ? "numa_add_cpu" : "numa_remove_cpu", 824 cpu, node, buf); 825 return; 826 } 827 828 # ifndef CONFIG_NUMA_EMU 829 static void numa_set_cpumask(int cpu, bool enable) 830 { 831 debug_cpumask_set_cpu(cpu, early_cpu_to_node(cpu), enable); 832 } 833 834 void numa_add_cpu(int cpu) 835 { 836 numa_set_cpumask(cpu, true); 837 } 838 839 void numa_remove_cpu(int cpu) 840 { 841 numa_set_cpumask(cpu, false); 842 } 843 # endif /* !CONFIG_NUMA_EMU */ 844 845 /* 846 * Returns a pointer to the bitmask of CPUs on Node 'node'. 847 */ 848 const struct cpumask *cpumask_of_node(int node) 849 { 850 if (node >= nr_node_ids) { 851 printk(KERN_WARNING 852 "cpumask_of_node(%d): node > nr_node_ids(%d)\n", 853 node, nr_node_ids); 854 dump_stack(); 855 return cpu_none_mask; 856 } 857 if (node_to_cpumask_map[node] == NULL) { 858 printk(KERN_WARNING 859 "cpumask_of_node(%d): no node_to_cpumask_map!\n", 860 node); 861 dump_stack(); 862 return cpu_online_mask; 863 } 864 return node_to_cpumask_map[node]; 865 } 866 EXPORT_SYMBOL(cpumask_of_node); 867 868 #endif /* !CONFIG_DEBUG_PER_CPU_MAPS */ 869 870 #ifdef CONFIG_MEMORY_HOTPLUG 871 int memory_add_physaddr_to_nid(u64 start) 872 { 873 struct numa_meminfo *mi = &numa_meminfo; 874 int nid = mi->blk[0].nid; 875 int i; 876 877 for (i = 0; i < mi->nr_blks; i++) 878 if (mi->blk[i].start <= start && mi->blk[i].end > start) 879 nid = mi->blk[i].nid; 880 return nid; 881 } 882 EXPORT_SYMBOL_GPL(memory_add_physaddr_to_nid); 883 #endif 884