1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Common code for 32 and 64-bit NUMA */ 3 #include <linux/acpi.h> 4 #include <linux/kernel.h> 5 #include <linux/mm.h> 6 #include <linux/of.h> 7 #include <linux/string.h> 8 #include <linux/init.h> 9 #include <linux/memblock.h> 10 #include <linux/mmzone.h> 11 #include <linux/ctype.h> 12 #include <linux/nodemask.h> 13 #include <linux/sched.h> 14 #include <linux/topology.h> 15 #include <linux/sort.h> 16 #include <linux/numa_memblks.h> 17 18 #include <asm/e820/api.h> 19 #include <asm/proto.h> 20 #include <asm/dma.h> 21 #include <asm/numa.h> 22 #include <asm/amd/nb.h> 23 24 #include "mm_internal.h" 25 26 int numa_off; 27 28 static __init int numa_setup(char *opt) 29 { 30 if (!opt) 31 return -EINVAL; 32 if (!strncmp(opt, "off", 3)) 33 numa_off = 1; 34 if (!strncmp(opt, "fake=", 5)) 35 return numa_emu_cmdline(opt + 5); 36 if (!strncmp(opt, "noacpi", 6)) 37 disable_srat(); 38 if (!strncmp(opt, "nohmat", 6)) 39 disable_hmat(); 40 return 0; 41 } 42 early_param("numa", numa_setup); 43 44 /* 45 * apicid, cpu, node mappings 46 */ 47 s16 __apicid_to_node[MAX_LOCAL_APIC] = { 48 [0 ... MAX_LOCAL_APIC-1] = NUMA_NO_NODE 49 }; 50 51 nodemask_t numa_phys_nodes_parsed __initdata; 52 53 int numa_cpu_node(int cpu) 54 { 55 u32 apicid = early_per_cpu(x86_cpu_to_apicid, cpu); 56 57 if (apicid != BAD_APICID) 58 return __apicid_to_node[apicid]; 59 return NUMA_NO_NODE; 60 } 61 62 int __init num_phys_nodes(void) 63 { 64 return bitmap_weight(numa_phys_nodes_parsed.bits, MAX_NUMNODES); 65 } 66 67 cpumask_var_t node_to_cpumask_map[MAX_NUMNODES]; 68 EXPORT_SYMBOL(node_to_cpumask_map); 69 70 /* 71 * Map cpu index to node index 72 */ 73 DEFINE_EARLY_PER_CPU(int, x86_cpu_to_node_map, NUMA_NO_NODE); 74 EXPORT_EARLY_PER_CPU_SYMBOL(x86_cpu_to_node_map); 75 76 void numa_set_node(int cpu, int node) 77 { 78 int *cpu_to_node_map = early_per_cpu_ptr(x86_cpu_to_node_map); 79 80 /* early setting, no percpu area yet */ 81 if (cpu_to_node_map) { 82 cpu_to_node_map[cpu] = node; 83 return; 84 } 85 86 #ifdef CONFIG_DEBUG_PER_CPU_MAPS 87 if (cpu >= nr_cpu_ids || !cpu_possible(cpu)) { 88 printk(KERN_ERR "numa_set_node: invalid cpu# (%d)\n", cpu); 89 dump_stack(); 90 return; 91 } 92 #endif 93 per_cpu(x86_cpu_to_node_map, cpu) = node; 94 95 set_cpu_numa_node(cpu, node); 96 } 97 98 void numa_clear_node(int cpu) 99 { 100 numa_set_node(cpu, NUMA_NO_NODE); 101 } 102 103 /* 104 * Allocate node_to_cpumask_map based on number of available nodes 105 * Requires node_possible_map to be valid. 106 * 107 * Note: cpumask_of_node() is not valid until after this is done. 108 * (Use CONFIG_DEBUG_PER_CPU_MAPS to check this.) 109 */ 110 void __init setup_node_to_cpumask_map(void) 111 { 112 unsigned int node; 113 114 /* setup nr_node_ids if not done yet */ 115 if (nr_node_ids == MAX_NUMNODES) 116 setup_nr_node_ids(); 117 118 /* allocate the map */ 119 for (node = 0; node < nr_node_ids; node++) 120 alloc_bootmem_cpumask_var(&node_to_cpumask_map[node]); 121 122 /* cpumask_of_node() will now work */ 123 pr_debug("Node to cpumask map for %u nodes\n", nr_node_ids); 124 } 125 126 static int __init numa_register_nodes(void) 127 { 128 int nid; 129 130 if (!memblock_validate_numa_coverage(SZ_1M)) 131 return -EINVAL; 132 133 /* Finally register nodes. */ 134 for_each_node_mask(nid, node_possible_map) { 135 unsigned long start_pfn, end_pfn; 136 137 /* 138 * Note, get_pfn_range_for_nid() depends on 139 * memblock_set_node() having already happened 140 */ 141 get_pfn_range_for_nid(nid, &start_pfn, &end_pfn); 142 if (start_pfn >= end_pfn) 143 continue; 144 145 alloc_node_data(nid); 146 node_set_online(nid); 147 } 148 149 /* Dump memblock with node info and return. */ 150 memblock_dump_all(); 151 return 0; 152 } 153 154 /* 155 * There are unfortunately some poorly designed mainboards around that 156 * only connect memory to a single CPU. This breaks the 1:1 cpu->node 157 * mapping. To avoid this fill in the mapping for all possible CPUs, 158 * as the number of CPUs is not known yet. We round robin the existing 159 * nodes. 160 */ 161 static void __init numa_init_array(void) 162 { 163 int rr, i; 164 165 rr = first_node(node_online_map); 166 for (i = 0; i < nr_cpu_ids; i++) { 167 if (early_cpu_to_node(i) != NUMA_NO_NODE) 168 continue; 169 numa_set_node(i, rr); 170 rr = next_node_in(rr, node_online_map); 171 } 172 } 173 174 static int __init numa_init(int (*init_func)(void)) 175 { 176 int i; 177 int ret; 178 179 for (i = 0; i < MAX_LOCAL_APIC; i++) 180 set_apicid_to_node(i, NUMA_NO_NODE); 181 182 ret = numa_memblks_init(init_func, /* memblock_force_top_down */ true); 183 if (ret < 0) 184 return ret; 185 186 ret = numa_register_nodes(); 187 if (ret < 0) 188 return ret; 189 190 for (i = 0; i < nr_cpu_ids; i++) { 191 int nid = early_cpu_to_node(i); 192 193 if (nid == NUMA_NO_NODE) 194 continue; 195 if (!node_online(nid)) 196 numa_clear_node(i); 197 } 198 numa_init_array(); 199 200 return 0; 201 } 202 203 /** 204 * dummy_numa_init - Fallback dummy NUMA init 205 * 206 * Used if there's no underlying NUMA architecture, NUMA initialization 207 * fails, or NUMA is disabled on the command line. 208 * 209 * Must online at least one node and add memory blocks that cover all 210 * allowed memory. This function must not fail. 211 */ 212 static int __init dummy_numa_init(void) 213 { 214 printk(KERN_INFO "%s\n", 215 numa_off ? "NUMA turned off" : "No NUMA configuration found"); 216 printk(KERN_INFO "Faking a node at [mem %#018Lx-%#018Lx]\n", 217 0LLU, PFN_PHYS(max_pfn) - 1); 218 219 node_set(0, numa_phys_nodes_parsed); 220 numa_add_memblk(0, 0, PFN_PHYS(max_pfn)); 221 222 return 0; 223 } 224 225 /** 226 * x86_numa_init - Initialize NUMA 227 * 228 * Try each configured NUMA initialization method until one succeeds. The 229 * last fallback is dummy single node config encompassing whole memory and 230 * never fails. 231 */ 232 void __init x86_numa_init(void) 233 { 234 if (!numa_off) { 235 #ifdef CONFIG_ACPI_NUMA 236 if (!numa_init(x86_acpi_numa_init)) 237 return; 238 #endif 239 #ifdef CONFIG_AMD_NUMA 240 if (!numa_init(amd_numa_init)) 241 return; 242 #endif 243 if (acpi_disabled && !numa_init(of_numa_init)) 244 return; 245 } 246 247 numa_init(dummy_numa_init); 248 } 249 250 251 /* 252 * A node may exist which has one or more Generic Initiators but no CPUs and no 253 * memory. 254 * 255 * This function must be called after init_cpu_to_node(), to ensure that any 256 * memoryless CPU nodes have already been brought online, and before the 257 * node_data[nid] is needed for zone list setup in build_all_zonelists(). 258 * 259 * When this function is called, any nodes containing either memory and/or CPUs 260 * will already be online and there is no need to do anything extra, even if 261 * they also contain one or more Generic Initiators. 262 */ 263 void __init init_gi_nodes(void) 264 { 265 int nid; 266 267 /* 268 * Exclude this node from 269 * bringup_nonboot_cpus 270 * cpu_up 271 * __try_online_node 272 * register_node 273 * because node_subsys is not initialized yet. 274 * TODO remove dependency on node_online 275 */ 276 for_each_node_state(nid, N_GENERIC_INITIATOR) 277 if (!node_online(nid)) 278 node_set_online(nid); 279 } 280 281 /* 282 * Setup early cpu_to_node. 283 * 284 * Populate cpu_to_node[] only if x86_cpu_to_apicid[], 285 * and apicid_to_node[] tables have valid entries for a CPU. 286 * This means we skip cpu_to_node[] initialisation for NUMA 287 * emulation and faking node case (when running a kernel compiled 288 * for NUMA on a non NUMA box), which is OK as cpu_to_node[] 289 * is already initialized in a round robin manner at numa_init_array, 290 * prior to this call, and this initialization is good enough 291 * for the fake NUMA cases. 292 * 293 * Called before the per_cpu areas are setup. 294 */ 295 void __init init_cpu_to_node(void) 296 { 297 int cpu; 298 u32 *cpu_to_apicid = early_per_cpu_ptr(x86_cpu_to_apicid); 299 300 BUG_ON(cpu_to_apicid == NULL); 301 302 for_each_possible_cpu(cpu) { 303 int node = numa_cpu_node(cpu); 304 305 if (node == NUMA_NO_NODE) 306 continue; 307 308 /* 309 * Exclude this node from 310 * bringup_nonboot_cpus 311 * cpu_up 312 * __try_online_node 313 * register_node 314 * because node_subsys is not initialized yet. 315 * TODO remove dependency on node_online 316 */ 317 if (!node_online(node)) 318 node_set_online(node); 319 320 numa_set_node(cpu, node); 321 } 322 } 323 324 #ifndef CONFIG_DEBUG_PER_CPU_MAPS 325 326 # ifndef CONFIG_NUMA_EMU 327 void numa_add_cpu(unsigned int cpu) 328 { 329 cpumask_set_cpu(cpu, node_to_cpumask_map[early_cpu_to_node(cpu)]); 330 } 331 332 void numa_remove_cpu(unsigned int cpu) 333 { 334 cpumask_clear_cpu(cpu, node_to_cpumask_map[early_cpu_to_node(cpu)]); 335 } 336 # endif /* !CONFIG_NUMA_EMU */ 337 338 #else /* !CONFIG_DEBUG_PER_CPU_MAPS */ 339 340 int __cpu_to_node(int cpu) 341 { 342 if (early_per_cpu_ptr(x86_cpu_to_node_map)) { 343 printk(KERN_WARNING 344 "cpu_to_node(%d): usage too early!\n", cpu); 345 dump_stack(); 346 return early_per_cpu_ptr(x86_cpu_to_node_map)[cpu]; 347 } 348 return per_cpu(x86_cpu_to_node_map, cpu); 349 } 350 EXPORT_SYMBOL(__cpu_to_node); 351 352 /* 353 * Same function as cpu_to_node() but used if called before the 354 * per_cpu areas are setup. 355 */ 356 int early_cpu_to_node(int cpu) 357 { 358 if (early_per_cpu_ptr(x86_cpu_to_node_map)) 359 return early_per_cpu_ptr(x86_cpu_to_node_map)[cpu]; 360 361 if (!cpu_possible(cpu)) { 362 printk(KERN_WARNING 363 "early_cpu_to_node(%d): no per_cpu area!\n", cpu); 364 dump_stack(); 365 return NUMA_NO_NODE; 366 } 367 return per_cpu(x86_cpu_to_node_map, cpu); 368 } 369 370 void debug_cpumask_set_cpu(unsigned int cpu, int node, bool enable) 371 { 372 struct cpumask *mask; 373 374 if (node == NUMA_NO_NODE) { 375 /* early_cpu_to_node() already emits a warning and trace */ 376 return; 377 } 378 mask = node_to_cpumask_map[node]; 379 if (!cpumask_available(mask)) { 380 pr_err("node_to_cpumask_map[%i] NULL\n", node); 381 dump_stack(); 382 return; 383 } 384 385 if (enable) 386 cpumask_set_cpu(cpu, mask); 387 else 388 cpumask_clear_cpu(cpu, mask); 389 390 printk(KERN_DEBUG "%s cpu %d node %d: mask now %*pbl\n", 391 enable ? "numa_add_cpu" : "numa_remove_cpu", 392 cpu, node, cpumask_pr_args(mask)); 393 return; 394 } 395 396 # ifndef CONFIG_NUMA_EMU 397 static void numa_set_cpumask(int cpu, bool enable) 398 { 399 debug_cpumask_set_cpu(cpu, early_cpu_to_node(cpu), enable); 400 } 401 402 void numa_add_cpu(unsigned int cpu) 403 { 404 numa_set_cpumask(cpu, true); 405 } 406 407 void numa_remove_cpu(unsigned int cpu) 408 { 409 numa_set_cpumask(cpu, false); 410 } 411 # endif /* !CONFIG_NUMA_EMU */ 412 413 /* 414 * Returns a pointer to the bitmask of CPUs on Node 'node'. 415 */ 416 const struct cpumask *cpumask_of_node(int node) 417 { 418 if ((unsigned)node >= nr_node_ids) { 419 printk(KERN_WARNING 420 "cpumask_of_node(%d): (unsigned)node >= nr_node_ids(%u)\n", 421 node, nr_node_ids); 422 dump_stack(); 423 return cpu_none_mask; 424 } 425 if (!cpumask_available(node_to_cpumask_map[node])) { 426 printk(KERN_WARNING 427 "cpumask_of_node(%d): no node_to_cpumask_map!\n", 428 node); 429 dump_stack(); 430 return cpu_online_mask; 431 } 432 return node_to_cpumask_map[node]; 433 } 434 EXPORT_SYMBOL(cpumask_of_node); 435 436 #endif /* !CONFIG_DEBUG_PER_CPU_MAPS */ 437 438 #ifdef CONFIG_NUMA_EMU 439 void __init numa_emu_update_cpu_to_node(int *emu_nid_to_phys, 440 unsigned int nr_emu_nids) 441 { 442 int i, j; 443 444 /* 445 * Transform __apicid_to_node table to use emulated nids by 446 * reverse-mapping phys_nid. The maps should always exist but fall 447 * back to zero just in case. 448 */ 449 for (i = 0; i < ARRAY_SIZE(__apicid_to_node); i++) { 450 if (__apicid_to_node[i] == NUMA_NO_NODE) 451 continue; 452 for (j = 0; j < nr_emu_nids; j++) 453 if (__apicid_to_node[i] == emu_nid_to_phys[j]) 454 break; 455 __apicid_to_node[i] = j < nr_emu_nids ? j : 0; 456 } 457 } 458 459 u64 __init numa_emu_dma_end(void) 460 { 461 return PFN_PHYS(MAX_DMA32_PFN); 462 } 463 #endif /* CONFIG_NUMA_EMU */ 464