1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * NUMA support, based on the x86 implementation. 4 * 5 * Copyright (C) 2015 Cavium Inc. 6 * Author: Ganapatrao Kulkarni <gkulkarni@cavium.com> 7 */ 8 9 #define pr_fmt(fmt) "NUMA: " fmt 10 11 #include <linux/acpi.h> 12 #include <linux/memblock.h> 13 #include <linux/module.h> 14 #include <linux/of.h> 15 #include <linux/numa_memblks.h> 16 17 #include <asm/sections.h> 18 19 static int cpu_to_node_map[NR_CPUS] = { [0 ... NR_CPUS-1] = NUMA_NO_NODE }; 20 21 bool numa_off; 22 23 static __init int numa_parse_early_param(char *opt) 24 { 25 if (!opt) 26 return -EINVAL; 27 if (str_has_prefix(opt, "off")) 28 numa_off = true; 29 if (!strncmp(opt, "fake=", 5)) 30 return numa_emu_cmdline(opt + 5); 31 32 return 0; 33 } 34 early_param("numa", numa_parse_early_param); 35 36 cpumask_var_t node_to_cpumask_map[MAX_NUMNODES]; 37 EXPORT_SYMBOL(node_to_cpumask_map); 38 39 #ifdef CONFIG_DEBUG_PER_CPU_MAPS 40 41 /* 42 * Returns a pointer to the bitmask of CPUs on Node 'node'. 43 */ 44 const struct cpumask *cpumask_of_node(int node) 45 { 46 47 if (node == NUMA_NO_NODE) 48 return cpu_all_mask; 49 50 if (WARN_ON(node < 0 || node >= nr_node_ids)) 51 return cpu_none_mask; 52 53 if (WARN_ON(node_to_cpumask_map[node] == NULL)) 54 return cpu_online_mask; 55 56 return node_to_cpumask_map[node]; 57 } 58 EXPORT_SYMBOL(cpumask_of_node); 59 60 #endif 61 62 #ifndef CONFIG_NUMA_EMU 63 static void numa_update_cpu(unsigned int cpu, bool remove) 64 { 65 int nid = cpu_to_node(cpu); 66 67 if (nid == NUMA_NO_NODE) 68 return; 69 70 if (remove) 71 cpumask_clear_cpu(cpu, node_to_cpumask_map[nid]); 72 else 73 cpumask_set_cpu(cpu, node_to_cpumask_map[nid]); 74 } 75 76 void numa_add_cpu(unsigned int cpu) 77 { 78 numa_update_cpu(cpu, false); 79 } 80 81 void numa_remove_cpu(unsigned int cpu) 82 { 83 numa_update_cpu(cpu, true); 84 } 85 #endif 86 87 void numa_clear_node(unsigned int cpu) 88 { 89 numa_remove_cpu(cpu); 90 set_cpu_numa_node(cpu, NUMA_NO_NODE); 91 } 92 93 /* 94 * Allocate node_to_cpumask_map based on number of available nodes 95 * Requires node_possible_map to be valid. 96 * 97 * Note: cpumask_of_node() is not valid until after this is done. 98 * (Use CONFIG_DEBUG_PER_CPU_MAPS to check this.) 99 */ 100 static void __init setup_node_to_cpumask_map(void) 101 { 102 int node; 103 104 /* setup nr_node_ids if not done yet */ 105 if (nr_node_ids == MAX_NUMNODES) 106 setup_nr_node_ids(); 107 108 /* 109 * This check should never be true but it makes it clear to compilers 110 * that node_to_cpumask_map is bound by nr_node_ids, avoiding false 111 * positive fortify warnings when accessing node_to_cpumask_map in the 112 * for loop below. 113 */ 114 if (unlikely(nr_node_ids > MAX_NUMNODES)) { 115 pr_err("nr_node_ids (%u) is larger than MAX_NUMNODES (%u)\n", 116 nr_node_ids, MAX_NUMNODES); 117 return; 118 } 119 120 /* allocate and clear the mapping */ 121 for (node = 0; node < nr_node_ids; node++) { 122 alloc_bootmem_cpumask_var(&node_to_cpumask_map[node]); 123 cpumask_clear(node_to_cpumask_map[node]); 124 } 125 126 /* cpumask_of_node() will now work */ 127 pr_debug("Node to cpumask map for %u nodes\n", nr_node_ids); 128 } 129 130 /* 131 * Set the cpu to node and mem mapping 132 */ 133 void numa_store_cpu_info(unsigned int cpu) 134 { 135 set_cpu_numa_node(cpu, cpu_to_node_map[cpu]); 136 } 137 138 void __init early_map_cpu_to_node(unsigned int cpu, int nid) 139 { 140 /* fallback to node 0 */ 141 if (nid < 0 || nid >= MAX_NUMNODES || numa_off) 142 nid = 0; 143 144 cpu_to_node_map[cpu] = nid; 145 146 /* 147 * We should set the numa node of cpu0 as soon as possible, because it 148 * has already been set up online before. cpu_to_node(0) will soon be 149 * called. 150 */ 151 if (!cpu) 152 set_cpu_numa_node(cpu, nid); 153 } 154 155 #ifdef CONFIG_HAVE_SETUP_PER_CPU_AREA 156 unsigned long __per_cpu_offset[NR_CPUS] __read_mostly; 157 EXPORT_SYMBOL(__per_cpu_offset); 158 159 int early_cpu_to_node(int cpu) 160 { 161 return cpu_to_node_map[cpu]; 162 } 163 164 static int __init pcpu_cpu_distance(unsigned int from, unsigned int to) 165 { 166 return node_distance(early_cpu_to_node(from), early_cpu_to_node(to)); 167 } 168 169 void __init setup_per_cpu_areas(void) 170 { 171 unsigned long delta; 172 unsigned int cpu; 173 int rc = -EINVAL; 174 175 if (pcpu_chosen_fc != PCPU_FC_PAGE) { 176 /* 177 * Always reserve area for module percpu variables. That's 178 * what the legacy allocator did. 179 */ 180 rc = pcpu_embed_first_chunk(PERCPU_MODULE_RESERVE, 181 PERCPU_DYNAMIC_RESERVE, PAGE_SIZE, 182 pcpu_cpu_distance, 183 early_cpu_to_node); 184 #ifdef CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK 185 if (rc < 0) 186 pr_warn("PERCPU: %s allocator failed (%d), falling back to page size\n", 187 pcpu_fc_names[pcpu_chosen_fc], rc); 188 #endif 189 } 190 191 #ifdef CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK 192 if (rc < 0) 193 rc = pcpu_page_first_chunk(PERCPU_MODULE_RESERVE, early_cpu_to_node); 194 #endif 195 if (rc < 0) 196 panic("Failed to initialize percpu areas (err=%d).", rc); 197 198 delta = (unsigned long)pcpu_base_addr - (unsigned long)__per_cpu_start; 199 for_each_possible_cpu(cpu) 200 __per_cpu_offset[cpu] = delta + pcpu_unit_offsets[cpu]; 201 } 202 #endif 203 204 /* 205 * Initialize NODE_DATA for a node on the local memory 206 */ 207 static void __init setup_node_data(int nid, u64 start_pfn, u64 end_pfn) 208 { 209 if (start_pfn >= end_pfn) 210 pr_info("Initmem setup node %d [<memory-less node>]\n", nid); 211 212 alloc_node_data(nid); 213 214 NODE_DATA(nid)->node_id = nid; 215 NODE_DATA(nid)->node_start_pfn = start_pfn; 216 NODE_DATA(nid)->node_spanned_pages = end_pfn - start_pfn; 217 } 218 219 static int __init numa_register_nodes(void) 220 { 221 int nid; 222 223 /* Check the validity of the memblock/node mapping */ 224 if (!memblock_validate_numa_coverage(0)) 225 return -EINVAL; 226 227 /* Finally register nodes. */ 228 for_each_node_mask(nid, numa_nodes_parsed) { 229 unsigned long start_pfn, end_pfn; 230 231 get_pfn_range_for_nid(nid, &start_pfn, &end_pfn); 232 setup_node_data(nid, start_pfn, end_pfn); 233 node_set_online(nid); 234 } 235 236 return 0; 237 } 238 239 static int __init numa_init(int (*init_func)(void)) 240 { 241 int ret; 242 243 ret = numa_memblks_init(init_func, /* memblock_force_top_down */ false); 244 if (ret < 0) 245 goto out_free_distance; 246 247 if (nodes_empty(numa_nodes_parsed)) { 248 pr_info("No NUMA configuration found\n"); 249 ret = -EINVAL; 250 goto out_free_distance; 251 } 252 253 ret = numa_register_nodes(); 254 if (ret < 0) 255 goto out_free_distance; 256 257 setup_node_to_cpumask_map(); 258 259 return 0; 260 out_free_distance: 261 numa_reset_distance(); 262 return ret; 263 } 264 265 /** 266 * dummy_numa_init() - Fallback dummy NUMA init 267 * 268 * Used if there's no underlying NUMA architecture, NUMA initialization 269 * fails, or NUMA is disabled on the command line. 270 * 271 * Must online at least one node (node 0) and add memory blocks that cover all 272 * allowed memory. It is unlikely that this function fails. 273 * 274 * Return: 0 on success, -errno on failure. 275 */ 276 static int __init dummy_numa_init(void) 277 { 278 phys_addr_t start = memblock_start_of_DRAM(); 279 phys_addr_t end = memblock_end_of_DRAM() - 1; 280 int ret; 281 282 if (numa_off) 283 pr_info("NUMA disabled\n"); /* Forced off on command line. */ 284 pr_info("Faking a node at [mem %pap-%pap]\n", &start, &end); 285 286 ret = numa_add_memblk(0, start, end + 1); 287 if (ret) { 288 pr_err("NUMA init failed\n"); 289 return ret; 290 } 291 292 numa_off = true; 293 return 0; 294 } 295 296 #ifdef CONFIG_ACPI_NUMA 297 static int __init arch_acpi_numa_init(void) 298 { 299 int ret; 300 301 ret = acpi_numa_init(); 302 if (ret) { 303 pr_debug("Failed to initialise from firmware\n"); 304 return ret; 305 } 306 307 return srat_disabled() ? -EINVAL : 0; 308 } 309 #else 310 static int __init arch_acpi_numa_init(void) 311 { 312 return -EOPNOTSUPP; 313 } 314 #endif 315 316 /** 317 * arch_numa_init() - Initialize NUMA 318 * 319 * Try each configured NUMA initialization method until one succeeds. The 320 * last fallback is dummy single node config encompassing whole memory. 321 */ 322 void __init arch_numa_init(void) 323 { 324 if (!numa_off) { 325 if (!acpi_disabled && !numa_init(arch_acpi_numa_init)) 326 return; 327 if (acpi_disabled && !numa_init(of_numa_init)) 328 return; 329 } 330 331 numa_init(dummy_numa_init); 332 } 333 334 #ifdef CONFIG_NUMA_EMU 335 void __init numa_emu_update_cpu_to_node(int *emu_nid_to_phys, 336 unsigned int nr_emu_nids) 337 { 338 int i, j; 339 340 /* 341 * Transform cpu_to_node_map table to use emulated nids by 342 * reverse-mapping phys_nid. The maps should always exist but fall 343 * back to zero just in case. 344 */ 345 for (i = 0; i < ARRAY_SIZE(cpu_to_node_map); i++) { 346 if (cpu_to_node_map[i] == NUMA_NO_NODE) 347 continue; 348 for (j = 0; j < nr_emu_nids; j++) 349 if (cpu_to_node_map[i] == emu_nid_to_phys[j]) 350 break; 351 cpu_to_node_map[i] = j < nr_emu_nids ? j : 0; 352 } 353 } 354 355 u64 __init numa_emu_dma_end(void) 356 { 357 return memblock_start_of_DRAM() + SZ_4G; 358 } 359 360 void debug_cpumask_set_cpu(unsigned int cpu, int node, bool enable) 361 { 362 struct cpumask *mask; 363 364 if (node == NUMA_NO_NODE) 365 return; 366 367 mask = node_to_cpumask_map[node]; 368 if (!cpumask_available(mask)) { 369 pr_err("node_to_cpumask_map[%i] NULL\n", node); 370 dump_stack(); 371 return; 372 } 373 374 if (enable) 375 cpumask_set_cpu(cpu, mask); 376 else 377 cpumask_clear_cpu(cpu, mask); 378 379 pr_debug("%s cpu %d node %d: mask now %*pbl\n", 380 enable ? "numa_add_cpu" : "numa_remove_cpu", 381 cpu, node, cpumask_pr_args(mask)); 382 } 383 #endif /* CONFIG_NUMA_EMU */ 384