12874c5fdSThomas Gleixner // SPDX-License-Identifier: GPL-2.0-or-later 2ab1f9dacSPaul Mackerras /* 3ab1f9dacSPaul Mackerras * pSeries NUMA support 4ab1f9dacSPaul Mackerras * 5ab1f9dacSPaul Mackerras * Copyright (C) 2002 Anton Blanchard <anton@au.ibm.com>, IBM 6ab1f9dacSPaul Mackerras */ 72d73bae1SNishanth Aravamudan #define pr_fmt(fmt) "numa: " fmt 82d73bae1SNishanth Aravamudan 9ab1f9dacSPaul Mackerras #include <linux/threads.h> 1057c8a661SMike Rapoport #include <linux/memblock.h> 11ab1f9dacSPaul Mackerras #include <linux/init.h> 12ab1f9dacSPaul Mackerras #include <linux/mm.h> 13ab1f9dacSPaul Mackerras #include <linux/mmzone.h> 144b16f8e2SPaul Gortmaker #include <linux/export.h> 15ab1f9dacSPaul Mackerras #include <linux/nodemask.h> 16ab1f9dacSPaul Mackerras #include <linux/cpu.h> 17ab1f9dacSPaul Mackerras #include <linux/notifier.h> 186df1646eSMichael Ellerman #include <linux/of.h> 192500763dSRob Herring #include <linux/of_address.h> 2006eccea6SDave Hansen #include <linux/pfn.h> 219eff1a38SJesse Larrew #include <linux/cpuset.h> 229eff1a38SJesse Larrew #include <linux/node.h> 2330c05350SNathan Fontenot #include <linux/stop_machine.h> 24e04fa612SNathan Fontenot #include <linux/proc_fs.h> 25e04fa612SNathan Fontenot #include <linux/seq_file.h> 26e04fa612SNathan Fontenot #include <linux/uaccess.h> 27191a7120SLinus Torvalds #include <linux/slab.h> 283be7db6aSRobert Jennings #include <asm/cputhreads.h> 2945fb6ceaSAnton Blanchard #include <asm/sparsemem.h> 302249ca9dSPaul Mackerras #include <asm/smp.h> 31d4edc5b6SSrivatsa S. Bhat #include <asm/topology.h> 329eff1a38SJesse Larrew #include <asm/firmware.h> 339eff1a38SJesse Larrew #include <asm/paca.h> 3439bf990eSJesse Larrew #include <asm/hvcall.h> 35ae3a197eSDavid Howells #include <asm/setup.h> 36176bbf14SJesse Larrew #include <asm/vdso.h> 37c040c748SMichael Ellerman #include <asm/vphn.h> 38514a9cb3SNathan Fontenot #include <asm/drmem.h> 39ab1f9dacSPaul Mackerras 40ab1f9dacSPaul Mackerras static int numa_enabled = 1; 41ab1f9dacSPaul Mackerras 421daa6d08SBalbir Singh static char *cmdline __initdata; 431daa6d08SBalbir Singh 4445fb6ceaSAnton Blanchard int numa_cpu_lookup_table[NR_CPUS]; 4525863de0SAnton Blanchard cpumask_var_t node_to_cpumask_map[MAX_NUMNODES]; 46ab1f9dacSPaul Mackerras struct pglist_data *node_data[MAX_NUMNODES]; 4745fb6ceaSAnton Blanchard 4845fb6ceaSAnton Blanchard EXPORT_SYMBOL(numa_cpu_lookup_table); 4925863de0SAnton Blanchard EXPORT_SYMBOL(node_to_cpumask_map); 5045fb6ceaSAnton Blanchard EXPORT_SYMBOL(node_data); 5145fb6ceaSAnton Blanchard 527e35ef66SAneesh Kumar K.V static int primary_domain_index; 53237a0989SMike Kravetz static int n_mem_addr_cells, n_mem_size_cells; 540eacd06bSAneesh Kumar K.V 550eacd06bSAneesh Kumar K.V #define FORM0_AFFINITY 0 560eacd06bSAneesh Kumar K.V #define FORM1_AFFINITY 1 571c6b5a7eSAneesh Kumar K.V #define FORM2_AFFINITY 2 580eacd06bSAneesh Kumar K.V static int affinity_form; 5941eab6f8SAnton Blanchard 6041eab6f8SAnton Blanchard #define MAX_DISTANCE_REF_POINTS 4 6141eab6f8SAnton Blanchard static int distance_ref_points_depth; 62b08a2a12SAlistair Popple static const __be32 *distance_ref_points; 6341eab6f8SAnton Blanchard static int distance_lookup_table[MAX_NUMNODES][MAX_DISTANCE_REF_POINTS]; 641c6b5a7eSAneesh Kumar K.V static int numa_distance_table[MAX_NUMNODES][MAX_NUMNODES] = { 651c6b5a7eSAneesh Kumar K.V [0 ... MAX_NUMNODES - 1] = { [0 ... MAX_NUMNODES - 1] = -1 } 661c6b5a7eSAneesh Kumar K.V }; 671c6b5a7eSAneesh Kumar K.V static int numa_id_index_table[MAX_NUMNODES] = { [0 ... MAX_NUMNODES - 1] = NUMA_NO_NODE }; 68ab1f9dacSPaul Mackerras 6925863de0SAnton Blanchard /* 7025863de0SAnton Blanchard * Allocate node_to_cpumask_map based on number of available nodes 7125863de0SAnton Blanchard * Requires node_possible_map to be valid. 7225863de0SAnton Blanchard * 739512938bSWanlong Gao * Note: cpumask_of_node() is not valid until after this is done. 7425863de0SAnton Blanchard */ 7525863de0SAnton Blanchard static void __init setup_node_to_cpumask_map(void) 7625863de0SAnton Blanchard { 77f9d531b8SCody P Schafer unsigned int node; 7825863de0SAnton Blanchard 7925863de0SAnton Blanchard /* setup nr_node_ids if not done yet */ 80f9d531b8SCody P Schafer if (nr_node_ids == MAX_NUMNODES) 81f9d531b8SCody P Schafer setup_nr_node_ids(); 8225863de0SAnton Blanchard 8325863de0SAnton Blanchard /* allocate the map */ 84c118baf8SRaghavendra K T for_each_node(node) 8525863de0SAnton Blanchard alloc_bootmem_cpumask_var(&node_to_cpumask_map[node]); 8625863de0SAnton Blanchard 8725863de0SAnton Blanchard /* cpumask_of_node() will now work */ 88544af642SSrikar Dronamraju pr_debug("Node to cpumask map for %u nodes\n", nr_node_ids); 8925863de0SAnton Blanchard } 9025863de0SAnton Blanchard 9155671f3cSStephen Rothwell static int __init fake_numa_create_new_node(unsigned long end_pfn, 921daa6d08SBalbir Singh unsigned int *nid) 931daa6d08SBalbir Singh { 941daa6d08SBalbir Singh unsigned long long mem; 951daa6d08SBalbir Singh char *p = cmdline; 961daa6d08SBalbir Singh static unsigned int fake_nid; 971daa6d08SBalbir Singh static unsigned long long curr_boundary; 981daa6d08SBalbir Singh 991daa6d08SBalbir Singh /* 1001daa6d08SBalbir Singh * Modify node id, iff we started creating NUMA nodes 1011daa6d08SBalbir Singh * We want to continue from where we left of the last time 1021daa6d08SBalbir Singh */ 1031daa6d08SBalbir Singh if (fake_nid) 1041daa6d08SBalbir Singh *nid = fake_nid; 1051daa6d08SBalbir Singh /* 1061daa6d08SBalbir Singh * In case there are no more arguments to parse, the 1071daa6d08SBalbir Singh * node_id should be the same as the last fake node id 1081daa6d08SBalbir Singh * (we've handled this above). 1091daa6d08SBalbir Singh */ 1101daa6d08SBalbir Singh if (!p) 1111daa6d08SBalbir Singh return 0; 1121daa6d08SBalbir Singh 1131daa6d08SBalbir Singh mem = memparse(p, &p); 1141daa6d08SBalbir Singh if (!mem) 1151daa6d08SBalbir Singh return 0; 1161daa6d08SBalbir Singh 1171daa6d08SBalbir Singh if (mem < curr_boundary) 1181daa6d08SBalbir Singh return 0; 1191daa6d08SBalbir Singh 1201daa6d08SBalbir Singh curr_boundary = mem; 1211daa6d08SBalbir Singh 1221daa6d08SBalbir Singh if ((end_pfn << PAGE_SHIFT) > mem) { 1231daa6d08SBalbir Singh /* 1241daa6d08SBalbir Singh * Skip commas and spaces 1251daa6d08SBalbir Singh */ 1261daa6d08SBalbir Singh while (*p == ',' || *p == ' ' || *p == '\t') 1271daa6d08SBalbir Singh p++; 1281daa6d08SBalbir Singh 1291daa6d08SBalbir Singh cmdline = p; 1301daa6d08SBalbir Singh fake_nid++; 1311daa6d08SBalbir Singh *nid = fake_nid; 132544af642SSrikar Dronamraju pr_debug("created new fake_node with id %d\n", fake_nid); 1331daa6d08SBalbir Singh return 1; 1341daa6d08SBalbir Singh } 1351daa6d08SBalbir Singh return 0; 1361daa6d08SBalbir Singh } 1371daa6d08SBalbir Singh 138c13f2b2bSNick Child static void __init reset_numa_cpu_lookup_table(void) 139d4edc5b6SSrivatsa S. Bhat { 140d4edc5b6SSrivatsa S. Bhat unsigned int cpu; 141d4edc5b6SSrivatsa S. Bhat 142d4edc5b6SSrivatsa S. Bhat for_each_possible_cpu(cpu) 143d4edc5b6SSrivatsa S. Bhat numa_cpu_lookup_table[cpu] = -1; 144d4edc5b6SSrivatsa S. Bhat } 145d4edc5b6SSrivatsa S. Bhat 1469a245d0eSSrikar Dronamraju void map_cpu_to_node(int cpu, int node) 147d4edc5b6SSrivatsa S. Bhat { 148d4edc5b6SSrivatsa S. Bhat update_numa_cpu_lookup_table(cpu, node); 14945fb6ceaSAnton Blanchard 150544a09eeSSrikar Dronamraju if (!(cpumask_test_cpu(cpu, node_to_cpumask_map[node]))) { 151544af642SSrikar Dronamraju pr_debug("adding cpu %d to node %d\n", cpu, node); 15225863de0SAnton Blanchard cpumask_set_cpu(cpu, node_to_cpumask_map[node]); 153ab1f9dacSPaul Mackerras } 154544a09eeSSrikar Dronamraju } 155ab1f9dacSPaul Mackerras 15639bf990eSJesse Larrew #if defined(CONFIG_HOTPLUG_CPU) || defined(CONFIG_PPC_SPLPAR) 1579a245d0eSSrikar Dronamraju void unmap_cpu_from_node(unsigned long cpu) 158ab1f9dacSPaul Mackerras { 159ab1f9dacSPaul Mackerras int node = numa_cpu_lookup_table[cpu]; 160ab1f9dacSPaul Mackerras 16125863de0SAnton Blanchard if (cpumask_test_cpu(cpu, node_to_cpumask_map[node])) { 162429f4d8dSAnton Blanchard cpumask_clear_cpu(cpu, node_to_cpumask_map[node]); 163544a09eeSSrikar Dronamraju pr_debug("removing cpu %lu from node %d\n", cpu, node); 164ab1f9dacSPaul Mackerras } else { 165506c2075SSrikar Dronamraju pr_warn("Warning: cpu %lu not found in node %d\n", cpu, node); 166ab1f9dacSPaul Mackerras } 167ab1f9dacSPaul Mackerras } 16839bf990eSJesse Larrew #endif /* CONFIG_HOTPLUG_CPU || CONFIG_PPC_SPLPAR */ 169ab1f9dacSPaul Mackerras 1708ddc6448SAneesh Kumar K.V static int __associativity_to_nid(const __be32 *associativity, 1718ddc6448SAneesh Kumar K.V int max_array_sz) 17241eab6f8SAnton Blanchard { 1738ddc6448SAneesh Kumar K.V int nid; 1748ddc6448SAneesh Kumar K.V /* 1758ddc6448SAneesh Kumar K.V * primary_domain_index is 1 based array index. 1768ddc6448SAneesh Kumar K.V */ 1778ddc6448SAneesh Kumar K.V int index = primary_domain_index - 1; 17841eab6f8SAnton Blanchard 1798ddc6448SAneesh Kumar K.V if (!numa_enabled || index >= max_array_sz) 1808ddc6448SAneesh Kumar K.V return NUMA_NO_NODE; 18141eab6f8SAnton Blanchard 1828ddc6448SAneesh Kumar K.V nid = of_read_number(&associativity[index], 1); 183b08a2a12SAlistair Popple 1848ddc6448SAneesh Kumar K.V /* POWER4 LPAR uses 0xffff as invalid node */ 1858ddc6448SAneesh Kumar K.V if (nid == 0xffff || nid >= nr_node_ids) 1868ddc6448SAneesh Kumar K.V nid = NUMA_NO_NODE; 1878ddc6448SAneesh Kumar K.V return nid; 18841eab6f8SAnton Blanchard } 189dbce4562SSrikar Dronamraju /* 190dbce4562SSrikar Dronamraju * Returns nid in the range [0..nr_node_ids], or -1 if no useful NUMA 191482ec7c4SNathan Lynch * info is found. 192482ec7c4SNathan Lynch */ 193b08a2a12SAlistair Popple static int associativity_to_nid(const __be32 *associativity) 194ab1f9dacSPaul Mackerras { 1958ddc6448SAneesh Kumar K.V int array_sz = of_read_number(associativity, 1); 196ab1f9dacSPaul Mackerras 1978ddc6448SAneesh Kumar K.V /* Skip the first element in the associativity array */ 1988ddc6448SAneesh Kumar K.V return __associativity_to_nid((associativity + 1), array_sz); 199ab1f9dacSPaul Mackerras } 200ab1f9dacSPaul Mackerras 2011c6b5a7eSAneesh Kumar K.V static int __cpu_form2_relative_distance(__be32 *cpu1_assoc, __be32 *cpu2_assoc) 2021c6b5a7eSAneesh Kumar K.V { 2031c6b5a7eSAneesh Kumar K.V int dist; 2041c6b5a7eSAneesh Kumar K.V int node1, node2; 2051c6b5a7eSAneesh Kumar K.V 2061c6b5a7eSAneesh Kumar K.V node1 = associativity_to_nid(cpu1_assoc); 2071c6b5a7eSAneesh Kumar K.V node2 = associativity_to_nid(cpu2_assoc); 2081c6b5a7eSAneesh Kumar K.V 2091c6b5a7eSAneesh Kumar K.V dist = numa_distance_table[node1][node2]; 2101c6b5a7eSAneesh Kumar K.V if (dist <= LOCAL_DISTANCE) 2111c6b5a7eSAneesh Kumar K.V return 0; 2121c6b5a7eSAneesh Kumar K.V else if (dist <= REMOTE_DISTANCE) 2131c6b5a7eSAneesh Kumar K.V return 1; 2141c6b5a7eSAneesh Kumar K.V else 2151c6b5a7eSAneesh Kumar K.V return 2; 2161c6b5a7eSAneesh Kumar K.V } 2171c6b5a7eSAneesh Kumar K.V 2181c6b5a7eSAneesh Kumar K.V static int __cpu_form1_relative_distance(__be32 *cpu1_assoc, __be32 *cpu2_assoc) 2191c6b5a7eSAneesh Kumar K.V { 2201c6b5a7eSAneesh Kumar K.V int dist = 0; 2211c6b5a7eSAneesh Kumar K.V 2221c6b5a7eSAneesh Kumar K.V int i, index; 2231c6b5a7eSAneesh Kumar K.V 2241c6b5a7eSAneesh Kumar K.V for (i = 0; i < distance_ref_points_depth; i++) { 2251c6b5a7eSAneesh Kumar K.V index = be32_to_cpu(distance_ref_points[i]); 2261c6b5a7eSAneesh Kumar K.V if (cpu1_assoc[index] == cpu2_assoc[index]) 2271c6b5a7eSAneesh Kumar K.V break; 2281c6b5a7eSAneesh Kumar K.V dist++; 2291c6b5a7eSAneesh Kumar K.V } 2301c6b5a7eSAneesh Kumar K.V 2311c6b5a7eSAneesh Kumar K.V return dist; 2321c6b5a7eSAneesh Kumar K.V } 2331c6b5a7eSAneesh Kumar K.V 2341c6b5a7eSAneesh Kumar K.V int cpu_relative_distance(__be32 *cpu1_assoc, __be32 *cpu2_assoc) 2351c6b5a7eSAneesh Kumar K.V { 2361c6b5a7eSAneesh Kumar K.V /* We should not get called with FORM0 */ 2371c6b5a7eSAneesh Kumar K.V VM_WARN_ON(affinity_form == FORM0_AFFINITY); 2381c6b5a7eSAneesh Kumar K.V if (affinity_form == FORM1_AFFINITY) 2391c6b5a7eSAneesh Kumar K.V return __cpu_form1_relative_distance(cpu1_assoc, cpu2_assoc); 2401c6b5a7eSAneesh Kumar K.V return __cpu_form2_relative_distance(cpu1_assoc, cpu2_assoc); 2411c6b5a7eSAneesh Kumar K.V } 2421c6b5a7eSAneesh Kumar K.V 2431c6b5a7eSAneesh Kumar K.V /* must hold reference to node during call */ 2441c6b5a7eSAneesh Kumar K.V static const __be32 *of_get_associativity(struct device_node *dev) 2451c6b5a7eSAneesh Kumar K.V { 2461c6b5a7eSAneesh Kumar K.V return of_get_property(dev, "ibm,associativity", NULL); 2471c6b5a7eSAneesh Kumar K.V } 2481c6b5a7eSAneesh Kumar K.V 2491c6b5a7eSAneesh Kumar K.V int __node_distance(int a, int b) 2501c6b5a7eSAneesh Kumar K.V { 2511c6b5a7eSAneesh Kumar K.V int i; 2521c6b5a7eSAneesh Kumar K.V int distance = LOCAL_DISTANCE; 2531c6b5a7eSAneesh Kumar K.V 2541c6b5a7eSAneesh Kumar K.V if (affinity_form == FORM2_AFFINITY) 2551c6b5a7eSAneesh Kumar K.V return numa_distance_table[a][b]; 2561c6b5a7eSAneesh Kumar K.V else if (affinity_form == FORM0_AFFINITY) 2571c6b5a7eSAneesh Kumar K.V return ((a == b) ? LOCAL_DISTANCE : REMOTE_DISTANCE); 2581c6b5a7eSAneesh Kumar K.V 2591c6b5a7eSAneesh Kumar K.V for (i = 0; i < distance_ref_points_depth; i++) { 2601c6b5a7eSAneesh Kumar K.V if (distance_lookup_table[a][i] == distance_lookup_table[b][i]) 2611c6b5a7eSAneesh Kumar K.V break; 2621c6b5a7eSAneesh Kumar K.V 2631c6b5a7eSAneesh Kumar K.V /* Double the distance for each NUMA level */ 2641c6b5a7eSAneesh Kumar K.V distance *= 2; 2651c6b5a7eSAneesh Kumar K.V } 2661c6b5a7eSAneesh Kumar K.V 2671c6b5a7eSAneesh Kumar K.V return distance; 2681c6b5a7eSAneesh Kumar K.V } 2691c6b5a7eSAneesh Kumar K.V EXPORT_SYMBOL(__node_distance); 2701c6b5a7eSAneesh Kumar K.V 2719eff1a38SJesse Larrew /* Returns the nid associated with the given device tree node, 2729eff1a38SJesse Larrew * or -1 if not found. 2739eff1a38SJesse Larrew */ 2749eff1a38SJesse Larrew static int of_node_to_nid_single(struct device_node *device) 2759eff1a38SJesse Larrew { 27698fa15f3SAnshuman Khandual int nid = NUMA_NO_NODE; 277b08a2a12SAlistair Popple const __be32 *tmp; 2789eff1a38SJesse Larrew 2799eff1a38SJesse Larrew tmp = of_get_associativity(device); 2809eff1a38SJesse Larrew if (tmp) 2819eff1a38SJesse Larrew nid = associativity_to_nid(tmp); 2829eff1a38SJesse Larrew return nid; 2839eff1a38SJesse Larrew } 2849eff1a38SJesse Larrew 285953039c8SJeremy Kerr /* Walk the device tree upwards, looking for an associativity id */ 286953039c8SJeremy Kerr int of_node_to_nid(struct device_node *device) 287953039c8SJeremy Kerr { 28898fa15f3SAnshuman Khandual int nid = NUMA_NO_NODE; 289953039c8SJeremy Kerr 290953039c8SJeremy Kerr of_node_get(device); 291953039c8SJeremy Kerr while (device) { 292953039c8SJeremy Kerr nid = of_node_to_nid_single(device); 293953039c8SJeremy Kerr if (nid != -1) 294953039c8SJeremy Kerr break; 295953039c8SJeremy Kerr 2961def3758SChristophe Jaillet device = of_get_next_parent(device); 297953039c8SJeremy Kerr } 298953039c8SJeremy Kerr of_node_put(device); 299953039c8SJeremy Kerr 300953039c8SJeremy Kerr return nid; 301953039c8SJeremy Kerr } 302be9ba9ffSShailendra Singh EXPORT_SYMBOL(of_node_to_nid); 303953039c8SJeremy Kerr 3048ddc6448SAneesh Kumar K.V static void __initialize_form1_numa_distance(const __be32 *associativity, 3058ddc6448SAneesh Kumar K.V int max_array_sz) 3068ddc6448SAneesh Kumar K.V { 3078ddc6448SAneesh Kumar K.V int i, nid; 3088ddc6448SAneesh Kumar K.V 3098ddc6448SAneesh Kumar K.V if (affinity_form != FORM1_AFFINITY) 3108ddc6448SAneesh Kumar K.V return; 3118ddc6448SAneesh Kumar K.V 3128ddc6448SAneesh Kumar K.V nid = __associativity_to_nid(associativity, max_array_sz); 3138ddc6448SAneesh Kumar K.V if (nid != NUMA_NO_NODE) { 3148ddc6448SAneesh Kumar K.V for (i = 0; i < distance_ref_points_depth; i++) { 3158ddc6448SAneesh Kumar K.V const __be32 *entry; 3168ddc6448SAneesh Kumar K.V int index = be32_to_cpu(distance_ref_points[i]) - 1; 3178ddc6448SAneesh Kumar K.V 3188ddc6448SAneesh Kumar K.V /* 3198ddc6448SAneesh Kumar K.V * broken hierarchy, return with broken distance table 3208ddc6448SAneesh Kumar K.V */ 3218ddc6448SAneesh Kumar K.V if (WARN(index >= max_array_sz, "Broken ibm,associativity property")) 3228ddc6448SAneesh Kumar K.V return; 3238ddc6448SAneesh Kumar K.V 3248ddc6448SAneesh Kumar K.V entry = &associativity[index]; 3258ddc6448SAneesh Kumar K.V distance_lookup_table[nid][i] = of_read_number(entry, 1); 3268ddc6448SAneesh Kumar K.V } 3278ddc6448SAneesh Kumar K.V } 3288ddc6448SAneesh Kumar K.V } 3298ddc6448SAneesh Kumar K.V 3308ddc6448SAneesh Kumar K.V static void initialize_form1_numa_distance(const __be32 *associativity) 3318ddc6448SAneesh Kumar K.V { 3328ddc6448SAneesh Kumar K.V int array_sz; 3338ddc6448SAneesh Kumar K.V 3348ddc6448SAneesh Kumar K.V array_sz = of_read_number(associativity, 1); 3358ddc6448SAneesh Kumar K.V /* Skip the first element in the associativity array */ 3368ddc6448SAneesh Kumar K.V __initialize_form1_numa_distance(associativity + 1, array_sz); 3378ddc6448SAneesh Kumar K.V } 3388ddc6448SAneesh Kumar K.V 3398ddc6448SAneesh Kumar K.V /* 3408ddc6448SAneesh Kumar K.V * Used to update distance information w.r.t newly added node. 3418ddc6448SAneesh Kumar K.V */ 3428ddc6448SAneesh Kumar K.V void update_numa_distance(struct device_node *node) 3438ddc6448SAneesh Kumar K.V { 3441c6b5a7eSAneesh Kumar K.V int nid; 3451c6b5a7eSAneesh Kumar K.V 3468ddc6448SAneesh Kumar K.V if (affinity_form == FORM0_AFFINITY) 3478ddc6448SAneesh Kumar K.V return; 3488ddc6448SAneesh Kumar K.V else if (affinity_form == FORM1_AFFINITY) { 3498ddc6448SAneesh Kumar K.V const __be32 *associativity; 3508ddc6448SAneesh Kumar K.V 3518ddc6448SAneesh Kumar K.V associativity = of_get_associativity(node); 3528ddc6448SAneesh Kumar K.V if (!associativity) 3538ddc6448SAneesh Kumar K.V return; 3548ddc6448SAneesh Kumar K.V 3558ddc6448SAneesh Kumar K.V initialize_form1_numa_distance(associativity); 3568ddc6448SAneesh Kumar K.V return; 3578ddc6448SAneesh Kumar K.V } 3581c6b5a7eSAneesh Kumar K.V 3591c6b5a7eSAneesh Kumar K.V /* FORM2 affinity */ 3601c6b5a7eSAneesh Kumar K.V nid = of_node_to_nid_single(node); 3611c6b5a7eSAneesh Kumar K.V if (nid == NUMA_NO_NODE) 3621c6b5a7eSAneesh Kumar K.V return; 3631c6b5a7eSAneesh Kumar K.V 3641c6b5a7eSAneesh Kumar K.V /* 3651c6b5a7eSAneesh Kumar K.V * With FORM2 we expect NUMA distance of all possible NUMA 3661c6b5a7eSAneesh Kumar K.V * nodes to be provided during boot. 3671c6b5a7eSAneesh Kumar K.V */ 3681c6b5a7eSAneesh Kumar K.V WARN(numa_distance_table[nid][nid] == -1, 3691c6b5a7eSAneesh Kumar K.V "NUMA distance details for node %d not provided\n", nid); 3701c6b5a7eSAneesh Kumar K.V } 371b277fc79SAneesh Kumar K.V EXPORT_SYMBOL_GPL(update_numa_distance); 3721c6b5a7eSAneesh Kumar K.V 3731c6b5a7eSAneesh Kumar K.V /* 3741c6b5a7eSAneesh Kumar K.V * ibm,numa-lookup-index-table= {N, domainid1, domainid2, ..... domainidN} 3751c6b5a7eSAneesh Kumar K.V * ibm,numa-distance-table = { N, 1, 2, 4, 5, 1, 6, .... N elements} 3761c6b5a7eSAneesh Kumar K.V */ 377c13f2b2bSNick Child static void __init initialize_form2_numa_distance_lookup_table(void) 3781c6b5a7eSAneesh Kumar K.V { 3791c6b5a7eSAneesh Kumar K.V int i, j; 3801c6b5a7eSAneesh Kumar K.V struct device_node *root; 3810bd81274SNicholas Piggin const __u8 *form2_distances; 3821c6b5a7eSAneesh Kumar K.V const __be32 *numa_lookup_index; 3830bd81274SNicholas Piggin int form2_distances_length; 3841c6b5a7eSAneesh Kumar K.V int max_numa_index, distance_index; 3851c6b5a7eSAneesh Kumar K.V 3861c6b5a7eSAneesh Kumar K.V if (firmware_has_feature(FW_FEATURE_OPAL)) 3871c6b5a7eSAneesh Kumar K.V root = of_find_node_by_path("/ibm,opal"); 3881c6b5a7eSAneesh Kumar K.V else 3891c6b5a7eSAneesh Kumar K.V root = of_find_node_by_path("/rtas"); 3901c6b5a7eSAneesh Kumar K.V if (!root) 3911c6b5a7eSAneesh Kumar K.V root = of_find_node_by_path("/"); 3921c6b5a7eSAneesh Kumar K.V 3931c6b5a7eSAneesh Kumar K.V numa_lookup_index = of_get_property(root, "ibm,numa-lookup-index-table", NULL); 3941c6b5a7eSAneesh Kumar K.V max_numa_index = of_read_number(&numa_lookup_index[0], 1); 3951c6b5a7eSAneesh Kumar K.V 3961c6b5a7eSAneesh Kumar K.V /* first element of the array is the size and is encode-int */ 3970bd81274SNicholas Piggin form2_distances = of_get_property(root, "ibm,numa-distance-table", NULL); 3980bd81274SNicholas Piggin form2_distances_length = of_read_number((const __be32 *)&form2_distances[0], 1); 3991c6b5a7eSAneesh Kumar K.V /* Skip the size which is encoded int */ 4000bd81274SNicholas Piggin form2_distances += sizeof(__be32); 4011c6b5a7eSAneesh Kumar K.V 4020bd81274SNicholas Piggin pr_debug("form2_distances_len = %d, numa_dist_indexes_len = %d\n", 4030bd81274SNicholas Piggin form2_distances_length, max_numa_index); 4041c6b5a7eSAneesh Kumar K.V 4051c6b5a7eSAneesh Kumar K.V for (i = 0; i < max_numa_index; i++) 4061c6b5a7eSAneesh Kumar K.V /* +1 skip the max_numa_index in the property */ 4071c6b5a7eSAneesh Kumar K.V numa_id_index_table[i] = of_read_number(&numa_lookup_index[i + 1], 1); 4081c6b5a7eSAneesh Kumar K.V 4091c6b5a7eSAneesh Kumar K.V 4100bd81274SNicholas Piggin if (form2_distances_length != max_numa_index * max_numa_index) { 4111c6b5a7eSAneesh Kumar K.V WARN(1, "Wrong NUMA distance information\n"); 41230203946SNicholas Piggin form2_distances = NULL; // don't use it 4131c6b5a7eSAneesh Kumar K.V } 4141c6b5a7eSAneesh Kumar K.V distance_index = 0; 4151c6b5a7eSAneesh Kumar K.V for (i = 0; i < max_numa_index; i++) { 4161c6b5a7eSAneesh Kumar K.V for (j = 0; j < max_numa_index; j++) { 4171c6b5a7eSAneesh Kumar K.V int nodeA = numa_id_index_table[i]; 4181c6b5a7eSAneesh Kumar K.V int nodeB = numa_id_index_table[j]; 41930203946SNicholas Piggin int dist; 4201c6b5a7eSAneesh Kumar K.V 42130203946SNicholas Piggin if (form2_distances) 42230203946SNicholas Piggin dist = form2_distances[distance_index++]; 42330203946SNicholas Piggin else if (nodeA == nodeB) 42430203946SNicholas Piggin dist = LOCAL_DISTANCE; 42530203946SNicholas Piggin else 42630203946SNicholas Piggin dist = REMOTE_DISTANCE; 42730203946SNicholas Piggin numa_distance_table[nodeA][nodeB] = dist; 42830203946SNicholas Piggin pr_debug("dist[%d][%d]=%d ", nodeA, nodeB, dist); 4291c6b5a7eSAneesh Kumar K.V } 4301c6b5a7eSAneesh Kumar K.V } 43130203946SNicholas Piggin 4321c6b5a7eSAneesh Kumar K.V of_node_put(root); 4338ddc6448SAneesh Kumar K.V } 4348ddc6448SAneesh Kumar K.V 4357e35ef66SAneesh Kumar K.V static int __init find_primary_domain_index(void) 436ab1f9dacSPaul Mackerras { 4377e35ef66SAneesh Kumar K.V int index; 438e70606ebSMichael Ellerman struct device_node *root; 439ab1f9dacSPaul Mackerras 4400eacd06bSAneesh Kumar K.V /* 4410eacd06bSAneesh Kumar K.V * Check for which form of affinity. 4420eacd06bSAneesh Kumar K.V */ 4430eacd06bSAneesh Kumar K.V if (firmware_has_feature(FW_FEATURE_OPAL)) { 4440eacd06bSAneesh Kumar K.V affinity_form = FORM1_AFFINITY; 4451c6b5a7eSAneesh Kumar K.V } else if (firmware_has_feature(FW_FEATURE_FORM2_AFFINITY)) { 446544af642SSrikar Dronamraju pr_debug("Using form 2 affinity\n"); 4471c6b5a7eSAneesh Kumar K.V affinity_form = FORM2_AFFINITY; 4480eacd06bSAneesh Kumar K.V } else if (firmware_has_feature(FW_FEATURE_FORM1_AFFINITY)) { 449544af642SSrikar Dronamraju pr_debug("Using form 1 affinity\n"); 4500eacd06bSAneesh Kumar K.V affinity_form = FORM1_AFFINITY; 4510eacd06bSAneesh Kumar K.V } else 4520eacd06bSAneesh Kumar K.V affinity_form = FORM0_AFFINITY; 4530eacd06bSAneesh Kumar K.V 4541c8ee733SDipankar Sarma if (firmware_has_feature(FW_FEATURE_OPAL)) 4551c8ee733SDipankar Sarma root = of_find_node_by_path("/ibm,opal"); 4561c8ee733SDipankar Sarma else 457e70606ebSMichael Ellerman root = of_find_node_by_path("/rtas"); 458e70606ebSMichael Ellerman if (!root) 459e70606ebSMichael Ellerman root = of_find_node_by_path("/"); 460ab1f9dacSPaul Mackerras 461ab1f9dacSPaul Mackerras /* 46241eab6f8SAnton Blanchard * This property is a set of 32-bit integers, each representing 46341eab6f8SAnton Blanchard * an index into the ibm,associativity nodes. 46441eab6f8SAnton Blanchard * 46541eab6f8SAnton Blanchard * With form 0 affinity the first integer is for an SMP configuration 46641eab6f8SAnton Blanchard * (should be all 0's) and the second is for a normal NUMA 46741eab6f8SAnton Blanchard * configuration. We have only one level of NUMA. 46841eab6f8SAnton Blanchard * 46941eab6f8SAnton Blanchard * With form 1 affinity the first integer is the most significant 47041eab6f8SAnton Blanchard * NUMA boundary and the following are progressively less significant 47141eab6f8SAnton Blanchard * boundaries. There can be more than one level of NUMA. 472ab1f9dacSPaul Mackerras */ 473e70606ebSMichael Ellerman distance_ref_points = of_get_property(root, 47441eab6f8SAnton Blanchard "ibm,associativity-reference-points", 47541eab6f8SAnton Blanchard &distance_ref_points_depth); 476ab1f9dacSPaul Mackerras 47741eab6f8SAnton Blanchard if (!distance_ref_points) { 478506c2075SSrikar Dronamraju pr_debug("ibm,associativity-reference-points not found.\n"); 47941eab6f8SAnton Blanchard goto err; 48041eab6f8SAnton Blanchard } 48141eab6f8SAnton Blanchard 48241eab6f8SAnton Blanchard distance_ref_points_depth /= sizeof(int); 4830eacd06bSAneesh Kumar K.V if (affinity_form == FORM0_AFFINITY) { 48441eab6f8SAnton Blanchard if (distance_ref_points_depth < 2) { 485506c2075SSrikar Dronamraju pr_warn("short ibm,associativity-reference-points\n"); 48641eab6f8SAnton Blanchard goto err; 487ab1f9dacSPaul Mackerras } 488ab1f9dacSPaul Mackerras 4897e35ef66SAneesh Kumar K.V index = of_read_number(&distance_ref_points[1], 1); 4900eacd06bSAneesh Kumar K.V } else { 4911c6b5a7eSAneesh Kumar K.V /* 4921c6b5a7eSAneesh Kumar K.V * Both FORM1 and FORM2 affinity find the primary domain details 4931c6b5a7eSAneesh Kumar K.V * at the same offset. 4941c6b5a7eSAneesh Kumar K.V */ 4950eacd06bSAneesh Kumar K.V index = of_read_number(distance_ref_points, 1); 49641eab6f8SAnton Blanchard } 49741eab6f8SAnton Blanchard /* 49841eab6f8SAnton Blanchard * Warn and cap if the hardware supports more than 49941eab6f8SAnton Blanchard * MAX_DISTANCE_REF_POINTS domains. 50041eab6f8SAnton Blanchard */ 50141eab6f8SAnton Blanchard if (distance_ref_points_depth > MAX_DISTANCE_REF_POINTS) { 502506c2075SSrikar Dronamraju pr_warn("distance array capped at %d entries\n", 503506c2075SSrikar Dronamraju MAX_DISTANCE_REF_POINTS); 50441eab6f8SAnton Blanchard distance_ref_points_depth = MAX_DISTANCE_REF_POINTS; 50541eab6f8SAnton Blanchard } 50641eab6f8SAnton Blanchard 507e70606ebSMichael Ellerman of_node_put(root); 5087e35ef66SAneesh Kumar K.V return index; 50941eab6f8SAnton Blanchard 51041eab6f8SAnton Blanchard err: 511e70606ebSMichael Ellerman of_node_put(root); 51241eab6f8SAnton Blanchard return -1; 513ab1f9dacSPaul Mackerras } 514ab1f9dacSPaul Mackerras 51584c9fdd1SMike Kravetz static void __init get_n_mem_cells(int *n_addr_cells, int *n_size_cells) 516ab1f9dacSPaul Mackerras { 517ab1f9dacSPaul Mackerras struct device_node *memory = NULL; 518ab1f9dacSPaul Mackerras 519ab1f9dacSPaul Mackerras memory = of_find_node_by_type(memory, "memory"); 52054c23310SPaul Mackerras if (!memory) 52184c9fdd1SMike Kravetz panic("numa.c: No memory nodes found!"); 52254c23310SPaul Mackerras 523a8bda5ddSStephen Rothwell *n_addr_cells = of_n_addr_cells(memory); 5249213feeaSStephen Rothwell *n_size_cells = of_n_size_cells(memory); 52584c9fdd1SMike Kravetz of_node_put(memory); 526ab1f9dacSPaul Mackerras } 527ab1f9dacSPaul Mackerras 528b08a2a12SAlistair Popple static unsigned long read_n_cells(int n, const __be32 **buf) 529ab1f9dacSPaul Mackerras { 530ab1f9dacSPaul Mackerras unsigned long result = 0; 531ab1f9dacSPaul Mackerras 532ab1f9dacSPaul Mackerras while (n--) { 533b08a2a12SAlistair Popple result = (result << 32) | of_read_number(*buf, 1); 534ab1f9dacSPaul Mackerras (*buf)++; 535ab1f9dacSPaul Mackerras } 536ab1f9dacSPaul Mackerras return result; 537ab1f9dacSPaul Mackerras } 538ab1f9dacSPaul Mackerras 5398342681dSNathan Fontenot struct assoc_arrays { 5408342681dSNathan Fontenot u32 n_arrays; 5418342681dSNathan Fontenot u32 array_sz; 542b08a2a12SAlistair Popple const __be32 *arrays; 5438342681dSNathan Fontenot }; 5448342681dSNathan Fontenot 5458342681dSNathan Fontenot /* 54625985edcSLucas De Marchi * Retrieve and validate the list of associativity arrays for drconf 5478342681dSNathan Fontenot * memory from the ibm,associativity-lookup-arrays property of the 5488342681dSNathan Fontenot * device tree.. 5498342681dSNathan Fontenot * 5508342681dSNathan Fontenot * The layout of the ibm,associativity-lookup-arrays property is a number N 5518342681dSNathan Fontenot * indicating the number of associativity arrays, followed by a number M 5528342681dSNathan Fontenot * indicating the size of each associativity array, followed by a list 5538342681dSNathan Fontenot * of N associativity arrays. 5548342681dSNathan Fontenot */ 55535f80debSNathan Fontenot static int of_get_assoc_arrays(struct assoc_arrays *aa) 5568342681dSNathan Fontenot { 55735f80debSNathan Fontenot struct device_node *memory; 558b08a2a12SAlistair Popple const __be32 *prop; 5598342681dSNathan Fontenot u32 len; 5608342681dSNathan Fontenot 56135f80debSNathan Fontenot memory = of_find_node_by_path("/ibm,dynamic-reconfiguration-memory"); 56235f80debSNathan Fontenot if (!memory) 5638342681dSNathan Fontenot return -1; 5648342681dSNathan Fontenot 56535f80debSNathan Fontenot prop = of_get_property(memory, "ibm,associativity-lookup-arrays", &len); 56635f80debSNathan Fontenot if (!prop || len < 2 * sizeof(unsigned int)) { 56735f80debSNathan Fontenot of_node_put(memory); 56835f80debSNathan Fontenot return -1; 56935f80debSNathan Fontenot } 57035f80debSNathan Fontenot 571b08a2a12SAlistair Popple aa->n_arrays = of_read_number(prop++, 1); 572b08a2a12SAlistair Popple aa->array_sz = of_read_number(prop++, 1); 5738342681dSNathan Fontenot 57435f80debSNathan Fontenot of_node_put(memory); 57535f80debSNathan Fontenot 57642b2aa86SJustin P. Mattock /* Now that we know the number of arrays and size of each array, 5778342681dSNathan Fontenot * revalidate the size of the property read in. 5788342681dSNathan Fontenot */ 5798342681dSNathan Fontenot if (len < (aa->n_arrays * aa->array_sz + 2) * sizeof(unsigned int)) 5808342681dSNathan Fontenot return -1; 5818342681dSNathan Fontenot 5828342681dSNathan Fontenot aa->arrays = prop; 5838342681dSNathan Fontenot return 0; 5848342681dSNathan Fontenot } 5858342681dSNathan Fontenot 586c13f2b2bSNick Child static int __init get_nid_and_numa_distance(struct drmem_lmb *lmb) 5878ddc6448SAneesh Kumar K.V { 5888ddc6448SAneesh Kumar K.V struct assoc_arrays aa = { .arrays = NULL }; 5898ddc6448SAneesh Kumar K.V int default_nid = NUMA_NO_NODE; 5908ddc6448SAneesh Kumar K.V int nid = default_nid; 5918ddc6448SAneesh Kumar K.V int rc, index; 5928ddc6448SAneesh Kumar K.V 5938ddc6448SAneesh Kumar K.V if ((primary_domain_index < 0) || !numa_enabled) 5948ddc6448SAneesh Kumar K.V return default_nid; 5958ddc6448SAneesh Kumar K.V 5968ddc6448SAneesh Kumar K.V rc = of_get_assoc_arrays(&aa); 5978ddc6448SAneesh Kumar K.V if (rc) 5988ddc6448SAneesh Kumar K.V return default_nid; 5998ddc6448SAneesh Kumar K.V 6008ddc6448SAneesh Kumar K.V if (primary_domain_index <= aa.array_sz && 6018ddc6448SAneesh Kumar K.V !(lmb->flags & DRCONF_MEM_AI_INVALID) && lmb->aa_index < aa.n_arrays) { 6028ddc6448SAneesh Kumar K.V const __be32 *associativity; 6038ddc6448SAneesh Kumar K.V 6048ddc6448SAneesh Kumar K.V index = lmb->aa_index * aa.array_sz; 6058ddc6448SAneesh Kumar K.V associativity = &aa.arrays[index]; 6068ddc6448SAneesh Kumar K.V nid = __associativity_to_nid(associativity, aa.array_sz); 6078ddc6448SAneesh Kumar K.V if (nid > 0 && affinity_form == FORM1_AFFINITY) { 6088ddc6448SAneesh Kumar K.V /* 6098ddc6448SAneesh Kumar K.V * lookup array associativity entries have 6108ddc6448SAneesh Kumar K.V * no length of the array as the first element. 6118ddc6448SAneesh Kumar K.V */ 6128ddc6448SAneesh Kumar K.V __initialize_form1_numa_distance(associativity, aa.array_sz); 6138ddc6448SAneesh Kumar K.V } 6148ddc6448SAneesh Kumar K.V } 6158ddc6448SAneesh Kumar K.V return nid; 6168ddc6448SAneesh Kumar K.V } 6178ddc6448SAneesh Kumar K.V 6188342681dSNathan Fontenot /* 6198342681dSNathan Fontenot * This is like of_node_to_nid_single() for memory represented in the 6208342681dSNathan Fontenot * ibm,dynamic-reconfiguration-memory node. 6218342681dSNathan Fontenot */ 62272cdd117SScott Cheloha int of_drconf_to_nid_single(struct drmem_lmb *lmb) 6238342681dSNathan Fontenot { 624b88fc309SNathan Fontenot struct assoc_arrays aa = { .arrays = NULL }; 625ea9f5b70SAneesh Kumar K.V int default_nid = NUMA_NO_NODE; 6268342681dSNathan Fontenot int nid = default_nid; 627b88fc309SNathan Fontenot int rc, index; 6288342681dSNathan Fontenot 6297e35ef66SAneesh Kumar K.V if ((primary_domain_index < 0) || !numa_enabled) 630ea9f5b70SAneesh Kumar K.V return default_nid; 631ea9f5b70SAneesh Kumar K.V 632b88fc309SNathan Fontenot rc = of_get_assoc_arrays(&aa); 633b88fc309SNathan Fontenot if (rc) 634b88fc309SNathan Fontenot return default_nid; 635b88fc309SNathan Fontenot 6367e35ef66SAneesh Kumar K.V if (primary_domain_index <= aa.array_sz && 637ea9f5b70SAneesh Kumar K.V !(lmb->flags & DRCONF_MEM_AI_INVALID) && lmb->aa_index < aa.n_arrays) { 6388ddc6448SAneesh Kumar K.V const __be32 *associativity; 6398342681dSNathan Fontenot 640514a9cb3SNathan Fontenot index = lmb->aa_index * aa.array_sz; 6418ddc6448SAneesh Kumar K.V associativity = &aa.arrays[index]; 6428ddc6448SAneesh Kumar K.V nid = __associativity_to_nid(associativity, aa.array_sz); 6431d805440SNikunj A Dadhania } 6448342681dSNathan Fontenot return nid; 6458342681dSNathan Fontenot } 6468342681dSNathan Fontenot 647dc909d8bSSrikar Dronamraju #ifdef CONFIG_PPC_SPLPAR 6488ddc6448SAneesh Kumar K.V 6498ddc6448SAneesh Kumar K.V static int __vphn_get_associativity(long lcpu, __be32 *associativity) 650dc909d8bSSrikar Dronamraju { 651dc909d8bSSrikar Dronamraju long rc, hwid; 652dc909d8bSSrikar Dronamraju 653dc909d8bSSrikar Dronamraju /* 654dc909d8bSSrikar Dronamraju * On a shared lpar, device tree will not have node associativity. 655dc909d8bSSrikar Dronamraju * At this time lppaca, or its __old_status field may not be 656dc909d8bSSrikar Dronamraju * updated. Hence kernel cannot detect if its on a shared lpar. So 657dc909d8bSSrikar Dronamraju * request an explicit associativity irrespective of whether the 658dc909d8bSSrikar Dronamraju * lpar is shared or dedicated. Use the device tree property as a 659dc909d8bSSrikar Dronamraju * fallback. cpu_to_phys_id is only valid between 660dc909d8bSSrikar Dronamraju * smp_setup_cpu_maps() and smp_setup_pacas(). 661dc909d8bSSrikar Dronamraju */ 662dc909d8bSSrikar Dronamraju if (firmware_has_feature(FW_FEATURE_VPHN)) { 663dc909d8bSSrikar Dronamraju if (cpu_to_phys_id) 664dc909d8bSSrikar Dronamraju hwid = cpu_to_phys_id[lcpu]; 665dc909d8bSSrikar Dronamraju else 666dc909d8bSSrikar Dronamraju hwid = get_hard_smp_processor_id(lcpu); 667dc909d8bSSrikar Dronamraju 668dc909d8bSSrikar Dronamraju rc = hcall_vphn(hwid, VPHN_FLAG_VCPU, associativity); 669dc909d8bSSrikar Dronamraju if (rc == H_SUCCESS) 6708ddc6448SAneesh Kumar K.V return 0; 671dc909d8bSSrikar Dronamraju } 672dc909d8bSSrikar Dronamraju 6738ddc6448SAneesh Kumar K.V return -1; 6748ddc6448SAneesh Kumar K.V } 6758ddc6448SAneesh Kumar K.V 6768ddc6448SAneesh Kumar K.V static int vphn_get_nid(long lcpu) 6778ddc6448SAneesh Kumar K.V { 6788ddc6448SAneesh Kumar K.V __be32 associativity[VPHN_ASSOC_BUFSIZE] = {0}; 6798ddc6448SAneesh Kumar K.V 6808ddc6448SAneesh Kumar K.V 6818ddc6448SAneesh Kumar K.V if (!__vphn_get_associativity(lcpu, associativity)) 6828ddc6448SAneesh Kumar K.V return associativity_to_nid(associativity); 6838ddc6448SAneesh Kumar K.V 684dc909d8bSSrikar Dronamraju return NUMA_NO_NODE; 6858ddc6448SAneesh Kumar K.V 686dc909d8bSSrikar Dronamraju } 687dc909d8bSSrikar Dronamraju #else 6888ddc6448SAneesh Kumar K.V 6898ddc6448SAneesh Kumar K.V static int __vphn_get_associativity(long lcpu, __be32 *associativity) 6908ddc6448SAneesh Kumar K.V { 6918ddc6448SAneesh Kumar K.V return -1; 6928ddc6448SAneesh Kumar K.V } 6938ddc6448SAneesh Kumar K.V 694dc909d8bSSrikar Dronamraju static int vphn_get_nid(long unused) 695dc909d8bSSrikar Dronamraju { 696dc909d8bSSrikar Dronamraju return NUMA_NO_NODE; 697dc909d8bSSrikar Dronamraju } 698dc909d8bSSrikar Dronamraju #endif /* CONFIG_PPC_SPLPAR */ 699dc909d8bSSrikar Dronamraju 700ab1f9dacSPaul Mackerras /* 701ab1f9dacSPaul Mackerras * Figure out to which domain a cpu belongs and stick it there. 702ab1f9dacSPaul Mackerras * Return the id of the domain used. 703ab1f9dacSPaul Mackerras */ 704061d19f2SPaul Gortmaker static int numa_setup_cpu(unsigned long lcpu) 705ab1f9dacSPaul Mackerras { 706d4edc5b6SSrivatsa S. Bhat struct device_node *cpu; 707413e4055SSrikar Dronamraju int fcpu = cpu_first_thread_sibling(lcpu); 708413e4055SSrikar Dronamraju int nid = NUMA_NO_NODE; 709d4edc5b6SSrivatsa S. Bhat 710a874f100SSrikar Dronamraju if (!cpu_present(lcpu)) { 711a874f100SSrikar Dronamraju set_cpu_numa_node(lcpu, first_online_node); 712a874f100SSrikar Dronamraju return first_online_node; 713a874f100SSrikar Dronamraju } 714a874f100SSrikar Dronamraju 715d4edc5b6SSrivatsa S. Bhat /* 716d4edc5b6SSrivatsa S. Bhat * If a valid cpu-to-node mapping is already available, use it 717d4edc5b6SSrivatsa S. Bhat * directly instead of querying the firmware, since it represents 718d4edc5b6SSrivatsa S. Bhat * the most recent mapping notified to us by the platform (eg: VPHN). 719413e4055SSrikar Dronamraju * Since cpu_to_node binding remains the same for all threads in the 720413e4055SSrikar Dronamraju * core. If a valid cpu-to-node mapping is already available, for 721413e4055SSrikar Dronamraju * the first thread in the core, use it. 722d4edc5b6SSrivatsa S. Bhat */ 723413e4055SSrikar Dronamraju nid = numa_cpu_lookup_table[fcpu]; 724413e4055SSrikar Dronamraju if (nid >= 0) { 725d4edc5b6SSrivatsa S. Bhat map_cpu_to_node(lcpu, nid); 726d4edc5b6SSrivatsa S. Bhat return nid; 727d4edc5b6SSrivatsa S. Bhat } 728d4edc5b6SSrivatsa S. Bhat 729dc909d8bSSrikar Dronamraju nid = vphn_get_nid(lcpu); 730dc909d8bSSrikar Dronamraju if (nid != NUMA_NO_NODE) 731dc909d8bSSrikar Dronamraju goto out_present; 732dc909d8bSSrikar Dronamraju 733d4edc5b6SSrivatsa S. Bhat cpu = of_get_cpu_node(lcpu, NULL); 734ab1f9dacSPaul Mackerras 735ab1f9dacSPaul Mackerras if (!cpu) { 736ab1f9dacSPaul Mackerras WARN_ON(1); 737297cf502SLi Zhong if (cpu_present(lcpu)) 738297cf502SLi Zhong goto out_present; 739297cf502SLi Zhong else 740ab1f9dacSPaul Mackerras goto out; 741ab1f9dacSPaul Mackerras } 742ab1f9dacSPaul Mackerras 743953039c8SJeremy Kerr nid = of_node_to_nid_single(cpu); 744dc909d8bSSrikar Dronamraju of_node_put(cpu); 745ab1f9dacSPaul Mackerras 746297cf502SLi Zhong out_present: 747ea05ba7cSMichael Bringmann if (nid < 0 || !node_possible(nid)) 74872c33688SH Hartley Sweeten nid = first_online_node; 749297cf502SLi Zhong 750413e4055SSrikar Dronamraju /* 751413e4055SSrikar Dronamraju * Update for the first thread of the core. All threads of a core 752413e4055SSrikar Dronamraju * have to be part of the same node. This not only avoids querying 753413e4055SSrikar Dronamraju * for every other thread in the core, but always avoids a case 754413e4055SSrikar Dronamraju * where virtual node associativity change causes subsequent threads 755413e4055SSrikar Dronamraju * of a core to be associated with different nid. However if first 756413e4055SSrikar Dronamraju * thread is already online, expect it to have a valid mapping. 757413e4055SSrikar Dronamraju */ 758413e4055SSrikar Dronamraju if (fcpu != lcpu) { 759413e4055SSrikar Dronamraju WARN_ON(cpu_online(fcpu)); 760413e4055SSrikar Dronamraju map_cpu_to_node(fcpu, nid); 761413e4055SSrikar Dronamraju } 762413e4055SSrikar Dronamraju 763cf950b7aSNathan Lynch map_cpu_to_node(lcpu, nid); 764297cf502SLi Zhong out: 765cf950b7aSNathan Lynch return nid; 766ab1f9dacSPaul Mackerras } 767ab1f9dacSPaul Mackerras 76868fb18aaSSrivatsa S. Bhat static void verify_cpu_node_mapping(int cpu, int node) 76968fb18aaSSrivatsa S. Bhat { 77068fb18aaSSrivatsa S. Bhat int base, sibling, i; 77168fb18aaSSrivatsa S. Bhat 77268fb18aaSSrivatsa S. Bhat /* Verify that all the threads in the core belong to the same node */ 77368fb18aaSSrivatsa S. Bhat base = cpu_first_thread_sibling(cpu); 77468fb18aaSSrivatsa S. Bhat 77568fb18aaSSrivatsa S. Bhat for (i = 0; i < threads_per_core; i++) { 77668fb18aaSSrivatsa S. Bhat sibling = base + i; 77768fb18aaSSrivatsa S. Bhat 77868fb18aaSSrivatsa S. Bhat if (sibling == cpu || cpu_is_offline(sibling)) 77968fb18aaSSrivatsa S. Bhat continue; 78068fb18aaSSrivatsa S. Bhat 78168fb18aaSSrivatsa S. Bhat if (cpu_to_node(sibling) != node) { 78268fb18aaSSrivatsa S. Bhat WARN(1, "CPU thread siblings %d and %d don't belong" 78368fb18aaSSrivatsa S. Bhat " to the same node!\n", cpu, sibling); 78468fb18aaSSrivatsa S. Bhat break; 78568fb18aaSSrivatsa S. Bhat } 78668fb18aaSSrivatsa S. Bhat } 78768fb18aaSSrivatsa S. Bhat } 78868fb18aaSSrivatsa S. Bhat 789bdab88e0SSebastian Andrzej Siewior /* Must run before sched domains notifier. */ 790bdab88e0SSebastian Andrzej Siewior static int ppc_numa_cpu_prepare(unsigned int cpu) 791ab1f9dacSPaul Mackerras { 792bdab88e0SSebastian Andrzej Siewior int nid; 793ab1f9dacSPaul Mackerras 794bdab88e0SSebastian Andrzej Siewior nid = numa_setup_cpu(cpu); 795bdab88e0SSebastian Andrzej Siewior verify_cpu_node_mapping(cpu, nid); 796bdab88e0SSebastian Andrzej Siewior return 0; 797ab1f9dacSPaul Mackerras } 798bdab88e0SSebastian Andrzej Siewior 799bdab88e0SSebastian Andrzej Siewior static int ppc_numa_cpu_dead(unsigned int cpu) 800bdab88e0SSebastian Andrzej Siewior { 801bdab88e0SSebastian Andrzej Siewior return 0; 802ab1f9dacSPaul Mackerras } 803ab1f9dacSPaul Mackerras 804ab1f9dacSPaul Mackerras /* 805ab1f9dacSPaul Mackerras * Check and possibly modify a memory region to enforce the memory limit. 806ab1f9dacSPaul Mackerras * 807ab1f9dacSPaul Mackerras * Returns the size the region should have to enforce the memory limit. 808ab1f9dacSPaul Mackerras * This will either be the original value of size, a truncated value, 809ab1f9dacSPaul Mackerras * or zero. If the returned value of size is 0 the region should be 81025985edcSLucas De Marchi * discarded as it lies wholly above the memory limit. 811ab1f9dacSPaul Mackerras */ 81245fb6ceaSAnton Blanchard static unsigned long __init numa_enforce_memory_limit(unsigned long start, 81345fb6ceaSAnton Blanchard unsigned long size) 814ab1f9dacSPaul Mackerras { 815ab1f9dacSPaul Mackerras /* 81695f72d1eSYinghai Lu * We use memblock_end_of_DRAM() in here instead of memory_limit because 817ab1f9dacSPaul Mackerras * we've already adjusted it for the limit and it takes care of 818fe55249dSMilton Miller * having memory holes below the limit. Also, in the case of 819fe55249dSMilton Miller * iommu_is_off, memory_limit is not set but is implicitly enforced. 820ab1f9dacSPaul Mackerras */ 821ab1f9dacSPaul Mackerras 82295f72d1eSYinghai Lu if (start + size <= memblock_end_of_DRAM()) 823ab1f9dacSPaul Mackerras return size; 824ab1f9dacSPaul Mackerras 82595f72d1eSYinghai Lu if (start >= memblock_end_of_DRAM()) 826ab1f9dacSPaul Mackerras return 0; 827ab1f9dacSPaul Mackerras 82895f72d1eSYinghai Lu return memblock_end_of_DRAM() - start; 829ab1f9dacSPaul Mackerras } 830ab1f9dacSPaul Mackerras 8310204568aSPaul Mackerras /* 832cf00085dSChandru * Reads the counter for a given entry in 833cf00085dSChandru * linux,drconf-usable-memory property 834cf00085dSChandru */ 835b08a2a12SAlistair Popple static inline int __init read_usm_ranges(const __be32 **usm) 836cf00085dSChandru { 837cf00085dSChandru /* 8383fdfd990SBenjamin Herrenschmidt * For each lmb in ibm,dynamic-memory a corresponding 839cf00085dSChandru * entry in linux,drconf-usable-memory property contains 840cf00085dSChandru * a counter followed by that many (base, size) duple. 841cf00085dSChandru * read the counter from linux,drconf-usable-memory 842cf00085dSChandru */ 843cf00085dSChandru return read_n_cells(n_mem_size_cells, usm); 844cf00085dSChandru } 845cf00085dSChandru 846cf00085dSChandru /* 8470204568aSPaul Mackerras * Extract NUMA information from the ibm,dynamic-reconfiguration-memory 8480204568aSPaul Mackerras * node. This assumes n_mem_{addr,size}_cells have been set. 8490204568aSPaul Mackerras */ 850adfefc60SHari Bathini static int __init numa_setup_drmem_lmb(struct drmem_lmb *lmb, 851adfefc60SHari Bathini const __be32 **usm, 852adfefc60SHari Bathini void *data) 8530204568aSPaul Mackerras { 854514a9cb3SNathan Fontenot unsigned int ranges, is_kexec_kdump = 0; 855514a9cb3SNathan Fontenot unsigned long base, size, sz; 8568342681dSNathan Fontenot int nid; 8570204568aSPaul Mackerras 858514a9cb3SNathan Fontenot /* 859514a9cb3SNathan Fontenot * Skip this block if the reserved bit is set in flags (0x80) 860514a9cb3SNathan Fontenot * or if the block is not assigned to this partition (0x8) 861514a9cb3SNathan Fontenot */ 862514a9cb3SNathan Fontenot if ((lmb->flags & DRCONF_MEM_RESERVED) 863514a9cb3SNathan Fontenot || !(lmb->flags & DRCONF_MEM_ASSIGNED)) 864adfefc60SHari Bathini return 0; 8650204568aSPaul Mackerras 866514a9cb3SNathan Fontenot if (*usm) 867cf00085dSChandru is_kexec_kdump = 1; 868cf00085dSChandru 869514a9cb3SNathan Fontenot base = lmb->base_addr; 870514a9cb3SNathan Fontenot size = drmem_lmb_size(); 871cf00085dSChandru ranges = 1; 8728342681dSNathan Fontenot 873cf00085dSChandru if (is_kexec_kdump) { 874514a9cb3SNathan Fontenot ranges = read_usm_ranges(usm); 875cf00085dSChandru if (!ranges) /* there are no (base, size) duple */ 876adfefc60SHari Bathini return 0; 877cf00085dSChandru } 878514a9cb3SNathan Fontenot 879cf00085dSChandru do { 880cf00085dSChandru if (is_kexec_kdump) { 881514a9cb3SNathan Fontenot base = read_n_cells(n_mem_addr_cells, usm); 882514a9cb3SNathan Fontenot size = read_n_cells(n_mem_size_cells, usm); 883cf00085dSChandru } 884514a9cb3SNathan Fontenot 8858ddc6448SAneesh Kumar K.V nid = get_nid_and_numa_distance(lmb); 886514a9cb3SNathan Fontenot fake_numa_create_new_node(((base + size) >> PAGE_SHIFT), 887cf00085dSChandru &nid); 888cf00085dSChandru node_set_online(nid); 889cf00085dSChandru sz = numa_enforce_memory_limit(base, size); 890cf00085dSChandru if (sz) 891514a9cb3SNathan Fontenot memblock_set_node(base, sz, &memblock.memory, nid); 892cf00085dSChandru } while (--ranges); 893adfefc60SHari Bathini 894adfefc60SHari Bathini return 0; 8950204568aSPaul Mackerras } 8960204568aSPaul Mackerras 897ab1f9dacSPaul Mackerras static int __init parse_numa_properties(void) 898ab1f9dacSPaul Mackerras { 899*11981816SNilay Shroff struct device_node *memory, *pci; 900482ec7c4SNathan Lynch int default_nid = 0; 901ab1f9dacSPaul Mackerras unsigned long i; 9028ddc6448SAneesh Kumar K.V const __be32 *associativity; 903ab1f9dacSPaul Mackerras 904ab1f9dacSPaul Mackerras if (numa_enabled == 0) { 905506c2075SSrikar Dronamraju pr_warn("disabled by user\n"); 906ab1f9dacSPaul Mackerras return -1; 907ab1f9dacSPaul Mackerras } 908ab1f9dacSPaul Mackerras 9097e35ef66SAneesh Kumar K.V primary_domain_index = find_primary_domain_index(); 910ab1f9dacSPaul Mackerras 9117e35ef66SAneesh Kumar K.V if (primary_domain_index < 0) { 912495c2ff4SAneesh Kumar K.V /* 9137e35ef66SAneesh Kumar K.V * if we fail to parse primary_domain_index from device tree 914495c2ff4SAneesh Kumar K.V * mark the numa disabled, boot with numa disabled. 915495c2ff4SAneesh Kumar K.V */ 916495c2ff4SAneesh Kumar K.V numa_enabled = false; 9177e35ef66SAneesh Kumar K.V return primary_domain_index; 918495c2ff4SAneesh Kumar K.V } 919ab1f9dacSPaul Mackerras 920506c2075SSrikar Dronamraju pr_debug("associativity depth for CPU/Memory: %d\n", primary_domain_index); 921bf4b85b0SNathan Lynch 922ab1f9dacSPaul Mackerras /* 9231c6b5a7eSAneesh Kumar K.V * If it is FORM2 initialize the distance table here. 9241c6b5a7eSAneesh Kumar K.V */ 9251c6b5a7eSAneesh Kumar K.V if (affinity_form == FORM2_AFFINITY) 9261c6b5a7eSAneesh Kumar K.V initialize_form2_numa_distance_lookup_table(); 9271c6b5a7eSAneesh Kumar K.V 9281c6b5a7eSAneesh Kumar K.V /* 929482ec7c4SNathan Lynch * Even though we connect cpus to numa domains later in SMP 930482ec7c4SNathan Lynch * init, we need to know the node ids now. This is because 931482ec7c4SNathan Lynch * each node to be onlined must have NODE_DATA etc backing it. 932ab1f9dacSPaul Mackerras */ 933482ec7c4SNathan Lynch for_each_present_cpu(i) { 9348ddc6448SAneesh Kumar K.V __be32 vphn_assoc[VPHN_ASSOC_BUFSIZE]; 935dfbe93a2SAnton Blanchard struct device_node *cpu; 9368ddc6448SAneesh Kumar K.V int nid = NUMA_NO_NODE; 9378ddc6448SAneesh Kumar K.V 9388ddc6448SAneesh Kumar K.V memset(vphn_assoc, 0, VPHN_ASSOC_BUFSIZE * sizeof(__be32)); 9398ddc6448SAneesh Kumar K.V 9408ddc6448SAneesh Kumar K.V if (__vphn_get_associativity(i, vphn_assoc) == 0) { 9418ddc6448SAneesh Kumar K.V nid = associativity_to_nid(vphn_assoc); 9428ddc6448SAneesh Kumar K.V initialize_form1_numa_distance(vphn_assoc); 9438ddc6448SAneesh Kumar K.V } else { 944ab1f9dacSPaul Mackerras 945482ec7c4SNathan Lynch /* 946482ec7c4SNathan Lynch * Don't fall back to default_nid yet -- we will plug 947482ec7c4SNathan Lynch * cpus into nodes once the memory scan has discovered 948482ec7c4SNathan Lynch * the topology. 949482ec7c4SNathan Lynch */ 9506398eaa2SSrikar Dronamraju cpu = of_get_cpu_node(i, NULL); 9516398eaa2SSrikar Dronamraju BUG_ON(!cpu); 9528ddc6448SAneesh Kumar K.V 9538ddc6448SAneesh Kumar K.V associativity = of_get_associativity(cpu); 9548ddc6448SAneesh Kumar K.V if (associativity) { 9558ddc6448SAneesh Kumar K.V nid = associativity_to_nid(associativity); 9568ddc6448SAneesh Kumar K.V initialize_form1_numa_distance(associativity); 9578ddc6448SAneesh Kumar K.V } 9586398eaa2SSrikar Dronamraju of_node_put(cpu); 9596398eaa2SSrikar Dronamraju } 9606398eaa2SSrikar Dronamraju 961749ed4a2SDaniel Henrique Barboza /* node_set_online() is an UB if 'nid' is negative */ 962749ed4a2SDaniel Henrique Barboza if (likely(nid >= 0)) 963482ec7c4SNathan Lynch node_set_online(nid); 964ab1f9dacSPaul Mackerras } 965ab1f9dacSPaul Mackerras 966237a0989SMike Kravetz get_n_mem_cells(&n_mem_addr_cells, &n_mem_size_cells); 96794db7c5eSAnton Blanchard 96894db7c5eSAnton Blanchard for_each_node_by_type(memory, "memory") { 969ab1f9dacSPaul Mackerras unsigned long start; 970ab1f9dacSPaul Mackerras unsigned long size; 971cf950b7aSNathan Lynch int nid; 972ab1f9dacSPaul Mackerras int ranges; 973b08a2a12SAlistair Popple const __be32 *memcell_buf; 974ab1f9dacSPaul Mackerras unsigned int len; 975ab1f9dacSPaul Mackerras 976e2eb6392SStephen Rothwell memcell_buf = of_get_property(memory, 977ba759485SMichael Ellerman "linux,usable-memory", &len); 978ba759485SMichael Ellerman if (!memcell_buf || len <= 0) 979e2eb6392SStephen Rothwell memcell_buf = of_get_property(memory, "reg", &len); 980ab1f9dacSPaul Mackerras if (!memcell_buf || len <= 0) 981ab1f9dacSPaul Mackerras continue; 982ab1f9dacSPaul Mackerras 983cc5d0189SBenjamin Herrenschmidt /* ranges in cell */ 984cc5d0189SBenjamin Herrenschmidt ranges = (len >> 2) / (n_mem_addr_cells + n_mem_size_cells); 985ab1f9dacSPaul Mackerras new_range: 986ab1f9dacSPaul Mackerras /* these are order-sensitive, and modify the buffer pointer */ 987237a0989SMike Kravetz start = read_n_cells(n_mem_addr_cells, &memcell_buf); 988237a0989SMike Kravetz size = read_n_cells(n_mem_size_cells, &memcell_buf); 989ab1f9dacSPaul Mackerras 990482ec7c4SNathan Lynch /* 991482ec7c4SNathan Lynch * Assumption: either all memory nodes or none will 992482ec7c4SNathan Lynch * have associativity properties. If none, then 993482ec7c4SNathan Lynch * everything goes to default_nid. 994482ec7c4SNathan Lynch */ 9958ddc6448SAneesh Kumar K.V associativity = of_get_associativity(memory); 9968ddc6448SAneesh Kumar K.V if (associativity) { 9978ddc6448SAneesh Kumar K.V nid = associativity_to_nid(associativity); 9988ddc6448SAneesh Kumar K.V initialize_form1_numa_distance(associativity); 9998ddc6448SAneesh Kumar K.V } else 1000482ec7c4SNathan Lynch nid = default_nid; 10011daa6d08SBalbir Singh 10021daa6d08SBalbir Singh fake_numa_create_new_node(((start + size) >> PAGE_SHIFT), &nid); 1003482ec7c4SNathan Lynch node_set_online(nid); 1004ab1f9dacSPaul Mackerras 10057656cd8eSReza Arbab size = numa_enforce_memory_limit(start, size); 10067656cd8eSReza Arbab if (size) 1007e7e8de59STang Chen memblock_set_node(start, size, &memblock.memory, nid); 1008ab1f9dacSPaul Mackerras 1009ab1f9dacSPaul Mackerras if (--ranges) 1010ab1f9dacSPaul Mackerras goto new_range; 1011ab1f9dacSPaul Mackerras } 1012ab1f9dacSPaul Mackerras 1013*11981816SNilay Shroff for_each_node_by_name(pci, "pci") { 1014*11981816SNilay Shroff int nid = NUMA_NO_NODE; 1015*11981816SNilay Shroff 1016*11981816SNilay Shroff associativity = of_get_associativity(pci); 1017*11981816SNilay Shroff if (associativity) { 1018*11981816SNilay Shroff nid = associativity_to_nid(associativity); 1019*11981816SNilay Shroff initialize_form1_numa_distance(associativity); 1020*11981816SNilay Shroff } 1021*11981816SNilay Shroff if (likely(nid >= 0) && !node_online(nid)) 1022*11981816SNilay Shroff node_set_online(nid); 1023*11981816SNilay Shroff } 1024*11981816SNilay Shroff 10250204568aSPaul Mackerras /* 1026dfbe93a2SAnton Blanchard * Now do the same thing for each MEMBLOCK listed in the 1027dfbe93a2SAnton Blanchard * ibm,dynamic-memory property in the 1028dfbe93a2SAnton Blanchard * ibm,dynamic-reconfiguration-memory node. 10290204568aSPaul Mackerras */ 10300204568aSPaul Mackerras memory = of_find_node_by_path("/ibm,dynamic-reconfiguration-memory"); 1031514a9cb3SNathan Fontenot if (memory) { 1032adfefc60SHari Bathini walk_drmem_lmbs(memory, NULL, numa_setup_drmem_lmb); 1033514a9cb3SNathan Fontenot of_node_put(memory); 1034514a9cb3SNathan Fontenot } 10350204568aSPaul Mackerras 1036ab1f9dacSPaul Mackerras return 0; 1037ab1f9dacSPaul Mackerras } 1038ab1f9dacSPaul Mackerras 1039ab1f9dacSPaul Mackerras static void __init setup_nonnuma(void) 1040ab1f9dacSPaul Mackerras { 104195f72d1eSYinghai Lu unsigned long top_of_ram = memblock_end_of_DRAM(); 104295f72d1eSYinghai Lu unsigned long total_ram = memblock_phys_mem_size(); 1043c67c3cb4SMel Gorman unsigned long start_pfn, end_pfn; 104428be7072SBenjamin Herrenschmidt unsigned int nid = 0; 1045c9118e6cSMike Rapoport int i; 1046ab1f9dacSPaul Mackerras 1047506c2075SSrikar Dronamraju pr_debug("Top of RAM: 0x%lx, Total RAM: 0x%lx\n", top_of_ram, total_ram); 1048506c2075SSrikar Dronamraju pr_debug("Memory hole size: %ldMB\n", (top_of_ram - total_ram) >> 20); 1049ab1f9dacSPaul Mackerras 1050c9118e6cSMike Rapoport for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, NULL) { 10511daa6d08SBalbir Singh fake_numa_create_new_node(end_pfn, &nid); 10521d7cfe18STejun Heo memblock_set_node(PFN_PHYS(start_pfn), 1053e7e8de59STang Chen PFN_PHYS(end_pfn - start_pfn), 1054e7e8de59STang Chen &memblock.memory, nid); 10551daa6d08SBalbir Singh node_set_online(nid); 1056c67c3cb4SMel Gorman } 1057ab1f9dacSPaul Mackerras } 1058ab1f9dacSPaul Mackerras 10594b703a23SAnton Blanchard void __init dump_numa_cpu_topology(void) 10604b703a23SAnton Blanchard { 10614b703a23SAnton Blanchard unsigned int node; 10624b703a23SAnton Blanchard unsigned int cpu, count; 10634b703a23SAnton Blanchard 1064495c2ff4SAneesh Kumar K.V if (!numa_enabled) 10654b703a23SAnton Blanchard return; 10664b703a23SAnton Blanchard 10674b703a23SAnton Blanchard for_each_online_node(node) { 10688467801cSAneesh Kumar K.V pr_info("Node %d CPUs:", node); 10694b703a23SAnton Blanchard 10704b703a23SAnton Blanchard count = 0; 10714b703a23SAnton Blanchard /* 10724b703a23SAnton Blanchard * If we used a CPU iterator here we would miss printing 10734b703a23SAnton Blanchard * the holes in the cpumap. 10744b703a23SAnton Blanchard */ 107525863de0SAnton Blanchard for (cpu = 0; cpu < nr_cpu_ids; cpu++) { 107625863de0SAnton Blanchard if (cpumask_test_cpu(cpu, 107725863de0SAnton Blanchard node_to_cpumask_map[node])) { 10784b703a23SAnton Blanchard if (count == 0) 10798467801cSAneesh Kumar K.V pr_cont(" %u", cpu); 10804b703a23SAnton Blanchard ++count; 10814b703a23SAnton Blanchard } else { 10824b703a23SAnton Blanchard if (count > 1) 10838467801cSAneesh Kumar K.V pr_cont("-%u", cpu - 1); 10844b703a23SAnton Blanchard count = 0; 10854b703a23SAnton Blanchard } 10864b703a23SAnton Blanchard } 10874b703a23SAnton Blanchard 10884b703a23SAnton Blanchard if (count > 1) 10898467801cSAneesh Kumar K.V pr_cont("-%u", nr_cpu_ids - 1); 10908467801cSAneesh Kumar K.V pr_cont("\n"); 10914b703a23SAnton Blanchard } 10924b703a23SAnton Blanchard } 10934b703a23SAnton Blanchard 109410239733SAnton Blanchard /* Initialize NODE_DATA for a node on the local memory */ 109510239733SAnton Blanchard static void __init setup_node_data(int nid, u64 start_pfn, u64 end_pfn) 1096ab1f9dacSPaul Mackerras { 109710239733SAnton Blanchard u64 spanned_pages = end_pfn - start_pfn; 109810239733SAnton Blanchard const size_t nd_size = roundup(sizeof(pg_data_t), SMP_CACHE_BYTES); 109910239733SAnton Blanchard u64 nd_pa; 110010239733SAnton Blanchard void *nd; 110110239733SAnton Blanchard int tnid; 1102ab1f9dacSPaul Mackerras 11039a8dd708SMike Rapoport nd_pa = memblock_phys_alloc_try_nid(nd_size, SMP_CACHE_BYTES, nid); 110433755574SMike Rapoport if (!nd_pa) 110533755574SMike Rapoport panic("Cannot allocate %zu bytes for node %d data\n", 110633755574SMike Rapoport nd_size, nid); 110733755574SMike Rapoport 110810239733SAnton Blanchard nd = __va(nd_pa); 1109ab1f9dacSPaul Mackerras 111010239733SAnton Blanchard /* report and initialize */ 111110239733SAnton Blanchard pr_info(" NODE_DATA [mem %#010Lx-%#010Lx]\n", 111210239733SAnton Blanchard nd_pa, nd_pa + nd_size - 1); 111310239733SAnton Blanchard tnid = early_pfn_to_nid(nd_pa >> PAGE_SHIFT); 111410239733SAnton Blanchard if (tnid != nid) 111510239733SAnton Blanchard pr_info(" NODE_DATA(%d) on node %d\n", nid, tnid); 11168f64e1f2SJon Tollefson 111710239733SAnton Blanchard node_data[nid] = nd; 111810239733SAnton Blanchard memset(NODE_DATA(nid), 0, sizeof(pg_data_t)); 111910239733SAnton Blanchard NODE_DATA(nid)->node_id = nid; 112010239733SAnton Blanchard NODE_DATA(nid)->node_start_pfn = start_pfn; 112110239733SAnton Blanchard NODE_DATA(nid)->node_spanned_pages = spanned_pages; 1122ab1f9dacSPaul Mackerras } 11238f64e1f2SJon Tollefson 1124a346137eSMichael Bringmann static void __init find_possible_nodes(void) 1125a346137eSMichael Bringmann { 11262a066ae1SChristophe Leroy struct device_node *rtas, *root; 11279c7248bbSLaurent Dufour const __be32 *domains = NULL; 1128f9f130ffSSrikar Dronamraju int prop_length, max_nodes; 1129f9f130ffSSrikar Dronamraju u32 i; 1130a346137eSMichael Bringmann 1131495c2ff4SAneesh Kumar K.V if (!numa_enabled) 1132a346137eSMichael Bringmann return; 1133a346137eSMichael Bringmann 1134a346137eSMichael Bringmann rtas = of_find_node_by_path("/rtas"); 1135a346137eSMichael Bringmann if (!rtas) 1136a346137eSMichael Bringmann return; 1137a346137eSMichael Bringmann 113867df7784SSrikar Dronamraju /* 1139f9f130ffSSrikar Dronamraju * ibm,current-associativity-domains is a fairly recent property. If 1140f9f130ffSSrikar Dronamraju * it doesn't exist, then fallback on ibm,max-associativity-domains. 1141f9f130ffSSrikar Dronamraju * Current denotes what the platform can support compared to max 1142f9f130ffSSrikar Dronamraju * which denotes what the Hypervisor can support. 11439c7248bbSLaurent Dufour * 11449c7248bbSLaurent Dufour * If the LPAR is migratable, new nodes might be activated after a LPM, 11459c7248bbSLaurent Dufour * so we should consider the max number in that case. 114667df7784SSrikar Dronamraju */ 11472a066ae1SChristophe Leroy root = of_find_node_by_path("/"); 11482a066ae1SChristophe Leroy if (!of_get_property(root, "ibm,migratable-partition", NULL)) 11499c7248bbSLaurent Dufour domains = of_get_property(rtas, 11509c7248bbSLaurent Dufour "ibm,current-associativity-domains", 1151f9f130ffSSrikar Dronamraju &prop_length); 11522a066ae1SChristophe Leroy of_node_put(root); 1153f9f130ffSSrikar Dronamraju if (!domains) { 1154f9f130ffSSrikar Dronamraju domains = of_get_property(rtas, "ibm,max-associativity-domains", 1155f9f130ffSSrikar Dronamraju &prop_length); 1156f9f130ffSSrikar Dronamraju if (!domains) 1157a346137eSMichael Bringmann goto out; 115867df7784SSrikar Dronamraju } 1159a346137eSMichael Bringmann 11607e35ef66SAneesh Kumar K.V max_nodes = of_read_number(&domains[primary_domain_index], 1); 11619c7248bbSLaurent Dufour pr_info("Partition configured for %d NUMA nodes.\n", max_nodes); 11629c7248bbSLaurent Dufour 1163f9f130ffSSrikar Dronamraju for (i = 0; i < max_nodes; i++) { 1164ea05ba7cSMichael Bringmann if (!node_possible(i)) 1165a346137eSMichael Bringmann node_set(i, node_possible_map); 1166a346137eSMichael Bringmann } 1167a346137eSMichael Bringmann 1168f9f130ffSSrikar Dronamraju prop_length /= sizeof(int); 11697e35ef66SAneesh Kumar K.V if (prop_length > primary_domain_index + 2) 1170f9f130ffSSrikar Dronamraju coregroup_enabled = 1; 1171f9f130ffSSrikar Dronamraju 1172a346137eSMichael Bringmann out: 1173a346137eSMichael Bringmann of_node_put(rtas); 1174a346137eSMichael Bringmann } 1175a346137eSMichael Bringmann 11769bd9be00SNicholas Piggin void __init mem_topology_setup(void) 11774a618669SDave Hansen { 11789bd9be00SNicholas Piggin int cpu; 11794a618669SDave Hansen 11807b31f7daSAneesh Kumar K.V max_low_pfn = max_pfn = memblock_end_of_DRAM() >> PAGE_SHIFT; 11817b31f7daSAneesh Kumar K.V min_low_pfn = MEMORY_START >> PAGE_SHIFT; 11827b31f7daSAneesh Kumar K.V 1183e75130f2SSrikar Dronamraju /* 1184e75130f2SSrikar Dronamraju * Linux/mm assumes node 0 to be online at boot. However this is not 1185e75130f2SSrikar Dronamraju * true on PowerPC, where node 0 is similar to any other node, it 1186e75130f2SSrikar Dronamraju * could be cpuless, memoryless node. So force node 0 to be offline 1187e75130f2SSrikar Dronamraju * for now. This will prevent cpuless, memoryless node 0 showing up 1188e75130f2SSrikar Dronamraju * unnecessarily as online. If a node has cpus or memory that need 1189e75130f2SSrikar Dronamraju * to be online, then node will anyway be marked online. 1190e75130f2SSrikar Dronamraju */ 1191e75130f2SSrikar Dronamraju node_set_offline(0); 1192e75130f2SSrikar Dronamraju 11934a618669SDave Hansen if (parse_numa_properties()) 11944a618669SDave Hansen setup_nonnuma(); 11954a618669SDave Hansen 11963af229f2SNishanth Aravamudan /* 1197a346137eSMichael Bringmann * Modify the set of possible NUMA nodes to reflect information 1198a346137eSMichael Bringmann * available about the set of online nodes, and the set of nodes 1199a346137eSMichael Bringmann * that we expect to make use of for this platform's affinity 1200a346137eSMichael Bringmann * calculations. 12013af229f2SNishanth Aravamudan */ 12023af229f2SNishanth Aravamudan nodes_and(node_possible_map, node_possible_map, node_online_map); 12033af229f2SNishanth Aravamudan 1204a346137eSMichael Bringmann find_possible_nodes(); 1205a346137eSMichael Bringmann 12069bd9be00SNicholas Piggin setup_node_to_cpumask_map(); 12079bd9be00SNicholas Piggin 12089bd9be00SNicholas Piggin reset_numa_cpu_lookup_table(); 12099bd9be00SNicholas Piggin 1210a874f100SSrikar Dronamraju for_each_possible_cpu(cpu) { 1211a874f100SSrikar Dronamraju /* 1212a874f100SSrikar Dronamraju * Powerpc with CONFIG_NUMA always used to have a node 0, 1213a874f100SSrikar Dronamraju * even if it was memoryless or cpuless. For all cpus that 1214a874f100SSrikar Dronamraju * are possible but not present, cpu_to_node() would point 1215a874f100SSrikar Dronamraju * to node 0. To remove a cpuless, memoryless dummy node, 1216a874f100SSrikar Dronamraju * powerpc need to make sure all possible but not present 1217a874f100SSrikar Dronamraju * cpu_to_node are set to a proper node. 1218a874f100SSrikar Dronamraju */ 12199bd9be00SNicholas Piggin numa_setup_cpu(cpu); 12209bd9be00SNicholas Piggin } 1221a874f100SSrikar Dronamraju } 12229bd9be00SNicholas Piggin 12239bd9be00SNicholas Piggin void __init initmem_init(void) 12249bd9be00SNicholas Piggin { 12259bd9be00SNicholas Piggin int nid; 12269bd9be00SNicholas Piggin 12279bd9be00SNicholas Piggin memblock_dump_all(); 12289bd9be00SNicholas Piggin 12294a618669SDave Hansen for_each_online_node(nid) { 12304a618669SDave Hansen unsigned long start_pfn, end_pfn; 12314a618669SDave Hansen 12324a618669SDave Hansen get_pfn_range_for_nid(nid, &start_pfn, &end_pfn); 123310239733SAnton Blanchard setup_node_data(nid, start_pfn, end_pfn); 1234ab1f9dacSPaul Mackerras } 1235d3f6204aSBenjamin Herrenschmidt 123621098b9eSAnton Blanchard sparse_init(); 123725863de0SAnton Blanchard 12382fabf084SNishanth Aravamudan /* 12392fabf084SNishanth Aravamudan * We need the numa_cpu_lookup_table to be accurate for all CPUs, 12402fabf084SNishanth Aravamudan * even before we online them, so that we can use cpu_to_{node,mem} 12412fabf084SNishanth Aravamudan * early in boot, cf. smp_prepare_cpus(). 1242bdab88e0SSebastian Andrzej Siewior * _nocalls() + manual invocation is used because cpuhp is not yet 1243bdab88e0SSebastian Andrzej Siewior * initialized for the boot CPU. 12442fabf084SNishanth Aravamudan */ 124573c1b41eSThomas Gleixner cpuhp_setup_state_nocalls(CPUHP_POWER_NUMA_PREPARE, "powerpc/numa:prepare", 1246bdab88e0SSebastian Andrzej Siewior ppc_numa_cpu_prepare, ppc_numa_cpu_dead); 12474a618669SDave Hansen } 1248ab1f9dacSPaul Mackerras 1249ab1f9dacSPaul Mackerras static int __init early_numa(char *p) 1250ab1f9dacSPaul Mackerras { 1251ab1f9dacSPaul Mackerras if (!p) 1252ab1f9dacSPaul Mackerras return 0; 1253ab1f9dacSPaul Mackerras 1254ab1f9dacSPaul Mackerras if (strstr(p, "off")) 1255ab1f9dacSPaul Mackerras numa_enabled = 0; 1256ab1f9dacSPaul Mackerras 12571daa6d08SBalbir Singh p = strstr(p, "fake="); 12581daa6d08SBalbir Singh if (p) 12591daa6d08SBalbir Singh cmdline = p + strlen("fake="); 12601daa6d08SBalbir Singh 1261ab1f9dacSPaul Mackerras return 0; 1262ab1f9dacSPaul Mackerras } 1263ab1f9dacSPaul Mackerras early_param("numa", early_numa); 1264237a0989SMike Kravetz 1265237a0989SMike Kravetz #ifdef CONFIG_MEMORY_HOTPLUG 1266237a0989SMike Kravetz /* 12670f16ef7fSNathan Fontenot * Find the node associated with a hot added memory section for 12680f16ef7fSNathan Fontenot * memory represented in the device tree by the property 12690f16ef7fSNathan Fontenot * ibm,dynamic-reconfiguration-memory/ibm,dynamic-memory. 12700db9360aSNathan Fontenot */ 1271514a9cb3SNathan Fontenot static int hot_add_drconf_scn_to_nid(unsigned long scn_addr) 12720db9360aSNathan Fontenot { 1273514a9cb3SNathan Fontenot struct drmem_lmb *lmb; 12743fdfd990SBenjamin Herrenschmidt unsigned long lmb_size; 127598fa15f3SAnshuman Khandual int nid = NUMA_NO_NODE; 12760db9360aSNathan Fontenot 1277514a9cb3SNathan Fontenot lmb_size = drmem_lmb_size(); 12780db9360aSNathan Fontenot 1279514a9cb3SNathan Fontenot for_each_drmem_lmb(lmb) { 12800db9360aSNathan Fontenot /* skip this block if it is reserved or not assigned to 12810db9360aSNathan Fontenot * this partition */ 1282514a9cb3SNathan Fontenot if ((lmb->flags & DRCONF_MEM_RESERVED) 1283514a9cb3SNathan Fontenot || !(lmb->flags & DRCONF_MEM_ASSIGNED)) 12840db9360aSNathan Fontenot continue; 12850db9360aSNathan Fontenot 1286514a9cb3SNathan Fontenot if ((scn_addr < lmb->base_addr) 1287514a9cb3SNathan Fontenot || (scn_addr >= (lmb->base_addr + lmb_size))) 12880f16ef7fSNathan Fontenot continue; 12890db9360aSNathan Fontenot 1290514a9cb3SNathan Fontenot nid = of_drconf_to_nid_single(lmb); 12910f16ef7fSNathan Fontenot break; 12920db9360aSNathan Fontenot } 12930db9360aSNathan Fontenot 12940f16ef7fSNathan Fontenot return nid; 12950db9360aSNathan Fontenot } 12960db9360aSNathan Fontenot 12970db9360aSNathan Fontenot /* 12980f16ef7fSNathan Fontenot * Find the node associated with a hot added memory section for memory 12990f16ef7fSNathan Fontenot * represented in the device tree as a node (i.e. memory@XXXX) for 130095f72d1eSYinghai Lu * each memblock. 1301237a0989SMike Kravetz */ 1302ec32dd66SRobert Jennings static int hot_add_node_scn_to_nid(unsigned long scn_addr) 1303237a0989SMike Kravetz { 130494db7c5eSAnton Blanchard struct device_node *memory; 130598fa15f3SAnshuman Khandual int nid = NUMA_NO_NODE; 1306237a0989SMike Kravetz 130794db7c5eSAnton Blanchard for_each_node_by_type(memory, "memory") { 13082500763dSRob Herring int i = 0; 1309237a0989SMike Kravetz 13102500763dSRob Herring while (1) { 13112500763dSRob Herring struct resource res; 1312237a0989SMike Kravetz 13132500763dSRob Herring if (of_address_to_resource(memory, i++, &res)) 13142500763dSRob Herring break; 13150f16ef7fSNathan Fontenot 13162500763dSRob Herring if ((scn_addr < res.start) || (scn_addr > res.end)) 13170f16ef7fSNathan Fontenot continue; 13180f16ef7fSNathan Fontenot 13190f16ef7fSNathan Fontenot nid = of_node_to_nid_single(memory); 13200f16ef7fSNathan Fontenot break; 13210f16ef7fSNathan Fontenot } 13220f16ef7fSNathan Fontenot 13230f16ef7fSNathan Fontenot if (nid >= 0) 13240f16ef7fSNathan Fontenot break; 13250f16ef7fSNathan Fontenot } 13260f16ef7fSNathan Fontenot 132760831842SAnton Blanchard of_node_put(memory); 132860831842SAnton Blanchard 13290db9360aSNathan Fontenot return nid; 1330237a0989SMike Kravetz } 1331237a0989SMike Kravetz 13320f16ef7fSNathan Fontenot /* 13330f16ef7fSNathan Fontenot * Find the node associated with a hot added memory section. Section 133495f72d1eSYinghai Lu * corresponds to a SPARSEMEM section, not an MEMBLOCK. It is assumed that 133595f72d1eSYinghai Lu * sections are fully contained within a single MEMBLOCK. 13360f16ef7fSNathan Fontenot */ 13370f16ef7fSNathan Fontenot int hot_add_scn_to_nid(unsigned long scn_addr) 13380f16ef7fSNathan Fontenot { 13390f16ef7fSNathan Fontenot struct device_node *memory = NULL; 13404a3bac4eSReza Arbab int nid; 13410f16ef7fSNathan Fontenot 1342495c2ff4SAneesh Kumar K.V if (!numa_enabled) 134372c33688SH Hartley Sweeten return first_online_node; 13440f16ef7fSNathan Fontenot 13450f16ef7fSNathan Fontenot memory = of_find_node_by_path("/ibm,dynamic-reconfiguration-memory"); 13460f16ef7fSNathan Fontenot if (memory) { 1347514a9cb3SNathan Fontenot nid = hot_add_drconf_scn_to_nid(scn_addr); 13480f16ef7fSNathan Fontenot of_node_put(memory); 13490f16ef7fSNathan Fontenot } else { 13500f16ef7fSNathan Fontenot nid = hot_add_node_scn_to_nid(scn_addr); 1351237a0989SMike Kravetz } 13520f16ef7fSNathan Fontenot 13532a8628d4SReza Arbab if (nid < 0 || !node_possible(nid)) 135472c33688SH Hartley Sweeten nid = first_online_node; 13550f16ef7fSNathan Fontenot 13560f16ef7fSNathan Fontenot return nid; 13570f16ef7fSNathan Fontenot } 13580f16ef7fSNathan Fontenot 1359cd34206eSNishanth Aravamudan static u64 hot_add_drconf_memory_max(void) 1360cd34206eSNishanth Aravamudan { 1361cd34206eSNishanth Aravamudan struct device_node *memory = NULL; 136245b64ee6SBharata B Rao struct device_node *dn = NULL; 136345b64ee6SBharata B Rao const __be64 *lrdr = NULL; 136445b64ee6SBharata B Rao 136545b64ee6SBharata B Rao dn = of_find_node_by_path("/rtas"); 136645b64ee6SBharata B Rao if (dn) { 136745b64ee6SBharata B Rao lrdr = of_get_property(dn, "ibm,lrdr-capacity", NULL); 136845b64ee6SBharata B Rao of_node_put(dn); 136945b64ee6SBharata B Rao if (lrdr) 137045b64ee6SBharata B Rao return be64_to_cpup(lrdr); 137145b64ee6SBharata B Rao } 1372cd34206eSNishanth Aravamudan 1373cd34206eSNishanth Aravamudan memory = of_find_node_by_path("/ibm,dynamic-reconfiguration-memory"); 1374cd34206eSNishanth Aravamudan if (memory) { 1375cd34206eSNishanth Aravamudan of_node_put(memory); 1376514a9cb3SNathan Fontenot return drmem_lmb_memory_max(); 1377cd34206eSNishanth Aravamudan } 137845b64ee6SBharata B Rao return 0; 1379cd34206eSNishanth Aravamudan } 1380cd34206eSNishanth Aravamudan 1381cd34206eSNishanth Aravamudan /* 1382cd34206eSNishanth Aravamudan * memory_hotplug_max - return max address of memory that may be added 1383cd34206eSNishanth Aravamudan * 1384cd34206eSNishanth Aravamudan * This is currently only used on systems that support drconfig memory 1385cd34206eSNishanth Aravamudan * hotplug. 1386cd34206eSNishanth Aravamudan */ 1387cd34206eSNishanth Aravamudan u64 memory_hotplug_max(void) 1388cd34206eSNishanth Aravamudan { 1389cd34206eSNishanth Aravamudan return max(hot_add_drconf_memory_max(), memblock_end_of_DRAM()); 1390cd34206eSNishanth Aravamudan } 1391237a0989SMike Kravetz #endif /* CONFIG_MEMORY_HOTPLUG */ 13929eff1a38SJesse Larrew 1393bd03403aSJesse Larrew /* Virtual Processor Home Node (VPHN) support */ 139439bf990eSJesse Larrew #ifdef CONFIG_PPC_SPLPAR 139517f444c0SMichael Bringmann static int topology_inited; 13969eff1a38SJesse Larrew 13979eff1a38SJesse Larrew /* 13989eff1a38SJesse Larrew * Retrieve the new associativity information for a virtual processor's 13999eff1a38SJesse Larrew * home node. 14009eff1a38SJesse Larrew */ 14019eff1a38SJesse Larrew static long vphn_get_associativity(unsigned long cpu, 1402b08a2a12SAlistair Popple __be32 *associativity) 14039eff1a38SJesse Larrew { 1404cd9d6cc7SJesse Larrew long rc; 14059eff1a38SJesse Larrew 1406ef34e0efSNaveen N. Rao rc = hcall_vphn(get_hard_smp_processor_id(cpu), 1407ef34e0efSNaveen N. Rao VPHN_FLAG_VCPU, associativity); 14089eff1a38SJesse Larrew 14099eff1a38SJesse Larrew switch (rc) { 141017f444c0SMichael Bringmann case H_SUCCESS: 1411544af642SSrikar Dronamraju pr_debug("VPHN hcall succeeded. Reset polling...\n"); 141276b7bfb1SSrikar Dronamraju goto out; 141376b7bfb1SSrikar Dronamraju 141476b7bfb1SSrikar Dronamraju case H_FUNCTION: 141576b7bfb1SSrikar Dronamraju pr_err_ratelimited("VPHN unsupported. Disabling polling...\n"); 141676b7bfb1SSrikar Dronamraju break; 141776b7bfb1SSrikar Dronamraju case H_HARDWARE: 141876b7bfb1SSrikar Dronamraju pr_err_ratelimited("hcall_vphn() experienced a hardware fault " 141976b7bfb1SSrikar Dronamraju "preventing VPHN. Disabling polling...\n"); 142076b7bfb1SSrikar Dronamraju break; 142176b7bfb1SSrikar Dronamraju case H_PARAMETER: 142276b7bfb1SSrikar Dronamraju pr_err_ratelimited("hcall_vphn() was passed an invalid parameter. " 142376b7bfb1SSrikar Dronamraju "Disabling polling...\n"); 142476b7bfb1SSrikar Dronamraju break; 142576b7bfb1SSrikar Dronamraju default: 142676b7bfb1SSrikar Dronamraju pr_err_ratelimited("hcall_vphn() returned %ld. Disabling polling...\n" 142776b7bfb1SSrikar Dronamraju , rc); 142817f444c0SMichael Bringmann break; 14299eff1a38SJesse Larrew } 143076b7bfb1SSrikar Dronamraju out: 14319eff1a38SJesse Larrew return rc; 14329eff1a38SJesse Larrew } 14339eff1a38SJesse Larrew 143448b63961SOscar Salvador void find_and_update_cpu_nid(int cpu) 1435ea05ba7cSMichael Bringmann { 1436ea05ba7cSMichael Bringmann __be32 associativity[VPHN_ASSOC_BUFSIZE] = {0}; 1437ea05ba7cSMichael Bringmann int new_nid; 1438ea05ba7cSMichael Bringmann 1439ea05ba7cSMichael Bringmann /* Use associativity from first thread for all siblings */ 14402483ef05SSrikar Dronamraju if (vphn_get_associativity(cpu, associativity)) 144148b63961SOscar Salvador return; 14422483ef05SSrikar Dronamraju 144348b63961SOscar Salvador /* Do not have previous associativity, so find it now. */ 1444ea05ba7cSMichael Bringmann new_nid = associativity_to_nid(associativity); 144548b63961SOscar Salvador 1446ea05ba7cSMichael Bringmann if (new_nid < 0 || !node_possible(new_nid)) 1447ea05ba7cSMichael Bringmann new_nid = first_online_node; 144848b63961SOscar Salvador else 144948b63961SOscar Salvador // Associate node <-> cpu, so cpu_up() calls 145048b63961SOscar Salvador // try_online_node() on the right node. 145148b63961SOscar Salvador set_cpu_numa_node(cpu, new_nid); 1452ea05ba7cSMichael Bringmann 1453e62520b8SDwaipayan Ray pr_debug("%s:%d cpu %d nid %d\n", __func__, __LINE__, cpu, new_nid); 1454ea05ba7cSMichael Bringmann } 1455ea05ba7cSMichael Bringmann 145672730bfcSSrikar Dronamraju int cpu_to_coregroup_id(int cpu) 145772730bfcSSrikar Dronamraju { 1458fa35e868SSrikar Dronamraju __be32 associativity[VPHN_ASSOC_BUFSIZE] = {0}; 1459fa35e868SSrikar Dronamraju int index; 1460fa35e868SSrikar Dronamraju 1461fa35e868SSrikar Dronamraju if (cpu < 0 || cpu > nr_cpu_ids) 1462fa35e868SSrikar Dronamraju return -1; 1463fa35e868SSrikar Dronamraju 1464fa35e868SSrikar Dronamraju if (!coregroup_enabled) 1465fa35e868SSrikar Dronamraju goto out; 1466fa35e868SSrikar Dronamraju 1467fa35e868SSrikar Dronamraju if (!firmware_has_feature(FW_FEATURE_VPHN)) 1468fa35e868SSrikar Dronamraju goto out; 1469fa35e868SSrikar Dronamraju 1470fa35e868SSrikar Dronamraju if (vphn_get_associativity(cpu, associativity)) 1471fa35e868SSrikar Dronamraju goto out; 1472fa35e868SSrikar Dronamraju 1473fa35e868SSrikar Dronamraju index = of_read_number(associativity, 1); 14747e35ef66SAneesh Kumar K.V if (index > primary_domain_index + 1) 1475fa35e868SSrikar Dronamraju return of_read_number(&associativity[index - 1], 1); 1476fa35e868SSrikar Dronamraju 1477fa35e868SSrikar Dronamraju out: 147872730bfcSSrikar Dronamraju return cpu_to_core_id(cpu); 147972730bfcSSrikar Dronamraju } 148072730bfcSSrikar Dronamraju 1481e04fa612SNathan Fontenot static int topology_update_init(void) 1482e04fa612SNathan Fontenot { 148317f444c0SMichael Bringmann topology_inited = 1; 1484e04fa612SNathan Fontenot return 0; 1485e04fa612SNathan Fontenot } 1486e04fa612SNathan Fontenot device_initcall(topology_update_init); 148739bf990eSJesse Larrew #endif /* CONFIG_PPC_SPLPAR */ 1488