1ab1f9dacSPaul Mackerras /* 2ab1f9dacSPaul Mackerras * pSeries NUMA support 3ab1f9dacSPaul Mackerras * 4ab1f9dacSPaul Mackerras * Copyright (C) 2002 Anton Blanchard <anton@au.ibm.com>, IBM 5ab1f9dacSPaul Mackerras * 6ab1f9dacSPaul Mackerras * This program is free software; you can redistribute it and/or 7ab1f9dacSPaul Mackerras * modify it under the terms of the GNU General Public License 8ab1f9dacSPaul Mackerras * as published by the Free Software Foundation; either version 9ab1f9dacSPaul Mackerras * 2 of the License, or (at your option) any later version. 10ab1f9dacSPaul Mackerras */ 11ab1f9dacSPaul Mackerras #include <linux/threads.h> 12ab1f9dacSPaul Mackerras #include <linux/bootmem.h> 13ab1f9dacSPaul Mackerras #include <linux/init.h> 14ab1f9dacSPaul Mackerras #include <linux/mm.h> 15ab1f9dacSPaul Mackerras #include <linux/mmzone.h> 164b16f8e2SPaul Gortmaker #include <linux/export.h> 17ab1f9dacSPaul Mackerras #include <linux/nodemask.h> 18ab1f9dacSPaul Mackerras #include <linux/cpu.h> 19ab1f9dacSPaul Mackerras #include <linux/notifier.h> 2095f72d1eSYinghai Lu #include <linux/memblock.h> 216df1646eSMichael Ellerman #include <linux/of.h> 2206eccea6SDave Hansen #include <linux/pfn.h> 239eff1a38SJesse Larrew #include <linux/cpuset.h> 249eff1a38SJesse Larrew #include <linux/node.h> 2530c05350SNathan Fontenot #include <linux/stop_machine.h> 26e04fa612SNathan Fontenot #include <linux/proc_fs.h> 27e04fa612SNathan Fontenot #include <linux/seq_file.h> 28e04fa612SNathan Fontenot #include <linux/uaccess.h> 29191a7120SLinus Torvalds #include <linux/slab.h> 303be7db6aSRobert Jennings #include <asm/cputhreads.h> 3145fb6ceaSAnton Blanchard #include <asm/sparsemem.h> 32d9b2b2a2SDavid S. Miller #include <asm/prom.h> 332249ca9dSPaul Mackerras #include <asm/smp.h> 34d4edc5b6SSrivatsa S. Bhat #include <asm/cputhreads.h> 35d4edc5b6SSrivatsa S. Bhat #include <asm/topology.h> 369eff1a38SJesse Larrew #include <asm/firmware.h> 379eff1a38SJesse Larrew #include <asm/paca.h> 3839bf990eSJesse Larrew #include <asm/hvcall.h> 39ae3a197eSDavid Howells #include <asm/setup.h> 40176bbf14SJesse Larrew #include <asm/vdso.h> 41ab1f9dacSPaul Mackerras 42ab1f9dacSPaul Mackerras static int numa_enabled = 1; 43ab1f9dacSPaul Mackerras 441daa6d08SBalbir Singh static char *cmdline __initdata; 451daa6d08SBalbir Singh 46ab1f9dacSPaul Mackerras static int numa_debug; 47ab1f9dacSPaul Mackerras #define dbg(args...) if (numa_debug) { printk(KERN_INFO args); } 48ab1f9dacSPaul Mackerras 4945fb6ceaSAnton Blanchard int numa_cpu_lookup_table[NR_CPUS]; 5025863de0SAnton Blanchard cpumask_var_t node_to_cpumask_map[MAX_NUMNODES]; 51ab1f9dacSPaul Mackerras struct pglist_data *node_data[MAX_NUMNODES]; 5245fb6ceaSAnton Blanchard 5345fb6ceaSAnton Blanchard EXPORT_SYMBOL(numa_cpu_lookup_table); 5425863de0SAnton Blanchard EXPORT_SYMBOL(node_to_cpumask_map); 5545fb6ceaSAnton Blanchard EXPORT_SYMBOL(node_data); 5645fb6ceaSAnton Blanchard 57ab1f9dacSPaul Mackerras static int min_common_depth; 58237a0989SMike Kravetz static int n_mem_addr_cells, n_mem_size_cells; 5941eab6f8SAnton Blanchard static int form1_affinity; 6041eab6f8SAnton Blanchard 6141eab6f8SAnton Blanchard #define MAX_DISTANCE_REF_POINTS 4 6241eab6f8SAnton Blanchard static int distance_ref_points_depth; 63b08a2a12SAlistair Popple static const __be32 *distance_ref_points; 6441eab6f8SAnton Blanchard static int distance_lookup_table[MAX_NUMNODES][MAX_DISTANCE_REF_POINTS]; 65ab1f9dacSPaul Mackerras 6625863de0SAnton Blanchard /* 6725863de0SAnton Blanchard * Allocate node_to_cpumask_map based on number of available nodes 6825863de0SAnton Blanchard * Requires node_possible_map to be valid. 6925863de0SAnton Blanchard * 709512938bSWanlong Gao * Note: cpumask_of_node() is not valid until after this is done. 7125863de0SAnton Blanchard */ 7225863de0SAnton Blanchard static void __init setup_node_to_cpumask_map(void) 7325863de0SAnton Blanchard { 74f9d531b8SCody P Schafer unsigned int node; 7525863de0SAnton Blanchard 7625863de0SAnton Blanchard /* setup nr_node_ids if not done yet */ 77f9d531b8SCody P Schafer if (nr_node_ids == MAX_NUMNODES) 78f9d531b8SCody P Schafer setup_nr_node_ids(); 7925863de0SAnton Blanchard 8025863de0SAnton Blanchard /* allocate the map */ 8125863de0SAnton Blanchard for (node = 0; node < nr_node_ids; node++) 8225863de0SAnton Blanchard alloc_bootmem_cpumask_var(&node_to_cpumask_map[node]); 8325863de0SAnton Blanchard 8425863de0SAnton Blanchard /* cpumask_of_node() will now work */ 8525863de0SAnton Blanchard dbg("Node to cpumask map for %d nodes\n", nr_node_ids); 8625863de0SAnton Blanchard } 8725863de0SAnton Blanchard 8855671f3cSStephen Rothwell static int __init fake_numa_create_new_node(unsigned long end_pfn, 891daa6d08SBalbir Singh unsigned int *nid) 901daa6d08SBalbir Singh { 911daa6d08SBalbir Singh unsigned long long mem; 921daa6d08SBalbir Singh char *p = cmdline; 931daa6d08SBalbir Singh static unsigned int fake_nid; 941daa6d08SBalbir Singh static unsigned long long curr_boundary; 951daa6d08SBalbir Singh 961daa6d08SBalbir Singh /* 971daa6d08SBalbir Singh * Modify node id, iff we started creating NUMA nodes 981daa6d08SBalbir Singh * We want to continue from where we left of the last time 991daa6d08SBalbir Singh */ 1001daa6d08SBalbir Singh if (fake_nid) 1011daa6d08SBalbir Singh *nid = fake_nid; 1021daa6d08SBalbir Singh /* 1031daa6d08SBalbir Singh * In case there are no more arguments to parse, the 1041daa6d08SBalbir Singh * node_id should be the same as the last fake node id 1051daa6d08SBalbir Singh * (we've handled this above). 1061daa6d08SBalbir Singh */ 1071daa6d08SBalbir Singh if (!p) 1081daa6d08SBalbir Singh return 0; 1091daa6d08SBalbir Singh 1101daa6d08SBalbir Singh mem = memparse(p, &p); 1111daa6d08SBalbir Singh if (!mem) 1121daa6d08SBalbir Singh return 0; 1131daa6d08SBalbir Singh 1141daa6d08SBalbir Singh if (mem < curr_boundary) 1151daa6d08SBalbir Singh return 0; 1161daa6d08SBalbir Singh 1171daa6d08SBalbir Singh curr_boundary = mem; 1181daa6d08SBalbir Singh 1191daa6d08SBalbir Singh if ((end_pfn << PAGE_SHIFT) > mem) { 1201daa6d08SBalbir Singh /* 1211daa6d08SBalbir Singh * Skip commas and spaces 1221daa6d08SBalbir Singh */ 1231daa6d08SBalbir Singh while (*p == ',' || *p == ' ' || *p == '\t') 1241daa6d08SBalbir Singh p++; 1251daa6d08SBalbir Singh 1261daa6d08SBalbir Singh cmdline = p; 1271daa6d08SBalbir Singh fake_nid++; 1281daa6d08SBalbir Singh *nid = fake_nid; 1291daa6d08SBalbir Singh dbg("created new fake_node with id %d\n", fake_nid); 1301daa6d08SBalbir Singh return 1; 1311daa6d08SBalbir Singh } 1321daa6d08SBalbir Singh return 0; 1331daa6d08SBalbir Singh } 1341daa6d08SBalbir Singh 1358f64e1f2SJon Tollefson /* 1365dfe8660STejun Heo * get_node_active_region - Return active region containing pfn 137e8170372SJon Tollefson * Active range returned is empty if none found. 1385dfe8660STejun Heo * @pfn: The page to return the region for 1395dfe8660STejun Heo * @node_ar: Returned set to the active region containing @pfn 1408f64e1f2SJon Tollefson */ 1415dfe8660STejun Heo static void __init get_node_active_region(unsigned long pfn, 1428f64e1f2SJon Tollefson struct node_active_region *node_ar) 1438f64e1f2SJon Tollefson { 1445dfe8660STejun Heo unsigned long start_pfn, end_pfn; 1455dfe8660STejun Heo int i, nid; 1468f64e1f2SJon Tollefson 1475dfe8660STejun Heo for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) { 1485dfe8660STejun Heo if (pfn >= start_pfn && pfn < end_pfn) { 1498f64e1f2SJon Tollefson node_ar->nid = nid; 1508f64e1f2SJon Tollefson node_ar->start_pfn = start_pfn; 1515dfe8660STejun Heo node_ar->end_pfn = end_pfn; 1525dfe8660STejun Heo break; 1535dfe8660STejun Heo } 1545dfe8660STejun Heo } 1558f64e1f2SJon Tollefson } 1568f64e1f2SJon Tollefson 157d4edc5b6SSrivatsa S. Bhat static void reset_numa_cpu_lookup_table(void) 158d4edc5b6SSrivatsa S. Bhat { 159d4edc5b6SSrivatsa S. Bhat unsigned int cpu; 160d4edc5b6SSrivatsa S. Bhat 161d4edc5b6SSrivatsa S. Bhat for_each_possible_cpu(cpu) 162d4edc5b6SSrivatsa S. Bhat numa_cpu_lookup_table[cpu] = -1; 163d4edc5b6SSrivatsa S. Bhat } 164d4edc5b6SSrivatsa S. Bhat 165d4edc5b6SSrivatsa S. Bhat static void update_numa_cpu_lookup_table(unsigned int cpu, int node) 166ab1f9dacSPaul Mackerras { 167ab1f9dacSPaul Mackerras numa_cpu_lookup_table[cpu] = node; 168d4edc5b6SSrivatsa S. Bhat } 169d4edc5b6SSrivatsa S. Bhat 170d4edc5b6SSrivatsa S. Bhat static void map_cpu_to_node(int cpu, int node) 171d4edc5b6SSrivatsa S. Bhat { 172d4edc5b6SSrivatsa S. Bhat update_numa_cpu_lookup_table(cpu, node); 17345fb6ceaSAnton Blanchard 174bf4b85b0SNathan Lynch dbg("adding cpu %d to node %d\n", cpu, node); 175bf4b85b0SNathan Lynch 17625863de0SAnton Blanchard if (!(cpumask_test_cpu(cpu, node_to_cpumask_map[node]))) 17725863de0SAnton Blanchard cpumask_set_cpu(cpu, node_to_cpumask_map[node]); 178ab1f9dacSPaul Mackerras } 179ab1f9dacSPaul Mackerras 18039bf990eSJesse Larrew #if defined(CONFIG_HOTPLUG_CPU) || defined(CONFIG_PPC_SPLPAR) 181ab1f9dacSPaul Mackerras static void unmap_cpu_from_node(unsigned long cpu) 182ab1f9dacSPaul Mackerras { 183ab1f9dacSPaul Mackerras int node = numa_cpu_lookup_table[cpu]; 184ab1f9dacSPaul Mackerras 185ab1f9dacSPaul Mackerras dbg("removing cpu %lu from node %d\n", cpu, node); 186ab1f9dacSPaul Mackerras 18725863de0SAnton Blanchard if (cpumask_test_cpu(cpu, node_to_cpumask_map[node])) { 188429f4d8dSAnton Blanchard cpumask_clear_cpu(cpu, node_to_cpumask_map[node]); 189ab1f9dacSPaul Mackerras } else { 190ab1f9dacSPaul Mackerras printk(KERN_ERR "WARNING: cpu %lu not found in node %d\n", 191ab1f9dacSPaul Mackerras cpu, node); 192ab1f9dacSPaul Mackerras } 193ab1f9dacSPaul Mackerras } 19439bf990eSJesse Larrew #endif /* CONFIG_HOTPLUG_CPU || CONFIG_PPC_SPLPAR */ 195ab1f9dacSPaul Mackerras 196ab1f9dacSPaul Mackerras /* must hold reference to node during call */ 197b08a2a12SAlistair Popple static const __be32 *of_get_associativity(struct device_node *dev) 198ab1f9dacSPaul Mackerras { 199e2eb6392SStephen Rothwell return of_get_property(dev, "ibm,associativity", NULL); 200ab1f9dacSPaul Mackerras } 201ab1f9dacSPaul Mackerras 202cf00085dSChandru /* 203cf00085dSChandru * Returns the property linux,drconf-usable-memory if 204cf00085dSChandru * it exists (the property exists only in kexec/kdump kernels, 205cf00085dSChandru * added by kexec-tools) 206cf00085dSChandru */ 207b08a2a12SAlistair Popple static const __be32 *of_get_usable_memory(struct device_node *memory) 208cf00085dSChandru { 209b08a2a12SAlistair Popple const __be32 *prop; 210cf00085dSChandru u32 len; 211cf00085dSChandru prop = of_get_property(memory, "linux,drconf-usable-memory", &len); 212cf00085dSChandru if (!prop || len < sizeof(unsigned int)) 213ec32dd66SRobert Jennings return NULL; 214cf00085dSChandru return prop; 215cf00085dSChandru } 216cf00085dSChandru 21741eab6f8SAnton Blanchard int __node_distance(int a, int b) 21841eab6f8SAnton Blanchard { 21941eab6f8SAnton Blanchard int i; 22041eab6f8SAnton Blanchard int distance = LOCAL_DISTANCE; 22141eab6f8SAnton Blanchard 22241eab6f8SAnton Blanchard if (!form1_affinity) 2237122beeeSVaidyanathan Srinivasan return ((a == b) ? LOCAL_DISTANCE : REMOTE_DISTANCE); 22441eab6f8SAnton Blanchard 22541eab6f8SAnton Blanchard for (i = 0; i < distance_ref_points_depth; i++) { 22641eab6f8SAnton Blanchard if (distance_lookup_table[a][i] == distance_lookup_table[b][i]) 22741eab6f8SAnton Blanchard break; 22841eab6f8SAnton Blanchard 22941eab6f8SAnton Blanchard /* Double the distance for each NUMA level */ 23041eab6f8SAnton Blanchard distance *= 2; 23141eab6f8SAnton Blanchard } 23241eab6f8SAnton Blanchard 23341eab6f8SAnton Blanchard return distance; 23441eab6f8SAnton Blanchard } 23512c743ebSMike Qiu EXPORT_SYMBOL(__node_distance); 23641eab6f8SAnton Blanchard 23741eab6f8SAnton Blanchard static void initialize_distance_lookup_table(int nid, 238b08a2a12SAlistair Popple const __be32 *associativity) 23941eab6f8SAnton Blanchard { 24041eab6f8SAnton Blanchard int i; 24141eab6f8SAnton Blanchard 24241eab6f8SAnton Blanchard if (!form1_affinity) 24341eab6f8SAnton Blanchard return; 24441eab6f8SAnton Blanchard 24541eab6f8SAnton Blanchard for (i = 0; i < distance_ref_points_depth; i++) { 246b08a2a12SAlistair Popple const __be32 *entry; 247b08a2a12SAlistair Popple 248b08a2a12SAlistair Popple entry = &associativity[be32_to_cpu(distance_ref_points[i])]; 249b08a2a12SAlistair Popple distance_lookup_table[nid][i] = of_read_number(entry, 1); 25041eab6f8SAnton Blanchard } 25141eab6f8SAnton Blanchard } 25241eab6f8SAnton Blanchard 253482ec7c4SNathan Lynch /* Returns nid in the range [0..MAX_NUMNODES-1], or -1 if no useful numa 254482ec7c4SNathan Lynch * info is found. 255482ec7c4SNathan Lynch */ 256b08a2a12SAlistair Popple static int associativity_to_nid(const __be32 *associativity) 257ab1f9dacSPaul Mackerras { 258482ec7c4SNathan Lynch int nid = -1; 259ab1f9dacSPaul Mackerras 260ab1f9dacSPaul Mackerras if (min_common_depth == -1) 261482ec7c4SNathan Lynch goto out; 262ab1f9dacSPaul Mackerras 263b08a2a12SAlistair Popple if (of_read_number(associativity, 1) >= min_common_depth) 264b08a2a12SAlistair Popple nid = of_read_number(&associativity[min_common_depth], 1); 265bc16a759SNathan Lynch 266bc16a759SNathan Lynch /* POWER4 LPAR uses 0xffff as invalid node */ 267482ec7c4SNathan Lynch if (nid == 0xffff || nid >= MAX_NUMNODES) 268482ec7c4SNathan Lynch nid = -1; 26941eab6f8SAnton Blanchard 270b08a2a12SAlistair Popple if (nid > 0 && 271b08a2a12SAlistair Popple of_read_number(associativity, 1) >= distance_ref_points_depth) 2729eff1a38SJesse Larrew initialize_distance_lookup_table(nid, associativity); 27341eab6f8SAnton Blanchard 274482ec7c4SNathan Lynch out: 275cf950b7aSNathan Lynch return nid; 276ab1f9dacSPaul Mackerras } 277ab1f9dacSPaul Mackerras 2789eff1a38SJesse Larrew /* Returns the nid associated with the given device tree node, 2799eff1a38SJesse Larrew * or -1 if not found. 2809eff1a38SJesse Larrew */ 2819eff1a38SJesse Larrew static int of_node_to_nid_single(struct device_node *device) 2829eff1a38SJesse Larrew { 2839eff1a38SJesse Larrew int nid = -1; 284b08a2a12SAlistair Popple const __be32 *tmp; 2859eff1a38SJesse Larrew 2869eff1a38SJesse Larrew tmp = of_get_associativity(device); 2879eff1a38SJesse Larrew if (tmp) 2889eff1a38SJesse Larrew nid = associativity_to_nid(tmp); 2899eff1a38SJesse Larrew return nid; 2909eff1a38SJesse Larrew } 2919eff1a38SJesse Larrew 292953039c8SJeremy Kerr /* Walk the device tree upwards, looking for an associativity id */ 293953039c8SJeremy Kerr int of_node_to_nid(struct device_node *device) 294953039c8SJeremy Kerr { 295953039c8SJeremy Kerr struct device_node *tmp; 296953039c8SJeremy Kerr int nid = -1; 297953039c8SJeremy Kerr 298953039c8SJeremy Kerr of_node_get(device); 299953039c8SJeremy Kerr while (device) { 300953039c8SJeremy Kerr nid = of_node_to_nid_single(device); 301953039c8SJeremy Kerr if (nid != -1) 302953039c8SJeremy Kerr break; 303953039c8SJeremy Kerr 304953039c8SJeremy Kerr tmp = device; 305953039c8SJeremy Kerr device = of_get_parent(tmp); 306953039c8SJeremy Kerr of_node_put(tmp); 307953039c8SJeremy Kerr } 308953039c8SJeremy Kerr of_node_put(device); 309953039c8SJeremy Kerr 310953039c8SJeremy Kerr return nid; 311953039c8SJeremy Kerr } 312953039c8SJeremy Kerr EXPORT_SYMBOL_GPL(of_node_to_nid); 313953039c8SJeremy Kerr 314ab1f9dacSPaul Mackerras static int __init find_min_common_depth(void) 315ab1f9dacSPaul Mackerras { 31641eab6f8SAnton Blanchard int depth; 317e70606ebSMichael Ellerman struct device_node *root; 318ab1f9dacSPaul Mackerras 3191c8ee733SDipankar Sarma if (firmware_has_feature(FW_FEATURE_OPAL)) 3201c8ee733SDipankar Sarma root = of_find_node_by_path("/ibm,opal"); 3211c8ee733SDipankar Sarma else 322e70606ebSMichael Ellerman root = of_find_node_by_path("/rtas"); 323e70606ebSMichael Ellerman if (!root) 324e70606ebSMichael Ellerman root = of_find_node_by_path("/"); 325ab1f9dacSPaul Mackerras 326ab1f9dacSPaul Mackerras /* 32741eab6f8SAnton Blanchard * This property is a set of 32-bit integers, each representing 32841eab6f8SAnton Blanchard * an index into the ibm,associativity nodes. 32941eab6f8SAnton Blanchard * 33041eab6f8SAnton Blanchard * With form 0 affinity the first integer is for an SMP configuration 33141eab6f8SAnton Blanchard * (should be all 0's) and the second is for a normal NUMA 33241eab6f8SAnton Blanchard * configuration. We have only one level of NUMA. 33341eab6f8SAnton Blanchard * 33441eab6f8SAnton Blanchard * With form 1 affinity the first integer is the most significant 33541eab6f8SAnton Blanchard * NUMA boundary and the following are progressively less significant 33641eab6f8SAnton Blanchard * boundaries. There can be more than one level of NUMA. 337ab1f9dacSPaul Mackerras */ 338e70606ebSMichael Ellerman distance_ref_points = of_get_property(root, 33941eab6f8SAnton Blanchard "ibm,associativity-reference-points", 34041eab6f8SAnton Blanchard &distance_ref_points_depth); 341ab1f9dacSPaul Mackerras 34241eab6f8SAnton Blanchard if (!distance_ref_points) { 34341eab6f8SAnton Blanchard dbg("NUMA: ibm,associativity-reference-points not found.\n"); 34441eab6f8SAnton Blanchard goto err; 34541eab6f8SAnton Blanchard } 34641eab6f8SAnton Blanchard 34741eab6f8SAnton Blanchard distance_ref_points_depth /= sizeof(int); 34841eab6f8SAnton Blanchard 3498002b0c5SNathan Fontenot if (firmware_has_feature(FW_FEATURE_OPAL) || 3508002b0c5SNathan Fontenot firmware_has_feature(FW_FEATURE_TYPE1_AFFINITY)) { 351bc8449ccSAnton Blanchard dbg("Using form 1 affinity\n"); 35241eab6f8SAnton Blanchard form1_affinity = 1; 3534b83c330SAnton Blanchard } 3545b958a7eSGavin Shan 35541eab6f8SAnton Blanchard if (form1_affinity) { 356b08a2a12SAlistair Popple depth = of_read_number(distance_ref_points, 1); 357ab1f9dacSPaul Mackerras } else { 35841eab6f8SAnton Blanchard if (distance_ref_points_depth < 2) { 35941eab6f8SAnton Blanchard printk(KERN_WARNING "NUMA: " 36041eab6f8SAnton Blanchard "short ibm,associativity-reference-points\n"); 36141eab6f8SAnton Blanchard goto err; 362ab1f9dacSPaul Mackerras } 363ab1f9dacSPaul Mackerras 364b08a2a12SAlistair Popple depth = of_read_number(&distance_ref_points[1], 1); 36541eab6f8SAnton Blanchard } 36641eab6f8SAnton Blanchard 36741eab6f8SAnton Blanchard /* 36841eab6f8SAnton Blanchard * Warn and cap if the hardware supports more than 36941eab6f8SAnton Blanchard * MAX_DISTANCE_REF_POINTS domains. 37041eab6f8SAnton Blanchard */ 37141eab6f8SAnton Blanchard if (distance_ref_points_depth > MAX_DISTANCE_REF_POINTS) { 37241eab6f8SAnton Blanchard printk(KERN_WARNING "NUMA: distance array capped at " 37341eab6f8SAnton Blanchard "%d entries\n", MAX_DISTANCE_REF_POINTS); 37441eab6f8SAnton Blanchard distance_ref_points_depth = MAX_DISTANCE_REF_POINTS; 37541eab6f8SAnton Blanchard } 37641eab6f8SAnton Blanchard 377e70606ebSMichael Ellerman of_node_put(root); 378ab1f9dacSPaul Mackerras return depth; 37941eab6f8SAnton Blanchard 38041eab6f8SAnton Blanchard err: 381e70606ebSMichael Ellerman of_node_put(root); 38241eab6f8SAnton Blanchard return -1; 383ab1f9dacSPaul Mackerras } 384ab1f9dacSPaul Mackerras 38584c9fdd1SMike Kravetz static void __init get_n_mem_cells(int *n_addr_cells, int *n_size_cells) 386ab1f9dacSPaul Mackerras { 387ab1f9dacSPaul Mackerras struct device_node *memory = NULL; 388ab1f9dacSPaul Mackerras 389ab1f9dacSPaul Mackerras memory = of_find_node_by_type(memory, "memory"); 39054c23310SPaul Mackerras if (!memory) 39184c9fdd1SMike Kravetz panic("numa.c: No memory nodes found!"); 39254c23310SPaul Mackerras 393a8bda5ddSStephen Rothwell *n_addr_cells = of_n_addr_cells(memory); 3949213feeaSStephen Rothwell *n_size_cells = of_n_size_cells(memory); 39584c9fdd1SMike Kravetz of_node_put(memory); 396ab1f9dacSPaul Mackerras } 397ab1f9dacSPaul Mackerras 398b08a2a12SAlistair Popple static unsigned long read_n_cells(int n, const __be32 **buf) 399ab1f9dacSPaul Mackerras { 400ab1f9dacSPaul Mackerras unsigned long result = 0; 401ab1f9dacSPaul Mackerras 402ab1f9dacSPaul Mackerras while (n--) { 403b08a2a12SAlistair Popple result = (result << 32) | of_read_number(*buf, 1); 404ab1f9dacSPaul Mackerras (*buf)++; 405ab1f9dacSPaul Mackerras } 406ab1f9dacSPaul Mackerras return result; 407ab1f9dacSPaul Mackerras } 408ab1f9dacSPaul Mackerras 4098342681dSNathan Fontenot /* 41095f72d1eSYinghai Lu * Read the next memblock list entry from the ibm,dynamic-memory property 4118342681dSNathan Fontenot * and return the information in the provided of_drconf_cell structure. 4128342681dSNathan Fontenot */ 413b08a2a12SAlistair Popple static void read_drconf_cell(struct of_drconf_cell *drmem, const __be32 **cellp) 4148342681dSNathan Fontenot { 415b08a2a12SAlistair Popple const __be32 *cp; 4168342681dSNathan Fontenot 4178342681dSNathan Fontenot drmem->base_addr = read_n_cells(n_mem_addr_cells, cellp); 4188342681dSNathan Fontenot 4198342681dSNathan Fontenot cp = *cellp; 420b08a2a12SAlistair Popple drmem->drc_index = of_read_number(cp, 1); 421b08a2a12SAlistair Popple drmem->reserved = of_read_number(&cp[1], 1); 422b08a2a12SAlistair Popple drmem->aa_index = of_read_number(&cp[2], 1); 423b08a2a12SAlistair Popple drmem->flags = of_read_number(&cp[3], 1); 4248342681dSNathan Fontenot 4258342681dSNathan Fontenot *cellp = cp + 4; 4268342681dSNathan Fontenot } 4278342681dSNathan Fontenot 4288342681dSNathan Fontenot /* 42925985edcSLucas De Marchi * Retrieve and validate the ibm,dynamic-memory property of the device tree. 4308342681dSNathan Fontenot * 43195f72d1eSYinghai Lu * The layout of the ibm,dynamic-memory property is a number N of memblock 43295f72d1eSYinghai Lu * list entries followed by N memblock list entries. Each memblock list entry 43325985edcSLucas De Marchi * contains information as laid out in the of_drconf_cell struct above. 4348342681dSNathan Fontenot */ 435b08a2a12SAlistair Popple static int of_get_drconf_memory(struct device_node *memory, const __be32 **dm) 4368342681dSNathan Fontenot { 437b08a2a12SAlistair Popple const __be32 *prop; 4388342681dSNathan Fontenot u32 len, entries; 4398342681dSNathan Fontenot 4408342681dSNathan Fontenot prop = of_get_property(memory, "ibm,dynamic-memory", &len); 4418342681dSNathan Fontenot if (!prop || len < sizeof(unsigned int)) 4428342681dSNathan Fontenot return 0; 4438342681dSNathan Fontenot 444b08a2a12SAlistair Popple entries = of_read_number(prop++, 1); 4458342681dSNathan Fontenot 4468342681dSNathan Fontenot /* Now that we know the number of entries, revalidate the size 4478342681dSNathan Fontenot * of the property read in to ensure we have everything 4488342681dSNathan Fontenot */ 4498342681dSNathan Fontenot if (len < (entries * (n_mem_addr_cells + 4) + 1) * sizeof(unsigned int)) 4508342681dSNathan Fontenot return 0; 4518342681dSNathan Fontenot 4528342681dSNathan Fontenot *dm = prop; 4538342681dSNathan Fontenot return entries; 4548342681dSNathan Fontenot } 4558342681dSNathan Fontenot 4568342681dSNathan Fontenot /* 45725985edcSLucas De Marchi * Retrieve and validate the ibm,lmb-size property for drconf memory 4588342681dSNathan Fontenot * from the device tree. 4598342681dSNathan Fontenot */ 4603fdfd990SBenjamin Herrenschmidt static u64 of_get_lmb_size(struct device_node *memory) 4618342681dSNathan Fontenot { 462b08a2a12SAlistair Popple const __be32 *prop; 4638342681dSNathan Fontenot u32 len; 4648342681dSNathan Fontenot 4653fdfd990SBenjamin Herrenschmidt prop = of_get_property(memory, "ibm,lmb-size", &len); 4668342681dSNathan Fontenot if (!prop || len < sizeof(unsigned int)) 4678342681dSNathan Fontenot return 0; 4688342681dSNathan Fontenot 4698342681dSNathan Fontenot return read_n_cells(n_mem_size_cells, &prop); 4708342681dSNathan Fontenot } 4718342681dSNathan Fontenot 4728342681dSNathan Fontenot struct assoc_arrays { 4738342681dSNathan Fontenot u32 n_arrays; 4748342681dSNathan Fontenot u32 array_sz; 475b08a2a12SAlistair Popple const __be32 *arrays; 4768342681dSNathan Fontenot }; 4778342681dSNathan Fontenot 4788342681dSNathan Fontenot /* 47925985edcSLucas De Marchi * Retrieve and validate the list of associativity arrays for drconf 4808342681dSNathan Fontenot * memory from the ibm,associativity-lookup-arrays property of the 4818342681dSNathan Fontenot * device tree.. 4828342681dSNathan Fontenot * 4838342681dSNathan Fontenot * The layout of the ibm,associativity-lookup-arrays property is a number N 4848342681dSNathan Fontenot * indicating the number of associativity arrays, followed by a number M 4858342681dSNathan Fontenot * indicating the size of each associativity array, followed by a list 4868342681dSNathan Fontenot * of N associativity arrays. 4878342681dSNathan Fontenot */ 4888342681dSNathan Fontenot static int of_get_assoc_arrays(struct device_node *memory, 4898342681dSNathan Fontenot struct assoc_arrays *aa) 4908342681dSNathan Fontenot { 491b08a2a12SAlistair Popple const __be32 *prop; 4928342681dSNathan Fontenot u32 len; 4938342681dSNathan Fontenot 4948342681dSNathan Fontenot prop = of_get_property(memory, "ibm,associativity-lookup-arrays", &len); 4958342681dSNathan Fontenot if (!prop || len < 2 * sizeof(unsigned int)) 4968342681dSNathan Fontenot return -1; 4978342681dSNathan Fontenot 498b08a2a12SAlistair Popple aa->n_arrays = of_read_number(prop++, 1); 499b08a2a12SAlistair Popple aa->array_sz = of_read_number(prop++, 1); 5008342681dSNathan Fontenot 50142b2aa86SJustin P. Mattock /* Now that we know the number of arrays and size of each array, 5028342681dSNathan Fontenot * revalidate the size of the property read in. 5038342681dSNathan Fontenot */ 5048342681dSNathan Fontenot if (len < (aa->n_arrays * aa->array_sz + 2) * sizeof(unsigned int)) 5058342681dSNathan Fontenot return -1; 5068342681dSNathan Fontenot 5078342681dSNathan Fontenot aa->arrays = prop; 5088342681dSNathan Fontenot return 0; 5098342681dSNathan Fontenot } 5108342681dSNathan Fontenot 5118342681dSNathan Fontenot /* 5128342681dSNathan Fontenot * This is like of_node_to_nid_single() for memory represented in the 5138342681dSNathan Fontenot * ibm,dynamic-reconfiguration-memory node. 5148342681dSNathan Fontenot */ 5158342681dSNathan Fontenot static int of_drconf_to_nid_single(struct of_drconf_cell *drmem, 5168342681dSNathan Fontenot struct assoc_arrays *aa) 5178342681dSNathan Fontenot { 5188342681dSNathan Fontenot int default_nid = 0; 5198342681dSNathan Fontenot int nid = default_nid; 5208342681dSNathan Fontenot int index; 5218342681dSNathan Fontenot 5228342681dSNathan Fontenot if (min_common_depth > 0 && min_common_depth <= aa->array_sz && 5238342681dSNathan Fontenot !(drmem->flags & DRCONF_MEM_AI_INVALID) && 5248342681dSNathan Fontenot drmem->aa_index < aa->n_arrays) { 5258342681dSNathan Fontenot index = drmem->aa_index * aa->array_sz + min_common_depth - 1; 526b08a2a12SAlistair Popple nid = of_read_number(&aa->arrays[index], 1); 5278342681dSNathan Fontenot 5288342681dSNathan Fontenot if (nid == 0xffff || nid >= MAX_NUMNODES) 5298342681dSNathan Fontenot nid = default_nid; 5308342681dSNathan Fontenot } 5318342681dSNathan Fontenot 5328342681dSNathan Fontenot return nid; 5338342681dSNathan Fontenot } 5348342681dSNathan Fontenot 535ab1f9dacSPaul Mackerras /* 536ab1f9dacSPaul Mackerras * Figure out to which domain a cpu belongs and stick it there. 537ab1f9dacSPaul Mackerras * Return the id of the domain used. 538ab1f9dacSPaul Mackerras */ 539061d19f2SPaul Gortmaker static int numa_setup_cpu(unsigned long lcpu) 540ab1f9dacSPaul Mackerras { 541297cf502SLi Zhong int nid = -1; 542d4edc5b6SSrivatsa S. Bhat struct device_node *cpu; 543d4edc5b6SSrivatsa S. Bhat 544d4edc5b6SSrivatsa S. Bhat /* 545d4edc5b6SSrivatsa S. Bhat * If a valid cpu-to-node mapping is already available, use it 546d4edc5b6SSrivatsa S. Bhat * directly instead of querying the firmware, since it represents 547d4edc5b6SSrivatsa S. Bhat * the most recent mapping notified to us by the platform (eg: VPHN). 548d4edc5b6SSrivatsa S. Bhat */ 549d4edc5b6SSrivatsa S. Bhat if ((nid = numa_cpu_lookup_table[lcpu]) >= 0) { 550d4edc5b6SSrivatsa S. Bhat map_cpu_to_node(lcpu, nid); 551d4edc5b6SSrivatsa S. Bhat return nid; 552d4edc5b6SSrivatsa S. Bhat } 553d4edc5b6SSrivatsa S. Bhat 554d4edc5b6SSrivatsa S. Bhat cpu = of_get_cpu_node(lcpu, NULL); 555ab1f9dacSPaul Mackerras 556ab1f9dacSPaul Mackerras if (!cpu) { 557ab1f9dacSPaul Mackerras WARN_ON(1); 558297cf502SLi Zhong if (cpu_present(lcpu)) 559297cf502SLi Zhong goto out_present; 560297cf502SLi Zhong else 561ab1f9dacSPaul Mackerras goto out; 562ab1f9dacSPaul Mackerras } 563ab1f9dacSPaul Mackerras 564953039c8SJeremy Kerr nid = of_node_to_nid_single(cpu); 565ab1f9dacSPaul Mackerras 566297cf502SLi Zhong out_present: 567482ec7c4SNathan Lynch if (nid < 0 || !node_online(nid)) 56872c33688SH Hartley Sweeten nid = first_online_node; 569297cf502SLi Zhong 570cf950b7aSNathan Lynch map_cpu_to_node(lcpu, nid); 571ab1f9dacSPaul Mackerras of_node_put(cpu); 572297cf502SLi Zhong out: 573cf950b7aSNathan Lynch return nid; 574ab1f9dacSPaul Mackerras } 575ab1f9dacSPaul Mackerras 57668fb18aaSSrivatsa S. Bhat static void verify_cpu_node_mapping(int cpu, int node) 57768fb18aaSSrivatsa S. Bhat { 57868fb18aaSSrivatsa S. Bhat int base, sibling, i; 57968fb18aaSSrivatsa S. Bhat 58068fb18aaSSrivatsa S. Bhat /* Verify that all the threads in the core belong to the same node */ 58168fb18aaSSrivatsa S. Bhat base = cpu_first_thread_sibling(cpu); 58268fb18aaSSrivatsa S. Bhat 58368fb18aaSSrivatsa S. Bhat for (i = 0; i < threads_per_core; i++) { 58468fb18aaSSrivatsa S. Bhat sibling = base + i; 58568fb18aaSSrivatsa S. Bhat 58668fb18aaSSrivatsa S. Bhat if (sibling == cpu || cpu_is_offline(sibling)) 58768fb18aaSSrivatsa S. Bhat continue; 58868fb18aaSSrivatsa S. Bhat 58968fb18aaSSrivatsa S. Bhat if (cpu_to_node(sibling) != node) { 59068fb18aaSSrivatsa S. Bhat WARN(1, "CPU thread siblings %d and %d don't belong" 59168fb18aaSSrivatsa S. Bhat " to the same node!\n", cpu, sibling); 59268fb18aaSSrivatsa S. Bhat break; 59368fb18aaSSrivatsa S. Bhat } 59468fb18aaSSrivatsa S. Bhat } 59568fb18aaSSrivatsa S. Bhat } 59668fb18aaSSrivatsa S. Bhat 597061d19f2SPaul Gortmaker static int cpu_numa_callback(struct notifier_block *nfb, unsigned long action, 598ab1f9dacSPaul Mackerras void *hcpu) 599ab1f9dacSPaul Mackerras { 600ab1f9dacSPaul Mackerras unsigned long lcpu = (unsigned long)hcpu; 60168fb18aaSSrivatsa S. Bhat int ret = NOTIFY_DONE, nid; 602ab1f9dacSPaul Mackerras 603ab1f9dacSPaul Mackerras switch (action) { 604ab1f9dacSPaul Mackerras case CPU_UP_PREPARE: 6058bb78442SRafael J. Wysocki case CPU_UP_PREPARE_FROZEN: 60668fb18aaSSrivatsa S. Bhat nid = numa_setup_cpu(lcpu); 60768fb18aaSSrivatsa S. Bhat verify_cpu_node_mapping((int)lcpu, nid); 608ab1f9dacSPaul Mackerras ret = NOTIFY_OK; 609ab1f9dacSPaul Mackerras break; 610ab1f9dacSPaul Mackerras #ifdef CONFIG_HOTPLUG_CPU 611ab1f9dacSPaul Mackerras case CPU_DEAD: 6128bb78442SRafael J. Wysocki case CPU_DEAD_FROZEN: 613ab1f9dacSPaul Mackerras case CPU_UP_CANCELED: 6148bb78442SRafael J. Wysocki case CPU_UP_CANCELED_FROZEN: 615ab1f9dacSPaul Mackerras unmap_cpu_from_node(lcpu); 616ab1f9dacSPaul Mackerras ret = NOTIFY_OK; 617b00fc6ecSAndrey Utkin break; 618ab1f9dacSPaul Mackerras #endif 619ab1f9dacSPaul Mackerras } 620ab1f9dacSPaul Mackerras return ret; 621ab1f9dacSPaul Mackerras } 622ab1f9dacSPaul Mackerras 623ab1f9dacSPaul Mackerras /* 624ab1f9dacSPaul Mackerras * Check and possibly modify a memory region to enforce the memory limit. 625ab1f9dacSPaul Mackerras * 626ab1f9dacSPaul Mackerras * Returns the size the region should have to enforce the memory limit. 627ab1f9dacSPaul Mackerras * This will either be the original value of size, a truncated value, 628ab1f9dacSPaul Mackerras * or zero. If the returned value of size is 0 the region should be 62925985edcSLucas De Marchi * discarded as it lies wholly above the memory limit. 630ab1f9dacSPaul Mackerras */ 63145fb6ceaSAnton Blanchard static unsigned long __init numa_enforce_memory_limit(unsigned long start, 63245fb6ceaSAnton Blanchard unsigned long size) 633ab1f9dacSPaul Mackerras { 634ab1f9dacSPaul Mackerras /* 63595f72d1eSYinghai Lu * We use memblock_end_of_DRAM() in here instead of memory_limit because 636ab1f9dacSPaul Mackerras * we've already adjusted it for the limit and it takes care of 637fe55249dSMilton Miller * having memory holes below the limit. Also, in the case of 638fe55249dSMilton Miller * iommu_is_off, memory_limit is not set but is implicitly enforced. 639ab1f9dacSPaul Mackerras */ 640ab1f9dacSPaul Mackerras 64195f72d1eSYinghai Lu if (start + size <= memblock_end_of_DRAM()) 642ab1f9dacSPaul Mackerras return size; 643ab1f9dacSPaul Mackerras 64495f72d1eSYinghai Lu if (start >= memblock_end_of_DRAM()) 645ab1f9dacSPaul Mackerras return 0; 646ab1f9dacSPaul Mackerras 64795f72d1eSYinghai Lu return memblock_end_of_DRAM() - start; 648ab1f9dacSPaul Mackerras } 649ab1f9dacSPaul Mackerras 6500204568aSPaul Mackerras /* 651cf00085dSChandru * Reads the counter for a given entry in 652cf00085dSChandru * linux,drconf-usable-memory property 653cf00085dSChandru */ 654b08a2a12SAlistair Popple static inline int __init read_usm_ranges(const __be32 **usm) 655cf00085dSChandru { 656cf00085dSChandru /* 6573fdfd990SBenjamin Herrenschmidt * For each lmb in ibm,dynamic-memory a corresponding 658cf00085dSChandru * entry in linux,drconf-usable-memory property contains 659cf00085dSChandru * a counter followed by that many (base, size) duple. 660cf00085dSChandru * read the counter from linux,drconf-usable-memory 661cf00085dSChandru */ 662cf00085dSChandru return read_n_cells(n_mem_size_cells, usm); 663cf00085dSChandru } 664cf00085dSChandru 665cf00085dSChandru /* 6660204568aSPaul Mackerras * Extract NUMA information from the ibm,dynamic-reconfiguration-memory 6670204568aSPaul Mackerras * node. This assumes n_mem_{addr,size}_cells have been set. 6680204568aSPaul Mackerras */ 6690204568aSPaul Mackerras static void __init parse_drconf_memory(struct device_node *memory) 6700204568aSPaul Mackerras { 671b08a2a12SAlistair Popple const __be32 *uninitialized_var(dm), *usm; 672cf00085dSChandru unsigned int n, rc, ranges, is_kexec_kdump = 0; 6733fdfd990SBenjamin Herrenschmidt unsigned long lmb_size, base, size, sz; 6748342681dSNathan Fontenot int nid; 675aa709f3bSBenjamin Herrenschmidt struct assoc_arrays aa = { .arrays = NULL }; 6760204568aSPaul Mackerras 6778342681dSNathan Fontenot n = of_get_drconf_memory(memory, &dm); 6788342681dSNathan Fontenot if (!n) 6790204568aSPaul Mackerras return; 6800204568aSPaul Mackerras 6813fdfd990SBenjamin Herrenschmidt lmb_size = of_get_lmb_size(memory); 6823fdfd990SBenjamin Herrenschmidt if (!lmb_size) 6838342681dSNathan Fontenot return; 6848342681dSNathan Fontenot 6858342681dSNathan Fontenot rc = of_get_assoc_arrays(memory, &aa); 6868342681dSNathan Fontenot if (rc) 6870204568aSPaul Mackerras return; 6880204568aSPaul Mackerras 689cf00085dSChandru /* check if this is a kexec/kdump kernel */ 690cf00085dSChandru usm = of_get_usable_memory(memory); 691cf00085dSChandru if (usm != NULL) 692cf00085dSChandru is_kexec_kdump = 1; 693cf00085dSChandru 6940204568aSPaul Mackerras for (; n != 0; --n) { 6958342681dSNathan Fontenot struct of_drconf_cell drmem; 6961daa6d08SBalbir Singh 6978342681dSNathan Fontenot read_drconf_cell(&drmem, &dm); 6988342681dSNathan Fontenot 6998342681dSNathan Fontenot /* skip this block if the reserved bit is set in flags (0x80) 7008342681dSNathan Fontenot or if the block is not assigned to this partition (0x8) */ 7018342681dSNathan Fontenot if ((drmem.flags & DRCONF_MEM_RESERVED) 7028342681dSNathan Fontenot || !(drmem.flags & DRCONF_MEM_ASSIGNED)) 7038342681dSNathan Fontenot continue; 7048342681dSNathan Fontenot 705cf00085dSChandru base = drmem.base_addr; 7063fdfd990SBenjamin Herrenschmidt size = lmb_size; 707cf00085dSChandru ranges = 1; 7088342681dSNathan Fontenot 709cf00085dSChandru if (is_kexec_kdump) { 710cf00085dSChandru ranges = read_usm_ranges(&usm); 711cf00085dSChandru if (!ranges) /* there are no (base, size) duple */ 7120204568aSPaul Mackerras continue; 713cf00085dSChandru } 714cf00085dSChandru do { 715cf00085dSChandru if (is_kexec_kdump) { 716cf00085dSChandru base = read_n_cells(n_mem_addr_cells, &usm); 717cf00085dSChandru size = read_n_cells(n_mem_size_cells, &usm); 718cf00085dSChandru } 719cf00085dSChandru nid = of_drconf_to_nid_single(&drmem, &aa); 720cf00085dSChandru fake_numa_create_new_node( 721cf00085dSChandru ((base + size) >> PAGE_SHIFT), 722cf00085dSChandru &nid); 723cf00085dSChandru node_set_online(nid); 724cf00085dSChandru sz = numa_enforce_memory_limit(base, size); 725cf00085dSChandru if (sz) 726e7e8de59STang Chen memblock_set_node(base, sz, 727e7e8de59STang Chen &memblock.memory, nid); 728cf00085dSChandru } while (--ranges); 7290204568aSPaul Mackerras } 7300204568aSPaul Mackerras } 7310204568aSPaul Mackerras 732ab1f9dacSPaul Mackerras static int __init parse_numa_properties(void) 733ab1f9dacSPaul Mackerras { 73494db7c5eSAnton Blanchard struct device_node *memory; 735482ec7c4SNathan Lynch int default_nid = 0; 736ab1f9dacSPaul Mackerras unsigned long i; 737ab1f9dacSPaul Mackerras 738ab1f9dacSPaul Mackerras if (numa_enabled == 0) { 739ab1f9dacSPaul Mackerras printk(KERN_WARNING "NUMA disabled by user\n"); 740ab1f9dacSPaul Mackerras return -1; 741ab1f9dacSPaul Mackerras } 742ab1f9dacSPaul Mackerras 743ab1f9dacSPaul Mackerras min_common_depth = find_min_common_depth(); 744ab1f9dacSPaul Mackerras 745ab1f9dacSPaul Mackerras if (min_common_depth < 0) 746ab1f9dacSPaul Mackerras return min_common_depth; 747ab1f9dacSPaul Mackerras 748bf4b85b0SNathan Lynch dbg("NUMA associativity depth for CPU/Memory: %d\n", min_common_depth); 749bf4b85b0SNathan Lynch 750ab1f9dacSPaul Mackerras /* 751482ec7c4SNathan Lynch * Even though we connect cpus to numa domains later in SMP 752482ec7c4SNathan Lynch * init, we need to know the node ids now. This is because 753482ec7c4SNathan Lynch * each node to be onlined must have NODE_DATA etc backing it. 754ab1f9dacSPaul Mackerras */ 755482ec7c4SNathan Lynch for_each_present_cpu(i) { 756dfbe93a2SAnton Blanchard struct device_node *cpu; 757cf950b7aSNathan Lynch int nid; 758ab1f9dacSPaul Mackerras 7598b16cd23SMilton Miller cpu = of_get_cpu_node(i, NULL); 760482ec7c4SNathan Lynch BUG_ON(!cpu); 761953039c8SJeremy Kerr nid = of_node_to_nid_single(cpu); 762ab1f9dacSPaul Mackerras of_node_put(cpu); 763ab1f9dacSPaul Mackerras 764482ec7c4SNathan Lynch /* 765482ec7c4SNathan Lynch * Don't fall back to default_nid yet -- we will plug 766482ec7c4SNathan Lynch * cpus into nodes once the memory scan has discovered 767482ec7c4SNathan Lynch * the topology. 768482ec7c4SNathan Lynch */ 769482ec7c4SNathan Lynch if (nid < 0) 770482ec7c4SNathan Lynch continue; 771482ec7c4SNathan Lynch node_set_online(nid); 772ab1f9dacSPaul Mackerras } 773ab1f9dacSPaul Mackerras 774237a0989SMike Kravetz get_n_mem_cells(&n_mem_addr_cells, &n_mem_size_cells); 77594db7c5eSAnton Blanchard 77694db7c5eSAnton Blanchard for_each_node_by_type(memory, "memory") { 777ab1f9dacSPaul Mackerras unsigned long start; 778ab1f9dacSPaul Mackerras unsigned long size; 779cf950b7aSNathan Lynch int nid; 780ab1f9dacSPaul Mackerras int ranges; 781b08a2a12SAlistair Popple const __be32 *memcell_buf; 782ab1f9dacSPaul Mackerras unsigned int len; 783ab1f9dacSPaul Mackerras 784e2eb6392SStephen Rothwell memcell_buf = of_get_property(memory, 785ba759485SMichael Ellerman "linux,usable-memory", &len); 786ba759485SMichael Ellerman if (!memcell_buf || len <= 0) 787e2eb6392SStephen Rothwell memcell_buf = of_get_property(memory, "reg", &len); 788ab1f9dacSPaul Mackerras if (!memcell_buf || len <= 0) 789ab1f9dacSPaul Mackerras continue; 790ab1f9dacSPaul Mackerras 791cc5d0189SBenjamin Herrenschmidt /* ranges in cell */ 792cc5d0189SBenjamin Herrenschmidt ranges = (len >> 2) / (n_mem_addr_cells + n_mem_size_cells); 793ab1f9dacSPaul Mackerras new_range: 794ab1f9dacSPaul Mackerras /* these are order-sensitive, and modify the buffer pointer */ 795237a0989SMike Kravetz start = read_n_cells(n_mem_addr_cells, &memcell_buf); 796237a0989SMike Kravetz size = read_n_cells(n_mem_size_cells, &memcell_buf); 797ab1f9dacSPaul Mackerras 798482ec7c4SNathan Lynch /* 799482ec7c4SNathan Lynch * Assumption: either all memory nodes or none will 800482ec7c4SNathan Lynch * have associativity properties. If none, then 801482ec7c4SNathan Lynch * everything goes to default_nid. 802482ec7c4SNathan Lynch */ 803953039c8SJeremy Kerr nid = of_node_to_nid_single(memory); 804482ec7c4SNathan Lynch if (nid < 0) 805482ec7c4SNathan Lynch nid = default_nid; 8061daa6d08SBalbir Singh 8071daa6d08SBalbir Singh fake_numa_create_new_node(((start + size) >> PAGE_SHIFT), &nid); 808482ec7c4SNathan Lynch node_set_online(nid); 809ab1f9dacSPaul Mackerras 810ab1f9dacSPaul Mackerras if (!(size = numa_enforce_memory_limit(start, size))) { 811ab1f9dacSPaul Mackerras if (--ranges) 812ab1f9dacSPaul Mackerras goto new_range; 813ab1f9dacSPaul Mackerras else 814ab1f9dacSPaul Mackerras continue; 815ab1f9dacSPaul Mackerras } 816ab1f9dacSPaul Mackerras 817e7e8de59STang Chen memblock_set_node(start, size, &memblock.memory, nid); 818ab1f9dacSPaul Mackerras 819ab1f9dacSPaul Mackerras if (--ranges) 820ab1f9dacSPaul Mackerras goto new_range; 821ab1f9dacSPaul Mackerras } 822ab1f9dacSPaul Mackerras 8230204568aSPaul Mackerras /* 824dfbe93a2SAnton Blanchard * Now do the same thing for each MEMBLOCK listed in the 825dfbe93a2SAnton Blanchard * ibm,dynamic-memory property in the 826dfbe93a2SAnton Blanchard * ibm,dynamic-reconfiguration-memory node. 8270204568aSPaul Mackerras */ 8280204568aSPaul Mackerras memory = of_find_node_by_path("/ibm,dynamic-reconfiguration-memory"); 8290204568aSPaul Mackerras if (memory) 8300204568aSPaul Mackerras parse_drconf_memory(memory); 8310204568aSPaul Mackerras 832ab1f9dacSPaul Mackerras return 0; 833ab1f9dacSPaul Mackerras } 834ab1f9dacSPaul Mackerras 835ab1f9dacSPaul Mackerras static void __init setup_nonnuma(void) 836ab1f9dacSPaul Mackerras { 83795f72d1eSYinghai Lu unsigned long top_of_ram = memblock_end_of_DRAM(); 83895f72d1eSYinghai Lu unsigned long total_ram = memblock_phys_mem_size(); 839c67c3cb4SMel Gorman unsigned long start_pfn, end_pfn; 84028be7072SBenjamin Herrenschmidt unsigned int nid = 0; 84128be7072SBenjamin Herrenschmidt struct memblock_region *reg; 842ab1f9dacSPaul Mackerras 843e110b281SOlof Johansson printk(KERN_DEBUG "Top of RAM: 0x%lx, Total RAM: 0x%lx\n", 844ab1f9dacSPaul Mackerras top_of_ram, total_ram); 845e110b281SOlof Johansson printk(KERN_DEBUG "Memory hole size: %ldMB\n", 846ab1f9dacSPaul Mackerras (top_of_ram - total_ram) >> 20); 847ab1f9dacSPaul Mackerras 84828be7072SBenjamin Herrenschmidt for_each_memblock(memory, reg) { 849c7fc2de0SYinghai Lu start_pfn = memblock_region_memory_base_pfn(reg); 850c7fc2de0SYinghai Lu end_pfn = memblock_region_memory_end_pfn(reg); 8511daa6d08SBalbir Singh 8521daa6d08SBalbir Singh fake_numa_create_new_node(end_pfn, &nid); 8531d7cfe18STejun Heo memblock_set_node(PFN_PHYS(start_pfn), 854e7e8de59STang Chen PFN_PHYS(end_pfn - start_pfn), 855e7e8de59STang Chen &memblock.memory, nid); 8561daa6d08SBalbir Singh node_set_online(nid); 857c67c3cb4SMel Gorman } 858ab1f9dacSPaul Mackerras } 859ab1f9dacSPaul Mackerras 8604b703a23SAnton Blanchard void __init dump_numa_cpu_topology(void) 8614b703a23SAnton Blanchard { 8624b703a23SAnton Blanchard unsigned int node; 8634b703a23SAnton Blanchard unsigned int cpu, count; 8644b703a23SAnton Blanchard 8654b703a23SAnton Blanchard if (min_common_depth == -1 || !numa_enabled) 8664b703a23SAnton Blanchard return; 8674b703a23SAnton Blanchard 8684b703a23SAnton Blanchard for_each_online_node(node) { 869e110b281SOlof Johansson printk(KERN_DEBUG "Node %d CPUs:", node); 8704b703a23SAnton Blanchard 8714b703a23SAnton Blanchard count = 0; 8724b703a23SAnton Blanchard /* 8734b703a23SAnton Blanchard * If we used a CPU iterator here we would miss printing 8744b703a23SAnton Blanchard * the holes in the cpumap. 8754b703a23SAnton Blanchard */ 87625863de0SAnton Blanchard for (cpu = 0; cpu < nr_cpu_ids; cpu++) { 87725863de0SAnton Blanchard if (cpumask_test_cpu(cpu, 87825863de0SAnton Blanchard node_to_cpumask_map[node])) { 8794b703a23SAnton Blanchard if (count == 0) 8804b703a23SAnton Blanchard printk(" %u", cpu); 8814b703a23SAnton Blanchard ++count; 8824b703a23SAnton Blanchard } else { 8834b703a23SAnton Blanchard if (count > 1) 8844b703a23SAnton Blanchard printk("-%u", cpu - 1); 8854b703a23SAnton Blanchard count = 0; 8864b703a23SAnton Blanchard } 8874b703a23SAnton Blanchard } 8884b703a23SAnton Blanchard 8894b703a23SAnton Blanchard if (count > 1) 89025863de0SAnton Blanchard printk("-%u", nr_cpu_ids - 1); 8914b703a23SAnton Blanchard printk("\n"); 8924b703a23SAnton Blanchard } 8934b703a23SAnton Blanchard } 8944b703a23SAnton Blanchard 8954b703a23SAnton Blanchard static void __init dump_numa_memory_topology(void) 896ab1f9dacSPaul Mackerras { 897ab1f9dacSPaul Mackerras unsigned int node; 898ab1f9dacSPaul Mackerras unsigned int count; 899ab1f9dacSPaul Mackerras 900ab1f9dacSPaul Mackerras if (min_common_depth == -1 || !numa_enabled) 901ab1f9dacSPaul Mackerras return; 902ab1f9dacSPaul Mackerras 903ab1f9dacSPaul Mackerras for_each_online_node(node) { 904ab1f9dacSPaul Mackerras unsigned long i; 905ab1f9dacSPaul Mackerras 906e110b281SOlof Johansson printk(KERN_DEBUG "Node %d Memory:", node); 907ab1f9dacSPaul Mackerras 908ab1f9dacSPaul Mackerras count = 0; 909ab1f9dacSPaul Mackerras 91095f72d1eSYinghai Lu for (i = 0; i < memblock_end_of_DRAM(); 91145fb6ceaSAnton Blanchard i += (1 << SECTION_SIZE_BITS)) { 91245fb6ceaSAnton Blanchard if (early_pfn_to_nid(i >> PAGE_SHIFT) == node) { 913ab1f9dacSPaul Mackerras if (count == 0) 914ab1f9dacSPaul Mackerras printk(" 0x%lx", i); 915ab1f9dacSPaul Mackerras ++count; 916ab1f9dacSPaul Mackerras } else { 917ab1f9dacSPaul Mackerras if (count > 0) 918ab1f9dacSPaul Mackerras printk("-0x%lx", i); 919ab1f9dacSPaul Mackerras count = 0; 920ab1f9dacSPaul Mackerras } 921ab1f9dacSPaul Mackerras } 922ab1f9dacSPaul Mackerras 923ab1f9dacSPaul Mackerras if (count > 0) 924ab1f9dacSPaul Mackerras printk("-0x%lx", i); 925ab1f9dacSPaul Mackerras printk("\n"); 926ab1f9dacSPaul Mackerras } 927ab1f9dacSPaul Mackerras } 928ab1f9dacSPaul Mackerras 929ab1f9dacSPaul Mackerras /* 93095f72d1eSYinghai Lu * Allocate some memory, satisfying the memblock or bootmem allocator where 931ab1f9dacSPaul Mackerras * required. nid is the preferred node and end is the physical address of 932ab1f9dacSPaul Mackerras * the highest address in the node. 933ab1f9dacSPaul Mackerras * 9340be210fdSDave Hansen * Returns the virtual address of the memory. 935ab1f9dacSPaul Mackerras */ 936893473dfSDave Hansen static void __init *careful_zallocation(int nid, unsigned long size, 93745fb6ceaSAnton Blanchard unsigned long align, 93845fb6ceaSAnton Blanchard unsigned long end_pfn) 939ab1f9dacSPaul Mackerras { 9400be210fdSDave Hansen void *ret; 94145fb6ceaSAnton Blanchard int new_nid; 9420be210fdSDave Hansen unsigned long ret_paddr; 9430be210fdSDave Hansen 94495f72d1eSYinghai Lu ret_paddr = __memblock_alloc_base(size, align, end_pfn << PAGE_SHIFT); 945ab1f9dacSPaul Mackerras 946ab1f9dacSPaul Mackerras /* retry over all memory */ 9470be210fdSDave Hansen if (!ret_paddr) 94895f72d1eSYinghai Lu ret_paddr = __memblock_alloc_base(size, align, memblock_end_of_DRAM()); 949ab1f9dacSPaul Mackerras 9500be210fdSDave Hansen if (!ret_paddr) 9515d21ea2bSDave Hansen panic("numa.c: cannot allocate %lu bytes for node %d", 952ab1f9dacSPaul Mackerras size, nid); 953ab1f9dacSPaul Mackerras 9540be210fdSDave Hansen ret = __va(ret_paddr); 9550be210fdSDave Hansen 956ab1f9dacSPaul Mackerras /* 957c555e520SDave Hansen * We initialize the nodes in numeric order: 0, 1, 2... 95895f72d1eSYinghai Lu * and hand over control from the MEMBLOCK allocator to the 959c555e520SDave Hansen * bootmem allocator. If this function is called for 960c555e520SDave Hansen * node 5, then we know that all nodes <5 are using the 96195f72d1eSYinghai Lu * bootmem allocator instead of the MEMBLOCK allocator. 962c555e520SDave Hansen * 963c555e520SDave Hansen * So, check the nid from which this allocation came 964c555e520SDave Hansen * and double check to see if we need to use bootmem 96595f72d1eSYinghai Lu * instead of the MEMBLOCK. We don't free the MEMBLOCK memory 966c555e520SDave Hansen * since it would be useless. 967ab1f9dacSPaul Mackerras */ 9680be210fdSDave Hansen new_nid = early_pfn_to_nid(ret_paddr >> PAGE_SHIFT); 96945fb6ceaSAnton Blanchard if (new_nid < nid) { 9700be210fdSDave Hansen ret = __alloc_bootmem_node(NODE_DATA(new_nid), 971ab1f9dacSPaul Mackerras size, align, 0); 972ab1f9dacSPaul Mackerras 9730be210fdSDave Hansen dbg("alloc_bootmem %p %lx\n", ret, size); 974ab1f9dacSPaul Mackerras } 975ab1f9dacSPaul Mackerras 976893473dfSDave Hansen memset(ret, 0, size); 9770be210fdSDave Hansen return ret; 978ab1f9dacSPaul Mackerras } 979ab1f9dacSPaul Mackerras 980061d19f2SPaul Gortmaker static struct notifier_block ppc64_numa_nb = { 98174b85f37SChandra Seetharaman .notifier_call = cpu_numa_callback, 98274b85f37SChandra Seetharaman .priority = 1 /* Must run before sched domains notifier. */ 98374b85f37SChandra Seetharaman }; 98474b85f37SChandra Seetharaman 98528e86bdbSDavid Rientjes static void __init mark_reserved_regions_for_nid(int nid) 986ab1f9dacSPaul Mackerras { 9874a618669SDave Hansen struct pglist_data *node = NODE_DATA(nid); 98828be7072SBenjamin Herrenschmidt struct memblock_region *reg; 989ab1f9dacSPaul Mackerras 99028be7072SBenjamin Herrenschmidt for_each_memblock(reserved, reg) { 99128be7072SBenjamin Herrenschmidt unsigned long physbase = reg->base; 99228be7072SBenjamin Herrenschmidt unsigned long size = reg->size; 9938f64e1f2SJon Tollefson unsigned long start_pfn = physbase >> PAGE_SHIFT; 99406eccea6SDave Hansen unsigned long end_pfn = PFN_UP(physbase + size); 9958f64e1f2SJon Tollefson struct node_active_region node_ar; 9966408068eSXishi Qiu unsigned long node_end_pfn = pgdat_end_pfn(node); 9974a618669SDave Hansen 9984a618669SDave Hansen /* 99995f72d1eSYinghai Lu * Check to make sure that this memblock.reserved area is 10004a618669SDave Hansen * within the bounds of the node that we care about. 10014a618669SDave Hansen * Checking the nid of the start and end points is not 10024a618669SDave Hansen * sufficient because the reserved area could span the 10034a618669SDave Hansen * entire node. 10044a618669SDave Hansen */ 10054a618669SDave Hansen if (end_pfn <= node->node_start_pfn || 10064a618669SDave Hansen start_pfn >= node_end_pfn) 10074a618669SDave Hansen continue; 1008ab1f9dacSPaul Mackerras 10098f64e1f2SJon Tollefson get_node_active_region(start_pfn, &node_ar); 1010e8170372SJon Tollefson while (start_pfn < end_pfn && 1011e8170372SJon Tollefson node_ar.start_pfn < node_ar.end_pfn) { 1012e8170372SJon Tollefson unsigned long reserve_size = size; 10138f64e1f2SJon Tollefson /* 10148f64e1f2SJon Tollefson * if reserved region extends past active region 10158f64e1f2SJon Tollefson * then trim size to active region 10168f64e1f2SJon Tollefson */ 10178f64e1f2SJon Tollefson if (end_pfn > node_ar.end_pfn) 1018e8170372SJon Tollefson reserve_size = (node_ar.end_pfn << PAGE_SHIFT) 101906eccea6SDave Hansen - physbase; 1020a4c74dddSDave Hansen /* 1021a4c74dddSDave Hansen * Only worry about *this* node, others may not 1022a4c74dddSDave Hansen * yet have valid NODE_DATA(). 1023a4c74dddSDave Hansen */ 1024a4c74dddSDave Hansen if (node_ar.nid == nid) { 1025a4c74dddSDave Hansen dbg("reserve_bootmem %lx %lx nid=%d\n", 1026a4c74dddSDave Hansen physbase, reserve_size, node_ar.nid); 1027a4c74dddSDave Hansen reserve_bootmem_node(NODE_DATA(node_ar.nid), 1028a4c74dddSDave Hansen physbase, reserve_size, 1029a4c74dddSDave Hansen BOOTMEM_DEFAULT); 1030a4c74dddSDave Hansen } 10318f64e1f2SJon Tollefson /* 10328f64e1f2SJon Tollefson * if reserved region is contained in the active region 10338f64e1f2SJon Tollefson * then done. 10348f64e1f2SJon Tollefson */ 10358f64e1f2SJon Tollefson if (end_pfn <= node_ar.end_pfn) 10368f64e1f2SJon Tollefson break; 10378f64e1f2SJon Tollefson 10388f64e1f2SJon Tollefson /* 10398f64e1f2SJon Tollefson * reserved region extends past the active region 10408f64e1f2SJon Tollefson * get next active region that contains this 10418f64e1f2SJon Tollefson * reserved region 10428f64e1f2SJon Tollefson */ 10438f64e1f2SJon Tollefson start_pfn = node_ar.end_pfn; 10448f64e1f2SJon Tollefson physbase = start_pfn << PAGE_SHIFT; 1045e8170372SJon Tollefson size = size - reserve_size; 10468f64e1f2SJon Tollefson get_node_active_region(start_pfn, &node_ar); 1047ab1f9dacSPaul Mackerras } 10484a618669SDave Hansen } 1049ab1f9dacSPaul Mackerras } 10508f64e1f2SJon Tollefson 10514a618669SDave Hansen 10524a618669SDave Hansen void __init do_init_bootmem(void) 10534a618669SDave Hansen { 10542fabf084SNishanth Aravamudan int nid, cpu; 10554a618669SDave Hansen 10564a618669SDave Hansen min_low_pfn = 0; 105795f72d1eSYinghai Lu max_low_pfn = memblock_end_of_DRAM() >> PAGE_SHIFT; 10584a618669SDave Hansen max_pfn = max_low_pfn; 10594a618669SDave Hansen 10604a618669SDave Hansen if (parse_numa_properties()) 10614a618669SDave Hansen setup_nonnuma(); 10624a618669SDave Hansen else 10634a618669SDave Hansen dump_numa_memory_topology(); 10644a618669SDave Hansen 10654a618669SDave Hansen for_each_online_node(nid) { 10664a618669SDave Hansen unsigned long start_pfn, end_pfn; 10670be210fdSDave Hansen void *bootmem_vaddr; 10684a618669SDave Hansen unsigned long bootmap_pages; 10694a618669SDave Hansen 10704a618669SDave Hansen get_pfn_range_for_nid(nid, &start_pfn, &end_pfn); 10714a618669SDave Hansen 10724a618669SDave Hansen /* 10734a618669SDave Hansen * Allocate the node structure node local if possible 10744a618669SDave Hansen * 10754a618669SDave Hansen * Be careful moving this around, as it relies on all 10764a618669SDave Hansen * previous nodes' bootmem to be initialized and have 10774a618669SDave Hansen * all reserved areas marked. 10784a618669SDave Hansen */ 1079893473dfSDave Hansen NODE_DATA(nid) = careful_zallocation(nid, 10804a618669SDave Hansen sizeof(struct pglist_data), 10814a618669SDave Hansen SMP_CACHE_BYTES, end_pfn); 10824a618669SDave Hansen 10834a618669SDave Hansen dbg("node %d\n", nid); 10844a618669SDave Hansen dbg("NODE_DATA() = %p\n", NODE_DATA(nid)); 10854a618669SDave Hansen 10864a618669SDave Hansen NODE_DATA(nid)->bdata = &bootmem_node_data[nid]; 10874a618669SDave Hansen NODE_DATA(nid)->node_start_pfn = start_pfn; 10884a618669SDave Hansen NODE_DATA(nid)->node_spanned_pages = end_pfn - start_pfn; 10894a618669SDave Hansen 10904a618669SDave Hansen if (NODE_DATA(nid)->node_spanned_pages == 0) 10914a618669SDave Hansen continue; 10924a618669SDave Hansen 10934a618669SDave Hansen dbg("start_paddr = %lx\n", start_pfn << PAGE_SHIFT); 10944a618669SDave Hansen dbg("end_paddr = %lx\n", end_pfn << PAGE_SHIFT); 10954a618669SDave Hansen 10964a618669SDave Hansen bootmap_pages = bootmem_bootmap_pages(end_pfn - start_pfn); 1097893473dfSDave Hansen bootmem_vaddr = careful_zallocation(nid, 10984a618669SDave Hansen bootmap_pages << PAGE_SHIFT, 10994a618669SDave Hansen PAGE_SIZE, end_pfn); 11004a618669SDave Hansen 11010be210fdSDave Hansen dbg("bootmap_vaddr = %p\n", bootmem_vaddr); 11024a618669SDave Hansen 11030be210fdSDave Hansen init_bootmem_node(NODE_DATA(nid), 11040be210fdSDave Hansen __pa(bootmem_vaddr) >> PAGE_SHIFT, 11054a618669SDave Hansen start_pfn, end_pfn); 11064a618669SDave Hansen 11074a618669SDave Hansen free_bootmem_with_active_regions(nid, end_pfn); 11084a618669SDave Hansen /* 11094a618669SDave Hansen * Be very careful about moving this around. Future 1110893473dfSDave Hansen * calls to careful_zallocation() depend on this getting 11114a618669SDave Hansen * done correctly. 11124a618669SDave Hansen */ 11134a618669SDave Hansen mark_reserved_regions_for_nid(nid); 11148f64e1f2SJon Tollefson sparse_memory_present_with_active_regions(nid); 1115ab1f9dacSPaul Mackerras } 1116d3f6204aSBenjamin Herrenschmidt 1117d3f6204aSBenjamin Herrenschmidt init_bootmem_done = 1; 111825863de0SAnton Blanchard 111925863de0SAnton Blanchard /* 112025863de0SAnton Blanchard * Now bootmem is initialised we can create the node to cpumask 112125863de0SAnton Blanchard * lookup tables and setup the cpu callback to populate them. 112225863de0SAnton Blanchard */ 112325863de0SAnton Blanchard setup_node_to_cpumask_map(); 112425863de0SAnton Blanchard 1125d4edc5b6SSrivatsa S. Bhat reset_numa_cpu_lookup_table(); 112625863de0SAnton Blanchard register_cpu_notifier(&ppc64_numa_nb); 11272fabf084SNishanth Aravamudan /* 11282fabf084SNishanth Aravamudan * We need the numa_cpu_lookup_table to be accurate for all CPUs, 11292fabf084SNishanth Aravamudan * even before we online them, so that we can use cpu_to_{node,mem} 11302fabf084SNishanth Aravamudan * early in boot, cf. smp_prepare_cpus(). 11312fabf084SNishanth Aravamudan */ 1132bc3c4327SLi Zhong for_each_present_cpu(cpu) { 113370ad2375SLi Zhong numa_setup_cpu((unsigned long)cpu); 11342fabf084SNishanth Aravamudan } 11354a618669SDave Hansen } 1136ab1f9dacSPaul Mackerras 1137ab1f9dacSPaul Mackerras static int __init early_numa(char *p) 1138ab1f9dacSPaul Mackerras { 1139ab1f9dacSPaul Mackerras if (!p) 1140ab1f9dacSPaul Mackerras return 0; 1141ab1f9dacSPaul Mackerras 1142ab1f9dacSPaul Mackerras if (strstr(p, "off")) 1143ab1f9dacSPaul Mackerras numa_enabled = 0; 1144ab1f9dacSPaul Mackerras 1145ab1f9dacSPaul Mackerras if (strstr(p, "debug")) 1146ab1f9dacSPaul Mackerras numa_debug = 1; 1147ab1f9dacSPaul Mackerras 11481daa6d08SBalbir Singh p = strstr(p, "fake="); 11491daa6d08SBalbir Singh if (p) 11501daa6d08SBalbir Singh cmdline = p + strlen("fake="); 11511daa6d08SBalbir Singh 1152ab1f9dacSPaul Mackerras return 0; 1153ab1f9dacSPaul Mackerras } 1154ab1f9dacSPaul Mackerras early_param("numa", early_numa); 1155237a0989SMike Kravetz 1156237a0989SMike Kravetz #ifdef CONFIG_MEMORY_HOTPLUG 1157237a0989SMike Kravetz /* 11580f16ef7fSNathan Fontenot * Find the node associated with a hot added memory section for 11590f16ef7fSNathan Fontenot * memory represented in the device tree by the property 11600f16ef7fSNathan Fontenot * ibm,dynamic-reconfiguration-memory/ibm,dynamic-memory. 11610db9360aSNathan Fontenot */ 11620db9360aSNathan Fontenot static int hot_add_drconf_scn_to_nid(struct device_node *memory, 11630db9360aSNathan Fontenot unsigned long scn_addr) 11640db9360aSNathan Fontenot { 1165b08a2a12SAlistair Popple const __be32 *dm; 11660f16ef7fSNathan Fontenot unsigned int drconf_cell_cnt, rc; 11673fdfd990SBenjamin Herrenschmidt unsigned long lmb_size; 11680db9360aSNathan Fontenot struct assoc_arrays aa; 11690f16ef7fSNathan Fontenot int nid = -1; 11700db9360aSNathan Fontenot 11710f16ef7fSNathan Fontenot drconf_cell_cnt = of_get_drconf_memory(memory, &dm); 11720f16ef7fSNathan Fontenot if (!drconf_cell_cnt) 11730f16ef7fSNathan Fontenot return -1; 11740db9360aSNathan Fontenot 11753fdfd990SBenjamin Herrenschmidt lmb_size = of_get_lmb_size(memory); 11763fdfd990SBenjamin Herrenschmidt if (!lmb_size) 11770f16ef7fSNathan Fontenot return -1; 11780db9360aSNathan Fontenot 11790db9360aSNathan Fontenot rc = of_get_assoc_arrays(memory, &aa); 11800db9360aSNathan Fontenot if (rc) 11810f16ef7fSNathan Fontenot return -1; 11820db9360aSNathan Fontenot 11830f16ef7fSNathan Fontenot for (; drconf_cell_cnt != 0; --drconf_cell_cnt) { 11840db9360aSNathan Fontenot struct of_drconf_cell drmem; 11850db9360aSNathan Fontenot 11860db9360aSNathan Fontenot read_drconf_cell(&drmem, &dm); 11870db9360aSNathan Fontenot 11880db9360aSNathan Fontenot /* skip this block if it is reserved or not assigned to 11890db9360aSNathan Fontenot * this partition */ 11900db9360aSNathan Fontenot if ((drmem.flags & DRCONF_MEM_RESERVED) 11910db9360aSNathan Fontenot || !(drmem.flags & DRCONF_MEM_ASSIGNED)) 11920db9360aSNathan Fontenot continue; 11930db9360aSNathan Fontenot 11940f16ef7fSNathan Fontenot if ((scn_addr < drmem.base_addr) 11953fdfd990SBenjamin Herrenschmidt || (scn_addr >= (drmem.base_addr + lmb_size))) 11960f16ef7fSNathan Fontenot continue; 11970db9360aSNathan Fontenot 11980f16ef7fSNathan Fontenot nid = of_drconf_to_nid_single(&drmem, &aa); 11990f16ef7fSNathan Fontenot break; 12000db9360aSNathan Fontenot } 12010db9360aSNathan Fontenot 12020f16ef7fSNathan Fontenot return nid; 12030db9360aSNathan Fontenot } 12040db9360aSNathan Fontenot 12050db9360aSNathan Fontenot /* 12060f16ef7fSNathan Fontenot * Find the node associated with a hot added memory section for memory 12070f16ef7fSNathan Fontenot * represented in the device tree as a node (i.e. memory@XXXX) for 120895f72d1eSYinghai Lu * each memblock. 1209237a0989SMike Kravetz */ 1210ec32dd66SRobert Jennings static int hot_add_node_scn_to_nid(unsigned long scn_addr) 1211237a0989SMike Kravetz { 121294db7c5eSAnton Blanchard struct device_node *memory; 12130f16ef7fSNathan Fontenot int nid = -1; 1214237a0989SMike Kravetz 121594db7c5eSAnton Blanchard for_each_node_by_type(memory, "memory") { 1216237a0989SMike Kravetz unsigned long start, size; 1217b226e462SMike Kravetz int ranges; 1218b08a2a12SAlistair Popple const __be32 *memcell_buf; 1219237a0989SMike Kravetz unsigned int len; 1220237a0989SMike Kravetz 1221e2eb6392SStephen Rothwell memcell_buf = of_get_property(memory, "reg", &len); 1222237a0989SMike Kravetz if (!memcell_buf || len <= 0) 1223237a0989SMike Kravetz continue; 1224237a0989SMike Kravetz 1225cc5d0189SBenjamin Herrenschmidt /* ranges in cell */ 1226cc5d0189SBenjamin Herrenschmidt ranges = (len >> 2) / (n_mem_addr_cells + n_mem_size_cells); 12270f16ef7fSNathan Fontenot 12280f16ef7fSNathan Fontenot while (ranges--) { 1229237a0989SMike Kravetz start = read_n_cells(n_mem_addr_cells, &memcell_buf); 1230237a0989SMike Kravetz size = read_n_cells(n_mem_size_cells, &memcell_buf); 1231237a0989SMike Kravetz 12320f16ef7fSNathan Fontenot if ((scn_addr < start) || (scn_addr >= (start + size))) 12330f16ef7fSNathan Fontenot continue; 12340f16ef7fSNathan Fontenot 12350f16ef7fSNathan Fontenot nid = of_node_to_nid_single(memory); 12360f16ef7fSNathan Fontenot break; 12370f16ef7fSNathan Fontenot } 12380f16ef7fSNathan Fontenot 12390f16ef7fSNathan Fontenot if (nid >= 0) 12400f16ef7fSNathan Fontenot break; 12410f16ef7fSNathan Fontenot } 12420f16ef7fSNathan Fontenot 124360831842SAnton Blanchard of_node_put(memory); 124460831842SAnton Blanchard 12450db9360aSNathan Fontenot return nid; 1246237a0989SMike Kravetz } 1247237a0989SMike Kravetz 12480f16ef7fSNathan Fontenot /* 12490f16ef7fSNathan Fontenot * Find the node associated with a hot added memory section. Section 125095f72d1eSYinghai Lu * corresponds to a SPARSEMEM section, not an MEMBLOCK. It is assumed that 125195f72d1eSYinghai Lu * sections are fully contained within a single MEMBLOCK. 12520f16ef7fSNathan Fontenot */ 12530f16ef7fSNathan Fontenot int hot_add_scn_to_nid(unsigned long scn_addr) 12540f16ef7fSNathan Fontenot { 12550f16ef7fSNathan Fontenot struct device_node *memory = NULL; 12560f16ef7fSNathan Fontenot int nid, found = 0; 12570f16ef7fSNathan Fontenot 12580f16ef7fSNathan Fontenot if (!numa_enabled || (min_common_depth < 0)) 125972c33688SH Hartley Sweeten return first_online_node; 12600f16ef7fSNathan Fontenot 12610f16ef7fSNathan Fontenot memory = of_find_node_by_path("/ibm,dynamic-reconfiguration-memory"); 12620f16ef7fSNathan Fontenot if (memory) { 12630f16ef7fSNathan Fontenot nid = hot_add_drconf_scn_to_nid(memory, scn_addr); 12640f16ef7fSNathan Fontenot of_node_put(memory); 12650f16ef7fSNathan Fontenot } else { 12660f16ef7fSNathan Fontenot nid = hot_add_node_scn_to_nid(scn_addr); 1267237a0989SMike Kravetz } 12680f16ef7fSNathan Fontenot 12690f16ef7fSNathan Fontenot if (nid < 0 || !node_online(nid)) 127072c33688SH Hartley Sweeten nid = first_online_node; 12710f16ef7fSNathan Fontenot 12720f16ef7fSNathan Fontenot if (NODE_DATA(nid)->node_spanned_pages) 12730f16ef7fSNathan Fontenot return nid; 12740f16ef7fSNathan Fontenot 12750f16ef7fSNathan Fontenot for_each_online_node(nid) { 12760f16ef7fSNathan Fontenot if (NODE_DATA(nid)->node_spanned_pages) { 12770f16ef7fSNathan Fontenot found = 1; 12780f16ef7fSNathan Fontenot break; 1279237a0989SMike Kravetz } 12800f16ef7fSNathan Fontenot } 12810f16ef7fSNathan Fontenot 12820f16ef7fSNathan Fontenot BUG_ON(!found); 12830f16ef7fSNathan Fontenot return nid; 12840f16ef7fSNathan Fontenot } 12850f16ef7fSNathan Fontenot 1286cd34206eSNishanth Aravamudan static u64 hot_add_drconf_memory_max(void) 1287cd34206eSNishanth Aravamudan { 1288cd34206eSNishanth Aravamudan struct device_node *memory = NULL; 1289cd34206eSNishanth Aravamudan unsigned int drconf_cell_cnt = 0; 1290cd34206eSNishanth Aravamudan u64 lmb_size = 0; 1291ec32dd66SRobert Jennings const __be32 *dm = NULL; 1292cd34206eSNishanth Aravamudan 1293cd34206eSNishanth Aravamudan memory = of_find_node_by_path("/ibm,dynamic-reconfiguration-memory"); 1294cd34206eSNishanth Aravamudan if (memory) { 1295cd34206eSNishanth Aravamudan drconf_cell_cnt = of_get_drconf_memory(memory, &dm); 1296cd34206eSNishanth Aravamudan lmb_size = of_get_lmb_size(memory); 1297cd34206eSNishanth Aravamudan of_node_put(memory); 1298cd34206eSNishanth Aravamudan } 1299cd34206eSNishanth Aravamudan return lmb_size * drconf_cell_cnt; 1300cd34206eSNishanth Aravamudan } 1301cd34206eSNishanth Aravamudan 1302cd34206eSNishanth Aravamudan /* 1303cd34206eSNishanth Aravamudan * memory_hotplug_max - return max address of memory that may be added 1304cd34206eSNishanth Aravamudan * 1305cd34206eSNishanth Aravamudan * This is currently only used on systems that support drconfig memory 1306cd34206eSNishanth Aravamudan * hotplug. 1307cd34206eSNishanth Aravamudan */ 1308cd34206eSNishanth Aravamudan u64 memory_hotplug_max(void) 1309cd34206eSNishanth Aravamudan { 1310cd34206eSNishanth Aravamudan return max(hot_add_drconf_memory_max(), memblock_end_of_DRAM()); 1311cd34206eSNishanth Aravamudan } 1312237a0989SMike Kravetz #endif /* CONFIG_MEMORY_HOTPLUG */ 13139eff1a38SJesse Larrew 1314bd03403aSJesse Larrew /* Virtual Processor Home Node (VPHN) support */ 131539bf990eSJesse Larrew #ifdef CONFIG_PPC_SPLPAR 131630c05350SNathan Fontenot struct topology_update_data { 131730c05350SNathan Fontenot struct topology_update_data *next; 131830c05350SNathan Fontenot unsigned int cpu; 131930c05350SNathan Fontenot int old_nid; 132030c05350SNathan Fontenot int new_nid; 132130c05350SNathan Fontenot }; 132230c05350SNathan Fontenot 13235de16699SAnton Blanchard static u8 vphn_cpu_change_counts[NR_CPUS][MAX_DISTANCE_REF_POINTS]; 13249eff1a38SJesse Larrew static cpumask_t cpu_associativity_changes_mask; 13259eff1a38SJesse Larrew static int vphn_enabled; 13265d88aa85SJesse Larrew static int prrn_enabled; 13275d88aa85SJesse Larrew static void reset_topology_timer(void); 13289eff1a38SJesse Larrew 13299eff1a38SJesse Larrew /* 13309eff1a38SJesse Larrew * Store the current values of the associativity change counters in the 13319eff1a38SJesse Larrew * hypervisor. 13329eff1a38SJesse Larrew */ 13339eff1a38SJesse Larrew static void setup_cpu_associativity_change_counters(void) 13349eff1a38SJesse Larrew { 1335cd9d6cc7SJesse Larrew int cpu; 13369eff1a38SJesse Larrew 13375de16699SAnton Blanchard /* The VPHN feature supports a maximum of 8 reference points */ 13385de16699SAnton Blanchard BUILD_BUG_ON(MAX_DISTANCE_REF_POINTS > 8); 13395de16699SAnton Blanchard 13409eff1a38SJesse Larrew for_each_possible_cpu(cpu) { 1341cd9d6cc7SJesse Larrew int i; 13429eff1a38SJesse Larrew u8 *counts = vphn_cpu_change_counts[cpu]; 13439eff1a38SJesse Larrew volatile u8 *hypervisor_counts = lppaca[cpu].vphn_assoc_counts; 13449eff1a38SJesse Larrew 13455de16699SAnton Blanchard for (i = 0; i < distance_ref_points_depth; i++) 13469eff1a38SJesse Larrew counts[i] = hypervisor_counts[i]; 13479eff1a38SJesse Larrew } 13489eff1a38SJesse Larrew } 13499eff1a38SJesse Larrew 13509eff1a38SJesse Larrew /* 13519eff1a38SJesse Larrew * The hypervisor maintains a set of 8 associativity change counters in 13529eff1a38SJesse Larrew * the VPA of each cpu that correspond to the associativity levels in the 13539eff1a38SJesse Larrew * ibm,associativity-reference-points property. When an associativity 13549eff1a38SJesse Larrew * level changes, the corresponding counter is incremented. 13559eff1a38SJesse Larrew * 13569eff1a38SJesse Larrew * Set a bit in cpu_associativity_changes_mask for each cpu whose home 13579eff1a38SJesse Larrew * node associativity levels have changed. 13589eff1a38SJesse Larrew * 13599eff1a38SJesse Larrew * Returns the number of cpus with unhandled associativity changes. 13609eff1a38SJesse Larrew */ 13619eff1a38SJesse Larrew static int update_cpu_associativity_changes_mask(void) 13629eff1a38SJesse Larrew { 13635d88aa85SJesse Larrew int cpu; 13649eff1a38SJesse Larrew cpumask_t *changes = &cpu_associativity_changes_mask; 13659eff1a38SJesse Larrew 13669eff1a38SJesse Larrew for_each_possible_cpu(cpu) { 13679eff1a38SJesse Larrew int i, changed = 0; 13689eff1a38SJesse Larrew u8 *counts = vphn_cpu_change_counts[cpu]; 13699eff1a38SJesse Larrew volatile u8 *hypervisor_counts = lppaca[cpu].vphn_assoc_counts; 13709eff1a38SJesse Larrew 13715de16699SAnton Blanchard for (i = 0; i < distance_ref_points_depth; i++) { 1372d69043e8SAnton Blanchard if (hypervisor_counts[i] != counts[i]) { 13739eff1a38SJesse Larrew counts[i] = hypervisor_counts[i]; 13749eff1a38SJesse Larrew changed = 1; 13759eff1a38SJesse Larrew } 13769eff1a38SJesse Larrew } 13779eff1a38SJesse Larrew if (changed) { 13783be7db6aSRobert Jennings cpumask_or(changes, changes, cpu_sibling_mask(cpu)); 13793be7db6aSRobert Jennings cpu = cpu_last_thread_sibling(cpu); 13809eff1a38SJesse Larrew } 13819eff1a38SJesse Larrew } 13829eff1a38SJesse Larrew 13835d88aa85SJesse Larrew return cpumask_weight(changes); 13849eff1a38SJesse Larrew } 13859eff1a38SJesse Larrew 1386c0e5e46fSAnton Blanchard /* 1387c0e5e46fSAnton Blanchard * 6 64-bit registers unpacked into 12 32-bit associativity values. To form 1388c0e5e46fSAnton Blanchard * the complete property we have to add the length in the first cell. 1389c0e5e46fSAnton Blanchard */ 1390c0e5e46fSAnton Blanchard #define VPHN_ASSOC_BUFSIZE (6*sizeof(u64)/sizeof(u32) + 1) 13919eff1a38SJesse Larrew 13929eff1a38SJesse Larrew /* 13939eff1a38SJesse Larrew * Convert the associativity domain numbers returned from the hypervisor 13949eff1a38SJesse Larrew * to the sequence they would appear in the ibm,associativity property. 13959eff1a38SJesse Larrew */ 1396b08a2a12SAlistair Popple static int vphn_unpack_associativity(const long *packed, __be32 *unpacked) 13979eff1a38SJesse Larrew { 1398cd9d6cc7SJesse Larrew int i, nr_assoc_doms = 0; 1399b08a2a12SAlistair Popple const __be16 *field = (const __be16 *) packed; 14009eff1a38SJesse Larrew 14019eff1a38SJesse Larrew #define VPHN_FIELD_UNUSED (0xffff) 14029eff1a38SJesse Larrew #define VPHN_FIELD_MSB (0x8000) 14039eff1a38SJesse Larrew #define VPHN_FIELD_MASK (~VPHN_FIELD_MSB) 14049eff1a38SJesse Larrew 1405c0e5e46fSAnton Blanchard for (i = 1; i < VPHN_ASSOC_BUFSIZE; i++) { 1406b08a2a12SAlistair Popple if (be16_to_cpup(field) == VPHN_FIELD_UNUSED) { 14079eff1a38SJesse Larrew /* All significant fields processed, and remaining 14089eff1a38SJesse Larrew * fields contain the reserved value of all 1's. 14099eff1a38SJesse Larrew * Just store them. 14109eff1a38SJesse Larrew */ 1411b08a2a12SAlistair Popple unpacked[i] = *((__be32 *)field); 14129eff1a38SJesse Larrew field += 2; 1413b08a2a12SAlistair Popple } else if (be16_to_cpup(field) & VPHN_FIELD_MSB) { 14149eff1a38SJesse Larrew /* Data is in the lower 15 bits of this field */ 1415b08a2a12SAlistair Popple unpacked[i] = cpu_to_be32( 1416b08a2a12SAlistair Popple be16_to_cpup(field) & VPHN_FIELD_MASK); 14179eff1a38SJesse Larrew field++; 14189eff1a38SJesse Larrew nr_assoc_doms++; 14197639adaaSJesse Larrew } else { 14209eff1a38SJesse Larrew /* Data is in the lower 15 bits of this field 14219eff1a38SJesse Larrew * concatenated with the next 16 bit field 14229eff1a38SJesse Larrew */ 1423b08a2a12SAlistair Popple unpacked[i] = *((__be32 *)field); 14249eff1a38SJesse Larrew field += 2; 14259eff1a38SJesse Larrew nr_assoc_doms++; 14269eff1a38SJesse Larrew } 14279eff1a38SJesse Larrew } 14289eff1a38SJesse Larrew 1429c0e5e46fSAnton Blanchard /* The first cell contains the length of the property */ 1430b08a2a12SAlistair Popple unpacked[0] = cpu_to_be32(nr_assoc_doms); 1431c0e5e46fSAnton Blanchard 14329eff1a38SJesse Larrew return nr_assoc_doms; 14339eff1a38SJesse Larrew } 14349eff1a38SJesse Larrew 14359eff1a38SJesse Larrew /* 14369eff1a38SJesse Larrew * Retrieve the new associativity information for a virtual processor's 14379eff1a38SJesse Larrew * home node. 14389eff1a38SJesse Larrew */ 1439b08a2a12SAlistair Popple static long hcall_vphn(unsigned long cpu, __be32 *associativity) 14409eff1a38SJesse Larrew { 1441cd9d6cc7SJesse Larrew long rc; 14429eff1a38SJesse Larrew long retbuf[PLPAR_HCALL9_BUFSIZE] = {0}; 14439eff1a38SJesse Larrew u64 flags = 1; 14449eff1a38SJesse Larrew int hwcpu = get_hard_smp_processor_id(cpu); 14459eff1a38SJesse Larrew 14469eff1a38SJesse Larrew rc = plpar_hcall9(H_HOME_NODE_ASSOCIATIVITY, retbuf, flags, hwcpu); 14479eff1a38SJesse Larrew vphn_unpack_associativity(retbuf, associativity); 14489eff1a38SJesse Larrew 14499eff1a38SJesse Larrew return rc; 14509eff1a38SJesse Larrew } 14519eff1a38SJesse Larrew 14529eff1a38SJesse Larrew static long vphn_get_associativity(unsigned long cpu, 1453b08a2a12SAlistair Popple __be32 *associativity) 14549eff1a38SJesse Larrew { 1455cd9d6cc7SJesse Larrew long rc; 14569eff1a38SJesse Larrew 14579eff1a38SJesse Larrew rc = hcall_vphn(cpu, associativity); 14589eff1a38SJesse Larrew 14599eff1a38SJesse Larrew switch (rc) { 14609eff1a38SJesse Larrew case H_FUNCTION: 14619eff1a38SJesse Larrew printk(KERN_INFO 14629eff1a38SJesse Larrew "VPHN is not supported. Disabling polling...\n"); 14639eff1a38SJesse Larrew stop_topology_update(); 14649eff1a38SJesse Larrew break; 14659eff1a38SJesse Larrew case H_HARDWARE: 14669eff1a38SJesse Larrew printk(KERN_ERR 14679eff1a38SJesse Larrew "hcall_vphn() experienced a hardware fault " 14689eff1a38SJesse Larrew "preventing VPHN. Disabling polling...\n"); 14699eff1a38SJesse Larrew stop_topology_update(); 14709eff1a38SJesse Larrew } 14719eff1a38SJesse Larrew 14729eff1a38SJesse Larrew return rc; 14739eff1a38SJesse Larrew } 14749eff1a38SJesse Larrew 14759eff1a38SJesse Larrew /* 147630c05350SNathan Fontenot * Update the CPU maps and sysfs entries for a single CPU when its NUMA 147730c05350SNathan Fontenot * characteristics change. This function doesn't perform any locking and is 147830c05350SNathan Fontenot * only safe to call from stop_machine(). 147930c05350SNathan Fontenot */ 148030c05350SNathan Fontenot static int update_cpu_topology(void *data) 148130c05350SNathan Fontenot { 148230c05350SNathan Fontenot struct topology_update_data *update; 148330c05350SNathan Fontenot unsigned long cpu; 148430c05350SNathan Fontenot 148530c05350SNathan Fontenot if (!data) 148630c05350SNathan Fontenot return -EINVAL; 148730c05350SNathan Fontenot 14883be7db6aSRobert Jennings cpu = smp_processor_id(); 148930c05350SNathan Fontenot 149030c05350SNathan Fontenot for (update = data; update; update = update->next) { 149130c05350SNathan Fontenot if (cpu != update->cpu) 149230c05350SNathan Fontenot continue; 149330c05350SNathan Fontenot 149430c05350SNathan Fontenot unmap_cpu_from_node(update->cpu); 149530c05350SNathan Fontenot map_cpu_to_node(update->cpu, update->new_nid); 1496176bbf14SJesse Larrew vdso_getcpu_init(); 149730c05350SNathan Fontenot } 149830c05350SNathan Fontenot 149930c05350SNathan Fontenot return 0; 150030c05350SNathan Fontenot } 150130c05350SNathan Fontenot 1502d4edc5b6SSrivatsa S. Bhat static int update_lookup_table(void *data) 1503d4edc5b6SSrivatsa S. Bhat { 1504d4edc5b6SSrivatsa S. Bhat struct topology_update_data *update; 1505d4edc5b6SSrivatsa S. Bhat 1506d4edc5b6SSrivatsa S. Bhat if (!data) 1507d4edc5b6SSrivatsa S. Bhat return -EINVAL; 1508d4edc5b6SSrivatsa S. Bhat 1509d4edc5b6SSrivatsa S. Bhat /* 1510d4edc5b6SSrivatsa S. Bhat * Upon topology update, the numa-cpu lookup table needs to be updated 1511d4edc5b6SSrivatsa S. Bhat * for all threads in the core, including offline CPUs, to ensure that 1512d4edc5b6SSrivatsa S. Bhat * future hotplug operations respect the cpu-to-node associativity 1513d4edc5b6SSrivatsa S. Bhat * properly. 1514d4edc5b6SSrivatsa S. Bhat */ 1515d4edc5b6SSrivatsa S. Bhat for (update = data; update; update = update->next) { 1516d4edc5b6SSrivatsa S. Bhat int nid, base, j; 1517d4edc5b6SSrivatsa S. Bhat 1518d4edc5b6SSrivatsa S. Bhat nid = update->new_nid; 1519d4edc5b6SSrivatsa S. Bhat base = cpu_first_thread_sibling(update->cpu); 1520d4edc5b6SSrivatsa S. Bhat 1521d4edc5b6SSrivatsa S. Bhat for (j = 0; j < threads_per_core; j++) { 1522d4edc5b6SSrivatsa S. Bhat update_numa_cpu_lookup_table(base + j, nid); 1523d4edc5b6SSrivatsa S. Bhat } 1524d4edc5b6SSrivatsa S. Bhat } 1525d4edc5b6SSrivatsa S. Bhat 1526d4edc5b6SSrivatsa S. Bhat return 0; 1527d4edc5b6SSrivatsa S. Bhat } 1528d4edc5b6SSrivatsa S. Bhat 152930c05350SNathan Fontenot /* 15309eff1a38SJesse Larrew * Update the node maps and sysfs entries for each cpu whose home node 153179c5fcebSJesse Larrew * has changed. Returns 1 when the topology has changed, and 0 otherwise. 15329eff1a38SJesse Larrew */ 15339eff1a38SJesse Larrew int arch_update_cpu_topology(void) 15349eff1a38SJesse Larrew { 15353be7db6aSRobert Jennings unsigned int cpu, sibling, changed = 0; 153630c05350SNathan Fontenot struct topology_update_data *updates, *ud; 1537b08a2a12SAlistair Popple __be32 associativity[VPHN_ASSOC_BUFSIZE] = {0}; 1538176bbf14SJesse Larrew cpumask_t updated_cpus; 15398a25a2fdSKay Sievers struct device *dev; 15403be7db6aSRobert Jennings int weight, new_nid, i = 0; 154130c05350SNathan Fontenot 154230c05350SNathan Fontenot weight = cpumask_weight(&cpu_associativity_changes_mask); 154330c05350SNathan Fontenot if (!weight) 154430c05350SNathan Fontenot return 0; 154530c05350SNathan Fontenot 154630c05350SNathan Fontenot updates = kzalloc(weight * (sizeof(*updates)), GFP_KERNEL); 154730c05350SNathan Fontenot if (!updates) 154830c05350SNathan Fontenot return 0; 15499eff1a38SJesse Larrew 1550176bbf14SJesse Larrew cpumask_clear(&updated_cpus); 15519eff1a38SJesse Larrew 1552104699c0SKOSAKI Motohiro for_each_cpu(cpu, &cpu_associativity_changes_mask) { 15533be7db6aSRobert Jennings /* 15543be7db6aSRobert Jennings * If siblings aren't flagged for changes, updates list 15553be7db6aSRobert Jennings * will be too short. Skip on this update and set for next 15563be7db6aSRobert Jennings * update. 15573be7db6aSRobert Jennings */ 15583be7db6aSRobert Jennings if (!cpumask_subset(cpu_sibling_mask(cpu), 15593be7db6aSRobert Jennings &cpu_associativity_changes_mask)) { 15603be7db6aSRobert Jennings pr_info("Sibling bits not set for associativity " 15613be7db6aSRobert Jennings "change, cpu%d\n", cpu); 15623be7db6aSRobert Jennings cpumask_or(&cpu_associativity_changes_mask, 15633be7db6aSRobert Jennings &cpu_associativity_changes_mask, 15643be7db6aSRobert Jennings cpu_sibling_mask(cpu)); 15653be7db6aSRobert Jennings cpu = cpu_last_thread_sibling(cpu); 15663be7db6aSRobert Jennings continue; 15673be7db6aSRobert Jennings } 15683be7db6aSRobert Jennings 15693be7db6aSRobert Jennings /* Use associativity from first thread for all siblings */ 15709eff1a38SJesse Larrew vphn_get_associativity(cpu, associativity); 15713be7db6aSRobert Jennings new_nid = associativity_to_nid(associativity); 15723be7db6aSRobert Jennings if (new_nid < 0 || !node_online(new_nid)) 15733be7db6aSRobert Jennings new_nid = first_online_node; 15749eff1a38SJesse Larrew 15753be7db6aSRobert Jennings if (new_nid == numa_cpu_lookup_table[cpu]) { 15763be7db6aSRobert Jennings cpumask_andnot(&cpu_associativity_changes_mask, 15773be7db6aSRobert Jennings &cpu_associativity_changes_mask, 15783be7db6aSRobert Jennings cpu_sibling_mask(cpu)); 15793be7db6aSRobert Jennings cpu = cpu_last_thread_sibling(cpu); 15803be7db6aSRobert Jennings continue; 15813be7db6aSRobert Jennings } 15829eff1a38SJesse Larrew 15833be7db6aSRobert Jennings for_each_cpu(sibling, cpu_sibling_mask(cpu)) { 15843be7db6aSRobert Jennings ud = &updates[i++]; 15853be7db6aSRobert Jennings ud->cpu = sibling; 15863be7db6aSRobert Jennings ud->new_nid = new_nid; 15873be7db6aSRobert Jennings ud->old_nid = numa_cpu_lookup_table[sibling]; 15883be7db6aSRobert Jennings cpumask_set_cpu(sibling, &updated_cpus); 158930c05350SNathan Fontenot if (i < weight) 159030c05350SNathan Fontenot ud->next = &updates[i]; 159130c05350SNathan Fontenot } 15923be7db6aSRobert Jennings cpu = cpu_last_thread_sibling(cpu); 15933be7db6aSRobert Jennings } 15949eff1a38SJesse Larrew 15959a013361SMichael Wang /* 15969a013361SMichael Wang * In cases where we have nothing to update (because the updates list 15979a013361SMichael Wang * is too short or because the new topology is same as the old one), 15989a013361SMichael Wang * skip invoking update_cpu_topology() via stop-machine(). This is 15999a013361SMichael Wang * necessary (and not just a fast-path optimization) since stop-machine 16009a013361SMichael Wang * can end up electing a random CPU to run update_cpu_topology(), and 16019a013361SMichael Wang * thus trick us into setting up incorrect cpu-node mappings (since 16029a013361SMichael Wang * 'updates' is kzalloc()'ed). 16039a013361SMichael Wang * 16049a013361SMichael Wang * And for the similar reason, we will skip all the following updating. 16059a013361SMichael Wang */ 16069a013361SMichael Wang if (!cpumask_weight(&updated_cpus)) 16079a013361SMichael Wang goto out; 16089a013361SMichael Wang 1609176bbf14SJesse Larrew stop_machine(update_cpu_topology, &updates[0], &updated_cpus); 161030c05350SNathan Fontenot 1611d4edc5b6SSrivatsa S. Bhat /* 1612d4edc5b6SSrivatsa S. Bhat * Update the numa-cpu lookup table with the new mappings, even for 1613d4edc5b6SSrivatsa S. Bhat * offline CPUs. It is best to perform this update from the stop- 1614d4edc5b6SSrivatsa S. Bhat * machine context. 1615d4edc5b6SSrivatsa S. Bhat */ 1616d4edc5b6SSrivatsa S. Bhat stop_machine(update_lookup_table, &updates[0], 1617d4edc5b6SSrivatsa S. Bhat cpumask_of(raw_smp_processor_id())); 1618d4edc5b6SSrivatsa S. Bhat 161930c05350SNathan Fontenot for (ud = &updates[0]; ud; ud = ud->next) { 1620dd023217SNathan Fontenot unregister_cpu_under_node(ud->cpu, ud->old_nid); 1621dd023217SNathan Fontenot register_cpu_under_node(ud->cpu, ud->new_nid); 1622dd023217SNathan Fontenot 162330c05350SNathan Fontenot dev = get_cpu_device(ud->cpu); 16248a25a2fdSKay Sievers if (dev) 16258a25a2fdSKay Sievers kobject_uevent(&dev->kobj, KOBJ_CHANGE); 162630c05350SNathan Fontenot cpumask_clear_cpu(ud->cpu, &cpu_associativity_changes_mask); 162779c5fcebSJesse Larrew changed = 1; 16289eff1a38SJesse Larrew } 16299eff1a38SJesse Larrew 16309a013361SMichael Wang out: 163130c05350SNathan Fontenot kfree(updates); 163279c5fcebSJesse Larrew return changed; 16339eff1a38SJesse Larrew } 16349eff1a38SJesse Larrew 16359eff1a38SJesse Larrew static void topology_work_fn(struct work_struct *work) 16369eff1a38SJesse Larrew { 16379eff1a38SJesse Larrew rebuild_sched_domains(); 16389eff1a38SJesse Larrew } 16399eff1a38SJesse Larrew static DECLARE_WORK(topology_work, topology_work_fn); 16409eff1a38SJesse Larrew 1641ec32dd66SRobert Jennings static void topology_schedule_update(void) 16429eff1a38SJesse Larrew { 16439eff1a38SJesse Larrew schedule_work(&topology_work); 16449eff1a38SJesse Larrew } 16459eff1a38SJesse Larrew 16469eff1a38SJesse Larrew static void topology_timer_fn(unsigned long ignored) 16479eff1a38SJesse Larrew { 16485d88aa85SJesse Larrew if (prrn_enabled && cpumask_weight(&cpu_associativity_changes_mask)) 16495d88aa85SJesse Larrew topology_schedule_update(); 16505d88aa85SJesse Larrew else if (vphn_enabled) { 16519eff1a38SJesse Larrew if (update_cpu_associativity_changes_mask() > 0) 16529eff1a38SJesse Larrew topology_schedule_update(); 16535d88aa85SJesse Larrew reset_topology_timer(); 16545d88aa85SJesse Larrew } 16559eff1a38SJesse Larrew } 16569eff1a38SJesse Larrew static struct timer_list topology_timer = 16579eff1a38SJesse Larrew TIMER_INITIALIZER(topology_timer_fn, 0, 0); 16589eff1a38SJesse Larrew 16595d88aa85SJesse Larrew static void reset_topology_timer(void) 16609eff1a38SJesse Larrew { 16619eff1a38SJesse Larrew topology_timer.data = 0; 16629eff1a38SJesse Larrew topology_timer.expires = jiffies + 60 * HZ; 16635d88aa85SJesse Larrew mod_timer(&topology_timer, topology_timer.expires); 16649eff1a38SJesse Larrew } 16659eff1a38SJesse Larrew 1666601abdc3SNathan Fontenot #ifdef CONFIG_SMP 1667601abdc3SNathan Fontenot 16685d88aa85SJesse Larrew static void stage_topology_update(int core_id) 16695d88aa85SJesse Larrew { 16705d88aa85SJesse Larrew cpumask_or(&cpu_associativity_changes_mask, 16715d88aa85SJesse Larrew &cpu_associativity_changes_mask, cpu_sibling_mask(core_id)); 16725d88aa85SJesse Larrew reset_topology_timer(); 16735d88aa85SJesse Larrew } 16745d88aa85SJesse Larrew 16755d88aa85SJesse Larrew static int dt_update_callback(struct notifier_block *nb, 16765d88aa85SJesse Larrew unsigned long action, void *data) 16775d88aa85SJesse Larrew { 16785d88aa85SJesse Larrew struct of_prop_reconfig *update; 16795d88aa85SJesse Larrew int rc = NOTIFY_DONE; 16805d88aa85SJesse Larrew 16815d88aa85SJesse Larrew switch (action) { 16825d88aa85SJesse Larrew case OF_RECONFIG_UPDATE_PROPERTY: 16835d88aa85SJesse Larrew update = (struct of_prop_reconfig *)data; 168430c05350SNathan Fontenot if (!of_prop_cmp(update->dn->type, "cpu") && 168530c05350SNathan Fontenot !of_prop_cmp(update->prop->name, "ibm,associativity")) { 16865d88aa85SJesse Larrew u32 core_id; 16875d88aa85SJesse Larrew of_property_read_u32(update->dn, "reg", &core_id); 16885d88aa85SJesse Larrew stage_topology_update(core_id); 16895d88aa85SJesse Larrew rc = NOTIFY_OK; 16905d88aa85SJesse Larrew } 16915d88aa85SJesse Larrew break; 16925d88aa85SJesse Larrew } 16935d88aa85SJesse Larrew 16945d88aa85SJesse Larrew return rc; 16955d88aa85SJesse Larrew } 16965d88aa85SJesse Larrew 16975d88aa85SJesse Larrew static struct notifier_block dt_update_nb = { 16985d88aa85SJesse Larrew .notifier_call = dt_update_callback, 16995d88aa85SJesse Larrew }; 17005d88aa85SJesse Larrew 1701601abdc3SNathan Fontenot #endif 1702601abdc3SNathan Fontenot 17039eff1a38SJesse Larrew /* 17045d88aa85SJesse Larrew * Start polling for associativity changes. 17059eff1a38SJesse Larrew */ 17069eff1a38SJesse Larrew int start_topology_update(void) 17079eff1a38SJesse Larrew { 17089eff1a38SJesse Larrew int rc = 0; 17099eff1a38SJesse Larrew 17105d88aa85SJesse Larrew if (firmware_has_feature(FW_FEATURE_PRRN)) { 17115d88aa85SJesse Larrew if (!prrn_enabled) { 17125d88aa85SJesse Larrew prrn_enabled = 1; 17135d88aa85SJesse Larrew vphn_enabled = 0; 1714601abdc3SNathan Fontenot #ifdef CONFIG_SMP 17155d88aa85SJesse Larrew rc = of_reconfig_notifier_register(&dt_update_nb); 1716601abdc3SNathan Fontenot #endif 17175d88aa85SJesse Larrew } 1718b7abef04SJesse Larrew } else if (firmware_has_feature(FW_FEATURE_VPHN) && 1719f13c13a0SAnton Blanchard lppaca_shared_proc(get_lppaca())) { 17205d88aa85SJesse Larrew if (!vphn_enabled) { 17215d88aa85SJesse Larrew prrn_enabled = 0; 17229eff1a38SJesse Larrew vphn_enabled = 1; 17239eff1a38SJesse Larrew setup_cpu_associativity_change_counters(); 17249eff1a38SJesse Larrew init_timer_deferrable(&topology_timer); 17255d88aa85SJesse Larrew reset_topology_timer(); 17265d88aa85SJesse Larrew } 17279eff1a38SJesse Larrew } 17289eff1a38SJesse Larrew 17299eff1a38SJesse Larrew return rc; 17309eff1a38SJesse Larrew } 17319eff1a38SJesse Larrew 17329eff1a38SJesse Larrew /* 17339eff1a38SJesse Larrew * Disable polling for VPHN associativity changes. 17349eff1a38SJesse Larrew */ 17359eff1a38SJesse Larrew int stop_topology_update(void) 17369eff1a38SJesse Larrew { 17375d88aa85SJesse Larrew int rc = 0; 17385d88aa85SJesse Larrew 17395d88aa85SJesse Larrew if (prrn_enabled) { 17405d88aa85SJesse Larrew prrn_enabled = 0; 1741601abdc3SNathan Fontenot #ifdef CONFIG_SMP 17425d88aa85SJesse Larrew rc = of_reconfig_notifier_unregister(&dt_update_nb); 1743601abdc3SNathan Fontenot #endif 17445d88aa85SJesse Larrew } else if (vphn_enabled) { 17459eff1a38SJesse Larrew vphn_enabled = 0; 17465d88aa85SJesse Larrew rc = del_timer_sync(&topology_timer); 17479eff1a38SJesse Larrew } 17485d88aa85SJesse Larrew 17495d88aa85SJesse Larrew return rc; 1750ab1f9dacSPaul Mackerras } 1751e04fa612SNathan Fontenot 1752e04fa612SNathan Fontenot int prrn_is_enabled(void) 1753e04fa612SNathan Fontenot { 1754e04fa612SNathan Fontenot return prrn_enabled; 1755e04fa612SNathan Fontenot } 1756e04fa612SNathan Fontenot 1757e04fa612SNathan Fontenot static int topology_read(struct seq_file *file, void *v) 1758e04fa612SNathan Fontenot { 1759e04fa612SNathan Fontenot if (vphn_enabled || prrn_enabled) 1760e04fa612SNathan Fontenot seq_puts(file, "on\n"); 1761e04fa612SNathan Fontenot else 1762e04fa612SNathan Fontenot seq_puts(file, "off\n"); 1763e04fa612SNathan Fontenot 1764e04fa612SNathan Fontenot return 0; 1765e04fa612SNathan Fontenot } 1766e04fa612SNathan Fontenot 1767e04fa612SNathan Fontenot static int topology_open(struct inode *inode, struct file *file) 1768e04fa612SNathan Fontenot { 1769e04fa612SNathan Fontenot return single_open(file, topology_read, NULL); 1770e04fa612SNathan Fontenot } 1771e04fa612SNathan Fontenot 1772e04fa612SNathan Fontenot static ssize_t topology_write(struct file *file, const char __user *buf, 1773e04fa612SNathan Fontenot size_t count, loff_t *off) 1774e04fa612SNathan Fontenot { 1775e04fa612SNathan Fontenot char kbuf[4]; /* "on" or "off" plus null. */ 1776e04fa612SNathan Fontenot int read_len; 1777e04fa612SNathan Fontenot 1778e04fa612SNathan Fontenot read_len = count < 3 ? count : 3; 1779e04fa612SNathan Fontenot if (copy_from_user(kbuf, buf, read_len)) 1780e04fa612SNathan Fontenot return -EINVAL; 1781e04fa612SNathan Fontenot 1782e04fa612SNathan Fontenot kbuf[read_len] = '\0'; 1783e04fa612SNathan Fontenot 1784e04fa612SNathan Fontenot if (!strncmp(kbuf, "on", 2)) 1785e04fa612SNathan Fontenot start_topology_update(); 1786e04fa612SNathan Fontenot else if (!strncmp(kbuf, "off", 3)) 1787e04fa612SNathan Fontenot stop_topology_update(); 1788e04fa612SNathan Fontenot else 1789e04fa612SNathan Fontenot return -EINVAL; 1790e04fa612SNathan Fontenot 1791e04fa612SNathan Fontenot return count; 1792e04fa612SNathan Fontenot } 1793e04fa612SNathan Fontenot 1794e04fa612SNathan Fontenot static const struct file_operations topology_ops = { 1795e04fa612SNathan Fontenot .read = seq_read, 1796e04fa612SNathan Fontenot .write = topology_write, 1797e04fa612SNathan Fontenot .open = topology_open, 1798e04fa612SNathan Fontenot .release = single_release 1799e04fa612SNathan Fontenot }; 1800e04fa612SNathan Fontenot 1801e04fa612SNathan Fontenot static int topology_update_init(void) 1802e04fa612SNathan Fontenot { 1803e04fa612SNathan Fontenot start_topology_update(); 1804*2d15b9b4SNishanth Aravamudan if (!proc_create("powerpc/topology_updates", 0644, NULL, &topology_ops)) 1805*2d15b9b4SNishanth Aravamudan return -ENOMEM; 1806e04fa612SNathan Fontenot 1807e04fa612SNathan Fontenot return 0; 1808e04fa612SNathan Fontenot } 1809e04fa612SNathan Fontenot device_initcall(topology_update_init); 181039bf990eSJesse Larrew #endif /* CONFIG_PPC_SPLPAR */ 1811