1ab1f9dacSPaul Mackerras /* 2ab1f9dacSPaul Mackerras * pSeries NUMA support 3ab1f9dacSPaul Mackerras * 4ab1f9dacSPaul Mackerras * Copyright (C) 2002 Anton Blanchard <anton@au.ibm.com>, IBM 5ab1f9dacSPaul Mackerras * 6ab1f9dacSPaul Mackerras * This program is free software; you can redistribute it and/or 7ab1f9dacSPaul Mackerras * modify it under the terms of the GNU General Public License 8ab1f9dacSPaul Mackerras * as published by the Free Software Foundation; either version 9ab1f9dacSPaul Mackerras * 2 of the License, or (at your option) any later version. 10ab1f9dacSPaul Mackerras */ 11ab1f9dacSPaul Mackerras #include <linux/threads.h> 12ab1f9dacSPaul Mackerras #include <linux/bootmem.h> 13ab1f9dacSPaul Mackerras #include <linux/init.h> 14ab1f9dacSPaul Mackerras #include <linux/mm.h> 15ab1f9dacSPaul Mackerras #include <linux/mmzone.h> 16ab1f9dacSPaul Mackerras #include <linux/module.h> 17ab1f9dacSPaul Mackerras #include <linux/nodemask.h> 18ab1f9dacSPaul Mackerras #include <linux/cpu.h> 19ab1f9dacSPaul Mackerras #include <linux/notifier.h> 2095f72d1eSYinghai Lu #include <linux/memblock.h> 216df1646eSMichael Ellerman #include <linux/of.h> 2206eccea6SDave Hansen #include <linux/pfn.h> 239eff1a38SJesse Larrew #include <linux/cpuset.h> 249eff1a38SJesse Larrew #include <linux/node.h> 2545fb6ceaSAnton Blanchard #include <asm/sparsemem.h> 26d9b2b2a2SDavid S. Miller #include <asm/prom.h> 27cf00a8d1SPaul Mackerras #include <asm/system.h> 282249ca9dSPaul Mackerras #include <asm/smp.h> 299eff1a38SJesse Larrew #include <asm/firmware.h> 309eff1a38SJesse Larrew #include <asm/paca.h> 3139bf990eSJesse Larrew #include <asm/hvcall.h> 32ab1f9dacSPaul Mackerras 33ab1f9dacSPaul Mackerras static int numa_enabled = 1; 34ab1f9dacSPaul Mackerras 351daa6d08SBalbir Singh static char *cmdline __initdata; 361daa6d08SBalbir Singh 37ab1f9dacSPaul Mackerras static int numa_debug; 38ab1f9dacSPaul Mackerras #define dbg(args...) if (numa_debug) { printk(KERN_INFO args); } 39ab1f9dacSPaul Mackerras 4045fb6ceaSAnton Blanchard int numa_cpu_lookup_table[NR_CPUS]; 4125863de0SAnton Blanchard cpumask_var_t node_to_cpumask_map[MAX_NUMNODES]; 42ab1f9dacSPaul Mackerras struct pglist_data *node_data[MAX_NUMNODES]; 4345fb6ceaSAnton Blanchard 4445fb6ceaSAnton Blanchard EXPORT_SYMBOL(numa_cpu_lookup_table); 4525863de0SAnton Blanchard EXPORT_SYMBOL(node_to_cpumask_map); 4645fb6ceaSAnton Blanchard EXPORT_SYMBOL(node_data); 4745fb6ceaSAnton Blanchard 48ab1f9dacSPaul Mackerras static int min_common_depth; 49237a0989SMike Kravetz static int n_mem_addr_cells, n_mem_size_cells; 5041eab6f8SAnton Blanchard static int form1_affinity; 5141eab6f8SAnton Blanchard 5241eab6f8SAnton Blanchard #define MAX_DISTANCE_REF_POINTS 4 5341eab6f8SAnton Blanchard static int distance_ref_points_depth; 5441eab6f8SAnton Blanchard static const unsigned int *distance_ref_points; 5541eab6f8SAnton Blanchard static int distance_lookup_table[MAX_NUMNODES][MAX_DISTANCE_REF_POINTS]; 56ab1f9dacSPaul Mackerras 5725863de0SAnton Blanchard /* 5825863de0SAnton Blanchard * Allocate node_to_cpumask_map based on number of available nodes 5925863de0SAnton Blanchard * Requires node_possible_map to be valid. 6025863de0SAnton Blanchard * 6125863de0SAnton Blanchard * Note: node_to_cpumask() is not valid until after this is done. 6225863de0SAnton Blanchard */ 6325863de0SAnton Blanchard static void __init setup_node_to_cpumask_map(void) 6425863de0SAnton Blanchard { 6525863de0SAnton Blanchard unsigned int node, num = 0; 6625863de0SAnton Blanchard 6725863de0SAnton Blanchard /* setup nr_node_ids if not done yet */ 6825863de0SAnton Blanchard if (nr_node_ids == MAX_NUMNODES) { 6925863de0SAnton Blanchard for_each_node_mask(node, node_possible_map) 7025863de0SAnton Blanchard num = node; 7125863de0SAnton Blanchard nr_node_ids = num + 1; 7225863de0SAnton Blanchard } 7325863de0SAnton Blanchard 7425863de0SAnton Blanchard /* allocate the map */ 7525863de0SAnton Blanchard for (node = 0; node < nr_node_ids; node++) 7625863de0SAnton Blanchard alloc_bootmem_cpumask_var(&node_to_cpumask_map[node]); 7725863de0SAnton Blanchard 7825863de0SAnton Blanchard /* cpumask_of_node() will now work */ 7925863de0SAnton Blanchard dbg("Node to cpumask map for %d nodes\n", nr_node_ids); 8025863de0SAnton Blanchard } 8125863de0SAnton Blanchard 821daa6d08SBalbir Singh static int __cpuinit fake_numa_create_new_node(unsigned long end_pfn, 831daa6d08SBalbir Singh unsigned int *nid) 841daa6d08SBalbir Singh { 851daa6d08SBalbir Singh unsigned long long mem; 861daa6d08SBalbir Singh char *p = cmdline; 871daa6d08SBalbir Singh static unsigned int fake_nid; 881daa6d08SBalbir Singh static unsigned long long curr_boundary; 891daa6d08SBalbir Singh 901daa6d08SBalbir Singh /* 911daa6d08SBalbir Singh * Modify node id, iff we started creating NUMA nodes 921daa6d08SBalbir Singh * We want to continue from where we left of the last time 931daa6d08SBalbir Singh */ 941daa6d08SBalbir Singh if (fake_nid) 951daa6d08SBalbir Singh *nid = fake_nid; 961daa6d08SBalbir Singh /* 971daa6d08SBalbir Singh * In case there are no more arguments to parse, the 981daa6d08SBalbir Singh * node_id should be the same as the last fake node id 991daa6d08SBalbir Singh * (we've handled this above). 1001daa6d08SBalbir Singh */ 1011daa6d08SBalbir Singh if (!p) 1021daa6d08SBalbir Singh return 0; 1031daa6d08SBalbir Singh 1041daa6d08SBalbir Singh mem = memparse(p, &p); 1051daa6d08SBalbir Singh if (!mem) 1061daa6d08SBalbir Singh return 0; 1071daa6d08SBalbir Singh 1081daa6d08SBalbir Singh if (mem < curr_boundary) 1091daa6d08SBalbir Singh return 0; 1101daa6d08SBalbir Singh 1111daa6d08SBalbir Singh curr_boundary = mem; 1121daa6d08SBalbir Singh 1131daa6d08SBalbir Singh if ((end_pfn << PAGE_SHIFT) > mem) { 1141daa6d08SBalbir Singh /* 1151daa6d08SBalbir Singh * Skip commas and spaces 1161daa6d08SBalbir Singh */ 1171daa6d08SBalbir Singh while (*p == ',' || *p == ' ' || *p == '\t') 1181daa6d08SBalbir Singh p++; 1191daa6d08SBalbir Singh 1201daa6d08SBalbir Singh cmdline = p; 1211daa6d08SBalbir Singh fake_nid++; 1221daa6d08SBalbir Singh *nid = fake_nid; 1231daa6d08SBalbir Singh dbg("created new fake_node with id %d\n", fake_nid); 1241daa6d08SBalbir Singh return 1; 1251daa6d08SBalbir Singh } 1261daa6d08SBalbir Singh return 0; 1271daa6d08SBalbir Singh } 1281daa6d08SBalbir Singh 1298f64e1f2SJon Tollefson /* 130*5dfe8660STejun Heo * get_node_active_region - Return active region containing pfn 131e8170372SJon Tollefson * Active range returned is empty if none found. 132*5dfe8660STejun Heo * @pfn: The page to return the region for 133*5dfe8660STejun Heo * @node_ar: Returned set to the active region containing @pfn 1348f64e1f2SJon Tollefson */ 135*5dfe8660STejun Heo static void __init get_node_active_region(unsigned long pfn, 1368f64e1f2SJon Tollefson struct node_active_region *node_ar) 1378f64e1f2SJon Tollefson { 138*5dfe8660STejun Heo unsigned long start_pfn, end_pfn; 139*5dfe8660STejun Heo int i, nid; 1408f64e1f2SJon Tollefson 141*5dfe8660STejun Heo for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) { 142*5dfe8660STejun Heo if (pfn >= start_pfn && pfn < end_pfn) { 1438f64e1f2SJon Tollefson node_ar->nid = nid; 1448f64e1f2SJon Tollefson node_ar->start_pfn = start_pfn; 145*5dfe8660STejun Heo node_ar->end_pfn = end_pfn; 146*5dfe8660STejun Heo break; 147*5dfe8660STejun Heo } 148*5dfe8660STejun Heo } 1498f64e1f2SJon Tollefson } 1508f64e1f2SJon Tollefson 15139bf990eSJesse Larrew static void map_cpu_to_node(int cpu, int node) 152ab1f9dacSPaul Mackerras { 153ab1f9dacSPaul Mackerras numa_cpu_lookup_table[cpu] = node; 15445fb6ceaSAnton Blanchard 155bf4b85b0SNathan Lynch dbg("adding cpu %d to node %d\n", cpu, node); 156bf4b85b0SNathan Lynch 15725863de0SAnton Blanchard if (!(cpumask_test_cpu(cpu, node_to_cpumask_map[node]))) 15825863de0SAnton Blanchard cpumask_set_cpu(cpu, node_to_cpumask_map[node]); 159ab1f9dacSPaul Mackerras } 160ab1f9dacSPaul Mackerras 16139bf990eSJesse Larrew #if defined(CONFIG_HOTPLUG_CPU) || defined(CONFIG_PPC_SPLPAR) 162ab1f9dacSPaul Mackerras static void unmap_cpu_from_node(unsigned long cpu) 163ab1f9dacSPaul Mackerras { 164ab1f9dacSPaul Mackerras int node = numa_cpu_lookup_table[cpu]; 165ab1f9dacSPaul Mackerras 166ab1f9dacSPaul Mackerras dbg("removing cpu %lu from node %d\n", cpu, node); 167ab1f9dacSPaul Mackerras 16825863de0SAnton Blanchard if (cpumask_test_cpu(cpu, node_to_cpumask_map[node])) { 169429f4d8dSAnton Blanchard cpumask_clear_cpu(cpu, node_to_cpumask_map[node]); 170ab1f9dacSPaul Mackerras } else { 171ab1f9dacSPaul Mackerras printk(KERN_ERR "WARNING: cpu %lu not found in node %d\n", 172ab1f9dacSPaul Mackerras cpu, node); 173ab1f9dacSPaul Mackerras } 174ab1f9dacSPaul Mackerras } 17539bf990eSJesse Larrew #endif /* CONFIG_HOTPLUG_CPU || CONFIG_PPC_SPLPAR */ 176ab1f9dacSPaul Mackerras 177ab1f9dacSPaul Mackerras /* must hold reference to node during call */ 178a7f67bdfSJeremy Kerr static const int *of_get_associativity(struct device_node *dev) 179ab1f9dacSPaul Mackerras { 180e2eb6392SStephen Rothwell return of_get_property(dev, "ibm,associativity", NULL); 181ab1f9dacSPaul Mackerras } 182ab1f9dacSPaul Mackerras 183cf00085dSChandru /* 184cf00085dSChandru * Returns the property linux,drconf-usable-memory if 185cf00085dSChandru * it exists (the property exists only in kexec/kdump kernels, 186cf00085dSChandru * added by kexec-tools) 187cf00085dSChandru */ 188cf00085dSChandru static const u32 *of_get_usable_memory(struct device_node *memory) 189cf00085dSChandru { 190cf00085dSChandru const u32 *prop; 191cf00085dSChandru u32 len; 192cf00085dSChandru prop = of_get_property(memory, "linux,drconf-usable-memory", &len); 193cf00085dSChandru if (!prop || len < sizeof(unsigned int)) 194cf00085dSChandru return 0; 195cf00085dSChandru return prop; 196cf00085dSChandru } 197cf00085dSChandru 19841eab6f8SAnton Blanchard int __node_distance(int a, int b) 19941eab6f8SAnton Blanchard { 20041eab6f8SAnton Blanchard int i; 20141eab6f8SAnton Blanchard int distance = LOCAL_DISTANCE; 20241eab6f8SAnton Blanchard 20341eab6f8SAnton Blanchard if (!form1_affinity) 20441eab6f8SAnton Blanchard return distance; 20541eab6f8SAnton Blanchard 20641eab6f8SAnton Blanchard for (i = 0; i < distance_ref_points_depth; i++) { 20741eab6f8SAnton Blanchard if (distance_lookup_table[a][i] == distance_lookup_table[b][i]) 20841eab6f8SAnton Blanchard break; 20941eab6f8SAnton Blanchard 21041eab6f8SAnton Blanchard /* Double the distance for each NUMA level */ 21141eab6f8SAnton Blanchard distance *= 2; 21241eab6f8SAnton Blanchard } 21341eab6f8SAnton Blanchard 21441eab6f8SAnton Blanchard return distance; 21541eab6f8SAnton Blanchard } 21641eab6f8SAnton Blanchard 21741eab6f8SAnton Blanchard static void initialize_distance_lookup_table(int nid, 21841eab6f8SAnton Blanchard const unsigned int *associativity) 21941eab6f8SAnton Blanchard { 22041eab6f8SAnton Blanchard int i; 22141eab6f8SAnton Blanchard 22241eab6f8SAnton Blanchard if (!form1_affinity) 22341eab6f8SAnton Blanchard return; 22441eab6f8SAnton Blanchard 22541eab6f8SAnton Blanchard for (i = 0; i < distance_ref_points_depth; i++) { 22641eab6f8SAnton Blanchard distance_lookup_table[nid][i] = 22741eab6f8SAnton Blanchard associativity[distance_ref_points[i]]; 22841eab6f8SAnton Blanchard } 22941eab6f8SAnton Blanchard } 23041eab6f8SAnton Blanchard 231482ec7c4SNathan Lynch /* Returns nid in the range [0..MAX_NUMNODES-1], or -1 if no useful numa 232482ec7c4SNathan Lynch * info is found. 233482ec7c4SNathan Lynch */ 2349eff1a38SJesse Larrew static int associativity_to_nid(const unsigned int *associativity) 235ab1f9dacSPaul Mackerras { 236482ec7c4SNathan Lynch int nid = -1; 237ab1f9dacSPaul Mackerras 238ab1f9dacSPaul Mackerras if (min_common_depth == -1) 239482ec7c4SNathan Lynch goto out; 240ab1f9dacSPaul Mackerras 2419eff1a38SJesse Larrew if (associativity[0] >= min_common_depth) 2429eff1a38SJesse Larrew nid = associativity[min_common_depth]; 243bc16a759SNathan Lynch 244bc16a759SNathan Lynch /* POWER4 LPAR uses 0xffff as invalid node */ 245482ec7c4SNathan Lynch if (nid == 0xffff || nid >= MAX_NUMNODES) 246482ec7c4SNathan Lynch nid = -1; 24741eab6f8SAnton Blanchard 2489eff1a38SJesse Larrew if (nid > 0 && associativity[0] >= distance_ref_points_depth) 2499eff1a38SJesse Larrew initialize_distance_lookup_table(nid, associativity); 25041eab6f8SAnton Blanchard 251482ec7c4SNathan Lynch out: 252cf950b7aSNathan Lynch return nid; 253ab1f9dacSPaul Mackerras } 254ab1f9dacSPaul Mackerras 2559eff1a38SJesse Larrew /* Returns the nid associated with the given device tree node, 2569eff1a38SJesse Larrew * or -1 if not found. 2579eff1a38SJesse Larrew */ 2589eff1a38SJesse Larrew static int of_node_to_nid_single(struct device_node *device) 2599eff1a38SJesse Larrew { 2609eff1a38SJesse Larrew int nid = -1; 2619eff1a38SJesse Larrew const unsigned int *tmp; 2629eff1a38SJesse Larrew 2639eff1a38SJesse Larrew tmp = of_get_associativity(device); 2649eff1a38SJesse Larrew if (tmp) 2659eff1a38SJesse Larrew nid = associativity_to_nid(tmp); 2669eff1a38SJesse Larrew return nid; 2679eff1a38SJesse Larrew } 2689eff1a38SJesse Larrew 269953039c8SJeremy Kerr /* Walk the device tree upwards, looking for an associativity id */ 270953039c8SJeremy Kerr int of_node_to_nid(struct device_node *device) 271953039c8SJeremy Kerr { 272953039c8SJeremy Kerr struct device_node *tmp; 273953039c8SJeremy Kerr int nid = -1; 274953039c8SJeremy Kerr 275953039c8SJeremy Kerr of_node_get(device); 276953039c8SJeremy Kerr while (device) { 277953039c8SJeremy Kerr nid = of_node_to_nid_single(device); 278953039c8SJeremy Kerr if (nid != -1) 279953039c8SJeremy Kerr break; 280953039c8SJeremy Kerr 281953039c8SJeremy Kerr tmp = device; 282953039c8SJeremy Kerr device = of_get_parent(tmp); 283953039c8SJeremy Kerr of_node_put(tmp); 284953039c8SJeremy Kerr } 285953039c8SJeremy Kerr of_node_put(device); 286953039c8SJeremy Kerr 287953039c8SJeremy Kerr return nid; 288953039c8SJeremy Kerr } 289953039c8SJeremy Kerr EXPORT_SYMBOL_GPL(of_node_to_nid); 290953039c8SJeremy Kerr 291ab1f9dacSPaul Mackerras static int __init find_min_common_depth(void) 292ab1f9dacSPaul Mackerras { 29341eab6f8SAnton Blanchard int depth; 294bc8449ccSAnton Blanchard struct device_node *chosen; 295e70606ebSMichael Ellerman struct device_node *root; 296bc8449ccSAnton Blanchard const char *vec5; 297ab1f9dacSPaul Mackerras 298e70606ebSMichael Ellerman root = of_find_node_by_path("/rtas"); 299e70606ebSMichael Ellerman if (!root) 300e70606ebSMichael Ellerman root = of_find_node_by_path("/"); 301ab1f9dacSPaul Mackerras 302ab1f9dacSPaul Mackerras /* 30341eab6f8SAnton Blanchard * This property is a set of 32-bit integers, each representing 30441eab6f8SAnton Blanchard * an index into the ibm,associativity nodes. 30541eab6f8SAnton Blanchard * 30641eab6f8SAnton Blanchard * With form 0 affinity the first integer is for an SMP configuration 30741eab6f8SAnton Blanchard * (should be all 0's) and the second is for a normal NUMA 30841eab6f8SAnton Blanchard * configuration. We have only one level of NUMA. 30941eab6f8SAnton Blanchard * 31041eab6f8SAnton Blanchard * With form 1 affinity the first integer is the most significant 31141eab6f8SAnton Blanchard * NUMA boundary and the following are progressively less significant 31241eab6f8SAnton Blanchard * boundaries. There can be more than one level of NUMA. 313ab1f9dacSPaul Mackerras */ 314e70606ebSMichael Ellerman distance_ref_points = of_get_property(root, 31541eab6f8SAnton Blanchard "ibm,associativity-reference-points", 31641eab6f8SAnton Blanchard &distance_ref_points_depth); 317ab1f9dacSPaul Mackerras 31841eab6f8SAnton Blanchard if (!distance_ref_points) { 31941eab6f8SAnton Blanchard dbg("NUMA: ibm,associativity-reference-points not found.\n"); 32041eab6f8SAnton Blanchard goto err; 32141eab6f8SAnton Blanchard } 32241eab6f8SAnton Blanchard 32341eab6f8SAnton Blanchard distance_ref_points_depth /= sizeof(int); 32441eab6f8SAnton Blanchard 325bc8449ccSAnton Blanchard #define VEC5_AFFINITY_BYTE 5 326bc8449ccSAnton Blanchard #define VEC5_AFFINITY 0x80 327bc8449ccSAnton Blanchard chosen = of_find_node_by_path("/chosen"); 328bc8449ccSAnton Blanchard if (chosen) { 329bc8449ccSAnton Blanchard vec5 = of_get_property(chosen, "ibm,architecture-vec-5", NULL); 330bc8449ccSAnton Blanchard if (vec5 && (vec5[VEC5_AFFINITY_BYTE] & VEC5_AFFINITY)) { 331bc8449ccSAnton Blanchard dbg("Using form 1 affinity\n"); 33241eab6f8SAnton Blanchard form1_affinity = 1; 3334b83c330SAnton Blanchard } 334bc8449ccSAnton Blanchard } 3354b83c330SAnton Blanchard 33641eab6f8SAnton Blanchard if (form1_affinity) { 33741eab6f8SAnton Blanchard depth = distance_ref_points[0]; 338ab1f9dacSPaul Mackerras } else { 33941eab6f8SAnton Blanchard if (distance_ref_points_depth < 2) { 34041eab6f8SAnton Blanchard printk(KERN_WARNING "NUMA: " 34141eab6f8SAnton Blanchard "short ibm,associativity-reference-points\n"); 34241eab6f8SAnton Blanchard goto err; 343ab1f9dacSPaul Mackerras } 344ab1f9dacSPaul Mackerras 34541eab6f8SAnton Blanchard depth = distance_ref_points[1]; 34641eab6f8SAnton Blanchard } 34741eab6f8SAnton Blanchard 34841eab6f8SAnton Blanchard /* 34941eab6f8SAnton Blanchard * Warn and cap if the hardware supports more than 35041eab6f8SAnton Blanchard * MAX_DISTANCE_REF_POINTS domains. 35141eab6f8SAnton Blanchard */ 35241eab6f8SAnton Blanchard if (distance_ref_points_depth > MAX_DISTANCE_REF_POINTS) { 35341eab6f8SAnton Blanchard printk(KERN_WARNING "NUMA: distance array capped at " 35441eab6f8SAnton Blanchard "%d entries\n", MAX_DISTANCE_REF_POINTS); 35541eab6f8SAnton Blanchard distance_ref_points_depth = MAX_DISTANCE_REF_POINTS; 35641eab6f8SAnton Blanchard } 35741eab6f8SAnton Blanchard 358e70606ebSMichael Ellerman of_node_put(root); 359ab1f9dacSPaul Mackerras return depth; 36041eab6f8SAnton Blanchard 36141eab6f8SAnton Blanchard err: 362e70606ebSMichael Ellerman of_node_put(root); 36341eab6f8SAnton Blanchard return -1; 364ab1f9dacSPaul Mackerras } 365ab1f9dacSPaul Mackerras 36684c9fdd1SMike Kravetz static void __init get_n_mem_cells(int *n_addr_cells, int *n_size_cells) 367ab1f9dacSPaul Mackerras { 368ab1f9dacSPaul Mackerras struct device_node *memory = NULL; 369ab1f9dacSPaul Mackerras 370ab1f9dacSPaul Mackerras memory = of_find_node_by_type(memory, "memory"); 37154c23310SPaul Mackerras if (!memory) 37284c9fdd1SMike Kravetz panic("numa.c: No memory nodes found!"); 37354c23310SPaul Mackerras 374a8bda5ddSStephen Rothwell *n_addr_cells = of_n_addr_cells(memory); 3759213feeaSStephen Rothwell *n_size_cells = of_n_size_cells(memory); 37684c9fdd1SMike Kravetz of_node_put(memory); 377ab1f9dacSPaul Mackerras } 378ab1f9dacSPaul Mackerras 379a7f67bdfSJeremy Kerr static unsigned long __devinit read_n_cells(int n, const unsigned int **buf) 380ab1f9dacSPaul Mackerras { 381ab1f9dacSPaul Mackerras unsigned long result = 0; 382ab1f9dacSPaul Mackerras 383ab1f9dacSPaul Mackerras while (n--) { 384ab1f9dacSPaul Mackerras result = (result << 32) | **buf; 385ab1f9dacSPaul Mackerras (*buf)++; 386ab1f9dacSPaul Mackerras } 387ab1f9dacSPaul Mackerras return result; 388ab1f9dacSPaul Mackerras } 389ab1f9dacSPaul Mackerras 3908342681dSNathan Fontenot struct of_drconf_cell { 3918342681dSNathan Fontenot u64 base_addr; 3928342681dSNathan Fontenot u32 drc_index; 3938342681dSNathan Fontenot u32 reserved; 3948342681dSNathan Fontenot u32 aa_index; 3958342681dSNathan Fontenot u32 flags; 3968342681dSNathan Fontenot }; 3978342681dSNathan Fontenot 3988342681dSNathan Fontenot #define DRCONF_MEM_ASSIGNED 0x00000008 3998342681dSNathan Fontenot #define DRCONF_MEM_AI_INVALID 0x00000040 4008342681dSNathan Fontenot #define DRCONF_MEM_RESERVED 0x00000080 4018342681dSNathan Fontenot 4028342681dSNathan Fontenot /* 40395f72d1eSYinghai Lu * Read the next memblock list entry from the ibm,dynamic-memory property 4048342681dSNathan Fontenot * and return the information in the provided of_drconf_cell structure. 4058342681dSNathan Fontenot */ 4068342681dSNathan Fontenot static void read_drconf_cell(struct of_drconf_cell *drmem, const u32 **cellp) 4078342681dSNathan Fontenot { 4088342681dSNathan Fontenot const u32 *cp; 4098342681dSNathan Fontenot 4108342681dSNathan Fontenot drmem->base_addr = read_n_cells(n_mem_addr_cells, cellp); 4118342681dSNathan Fontenot 4128342681dSNathan Fontenot cp = *cellp; 4138342681dSNathan Fontenot drmem->drc_index = cp[0]; 4148342681dSNathan Fontenot drmem->reserved = cp[1]; 4158342681dSNathan Fontenot drmem->aa_index = cp[2]; 4168342681dSNathan Fontenot drmem->flags = cp[3]; 4178342681dSNathan Fontenot 4188342681dSNathan Fontenot *cellp = cp + 4; 4198342681dSNathan Fontenot } 4208342681dSNathan Fontenot 4218342681dSNathan Fontenot /* 42225985edcSLucas De Marchi * Retrieve and validate the ibm,dynamic-memory property of the device tree. 4238342681dSNathan Fontenot * 42495f72d1eSYinghai Lu * The layout of the ibm,dynamic-memory property is a number N of memblock 42595f72d1eSYinghai Lu * list entries followed by N memblock list entries. Each memblock list entry 42625985edcSLucas De Marchi * contains information as laid out in the of_drconf_cell struct above. 4278342681dSNathan Fontenot */ 4288342681dSNathan Fontenot static int of_get_drconf_memory(struct device_node *memory, const u32 **dm) 4298342681dSNathan Fontenot { 4308342681dSNathan Fontenot const u32 *prop; 4318342681dSNathan Fontenot u32 len, entries; 4328342681dSNathan Fontenot 4338342681dSNathan Fontenot prop = of_get_property(memory, "ibm,dynamic-memory", &len); 4348342681dSNathan Fontenot if (!prop || len < sizeof(unsigned int)) 4358342681dSNathan Fontenot return 0; 4368342681dSNathan Fontenot 4378342681dSNathan Fontenot entries = *prop++; 4388342681dSNathan Fontenot 4398342681dSNathan Fontenot /* Now that we know the number of entries, revalidate the size 4408342681dSNathan Fontenot * of the property read in to ensure we have everything 4418342681dSNathan Fontenot */ 4428342681dSNathan Fontenot if (len < (entries * (n_mem_addr_cells + 4) + 1) * sizeof(unsigned int)) 4438342681dSNathan Fontenot return 0; 4448342681dSNathan Fontenot 4458342681dSNathan Fontenot *dm = prop; 4468342681dSNathan Fontenot return entries; 4478342681dSNathan Fontenot } 4488342681dSNathan Fontenot 4498342681dSNathan Fontenot /* 45025985edcSLucas De Marchi * Retrieve and validate the ibm,lmb-size property for drconf memory 4518342681dSNathan Fontenot * from the device tree. 4528342681dSNathan Fontenot */ 4533fdfd990SBenjamin Herrenschmidt static u64 of_get_lmb_size(struct device_node *memory) 4548342681dSNathan Fontenot { 4558342681dSNathan Fontenot const u32 *prop; 4568342681dSNathan Fontenot u32 len; 4578342681dSNathan Fontenot 4583fdfd990SBenjamin Herrenschmidt prop = of_get_property(memory, "ibm,lmb-size", &len); 4598342681dSNathan Fontenot if (!prop || len < sizeof(unsigned int)) 4608342681dSNathan Fontenot return 0; 4618342681dSNathan Fontenot 4628342681dSNathan Fontenot return read_n_cells(n_mem_size_cells, &prop); 4638342681dSNathan Fontenot } 4648342681dSNathan Fontenot 4658342681dSNathan Fontenot struct assoc_arrays { 4668342681dSNathan Fontenot u32 n_arrays; 4678342681dSNathan Fontenot u32 array_sz; 4688342681dSNathan Fontenot const u32 *arrays; 4698342681dSNathan Fontenot }; 4708342681dSNathan Fontenot 4718342681dSNathan Fontenot /* 47225985edcSLucas De Marchi * Retrieve and validate the list of associativity arrays for drconf 4738342681dSNathan Fontenot * memory from the ibm,associativity-lookup-arrays property of the 4748342681dSNathan Fontenot * device tree.. 4758342681dSNathan Fontenot * 4768342681dSNathan Fontenot * The layout of the ibm,associativity-lookup-arrays property is a number N 4778342681dSNathan Fontenot * indicating the number of associativity arrays, followed by a number M 4788342681dSNathan Fontenot * indicating the size of each associativity array, followed by a list 4798342681dSNathan Fontenot * of N associativity arrays. 4808342681dSNathan Fontenot */ 4818342681dSNathan Fontenot static int of_get_assoc_arrays(struct device_node *memory, 4828342681dSNathan Fontenot struct assoc_arrays *aa) 4838342681dSNathan Fontenot { 4848342681dSNathan Fontenot const u32 *prop; 4858342681dSNathan Fontenot u32 len; 4868342681dSNathan Fontenot 4878342681dSNathan Fontenot prop = of_get_property(memory, "ibm,associativity-lookup-arrays", &len); 4888342681dSNathan Fontenot if (!prop || len < 2 * sizeof(unsigned int)) 4898342681dSNathan Fontenot return -1; 4908342681dSNathan Fontenot 4918342681dSNathan Fontenot aa->n_arrays = *prop++; 4928342681dSNathan Fontenot aa->array_sz = *prop++; 4938342681dSNathan Fontenot 4948342681dSNathan Fontenot /* Now that we know the number of arrrays and size of each array, 4958342681dSNathan Fontenot * revalidate the size of the property read in. 4968342681dSNathan Fontenot */ 4978342681dSNathan Fontenot if (len < (aa->n_arrays * aa->array_sz + 2) * sizeof(unsigned int)) 4988342681dSNathan Fontenot return -1; 4998342681dSNathan Fontenot 5008342681dSNathan Fontenot aa->arrays = prop; 5018342681dSNathan Fontenot return 0; 5028342681dSNathan Fontenot } 5038342681dSNathan Fontenot 5048342681dSNathan Fontenot /* 5058342681dSNathan Fontenot * This is like of_node_to_nid_single() for memory represented in the 5068342681dSNathan Fontenot * ibm,dynamic-reconfiguration-memory node. 5078342681dSNathan Fontenot */ 5088342681dSNathan Fontenot static int of_drconf_to_nid_single(struct of_drconf_cell *drmem, 5098342681dSNathan Fontenot struct assoc_arrays *aa) 5108342681dSNathan Fontenot { 5118342681dSNathan Fontenot int default_nid = 0; 5128342681dSNathan Fontenot int nid = default_nid; 5138342681dSNathan Fontenot int index; 5148342681dSNathan Fontenot 5158342681dSNathan Fontenot if (min_common_depth > 0 && min_common_depth <= aa->array_sz && 5168342681dSNathan Fontenot !(drmem->flags & DRCONF_MEM_AI_INVALID) && 5178342681dSNathan Fontenot drmem->aa_index < aa->n_arrays) { 5188342681dSNathan Fontenot index = drmem->aa_index * aa->array_sz + min_common_depth - 1; 5198342681dSNathan Fontenot nid = aa->arrays[index]; 5208342681dSNathan Fontenot 5218342681dSNathan Fontenot if (nid == 0xffff || nid >= MAX_NUMNODES) 5228342681dSNathan Fontenot nid = default_nid; 5238342681dSNathan Fontenot } 5248342681dSNathan Fontenot 5258342681dSNathan Fontenot return nid; 5268342681dSNathan Fontenot } 5278342681dSNathan Fontenot 528ab1f9dacSPaul Mackerras /* 529ab1f9dacSPaul Mackerras * Figure out to which domain a cpu belongs and stick it there. 530ab1f9dacSPaul Mackerras * Return the id of the domain used. 531ab1f9dacSPaul Mackerras */ 5322e5ce39dSNathan Lynch static int __cpuinit numa_setup_cpu(unsigned long lcpu) 533ab1f9dacSPaul Mackerras { 534cf950b7aSNathan Lynch int nid = 0; 5358b16cd23SMilton Miller struct device_node *cpu = of_get_cpu_node(lcpu, NULL); 536ab1f9dacSPaul Mackerras 537ab1f9dacSPaul Mackerras if (!cpu) { 538ab1f9dacSPaul Mackerras WARN_ON(1); 539ab1f9dacSPaul Mackerras goto out; 540ab1f9dacSPaul Mackerras } 541ab1f9dacSPaul Mackerras 542953039c8SJeremy Kerr nid = of_node_to_nid_single(cpu); 543ab1f9dacSPaul Mackerras 544482ec7c4SNathan Lynch if (nid < 0 || !node_online(nid)) 54572c33688SH Hartley Sweeten nid = first_online_node; 546ab1f9dacSPaul Mackerras out: 547cf950b7aSNathan Lynch map_cpu_to_node(lcpu, nid); 548ab1f9dacSPaul Mackerras 549ab1f9dacSPaul Mackerras of_node_put(cpu); 550ab1f9dacSPaul Mackerras 551cf950b7aSNathan Lynch return nid; 552ab1f9dacSPaul Mackerras } 553ab1f9dacSPaul Mackerras 55474b85f37SChandra Seetharaman static int __cpuinit cpu_numa_callback(struct notifier_block *nfb, 555ab1f9dacSPaul Mackerras unsigned long action, 556ab1f9dacSPaul Mackerras void *hcpu) 557ab1f9dacSPaul Mackerras { 558ab1f9dacSPaul Mackerras unsigned long lcpu = (unsigned long)hcpu; 559ab1f9dacSPaul Mackerras int ret = NOTIFY_DONE; 560ab1f9dacSPaul Mackerras 561ab1f9dacSPaul Mackerras switch (action) { 562ab1f9dacSPaul Mackerras case CPU_UP_PREPARE: 5638bb78442SRafael J. Wysocki case CPU_UP_PREPARE_FROZEN: 564ab1f9dacSPaul Mackerras numa_setup_cpu(lcpu); 565ab1f9dacSPaul Mackerras ret = NOTIFY_OK; 566ab1f9dacSPaul Mackerras break; 567ab1f9dacSPaul Mackerras #ifdef CONFIG_HOTPLUG_CPU 568ab1f9dacSPaul Mackerras case CPU_DEAD: 5698bb78442SRafael J. Wysocki case CPU_DEAD_FROZEN: 570ab1f9dacSPaul Mackerras case CPU_UP_CANCELED: 5718bb78442SRafael J. Wysocki case CPU_UP_CANCELED_FROZEN: 572ab1f9dacSPaul Mackerras unmap_cpu_from_node(lcpu); 573ab1f9dacSPaul Mackerras break; 574ab1f9dacSPaul Mackerras ret = NOTIFY_OK; 575ab1f9dacSPaul Mackerras #endif 576ab1f9dacSPaul Mackerras } 577ab1f9dacSPaul Mackerras return ret; 578ab1f9dacSPaul Mackerras } 579ab1f9dacSPaul Mackerras 580ab1f9dacSPaul Mackerras /* 581ab1f9dacSPaul Mackerras * Check and possibly modify a memory region to enforce the memory limit. 582ab1f9dacSPaul Mackerras * 583ab1f9dacSPaul Mackerras * Returns the size the region should have to enforce the memory limit. 584ab1f9dacSPaul Mackerras * This will either be the original value of size, a truncated value, 585ab1f9dacSPaul Mackerras * or zero. If the returned value of size is 0 the region should be 58625985edcSLucas De Marchi * discarded as it lies wholly above the memory limit. 587ab1f9dacSPaul Mackerras */ 58845fb6ceaSAnton Blanchard static unsigned long __init numa_enforce_memory_limit(unsigned long start, 58945fb6ceaSAnton Blanchard unsigned long size) 590ab1f9dacSPaul Mackerras { 591ab1f9dacSPaul Mackerras /* 59295f72d1eSYinghai Lu * We use memblock_end_of_DRAM() in here instead of memory_limit because 593ab1f9dacSPaul Mackerras * we've already adjusted it for the limit and it takes care of 594fe55249dSMilton Miller * having memory holes below the limit. Also, in the case of 595fe55249dSMilton Miller * iommu_is_off, memory_limit is not set but is implicitly enforced. 596ab1f9dacSPaul Mackerras */ 597ab1f9dacSPaul Mackerras 59895f72d1eSYinghai Lu if (start + size <= memblock_end_of_DRAM()) 599ab1f9dacSPaul Mackerras return size; 600ab1f9dacSPaul Mackerras 60195f72d1eSYinghai Lu if (start >= memblock_end_of_DRAM()) 602ab1f9dacSPaul Mackerras return 0; 603ab1f9dacSPaul Mackerras 60495f72d1eSYinghai Lu return memblock_end_of_DRAM() - start; 605ab1f9dacSPaul Mackerras } 606ab1f9dacSPaul Mackerras 6070204568aSPaul Mackerras /* 608cf00085dSChandru * Reads the counter for a given entry in 609cf00085dSChandru * linux,drconf-usable-memory property 610cf00085dSChandru */ 611cf00085dSChandru static inline int __init read_usm_ranges(const u32 **usm) 612cf00085dSChandru { 613cf00085dSChandru /* 6143fdfd990SBenjamin Herrenschmidt * For each lmb in ibm,dynamic-memory a corresponding 615cf00085dSChandru * entry in linux,drconf-usable-memory property contains 616cf00085dSChandru * a counter followed by that many (base, size) duple. 617cf00085dSChandru * read the counter from linux,drconf-usable-memory 618cf00085dSChandru */ 619cf00085dSChandru return read_n_cells(n_mem_size_cells, usm); 620cf00085dSChandru } 621cf00085dSChandru 622cf00085dSChandru /* 6230204568aSPaul Mackerras * Extract NUMA information from the ibm,dynamic-reconfiguration-memory 6240204568aSPaul Mackerras * node. This assumes n_mem_{addr,size}_cells have been set. 6250204568aSPaul Mackerras */ 6260204568aSPaul Mackerras static void __init parse_drconf_memory(struct device_node *memory) 6270204568aSPaul Mackerras { 628cf00085dSChandru const u32 *dm, *usm; 629cf00085dSChandru unsigned int n, rc, ranges, is_kexec_kdump = 0; 6303fdfd990SBenjamin Herrenschmidt unsigned long lmb_size, base, size, sz; 6318342681dSNathan Fontenot int nid; 6328342681dSNathan Fontenot struct assoc_arrays aa; 6330204568aSPaul Mackerras 6348342681dSNathan Fontenot n = of_get_drconf_memory(memory, &dm); 6358342681dSNathan Fontenot if (!n) 6360204568aSPaul Mackerras return; 6370204568aSPaul Mackerras 6383fdfd990SBenjamin Herrenschmidt lmb_size = of_get_lmb_size(memory); 6393fdfd990SBenjamin Herrenschmidt if (!lmb_size) 6408342681dSNathan Fontenot return; 6418342681dSNathan Fontenot 6428342681dSNathan Fontenot rc = of_get_assoc_arrays(memory, &aa); 6438342681dSNathan Fontenot if (rc) 6440204568aSPaul Mackerras return; 6450204568aSPaul Mackerras 646cf00085dSChandru /* check if this is a kexec/kdump kernel */ 647cf00085dSChandru usm = of_get_usable_memory(memory); 648cf00085dSChandru if (usm != NULL) 649cf00085dSChandru is_kexec_kdump = 1; 650cf00085dSChandru 6510204568aSPaul Mackerras for (; n != 0; --n) { 6528342681dSNathan Fontenot struct of_drconf_cell drmem; 6531daa6d08SBalbir Singh 6548342681dSNathan Fontenot read_drconf_cell(&drmem, &dm); 6558342681dSNathan Fontenot 6568342681dSNathan Fontenot /* skip this block if the reserved bit is set in flags (0x80) 6578342681dSNathan Fontenot or if the block is not assigned to this partition (0x8) */ 6588342681dSNathan Fontenot if ((drmem.flags & DRCONF_MEM_RESERVED) 6598342681dSNathan Fontenot || !(drmem.flags & DRCONF_MEM_ASSIGNED)) 6608342681dSNathan Fontenot continue; 6618342681dSNathan Fontenot 662cf00085dSChandru base = drmem.base_addr; 6633fdfd990SBenjamin Herrenschmidt size = lmb_size; 664cf00085dSChandru ranges = 1; 6658342681dSNathan Fontenot 666cf00085dSChandru if (is_kexec_kdump) { 667cf00085dSChandru ranges = read_usm_ranges(&usm); 668cf00085dSChandru if (!ranges) /* there are no (base, size) duple */ 6690204568aSPaul Mackerras continue; 670cf00085dSChandru } 671cf00085dSChandru do { 672cf00085dSChandru if (is_kexec_kdump) { 673cf00085dSChandru base = read_n_cells(n_mem_addr_cells, &usm); 674cf00085dSChandru size = read_n_cells(n_mem_size_cells, &usm); 675cf00085dSChandru } 676cf00085dSChandru nid = of_drconf_to_nid_single(&drmem, &aa); 677cf00085dSChandru fake_numa_create_new_node( 678cf00085dSChandru ((base + size) >> PAGE_SHIFT), 679cf00085dSChandru &nid); 680cf00085dSChandru node_set_online(nid); 681cf00085dSChandru sz = numa_enforce_memory_limit(base, size); 682cf00085dSChandru if (sz) 683cf00085dSChandru add_active_range(nid, base >> PAGE_SHIFT, 684cf00085dSChandru (base >> PAGE_SHIFT) 685cf00085dSChandru + (sz >> PAGE_SHIFT)); 686cf00085dSChandru } while (--ranges); 6870204568aSPaul Mackerras } 6880204568aSPaul Mackerras } 6890204568aSPaul Mackerras 690ab1f9dacSPaul Mackerras static int __init parse_numa_properties(void) 691ab1f9dacSPaul Mackerras { 692ab1f9dacSPaul Mackerras struct device_node *cpu = NULL; 693ab1f9dacSPaul Mackerras struct device_node *memory = NULL; 694482ec7c4SNathan Lynch int default_nid = 0; 695ab1f9dacSPaul Mackerras unsigned long i; 696ab1f9dacSPaul Mackerras 697ab1f9dacSPaul Mackerras if (numa_enabled == 0) { 698ab1f9dacSPaul Mackerras printk(KERN_WARNING "NUMA disabled by user\n"); 699ab1f9dacSPaul Mackerras return -1; 700ab1f9dacSPaul Mackerras } 701ab1f9dacSPaul Mackerras 702ab1f9dacSPaul Mackerras min_common_depth = find_min_common_depth(); 703ab1f9dacSPaul Mackerras 704ab1f9dacSPaul Mackerras if (min_common_depth < 0) 705ab1f9dacSPaul Mackerras return min_common_depth; 706ab1f9dacSPaul Mackerras 707bf4b85b0SNathan Lynch dbg("NUMA associativity depth for CPU/Memory: %d\n", min_common_depth); 708bf4b85b0SNathan Lynch 709ab1f9dacSPaul Mackerras /* 710482ec7c4SNathan Lynch * Even though we connect cpus to numa domains later in SMP 711482ec7c4SNathan Lynch * init, we need to know the node ids now. This is because 712482ec7c4SNathan Lynch * each node to be onlined must have NODE_DATA etc backing it. 713ab1f9dacSPaul Mackerras */ 714482ec7c4SNathan Lynch for_each_present_cpu(i) { 715cf950b7aSNathan Lynch int nid; 716ab1f9dacSPaul Mackerras 7178b16cd23SMilton Miller cpu = of_get_cpu_node(i, NULL); 718482ec7c4SNathan Lynch BUG_ON(!cpu); 719953039c8SJeremy Kerr nid = of_node_to_nid_single(cpu); 720ab1f9dacSPaul Mackerras of_node_put(cpu); 721ab1f9dacSPaul Mackerras 722482ec7c4SNathan Lynch /* 723482ec7c4SNathan Lynch * Don't fall back to default_nid yet -- we will plug 724482ec7c4SNathan Lynch * cpus into nodes once the memory scan has discovered 725482ec7c4SNathan Lynch * the topology. 726482ec7c4SNathan Lynch */ 727482ec7c4SNathan Lynch if (nid < 0) 728482ec7c4SNathan Lynch continue; 729482ec7c4SNathan Lynch node_set_online(nid); 730ab1f9dacSPaul Mackerras } 731ab1f9dacSPaul Mackerras 732237a0989SMike Kravetz get_n_mem_cells(&n_mem_addr_cells, &n_mem_size_cells); 733ab1f9dacSPaul Mackerras memory = NULL; 734ab1f9dacSPaul Mackerras while ((memory = of_find_node_by_type(memory, "memory")) != NULL) { 735ab1f9dacSPaul Mackerras unsigned long start; 736ab1f9dacSPaul Mackerras unsigned long size; 737cf950b7aSNathan Lynch int nid; 738ab1f9dacSPaul Mackerras int ranges; 739a7f67bdfSJeremy Kerr const unsigned int *memcell_buf; 740ab1f9dacSPaul Mackerras unsigned int len; 741ab1f9dacSPaul Mackerras 742e2eb6392SStephen Rothwell memcell_buf = of_get_property(memory, 743ba759485SMichael Ellerman "linux,usable-memory", &len); 744ba759485SMichael Ellerman if (!memcell_buf || len <= 0) 745e2eb6392SStephen Rothwell memcell_buf = of_get_property(memory, "reg", &len); 746ab1f9dacSPaul Mackerras if (!memcell_buf || len <= 0) 747ab1f9dacSPaul Mackerras continue; 748ab1f9dacSPaul Mackerras 749cc5d0189SBenjamin Herrenschmidt /* ranges in cell */ 750cc5d0189SBenjamin Herrenschmidt ranges = (len >> 2) / (n_mem_addr_cells + n_mem_size_cells); 751ab1f9dacSPaul Mackerras new_range: 752ab1f9dacSPaul Mackerras /* these are order-sensitive, and modify the buffer pointer */ 753237a0989SMike Kravetz start = read_n_cells(n_mem_addr_cells, &memcell_buf); 754237a0989SMike Kravetz size = read_n_cells(n_mem_size_cells, &memcell_buf); 755ab1f9dacSPaul Mackerras 756482ec7c4SNathan Lynch /* 757482ec7c4SNathan Lynch * Assumption: either all memory nodes or none will 758482ec7c4SNathan Lynch * have associativity properties. If none, then 759482ec7c4SNathan Lynch * everything goes to default_nid. 760482ec7c4SNathan Lynch */ 761953039c8SJeremy Kerr nid = of_node_to_nid_single(memory); 762482ec7c4SNathan Lynch if (nid < 0) 763482ec7c4SNathan Lynch nid = default_nid; 7641daa6d08SBalbir Singh 7651daa6d08SBalbir Singh fake_numa_create_new_node(((start + size) >> PAGE_SHIFT), &nid); 766482ec7c4SNathan Lynch node_set_online(nid); 767ab1f9dacSPaul Mackerras 768ab1f9dacSPaul Mackerras if (!(size = numa_enforce_memory_limit(start, size))) { 769ab1f9dacSPaul Mackerras if (--ranges) 770ab1f9dacSPaul Mackerras goto new_range; 771ab1f9dacSPaul Mackerras else 772ab1f9dacSPaul Mackerras continue; 773ab1f9dacSPaul Mackerras } 774ab1f9dacSPaul Mackerras 775c67c3cb4SMel Gorman add_active_range(nid, start >> PAGE_SHIFT, 776c67c3cb4SMel Gorman (start >> PAGE_SHIFT) + (size >> PAGE_SHIFT)); 777ab1f9dacSPaul Mackerras 778ab1f9dacSPaul Mackerras if (--ranges) 779ab1f9dacSPaul Mackerras goto new_range; 780ab1f9dacSPaul Mackerras } 781ab1f9dacSPaul Mackerras 7820204568aSPaul Mackerras /* 78395f72d1eSYinghai Lu * Now do the same thing for each MEMBLOCK listed in the ibm,dynamic-memory 7840204568aSPaul Mackerras * property in the ibm,dynamic-reconfiguration-memory node. 7850204568aSPaul Mackerras */ 7860204568aSPaul Mackerras memory = of_find_node_by_path("/ibm,dynamic-reconfiguration-memory"); 7870204568aSPaul Mackerras if (memory) 7880204568aSPaul Mackerras parse_drconf_memory(memory); 7890204568aSPaul Mackerras 790ab1f9dacSPaul Mackerras return 0; 791ab1f9dacSPaul Mackerras } 792ab1f9dacSPaul Mackerras 793ab1f9dacSPaul Mackerras static void __init setup_nonnuma(void) 794ab1f9dacSPaul Mackerras { 79595f72d1eSYinghai Lu unsigned long top_of_ram = memblock_end_of_DRAM(); 79695f72d1eSYinghai Lu unsigned long total_ram = memblock_phys_mem_size(); 797c67c3cb4SMel Gorman unsigned long start_pfn, end_pfn; 79828be7072SBenjamin Herrenschmidt unsigned int nid = 0; 79928be7072SBenjamin Herrenschmidt struct memblock_region *reg; 800ab1f9dacSPaul Mackerras 801e110b281SOlof Johansson printk(KERN_DEBUG "Top of RAM: 0x%lx, Total RAM: 0x%lx\n", 802ab1f9dacSPaul Mackerras top_of_ram, total_ram); 803e110b281SOlof Johansson printk(KERN_DEBUG "Memory hole size: %ldMB\n", 804ab1f9dacSPaul Mackerras (top_of_ram - total_ram) >> 20); 805ab1f9dacSPaul Mackerras 80628be7072SBenjamin Herrenschmidt for_each_memblock(memory, reg) { 807c7fc2de0SYinghai Lu start_pfn = memblock_region_memory_base_pfn(reg); 808c7fc2de0SYinghai Lu end_pfn = memblock_region_memory_end_pfn(reg); 8091daa6d08SBalbir Singh 8101daa6d08SBalbir Singh fake_numa_create_new_node(end_pfn, &nid); 8111daa6d08SBalbir Singh add_active_range(nid, start_pfn, end_pfn); 8121daa6d08SBalbir Singh node_set_online(nid); 813c67c3cb4SMel Gorman } 814ab1f9dacSPaul Mackerras } 815ab1f9dacSPaul Mackerras 8164b703a23SAnton Blanchard void __init dump_numa_cpu_topology(void) 8174b703a23SAnton Blanchard { 8184b703a23SAnton Blanchard unsigned int node; 8194b703a23SAnton Blanchard unsigned int cpu, count; 8204b703a23SAnton Blanchard 8214b703a23SAnton Blanchard if (min_common_depth == -1 || !numa_enabled) 8224b703a23SAnton Blanchard return; 8234b703a23SAnton Blanchard 8244b703a23SAnton Blanchard for_each_online_node(node) { 825e110b281SOlof Johansson printk(KERN_DEBUG "Node %d CPUs:", node); 8264b703a23SAnton Blanchard 8274b703a23SAnton Blanchard count = 0; 8284b703a23SAnton Blanchard /* 8294b703a23SAnton Blanchard * If we used a CPU iterator here we would miss printing 8304b703a23SAnton Blanchard * the holes in the cpumap. 8314b703a23SAnton Blanchard */ 83225863de0SAnton Blanchard for (cpu = 0; cpu < nr_cpu_ids; cpu++) { 83325863de0SAnton Blanchard if (cpumask_test_cpu(cpu, 83425863de0SAnton Blanchard node_to_cpumask_map[node])) { 8354b703a23SAnton Blanchard if (count == 0) 8364b703a23SAnton Blanchard printk(" %u", cpu); 8374b703a23SAnton Blanchard ++count; 8384b703a23SAnton Blanchard } else { 8394b703a23SAnton Blanchard if (count > 1) 8404b703a23SAnton Blanchard printk("-%u", cpu - 1); 8414b703a23SAnton Blanchard count = 0; 8424b703a23SAnton Blanchard } 8434b703a23SAnton Blanchard } 8444b703a23SAnton Blanchard 8454b703a23SAnton Blanchard if (count > 1) 84625863de0SAnton Blanchard printk("-%u", nr_cpu_ids - 1); 8474b703a23SAnton Blanchard printk("\n"); 8484b703a23SAnton Blanchard } 8494b703a23SAnton Blanchard } 8504b703a23SAnton Blanchard 8514b703a23SAnton Blanchard static void __init dump_numa_memory_topology(void) 852ab1f9dacSPaul Mackerras { 853ab1f9dacSPaul Mackerras unsigned int node; 854ab1f9dacSPaul Mackerras unsigned int count; 855ab1f9dacSPaul Mackerras 856ab1f9dacSPaul Mackerras if (min_common_depth == -1 || !numa_enabled) 857ab1f9dacSPaul Mackerras return; 858ab1f9dacSPaul Mackerras 859ab1f9dacSPaul Mackerras for_each_online_node(node) { 860ab1f9dacSPaul Mackerras unsigned long i; 861ab1f9dacSPaul Mackerras 862e110b281SOlof Johansson printk(KERN_DEBUG "Node %d Memory:", node); 863ab1f9dacSPaul Mackerras 864ab1f9dacSPaul Mackerras count = 0; 865ab1f9dacSPaul Mackerras 86695f72d1eSYinghai Lu for (i = 0; i < memblock_end_of_DRAM(); 86745fb6ceaSAnton Blanchard i += (1 << SECTION_SIZE_BITS)) { 86845fb6ceaSAnton Blanchard if (early_pfn_to_nid(i >> PAGE_SHIFT) == node) { 869ab1f9dacSPaul Mackerras if (count == 0) 870ab1f9dacSPaul Mackerras printk(" 0x%lx", i); 871ab1f9dacSPaul Mackerras ++count; 872ab1f9dacSPaul Mackerras } else { 873ab1f9dacSPaul Mackerras if (count > 0) 874ab1f9dacSPaul Mackerras printk("-0x%lx", i); 875ab1f9dacSPaul Mackerras count = 0; 876ab1f9dacSPaul Mackerras } 877ab1f9dacSPaul Mackerras } 878ab1f9dacSPaul Mackerras 879ab1f9dacSPaul Mackerras if (count > 0) 880ab1f9dacSPaul Mackerras printk("-0x%lx", i); 881ab1f9dacSPaul Mackerras printk("\n"); 882ab1f9dacSPaul Mackerras } 883ab1f9dacSPaul Mackerras } 884ab1f9dacSPaul Mackerras 885ab1f9dacSPaul Mackerras /* 88695f72d1eSYinghai Lu * Allocate some memory, satisfying the memblock or bootmem allocator where 887ab1f9dacSPaul Mackerras * required. nid is the preferred node and end is the physical address of 888ab1f9dacSPaul Mackerras * the highest address in the node. 889ab1f9dacSPaul Mackerras * 8900be210fdSDave Hansen * Returns the virtual address of the memory. 891ab1f9dacSPaul Mackerras */ 892893473dfSDave Hansen static void __init *careful_zallocation(int nid, unsigned long size, 89345fb6ceaSAnton Blanchard unsigned long align, 89445fb6ceaSAnton Blanchard unsigned long end_pfn) 895ab1f9dacSPaul Mackerras { 8960be210fdSDave Hansen void *ret; 89745fb6ceaSAnton Blanchard int new_nid; 8980be210fdSDave Hansen unsigned long ret_paddr; 8990be210fdSDave Hansen 90095f72d1eSYinghai Lu ret_paddr = __memblock_alloc_base(size, align, end_pfn << PAGE_SHIFT); 901ab1f9dacSPaul Mackerras 902ab1f9dacSPaul Mackerras /* retry over all memory */ 9030be210fdSDave Hansen if (!ret_paddr) 90495f72d1eSYinghai Lu ret_paddr = __memblock_alloc_base(size, align, memblock_end_of_DRAM()); 905ab1f9dacSPaul Mackerras 9060be210fdSDave Hansen if (!ret_paddr) 9075d21ea2bSDave Hansen panic("numa.c: cannot allocate %lu bytes for node %d", 908ab1f9dacSPaul Mackerras size, nid); 909ab1f9dacSPaul Mackerras 9100be210fdSDave Hansen ret = __va(ret_paddr); 9110be210fdSDave Hansen 912ab1f9dacSPaul Mackerras /* 913c555e520SDave Hansen * We initialize the nodes in numeric order: 0, 1, 2... 91495f72d1eSYinghai Lu * and hand over control from the MEMBLOCK allocator to the 915c555e520SDave Hansen * bootmem allocator. If this function is called for 916c555e520SDave Hansen * node 5, then we know that all nodes <5 are using the 91795f72d1eSYinghai Lu * bootmem allocator instead of the MEMBLOCK allocator. 918c555e520SDave Hansen * 919c555e520SDave Hansen * So, check the nid from which this allocation came 920c555e520SDave Hansen * and double check to see if we need to use bootmem 92195f72d1eSYinghai Lu * instead of the MEMBLOCK. We don't free the MEMBLOCK memory 922c555e520SDave Hansen * since it would be useless. 923ab1f9dacSPaul Mackerras */ 9240be210fdSDave Hansen new_nid = early_pfn_to_nid(ret_paddr >> PAGE_SHIFT); 92545fb6ceaSAnton Blanchard if (new_nid < nid) { 9260be210fdSDave Hansen ret = __alloc_bootmem_node(NODE_DATA(new_nid), 927ab1f9dacSPaul Mackerras size, align, 0); 928ab1f9dacSPaul Mackerras 9290be210fdSDave Hansen dbg("alloc_bootmem %p %lx\n", ret, size); 930ab1f9dacSPaul Mackerras } 931ab1f9dacSPaul Mackerras 932893473dfSDave Hansen memset(ret, 0, size); 9330be210fdSDave Hansen return ret; 934ab1f9dacSPaul Mackerras } 935ab1f9dacSPaul Mackerras 93674b85f37SChandra Seetharaman static struct notifier_block __cpuinitdata ppc64_numa_nb = { 93774b85f37SChandra Seetharaman .notifier_call = cpu_numa_callback, 93874b85f37SChandra Seetharaman .priority = 1 /* Must run before sched domains notifier. */ 93974b85f37SChandra Seetharaman }; 94074b85f37SChandra Seetharaman 9414a618669SDave Hansen static void mark_reserved_regions_for_nid(int nid) 942ab1f9dacSPaul Mackerras { 9434a618669SDave Hansen struct pglist_data *node = NODE_DATA(nid); 94428be7072SBenjamin Herrenschmidt struct memblock_region *reg; 945ab1f9dacSPaul Mackerras 94628be7072SBenjamin Herrenschmidt for_each_memblock(reserved, reg) { 94728be7072SBenjamin Herrenschmidt unsigned long physbase = reg->base; 94828be7072SBenjamin Herrenschmidt unsigned long size = reg->size; 9498f64e1f2SJon Tollefson unsigned long start_pfn = physbase >> PAGE_SHIFT; 95006eccea6SDave Hansen unsigned long end_pfn = PFN_UP(physbase + size); 9518f64e1f2SJon Tollefson struct node_active_region node_ar; 9524a618669SDave Hansen unsigned long node_end_pfn = node->node_start_pfn + 9534a618669SDave Hansen node->node_spanned_pages; 9544a618669SDave Hansen 9554a618669SDave Hansen /* 95695f72d1eSYinghai Lu * Check to make sure that this memblock.reserved area is 9574a618669SDave Hansen * within the bounds of the node that we care about. 9584a618669SDave Hansen * Checking the nid of the start and end points is not 9594a618669SDave Hansen * sufficient because the reserved area could span the 9604a618669SDave Hansen * entire node. 9614a618669SDave Hansen */ 9624a618669SDave Hansen if (end_pfn <= node->node_start_pfn || 9634a618669SDave Hansen start_pfn >= node_end_pfn) 9644a618669SDave Hansen continue; 965ab1f9dacSPaul Mackerras 9668f64e1f2SJon Tollefson get_node_active_region(start_pfn, &node_ar); 967e8170372SJon Tollefson while (start_pfn < end_pfn && 968e8170372SJon Tollefson node_ar.start_pfn < node_ar.end_pfn) { 969e8170372SJon Tollefson unsigned long reserve_size = size; 9708f64e1f2SJon Tollefson /* 9718f64e1f2SJon Tollefson * if reserved region extends past active region 9728f64e1f2SJon Tollefson * then trim size to active region 9738f64e1f2SJon Tollefson */ 9748f64e1f2SJon Tollefson if (end_pfn > node_ar.end_pfn) 975e8170372SJon Tollefson reserve_size = (node_ar.end_pfn << PAGE_SHIFT) 97606eccea6SDave Hansen - physbase; 977a4c74dddSDave Hansen /* 978a4c74dddSDave Hansen * Only worry about *this* node, others may not 979a4c74dddSDave Hansen * yet have valid NODE_DATA(). 980a4c74dddSDave Hansen */ 981a4c74dddSDave Hansen if (node_ar.nid == nid) { 982a4c74dddSDave Hansen dbg("reserve_bootmem %lx %lx nid=%d\n", 983a4c74dddSDave Hansen physbase, reserve_size, node_ar.nid); 984a4c74dddSDave Hansen reserve_bootmem_node(NODE_DATA(node_ar.nid), 985a4c74dddSDave Hansen physbase, reserve_size, 986a4c74dddSDave Hansen BOOTMEM_DEFAULT); 987a4c74dddSDave Hansen } 9888f64e1f2SJon Tollefson /* 9898f64e1f2SJon Tollefson * if reserved region is contained in the active region 9908f64e1f2SJon Tollefson * then done. 9918f64e1f2SJon Tollefson */ 9928f64e1f2SJon Tollefson if (end_pfn <= node_ar.end_pfn) 9938f64e1f2SJon Tollefson break; 9948f64e1f2SJon Tollefson 9958f64e1f2SJon Tollefson /* 9968f64e1f2SJon Tollefson * reserved region extends past the active region 9978f64e1f2SJon Tollefson * get next active region that contains this 9988f64e1f2SJon Tollefson * reserved region 9998f64e1f2SJon Tollefson */ 10008f64e1f2SJon Tollefson start_pfn = node_ar.end_pfn; 10018f64e1f2SJon Tollefson physbase = start_pfn << PAGE_SHIFT; 1002e8170372SJon Tollefson size = size - reserve_size; 10038f64e1f2SJon Tollefson get_node_active_region(start_pfn, &node_ar); 1004ab1f9dacSPaul Mackerras } 10054a618669SDave Hansen } 1006ab1f9dacSPaul Mackerras } 10078f64e1f2SJon Tollefson 10084a618669SDave Hansen 10094a618669SDave Hansen void __init do_init_bootmem(void) 10104a618669SDave Hansen { 10114a618669SDave Hansen int nid; 10124a618669SDave Hansen 10134a618669SDave Hansen min_low_pfn = 0; 101495f72d1eSYinghai Lu max_low_pfn = memblock_end_of_DRAM() >> PAGE_SHIFT; 10154a618669SDave Hansen max_pfn = max_low_pfn; 10164a618669SDave Hansen 10174a618669SDave Hansen if (parse_numa_properties()) 10184a618669SDave Hansen setup_nonnuma(); 10194a618669SDave Hansen else 10204a618669SDave Hansen dump_numa_memory_topology(); 10214a618669SDave Hansen 10224a618669SDave Hansen for_each_online_node(nid) { 10234a618669SDave Hansen unsigned long start_pfn, end_pfn; 10240be210fdSDave Hansen void *bootmem_vaddr; 10254a618669SDave Hansen unsigned long bootmap_pages; 10264a618669SDave Hansen 10274a618669SDave Hansen get_pfn_range_for_nid(nid, &start_pfn, &end_pfn); 10284a618669SDave Hansen 10294a618669SDave Hansen /* 10304a618669SDave Hansen * Allocate the node structure node local if possible 10314a618669SDave Hansen * 10324a618669SDave Hansen * Be careful moving this around, as it relies on all 10334a618669SDave Hansen * previous nodes' bootmem to be initialized and have 10344a618669SDave Hansen * all reserved areas marked. 10354a618669SDave Hansen */ 1036893473dfSDave Hansen NODE_DATA(nid) = careful_zallocation(nid, 10374a618669SDave Hansen sizeof(struct pglist_data), 10384a618669SDave Hansen SMP_CACHE_BYTES, end_pfn); 10394a618669SDave Hansen 10404a618669SDave Hansen dbg("node %d\n", nid); 10414a618669SDave Hansen dbg("NODE_DATA() = %p\n", NODE_DATA(nid)); 10424a618669SDave Hansen 10434a618669SDave Hansen NODE_DATA(nid)->bdata = &bootmem_node_data[nid]; 10444a618669SDave Hansen NODE_DATA(nid)->node_start_pfn = start_pfn; 10454a618669SDave Hansen NODE_DATA(nid)->node_spanned_pages = end_pfn - start_pfn; 10464a618669SDave Hansen 10474a618669SDave Hansen if (NODE_DATA(nid)->node_spanned_pages == 0) 10484a618669SDave Hansen continue; 10494a618669SDave Hansen 10504a618669SDave Hansen dbg("start_paddr = %lx\n", start_pfn << PAGE_SHIFT); 10514a618669SDave Hansen dbg("end_paddr = %lx\n", end_pfn << PAGE_SHIFT); 10524a618669SDave Hansen 10534a618669SDave Hansen bootmap_pages = bootmem_bootmap_pages(end_pfn - start_pfn); 1054893473dfSDave Hansen bootmem_vaddr = careful_zallocation(nid, 10554a618669SDave Hansen bootmap_pages << PAGE_SHIFT, 10564a618669SDave Hansen PAGE_SIZE, end_pfn); 10574a618669SDave Hansen 10580be210fdSDave Hansen dbg("bootmap_vaddr = %p\n", bootmem_vaddr); 10594a618669SDave Hansen 10600be210fdSDave Hansen init_bootmem_node(NODE_DATA(nid), 10610be210fdSDave Hansen __pa(bootmem_vaddr) >> PAGE_SHIFT, 10624a618669SDave Hansen start_pfn, end_pfn); 10634a618669SDave Hansen 10644a618669SDave Hansen free_bootmem_with_active_regions(nid, end_pfn); 10654a618669SDave Hansen /* 10664a618669SDave Hansen * Be very careful about moving this around. Future 1067893473dfSDave Hansen * calls to careful_zallocation() depend on this getting 10684a618669SDave Hansen * done correctly. 10694a618669SDave Hansen */ 10704a618669SDave Hansen mark_reserved_regions_for_nid(nid); 10718f64e1f2SJon Tollefson sparse_memory_present_with_active_regions(nid); 1072ab1f9dacSPaul Mackerras } 1073d3f6204aSBenjamin Herrenschmidt 1074d3f6204aSBenjamin Herrenschmidt init_bootmem_done = 1; 107525863de0SAnton Blanchard 107625863de0SAnton Blanchard /* 107725863de0SAnton Blanchard * Now bootmem is initialised we can create the node to cpumask 107825863de0SAnton Blanchard * lookup tables and setup the cpu callback to populate them. 107925863de0SAnton Blanchard */ 108025863de0SAnton Blanchard setup_node_to_cpumask_map(); 108125863de0SAnton Blanchard 108225863de0SAnton Blanchard register_cpu_notifier(&ppc64_numa_nb); 108325863de0SAnton Blanchard cpu_numa_callback(&ppc64_numa_nb, CPU_UP_PREPARE, 108425863de0SAnton Blanchard (void *)(unsigned long)boot_cpuid); 10854a618669SDave Hansen } 1086ab1f9dacSPaul Mackerras 1087ab1f9dacSPaul Mackerras void __init paging_init(void) 1088ab1f9dacSPaul Mackerras { 10896391af17SMel Gorman unsigned long max_zone_pfns[MAX_NR_ZONES]; 10906391af17SMel Gorman memset(max_zone_pfns, 0, sizeof(max_zone_pfns)); 109195f72d1eSYinghai Lu max_zone_pfns[ZONE_DMA] = memblock_end_of_DRAM() >> PAGE_SHIFT; 1092c67c3cb4SMel Gorman free_area_init_nodes(max_zone_pfns); 1093ab1f9dacSPaul Mackerras } 1094ab1f9dacSPaul Mackerras 1095ab1f9dacSPaul Mackerras static int __init early_numa(char *p) 1096ab1f9dacSPaul Mackerras { 1097ab1f9dacSPaul Mackerras if (!p) 1098ab1f9dacSPaul Mackerras return 0; 1099ab1f9dacSPaul Mackerras 1100ab1f9dacSPaul Mackerras if (strstr(p, "off")) 1101ab1f9dacSPaul Mackerras numa_enabled = 0; 1102ab1f9dacSPaul Mackerras 1103ab1f9dacSPaul Mackerras if (strstr(p, "debug")) 1104ab1f9dacSPaul Mackerras numa_debug = 1; 1105ab1f9dacSPaul Mackerras 11061daa6d08SBalbir Singh p = strstr(p, "fake="); 11071daa6d08SBalbir Singh if (p) 11081daa6d08SBalbir Singh cmdline = p + strlen("fake="); 11091daa6d08SBalbir Singh 1110ab1f9dacSPaul Mackerras return 0; 1111ab1f9dacSPaul Mackerras } 1112ab1f9dacSPaul Mackerras early_param("numa", early_numa); 1113237a0989SMike Kravetz 1114237a0989SMike Kravetz #ifdef CONFIG_MEMORY_HOTPLUG 1115237a0989SMike Kravetz /* 11160f16ef7fSNathan Fontenot * Find the node associated with a hot added memory section for 11170f16ef7fSNathan Fontenot * memory represented in the device tree by the property 11180f16ef7fSNathan Fontenot * ibm,dynamic-reconfiguration-memory/ibm,dynamic-memory. 11190db9360aSNathan Fontenot */ 11200db9360aSNathan Fontenot static int hot_add_drconf_scn_to_nid(struct device_node *memory, 11210db9360aSNathan Fontenot unsigned long scn_addr) 11220db9360aSNathan Fontenot { 11230db9360aSNathan Fontenot const u32 *dm; 11240f16ef7fSNathan Fontenot unsigned int drconf_cell_cnt, rc; 11253fdfd990SBenjamin Herrenschmidt unsigned long lmb_size; 11260db9360aSNathan Fontenot struct assoc_arrays aa; 11270f16ef7fSNathan Fontenot int nid = -1; 11280db9360aSNathan Fontenot 11290f16ef7fSNathan Fontenot drconf_cell_cnt = of_get_drconf_memory(memory, &dm); 11300f16ef7fSNathan Fontenot if (!drconf_cell_cnt) 11310f16ef7fSNathan Fontenot return -1; 11320db9360aSNathan Fontenot 11333fdfd990SBenjamin Herrenschmidt lmb_size = of_get_lmb_size(memory); 11343fdfd990SBenjamin Herrenschmidt if (!lmb_size) 11350f16ef7fSNathan Fontenot return -1; 11360db9360aSNathan Fontenot 11370db9360aSNathan Fontenot rc = of_get_assoc_arrays(memory, &aa); 11380db9360aSNathan Fontenot if (rc) 11390f16ef7fSNathan Fontenot return -1; 11400db9360aSNathan Fontenot 11410f16ef7fSNathan Fontenot for (; drconf_cell_cnt != 0; --drconf_cell_cnt) { 11420db9360aSNathan Fontenot struct of_drconf_cell drmem; 11430db9360aSNathan Fontenot 11440db9360aSNathan Fontenot read_drconf_cell(&drmem, &dm); 11450db9360aSNathan Fontenot 11460db9360aSNathan Fontenot /* skip this block if it is reserved or not assigned to 11470db9360aSNathan Fontenot * this partition */ 11480db9360aSNathan Fontenot if ((drmem.flags & DRCONF_MEM_RESERVED) 11490db9360aSNathan Fontenot || !(drmem.flags & DRCONF_MEM_ASSIGNED)) 11500db9360aSNathan Fontenot continue; 11510db9360aSNathan Fontenot 11520f16ef7fSNathan Fontenot if ((scn_addr < drmem.base_addr) 11533fdfd990SBenjamin Herrenschmidt || (scn_addr >= (drmem.base_addr + lmb_size))) 11540f16ef7fSNathan Fontenot continue; 11550db9360aSNathan Fontenot 11560f16ef7fSNathan Fontenot nid = of_drconf_to_nid_single(&drmem, &aa); 11570f16ef7fSNathan Fontenot break; 11580db9360aSNathan Fontenot } 11590db9360aSNathan Fontenot 11600f16ef7fSNathan Fontenot return nid; 11610db9360aSNathan Fontenot } 11620db9360aSNathan Fontenot 11630db9360aSNathan Fontenot /* 11640f16ef7fSNathan Fontenot * Find the node associated with a hot added memory section for memory 11650f16ef7fSNathan Fontenot * represented in the device tree as a node (i.e. memory@XXXX) for 116695f72d1eSYinghai Lu * each memblock. 1167237a0989SMike Kravetz */ 11680f16ef7fSNathan Fontenot int hot_add_node_scn_to_nid(unsigned long scn_addr) 1169237a0989SMike Kravetz { 1170237a0989SMike Kravetz struct device_node *memory = NULL; 11710f16ef7fSNathan Fontenot int nid = -1; 1172237a0989SMike Kravetz 1173237a0989SMike Kravetz while ((memory = of_find_node_by_type(memory, "memory")) != NULL) { 1174237a0989SMike Kravetz unsigned long start, size; 1175b226e462SMike Kravetz int ranges; 1176a7f67bdfSJeremy Kerr const unsigned int *memcell_buf; 1177237a0989SMike Kravetz unsigned int len; 1178237a0989SMike Kravetz 1179e2eb6392SStephen Rothwell memcell_buf = of_get_property(memory, "reg", &len); 1180237a0989SMike Kravetz if (!memcell_buf || len <= 0) 1181237a0989SMike Kravetz continue; 1182237a0989SMike Kravetz 1183cc5d0189SBenjamin Herrenschmidt /* ranges in cell */ 1184cc5d0189SBenjamin Herrenschmidt ranges = (len >> 2) / (n_mem_addr_cells + n_mem_size_cells); 11850f16ef7fSNathan Fontenot 11860f16ef7fSNathan Fontenot while (ranges--) { 1187237a0989SMike Kravetz start = read_n_cells(n_mem_addr_cells, &memcell_buf); 1188237a0989SMike Kravetz size = read_n_cells(n_mem_size_cells, &memcell_buf); 1189237a0989SMike Kravetz 11900f16ef7fSNathan Fontenot if ((scn_addr < start) || (scn_addr >= (start + size))) 11910f16ef7fSNathan Fontenot continue; 11920f16ef7fSNathan Fontenot 11930f16ef7fSNathan Fontenot nid = of_node_to_nid_single(memory); 11940f16ef7fSNathan Fontenot break; 11950f16ef7fSNathan Fontenot } 11960f16ef7fSNathan Fontenot 1197237a0989SMike Kravetz of_node_put(memory); 11980f16ef7fSNathan Fontenot if (nid >= 0) 11990f16ef7fSNathan Fontenot break; 12000f16ef7fSNathan Fontenot } 12010f16ef7fSNathan Fontenot 12020db9360aSNathan Fontenot return nid; 1203237a0989SMike Kravetz } 1204237a0989SMike Kravetz 12050f16ef7fSNathan Fontenot /* 12060f16ef7fSNathan Fontenot * Find the node associated with a hot added memory section. Section 120795f72d1eSYinghai Lu * corresponds to a SPARSEMEM section, not an MEMBLOCK. It is assumed that 120895f72d1eSYinghai Lu * sections are fully contained within a single MEMBLOCK. 12090f16ef7fSNathan Fontenot */ 12100f16ef7fSNathan Fontenot int hot_add_scn_to_nid(unsigned long scn_addr) 12110f16ef7fSNathan Fontenot { 12120f16ef7fSNathan Fontenot struct device_node *memory = NULL; 12130f16ef7fSNathan Fontenot int nid, found = 0; 12140f16ef7fSNathan Fontenot 12150f16ef7fSNathan Fontenot if (!numa_enabled || (min_common_depth < 0)) 121672c33688SH Hartley Sweeten return first_online_node; 12170f16ef7fSNathan Fontenot 12180f16ef7fSNathan Fontenot memory = of_find_node_by_path("/ibm,dynamic-reconfiguration-memory"); 12190f16ef7fSNathan Fontenot if (memory) { 12200f16ef7fSNathan Fontenot nid = hot_add_drconf_scn_to_nid(memory, scn_addr); 12210f16ef7fSNathan Fontenot of_node_put(memory); 12220f16ef7fSNathan Fontenot } else { 12230f16ef7fSNathan Fontenot nid = hot_add_node_scn_to_nid(scn_addr); 1224237a0989SMike Kravetz } 12250f16ef7fSNathan Fontenot 12260f16ef7fSNathan Fontenot if (nid < 0 || !node_online(nid)) 122772c33688SH Hartley Sweeten nid = first_online_node; 12280f16ef7fSNathan Fontenot 12290f16ef7fSNathan Fontenot if (NODE_DATA(nid)->node_spanned_pages) 12300f16ef7fSNathan Fontenot return nid; 12310f16ef7fSNathan Fontenot 12320f16ef7fSNathan Fontenot for_each_online_node(nid) { 12330f16ef7fSNathan Fontenot if (NODE_DATA(nid)->node_spanned_pages) { 12340f16ef7fSNathan Fontenot found = 1; 12350f16ef7fSNathan Fontenot break; 1236237a0989SMike Kravetz } 12370f16ef7fSNathan Fontenot } 12380f16ef7fSNathan Fontenot 12390f16ef7fSNathan Fontenot BUG_ON(!found); 12400f16ef7fSNathan Fontenot return nid; 12410f16ef7fSNathan Fontenot } 12420f16ef7fSNathan Fontenot 1243cd34206eSNishanth Aravamudan static u64 hot_add_drconf_memory_max(void) 1244cd34206eSNishanth Aravamudan { 1245cd34206eSNishanth Aravamudan struct device_node *memory = NULL; 1246cd34206eSNishanth Aravamudan unsigned int drconf_cell_cnt = 0; 1247cd34206eSNishanth Aravamudan u64 lmb_size = 0; 1248cd34206eSNishanth Aravamudan const u32 *dm = 0; 1249cd34206eSNishanth Aravamudan 1250cd34206eSNishanth Aravamudan memory = of_find_node_by_path("/ibm,dynamic-reconfiguration-memory"); 1251cd34206eSNishanth Aravamudan if (memory) { 1252cd34206eSNishanth Aravamudan drconf_cell_cnt = of_get_drconf_memory(memory, &dm); 1253cd34206eSNishanth Aravamudan lmb_size = of_get_lmb_size(memory); 1254cd34206eSNishanth Aravamudan of_node_put(memory); 1255cd34206eSNishanth Aravamudan } 1256cd34206eSNishanth Aravamudan return lmb_size * drconf_cell_cnt; 1257cd34206eSNishanth Aravamudan } 1258cd34206eSNishanth Aravamudan 1259cd34206eSNishanth Aravamudan /* 1260cd34206eSNishanth Aravamudan * memory_hotplug_max - return max address of memory that may be added 1261cd34206eSNishanth Aravamudan * 1262cd34206eSNishanth Aravamudan * This is currently only used on systems that support drconfig memory 1263cd34206eSNishanth Aravamudan * hotplug. 1264cd34206eSNishanth Aravamudan */ 1265cd34206eSNishanth Aravamudan u64 memory_hotplug_max(void) 1266cd34206eSNishanth Aravamudan { 1267cd34206eSNishanth Aravamudan return max(hot_add_drconf_memory_max(), memblock_end_of_DRAM()); 1268cd34206eSNishanth Aravamudan } 1269237a0989SMike Kravetz #endif /* CONFIG_MEMORY_HOTPLUG */ 12709eff1a38SJesse Larrew 1271bd03403aSJesse Larrew /* Virtual Processor Home Node (VPHN) support */ 127239bf990eSJesse Larrew #ifdef CONFIG_PPC_SPLPAR 12735de16699SAnton Blanchard static u8 vphn_cpu_change_counts[NR_CPUS][MAX_DISTANCE_REF_POINTS]; 12749eff1a38SJesse Larrew static cpumask_t cpu_associativity_changes_mask; 12759eff1a38SJesse Larrew static int vphn_enabled; 12769eff1a38SJesse Larrew static void set_topology_timer(void); 12779eff1a38SJesse Larrew 12789eff1a38SJesse Larrew /* 12799eff1a38SJesse Larrew * Store the current values of the associativity change counters in the 12809eff1a38SJesse Larrew * hypervisor. 12819eff1a38SJesse Larrew */ 12829eff1a38SJesse Larrew static void setup_cpu_associativity_change_counters(void) 12839eff1a38SJesse Larrew { 1284cd9d6cc7SJesse Larrew int cpu; 12859eff1a38SJesse Larrew 12865de16699SAnton Blanchard /* The VPHN feature supports a maximum of 8 reference points */ 12875de16699SAnton Blanchard BUILD_BUG_ON(MAX_DISTANCE_REF_POINTS > 8); 12885de16699SAnton Blanchard 12899eff1a38SJesse Larrew for_each_possible_cpu(cpu) { 1290cd9d6cc7SJesse Larrew int i; 12919eff1a38SJesse Larrew u8 *counts = vphn_cpu_change_counts[cpu]; 12929eff1a38SJesse Larrew volatile u8 *hypervisor_counts = lppaca[cpu].vphn_assoc_counts; 12939eff1a38SJesse Larrew 12945de16699SAnton Blanchard for (i = 0; i < distance_ref_points_depth; i++) 12959eff1a38SJesse Larrew counts[i] = hypervisor_counts[i]; 12969eff1a38SJesse Larrew } 12979eff1a38SJesse Larrew } 12989eff1a38SJesse Larrew 12999eff1a38SJesse Larrew /* 13009eff1a38SJesse Larrew * The hypervisor maintains a set of 8 associativity change counters in 13019eff1a38SJesse Larrew * the VPA of each cpu that correspond to the associativity levels in the 13029eff1a38SJesse Larrew * ibm,associativity-reference-points property. When an associativity 13039eff1a38SJesse Larrew * level changes, the corresponding counter is incremented. 13049eff1a38SJesse Larrew * 13059eff1a38SJesse Larrew * Set a bit in cpu_associativity_changes_mask for each cpu whose home 13069eff1a38SJesse Larrew * node associativity levels have changed. 13079eff1a38SJesse Larrew * 13089eff1a38SJesse Larrew * Returns the number of cpus with unhandled associativity changes. 13099eff1a38SJesse Larrew */ 13109eff1a38SJesse Larrew static int update_cpu_associativity_changes_mask(void) 13119eff1a38SJesse Larrew { 1312cd9d6cc7SJesse Larrew int cpu, nr_cpus = 0; 13139eff1a38SJesse Larrew cpumask_t *changes = &cpu_associativity_changes_mask; 13149eff1a38SJesse Larrew 13159eff1a38SJesse Larrew cpumask_clear(changes); 13169eff1a38SJesse Larrew 13179eff1a38SJesse Larrew for_each_possible_cpu(cpu) { 13189eff1a38SJesse Larrew int i, changed = 0; 13199eff1a38SJesse Larrew u8 *counts = vphn_cpu_change_counts[cpu]; 13209eff1a38SJesse Larrew volatile u8 *hypervisor_counts = lppaca[cpu].vphn_assoc_counts; 13219eff1a38SJesse Larrew 13225de16699SAnton Blanchard for (i = 0; i < distance_ref_points_depth; i++) { 1323d69043e8SAnton Blanchard if (hypervisor_counts[i] != counts[i]) { 13249eff1a38SJesse Larrew counts[i] = hypervisor_counts[i]; 13259eff1a38SJesse Larrew changed = 1; 13269eff1a38SJesse Larrew } 13279eff1a38SJesse Larrew } 13289eff1a38SJesse Larrew if (changed) { 13299eff1a38SJesse Larrew cpumask_set_cpu(cpu, changes); 13309eff1a38SJesse Larrew nr_cpus++; 13319eff1a38SJesse Larrew } 13329eff1a38SJesse Larrew } 13339eff1a38SJesse Larrew 13349eff1a38SJesse Larrew return nr_cpus; 13359eff1a38SJesse Larrew } 13369eff1a38SJesse Larrew 1337c0e5e46fSAnton Blanchard /* 1338c0e5e46fSAnton Blanchard * 6 64-bit registers unpacked into 12 32-bit associativity values. To form 1339c0e5e46fSAnton Blanchard * the complete property we have to add the length in the first cell. 1340c0e5e46fSAnton Blanchard */ 1341c0e5e46fSAnton Blanchard #define VPHN_ASSOC_BUFSIZE (6*sizeof(u64)/sizeof(u32) + 1) 13429eff1a38SJesse Larrew 13439eff1a38SJesse Larrew /* 13449eff1a38SJesse Larrew * Convert the associativity domain numbers returned from the hypervisor 13459eff1a38SJesse Larrew * to the sequence they would appear in the ibm,associativity property. 13469eff1a38SJesse Larrew */ 13479eff1a38SJesse Larrew static int vphn_unpack_associativity(const long *packed, unsigned int *unpacked) 13489eff1a38SJesse Larrew { 1349cd9d6cc7SJesse Larrew int i, nr_assoc_doms = 0; 13509eff1a38SJesse Larrew const u16 *field = (const u16*) packed; 13519eff1a38SJesse Larrew 13529eff1a38SJesse Larrew #define VPHN_FIELD_UNUSED (0xffff) 13539eff1a38SJesse Larrew #define VPHN_FIELD_MSB (0x8000) 13549eff1a38SJesse Larrew #define VPHN_FIELD_MASK (~VPHN_FIELD_MSB) 13559eff1a38SJesse Larrew 1356c0e5e46fSAnton Blanchard for (i = 1; i < VPHN_ASSOC_BUFSIZE; i++) { 13579eff1a38SJesse Larrew if (*field == VPHN_FIELD_UNUSED) { 13589eff1a38SJesse Larrew /* All significant fields processed, and remaining 13599eff1a38SJesse Larrew * fields contain the reserved value of all 1's. 13609eff1a38SJesse Larrew * Just store them. 13619eff1a38SJesse Larrew */ 13629eff1a38SJesse Larrew unpacked[i] = *((u32*)field); 13639eff1a38SJesse Larrew field += 2; 13647639adaaSJesse Larrew } else if (*field & VPHN_FIELD_MSB) { 13659eff1a38SJesse Larrew /* Data is in the lower 15 bits of this field */ 13669eff1a38SJesse Larrew unpacked[i] = *field & VPHN_FIELD_MASK; 13679eff1a38SJesse Larrew field++; 13689eff1a38SJesse Larrew nr_assoc_doms++; 13697639adaaSJesse Larrew } else { 13709eff1a38SJesse Larrew /* Data is in the lower 15 bits of this field 13719eff1a38SJesse Larrew * concatenated with the next 16 bit field 13729eff1a38SJesse Larrew */ 13739eff1a38SJesse Larrew unpacked[i] = *((u32*)field); 13749eff1a38SJesse Larrew field += 2; 13759eff1a38SJesse Larrew nr_assoc_doms++; 13769eff1a38SJesse Larrew } 13779eff1a38SJesse Larrew } 13789eff1a38SJesse Larrew 1379c0e5e46fSAnton Blanchard /* The first cell contains the length of the property */ 1380c0e5e46fSAnton Blanchard unpacked[0] = nr_assoc_doms; 1381c0e5e46fSAnton Blanchard 13829eff1a38SJesse Larrew return nr_assoc_doms; 13839eff1a38SJesse Larrew } 13849eff1a38SJesse Larrew 13859eff1a38SJesse Larrew /* 13869eff1a38SJesse Larrew * Retrieve the new associativity information for a virtual processor's 13879eff1a38SJesse Larrew * home node. 13889eff1a38SJesse Larrew */ 13899eff1a38SJesse Larrew static long hcall_vphn(unsigned long cpu, unsigned int *associativity) 13909eff1a38SJesse Larrew { 1391cd9d6cc7SJesse Larrew long rc; 13929eff1a38SJesse Larrew long retbuf[PLPAR_HCALL9_BUFSIZE] = {0}; 13939eff1a38SJesse Larrew u64 flags = 1; 13949eff1a38SJesse Larrew int hwcpu = get_hard_smp_processor_id(cpu); 13959eff1a38SJesse Larrew 13969eff1a38SJesse Larrew rc = plpar_hcall9(H_HOME_NODE_ASSOCIATIVITY, retbuf, flags, hwcpu); 13979eff1a38SJesse Larrew vphn_unpack_associativity(retbuf, associativity); 13989eff1a38SJesse Larrew 13999eff1a38SJesse Larrew return rc; 14009eff1a38SJesse Larrew } 14019eff1a38SJesse Larrew 14029eff1a38SJesse Larrew static long vphn_get_associativity(unsigned long cpu, 14039eff1a38SJesse Larrew unsigned int *associativity) 14049eff1a38SJesse Larrew { 1405cd9d6cc7SJesse Larrew long rc; 14069eff1a38SJesse Larrew 14079eff1a38SJesse Larrew rc = hcall_vphn(cpu, associativity); 14089eff1a38SJesse Larrew 14099eff1a38SJesse Larrew switch (rc) { 14109eff1a38SJesse Larrew case H_FUNCTION: 14119eff1a38SJesse Larrew printk(KERN_INFO 14129eff1a38SJesse Larrew "VPHN is not supported. Disabling polling...\n"); 14139eff1a38SJesse Larrew stop_topology_update(); 14149eff1a38SJesse Larrew break; 14159eff1a38SJesse Larrew case H_HARDWARE: 14169eff1a38SJesse Larrew printk(KERN_ERR 14179eff1a38SJesse Larrew "hcall_vphn() experienced a hardware fault " 14189eff1a38SJesse Larrew "preventing VPHN. Disabling polling...\n"); 14199eff1a38SJesse Larrew stop_topology_update(); 14209eff1a38SJesse Larrew } 14219eff1a38SJesse Larrew 14229eff1a38SJesse Larrew return rc; 14239eff1a38SJesse Larrew } 14249eff1a38SJesse Larrew 14259eff1a38SJesse Larrew /* 14269eff1a38SJesse Larrew * Update the node maps and sysfs entries for each cpu whose home node 14279eff1a38SJesse Larrew * has changed. 14289eff1a38SJesse Larrew */ 14299eff1a38SJesse Larrew int arch_update_cpu_topology(void) 14309eff1a38SJesse Larrew { 1431cd9d6cc7SJesse Larrew int cpu, nid, old_nid; 14329eff1a38SJesse Larrew unsigned int associativity[VPHN_ASSOC_BUFSIZE] = {0}; 1433cd9d6cc7SJesse Larrew struct sys_device *sysdev; 14349eff1a38SJesse Larrew 1435104699c0SKOSAKI Motohiro for_each_cpu(cpu,&cpu_associativity_changes_mask) { 14369eff1a38SJesse Larrew vphn_get_associativity(cpu, associativity); 14379eff1a38SJesse Larrew nid = associativity_to_nid(associativity); 14389eff1a38SJesse Larrew 14399eff1a38SJesse Larrew if (nid < 0 || !node_online(nid)) 14409eff1a38SJesse Larrew nid = first_online_node; 14419eff1a38SJesse Larrew 14429eff1a38SJesse Larrew old_nid = numa_cpu_lookup_table[cpu]; 14439eff1a38SJesse Larrew 14449eff1a38SJesse Larrew /* Disable hotplug while we update the cpu 14459eff1a38SJesse Larrew * masks and sysfs. 14469eff1a38SJesse Larrew */ 14479eff1a38SJesse Larrew get_online_cpus(); 14489eff1a38SJesse Larrew unregister_cpu_under_node(cpu, old_nid); 14499eff1a38SJesse Larrew unmap_cpu_from_node(cpu); 14509eff1a38SJesse Larrew map_cpu_to_node(cpu, nid); 14519eff1a38SJesse Larrew register_cpu_under_node(cpu, nid); 14529eff1a38SJesse Larrew put_online_cpus(); 14539eff1a38SJesse Larrew 14549eff1a38SJesse Larrew sysdev = get_cpu_sysdev(cpu); 14559eff1a38SJesse Larrew if (sysdev) 14569eff1a38SJesse Larrew kobject_uevent(&sysdev->kobj, KOBJ_CHANGE); 14579eff1a38SJesse Larrew } 14589eff1a38SJesse Larrew 14599eff1a38SJesse Larrew return 1; 14609eff1a38SJesse Larrew } 14619eff1a38SJesse Larrew 14629eff1a38SJesse Larrew static void topology_work_fn(struct work_struct *work) 14639eff1a38SJesse Larrew { 14649eff1a38SJesse Larrew rebuild_sched_domains(); 14659eff1a38SJesse Larrew } 14669eff1a38SJesse Larrew static DECLARE_WORK(topology_work, topology_work_fn); 14679eff1a38SJesse Larrew 14689eff1a38SJesse Larrew void topology_schedule_update(void) 14699eff1a38SJesse Larrew { 14709eff1a38SJesse Larrew schedule_work(&topology_work); 14719eff1a38SJesse Larrew } 14729eff1a38SJesse Larrew 14739eff1a38SJesse Larrew static void topology_timer_fn(unsigned long ignored) 14749eff1a38SJesse Larrew { 14759eff1a38SJesse Larrew if (!vphn_enabled) 14769eff1a38SJesse Larrew return; 14779eff1a38SJesse Larrew if (update_cpu_associativity_changes_mask() > 0) 14789eff1a38SJesse Larrew topology_schedule_update(); 14799eff1a38SJesse Larrew set_topology_timer(); 14809eff1a38SJesse Larrew } 14819eff1a38SJesse Larrew static struct timer_list topology_timer = 14829eff1a38SJesse Larrew TIMER_INITIALIZER(topology_timer_fn, 0, 0); 14839eff1a38SJesse Larrew 14849eff1a38SJesse Larrew static void set_topology_timer(void) 14859eff1a38SJesse Larrew { 14869eff1a38SJesse Larrew topology_timer.data = 0; 14879eff1a38SJesse Larrew topology_timer.expires = jiffies + 60 * HZ; 14889eff1a38SJesse Larrew add_timer(&topology_timer); 14899eff1a38SJesse Larrew } 14909eff1a38SJesse Larrew 14919eff1a38SJesse Larrew /* 14929eff1a38SJesse Larrew * Start polling for VPHN associativity changes. 14939eff1a38SJesse Larrew */ 14949eff1a38SJesse Larrew int start_topology_update(void) 14959eff1a38SJesse Larrew { 14969eff1a38SJesse Larrew int rc = 0; 14979eff1a38SJesse Larrew 149836e8695cSBenjamin Herrenschmidt /* Disabled until races with load balancing are fixed */ 149936e8695cSBenjamin Herrenschmidt if (0 && firmware_has_feature(FW_FEATURE_VPHN) && 1500fe5cfd63SAnton Blanchard get_lppaca()->shared_proc) { 15019eff1a38SJesse Larrew vphn_enabled = 1; 15029eff1a38SJesse Larrew setup_cpu_associativity_change_counters(); 15039eff1a38SJesse Larrew init_timer_deferrable(&topology_timer); 15049eff1a38SJesse Larrew set_topology_timer(); 15059eff1a38SJesse Larrew rc = 1; 15069eff1a38SJesse Larrew } 15079eff1a38SJesse Larrew 15089eff1a38SJesse Larrew return rc; 15099eff1a38SJesse Larrew } 15109eff1a38SJesse Larrew __initcall(start_topology_update); 15119eff1a38SJesse Larrew 15129eff1a38SJesse Larrew /* 15139eff1a38SJesse Larrew * Disable polling for VPHN associativity changes. 15149eff1a38SJesse Larrew */ 15159eff1a38SJesse Larrew int stop_topology_update(void) 15169eff1a38SJesse Larrew { 15179eff1a38SJesse Larrew vphn_enabled = 0; 15189eff1a38SJesse Larrew return del_timer_sync(&topology_timer); 15199eff1a38SJesse Larrew } 152039bf990eSJesse Larrew #endif /* CONFIG_PPC_SPLPAR */ 1521