17c478bd9Sstevel@tonic-gate /* 27c478bd9Sstevel@tonic-gate * CDDL HEADER START 37c478bd9Sstevel@tonic-gate * 47c478bd9Sstevel@tonic-gate * The contents of this file are subject to the terms of the 5c39996a7Sstevel * Common Development and Distribution License (the "License"). 6c39996a7Sstevel * You may not use this file except in compliance with the License. 77c478bd9Sstevel@tonic-gate * 87c478bd9Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 97c478bd9Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing. 107c478bd9Sstevel@tonic-gate * See the License for the specific language governing permissions 117c478bd9Sstevel@tonic-gate * and limitations under the License. 127c478bd9Sstevel@tonic-gate * 137c478bd9Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each 147c478bd9Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 157c478bd9Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the 167c478bd9Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying 177c478bd9Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner] 187c478bd9Sstevel@tonic-gate * 197c478bd9Sstevel@tonic-gate * CDDL HEADER END 207c478bd9Sstevel@tonic-gate */ 21c39996a7Sstevel 227c478bd9Sstevel@tonic-gate /* 23472714d6Skchow * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 247c478bd9Sstevel@tonic-gate * Use is subject to license terms. 257c478bd9Sstevel@tonic-gate */ 267c478bd9Sstevel@tonic-gate 277c478bd9Sstevel@tonic-gate #pragma ident "%Z%%M% %I% %E% SMI" 287c478bd9Sstevel@tonic-gate 297c478bd9Sstevel@tonic-gate 302e2c009bSjjc /* 312e2c009bSjjc * LOCALITY GROUP (LGROUP) PLATFORM SUPPORT FOR X86/AMD64 PLATFORMS 322e2c009bSjjc * ================================================================ 332e2c009bSjjc * Multiprocessor AMD and Intel systems may have Non Uniform Memory Access 342e2c009bSjjc * (NUMA). A NUMA machine consists of one or more "nodes" that each consist of 352e2c009bSjjc * one or more CPUs and some local memory. The CPUs in each node can access 362e2c009bSjjc * the memory in the other nodes but at a higher latency than accessing their 372e2c009bSjjc * local memory. Typically, a system with only one node has Uniform Memory 382e2c009bSjjc * Access (UMA), but it may be possible to have a one node system that has 392e2c009bSjjc * some global memory outside of the node which is higher latency. 402e2c009bSjjc * 412e2c009bSjjc * Module Description 422e2c009bSjjc * ------------------ 432e2c009bSjjc * This module provides a platform interface for determining which CPUs and 442e2c009bSjjc * which memory (and how much) are in a NUMA node and how far each node is from 452e2c009bSjjc * each other. The interface is used by the Virtual Memory (VM) system and the 462e2c009bSjjc * common lgroup framework. The VM system uses the plat_*() routines to fill 472e2c009bSjjc * in its memory node (memnode) array with the physical address range spanned 482e2c009bSjjc * by each NUMA node to know which memory belongs to which node, so it can 492e2c009bSjjc * build and manage a physical page free list for each NUMA node and allocate 502e2c009bSjjc * local memory from each node as needed. The common lgroup framework uses the 512e2c009bSjjc * exported lgrp_plat_*() routines to figure out which CPUs and memory belong 522e2c009bSjjc * to each node (leaf lgroup) and how far each node is from each other, so it 532e2c009bSjjc * can build the latency (lgroup) topology for the machine in order to optimize 542e2c009bSjjc * for locality. Also, an lgroup platform handle instead of lgroups are used 552e2c009bSjjc * in the interface with this module, so this module shouldn't need to know 562e2c009bSjjc * anything about lgroups. Instead, it just needs to know which CPUs, memory, 572e2c009bSjjc * etc. are in each NUMA node, how far each node is from each other, and to use 582e2c009bSjjc * a unique lgroup platform handle to refer to each node through the interface. 592e2c009bSjjc * 602e2c009bSjjc * Determining NUMA Configuration 612e2c009bSjjc * ------------------------------ 622e2c009bSjjc * By default, this module will try to determine the NUMA configuration of the 632e2c009bSjjc * machine by reading the ACPI System Resource Affinity Table (SRAT) and System 642e2c009bSjjc * Locality Information Table (SLIT). The SRAT contains info to tell which 652e2c009bSjjc * CPUs and memory are local to a given proximity domain (NUMA node). The SLIT 662e2c009bSjjc * is a matrix that gives the distance between each system locality (which is 672e2c009bSjjc * a NUMA node and should correspond to proximity domains in the SRAT). For 682e2c009bSjjc * more details on the SRAT and SLIT, please refer to an ACPI 3.0 or newer 692e2c009bSjjc * specification. 702e2c009bSjjc * 712e2c009bSjjc * If the SRAT doesn't exist on a system with AMD Opteron processors, we 722e2c009bSjjc * examine registers in PCI configuration space to determine how many nodes are 732e2c009bSjjc * in the system and which CPUs and memory are in each node. 742e2c009bSjjc * do while booting the kernel. 752e2c009bSjjc * 762e2c009bSjjc * NOTE: Using these PCI configuration space registers to determine this 772e2c009bSjjc * locality info is not guaranteed to work or be compatible across all 782e2c009bSjjc * Opteron processor families. 792e2c009bSjjc * 802e2c009bSjjc * If the SLIT does not exist or look right, the kernel will probe to determine 812e2c009bSjjc * the distance between nodes as long as the NUMA CPU and memory configuration 822e2c009bSjjc * has been determined (see lgrp_plat_probe() for details). 832e2c009bSjjc * 842e2c009bSjjc * Data Structures 852e2c009bSjjc * --------------- 862e2c009bSjjc * The main data structures used by this code are the following: 872e2c009bSjjc * 88dae2fa37Sjjc * - lgrp_plat_cpu_node[] CPU to node ID mapping table indexed by 89dae2fa37Sjjc * CPU ID (only used for SRAT) 902e2c009bSjjc * 912e2c009bSjjc * - lgrp_plat_lat_stats.latencies[][] Table of latencies between same and 922e2c009bSjjc * different nodes indexed by node ID 932e2c009bSjjc * 942e2c009bSjjc * - lgrp_plat_node_cnt Number of NUMA nodes in system 952e2c009bSjjc * 962e2c009bSjjc * - lgrp_plat_node_domain[] Node ID to proximity domain ID mapping 972e2c009bSjjc * table indexed by node ID (only used 982e2c009bSjjc * for SRAT) 992e2c009bSjjc * 1002e2c009bSjjc * - lgrp_plat_node_memory[] Table with physical address range for 1012e2c009bSjjc * each node indexed by node ID 1022e2c009bSjjc * 1032e2c009bSjjc * The code is implemented to make the following always be true: 1042e2c009bSjjc * 1052e2c009bSjjc * lgroup platform handle == node ID == memnode ID 1062e2c009bSjjc * 1072e2c009bSjjc * Moreover, it allows for the proximity domain ID to be equal to all of the 1082e2c009bSjjc * above as long as the proximity domains IDs are numbered from 0 to <number of 1092e2c009bSjjc * nodes - 1>. This is done by hashing each proximity domain ID into the range 1102e2c009bSjjc * from 0 to <number of nodes - 1>. Then proximity ID N will hash into node ID 1112e2c009bSjjc * N and proximity domain ID N will be entered into lgrp_plat_node_domain[N] 1122e2c009bSjjc * and be assigned node ID N. If the proximity domain IDs aren't numbered 1132e2c009bSjjc * from 0 to <number of nodes - 1>, then hashing the proximity domain IDs into 1142e2c009bSjjc * lgrp_plat_node_domain[] will still work for assigning proximity domain IDs 1152e2c009bSjjc * to node IDs. However, the proximity domain IDs may not map to the 1162e2c009bSjjc * equivalent node ID since we want to keep the node IDs numbered from 0 to 1172e2c009bSjjc * <number of nodes - 1> to minimize cost of searching and potentially space. 1182e2c009bSjjc */ 1192e2c009bSjjc 1202e2c009bSjjc 1217c478bd9Sstevel@tonic-gate #include <sys/archsystm.h> /* for {in,out}{b,w,l}() */ 122dae2fa37Sjjc #include <sys/bootconf.h> 1237c478bd9Sstevel@tonic-gate #include <sys/cmn_err.h> 124f78a91cdSjjc #include <sys/controlregs.h> 1257c478bd9Sstevel@tonic-gate #include <sys/cpupart.h> 1267c478bd9Sstevel@tonic-gate #include <sys/cpuvar.h> 1277c478bd9Sstevel@tonic-gate #include <sys/lgrp.h> 1287c478bd9Sstevel@tonic-gate #include <sys/machsystm.h> 1297c478bd9Sstevel@tonic-gate #include <sys/memlist.h> 1307c478bd9Sstevel@tonic-gate #include <sys/memnode.h> 1317c478bd9Sstevel@tonic-gate #include <sys/mman.h> 132ef50d8c0Sesaxe #include <sys/pci_cfgspace.h> 133ef50d8c0Sesaxe #include <sys/pci_impl.h> 1347c478bd9Sstevel@tonic-gate #include <sys/param.h> 135fb2f18f8Sesaxe #include <sys/pghw.h> 1367c478bd9Sstevel@tonic-gate #include <sys/promif.h> /* for prom_printf() */ 1372e2c009bSjjc #include <sys/sysmacros.h> 1387c478bd9Sstevel@tonic-gate #include <sys/systm.h> 1397c478bd9Sstevel@tonic-gate #include <sys/thread.h> 1407c478bd9Sstevel@tonic-gate #include <sys/types.h> 1417c478bd9Sstevel@tonic-gate #include <sys/var.h> 1427c478bd9Sstevel@tonic-gate #include <sys/x86_archext.h> /* for x86_feature and X86_AMD */ 1437c478bd9Sstevel@tonic-gate #include <vm/hat_i86.h> 1447c478bd9Sstevel@tonic-gate #include <vm/seg_kmem.h> 145affbd3ccSkchow #include <vm/vm_dep.h> 1467c478bd9Sstevel@tonic-gate 1472e2c009bSjjc #include "acpi_fw.h" /* for SRAT and SLIT */ 1487c478bd9Sstevel@tonic-gate 1497c478bd9Sstevel@tonic-gate 1507c478bd9Sstevel@tonic-gate #define MAX_NODES 8 1517c478bd9Sstevel@tonic-gate #define NLGRP (MAX_NODES * (MAX_NODES - 1) + 1) 1527c478bd9Sstevel@tonic-gate 1532e2c009bSjjc /* 1542e2c009bSjjc * Constants for configuring probing 1552e2c009bSjjc */ 1567c478bd9Sstevel@tonic-gate #define LGRP_PLAT_PROBE_NROUNDS 64 /* default laps for probing */ 1577c478bd9Sstevel@tonic-gate #define LGRP_PLAT_PROBE_NSAMPLES 1 /* default samples to take */ 1588949bcd6Sandrei #define LGRP_PLAT_PROBE_NREADS 256 /* number of vendor ID reads */ 1597c478bd9Sstevel@tonic-gate 1607c478bd9Sstevel@tonic-gate /* 1612e2c009bSjjc * Flags for probing 1622e2c009bSjjc */ 1632e2c009bSjjc #define LGRP_PLAT_PROBE_ENABLE 0x1 /* enable probing */ 1642e2c009bSjjc #define LGRP_PLAT_PROBE_PGCPY 0x2 /* probe using page copy */ 1652e2c009bSjjc #define LGRP_PLAT_PROBE_VENDOR 0x4 /* probe vendor ID register */ 1662e2c009bSjjc 1672e2c009bSjjc /* 1682e2c009bSjjc * Hash proximity domain ID into node to domain mapping table using to minimize 1692e2c009bSjjc * span of entries used 1702e2c009bSjjc */ 171d821f0f0Sjjc #define NODE_DOMAIN_HASH(domain, node_cnt) ((domain) % node_cnt) 1722e2c009bSjjc 1732e2c009bSjjc 1742e2c009bSjjc /* 175dae2fa37Sjjc * CPU to node ID mapping structure (only used with SRAT) 1762e2c009bSjjc */ 1772e2c009bSjjc typedef struct cpu_node_map { 1782e2c009bSjjc int exists; 1792e2c009bSjjc uint_t node; 1802e2c009bSjjc uint32_t apicid; 1812e2c009bSjjc uint32_t prox_domain; 1822e2c009bSjjc } cpu_node_map_t; 1832e2c009bSjjc 1842e2c009bSjjc /* 1852e2c009bSjjc * Latency statistics 1862e2c009bSjjc */ 1872e2c009bSjjc typedef struct lgrp_plat_latency_stats { 1882e2c009bSjjc hrtime_t latencies[MAX_NODES][MAX_NODES]; 1892e2c009bSjjc hrtime_t latency_max; 1902e2c009bSjjc hrtime_t latency_min; 1912e2c009bSjjc } lgrp_plat_latency_stats_t; 1922e2c009bSjjc 1932e2c009bSjjc /* 1942e2c009bSjjc * Memory configuration for probing 1952e2c009bSjjc */ 1962e2c009bSjjc typedef struct lgrp_plat_probe_mem_config { 1972e2c009bSjjc size_t probe_memsize; /* how much memory to probe per node */ 1982e2c009bSjjc caddr_t probe_va[MAX_NODES]; /* where memory mapped for probing */ 1992e2c009bSjjc pfn_t probe_pfn[MAX_NODES]; /* physical pages to map for probing */ 2002e2c009bSjjc } lgrp_plat_probe_mem_config_t; 2012e2c009bSjjc 2022e2c009bSjjc /* 2032e2c009bSjjc * Statistics kept for probing 2042e2c009bSjjc */ 2052e2c009bSjjc typedef struct lgrp_plat_probe_stats { 2062e2c009bSjjc hrtime_t flush_cost; 2072e2c009bSjjc hrtime_t probe_cost; 2082e2c009bSjjc hrtime_t probe_cost_total; 2092e2c009bSjjc hrtime_t probe_error_code; 2102e2c009bSjjc hrtime_t probe_errors[MAX_NODES][MAX_NODES]; 2112e2c009bSjjc int probe_suspect[MAX_NODES][MAX_NODES]; 2122e2c009bSjjc hrtime_t probe_max[MAX_NODES][MAX_NODES]; 2132e2c009bSjjc hrtime_t probe_min[MAX_NODES][MAX_NODES]; 2142e2c009bSjjc } lgrp_plat_probe_stats_t; 2152e2c009bSjjc 2162e2c009bSjjc /* 2172e2c009bSjjc * Node to proximity domain ID mapping structure (only used with SRAT) 2182e2c009bSjjc */ 2192e2c009bSjjc typedef struct node_domain_map { 2202e2c009bSjjc int exists; 2212e2c009bSjjc uint32_t prox_domain; 2222e2c009bSjjc } node_domain_map_t; 2232e2c009bSjjc 2242e2c009bSjjc /* 2252e2c009bSjjc * Node ID and starting and ending page for physical memory in node 2262e2c009bSjjc */ 2272e2c009bSjjc typedef struct node_phys_addr_map { 2282e2c009bSjjc pfn_t start; 2292e2c009bSjjc pfn_t end; 2302e2c009bSjjc int exists; 2312e2c009bSjjc uint32_t prox_domain; 2322e2c009bSjjc } node_phys_addr_map_t; 2332e2c009bSjjc 234dae2fa37Sjjc /* 235d821f0f0Sjjc * Number of CPUs for which we got APIC IDs 236dae2fa37Sjjc */ 237d821f0f0Sjjc static int lgrp_plat_apic_ncpus = 0; 2382e2c009bSjjc 2392e2c009bSjjc /* 240dae2fa37Sjjc * CPU to node ID mapping table (only used for SRAT) 2412e2c009bSjjc */ 2422e2c009bSjjc static cpu_node_map_t lgrp_plat_cpu_node[NCPU]; 2432e2c009bSjjc 2442e2c009bSjjc /* 2452e2c009bSjjc * Latency statistics 2462e2c009bSjjc */ 2472e2c009bSjjc lgrp_plat_latency_stats_t lgrp_plat_lat_stats; 2482e2c009bSjjc 2492e2c009bSjjc /* 2502e2c009bSjjc * Whether memory is interleaved across nodes causing MPO to be disabled 2512e2c009bSjjc */ 2522e2c009bSjjc static int lgrp_plat_mem_intrlv = 0; 2532e2c009bSjjc 2542e2c009bSjjc /* 2552e2c009bSjjc * Node ID to proximity domain ID mapping table (only used for SRAT) 2562e2c009bSjjc */ 2572e2c009bSjjc static node_domain_map_t lgrp_plat_node_domain[MAX_NODES]; 2582e2c009bSjjc 2592e2c009bSjjc /* 2602e2c009bSjjc * Physical address range for memory in each node 2612e2c009bSjjc */ 2622e2c009bSjjc static node_phys_addr_map_t lgrp_plat_node_memory[MAX_NODES]; 2632e2c009bSjjc 2642e2c009bSjjc /* 2652e2c009bSjjc * Statistics gotten from probing 2662e2c009bSjjc */ 2672e2c009bSjjc static lgrp_plat_probe_stats_t lgrp_plat_probe_stats; 2682e2c009bSjjc 2692e2c009bSjjc /* 2702e2c009bSjjc * Memory configuration for probing 2712e2c009bSjjc */ 2722e2c009bSjjc static lgrp_plat_probe_mem_config_t lgrp_plat_probe_mem_config; 2732e2c009bSjjc 2742e2c009bSjjc /* 2752e2c009bSjjc * Error code from processing ACPI SRAT 2762e2c009bSjjc */ 2772e2c009bSjjc static int lgrp_plat_srat_error = 0; 2782e2c009bSjjc 2792e2c009bSjjc /* 2802e2c009bSjjc * Error code from processing ACPI SLIT 2812e2c009bSjjc */ 2822e2c009bSjjc static int lgrp_plat_slit_error = 0; 2832e2c009bSjjc 2842e2c009bSjjc /* 2852e2c009bSjjc * Allocate lgroup array statically 2862e2c009bSjjc */ 2872e2c009bSjjc static lgrp_t lgrp_space[NLGRP]; 2882e2c009bSjjc static int nlgrps_alloc; 2892e2c009bSjjc 2902e2c009bSjjc 2912e2c009bSjjc /* 2922e2c009bSjjc * Number of nodes in system 2932e2c009bSjjc */ 2942e2c009bSjjc uint_t lgrp_plat_node_cnt = 1; 2952e2c009bSjjc 2962e2c009bSjjc /* 2972e2c009bSjjc * Configuration Parameters for Probing 2982e2c009bSjjc * - lgrp_plat_probe_flags Flags to specify enabling probing, probe 2992e2c009bSjjc * operation, etc. 3002e2c009bSjjc * - lgrp_plat_probe_nrounds How many rounds of probing to do 3012e2c009bSjjc * - lgrp_plat_probe_nsamples Number of samples to take when probing each 3022e2c009bSjjc * node 3032e2c009bSjjc * - lgrp_plat_probe_nreads Number of times to read vendor ID from 3042e2c009bSjjc * Northbridge for each probe 3052e2c009bSjjc */ 3062e2c009bSjjc uint_t lgrp_plat_probe_flags = 0; 3072e2c009bSjjc int lgrp_plat_probe_nrounds = LGRP_PLAT_PROBE_NROUNDS; 3082e2c009bSjjc int lgrp_plat_probe_nsamples = LGRP_PLAT_PROBE_NSAMPLES; 3092e2c009bSjjc int lgrp_plat_probe_nreads = LGRP_PLAT_PROBE_NREADS; 3102e2c009bSjjc 3112e2c009bSjjc /* 3122e2c009bSjjc * Enable use of ACPI System Resource Affinity Table (SRAT) and System 3132e2c009bSjjc * Locality Information Table (SLIT) 3142e2c009bSjjc */ 3152e2c009bSjjc int lgrp_plat_srat_enable = 1; 3162e2c009bSjjc int lgrp_plat_slit_enable = 1; 3172e2c009bSjjc 3182e2c009bSjjc /* 3192e2c009bSjjc * Static array to hold lgroup statistics 3202e2c009bSjjc */ 3212e2c009bSjjc struct lgrp_stats lgrp_stats[NLGRP]; 3222e2c009bSjjc 3232e2c009bSjjc 3242e2c009bSjjc /* 3252e2c009bSjjc * Forward declarations of platform interface routines 3262e2c009bSjjc */ 3272e2c009bSjjc void plat_build_mem_nodes(struct memlist *list); 3282e2c009bSjjc 3292e2c009bSjjc int plat_lgrphand_to_mem_node(lgrp_handle_t hand); 3302e2c009bSjjc 3312e2c009bSjjc lgrp_handle_t plat_mem_node_to_lgrphand(int mnode); 3322e2c009bSjjc 3332e2c009bSjjc int plat_mnode_xcheck(pfn_t pfncnt); 3342e2c009bSjjc 3352e2c009bSjjc int plat_pfn_to_mem_node(pfn_t pfn); 3362e2c009bSjjc 3372e2c009bSjjc /* 3382e2c009bSjjc * Forward declarations of lgroup platform interface routines 3392e2c009bSjjc */ 3402e2c009bSjjc lgrp_t *lgrp_plat_alloc(lgrp_id_t lgrpid); 3412e2c009bSjjc 3422e2c009bSjjc void lgrp_plat_config(lgrp_config_flag_t flag, uintptr_t arg); 3432e2c009bSjjc 3442e2c009bSjjc lgrp_handle_t lgrp_plat_cpu_to_hand(processorid_t id); 3452e2c009bSjjc 3462e2c009bSjjc void lgrp_plat_init(void); 3472e2c009bSjjc 3482e2c009bSjjc int lgrp_plat_latency(lgrp_handle_t from, lgrp_handle_t to); 3492e2c009bSjjc 3502e2c009bSjjc void lgrp_plat_main_init(void); 3512e2c009bSjjc 3522e2c009bSjjc int lgrp_plat_max_lgrps(void); 3532e2c009bSjjc 3542e2c009bSjjc pgcnt_t lgrp_plat_mem_size(lgrp_handle_t plathand, 3552e2c009bSjjc lgrp_mem_query_t query); 3562e2c009bSjjc 3572e2c009bSjjc lgrp_handle_t lgrp_plat_pfn_to_hand(pfn_t pfn); 3582e2c009bSjjc 3592e2c009bSjjc void lgrp_plat_probe(void); 3602e2c009bSjjc 3612e2c009bSjjc lgrp_handle_t lgrp_plat_root_hand(void); 3622e2c009bSjjc 3632e2c009bSjjc 3642e2c009bSjjc /* 3652e2c009bSjjc * Forward declarations of local routines 3662e2c009bSjjc */ 3672e2c009bSjjc static int is_opteron(void); 3682e2c009bSjjc 369dae2fa37Sjjc static int lgrp_plat_cpu_node_update(node_domain_map_t *node_domain, 370d821f0f0Sjjc int node_cnt, cpu_node_map_t *cpu_node, int nentries, uint32_t apicid, 371d821f0f0Sjjc uint32_t domain); 372dae2fa37Sjjc 3732e2c009bSjjc static int lgrp_plat_cpu_to_node(cpu_t *cp, cpu_node_map_t *cpu_node); 3742e2c009bSjjc 3752e2c009bSjjc static int lgrp_plat_domain_to_node(node_domain_map_t *node_domain, 376d821f0f0Sjjc int node_cnt, uint32_t domain); 3772e2c009bSjjc 3782e2c009bSjjc static void lgrp_plat_latency_adjust(node_phys_addr_map_t *node_memory, 3792e2c009bSjjc lgrp_plat_latency_stats_t *lat_stats, 3802e2c009bSjjc lgrp_plat_probe_stats_t *probe_stats); 3812e2c009bSjjc 3822e2c009bSjjc static int lgrp_plat_latency_verify(node_phys_addr_map_t *node_memory, 3832e2c009bSjjc lgrp_plat_latency_stats_t *lat_stats); 3842e2c009bSjjc 3852e2c009bSjjc static pgcnt_t lgrp_plat_mem_size_default(lgrp_handle_t, lgrp_mem_query_t); 3862e2c009bSjjc 3872e2c009bSjjc static int lgrp_plat_node_domain_update(node_domain_map_t *node_domain, 388d821f0f0Sjjc int node_cnt, uint32_t domain); 3892e2c009bSjjc 3902e2c009bSjjc static int lgrp_plat_node_memory_update(node_domain_map_t *node_domain, 391d821f0f0Sjjc int node_cnt, node_phys_addr_map_t *node_memory, uint64_t start, 392d821f0f0Sjjc uint64_t end, uint32_t domain); 3932e2c009bSjjc 3942e2c009bSjjc static hrtime_t lgrp_plat_probe_time(int to, cpu_node_map_t *cpu_node, 3952e2c009bSjjc lgrp_plat_probe_mem_config_t *probe_mem_config, 3962e2c009bSjjc lgrp_plat_latency_stats_t *lat_stats, 3972e2c009bSjjc lgrp_plat_probe_stats_t *probe_stats); 3982e2c009bSjjc 399d821f0f0Sjjc static int lgrp_plat_process_cpu_apicids(cpu_node_map_t *cpu_node); 400dae2fa37Sjjc 4012e2c009bSjjc static int lgrp_plat_process_slit(struct slit *tp, uint_t node_cnt, 4022e2c009bSjjc node_phys_addr_map_t *node_memory, lgrp_plat_latency_stats_t *lat_stats); 4032e2c009bSjjc 404d821f0f0Sjjc static int lgrp_plat_process_srat(struct srat *tp, 405d821f0f0Sjjc node_domain_map_t *node_domain, cpu_node_map_t *cpu_node, int cpu_count, 4062e2c009bSjjc node_phys_addr_map_t *node_memory); 4072e2c009bSjjc 4082e2c009bSjjc static int lgrp_plat_srat_domains(struct srat *tp); 4092e2c009bSjjc 4102e2c009bSjjc static void lgrp_plat_2level_setup(node_phys_addr_map_t *node_memory, 4112e2c009bSjjc lgrp_plat_latency_stats_t *lat_stats); 4122e2c009bSjjc 4132e2c009bSjjc static void opt_get_numa_config(uint_t *node_cnt, int *mem_intrlv, 4142e2c009bSjjc node_phys_addr_map_t *node_memory); 4152e2c009bSjjc 4162e2c009bSjjc static hrtime_t opt_probe_vendor(int dest_node, int nreads); 4172e2c009bSjjc 4182e2c009bSjjc 4192e2c009bSjjc /* 4202e2c009bSjjc * PLATFORM INTERFACE ROUTINES 4217c478bd9Sstevel@tonic-gate */ 4227c478bd9Sstevel@tonic-gate 4237c478bd9Sstevel@tonic-gate /* 4242e2c009bSjjc * Configure memory nodes for machines with more than one node (ie NUMA) 4252e2c009bSjjc */ 4262e2c009bSjjc void 4272e2c009bSjjc plat_build_mem_nodes(struct memlist *list) 4282e2c009bSjjc { 4292e2c009bSjjc pfn_t cur_start; /* start addr of subrange */ 4302e2c009bSjjc pfn_t cur_end; /* end addr of subrange */ 4312e2c009bSjjc pfn_t start; /* start addr of whole range */ 4322e2c009bSjjc pfn_t end; /* end addr of whole range */ 4332e2c009bSjjc 4342e2c009bSjjc /* 4352e2c009bSjjc * Boot install lists are arranged <addr, len>, ... 4362e2c009bSjjc */ 4372e2c009bSjjc while (list) { 4382e2c009bSjjc int node; 4392e2c009bSjjc 4402e2c009bSjjc start = list->address >> PAGESHIFT; 4412e2c009bSjjc end = (list->address + list->size - 1) >> PAGESHIFT; 4422e2c009bSjjc 4432e2c009bSjjc if (start > physmax) { 4442e2c009bSjjc list = list->next; 4452e2c009bSjjc continue; 4462e2c009bSjjc } 4472e2c009bSjjc if (end > physmax) 4482e2c009bSjjc end = physmax; 4492e2c009bSjjc 4502e2c009bSjjc /* 4512e2c009bSjjc * When there is only one memnode, just add memory to memnode 4522e2c009bSjjc */ 4532e2c009bSjjc if (max_mem_nodes == 1) { 4542e2c009bSjjc mem_node_add_slice(start, end); 4552e2c009bSjjc list = list->next; 4562e2c009bSjjc continue; 4572e2c009bSjjc } 4582e2c009bSjjc 4592e2c009bSjjc /* 4602e2c009bSjjc * mem_node_add_slice() expects to get a memory range that 4612e2c009bSjjc * is within one memnode, so need to split any memory range 4622e2c009bSjjc * that spans multiple memnodes into subranges that are each 4632e2c009bSjjc * contained within one memnode when feeding them to 4642e2c009bSjjc * mem_node_add_slice() 4652e2c009bSjjc */ 4662e2c009bSjjc cur_start = start; 4672e2c009bSjjc do { 4682e2c009bSjjc node = plat_pfn_to_mem_node(cur_start); 4692e2c009bSjjc 4702e2c009bSjjc /* 4712e2c009bSjjc * Panic if DRAM address map registers or SRAT say 4722e2c009bSjjc * memory in node doesn't exist or address from 4732e2c009bSjjc * boot installed memory list entry isn't in this node. 4742e2c009bSjjc * This shouldn't happen and rest of code can't deal 4752e2c009bSjjc * with this if it does. 4762e2c009bSjjc */ 4772e2c009bSjjc if (node < 0 || node >= lgrp_plat_node_cnt || 4782e2c009bSjjc !lgrp_plat_node_memory[node].exists || 4792e2c009bSjjc cur_start < lgrp_plat_node_memory[node].start || 4802e2c009bSjjc cur_start > lgrp_plat_node_memory[node].end) { 4812e2c009bSjjc cmn_err(CE_PANIC, "Don't know which memnode " 4822e2c009bSjjc "to add installed memory address 0x%lx\n", 4832e2c009bSjjc cur_start); 4842e2c009bSjjc } 4852e2c009bSjjc 4862e2c009bSjjc /* 4872e2c009bSjjc * End of current subrange should not span memnodes 4882e2c009bSjjc */ 4892e2c009bSjjc cur_end = end; 4902e2c009bSjjc if (lgrp_plat_node_memory[node].exists && 4912e2c009bSjjc cur_end > lgrp_plat_node_memory[node].end) 4922e2c009bSjjc cur_end = lgrp_plat_node_memory[node].end; 4932e2c009bSjjc 4942e2c009bSjjc mem_node_add_slice(cur_start, cur_end); 4952e2c009bSjjc 4962e2c009bSjjc /* 4972e2c009bSjjc * Next subrange starts after end of current one 4982e2c009bSjjc */ 4992e2c009bSjjc cur_start = cur_end + 1; 5002e2c009bSjjc } while (cur_end < end); 5012e2c009bSjjc 5022e2c009bSjjc list = list->next; 5032e2c009bSjjc } 5042e2c009bSjjc mem_node_physalign = 0; 5052e2c009bSjjc mem_node_pfn_shift = 0; 5062e2c009bSjjc } 5072e2c009bSjjc 5082e2c009bSjjc 5092e2c009bSjjc int 5102e2c009bSjjc plat_lgrphand_to_mem_node(lgrp_handle_t hand) 5112e2c009bSjjc { 5122e2c009bSjjc if (max_mem_nodes == 1) 5132e2c009bSjjc return (0); 5142e2c009bSjjc 5152e2c009bSjjc return ((int)hand); 5162e2c009bSjjc } 5172e2c009bSjjc 5182e2c009bSjjc 5192e2c009bSjjc /* 5202e2c009bSjjc * plat_mnode_xcheck: checks the node memory ranges to see if there is a pfncnt 5212e2c009bSjjc * range of pages aligned on pfncnt that crosses an node boundary. Returns 1 if 5222e2c009bSjjc * a crossing is found and returns 0 otherwise. 5232e2c009bSjjc */ 5242e2c009bSjjc int 5252e2c009bSjjc plat_mnode_xcheck(pfn_t pfncnt) 5262e2c009bSjjc { 5272e2c009bSjjc int node, prevnode = -1, basenode; 5282e2c009bSjjc pfn_t ea, sa; 5292e2c009bSjjc 5302e2c009bSjjc for (node = 0; node < lgrp_plat_node_cnt; node++) { 5312e2c009bSjjc 5322e2c009bSjjc if (lgrp_plat_node_memory[node].exists == 0) 5332e2c009bSjjc continue; 5342e2c009bSjjc 5352e2c009bSjjc if (prevnode == -1) { 5362e2c009bSjjc prevnode = node; 5372e2c009bSjjc basenode = node; 5382e2c009bSjjc continue; 5392e2c009bSjjc } 5402e2c009bSjjc 5412e2c009bSjjc /* assume x86 node pfn ranges are in increasing order */ 5422e2c009bSjjc ASSERT(lgrp_plat_node_memory[node].start > 5432e2c009bSjjc lgrp_plat_node_memory[prevnode].end); 5442e2c009bSjjc 5452e2c009bSjjc /* 5462e2c009bSjjc * continue if the starting address of node is not contiguous 5472e2c009bSjjc * with the previous node. 5482e2c009bSjjc */ 5492e2c009bSjjc 5502e2c009bSjjc if (lgrp_plat_node_memory[node].start != 5512e2c009bSjjc (lgrp_plat_node_memory[prevnode].end + 1)) { 5522e2c009bSjjc basenode = node; 5532e2c009bSjjc prevnode = node; 5542e2c009bSjjc continue; 5552e2c009bSjjc } 5562e2c009bSjjc 5572e2c009bSjjc /* check if the starting address of node is pfncnt aligned */ 5582e2c009bSjjc if ((lgrp_plat_node_memory[node].start & (pfncnt - 1)) != 0) { 5592e2c009bSjjc 5602e2c009bSjjc /* 5612e2c009bSjjc * at this point, node starts at an unaligned boundary 5622e2c009bSjjc * and is contiguous with the previous node(s) to 5632e2c009bSjjc * basenode. Check if there is an aligned contiguous 5642e2c009bSjjc * range of length pfncnt that crosses this boundary. 5652e2c009bSjjc */ 5662e2c009bSjjc 5672e2c009bSjjc sa = P2ALIGN(lgrp_plat_node_memory[prevnode].end, 5682e2c009bSjjc pfncnt); 5692e2c009bSjjc ea = P2ROUNDUP((lgrp_plat_node_memory[node].start), 5702e2c009bSjjc pfncnt); 5712e2c009bSjjc 5722e2c009bSjjc ASSERT((ea - sa) == pfncnt); 5732e2c009bSjjc if (sa >= lgrp_plat_node_memory[basenode].start && 5742e2c009bSjjc ea <= (lgrp_plat_node_memory[node].end + 1)) 5752e2c009bSjjc return (1); 5762e2c009bSjjc } 5772e2c009bSjjc prevnode = node; 5782e2c009bSjjc } 5792e2c009bSjjc return (0); 5802e2c009bSjjc } 5812e2c009bSjjc 5822e2c009bSjjc 5832e2c009bSjjc lgrp_handle_t 5842e2c009bSjjc plat_mem_node_to_lgrphand(int mnode) 5852e2c009bSjjc { 5862e2c009bSjjc if (max_mem_nodes == 1) 5872e2c009bSjjc return (LGRP_DEFAULT_HANDLE); 5882e2c009bSjjc 5892e2c009bSjjc return ((lgrp_handle_t)mnode); 5902e2c009bSjjc } 5912e2c009bSjjc 5922e2c009bSjjc 5932e2c009bSjjc int 5942e2c009bSjjc plat_pfn_to_mem_node(pfn_t pfn) 5952e2c009bSjjc { 5962e2c009bSjjc int node; 5972e2c009bSjjc 5982e2c009bSjjc if (max_mem_nodes == 1) 5992e2c009bSjjc return (0); 6002e2c009bSjjc 6012e2c009bSjjc for (node = 0; node < lgrp_plat_node_cnt; node++) { 6022e2c009bSjjc /* 6032e2c009bSjjc * Skip nodes with no memory 6042e2c009bSjjc */ 6052e2c009bSjjc if (!lgrp_plat_node_memory[node].exists) 6062e2c009bSjjc continue; 6072e2c009bSjjc 6082e2c009bSjjc if (pfn >= lgrp_plat_node_memory[node].start && 6092e2c009bSjjc pfn <= lgrp_plat_node_memory[node].end) 6102e2c009bSjjc return (node); 6112e2c009bSjjc } 6122e2c009bSjjc 6132e2c009bSjjc /* 6142e2c009bSjjc * Didn't find memnode where this PFN lives which should never happen 6152e2c009bSjjc */ 6162e2c009bSjjc ASSERT(node < lgrp_plat_node_cnt); 6172e2c009bSjjc return (-1); 6182e2c009bSjjc } 6192e2c009bSjjc 6202e2c009bSjjc 6212e2c009bSjjc /* 6222e2c009bSjjc * LGROUP PLATFORM INTERFACE ROUTINES 6232e2c009bSjjc */ 6242e2c009bSjjc 6252e2c009bSjjc /* 6262e2c009bSjjc * Allocate additional space for an lgroup. 6272e2c009bSjjc */ 6282e2c009bSjjc /* ARGSUSED */ 6292e2c009bSjjc lgrp_t * 6302e2c009bSjjc lgrp_plat_alloc(lgrp_id_t lgrpid) 6312e2c009bSjjc { 6322e2c009bSjjc lgrp_t *lgrp; 6332e2c009bSjjc 6342e2c009bSjjc lgrp = &lgrp_space[nlgrps_alloc++]; 6352e2c009bSjjc if (lgrpid >= NLGRP || nlgrps_alloc > NLGRP) 6362e2c009bSjjc return (NULL); 6372e2c009bSjjc return (lgrp); 6382e2c009bSjjc } 6392e2c009bSjjc 6402e2c009bSjjc 6412e2c009bSjjc /* 6422e2c009bSjjc * Platform handling for (re)configuration changes 6432e2c009bSjjc */ 6442e2c009bSjjc /* ARGSUSED */ 6452e2c009bSjjc void 6462e2c009bSjjc lgrp_plat_config(lgrp_config_flag_t flag, uintptr_t arg) 6472e2c009bSjjc { 6482e2c009bSjjc } 6492e2c009bSjjc 6502e2c009bSjjc 6512e2c009bSjjc /* 6522e2c009bSjjc * Return the platform handle for the lgroup containing the given CPU 6532e2c009bSjjc */ 6542e2c009bSjjc /* ARGSUSED */ 6552e2c009bSjjc lgrp_handle_t 6562e2c009bSjjc lgrp_plat_cpu_to_hand(processorid_t id) 6572e2c009bSjjc { 6582e2c009bSjjc lgrp_handle_t hand; 6592e2c009bSjjc 6602e2c009bSjjc if (lgrp_plat_node_cnt == 1) 6612e2c009bSjjc return (LGRP_DEFAULT_HANDLE); 6622e2c009bSjjc 6632e2c009bSjjc hand = (lgrp_handle_t)lgrp_plat_cpu_to_node(cpu[id], 6642e2c009bSjjc lgrp_plat_cpu_node); 6652e2c009bSjjc 6662e2c009bSjjc ASSERT(hand != (lgrp_handle_t)-1); 6672e2c009bSjjc if (hand == (lgrp_handle_t)-1) 6682e2c009bSjjc return (LGRP_NULL_HANDLE); 6692e2c009bSjjc 6702e2c009bSjjc return (hand); 6712e2c009bSjjc } 6722e2c009bSjjc 6732e2c009bSjjc 6742e2c009bSjjc /* 6752e2c009bSjjc * Platform-specific initialization of lgroups 6762e2c009bSjjc */ 6772e2c009bSjjc void 6782e2c009bSjjc lgrp_plat_init(void) 6792e2c009bSjjc { 6802e2c009bSjjc #if defined(__xpv) 6812e2c009bSjjc /* 6822e2c009bSjjc * XXPV For now, the hypervisor treats all memory equally. 6832e2c009bSjjc */ 6842e2c009bSjjc lgrp_plat_node_cnt = max_mem_nodes = 1; 6852e2c009bSjjc #else /* __xpv */ 6862e2c009bSjjc uint_t probe_op; 6872e2c009bSjjc 6882e2c009bSjjc /* 6892e2c009bSjjc * Initialize as a UMA machine 6902e2c009bSjjc */ 6912e2c009bSjjc if (lgrp_topo_ht_limit() == 1) { 6922e2c009bSjjc lgrp_plat_node_cnt = max_mem_nodes = 1; 6932e2c009bSjjc return; 6942e2c009bSjjc } 6952e2c009bSjjc 6962e2c009bSjjc /* 697dae2fa37Sjjc * Read boot property with CPU to APIC ID mapping table/array and fill 698dae2fa37Sjjc * in CPU to node ID mapping table with APIC ID for each CPU 699dae2fa37Sjjc */ 700d821f0f0Sjjc lgrp_plat_apic_ncpus = 701d821f0f0Sjjc lgrp_plat_process_cpu_apicids(lgrp_plat_cpu_node); 702dae2fa37Sjjc 703dae2fa37Sjjc /* 7042e2c009bSjjc * Determine which CPUs and memory are local to each other and number 7052e2c009bSjjc * of NUMA nodes by reading ACPI System Resource Affinity Table (SRAT) 7062e2c009bSjjc */ 707d821f0f0Sjjc if (lgrp_plat_apic_ncpus > 0) { 708d821f0f0Sjjc int retval; 709d821f0f0Sjjc 710d821f0f0Sjjc retval = lgrp_plat_process_srat(srat_ptr, 711d821f0f0Sjjc lgrp_plat_node_domain, lgrp_plat_cpu_node, 712d821f0f0Sjjc lgrp_plat_apic_ncpus, lgrp_plat_node_memory); 713d821f0f0Sjjc if (retval <= 0) { 714d821f0f0Sjjc lgrp_plat_srat_error = retval; 715d821f0f0Sjjc lgrp_plat_node_cnt = 1; 716d821f0f0Sjjc } else { 717d821f0f0Sjjc lgrp_plat_srat_error = 0; 718d821f0f0Sjjc lgrp_plat_node_cnt = retval; 719d821f0f0Sjjc } 720dae2fa37Sjjc } 7212e2c009bSjjc 7222e2c009bSjjc /* 723dae2fa37Sjjc * Try to use PCI config space registers on Opteron if there's an error 724dae2fa37Sjjc * processing CPU to APIC ID mapping or SRAT 7252e2c009bSjjc */ 726d821f0f0Sjjc if ((lgrp_plat_apic_ncpus <= 0 || lgrp_plat_srat_error != 0) && 727dae2fa37Sjjc is_opteron()) 7282e2c009bSjjc opt_get_numa_config(&lgrp_plat_node_cnt, &lgrp_plat_mem_intrlv, 7292e2c009bSjjc lgrp_plat_node_memory); 7302e2c009bSjjc 7312e2c009bSjjc /* 7322e2c009bSjjc * Don't bother to setup system for multiple lgroups and only use one 7332e2c009bSjjc * memory node when memory is interleaved between any nodes or there is 7342e2c009bSjjc * only one NUMA node 7352e2c009bSjjc * 7362e2c009bSjjc * NOTE: May need to change this for Dynamic Reconfiguration (DR) 7372e2c009bSjjc * when and if it happens for x86/x64 7382e2c009bSjjc */ 7392e2c009bSjjc if (lgrp_plat_mem_intrlv || lgrp_plat_node_cnt == 1) { 7402e2c009bSjjc lgrp_plat_node_cnt = max_mem_nodes = 1; 7412e2c009bSjjc (void) lgrp_topo_ht_limit_set(1); 7422e2c009bSjjc return; 7432e2c009bSjjc } 7442e2c009bSjjc 7452e2c009bSjjc /* 7462e2c009bSjjc * Leaf lgroups on x86/x64 architectures contain one physical 7472e2c009bSjjc * processor chip. Tune lgrp_expand_proc_thresh and 7482e2c009bSjjc * lgrp_expand_proc_diff so that lgrp_choose() will spread 7492e2c009bSjjc * things out aggressively. 7502e2c009bSjjc */ 7512e2c009bSjjc lgrp_expand_proc_thresh = LGRP_LOADAVG_THREAD_MAX / 2; 7522e2c009bSjjc lgrp_expand_proc_diff = 0; 7532e2c009bSjjc 7542e2c009bSjjc /* 7552e2c009bSjjc * There should be one memnode (physical page free list(s)) for 7562e2c009bSjjc * each node 7572e2c009bSjjc */ 7582e2c009bSjjc max_mem_nodes = lgrp_plat_node_cnt; 7592e2c009bSjjc 7602e2c009bSjjc /* 7615b7cf7f0Sjjc * Initialize min and max latency before reading SLIT or probing 7625b7cf7f0Sjjc */ 7635b7cf7f0Sjjc lgrp_plat_lat_stats.latency_min = -1; 7645b7cf7f0Sjjc lgrp_plat_lat_stats.latency_max = 0; 7655b7cf7f0Sjjc 7665b7cf7f0Sjjc /* 7672e2c009bSjjc * Determine how far each NUMA node is from each other by 7682e2c009bSjjc * reading ACPI System Locality Information Table (SLIT) if it 7692e2c009bSjjc * exists 7702e2c009bSjjc */ 7712e2c009bSjjc lgrp_plat_slit_error = lgrp_plat_process_slit(slit_ptr, 7722e2c009bSjjc lgrp_plat_node_cnt, lgrp_plat_node_memory, 7732e2c009bSjjc &lgrp_plat_lat_stats); 7742e2c009bSjjc if (lgrp_plat_slit_error == 0) 7752e2c009bSjjc return; 7762e2c009bSjjc 7772e2c009bSjjc /* 7782e2c009bSjjc * Probe to determine latency between NUMA nodes when SLIT 7792e2c009bSjjc * doesn't exist or make sense 7802e2c009bSjjc */ 7812e2c009bSjjc lgrp_plat_probe_flags |= LGRP_PLAT_PROBE_ENABLE; 7822e2c009bSjjc 7832e2c009bSjjc /* 7842e2c009bSjjc * Specify whether to probe using vendor ID register or page copy 7852e2c009bSjjc * if hasn't been specified already or is overspecified 7862e2c009bSjjc */ 7872e2c009bSjjc probe_op = lgrp_plat_probe_flags & 7882e2c009bSjjc (LGRP_PLAT_PROBE_PGCPY|LGRP_PLAT_PROBE_VENDOR); 7892e2c009bSjjc 7902e2c009bSjjc if (probe_op == 0 || 7912e2c009bSjjc probe_op == (LGRP_PLAT_PROBE_PGCPY|LGRP_PLAT_PROBE_VENDOR)) { 7922e2c009bSjjc lgrp_plat_probe_flags &= 7932e2c009bSjjc ~(LGRP_PLAT_PROBE_PGCPY|LGRP_PLAT_PROBE_VENDOR); 7942e2c009bSjjc if (is_opteron()) 7952e2c009bSjjc lgrp_plat_probe_flags |= 7962e2c009bSjjc LGRP_PLAT_PROBE_VENDOR; 7972e2c009bSjjc else 7982e2c009bSjjc lgrp_plat_probe_flags |= LGRP_PLAT_PROBE_PGCPY; 7992e2c009bSjjc } 8002e2c009bSjjc 8012e2c009bSjjc /* 8022e2c009bSjjc * Probing errors can mess up the lgroup topology and 8032e2c009bSjjc * force us fall back to a 2 level lgroup topology. 8042e2c009bSjjc * Here we bound how tall the lgroup topology can grow 8052e2c009bSjjc * in hopes of avoiding any anamolies in probing from 8062e2c009bSjjc * messing up the lgroup topology by limiting the 8072e2c009bSjjc * accuracy of the latency topology. 8082e2c009bSjjc * 8092e2c009bSjjc * Assume that nodes will at least be configured in a 8102e2c009bSjjc * ring, so limit height of lgroup topology to be less 8112e2c009bSjjc * than number of nodes on a system with 4 or more 8122e2c009bSjjc * nodes 8132e2c009bSjjc */ 8142e2c009bSjjc if (lgrp_plat_node_cnt >= 4 && lgrp_topo_ht_limit() == 8152e2c009bSjjc lgrp_topo_ht_limit_default()) 8162e2c009bSjjc (void) lgrp_topo_ht_limit_set(lgrp_plat_node_cnt - 1); 8172e2c009bSjjc #endif /* __xpv */ 8182e2c009bSjjc } 8192e2c009bSjjc 8202e2c009bSjjc 8212e2c009bSjjc /* 8222e2c009bSjjc * Return latency between "from" and "to" lgroups 8232e2c009bSjjc * 8242e2c009bSjjc * This latency number can only be used for relative comparison 8252e2c009bSjjc * between lgroups on the running system, cannot be used across platforms, 8262e2c009bSjjc * and may not reflect the actual latency. It is platform and implementation 8272e2c009bSjjc * specific, so platform gets to decide its value. It would be nice if the 8282e2c009bSjjc * number was at least proportional to make comparisons more meaningful though. 8292e2c009bSjjc */ 8302e2c009bSjjc /* ARGSUSED */ 8312e2c009bSjjc int 8322e2c009bSjjc lgrp_plat_latency(lgrp_handle_t from, lgrp_handle_t to) 8332e2c009bSjjc { 8342e2c009bSjjc lgrp_handle_t src, dest; 8352e2c009bSjjc int node; 8362e2c009bSjjc 8372e2c009bSjjc if (max_mem_nodes == 1) 8382e2c009bSjjc return (0); 8392e2c009bSjjc 8402e2c009bSjjc /* 8412e2c009bSjjc * Return max latency for root lgroup 8422e2c009bSjjc */ 8432e2c009bSjjc if (from == LGRP_DEFAULT_HANDLE || to == LGRP_DEFAULT_HANDLE) 8442e2c009bSjjc return (lgrp_plat_lat_stats.latency_max); 8452e2c009bSjjc 8462e2c009bSjjc src = from; 8472e2c009bSjjc dest = to; 8482e2c009bSjjc 8492e2c009bSjjc /* 8502e2c009bSjjc * Return 0 for nodes (lgroup platform handles) out of range 8512e2c009bSjjc */ 8522e2c009bSjjc if (src < 0 || src >= MAX_NODES || dest < 0 || dest >= MAX_NODES) 8532e2c009bSjjc return (0); 8542e2c009bSjjc 8552e2c009bSjjc /* 8562e2c009bSjjc * Probe from current CPU if its lgroup latencies haven't been set yet 8572e2c009bSjjc * and we are trying to get latency from current CPU to some node 8582e2c009bSjjc */ 8592e2c009bSjjc node = lgrp_plat_cpu_to_node(CPU, lgrp_plat_cpu_node); 8602e2c009bSjjc ASSERT(node >= 0 && node < lgrp_plat_node_cnt); 8612e2c009bSjjc if (lgrp_plat_lat_stats.latencies[src][src] == 0 && node == src) 8622e2c009bSjjc lgrp_plat_probe(); 8632e2c009bSjjc 8642e2c009bSjjc return (lgrp_plat_lat_stats.latencies[src][dest]); 8652e2c009bSjjc } 8662e2c009bSjjc 8672e2c009bSjjc 8682e2c009bSjjc /* 8692e2c009bSjjc * Platform-specific initialization 8702e2c009bSjjc */ 8712e2c009bSjjc void 8722e2c009bSjjc lgrp_plat_main_init(void) 8732e2c009bSjjc { 8742e2c009bSjjc int curnode; 8752e2c009bSjjc int ht_limit; 8762e2c009bSjjc int i; 8772e2c009bSjjc 8782e2c009bSjjc /* 8792e2c009bSjjc * Print a notice that MPO is disabled when memory is interleaved 8802e2c009bSjjc * across nodes....Would do this when it is discovered, but can't 8812e2c009bSjjc * because it happens way too early during boot.... 8822e2c009bSjjc */ 8832e2c009bSjjc if (lgrp_plat_mem_intrlv) 8842e2c009bSjjc cmn_err(CE_NOTE, 8852e2c009bSjjc "MPO disabled because memory is interleaved\n"); 8862e2c009bSjjc 8872e2c009bSjjc /* 8882e2c009bSjjc * Don't bother to do any probing if it is disabled, there is only one 8892e2c009bSjjc * node, or the height of the lgroup topology less than or equal to 2 8902e2c009bSjjc */ 8912e2c009bSjjc ht_limit = lgrp_topo_ht_limit(); 8922e2c009bSjjc if (!(lgrp_plat_probe_flags & LGRP_PLAT_PROBE_ENABLE) || 8932e2c009bSjjc max_mem_nodes == 1 || ht_limit <= 2) { 8942e2c009bSjjc /* 8952e2c009bSjjc * Setup lgroup latencies for 2 level lgroup topology 8962e2c009bSjjc * (ie. local and remote only) if they haven't been set yet 8972e2c009bSjjc */ 8982e2c009bSjjc if (ht_limit == 2 && lgrp_plat_lat_stats.latency_min == -1 && 8992e2c009bSjjc lgrp_plat_lat_stats.latency_max == 0) 9002e2c009bSjjc lgrp_plat_2level_setup(lgrp_plat_node_memory, 9012e2c009bSjjc &lgrp_plat_lat_stats); 9022e2c009bSjjc return; 9032e2c009bSjjc } 9042e2c009bSjjc 9052e2c009bSjjc if (lgrp_plat_probe_flags & LGRP_PLAT_PROBE_VENDOR) { 9062e2c009bSjjc /* 9072e2c009bSjjc * Should have been able to probe from CPU 0 when it was added 9082e2c009bSjjc * to lgroup hierarchy, but may not have been able to then 9092e2c009bSjjc * because it happens so early in boot that gethrtime() hasn't 9102e2c009bSjjc * been initialized. (:-( 9112e2c009bSjjc */ 9122e2c009bSjjc curnode = lgrp_plat_cpu_to_node(CPU, lgrp_plat_cpu_node); 9132e2c009bSjjc ASSERT(curnode >= 0 && curnode < lgrp_plat_node_cnt); 9142e2c009bSjjc if (lgrp_plat_lat_stats.latencies[curnode][curnode] == 0) 9152e2c009bSjjc lgrp_plat_probe(); 9162e2c009bSjjc 9172e2c009bSjjc return; 9182e2c009bSjjc } 9192e2c009bSjjc 9202e2c009bSjjc /* 9212e2c009bSjjc * When probing memory, use one page for every sample to determine 9222e2c009bSjjc * lgroup topology and taking multiple samples 9232e2c009bSjjc */ 9242e2c009bSjjc if (lgrp_plat_probe_mem_config.probe_memsize == 0) 9252e2c009bSjjc lgrp_plat_probe_mem_config.probe_memsize = PAGESIZE * 9262e2c009bSjjc lgrp_plat_probe_nsamples; 9272e2c009bSjjc 9282e2c009bSjjc /* 9292e2c009bSjjc * Map memory in each node needed for probing to determine latency 9302e2c009bSjjc * topology 9312e2c009bSjjc */ 9322e2c009bSjjc for (i = 0; i < lgrp_plat_node_cnt; i++) { 9332e2c009bSjjc int mnode; 9342e2c009bSjjc 9352e2c009bSjjc /* 9362e2c009bSjjc * Skip this node and leave its probe page NULL 9372e2c009bSjjc * if it doesn't have any memory 9382e2c009bSjjc */ 9392e2c009bSjjc mnode = plat_lgrphand_to_mem_node((lgrp_handle_t)i); 9402e2c009bSjjc if (!mem_node_config[mnode].exists) { 9412e2c009bSjjc lgrp_plat_probe_mem_config.probe_va[i] = NULL; 9422e2c009bSjjc continue; 9432e2c009bSjjc } 9442e2c009bSjjc 9452e2c009bSjjc /* 9462e2c009bSjjc * Allocate one kernel virtual page 9472e2c009bSjjc */ 9482e2c009bSjjc lgrp_plat_probe_mem_config.probe_va[i] = vmem_alloc(heap_arena, 9492e2c009bSjjc lgrp_plat_probe_mem_config.probe_memsize, VM_NOSLEEP); 9502e2c009bSjjc if (lgrp_plat_probe_mem_config.probe_va[i] == NULL) { 9512e2c009bSjjc cmn_err(CE_WARN, 9522e2c009bSjjc "lgrp_plat_main_init: couldn't allocate memory"); 9532e2c009bSjjc return; 9542e2c009bSjjc } 9552e2c009bSjjc 9562e2c009bSjjc /* 9572e2c009bSjjc * Get PFN for first page in each node 9582e2c009bSjjc */ 9592e2c009bSjjc lgrp_plat_probe_mem_config.probe_pfn[i] = 9602e2c009bSjjc mem_node_config[mnode].physbase; 9612e2c009bSjjc 9622e2c009bSjjc /* 9632e2c009bSjjc * Map virtual page to first page in node 9642e2c009bSjjc */ 9652e2c009bSjjc hat_devload(kas.a_hat, lgrp_plat_probe_mem_config.probe_va[i], 9662e2c009bSjjc lgrp_plat_probe_mem_config.probe_memsize, 9672e2c009bSjjc lgrp_plat_probe_mem_config.probe_pfn[i], 9682e2c009bSjjc PROT_READ | PROT_WRITE | HAT_PLAT_NOCACHE, 9692e2c009bSjjc HAT_LOAD_NOCONSIST); 9702e2c009bSjjc } 9712e2c009bSjjc 9722e2c009bSjjc /* 9732e2c009bSjjc * Probe from current CPU 9742e2c009bSjjc */ 9752e2c009bSjjc lgrp_plat_probe(); 9762e2c009bSjjc } 9772e2c009bSjjc 9782e2c009bSjjc 9792e2c009bSjjc /* 9802e2c009bSjjc * Return the maximum number of lgrps supported by the platform. 9812e2c009bSjjc * Before lgrp topology is known it returns an estimate based on the number of 9822e2c009bSjjc * nodes. Once topology is known it returns the actual maximim number of lgrps 9832e2c009bSjjc * created. Since x86/x64 doesn't support Dynamic Reconfiguration (DR) and 9842e2c009bSjjc * dynamic addition of new nodes, this number may not grow during system 9852e2c009bSjjc * lifetime (yet). 9862e2c009bSjjc */ 9872e2c009bSjjc int 9882e2c009bSjjc lgrp_plat_max_lgrps(void) 9892e2c009bSjjc { 9902e2c009bSjjc return (lgrp_topo_initialized ? 9912e2c009bSjjc lgrp_alloc_max + 1 : 9922e2c009bSjjc lgrp_plat_node_cnt * (lgrp_plat_node_cnt - 1) + 1); 9932e2c009bSjjc } 9942e2c009bSjjc 9952e2c009bSjjc 9962e2c009bSjjc /* 9972e2c009bSjjc * Return the number of free pages in an lgroup. 9982e2c009bSjjc * 9992e2c009bSjjc * For query of LGRP_MEM_SIZE_FREE, return the number of base pagesize 10002e2c009bSjjc * pages on freelists. For query of LGRP_MEM_SIZE_AVAIL, return the 10012e2c009bSjjc * number of allocatable base pagesize pages corresponding to the 10022e2c009bSjjc * lgroup (e.g. do not include page_t's, BOP_ALLOC()'ed memory, ..) 10032e2c009bSjjc * For query of LGRP_MEM_SIZE_INSTALL, return the amount of physical 10042e2c009bSjjc * memory installed, regardless of whether or not it's usable. 10052e2c009bSjjc */ 10062e2c009bSjjc pgcnt_t 10072e2c009bSjjc lgrp_plat_mem_size(lgrp_handle_t plathand, lgrp_mem_query_t query) 10082e2c009bSjjc { 10092e2c009bSjjc int mnode; 10102e2c009bSjjc pgcnt_t npgs = (pgcnt_t)0; 10112e2c009bSjjc extern struct memlist *phys_avail; 10122e2c009bSjjc extern struct memlist *phys_install; 10132e2c009bSjjc 10142e2c009bSjjc 10152e2c009bSjjc if (plathand == LGRP_DEFAULT_HANDLE) 10162e2c009bSjjc return (lgrp_plat_mem_size_default(plathand, query)); 10172e2c009bSjjc 10182e2c009bSjjc if (plathand != LGRP_NULL_HANDLE) { 10192e2c009bSjjc mnode = plat_lgrphand_to_mem_node(plathand); 10202e2c009bSjjc if (mnode >= 0 && mem_node_config[mnode].exists) { 10212e2c009bSjjc switch (query) { 10222e2c009bSjjc case LGRP_MEM_SIZE_FREE: 10232e2c009bSjjc npgs = MNODE_PGCNT(mnode); 10242e2c009bSjjc break; 10252e2c009bSjjc case LGRP_MEM_SIZE_AVAIL: 10262e2c009bSjjc npgs = mem_node_memlist_pages(mnode, 10272e2c009bSjjc phys_avail); 10282e2c009bSjjc break; 10292e2c009bSjjc case LGRP_MEM_SIZE_INSTALL: 10302e2c009bSjjc npgs = mem_node_memlist_pages(mnode, 10312e2c009bSjjc phys_install); 10322e2c009bSjjc break; 10332e2c009bSjjc default: 10342e2c009bSjjc break; 10352e2c009bSjjc } 10362e2c009bSjjc } 10372e2c009bSjjc } 10382e2c009bSjjc return (npgs); 10392e2c009bSjjc } 10402e2c009bSjjc 10412e2c009bSjjc 10422e2c009bSjjc /* 10432e2c009bSjjc * Return the platform handle of the lgroup that contains the physical memory 10442e2c009bSjjc * corresponding to the given page frame number 10452e2c009bSjjc */ 10462e2c009bSjjc /* ARGSUSED */ 10472e2c009bSjjc lgrp_handle_t 10482e2c009bSjjc lgrp_plat_pfn_to_hand(pfn_t pfn) 10492e2c009bSjjc { 10502e2c009bSjjc int mnode; 10512e2c009bSjjc 10522e2c009bSjjc if (max_mem_nodes == 1) 10532e2c009bSjjc return (LGRP_DEFAULT_HANDLE); 10542e2c009bSjjc 10552e2c009bSjjc if (pfn > physmax) 10562e2c009bSjjc return (LGRP_NULL_HANDLE); 10572e2c009bSjjc 10582e2c009bSjjc mnode = plat_pfn_to_mem_node(pfn); 10592e2c009bSjjc if (mnode < 0) 10602e2c009bSjjc return (LGRP_NULL_HANDLE); 10612e2c009bSjjc 10622e2c009bSjjc return (MEM_NODE_2_LGRPHAND(mnode)); 10632e2c009bSjjc } 10642e2c009bSjjc 10652e2c009bSjjc 10662e2c009bSjjc /* 10672e2c009bSjjc * Probe memory in each node from current CPU to determine latency topology 10682e2c009bSjjc * 10692e2c009bSjjc * The probing code will probe the vendor ID register on the Northbridge of 10702e2c009bSjjc * Opteron processors and probe memory for other processors by default. 10712e2c009bSjjc * 10722e2c009bSjjc * Since probing is inherently error prone, the code takes laps across all the 10732e2c009bSjjc * nodes probing from each node to each of the other nodes some number of 10742e2c009bSjjc * times. Furthermore, each node is probed some number of times before moving 10752e2c009bSjjc * onto the next one during each lap. The minimum latency gotten between nodes 10762e2c009bSjjc * is kept as the latency between the nodes. 10772e2c009bSjjc * 10782e2c009bSjjc * After all that, the probe times are adjusted by normalizing values that are 10792e2c009bSjjc * close to each other and local latencies are made the same. Lastly, the 10802e2c009bSjjc * latencies are verified to make sure that certain conditions are met (eg. 10812e2c009bSjjc * local < remote, latency(a, b) == latency(b, a), etc.). 10822e2c009bSjjc * 10832e2c009bSjjc * If any of the conditions aren't met, the code will export a NUMA 10842e2c009bSjjc * configuration with the local CPUs and memory given by the SRAT or PCI config 10852e2c009bSjjc * space registers and one remote memory latency since it can't tell exactly 10862e2c009bSjjc * how far each node is from each other. 10872e2c009bSjjc */ 10882e2c009bSjjc void 10892e2c009bSjjc lgrp_plat_probe(void) 10902e2c009bSjjc { 10912e2c009bSjjc int from; 10922e2c009bSjjc int i; 10932e2c009bSjjc lgrp_plat_latency_stats_t *lat_stats; 10942e2c009bSjjc hrtime_t probe_time; 10952e2c009bSjjc int to; 10962e2c009bSjjc 10972e2c009bSjjc if (!(lgrp_plat_probe_flags & LGRP_PLAT_PROBE_ENABLE) || 10982e2c009bSjjc max_mem_nodes == 1 || lgrp_topo_ht_limit() <= 2) 10992e2c009bSjjc return; 11002e2c009bSjjc 11012e2c009bSjjc /* 11022e2c009bSjjc * Determine ID of node containing current CPU 11032e2c009bSjjc */ 11042e2c009bSjjc from = lgrp_plat_cpu_to_node(CPU, lgrp_plat_cpu_node); 11052e2c009bSjjc ASSERT(from >= 0 && from < lgrp_plat_node_cnt); 11062e2c009bSjjc if (srat_ptr && lgrp_plat_srat_enable && !lgrp_plat_srat_error) 11072e2c009bSjjc ASSERT(lgrp_plat_node_domain[from].exists); 11082e2c009bSjjc 11092e2c009bSjjc /* 11102e2c009bSjjc * Don't need to probe if got times already 11112e2c009bSjjc */ 11122e2c009bSjjc lat_stats = &lgrp_plat_lat_stats; 11132e2c009bSjjc if (lat_stats->latencies[from][from] != 0) 11142e2c009bSjjc return; 11152e2c009bSjjc 11162e2c009bSjjc /* 11172e2c009bSjjc * Read vendor ID in Northbridge or read and write page(s) 11182e2c009bSjjc * in each node from current CPU and remember how long it takes, 11192e2c009bSjjc * so we can build latency topology of machine later. 11202e2c009bSjjc * This should approximate the memory latency between each node. 11212e2c009bSjjc */ 11222e2c009bSjjc for (i = 0; i < lgrp_plat_probe_nrounds; i++) { 11232e2c009bSjjc for (to = 0; to < lgrp_plat_node_cnt; to++) { 11242e2c009bSjjc /* 11252e2c009bSjjc * Get probe time and bail out if can't get it yet 11262e2c009bSjjc */ 11272e2c009bSjjc probe_time = lgrp_plat_probe_time(to, 11282e2c009bSjjc lgrp_plat_cpu_node, &lgrp_plat_probe_mem_config, 11292e2c009bSjjc &lgrp_plat_lat_stats, &lgrp_plat_probe_stats); 11302e2c009bSjjc if (probe_time == 0) 11312e2c009bSjjc return; 11322e2c009bSjjc 11332e2c009bSjjc /* 11342e2c009bSjjc * Keep lowest probe time as latency between nodes 11352e2c009bSjjc */ 11362e2c009bSjjc if (lat_stats->latencies[from][to] == 0 || 11372e2c009bSjjc probe_time < lat_stats->latencies[from][to]) 11382e2c009bSjjc lat_stats->latencies[from][to] = probe_time; 11392e2c009bSjjc 11402e2c009bSjjc /* 11412e2c009bSjjc * Update overall minimum and maximum probe times 11422e2c009bSjjc * across all nodes 11432e2c009bSjjc */ 11442e2c009bSjjc if (probe_time < lat_stats->latency_min || 11452e2c009bSjjc lat_stats->latency_min == -1) 11462e2c009bSjjc lat_stats->latency_min = probe_time; 11472e2c009bSjjc if (probe_time > lat_stats->latency_max) 11482e2c009bSjjc lat_stats->latency_max = probe_time; 11492e2c009bSjjc } 11502e2c009bSjjc } 11512e2c009bSjjc 11522e2c009bSjjc /* 11532e2c009bSjjc * - Fix up latencies such that local latencies are same, 11542e2c009bSjjc * latency(i, j) == latency(j, i), etc. (if possible) 11552e2c009bSjjc * 11562e2c009bSjjc * - Verify that latencies look ok 11572e2c009bSjjc * 11582e2c009bSjjc * - Fallback to just optimizing for local and remote if 11592e2c009bSjjc * latencies didn't look right 11602e2c009bSjjc */ 11612e2c009bSjjc lgrp_plat_latency_adjust(lgrp_plat_node_memory, &lgrp_plat_lat_stats, 11622e2c009bSjjc &lgrp_plat_probe_stats); 11632e2c009bSjjc lgrp_plat_probe_stats.probe_error_code = 11642e2c009bSjjc lgrp_plat_latency_verify(lgrp_plat_node_memory, 11652e2c009bSjjc &lgrp_plat_lat_stats); 11662e2c009bSjjc if (lgrp_plat_probe_stats.probe_error_code) 11672e2c009bSjjc lgrp_plat_2level_setup(lgrp_plat_node_memory, 11682e2c009bSjjc &lgrp_plat_lat_stats); 11692e2c009bSjjc } 11702e2c009bSjjc 11712e2c009bSjjc 11722e2c009bSjjc /* 11732e2c009bSjjc * Return platform handle for root lgroup 11742e2c009bSjjc */ 11752e2c009bSjjc lgrp_handle_t 11762e2c009bSjjc lgrp_plat_root_hand(void) 11772e2c009bSjjc { 11782e2c009bSjjc return (LGRP_DEFAULT_HANDLE); 11792e2c009bSjjc } 11802e2c009bSjjc 11812e2c009bSjjc 11822e2c009bSjjc /* 11832e2c009bSjjc * INTERNAL ROUTINES 11842e2c009bSjjc */ 11852e2c009bSjjc 11862e2c009bSjjc 11872e2c009bSjjc /* 11882e2c009bSjjc * Update CPU to node mapping for given CPU and proximity domain (and returns 11892e2c009bSjjc * negative numbers for errors and positive ones for success) 11902e2c009bSjjc */ 11912e2c009bSjjc static int 1192d821f0f0Sjjc lgrp_plat_cpu_node_update(node_domain_map_t *node_domain, int node_cnt, 1193dae2fa37Sjjc cpu_node_map_t *cpu_node, int nentries, uint32_t apicid, uint32_t domain) 11942e2c009bSjjc { 11952e2c009bSjjc uint_t i; 11962e2c009bSjjc int node; 11972e2c009bSjjc 11982e2c009bSjjc /* 11992e2c009bSjjc * Get node number for proximity domain 12002e2c009bSjjc */ 1201d821f0f0Sjjc node = lgrp_plat_domain_to_node(node_domain, node_cnt, domain); 12022e2c009bSjjc if (node == -1) { 1203d821f0f0Sjjc node = lgrp_plat_node_domain_update(node_domain, node_cnt, 1204d821f0f0Sjjc domain); 12052e2c009bSjjc if (node == -1) 12062e2c009bSjjc return (-1); 12072e2c009bSjjc } 12082e2c009bSjjc 12092e2c009bSjjc /* 1210dae2fa37Sjjc * Search for entry with given APIC ID and fill in its node and 1211dae2fa37Sjjc * proximity domain IDs (if they haven't been set already) 12122e2c009bSjjc */ 1213dae2fa37Sjjc for (i = 0; i < nentries; i++) { 12142e2c009bSjjc /* 1215dae2fa37Sjjc * Skip nonexistent entries and ones without matching APIC ID 12162e2c009bSjjc */ 1217dae2fa37Sjjc if (!cpu_node[i].exists || cpu_node[i].apicid != apicid) 1218dae2fa37Sjjc continue; 1219dae2fa37Sjjc 12202e2c009bSjjc /* 1221dae2fa37Sjjc * Just return if entry completely and correctly filled in 1222dae2fa37Sjjc * already 12232e2c009bSjjc */ 12242e2c009bSjjc if (cpu_node[i].prox_domain == domain && 12252e2c009bSjjc cpu_node[i].node == node) 12262e2c009bSjjc return (1); 12272e2c009bSjjc 12282e2c009bSjjc /* 1229dae2fa37Sjjc * Fill in node and proximity domain IDs 12302e2c009bSjjc */ 12312e2c009bSjjc cpu_node[i].prox_domain = domain; 12322e2c009bSjjc cpu_node[i].node = node; 1233dae2fa37Sjjc 12342e2c009bSjjc return (0); 12352e2c009bSjjc } 12362e2c009bSjjc 12372e2c009bSjjc /* 1238dae2fa37Sjjc * Return error when entry for APIC ID wasn't found in table 12392e2c009bSjjc */ 1240dae2fa37Sjjc return (-2); 12412e2c009bSjjc } 12422e2c009bSjjc 12432e2c009bSjjc 12442e2c009bSjjc /* 1245dae2fa37Sjjc * Get node ID for given CPU 12462e2c009bSjjc */ 12472e2c009bSjjc static int 12482e2c009bSjjc lgrp_plat_cpu_to_node(cpu_t *cp, cpu_node_map_t *cpu_node) 12492e2c009bSjjc { 1250dae2fa37Sjjc processorid_t cpuid; 12512e2c009bSjjc 12522e2c009bSjjc if (cp == NULL) 12532e2c009bSjjc return (-1); 12542e2c009bSjjc 1255dae2fa37Sjjc cpuid = cp->cpu_id; 1256dae2fa37Sjjc if (cpuid < 0 || cpuid >= max_ncpus) 1257dae2fa37Sjjc return (-1); 1258dae2fa37Sjjc 12592e2c009bSjjc /* 12602e2c009bSjjc * SRAT doesn't exist, isn't enabled, or there was an error processing 12612e2c009bSjjc * it, so return chip ID for Opteron and -1 otherwise. 12622e2c009bSjjc */ 12632e2c009bSjjc if (srat_ptr == NULL || !lgrp_plat_srat_enable || 12642e2c009bSjjc lgrp_plat_srat_error) { 12652e2c009bSjjc if (is_opteron()) 12662e2c009bSjjc return (pg_plat_hw_instance_id(cp, PGHW_CHIP)); 12672e2c009bSjjc return (-1); 12682e2c009bSjjc } 12692e2c009bSjjc 12702e2c009bSjjc /* 1271dae2fa37Sjjc * Return -1 when CPU to node ID mapping entry doesn't exist for given 1272dae2fa37Sjjc * CPU 12732e2c009bSjjc */ 1274dae2fa37Sjjc if (!cpu_node[cpuid].exists) 12752e2c009bSjjc return (-1); 1276dae2fa37Sjjc 1277dae2fa37Sjjc return (cpu_node[cpuid].node); 12782e2c009bSjjc } 12792e2c009bSjjc 12802e2c009bSjjc 12812e2c009bSjjc /* 12822e2c009bSjjc * Return node number for given proximity domain/system locality 12832e2c009bSjjc */ 12842e2c009bSjjc static int 1285d821f0f0Sjjc lgrp_plat_domain_to_node(node_domain_map_t *node_domain, int node_cnt, 1286d821f0f0Sjjc uint32_t domain) 12872e2c009bSjjc { 12882e2c009bSjjc uint_t node; 12892e2c009bSjjc uint_t start; 12902e2c009bSjjc 12912e2c009bSjjc /* 12922e2c009bSjjc * Hash proximity domain ID into node to domain mapping table (array), 12932e2c009bSjjc * search for entry with matching proximity domain ID, and return index 12942e2c009bSjjc * of matching entry as node ID. 12952e2c009bSjjc */ 1296d821f0f0Sjjc node = start = NODE_DOMAIN_HASH(domain, node_cnt); 12972e2c009bSjjc do { 12982e2c009bSjjc if (node_domain[node].prox_domain == domain && 12992e2c009bSjjc node_domain[node].exists) 13002e2c009bSjjc return (node); 1301d821f0f0Sjjc node = NODE_DOMAIN_HASH(node + 1, node_cnt); 13022e2c009bSjjc } while (node != start); 13032e2c009bSjjc return (-1); 13042e2c009bSjjc } 13052e2c009bSjjc 13062e2c009bSjjc 13072e2c009bSjjc /* 13082e2c009bSjjc * Latencies must be within 1/(2**LGRP_LAT_TOLERANCE_SHIFT) of each other to 13092e2c009bSjjc * be considered same 13102e2c009bSjjc */ 13112e2c009bSjjc #define LGRP_LAT_TOLERANCE_SHIFT 4 13122e2c009bSjjc 13132e2c009bSjjc int lgrp_plat_probe_lt_shift = LGRP_LAT_TOLERANCE_SHIFT; 13142e2c009bSjjc 13152e2c009bSjjc 13162e2c009bSjjc /* 13172e2c009bSjjc * Adjust latencies between nodes to be symmetric, normalize latencies between 13182e2c009bSjjc * any nodes that are within some tolerance to be same, and make local 13192e2c009bSjjc * latencies be same 13202e2c009bSjjc */ 13212e2c009bSjjc static void 13222e2c009bSjjc lgrp_plat_latency_adjust(node_phys_addr_map_t *node_memory, 13232e2c009bSjjc lgrp_plat_latency_stats_t *lat_stats, lgrp_plat_probe_stats_t *probe_stats) 13242e2c009bSjjc { 13252e2c009bSjjc int i; 13262e2c009bSjjc int j; 13272e2c009bSjjc int k; 13282e2c009bSjjc int l; 13292e2c009bSjjc u_longlong_t max; 13302e2c009bSjjc u_longlong_t min; 13312e2c009bSjjc u_longlong_t t; 13322e2c009bSjjc u_longlong_t t1; 13332e2c009bSjjc u_longlong_t t2; 13342e2c009bSjjc const lgrp_config_flag_t cflag = LGRP_CONFIG_LAT_CHANGE_ALL; 13352e2c009bSjjc int lat_corrected[MAX_NODES][MAX_NODES]; 13362e2c009bSjjc 13372e2c009bSjjc /* 13382e2c009bSjjc * Nothing to do when this is an UMA machine or don't have args needed 13392e2c009bSjjc */ 13402e2c009bSjjc if (max_mem_nodes == 1) 13412e2c009bSjjc return; 13422e2c009bSjjc 13432e2c009bSjjc ASSERT(node_memory != NULL && lat_stats != NULL && 13442e2c009bSjjc probe_stats != NULL); 13452e2c009bSjjc 13462e2c009bSjjc /* 13472e2c009bSjjc * Make sure that latencies are symmetric between any two nodes 13482e2c009bSjjc * (ie. latency(node0, node1) == latency(node1, node0)) 13492e2c009bSjjc */ 13502e2c009bSjjc for (i = 0; i < lgrp_plat_node_cnt; i++) { 13512e2c009bSjjc if (!node_memory[i].exists) 13522e2c009bSjjc continue; 13532e2c009bSjjc 13542e2c009bSjjc for (j = 0; j < lgrp_plat_node_cnt; j++) { 13552e2c009bSjjc if (!node_memory[j].exists) 13562e2c009bSjjc continue; 13572e2c009bSjjc 13582e2c009bSjjc t1 = lat_stats->latencies[i][j]; 13592e2c009bSjjc t2 = lat_stats->latencies[j][i]; 13602e2c009bSjjc 13612e2c009bSjjc if (t1 == 0 || t2 == 0 || t1 == t2) 13622e2c009bSjjc continue; 13632e2c009bSjjc 13642e2c009bSjjc /* 13652e2c009bSjjc * Latencies should be same 13662e2c009bSjjc * - Use minimum of two latencies which should be same 13672e2c009bSjjc * - Track suspect probe times not within tolerance of 13682e2c009bSjjc * min value 13692e2c009bSjjc * - Remember how much values are corrected by 13702e2c009bSjjc */ 13712e2c009bSjjc if (t1 > t2) { 13722e2c009bSjjc t = t2; 13732e2c009bSjjc probe_stats->probe_errors[i][j] += t1 - t2; 13742e2c009bSjjc if (t1 - t2 > t2 >> lgrp_plat_probe_lt_shift) { 13752e2c009bSjjc probe_stats->probe_suspect[i][j]++; 13762e2c009bSjjc probe_stats->probe_suspect[j][i]++; 13772e2c009bSjjc } 13782e2c009bSjjc } else if (t2 > t1) { 13792e2c009bSjjc t = t1; 13802e2c009bSjjc probe_stats->probe_errors[j][i] += t2 - t1; 13812e2c009bSjjc if (t2 - t1 > t1 >> lgrp_plat_probe_lt_shift) { 13822e2c009bSjjc probe_stats->probe_suspect[i][j]++; 13832e2c009bSjjc probe_stats->probe_suspect[j][i]++; 13842e2c009bSjjc } 13852e2c009bSjjc } 13862e2c009bSjjc 13872e2c009bSjjc lat_stats->latencies[i][j] = 13882e2c009bSjjc lat_stats->latencies[j][i] = t; 13892e2c009bSjjc lgrp_config(cflag, t1, t); 13902e2c009bSjjc lgrp_config(cflag, t2, t); 13912e2c009bSjjc } 13922e2c009bSjjc } 13932e2c009bSjjc 13942e2c009bSjjc /* 13952e2c009bSjjc * Keep track of which latencies get corrected 13962e2c009bSjjc */ 13972e2c009bSjjc for (i = 0; i < MAX_NODES; i++) 13982e2c009bSjjc for (j = 0; j < MAX_NODES; j++) 13992e2c009bSjjc lat_corrected[i][j] = 0; 14002e2c009bSjjc 14012e2c009bSjjc /* 14022e2c009bSjjc * For every two nodes, see whether there is another pair of nodes which 14032e2c009bSjjc * are about the same distance apart and make the latencies be the same 14042e2c009bSjjc * if they are close enough together 14052e2c009bSjjc */ 14062e2c009bSjjc for (i = 0; i < lgrp_plat_node_cnt; i++) { 14072e2c009bSjjc if (!node_memory[i].exists) 14082e2c009bSjjc continue; 14092e2c009bSjjc for (j = 0; j < lgrp_plat_node_cnt; j++) { 14102e2c009bSjjc if (!node_memory[j].exists) 14112e2c009bSjjc continue; 14122e2c009bSjjc /* 14132e2c009bSjjc * Pick one pair of nodes (i, j) 14142e2c009bSjjc * and get latency between them 14152e2c009bSjjc */ 14162e2c009bSjjc t1 = lat_stats->latencies[i][j]; 14172e2c009bSjjc 14182e2c009bSjjc /* 14192e2c009bSjjc * Skip this pair of nodes if there isn't a latency 14202e2c009bSjjc * for it yet 14212e2c009bSjjc */ 14222e2c009bSjjc if (t1 == 0) 14232e2c009bSjjc continue; 14242e2c009bSjjc 14252e2c009bSjjc for (k = 0; k < lgrp_plat_node_cnt; k++) { 14262e2c009bSjjc if (!node_memory[k].exists) 14272e2c009bSjjc continue; 14282e2c009bSjjc for (l = 0; l < lgrp_plat_node_cnt; l++) { 14292e2c009bSjjc if (!node_memory[l].exists) 14302e2c009bSjjc continue; 14312e2c009bSjjc /* 14322e2c009bSjjc * Pick another pair of nodes (k, l) 14332e2c009bSjjc * not same as (i, j) and get latency 14342e2c009bSjjc * between them 14352e2c009bSjjc */ 14362e2c009bSjjc if (k == i && l == j) 14372e2c009bSjjc continue; 14382e2c009bSjjc 14392e2c009bSjjc t2 = lat_stats->latencies[k][l]; 14402e2c009bSjjc 14412e2c009bSjjc /* 14422e2c009bSjjc * Skip this pair of nodes if there 14432e2c009bSjjc * isn't a latency for it yet 14442e2c009bSjjc */ 14452e2c009bSjjc 14462e2c009bSjjc if (t2 == 0) 14472e2c009bSjjc continue; 14482e2c009bSjjc 14492e2c009bSjjc /* 14502e2c009bSjjc * Skip nodes (k, l) if they already 14512e2c009bSjjc * have same latency as (i, j) or 14522e2c009bSjjc * their latency isn't close enough to 14532e2c009bSjjc * be considered/made the same 14542e2c009bSjjc */ 14552e2c009bSjjc if (t1 == t2 || (t1 > t2 && t1 - t2 > 14562e2c009bSjjc t1 >> lgrp_plat_probe_lt_shift) || 14572e2c009bSjjc (t2 > t1 && t2 - t1 > 14582e2c009bSjjc t2 >> lgrp_plat_probe_lt_shift)) 14592e2c009bSjjc continue; 14602e2c009bSjjc 14612e2c009bSjjc /* 14622e2c009bSjjc * Make latency(i, j) same as 14632e2c009bSjjc * latency(k, l), try to use latency 14642e2c009bSjjc * that has been adjusted already to get 14652e2c009bSjjc * more consistency (if possible), and 14662e2c009bSjjc * remember which latencies were 14672e2c009bSjjc * adjusted for next time 14682e2c009bSjjc */ 14692e2c009bSjjc if (lat_corrected[i][j]) { 14702e2c009bSjjc t = t1; 14712e2c009bSjjc lgrp_config(cflag, t2, t); 14722e2c009bSjjc t2 = t; 14732e2c009bSjjc } else if (lat_corrected[k][l]) { 14742e2c009bSjjc t = t2; 14752e2c009bSjjc lgrp_config(cflag, t1, t); 14762e2c009bSjjc t1 = t; 14772e2c009bSjjc } else { 14782e2c009bSjjc if (t1 > t2) 14792e2c009bSjjc t = t2; 14802e2c009bSjjc else 14812e2c009bSjjc t = t1; 14822e2c009bSjjc lgrp_config(cflag, t1, t); 14832e2c009bSjjc lgrp_config(cflag, t2, t); 14842e2c009bSjjc t1 = t2 = t; 14852e2c009bSjjc } 14862e2c009bSjjc 14872e2c009bSjjc lat_stats->latencies[i][j] = 14882e2c009bSjjc lat_stats->latencies[k][l] = t; 14892e2c009bSjjc 14902e2c009bSjjc lat_corrected[i][j] = 14912e2c009bSjjc lat_corrected[k][l] = 1; 14922e2c009bSjjc } 14932e2c009bSjjc } 14942e2c009bSjjc } 14952e2c009bSjjc } 14962e2c009bSjjc 14972e2c009bSjjc /* 14982e2c009bSjjc * Local latencies should be same 14992e2c009bSjjc * - Find min and max local latencies 15002e2c009bSjjc * - Make all local latencies be minimum 15012e2c009bSjjc */ 15022e2c009bSjjc min = -1; 15032e2c009bSjjc max = 0; 15042e2c009bSjjc for (i = 0; i < lgrp_plat_node_cnt; i++) { 15052e2c009bSjjc if (!node_memory[i].exists) 15062e2c009bSjjc continue; 15072e2c009bSjjc t = lat_stats->latencies[i][i]; 15082e2c009bSjjc if (t == 0) 15092e2c009bSjjc continue; 15102e2c009bSjjc if (min == -1 || t < min) 15112e2c009bSjjc min = t; 15122e2c009bSjjc if (t > max) 15132e2c009bSjjc max = t; 15142e2c009bSjjc } 15152e2c009bSjjc if (min != max) { 15162e2c009bSjjc for (i = 0; i < lgrp_plat_node_cnt; i++) { 15172e2c009bSjjc int local; 15182e2c009bSjjc 15192e2c009bSjjc if (!node_memory[i].exists) 15202e2c009bSjjc continue; 15212e2c009bSjjc 15222e2c009bSjjc local = lat_stats->latencies[i][i]; 15232e2c009bSjjc if (local == 0) 15242e2c009bSjjc continue; 15252e2c009bSjjc 15262e2c009bSjjc /* 15272e2c009bSjjc * Track suspect probe times that aren't within 15282e2c009bSjjc * tolerance of minimum local latency and how much 15292e2c009bSjjc * probe times are corrected by 15302e2c009bSjjc */ 15312e2c009bSjjc if (local - min > min >> lgrp_plat_probe_lt_shift) 15322e2c009bSjjc probe_stats->probe_suspect[i][i]++; 15332e2c009bSjjc 15342e2c009bSjjc probe_stats->probe_errors[i][i] += local - min; 15352e2c009bSjjc 15362e2c009bSjjc /* 15372e2c009bSjjc * Make local latencies be minimum 15382e2c009bSjjc */ 15392e2c009bSjjc lgrp_config(LGRP_CONFIG_LAT_CHANGE, i, min); 15402e2c009bSjjc lat_stats->latencies[i][i] = min; 15412e2c009bSjjc } 15422e2c009bSjjc } 15432e2c009bSjjc 15442e2c009bSjjc /* 15452e2c009bSjjc * Determine max probe time again since just adjusted latencies 15462e2c009bSjjc */ 15472e2c009bSjjc lat_stats->latency_max = 0; 15482e2c009bSjjc for (i = 0; i < lgrp_plat_node_cnt; i++) { 15492e2c009bSjjc if (!node_memory[i].exists) 15502e2c009bSjjc continue; 15512e2c009bSjjc for (j = 0; j < lgrp_plat_node_cnt; j++) { 15522e2c009bSjjc if (!node_memory[j].exists) 15532e2c009bSjjc continue; 15542e2c009bSjjc t = lat_stats->latencies[i][j]; 15552e2c009bSjjc if (t > lat_stats->latency_max) 15562e2c009bSjjc lat_stats->latency_max = t; 15572e2c009bSjjc } 15582e2c009bSjjc } 15592e2c009bSjjc } 15602e2c009bSjjc 15612e2c009bSjjc 15622e2c009bSjjc /* 15632e2c009bSjjc * Verify following about latencies between nodes: 15642e2c009bSjjc * 15652e2c009bSjjc * - Latencies should be symmetric (ie. latency(a, b) == latency(b, a)) 15662e2c009bSjjc * - Local latencies same 15672e2c009bSjjc * - Local < remote 15682e2c009bSjjc * - Number of latencies seen is reasonable 15692e2c009bSjjc * - Number of occurrences of a given latency should be more than 1 15702e2c009bSjjc * 15712e2c009bSjjc * Returns: 15722e2c009bSjjc * 0 Success 15732e2c009bSjjc * -1 Not symmetric 15742e2c009bSjjc * -2 Local latencies not same 15752e2c009bSjjc * -3 Local >= remote 15762e2c009bSjjc */ 15772e2c009bSjjc static int 15782e2c009bSjjc lgrp_plat_latency_verify(node_phys_addr_map_t *node_memory, 15792e2c009bSjjc lgrp_plat_latency_stats_t *lat_stats) 15802e2c009bSjjc { 15812e2c009bSjjc int i; 15822e2c009bSjjc int j; 15832e2c009bSjjc u_longlong_t t1; 15842e2c009bSjjc u_longlong_t t2; 15852e2c009bSjjc 15862e2c009bSjjc ASSERT(node_memory != NULL && lat_stats != NULL); 15872e2c009bSjjc 15882e2c009bSjjc /* 15892e2c009bSjjc * Nothing to do when this is an UMA machine, lgroup topology is 15902e2c009bSjjc * limited to 2 levels, or there aren't any probe times yet 15912e2c009bSjjc */ 15922e2c009bSjjc if (max_mem_nodes == 1 || lgrp_topo_levels < 2 || 15932e2c009bSjjc lat_stats->latencies[0][0] == 0) 15942e2c009bSjjc return (0); 15952e2c009bSjjc 15962e2c009bSjjc /* 15972e2c009bSjjc * Make sure that latencies are symmetric between any two nodes 15982e2c009bSjjc * (ie. latency(node0, node1) == latency(node1, node0)) 15992e2c009bSjjc */ 16002e2c009bSjjc for (i = 0; i < lgrp_plat_node_cnt; i++) { 16012e2c009bSjjc if (!node_memory[i].exists) 16022e2c009bSjjc continue; 16032e2c009bSjjc for (j = 0; j < lgrp_plat_node_cnt; j++) { 16042e2c009bSjjc if (!node_memory[j].exists) 16052e2c009bSjjc continue; 16062e2c009bSjjc t1 = lat_stats->latencies[i][j]; 16072e2c009bSjjc t2 = lat_stats->latencies[j][i]; 16082e2c009bSjjc 16092e2c009bSjjc if (t1 == 0 || t2 == 0 || t1 == t2) 16102e2c009bSjjc continue; 16112e2c009bSjjc 16122e2c009bSjjc return (-1); 16132e2c009bSjjc } 16142e2c009bSjjc } 16152e2c009bSjjc 16162e2c009bSjjc /* 16172e2c009bSjjc * Local latencies should be same 16182e2c009bSjjc */ 16192e2c009bSjjc t1 = lat_stats->latencies[0][0]; 16202e2c009bSjjc for (i = 1; i < lgrp_plat_node_cnt; i++) { 16212e2c009bSjjc if (!node_memory[i].exists) 16222e2c009bSjjc continue; 16232e2c009bSjjc 16242e2c009bSjjc t2 = lat_stats->latencies[i][i]; 16252e2c009bSjjc if (t2 == 0) 16262e2c009bSjjc continue; 16272e2c009bSjjc 16282e2c009bSjjc if (t1 == 0) { 16292e2c009bSjjc t1 = t2; 16302e2c009bSjjc continue; 16312e2c009bSjjc } 16322e2c009bSjjc 16332e2c009bSjjc if (t1 != t2) 16342e2c009bSjjc return (-2); 16352e2c009bSjjc } 16362e2c009bSjjc 16372e2c009bSjjc /* 16382e2c009bSjjc * Local latencies should be less than remote 16392e2c009bSjjc */ 16402e2c009bSjjc if (t1) { 16412e2c009bSjjc for (i = 0; i < lgrp_plat_node_cnt; i++) { 16422e2c009bSjjc if (!node_memory[i].exists) 16432e2c009bSjjc continue; 16442e2c009bSjjc for (j = 0; j < lgrp_plat_node_cnt; j++) { 16452e2c009bSjjc if (!node_memory[j].exists) 16462e2c009bSjjc continue; 16472e2c009bSjjc t2 = lat_stats->latencies[i][j]; 16482e2c009bSjjc if (i == j || t2 == 0) 16492e2c009bSjjc continue; 16502e2c009bSjjc 16512e2c009bSjjc if (t1 >= t2) 16522e2c009bSjjc return (-3); 16532e2c009bSjjc } 16542e2c009bSjjc } 16552e2c009bSjjc } 16562e2c009bSjjc 16572e2c009bSjjc return (0); 16582e2c009bSjjc } 16592e2c009bSjjc 16602e2c009bSjjc 16612e2c009bSjjc /* 16622e2c009bSjjc * Return the number of free, allocatable, or installed 16632e2c009bSjjc * pages in an lgroup 16642e2c009bSjjc * This is a copy of the MAX_MEM_NODES == 1 version of the routine 16652e2c009bSjjc * used when MPO is disabled (i.e. single lgroup) or this is the root lgroup 16662e2c009bSjjc */ 16672e2c009bSjjc /* ARGSUSED */ 16682e2c009bSjjc static pgcnt_t 16692e2c009bSjjc lgrp_plat_mem_size_default(lgrp_handle_t lgrphand, lgrp_mem_query_t query) 16702e2c009bSjjc { 16712e2c009bSjjc struct memlist *mlist; 16722e2c009bSjjc pgcnt_t npgs = 0; 16732e2c009bSjjc extern struct memlist *phys_avail; 16742e2c009bSjjc extern struct memlist *phys_install; 16752e2c009bSjjc 16762e2c009bSjjc switch (query) { 16772e2c009bSjjc case LGRP_MEM_SIZE_FREE: 16782e2c009bSjjc return ((pgcnt_t)freemem); 16792e2c009bSjjc case LGRP_MEM_SIZE_AVAIL: 16802e2c009bSjjc memlist_read_lock(); 16812e2c009bSjjc for (mlist = phys_avail; mlist; mlist = mlist->next) 16822e2c009bSjjc npgs += btop(mlist->size); 16832e2c009bSjjc memlist_read_unlock(); 16842e2c009bSjjc return (npgs); 16852e2c009bSjjc case LGRP_MEM_SIZE_INSTALL: 16862e2c009bSjjc memlist_read_lock(); 16872e2c009bSjjc for (mlist = phys_install; mlist; mlist = mlist->next) 16882e2c009bSjjc npgs += btop(mlist->size); 16892e2c009bSjjc memlist_read_unlock(); 16902e2c009bSjjc return (npgs); 16912e2c009bSjjc default: 16922e2c009bSjjc return ((pgcnt_t)0); 16932e2c009bSjjc } 16942e2c009bSjjc } 16952e2c009bSjjc 16962e2c009bSjjc 16972e2c009bSjjc /* 16982e2c009bSjjc * Update node to proximity domain mappings for given domain and return node ID 16992e2c009bSjjc */ 17002e2c009bSjjc static int 1701d821f0f0Sjjc lgrp_plat_node_domain_update(node_domain_map_t *node_domain, int node_cnt, 1702d821f0f0Sjjc uint32_t domain) 17032e2c009bSjjc { 17042e2c009bSjjc uint_t node; 17052e2c009bSjjc uint_t start; 17062e2c009bSjjc 17072e2c009bSjjc /* 17082e2c009bSjjc * Hash proximity domain ID into node to domain mapping table (array) 17092e2c009bSjjc * and add entry for it into first non-existent or matching entry found 17102e2c009bSjjc */ 1711d821f0f0Sjjc node = start = NODE_DOMAIN_HASH(domain, node_cnt); 17122e2c009bSjjc do { 17132e2c009bSjjc /* 17142e2c009bSjjc * Entry doesn't exist yet, so create one for this proximity 17152e2c009bSjjc * domain and return node ID which is index into mapping table. 17162e2c009bSjjc */ 17172e2c009bSjjc if (!node_domain[node].exists) { 17182e2c009bSjjc node_domain[node].exists = 1; 17192e2c009bSjjc node_domain[node].prox_domain = domain; 17202e2c009bSjjc return (node); 17212e2c009bSjjc } 17222e2c009bSjjc 17232e2c009bSjjc /* 17242e2c009bSjjc * Entry exists for this proximity domain already, so just 17252e2c009bSjjc * return node ID (index into table). 17262e2c009bSjjc */ 17272e2c009bSjjc if (node_domain[node].prox_domain == domain) 17282e2c009bSjjc return (node); 1729d821f0f0Sjjc node = NODE_DOMAIN_HASH(node + 1, node_cnt); 17302e2c009bSjjc } while (node != start); 17312e2c009bSjjc 17322e2c009bSjjc /* 17332e2c009bSjjc * Ran out of supported number of entries which shouldn't happen.... 17342e2c009bSjjc */ 17352e2c009bSjjc ASSERT(node != start); 17362e2c009bSjjc return (-1); 17372e2c009bSjjc } 17382e2c009bSjjc 17392e2c009bSjjc 17402e2c009bSjjc /* 17412e2c009bSjjc * Update node memory information for given proximity domain with specified 17422e2c009bSjjc * starting and ending physical address range (and return positive numbers for 17432e2c009bSjjc * success and negative ones for errors) 17442e2c009bSjjc */ 17452e2c009bSjjc static int 1746d821f0f0Sjjc lgrp_plat_node_memory_update(node_domain_map_t *node_domain, int node_cnt, 1747e9dd3ea3Sjjc node_phys_addr_map_t *node_memory, uint64_t start, uint64_t end, 17482e2c009bSjjc uint32_t domain) 17492e2c009bSjjc { 17502e2c009bSjjc int node; 17512e2c009bSjjc 17522e2c009bSjjc /* 17532e2c009bSjjc * Get node number for proximity domain 17542e2c009bSjjc */ 1755d821f0f0Sjjc node = lgrp_plat_domain_to_node(node_domain, node_cnt, domain); 17562e2c009bSjjc if (node == -1) { 1757d821f0f0Sjjc node = lgrp_plat_node_domain_update(node_domain, node_cnt, 1758d821f0f0Sjjc domain); 17592e2c009bSjjc if (node == -1) 17602e2c009bSjjc return (-1); 17612e2c009bSjjc } 17622e2c009bSjjc 17632e2c009bSjjc /* 17642e2c009bSjjc * Create entry in table for node if it doesn't exist 17652e2c009bSjjc */ 17662e2c009bSjjc if (!node_memory[node].exists) { 17672e2c009bSjjc node_memory[node].exists = 1; 17682e2c009bSjjc node_memory[node].start = btop(start); 17692e2c009bSjjc node_memory[node].end = btop(end); 17702e2c009bSjjc node_memory[node].prox_domain = domain; 17712e2c009bSjjc return (0); 17722e2c009bSjjc } 17732e2c009bSjjc 17742e2c009bSjjc /* 17752e2c009bSjjc * Entry already exists for this proximity domain 17762e2c009bSjjc * 17772e2c009bSjjc * There may be more than one SRAT memory entry for a domain, so we may 17782e2c009bSjjc * need to update existing start or end address for the node. 17792e2c009bSjjc */ 17802e2c009bSjjc if (node_memory[node].prox_domain == domain) { 17812e2c009bSjjc if (btop(start) < node_memory[node].start) 17822e2c009bSjjc node_memory[node].start = btop(start); 17832e2c009bSjjc if (btop(end) > node_memory[node].end) 17842e2c009bSjjc node_memory[node].end = btop(end); 17852e2c009bSjjc return (1); 17862e2c009bSjjc } 17872e2c009bSjjc return (-2); 17882e2c009bSjjc } 17892e2c009bSjjc 17902e2c009bSjjc 17912e2c009bSjjc /* 17922e2c009bSjjc * Return time needed to probe from current CPU to memory in given node 17932e2c009bSjjc */ 17942e2c009bSjjc static hrtime_t 17952e2c009bSjjc lgrp_plat_probe_time(int to, cpu_node_map_t *cpu_node, 17962e2c009bSjjc lgrp_plat_probe_mem_config_t *probe_mem_config, 17972e2c009bSjjc lgrp_plat_latency_stats_t *lat_stats, lgrp_plat_probe_stats_t *probe_stats) 17982e2c009bSjjc { 17992e2c009bSjjc caddr_t buf; 18002e2c009bSjjc hrtime_t elapsed; 18012e2c009bSjjc hrtime_t end; 18022e2c009bSjjc int from; 18032e2c009bSjjc int i; 18042e2c009bSjjc int ipl; 18052e2c009bSjjc hrtime_t max; 18062e2c009bSjjc hrtime_t min; 18072e2c009bSjjc hrtime_t start; 18082e2c009bSjjc extern int use_sse_pagecopy; 18092e2c009bSjjc 18102e2c009bSjjc /* 18112e2c009bSjjc * Determine ID of node containing current CPU 18122e2c009bSjjc */ 18132e2c009bSjjc from = lgrp_plat_cpu_to_node(CPU, cpu_node); 18142e2c009bSjjc ASSERT(from >= 0 && from < lgrp_plat_node_cnt); 18152e2c009bSjjc 18162e2c009bSjjc /* 18172e2c009bSjjc * Do common work for probing main memory 18182e2c009bSjjc */ 18192e2c009bSjjc if (lgrp_plat_probe_flags & LGRP_PLAT_PROBE_PGCPY) { 18202e2c009bSjjc /* 18212e2c009bSjjc * Skip probing any nodes without memory and 18222e2c009bSjjc * set probe time to 0 18232e2c009bSjjc */ 18242e2c009bSjjc if (probe_mem_config->probe_va[to] == NULL) { 18252e2c009bSjjc lat_stats->latencies[from][to] = 0; 18262e2c009bSjjc return (0); 18272e2c009bSjjc } 18282e2c009bSjjc 18292e2c009bSjjc /* 18302e2c009bSjjc * Invalidate caches once instead of once every sample 18312e2c009bSjjc * which should cut cost of probing by a lot 18322e2c009bSjjc */ 18332e2c009bSjjc probe_stats->flush_cost = gethrtime(); 18342e2c009bSjjc invalidate_cache(); 18352e2c009bSjjc probe_stats->flush_cost = gethrtime() - 18362e2c009bSjjc probe_stats->flush_cost; 18372e2c009bSjjc probe_stats->probe_cost_total += probe_stats->flush_cost; 18382e2c009bSjjc } 18392e2c009bSjjc 18402e2c009bSjjc /* 18412e2c009bSjjc * Probe from current CPU to given memory using specified operation 18422e2c009bSjjc * and take specified number of samples 18432e2c009bSjjc */ 18442e2c009bSjjc max = 0; 18452e2c009bSjjc min = -1; 18462e2c009bSjjc for (i = 0; i < lgrp_plat_probe_nsamples; i++) { 18472e2c009bSjjc probe_stats->probe_cost = gethrtime(); 18482e2c009bSjjc 18492e2c009bSjjc /* 18502e2c009bSjjc * Can't measure probe time if gethrtime() isn't working yet 18512e2c009bSjjc */ 18522e2c009bSjjc if (probe_stats->probe_cost == 0 && gethrtime() == 0) 18532e2c009bSjjc return (0); 18542e2c009bSjjc 18552e2c009bSjjc if (lgrp_plat_probe_flags & LGRP_PLAT_PROBE_VENDOR) { 18562e2c009bSjjc /* 18572e2c009bSjjc * Measure how long it takes to read vendor ID from 18582e2c009bSjjc * Northbridge 18592e2c009bSjjc */ 18602e2c009bSjjc elapsed = opt_probe_vendor(to, lgrp_plat_probe_nreads); 18612e2c009bSjjc } else { 18622e2c009bSjjc /* 18632e2c009bSjjc * Measure how long it takes to copy page 18642e2c009bSjjc * on top of itself 18652e2c009bSjjc */ 18662e2c009bSjjc buf = probe_mem_config->probe_va[to] + (i * PAGESIZE); 18672e2c009bSjjc 18682e2c009bSjjc kpreempt_disable(); 18692e2c009bSjjc ipl = splhigh(); 18702e2c009bSjjc start = gethrtime(); 18712e2c009bSjjc if (use_sse_pagecopy) 18722e2c009bSjjc hwblkpagecopy(buf, buf); 18732e2c009bSjjc else 18742e2c009bSjjc bcopy(buf, buf, PAGESIZE); 18752e2c009bSjjc end = gethrtime(); 18762e2c009bSjjc elapsed = end - start; 18772e2c009bSjjc splx(ipl); 18782e2c009bSjjc kpreempt_enable(); 18792e2c009bSjjc } 18802e2c009bSjjc 18812e2c009bSjjc probe_stats->probe_cost = gethrtime() - 18822e2c009bSjjc probe_stats->probe_cost; 18832e2c009bSjjc probe_stats->probe_cost_total += probe_stats->probe_cost; 18842e2c009bSjjc 18852e2c009bSjjc if (min == -1 || elapsed < min) 18862e2c009bSjjc min = elapsed; 18872e2c009bSjjc if (elapsed > max) 18882e2c009bSjjc max = elapsed; 18892e2c009bSjjc } 18902e2c009bSjjc 18912e2c009bSjjc /* 18922e2c009bSjjc * Update minimum and maximum probe times between 18932e2c009bSjjc * these two nodes 18942e2c009bSjjc */ 18952e2c009bSjjc if (min < probe_stats->probe_min[from][to] || 18962e2c009bSjjc probe_stats->probe_min[from][to] == 0) 18972e2c009bSjjc probe_stats->probe_min[from][to] = min; 18982e2c009bSjjc 18992e2c009bSjjc if (max > probe_stats->probe_max[from][to]) 19002e2c009bSjjc probe_stats->probe_max[from][to] = max; 19012e2c009bSjjc 19022e2c009bSjjc return (min); 19032e2c009bSjjc } 19042e2c009bSjjc 19052e2c009bSjjc 19062e2c009bSjjc /* 1907d821f0f0Sjjc * Read boot property with CPU to APIC ID array, fill in CPU to node ID 1908d821f0f0Sjjc * mapping table with APIC ID for each CPU, and return number of CPU APIC IDs. 1909dae2fa37Sjjc * 1910dae2fa37Sjjc * NOTE: This code assumes that CPU IDs are assigned in order that they appear 1911dae2fa37Sjjc * in in cpu_apicid_array boot property which is based on and follows 1912dae2fa37Sjjc * same ordering as processor list in ACPI MADT. If the code in 1913dae2fa37Sjjc * usr/src/uts/i86pc/io/pcplusmp/apic.c that reads MADT and assigns 1914dae2fa37Sjjc * CPU IDs ever changes, then this code will need to change too.... 1915dae2fa37Sjjc */ 1916dae2fa37Sjjc static int 1917d821f0f0Sjjc lgrp_plat_process_cpu_apicids(cpu_node_map_t *cpu_node) 1918dae2fa37Sjjc { 1919d821f0f0Sjjc int boot_prop_len; 1920dae2fa37Sjjc char *boot_prop_name = BP_CPU_APICID_ARRAY; 1921dae2fa37Sjjc uint8_t cpu_apicid_array[UINT8_MAX + 1]; 1922dae2fa37Sjjc int i; 1923d821f0f0Sjjc int n; 1924dae2fa37Sjjc 1925dae2fa37Sjjc /* 1926dae2fa37Sjjc * Nothing to do when no array to fill in or not enough CPUs 1927dae2fa37Sjjc */ 1928d821f0f0Sjjc if (cpu_node == NULL) 1929d821f0f0Sjjc return (-1); 1930dae2fa37Sjjc 1931dae2fa37Sjjc /* 1932dae2fa37Sjjc * Check length of property value 1933dae2fa37Sjjc */ 1934dae2fa37Sjjc boot_prop_len = BOP_GETPROPLEN(bootops, boot_prop_name); 1935d821f0f0Sjjc if (boot_prop_len <= 0 || boot_prop_len > sizeof (cpu_apicid_array)) 1936d821f0f0Sjjc return (-2); 1937d821f0f0Sjjc 1938d821f0f0Sjjc /* 1939d821f0f0Sjjc * Calculate number of entries in array and return when there's just 1940d821f0f0Sjjc * one CPU since that's not very interesting for NUMA 1941d821f0f0Sjjc */ 1942d821f0f0Sjjc n = boot_prop_len / sizeof (uint8_t); 1943d821f0f0Sjjc if (n == 1) 1944d821f0f0Sjjc return (-3); 1945dae2fa37Sjjc 1946dae2fa37Sjjc /* 1947dae2fa37Sjjc * Get CPU to APIC ID property value 1948dae2fa37Sjjc */ 1949dae2fa37Sjjc if (BOP_GETPROP(bootops, boot_prop_name, cpu_apicid_array) < 0) 1950d821f0f0Sjjc return (-4); 1951dae2fa37Sjjc 1952dae2fa37Sjjc /* 1953dae2fa37Sjjc * Fill in CPU to node ID mapping table with APIC ID for each CPU 1954dae2fa37Sjjc */ 1955d821f0f0Sjjc for (i = 0; i < n; i++) { 1956dae2fa37Sjjc cpu_node[i].exists = 1; 1957dae2fa37Sjjc cpu_node[i].apicid = cpu_apicid_array[i]; 1958dae2fa37Sjjc } 1959dae2fa37Sjjc 1960d821f0f0Sjjc /* 1961d821f0f0Sjjc * Return number of CPUs based on number of APIC IDs 1962d821f0f0Sjjc */ 1963d821f0f0Sjjc return (n); 1964dae2fa37Sjjc } 1965dae2fa37Sjjc 1966dae2fa37Sjjc 1967dae2fa37Sjjc /* 19682e2c009bSjjc * Read ACPI System Locality Information Table (SLIT) to determine how far each 19692e2c009bSjjc * NUMA node is from each other 19702e2c009bSjjc */ 19712e2c009bSjjc static int 19722e2c009bSjjc lgrp_plat_process_slit(struct slit *tp, uint_t node_cnt, 19732e2c009bSjjc node_phys_addr_map_t *node_memory, lgrp_plat_latency_stats_t *lat_stats) 19742e2c009bSjjc { 19752e2c009bSjjc int i; 19762e2c009bSjjc int j; 19772e2c009bSjjc int localities; 19782e2c009bSjjc hrtime_t max; 19792e2c009bSjjc hrtime_t min; 19802e2c009bSjjc int retval; 19812e2c009bSjjc uint8_t *slit_entries; 19822e2c009bSjjc 19832e2c009bSjjc if (tp == NULL || !lgrp_plat_slit_enable) 19842e2c009bSjjc return (1); 19852e2c009bSjjc 19862e2c009bSjjc if (lat_stats == NULL) 19872e2c009bSjjc return (2); 19882e2c009bSjjc 19892e2c009bSjjc localities = tp->number; 19902e2c009bSjjc if (localities != node_cnt) 19912e2c009bSjjc return (3); 19922e2c009bSjjc 19932e2c009bSjjc min = lat_stats->latency_min; 19942e2c009bSjjc max = lat_stats->latency_max; 19952e2c009bSjjc 19962e2c009bSjjc /* 19972e2c009bSjjc * Fill in latency matrix based on SLIT entries 19982e2c009bSjjc */ 19992e2c009bSjjc slit_entries = tp->entry; 20002e2c009bSjjc for (i = 0; i < localities; i++) { 20012e2c009bSjjc for (j = 0; j < localities; j++) { 20022e2c009bSjjc uint8_t latency; 20032e2c009bSjjc 20042e2c009bSjjc latency = slit_entries[(i * localities) + j]; 20052e2c009bSjjc lat_stats->latencies[i][j] = latency; 20065b7cf7f0Sjjc if (latency < min || min == -1) 20072e2c009bSjjc min = latency; 20082e2c009bSjjc if (latency > max) 20092e2c009bSjjc max = latency; 20102e2c009bSjjc } 20112e2c009bSjjc } 20122e2c009bSjjc 20132e2c009bSjjc /* 20142e2c009bSjjc * Verify that latencies/distances given in SLIT look reasonable 20152e2c009bSjjc */ 20162e2c009bSjjc retval = lgrp_plat_latency_verify(node_memory, lat_stats); 20172e2c009bSjjc 20182e2c009bSjjc if (retval) { 20192e2c009bSjjc /* 20202e2c009bSjjc * Reinitialize (zero) latency table since SLIT doesn't look 20212e2c009bSjjc * right 20222e2c009bSjjc */ 20232e2c009bSjjc for (i = 0; i < localities; i++) { 20242e2c009bSjjc for (j = 0; j < localities; j++) 20252e2c009bSjjc lat_stats->latencies[i][j] = 0; 20262e2c009bSjjc } 20272e2c009bSjjc } else { 20282e2c009bSjjc /* 20292e2c009bSjjc * Update min and max latencies seen since SLIT looks valid 20302e2c009bSjjc */ 20312e2c009bSjjc lat_stats->latency_min = min; 20322e2c009bSjjc lat_stats->latency_max = max; 20332e2c009bSjjc } 20342e2c009bSjjc 20352e2c009bSjjc return (retval); 20362e2c009bSjjc } 20372e2c009bSjjc 20382e2c009bSjjc 20392e2c009bSjjc /* 20402e2c009bSjjc * Read ACPI System Resource Affinity Table (SRAT) to determine which CPUs 2041d821f0f0Sjjc * and memory are local to each other in the same NUMA node and return number 2042d821f0f0Sjjc * of nodes 20432e2c009bSjjc */ 20442e2c009bSjjc static int 2045d821f0f0Sjjc lgrp_plat_process_srat(struct srat *tp, node_domain_map_t *node_domain, 2046d821f0f0Sjjc cpu_node_map_t *cpu_node, int cpu_count, node_phys_addr_map_t *node_memory) 20472e2c009bSjjc { 20485b7cf7f0Sjjc struct srat_item *srat_end; 20492e2c009bSjjc int i; 20502e2c009bSjjc struct srat_item *item; 2051d821f0f0Sjjc int node_cnt; 2052dae2fa37Sjjc int proc_entry_count; 20532e2c009bSjjc 2054d821f0f0Sjjc /* 2055d821f0f0Sjjc * Nothing to do when no SRAT or disabled 2056d821f0f0Sjjc */ 20572e2c009bSjjc if (tp == NULL || !lgrp_plat_srat_enable) 2058d821f0f0Sjjc return (-1); 20592e2c009bSjjc 20602e2c009bSjjc /* 20612e2c009bSjjc * Determine number of nodes by counting number of proximity domains in 2062d821f0f0Sjjc * SRAT and return if number of nodes is 1 or less since don't need to 2063d821f0f0Sjjc * read SRAT then 20642e2c009bSjjc */ 2065d821f0f0Sjjc node_cnt = lgrp_plat_srat_domains(tp); 2066d821f0f0Sjjc if (node_cnt == 1) 2067d821f0f0Sjjc return (1); 2068d821f0f0Sjjc else if (node_cnt <= 0) 2069d821f0f0Sjjc return (-2); 20702e2c009bSjjc 20712e2c009bSjjc /* 20722e2c009bSjjc * Walk through SRAT, examining each CPU and memory entry to determine 20732e2c009bSjjc * which CPUs and memory belong to which node. 20742e2c009bSjjc */ 20752e2c009bSjjc item = tp->list; 20765b7cf7f0Sjjc srat_end = (struct srat_item *)(tp->hdr.len + (uintptr_t)tp); 2077dae2fa37Sjjc proc_entry_count = 0; 20785b7cf7f0Sjjc while (item < srat_end) { 20792e2c009bSjjc uint32_t apic_id; 20802e2c009bSjjc uint32_t domain; 20812e2c009bSjjc uint64_t end; 20822e2c009bSjjc uint64_t length; 20832e2c009bSjjc uint64_t start; 20842e2c009bSjjc 20852e2c009bSjjc switch (item->type) { 20862e2c009bSjjc case SRAT_PROCESSOR: /* CPU entry */ 20872e2c009bSjjc if (!(item->i.p.flags & SRAT_ENABLED) || 20882e2c009bSjjc cpu_node == NULL) 20892e2c009bSjjc break; 20902e2c009bSjjc 20912e2c009bSjjc /* 20922e2c009bSjjc * Calculate domain (node) ID and fill in APIC ID to 20932e2c009bSjjc * domain/node mapping table 20942e2c009bSjjc */ 20952e2c009bSjjc domain = item->i.p.domain1; 20962e2c009bSjjc for (i = 0; i < 3; i++) { 20972e2c009bSjjc domain += item->i.p.domain2[i] << 20982e2c009bSjjc ((i + 1) * 8); 20992e2c009bSjjc } 21002e2c009bSjjc apic_id = item->i.p.apic_id; 21012e2c009bSjjc 2102d821f0f0Sjjc if (lgrp_plat_cpu_node_update(node_domain, node_cnt, 2103d821f0f0Sjjc cpu_node, cpu_count, apic_id, domain) < 0) 2104d821f0f0Sjjc return (-3); 2105dae2fa37Sjjc 2106dae2fa37Sjjc proc_entry_count++; 21072e2c009bSjjc break; 21082e2c009bSjjc 21092e2c009bSjjc case SRAT_MEMORY: /* memory entry */ 21102e2c009bSjjc if (!(item->i.m.flags & SRAT_ENABLED) || 21112e2c009bSjjc node_memory == NULL) 21122e2c009bSjjc break; 21132e2c009bSjjc 21142e2c009bSjjc /* 21152e2c009bSjjc * Get domain (node) ID and fill in domain/node 21162e2c009bSjjc * to memory mapping table 21172e2c009bSjjc */ 21182e2c009bSjjc domain = item->i.m.domain; 21192e2c009bSjjc start = item->i.m.base_addr; 21202e2c009bSjjc length = item->i.m.len; 21212e2c009bSjjc end = start + length - 1; 21222e2c009bSjjc 2123d821f0f0Sjjc if (lgrp_plat_node_memory_update(node_domain, node_cnt, 21242e2c009bSjjc node_memory, start, end, domain) < 0) 2125d821f0f0Sjjc return (-4); 21262e2c009bSjjc break; 2127*b6917abeSmishra case SRAT_X2APIC: /* x2apic CPU entry */ 2128*b6917abeSmishra if (!(item->i.xp.flags & SRAT_ENABLED) || 2129*b6917abeSmishra cpu_node == NULL) 2130*b6917abeSmishra break; 2131*b6917abeSmishra 2132*b6917abeSmishra /* 2133*b6917abeSmishra * Calculate domain (node) ID and fill in APIC ID to 2134*b6917abeSmishra * domain/node mapping table 2135*b6917abeSmishra */ 2136*b6917abeSmishra domain = item->i.xp.domain; 2137*b6917abeSmishra apic_id = item->i.xp.x2apic_id; 2138*b6917abeSmishra 2139*b6917abeSmishra if (lgrp_plat_cpu_node_update(node_domain, node_cnt, 2140*b6917abeSmishra cpu_node, cpu_count, apic_id, domain) < 0) 2141*b6917abeSmishra return (-3); 2142*b6917abeSmishra 2143*b6917abeSmishra proc_entry_count++; 2144*b6917abeSmishra break; 21452e2c009bSjjc 21462e2c009bSjjc default: 21472e2c009bSjjc break; 21482e2c009bSjjc } 21492e2c009bSjjc 21502e2c009bSjjc item = (struct srat_item *)((uintptr_t)item + item->len); 21512e2c009bSjjc } 2152dae2fa37Sjjc 2153dae2fa37Sjjc /* 2154dae2fa37Sjjc * Should have seen at least as many SRAT processor entries as CPUs 2155dae2fa37Sjjc */ 2156d821f0f0Sjjc if (proc_entry_count < cpu_count) 2157d821f0f0Sjjc return (-5); 2158dae2fa37Sjjc 2159d821f0f0Sjjc return (node_cnt); 21602e2c009bSjjc } 21612e2c009bSjjc 21622e2c009bSjjc 21632e2c009bSjjc /* 21642e2c009bSjjc * Return number of proximity domains given in ACPI SRAT 21652e2c009bSjjc */ 21662e2c009bSjjc static int 21672e2c009bSjjc lgrp_plat_srat_domains(struct srat *tp) 21682e2c009bSjjc { 21692e2c009bSjjc int domain_cnt; 21702e2c009bSjjc struct srat_item *end; 21712e2c009bSjjc int i; 21722e2c009bSjjc struct srat_item *item; 21732e2c009bSjjc node_domain_map_t node_domain[MAX_NODES]; 21742e2c009bSjjc 21752e2c009bSjjc 21762e2c009bSjjc if (tp == NULL || !lgrp_plat_srat_enable) 21772e2c009bSjjc return (1); 21782e2c009bSjjc 21792e2c009bSjjc /* 21802e2c009bSjjc * Walk through SRAT, examining each CPU and memory entry to determine 21812e2c009bSjjc * proximity domain ID for each. 21822e2c009bSjjc */ 21832e2c009bSjjc domain_cnt = 0; 21842e2c009bSjjc item = tp->list; 21852e2c009bSjjc end = (struct srat_item *)(tp->hdr.len + (uintptr_t)tp); 21862e2c009bSjjc bzero(node_domain, MAX_NODES * sizeof (node_domain_map_t)); 21872e2c009bSjjc while (item < end) { 21882e2c009bSjjc uint32_t domain; 21892e2c009bSjjc boolean_t overflow; 21902e2c009bSjjc uint_t start; 21912e2c009bSjjc 21922e2c009bSjjc switch (item->type) { 21932e2c009bSjjc case SRAT_PROCESSOR: /* CPU entry */ 21942e2c009bSjjc if (!(item->i.p.flags & SRAT_ENABLED)) 21952e2c009bSjjc break; 21962e2c009bSjjc domain = item->i.p.domain1; 21972e2c009bSjjc for (i = 0; i < 3; i++) { 21982e2c009bSjjc domain += item->i.p.domain2[i] << 21992e2c009bSjjc ((i + 1) * 8); 22002e2c009bSjjc } 22012e2c009bSjjc break; 22022e2c009bSjjc 22032e2c009bSjjc case SRAT_MEMORY: /* memory entry */ 22042e2c009bSjjc if (!(item->i.m.flags & SRAT_ENABLED)) 22052e2c009bSjjc break; 22062e2c009bSjjc domain = item->i.m.domain; 22072e2c009bSjjc break; 22082e2c009bSjjc 2209*b6917abeSmishra case SRAT_X2APIC: /* x2apic CPU entry */ 2210*b6917abeSmishra if (!(item->i.xp.flags & SRAT_ENABLED)) 2211*b6917abeSmishra break; 2212*b6917abeSmishra domain = item->i.xp.domain; 2213*b6917abeSmishra break; 2214*b6917abeSmishra 22152e2c009bSjjc default: 22162e2c009bSjjc break; 22172e2c009bSjjc } 22182e2c009bSjjc 22192e2c009bSjjc /* 22202e2c009bSjjc * Count and keep track of which proximity domain IDs seen 22212e2c009bSjjc */ 22222e2c009bSjjc start = i = domain % MAX_NODES; 22232e2c009bSjjc overflow = B_TRUE; 22242e2c009bSjjc do { 22252e2c009bSjjc /* 22262e2c009bSjjc * Create entry for proximity domain and increment 22272e2c009bSjjc * count when no entry exists where proximity domain 22282e2c009bSjjc * hashed 22292e2c009bSjjc */ 22302e2c009bSjjc if (!node_domain[i].exists) { 22312e2c009bSjjc node_domain[i].exists = 1; 22322e2c009bSjjc node_domain[i].prox_domain = domain; 22332e2c009bSjjc domain_cnt++; 22342e2c009bSjjc overflow = B_FALSE; 22352e2c009bSjjc break; 22362e2c009bSjjc } 22372e2c009bSjjc 22382e2c009bSjjc /* 22392e2c009bSjjc * Nothing to do when proximity domain seen already 22402e2c009bSjjc * and its entry exists 22412e2c009bSjjc */ 22422e2c009bSjjc if (node_domain[i].prox_domain == domain) { 22432e2c009bSjjc overflow = B_FALSE; 22442e2c009bSjjc break; 22452e2c009bSjjc } 22462e2c009bSjjc 22472e2c009bSjjc /* 22482e2c009bSjjc * Entry exists where proximity domain hashed, but for 22492e2c009bSjjc * different proximity domain so keep search for empty 22502e2c009bSjjc * slot to put it or matching entry whichever comes 22512e2c009bSjjc * first. 22522e2c009bSjjc */ 22532e2c009bSjjc i = (i + 1) % MAX_NODES; 22542e2c009bSjjc } while (i != start); 22552e2c009bSjjc 22562e2c009bSjjc /* 22572e2c009bSjjc * Didn't find empty or matching entry which means have more 22582e2c009bSjjc * proximity domains than supported nodes (:-( 22592e2c009bSjjc */ 22602e2c009bSjjc ASSERT(overflow != B_TRUE); 22612e2c009bSjjc if (overflow == B_TRUE) 22622e2c009bSjjc return (-1); 22632e2c009bSjjc 22642e2c009bSjjc item = (struct srat_item *)((uintptr_t)item + item->len); 22652e2c009bSjjc } 22662e2c009bSjjc return (domain_cnt); 22672e2c009bSjjc } 22682e2c009bSjjc 22692e2c009bSjjc 22702e2c009bSjjc /* 22712e2c009bSjjc * Set lgroup latencies for 2 level lgroup topology 22722e2c009bSjjc */ 22732e2c009bSjjc static void 22742e2c009bSjjc lgrp_plat_2level_setup(node_phys_addr_map_t *node_memory, 22752e2c009bSjjc lgrp_plat_latency_stats_t *lat_stats) 22762e2c009bSjjc { 22772e2c009bSjjc int i; 22782e2c009bSjjc 22792e2c009bSjjc ASSERT(node_memory != NULL && lat_stats != NULL); 22802e2c009bSjjc 22812e2c009bSjjc if (lgrp_plat_node_cnt >= 4) 22822e2c009bSjjc cmn_err(CE_NOTE, 22832e2c009bSjjc "MPO only optimizing for local and remote\n"); 22842e2c009bSjjc for (i = 0; i < lgrp_plat_node_cnt; i++) { 22852e2c009bSjjc int j; 22862e2c009bSjjc 22872e2c009bSjjc if (!node_memory[i].exists) 22882e2c009bSjjc continue; 22892e2c009bSjjc for (j = 0; j < lgrp_plat_node_cnt; j++) { 22902e2c009bSjjc if (!node_memory[j].exists) 22912e2c009bSjjc continue; 22922e2c009bSjjc if (i == j) 22932e2c009bSjjc lat_stats->latencies[i][j] = 2; 22942e2c009bSjjc else 22952e2c009bSjjc lat_stats->latencies[i][j] = 3; 22962e2c009bSjjc } 22972e2c009bSjjc } 22982e2c009bSjjc lat_stats->latency_min = 2; 22992e2c009bSjjc lat_stats->latency_max = 3; 23002e2c009bSjjc lgrp_config(LGRP_CONFIG_FLATTEN, 2, 0); 23012e2c009bSjjc } 23022e2c009bSjjc 23032e2c009bSjjc 23042e2c009bSjjc /* 23052e2c009bSjjc * The following Opteron specific constants, macros, types, and routines define 23062e2c009bSjjc * PCI configuration space registers and how to read them to determine the NUMA 23072e2c009bSjjc * configuration of *supported* Opteron processors. They provide the same 23082e2c009bSjjc * information that may be gotten from the ACPI System Resource Affinity Table 23092e2c009bSjjc * (SRAT) if it exists on the machine of interest. 23102e2c009bSjjc * 23112e2c009bSjjc * The AMD BIOS and Kernel Developer's Guide (BKDG) for the processor family 23122e2c009bSjjc * of interest describes all of these registers and their contents. The main 23132e2c009bSjjc * registers used by this code to determine the NUMA configuration of the 23142e2c009bSjjc * machine are the node ID register for the number of NUMA nodes and the DRAM 23152e2c009bSjjc * address map registers for the physical address range of each node. 23162e2c009bSjjc * 23172e2c009bSjjc * NOTE: The format and how to determine the NUMA configuration using PCI 23182e2c009bSjjc * config space registers may change or may not be supported in future 23192e2c009bSjjc * Opteron processor families. 23207c478bd9Sstevel@tonic-gate */ 23217c478bd9Sstevel@tonic-gate 23227c478bd9Sstevel@tonic-gate /* 23237c478bd9Sstevel@tonic-gate * How many bits to shift Opteron DRAM Address Map base and limit registers 23247c478bd9Sstevel@tonic-gate * to get actual value 23257c478bd9Sstevel@tonic-gate */ 2326f78a91cdSjjc #define OPT_DRAMADDR_HI_LSHIFT_ADDR 40 /* shift left for address */ 2327f78a91cdSjjc #define OPT_DRAMADDR_LO_LSHIFT_ADDR 8 /* shift left for address */ 23287c478bd9Sstevel@tonic-gate 2329f78a91cdSjjc #define OPT_DRAMADDR_HI_MASK_ADDR 0x000000FF /* address bits 47-40 */ 2330f78a91cdSjjc #define OPT_DRAMADDR_LO_MASK_ADDR 0xFFFF0000 /* address bits 39-24 */ 2331f78a91cdSjjc 2332f78a91cdSjjc #define OPT_DRAMADDR_LO_MASK_OFF 0xFFFFFF /* offset for address */ 2333f78a91cdSjjc 2334f78a91cdSjjc /* 2335f78a91cdSjjc * Macros to derive addresses from Opteron DRAM Address Map registers 2336f78a91cdSjjc */ 2337f78a91cdSjjc #define OPT_DRAMADDR_HI(reg) \ 2338f78a91cdSjjc (((u_longlong_t)reg & OPT_DRAMADDR_HI_MASK_ADDR) << \ 2339f78a91cdSjjc OPT_DRAMADDR_HI_LSHIFT_ADDR) 2340f78a91cdSjjc 2341f78a91cdSjjc #define OPT_DRAMADDR_LO(reg) \ 2342f78a91cdSjjc (((u_longlong_t)reg & OPT_DRAMADDR_LO_MASK_ADDR) << \ 2343f78a91cdSjjc OPT_DRAMADDR_LO_LSHIFT_ADDR) 2344f78a91cdSjjc 2345f78a91cdSjjc #define OPT_DRAMADDR(high, low) \ 2346f78a91cdSjjc (OPT_DRAMADDR_HI(high) | OPT_DRAMADDR_LO(low)) 23477c478bd9Sstevel@tonic-gate 23487c478bd9Sstevel@tonic-gate /* 23497c478bd9Sstevel@tonic-gate * Bit masks defining what's in Opteron DRAM Address Map base register 23507c478bd9Sstevel@tonic-gate */ 2351f78a91cdSjjc #define OPT_DRAMBASE_LO_MASK_RE 0x1 /* read enable */ 2352f78a91cdSjjc #define OPT_DRAMBASE_LO_MASK_WE 0x2 /* write enable */ 2353f78a91cdSjjc #define OPT_DRAMBASE_LO_MASK_INTRLVEN 0x700 /* interleave */ 23547c478bd9Sstevel@tonic-gate 23557c478bd9Sstevel@tonic-gate /* 23567c478bd9Sstevel@tonic-gate * Bit masks defining what's in Opteron DRAM Address Map limit register 23577c478bd9Sstevel@tonic-gate */ 2358f78a91cdSjjc #define OPT_DRAMLIMIT_LO_MASK_DSTNODE 0x7 /* destination node */ 2359f78a91cdSjjc #define OPT_DRAMLIMIT_LO_MASK_INTRLVSEL 0x700 /* interleave select */ 23607c478bd9Sstevel@tonic-gate 23617c478bd9Sstevel@tonic-gate 23627c478bd9Sstevel@tonic-gate /* 23637c478bd9Sstevel@tonic-gate * Opteron Node ID register in PCI configuration space contains 23647c478bd9Sstevel@tonic-gate * number of nodes in system, etc. for Opteron K8. The following 23657c478bd9Sstevel@tonic-gate * constants and macros define its contents, structure, and access. 23667c478bd9Sstevel@tonic-gate */ 23677c478bd9Sstevel@tonic-gate 23687c478bd9Sstevel@tonic-gate /* 23697c478bd9Sstevel@tonic-gate * Bit masks defining what's in Opteron Node ID register 23707c478bd9Sstevel@tonic-gate */ 23717c478bd9Sstevel@tonic-gate #define OPT_NODE_MASK_ID 0x7 /* node ID */ 23727c478bd9Sstevel@tonic-gate #define OPT_NODE_MASK_CNT 0x70 /* node count */ 23737c478bd9Sstevel@tonic-gate #define OPT_NODE_MASK_IONODE 0x700 /* Hypertransport I/O hub node ID */ 23747c478bd9Sstevel@tonic-gate #define OPT_NODE_MASK_LCKNODE 0x7000 /* lock controller node ID */ 23757c478bd9Sstevel@tonic-gate #define OPT_NODE_MASK_CPUCNT 0xF0000 /* CPUs in system (0 means 1 CPU) */ 23767c478bd9Sstevel@tonic-gate 23777c478bd9Sstevel@tonic-gate /* 23787c478bd9Sstevel@tonic-gate * How many bits in Opteron Node ID register to shift right to get actual value 23797c478bd9Sstevel@tonic-gate */ 23807c478bd9Sstevel@tonic-gate #define OPT_NODE_RSHIFT_CNT 0x4 /* shift right for node count value */ 23817c478bd9Sstevel@tonic-gate 23827c478bd9Sstevel@tonic-gate /* 23837c478bd9Sstevel@tonic-gate * Macros to get values from Opteron Node ID register 23847c478bd9Sstevel@tonic-gate */ 23857c478bd9Sstevel@tonic-gate #define OPT_NODE_CNT(reg) \ 23867c478bd9Sstevel@tonic-gate ((reg & OPT_NODE_MASK_CNT) >> OPT_NODE_RSHIFT_CNT) 23877c478bd9Sstevel@tonic-gate 2388f78a91cdSjjc /* 2389f78a91cdSjjc * Macro to setup PCI Extended Configuration Space (ECS) address to give to 2390f78a91cdSjjc * "in/out" instructions 2391f78a91cdSjjc * 2392f78a91cdSjjc * NOTE: Should only be used in lgrp_plat_init() before MMIO setup because any 2393f78a91cdSjjc * other uses should just do MMIO to access PCI ECS. 2394f78a91cdSjjc * Must enable special bit in Northbridge Configuration Register on 2395f78a91cdSjjc * Greyhound for extended CF8 space access to be able to access PCI ECS 2396f78a91cdSjjc * using "in/out" instructions and restore special bit after done 2397f78a91cdSjjc * accessing PCI ECS. 2398f78a91cdSjjc */ 2399f78a91cdSjjc #define OPT_PCI_ECS_ADDR(bus, device, function, reg) \ 2400f78a91cdSjjc (PCI_CONE | (((bus) & 0xff) << 16) | (((device & 0x1f)) << 11) | \ 2401f78a91cdSjjc (((function) & 0x7) << 8) | ((reg) & 0xfc) | \ 2402f78a91cdSjjc ((((reg) >> 8) & 0xf) << 24)) 24037c478bd9Sstevel@tonic-gate 24047c478bd9Sstevel@tonic-gate /* 24057c478bd9Sstevel@tonic-gate * PCI configuration space registers accessed by specifying 24067c478bd9Sstevel@tonic-gate * a bus, device, function, and offset. The following constants 24077c478bd9Sstevel@tonic-gate * define the values needed to access Opteron K8 configuration 24087c478bd9Sstevel@tonic-gate * info to determine its node topology 24097c478bd9Sstevel@tonic-gate */ 24107c478bd9Sstevel@tonic-gate 24117c478bd9Sstevel@tonic-gate #define OPT_PCS_BUS_CONFIG 0 /* Hypertransport config space bus */ 24127c478bd9Sstevel@tonic-gate 24137c478bd9Sstevel@tonic-gate /* 24147c478bd9Sstevel@tonic-gate * Opteron PCI configuration space register function values 24157c478bd9Sstevel@tonic-gate */ 24167c478bd9Sstevel@tonic-gate #define OPT_PCS_FUNC_HT 0 /* Hypertransport configuration */ 24177c478bd9Sstevel@tonic-gate #define OPT_PCS_FUNC_ADDRMAP 1 /* Address map configuration */ 24187c478bd9Sstevel@tonic-gate #define OPT_PCS_FUNC_DRAM 2 /* DRAM configuration */ 24197c478bd9Sstevel@tonic-gate #define OPT_PCS_FUNC_MISC 3 /* Miscellaneous configuration */ 24207c478bd9Sstevel@tonic-gate 24217c478bd9Sstevel@tonic-gate /* 24227c478bd9Sstevel@tonic-gate * PCI Configuration Space register offsets 24237c478bd9Sstevel@tonic-gate */ 24247c478bd9Sstevel@tonic-gate #define OPT_PCS_OFF_VENDOR 0x0 /* device/vendor ID register */ 2425f78a91cdSjjc #define OPT_PCS_OFF_DRAMBASE_HI 0x140 /* DRAM Base register (node 0) */ 2426f78a91cdSjjc #define OPT_PCS_OFF_DRAMBASE_LO 0x40 /* DRAM Base register (node 0) */ 24277c478bd9Sstevel@tonic-gate #define OPT_PCS_OFF_NODEID 0x60 /* Node ID register */ 24287c478bd9Sstevel@tonic-gate 24297c478bd9Sstevel@tonic-gate /* 24307c478bd9Sstevel@tonic-gate * Opteron PCI Configuration Space device IDs for nodes 24317c478bd9Sstevel@tonic-gate */ 24327c478bd9Sstevel@tonic-gate #define OPT_PCS_DEV_NODE0 24 /* device number for node 0 */ 24337c478bd9Sstevel@tonic-gate 24347c478bd9Sstevel@tonic-gate 24357c478bd9Sstevel@tonic-gate /* 24367c478bd9Sstevel@tonic-gate * Opteron DRAM address map gives base and limit for physical memory in a node 24377c478bd9Sstevel@tonic-gate */ 24387c478bd9Sstevel@tonic-gate typedef struct opt_dram_addr_map { 2439f78a91cdSjjc uint32_t base_hi; 2440f78a91cdSjjc uint32_t base_lo; 2441f78a91cdSjjc uint32_t limit_hi; 2442f78a91cdSjjc uint32_t limit_lo; 24437c478bd9Sstevel@tonic-gate } opt_dram_addr_map_t; 24447c478bd9Sstevel@tonic-gate 24457c478bd9Sstevel@tonic-gate 24467c478bd9Sstevel@tonic-gate /* 2447f78a91cdSjjc * Supported AMD processor families 2448f78a91cdSjjc */ 2449f78a91cdSjjc #define AMD_FAMILY_HAMMER 15 2450f78a91cdSjjc #define AMD_FAMILY_GREYHOUND 16 24517c478bd9Sstevel@tonic-gate 2452f78a91cdSjjc /* 24532e2c009bSjjc * Whether to have is_opteron() return 1 even when processor isn't supported 2454f78a91cdSjjc */ 2455f78a91cdSjjc uint_t is_opteron_override = 0; 2456f78a91cdSjjc 2457f78a91cdSjjc /* 2458f78a91cdSjjc * AMD processor family for current CPU 2459f78a91cdSjjc */ 24607c478bd9Sstevel@tonic-gate uint_t opt_family = 0; 2461f78a91cdSjjc 24627c478bd9Sstevel@tonic-gate 24637c478bd9Sstevel@tonic-gate /* 2464f78a91cdSjjc * Determine whether we're running on a supported AMD Opteron since reading 2465f78a91cdSjjc * node count and DRAM address map registers may have different format or 24662e2c009bSjjc * may not be supported across processor families 24677c478bd9Sstevel@tonic-gate */ 24682e2c009bSjjc static int 24697c478bd9Sstevel@tonic-gate is_opteron(void) 24707c478bd9Sstevel@tonic-gate { 2471f78a91cdSjjc 24727c478bd9Sstevel@tonic-gate if (x86_vendor != X86_VENDOR_AMD) 24737c478bd9Sstevel@tonic-gate return (0); 24747c478bd9Sstevel@tonic-gate 2475f78a91cdSjjc opt_family = cpuid_getfamily(CPU); 2476f78a91cdSjjc if (opt_family == AMD_FAMILY_HAMMER || 2477f78a91cdSjjc opt_family == AMD_FAMILY_GREYHOUND || is_opteron_override) 24787c478bd9Sstevel@tonic-gate return (1); 24797c478bd9Sstevel@tonic-gate else 24807c478bd9Sstevel@tonic-gate return (0); 24817c478bd9Sstevel@tonic-gate } 24827c478bd9Sstevel@tonic-gate 24832e2c009bSjjc 24842e2c009bSjjc /* 24852e2c009bSjjc * Determine NUMA configuration for Opteron from registers that live in PCI 24862e2c009bSjjc * configuration space 24872e2c009bSjjc */ 24882e2c009bSjjc static void 24892e2c009bSjjc opt_get_numa_config(uint_t *node_cnt, int *mem_intrlv, 24902e2c009bSjjc node_phys_addr_map_t *node_memory) 24917c478bd9Sstevel@tonic-gate { 24927c478bd9Sstevel@tonic-gate uint_t bus; 24937c478bd9Sstevel@tonic-gate uint_t dev; 24942e2c009bSjjc struct opt_dram_addr_map dram_map[MAX_NODES]; 24957c478bd9Sstevel@tonic-gate uint_t node; 24962e2c009bSjjc uint_t node_info[MAX_NODES]; 2497f78a91cdSjjc uint_t off_hi; 2498f78a91cdSjjc uint_t off_lo; 2499f78a91cdSjjc uint64_t nb_cfg_reg; 25007c478bd9Sstevel@tonic-gate 25017c478bd9Sstevel@tonic-gate /* 25027c478bd9Sstevel@tonic-gate * Read configuration registers from PCI configuration space to 25037c478bd9Sstevel@tonic-gate * determine node information, which memory is in each node, etc. 25047c478bd9Sstevel@tonic-gate * 25057c478bd9Sstevel@tonic-gate * Write to PCI configuration space address register to specify 25067c478bd9Sstevel@tonic-gate * which configuration register to read and read/write PCI 25077c478bd9Sstevel@tonic-gate * configuration space data register to get/set contents 25087c478bd9Sstevel@tonic-gate */ 25097c478bd9Sstevel@tonic-gate bus = OPT_PCS_BUS_CONFIG; 25107c478bd9Sstevel@tonic-gate dev = OPT_PCS_DEV_NODE0; 2511f78a91cdSjjc off_hi = OPT_PCS_OFF_DRAMBASE_HI; 2512f78a91cdSjjc off_lo = OPT_PCS_OFF_DRAMBASE_LO; 25137c478bd9Sstevel@tonic-gate 25147c478bd9Sstevel@tonic-gate /* 25157c478bd9Sstevel@tonic-gate * Read node ID register for node 0 to get node count 25167c478bd9Sstevel@tonic-gate */ 25172e2c009bSjjc node_info[0] = pci_getl_func(bus, dev, OPT_PCS_FUNC_HT, 2518ef50d8c0Sesaxe OPT_PCS_OFF_NODEID); 25192e2c009bSjjc *node_cnt = OPT_NODE_CNT(node_info[0]) + 1; 25202e2c009bSjjc 25212e2c009bSjjc /* 25222e2c009bSjjc * If number of nodes is more than maximum supported, then set node 25232e2c009bSjjc * count to 1 and treat system as UMA instead of NUMA. 25242e2c009bSjjc */ 25252e2c009bSjjc if (*node_cnt > MAX_NODES) { 25262e2c009bSjjc *node_cnt = 1; 25272e2c009bSjjc return; 25282e2c009bSjjc } 25297c478bd9Sstevel@tonic-gate 2530f78a91cdSjjc /* 2531f78a91cdSjjc * For Greyhound, PCI Extended Configuration Space must be enabled to 2532f78a91cdSjjc * read high DRAM address map base and limit registers 2533f78a91cdSjjc */ 2534f78a91cdSjjc if (opt_family == AMD_FAMILY_GREYHOUND) { 2535f78a91cdSjjc nb_cfg_reg = rdmsr(MSR_AMD_NB_CFG); 2536f78a91cdSjjc if ((nb_cfg_reg & AMD_GH_NB_CFG_EN_ECS) == 0) 2537f78a91cdSjjc wrmsr(MSR_AMD_NB_CFG, 2538f78a91cdSjjc nb_cfg_reg | AMD_GH_NB_CFG_EN_ECS); 2539f78a91cdSjjc } 2540f78a91cdSjjc 25412e2c009bSjjc for (node = 0; node < *node_cnt; node++) { 2542f78a91cdSjjc uint32_t base_hi; 2543f78a91cdSjjc uint32_t base_lo; 2544f78a91cdSjjc uint32_t limit_hi; 2545f78a91cdSjjc uint32_t limit_lo; 2546f78a91cdSjjc 25477c478bd9Sstevel@tonic-gate /* 25487c478bd9Sstevel@tonic-gate * Read node ID register (except for node 0 which we just read) 25497c478bd9Sstevel@tonic-gate */ 25507c478bd9Sstevel@tonic-gate if (node > 0) { 25512e2c009bSjjc node_info[node] = pci_getl_func(bus, dev, 2552ef50d8c0Sesaxe OPT_PCS_FUNC_HT, OPT_PCS_OFF_NODEID); 25537c478bd9Sstevel@tonic-gate } 25547c478bd9Sstevel@tonic-gate 25557c478bd9Sstevel@tonic-gate /* 25567c478bd9Sstevel@tonic-gate * Read DRAM base and limit registers which specify 25577c478bd9Sstevel@tonic-gate * physical memory range of each node 25587c478bd9Sstevel@tonic-gate */ 2559f78a91cdSjjc if (opt_family != AMD_FAMILY_GREYHOUND) 2560f78a91cdSjjc base_hi = 0; 2561f78a91cdSjjc else { 2562f78a91cdSjjc outl(PCI_CONFADD, OPT_PCI_ECS_ADDR(bus, dev, 2563f78a91cdSjjc OPT_PCS_FUNC_ADDRMAP, off_hi)); 25642e2c009bSjjc base_hi = dram_map[node].base_hi = 2565f78a91cdSjjc inl(PCI_CONFDATA); 2566f78a91cdSjjc } 25672e2c009bSjjc base_lo = dram_map[node].base_lo = pci_getl_func(bus, dev, 2568f78a91cdSjjc OPT_PCS_FUNC_ADDRMAP, off_lo); 2569f78a91cdSjjc 25702e2c009bSjjc if ((dram_map[node].base_lo & OPT_DRAMBASE_LO_MASK_INTRLVEN) && 25712e2c009bSjjc mem_intrlv) 25722e2c009bSjjc *mem_intrlv = *mem_intrlv + 1; 25737c478bd9Sstevel@tonic-gate 2574f78a91cdSjjc off_hi += 4; /* high limit register offset */ 2575f78a91cdSjjc if (opt_family != AMD_FAMILY_GREYHOUND) 2576f78a91cdSjjc limit_hi = 0; 2577f78a91cdSjjc else { 2578f78a91cdSjjc outl(PCI_CONFADD, OPT_PCI_ECS_ADDR(bus, dev, 2579f78a91cdSjjc OPT_PCS_FUNC_ADDRMAP, off_hi)); 25802e2c009bSjjc limit_hi = dram_map[node].limit_hi = 2581f78a91cdSjjc inl(PCI_CONFDATA); 2582f78a91cdSjjc } 2583f78a91cdSjjc 2584f78a91cdSjjc off_lo += 4; /* low limit register offset */ 25852e2c009bSjjc limit_lo = dram_map[node].limit_lo = pci_getl_func(bus, 2586f78a91cdSjjc dev, OPT_PCS_FUNC_ADDRMAP, off_lo); 25877c478bd9Sstevel@tonic-gate 25887c478bd9Sstevel@tonic-gate /* 2589f78a91cdSjjc * Increment device number to next node and register offsets 2590f78a91cdSjjc * for DRAM base register of next node 25917c478bd9Sstevel@tonic-gate */ 2592f78a91cdSjjc off_hi += 4; 2593f78a91cdSjjc off_lo += 4; 25947c478bd9Sstevel@tonic-gate dev++; 25957c478bd9Sstevel@tonic-gate 25967c478bd9Sstevel@tonic-gate /* 2597a940d195Sjjc * Both read and write enable bits must be enabled in DRAM 2598a940d195Sjjc * address map base register for physical memory to exist in 2599a940d195Sjjc * node 2600a940d195Sjjc */ 2601f78a91cdSjjc if ((base_lo & OPT_DRAMBASE_LO_MASK_RE) == 0 || 2602f78a91cdSjjc (base_lo & OPT_DRAMBASE_LO_MASK_WE) == 0) { 2603a940d195Sjjc /* 2604a940d195Sjjc * Mark node memory as non-existent and set start and 26052e2c009bSjjc * end addresses to be same in node_memory[] 2606a940d195Sjjc */ 26072e2c009bSjjc node_memory[node].exists = 0; 26082e2c009bSjjc node_memory[node].start = node_memory[node].end = 26092e2c009bSjjc (pfn_t)-1; 2610a940d195Sjjc continue; 2611a940d195Sjjc } 2612a940d195Sjjc 2613a940d195Sjjc /* 2614a940d195Sjjc * Mark node memory as existing and remember physical address 2615a940d195Sjjc * range of each node for use later 26167c478bd9Sstevel@tonic-gate */ 26172e2c009bSjjc node_memory[node].exists = 1; 2618f78a91cdSjjc 26192e2c009bSjjc node_memory[node].start = btop(OPT_DRAMADDR(base_hi, base_lo)); 2620f78a91cdSjjc 26212e2c009bSjjc node_memory[node].end = btop(OPT_DRAMADDR(limit_hi, limit_lo) | 2622f78a91cdSjjc OPT_DRAMADDR_LO_MASK_OFF); 2623f78a91cdSjjc } 2624f78a91cdSjjc 2625f78a91cdSjjc /* 2626f78a91cdSjjc * Restore PCI Extended Configuration Space enable bit 2627f78a91cdSjjc */ 2628f78a91cdSjjc if (opt_family == AMD_FAMILY_GREYHOUND) { 2629f78a91cdSjjc if ((nb_cfg_reg & AMD_GH_NB_CFG_EN_ECS) == 0) 2630f78a91cdSjjc wrmsr(MSR_AMD_NB_CFG, nb_cfg_reg); 26317c478bd9Sstevel@tonic-gate } 26327c478bd9Sstevel@tonic-gate } 26337c478bd9Sstevel@tonic-gate 26347c478bd9Sstevel@tonic-gate 26357c478bd9Sstevel@tonic-gate /* 26362e2c009bSjjc * Return average amount of time to read vendor ID register on Northbridge 26372e2c009bSjjc * N times on specified destination node from current CPU 26387c478bd9Sstevel@tonic-gate */ 26397c478bd9Sstevel@tonic-gate static hrtime_t 26402e2c009bSjjc opt_probe_vendor(int dest_node, int nreads) 26417c478bd9Sstevel@tonic-gate { 26422e2c009bSjjc int cnt; 26437c478bd9Sstevel@tonic-gate uint_t dev; 26447c478bd9Sstevel@tonic-gate /* LINTED: set but not used in function */ 26457c478bd9Sstevel@tonic-gate volatile uint_t dev_vendor; 26467c478bd9Sstevel@tonic-gate hrtime_t elapsed; 26477c478bd9Sstevel@tonic-gate hrtime_t end; 26487c478bd9Sstevel@tonic-gate int ipl; 26497c478bd9Sstevel@tonic-gate hrtime_t start; 26507c478bd9Sstevel@tonic-gate 26512e2c009bSjjc dev = OPT_PCS_DEV_NODE0 + dest_node; 26527c478bd9Sstevel@tonic-gate kpreempt_disable(); 26537c478bd9Sstevel@tonic-gate ipl = spl8(); 26542e2c009bSjjc outl(PCI_CONFADD, PCI_CADDR1(0, dev, OPT_PCS_FUNC_DRAM, 26557c478bd9Sstevel@tonic-gate OPT_PCS_OFF_VENDOR)); 26567c478bd9Sstevel@tonic-gate start = gethrtime(); 26572e2c009bSjjc for (cnt = 0; cnt < nreads; cnt++) 26587c478bd9Sstevel@tonic-gate dev_vendor = inl(PCI_CONFDATA); 26597c478bd9Sstevel@tonic-gate end = gethrtime(); 26602e2c009bSjjc elapsed = (end - start) / nreads; 26617c478bd9Sstevel@tonic-gate splx(ipl); 26627c478bd9Sstevel@tonic-gate kpreempt_enable(); 26632e2c009bSjjc return (elapsed); 26647c478bd9Sstevel@tonic-gate } 2665