xref: /titanic_50/usr/src/uts/i86pc/os/lgrpplat.c (revision b6917abefc343244b784f0cc34bc65b01469c3bf)
17c478bd9Sstevel@tonic-gate /*
27c478bd9Sstevel@tonic-gate  * CDDL HEADER START
37c478bd9Sstevel@tonic-gate  *
47c478bd9Sstevel@tonic-gate  * The contents of this file are subject to the terms of the
5c39996a7Sstevel  * Common Development and Distribution License (the "License").
6c39996a7Sstevel  * You may not use this file except in compliance with the License.
77c478bd9Sstevel@tonic-gate  *
87c478bd9Sstevel@tonic-gate  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
97c478bd9Sstevel@tonic-gate  * or http://www.opensolaris.org/os/licensing.
107c478bd9Sstevel@tonic-gate  * See the License for the specific language governing permissions
117c478bd9Sstevel@tonic-gate  * and limitations under the License.
127c478bd9Sstevel@tonic-gate  *
137c478bd9Sstevel@tonic-gate  * When distributing Covered Code, include this CDDL HEADER in each
147c478bd9Sstevel@tonic-gate  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
157c478bd9Sstevel@tonic-gate  * If applicable, add the following below this CDDL HEADER, with the
167c478bd9Sstevel@tonic-gate  * fields enclosed by brackets "[]" replaced with your own identifying
177c478bd9Sstevel@tonic-gate  * information: Portions Copyright [yyyy] [name of copyright owner]
187c478bd9Sstevel@tonic-gate  *
197c478bd9Sstevel@tonic-gate  * CDDL HEADER END
207c478bd9Sstevel@tonic-gate  */
21c39996a7Sstevel 
227c478bd9Sstevel@tonic-gate /*
23472714d6Skchow  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
247c478bd9Sstevel@tonic-gate  * Use is subject to license terms.
257c478bd9Sstevel@tonic-gate  */
267c478bd9Sstevel@tonic-gate 
277c478bd9Sstevel@tonic-gate #pragma ident	"%Z%%M%	%I%	%E% SMI"
287c478bd9Sstevel@tonic-gate 
297c478bd9Sstevel@tonic-gate 
302e2c009bSjjc /*
312e2c009bSjjc  * LOCALITY GROUP (LGROUP) PLATFORM SUPPORT FOR X86/AMD64 PLATFORMS
322e2c009bSjjc  * ================================================================
332e2c009bSjjc  * Multiprocessor AMD and Intel systems may have Non Uniform Memory Access
342e2c009bSjjc  * (NUMA).  A NUMA machine consists of one or more "nodes" that each consist of
352e2c009bSjjc  * one or more CPUs and some local memory.  The CPUs in each node can access
362e2c009bSjjc  * the memory in the other nodes but at a higher latency than accessing their
372e2c009bSjjc  * local memory.  Typically, a system with only one node has Uniform Memory
382e2c009bSjjc  * Access (UMA), but it may be possible to have a one node system that has
392e2c009bSjjc  * some global memory outside of the node which is higher latency.
402e2c009bSjjc  *
412e2c009bSjjc  * Module Description
422e2c009bSjjc  * ------------------
432e2c009bSjjc  * This module provides a platform interface for determining which CPUs and
442e2c009bSjjc  * which memory (and how much) are in a NUMA node and how far each node is from
452e2c009bSjjc  * each other.  The interface is used by the Virtual Memory (VM) system and the
462e2c009bSjjc  * common lgroup framework.  The VM system uses the plat_*() routines to fill
472e2c009bSjjc  * in its memory node (memnode) array with the physical address range spanned
482e2c009bSjjc  * by each NUMA node to know which memory belongs to which node, so it can
492e2c009bSjjc  * build and manage a physical page free list for each NUMA node and allocate
502e2c009bSjjc  * local memory from each node as needed.  The common lgroup framework uses the
512e2c009bSjjc  * exported lgrp_plat_*() routines to figure out which CPUs and memory belong
522e2c009bSjjc  * to each node (leaf lgroup) and how far each node is from each other, so it
532e2c009bSjjc  * can build the latency (lgroup) topology for the machine in order to optimize
542e2c009bSjjc  * for locality.  Also, an lgroup platform handle instead of lgroups are used
552e2c009bSjjc  * in the interface with this module, so this module shouldn't need to know
562e2c009bSjjc  * anything about lgroups.  Instead, it just needs to know which CPUs, memory,
572e2c009bSjjc  * etc. are in each NUMA node, how far each node is from each other, and to use
582e2c009bSjjc  * a unique lgroup platform handle to refer to each node through the interface.
592e2c009bSjjc  *
602e2c009bSjjc  * Determining NUMA Configuration
612e2c009bSjjc  * ------------------------------
622e2c009bSjjc  * By default, this module will try to determine the NUMA configuration of the
632e2c009bSjjc  * machine by reading the ACPI System Resource Affinity Table (SRAT) and System
642e2c009bSjjc  * Locality Information Table (SLIT).  The SRAT contains info to tell which
652e2c009bSjjc  * CPUs and memory are local to a given proximity domain (NUMA node).  The SLIT
662e2c009bSjjc  * is a matrix that gives the distance between each system locality (which is
672e2c009bSjjc  * a NUMA node and should correspond to proximity domains in the SRAT).  For
682e2c009bSjjc  * more details on the SRAT and SLIT, please refer to an ACPI 3.0 or newer
692e2c009bSjjc  * specification.
702e2c009bSjjc  *
712e2c009bSjjc  * If the SRAT doesn't exist on a system with AMD Opteron processors, we
722e2c009bSjjc  * examine registers in PCI configuration space to determine how many nodes are
732e2c009bSjjc  * in the system and which CPUs and memory are in each node.
742e2c009bSjjc  * do while booting the kernel.
752e2c009bSjjc  *
762e2c009bSjjc  * NOTE: Using these PCI configuration space registers to determine this
772e2c009bSjjc  *       locality info is not guaranteed to work or be compatible across all
782e2c009bSjjc  *	 Opteron processor families.
792e2c009bSjjc  *
802e2c009bSjjc  * If the SLIT does not exist or look right, the kernel will probe to determine
812e2c009bSjjc  * the distance between nodes as long as the NUMA CPU and memory configuration
822e2c009bSjjc  * has been determined (see lgrp_plat_probe() for details).
832e2c009bSjjc  *
842e2c009bSjjc  * Data Structures
852e2c009bSjjc  * ---------------
862e2c009bSjjc  * The main data structures used by this code are the following:
872e2c009bSjjc  *
88dae2fa37Sjjc  * - lgrp_plat_cpu_node[]		CPU to node ID mapping table indexed by
89dae2fa37Sjjc  *					CPU ID (only used for SRAT)
902e2c009bSjjc  *
912e2c009bSjjc  * - lgrp_plat_lat_stats.latencies[][]	Table of latencies between same and
922e2c009bSjjc  *					different nodes indexed by node ID
932e2c009bSjjc  *
942e2c009bSjjc  * - lgrp_plat_node_cnt			Number of NUMA nodes in system
952e2c009bSjjc  *
962e2c009bSjjc  * - lgrp_plat_node_domain[]		Node ID to proximity domain ID mapping
972e2c009bSjjc  *					table indexed by node ID (only used
982e2c009bSjjc  *					for SRAT)
992e2c009bSjjc  *
1002e2c009bSjjc  * - lgrp_plat_node_memory[]		Table with physical address range for
1012e2c009bSjjc  *					each node indexed by node ID
1022e2c009bSjjc  *
1032e2c009bSjjc  * The code is implemented to make the following always be true:
1042e2c009bSjjc  *
1052e2c009bSjjc  *	lgroup platform handle == node ID == memnode ID
1062e2c009bSjjc  *
1072e2c009bSjjc  * Moreover, it allows for the proximity domain ID to be equal to all of the
1082e2c009bSjjc  * above as long as the proximity domains IDs are numbered from 0 to <number of
1092e2c009bSjjc  * nodes - 1>.  This is done by hashing each proximity domain ID into the range
1102e2c009bSjjc  * from 0 to <number of nodes - 1>.  Then proximity ID N will hash into node ID
1112e2c009bSjjc  * N and proximity domain ID N will be entered into lgrp_plat_node_domain[N]
1122e2c009bSjjc  * and be assigned node ID N.  If the proximity domain IDs aren't numbered
1132e2c009bSjjc  * from 0 to <number of nodes - 1>, then hashing the proximity domain IDs into
1142e2c009bSjjc  * lgrp_plat_node_domain[] will still work for assigning proximity domain IDs
1152e2c009bSjjc  * to node IDs.  However, the proximity domain IDs may not map to the
1162e2c009bSjjc  * equivalent node ID since we want to keep the node IDs numbered from 0 to
1172e2c009bSjjc  * <number of nodes - 1> to minimize cost of searching and potentially space.
1182e2c009bSjjc  */
1192e2c009bSjjc 
1202e2c009bSjjc 
1217c478bd9Sstevel@tonic-gate #include <sys/archsystm.h>	/* for {in,out}{b,w,l}() */
122dae2fa37Sjjc #include <sys/bootconf.h>
1237c478bd9Sstevel@tonic-gate #include <sys/cmn_err.h>
124f78a91cdSjjc #include <sys/controlregs.h>
1257c478bd9Sstevel@tonic-gate #include <sys/cpupart.h>
1267c478bd9Sstevel@tonic-gate #include <sys/cpuvar.h>
1277c478bd9Sstevel@tonic-gate #include <sys/lgrp.h>
1287c478bd9Sstevel@tonic-gate #include <sys/machsystm.h>
1297c478bd9Sstevel@tonic-gate #include <sys/memlist.h>
1307c478bd9Sstevel@tonic-gate #include <sys/memnode.h>
1317c478bd9Sstevel@tonic-gate #include <sys/mman.h>
132ef50d8c0Sesaxe #include <sys/pci_cfgspace.h>
133ef50d8c0Sesaxe #include <sys/pci_impl.h>
1347c478bd9Sstevel@tonic-gate #include <sys/param.h>
135fb2f18f8Sesaxe #include <sys/pghw.h>
1367c478bd9Sstevel@tonic-gate #include <sys/promif.h>		/* for prom_printf() */
1372e2c009bSjjc #include <sys/sysmacros.h>
1387c478bd9Sstevel@tonic-gate #include <sys/systm.h>
1397c478bd9Sstevel@tonic-gate #include <sys/thread.h>
1407c478bd9Sstevel@tonic-gate #include <sys/types.h>
1417c478bd9Sstevel@tonic-gate #include <sys/var.h>
1427c478bd9Sstevel@tonic-gate #include <sys/x86_archext.h>	/* for x86_feature and X86_AMD */
1437c478bd9Sstevel@tonic-gate #include <vm/hat_i86.h>
1447c478bd9Sstevel@tonic-gate #include <vm/seg_kmem.h>
145affbd3ccSkchow #include <vm/vm_dep.h>
1467c478bd9Sstevel@tonic-gate 
1472e2c009bSjjc #include "acpi_fw.h"		/* for SRAT and SLIT */
1487c478bd9Sstevel@tonic-gate 
1497c478bd9Sstevel@tonic-gate 
1507c478bd9Sstevel@tonic-gate #define	MAX_NODES		8
1517c478bd9Sstevel@tonic-gate #define	NLGRP			(MAX_NODES * (MAX_NODES - 1) + 1)
1527c478bd9Sstevel@tonic-gate 
1532e2c009bSjjc /*
1542e2c009bSjjc  * Constants for configuring probing
1552e2c009bSjjc  */
1567c478bd9Sstevel@tonic-gate #define	LGRP_PLAT_PROBE_NROUNDS		64	/* default laps for probing */
1577c478bd9Sstevel@tonic-gate #define	LGRP_PLAT_PROBE_NSAMPLES	1	/* default samples to take */
1588949bcd6Sandrei #define	LGRP_PLAT_PROBE_NREADS		256	/* number of vendor ID reads */
1597c478bd9Sstevel@tonic-gate 
1607c478bd9Sstevel@tonic-gate /*
1612e2c009bSjjc  * Flags for probing
1622e2c009bSjjc  */
1632e2c009bSjjc #define	LGRP_PLAT_PROBE_ENABLE		0x1	/* enable probing */
1642e2c009bSjjc #define	LGRP_PLAT_PROBE_PGCPY		0x2	/* probe using page copy */
1652e2c009bSjjc #define	LGRP_PLAT_PROBE_VENDOR		0x4	/* probe vendor ID register */
1662e2c009bSjjc 
1672e2c009bSjjc /*
1682e2c009bSjjc  * Hash proximity domain ID into node to domain mapping table using to minimize
1692e2c009bSjjc  * span of entries used
1702e2c009bSjjc  */
171d821f0f0Sjjc #define	NODE_DOMAIN_HASH(domain, node_cnt)	((domain) % node_cnt)
1722e2c009bSjjc 
1732e2c009bSjjc 
1742e2c009bSjjc /*
175dae2fa37Sjjc  * CPU to node ID mapping structure (only used with SRAT)
1762e2c009bSjjc  */
1772e2c009bSjjc typedef	struct cpu_node_map {
1782e2c009bSjjc 	int		exists;
1792e2c009bSjjc 	uint_t		node;
1802e2c009bSjjc 	uint32_t	apicid;
1812e2c009bSjjc 	uint32_t	prox_domain;
1822e2c009bSjjc } cpu_node_map_t;
1832e2c009bSjjc 
1842e2c009bSjjc /*
1852e2c009bSjjc  * Latency statistics
1862e2c009bSjjc  */
1872e2c009bSjjc typedef struct lgrp_plat_latency_stats {
1882e2c009bSjjc 	hrtime_t	latencies[MAX_NODES][MAX_NODES];
1892e2c009bSjjc 	hrtime_t	latency_max;
1902e2c009bSjjc 	hrtime_t	latency_min;
1912e2c009bSjjc } lgrp_plat_latency_stats_t;
1922e2c009bSjjc 
1932e2c009bSjjc /*
1942e2c009bSjjc  * Memory configuration for probing
1952e2c009bSjjc  */
1962e2c009bSjjc typedef struct lgrp_plat_probe_mem_config {
1972e2c009bSjjc 	size_t	probe_memsize;		/* how much memory to probe per node */
1982e2c009bSjjc 	caddr_t	probe_va[MAX_NODES];	/* where memory mapped for probing */
1992e2c009bSjjc 	pfn_t	probe_pfn[MAX_NODES];	/* physical pages to map for probing */
2002e2c009bSjjc } lgrp_plat_probe_mem_config_t;
2012e2c009bSjjc 
2022e2c009bSjjc /*
2032e2c009bSjjc  * Statistics kept for probing
2042e2c009bSjjc  */
2052e2c009bSjjc typedef struct lgrp_plat_probe_stats {
2062e2c009bSjjc 	hrtime_t	flush_cost;
2072e2c009bSjjc 	hrtime_t	probe_cost;
2082e2c009bSjjc 	hrtime_t	probe_cost_total;
2092e2c009bSjjc 	hrtime_t	probe_error_code;
2102e2c009bSjjc 	hrtime_t	probe_errors[MAX_NODES][MAX_NODES];
2112e2c009bSjjc 	int		probe_suspect[MAX_NODES][MAX_NODES];
2122e2c009bSjjc 	hrtime_t	probe_max[MAX_NODES][MAX_NODES];
2132e2c009bSjjc 	hrtime_t	probe_min[MAX_NODES][MAX_NODES];
2142e2c009bSjjc } lgrp_plat_probe_stats_t;
2152e2c009bSjjc 
2162e2c009bSjjc /*
2172e2c009bSjjc  * Node to proximity domain ID mapping structure (only used with SRAT)
2182e2c009bSjjc  */
2192e2c009bSjjc typedef	struct node_domain_map {
2202e2c009bSjjc 	int		exists;
2212e2c009bSjjc 	uint32_t	prox_domain;
2222e2c009bSjjc } node_domain_map_t;
2232e2c009bSjjc 
2242e2c009bSjjc /*
2252e2c009bSjjc  * Node ID and starting and ending page for physical memory in node
2262e2c009bSjjc  */
2272e2c009bSjjc typedef	struct node_phys_addr_map {
2282e2c009bSjjc 	pfn_t		start;
2292e2c009bSjjc 	pfn_t		end;
2302e2c009bSjjc 	int		exists;
2312e2c009bSjjc 	uint32_t	prox_domain;
2322e2c009bSjjc } node_phys_addr_map_t;
2332e2c009bSjjc 
234dae2fa37Sjjc /*
235d821f0f0Sjjc  * Number of CPUs for which we got APIC IDs
236dae2fa37Sjjc  */
237d821f0f0Sjjc static int				lgrp_plat_apic_ncpus = 0;
2382e2c009bSjjc 
2392e2c009bSjjc /*
240dae2fa37Sjjc  * CPU to node ID mapping table (only used for SRAT)
2412e2c009bSjjc  */
2422e2c009bSjjc static cpu_node_map_t			lgrp_plat_cpu_node[NCPU];
2432e2c009bSjjc 
2442e2c009bSjjc /*
2452e2c009bSjjc  * Latency statistics
2462e2c009bSjjc  */
2472e2c009bSjjc lgrp_plat_latency_stats_t		lgrp_plat_lat_stats;
2482e2c009bSjjc 
2492e2c009bSjjc /*
2502e2c009bSjjc  * Whether memory is interleaved across nodes causing MPO to be disabled
2512e2c009bSjjc  */
2522e2c009bSjjc static int				lgrp_plat_mem_intrlv = 0;
2532e2c009bSjjc 
2542e2c009bSjjc /*
2552e2c009bSjjc  * Node ID to proximity domain ID mapping table (only used for SRAT)
2562e2c009bSjjc  */
2572e2c009bSjjc static node_domain_map_t		lgrp_plat_node_domain[MAX_NODES];
2582e2c009bSjjc 
2592e2c009bSjjc /*
2602e2c009bSjjc  * Physical address range for memory in each node
2612e2c009bSjjc  */
2622e2c009bSjjc static node_phys_addr_map_t		lgrp_plat_node_memory[MAX_NODES];
2632e2c009bSjjc 
2642e2c009bSjjc /*
2652e2c009bSjjc  * Statistics gotten from probing
2662e2c009bSjjc  */
2672e2c009bSjjc static lgrp_plat_probe_stats_t		lgrp_plat_probe_stats;
2682e2c009bSjjc 
2692e2c009bSjjc /*
2702e2c009bSjjc  * Memory configuration for probing
2712e2c009bSjjc  */
2722e2c009bSjjc static lgrp_plat_probe_mem_config_t	lgrp_plat_probe_mem_config;
2732e2c009bSjjc 
2742e2c009bSjjc /*
2752e2c009bSjjc  * Error code from processing ACPI SRAT
2762e2c009bSjjc  */
2772e2c009bSjjc static int				lgrp_plat_srat_error = 0;
2782e2c009bSjjc 
2792e2c009bSjjc /*
2802e2c009bSjjc  * Error code from processing ACPI SLIT
2812e2c009bSjjc  */
2822e2c009bSjjc static int				lgrp_plat_slit_error = 0;
2832e2c009bSjjc 
2842e2c009bSjjc /*
2852e2c009bSjjc  * Allocate lgroup array statically
2862e2c009bSjjc  */
2872e2c009bSjjc static lgrp_t				lgrp_space[NLGRP];
2882e2c009bSjjc static int				nlgrps_alloc;
2892e2c009bSjjc 
2902e2c009bSjjc 
2912e2c009bSjjc /*
2922e2c009bSjjc  * Number of nodes in system
2932e2c009bSjjc  */
2942e2c009bSjjc uint_t			lgrp_plat_node_cnt = 1;
2952e2c009bSjjc 
2962e2c009bSjjc /*
2972e2c009bSjjc  * Configuration Parameters for Probing
2982e2c009bSjjc  * - lgrp_plat_probe_flags	Flags to specify enabling probing, probe
2992e2c009bSjjc  *				operation, etc.
3002e2c009bSjjc  * - lgrp_plat_probe_nrounds	How many rounds of probing to do
3012e2c009bSjjc  * - lgrp_plat_probe_nsamples	Number of samples to take when probing each
3022e2c009bSjjc  *				node
3032e2c009bSjjc  * - lgrp_plat_probe_nreads	Number of times to read vendor ID from
3042e2c009bSjjc  *				Northbridge for each probe
3052e2c009bSjjc  */
3062e2c009bSjjc uint_t			lgrp_plat_probe_flags = 0;
3072e2c009bSjjc int			lgrp_plat_probe_nrounds = LGRP_PLAT_PROBE_NROUNDS;
3082e2c009bSjjc int			lgrp_plat_probe_nsamples = LGRP_PLAT_PROBE_NSAMPLES;
3092e2c009bSjjc int			lgrp_plat_probe_nreads = LGRP_PLAT_PROBE_NREADS;
3102e2c009bSjjc 
3112e2c009bSjjc /*
3122e2c009bSjjc  * Enable use of ACPI System Resource Affinity Table (SRAT) and System
3132e2c009bSjjc  * Locality Information Table (SLIT)
3142e2c009bSjjc  */
3152e2c009bSjjc int			lgrp_plat_srat_enable = 1;
3162e2c009bSjjc int			lgrp_plat_slit_enable = 1;
3172e2c009bSjjc 
3182e2c009bSjjc /*
3192e2c009bSjjc  * Static array to hold lgroup statistics
3202e2c009bSjjc  */
3212e2c009bSjjc struct lgrp_stats	lgrp_stats[NLGRP];
3222e2c009bSjjc 
3232e2c009bSjjc 
3242e2c009bSjjc /*
3252e2c009bSjjc  * Forward declarations of platform interface routines
3262e2c009bSjjc  */
3272e2c009bSjjc void		plat_build_mem_nodes(struct memlist *list);
3282e2c009bSjjc 
3292e2c009bSjjc int		plat_lgrphand_to_mem_node(lgrp_handle_t hand);
3302e2c009bSjjc 
3312e2c009bSjjc lgrp_handle_t	plat_mem_node_to_lgrphand(int mnode);
3322e2c009bSjjc 
3332e2c009bSjjc int		plat_mnode_xcheck(pfn_t pfncnt);
3342e2c009bSjjc 
3352e2c009bSjjc int		plat_pfn_to_mem_node(pfn_t pfn);
3362e2c009bSjjc 
3372e2c009bSjjc /*
3382e2c009bSjjc  * Forward declarations of lgroup platform interface routines
3392e2c009bSjjc  */
3402e2c009bSjjc lgrp_t		*lgrp_plat_alloc(lgrp_id_t lgrpid);
3412e2c009bSjjc 
3422e2c009bSjjc void		lgrp_plat_config(lgrp_config_flag_t flag, uintptr_t arg);
3432e2c009bSjjc 
3442e2c009bSjjc lgrp_handle_t	lgrp_plat_cpu_to_hand(processorid_t id);
3452e2c009bSjjc 
3462e2c009bSjjc void		lgrp_plat_init(void);
3472e2c009bSjjc 
3482e2c009bSjjc int		lgrp_plat_latency(lgrp_handle_t from, lgrp_handle_t to);
3492e2c009bSjjc 
3502e2c009bSjjc void		lgrp_plat_main_init(void);
3512e2c009bSjjc 
3522e2c009bSjjc int		lgrp_plat_max_lgrps(void);
3532e2c009bSjjc 
3542e2c009bSjjc pgcnt_t		lgrp_plat_mem_size(lgrp_handle_t plathand,
3552e2c009bSjjc     lgrp_mem_query_t query);
3562e2c009bSjjc 
3572e2c009bSjjc lgrp_handle_t	lgrp_plat_pfn_to_hand(pfn_t pfn);
3582e2c009bSjjc 
3592e2c009bSjjc void		lgrp_plat_probe(void);
3602e2c009bSjjc 
3612e2c009bSjjc lgrp_handle_t	lgrp_plat_root_hand(void);
3622e2c009bSjjc 
3632e2c009bSjjc 
3642e2c009bSjjc /*
3652e2c009bSjjc  * Forward declarations of local routines
3662e2c009bSjjc  */
3672e2c009bSjjc static int	is_opteron(void);
3682e2c009bSjjc 
369dae2fa37Sjjc static int	lgrp_plat_cpu_node_update(node_domain_map_t *node_domain,
370d821f0f0Sjjc     int node_cnt, cpu_node_map_t *cpu_node, int nentries, uint32_t apicid,
371d821f0f0Sjjc     uint32_t domain);
372dae2fa37Sjjc 
3732e2c009bSjjc static int	lgrp_plat_cpu_to_node(cpu_t *cp, cpu_node_map_t *cpu_node);
3742e2c009bSjjc 
3752e2c009bSjjc static int	lgrp_plat_domain_to_node(node_domain_map_t *node_domain,
376d821f0f0Sjjc     int node_cnt, uint32_t domain);
3772e2c009bSjjc 
3782e2c009bSjjc static void	lgrp_plat_latency_adjust(node_phys_addr_map_t *node_memory,
3792e2c009bSjjc     lgrp_plat_latency_stats_t *lat_stats,
3802e2c009bSjjc     lgrp_plat_probe_stats_t *probe_stats);
3812e2c009bSjjc 
3822e2c009bSjjc static int	lgrp_plat_latency_verify(node_phys_addr_map_t *node_memory,
3832e2c009bSjjc     lgrp_plat_latency_stats_t *lat_stats);
3842e2c009bSjjc 
3852e2c009bSjjc static pgcnt_t	lgrp_plat_mem_size_default(lgrp_handle_t, lgrp_mem_query_t);
3862e2c009bSjjc 
3872e2c009bSjjc static int	lgrp_plat_node_domain_update(node_domain_map_t *node_domain,
388d821f0f0Sjjc     int node_cnt, uint32_t domain);
3892e2c009bSjjc 
3902e2c009bSjjc static int	lgrp_plat_node_memory_update(node_domain_map_t *node_domain,
391d821f0f0Sjjc     int node_cnt, node_phys_addr_map_t *node_memory, uint64_t start,
392d821f0f0Sjjc     uint64_t end, uint32_t domain);
3932e2c009bSjjc 
3942e2c009bSjjc static hrtime_t	lgrp_plat_probe_time(int to, cpu_node_map_t *cpu_node,
3952e2c009bSjjc     lgrp_plat_probe_mem_config_t *probe_mem_config,
3962e2c009bSjjc     lgrp_plat_latency_stats_t *lat_stats,
3972e2c009bSjjc     lgrp_plat_probe_stats_t *probe_stats);
3982e2c009bSjjc 
399d821f0f0Sjjc static int	lgrp_plat_process_cpu_apicids(cpu_node_map_t *cpu_node);
400dae2fa37Sjjc 
4012e2c009bSjjc static int	lgrp_plat_process_slit(struct slit *tp, uint_t node_cnt,
4022e2c009bSjjc     node_phys_addr_map_t *node_memory, lgrp_plat_latency_stats_t *lat_stats);
4032e2c009bSjjc 
404d821f0f0Sjjc static int	lgrp_plat_process_srat(struct srat *tp,
405d821f0f0Sjjc     node_domain_map_t *node_domain, cpu_node_map_t *cpu_node, int cpu_count,
4062e2c009bSjjc     node_phys_addr_map_t *node_memory);
4072e2c009bSjjc 
4082e2c009bSjjc static int	lgrp_plat_srat_domains(struct srat *tp);
4092e2c009bSjjc 
4102e2c009bSjjc static void	lgrp_plat_2level_setup(node_phys_addr_map_t *node_memory,
4112e2c009bSjjc     lgrp_plat_latency_stats_t *lat_stats);
4122e2c009bSjjc 
4132e2c009bSjjc static void	opt_get_numa_config(uint_t *node_cnt, int *mem_intrlv,
4142e2c009bSjjc     node_phys_addr_map_t *node_memory);
4152e2c009bSjjc 
4162e2c009bSjjc static hrtime_t	opt_probe_vendor(int dest_node, int nreads);
4172e2c009bSjjc 
4182e2c009bSjjc 
4192e2c009bSjjc /*
4202e2c009bSjjc  * PLATFORM INTERFACE ROUTINES
4217c478bd9Sstevel@tonic-gate  */
4227c478bd9Sstevel@tonic-gate 
4237c478bd9Sstevel@tonic-gate /*
4242e2c009bSjjc  * Configure memory nodes for machines with more than one node (ie NUMA)
4252e2c009bSjjc  */
4262e2c009bSjjc void
4272e2c009bSjjc plat_build_mem_nodes(struct memlist *list)
4282e2c009bSjjc {
4292e2c009bSjjc 	pfn_t		cur_start;	/* start addr of subrange */
4302e2c009bSjjc 	pfn_t		cur_end;	/* end addr of subrange */
4312e2c009bSjjc 	pfn_t		start;		/* start addr of whole range */
4322e2c009bSjjc 	pfn_t		end;		/* end addr of whole range */
4332e2c009bSjjc 
4342e2c009bSjjc 	/*
4352e2c009bSjjc 	 * Boot install lists are arranged <addr, len>, ...
4362e2c009bSjjc 	 */
4372e2c009bSjjc 	while (list) {
4382e2c009bSjjc 		int	node;
4392e2c009bSjjc 
4402e2c009bSjjc 		start = list->address >> PAGESHIFT;
4412e2c009bSjjc 		end = (list->address + list->size - 1) >> PAGESHIFT;
4422e2c009bSjjc 
4432e2c009bSjjc 		if (start > physmax) {
4442e2c009bSjjc 			list = list->next;
4452e2c009bSjjc 			continue;
4462e2c009bSjjc 		}
4472e2c009bSjjc 		if (end > physmax)
4482e2c009bSjjc 			end = physmax;
4492e2c009bSjjc 
4502e2c009bSjjc 		/*
4512e2c009bSjjc 		 * When there is only one memnode, just add memory to memnode
4522e2c009bSjjc 		 */
4532e2c009bSjjc 		if (max_mem_nodes == 1) {
4542e2c009bSjjc 			mem_node_add_slice(start, end);
4552e2c009bSjjc 			list = list->next;
4562e2c009bSjjc 			continue;
4572e2c009bSjjc 		}
4582e2c009bSjjc 
4592e2c009bSjjc 		/*
4602e2c009bSjjc 		 * mem_node_add_slice() expects to get a memory range that
4612e2c009bSjjc 		 * is within one memnode, so need to split any memory range
4622e2c009bSjjc 		 * that spans multiple memnodes into subranges that are each
4632e2c009bSjjc 		 * contained within one memnode when feeding them to
4642e2c009bSjjc 		 * mem_node_add_slice()
4652e2c009bSjjc 		 */
4662e2c009bSjjc 		cur_start = start;
4672e2c009bSjjc 		do {
4682e2c009bSjjc 			node = plat_pfn_to_mem_node(cur_start);
4692e2c009bSjjc 
4702e2c009bSjjc 			/*
4712e2c009bSjjc 			 * Panic if DRAM address map registers or SRAT say
4722e2c009bSjjc 			 * memory in node doesn't exist or address from
4732e2c009bSjjc 			 * boot installed memory list entry isn't in this node.
4742e2c009bSjjc 			 * This shouldn't happen and rest of code can't deal
4752e2c009bSjjc 			 * with this if it does.
4762e2c009bSjjc 			 */
4772e2c009bSjjc 			if (node < 0 || node >= lgrp_plat_node_cnt ||
4782e2c009bSjjc 			    !lgrp_plat_node_memory[node].exists ||
4792e2c009bSjjc 			    cur_start < lgrp_plat_node_memory[node].start ||
4802e2c009bSjjc 			    cur_start > lgrp_plat_node_memory[node].end) {
4812e2c009bSjjc 				cmn_err(CE_PANIC, "Don't know which memnode "
4822e2c009bSjjc 				    "to add installed memory address 0x%lx\n",
4832e2c009bSjjc 				    cur_start);
4842e2c009bSjjc 			}
4852e2c009bSjjc 
4862e2c009bSjjc 			/*
4872e2c009bSjjc 			 * End of current subrange should not span memnodes
4882e2c009bSjjc 			 */
4892e2c009bSjjc 			cur_end = end;
4902e2c009bSjjc 			if (lgrp_plat_node_memory[node].exists &&
4912e2c009bSjjc 			    cur_end > lgrp_plat_node_memory[node].end)
4922e2c009bSjjc 				cur_end = lgrp_plat_node_memory[node].end;
4932e2c009bSjjc 
4942e2c009bSjjc 			mem_node_add_slice(cur_start, cur_end);
4952e2c009bSjjc 
4962e2c009bSjjc 			/*
4972e2c009bSjjc 			 * Next subrange starts after end of current one
4982e2c009bSjjc 			 */
4992e2c009bSjjc 			cur_start = cur_end + 1;
5002e2c009bSjjc 		} while (cur_end < end);
5012e2c009bSjjc 
5022e2c009bSjjc 		list = list->next;
5032e2c009bSjjc 	}
5042e2c009bSjjc 	mem_node_physalign = 0;
5052e2c009bSjjc 	mem_node_pfn_shift = 0;
5062e2c009bSjjc }
5072e2c009bSjjc 
5082e2c009bSjjc 
5092e2c009bSjjc int
5102e2c009bSjjc plat_lgrphand_to_mem_node(lgrp_handle_t hand)
5112e2c009bSjjc {
5122e2c009bSjjc 	if (max_mem_nodes == 1)
5132e2c009bSjjc 		return (0);
5142e2c009bSjjc 
5152e2c009bSjjc 	return ((int)hand);
5162e2c009bSjjc }
5172e2c009bSjjc 
5182e2c009bSjjc 
5192e2c009bSjjc /*
5202e2c009bSjjc  * plat_mnode_xcheck: checks the node memory ranges to see if there is a pfncnt
5212e2c009bSjjc  * range of pages aligned on pfncnt that crosses an node boundary. Returns 1 if
5222e2c009bSjjc  * a crossing is found and returns 0 otherwise.
5232e2c009bSjjc  */
5242e2c009bSjjc int
5252e2c009bSjjc plat_mnode_xcheck(pfn_t pfncnt)
5262e2c009bSjjc {
5272e2c009bSjjc 	int	node, prevnode = -1, basenode;
5282e2c009bSjjc 	pfn_t	ea, sa;
5292e2c009bSjjc 
5302e2c009bSjjc 	for (node = 0; node < lgrp_plat_node_cnt; node++) {
5312e2c009bSjjc 
5322e2c009bSjjc 		if (lgrp_plat_node_memory[node].exists == 0)
5332e2c009bSjjc 			continue;
5342e2c009bSjjc 
5352e2c009bSjjc 		if (prevnode == -1) {
5362e2c009bSjjc 			prevnode = node;
5372e2c009bSjjc 			basenode = node;
5382e2c009bSjjc 			continue;
5392e2c009bSjjc 		}
5402e2c009bSjjc 
5412e2c009bSjjc 		/* assume x86 node pfn ranges are in increasing order */
5422e2c009bSjjc 		ASSERT(lgrp_plat_node_memory[node].start >
5432e2c009bSjjc 		    lgrp_plat_node_memory[prevnode].end);
5442e2c009bSjjc 
5452e2c009bSjjc 		/*
5462e2c009bSjjc 		 * continue if the starting address of node is not contiguous
5472e2c009bSjjc 		 * with the previous node.
5482e2c009bSjjc 		 */
5492e2c009bSjjc 
5502e2c009bSjjc 		if (lgrp_plat_node_memory[node].start !=
5512e2c009bSjjc 		    (lgrp_plat_node_memory[prevnode].end + 1)) {
5522e2c009bSjjc 			basenode = node;
5532e2c009bSjjc 			prevnode = node;
5542e2c009bSjjc 			continue;
5552e2c009bSjjc 		}
5562e2c009bSjjc 
5572e2c009bSjjc 		/* check if the starting address of node is pfncnt aligned */
5582e2c009bSjjc 		if ((lgrp_plat_node_memory[node].start & (pfncnt - 1)) != 0) {
5592e2c009bSjjc 
5602e2c009bSjjc 			/*
5612e2c009bSjjc 			 * at this point, node starts at an unaligned boundary
5622e2c009bSjjc 			 * and is contiguous with the previous node(s) to
5632e2c009bSjjc 			 * basenode. Check if there is an aligned contiguous
5642e2c009bSjjc 			 * range of length pfncnt that crosses this boundary.
5652e2c009bSjjc 			 */
5662e2c009bSjjc 
5672e2c009bSjjc 			sa = P2ALIGN(lgrp_plat_node_memory[prevnode].end,
5682e2c009bSjjc 			    pfncnt);
5692e2c009bSjjc 			ea = P2ROUNDUP((lgrp_plat_node_memory[node].start),
5702e2c009bSjjc 			    pfncnt);
5712e2c009bSjjc 
5722e2c009bSjjc 			ASSERT((ea - sa) == pfncnt);
5732e2c009bSjjc 			if (sa >= lgrp_plat_node_memory[basenode].start &&
5742e2c009bSjjc 			    ea <= (lgrp_plat_node_memory[node].end + 1))
5752e2c009bSjjc 				return (1);
5762e2c009bSjjc 		}
5772e2c009bSjjc 		prevnode = node;
5782e2c009bSjjc 	}
5792e2c009bSjjc 	return (0);
5802e2c009bSjjc }
5812e2c009bSjjc 
5822e2c009bSjjc 
5832e2c009bSjjc lgrp_handle_t
5842e2c009bSjjc plat_mem_node_to_lgrphand(int mnode)
5852e2c009bSjjc {
5862e2c009bSjjc 	if (max_mem_nodes == 1)
5872e2c009bSjjc 		return (LGRP_DEFAULT_HANDLE);
5882e2c009bSjjc 
5892e2c009bSjjc 	return ((lgrp_handle_t)mnode);
5902e2c009bSjjc }
5912e2c009bSjjc 
5922e2c009bSjjc 
5932e2c009bSjjc int
5942e2c009bSjjc plat_pfn_to_mem_node(pfn_t pfn)
5952e2c009bSjjc {
5962e2c009bSjjc 	int	node;
5972e2c009bSjjc 
5982e2c009bSjjc 	if (max_mem_nodes == 1)
5992e2c009bSjjc 		return (0);
6002e2c009bSjjc 
6012e2c009bSjjc 	for (node = 0; node < lgrp_plat_node_cnt; node++) {
6022e2c009bSjjc 		/*
6032e2c009bSjjc 		 * Skip nodes with no memory
6042e2c009bSjjc 		 */
6052e2c009bSjjc 		if (!lgrp_plat_node_memory[node].exists)
6062e2c009bSjjc 			continue;
6072e2c009bSjjc 
6082e2c009bSjjc 		if (pfn >= lgrp_plat_node_memory[node].start &&
6092e2c009bSjjc 		    pfn <= lgrp_plat_node_memory[node].end)
6102e2c009bSjjc 			return (node);
6112e2c009bSjjc 	}
6122e2c009bSjjc 
6132e2c009bSjjc 	/*
6142e2c009bSjjc 	 * Didn't find memnode where this PFN lives which should never happen
6152e2c009bSjjc 	 */
6162e2c009bSjjc 	ASSERT(node < lgrp_plat_node_cnt);
6172e2c009bSjjc 	return (-1);
6182e2c009bSjjc }
6192e2c009bSjjc 
6202e2c009bSjjc 
6212e2c009bSjjc /*
6222e2c009bSjjc  * LGROUP PLATFORM INTERFACE ROUTINES
6232e2c009bSjjc  */
6242e2c009bSjjc 
6252e2c009bSjjc /*
6262e2c009bSjjc  * Allocate additional space for an lgroup.
6272e2c009bSjjc  */
6282e2c009bSjjc /* ARGSUSED */
6292e2c009bSjjc lgrp_t *
6302e2c009bSjjc lgrp_plat_alloc(lgrp_id_t lgrpid)
6312e2c009bSjjc {
6322e2c009bSjjc 	lgrp_t *lgrp;
6332e2c009bSjjc 
6342e2c009bSjjc 	lgrp = &lgrp_space[nlgrps_alloc++];
6352e2c009bSjjc 	if (lgrpid >= NLGRP || nlgrps_alloc > NLGRP)
6362e2c009bSjjc 		return (NULL);
6372e2c009bSjjc 	return (lgrp);
6382e2c009bSjjc }
6392e2c009bSjjc 
6402e2c009bSjjc 
6412e2c009bSjjc /*
6422e2c009bSjjc  * Platform handling for (re)configuration changes
6432e2c009bSjjc  */
6442e2c009bSjjc /* ARGSUSED */
6452e2c009bSjjc void
6462e2c009bSjjc lgrp_plat_config(lgrp_config_flag_t flag, uintptr_t arg)
6472e2c009bSjjc {
6482e2c009bSjjc }
6492e2c009bSjjc 
6502e2c009bSjjc 
6512e2c009bSjjc /*
6522e2c009bSjjc  * Return the platform handle for the lgroup containing the given CPU
6532e2c009bSjjc  */
6542e2c009bSjjc /* ARGSUSED */
6552e2c009bSjjc lgrp_handle_t
6562e2c009bSjjc lgrp_plat_cpu_to_hand(processorid_t id)
6572e2c009bSjjc {
6582e2c009bSjjc 	lgrp_handle_t	hand;
6592e2c009bSjjc 
6602e2c009bSjjc 	if (lgrp_plat_node_cnt == 1)
6612e2c009bSjjc 		return (LGRP_DEFAULT_HANDLE);
6622e2c009bSjjc 
6632e2c009bSjjc 	hand = (lgrp_handle_t)lgrp_plat_cpu_to_node(cpu[id],
6642e2c009bSjjc 	    lgrp_plat_cpu_node);
6652e2c009bSjjc 
6662e2c009bSjjc 	ASSERT(hand != (lgrp_handle_t)-1);
6672e2c009bSjjc 	if (hand == (lgrp_handle_t)-1)
6682e2c009bSjjc 		return (LGRP_NULL_HANDLE);
6692e2c009bSjjc 
6702e2c009bSjjc 	return (hand);
6712e2c009bSjjc }
6722e2c009bSjjc 
6732e2c009bSjjc 
6742e2c009bSjjc /*
6752e2c009bSjjc  * Platform-specific initialization of lgroups
6762e2c009bSjjc  */
6772e2c009bSjjc void
6782e2c009bSjjc lgrp_plat_init(void)
6792e2c009bSjjc {
6802e2c009bSjjc #if defined(__xpv)
6812e2c009bSjjc 	/*
6822e2c009bSjjc 	 * XXPV	For now, the hypervisor treats all memory equally.
6832e2c009bSjjc 	 */
6842e2c009bSjjc 	lgrp_plat_node_cnt = max_mem_nodes = 1;
6852e2c009bSjjc #else	/* __xpv */
6862e2c009bSjjc 	uint_t	probe_op;
6872e2c009bSjjc 
6882e2c009bSjjc 	/*
6892e2c009bSjjc 	 * Initialize as a UMA machine
6902e2c009bSjjc 	 */
6912e2c009bSjjc 	if (lgrp_topo_ht_limit() == 1) {
6922e2c009bSjjc 		lgrp_plat_node_cnt = max_mem_nodes = 1;
6932e2c009bSjjc 		return;
6942e2c009bSjjc 	}
6952e2c009bSjjc 
6962e2c009bSjjc 	/*
697dae2fa37Sjjc 	 * Read boot property with CPU to APIC ID mapping table/array and fill
698dae2fa37Sjjc 	 * in CPU to node ID mapping table with APIC ID for each CPU
699dae2fa37Sjjc 	 */
700d821f0f0Sjjc 	lgrp_plat_apic_ncpus =
701d821f0f0Sjjc 	    lgrp_plat_process_cpu_apicids(lgrp_plat_cpu_node);
702dae2fa37Sjjc 
703dae2fa37Sjjc 	/*
7042e2c009bSjjc 	 * Determine which CPUs and memory are local to each other and number
7052e2c009bSjjc 	 * of NUMA nodes by reading ACPI System Resource Affinity Table (SRAT)
7062e2c009bSjjc 	 */
707d821f0f0Sjjc 	if (lgrp_plat_apic_ncpus > 0) {
708d821f0f0Sjjc 		int	retval;
709d821f0f0Sjjc 
710d821f0f0Sjjc 		retval = lgrp_plat_process_srat(srat_ptr,
711d821f0f0Sjjc 		    lgrp_plat_node_domain, lgrp_plat_cpu_node,
712d821f0f0Sjjc 		    lgrp_plat_apic_ncpus, lgrp_plat_node_memory);
713d821f0f0Sjjc 		if (retval <= 0) {
714d821f0f0Sjjc 			lgrp_plat_srat_error = retval;
715d821f0f0Sjjc 			lgrp_plat_node_cnt = 1;
716d821f0f0Sjjc 		} else {
717d821f0f0Sjjc 			lgrp_plat_srat_error = 0;
718d821f0f0Sjjc 			lgrp_plat_node_cnt = retval;
719d821f0f0Sjjc 		}
720dae2fa37Sjjc 	}
7212e2c009bSjjc 
7222e2c009bSjjc 	/*
723dae2fa37Sjjc 	 * Try to use PCI config space registers on Opteron if there's an error
724dae2fa37Sjjc 	 * processing CPU to APIC ID mapping or SRAT
7252e2c009bSjjc 	 */
726d821f0f0Sjjc 	if ((lgrp_plat_apic_ncpus <= 0 || lgrp_plat_srat_error != 0) &&
727dae2fa37Sjjc 	    is_opteron())
7282e2c009bSjjc 		opt_get_numa_config(&lgrp_plat_node_cnt, &lgrp_plat_mem_intrlv,
7292e2c009bSjjc 		    lgrp_plat_node_memory);
7302e2c009bSjjc 
7312e2c009bSjjc 	/*
7322e2c009bSjjc 	 * Don't bother to setup system for multiple lgroups and only use one
7332e2c009bSjjc 	 * memory node when memory is interleaved between any nodes or there is
7342e2c009bSjjc 	 * only one NUMA node
7352e2c009bSjjc 	 *
7362e2c009bSjjc 	 * NOTE: May need to change this for Dynamic Reconfiguration (DR)
7372e2c009bSjjc 	 *	 when and if it happens for x86/x64
7382e2c009bSjjc 	 */
7392e2c009bSjjc 	if (lgrp_plat_mem_intrlv || lgrp_plat_node_cnt == 1) {
7402e2c009bSjjc 		lgrp_plat_node_cnt = max_mem_nodes = 1;
7412e2c009bSjjc 		(void) lgrp_topo_ht_limit_set(1);
7422e2c009bSjjc 		return;
7432e2c009bSjjc 	}
7442e2c009bSjjc 
7452e2c009bSjjc 	/*
7462e2c009bSjjc 	 * Leaf lgroups on x86/x64 architectures contain one physical
7472e2c009bSjjc 	 * processor chip. Tune lgrp_expand_proc_thresh and
7482e2c009bSjjc 	 * lgrp_expand_proc_diff so that lgrp_choose() will spread
7492e2c009bSjjc 	 * things out aggressively.
7502e2c009bSjjc 	 */
7512e2c009bSjjc 	lgrp_expand_proc_thresh = LGRP_LOADAVG_THREAD_MAX / 2;
7522e2c009bSjjc 	lgrp_expand_proc_diff = 0;
7532e2c009bSjjc 
7542e2c009bSjjc 	/*
7552e2c009bSjjc 	 * There should be one memnode (physical page free list(s)) for
7562e2c009bSjjc 	 * each node
7572e2c009bSjjc 	 */
7582e2c009bSjjc 	max_mem_nodes = lgrp_plat_node_cnt;
7592e2c009bSjjc 
7602e2c009bSjjc 	/*
7615b7cf7f0Sjjc 	 * Initialize min and max latency before reading SLIT or probing
7625b7cf7f0Sjjc 	 */
7635b7cf7f0Sjjc 	lgrp_plat_lat_stats.latency_min = -1;
7645b7cf7f0Sjjc 	lgrp_plat_lat_stats.latency_max = 0;
7655b7cf7f0Sjjc 
7665b7cf7f0Sjjc 	/*
7672e2c009bSjjc 	 * Determine how far each NUMA node is from each other by
7682e2c009bSjjc 	 * reading ACPI System Locality Information Table (SLIT) if it
7692e2c009bSjjc 	 * exists
7702e2c009bSjjc 	 */
7712e2c009bSjjc 	lgrp_plat_slit_error = lgrp_plat_process_slit(slit_ptr,
7722e2c009bSjjc 	    lgrp_plat_node_cnt, lgrp_plat_node_memory,
7732e2c009bSjjc 	    &lgrp_plat_lat_stats);
7742e2c009bSjjc 	if (lgrp_plat_slit_error == 0)
7752e2c009bSjjc 		return;
7762e2c009bSjjc 
7772e2c009bSjjc 	/*
7782e2c009bSjjc 	 * Probe to determine latency between NUMA nodes when SLIT
7792e2c009bSjjc 	 * doesn't exist or make sense
7802e2c009bSjjc 	 */
7812e2c009bSjjc 	lgrp_plat_probe_flags |= LGRP_PLAT_PROBE_ENABLE;
7822e2c009bSjjc 
7832e2c009bSjjc 	/*
7842e2c009bSjjc 	 * Specify whether to probe using vendor ID register or page copy
7852e2c009bSjjc 	 * if hasn't been specified already or is overspecified
7862e2c009bSjjc 	 */
7872e2c009bSjjc 	probe_op = lgrp_plat_probe_flags &
7882e2c009bSjjc 	    (LGRP_PLAT_PROBE_PGCPY|LGRP_PLAT_PROBE_VENDOR);
7892e2c009bSjjc 
7902e2c009bSjjc 	if (probe_op == 0 ||
7912e2c009bSjjc 	    probe_op == (LGRP_PLAT_PROBE_PGCPY|LGRP_PLAT_PROBE_VENDOR)) {
7922e2c009bSjjc 		lgrp_plat_probe_flags &=
7932e2c009bSjjc 		    ~(LGRP_PLAT_PROBE_PGCPY|LGRP_PLAT_PROBE_VENDOR);
7942e2c009bSjjc 		if (is_opteron())
7952e2c009bSjjc 			lgrp_plat_probe_flags |=
7962e2c009bSjjc 			    LGRP_PLAT_PROBE_VENDOR;
7972e2c009bSjjc 		else
7982e2c009bSjjc 			lgrp_plat_probe_flags |= LGRP_PLAT_PROBE_PGCPY;
7992e2c009bSjjc 	}
8002e2c009bSjjc 
8012e2c009bSjjc 	/*
8022e2c009bSjjc 	 * Probing errors can mess up the lgroup topology and
8032e2c009bSjjc 	 * force us fall back to a 2 level lgroup topology.
8042e2c009bSjjc 	 * Here we bound how tall the lgroup topology can grow
8052e2c009bSjjc 	 * in hopes of avoiding any anamolies in probing from
8062e2c009bSjjc 	 * messing up the lgroup topology by limiting the
8072e2c009bSjjc 	 * accuracy of the latency topology.
8082e2c009bSjjc 	 *
8092e2c009bSjjc 	 * Assume that nodes will at least be configured in a
8102e2c009bSjjc 	 * ring, so limit height of lgroup topology to be less
8112e2c009bSjjc 	 * than number of nodes on a system with 4 or more
8122e2c009bSjjc 	 * nodes
8132e2c009bSjjc 	 */
8142e2c009bSjjc 	if (lgrp_plat_node_cnt >= 4 && lgrp_topo_ht_limit() ==
8152e2c009bSjjc 	    lgrp_topo_ht_limit_default())
8162e2c009bSjjc 		(void) lgrp_topo_ht_limit_set(lgrp_plat_node_cnt - 1);
8172e2c009bSjjc #endif	/* __xpv */
8182e2c009bSjjc }
8192e2c009bSjjc 
8202e2c009bSjjc 
8212e2c009bSjjc /*
8222e2c009bSjjc  * Return latency between "from" and "to" lgroups
8232e2c009bSjjc  *
8242e2c009bSjjc  * This latency number can only be used for relative comparison
8252e2c009bSjjc  * between lgroups on the running system, cannot be used across platforms,
8262e2c009bSjjc  * and may not reflect the actual latency.  It is platform and implementation
8272e2c009bSjjc  * specific, so platform gets to decide its value.  It would be nice if the
8282e2c009bSjjc  * number was at least proportional to make comparisons more meaningful though.
8292e2c009bSjjc  */
8302e2c009bSjjc /* ARGSUSED */
8312e2c009bSjjc int
8322e2c009bSjjc lgrp_plat_latency(lgrp_handle_t from, lgrp_handle_t to)
8332e2c009bSjjc {
8342e2c009bSjjc 	lgrp_handle_t	src, dest;
8352e2c009bSjjc 	int		node;
8362e2c009bSjjc 
8372e2c009bSjjc 	if (max_mem_nodes == 1)
8382e2c009bSjjc 		return (0);
8392e2c009bSjjc 
8402e2c009bSjjc 	/*
8412e2c009bSjjc 	 * Return max latency for root lgroup
8422e2c009bSjjc 	 */
8432e2c009bSjjc 	if (from == LGRP_DEFAULT_HANDLE || to == LGRP_DEFAULT_HANDLE)
8442e2c009bSjjc 		return (lgrp_plat_lat_stats.latency_max);
8452e2c009bSjjc 
8462e2c009bSjjc 	src = from;
8472e2c009bSjjc 	dest = to;
8482e2c009bSjjc 
8492e2c009bSjjc 	/*
8502e2c009bSjjc 	 * Return 0 for nodes (lgroup platform handles) out of range
8512e2c009bSjjc 	 */
8522e2c009bSjjc 	if (src < 0 || src >= MAX_NODES || dest < 0 || dest >= MAX_NODES)
8532e2c009bSjjc 		return (0);
8542e2c009bSjjc 
8552e2c009bSjjc 	/*
8562e2c009bSjjc 	 * Probe from current CPU if its lgroup latencies haven't been set yet
8572e2c009bSjjc 	 * and we are trying to get latency from current CPU to some node
8582e2c009bSjjc 	 */
8592e2c009bSjjc 	node = lgrp_plat_cpu_to_node(CPU, lgrp_plat_cpu_node);
8602e2c009bSjjc 	ASSERT(node >= 0 && node < lgrp_plat_node_cnt);
8612e2c009bSjjc 	if (lgrp_plat_lat_stats.latencies[src][src] == 0 && node == src)
8622e2c009bSjjc 		lgrp_plat_probe();
8632e2c009bSjjc 
8642e2c009bSjjc 	return (lgrp_plat_lat_stats.latencies[src][dest]);
8652e2c009bSjjc }
8662e2c009bSjjc 
8672e2c009bSjjc 
8682e2c009bSjjc /*
8692e2c009bSjjc  * Platform-specific initialization
8702e2c009bSjjc  */
8712e2c009bSjjc void
8722e2c009bSjjc lgrp_plat_main_init(void)
8732e2c009bSjjc {
8742e2c009bSjjc 	int	curnode;
8752e2c009bSjjc 	int	ht_limit;
8762e2c009bSjjc 	int	i;
8772e2c009bSjjc 
8782e2c009bSjjc 	/*
8792e2c009bSjjc 	 * Print a notice that MPO is disabled when memory is interleaved
8802e2c009bSjjc 	 * across nodes....Would do this when it is discovered, but can't
8812e2c009bSjjc 	 * because it happens way too early during boot....
8822e2c009bSjjc 	 */
8832e2c009bSjjc 	if (lgrp_plat_mem_intrlv)
8842e2c009bSjjc 		cmn_err(CE_NOTE,
8852e2c009bSjjc 		    "MPO disabled because memory is interleaved\n");
8862e2c009bSjjc 
8872e2c009bSjjc 	/*
8882e2c009bSjjc 	 * Don't bother to do any probing if it is disabled, there is only one
8892e2c009bSjjc 	 * node, or the height of the lgroup topology less than or equal to 2
8902e2c009bSjjc 	 */
8912e2c009bSjjc 	ht_limit = lgrp_topo_ht_limit();
8922e2c009bSjjc 	if (!(lgrp_plat_probe_flags & LGRP_PLAT_PROBE_ENABLE) ||
8932e2c009bSjjc 	    max_mem_nodes == 1 || ht_limit <= 2) {
8942e2c009bSjjc 		/*
8952e2c009bSjjc 		 * Setup lgroup latencies for 2 level lgroup topology
8962e2c009bSjjc 		 * (ie. local and remote only) if they haven't been set yet
8972e2c009bSjjc 		 */
8982e2c009bSjjc 		if (ht_limit == 2 && lgrp_plat_lat_stats.latency_min == -1 &&
8992e2c009bSjjc 		    lgrp_plat_lat_stats.latency_max == 0)
9002e2c009bSjjc 			lgrp_plat_2level_setup(lgrp_plat_node_memory,
9012e2c009bSjjc 			    &lgrp_plat_lat_stats);
9022e2c009bSjjc 		return;
9032e2c009bSjjc 	}
9042e2c009bSjjc 
9052e2c009bSjjc 	if (lgrp_plat_probe_flags & LGRP_PLAT_PROBE_VENDOR) {
9062e2c009bSjjc 		/*
9072e2c009bSjjc 		 * Should have been able to probe from CPU 0 when it was added
9082e2c009bSjjc 		 * to lgroup hierarchy, but may not have been able to then
9092e2c009bSjjc 		 * because it happens so early in boot that gethrtime() hasn't
9102e2c009bSjjc 		 * been initialized.  (:-(
9112e2c009bSjjc 		 */
9122e2c009bSjjc 		curnode = lgrp_plat_cpu_to_node(CPU, lgrp_plat_cpu_node);
9132e2c009bSjjc 		ASSERT(curnode >= 0 && curnode < lgrp_plat_node_cnt);
9142e2c009bSjjc 		if (lgrp_plat_lat_stats.latencies[curnode][curnode] == 0)
9152e2c009bSjjc 			lgrp_plat_probe();
9162e2c009bSjjc 
9172e2c009bSjjc 		return;
9182e2c009bSjjc 	}
9192e2c009bSjjc 
9202e2c009bSjjc 	/*
9212e2c009bSjjc 	 * When probing memory, use one page for every sample to determine
9222e2c009bSjjc 	 * lgroup topology and taking multiple samples
9232e2c009bSjjc 	 */
9242e2c009bSjjc 	if (lgrp_plat_probe_mem_config.probe_memsize == 0)
9252e2c009bSjjc 		lgrp_plat_probe_mem_config.probe_memsize = PAGESIZE *
9262e2c009bSjjc 		    lgrp_plat_probe_nsamples;
9272e2c009bSjjc 
9282e2c009bSjjc 	/*
9292e2c009bSjjc 	 * Map memory in each node needed for probing to determine latency
9302e2c009bSjjc 	 * topology
9312e2c009bSjjc 	 */
9322e2c009bSjjc 	for (i = 0; i < lgrp_plat_node_cnt; i++) {
9332e2c009bSjjc 		int	mnode;
9342e2c009bSjjc 
9352e2c009bSjjc 		/*
9362e2c009bSjjc 		 * Skip this node and leave its probe page NULL
9372e2c009bSjjc 		 * if it doesn't have any memory
9382e2c009bSjjc 		 */
9392e2c009bSjjc 		mnode = plat_lgrphand_to_mem_node((lgrp_handle_t)i);
9402e2c009bSjjc 		if (!mem_node_config[mnode].exists) {
9412e2c009bSjjc 			lgrp_plat_probe_mem_config.probe_va[i] = NULL;
9422e2c009bSjjc 			continue;
9432e2c009bSjjc 		}
9442e2c009bSjjc 
9452e2c009bSjjc 		/*
9462e2c009bSjjc 		 * Allocate one kernel virtual page
9472e2c009bSjjc 		 */
9482e2c009bSjjc 		lgrp_plat_probe_mem_config.probe_va[i] = vmem_alloc(heap_arena,
9492e2c009bSjjc 		    lgrp_plat_probe_mem_config.probe_memsize, VM_NOSLEEP);
9502e2c009bSjjc 		if (lgrp_plat_probe_mem_config.probe_va[i] == NULL) {
9512e2c009bSjjc 			cmn_err(CE_WARN,
9522e2c009bSjjc 			    "lgrp_plat_main_init: couldn't allocate memory");
9532e2c009bSjjc 			return;
9542e2c009bSjjc 		}
9552e2c009bSjjc 
9562e2c009bSjjc 		/*
9572e2c009bSjjc 		 * Get PFN for first page in each node
9582e2c009bSjjc 		 */
9592e2c009bSjjc 		lgrp_plat_probe_mem_config.probe_pfn[i] =
9602e2c009bSjjc 		    mem_node_config[mnode].physbase;
9612e2c009bSjjc 
9622e2c009bSjjc 		/*
9632e2c009bSjjc 		 * Map virtual page to first page in node
9642e2c009bSjjc 		 */
9652e2c009bSjjc 		hat_devload(kas.a_hat, lgrp_plat_probe_mem_config.probe_va[i],
9662e2c009bSjjc 		    lgrp_plat_probe_mem_config.probe_memsize,
9672e2c009bSjjc 		    lgrp_plat_probe_mem_config.probe_pfn[i],
9682e2c009bSjjc 		    PROT_READ | PROT_WRITE | HAT_PLAT_NOCACHE,
9692e2c009bSjjc 		    HAT_LOAD_NOCONSIST);
9702e2c009bSjjc 	}
9712e2c009bSjjc 
9722e2c009bSjjc 	/*
9732e2c009bSjjc 	 * Probe from current CPU
9742e2c009bSjjc 	 */
9752e2c009bSjjc 	lgrp_plat_probe();
9762e2c009bSjjc }
9772e2c009bSjjc 
9782e2c009bSjjc 
9792e2c009bSjjc /*
9802e2c009bSjjc  * Return the maximum number of lgrps supported by the platform.
9812e2c009bSjjc  * Before lgrp topology is known it returns an estimate based on the number of
9822e2c009bSjjc  * nodes. Once topology is known it returns the actual maximim number of lgrps
9832e2c009bSjjc  * created. Since x86/x64 doesn't support Dynamic Reconfiguration (DR) and
9842e2c009bSjjc  * dynamic addition of new nodes, this number may not grow during system
9852e2c009bSjjc  * lifetime (yet).
9862e2c009bSjjc  */
9872e2c009bSjjc int
9882e2c009bSjjc lgrp_plat_max_lgrps(void)
9892e2c009bSjjc {
9902e2c009bSjjc 	return (lgrp_topo_initialized ?
9912e2c009bSjjc 	    lgrp_alloc_max + 1 :
9922e2c009bSjjc 	    lgrp_plat_node_cnt * (lgrp_plat_node_cnt - 1) + 1);
9932e2c009bSjjc }
9942e2c009bSjjc 
9952e2c009bSjjc 
9962e2c009bSjjc /*
9972e2c009bSjjc  * Return the number of free pages in an lgroup.
9982e2c009bSjjc  *
9992e2c009bSjjc  * For query of LGRP_MEM_SIZE_FREE, return the number of base pagesize
10002e2c009bSjjc  * pages on freelists.  For query of LGRP_MEM_SIZE_AVAIL, return the
10012e2c009bSjjc  * number of allocatable base pagesize pages corresponding to the
10022e2c009bSjjc  * lgroup (e.g. do not include page_t's, BOP_ALLOC()'ed memory, ..)
10032e2c009bSjjc  * For query of LGRP_MEM_SIZE_INSTALL, return the amount of physical
10042e2c009bSjjc  * memory installed, regardless of whether or not it's usable.
10052e2c009bSjjc  */
10062e2c009bSjjc pgcnt_t
10072e2c009bSjjc lgrp_plat_mem_size(lgrp_handle_t plathand, lgrp_mem_query_t query)
10082e2c009bSjjc {
10092e2c009bSjjc 	int	mnode;
10102e2c009bSjjc 	pgcnt_t npgs = (pgcnt_t)0;
10112e2c009bSjjc 	extern struct memlist *phys_avail;
10122e2c009bSjjc 	extern struct memlist *phys_install;
10132e2c009bSjjc 
10142e2c009bSjjc 
10152e2c009bSjjc 	if (plathand == LGRP_DEFAULT_HANDLE)
10162e2c009bSjjc 		return (lgrp_plat_mem_size_default(plathand, query));
10172e2c009bSjjc 
10182e2c009bSjjc 	if (plathand != LGRP_NULL_HANDLE) {
10192e2c009bSjjc 		mnode = plat_lgrphand_to_mem_node(plathand);
10202e2c009bSjjc 		if (mnode >= 0 && mem_node_config[mnode].exists) {
10212e2c009bSjjc 			switch (query) {
10222e2c009bSjjc 			case LGRP_MEM_SIZE_FREE:
10232e2c009bSjjc 				npgs = MNODE_PGCNT(mnode);
10242e2c009bSjjc 				break;
10252e2c009bSjjc 			case LGRP_MEM_SIZE_AVAIL:
10262e2c009bSjjc 				npgs = mem_node_memlist_pages(mnode,
10272e2c009bSjjc 				    phys_avail);
10282e2c009bSjjc 				break;
10292e2c009bSjjc 			case LGRP_MEM_SIZE_INSTALL:
10302e2c009bSjjc 				npgs = mem_node_memlist_pages(mnode,
10312e2c009bSjjc 				    phys_install);
10322e2c009bSjjc 				break;
10332e2c009bSjjc 			default:
10342e2c009bSjjc 				break;
10352e2c009bSjjc 			}
10362e2c009bSjjc 		}
10372e2c009bSjjc 	}
10382e2c009bSjjc 	return (npgs);
10392e2c009bSjjc }
10402e2c009bSjjc 
10412e2c009bSjjc 
10422e2c009bSjjc /*
10432e2c009bSjjc  * Return the platform handle of the lgroup that contains the physical memory
10442e2c009bSjjc  * corresponding to the given page frame number
10452e2c009bSjjc  */
10462e2c009bSjjc /* ARGSUSED */
10472e2c009bSjjc lgrp_handle_t
10482e2c009bSjjc lgrp_plat_pfn_to_hand(pfn_t pfn)
10492e2c009bSjjc {
10502e2c009bSjjc 	int	mnode;
10512e2c009bSjjc 
10522e2c009bSjjc 	if (max_mem_nodes == 1)
10532e2c009bSjjc 		return (LGRP_DEFAULT_HANDLE);
10542e2c009bSjjc 
10552e2c009bSjjc 	if (pfn > physmax)
10562e2c009bSjjc 		return (LGRP_NULL_HANDLE);
10572e2c009bSjjc 
10582e2c009bSjjc 	mnode = plat_pfn_to_mem_node(pfn);
10592e2c009bSjjc 	if (mnode < 0)
10602e2c009bSjjc 		return (LGRP_NULL_HANDLE);
10612e2c009bSjjc 
10622e2c009bSjjc 	return (MEM_NODE_2_LGRPHAND(mnode));
10632e2c009bSjjc }
10642e2c009bSjjc 
10652e2c009bSjjc 
10662e2c009bSjjc /*
10672e2c009bSjjc  * Probe memory in each node from current CPU to determine latency topology
10682e2c009bSjjc  *
10692e2c009bSjjc  * The probing code will probe the vendor ID register on the Northbridge of
10702e2c009bSjjc  * Opteron processors and probe memory for other processors by default.
10712e2c009bSjjc  *
10722e2c009bSjjc  * Since probing is inherently error prone, the code takes laps across all the
10732e2c009bSjjc  * nodes probing from each node to each of the other nodes some number of
10742e2c009bSjjc  * times.  Furthermore, each node is probed some number of times before moving
10752e2c009bSjjc  * onto the next one during each lap.  The minimum latency gotten between nodes
10762e2c009bSjjc  * is kept as the latency between the nodes.
10772e2c009bSjjc  *
10782e2c009bSjjc  * After all that,  the probe times are adjusted by normalizing values that are
10792e2c009bSjjc  * close to each other and local latencies are made the same.  Lastly, the
10802e2c009bSjjc  * latencies are verified to make sure that certain conditions are met (eg.
10812e2c009bSjjc  * local < remote, latency(a, b) == latency(b, a), etc.).
10822e2c009bSjjc  *
10832e2c009bSjjc  * If any of the conditions aren't met, the code will export a NUMA
10842e2c009bSjjc  * configuration with the local CPUs and memory given by the SRAT or PCI config
10852e2c009bSjjc  * space registers and one remote memory latency since it can't tell exactly
10862e2c009bSjjc  * how far each node is from each other.
10872e2c009bSjjc  */
10882e2c009bSjjc void
10892e2c009bSjjc lgrp_plat_probe(void)
10902e2c009bSjjc {
10912e2c009bSjjc 	int				from;
10922e2c009bSjjc 	int				i;
10932e2c009bSjjc 	lgrp_plat_latency_stats_t	*lat_stats;
10942e2c009bSjjc 	hrtime_t			probe_time;
10952e2c009bSjjc 	int				to;
10962e2c009bSjjc 
10972e2c009bSjjc 	if (!(lgrp_plat_probe_flags & LGRP_PLAT_PROBE_ENABLE) ||
10982e2c009bSjjc 	    max_mem_nodes == 1 || lgrp_topo_ht_limit() <= 2)
10992e2c009bSjjc 		return;
11002e2c009bSjjc 
11012e2c009bSjjc 	/*
11022e2c009bSjjc 	 * Determine ID of node containing current CPU
11032e2c009bSjjc 	 */
11042e2c009bSjjc 	from = lgrp_plat_cpu_to_node(CPU, lgrp_plat_cpu_node);
11052e2c009bSjjc 	ASSERT(from >= 0 && from < lgrp_plat_node_cnt);
11062e2c009bSjjc 	if (srat_ptr && lgrp_plat_srat_enable && !lgrp_plat_srat_error)
11072e2c009bSjjc 		ASSERT(lgrp_plat_node_domain[from].exists);
11082e2c009bSjjc 
11092e2c009bSjjc 	/*
11102e2c009bSjjc 	 * Don't need to probe if got times already
11112e2c009bSjjc 	 */
11122e2c009bSjjc 	lat_stats = &lgrp_plat_lat_stats;
11132e2c009bSjjc 	if (lat_stats->latencies[from][from] != 0)
11142e2c009bSjjc 		return;
11152e2c009bSjjc 
11162e2c009bSjjc 	/*
11172e2c009bSjjc 	 * Read vendor ID in Northbridge or read and write page(s)
11182e2c009bSjjc 	 * in each node from current CPU and remember how long it takes,
11192e2c009bSjjc 	 * so we can build latency topology of machine later.
11202e2c009bSjjc 	 * This should approximate the memory latency between each node.
11212e2c009bSjjc 	 */
11222e2c009bSjjc 	for (i = 0; i < lgrp_plat_probe_nrounds; i++) {
11232e2c009bSjjc 		for (to = 0; to < lgrp_plat_node_cnt; to++) {
11242e2c009bSjjc 			/*
11252e2c009bSjjc 			 * Get probe time and bail out if can't get it yet
11262e2c009bSjjc 			 */
11272e2c009bSjjc 			probe_time = lgrp_plat_probe_time(to,
11282e2c009bSjjc 			    lgrp_plat_cpu_node, &lgrp_plat_probe_mem_config,
11292e2c009bSjjc 			    &lgrp_plat_lat_stats, &lgrp_plat_probe_stats);
11302e2c009bSjjc 			if (probe_time == 0)
11312e2c009bSjjc 				return;
11322e2c009bSjjc 
11332e2c009bSjjc 			/*
11342e2c009bSjjc 			 * Keep lowest probe time as latency between nodes
11352e2c009bSjjc 			 */
11362e2c009bSjjc 			if (lat_stats->latencies[from][to] == 0 ||
11372e2c009bSjjc 			    probe_time < lat_stats->latencies[from][to])
11382e2c009bSjjc 				lat_stats->latencies[from][to] = probe_time;
11392e2c009bSjjc 
11402e2c009bSjjc 			/*
11412e2c009bSjjc 			 * Update overall minimum and maximum probe times
11422e2c009bSjjc 			 * across all nodes
11432e2c009bSjjc 			 */
11442e2c009bSjjc 			if (probe_time < lat_stats->latency_min ||
11452e2c009bSjjc 			    lat_stats->latency_min == -1)
11462e2c009bSjjc 				lat_stats->latency_min = probe_time;
11472e2c009bSjjc 			if (probe_time > lat_stats->latency_max)
11482e2c009bSjjc 				lat_stats->latency_max = probe_time;
11492e2c009bSjjc 		}
11502e2c009bSjjc 	}
11512e2c009bSjjc 
11522e2c009bSjjc 	/*
11532e2c009bSjjc 	 * - Fix up latencies such that local latencies are same,
11542e2c009bSjjc 	 *   latency(i, j) == latency(j, i), etc. (if possible)
11552e2c009bSjjc 	 *
11562e2c009bSjjc 	 * - Verify that latencies look ok
11572e2c009bSjjc 	 *
11582e2c009bSjjc 	 * - Fallback to just optimizing for local and remote if
11592e2c009bSjjc 	 *   latencies didn't look right
11602e2c009bSjjc 	 */
11612e2c009bSjjc 	lgrp_plat_latency_adjust(lgrp_plat_node_memory, &lgrp_plat_lat_stats,
11622e2c009bSjjc 	    &lgrp_plat_probe_stats);
11632e2c009bSjjc 	lgrp_plat_probe_stats.probe_error_code =
11642e2c009bSjjc 	    lgrp_plat_latency_verify(lgrp_plat_node_memory,
11652e2c009bSjjc 	    &lgrp_plat_lat_stats);
11662e2c009bSjjc 	if (lgrp_plat_probe_stats.probe_error_code)
11672e2c009bSjjc 		lgrp_plat_2level_setup(lgrp_plat_node_memory,
11682e2c009bSjjc 		    &lgrp_plat_lat_stats);
11692e2c009bSjjc }
11702e2c009bSjjc 
11712e2c009bSjjc 
11722e2c009bSjjc /*
11732e2c009bSjjc  * Return platform handle for root lgroup
11742e2c009bSjjc  */
11752e2c009bSjjc lgrp_handle_t
11762e2c009bSjjc lgrp_plat_root_hand(void)
11772e2c009bSjjc {
11782e2c009bSjjc 	return (LGRP_DEFAULT_HANDLE);
11792e2c009bSjjc }
11802e2c009bSjjc 
11812e2c009bSjjc 
11822e2c009bSjjc /*
11832e2c009bSjjc  * INTERNAL ROUTINES
11842e2c009bSjjc  */
11852e2c009bSjjc 
11862e2c009bSjjc 
11872e2c009bSjjc /*
11882e2c009bSjjc  * Update CPU to node mapping for given CPU and proximity domain (and returns
11892e2c009bSjjc  * negative numbers for errors and positive ones for success)
11902e2c009bSjjc  */
11912e2c009bSjjc static int
1192d821f0f0Sjjc lgrp_plat_cpu_node_update(node_domain_map_t *node_domain, int node_cnt,
1193dae2fa37Sjjc     cpu_node_map_t *cpu_node, int nentries, uint32_t apicid, uint32_t domain)
11942e2c009bSjjc {
11952e2c009bSjjc 	uint_t	i;
11962e2c009bSjjc 	int	node;
11972e2c009bSjjc 
11982e2c009bSjjc 	/*
11992e2c009bSjjc 	 * Get node number for proximity domain
12002e2c009bSjjc 	 */
1201d821f0f0Sjjc 	node = lgrp_plat_domain_to_node(node_domain, node_cnt, domain);
12022e2c009bSjjc 	if (node == -1) {
1203d821f0f0Sjjc 		node = lgrp_plat_node_domain_update(node_domain, node_cnt,
1204d821f0f0Sjjc 		    domain);
12052e2c009bSjjc 		if (node == -1)
12062e2c009bSjjc 			return (-1);
12072e2c009bSjjc 	}
12082e2c009bSjjc 
12092e2c009bSjjc 	/*
1210dae2fa37Sjjc 	 * Search for entry with given APIC ID and fill in its node and
1211dae2fa37Sjjc 	 * proximity domain IDs (if they haven't been set already)
12122e2c009bSjjc 	 */
1213dae2fa37Sjjc 	for (i = 0; i < nentries; i++) {
12142e2c009bSjjc 		/*
1215dae2fa37Sjjc 		 * Skip nonexistent entries and ones without matching APIC ID
12162e2c009bSjjc 		 */
1217dae2fa37Sjjc 		if (!cpu_node[i].exists || cpu_node[i].apicid != apicid)
1218dae2fa37Sjjc 			continue;
1219dae2fa37Sjjc 
12202e2c009bSjjc 		/*
1221dae2fa37Sjjc 		 * Just return if entry completely and correctly filled in
1222dae2fa37Sjjc 		 * already
12232e2c009bSjjc 		 */
12242e2c009bSjjc 		if (cpu_node[i].prox_domain == domain &&
12252e2c009bSjjc 		    cpu_node[i].node == node)
12262e2c009bSjjc 			return (1);
12272e2c009bSjjc 
12282e2c009bSjjc 		/*
1229dae2fa37Sjjc 		 * Fill in node and proximity domain IDs
12302e2c009bSjjc 		 */
12312e2c009bSjjc 		cpu_node[i].prox_domain = domain;
12322e2c009bSjjc 		cpu_node[i].node = node;
1233dae2fa37Sjjc 
12342e2c009bSjjc 		return (0);
12352e2c009bSjjc 	}
12362e2c009bSjjc 
12372e2c009bSjjc 	/*
1238dae2fa37Sjjc 	 * Return error when entry for APIC ID wasn't found in table
12392e2c009bSjjc 	 */
1240dae2fa37Sjjc 	return (-2);
12412e2c009bSjjc }
12422e2c009bSjjc 
12432e2c009bSjjc 
12442e2c009bSjjc /*
1245dae2fa37Sjjc  * Get node ID for given CPU
12462e2c009bSjjc  */
12472e2c009bSjjc static int
12482e2c009bSjjc lgrp_plat_cpu_to_node(cpu_t *cp, cpu_node_map_t *cpu_node)
12492e2c009bSjjc {
1250dae2fa37Sjjc 	processorid_t	cpuid;
12512e2c009bSjjc 
12522e2c009bSjjc 	if (cp == NULL)
12532e2c009bSjjc 		return (-1);
12542e2c009bSjjc 
1255dae2fa37Sjjc 	cpuid = cp->cpu_id;
1256dae2fa37Sjjc 	if (cpuid < 0 || cpuid >= max_ncpus)
1257dae2fa37Sjjc 		return (-1);
1258dae2fa37Sjjc 
12592e2c009bSjjc 	/*
12602e2c009bSjjc 	 * SRAT doesn't exist, isn't enabled, or there was an error processing
12612e2c009bSjjc 	 * it, so return chip ID for Opteron and -1 otherwise.
12622e2c009bSjjc 	 */
12632e2c009bSjjc 	if (srat_ptr == NULL || !lgrp_plat_srat_enable ||
12642e2c009bSjjc 	    lgrp_plat_srat_error) {
12652e2c009bSjjc 		if (is_opteron())
12662e2c009bSjjc 			return (pg_plat_hw_instance_id(cp, PGHW_CHIP));
12672e2c009bSjjc 		return (-1);
12682e2c009bSjjc 	}
12692e2c009bSjjc 
12702e2c009bSjjc 	/*
1271dae2fa37Sjjc 	 * Return -1 when CPU to node ID mapping entry doesn't exist for given
1272dae2fa37Sjjc 	 * CPU
12732e2c009bSjjc 	 */
1274dae2fa37Sjjc 	if (!cpu_node[cpuid].exists)
12752e2c009bSjjc 		return (-1);
1276dae2fa37Sjjc 
1277dae2fa37Sjjc 	return (cpu_node[cpuid].node);
12782e2c009bSjjc }
12792e2c009bSjjc 
12802e2c009bSjjc 
12812e2c009bSjjc /*
12822e2c009bSjjc  * Return node number for given proximity domain/system locality
12832e2c009bSjjc  */
12842e2c009bSjjc static int
1285d821f0f0Sjjc lgrp_plat_domain_to_node(node_domain_map_t *node_domain, int node_cnt,
1286d821f0f0Sjjc     uint32_t domain)
12872e2c009bSjjc {
12882e2c009bSjjc 	uint_t	node;
12892e2c009bSjjc 	uint_t	start;
12902e2c009bSjjc 
12912e2c009bSjjc 	/*
12922e2c009bSjjc 	 * Hash proximity domain ID into node to domain mapping table (array),
12932e2c009bSjjc 	 * search for entry with matching proximity domain ID, and return index
12942e2c009bSjjc 	 * of matching entry as node ID.
12952e2c009bSjjc 	 */
1296d821f0f0Sjjc 	node = start = NODE_DOMAIN_HASH(domain, node_cnt);
12972e2c009bSjjc 	do {
12982e2c009bSjjc 		if (node_domain[node].prox_domain == domain &&
12992e2c009bSjjc 		    node_domain[node].exists)
13002e2c009bSjjc 			return (node);
1301d821f0f0Sjjc 		node = NODE_DOMAIN_HASH(node + 1, node_cnt);
13022e2c009bSjjc 	} while (node != start);
13032e2c009bSjjc 	return (-1);
13042e2c009bSjjc }
13052e2c009bSjjc 
13062e2c009bSjjc 
13072e2c009bSjjc /*
13082e2c009bSjjc  * Latencies must be within 1/(2**LGRP_LAT_TOLERANCE_SHIFT) of each other to
13092e2c009bSjjc  * be considered same
13102e2c009bSjjc  */
13112e2c009bSjjc #define	LGRP_LAT_TOLERANCE_SHIFT	4
13122e2c009bSjjc 
13132e2c009bSjjc int	lgrp_plat_probe_lt_shift = LGRP_LAT_TOLERANCE_SHIFT;
13142e2c009bSjjc 
13152e2c009bSjjc 
13162e2c009bSjjc /*
13172e2c009bSjjc  * Adjust latencies between nodes to be symmetric, normalize latencies between
13182e2c009bSjjc  * any nodes that are within some tolerance to be same, and make local
13192e2c009bSjjc  * latencies be same
13202e2c009bSjjc  */
13212e2c009bSjjc static void
13222e2c009bSjjc lgrp_plat_latency_adjust(node_phys_addr_map_t *node_memory,
13232e2c009bSjjc     lgrp_plat_latency_stats_t *lat_stats, lgrp_plat_probe_stats_t *probe_stats)
13242e2c009bSjjc {
13252e2c009bSjjc 	int				i;
13262e2c009bSjjc 	int				j;
13272e2c009bSjjc 	int				k;
13282e2c009bSjjc 	int				l;
13292e2c009bSjjc 	u_longlong_t			max;
13302e2c009bSjjc 	u_longlong_t			min;
13312e2c009bSjjc 	u_longlong_t			t;
13322e2c009bSjjc 	u_longlong_t			t1;
13332e2c009bSjjc 	u_longlong_t			t2;
13342e2c009bSjjc 	const lgrp_config_flag_t	cflag = LGRP_CONFIG_LAT_CHANGE_ALL;
13352e2c009bSjjc 	int				lat_corrected[MAX_NODES][MAX_NODES];
13362e2c009bSjjc 
13372e2c009bSjjc 	/*
13382e2c009bSjjc 	 * Nothing to do when this is an UMA machine or don't have args needed
13392e2c009bSjjc 	 */
13402e2c009bSjjc 	if (max_mem_nodes == 1)
13412e2c009bSjjc 		return;
13422e2c009bSjjc 
13432e2c009bSjjc 	ASSERT(node_memory != NULL && lat_stats != NULL &&
13442e2c009bSjjc 	    probe_stats != NULL);
13452e2c009bSjjc 
13462e2c009bSjjc 	/*
13472e2c009bSjjc 	 * Make sure that latencies are symmetric between any two nodes
13482e2c009bSjjc 	 * (ie. latency(node0, node1) == latency(node1, node0))
13492e2c009bSjjc 	 */
13502e2c009bSjjc 	for (i = 0; i < lgrp_plat_node_cnt; i++) {
13512e2c009bSjjc 		if (!node_memory[i].exists)
13522e2c009bSjjc 			continue;
13532e2c009bSjjc 
13542e2c009bSjjc 		for (j = 0; j < lgrp_plat_node_cnt; j++) {
13552e2c009bSjjc 			if (!node_memory[j].exists)
13562e2c009bSjjc 				continue;
13572e2c009bSjjc 
13582e2c009bSjjc 			t1 = lat_stats->latencies[i][j];
13592e2c009bSjjc 			t2 = lat_stats->latencies[j][i];
13602e2c009bSjjc 
13612e2c009bSjjc 			if (t1 == 0 || t2 == 0 || t1 == t2)
13622e2c009bSjjc 				continue;
13632e2c009bSjjc 
13642e2c009bSjjc 			/*
13652e2c009bSjjc 			 * Latencies should be same
13662e2c009bSjjc 			 * - Use minimum of two latencies which should be same
13672e2c009bSjjc 			 * - Track suspect probe times not within tolerance of
13682e2c009bSjjc 			 *   min value
13692e2c009bSjjc 			 * - Remember how much values are corrected by
13702e2c009bSjjc 			 */
13712e2c009bSjjc 			if (t1 > t2) {
13722e2c009bSjjc 				t = t2;
13732e2c009bSjjc 				probe_stats->probe_errors[i][j] += t1 - t2;
13742e2c009bSjjc 				if (t1 - t2 > t2 >> lgrp_plat_probe_lt_shift) {
13752e2c009bSjjc 					probe_stats->probe_suspect[i][j]++;
13762e2c009bSjjc 					probe_stats->probe_suspect[j][i]++;
13772e2c009bSjjc 				}
13782e2c009bSjjc 			} else if (t2 > t1) {
13792e2c009bSjjc 				t = t1;
13802e2c009bSjjc 				probe_stats->probe_errors[j][i] += t2 - t1;
13812e2c009bSjjc 				if (t2 - t1 > t1 >> lgrp_plat_probe_lt_shift) {
13822e2c009bSjjc 					probe_stats->probe_suspect[i][j]++;
13832e2c009bSjjc 					probe_stats->probe_suspect[j][i]++;
13842e2c009bSjjc 				}
13852e2c009bSjjc 			}
13862e2c009bSjjc 
13872e2c009bSjjc 			lat_stats->latencies[i][j] =
13882e2c009bSjjc 			    lat_stats->latencies[j][i] = t;
13892e2c009bSjjc 			lgrp_config(cflag, t1, t);
13902e2c009bSjjc 			lgrp_config(cflag, t2, t);
13912e2c009bSjjc 		}
13922e2c009bSjjc 	}
13932e2c009bSjjc 
13942e2c009bSjjc 	/*
13952e2c009bSjjc 	 * Keep track of which latencies get corrected
13962e2c009bSjjc 	 */
13972e2c009bSjjc 	for (i = 0; i < MAX_NODES; i++)
13982e2c009bSjjc 		for (j = 0; j < MAX_NODES; j++)
13992e2c009bSjjc 			lat_corrected[i][j] = 0;
14002e2c009bSjjc 
14012e2c009bSjjc 	/*
14022e2c009bSjjc 	 * For every two nodes, see whether there is another pair of nodes which
14032e2c009bSjjc 	 * are about the same distance apart and make the latencies be the same
14042e2c009bSjjc 	 * if they are close enough together
14052e2c009bSjjc 	 */
14062e2c009bSjjc 	for (i = 0; i < lgrp_plat_node_cnt; i++) {
14072e2c009bSjjc 		if (!node_memory[i].exists)
14082e2c009bSjjc 			continue;
14092e2c009bSjjc 		for (j = 0; j < lgrp_plat_node_cnt; j++) {
14102e2c009bSjjc 			if (!node_memory[j].exists)
14112e2c009bSjjc 				continue;
14122e2c009bSjjc 			/*
14132e2c009bSjjc 			 * Pick one pair of nodes (i, j)
14142e2c009bSjjc 			 * and get latency between them
14152e2c009bSjjc 			 */
14162e2c009bSjjc 			t1 = lat_stats->latencies[i][j];
14172e2c009bSjjc 
14182e2c009bSjjc 			/*
14192e2c009bSjjc 			 * Skip this pair of nodes if there isn't a latency
14202e2c009bSjjc 			 * for it yet
14212e2c009bSjjc 			 */
14222e2c009bSjjc 			if (t1 == 0)
14232e2c009bSjjc 				continue;
14242e2c009bSjjc 
14252e2c009bSjjc 			for (k = 0; k < lgrp_plat_node_cnt; k++) {
14262e2c009bSjjc 				if (!node_memory[k].exists)
14272e2c009bSjjc 					continue;
14282e2c009bSjjc 				for (l = 0; l < lgrp_plat_node_cnt; l++) {
14292e2c009bSjjc 					if (!node_memory[l].exists)
14302e2c009bSjjc 						continue;
14312e2c009bSjjc 					/*
14322e2c009bSjjc 					 * Pick another pair of nodes (k, l)
14332e2c009bSjjc 					 * not same as (i, j) and get latency
14342e2c009bSjjc 					 * between them
14352e2c009bSjjc 					 */
14362e2c009bSjjc 					if (k == i && l == j)
14372e2c009bSjjc 						continue;
14382e2c009bSjjc 
14392e2c009bSjjc 					t2 = lat_stats->latencies[k][l];
14402e2c009bSjjc 
14412e2c009bSjjc 					/*
14422e2c009bSjjc 					 * Skip this pair of nodes if there
14432e2c009bSjjc 					 * isn't a latency for it yet
14442e2c009bSjjc 					 */
14452e2c009bSjjc 
14462e2c009bSjjc 					if (t2 == 0)
14472e2c009bSjjc 						continue;
14482e2c009bSjjc 
14492e2c009bSjjc 					/*
14502e2c009bSjjc 					 * Skip nodes (k, l) if they already
14512e2c009bSjjc 					 * have same latency as (i, j) or
14522e2c009bSjjc 					 * their latency isn't close enough to
14532e2c009bSjjc 					 * be considered/made the same
14542e2c009bSjjc 					 */
14552e2c009bSjjc 					if (t1 == t2 || (t1 > t2 && t1 - t2 >
14562e2c009bSjjc 					    t1 >> lgrp_plat_probe_lt_shift) ||
14572e2c009bSjjc 					    (t2 > t1 && t2 - t1 >
14582e2c009bSjjc 					    t2 >> lgrp_plat_probe_lt_shift))
14592e2c009bSjjc 						continue;
14602e2c009bSjjc 
14612e2c009bSjjc 					/*
14622e2c009bSjjc 					 * Make latency(i, j) same as
14632e2c009bSjjc 					 * latency(k, l), try to use latency
14642e2c009bSjjc 					 * that has been adjusted already to get
14652e2c009bSjjc 					 * more consistency (if possible), and
14662e2c009bSjjc 					 * remember which latencies were
14672e2c009bSjjc 					 * adjusted for next time
14682e2c009bSjjc 					 */
14692e2c009bSjjc 					if (lat_corrected[i][j]) {
14702e2c009bSjjc 						t = t1;
14712e2c009bSjjc 						lgrp_config(cflag, t2, t);
14722e2c009bSjjc 						t2 = t;
14732e2c009bSjjc 					} else if (lat_corrected[k][l]) {
14742e2c009bSjjc 						t = t2;
14752e2c009bSjjc 						lgrp_config(cflag, t1, t);
14762e2c009bSjjc 						t1 = t;
14772e2c009bSjjc 					} else {
14782e2c009bSjjc 						if (t1 > t2)
14792e2c009bSjjc 							t = t2;
14802e2c009bSjjc 						else
14812e2c009bSjjc 							t = t1;
14822e2c009bSjjc 						lgrp_config(cflag, t1, t);
14832e2c009bSjjc 						lgrp_config(cflag, t2, t);
14842e2c009bSjjc 						t1 = t2 = t;
14852e2c009bSjjc 					}
14862e2c009bSjjc 
14872e2c009bSjjc 					lat_stats->latencies[i][j] =
14882e2c009bSjjc 					    lat_stats->latencies[k][l] = t;
14892e2c009bSjjc 
14902e2c009bSjjc 					lat_corrected[i][j] =
14912e2c009bSjjc 					    lat_corrected[k][l] = 1;
14922e2c009bSjjc 				}
14932e2c009bSjjc 			}
14942e2c009bSjjc 		}
14952e2c009bSjjc 	}
14962e2c009bSjjc 
14972e2c009bSjjc 	/*
14982e2c009bSjjc 	 * Local latencies should be same
14992e2c009bSjjc 	 * - Find min and max local latencies
15002e2c009bSjjc 	 * - Make all local latencies be minimum
15012e2c009bSjjc 	 */
15022e2c009bSjjc 	min = -1;
15032e2c009bSjjc 	max = 0;
15042e2c009bSjjc 	for (i = 0; i < lgrp_plat_node_cnt; i++) {
15052e2c009bSjjc 		if (!node_memory[i].exists)
15062e2c009bSjjc 			continue;
15072e2c009bSjjc 		t = lat_stats->latencies[i][i];
15082e2c009bSjjc 		if (t == 0)
15092e2c009bSjjc 			continue;
15102e2c009bSjjc 		if (min == -1 || t < min)
15112e2c009bSjjc 			min = t;
15122e2c009bSjjc 		if (t > max)
15132e2c009bSjjc 			max = t;
15142e2c009bSjjc 	}
15152e2c009bSjjc 	if (min != max) {
15162e2c009bSjjc 		for (i = 0; i < lgrp_plat_node_cnt; i++) {
15172e2c009bSjjc 			int	local;
15182e2c009bSjjc 
15192e2c009bSjjc 			if (!node_memory[i].exists)
15202e2c009bSjjc 				continue;
15212e2c009bSjjc 
15222e2c009bSjjc 			local = lat_stats->latencies[i][i];
15232e2c009bSjjc 			if (local == 0)
15242e2c009bSjjc 				continue;
15252e2c009bSjjc 
15262e2c009bSjjc 			/*
15272e2c009bSjjc 			 * Track suspect probe times that aren't within
15282e2c009bSjjc 			 * tolerance of minimum local latency and how much
15292e2c009bSjjc 			 * probe times are corrected by
15302e2c009bSjjc 			 */
15312e2c009bSjjc 			if (local - min > min >> lgrp_plat_probe_lt_shift)
15322e2c009bSjjc 				probe_stats->probe_suspect[i][i]++;
15332e2c009bSjjc 
15342e2c009bSjjc 			probe_stats->probe_errors[i][i] += local - min;
15352e2c009bSjjc 
15362e2c009bSjjc 			/*
15372e2c009bSjjc 			 * Make local latencies be minimum
15382e2c009bSjjc 			 */
15392e2c009bSjjc 			lgrp_config(LGRP_CONFIG_LAT_CHANGE, i, min);
15402e2c009bSjjc 			lat_stats->latencies[i][i] = min;
15412e2c009bSjjc 		}
15422e2c009bSjjc 	}
15432e2c009bSjjc 
15442e2c009bSjjc 	/*
15452e2c009bSjjc 	 * Determine max probe time again since just adjusted latencies
15462e2c009bSjjc 	 */
15472e2c009bSjjc 	lat_stats->latency_max = 0;
15482e2c009bSjjc 	for (i = 0; i < lgrp_plat_node_cnt; i++) {
15492e2c009bSjjc 		if (!node_memory[i].exists)
15502e2c009bSjjc 			continue;
15512e2c009bSjjc 		for (j = 0; j < lgrp_plat_node_cnt; j++) {
15522e2c009bSjjc 			if (!node_memory[j].exists)
15532e2c009bSjjc 				continue;
15542e2c009bSjjc 			t = lat_stats->latencies[i][j];
15552e2c009bSjjc 			if (t > lat_stats->latency_max)
15562e2c009bSjjc 				lat_stats->latency_max = t;
15572e2c009bSjjc 		}
15582e2c009bSjjc 	}
15592e2c009bSjjc }
15602e2c009bSjjc 
15612e2c009bSjjc 
15622e2c009bSjjc /*
15632e2c009bSjjc  * Verify following about latencies between nodes:
15642e2c009bSjjc  *
15652e2c009bSjjc  * - Latencies should be symmetric (ie. latency(a, b) == latency(b, a))
15662e2c009bSjjc  * - Local latencies same
15672e2c009bSjjc  * - Local < remote
15682e2c009bSjjc  * - Number of latencies seen is reasonable
15692e2c009bSjjc  * - Number of occurrences of a given latency should be more than 1
15702e2c009bSjjc  *
15712e2c009bSjjc  * Returns:
15722e2c009bSjjc  *	0	Success
15732e2c009bSjjc  *	-1	Not symmetric
15742e2c009bSjjc  *	-2	Local latencies not same
15752e2c009bSjjc  *	-3	Local >= remote
15762e2c009bSjjc  */
15772e2c009bSjjc static int
15782e2c009bSjjc lgrp_plat_latency_verify(node_phys_addr_map_t *node_memory,
15792e2c009bSjjc     lgrp_plat_latency_stats_t *lat_stats)
15802e2c009bSjjc {
15812e2c009bSjjc 	int				i;
15822e2c009bSjjc 	int				j;
15832e2c009bSjjc 	u_longlong_t			t1;
15842e2c009bSjjc 	u_longlong_t			t2;
15852e2c009bSjjc 
15862e2c009bSjjc 	ASSERT(node_memory != NULL && lat_stats != NULL);
15872e2c009bSjjc 
15882e2c009bSjjc 	/*
15892e2c009bSjjc 	 * Nothing to do when this is an UMA machine, lgroup topology is
15902e2c009bSjjc 	 * limited to 2 levels, or there aren't any probe times yet
15912e2c009bSjjc 	 */
15922e2c009bSjjc 	if (max_mem_nodes == 1 || lgrp_topo_levels < 2 ||
15932e2c009bSjjc 	    lat_stats->latencies[0][0] == 0)
15942e2c009bSjjc 		return (0);
15952e2c009bSjjc 
15962e2c009bSjjc 	/*
15972e2c009bSjjc 	 * Make sure that latencies are symmetric between any two nodes
15982e2c009bSjjc 	 * (ie. latency(node0, node1) == latency(node1, node0))
15992e2c009bSjjc 	 */
16002e2c009bSjjc 	for (i = 0; i < lgrp_plat_node_cnt; i++) {
16012e2c009bSjjc 		if (!node_memory[i].exists)
16022e2c009bSjjc 			continue;
16032e2c009bSjjc 		for (j = 0; j < lgrp_plat_node_cnt; j++) {
16042e2c009bSjjc 			if (!node_memory[j].exists)
16052e2c009bSjjc 				continue;
16062e2c009bSjjc 			t1 = lat_stats->latencies[i][j];
16072e2c009bSjjc 			t2 = lat_stats->latencies[j][i];
16082e2c009bSjjc 
16092e2c009bSjjc 			if (t1 == 0 || t2 == 0 || t1 == t2)
16102e2c009bSjjc 				continue;
16112e2c009bSjjc 
16122e2c009bSjjc 			return (-1);
16132e2c009bSjjc 		}
16142e2c009bSjjc 	}
16152e2c009bSjjc 
16162e2c009bSjjc 	/*
16172e2c009bSjjc 	 * Local latencies should be same
16182e2c009bSjjc 	 */
16192e2c009bSjjc 	t1 = lat_stats->latencies[0][0];
16202e2c009bSjjc 	for (i = 1; i < lgrp_plat_node_cnt; i++) {
16212e2c009bSjjc 		if (!node_memory[i].exists)
16222e2c009bSjjc 			continue;
16232e2c009bSjjc 
16242e2c009bSjjc 		t2 = lat_stats->latencies[i][i];
16252e2c009bSjjc 		if (t2 == 0)
16262e2c009bSjjc 			continue;
16272e2c009bSjjc 
16282e2c009bSjjc 		if (t1 == 0) {
16292e2c009bSjjc 			t1 = t2;
16302e2c009bSjjc 			continue;
16312e2c009bSjjc 		}
16322e2c009bSjjc 
16332e2c009bSjjc 		if (t1 != t2)
16342e2c009bSjjc 			return (-2);
16352e2c009bSjjc 	}
16362e2c009bSjjc 
16372e2c009bSjjc 	/*
16382e2c009bSjjc 	 * Local latencies should be less than remote
16392e2c009bSjjc 	 */
16402e2c009bSjjc 	if (t1) {
16412e2c009bSjjc 		for (i = 0; i < lgrp_plat_node_cnt; i++) {
16422e2c009bSjjc 			if (!node_memory[i].exists)
16432e2c009bSjjc 				continue;
16442e2c009bSjjc 			for (j = 0; j < lgrp_plat_node_cnt; j++) {
16452e2c009bSjjc 				if (!node_memory[j].exists)
16462e2c009bSjjc 					continue;
16472e2c009bSjjc 				t2 = lat_stats->latencies[i][j];
16482e2c009bSjjc 				if (i == j || t2 == 0)
16492e2c009bSjjc 					continue;
16502e2c009bSjjc 
16512e2c009bSjjc 				if (t1 >= t2)
16522e2c009bSjjc 					return (-3);
16532e2c009bSjjc 			}
16542e2c009bSjjc 		}
16552e2c009bSjjc 	}
16562e2c009bSjjc 
16572e2c009bSjjc 	return (0);
16582e2c009bSjjc }
16592e2c009bSjjc 
16602e2c009bSjjc 
16612e2c009bSjjc /*
16622e2c009bSjjc  * Return the number of free, allocatable, or installed
16632e2c009bSjjc  * pages in an lgroup
16642e2c009bSjjc  * This is a copy of the MAX_MEM_NODES == 1 version of the routine
16652e2c009bSjjc  * used when MPO is disabled (i.e. single lgroup) or this is the root lgroup
16662e2c009bSjjc  */
16672e2c009bSjjc /* ARGSUSED */
16682e2c009bSjjc static pgcnt_t
16692e2c009bSjjc lgrp_plat_mem_size_default(lgrp_handle_t lgrphand, lgrp_mem_query_t query)
16702e2c009bSjjc {
16712e2c009bSjjc 	struct memlist *mlist;
16722e2c009bSjjc 	pgcnt_t npgs = 0;
16732e2c009bSjjc 	extern struct memlist *phys_avail;
16742e2c009bSjjc 	extern struct memlist *phys_install;
16752e2c009bSjjc 
16762e2c009bSjjc 	switch (query) {
16772e2c009bSjjc 	case LGRP_MEM_SIZE_FREE:
16782e2c009bSjjc 		return ((pgcnt_t)freemem);
16792e2c009bSjjc 	case LGRP_MEM_SIZE_AVAIL:
16802e2c009bSjjc 		memlist_read_lock();
16812e2c009bSjjc 		for (mlist = phys_avail; mlist; mlist = mlist->next)
16822e2c009bSjjc 			npgs += btop(mlist->size);
16832e2c009bSjjc 		memlist_read_unlock();
16842e2c009bSjjc 		return (npgs);
16852e2c009bSjjc 	case LGRP_MEM_SIZE_INSTALL:
16862e2c009bSjjc 		memlist_read_lock();
16872e2c009bSjjc 		for (mlist = phys_install; mlist; mlist = mlist->next)
16882e2c009bSjjc 			npgs += btop(mlist->size);
16892e2c009bSjjc 		memlist_read_unlock();
16902e2c009bSjjc 		return (npgs);
16912e2c009bSjjc 	default:
16922e2c009bSjjc 		return ((pgcnt_t)0);
16932e2c009bSjjc 	}
16942e2c009bSjjc }
16952e2c009bSjjc 
16962e2c009bSjjc 
16972e2c009bSjjc /*
16982e2c009bSjjc  * Update node to proximity domain mappings for given domain and return node ID
16992e2c009bSjjc  */
17002e2c009bSjjc static int
1701d821f0f0Sjjc lgrp_plat_node_domain_update(node_domain_map_t *node_domain, int node_cnt,
1702d821f0f0Sjjc     uint32_t domain)
17032e2c009bSjjc {
17042e2c009bSjjc 	uint_t	node;
17052e2c009bSjjc 	uint_t	start;
17062e2c009bSjjc 
17072e2c009bSjjc 	/*
17082e2c009bSjjc 	 * Hash proximity domain ID into node to domain mapping table (array)
17092e2c009bSjjc 	 * and add entry for it into first non-existent or matching entry found
17102e2c009bSjjc 	 */
1711d821f0f0Sjjc 	node = start = NODE_DOMAIN_HASH(domain, node_cnt);
17122e2c009bSjjc 	do {
17132e2c009bSjjc 		/*
17142e2c009bSjjc 		 * Entry doesn't exist yet, so create one for this proximity
17152e2c009bSjjc 		 * domain and return node ID which is index into mapping table.
17162e2c009bSjjc 		 */
17172e2c009bSjjc 		if (!node_domain[node].exists) {
17182e2c009bSjjc 			node_domain[node].exists = 1;
17192e2c009bSjjc 			node_domain[node].prox_domain = domain;
17202e2c009bSjjc 			return (node);
17212e2c009bSjjc 		}
17222e2c009bSjjc 
17232e2c009bSjjc 		/*
17242e2c009bSjjc 		 * Entry exists for this proximity domain already, so just
17252e2c009bSjjc 		 * return node ID (index into table).
17262e2c009bSjjc 		 */
17272e2c009bSjjc 		if (node_domain[node].prox_domain == domain)
17282e2c009bSjjc 			return (node);
1729d821f0f0Sjjc 		node = NODE_DOMAIN_HASH(node + 1, node_cnt);
17302e2c009bSjjc 	} while (node != start);
17312e2c009bSjjc 
17322e2c009bSjjc 	/*
17332e2c009bSjjc 	 * Ran out of supported number of entries which shouldn't happen....
17342e2c009bSjjc 	 */
17352e2c009bSjjc 	ASSERT(node != start);
17362e2c009bSjjc 	return (-1);
17372e2c009bSjjc }
17382e2c009bSjjc 
17392e2c009bSjjc 
17402e2c009bSjjc /*
17412e2c009bSjjc  * Update node memory information for given proximity domain with specified
17422e2c009bSjjc  * starting and ending physical address range (and return positive numbers for
17432e2c009bSjjc  * success and negative ones for errors)
17442e2c009bSjjc  */
17452e2c009bSjjc static int
1746d821f0f0Sjjc lgrp_plat_node_memory_update(node_domain_map_t *node_domain, int node_cnt,
1747e9dd3ea3Sjjc     node_phys_addr_map_t *node_memory, uint64_t start, uint64_t end,
17482e2c009bSjjc     uint32_t domain)
17492e2c009bSjjc {
17502e2c009bSjjc 	int	node;
17512e2c009bSjjc 
17522e2c009bSjjc 	/*
17532e2c009bSjjc 	 * Get node number for proximity domain
17542e2c009bSjjc 	 */
1755d821f0f0Sjjc 	node = lgrp_plat_domain_to_node(node_domain, node_cnt, domain);
17562e2c009bSjjc 	if (node == -1) {
1757d821f0f0Sjjc 		node = lgrp_plat_node_domain_update(node_domain, node_cnt,
1758d821f0f0Sjjc 		    domain);
17592e2c009bSjjc 		if (node == -1)
17602e2c009bSjjc 			return (-1);
17612e2c009bSjjc 	}
17622e2c009bSjjc 
17632e2c009bSjjc 	/*
17642e2c009bSjjc 	 * Create entry in table for node if it doesn't exist
17652e2c009bSjjc 	 */
17662e2c009bSjjc 	if (!node_memory[node].exists) {
17672e2c009bSjjc 		node_memory[node].exists = 1;
17682e2c009bSjjc 		node_memory[node].start = btop(start);
17692e2c009bSjjc 		node_memory[node].end = btop(end);
17702e2c009bSjjc 		node_memory[node].prox_domain = domain;
17712e2c009bSjjc 		return (0);
17722e2c009bSjjc 	}
17732e2c009bSjjc 
17742e2c009bSjjc 	/*
17752e2c009bSjjc 	 * Entry already exists for this proximity domain
17762e2c009bSjjc 	 *
17772e2c009bSjjc 	 * There may be more than one SRAT memory entry for a domain, so we may
17782e2c009bSjjc 	 * need to update existing start or end address for the node.
17792e2c009bSjjc 	 */
17802e2c009bSjjc 	if (node_memory[node].prox_domain == domain) {
17812e2c009bSjjc 		if (btop(start) < node_memory[node].start)
17822e2c009bSjjc 			node_memory[node].start = btop(start);
17832e2c009bSjjc 		if (btop(end) > node_memory[node].end)
17842e2c009bSjjc 			node_memory[node].end = btop(end);
17852e2c009bSjjc 		return (1);
17862e2c009bSjjc 	}
17872e2c009bSjjc 	return (-2);
17882e2c009bSjjc }
17892e2c009bSjjc 
17902e2c009bSjjc 
17912e2c009bSjjc /*
17922e2c009bSjjc  * Return time needed to probe from current CPU to memory in given node
17932e2c009bSjjc  */
17942e2c009bSjjc static hrtime_t
17952e2c009bSjjc lgrp_plat_probe_time(int to, cpu_node_map_t *cpu_node,
17962e2c009bSjjc     lgrp_plat_probe_mem_config_t *probe_mem_config,
17972e2c009bSjjc     lgrp_plat_latency_stats_t *lat_stats, lgrp_plat_probe_stats_t *probe_stats)
17982e2c009bSjjc {
17992e2c009bSjjc 	caddr_t			buf;
18002e2c009bSjjc 	hrtime_t		elapsed;
18012e2c009bSjjc 	hrtime_t		end;
18022e2c009bSjjc 	int			from;
18032e2c009bSjjc 	int			i;
18042e2c009bSjjc 	int			ipl;
18052e2c009bSjjc 	hrtime_t		max;
18062e2c009bSjjc 	hrtime_t		min;
18072e2c009bSjjc 	hrtime_t		start;
18082e2c009bSjjc 	extern int		use_sse_pagecopy;
18092e2c009bSjjc 
18102e2c009bSjjc 	/*
18112e2c009bSjjc 	 * Determine ID of node containing current CPU
18122e2c009bSjjc 	 */
18132e2c009bSjjc 	from = lgrp_plat_cpu_to_node(CPU, cpu_node);
18142e2c009bSjjc 	ASSERT(from >= 0 && from < lgrp_plat_node_cnt);
18152e2c009bSjjc 
18162e2c009bSjjc 	/*
18172e2c009bSjjc 	 * Do common work for probing main memory
18182e2c009bSjjc 	 */
18192e2c009bSjjc 	if (lgrp_plat_probe_flags & LGRP_PLAT_PROBE_PGCPY) {
18202e2c009bSjjc 		/*
18212e2c009bSjjc 		 * Skip probing any nodes without memory and
18222e2c009bSjjc 		 * set probe time to 0
18232e2c009bSjjc 		 */
18242e2c009bSjjc 		if (probe_mem_config->probe_va[to] == NULL) {
18252e2c009bSjjc 			lat_stats->latencies[from][to] = 0;
18262e2c009bSjjc 			return (0);
18272e2c009bSjjc 		}
18282e2c009bSjjc 
18292e2c009bSjjc 		/*
18302e2c009bSjjc 		 * Invalidate caches once instead of once every sample
18312e2c009bSjjc 		 * which should cut cost of probing by a lot
18322e2c009bSjjc 		 */
18332e2c009bSjjc 		probe_stats->flush_cost = gethrtime();
18342e2c009bSjjc 		invalidate_cache();
18352e2c009bSjjc 		probe_stats->flush_cost = gethrtime() -
18362e2c009bSjjc 		    probe_stats->flush_cost;
18372e2c009bSjjc 		probe_stats->probe_cost_total += probe_stats->flush_cost;
18382e2c009bSjjc 	}
18392e2c009bSjjc 
18402e2c009bSjjc 	/*
18412e2c009bSjjc 	 * Probe from current CPU to given memory using specified operation
18422e2c009bSjjc 	 * and take specified number of samples
18432e2c009bSjjc 	 */
18442e2c009bSjjc 	max = 0;
18452e2c009bSjjc 	min = -1;
18462e2c009bSjjc 	for (i = 0; i < lgrp_plat_probe_nsamples; i++) {
18472e2c009bSjjc 		probe_stats->probe_cost = gethrtime();
18482e2c009bSjjc 
18492e2c009bSjjc 		/*
18502e2c009bSjjc 		 * Can't measure probe time if gethrtime() isn't working yet
18512e2c009bSjjc 		 */
18522e2c009bSjjc 		if (probe_stats->probe_cost == 0 && gethrtime() == 0)
18532e2c009bSjjc 			return (0);
18542e2c009bSjjc 
18552e2c009bSjjc 		if (lgrp_plat_probe_flags & LGRP_PLAT_PROBE_VENDOR) {
18562e2c009bSjjc 			/*
18572e2c009bSjjc 			 * Measure how long it takes to read vendor ID from
18582e2c009bSjjc 			 * Northbridge
18592e2c009bSjjc 			 */
18602e2c009bSjjc 			elapsed = opt_probe_vendor(to, lgrp_plat_probe_nreads);
18612e2c009bSjjc 		} else {
18622e2c009bSjjc 			/*
18632e2c009bSjjc 			 * Measure how long it takes to copy page
18642e2c009bSjjc 			 * on top of itself
18652e2c009bSjjc 			 */
18662e2c009bSjjc 			buf = probe_mem_config->probe_va[to] + (i * PAGESIZE);
18672e2c009bSjjc 
18682e2c009bSjjc 			kpreempt_disable();
18692e2c009bSjjc 			ipl = splhigh();
18702e2c009bSjjc 			start = gethrtime();
18712e2c009bSjjc 			if (use_sse_pagecopy)
18722e2c009bSjjc 				hwblkpagecopy(buf, buf);
18732e2c009bSjjc 			else
18742e2c009bSjjc 				bcopy(buf, buf, PAGESIZE);
18752e2c009bSjjc 			end = gethrtime();
18762e2c009bSjjc 			elapsed = end - start;
18772e2c009bSjjc 			splx(ipl);
18782e2c009bSjjc 			kpreempt_enable();
18792e2c009bSjjc 		}
18802e2c009bSjjc 
18812e2c009bSjjc 		probe_stats->probe_cost = gethrtime() -
18822e2c009bSjjc 		    probe_stats->probe_cost;
18832e2c009bSjjc 		probe_stats->probe_cost_total += probe_stats->probe_cost;
18842e2c009bSjjc 
18852e2c009bSjjc 		if (min == -1 || elapsed < min)
18862e2c009bSjjc 			min = elapsed;
18872e2c009bSjjc 		if (elapsed > max)
18882e2c009bSjjc 			max = elapsed;
18892e2c009bSjjc 	}
18902e2c009bSjjc 
18912e2c009bSjjc 	/*
18922e2c009bSjjc 	 * Update minimum and maximum probe times between
18932e2c009bSjjc 	 * these two nodes
18942e2c009bSjjc 	 */
18952e2c009bSjjc 	if (min < probe_stats->probe_min[from][to] ||
18962e2c009bSjjc 	    probe_stats->probe_min[from][to] == 0)
18972e2c009bSjjc 		probe_stats->probe_min[from][to] = min;
18982e2c009bSjjc 
18992e2c009bSjjc 	if (max > probe_stats->probe_max[from][to])
19002e2c009bSjjc 		probe_stats->probe_max[from][to] = max;
19012e2c009bSjjc 
19022e2c009bSjjc 	return (min);
19032e2c009bSjjc }
19042e2c009bSjjc 
19052e2c009bSjjc 
19062e2c009bSjjc /*
1907d821f0f0Sjjc  * Read boot property with CPU to APIC ID array, fill in CPU to node ID
1908d821f0f0Sjjc  * mapping table with APIC ID for each CPU, and return number of CPU APIC IDs.
1909dae2fa37Sjjc  *
1910dae2fa37Sjjc  * NOTE: This code assumes that CPU IDs are assigned in order that they appear
1911dae2fa37Sjjc  *       in in cpu_apicid_array boot property which is based on and follows
1912dae2fa37Sjjc  *	 same ordering as processor list in ACPI MADT.  If the code in
1913dae2fa37Sjjc  *	 usr/src/uts/i86pc/io/pcplusmp/apic.c that reads MADT and assigns
1914dae2fa37Sjjc  *	 CPU IDs ever changes, then this code will need to change too....
1915dae2fa37Sjjc  */
1916dae2fa37Sjjc static int
1917d821f0f0Sjjc lgrp_plat_process_cpu_apicids(cpu_node_map_t *cpu_node)
1918dae2fa37Sjjc {
1919d821f0f0Sjjc 	int	boot_prop_len;
1920dae2fa37Sjjc 	char	*boot_prop_name = BP_CPU_APICID_ARRAY;
1921dae2fa37Sjjc 	uint8_t	cpu_apicid_array[UINT8_MAX + 1];
1922dae2fa37Sjjc 	int	i;
1923d821f0f0Sjjc 	int	n;
1924dae2fa37Sjjc 
1925dae2fa37Sjjc 	/*
1926dae2fa37Sjjc 	 * Nothing to do when no array to fill in or not enough CPUs
1927dae2fa37Sjjc 	 */
1928d821f0f0Sjjc 	if (cpu_node == NULL)
1929d821f0f0Sjjc 		return (-1);
1930dae2fa37Sjjc 
1931dae2fa37Sjjc 	/*
1932dae2fa37Sjjc 	 * Check length of property value
1933dae2fa37Sjjc 	 */
1934dae2fa37Sjjc 	boot_prop_len = BOP_GETPROPLEN(bootops, boot_prop_name);
1935d821f0f0Sjjc 	if (boot_prop_len <= 0 || boot_prop_len > sizeof (cpu_apicid_array))
1936d821f0f0Sjjc 		return (-2);
1937d821f0f0Sjjc 
1938d821f0f0Sjjc 	/*
1939d821f0f0Sjjc 	 * Calculate number of entries in array and return when there's just
1940d821f0f0Sjjc 	 * one CPU since that's not very interesting for NUMA
1941d821f0f0Sjjc 	 */
1942d821f0f0Sjjc 	n = boot_prop_len / sizeof (uint8_t);
1943d821f0f0Sjjc 	if (n == 1)
1944d821f0f0Sjjc 		return (-3);
1945dae2fa37Sjjc 
1946dae2fa37Sjjc 	/*
1947dae2fa37Sjjc 	 * Get CPU to APIC ID property value
1948dae2fa37Sjjc 	 */
1949dae2fa37Sjjc 	if (BOP_GETPROP(bootops, boot_prop_name, cpu_apicid_array) < 0)
1950d821f0f0Sjjc 		return (-4);
1951dae2fa37Sjjc 
1952dae2fa37Sjjc 	/*
1953dae2fa37Sjjc 	 * Fill in CPU to node ID mapping table with APIC ID for each CPU
1954dae2fa37Sjjc 	 */
1955d821f0f0Sjjc 	for (i = 0; i < n; i++) {
1956dae2fa37Sjjc 		cpu_node[i].exists = 1;
1957dae2fa37Sjjc 		cpu_node[i].apicid = cpu_apicid_array[i];
1958dae2fa37Sjjc 	}
1959dae2fa37Sjjc 
1960d821f0f0Sjjc 	/*
1961d821f0f0Sjjc 	 * Return number of CPUs based on number of APIC IDs
1962d821f0f0Sjjc 	 */
1963d821f0f0Sjjc 	return (n);
1964dae2fa37Sjjc }
1965dae2fa37Sjjc 
1966dae2fa37Sjjc 
1967dae2fa37Sjjc /*
19682e2c009bSjjc  * Read ACPI System Locality Information Table (SLIT) to determine how far each
19692e2c009bSjjc  * NUMA node is from each other
19702e2c009bSjjc  */
19712e2c009bSjjc static int
19722e2c009bSjjc lgrp_plat_process_slit(struct slit *tp, uint_t node_cnt,
19732e2c009bSjjc     node_phys_addr_map_t *node_memory, lgrp_plat_latency_stats_t *lat_stats)
19742e2c009bSjjc {
19752e2c009bSjjc 	int		i;
19762e2c009bSjjc 	int		j;
19772e2c009bSjjc 	int		localities;
19782e2c009bSjjc 	hrtime_t	max;
19792e2c009bSjjc 	hrtime_t	min;
19802e2c009bSjjc 	int		retval;
19812e2c009bSjjc 	uint8_t		*slit_entries;
19822e2c009bSjjc 
19832e2c009bSjjc 	if (tp == NULL || !lgrp_plat_slit_enable)
19842e2c009bSjjc 		return (1);
19852e2c009bSjjc 
19862e2c009bSjjc 	if (lat_stats == NULL)
19872e2c009bSjjc 		return (2);
19882e2c009bSjjc 
19892e2c009bSjjc 	localities = tp->number;
19902e2c009bSjjc 	if (localities != node_cnt)
19912e2c009bSjjc 		return (3);
19922e2c009bSjjc 
19932e2c009bSjjc 	min = lat_stats->latency_min;
19942e2c009bSjjc 	max = lat_stats->latency_max;
19952e2c009bSjjc 
19962e2c009bSjjc 	/*
19972e2c009bSjjc 	 * Fill in latency matrix based on SLIT entries
19982e2c009bSjjc 	 */
19992e2c009bSjjc 	slit_entries = tp->entry;
20002e2c009bSjjc 	for (i = 0; i < localities; i++) {
20012e2c009bSjjc 		for (j = 0; j < localities; j++) {
20022e2c009bSjjc 			uint8_t	latency;
20032e2c009bSjjc 
20042e2c009bSjjc 			latency = slit_entries[(i * localities) + j];
20052e2c009bSjjc 			lat_stats->latencies[i][j] = latency;
20065b7cf7f0Sjjc 			if (latency < min || min == -1)
20072e2c009bSjjc 				min = latency;
20082e2c009bSjjc 			if (latency > max)
20092e2c009bSjjc 				max = latency;
20102e2c009bSjjc 		}
20112e2c009bSjjc 	}
20122e2c009bSjjc 
20132e2c009bSjjc 	/*
20142e2c009bSjjc 	 * Verify that latencies/distances given in SLIT look reasonable
20152e2c009bSjjc 	 */
20162e2c009bSjjc 	retval = lgrp_plat_latency_verify(node_memory, lat_stats);
20172e2c009bSjjc 
20182e2c009bSjjc 	if (retval) {
20192e2c009bSjjc 		/*
20202e2c009bSjjc 		 * Reinitialize (zero) latency table since SLIT doesn't look
20212e2c009bSjjc 		 * right
20222e2c009bSjjc 		 */
20232e2c009bSjjc 		for (i = 0; i < localities; i++) {
20242e2c009bSjjc 			for (j = 0; j < localities; j++)
20252e2c009bSjjc 				lat_stats->latencies[i][j] = 0;
20262e2c009bSjjc 		}
20272e2c009bSjjc 	} else {
20282e2c009bSjjc 		/*
20292e2c009bSjjc 		 * Update min and max latencies seen since SLIT looks valid
20302e2c009bSjjc 		 */
20312e2c009bSjjc 		lat_stats->latency_min = min;
20322e2c009bSjjc 		lat_stats->latency_max = max;
20332e2c009bSjjc 	}
20342e2c009bSjjc 
20352e2c009bSjjc 	return (retval);
20362e2c009bSjjc }
20372e2c009bSjjc 
20382e2c009bSjjc 
20392e2c009bSjjc /*
20402e2c009bSjjc  * Read ACPI System Resource Affinity Table (SRAT) to determine which CPUs
2041d821f0f0Sjjc  * and memory are local to each other in the same NUMA node and return number
2042d821f0f0Sjjc  * of nodes
20432e2c009bSjjc  */
20442e2c009bSjjc static int
2045d821f0f0Sjjc lgrp_plat_process_srat(struct srat *tp, node_domain_map_t *node_domain,
2046d821f0f0Sjjc     cpu_node_map_t *cpu_node, int cpu_count, node_phys_addr_map_t *node_memory)
20472e2c009bSjjc {
20485b7cf7f0Sjjc 	struct srat_item	*srat_end;
20492e2c009bSjjc 	int			i;
20502e2c009bSjjc 	struct srat_item	*item;
2051d821f0f0Sjjc 	int			node_cnt;
2052dae2fa37Sjjc 	int			proc_entry_count;
20532e2c009bSjjc 
2054d821f0f0Sjjc 	/*
2055d821f0f0Sjjc 	 * Nothing to do when no SRAT or disabled
2056d821f0f0Sjjc 	 */
20572e2c009bSjjc 	if (tp == NULL || !lgrp_plat_srat_enable)
2058d821f0f0Sjjc 		return (-1);
20592e2c009bSjjc 
20602e2c009bSjjc 	/*
20612e2c009bSjjc 	 * Determine number of nodes by counting number of proximity domains in
2062d821f0f0Sjjc 	 * SRAT and return if number of nodes is 1 or less since don't need to
2063d821f0f0Sjjc 	 * read SRAT then
20642e2c009bSjjc 	 */
2065d821f0f0Sjjc 	node_cnt = lgrp_plat_srat_domains(tp);
2066d821f0f0Sjjc 	if (node_cnt == 1)
2067d821f0f0Sjjc 		return (1);
2068d821f0f0Sjjc 	else if (node_cnt <= 0)
2069d821f0f0Sjjc 		return (-2);
20702e2c009bSjjc 
20712e2c009bSjjc 	/*
20722e2c009bSjjc 	 * Walk through SRAT, examining each CPU and memory entry to determine
20732e2c009bSjjc 	 * which CPUs and memory belong to which node.
20742e2c009bSjjc 	 */
20752e2c009bSjjc 	item = tp->list;
20765b7cf7f0Sjjc 	srat_end = (struct srat_item *)(tp->hdr.len + (uintptr_t)tp);
2077dae2fa37Sjjc 	proc_entry_count = 0;
20785b7cf7f0Sjjc 	while (item < srat_end) {
20792e2c009bSjjc 		uint32_t	apic_id;
20802e2c009bSjjc 		uint32_t	domain;
20812e2c009bSjjc 		uint64_t	end;
20822e2c009bSjjc 		uint64_t	length;
20832e2c009bSjjc 		uint64_t	start;
20842e2c009bSjjc 
20852e2c009bSjjc 		switch (item->type) {
20862e2c009bSjjc 		case SRAT_PROCESSOR:	/* CPU entry */
20872e2c009bSjjc 			if (!(item->i.p.flags & SRAT_ENABLED) ||
20882e2c009bSjjc 			    cpu_node == NULL)
20892e2c009bSjjc 				break;
20902e2c009bSjjc 
20912e2c009bSjjc 			/*
20922e2c009bSjjc 			 * Calculate domain (node) ID and fill in APIC ID to
20932e2c009bSjjc 			 * domain/node mapping table
20942e2c009bSjjc 			 */
20952e2c009bSjjc 			domain = item->i.p.domain1;
20962e2c009bSjjc 			for (i = 0; i < 3; i++) {
20972e2c009bSjjc 				domain += item->i.p.domain2[i] <<
20982e2c009bSjjc 				    ((i + 1) * 8);
20992e2c009bSjjc 			}
21002e2c009bSjjc 			apic_id = item->i.p.apic_id;
21012e2c009bSjjc 
2102d821f0f0Sjjc 			if (lgrp_plat_cpu_node_update(node_domain, node_cnt,
2103d821f0f0Sjjc 			    cpu_node, cpu_count, apic_id, domain) < 0)
2104d821f0f0Sjjc 				return (-3);
2105dae2fa37Sjjc 
2106dae2fa37Sjjc 			proc_entry_count++;
21072e2c009bSjjc 			break;
21082e2c009bSjjc 
21092e2c009bSjjc 		case SRAT_MEMORY:	/* memory entry */
21102e2c009bSjjc 			if (!(item->i.m.flags & SRAT_ENABLED) ||
21112e2c009bSjjc 			    node_memory == NULL)
21122e2c009bSjjc 				break;
21132e2c009bSjjc 
21142e2c009bSjjc 			/*
21152e2c009bSjjc 			 * Get domain (node) ID and fill in domain/node
21162e2c009bSjjc 			 * to memory mapping table
21172e2c009bSjjc 			 */
21182e2c009bSjjc 			domain = item->i.m.domain;
21192e2c009bSjjc 			start = item->i.m.base_addr;
21202e2c009bSjjc 			length = item->i.m.len;
21212e2c009bSjjc 			end = start + length - 1;
21222e2c009bSjjc 
2123d821f0f0Sjjc 			if (lgrp_plat_node_memory_update(node_domain, node_cnt,
21242e2c009bSjjc 			    node_memory, start, end, domain) < 0)
2125d821f0f0Sjjc 				return (-4);
21262e2c009bSjjc 			break;
2127*b6917abeSmishra 		case SRAT_X2APIC:	/* x2apic CPU entry */
2128*b6917abeSmishra 			if (!(item->i.xp.flags & SRAT_ENABLED) ||
2129*b6917abeSmishra 			    cpu_node == NULL)
2130*b6917abeSmishra 				break;
2131*b6917abeSmishra 
2132*b6917abeSmishra 			/*
2133*b6917abeSmishra 			 * Calculate domain (node) ID and fill in APIC ID to
2134*b6917abeSmishra 			 * domain/node mapping table
2135*b6917abeSmishra 			 */
2136*b6917abeSmishra 			domain = item->i.xp.domain;
2137*b6917abeSmishra 			apic_id = item->i.xp.x2apic_id;
2138*b6917abeSmishra 
2139*b6917abeSmishra 			if (lgrp_plat_cpu_node_update(node_domain, node_cnt,
2140*b6917abeSmishra 			    cpu_node, cpu_count, apic_id, domain) < 0)
2141*b6917abeSmishra 				return (-3);
2142*b6917abeSmishra 
2143*b6917abeSmishra 			proc_entry_count++;
2144*b6917abeSmishra 			break;
21452e2c009bSjjc 
21462e2c009bSjjc 		default:
21472e2c009bSjjc 			break;
21482e2c009bSjjc 		}
21492e2c009bSjjc 
21502e2c009bSjjc 		item = (struct srat_item *)((uintptr_t)item + item->len);
21512e2c009bSjjc 	}
2152dae2fa37Sjjc 
2153dae2fa37Sjjc 	/*
2154dae2fa37Sjjc 	 * Should have seen at least as many SRAT processor entries as CPUs
2155dae2fa37Sjjc 	 */
2156d821f0f0Sjjc 	if (proc_entry_count < cpu_count)
2157d821f0f0Sjjc 		return (-5);
2158dae2fa37Sjjc 
2159d821f0f0Sjjc 	return (node_cnt);
21602e2c009bSjjc }
21612e2c009bSjjc 
21622e2c009bSjjc 
21632e2c009bSjjc /*
21642e2c009bSjjc  * Return number of proximity domains given in ACPI SRAT
21652e2c009bSjjc  */
21662e2c009bSjjc static int
21672e2c009bSjjc lgrp_plat_srat_domains(struct srat *tp)
21682e2c009bSjjc {
21692e2c009bSjjc 	int			domain_cnt;
21702e2c009bSjjc 	struct srat_item	*end;
21712e2c009bSjjc 	int			i;
21722e2c009bSjjc 	struct srat_item	*item;
21732e2c009bSjjc 	node_domain_map_t	node_domain[MAX_NODES];
21742e2c009bSjjc 
21752e2c009bSjjc 
21762e2c009bSjjc 	if (tp == NULL || !lgrp_plat_srat_enable)
21772e2c009bSjjc 		return (1);
21782e2c009bSjjc 
21792e2c009bSjjc 	/*
21802e2c009bSjjc 	 * Walk through SRAT, examining each CPU and memory entry to determine
21812e2c009bSjjc 	 * proximity domain ID for each.
21822e2c009bSjjc 	 */
21832e2c009bSjjc 	domain_cnt = 0;
21842e2c009bSjjc 	item = tp->list;
21852e2c009bSjjc 	end = (struct srat_item *)(tp->hdr.len + (uintptr_t)tp);
21862e2c009bSjjc 	bzero(node_domain, MAX_NODES * sizeof (node_domain_map_t));
21872e2c009bSjjc 	while (item < end) {
21882e2c009bSjjc 		uint32_t	domain;
21892e2c009bSjjc 		boolean_t	overflow;
21902e2c009bSjjc 		uint_t		start;
21912e2c009bSjjc 
21922e2c009bSjjc 		switch (item->type) {
21932e2c009bSjjc 		case SRAT_PROCESSOR:	/* CPU entry */
21942e2c009bSjjc 			if (!(item->i.p.flags & SRAT_ENABLED))
21952e2c009bSjjc 				break;
21962e2c009bSjjc 			domain = item->i.p.domain1;
21972e2c009bSjjc 			for (i = 0; i < 3; i++) {
21982e2c009bSjjc 				domain += item->i.p.domain2[i] <<
21992e2c009bSjjc 				    ((i + 1) * 8);
22002e2c009bSjjc 			}
22012e2c009bSjjc 			break;
22022e2c009bSjjc 
22032e2c009bSjjc 		case SRAT_MEMORY:	/* memory entry */
22042e2c009bSjjc 			if (!(item->i.m.flags & SRAT_ENABLED))
22052e2c009bSjjc 				break;
22062e2c009bSjjc 			domain = item->i.m.domain;
22072e2c009bSjjc 			break;
22082e2c009bSjjc 
2209*b6917abeSmishra 		case SRAT_X2APIC:	/* x2apic CPU entry */
2210*b6917abeSmishra 			if (!(item->i.xp.flags & SRAT_ENABLED))
2211*b6917abeSmishra 				break;
2212*b6917abeSmishra 			domain = item->i.xp.domain;
2213*b6917abeSmishra 			break;
2214*b6917abeSmishra 
22152e2c009bSjjc 		default:
22162e2c009bSjjc 			break;
22172e2c009bSjjc 		}
22182e2c009bSjjc 
22192e2c009bSjjc 		/*
22202e2c009bSjjc 		 * Count and keep track of which proximity domain IDs seen
22212e2c009bSjjc 		 */
22222e2c009bSjjc 		start = i = domain % MAX_NODES;
22232e2c009bSjjc 		overflow = B_TRUE;
22242e2c009bSjjc 		do {
22252e2c009bSjjc 			/*
22262e2c009bSjjc 			 * Create entry for proximity domain and increment
22272e2c009bSjjc 			 * count when no entry exists where proximity domain
22282e2c009bSjjc 			 * hashed
22292e2c009bSjjc 			 */
22302e2c009bSjjc 			if (!node_domain[i].exists) {
22312e2c009bSjjc 				node_domain[i].exists = 1;
22322e2c009bSjjc 				node_domain[i].prox_domain = domain;
22332e2c009bSjjc 				domain_cnt++;
22342e2c009bSjjc 				overflow = B_FALSE;
22352e2c009bSjjc 				break;
22362e2c009bSjjc 			}
22372e2c009bSjjc 
22382e2c009bSjjc 			/*
22392e2c009bSjjc 			 * Nothing to do when proximity domain seen already
22402e2c009bSjjc 			 * and its entry exists
22412e2c009bSjjc 			 */
22422e2c009bSjjc 			if (node_domain[i].prox_domain == domain) {
22432e2c009bSjjc 				overflow = B_FALSE;
22442e2c009bSjjc 				break;
22452e2c009bSjjc 			}
22462e2c009bSjjc 
22472e2c009bSjjc 			/*
22482e2c009bSjjc 			 * Entry exists where proximity domain hashed, but for
22492e2c009bSjjc 			 * different proximity domain so keep search for empty
22502e2c009bSjjc 			 * slot to put it or matching entry whichever comes
22512e2c009bSjjc 			 * first.
22522e2c009bSjjc 			 */
22532e2c009bSjjc 			i = (i + 1) % MAX_NODES;
22542e2c009bSjjc 		} while (i != start);
22552e2c009bSjjc 
22562e2c009bSjjc 		/*
22572e2c009bSjjc 		 * Didn't find empty or matching entry which means have more
22582e2c009bSjjc 		 * proximity domains than supported nodes (:-(
22592e2c009bSjjc 		 */
22602e2c009bSjjc 		ASSERT(overflow != B_TRUE);
22612e2c009bSjjc 		if (overflow == B_TRUE)
22622e2c009bSjjc 			return (-1);
22632e2c009bSjjc 
22642e2c009bSjjc 		item = (struct srat_item *)((uintptr_t)item + item->len);
22652e2c009bSjjc 	}
22662e2c009bSjjc 	return (domain_cnt);
22672e2c009bSjjc }
22682e2c009bSjjc 
22692e2c009bSjjc 
22702e2c009bSjjc /*
22712e2c009bSjjc  * Set lgroup latencies for 2 level lgroup topology
22722e2c009bSjjc  */
22732e2c009bSjjc static void
22742e2c009bSjjc lgrp_plat_2level_setup(node_phys_addr_map_t *node_memory,
22752e2c009bSjjc     lgrp_plat_latency_stats_t *lat_stats)
22762e2c009bSjjc {
22772e2c009bSjjc 	int	i;
22782e2c009bSjjc 
22792e2c009bSjjc 	ASSERT(node_memory != NULL && lat_stats != NULL);
22802e2c009bSjjc 
22812e2c009bSjjc 	if (lgrp_plat_node_cnt >= 4)
22822e2c009bSjjc 		cmn_err(CE_NOTE,
22832e2c009bSjjc 		    "MPO only optimizing for local and remote\n");
22842e2c009bSjjc 	for (i = 0; i < lgrp_plat_node_cnt; i++) {
22852e2c009bSjjc 		int	j;
22862e2c009bSjjc 
22872e2c009bSjjc 		if (!node_memory[i].exists)
22882e2c009bSjjc 			continue;
22892e2c009bSjjc 		for (j = 0; j < lgrp_plat_node_cnt; j++) {
22902e2c009bSjjc 			if (!node_memory[j].exists)
22912e2c009bSjjc 				continue;
22922e2c009bSjjc 			if (i == j)
22932e2c009bSjjc 				lat_stats->latencies[i][j] = 2;
22942e2c009bSjjc 			else
22952e2c009bSjjc 				lat_stats->latencies[i][j] = 3;
22962e2c009bSjjc 		}
22972e2c009bSjjc 	}
22982e2c009bSjjc 	lat_stats->latency_min = 2;
22992e2c009bSjjc 	lat_stats->latency_max = 3;
23002e2c009bSjjc 	lgrp_config(LGRP_CONFIG_FLATTEN, 2, 0);
23012e2c009bSjjc }
23022e2c009bSjjc 
23032e2c009bSjjc 
23042e2c009bSjjc /*
23052e2c009bSjjc  * The following Opteron specific constants, macros, types, and routines define
23062e2c009bSjjc  * PCI configuration space registers and how to read them to determine the NUMA
23072e2c009bSjjc  * configuration of *supported* Opteron processors.  They provide the same
23082e2c009bSjjc  * information that may be gotten from the ACPI System Resource Affinity Table
23092e2c009bSjjc  * (SRAT) if it exists on the machine of interest.
23102e2c009bSjjc  *
23112e2c009bSjjc  * The AMD BIOS and Kernel Developer's Guide (BKDG) for the processor family
23122e2c009bSjjc  * of interest describes all of these registers and their contents.  The main
23132e2c009bSjjc  * registers used by this code to determine the NUMA configuration of the
23142e2c009bSjjc  * machine are the node ID register for the number of NUMA nodes and the DRAM
23152e2c009bSjjc  * address map registers for the physical address range of each node.
23162e2c009bSjjc  *
23172e2c009bSjjc  * NOTE: The format and how to determine the NUMA configuration using PCI
23182e2c009bSjjc  *	 config space registers may change or may not be supported in future
23192e2c009bSjjc  *	 Opteron processor families.
23207c478bd9Sstevel@tonic-gate  */
23217c478bd9Sstevel@tonic-gate 
23227c478bd9Sstevel@tonic-gate /*
23237c478bd9Sstevel@tonic-gate  * How many bits to shift Opteron DRAM Address Map base and limit registers
23247c478bd9Sstevel@tonic-gate  * to get actual value
23257c478bd9Sstevel@tonic-gate  */
2326f78a91cdSjjc #define	OPT_DRAMADDR_HI_LSHIFT_ADDR	40	/* shift left for address */
2327f78a91cdSjjc #define	OPT_DRAMADDR_LO_LSHIFT_ADDR	8	/* shift left for address */
23287c478bd9Sstevel@tonic-gate 
2329f78a91cdSjjc #define	OPT_DRAMADDR_HI_MASK_ADDR	0x000000FF /* address bits 47-40 */
2330f78a91cdSjjc #define	OPT_DRAMADDR_LO_MASK_ADDR	0xFFFF0000 /* address bits 39-24 */
2331f78a91cdSjjc 
2332f78a91cdSjjc #define	OPT_DRAMADDR_LO_MASK_OFF	0xFFFFFF /* offset for address */
2333f78a91cdSjjc 
2334f78a91cdSjjc /*
2335f78a91cdSjjc  * Macros to derive addresses from Opteron DRAM Address Map registers
2336f78a91cdSjjc  */
2337f78a91cdSjjc #define	OPT_DRAMADDR_HI(reg) \
2338f78a91cdSjjc 	(((u_longlong_t)reg & OPT_DRAMADDR_HI_MASK_ADDR) << \
2339f78a91cdSjjc 	    OPT_DRAMADDR_HI_LSHIFT_ADDR)
2340f78a91cdSjjc 
2341f78a91cdSjjc #define	OPT_DRAMADDR_LO(reg) \
2342f78a91cdSjjc 	(((u_longlong_t)reg & OPT_DRAMADDR_LO_MASK_ADDR) << \
2343f78a91cdSjjc 	    OPT_DRAMADDR_LO_LSHIFT_ADDR)
2344f78a91cdSjjc 
2345f78a91cdSjjc #define	OPT_DRAMADDR(high, low) \
2346f78a91cdSjjc 	(OPT_DRAMADDR_HI(high) | OPT_DRAMADDR_LO(low))
23477c478bd9Sstevel@tonic-gate 
23487c478bd9Sstevel@tonic-gate /*
23497c478bd9Sstevel@tonic-gate  * Bit masks defining what's in Opteron DRAM Address Map base register
23507c478bd9Sstevel@tonic-gate  */
2351f78a91cdSjjc #define	OPT_DRAMBASE_LO_MASK_RE		0x1	/* read enable */
2352f78a91cdSjjc #define	OPT_DRAMBASE_LO_MASK_WE		0x2	/* write enable */
2353f78a91cdSjjc #define	OPT_DRAMBASE_LO_MASK_INTRLVEN	0x700	/* interleave */
23547c478bd9Sstevel@tonic-gate 
23557c478bd9Sstevel@tonic-gate /*
23567c478bd9Sstevel@tonic-gate  * Bit masks defining what's in Opteron DRAM Address Map limit register
23577c478bd9Sstevel@tonic-gate  */
2358f78a91cdSjjc #define	OPT_DRAMLIMIT_LO_MASK_DSTNODE	0x7		/* destination node */
2359f78a91cdSjjc #define	OPT_DRAMLIMIT_LO_MASK_INTRLVSEL	0x700		/* interleave select */
23607c478bd9Sstevel@tonic-gate 
23617c478bd9Sstevel@tonic-gate 
23627c478bd9Sstevel@tonic-gate /*
23637c478bd9Sstevel@tonic-gate  * Opteron Node ID register in PCI configuration space contains
23647c478bd9Sstevel@tonic-gate  * number of nodes in system, etc. for Opteron K8.  The following
23657c478bd9Sstevel@tonic-gate  * constants and macros define its contents, structure, and access.
23667c478bd9Sstevel@tonic-gate  */
23677c478bd9Sstevel@tonic-gate 
23687c478bd9Sstevel@tonic-gate /*
23697c478bd9Sstevel@tonic-gate  * Bit masks defining what's in Opteron Node ID register
23707c478bd9Sstevel@tonic-gate  */
23717c478bd9Sstevel@tonic-gate #define	OPT_NODE_MASK_ID	0x7	/* node ID */
23727c478bd9Sstevel@tonic-gate #define	OPT_NODE_MASK_CNT	0x70	/* node count */
23737c478bd9Sstevel@tonic-gate #define	OPT_NODE_MASK_IONODE	0x700	/* Hypertransport I/O hub node ID */
23747c478bd9Sstevel@tonic-gate #define	OPT_NODE_MASK_LCKNODE	0x7000	/* lock controller node ID */
23757c478bd9Sstevel@tonic-gate #define	OPT_NODE_MASK_CPUCNT	0xF0000	/* CPUs in system (0 means 1 CPU)  */
23767c478bd9Sstevel@tonic-gate 
23777c478bd9Sstevel@tonic-gate /*
23787c478bd9Sstevel@tonic-gate  * How many bits in Opteron Node ID register to shift right to get actual value
23797c478bd9Sstevel@tonic-gate  */
23807c478bd9Sstevel@tonic-gate #define	OPT_NODE_RSHIFT_CNT	0x4	/* shift right for node count value */
23817c478bd9Sstevel@tonic-gate 
23827c478bd9Sstevel@tonic-gate /*
23837c478bd9Sstevel@tonic-gate  * Macros to get values from Opteron Node ID register
23847c478bd9Sstevel@tonic-gate  */
23857c478bd9Sstevel@tonic-gate #define	OPT_NODE_CNT(reg) \
23867c478bd9Sstevel@tonic-gate 	((reg & OPT_NODE_MASK_CNT) >> OPT_NODE_RSHIFT_CNT)
23877c478bd9Sstevel@tonic-gate 
2388f78a91cdSjjc /*
2389f78a91cdSjjc  * Macro to setup PCI Extended Configuration Space (ECS) address to give to
2390f78a91cdSjjc  * "in/out" instructions
2391f78a91cdSjjc  *
2392f78a91cdSjjc  * NOTE: Should only be used in lgrp_plat_init() before MMIO setup because any
2393f78a91cdSjjc  *	 other uses should just do MMIO to access PCI ECS.
2394f78a91cdSjjc  *	 Must enable special bit in Northbridge Configuration Register on
2395f78a91cdSjjc  *	 Greyhound for extended CF8 space access to be able to access PCI ECS
2396f78a91cdSjjc  *	 using "in/out" instructions and restore special bit after done
2397f78a91cdSjjc  *	 accessing PCI ECS.
2398f78a91cdSjjc  */
2399f78a91cdSjjc #define	OPT_PCI_ECS_ADDR(bus, device, function, reg) \
2400f78a91cdSjjc 	(PCI_CONE | (((bus) & 0xff) << 16) | (((device & 0x1f)) << 11)  | \
2401f78a91cdSjjc 	    (((function) & 0x7) << 8) | ((reg) & 0xfc) | \
2402f78a91cdSjjc 	    ((((reg) >> 8) & 0xf) << 24))
24037c478bd9Sstevel@tonic-gate 
24047c478bd9Sstevel@tonic-gate /*
24057c478bd9Sstevel@tonic-gate  * PCI configuration space registers accessed by specifying
24067c478bd9Sstevel@tonic-gate  * a bus, device, function, and offset.  The following constants
24077c478bd9Sstevel@tonic-gate  * define the values needed to access Opteron K8 configuration
24087c478bd9Sstevel@tonic-gate  * info to determine its node topology
24097c478bd9Sstevel@tonic-gate  */
24107c478bd9Sstevel@tonic-gate 
24117c478bd9Sstevel@tonic-gate #define	OPT_PCS_BUS_CONFIG	0	/* Hypertransport config space bus */
24127c478bd9Sstevel@tonic-gate 
24137c478bd9Sstevel@tonic-gate /*
24147c478bd9Sstevel@tonic-gate  * Opteron PCI configuration space register function values
24157c478bd9Sstevel@tonic-gate  */
24167c478bd9Sstevel@tonic-gate #define	OPT_PCS_FUNC_HT		0	/* Hypertransport configuration */
24177c478bd9Sstevel@tonic-gate #define	OPT_PCS_FUNC_ADDRMAP	1	/* Address map configuration */
24187c478bd9Sstevel@tonic-gate #define	OPT_PCS_FUNC_DRAM	2	/* DRAM configuration */
24197c478bd9Sstevel@tonic-gate #define	OPT_PCS_FUNC_MISC	3	/* Miscellaneous configuration */
24207c478bd9Sstevel@tonic-gate 
24217c478bd9Sstevel@tonic-gate /*
24227c478bd9Sstevel@tonic-gate  * PCI Configuration Space register offsets
24237c478bd9Sstevel@tonic-gate  */
24247c478bd9Sstevel@tonic-gate #define	OPT_PCS_OFF_VENDOR	0x0	/* device/vendor ID register */
2425f78a91cdSjjc #define	OPT_PCS_OFF_DRAMBASE_HI	0x140	/* DRAM Base register (node 0) */
2426f78a91cdSjjc #define	OPT_PCS_OFF_DRAMBASE_LO	0x40	/* DRAM Base register (node 0) */
24277c478bd9Sstevel@tonic-gate #define	OPT_PCS_OFF_NODEID	0x60	/* Node ID register */
24287c478bd9Sstevel@tonic-gate 
24297c478bd9Sstevel@tonic-gate /*
24307c478bd9Sstevel@tonic-gate  * Opteron PCI Configuration Space device IDs for nodes
24317c478bd9Sstevel@tonic-gate  */
24327c478bd9Sstevel@tonic-gate #define	OPT_PCS_DEV_NODE0		24	/* device number for node 0 */
24337c478bd9Sstevel@tonic-gate 
24347c478bd9Sstevel@tonic-gate 
24357c478bd9Sstevel@tonic-gate /*
24367c478bd9Sstevel@tonic-gate  * Opteron DRAM address map gives base and limit for physical memory in a node
24377c478bd9Sstevel@tonic-gate  */
24387c478bd9Sstevel@tonic-gate typedef	struct opt_dram_addr_map {
2439f78a91cdSjjc 	uint32_t	base_hi;
2440f78a91cdSjjc 	uint32_t	base_lo;
2441f78a91cdSjjc 	uint32_t	limit_hi;
2442f78a91cdSjjc 	uint32_t	limit_lo;
24437c478bd9Sstevel@tonic-gate } opt_dram_addr_map_t;
24447c478bd9Sstevel@tonic-gate 
24457c478bd9Sstevel@tonic-gate 
24467c478bd9Sstevel@tonic-gate /*
2447f78a91cdSjjc  * Supported AMD processor families
2448f78a91cdSjjc  */
2449f78a91cdSjjc #define	AMD_FAMILY_HAMMER	15
2450f78a91cdSjjc #define	AMD_FAMILY_GREYHOUND	16
24517c478bd9Sstevel@tonic-gate 
2452f78a91cdSjjc /*
24532e2c009bSjjc  * Whether to have is_opteron() return 1 even when processor isn't supported
2454f78a91cdSjjc  */
2455f78a91cdSjjc uint_t	is_opteron_override = 0;
2456f78a91cdSjjc 
2457f78a91cdSjjc /*
2458f78a91cdSjjc  * AMD processor family for current CPU
2459f78a91cdSjjc  */
24607c478bd9Sstevel@tonic-gate uint_t	opt_family = 0;
2461f78a91cdSjjc 
24627c478bd9Sstevel@tonic-gate 
24637c478bd9Sstevel@tonic-gate /*
2464f78a91cdSjjc  * Determine whether we're running on a supported AMD Opteron since reading
2465f78a91cdSjjc  * node count and DRAM address map registers may have different format or
24662e2c009bSjjc  * may not be supported across processor families
24677c478bd9Sstevel@tonic-gate  */
24682e2c009bSjjc static int
24697c478bd9Sstevel@tonic-gate is_opteron(void)
24707c478bd9Sstevel@tonic-gate {
2471f78a91cdSjjc 
24727c478bd9Sstevel@tonic-gate 	if (x86_vendor != X86_VENDOR_AMD)
24737c478bd9Sstevel@tonic-gate 		return (0);
24747c478bd9Sstevel@tonic-gate 
2475f78a91cdSjjc 	opt_family = cpuid_getfamily(CPU);
2476f78a91cdSjjc 	if (opt_family == AMD_FAMILY_HAMMER ||
2477f78a91cdSjjc 	    opt_family == AMD_FAMILY_GREYHOUND || is_opteron_override)
24787c478bd9Sstevel@tonic-gate 		return (1);
24797c478bd9Sstevel@tonic-gate 	else
24807c478bd9Sstevel@tonic-gate 		return (0);
24817c478bd9Sstevel@tonic-gate }
24827c478bd9Sstevel@tonic-gate 
24832e2c009bSjjc 
24842e2c009bSjjc /*
24852e2c009bSjjc  * Determine NUMA configuration for Opteron from registers that live in PCI
24862e2c009bSjjc  * configuration space
24872e2c009bSjjc  */
24882e2c009bSjjc static void
24892e2c009bSjjc opt_get_numa_config(uint_t *node_cnt, int *mem_intrlv,
24902e2c009bSjjc     node_phys_addr_map_t *node_memory)
24917c478bd9Sstevel@tonic-gate {
24927c478bd9Sstevel@tonic-gate 	uint_t				bus;
24937c478bd9Sstevel@tonic-gate 	uint_t				dev;
24942e2c009bSjjc 	struct opt_dram_addr_map	dram_map[MAX_NODES];
24957c478bd9Sstevel@tonic-gate 	uint_t				node;
24962e2c009bSjjc 	uint_t				node_info[MAX_NODES];
2497f78a91cdSjjc 	uint_t				off_hi;
2498f78a91cdSjjc 	uint_t				off_lo;
2499f78a91cdSjjc 	uint64_t			nb_cfg_reg;
25007c478bd9Sstevel@tonic-gate 
25017c478bd9Sstevel@tonic-gate 	/*
25027c478bd9Sstevel@tonic-gate 	 * Read configuration registers from PCI configuration space to
25037c478bd9Sstevel@tonic-gate 	 * determine node information, which memory is in each node, etc.
25047c478bd9Sstevel@tonic-gate 	 *
25057c478bd9Sstevel@tonic-gate 	 * Write to PCI configuration space address register to specify
25067c478bd9Sstevel@tonic-gate 	 * which configuration register to read and read/write PCI
25077c478bd9Sstevel@tonic-gate 	 * configuration space data register to get/set contents
25087c478bd9Sstevel@tonic-gate 	 */
25097c478bd9Sstevel@tonic-gate 	bus = OPT_PCS_BUS_CONFIG;
25107c478bd9Sstevel@tonic-gate 	dev = OPT_PCS_DEV_NODE0;
2511f78a91cdSjjc 	off_hi = OPT_PCS_OFF_DRAMBASE_HI;
2512f78a91cdSjjc 	off_lo = OPT_PCS_OFF_DRAMBASE_LO;
25137c478bd9Sstevel@tonic-gate 
25147c478bd9Sstevel@tonic-gate 	/*
25157c478bd9Sstevel@tonic-gate 	 * Read node ID register for node 0 to get node count
25167c478bd9Sstevel@tonic-gate 	 */
25172e2c009bSjjc 	node_info[0] = pci_getl_func(bus, dev, OPT_PCS_FUNC_HT,
2518ef50d8c0Sesaxe 	    OPT_PCS_OFF_NODEID);
25192e2c009bSjjc 	*node_cnt = OPT_NODE_CNT(node_info[0]) + 1;
25202e2c009bSjjc 
25212e2c009bSjjc 	/*
25222e2c009bSjjc 	 * If number of nodes is more than maximum supported, then set node
25232e2c009bSjjc 	 * count to 1 and treat system as UMA instead of NUMA.
25242e2c009bSjjc 	 */
25252e2c009bSjjc 	if (*node_cnt > MAX_NODES) {
25262e2c009bSjjc 		*node_cnt = 1;
25272e2c009bSjjc 		return;
25282e2c009bSjjc 	}
25297c478bd9Sstevel@tonic-gate 
2530f78a91cdSjjc 	/*
2531f78a91cdSjjc 	 * For Greyhound, PCI Extended Configuration Space must be enabled to
2532f78a91cdSjjc 	 * read high DRAM address map base and limit registers
2533f78a91cdSjjc 	 */
2534f78a91cdSjjc 	if (opt_family == AMD_FAMILY_GREYHOUND) {
2535f78a91cdSjjc 		nb_cfg_reg = rdmsr(MSR_AMD_NB_CFG);
2536f78a91cdSjjc 		if ((nb_cfg_reg & AMD_GH_NB_CFG_EN_ECS) == 0)
2537f78a91cdSjjc 			wrmsr(MSR_AMD_NB_CFG,
2538f78a91cdSjjc 			    nb_cfg_reg | AMD_GH_NB_CFG_EN_ECS);
2539f78a91cdSjjc 	}
2540f78a91cdSjjc 
25412e2c009bSjjc 	for (node = 0; node < *node_cnt; node++) {
2542f78a91cdSjjc 		uint32_t	base_hi;
2543f78a91cdSjjc 		uint32_t	base_lo;
2544f78a91cdSjjc 		uint32_t	limit_hi;
2545f78a91cdSjjc 		uint32_t	limit_lo;
2546f78a91cdSjjc 
25477c478bd9Sstevel@tonic-gate 		/*
25487c478bd9Sstevel@tonic-gate 		 * Read node ID register (except for node 0 which we just read)
25497c478bd9Sstevel@tonic-gate 		 */
25507c478bd9Sstevel@tonic-gate 		if (node > 0) {
25512e2c009bSjjc 			node_info[node] = pci_getl_func(bus, dev,
2552ef50d8c0Sesaxe 			    OPT_PCS_FUNC_HT, OPT_PCS_OFF_NODEID);
25537c478bd9Sstevel@tonic-gate 		}
25547c478bd9Sstevel@tonic-gate 
25557c478bd9Sstevel@tonic-gate 		/*
25567c478bd9Sstevel@tonic-gate 		 * Read DRAM base and limit registers which specify
25577c478bd9Sstevel@tonic-gate 		 * physical memory range of each node
25587c478bd9Sstevel@tonic-gate 		 */
2559f78a91cdSjjc 		if (opt_family != AMD_FAMILY_GREYHOUND)
2560f78a91cdSjjc 			base_hi = 0;
2561f78a91cdSjjc 		else {
2562f78a91cdSjjc 			outl(PCI_CONFADD, OPT_PCI_ECS_ADDR(bus, dev,
2563f78a91cdSjjc 			    OPT_PCS_FUNC_ADDRMAP, off_hi));
25642e2c009bSjjc 			base_hi = dram_map[node].base_hi =
2565f78a91cdSjjc 			    inl(PCI_CONFDATA);
2566f78a91cdSjjc 		}
25672e2c009bSjjc 		base_lo = dram_map[node].base_lo = pci_getl_func(bus, dev,
2568f78a91cdSjjc 		    OPT_PCS_FUNC_ADDRMAP, off_lo);
2569f78a91cdSjjc 
25702e2c009bSjjc 		if ((dram_map[node].base_lo & OPT_DRAMBASE_LO_MASK_INTRLVEN) &&
25712e2c009bSjjc 		    mem_intrlv)
25722e2c009bSjjc 			*mem_intrlv = *mem_intrlv + 1;
25737c478bd9Sstevel@tonic-gate 
2574f78a91cdSjjc 		off_hi += 4;	/* high limit register offset */
2575f78a91cdSjjc 		if (opt_family != AMD_FAMILY_GREYHOUND)
2576f78a91cdSjjc 			limit_hi = 0;
2577f78a91cdSjjc 		else {
2578f78a91cdSjjc 			outl(PCI_CONFADD, OPT_PCI_ECS_ADDR(bus, dev,
2579f78a91cdSjjc 			    OPT_PCS_FUNC_ADDRMAP, off_hi));
25802e2c009bSjjc 			limit_hi = dram_map[node].limit_hi =
2581f78a91cdSjjc 			    inl(PCI_CONFDATA);
2582f78a91cdSjjc 		}
2583f78a91cdSjjc 
2584f78a91cdSjjc 		off_lo += 4;	/* low limit register offset */
25852e2c009bSjjc 		limit_lo = dram_map[node].limit_lo = pci_getl_func(bus,
2586f78a91cdSjjc 		    dev, OPT_PCS_FUNC_ADDRMAP, off_lo);
25877c478bd9Sstevel@tonic-gate 
25887c478bd9Sstevel@tonic-gate 		/*
2589f78a91cdSjjc 		 * Increment device number to next node and register offsets
2590f78a91cdSjjc 		 * for DRAM base register of next node
25917c478bd9Sstevel@tonic-gate 		 */
2592f78a91cdSjjc 		off_hi += 4;
2593f78a91cdSjjc 		off_lo += 4;
25947c478bd9Sstevel@tonic-gate 		dev++;
25957c478bd9Sstevel@tonic-gate 
25967c478bd9Sstevel@tonic-gate 		/*
2597a940d195Sjjc 		 * Both read and write enable bits must be enabled in DRAM
2598a940d195Sjjc 		 * address map base register for physical memory to exist in
2599a940d195Sjjc 		 * node
2600a940d195Sjjc 		 */
2601f78a91cdSjjc 		if ((base_lo & OPT_DRAMBASE_LO_MASK_RE) == 0 ||
2602f78a91cdSjjc 		    (base_lo & OPT_DRAMBASE_LO_MASK_WE) == 0) {
2603a940d195Sjjc 			/*
2604a940d195Sjjc 			 * Mark node memory as non-existent and set start and
26052e2c009bSjjc 			 * end addresses to be same in node_memory[]
2606a940d195Sjjc 			 */
26072e2c009bSjjc 			node_memory[node].exists = 0;
26082e2c009bSjjc 			node_memory[node].start = node_memory[node].end =
26092e2c009bSjjc 			    (pfn_t)-1;
2610a940d195Sjjc 			continue;
2611a940d195Sjjc 		}
2612a940d195Sjjc 
2613a940d195Sjjc 		/*
2614a940d195Sjjc 		 * Mark node memory as existing and remember physical address
2615a940d195Sjjc 		 * range of each node for use later
26167c478bd9Sstevel@tonic-gate 		 */
26172e2c009bSjjc 		node_memory[node].exists = 1;
2618f78a91cdSjjc 
26192e2c009bSjjc 		node_memory[node].start = btop(OPT_DRAMADDR(base_hi, base_lo));
2620f78a91cdSjjc 
26212e2c009bSjjc 		node_memory[node].end = btop(OPT_DRAMADDR(limit_hi, limit_lo) |
2622f78a91cdSjjc 		    OPT_DRAMADDR_LO_MASK_OFF);
2623f78a91cdSjjc 	}
2624f78a91cdSjjc 
2625f78a91cdSjjc 	/*
2626f78a91cdSjjc 	 * Restore PCI Extended Configuration Space enable bit
2627f78a91cdSjjc 	 */
2628f78a91cdSjjc 	if (opt_family == AMD_FAMILY_GREYHOUND) {
2629f78a91cdSjjc 		if ((nb_cfg_reg & AMD_GH_NB_CFG_EN_ECS) == 0)
2630f78a91cdSjjc 			wrmsr(MSR_AMD_NB_CFG, nb_cfg_reg);
26317c478bd9Sstevel@tonic-gate 	}
26327c478bd9Sstevel@tonic-gate }
26337c478bd9Sstevel@tonic-gate 
26347c478bd9Sstevel@tonic-gate 
26357c478bd9Sstevel@tonic-gate /*
26362e2c009bSjjc  * Return average amount of time to read vendor ID register on Northbridge
26372e2c009bSjjc  * N times on specified destination node from current CPU
26387c478bd9Sstevel@tonic-gate  */
26397c478bd9Sstevel@tonic-gate static hrtime_t
26402e2c009bSjjc opt_probe_vendor(int dest_node, int nreads)
26417c478bd9Sstevel@tonic-gate {
26422e2c009bSjjc 	int		cnt;
26437c478bd9Sstevel@tonic-gate 	uint_t		dev;
26447c478bd9Sstevel@tonic-gate 	/* LINTED: set but not used in function */
26457c478bd9Sstevel@tonic-gate 	volatile uint_t	dev_vendor;
26467c478bd9Sstevel@tonic-gate 	hrtime_t	elapsed;
26477c478bd9Sstevel@tonic-gate 	hrtime_t	end;
26487c478bd9Sstevel@tonic-gate 	int		ipl;
26497c478bd9Sstevel@tonic-gate 	hrtime_t	start;
26507c478bd9Sstevel@tonic-gate 
26512e2c009bSjjc 	dev = OPT_PCS_DEV_NODE0 + dest_node;
26527c478bd9Sstevel@tonic-gate 	kpreempt_disable();
26537c478bd9Sstevel@tonic-gate 	ipl = spl8();
26542e2c009bSjjc 	outl(PCI_CONFADD, PCI_CADDR1(0, dev, OPT_PCS_FUNC_DRAM,
26557c478bd9Sstevel@tonic-gate 	    OPT_PCS_OFF_VENDOR));
26567c478bd9Sstevel@tonic-gate 	start = gethrtime();
26572e2c009bSjjc 	for (cnt = 0; cnt < nreads; cnt++)
26587c478bd9Sstevel@tonic-gate 		dev_vendor = inl(PCI_CONFDATA);
26597c478bd9Sstevel@tonic-gate 	end = gethrtime();
26602e2c009bSjjc 	elapsed = (end - start) / nreads;
26617c478bd9Sstevel@tonic-gate 	splx(ipl);
26627c478bd9Sstevel@tonic-gate 	kpreempt_enable();
26632e2c009bSjjc 	return (elapsed);
26647c478bd9Sstevel@tonic-gate }
2665