xref: /titanic_41/usr/src/uts/common/os/lgrp.c (revision 2cb27123907a098a777e39eebc349d73e99a518f)
17c478bd9Sstevel@tonic-gate /*
27c478bd9Sstevel@tonic-gate  * CDDL HEADER START
37c478bd9Sstevel@tonic-gate  *
47c478bd9Sstevel@tonic-gate  * The contents of this file are subject to the terms of the
5ab761399Sesaxe  * Common Development and Distribution License (the "License").
6ab761399Sesaxe  * You may not use this file except in compliance with the License.
77c478bd9Sstevel@tonic-gate  *
87c478bd9Sstevel@tonic-gate  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
97c478bd9Sstevel@tonic-gate  * or http://www.opensolaris.org/os/licensing.
107c478bd9Sstevel@tonic-gate  * See the License for the specific language governing permissions
117c478bd9Sstevel@tonic-gate  * and limitations under the License.
127c478bd9Sstevel@tonic-gate  *
137c478bd9Sstevel@tonic-gate  * When distributing Covered Code, include this CDDL HEADER in each
147c478bd9Sstevel@tonic-gate  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
157c478bd9Sstevel@tonic-gate  * If applicable, add the following below this CDDL HEADER, with the
167c478bd9Sstevel@tonic-gate  * fields enclosed by brackets "[]" replaced with your own identifying
177c478bd9Sstevel@tonic-gate  * information: Portions Copyright [yyyy] [name of copyright owner]
187c478bd9Sstevel@tonic-gate  *
197c478bd9Sstevel@tonic-gate  * CDDL HEADER END
207c478bd9Sstevel@tonic-gate  */
217c478bd9Sstevel@tonic-gate /*
22fb2f18f8Sesaxe  * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
237c478bd9Sstevel@tonic-gate  * Use is subject to license terms.
247c478bd9Sstevel@tonic-gate  */
257c478bd9Sstevel@tonic-gate 
267c478bd9Sstevel@tonic-gate #pragma ident	"%Z%%M%	%I%	%E% SMI"
277c478bd9Sstevel@tonic-gate 
287c478bd9Sstevel@tonic-gate /*
297c478bd9Sstevel@tonic-gate  * Basic NUMA support in terms of locality groups
307c478bd9Sstevel@tonic-gate  *
317c478bd9Sstevel@tonic-gate  * Solaris needs to know which CPUs, memory, etc. are near each other to
327c478bd9Sstevel@tonic-gate  * provide good performance on NUMA machines by optimizing for locality.
337c478bd9Sstevel@tonic-gate  * In order to do this, a new abstraction called a "locality group (lgroup)"
347c478bd9Sstevel@tonic-gate  * has been introduced to keep track of which CPU-like and memory-like hardware
357c478bd9Sstevel@tonic-gate  * resources are close to each other.  Currently, latency is the only measure
367c478bd9Sstevel@tonic-gate  * used to determine how to group hardware resources into lgroups, but this
377c478bd9Sstevel@tonic-gate  * does not limit the groupings to be based solely on latency.  Other factors
387c478bd9Sstevel@tonic-gate  * may be used to determine the groupings in the future.
397c478bd9Sstevel@tonic-gate  *
407c478bd9Sstevel@tonic-gate  * Lgroups are organized into a hieararchy or topology that represents the
417c478bd9Sstevel@tonic-gate  * latency topology of the machine.  There is always at least a root lgroup in
427c478bd9Sstevel@tonic-gate  * the system.  It represents all the hardware resources in the machine at a
437c478bd9Sstevel@tonic-gate  * latency big enough that any hardware resource can at least access any other
447c478bd9Sstevel@tonic-gate  * hardware resource within that latency.  A Uniform Memory Access (UMA)
457c478bd9Sstevel@tonic-gate  * machine is represented with one lgroup (the root).  In contrast, a NUMA
467c478bd9Sstevel@tonic-gate  * machine is represented at least by the root lgroup and some number of leaf
477c478bd9Sstevel@tonic-gate  * lgroups where the leaf lgroups contain the hardware resources within the
487c478bd9Sstevel@tonic-gate  * least latency of each other and the root lgroup still contains all the
497c478bd9Sstevel@tonic-gate  * resources in the machine.  Some number of intermediate lgroups may exist
507c478bd9Sstevel@tonic-gate  * which represent more levels of locality than just the local latency of the
517c478bd9Sstevel@tonic-gate  * leaf lgroups and the system latency of the root lgroup.  Non-leaf lgroups
527c478bd9Sstevel@tonic-gate  * (eg. root and intermediate lgroups) contain the next nearest resources to
537c478bd9Sstevel@tonic-gate  * its children lgroups.  Thus, the lgroup hierarchy from a given leaf lgroup
547c478bd9Sstevel@tonic-gate  * to the root lgroup shows the hardware resources from closest to farthest
557c478bd9Sstevel@tonic-gate  * from the leaf lgroup such that each successive ancestor lgroup contains
567c478bd9Sstevel@tonic-gate  * the next nearest resources at the next level of locality from the previous.
577c478bd9Sstevel@tonic-gate  *
587c478bd9Sstevel@tonic-gate  * The kernel uses the lgroup abstraction to know how to allocate resources
597c478bd9Sstevel@tonic-gate  * near a given process/thread.  At fork() and lwp/thread_create() time, a
607c478bd9Sstevel@tonic-gate  * "home" lgroup is chosen for a thread.  This is done by picking the lgroup
617c478bd9Sstevel@tonic-gate  * with the lowest load average.  Binding to a processor or processor set will
627c478bd9Sstevel@tonic-gate  * change the home lgroup for a thread.  The scheduler has been modified to try
637c478bd9Sstevel@tonic-gate  * to dispatch a thread on a CPU in its home lgroup.  Physical memory
647c478bd9Sstevel@tonic-gate  * allocation is lgroup aware too, so memory will be allocated from the current
657c478bd9Sstevel@tonic-gate  * thread's home lgroup if possible.  If the desired resources are not
667c478bd9Sstevel@tonic-gate  * available, the kernel traverses the lgroup hierarchy going to the parent
677c478bd9Sstevel@tonic-gate  * lgroup to find resources at the next level of locality until it reaches the
687c478bd9Sstevel@tonic-gate  * root lgroup.
697c478bd9Sstevel@tonic-gate  */
707c478bd9Sstevel@tonic-gate 
717c478bd9Sstevel@tonic-gate #include <sys/lgrp.h>
727c478bd9Sstevel@tonic-gate #include <sys/lgrp_user.h>
737c478bd9Sstevel@tonic-gate #include <sys/types.h>
747c478bd9Sstevel@tonic-gate #include <sys/mman.h>
757c478bd9Sstevel@tonic-gate #include <sys/param.h>
767c478bd9Sstevel@tonic-gate #include <sys/var.h>
777c478bd9Sstevel@tonic-gate #include <sys/thread.h>
787c478bd9Sstevel@tonic-gate #include <sys/cpuvar.h>
797c478bd9Sstevel@tonic-gate #include <sys/cpupart.h>
807c478bd9Sstevel@tonic-gate #include <sys/kmem.h>
817c478bd9Sstevel@tonic-gate #include <vm/seg.h>
827c478bd9Sstevel@tonic-gate #include <vm/seg_kmem.h>
837c478bd9Sstevel@tonic-gate #include <vm/seg_spt.h>
847c478bd9Sstevel@tonic-gate #include <vm/seg_vn.h>
857c478bd9Sstevel@tonic-gate #include <vm/as.h>
867c478bd9Sstevel@tonic-gate #include <sys/atomic.h>
877c478bd9Sstevel@tonic-gate #include <sys/systm.h>
887c478bd9Sstevel@tonic-gate #include <sys/errno.h>
897c478bd9Sstevel@tonic-gate #include <sys/cmn_err.h>
907c478bd9Sstevel@tonic-gate #include <sys/kstat.h>
917c478bd9Sstevel@tonic-gate #include <sys/sysmacros.h>
92fb2f18f8Sesaxe #include <sys/pg.h>
937c478bd9Sstevel@tonic-gate #include <sys/promif.h>
947c478bd9Sstevel@tonic-gate #include <sys/sdt.h>
957c478bd9Sstevel@tonic-gate 
967c478bd9Sstevel@tonic-gate lgrp_gen_t	lgrp_gen = 0;		/* generation of lgroup hierarchy */
977c478bd9Sstevel@tonic-gate lgrp_t *lgrp_table[NLGRPS_MAX]; /* table of all initialized lgrp_t structs */
987c478bd9Sstevel@tonic-gate 				/* indexed by lgrp_id */
997c478bd9Sstevel@tonic-gate int	nlgrps;			/* number of lgroups in machine */
1007c478bd9Sstevel@tonic-gate int	lgrp_alloc_hint = -1;	/* hint for where to try to allocate next */
1017c478bd9Sstevel@tonic-gate int	lgrp_alloc_max = 0;	/* max lgroup ID allocated so far */
1027c478bd9Sstevel@tonic-gate 
1037c478bd9Sstevel@tonic-gate /*
1047c478bd9Sstevel@tonic-gate  * Kstat data for lgroups.
1057c478bd9Sstevel@tonic-gate  *
1067c478bd9Sstevel@tonic-gate  * Actual kstat data is collected in lgrp_stats array.
1077c478bd9Sstevel@tonic-gate  * The lgrp_kstat_data array of named kstats is used to extract data from
1087c478bd9Sstevel@tonic-gate  * lgrp_stats and present it to kstat framework. It is protected from partallel
1097c478bd9Sstevel@tonic-gate  * modifications by lgrp_kstat_mutex. This may cause some contention when
1107c478bd9Sstevel@tonic-gate  * several kstat commands run in parallel but this is not the
1117c478bd9Sstevel@tonic-gate  * performance-critical path.
1127c478bd9Sstevel@tonic-gate  */
1137c478bd9Sstevel@tonic-gate extern struct lgrp_stats lgrp_stats[];	/* table of per-lgrp stats */
1147c478bd9Sstevel@tonic-gate 
1157c478bd9Sstevel@tonic-gate /*
1167c478bd9Sstevel@tonic-gate  * Declare kstat names statically for enums as defined in the header file.
1177c478bd9Sstevel@tonic-gate  */
1187c478bd9Sstevel@tonic-gate LGRP_KSTAT_NAMES;
1197c478bd9Sstevel@tonic-gate 
1207c478bd9Sstevel@tonic-gate static void	lgrp_kstat_init(void);
1217c478bd9Sstevel@tonic-gate static int	lgrp_kstat_extract(kstat_t *, int);
1227c478bd9Sstevel@tonic-gate static void	lgrp_kstat_reset(lgrp_id_t);
1237c478bd9Sstevel@tonic-gate 
1247c478bd9Sstevel@tonic-gate static struct kstat_named lgrp_kstat_data[LGRP_NUM_STATS];
1257c478bd9Sstevel@tonic-gate static kmutex_t lgrp_kstat_mutex;
1267c478bd9Sstevel@tonic-gate 
1277c478bd9Sstevel@tonic-gate 
1287c478bd9Sstevel@tonic-gate /*
1297c478bd9Sstevel@tonic-gate  * max number of lgroups supported by the platform
1307c478bd9Sstevel@tonic-gate  */
1317c478bd9Sstevel@tonic-gate int	nlgrpsmax = 0;
1327c478bd9Sstevel@tonic-gate 
1337c478bd9Sstevel@tonic-gate /*
1347c478bd9Sstevel@tonic-gate  * The root lgroup. Represents the set of resources at the system wide
1357c478bd9Sstevel@tonic-gate  * level of locality.
1367c478bd9Sstevel@tonic-gate  */
1377c478bd9Sstevel@tonic-gate lgrp_t		*lgrp_root = NULL;
1387c478bd9Sstevel@tonic-gate 
1397c478bd9Sstevel@tonic-gate /*
1407c478bd9Sstevel@tonic-gate  * During system bootstrap cp_default does not contain the list of lgrp load
1417c478bd9Sstevel@tonic-gate  * averages (cp_lgrploads). The list is allocated after the first CPU is brought
1427c478bd9Sstevel@tonic-gate  * on-line when cp_default is initialized by cpupart_initialize_default().
1437c478bd9Sstevel@tonic-gate  * Configuring CPU0 may create a two-level topology with root and one leaf node
1447c478bd9Sstevel@tonic-gate  * containing CPU0. This topology is initially constructed in a special
1457c478bd9Sstevel@tonic-gate  * statically allocated 2-element lpl list lpl_bootstrap_list and later cloned
1467c478bd9Sstevel@tonic-gate  * to cp_default when cp_default is initialized. The lpl_bootstrap_list is used
1477c478bd9Sstevel@tonic-gate  * for all lpl operations until cp_default is fully constructed.
1487c478bd9Sstevel@tonic-gate  *
1497c478bd9Sstevel@tonic-gate  * The lpl_bootstrap_list is maintained by the code in lgrp.c. Every other
1507c478bd9Sstevel@tonic-gate  * consumer who needs default lpl should use lpl_bootstrap which is a pointer to
1517c478bd9Sstevel@tonic-gate  * the first element of lpl_bootstrap_list.
152394b433dSesaxe  *
153394b433dSesaxe  * CPUs that are added to the system, but have not yet been assigned to an
154394b433dSesaxe  * lgrp will use lpl_bootstrap as a default lpl. This is necessary because
155394b433dSesaxe  * on some architectures (x86) it's possible for the slave CPU startup thread
156394b433dSesaxe  * to enter the dispatcher or allocate memory before calling lgrp_cpu_init().
1577c478bd9Sstevel@tonic-gate  */
1587c478bd9Sstevel@tonic-gate #define	LPL_BOOTSTRAP_SIZE 2
1597c478bd9Sstevel@tonic-gate static lpl_t	lpl_bootstrap_list[LPL_BOOTSTRAP_SIZE];
1607c478bd9Sstevel@tonic-gate lpl_t		*lpl_bootstrap;
1617c478bd9Sstevel@tonic-gate 
162394b433dSesaxe /*
163394b433dSesaxe  * If cp still references the bootstrap lpl, it has not yet been added to
164394b433dSesaxe  * an lgrp. lgrp_mem_choose() uses this macro to detect the case where
165394b433dSesaxe  * a thread is trying to allocate memory close to a CPU that has no lgrp.
166394b433dSesaxe  */
167394b433dSesaxe #define	LGRP_CPU_HAS_NO_LGRP(cp)	((cp)->cpu_lpl == lpl_bootstrap)
168394b433dSesaxe 
1697c478bd9Sstevel@tonic-gate static lgrp_t	lroot;
1707c478bd9Sstevel@tonic-gate 
1717c478bd9Sstevel@tonic-gate /*
1727c478bd9Sstevel@tonic-gate  * Size, in bytes, beyond which random memory allocation policy is applied
1737c478bd9Sstevel@tonic-gate  * to non-shared memory.  Default is the maximum size, so random memory
1747c478bd9Sstevel@tonic-gate  * allocation won't be used for non-shared memory by default.
1757c478bd9Sstevel@tonic-gate  */
1767c478bd9Sstevel@tonic-gate size_t	lgrp_privm_random_thresh = (size_t)(-1);
1777c478bd9Sstevel@tonic-gate 
178c6402783Sakolb /* the maximum effect that a single thread can have on it's lgroup's load */
179c6402783Sakolb #define	LGRP_LOADAVG_MAX_EFFECT(ncpu) \
180c6402783Sakolb 	((lgrp_loadavg_max_effect) / (ncpu))
181c6402783Sakolb uint32_t	lgrp_loadavg_max_effect = LGRP_LOADAVG_THREAD_MAX;
182c6402783Sakolb 
183c6402783Sakolb 
1847c478bd9Sstevel@tonic-gate /*
1857c478bd9Sstevel@tonic-gate  * Size, in bytes, beyond which random memory allocation policy is applied to
1867c478bd9Sstevel@tonic-gate  * shared memory.  Default is 8MB (2 ISM pages).
1877c478bd9Sstevel@tonic-gate  */
1887c478bd9Sstevel@tonic-gate size_t	lgrp_shm_random_thresh = 8*1024*1024;
1897c478bd9Sstevel@tonic-gate 
1907c478bd9Sstevel@tonic-gate /*
1917c478bd9Sstevel@tonic-gate  * Whether to do processor set aware memory allocation by default
1927c478bd9Sstevel@tonic-gate  */
1937c478bd9Sstevel@tonic-gate int	lgrp_mem_pset_aware = 0;
1947c478bd9Sstevel@tonic-gate 
1957c478bd9Sstevel@tonic-gate /*
1967c478bd9Sstevel@tonic-gate  * Set the default memory allocation policy for root lgroup
1977c478bd9Sstevel@tonic-gate  */
1987c478bd9Sstevel@tonic-gate lgrp_mem_policy_t	lgrp_mem_policy_root = LGRP_MEM_POLICY_RANDOM;
1997c478bd9Sstevel@tonic-gate 
2007c478bd9Sstevel@tonic-gate /*
2017c478bd9Sstevel@tonic-gate  * Set the default memory allocation policy.  For most platforms,
2027c478bd9Sstevel@tonic-gate  * next touch is sufficient, but some platforms may wish to override
2037c478bd9Sstevel@tonic-gate  * this.
2047c478bd9Sstevel@tonic-gate  */
2057c478bd9Sstevel@tonic-gate lgrp_mem_policy_t	lgrp_mem_default_policy = LGRP_MEM_POLICY_NEXT;
2067c478bd9Sstevel@tonic-gate 
2077c478bd9Sstevel@tonic-gate 
2087c478bd9Sstevel@tonic-gate /*
2097c478bd9Sstevel@tonic-gate  * lgroup CPU event handlers
2107c478bd9Sstevel@tonic-gate  */
2117c478bd9Sstevel@tonic-gate static void	lgrp_cpu_init(struct cpu *);
2127c478bd9Sstevel@tonic-gate static void	lgrp_cpu_fini(struct cpu *, lgrp_id_t);
2137c478bd9Sstevel@tonic-gate static lgrp_t	*lgrp_cpu_to_lgrp(struct cpu *);
2147c478bd9Sstevel@tonic-gate 
2157c478bd9Sstevel@tonic-gate /*
2167c478bd9Sstevel@tonic-gate  * lgroup memory event handlers
2177c478bd9Sstevel@tonic-gate  */
2187c478bd9Sstevel@tonic-gate static void	lgrp_mem_init(int, lgrp_handle_t, boolean_t);
2197c478bd9Sstevel@tonic-gate static void	lgrp_mem_fini(int, lgrp_handle_t, boolean_t);
2207c478bd9Sstevel@tonic-gate static void	lgrp_mem_rename(int, lgrp_handle_t, lgrp_handle_t);
2217c478bd9Sstevel@tonic-gate 
2227c478bd9Sstevel@tonic-gate /*
2237c478bd9Sstevel@tonic-gate  * lgroup CPU partition event handlers
2247c478bd9Sstevel@tonic-gate  */
2257c478bd9Sstevel@tonic-gate static void	lgrp_part_add_cpu(struct cpu *, lgrp_id_t);
2267c478bd9Sstevel@tonic-gate static void	lgrp_part_del_cpu(struct cpu *);
2277c478bd9Sstevel@tonic-gate 
2287c478bd9Sstevel@tonic-gate static void	lgrp_root_init(void);
2297c478bd9Sstevel@tonic-gate 
2307c478bd9Sstevel@tonic-gate /*
2317c478bd9Sstevel@tonic-gate  * lpl topology
2327c478bd9Sstevel@tonic-gate  */
2337c478bd9Sstevel@tonic-gate static void	lpl_init(lpl_t *, lpl_t *, lgrp_t *);
2347c478bd9Sstevel@tonic-gate static void	lpl_clear(lpl_t *);
2357c478bd9Sstevel@tonic-gate static void	lpl_leaf_insert(lpl_t *, struct cpupart *);
2367c478bd9Sstevel@tonic-gate static void	lpl_leaf_remove(lpl_t *, struct cpupart *);
2377c478bd9Sstevel@tonic-gate static void	lpl_rset_add(lpl_t *, lpl_t *);
2387c478bd9Sstevel@tonic-gate static void	lpl_rset_del(lpl_t *, lpl_t *);
2397c478bd9Sstevel@tonic-gate static int	lpl_rset_contains(lpl_t *, lpl_t *);
2407c478bd9Sstevel@tonic-gate static void	lpl_cpu_adjcnt(lpl_act_t, struct cpu *);
2417c478bd9Sstevel@tonic-gate static void	lpl_child_update(lpl_t *, struct cpupart *);
2427c478bd9Sstevel@tonic-gate static int	lpl_pick(lpl_t *, lpl_t *);
2437c478bd9Sstevel@tonic-gate static void	lpl_verify_wrapper(struct cpupart *);
2447c478bd9Sstevel@tonic-gate 
2457c478bd9Sstevel@tonic-gate /*
2467c478bd9Sstevel@tonic-gate  * defines for lpl topology verifier return codes
2477c478bd9Sstevel@tonic-gate  */
2487c478bd9Sstevel@tonic-gate 
2497c478bd9Sstevel@tonic-gate #define	LPL_TOPO_CORRECT			0
2507c478bd9Sstevel@tonic-gate #define	LPL_TOPO_PART_HAS_NO_LPL		-1
2517c478bd9Sstevel@tonic-gate #define	LPL_TOPO_CPUS_NOT_EMPTY			-2
2527c478bd9Sstevel@tonic-gate #define	LPL_TOPO_LGRP_MISMATCH			-3
2537c478bd9Sstevel@tonic-gate #define	LPL_TOPO_MISSING_PARENT			-4
2547c478bd9Sstevel@tonic-gate #define	LPL_TOPO_PARENT_MISMATCH		-5
2557c478bd9Sstevel@tonic-gate #define	LPL_TOPO_BAD_CPUCNT			-6
2567c478bd9Sstevel@tonic-gate #define	LPL_TOPO_RSET_MISMATCH			-7
2577c478bd9Sstevel@tonic-gate #define	LPL_TOPO_LPL_ORPHANED			-8
2587c478bd9Sstevel@tonic-gate #define	LPL_TOPO_LPL_BAD_NCPU			-9
2597c478bd9Sstevel@tonic-gate #define	LPL_TOPO_RSET_MSSNG_LF			-10
2607c478bd9Sstevel@tonic-gate #define	LPL_TOPO_CPU_HAS_BAD_LPL		-11
2617c478bd9Sstevel@tonic-gate #define	LPL_TOPO_BOGUS_HINT			-12
2627c478bd9Sstevel@tonic-gate #define	LPL_TOPO_NONLEAF_HAS_CPUS		-13
2637c478bd9Sstevel@tonic-gate #define	LPL_TOPO_LGRP_NOT_LEAF			-14
2647c478bd9Sstevel@tonic-gate #define	LPL_TOPO_BAD_RSETCNT			-15
2657c478bd9Sstevel@tonic-gate 
2667c478bd9Sstevel@tonic-gate /*
2677c478bd9Sstevel@tonic-gate  * Return whether lgroup optimizations should be enabled on this system
2687c478bd9Sstevel@tonic-gate  */
2697c478bd9Sstevel@tonic-gate int
2707c478bd9Sstevel@tonic-gate lgrp_optimizations(void)
2717c478bd9Sstevel@tonic-gate {
2727c478bd9Sstevel@tonic-gate 	/*
2737c478bd9Sstevel@tonic-gate 	 * System must have more than 2 lgroups to enable lgroup optimizations
2747c478bd9Sstevel@tonic-gate 	 *
2757c478bd9Sstevel@tonic-gate 	 * XXX This assumes that a 2 lgroup system has an empty root lgroup
2767c478bd9Sstevel@tonic-gate 	 * with one child lgroup containing all the resources. A 2 lgroup
2777c478bd9Sstevel@tonic-gate 	 * system with a root lgroup directly containing CPUs or memory might
2787c478bd9Sstevel@tonic-gate 	 * need lgroup optimizations with its child lgroup, but there
2797c478bd9Sstevel@tonic-gate 	 * isn't such a machine for now....
2807c478bd9Sstevel@tonic-gate 	 */
2817c478bd9Sstevel@tonic-gate 	if (nlgrps > 2)
2827c478bd9Sstevel@tonic-gate 		return (1);
2837c478bd9Sstevel@tonic-gate 
2847c478bd9Sstevel@tonic-gate 	return (0);
2857c478bd9Sstevel@tonic-gate }
2867c478bd9Sstevel@tonic-gate 
2877c478bd9Sstevel@tonic-gate /*
2887c478bd9Sstevel@tonic-gate  * Build full lgroup topology
2897c478bd9Sstevel@tonic-gate  */
2907c478bd9Sstevel@tonic-gate static void
2917c478bd9Sstevel@tonic-gate lgrp_root_init(void)
2927c478bd9Sstevel@tonic-gate {
2937c478bd9Sstevel@tonic-gate 	lgrp_handle_t	hand;
2947c478bd9Sstevel@tonic-gate 	int		i;
2957c478bd9Sstevel@tonic-gate 	lgrp_id_t	id;
2967c478bd9Sstevel@tonic-gate 
2977c478bd9Sstevel@tonic-gate 	/*
2987c478bd9Sstevel@tonic-gate 	 * Create the "root" lgroup
2997c478bd9Sstevel@tonic-gate 	 */
3007c478bd9Sstevel@tonic-gate 	ASSERT(nlgrps == 0);
3017c478bd9Sstevel@tonic-gate 	id = nlgrps++;
3027c478bd9Sstevel@tonic-gate 
3037c478bd9Sstevel@tonic-gate 	lgrp_root = &lroot;
3047c478bd9Sstevel@tonic-gate 
3057c478bd9Sstevel@tonic-gate 	lgrp_root->lgrp_cpu = NULL;
3067c478bd9Sstevel@tonic-gate 	lgrp_root->lgrp_mnodes = 0;
3077c478bd9Sstevel@tonic-gate 	lgrp_root->lgrp_nmnodes = 0;
3087c478bd9Sstevel@tonic-gate 	hand = lgrp_plat_root_hand();
3097c478bd9Sstevel@tonic-gate 	lgrp_root->lgrp_plathand = hand;
3107c478bd9Sstevel@tonic-gate 
3117c478bd9Sstevel@tonic-gate 	lgrp_root->lgrp_id = id;
3127c478bd9Sstevel@tonic-gate 	lgrp_root->lgrp_cpucnt = 0;
3137c478bd9Sstevel@tonic-gate 	lgrp_root->lgrp_childcnt = 0;
3147c478bd9Sstevel@tonic-gate 	klgrpset_clear(lgrp_root->lgrp_children);
3157c478bd9Sstevel@tonic-gate 	klgrpset_clear(lgrp_root->lgrp_leaves);
3167c478bd9Sstevel@tonic-gate 	lgrp_root->lgrp_parent = NULL;
3177c478bd9Sstevel@tonic-gate 	lgrp_root->lgrp_latency = lgrp_plat_latency(hand, hand);
3187c478bd9Sstevel@tonic-gate 
3197c478bd9Sstevel@tonic-gate 	for (i = 0; i < LGRP_RSRC_COUNT; i++)
3207c478bd9Sstevel@tonic-gate 		klgrpset_clear(lgrp_root->lgrp_set[i]);
3217c478bd9Sstevel@tonic-gate 
3227c478bd9Sstevel@tonic-gate 	lgrp_root->lgrp_kstat = NULL;
3237c478bd9Sstevel@tonic-gate 
3247c478bd9Sstevel@tonic-gate 	lgrp_table[id] = lgrp_root;
3257c478bd9Sstevel@tonic-gate 
3267c478bd9Sstevel@tonic-gate 	/*
3277c478bd9Sstevel@tonic-gate 	 * Setup initial lpl list for CPU0 and initial t0 home.
3287c478bd9Sstevel@tonic-gate 	 * The only lpl space we have so far is lpl_bootstrap. It is used for
329394b433dSesaxe 	 * all topology operations until cp_default is initialized at which
330394b433dSesaxe 	 * point t0.t_lpl will be updated.
3317c478bd9Sstevel@tonic-gate 	 */
3327c478bd9Sstevel@tonic-gate 	lpl_bootstrap = lpl_bootstrap_list;
3337c478bd9Sstevel@tonic-gate 	t0.t_lpl = lpl_bootstrap;
3347c478bd9Sstevel@tonic-gate 	cp_default.cp_nlgrploads = LPL_BOOTSTRAP_SIZE;
3357c478bd9Sstevel@tonic-gate 	lpl_bootstrap_list[1].lpl_lgrpid = 1;
3367c478bd9Sstevel@tonic-gate 	cp_default.cp_lgrploads = lpl_bootstrap;
3377c478bd9Sstevel@tonic-gate }
3387c478bd9Sstevel@tonic-gate 
3397c478bd9Sstevel@tonic-gate /*
3407c478bd9Sstevel@tonic-gate  * Initialize the lgroup framework and allow the platform to do the same
3417c478bd9Sstevel@tonic-gate  */
3427c478bd9Sstevel@tonic-gate void
3437c478bd9Sstevel@tonic-gate lgrp_init(void)
3447c478bd9Sstevel@tonic-gate {
3457c478bd9Sstevel@tonic-gate 	/*
3467c478bd9Sstevel@tonic-gate 	 * Initialize the platform
3477c478bd9Sstevel@tonic-gate 	 */
3487c478bd9Sstevel@tonic-gate 	lgrp_plat_init();
3497c478bd9Sstevel@tonic-gate 
3507c478bd9Sstevel@tonic-gate 	/*
3517c478bd9Sstevel@tonic-gate 	 * Set max number of lgroups supported on this platform which must be
3527c478bd9Sstevel@tonic-gate 	 * less than the max number of lgroups supported by the common lgroup
3537c478bd9Sstevel@tonic-gate 	 * framework (eg. NLGRPS_MAX is max elements in lgrp_table[], etc.)
3547c478bd9Sstevel@tonic-gate 	 */
3557c478bd9Sstevel@tonic-gate 	nlgrpsmax = lgrp_plat_max_lgrps();
3567c478bd9Sstevel@tonic-gate 	ASSERT(nlgrpsmax <= NLGRPS_MAX);
3577c478bd9Sstevel@tonic-gate }
3587c478bd9Sstevel@tonic-gate 
3597c478bd9Sstevel@tonic-gate /*
3607c478bd9Sstevel@tonic-gate  * Create the root and cpu0's lgroup, and set t0's home.
3617c478bd9Sstevel@tonic-gate  */
3627c478bd9Sstevel@tonic-gate void
3637c478bd9Sstevel@tonic-gate lgrp_setup(void)
3647c478bd9Sstevel@tonic-gate {
3657c478bd9Sstevel@tonic-gate 	/*
3667c478bd9Sstevel@tonic-gate 	 * Setup the root lgroup
3677c478bd9Sstevel@tonic-gate 	 */
3687c478bd9Sstevel@tonic-gate 	lgrp_root_init();
3697c478bd9Sstevel@tonic-gate 
3707c478bd9Sstevel@tonic-gate 	/*
3717c478bd9Sstevel@tonic-gate 	 * Add cpu0 to an lgroup
3727c478bd9Sstevel@tonic-gate 	 */
3737c478bd9Sstevel@tonic-gate 	lgrp_config(LGRP_CONFIG_CPU_ADD, (uintptr_t)CPU, 0);
3747c478bd9Sstevel@tonic-gate 	lgrp_config(LGRP_CONFIG_CPU_ONLINE, (uintptr_t)CPU, 0);
3757c478bd9Sstevel@tonic-gate }
3767c478bd9Sstevel@tonic-gate 
3777c478bd9Sstevel@tonic-gate /*
3787c478bd9Sstevel@tonic-gate  * Lgroup initialization is split in two parts. The first part
3797c478bd9Sstevel@tonic-gate  * (lgrp_main_init()) is called right before start_other_cpus() in main. The
3807c478bd9Sstevel@tonic-gate  * second part (lgrp_main_mp_init()) is called right after start_other_cpus()
3817c478bd9Sstevel@tonic-gate  * when all CPUs are brought online and all distance information is available.
3827c478bd9Sstevel@tonic-gate  *
3837c478bd9Sstevel@tonic-gate  * When lgrp_main_init() is complete it sets lgrp_initialized. The
3847c478bd9Sstevel@tonic-gate  * lgrp_main_mp_init() sets lgrp_topo_initialized.
3857c478bd9Sstevel@tonic-gate  */
3867c478bd9Sstevel@tonic-gate 
3877c478bd9Sstevel@tonic-gate /*
3887c478bd9Sstevel@tonic-gate  * true when lgrp initialization has been completed.
3897c478bd9Sstevel@tonic-gate  */
3907c478bd9Sstevel@tonic-gate int	lgrp_initialized = 0;
3917c478bd9Sstevel@tonic-gate 
3927c478bd9Sstevel@tonic-gate /*
3937c478bd9Sstevel@tonic-gate  * True when lgrp topology is constructed.
3947c478bd9Sstevel@tonic-gate  */
3957c478bd9Sstevel@tonic-gate int	lgrp_topo_initialized = 0;
3967c478bd9Sstevel@tonic-gate 
3977c478bd9Sstevel@tonic-gate /*
3987c478bd9Sstevel@tonic-gate  * Init routine called after startup(), /etc/system has been processed,
3997c478bd9Sstevel@tonic-gate  * and cpu0 has been added to an lgroup.
4007c478bd9Sstevel@tonic-gate  */
4017c478bd9Sstevel@tonic-gate void
4027c478bd9Sstevel@tonic-gate lgrp_main_init(void)
4037c478bd9Sstevel@tonic-gate {
4047c478bd9Sstevel@tonic-gate 	cpu_t		*cp = CPU;
4057c478bd9Sstevel@tonic-gate 	lgrp_id_t	lgrpid;
4067c478bd9Sstevel@tonic-gate 	int		i;
407a6604450Sesaxe 	extern void	pg_cpu0_reinit();
408a6604450Sesaxe 
4097c478bd9Sstevel@tonic-gate 	/*
4107c478bd9Sstevel@tonic-gate 	 * Enforce a valid lgrp_mem_default_policy
4117c478bd9Sstevel@tonic-gate 	 */
4127c478bd9Sstevel@tonic-gate 	if ((lgrp_mem_default_policy <= LGRP_MEM_POLICY_DEFAULT) ||
413*2cb27123Saguzovsk 	    (lgrp_mem_default_policy >= LGRP_NUM_MEM_POLICIES) ||
414*2cb27123Saguzovsk 	    (lgrp_mem_default_policy == LGRP_MEM_POLICY_NEXT_SEG))
4157c478bd9Sstevel@tonic-gate 		lgrp_mem_default_policy = LGRP_MEM_POLICY_NEXT;
4167c478bd9Sstevel@tonic-gate 
4177c478bd9Sstevel@tonic-gate 	/*
4187c478bd9Sstevel@tonic-gate 	 * See if mpo should be disabled.
4197c478bd9Sstevel@tonic-gate 	 * This may happen in the case of null proc LPA on Starcat.
4207c478bd9Sstevel@tonic-gate 	 * The platform won't be able to detect null proc LPA until after
4217c478bd9Sstevel@tonic-gate 	 * cpu0 and memory have already been added to lgroups.
4227c478bd9Sstevel@tonic-gate 	 * When and if it is detected, the Starcat platform will return
4237c478bd9Sstevel@tonic-gate 	 * a different platform handle for cpu0 which is what we check for
4247c478bd9Sstevel@tonic-gate 	 * here. If mpo should be disabled move cpu0 to it's rightful place
4257c478bd9Sstevel@tonic-gate 	 * (the root), and destroy the remaining lgroups. This effectively
4267c478bd9Sstevel@tonic-gate 	 * provides an UMA lgroup topology.
4277c478bd9Sstevel@tonic-gate 	 */
4287c478bd9Sstevel@tonic-gate 	lgrpid = cp->cpu_lpl->lpl_lgrpid;
4297c478bd9Sstevel@tonic-gate 	if (lgrp_table[lgrpid]->lgrp_plathand !=
4307c478bd9Sstevel@tonic-gate 	    lgrp_plat_cpu_to_hand(cp->cpu_id)) {
4317c478bd9Sstevel@tonic-gate 		lgrp_part_del_cpu(cp);
4327c478bd9Sstevel@tonic-gate 		lgrp_cpu_fini(cp, lgrpid);
4337c478bd9Sstevel@tonic-gate 
4347c478bd9Sstevel@tonic-gate 		lgrp_cpu_init(cp);
4357c478bd9Sstevel@tonic-gate 		lgrp_part_add_cpu(cp, cp->cpu_lpl->lpl_lgrpid);
4367c478bd9Sstevel@tonic-gate 
4377c478bd9Sstevel@tonic-gate 		ASSERT(cp->cpu_lpl->lpl_lgrpid == LGRP_ROOTID);
4387c478bd9Sstevel@tonic-gate 
4398c6a5496Sjjc 		/*
440a6604450Sesaxe 		 * Notify the PG subsystem that the CPU's lgrp
441a6604450Sesaxe 		 * association has changed
442a6604450Sesaxe 		 */
443a6604450Sesaxe 		pg_cpu0_reinit();
444a6604450Sesaxe 
445a6604450Sesaxe 		/*
4468c6a5496Sjjc 		 * Destroy all lgroups except for root
4478c6a5496Sjjc 		 */
4487c478bd9Sstevel@tonic-gate 		for (i = 0; i <= lgrp_alloc_max; i++) {
4497c478bd9Sstevel@tonic-gate 			if (LGRP_EXISTS(lgrp_table[i]) &&
4507c478bd9Sstevel@tonic-gate 			    lgrp_table[i] != lgrp_root)
4517c478bd9Sstevel@tonic-gate 				lgrp_destroy(lgrp_table[i]);
4527c478bd9Sstevel@tonic-gate 		}
4538c6a5496Sjjc 
4548c6a5496Sjjc 		/*
4558c6a5496Sjjc 		 * Fix up root to point at itself for leaves and resources
4568c6a5496Sjjc 		 * and not have any children
4578c6a5496Sjjc 		 */
4588c6a5496Sjjc 		lgrp_root->lgrp_childcnt = 0;
4598c6a5496Sjjc 		klgrpset_clear(lgrp_root->lgrp_children);
4608c6a5496Sjjc 		klgrpset_clear(lgrp_root->lgrp_leaves);
4618c6a5496Sjjc 		klgrpset_add(lgrp_root->lgrp_leaves, LGRP_ROOTID);
4627c478bd9Sstevel@tonic-gate 		klgrpset_clear(lgrp_root->lgrp_set[LGRP_RSRC_MEM]);
4637c478bd9Sstevel@tonic-gate 		klgrpset_add(lgrp_root->lgrp_set[LGRP_RSRC_MEM], LGRP_ROOTID);
4647c478bd9Sstevel@tonic-gate 	}
4657c478bd9Sstevel@tonic-gate 
4667c478bd9Sstevel@tonic-gate 	/*
4677c478bd9Sstevel@tonic-gate 	 * Initialize kstats framework.
4687c478bd9Sstevel@tonic-gate 	 */
4697c478bd9Sstevel@tonic-gate 	lgrp_kstat_init();
4707c478bd9Sstevel@tonic-gate 	/*
4717c478bd9Sstevel@tonic-gate 	 * cpu0 is finally where it should be, so create it's lgroup's kstats
4727c478bd9Sstevel@tonic-gate 	 */
4737c478bd9Sstevel@tonic-gate 	mutex_enter(&cpu_lock);
4747c478bd9Sstevel@tonic-gate 	lgrp_kstat_create(cp);
4757c478bd9Sstevel@tonic-gate 	mutex_exit(&cpu_lock);
4767c478bd9Sstevel@tonic-gate 
4777c478bd9Sstevel@tonic-gate 	lgrp_plat_main_init();
4787c478bd9Sstevel@tonic-gate 	lgrp_initialized = 1;
4797c478bd9Sstevel@tonic-gate }
4807c478bd9Sstevel@tonic-gate 
4817c478bd9Sstevel@tonic-gate /*
4827c478bd9Sstevel@tonic-gate  * Finish lgrp initialization after all CPUS are brought on-line.
4837c478bd9Sstevel@tonic-gate  * This routine is called after start_other_cpus().
4847c478bd9Sstevel@tonic-gate  */
4857c478bd9Sstevel@tonic-gate void
4867c478bd9Sstevel@tonic-gate lgrp_main_mp_init(void)
4877c478bd9Sstevel@tonic-gate {
4887c478bd9Sstevel@tonic-gate 	klgrpset_t changed;
4897c478bd9Sstevel@tonic-gate 
4907c478bd9Sstevel@tonic-gate 	/*
4917c478bd9Sstevel@tonic-gate 	 * Update lgroup topology (if necessary)
4927c478bd9Sstevel@tonic-gate 	 */
4937c478bd9Sstevel@tonic-gate 	klgrpset_clear(changed);
4947c478bd9Sstevel@tonic-gate 	(void) lgrp_topo_update(lgrp_table, lgrp_alloc_max + 1, &changed);
4957c478bd9Sstevel@tonic-gate 	lgrp_topo_initialized = 1;
4967c478bd9Sstevel@tonic-gate }
4977c478bd9Sstevel@tonic-gate 
4987c478bd9Sstevel@tonic-gate /*
49903400a71Sjjc  * Change latency of lgroup with specified lgroup platform handle (if one is
50003400a71Sjjc  * given) or change all lgroups with old latency to new latency
50103400a71Sjjc  */
50203400a71Sjjc void
50303400a71Sjjc lgrp_latency_change(lgrp_handle_t hand, u_longlong_t oldtime,
50403400a71Sjjc     u_longlong_t newtime)
50503400a71Sjjc {
50603400a71Sjjc 	lgrp_t		*lgrp;
50703400a71Sjjc 	int		i;
50803400a71Sjjc 
50903400a71Sjjc 	for (i = 0; i <= lgrp_alloc_max; i++) {
51003400a71Sjjc 		lgrp = lgrp_table[i];
51103400a71Sjjc 
51203400a71Sjjc 		if (!LGRP_EXISTS(lgrp))
51303400a71Sjjc 			continue;
51403400a71Sjjc 
51503400a71Sjjc 		if ((hand == LGRP_NULL_HANDLE &&
51603400a71Sjjc 		    lgrp->lgrp_latency == oldtime) ||
51703400a71Sjjc 		    (hand != LGRP_NULL_HANDLE && lgrp->lgrp_plathand == hand))
51803400a71Sjjc 			lgrp->lgrp_latency = (int)newtime;
51903400a71Sjjc 	}
52003400a71Sjjc }
52103400a71Sjjc 
52203400a71Sjjc /*
5237c478bd9Sstevel@tonic-gate  * Handle lgroup (re)configuration events (eg. addition of CPU, etc.)
5247c478bd9Sstevel@tonic-gate  */
5257c478bd9Sstevel@tonic-gate void
5267c478bd9Sstevel@tonic-gate lgrp_config(lgrp_config_flag_t event, uintptr_t resource, uintptr_t where)
5277c478bd9Sstevel@tonic-gate {
5287c478bd9Sstevel@tonic-gate 	klgrpset_t	changed;
5297c478bd9Sstevel@tonic-gate 	cpu_t		*cp;
5307c478bd9Sstevel@tonic-gate 	lgrp_id_t	id;
5317c478bd9Sstevel@tonic-gate 	int		rc;
5327c478bd9Sstevel@tonic-gate 
5337c478bd9Sstevel@tonic-gate 	switch (event) {
5347c478bd9Sstevel@tonic-gate 	/*
5357c478bd9Sstevel@tonic-gate 	 * The following (re)configuration events are common code
5367c478bd9Sstevel@tonic-gate 	 * initiated. lgrp_plat_config() is called here to inform the
5377c478bd9Sstevel@tonic-gate 	 * platform of the reconfiguration event.
5387c478bd9Sstevel@tonic-gate 	 */
5397c478bd9Sstevel@tonic-gate 	case LGRP_CONFIG_CPU_ADD:
540394b433dSesaxe 		cp = (cpu_t *)resource;
541394b433dSesaxe 
542394b433dSesaxe 		/*
543394b433dSesaxe 		 * Initialize the new CPU's lgrp related next/prev
544394b433dSesaxe 		 * links, and give it a bootstrap lpl so that it can
545394b433dSesaxe 		 * survive should it need to enter the dispatcher.
546394b433dSesaxe 		 */
547394b433dSesaxe 		cp->cpu_next_lpl = cp;
548394b433dSesaxe 		cp->cpu_prev_lpl = cp;
549394b433dSesaxe 		cp->cpu_next_lgrp = cp;
550394b433dSesaxe 		cp->cpu_prev_lgrp = cp;
551394b433dSesaxe 		cp->cpu_lpl = lpl_bootstrap;
552394b433dSesaxe 
5537c478bd9Sstevel@tonic-gate 		lgrp_plat_config(event, resource);
5547c478bd9Sstevel@tonic-gate 		atomic_add_32(&lgrp_gen, 1);
5557c478bd9Sstevel@tonic-gate 
5567c478bd9Sstevel@tonic-gate 		break;
5577c478bd9Sstevel@tonic-gate 	case LGRP_CONFIG_CPU_DEL:
5587c478bd9Sstevel@tonic-gate 		lgrp_plat_config(event, resource);
5597c478bd9Sstevel@tonic-gate 		atomic_add_32(&lgrp_gen, 1);
5607c478bd9Sstevel@tonic-gate 
5617c478bd9Sstevel@tonic-gate 		break;
5627c478bd9Sstevel@tonic-gate 	case LGRP_CONFIG_CPU_ONLINE:
5637c478bd9Sstevel@tonic-gate 		cp = (cpu_t *)resource;
5647c478bd9Sstevel@tonic-gate 		lgrp_cpu_init(cp);
5657c478bd9Sstevel@tonic-gate 		lgrp_part_add_cpu(cp, cp->cpu_lpl->lpl_lgrpid);
5667c478bd9Sstevel@tonic-gate 		rc = lpl_topo_verify(cp->cpu_part);
5677c478bd9Sstevel@tonic-gate 		if (rc != LPL_TOPO_CORRECT) {
5687c478bd9Sstevel@tonic-gate 			panic("lpl_topo_verify failed: %d", rc);
5697c478bd9Sstevel@tonic-gate 		}
5707c478bd9Sstevel@tonic-gate 		lgrp_plat_config(event, resource);
5717c478bd9Sstevel@tonic-gate 		atomic_add_32(&lgrp_gen, 1);
5727c478bd9Sstevel@tonic-gate 
5737c478bd9Sstevel@tonic-gate 		break;
5747c478bd9Sstevel@tonic-gate 	case LGRP_CONFIG_CPU_OFFLINE:
5757c478bd9Sstevel@tonic-gate 		cp = (cpu_t *)resource;
5767c478bd9Sstevel@tonic-gate 		id = cp->cpu_lpl->lpl_lgrpid;
5777c478bd9Sstevel@tonic-gate 		lgrp_part_del_cpu(cp);
5787c478bd9Sstevel@tonic-gate 		lgrp_cpu_fini(cp, id);
5797c478bd9Sstevel@tonic-gate 		rc = lpl_topo_verify(cp->cpu_part);
5807c478bd9Sstevel@tonic-gate 		if (rc != LPL_TOPO_CORRECT) {
5817c478bd9Sstevel@tonic-gate 			panic("lpl_topo_verify failed: %d", rc);
5827c478bd9Sstevel@tonic-gate 		}
5837c478bd9Sstevel@tonic-gate 		lgrp_plat_config(event, resource);
5847c478bd9Sstevel@tonic-gate 		atomic_add_32(&lgrp_gen, 1);
5857c478bd9Sstevel@tonic-gate 
5867c478bd9Sstevel@tonic-gate 		break;
5877c478bd9Sstevel@tonic-gate 	case LGRP_CONFIG_CPUPART_ADD:
5887c478bd9Sstevel@tonic-gate 		cp = (cpu_t *)resource;
5897c478bd9Sstevel@tonic-gate 		lgrp_part_add_cpu((cpu_t *)resource, (lgrp_id_t)where);
5907c478bd9Sstevel@tonic-gate 		rc = lpl_topo_verify(cp->cpu_part);
5917c478bd9Sstevel@tonic-gate 		if (rc != LPL_TOPO_CORRECT) {
5927c478bd9Sstevel@tonic-gate 			panic("lpl_topo_verify failed: %d", rc);
5937c478bd9Sstevel@tonic-gate 		}
5947c478bd9Sstevel@tonic-gate 		lgrp_plat_config(event, resource);
5957c478bd9Sstevel@tonic-gate 
5967c478bd9Sstevel@tonic-gate 		break;
5977c478bd9Sstevel@tonic-gate 	case LGRP_CONFIG_CPUPART_DEL:
5987c478bd9Sstevel@tonic-gate 		cp = (cpu_t *)resource;
5997c478bd9Sstevel@tonic-gate 		lgrp_part_del_cpu((cpu_t *)resource);
6007c478bd9Sstevel@tonic-gate 		rc = lpl_topo_verify(cp->cpu_part);
6017c478bd9Sstevel@tonic-gate 		if (rc != LPL_TOPO_CORRECT) {
6027c478bd9Sstevel@tonic-gate 			panic("lpl_topo_verify failed: %d", rc);
6037c478bd9Sstevel@tonic-gate 		}
6047c478bd9Sstevel@tonic-gate 		lgrp_plat_config(event, resource);
6057c478bd9Sstevel@tonic-gate 
6067c478bd9Sstevel@tonic-gate 		break;
6077c478bd9Sstevel@tonic-gate 	/*
6087c478bd9Sstevel@tonic-gate 	 * The following events are initiated by the memnode
6097c478bd9Sstevel@tonic-gate 	 * subsystem.
6107c478bd9Sstevel@tonic-gate 	 */
6117c478bd9Sstevel@tonic-gate 	case LGRP_CONFIG_MEM_ADD:
6127c478bd9Sstevel@tonic-gate 		lgrp_mem_init((int)resource, where, B_FALSE);
6137c478bd9Sstevel@tonic-gate 		atomic_add_32(&lgrp_gen, 1);
6147c478bd9Sstevel@tonic-gate 
6157c478bd9Sstevel@tonic-gate 		break;
6167c478bd9Sstevel@tonic-gate 	case LGRP_CONFIG_MEM_DEL:
6177c478bd9Sstevel@tonic-gate 		lgrp_mem_fini((int)resource, where, B_FALSE);
6187c478bd9Sstevel@tonic-gate 		atomic_add_32(&lgrp_gen, 1);
6197c478bd9Sstevel@tonic-gate 
6207c478bd9Sstevel@tonic-gate 		break;
6217c478bd9Sstevel@tonic-gate 	case LGRP_CONFIG_MEM_RENAME: {
6227c478bd9Sstevel@tonic-gate 		lgrp_config_mem_rename_t *ren_arg =
6237c478bd9Sstevel@tonic-gate 		    (lgrp_config_mem_rename_t *)where;
6247c478bd9Sstevel@tonic-gate 
6257c478bd9Sstevel@tonic-gate 		lgrp_mem_rename((int)resource,
6267c478bd9Sstevel@tonic-gate 		    ren_arg->lmem_rename_from,
6277c478bd9Sstevel@tonic-gate 		    ren_arg->lmem_rename_to);
6287c478bd9Sstevel@tonic-gate 		atomic_add_32(&lgrp_gen, 1);
6297c478bd9Sstevel@tonic-gate 
6307c478bd9Sstevel@tonic-gate 		break;
6317c478bd9Sstevel@tonic-gate 	}
6327c478bd9Sstevel@tonic-gate 	case LGRP_CONFIG_GEN_UPDATE:
6337c478bd9Sstevel@tonic-gate 		atomic_add_32(&lgrp_gen, 1);
6347c478bd9Sstevel@tonic-gate 
6357c478bd9Sstevel@tonic-gate 		break;
6367c478bd9Sstevel@tonic-gate 	case LGRP_CONFIG_FLATTEN:
6377c478bd9Sstevel@tonic-gate 		if (where == 0)
6387c478bd9Sstevel@tonic-gate 			lgrp_topo_levels = (int)resource;
6397c478bd9Sstevel@tonic-gate 		else
6407c478bd9Sstevel@tonic-gate 			(void) lgrp_topo_flatten(resource,
6417c478bd9Sstevel@tonic-gate 			    lgrp_table, lgrp_alloc_max, &changed);
6427c478bd9Sstevel@tonic-gate 
6437c478bd9Sstevel@tonic-gate 		break;
6447c478bd9Sstevel@tonic-gate 	/*
64503400a71Sjjc 	 * Update any lgroups with old latency to new latency
6467c478bd9Sstevel@tonic-gate 	 */
64703400a71Sjjc 	case LGRP_CONFIG_LAT_CHANGE_ALL:
64803400a71Sjjc 		lgrp_latency_change(LGRP_NULL_HANDLE, (u_longlong_t)resource,
64903400a71Sjjc 		    (u_longlong_t)where);
65003400a71Sjjc 
65103400a71Sjjc 		break;
65203400a71Sjjc 	/*
65303400a71Sjjc 	 * Update lgroup with specified lgroup platform handle to have
65403400a71Sjjc 	 * new latency
65503400a71Sjjc 	 */
65603400a71Sjjc 	case LGRP_CONFIG_LAT_CHANGE:
65703400a71Sjjc 		lgrp_latency_change((lgrp_handle_t)resource, 0,
6587c478bd9Sstevel@tonic-gate 		    (u_longlong_t)where);
6597c478bd9Sstevel@tonic-gate 
6607c478bd9Sstevel@tonic-gate 		break;
6617c478bd9Sstevel@tonic-gate 	case LGRP_CONFIG_NOP:
6627c478bd9Sstevel@tonic-gate 
6637c478bd9Sstevel@tonic-gate 		break;
6647c478bd9Sstevel@tonic-gate 	default:
6657c478bd9Sstevel@tonic-gate 		break;
6667c478bd9Sstevel@tonic-gate 	}
6677c478bd9Sstevel@tonic-gate 
6687c478bd9Sstevel@tonic-gate }
6697c478bd9Sstevel@tonic-gate 
6707c478bd9Sstevel@tonic-gate /*
6717c478bd9Sstevel@tonic-gate  * Called to add lgrp info into cpu structure from cpu_add_unit;
6727c478bd9Sstevel@tonic-gate  * do not assume cpu is in cpu[] yet!
6737c478bd9Sstevel@tonic-gate  *
6747c478bd9Sstevel@tonic-gate  * CPUs are brought online with all other CPUs paused so we can't
6757c478bd9Sstevel@tonic-gate  * allocate memory or we could deadlock the system, so we rely on
6767c478bd9Sstevel@tonic-gate  * the platform to statically allocate as much space as we need
6777c478bd9Sstevel@tonic-gate  * for the lgrp structs and stats.
6787c478bd9Sstevel@tonic-gate  */
6797c478bd9Sstevel@tonic-gate static void
6807c478bd9Sstevel@tonic-gate lgrp_cpu_init(struct cpu *cp)
6817c478bd9Sstevel@tonic-gate {
6827c478bd9Sstevel@tonic-gate 	klgrpset_t	changed;
6837c478bd9Sstevel@tonic-gate 	int		count;
6847c478bd9Sstevel@tonic-gate 	lgrp_handle_t	hand;
6857c478bd9Sstevel@tonic-gate 	int		first_cpu;
6867c478bd9Sstevel@tonic-gate 	lgrp_t		*my_lgrp;
6877c478bd9Sstevel@tonic-gate 	lgrp_id_t	lgrpid;
6887c478bd9Sstevel@tonic-gate 	struct cpu	*cptr;
6897c478bd9Sstevel@tonic-gate 
6907c478bd9Sstevel@tonic-gate 	/*
6917c478bd9Sstevel@tonic-gate 	 * This is the first time through if the resource set
6927c478bd9Sstevel@tonic-gate 	 * for the root lgroup is empty. After cpu0 has been
6937c478bd9Sstevel@tonic-gate 	 * initially added to an lgroup, the root's CPU resource
6947c478bd9Sstevel@tonic-gate 	 * set can never be empty, since the system's last CPU
6957c478bd9Sstevel@tonic-gate 	 * cannot be offlined.
6967c478bd9Sstevel@tonic-gate 	 */
6977c478bd9Sstevel@tonic-gate 	if (klgrpset_isempty(lgrp_root->lgrp_set[LGRP_RSRC_CPU])) {
6987c478bd9Sstevel@tonic-gate 		/*
6997c478bd9Sstevel@tonic-gate 		 * First time through.
7007c478bd9Sstevel@tonic-gate 		 */
7017c478bd9Sstevel@tonic-gate 		first_cpu = 1;
7027c478bd9Sstevel@tonic-gate 	} else {
7037c478bd9Sstevel@tonic-gate 		/*
7047c478bd9Sstevel@tonic-gate 		 * If cpu0 needs to move lgroups, we may come
7057c478bd9Sstevel@tonic-gate 		 * through here again, at which time cpu_lock won't
7067c478bd9Sstevel@tonic-gate 		 * be held, and lgrp_initialized will be false.
7077c478bd9Sstevel@tonic-gate 		 */
7087c478bd9Sstevel@tonic-gate 		ASSERT(MUTEX_HELD(&cpu_lock) || !lgrp_initialized);
7097c478bd9Sstevel@tonic-gate 		ASSERT(cp->cpu_part != NULL);
7107c478bd9Sstevel@tonic-gate 		first_cpu = 0;
7117c478bd9Sstevel@tonic-gate 	}
7127c478bd9Sstevel@tonic-gate 
7137c478bd9Sstevel@tonic-gate 	hand = lgrp_plat_cpu_to_hand(cp->cpu_id);
7147c478bd9Sstevel@tonic-gate 	my_lgrp = lgrp_hand_to_lgrp(hand);
7157c478bd9Sstevel@tonic-gate 
7167c478bd9Sstevel@tonic-gate 	if (my_lgrp == NULL) {
7177c478bd9Sstevel@tonic-gate 		/*
7187c478bd9Sstevel@tonic-gate 		 * Create new lgrp and add it to lgroup topology
7197c478bd9Sstevel@tonic-gate 		 */
7207c478bd9Sstevel@tonic-gate 		my_lgrp = lgrp_create();
7217c478bd9Sstevel@tonic-gate 		my_lgrp->lgrp_plathand = hand;
7227c478bd9Sstevel@tonic-gate 		my_lgrp->lgrp_latency = lgrp_plat_latency(hand, hand);
7237c478bd9Sstevel@tonic-gate 		lgrpid = my_lgrp->lgrp_id;
7247c478bd9Sstevel@tonic-gate 		klgrpset_add(my_lgrp->lgrp_leaves, lgrpid);
7257c478bd9Sstevel@tonic-gate 		klgrpset_add(my_lgrp->lgrp_set[LGRP_RSRC_CPU], lgrpid);
7267c478bd9Sstevel@tonic-gate 
7277c478bd9Sstevel@tonic-gate 		count = 0;
7287c478bd9Sstevel@tonic-gate 		klgrpset_clear(changed);
7297c478bd9Sstevel@tonic-gate 		count += lgrp_leaf_add(my_lgrp, lgrp_table, lgrp_alloc_max + 1,
7307c478bd9Sstevel@tonic-gate 		    &changed);
7312dae3fb5Sjjc 		/*
7322dae3fb5Sjjc 		 * May have added new intermediate lgroups, so need to add
7332dae3fb5Sjjc 		 * resources other than CPUs which are added below
7342dae3fb5Sjjc 		 */
7352dae3fb5Sjjc 		(void) lgrp_mnode_update(changed, NULL);
7367c478bd9Sstevel@tonic-gate 	} else if (my_lgrp->lgrp_latency == 0 && lgrp_plat_latency(hand, hand)
7377c478bd9Sstevel@tonic-gate 	    > 0) {
7387c478bd9Sstevel@tonic-gate 		/*
7397c478bd9Sstevel@tonic-gate 		 * Leaf lgroup was created, but latency wasn't available
7407c478bd9Sstevel@tonic-gate 		 * then.  So, set latency for it and fill in rest of lgroup
7417c478bd9Sstevel@tonic-gate 		 * topology  now that we know how far it is from other leaf
7427c478bd9Sstevel@tonic-gate 		 * lgroups.
7437c478bd9Sstevel@tonic-gate 		 */
7447c478bd9Sstevel@tonic-gate 		lgrpid = my_lgrp->lgrp_id;
7457c478bd9Sstevel@tonic-gate 		klgrpset_clear(changed);
7467c478bd9Sstevel@tonic-gate 		if (!klgrpset_ismember(my_lgrp->lgrp_set[LGRP_RSRC_CPU],
7477c478bd9Sstevel@tonic-gate 		    lgrpid))
7487c478bd9Sstevel@tonic-gate 			klgrpset_add(my_lgrp->lgrp_set[LGRP_RSRC_CPU], lgrpid);
7497c478bd9Sstevel@tonic-gate 		count = lgrp_leaf_add(my_lgrp, lgrp_table, lgrp_alloc_max + 1,
7507c478bd9Sstevel@tonic-gate 		    &changed);
7517c478bd9Sstevel@tonic-gate 
7527c478bd9Sstevel@tonic-gate 		/*
7537c478bd9Sstevel@tonic-gate 		 * May have added new intermediate lgroups, so need to add
7547c478bd9Sstevel@tonic-gate 		 * resources other than CPUs which are added below
7557c478bd9Sstevel@tonic-gate 		 */
7567c478bd9Sstevel@tonic-gate 		(void) lgrp_mnode_update(changed, NULL);
7577c478bd9Sstevel@tonic-gate 	} else if (!klgrpset_ismember(my_lgrp->lgrp_set[LGRP_RSRC_CPU],
7587c478bd9Sstevel@tonic-gate 	    my_lgrp->lgrp_id)) {
7597c478bd9Sstevel@tonic-gate 		int	i;
7607c478bd9Sstevel@tonic-gate 
7617c478bd9Sstevel@tonic-gate 		/*
7627c478bd9Sstevel@tonic-gate 		 * Update existing lgroup and lgroups containing it with CPU
7637c478bd9Sstevel@tonic-gate 		 * resource
7647c478bd9Sstevel@tonic-gate 		 */
7657c478bd9Sstevel@tonic-gate 		lgrpid = my_lgrp->lgrp_id;
7667c478bd9Sstevel@tonic-gate 		klgrpset_add(my_lgrp->lgrp_set[LGRP_RSRC_CPU], lgrpid);
7677c478bd9Sstevel@tonic-gate 		for (i = 0; i <= lgrp_alloc_max; i++) {
7687c478bd9Sstevel@tonic-gate 			lgrp_t		*lgrp;
7697c478bd9Sstevel@tonic-gate 
7707c478bd9Sstevel@tonic-gate 			lgrp = lgrp_table[i];
7717c478bd9Sstevel@tonic-gate 			if (!LGRP_EXISTS(lgrp) ||
7727c478bd9Sstevel@tonic-gate 			    !lgrp_rsets_member(lgrp->lgrp_set, lgrpid))
7737c478bd9Sstevel@tonic-gate 				continue;
7747c478bd9Sstevel@tonic-gate 
7757c478bd9Sstevel@tonic-gate 			klgrpset_add(lgrp->lgrp_set[LGRP_RSRC_CPU], lgrpid);
7767c478bd9Sstevel@tonic-gate 		}
7777c478bd9Sstevel@tonic-gate 	}
7787c478bd9Sstevel@tonic-gate 
7797c478bd9Sstevel@tonic-gate 	lgrpid = my_lgrp->lgrp_id;
7807c478bd9Sstevel@tonic-gate 	cp->cpu_lpl = &cp->cpu_part->cp_lgrploads[lgrpid];
7817c478bd9Sstevel@tonic-gate 
7827c478bd9Sstevel@tonic-gate 	/*
7837c478bd9Sstevel@tonic-gate 	 * For multi-lgroup systems, need to setup lpl for CPU0 or CPU0 will
7847c478bd9Sstevel@tonic-gate 	 * end up in lpl for lgroup 0 whether it is supposed to be in there or
7857c478bd9Sstevel@tonic-gate 	 * not since none of lgroup IDs in the lpl's have been set yet.
7867c478bd9Sstevel@tonic-gate 	 */
7877c478bd9Sstevel@tonic-gate 	if (first_cpu && nlgrpsmax > 1 && lgrpid != cp->cpu_lpl->lpl_lgrpid)
7887c478bd9Sstevel@tonic-gate 		cp->cpu_lpl->lpl_lgrpid = lgrpid;
7897c478bd9Sstevel@tonic-gate 
7907c478bd9Sstevel@tonic-gate 	/*
7917c478bd9Sstevel@tonic-gate 	 * link the CPU into the lgrp's CPU list
7927c478bd9Sstevel@tonic-gate 	 */
7937c478bd9Sstevel@tonic-gate 	if (my_lgrp->lgrp_cpucnt == 0) {
7947c478bd9Sstevel@tonic-gate 		my_lgrp->lgrp_cpu = cp;
7957c478bd9Sstevel@tonic-gate 		cp->cpu_next_lgrp = cp->cpu_prev_lgrp = cp;
7967c478bd9Sstevel@tonic-gate 	} else {
7977c478bd9Sstevel@tonic-gate 		cptr = my_lgrp->lgrp_cpu;
7987c478bd9Sstevel@tonic-gate 		cp->cpu_next_lgrp = cptr;
7997c478bd9Sstevel@tonic-gate 		cp->cpu_prev_lgrp = cptr->cpu_prev_lgrp;
8007c478bd9Sstevel@tonic-gate 		cptr->cpu_prev_lgrp->cpu_next_lgrp = cp;
8017c478bd9Sstevel@tonic-gate 		cptr->cpu_prev_lgrp = cp;
8027c478bd9Sstevel@tonic-gate 	}
8037c478bd9Sstevel@tonic-gate 	my_lgrp->lgrp_cpucnt++;
8047c478bd9Sstevel@tonic-gate }
8057c478bd9Sstevel@tonic-gate 
8067c478bd9Sstevel@tonic-gate lgrp_t *
8077c478bd9Sstevel@tonic-gate lgrp_create(void)
8087c478bd9Sstevel@tonic-gate {
8097c478bd9Sstevel@tonic-gate 	lgrp_t		*my_lgrp;
8107c478bd9Sstevel@tonic-gate 	lgrp_id_t	lgrpid;
8117c478bd9Sstevel@tonic-gate 	int		i;
8127c478bd9Sstevel@tonic-gate 
8137c478bd9Sstevel@tonic-gate 	ASSERT(!lgrp_initialized || MUTEX_HELD(&cpu_lock));
8147c478bd9Sstevel@tonic-gate 
8157c478bd9Sstevel@tonic-gate 	/*
8167c478bd9Sstevel@tonic-gate 	 * Find an open slot in the lgroup table and recycle unused lgroup
8177c478bd9Sstevel@tonic-gate 	 * left there if any
8187c478bd9Sstevel@tonic-gate 	 */
8197c478bd9Sstevel@tonic-gate 	my_lgrp = NULL;
8207c478bd9Sstevel@tonic-gate 	if (lgrp_alloc_hint == -1)
8217c478bd9Sstevel@tonic-gate 		/*
8227c478bd9Sstevel@tonic-gate 		 * Allocate from end when hint not set yet because no lgroups
8237c478bd9Sstevel@tonic-gate 		 * have been deleted yet
8247c478bd9Sstevel@tonic-gate 		 */
8257c478bd9Sstevel@tonic-gate 		lgrpid = nlgrps++;
8267c478bd9Sstevel@tonic-gate 	else {
8277c478bd9Sstevel@tonic-gate 		/*
8287c478bd9Sstevel@tonic-gate 		 * Start looking for next open slot from hint and leave hint
8297c478bd9Sstevel@tonic-gate 		 * at slot allocated
8307c478bd9Sstevel@tonic-gate 		 */
8317c478bd9Sstevel@tonic-gate 		for (i = lgrp_alloc_hint; i < nlgrpsmax; i++) {
8327c478bd9Sstevel@tonic-gate 			my_lgrp = lgrp_table[i];
8337c478bd9Sstevel@tonic-gate 			if (!LGRP_EXISTS(my_lgrp)) {
8347c478bd9Sstevel@tonic-gate 				lgrpid = i;
8357c478bd9Sstevel@tonic-gate 				nlgrps++;
8367c478bd9Sstevel@tonic-gate 				break;
8377c478bd9Sstevel@tonic-gate 			}
8387c478bd9Sstevel@tonic-gate 		}
8397c478bd9Sstevel@tonic-gate 		lgrp_alloc_hint = lgrpid;
8407c478bd9Sstevel@tonic-gate 	}
8417c478bd9Sstevel@tonic-gate 
8427c478bd9Sstevel@tonic-gate 	/*
8437c478bd9Sstevel@tonic-gate 	 * Keep track of max lgroup ID allocated so far to cut down on searches
8447c478bd9Sstevel@tonic-gate 	 */
8457c478bd9Sstevel@tonic-gate 	if (lgrpid > lgrp_alloc_max)
8467c478bd9Sstevel@tonic-gate 		lgrp_alloc_max = lgrpid;
8477c478bd9Sstevel@tonic-gate 
8487c478bd9Sstevel@tonic-gate 	/*
8497c478bd9Sstevel@tonic-gate 	 * Need to allocate new lgroup if next open slot didn't have one
8507c478bd9Sstevel@tonic-gate 	 * for recycling
8517c478bd9Sstevel@tonic-gate 	 */
8527c478bd9Sstevel@tonic-gate 	if (my_lgrp == NULL)
8537c478bd9Sstevel@tonic-gate 		my_lgrp = lgrp_plat_alloc(lgrpid);
8547c478bd9Sstevel@tonic-gate 
8557c478bd9Sstevel@tonic-gate 	if (nlgrps > nlgrpsmax || my_lgrp == NULL)
8567c478bd9Sstevel@tonic-gate 		panic("Too many lgrps for platform (%d)", nlgrps);
8577c478bd9Sstevel@tonic-gate 
8587c478bd9Sstevel@tonic-gate 	my_lgrp->lgrp_id = lgrpid;
8597c478bd9Sstevel@tonic-gate 	my_lgrp->lgrp_latency = 0;
8607c478bd9Sstevel@tonic-gate 	my_lgrp->lgrp_plathand = LGRP_NULL_HANDLE;
8617c478bd9Sstevel@tonic-gate 	my_lgrp->lgrp_parent = NULL;
8627c478bd9Sstevel@tonic-gate 	my_lgrp->lgrp_childcnt = 0;
8637c478bd9Sstevel@tonic-gate 	my_lgrp->lgrp_mnodes = (mnodeset_t)0;
8647c478bd9Sstevel@tonic-gate 	my_lgrp->lgrp_nmnodes = 0;
8657c478bd9Sstevel@tonic-gate 	klgrpset_clear(my_lgrp->lgrp_children);
8667c478bd9Sstevel@tonic-gate 	klgrpset_clear(my_lgrp->lgrp_leaves);
8677c478bd9Sstevel@tonic-gate 	for (i = 0; i < LGRP_RSRC_COUNT; i++)
8687c478bd9Sstevel@tonic-gate 		klgrpset_clear(my_lgrp->lgrp_set[i]);
8697c478bd9Sstevel@tonic-gate 
8707c478bd9Sstevel@tonic-gate 	my_lgrp->lgrp_cpu = NULL;
8717c478bd9Sstevel@tonic-gate 	my_lgrp->lgrp_cpucnt = 0;
8727c478bd9Sstevel@tonic-gate 
8737c478bd9Sstevel@tonic-gate 	if (my_lgrp->lgrp_kstat != NULL)
8747c478bd9Sstevel@tonic-gate 		lgrp_kstat_reset(lgrpid);
8757c478bd9Sstevel@tonic-gate 
8767c478bd9Sstevel@tonic-gate 	lgrp_table[my_lgrp->lgrp_id] = my_lgrp;
8777c478bd9Sstevel@tonic-gate 
8787c478bd9Sstevel@tonic-gate 	return (my_lgrp);
8797c478bd9Sstevel@tonic-gate }
8807c478bd9Sstevel@tonic-gate 
8817c478bd9Sstevel@tonic-gate void
8827c478bd9Sstevel@tonic-gate lgrp_destroy(lgrp_t *lgrp)
8837c478bd9Sstevel@tonic-gate {
8847c478bd9Sstevel@tonic-gate 	int		i;
8857c478bd9Sstevel@tonic-gate 
8867c478bd9Sstevel@tonic-gate 	/*
8877c478bd9Sstevel@tonic-gate 	 * Unless this lgroup is being destroyed on behalf of
8887c478bd9Sstevel@tonic-gate 	 * the boot CPU, cpu_lock must be held
8897c478bd9Sstevel@tonic-gate 	 */
8907c478bd9Sstevel@tonic-gate 	ASSERT(!lgrp_initialized || MUTEX_HELD(&cpu_lock));
8917c478bd9Sstevel@tonic-gate 
8927c478bd9Sstevel@tonic-gate 	if (nlgrps == 1)
8937c478bd9Sstevel@tonic-gate 		cmn_err(CE_PANIC, "Can't destroy only lgroup!");
8947c478bd9Sstevel@tonic-gate 
8957c478bd9Sstevel@tonic-gate 	if (!LGRP_EXISTS(lgrp))
8967c478bd9Sstevel@tonic-gate 		return;
8977c478bd9Sstevel@tonic-gate 
8987c478bd9Sstevel@tonic-gate 	/*
8997c478bd9Sstevel@tonic-gate 	 * Set hint to lgroup being deleted and try to keep lower numbered
9007c478bd9Sstevel@tonic-gate 	 * hints to facilitate finding empty slots
9017c478bd9Sstevel@tonic-gate 	 */
9027c478bd9Sstevel@tonic-gate 	if (lgrp_alloc_hint == -1 || lgrp->lgrp_id < lgrp_alloc_hint)
9037c478bd9Sstevel@tonic-gate 		lgrp_alloc_hint = lgrp->lgrp_id;
9047c478bd9Sstevel@tonic-gate 
9057c478bd9Sstevel@tonic-gate 	/*
9067c478bd9Sstevel@tonic-gate 	 * Mark this lgroup to be recycled by setting its lgroup ID to
9077c478bd9Sstevel@tonic-gate 	 * LGRP_NONE and clear relevant fields
9087c478bd9Sstevel@tonic-gate 	 */
9097c478bd9Sstevel@tonic-gate 	lgrp->lgrp_id = LGRP_NONE;
9107c478bd9Sstevel@tonic-gate 	lgrp->lgrp_latency = 0;
9117c478bd9Sstevel@tonic-gate 	lgrp->lgrp_plathand = LGRP_NULL_HANDLE;
9127c478bd9Sstevel@tonic-gate 	lgrp->lgrp_parent = NULL;
9137c478bd9Sstevel@tonic-gate 	lgrp->lgrp_childcnt = 0;
9147c478bd9Sstevel@tonic-gate 
9157c478bd9Sstevel@tonic-gate 	klgrpset_clear(lgrp->lgrp_children);
9167c478bd9Sstevel@tonic-gate 	klgrpset_clear(lgrp->lgrp_leaves);
9177c478bd9Sstevel@tonic-gate 	for (i = 0; i < LGRP_RSRC_COUNT; i++)
9187c478bd9Sstevel@tonic-gate 		klgrpset_clear(lgrp->lgrp_set[i]);
9197c478bd9Sstevel@tonic-gate 
9207c478bd9Sstevel@tonic-gate 	lgrp->lgrp_mnodes = (mnodeset_t)0;
9217c478bd9Sstevel@tonic-gate 	lgrp->lgrp_nmnodes = 0;
9227c478bd9Sstevel@tonic-gate 
9237c478bd9Sstevel@tonic-gate 	lgrp->lgrp_cpu = NULL;
9247c478bd9Sstevel@tonic-gate 	lgrp->lgrp_cpucnt = 0;
9257c478bd9Sstevel@tonic-gate 
9267c478bd9Sstevel@tonic-gate 	nlgrps--;
9277c478bd9Sstevel@tonic-gate }
9287c478bd9Sstevel@tonic-gate 
9297c478bd9Sstevel@tonic-gate /*
9307c478bd9Sstevel@tonic-gate  * Initialize kstat data. Called from lgrp intialization code.
9317c478bd9Sstevel@tonic-gate  */
9327c478bd9Sstevel@tonic-gate static void
9337c478bd9Sstevel@tonic-gate lgrp_kstat_init(void)
9347c478bd9Sstevel@tonic-gate {
9357c478bd9Sstevel@tonic-gate 	lgrp_stat_t	stat;
9367c478bd9Sstevel@tonic-gate 
9377c478bd9Sstevel@tonic-gate 	mutex_init(&lgrp_kstat_mutex, NULL, MUTEX_DEFAULT, NULL);
9387c478bd9Sstevel@tonic-gate 
9397c478bd9Sstevel@tonic-gate 	for (stat = 0; stat < LGRP_NUM_STATS; stat++)
9407c478bd9Sstevel@tonic-gate 		kstat_named_init(&lgrp_kstat_data[stat],
9417c478bd9Sstevel@tonic-gate 		    lgrp_kstat_names[stat], KSTAT_DATA_INT64);
9427c478bd9Sstevel@tonic-gate }
9437c478bd9Sstevel@tonic-gate 
9447c478bd9Sstevel@tonic-gate /*
9457c478bd9Sstevel@tonic-gate  * initialize an lgrp's kstats if needed
9467c478bd9Sstevel@tonic-gate  * called with cpu_lock held but not with cpus paused.
9477c478bd9Sstevel@tonic-gate  * we don't tear these down now because we don't know about
9487c478bd9Sstevel@tonic-gate  * memory leaving the lgrp yet...
9497c478bd9Sstevel@tonic-gate  */
9507c478bd9Sstevel@tonic-gate 
9517c478bd9Sstevel@tonic-gate void
9527c478bd9Sstevel@tonic-gate lgrp_kstat_create(cpu_t *cp)
9537c478bd9Sstevel@tonic-gate {
9547c478bd9Sstevel@tonic-gate 	kstat_t		*lgrp_kstat;
9557c478bd9Sstevel@tonic-gate 	lgrp_id_t	lgrpid;
9567c478bd9Sstevel@tonic-gate 	lgrp_t		*my_lgrp;
9577c478bd9Sstevel@tonic-gate 
9587c478bd9Sstevel@tonic-gate 	ASSERT(MUTEX_HELD(&cpu_lock));
9597c478bd9Sstevel@tonic-gate 
9607c478bd9Sstevel@tonic-gate 	lgrpid = cp->cpu_lpl->lpl_lgrpid;
9617c478bd9Sstevel@tonic-gate 	my_lgrp = lgrp_table[lgrpid];
9627c478bd9Sstevel@tonic-gate 
9637c478bd9Sstevel@tonic-gate 	if (my_lgrp->lgrp_kstat != NULL)
9647c478bd9Sstevel@tonic-gate 		return; /* already initialized */
9657c478bd9Sstevel@tonic-gate 
9667c478bd9Sstevel@tonic-gate 	lgrp_kstat = kstat_create("lgrp", lgrpid, NULL, "misc",
9677c478bd9Sstevel@tonic-gate 	    KSTAT_TYPE_NAMED, LGRP_NUM_STATS,
9687c478bd9Sstevel@tonic-gate 	    KSTAT_FLAG_VIRTUAL | KSTAT_FLAG_WRITABLE);
9697c478bd9Sstevel@tonic-gate 
9707c478bd9Sstevel@tonic-gate 	if (lgrp_kstat != NULL) {
9717c478bd9Sstevel@tonic-gate 		lgrp_kstat->ks_lock = &lgrp_kstat_mutex;
9727c478bd9Sstevel@tonic-gate 		lgrp_kstat->ks_private = my_lgrp;
9737c478bd9Sstevel@tonic-gate 		lgrp_kstat->ks_data = &lgrp_kstat_data;
9747c478bd9Sstevel@tonic-gate 		lgrp_kstat->ks_update = lgrp_kstat_extract;
9757c478bd9Sstevel@tonic-gate 		my_lgrp->lgrp_kstat = lgrp_kstat;
9767c478bd9Sstevel@tonic-gate 		kstat_install(lgrp_kstat);
9777c478bd9Sstevel@tonic-gate 	}
9787c478bd9Sstevel@tonic-gate }
9797c478bd9Sstevel@tonic-gate 
9807c478bd9Sstevel@tonic-gate /*
9817c478bd9Sstevel@tonic-gate  * this will do something when we manage to remove now unused lgrps
9827c478bd9Sstevel@tonic-gate  */
9837c478bd9Sstevel@tonic-gate 
9847c478bd9Sstevel@tonic-gate /* ARGSUSED */
9857c478bd9Sstevel@tonic-gate void
9867c478bd9Sstevel@tonic-gate lgrp_kstat_destroy(cpu_t *cp)
9877c478bd9Sstevel@tonic-gate {
9887c478bd9Sstevel@tonic-gate 	ASSERT(MUTEX_HELD(&cpu_lock));
9897c478bd9Sstevel@tonic-gate }
9907c478bd9Sstevel@tonic-gate 
9917c478bd9Sstevel@tonic-gate /*
9927c478bd9Sstevel@tonic-gate  * Called when a CPU is off-lined.
9937c478bd9Sstevel@tonic-gate  */
9947c478bd9Sstevel@tonic-gate static void
9957c478bd9Sstevel@tonic-gate lgrp_cpu_fini(struct cpu *cp, lgrp_id_t lgrpid)
9967c478bd9Sstevel@tonic-gate {
9977c478bd9Sstevel@tonic-gate 	lgrp_t *my_lgrp;
9987c478bd9Sstevel@tonic-gate 	struct cpu *prev;
9997c478bd9Sstevel@tonic-gate 	struct cpu *next;
10007c478bd9Sstevel@tonic-gate 
10017c478bd9Sstevel@tonic-gate 	ASSERT(MUTEX_HELD(&cpu_lock) || !lgrp_initialized);
10027c478bd9Sstevel@tonic-gate 
10037c478bd9Sstevel@tonic-gate 	prev = cp->cpu_prev_lgrp;
10047c478bd9Sstevel@tonic-gate 	next = cp->cpu_next_lgrp;
10057c478bd9Sstevel@tonic-gate 
10067c478bd9Sstevel@tonic-gate 	prev->cpu_next_lgrp = next;
10077c478bd9Sstevel@tonic-gate 	next->cpu_prev_lgrp = prev;
10087c478bd9Sstevel@tonic-gate 
10097c478bd9Sstevel@tonic-gate 	/*
10107c478bd9Sstevel@tonic-gate 	 * just because I'm paranoid doesn't mean...
10117c478bd9Sstevel@tonic-gate 	 */
10127c478bd9Sstevel@tonic-gate 
10137c478bd9Sstevel@tonic-gate 	cp->cpu_next_lgrp = cp->cpu_prev_lgrp = NULL;
10147c478bd9Sstevel@tonic-gate 
10157c478bd9Sstevel@tonic-gate 	my_lgrp = lgrp_table[lgrpid];
10167c478bd9Sstevel@tonic-gate 	my_lgrp->lgrp_cpucnt--;
10177c478bd9Sstevel@tonic-gate 
10187c478bd9Sstevel@tonic-gate 	/*
10197c478bd9Sstevel@tonic-gate 	 * Removing last CPU in lgroup, so update lgroup topology
10207c478bd9Sstevel@tonic-gate 	 */
10217c478bd9Sstevel@tonic-gate 	if (my_lgrp->lgrp_cpucnt == 0) {
10227c478bd9Sstevel@tonic-gate 		klgrpset_t	changed;
10237c478bd9Sstevel@tonic-gate 		int		count;
10247c478bd9Sstevel@tonic-gate 		int		i;
10257c478bd9Sstevel@tonic-gate 
10267c478bd9Sstevel@tonic-gate 		my_lgrp->lgrp_cpu = NULL;
10277c478bd9Sstevel@tonic-gate 
10287c478bd9Sstevel@tonic-gate 		/*
10297c478bd9Sstevel@tonic-gate 		 * Remove this lgroup from its lgroup CPU resources and remove
10307c478bd9Sstevel@tonic-gate 		 * lgroup from lgroup topology if it doesn't have any more
10317c478bd9Sstevel@tonic-gate 		 * resources in it now
10327c478bd9Sstevel@tonic-gate 		 */
10337c478bd9Sstevel@tonic-gate 		klgrpset_del(my_lgrp->lgrp_set[LGRP_RSRC_CPU], lgrpid);
10347c478bd9Sstevel@tonic-gate 		if (lgrp_rsets_empty(my_lgrp->lgrp_set)) {
10357c478bd9Sstevel@tonic-gate 			count = 0;
10367c478bd9Sstevel@tonic-gate 			klgrpset_clear(changed);
10377c478bd9Sstevel@tonic-gate 			count += lgrp_leaf_delete(my_lgrp, lgrp_table,
10387c478bd9Sstevel@tonic-gate 			    lgrp_alloc_max + 1, &changed);
10397c478bd9Sstevel@tonic-gate 			return;
10407c478bd9Sstevel@tonic-gate 		}
10417c478bd9Sstevel@tonic-gate 
10427c478bd9Sstevel@tonic-gate 		/*
10437c478bd9Sstevel@tonic-gate 		 * This lgroup isn't empty, so just remove it from CPU
10447c478bd9Sstevel@tonic-gate 		 * resources of any lgroups that contain it as such
10457c478bd9Sstevel@tonic-gate 		 */
10467c478bd9Sstevel@tonic-gate 		for (i = 0; i <= lgrp_alloc_max; i++) {
10477c478bd9Sstevel@tonic-gate 			lgrp_t		*lgrp;
10487c478bd9Sstevel@tonic-gate 
10497c478bd9Sstevel@tonic-gate 			lgrp = lgrp_table[i];
10507c478bd9Sstevel@tonic-gate 			if (!LGRP_EXISTS(lgrp) ||
10517c478bd9Sstevel@tonic-gate 			    !klgrpset_ismember(lgrp->lgrp_set[LGRP_RSRC_CPU],
10527c478bd9Sstevel@tonic-gate 			    lgrpid))
10537c478bd9Sstevel@tonic-gate 				continue;
10547c478bd9Sstevel@tonic-gate 
10557c478bd9Sstevel@tonic-gate 			klgrpset_del(lgrp->lgrp_set[LGRP_RSRC_CPU], lgrpid);
10567c478bd9Sstevel@tonic-gate 		}
10577c478bd9Sstevel@tonic-gate 		return;
10587c478bd9Sstevel@tonic-gate 	}
10597c478bd9Sstevel@tonic-gate 
10607c478bd9Sstevel@tonic-gate 	if (my_lgrp->lgrp_cpu == cp)
10617c478bd9Sstevel@tonic-gate 		my_lgrp->lgrp_cpu = next;
10627c478bd9Sstevel@tonic-gate 
10637c478bd9Sstevel@tonic-gate }
10647c478bd9Sstevel@tonic-gate 
10657c478bd9Sstevel@tonic-gate /*
10667c478bd9Sstevel@tonic-gate  * Update memory nodes in target lgroups and return ones that get changed
10677c478bd9Sstevel@tonic-gate  */
10687c478bd9Sstevel@tonic-gate int
10697c478bd9Sstevel@tonic-gate lgrp_mnode_update(klgrpset_t target, klgrpset_t *changed)
10707c478bd9Sstevel@tonic-gate {
10717c478bd9Sstevel@tonic-gate 	int	count;
10727c478bd9Sstevel@tonic-gate 	int	i;
10737c478bd9Sstevel@tonic-gate 	int	j;
10747c478bd9Sstevel@tonic-gate 	lgrp_t	*lgrp;
10757c478bd9Sstevel@tonic-gate 	lgrp_t	*lgrp_rsrc;
10767c478bd9Sstevel@tonic-gate 
10777c478bd9Sstevel@tonic-gate 	count = 0;
10787c478bd9Sstevel@tonic-gate 	if (changed)
10797c478bd9Sstevel@tonic-gate 		klgrpset_clear(*changed);
10807c478bd9Sstevel@tonic-gate 
10817c478bd9Sstevel@tonic-gate 	if (klgrpset_isempty(target))
10827c478bd9Sstevel@tonic-gate 		return (0);
10837c478bd9Sstevel@tonic-gate 
10847c478bd9Sstevel@tonic-gate 	/*
10857c478bd9Sstevel@tonic-gate 	 * Find each lgroup in target lgroups
10867c478bd9Sstevel@tonic-gate 	 */
10877c478bd9Sstevel@tonic-gate 	for (i = 0; i <= lgrp_alloc_max; i++) {
10887c478bd9Sstevel@tonic-gate 		/*
10897c478bd9Sstevel@tonic-gate 		 * Skip any lgroups that don't exist or aren't in target group
10907c478bd9Sstevel@tonic-gate 		 */
10917c478bd9Sstevel@tonic-gate 		lgrp = lgrp_table[i];
10927c478bd9Sstevel@tonic-gate 		if (!klgrpset_ismember(target, i) || !LGRP_EXISTS(lgrp)) {
10937c478bd9Sstevel@tonic-gate 			continue;
10947c478bd9Sstevel@tonic-gate 		}
10957c478bd9Sstevel@tonic-gate 
10967c478bd9Sstevel@tonic-gate 		/*
10977c478bd9Sstevel@tonic-gate 		 * Initialize memnodes for intermediate lgroups to 0
10987c478bd9Sstevel@tonic-gate 		 * and update them from scratch since they may have completely
10997c478bd9Sstevel@tonic-gate 		 * changed
11007c478bd9Sstevel@tonic-gate 		 */
11017c478bd9Sstevel@tonic-gate 		if (lgrp->lgrp_childcnt && lgrp != lgrp_root) {
11027c478bd9Sstevel@tonic-gate 			lgrp->lgrp_mnodes = (mnodeset_t)0;
11037c478bd9Sstevel@tonic-gate 			lgrp->lgrp_nmnodes = 0;
11047c478bd9Sstevel@tonic-gate 		}
11057c478bd9Sstevel@tonic-gate 
11067c478bd9Sstevel@tonic-gate 		/*
11077c478bd9Sstevel@tonic-gate 		 * Update memory nodes of of target lgroup with memory nodes
11087c478bd9Sstevel@tonic-gate 		 * from each lgroup in its lgroup memory resource set
11097c478bd9Sstevel@tonic-gate 		 */
11107c478bd9Sstevel@tonic-gate 		for (j = 0; j <= lgrp_alloc_max; j++) {
11117c478bd9Sstevel@tonic-gate 			int	k;
11127c478bd9Sstevel@tonic-gate 
11137c478bd9Sstevel@tonic-gate 			/*
11147c478bd9Sstevel@tonic-gate 			 * Skip any lgroups that don't exist or aren't in
11157c478bd9Sstevel@tonic-gate 			 * memory resources of target lgroup
11167c478bd9Sstevel@tonic-gate 			 */
11177c478bd9Sstevel@tonic-gate 			lgrp_rsrc = lgrp_table[j];
11187c478bd9Sstevel@tonic-gate 			if (!LGRP_EXISTS(lgrp_rsrc) ||
11197c478bd9Sstevel@tonic-gate 			    !klgrpset_ismember(lgrp->lgrp_set[LGRP_RSRC_MEM],
11207c478bd9Sstevel@tonic-gate 			    j))
11217c478bd9Sstevel@tonic-gate 				continue;
11227c478bd9Sstevel@tonic-gate 
11237c478bd9Sstevel@tonic-gate 			/*
11247c478bd9Sstevel@tonic-gate 			 * Update target lgroup's memnodes to include memnodes
11257c478bd9Sstevel@tonic-gate 			 * of this lgroup
11267c478bd9Sstevel@tonic-gate 			 */
11277c478bd9Sstevel@tonic-gate 			for (k = 0; k < sizeof (mnodeset_t) * NBBY; k++) {
11287c478bd9Sstevel@tonic-gate 				mnodeset_t	mnode_mask;
11297c478bd9Sstevel@tonic-gate 
11307c478bd9Sstevel@tonic-gate 				mnode_mask = (mnodeset_t)1 << k;
11317c478bd9Sstevel@tonic-gate 				if ((lgrp_rsrc->lgrp_mnodes & mnode_mask) &&
11327c478bd9Sstevel@tonic-gate 				    !(lgrp->lgrp_mnodes & mnode_mask)) {
11337c478bd9Sstevel@tonic-gate 					lgrp->lgrp_mnodes |= mnode_mask;
11347c478bd9Sstevel@tonic-gate 					lgrp->lgrp_nmnodes++;
11357c478bd9Sstevel@tonic-gate 				}
11367c478bd9Sstevel@tonic-gate 			}
11377c478bd9Sstevel@tonic-gate 			count++;
11387c478bd9Sstevel@tonic-gate 			if (changed)
11397c478bd9Sstevel@tonic-gate 				klgrpset_add(*changed, lgrp->lgrp_id);
11407c478bd9Sstevel@tonic-gate 		}
11417c478bd9Sstevel@tonic-gate 	}
11427c478bd9Sstevel@tonic-gate 
11437c478bd9Sstevel@tonic-gate 	return (count);
11447c478bd9Sstevel@tonic-gate }
11457c478bd9Sstevel@tonic-gate 
11467c478bd9Sstevel@tonic-gate /*
11477c478bd9Sstevel@tonic-gate  * Memory copy-rename. Called when the "mnode" containing the kernel cage memory
11487c478bd9Sstevel@tonic-gate  * is moved from one board to another. The "from" and "to" arguments specify the
11497c478bd9Sstevel@tonic-gate  * source and the destination of the move.
11507c478bd9Sstevel@tonic-gate  *
11517c478bd9Sstevel@tonic-gate  * See plat_lgrp_config() for a detailed description of the copy-rename
11527c478bd9Sstevel@tonic-gate  * semantics.
11537c478bd9Sstevel@tonic-gate  *
11547c478bd9Sstevel@tonic-gate  * The lgrp_mem_rename() is called by the platform copy-rename code to update
11557c478bd9Sstevel@tonic-gate  * the lgroup topology which is changing as memory moves from one lgroup to
11567c478bd9Sstevel@tonic-gate  * another. It removes the mnode from the source lgroup and re-inserts it in the
11577c478bd9Sstevel@tonic-gate  * target lgroup.
11587c478bd9Sstevel@tonic-gate  *
11597c478bd9Sstevel@tonic-gate  * The lgrp_mem_rename() function passes a flag to lgrp_mem_init() and
11607c478bd9Sstevel@tonic-gate  * lgrp_mem_fini() telling that the insertion and deleteion are part of a DR
11617c478bd9Sstevel@tonic-gate  * copy-rename operation.
11627c478bd9Sstevel@tonic-gate  *
11637c478bd9Sstevel@tonic-gate  * There is one case which requires special handling. If the system contains
11647c478bd9Sstevel@tonic-gate  * only two boards (mnodes), the lgrp_mem_fini() removes the only mnode from the
11657c478bd9Sstevel@tonic-gate  * lgroup hierarchy. This mnode is soon re-inserted back in the hierarchy by
11667c478bd9Sstevel@tonic-gate  * lgrp_mem_init), but there is a window when the system has no memory in the
11677c478bd9Sstevel@tonic-gate  * lgroup hierarchy. If another thread tries to allocate memory during this
11687c478bd9Sstevel@tonic-gate  * window, the allocation will fail, although the system has physical memory.
11697c478bd9Sstevel@tonic-gate  * This may cause a system panic or a deadlock (some sleeping memory allocations
11707c478bd9Sstevel@tonic-gate  * happen with cpu_lock held which prevents lgrp_mem_init() from re-inserting
11717c478bd9Sstevel@tonic-gate  * the mnode back).
11727c478bd9Sstevel@tonic-gate  *
11737c478bd9Sstevel@tonic-gate  * The lgrp_memnode_choose() function walks the lgroup hierarchy looking for the
11747c478bd9Sstevel@tonic-gate  * lgrp with non-empty lgrp_mnodes. To deal with the special case above,
11757c478bd9Sstevel@tonic-gate  * lgrp_mem_fini() does not remove the last mnode from the lroot->lgrp_mnodes,
11767c478bd9Sstevel@tonic-gate  * but it updates the rest of the lgroup topology as if the mnode was actually
11777c478bd9Sstevel@tonic-gate  * removed. The lgrp_mem_init() function recognizes that the mnode being
11787c478bd9Sstevel@tonic-gate  * inserted represents such a special case and updates the topology
11797c478bd9Sstevel@tonic-gate  * appropriately.
11807c478bd9Sstevel@tonic-gate  */
11817c478bd9Sstevel@tonic-gate void
11827c478bd9Sstevel@tonic-gate lgrp_mem_rename(int mnode, lgrp_handle_t from, lgrp_handle_t to)
11837c478bd9Sstevel@tonic-gate {
11847c478bd9Sstevel@tonic-gate 	/*
11857c478bd9Sstevel@tonic-gate 	 * Remove the memory from the source node and add it to the destination
11867c478bd9Sstevel@tonic-gate 	 * node.
11877c478bd9Sstevel@tonic-gate 	 */
11887c478bd9Sstevel@tonic-gate 	lgrp_mem_fini(mnode, from, B_TRUE);
11897c478bd9Sstevel@tonic-gate 	lgrp_mem_init(mnode, to, B_TRUE);
11907c478bd9Sstevel@tonic-gate }
11917c478bd9Sstevel@tonic-gate 
11927c478bd9Sstevel@tonic-gate /*
11937c478bd9Sstevel@tonic-gate  * Called to indicate that the lgrp with platform handle "hand" now
11947c478bd9Sstevel@tonic-gate  * contains the memory identified by "mnode".
11957c478bd9Sstevel@tonic-gate  *
11967c478bd9Sstevel@tonic-gate  * LOCKING for this routine is a bit tricky. Usually it is called without
11977c478bd9Sstevel@tonic-gate  * cpu_lock and it must must grab cpu_lock here to prevent racing with other
11987c478bd9Sstevel@tonic-gate  * callers. During DR of the board containing the caged memory it may be called
11997c478bd9Sstevel@tonic-gate  * with cpu_lock already held and CPUs paused.
12007c478bd9Sstevel@tonic-gate  *
12017c478bd9Sstevel@tonic-gate  * If the insertion is part of the DR copy-rename and the inserted mnode (and
12027c478bd9Sstevel@tonic-gate  * only this mnode) is already present in the lgrp_root->lgrp_mnodes set, we are
12037c478bd9Sstevel@tonic-gate  * dealing with the special case of DR copy-rename described in
12047c478bd9Sstevel@tonic-gate  * lgrp_mem_rename().
12057c478bd9Sstevel@tonic-gate  */
12067c478bd9Sstevel@tonic-gate void
12077c478bd9Sstevel@tonic-gate lgrp_mem_init(int mnode, lgrp_handle_t hand, boolean_t is_copy_rename)
12087c478bd9Sstevel@tonic-gate {
12097c478bd9Sstevel@tonic-gate 	klgrpset_t	changed;
12107c478bd9Sstevel@tonic-gate 	int		count;
12117c478bd9Sstevel@tonic-gate 	int		i;
12127c478bd9Sstevel@tonic-gate 	lgrp_t		*my_lgrp;
12137c478bd9Sstevel@tonic-gate 	lgrp_id_t	lgrpid;
12147c478bd9Sstevel@tonic-gate 	mnodeset_t	mnodes_mask = ((mnodeset_t)1 << mnode);
12157c478bd9Sstevel@tonic-gate 	boolean_t	drop_lock = B_FALSE;
12167c478bd9Sstevel@tonic-gate 	boolean_t	need_synch = B_FALSE;
12177c478bd9Sstevel@tonic-gate 
12187c478bd9Sstevel@tonic-gate 	/*
12197c478bd9Sstevel@tonic-gate 	 * Grab CPU lock (if we haven't already)
12207c478bd9Sstevel@tonic-gate 	 */
12217c478bd9Sstevel@tonic-gate 	if (!MUTEX_HELD(&cpu_lock)) {
12227c478bd9Sstevel@tonic-gate 		mutex_enter(&cpu_lock);
12237c478bd9Sstevel@tonic-gate 		drop_lock = B_TRUE;
12247c478bd9Sstevel@tonic-gate 	}
12257c478bd9Sstevel@tonic-gate 
12267c478bd9Sstevel@tonic-gate 	/*
12277c478bd9Sstevel@tonic-gate 	 * This routine may be called from a context where we already
12287c478bd9Sstevel@tonic-gate 	 * hold cpu_lock, and have already paused cpus.
12297c478bd9Sstevel@tonic-gate 	 */
12307c478bd9Sstevel@tonic-gate 	if (!cpus_paused())
12317c478bd9Sstevel@tonic-gate 		need_synch = B_TRUE;
12327c478bd9Sstevel@tonic-gate 
12337c478bd9Sstevel@tonic-gate 	/*
12347c478bd9Sstevel@tonic-gate 	 * Check if this mnode is already configured and return immediately if
12357c478bd9Sstevel@tonic-gate 	 * it is.
12367c478bd9Sstevel@tonic-gate 	 *
12377c478bd9Sstevel@tonic-gate 	 * NOTE: in special case of copy-rename of the only remaining mnode,
12387c478bd9Sstevel@tonic-gate 	 * lgrp_mem_fini() refuses to remove the last mnode from the root, so we
12397c478bd9Sstevel@tonic-gate 	 * recognize this case and continue as usual, but skip the update to
12407c478bd9Sstevel@tonic-gate 	 * the lgrp_mnodes and the lgrp_nmnodes. This restores the inconsistency
12417c478bd9Sstevel@tonic-gate 	 * in topology, temporarily introduced by lgrp_mem_fini().
12427c478bd9Sstevel@tonic-gate 	 */
12437c478bd9Sstevel@tonic-gate 	if (! (is_copy_rename && (lgrp_root->lgrp_mnodes == mnodes_mask)) &&
12447c478bd9Sstevel@tonic-gate 	    lgrp_root->lgrp_mnodes & mnodes_mask) {
12457c478bd9Sstevel@tonic-gate 		if (drop_lock)
12467c478bd9Sstevel@tonic-gate 			mutex_exit(&cpu_lock);
12477c478bd9Sstevel@tonic-gate 		return;
12487c478bd9Sstevel@tonic-gate 	}
12497c478bd9Sstevel@tonic-gate 
12507c478bd9Sstevel@tonic-gate 	/*
12517c478bd9Sstevel@tonic-gate 	 * Update lgroup topology with new memory resources, keeping track of
12527c478bd9Sstevel@tonic-gate 	 * which lgroups change
12537c478bd9Sstevel@tonic-gate 	 */
12547c478bd9Sstevel@tonic-gate 	count = 0;
12557c478bd9Sstevel@tonic-gate 	klgrpset_clear(changed);
12567c478bd9Sstevel@tonic-gate 	my_lgrp = lgrp_hand_to_lgrp(hand);
12577c478bd9Sstevel@tonic-gate 	if (my_lgrp == NULL) {
12587c478bd9Sstevel@tonic-gate 		/* new lgrp */
12597c478bd9Sstevel@tonic-gate 		my_lgrp = lgrp_create();
12607c478bd9Sstevel@tonic-gate 		lgrpid = my_lgrp->lgrp_id;
12617c478bd9Sstevel@tonic-gate 		my_lgrp->lgrp_plathand = hand;
12627c478bd9Sstevel@tonic-gate 		my_lgrp->lgrp_latency = lgrp_plat_latency(hand, hand);
12637c478bd9Sstevel@tonic-gate 		klgrpset_add(my_lgrp->lgrp_leaves, lgrpid);
12647c478bd9Sstevel@tonic-gate 		klgrpset_add(my_lgrp->lgrp_set[LGRP_RSRC_MEM], lgrpid);
12657c478bd9Sstevel@tonic-gate 
12667c478bd9Sstevel@tonic-gate 		if (need_synch)
12677c478bd9Sstevel@tonic-gate 			pause_cpus(NULL);
12687c478bd9Sstevel@tonic-gate 		count = lgrp_leaf_add(my_lgrp, lgrp_table, lgrp_alloc_max + 1,
12697c478bd9Sstevel@tonic-gate 		    &changed);
12707c478bd9Sstevel@tonic-gate 		if (need_synch)
12717c478bd9Sstevel@tonic-gate 			start_cpus();
12727c478bd9Sstevel@tonic-gate 	} else if (my_lgrp->lgrp_latency == 0 && lgrp_plat_latency(hand, hand)
12737c478bd9Sstevel@tonic-gate 	    > 0) {
12747c478bd9Sstevel@tonic-gate 		/*
12757c478bd9Sstevel@tonic-gate 		 * Leaf lgroup was created, but latency wasn't available
12767c478bd9Sstevel@tonic-gate 		 * then.  So, set latency for it and fill in rest of lgroup
12777c478bd9Sstevel@tonic-gate 		 * topology  now that we know how far it is from other leaf
12787c478bd9Sstevel@tonic-gate 		 * lgroups.
12797c478bd9Sstevel@tonic-gate 		 */
12807c478bd9Sstevel@tonic-gate 		klgrpset_clear(changed);
12817c478bd9Sstevel@tonic-gate 		lgrpid = my_lgrp->lgrp_id;
12827c478bd9Sstevel@tonic-gate 		if (!klgrpset_ismember(my_lgrp->lgrp_set[LGRP_RSRC_MEM],
12837c478bd9Sstevel@tonic-gate 		    lgrpid))
12847c478bd9Sstevel@tonic-gate 			klgrpset_add(my_lgrp->lgrp_set[LGRP_RSRC_MEM], lgrpid);
12857c478bd9Sstevel@tonic-gate 		if (need_synch)
12867c478bd9Sstevel@tonic-gate 			pause_cpus(NULL);
12877c478bd9Sstevel@tonic-gate 		count = lgrp_leaf_add(my_lgrp, lgrp_table, lgrp_alloc_max + 1,
12887c478bd9Sstevel@tonic-gate 		    &changed);
12897c478bd9Sstevel@tonic-gate 		if (need_synch)
12907c478bd9Sstevel@tonic-gate 			start_cpus();
12917c478bd9Sstevel@tonic-gate 	} else if (!klgrpset_ismember(my_lgrp->lgrp_set[LGRP_RSRC_MEM],
12927c478bd9Sstevel@tonic-gate 	    my_lgrp->lgrp_id)) {
12932dae3fb5Sjjc 		/*
12942dae3fb5Sjjc 		 * Add new lgroup memory resource to existing lgroup
12952dae3fb5Sjjc 		 */
12967c478bd9Sstevel@tonic-gate 		lgrpid = my_lgrp->lgrp_id;
12977c478bd9Sstevel@tonic-gate 		klgrpset_add(my_lgrp->lgrp_set[LGRP_RSRC_MEM], lgrpid);
12987c478bd9Sstevel@tonic-gate 		klgrpset_add(changed, lgrpid);
12997c478bd9Sstevel@tonic-gate 		count++;
13007c478bd9Sstevel@tonic-gate 		for (i = 0; i <= lgrp_alloc_max; i++) {
13017c478bd9Sstevel@tonic-gate 			lgrp_t		*lgrp;
13027c478bd9Sstevel@tonic-gate 
13037c478bd9Sstevel@tonic-gate 			lgrp = lgrp_table[i];
13047c478bd9Sstevel@tonic-gate 			if (!LGRP_EXISTS(lgrp) ||
13057c478bd9Sstevel@tonic-gate 			    !lgrp_rsets_member(lgrp->lgrp_set, lgrpid))
13067c478bd9Sstevel@tonic-gate 				continue;
13077c478bd9Sstevel@tonic-gate 
13087c478bd9Sstevel@tonic-gate 			klgrpset_add(lgrp->lgrp_set[LGRP_RSRC_MEM], lgrpid);
13097c478bd9Sstevel@tonic-gate 			klgrpset_add(changed, lgrp->lgrp_id);
13107c478bd9Sstevel@tonic-gate 			count++;
13117c478bd9Sstevel@tonic-gate 		}
13127c478bd9Sstevel@tonic-gate 	}
13137c478bd9Sstevel@tonic-gate 
13147c478bd9Sstevel@tonic-gate 	/*
13157c478bd9Sstevel@tonic-gate 	 * Add memory node to lgroup and remove lgroup from ones that need
13167c478bd9Sstevel@tonic-gate 	 * to be updated
13177c478bd9Sstevel@tonic-gate 	 */
13187c478bd9Sstevel@tonic-gate 	if (!(my_lgrp->lgrp_mnodes & mnodes_mask)) {
13197c478bd9Sstevel@tonic-gate 		my_lgrp->lgrp_mnodes |= mnodes_mask;
13207c478bd9Sstevel@tonic-gate 		my_lgrp->lgrp_nmnodes++;
13217c478bd9Sstevel@tonic-gate 	}
13227c478bd9Sstevel@tonic-gate 	klgrpset_del(changed, lgrpid);
13237c478bd9Sstevel@tonic-gate 
13247c478bd9Sstevel@tonic-gate 	/*
13257c478bd9Sstevel@tonic-gate 	 * Update memory node information for all lgroups that changed and
13267c478bd9Sstevel@tonic-gate 	 * contain new memory node as a resource
13277c478bd9Sstevel@tonic-gate 	 */
13287c478bd9Sstevel@tonic-gate 	if (count)
13297c478bd9Sstevel@tonic-gate 		(void) lgrp_mnode_update(changed, NULL);
13307c478bd9Sstevel@tonic-gate 
13317c478bd9Sstevel@tonic-gate 	if (drop_lock)
13327c478bd9Sstevel@tonic-gate 		mutex_exit(&cpu_lock);
13337c478bd9Sstevel@tonic-gate }
13347c478bd9Sstevel@tonic-gate 
13357c478bd9Sstevel@tonic-gate /*
13367c478bd9Sstevel@tonic-gate  * Called to indicate that the lgroup associated with the platform
13377c478bd9Sstevel@tonic-gate  * handle "hand" no longer contains given memory node
13387c478bd9Sstevel@tonic-gate  *
13397c478bd9Sstevel@tonic-gate  * LOCKING for this routine is a bit tricky. Usually it is called without
13407c478bd9Sstevel@tonic-gate  * cpu_lock and it must must grab cpu_lock here to prevent racing with other
13417c478bd9Sstevel@tonic-gate  * callers. During DR of the board containing the caged memory it may be called
13427c478bd9Sstevel@tonic-gate  * with cpu_lock already held and CPUs paused.
13437c478bd9Sstevel@tonic-gate  *
13447c478bd9Sstevel@tonic-gate  * If the deletion is part of the DR copy-rename and the deleted mnode is the
13457c478bd9Sstevel@tonic-gate  * only one present in the lgrp_root->lgrp_mnodes, all the topology is updated,
13467c478bd9Sstevel@tonic-gate  * but lgrp_root->lgrp_mnodes is left intact. Later, lgrp_mem_init() will insert
13477c478bd9Sstevel@tonic-gate  * the same mnode back into the topology. See lgrp_mem_rename() and
13487c478bd9Sstevel@tonic-gate  * lgrp_mem_init() for additional details.
13497c478bd9Sstevel@tonic-gate  */
13507c478bd9Sstevel@tonic-gate void
13517c478bd9Sstevel@tonic-gate lgrp_mem_fini(int mnode, lgrp_handle_t hand, boolean_t is_copy_rename)
13527c478bd9Sstevel@tonic-gate {
13537c478bd9Sstevel@tonic-gate 	klgrpset_t	changed;
13547c478bd9Sstevel@tonic-gate 	int		count;
13557c478bd9Sstevel@tonic-gate 	int		i;
13567c478bd9Sstevel@tonic-gate 	lgrp_t		*my_lgrp;
13577c478bd9Sstevel@tonic-gate 	lgrp_id_t	lgrpid;
13587c478bd9Sstevel@tonic-gate 	mnodeset_t	mnodes_mask;
13597c478bd9Sstevel@tonic-gate 	boolean_t	drop_lock = B_FALSE;
13607c478bd9Sstevel@tonic-gate 	boolean_t	need_synch = B_FALSE;
13617c478bd9Sstevel@tonic-gate 
13627c478bd9Sstevel@tonic-gate 	/*
13637c478bd9Sstevel@tonic-gate 	 * Grab CPU lock (if we haven't already)
13647c478bd9Sstevel@tonic-gate 	 */
13657c478bd9Sstevel@tonic-gate 	if (!MUTEX_HELD(&cpu_lock)) {
13667c478bd9Sstevel@tonic-gate 		mutex_enter(&cpu_lock);
13677c478bd9Sstevel@tonic-gate 		drop_lock = B_TRUE;
13687c478bd9Sstevel@tonic-gate 	}
13697c478bd9Sstevel@tonic-gate 
13707c478bd9Sstevel@tonic-gate 	/*
13717c478bd9Sstevel@tonic-gate 	 * This routine may be called from a context where we already
13727c478bd9Sstevel@tonic-gate 	 * hold cpu_lock and have already paused cpus.
13737c478bd9Sstevel@tonic-gate 	 */
13747c478bd9Sstevel@tonic-gate 	if (!cpus_paused())
13757c478bd9Sstevel@tonic-gate 		need_synch = B_TRUE;
13767c478bd9Sstevel@tonic-gate 
13777c478bd9Sstevel@tonic-gate 	my_lgrp = lgrp_hand_to_lgrp(hand);
13787c478bd9Sstevel@tonic-gate 
13797c478bd9Sstevel@tonic-gate 	/*
13807c478bd9Sstevel@tonic-gate 	 * The lgrp *must* be pre-existing
13817c478bd9Sstevel@tonic-gate 	 */
13827c478bd9Sstevel@tonic-gate 	ASSERT(my_lgrp != NULL);
13837c478bd9Sstevel@tonic-gate 
13847c478bd9Sstevel@tonic-gate 	/*
13857c478bd9Sstevel@tonic-gate 	 * Delete memory node from lgroups which contain it
13867c478bd9Sstevel@tonic-gate 	 */
13877c478bd9Sstevel@tonic-gate 	mnodes_mask = ((mnodeset_t)1 << mnode);
13887c478bd9Sstevel@tonic-gate 	for (i = 0; i <= lgrp_alloc_max; i++) {
13897c478bd9Sstevel@tonic-gate 		lgrp_t *lgrp = lgrp_table[i];
13907c478bd9Sstevel@tonic-gate 		/*
13917c478bd9Sstevel@tonic-gate 		 * Skip any non-existent lgroups and any lgroups that don't
13927c478bd9Sstevel@tonic-gate 		 * contain leaf lgroup of memory as a memory resource
13937c478bd9Sstevel@tonic-gate 		 */
13947c478bd9Sstevel@tonic-gate 		if (!LGRP_EXISTS(lgrp) ||
13957c478bd9Sstevel@tonic-gate 		    !(lgrp->lgrp_mnodes & mnodes_mask))
13967c478bd9Sstevel@tonic-gate 			continue;
13977c478bd9Sstevel@tonic-gate 
13987c478bd9Sstevel@tonic-gate 		/*
13997c478bd9Sstevel@tonic-gate 		 * Avoid removing the last mnode from the root in the DR
14007c478bd9Sstevel@tonic-gate 		 * copy-rename case. See lgrp_mem_rename() for details.
14017c478bd9Sstevel@tonic-gate 		 */
14027c478bd9Sstevel@tonic-gate 		if (is_copy_rename &&
14037c478bd9Sstevel@tonic-gate 		    (lgrp == lgrp_root) && (lgrp->lgrp_mnodes == mnodes_mask))
14047c478bd9Sstevel@tonic-gate 			continue;
14057c478bd9Sstevel@tonic-gate 
14067c478bd9Sstevel@tonic-gate 		/*
14077c478bd9Sstevel@tonic-gate 		 * Remove memory node from lgroup.
14087c478bd9Sstevel@tonic-gate 		 */
14097c478bd9Sstevel@tonic-gate 		lgrp->lgrp_mnodes &= ~mnodes_mask;
14107c478bd9Sstevel@tonic-gate 		lgrp->lgrp_nmnodes--;
14117c478bd9Sstevel@tonic-gate 		ASSERT(lgrp->lgrp_nmnodes >= 0);
14127c478bd9Sstevel@tonic-gate 	}
14137c478bd9Sstevel@tonic-gate 	ASSERT(lgrp_root->lgrp_nmnodes > 0);
14147c478bd9Sstevel@tonic-gate 
14157c478bd9Sstevel@tonic-gate 	/*
14167c478bd9Sstevel@tonic-gate 	 * Don't need to update lgroup topology if this lgroup still has memory.
14177c478bd9Sstevel@tonic-gate 	 *
14187c478bd9Sstevel@tonic-gate 	 * In the special case of DR copy-rename with the only mnode being
14197c478bd9Sstevel@tonic-gate 	 * removed, the lgrp_mnodes for the root is always non-zero, but we
14207c478bd9Sstevel@tonic-gate 	 * still need to update the lgroup topology.
14217c478bd9Sstevel@tonic-gate 	 */
14227c478bd9Sstevel@tonic-gate 	if ((my_lgrp->lgrp_nmnodes > 0) &&
14237c478bd9Sstevel@tonic-gate 	    !(is_copy_rename &&
14247c478bd9Sstevel@tonic-gate 		(my_lgrp == lgrp_root) &&
14257c478bd9Sstevel@tonic-gate 		(my_lgrp->lgrp_mnodes == mnodes_mask))) {
14267c478bd9Sstevel@tonic-gate 		if (drop_lock)
14277c478bd9Sstevel@tonic-gate 			mutex_exit(&cpu_lock);
14287c478bd9Sstevel@tonic-gate 		return;
14297c478bd9Sstevel@tonic-gate 	}
14307c478bd9Sstevel@tonic-gate 
14317c478bd9Sstevel@tonic-gate 	/*
14327c478bd9Sstevel@tonic-gate 	 * This lgroup does not contain any memory now
14337c478bd9Sstevel@tonic-gate 	 */
14347c478bd9Sstevel@tonic-gate 	klgrpset_clear(my_lgrp->lgrp_set[LGRP_RSRC_MEM]);
14357c478bd9Sstevel@tonic-gate 
14367c478bd9Sstevel@tonic-gate 	/*
14377c478bd9Sstevel@tonic-gate 	 * Remove this lgroup from lgroup topology if it does not contain any
14387c478bd9Sstevel@tonic-gate 	 * resources now
14397c478bd9Sstevel@tonic-gate 	 */
14407c478bd9Sstevel@tonic-gate 	lgrpid = my_lgrp->lgrp_id;
14417c478bd9Sstevel@tonic-gate 	count = 0;
14427c478bd9Sstevel@tonic-gate 	klgrpset_clear(changed);
14437c478bd9Sstevel@tonic-gate 	if (lgrp_rsets_empty(my_lgrp->lgrp_set)) {
14447c478bd9Sstevel@tonic-gate 		/*
14457c478bd9Sstevel@tonic-gate 		 * Delete lgroup when no more resources
14467c478bd9Sstevel@tonic-gate 		 */
14477c478bd9Sstevel@tonic-gate 		if (need_synch)
14487c478bd9Sstevel@tonic-gate 			pause_cpus(NULL);
14497c478bd9Sstevel@tonic-gate 		count = lgrp_leaf_delete(my_lgrp, lgrp_table,
14507c478bd9Sstevel@tonic-gate 		    lgrp_alloc_max + 1, &changed);
14517c478bd9Sstevel@tonic-gate 		ASSERT(count > 0);
14527c478bd9Sstevel@tonic-gate 		if (need_synch)
14537c478bd9Sstevel@tonic-gate 			start_cpus();
14547c478bd9Sstevel@tonic-gate 	} else {
14557c478bd9Sstevel@tonic-gate 		/*
14567c478bd9Sstevel@tonic-gate 		 * Remove lgroup from memory resources of any lgroups that
14577c478bd9Sstevel@tonic-gate 		 * contain it as such
14587c478bd9Sstevel@tonic-gate 		 */
14597c478bd9Sstevel@tonic-gate 		for (i = 0; i <= lgrp_alloc_max; i++) {
14607c478bd9Sstevel@tonic-gate 			lgrp_t		*lgrp;
14617c478bd9Sstevel@tonic-gate 
14627c478bd9Sstevel@tonic-gate 			lgrp = lgrp_table[i];
14637c478bd9Sstevel@tonic-gate 			if (!LGRP_EXISTS(lgrp) ||
14647c478bd9Sstevel@tonic-gate 			    !klgrpset_ismember(lgrp->lgrp_set[LGRP_RSRC_MEM],
14657c478bd9Sstevel@tonic-gate 			    lgrpid))
14667c478bd9Sstevel@tonic-gate 				continue;
14677c478bd9Sstevel@tonic-gate 
14687c478bd9Sstevel@tonic-gate 			klgrpset_del(lgrp->lgrp_set[LGRP_RSRC_MEM], lgrpid);
14697c478bd9Sstevel@tonic-gate 		}
14707c478bd9Sstevel@tonic-gate 	}
14717c478bd9Sstevel@tonic-gate 	if (drop_lock)
14727c478bd9Sstevel@tonic-gate 		mutex_exit(&cpu_lock);
14737c478bd9Sstevel@tonic-gate }
14747c478bd9Sstevel@tonic-gate 
14757c478bd9Sstevel@tonic-gate /*
14767c478bd9Sstevel@tonic-gate  * Return lgroup with given platform handle
14777c478bd9Sstevel@tonic-gate  */
14787c478bd9Sstevel@tonic-gate lgrp_t *
14797c478bd9Sstevel@tonic-gate lgrp_hand_to_lgrp(lgrp_handle_t hand)
14807c478bd9Sstevel@tonic-gate {
14817c478bd9Sstevel@tonic-gate 	int	i;
14827c478bd9Sstevel@tonic-gate 	lgrp_t	*lgrp;
14837c478bd9Sstevel@tonic-gate 
14847c478bd9Sstevel@tonic-gate 	if (hand == LGRP_NULL_HANDLE)
14857c478bd9Sstevel@tonic-gate 		return (NULL);
14867c478bd9Sstevel@tonic-gate 
14877c478bd9Sstevel@tonic-gate 	for (i = 0; i <= lgrp_alloc_max; i++) {
14887c478bd9Sstevel@tonic-gate 		lgrp = lgrp_table[i];
14897c478bd9Sstevel@tonic-gate 		if (LGRP_EXISTS(lgrp) && lgrp->lgrp_plathand == hand)
14907c478bd9Sstevel@tonic-gate 			return (lgrp);
14917c478bd9Sstevel@tonic-gate 	}
14927c478bd9Sstevel@tonic-gate 	return (NULL);
14937c478bd9Sstevel@tonic-gate }
14947c478bd9Sstevel@tonic-gate 
14957c478bd9Sstevel@tonic-gate /*
14967c478bd9Sstevel@tonic-gate  * Return the home lgroup of the current thread.
14977c478bd9Sstevel@tonic-gate  * We must do this with kernel preemption disabled, since we don't want our
14987c478bd9Sstevel@tonic-gate  * thread to be re-homed while we're poking around with its lpl, and the lpl
14997c478bd9Sstevel@tonic-gate  * should never be NULL.
15007c478bd9Sstevel@tonic-gate  *
15017c478bd9Sstevel@tonic-gate  * NOTE: Can't guarantee that lgroup will be valid once kernel preemption
15027c478bd9Sstevel@tonic-gate  * is enabled because of DR.  Callers can use disable kernel preemption
15037c478bd9Sstevel@tonic-gate  * around this call to guarantee that the lgroup will be valid beyond this
15047c478bd9Sstevel@tonic-gate  * routine, since kernel preemption can be recursive.
15057c478bd9Sstevel@tonic-gate  */
15067c478bd9Sstevel@tonic-gate lgrp_t *
15077c478bd9Sstevel@tonic-gate lgrp_home_lgrp(void)
15087c478bd9Sstevel@tonic-gate {
15097c478bd9Sstevel@tonic-gate 	lgrp_t	*lgrp;
15107c478bd9Sstevel@tonic-gate 	lpl_t	*lpl;
15117c478bd9Sstevel@tonic-gate 
15127c478bd9Sstevel@tonic-gate 	kpreempt_disable();
15137c478bd9Sstevel@tonic-gate 
15147c478bd9Sstevel@tonic-gate 	lpl = curthread->t_lpl;
15157c478bd9Sstevel@tonic-gate 	ASSERT(lpl != NULL);
15167c478bd9Sstevel@tonic-gate 	ASSERT(lpl->lpl_lgrpid >= 0 && lpl->lpl_lgrpid <= lgrp_alloc_max);
15177c478bd9Sstevel@tonic-gate 	ASSERT(LGRP_EXISTS(lgrp_table[lpl->lpl_lgrpid]));
15187c478bd9Sstevel@tonic-gate 	lgrp = lgrp_table[lpl->lpl_lgrpid];
15197c478bd9Sstevel@tonic-gate 
15207c478bd9Sstevel@tonic-gate 	kpreempt_enable();
15217c478bd9Sstevel@tonic-gate 
15227c478bd9Sstevel@tonic-gate 	return (lgrp);
15237c478bd9Sstevel@tonic-gate }
15247c478bd9Sstevel@tonic-gate 
15257c478bd9Sstevel@tonic-gate /*
15267c478bd9Sstevel@tonic-gate  * Return ID of home lgroup for given thread
15277c478bd9Sstevel@tonic-gate  * (See comments for lgrp_home_lgrp() for special care and handling
15287c478bd9Sstevel@tonic-gate  * instructions)
15297c478bd9Sstevel@tonic-gate  */
15307c478bd9Sstevel@tonic-gate lgrp_id_t
15317c478bd9Sstevel@tonic-gate lgrp_home_id(kthread_t *t)
15327c478bd9Sstevel@tonic-gate {
15337c478bd9Sstevel@tonic-gate 	lgrp_id_t	lgrp;
15347c478bd9Sstevel@tonic-gate 	lpl_t		*lpl;
15357c478bd9Sstevel@tonic-gate 
15367c478bd9Sstevel@tonic-gate 	ASSERT(t != NULL);
15377c478bd9Sstevel@tonic-gate 	/*
15387c478bd9Sstevel@tonic-gate 	 * We'd like to ASSERT(MUTEX_HELD(&ttoproc(t)->p_lock)), but we
15397c478bd9Sstevel@tonic-gate 	 * cannot since the HAT layer can call into this routine to
15407c478bd9Sstevel@tonic-gate 	 * determine the locality for its data structures in the context
15417c478bd9Sstevel@tonic-gate 	 * of a page fault.
15427c478bd9Sstevel@tonic-gate 	 */
15437c478bd9Sstevel@tonic-gate 
15447c478bd9Sstevel@tonic-gate 	kpreempt_disable();
15457c478bd9Sstevel@tonic-gate 
15467c478bd9Sstevel@tonic-gate 	lpl = t->t_lpl;
15477c478bd9Sstevel@tonic-gate 	ASSERT(lpl != NULL);
15487c478bd9Sstevel@tonic-gate 	ASSERT(lpl->lpl_lgrpid >= 0 && lpl->lpl_lgrpid <= lgrp_alloc_max);
15497c478bd9Sstevel@tonic-gate 	lgrp = lpl->lpl_lgrpid;
15507c478bd9Sstevel@tonic-gate 
15517c478bd9Sstevel@tonic-gate 	kpreempt_enable();
15527c478bd9Sstevel@tonic-gate 
15537c478bd9Sstevel@tonic-gate 	return (lgrp);
15547c478bd9Sstevel@tonic-gate }
15557c478bd9Sstevel@tonic-gate 
15567c478bd9Sstevel@tonic-gate /*
15577c478bd9Sstevel@tonic-gate  * Return lgroup containing the physical memory for the given page frame number
15587c478bd9Sstevel@tonic-gate  */
15597c478bd9Sstevel@tonic-gate lgrp_t *
15607c478bd9Sstevel@tonic-gate lgrp_pfn_to_lgrp(pfn_t pfn)
15617c478bd9Sstevel@tonic-gate {
15627c478bd9Sstevel@tonic-gate 	lgrp_handle_t	hand;
15637c478bd9Sstevel@tonic-gate 	int		i;
15647c478bd9Sstevel@tonic-gate 	lgrp_t		*lgrp;
15657c478bd9Sstevel@tonic-gate 
15667c478bd9Sstevel@tonic-gate 	hand = lgrp_plat_pfn_to_hand(pfn);
15677c478bd9Sstevel@tonic-gate 	if (hand != LGRP_NULL_HANDLE)
15687c478bd9Sstevel@tonic-gate 		for (i = 0; i <= lgrp_alloc_max; i++) {
15697c478bd9Sstevel@tonic-gate 			lgrp = lgrp_table[i];
15707c478bd9Sstevel@tonic-gate 			if (LGRP_EXISTS(lgrp) && lgrp->lgrp_plathand == hand)
15717c478bd9Sstevel@tonic-gate 				return (lgrp);
15727c478bd9Sstevel@tonic-gate 		}
15737c478bd9Sstevel@tonic-gate 	return (NULL);
15747c478bd9Sstevel@tonic-gate }
15757c478bd9Sstevel@tonic-gate 
15767c478bd9Sstevel@tonic-gate /*
15777c478bd9Sstevel@tonic-gate  * Return lgroup containing the physical memory for the given page frame number
15787c478bd9Sstevel@tonic-gate  */
15797c478bd9Sstevel@tonic-gate lgrp_t *
15807c478bd9Sstevel@tonic-gate lgrp_phys_to_lgrp(u_longlong_t physaddr)
15817c478bd9Sstevel@tonic-gate {
15827c478bd9Sstevel@tonic-gate 	lgrp_handle_t	hand;
15837c478bd9Sstevel@tonic-gate 	int		i;
15847c478bd9Sstevel@tonic-gate 	lgrp_t		*lgrp;
15857c478bd9Sstevel@tonic-gate 	pfn_t		pfn;
15867c478bd9Sstevel@tonic-gate 
15877c478bd9Sstevel@tonic-gate 	pfn = btop(physaddr);
15887c478bd9Sstevel@tonic-gate 	hand = lgrp_plat_pfn_to_hand(pfn);
15897c478bd9Sstevel@tonic-gate 	if (hand != LGRP_NULL_HANDLE)
15907c478bd9Sstevel@tonic-gate 		for (i = 0; i <= lgrp_alloc_max; i++) {
15917c478bd9Sstevel@tonic-gate 			lgrp = lgrp_table[i];
15927c478bd9Sstevel@tonic-gate 			if (LGRP_EXISTS(lgrp) && lgrp->lgrp_plathand == hand)
15937c478bd9Sstevel@tonic-gate 				return (lgrp);
15947c478bd9Sstevel@tonic-gate 		}
15957c478bd9Sstevel@tonic-gate 	return (NULL);
15967c478bd9Sstevel@tonic-gate }
15977c478bd9Sstevel@tonic-gate 
15987c478bd9Sstevel@tonic-gate /*
15997c478bd9Sstevel@tonic-gate  * Return the leaf lgroup containing the given CPU
1600394b433dSesaxe  *
1601394b433dSesaxe  * The caller needs to take precautions necessary to prevent
1602fb2f18f8Sesaxe  * "cpu", and it's lpl from going away across a call to this function.
1603394b433dSesaxe  * hint: kpreempt_disable()/kpreempt_enable()
16047c478bd9Sstevel@tonic-gate  */
16057c478bd9Sstevel@tonic-gate static lgrp_t *
16067c478bd9Sstevel@tonic-gate lgrp_cpu_to_lgrp(cpu_t *cpu)
16077c478bd9Sstevel@tonic-gate {
1608ab761399Sesaxe 	return (cpu->cpu_lpl->lpl_lgrp);
16097c478bd9Sstevel@tonic-gate }
16107c478bd9Sstevel@tonic-gate 
16117c478bd9Sstevel@tonic-gate /*
16127c478bd9Sstevel@tonic-gate  * Return the sum of the partition loads in an lgrp divided by
16137c478bd9Sstevel@tonic-gate  * the number of CPUs in the lgrp.  This is our best approximation
16147c478bd9Sstevel@tonic-gate  * of an 'lgroup load average' for a useful per-lgroup kstat.
16157c478bd9Sstevel@tonic-gate  */
16167c478bd9Sstevel@tonic-gate static uint64_t
16177c478bd9Sstevel@tonic-gate lgrp_sum_loadavgs(lgrp_t *lgrp)
16187c478bd9Sstevel@tonic-gate {
16197c478bd9Sstevel@tonic-gate 	cpu_t *cpu;
16207c478bd9Sstevel@tonic-gate 	int ncpu;
16217c478bd9Sstevel@tonic-gate 	uint64_t loads = 0;
16227c478bd9Sstevel@tonic-gate 
16237c478bd9Sstevel@tonic-gate 	mutex_enter(&cpu_lock);
16247c478bd9Sstevel@tonic-gate 
16257c478bd9Sstevel@tonic-gate 	cpu = lgrp->lgrp_cpu;
16267c478bd9Sstevel@tonic-gate 	ncpu = lgrp->lgrp_cpucnt;
16277c478bd9Sstevel@tonic-gate 
16287c478bd9Sstevel@tonic-gate 	if (cpu == NULL || ncpu == 0) {
16297c478bd9Sstevel@tonic-gate 		mutex_exit(&cpu_lock);
16307c478bd9Sstevel@tonic-gate 		return (0ull);
16317c478bd9Sstevel@tonic-gate 	}
16327c478bd9Sstevel@tonic-gate 
16337c478bd9Sstevel@tonic-gate 	do {
16347c478bd9Sstevel@tonic-gate 		loads += cpu->cpu_lpl->lpl_loadavg;
16357c478bd9Sstevel@tonic-gate 		cpu = cpu->cpu_next_lgrp;
16367c478bd9Sstevel@tonic-gate 	} while (cpu != lgrp->lgrp_cpu);
16377c478bd9Sstevel@tonic-gate 
16387c478bd9Sstevel@tonic-gate 	mutex_exit(&cpu_lock);
16397c478bd9Sstevel@tonic-gate 
16407c478bd9Sstevel@tonic-gate 	return (loads / ncpu);
16417c478bd9Sstevel@tonic-gate }
16427c478bd9Sstevel@tonic-gate 
16437c478bd9Sstevel@tonic-gate void
16447c478bd9Sstevel@tonic-gate lgrp_stat_add(lgrp_id_t lgrpid, lgrp_stat_t stat, int64_t val)
16457c478bd9Sstevel@tonic-gate {
16467c478bd9Sstevel@tonic-gate 	struct lgrp_stats *pstats;
16477c478bd9Sstevel@tonic-gate 
16487c478bd9Sstevel@tonic-gate 	/*
16497c478bd9Sstevel@tonic-gate 	 * Verify that the caller isn't trying to add to
16507c478bd9Sstevel@tonic-gate 	 * a statistic for an lgroup that has gone away
16517c478bd9Sstevel@tonic-gate 	 */
16527c478bd9Sstevel@tonic-gate 	if (lgrpid < 0 || lgrpid > lgrp_alloc_max)
16537c478bd9Sstevel@tonic-gate 		return;
16547c478bd9Sstevel@tonic-gate 
16557c478bd9Sstevel@tonic-gate 	pstats = &lgrp_stats[lgrpid];
16567c478bd9Sstevel@tonic-gate 	atomic_add_64((uint64_t *)LGRP_STAT_WRITE_PTR(pstats, stat), val);
16577c478bd9Sstevel@tonic-gate }
16587c478bd9Sstevel@tonic-gate 
16597c478bd9Sstevel@tonic-gate int64_t
16607c478bd9Sstevel@tonic-gate lgrp_stat_read(lgrp_id_t lgrpid, lgrp_stat_t stat)
16617c478bd9Sstevel@tonic-gate {
16627c478bd9Sstevel@tonic-gate 	uint64_t val;
16637c478bd9Sstevel@tonic-gate 	struct lgrp_stats *pstats;
16647c478bd9Sstevel@tonic-gate 
16657c478bd9Sstevel@tonic-gate 	if (lgrpid < 0 || lgrpid > lgrp_alloc_max)
16667c478bd9Sstevel@tonic-gate 		return ((int64_t)0);
16677c478bd9Sstevel@tonic-gate 
16687c478bd9Sstevel@tonic-gate 	pstats = &lgrp_stats[lgrpid];
16697c478bd9Sstevel@tonic-gate 	LGRP_STAT_READ(pstats, stat, val);
16707c478bd9Sstevel@tonic-gate 	return (val);
16717c478bd9Sstevel@tonic-gate }
16727c478bd9Sstevel@tonic-gate 
16737c478bd9Sstevel@tonic-gate /*
16747c478bd9Sstevel@tonic-gate  * Reset all kstats for lgrp specified by its lgrpid.
16757c478bd9Sstevel@tonic-gate  */
16767c478bd9Sstevel@tonic-gate static void
16777c478bd9Sstevel@tonic-gate lgrp_kstat_reset(lgrp_id_t lgrpid)
16787c478bd9Sstevel@tonic-gate {
16797c478bd9Sstevel@tonic-gate 	lgrp_stat_t stat;
16807c478bd9Sstevel@tonic-gate 
16817c478bd9Sstevel@tonic-gate 	if (lgrpid < 0 || lgrpid > lgrp_alloc_max)
16827c478bd9Sstevel@tonic-gate 		return;
16837c478bd9Sstevel@tonic-gate 
16847c478bd9Sstevel@tonic-gate 	for (stat = 0; stat < LGRP_NUM_COUNTER_STATS; stat++) {
16857c478bd9Sstevel@tonic-gate 		LGRP_STAT_RESET(&lgrp_stats[lgrpid], stat);
16867c478bd9Sstevel@tonic-gate 	}
16877c478bd9Sstevel@tonic-gate }
16887c478bd9Sstevel@tonic-gate 
16897c478bd9Sstevel@tonic-gate /*
16907c478bd9Sstevel@tonic-gate  * Collect all per-lgrp statistics for the lgrp associated with this
16917c478bd9Sstevel@tonic-gate  * kstat, and store them in the ks_data array.
16927c478bd9Sstevel@tonic-gate  *
16937c478bd9Sstevel@tonic-gate  * The superuser can reset all the running counter statistics for an
16947c478bd9Sstevel@tonic-gate  * lgrp by writing to any of the lgrp's stats.
16957c478bd9Sstevel@tonic-gate  */
16967c478bd9Sstevel@tonic-gate static int
16977c478bd9Sstevel@tonic-gate lgrp_kstat_extract(kstat_t *ksp, int rw)
16987c478bd9Sstevel@tonic-gate {
16997c478bd9Sstevel@tonic-gate 	lgrp_stat_t		stat;
17007c478bd9Sstevel@tonic-gate 	struct kstat_named	*ksd;
17017c478bd9Sstevel@tonic-gate 	lgrp_t			*lgrp;
17027c478bd9Sstevel@tonic-gate 	lgrp_id_t		lgrpid;
17037c478bd9Sstevel@tonic-gate 
17047c478bd9Sstevel@tonic-gate 	lgrp = (lgrp_t *)ksp->ks_private;
17057c478bd9Sstevel@tonic-gate 
17067c478bd9Sstevel@tonic-gate 	ksd = (struct kstat_named *)ksp->ks_data;
17077c478bd9Sstevel@tonic-gate 	ASSERT(ksd == (struct kstat_named *)&lgrp_kstat_data);
17087c478bd9Sstevel@tonic-gate 
17097c478bd9Sstevel@tonic-gate 	lgrpid = lgrp->lgrp_id;
17107c478bd9Sstevel@tonic-gate 
17117c478bd9Sstevel@tonic-gate 	if (lgrpid == LGRP_NONE) {
17127c478bd9Sstevel@tonic-gate 		/*
17137c478bd9Sstevel@tonic-gate 		 * Return all zeroes as stats for freed lgrp.
17147c478bd9Sstevel@tonic-gate 		 */
17157c478bd9Sstevel@tonic-gate 		for (stat = 0; stat < LGRP_NUM_COUNTER_STATS; stat++) {
17167c478bd9Sstevel@tonic-gate 			ksd[stat].value.i64 = 0;
17177c478bd9Sstevel@tonic-gate 		}
17187c478bd9Sstevel@tonic-gate 		ksd[stat + LGRP_NUM_CPUS].value.i64 = 0;
17197c478bd9Sstevel@tonic-gate 		ksd[stat + LGRP_NUM_PG_INSTALL].value.i64 = 0;
17207c478bd9Sstevel@tonic-gate 		ksd[stat + LGRP_NUM_PG_AVAIL].value.i64 = 0;
17217c478bd9Sstevel@tonic-gate 		ksd[stat + LGRP_NUM_PG_FREE].value.i64 = 0;
17227c478bd9Sstevel@tonic-gate 		ksd[stat + LGRP_LOADAVG].value.i64 = 0;
17237c478bd9Sstevel@tonic-gate 	} else if (rw != KSTAT_WRITE) {
17247c478bd9Sstevel@tonic-gate 		/*
17257c478bd9Sstevel@tonic-gate 		 * Handle counter stats
17267c478bd9Sstevel@tonic-gate 		 */
17277c478bd9Sstevel@tonic-gate 		for (stat = 0; stat < LGRP_NUM_COUNTER_STATS; stat++) {
17287c478bd9Sstevel@tonic-gate 			ksd[stat].value.i64 = lgrp_stat_read(lgrpid, stat);
17297c478bd9Sstevel@tonic-gate 		}
17307c478bd9Sstevel@tonic-gate 
17317c478bd9Sstevel@tonic-gate 		/*
17327c478bd9Sstevel@tonic-gate 		 * Handle kernel data snapshot stats
17337c478bd9Sstevel@tonic-gate 		 */
17347c478bd9Sstevel@tonic-gate 		ksd[stat + LGRP_NUM_CPUS].value.i64 = lgrp->lgrp_cpucnt;
17357c478bd9Sstevel@tonic-gate 		ksd[stat + LGRP_NUM_PG_INSTALL].value.i64 =
17367c478bd9Sstevel@tonic-gate 		    lgrp_mem_size(lgrpid, LGRP_MEM_SIZE_INSTALL);
17377c478bd9Sstevel@tonic-gate 		ksd[stat + LGRP_NUM_PG_AVAIL].value.i64 =
17387c478bd9Sstevel@tonic-gate 		    lgrp_mem_size(lgrpid, LGRP_MEM_SIZE_AVAIL);
17397c478bd9Sstevel@tonic-gate 		ksd[stat + LGRP_NUM_PG_FREE].value.i64 =
17407c478bd9Sstevel@tonic-gate 		    lgrp_mem_size(lgrpid, LGRP_MEM_SIZE_FREE);
17417c478bd9Sstevel@tonic-gate 		ksd[stat + LGRP_LOADAVG].value.i64 = lgrp_sum_loadavgs(lgrp);
1742c6402783Sakolb 		ksd[stat + LGRP_LOADAVG_SCALE].value.i64 =
1743c6402783Sakolb 		    lgrp_loadavg_max_effect;
17447c478bd9Sstevel@tonic-gate 	} else {
17457c478bd9Sstevel@tonic-gate 		lgrp_kstat_reset(lgrpid);
17467c478bd9Sstevel@tonic-gate 	}
17477c478bd9Sstevel@tonic-gate 
17487c478bd9Sstevel@tonic-gate 	return (0);
17497c478bd9Sstevel@tonic-gate }
17507c478bd9Sstevel@tonic-gate 
17517c478bd9Sstevel@tonic-gate int
17527c478bd9Sstevel@tonic-gate lgrp_query_cpu(processorid_t id, lgrp_id_t *lp)
17537c478bd9Sstevel@tonic-gate {
17547c478bd9Sstevel@tonic-gate 	cpu_t	*cp;
17557c478bd9Sstevel@tonic-gate 
17567c478bd9Sstevel@tonic-gate 	mutex_enter(&cpu_lock);
17577c478bd9Sstevel@tonic-gate 
17587c478bd9Sstevel@tonic-gate 	if ((cp = cpu_get(id)) == NULL) {
17597c478bd9Sstevel@tonic-gate 		mutex_exit(&cpu_lock);
17607c478bd9Sstevel@tonic-gate 		return (EINVAL);
17617c478bd9Sstevel@tonic-gate 	}
17627c478bd9Sstevel@tonic-gate 
17637c478bd9Sstevel@tonic-gate 	if (cpu_is_offline(cp) || cpu_is_poweredoff(cp)) {
17647c478bd9Sstevel@tonic-gate 		mutex_exit(&cpu_lock);
17657c478bd9Sstevel@tonic-gate 		return (EINVAL);
17667c478bd9Sstevel@tonic-gate 	}
17677c478bd9Sstevel@tonic-gate 
17687c478bd9Sstevel@tonic-gate 	ASSERT(cp->cpu_lpl != NULL);
17697c478bd9Sstevel@tonic-gate 
17707c478bd9Sstevel@tonic-gate 	*lp = cp->cpu_lpl->lpl_lgrpid;
17717c478bd9Sstevel@tonic-gate 
17727c478bd9Sstevel@tonic-gate 	mutex_exit(&cpu_lock);
17737c478bd9Sstevel@tonic-gate 
17747c478bd9Sstevel@tonic-gate 	return (0);
17757c478bd9Sstevel@tonic-gate }
17767c478bd9Sstevel@tonic-gate 
17777c478bd9Sstevel@tonic-gate int
17787c478bd9Sstevel@tonic-gate lgrp_query_load(processorid_t id, lgrp_load_t *lp)
17797c478bd9Sstevel@tonic-gate {
17807c478bd9Sstevel@tonic-gate 	cpu_t *cp;
17817c478bd9Sstevel@tonic-gate 
17827c478bd9Sstevel@tonic-gate 	mutex_enter(&cpu_lock);
17837c478bd9Sstevel@tonic-gate 
17847c478bd9Sstevel@tonic-gate 	if ((cp = cpu_get(id)) == NULL) {
17857c478bd9Sstevel@tonic-gate 		mutex_exit(&cpu_lock);
17867c478bd9Sstevel@tonic-gate 		return (EINVAL);
17877c478bd9Sstevel@tonic-gate 	}
17887c478bd9Sstevel@tonic-gate 
17897c478bd9Sstevel@tonic-gate 	ASSERT(cp->cpu_lpl != NULL);
17907c478bd9Sstevel@tonic-gate 
17917c478bd9Sstevel@tonic-gate 	*lp = cp->cpu_lpl->lpl_loadavg;
17927c478bd9Sstevel@tonic-gate 
17937c478bd9Sstevel@tonic-gate 	mutex_exit(&cpu_lock);
17947c478bd9Sstevel@tonic-gate 
17957c478bd9Sstevel@tonic-gate 	return (0);
17967c478bd9Sstevel@tonic-gate }
17977c478bd9Sstevel@tonic-gate 
17987c478bd9Sstevel@tonic-gate /*
17997c478bd9Sstevel@tonic-gate  * Add a resource named by lpl_leaf to rset of lpl_target
18007c478bd9Sstevel@tonic-gate  *
18017c478bd9Sstevel@tonic-gate  * This routine also adjusts ncpu and nrset if the call succeeds in adding a
18027c478bd9Sstevel@tonic-gate  * resource. It is adjusted here, as this is presently the only place that we
18037c478bd9Sstevel@tonic-gate  * can be certain a resource addition has succeeded.
18047c478bd9Sstevel@tonic-gate  *
18057c478bd9Sstevel@tonic-gate  * We keep the list of rsets sorted so that the dispatcher can quickly walk the
18067c478bd9Sstevel@tonic-gate  * list in order until it reaches a NULL.  (This list is required to be NULL
18077c478bd9Sstevel@tonic-gate  * terminated, too).  This is done so that we can mark start pos + 1, so that
18087c478bd9Sstevel@tonic-gate  * each lpl is traversed sequentially, but in a different order.  We hope this
18097c478bd9Sstevel@tonic-gate  * will improve performance a bit.  (Hopefully, less read-to-own traffic...)
18107c478bd9Sstevel@tonic-gate  */
18117c478bd9Sstevel@tonic-gate 
18127c478bd9Sstevel@tonic-gate void
18137c478bd9Sstevel@tonic-gate lpl_rset_add(lpl_t *lpl_target, lpl_t *lpl_leaf)
18147c478bd9Sstevel@tonic-gate {
18157c478bd9Sstevel@tonic-gate 	int		i;
18167c478bd9Sstevel@tonic-gate 	int		entry_slot = 0;
18177c478bd9Sstevel@tonic-gate 
18187c478bd9Sstevel@tonic-gate 	/* return if leaf is already present */
18197c478bd9Sstevel@tonic-gate 	for (i = 0; i < lpl_target->lpl_nrset; i++) {
18207c478bd9Sstevel@tonic-gate 		if (lpl_target->lpl_rset[i] == lpl_leaf) {
18217c478bd9Sstevel@tonic-gate 			return;
18227c478bd9Sstevel@tonic-gate 		}
18237c478bd9Sstevel@tonic-gate 
18247c478bd9Sstevel@tonic-gate 		if (lpl_target->lpl_rset[i]->lpl_lgrpid >
18257c478bd9Sstevel@tonic-gate 		    lpl_leaf->lpl_lgrpid) {
18267c478bd9Sstevel@tonic-gate 			break;
18277c478bd9Sstevel@tonic-gate 		}
18287c478bd9Sstevel@tonic-gate 	}
18297c478bd9Sstevel@tonic-gate 
18307c478bd9Sstevel@tonic-gate 	/* insert leaf, update counts */
18317c478bd9Sstevel@tonic-gate 	entry_slot = i;
18327c478bd9Sstevel@tonic-gate 	i = lpl_target->lpl_nrset++;
18337c478bd9Sstevel@tonic-gate 	if (lpl_target->lpl_nrset >= LPL_RSET_MAX) {
18347c478bd9Sstevel@tonic-gate 		panic("More leaf lgrps in system than are supported!\n");
18357c478bd9Sstevel@tonic-gate 	}
18367c478bd9Sstevel@tonic-gate 
18377c478bd9Sstevel@tonic-gate 	/*
18387c478bd9Sstevel@tonic-gate 	 * Start at the end of the rset array and work backwards towards the
18397c478bd9Sstevel@tonic-gate 	 * slot into which the new lpl will be inserted. This effectively
18407c478bd9Sstevel@tonic-gate 	 * preserves the current ordering by scooting everybody over one entry,
18417c478bd9Sstevel@tonic-gate 	 * and placing the new entry into the space created.
18427c478bd9Sstevel@tonic-gate 	 */
18437c478bd9Sstevel@tonic-gate 
18447c478bd9Sstevel@tonic-gate 	while (i-- > entry_slot) {
18457c478bd9Sstevel@tonic-gate 		lpl_target->lpl_rset[i + 1] = lpl_target->lpl_rset[i];
18467c478bd9Sstevel@tonic-gate 	}
18477c478bd9Sstevel@tonic-gate 
18487c478bd9Sstevel@tonic-gate 	lpl_target->lpl_rset[entry_slot] = lpl_leaf;
18497c478bd9Sstevel@tonic-gate 	lpl_target->lpl_ncpu += lpl_leaf->lpl_ncpu;
18507c478bd9Sstevel@tonic-gate }
18517c478bd9Sstevel@tonic-gate 
18527c478bd9Sstevel@tonic-gate /*
18537c478bd9Sstevel@tonic-gate  * Update each of lpl_parent's children with a proper hint and
18547c478bd9Sstevel@tonic-gate  * a reference to their parent.
18557c478bd9Sstevel@tonic-gate  * The lgrp topology is used as the reference since it is fully
18567c478bd9Sstevel@tonic-gate  * consistent and correct at this point.
18577c478bd9Sstevel@tonic-gate  *
18587c478bd9Sstevel@tonic-gate  * Each child's hint will reference an element in lpl_parent's
18597c478bd9Sstevel@tonic-gate  * rset that designates where the child should start searching
18607c478bd9Sstevel@tonic-gate  * for CPU resources. The hint selected is the highest order leaf present
18617c478bd9Sstevel@tonic-gate  * in the child's lineage.
18627c478bd9Sstevel@tonic-gate  *
18637c478bd9Sstevel@tonic-gate  * This should be called after any potential change in lpl_parent's
18647c478bd9Sstevel@tonic-gate  * rset.
18657c478bd9Sstevel@tonic-gate  */
18667c478bd9Sstevel@tonic-gate static void
18677c478bd9Sstevel@tonic-gate lpl_child_update(lpl_t *lpl_parent, struct cpupart *cp)
18687c478bd9Sstevel@tonic-gate {
18697c478bd9Sstevel@tonic-gate 	klgrpset_t	children, leaves;
18707c478bd9Sstevel@tonic-gate 	lpl_t		*lpl;
18717c478bd9Sstevel@tonic-gate 	int		hint;
18727c478bd9Sstevel@tonic-gate 	int		i, j;
18737c478bd9Sstevel@tonic-gate 
18747c478bd9Sstevel@tonic-gate 	children = lgrp_table[lpl_parent->lpl_lgrpid]->lgrp_children;
18757c478bd9Sstevel@tonic-gate 	if (klgrpset_isempty(children))
18767c478bd9Sstevel@tonic-gate 		return; /* nothing to do */
18777c478bd9Sstevel@tonic-gate 
18787c478bd9Sstevel@tonic-gate 	for (i = 0; i <= lgrp_alloc_max; i++) {
18797c478bd9Sstevel@tonic-gate 		if (klgrpset_ismember(children, i)) {
18807c478bd9Sstevel@tonic-gate 
18817c478bd9Sstevel@tonic-gate 			/*
18827c478bd9Sstevel@tonic-gate 			 * Given the set of leaves in this child's lineage,
18837c478bd9Sstevel@tonic-gate 			 * find the highest order leaf present in the parent's
18847c478bd9Sstevel@tonic-gate 			 * rset. Select this as the hint for the child.
18857c478bd9Sstevel@tonic-gate 			 */
18867c478bd9Sstevel@tonic-gate 			leaves = lgrp_table[i]->lgrp_leaves;
18877c478bd9Sstevel@tonic-gate 			hint = 0;
18887c478bd9Sstevel@tonic-gate 			for (j = 0; j < lpl_parent->lpl_nrset; j++) {
18897c478bd9Sstevel@tonic-gate 				lpl = lpl_parent->lpl_rset[j];
18907c478bd9Sstevel@tonic-gate 				if (klgrpset_ismember(leaves, lpl->lpl_lgrpid))
18917c478bd9Sstevel@tonic-gate 					hint = j;
18927c478bd9Sstevel@tonic-gate 			}
18937c478bd9Sstevel@tonic-gate 			cp->cp_lgrploads[i].lpl_hint = hint;
18947c478bd9Sstevel@tonic-gate 
18957c478bd9Sstevel@tonic-gate 			/*
18967c478bd9Sstevel@tonic-gate 			 * (Re)set the parent. It may be incorrect if
18977c478bd9Sstevel@tonic-gate 			 * lpl_parent is new in the topology.
18987c478bd9Sstevel@tonic-gate 			 */
18997c478bd9Sstevel@tonic-gate 			cp->cp_lgrploads[i].lpl_parent = lpl_parent;
19007c478bd9Sstevel@tonic-gate 		}
19017c478bd9Sstevel@tonic-gate 	}
19027c478bd9Sstevel@tonic-gate }
19037c478bd9Sstevel@tonic-gate 
19047c478bd9Sstevel@tonic-gate /*
19057c478bd9Sstevel@tonic-gate  * Delete resource lpl_leaf from rset of lpl_target, assuming it's there.
19067c478bd9Sstevel@tonic-gate  *
19077c478bd9Sstevel@tonic-gate  * This routine also adjusts ncpu and nrset if the call succeeds in deleting a
19087c478bd9Sstevel@tonic-gate  * resource. The values are adjusted here, as this is the only place that we can
19097c478bd9Sstevel@tonic-gate  * be certain a resource was successfully deleted.
19107c478bd9Sstevel@tonic-gate  */
19117c478bd9Sstevel@tonic-gate void
19127c478bd9Sstevel@tonic-gate lpl_rset_del(lpl_t *lpl_target, lpl_t *lpl_leaf)
19137c478bd9Sstevel@tonic-gate {
19147c478bd9Sstevel@tonic-gate 	int i;
19157c478bd9Sstevel@tonic-gate 
19167c478bd9Sstevel@tonic-gate 	/* find leaf in intermediate node */
19177c478bd9Sstevel@tonic-gate 	for (i = 0; i < lpl_target->lpl_nrset; i++) {
19187c478bd9Sstevel@tonic-gate 		if (lpl_target->lpl_rset[i] == lpl_leaf)
19197c478bd9Sstevel@tonic-gate 			break;
19207c478bd9Sstevel@tonic-gate 	}
19217c478bd9Sstevel@tonic-gate 
19227c478bd9Sstevel@tonic-gate 	/* return if leaf not found */
19237c478bd9Sstevel@tonic-gate 	if (lpl_target->lpl_rset[i] != lpl_leaf)
19247c478bd9Sstevel@tonic-gate 		return;
19257c478bd9Sstevel@tonic-gate 
19267c478bd9Sstevel@tonic-gate 	/* prune leaf, compress array */
19277c478bd9Sstevel@tonic-gate 	ASSERT(lpl_target->lpl_nrset < LPL_RSET_MAX);
19287c478bd9Sstevel@tonic-gate 	lpl_target->lpl_rset[lpl_target->lpl_nrset--] = NULL;
19297c478bd9Sstevel@tonic-gate 	lpl_target->lpl_ncpu--;
19307c478bd9Sstevel@tonic-gate 	do {
19317c478bd9Sstevel@tonic-gate 		lpl_target->lpl_rset[i] = lpl_target->lpl_rset[i + 1];
19327c478bd9Sstevel@tonic-gate 	} while (i++ < lpl_target->lpl_nrset);
19337c478bd9Sstevel@tonic-gate }
19347c478bd9Sstevel@tonic-gate 
19357c478bd9Sstevel@tonic-gate /*
19367c478bd9Sstevel@tonic-gate  * Check to see if the resource set of the target lpl contains the
19377c478bd9Sstevel@tonic-gate  * supplied leaf lpl.  This returns 1 if the lpl is found, 0 if it is not.
19387c478bd9Sstevel@tonic-gate  */
19397c478bd9Sstevel@tonic-gate 
19407c478bd9Sstevel@tonic-gate int
19417c478bd9Sstevel@tonic-gate lpl_rset_contains(lpl_t *lpl_target, lpl_t *lpl_leaf)
19427c478bd9Sstevel@tonic-gate {
19437c478bd9Sstevel@tonic-gate 	int i;
19447c478bd9Sstevel@tonic-gate 
19457c478bd9Sstevel@tonic-gate 	for (i = 0; i < lpl_target->lpl_nrset; i++) {
19467c478bd9Sstevel@tonic-gate 		if (lpl_target->lpl_rset[i] == lpl_leaf)
19477c478bd9Sstevel@tonic-gate 			return (1);
19487c478bd9Sstevel@tonic-gate 	}
19497c478bd9Sstevel@tonic-gate 
19507c478bd9Sstevel@tonic-gate 	return (0);
19517c478bd9Sstevel@tonic-gate }
19527c478bd9Sstevel@tonic-gate 
19537c478bd9Sstevel@tonic-gate /*
19547c478bd9Sstevel@tonic-gate  * Called when we change cpu lpl membership.  This increments or decrements the
19557c478bd9Sstevel@tonic-gate  * per-cpu counter in every lpl in which our leaf appears.
19567c478bd9Sstevel@tonic-gate  */
19577c478bd9Sstevel@tonic-gate void
19587c478bd9Sstevel@tonic-gate lpl_cpu_adjcnt(lpl_act_t act, cpu_t *cp)
19597c478bd9Sstevel@tonic-gate {
19607c478bd9Sstevel@tonic-gate 	cpupart_t	*cpupart;
19617c478bd9Sstevel@tonic-gate 	lgrp_t		*lgrp_leaf;
19627c478bd9Sstevel@tonic-gate 	lgrp_t		*lgrp_cur;
19637c478bd9Sstevel@tonic-gate 	lpl_t		*lpl_leaf;
19647c478bd9Sstevel@tonic-gate 	lpl_t		*lpl_cur;
19657c478bd9Sstevel@tonic-gate 	int		i;
19667c478bd9Sstevel@tonic-gate 
19677c478bd9Sstevel@tonic-gate 	ASSERT(act == LPL_DECREMENT || act == LPL_INCREMENT);
19687c478bd9Sstevel@tonic-gate 
19697c478bd9Sstevel@tonic-gate 	cpupart = cp->cpu_part;
19707c478bd9Sstevel@tonic-gate 	lpl_leaf = cp->cpu_lpl;
19717c478bd9Sstevel@tonic-gate 	lgrp_leaf = lgrp_table[lpl_leaf->lpl_lgrpid];
19727c478bd9Sstevel@tonic-gate 
19737c478bd9Sstevel@tonic-gate 	for (i = 0; i <= lgrp_alloc_max; i++) {
19747c478bd9Sstevel@tonic-gate 		lgrp_cur = lgrp_table[i];
19757c478bd9Sstevel@tonic-gate 
19767c478bd9Sstevel@tonic-gate 		/*
19777c478bd9Sstevel@tonic-gate 		 * Don't adjust if the lgrp isn't there, if we're the leaf lpl
19787c478bd9Sstevel@tonic-gate 		 * for the cpu in question, or if the current lgrp and leaf
19797c478bd9Sstevel@tonic-gate 		 * don't share the same resources.
19807c478bd9Sstevel@tonic-gate 		 */
19817c478bd9Sstevel@tonic-gate 
19827c478bd9Sstevel@tonic-gate 		if (!LGRP_EXISTS(lgrp_cur) || (lgrp_cur == lgrp_leaf) ||
19837c478bd9Sstevel@tonic-gate 		    !klgrpset_intersects(lgrp_leaf->lgrp_set[LGRP_RSRC_CPU],
19847c478bd9Sstevel@tonic-gate 		    lgrp_cur->lgrp_set[LGRP_RSRC_CPU]))
19857c478bd9Sstevel@tonic-gate 			continue;
19867c478bd9Sstevel@tonic-gate 
19877c478bd9Sstevel@tonic-gate 
19887c478bd9Sstevel@tonic-gate 		lpl_cur = &cpupart->cp_lgrploads[lgrp_cur->lgrp_id];
19897c478bd9Sstevel@tonic-gate 
19907c478bd9Sstevel@tonic-gate 		if (lpl_cur->lpl_nrset > 0) {
19917c478bd9Sstevel@tonic-gate 			if (act == LPL_INCREMENT) {
19927c478bd9Sstevel@tonic-gate 				lpl_cur->lpl_ncpu++;
19937c478bd9Sstevel@tonic-gate 			} else if (act == LPL_DECREMENT) {
19947c478bd9Sstevel@tonic-gate 				lpl_cur->lpl_ncpu--;
19957c478bd9Sstevel@tonic-gate 			}
19967c478bd9Sstevel@tonic-gate 		}
19977c478bd9Sstevel@tonic-gate 	}
19987c478bd9Sstevel@tonic-gate }
19997c478bd9Sstevel@tonic-gate 
20007c478bd9Sstevel@tonic-gate /*
20017c478bd9Sstevel@tonic-gate  * Initialize lpl with given resources and specified lgrp
20027c478bd9Sstevel@tonic-gate  */
20037c478bd9Sstevel@tonic-gate 
20047c478bd9Sstevel@tonic-gate void
20057c478bd9Sstevel@tonic-gate lpl_init(lpl_t *lpl, lpl_t *lpl_leaf, lgrp_t *lgrp)
20067c478bd9Sstevel@tonic-gate {
20077c478bd9Sstevel@tonic-gate 	lpl->lpl_lgrpid = lgrp->lgrp_id;
20087c478bd9Sstevel@tonic-gate 	lpl->lpl_loadavg = 0;
20097c478bd9Sstevel@tonic-gate 	if (lpl == lpl_leaf)
20107c478bd9Sstevel@tonic-gate 		lpl->lpl_ncpu = 1;
20117c478bd9Sstevel@tonic-gate 	else
20127c478bd9Sstevel@tonic-gate 		lpl->lpl_ncpu = lpl_leaf->lpl_ncpu;
20137c478bd9Sstevel@tonic-gate 	lpl->lpl_nrset = 1;
20147c478bd9Sstevel@tonic-gate 	lpl->lpl_rset[0] = lpl_leaf;
20157c478bd9Sstevel@tonic-gate 	lpl->lpl_lgrp = lgrp;
20167c478bd9Sstevel@tonic-gate 	lpl->lpl_parent = NULL; /* set by lpl_leaf_insert() */
20177c478bd9Sstevel@tonic-gate 	lpl->lpl_cpus = NULL; /* set by lgrp_part_add_cpu() */
20187c478bd9Sstevel@tonic-gate }
20197c478bd9Sstevel@tonic-gate 
20207c478bd9Sstevel@tonic-gate /*
20217c478bd9Sstevel@tonic-gate  * Clear an unused lpl
20227c478bd9Sstevel@tonic-gate  */
20237c478bd9Sstevel@tonic-gate 
20247c478bd9Sstevel@tonic-gate void
20257c478bd9Sstevel@tonic-gate lpl_clear(lpl_t *lpl)
20267c478bd9Sstevel@tonic-gate {
2027ab761399Sesaxe 	lgrp_id_t	lid;
20287c478bd9Sstevel@tonic-gate 
20297c478bd9Sstevel@tonic-gate 	/* save lid for debugging purposes */
20307c478bd9Sstevel@tonic-gate 	lid = lpl->lpl_lgrpid;
20317c478bd9Sstevel@tonic-gate 	bzero(lpl, sizeof (lpl_t));
20327c478bd9Sstevel@tonic-gate 	lpl->lpl_lgrpid = lid;
20337c478bd9Sstevel@tonic-gate }
20347c478bd9Sstevel@tonic-gate 
20357c478bd9Sstevel@tonic-gate /*
20367c478bd9Sstevel@tonic-gate  * Given a CPU-partition, verify that the lpl topology in the CPU-partition
20377c478bd9Sstevel@tonic-gate  * is in sync with the lgroup toplogy in the system.  The lpl topology may not
20387c478bd9Sstevel@tonic-gate  * make full use of all of the lgroup topology, but this checks to make sure
20397c478bd9Sstevel@tonic-gate  * that for the parts that it does use, it has correctly understood the
20407c478bd9Sstevel@tonic-gate  * relationships that exist. This function returns
20417c478bd9Sstevel@tonic-gate  * 0 if the topology is correct, and a non-zero error code, for non-debug
20427c478bd9Sstevel@tonic-gate  * kernels if incorrect.  Asserts are spread throughout the code to aid in
20437c478bd9Sstevel@tonic-gate  * debugging on a DEBUG kernel.
20447c478bd9Sstevel@tonic-gate  */
20457c478bd9Sstevel@tonic-gate int
20467c478bd9Sstevel@tonic-gate lpl_topo_verify(cpupart_t *cpupart)
20477c478bd9Sstevel@tonic-gate {
20487c478bd9Sstevel@tonic-gate 	lgrp_t		*lgrp;
20497c478bd9Sstevel@tonic-gate 	lpl_t		*lpl;
20507c478bd9Sstevel@tonic-gate 	klgrpset_t	rset;
20517c478bd9Sstevel@tonic-gate 	klgrpset_t	cset;
20527c478bd9Sstevel@tonic-gate 	cpu_t		*cpu;
20537c478bd9Sstevel@tonic-gate 	cpu_t		*cp_start;
20547c478bd9Sstevel@tonic-gate 	int		i;
20557c478bd9Sstevel@tonic-gate 	int		j;
20567c478bd9Sstevel@tonic-gate 	int		sum;
20577c478bd9Sstevel@tonic-gate 
20587c478bd9Sstevel@tonic-gate 	/* topology can't be incorrect if it doesn't exist */
20597c478bd9Sstevel@tonic-gate 	if (!lgrp_topo_initialized || !lgrp_initialized)
20607c478bd9Sstevel@tonic-gate 		return (LPL_TOPO_CORRECT);
20617c478bd9Sstevel@tonic-gate 
20627c478bd9Sstevel@tonic-gate 	ASSERT(cpupart != NULL);
20637c478bd9Sstevel@tonic-gate 
20647c478bd9Sstevel@tonic-gate 	for (i = 0; i <= lgrp_alloc_max; i++) {
20657c478bd9Sstevel@tonic-gate 		lgrp = lgrp_table[i];
20667c478bd9Sstevel@tonic-gate 		lpl = NULL;
20677c478bd9Sstevel@tonic-gate 		/* make sure lpls are allocated */
20687c478bd9Sstevel@tonic-gate 		ASSERT(cpupart->cp_lgrploads);
20697c478bd9Sstevel@tonic-gate 		if (!cpupart->cp_lgrploads)
20707c478bd9Sstevel@tonic-gate 			return (LPL_TOPO_PART_HAS_NO_LPL);
20717c478bd9Sstevel@tonic-gate 
20727c478bd9Sstevel@tonic-gate 		lpl = &cpupart->cp_lgrploads[i];
20737c478bd9Sstevel@tonic-gate 		/* make sure our index is good */
20747c478bd9Sstevel@tonic-gate 		ASSERT(i < cpupart->cp_nlgrploads);
20757c478bd9Sstevel@tonic-gate 
20767c478bd9Sstevel@tonic-gate 		/* if lgroup doesn't exist, make sure lpl is empty */
20777c478bd9Sstevel@tonic-gate 		if (!LGRP_EXISTS(lgrp)) {
20787c478bd9Sstevel@tonic-gate 			ASSERT(lpl->lpl_ncpu == 0);
20797c478bd9Sstevel@tonic-gate 			if (lpl->lpl_ncpu > 0) {
20807c478bd9Sstevel@tonic-gate 				return (LPL_TOPO_CPUS_NOT_EMPTY);
20817c478bd9Sstevel@tonic-gate 			} else {
20827c478bd9Sstevel@tonic-gate 				continue;
20837c478bd9Sstevel@tonic-gate 			}
20847c478bd9Sstevel@tonic-gate 		}
20857c478bd9Sstevel@tonic-gate 
20867c478bd9Sstevel@tonic-gate 		/* verify that lgroup and lpl are identically numbered */
20877c478bd9Sstevel@tonic-gate 		ASSERT(lgrp->lgrp_id == lpl->lpl_lgrpid);
20887c478bd9Sstevel@tonic-gate 
20897c478bd9Sstevel@tonic-gate 		/* if lgroup isn't in our partition, make sure lpl is empty */
20907c478bd9Sstevel@tonic-gate 		if (!klgrpset_intersects(lgrp->lgrp_leaves,
20917c478bd9Sstevel@tonic-gate 		    cpupart->cp_lgrpset)) {
20927c478bd9Sstevel@tonic-gate 			ASSERT(lpl->lpl_ncpu == 0);
20937c478bd9Sstevel@tonic-gate 			if (lpl->lpl_ncpu > 0) {
20947c478bd9Sstevel@tonic-gate 				return (LPL_TOPO_CPUS_NOT_EMPTY);
20957c478bd9Sstevel@tonic-gate 			}
20967c478bd9Sstevel@tonic-gate 			/*
20977c478bd9Sstevel@tonic-gate 			 * lpl is empty, and lgroup isn't in partition.  verify
20987c478bd9Sstevel@tonic-gate 			 * that lpl doesn't show up in anyone else's rsets (in
20997c478bd9Sstevel@tonic-gate 			 * this partition, anyway)
21007c478bd9Sstevel@tonic-gate 			 */
21017c478bd9Sstevel@tonic-gate 
21027c478bd9Sstevel@tonic-gate 			for (j = 0; j < cpupart->cp_nlgrploads; j++) {
21037c478bd9Sstevel@tonic-gate 				lpl_t *i_lpl; /* lpl we're iterating over */
21047c478bd9Sstevel@tonic-gate 
21057c478bd9Sstevel@tonic-gate 				i_lpl = &cpupart->cp_lgrploads[j];
21067c478bd9Sstevel@tonic-gate 
21077c478bd9Sstevel@tonic-gate 				ASSERT(!lpl_rset_contains(i_lpl, lpl));
21087c478bd9Sstevel@tonic-gate 				if (lpl_rset_contains(i_lpl, lpl)) {
21097c478bd9Sstevel@tonic-gate 					return (LPL_TOPO_LPL_ORPHANED);
21107c478bd9Sstevel@tonic-gate 				}
21117c478bd9Sstevel@tonic-gate 			}
21127c478bd9Sstevel@tonic-gate 			/* lgroup is empty, and everything is ok. continue */
21137c478bd9Sstevel@tonic-gate 			continue;
21147c478bd9Sstevel@tonic-gate 		}
21157c478bd9Sstevel@tonic-gate 
21167c478bd9Sstevel@tonic-gate 
21177c478bd9Sstevel@tonic-gate 		/* lgroup is in this partition, now check it against lpl */
21187c478bd9Sstevel@tonic-gate 
21197c478bd9Sstevel@tonic-gate 		/* do both have matching lgrps? */
21207c478bd9Sstevel@tonic-gate 		ASSERT(lgrp == lpl->lpl_lgrp);
21217c478bd9Sstevel@tonic-gate 		if (lgrp != lpl->lpl_lgrp) {
21227c478bd9Sstevel@tonic-gate 			return (LPL_TOPO_LGRP_MISMATCH);
21237c478bd9Sstevel@tonic-gate 		}
21247c478bd9Sstevel@tonic-gate 
21257c478bd9Sstevel@tonic-gate 		/* do the parent lgroups exist and do they match? */
21267c478bd9Sstevel@tonic-gate 		if (lgrp->lgrp_parent) {
21277c478bd9Sstevel@tonic-gate 			ASSERT(lpl->lpl_parent);
21287c478bd9Sstevel@tonic-gate 			ASSERT(lgrp->lgrp_parent->lgrp_id ==
21297c478bd9Sstevel@tonic-gate 				    lpl->lpl_parent->lpl_lgrpid);
21307c478bd9Sstevel@tonic-gate 
21317c478bd9Sstevel@tonic-gate 			if (!lpl->lpl_parent) {
21327c478bd9Sstevel@tonic-gate 				return (LPL_TOPO_MISSING_PARENT);
21337c478bd9Sstevel@tonic-gate 			} else if (lgrp->lgrp_parent->lgrp_id !=
21347c478bd9Sstevel@tonic-gate 			    lpl->lpl_parent->lpl_lgrpid) {
21357c478bd9Sstevel@tonic-gate 				return (LPL_TOPO_PARENT_MISMATCH);
21367c478bd9Sstevel@tonic-gate 			}
21377c478bd9Sstevel@tonic-gate 		}
21387c478bd9Sstevel@tonic-gate 
21397c478bd9Sstevel@tonic-gate 		/* only leaf lgroups keep a cpucnt, only check leaves */
21407c478bd9Sstevel@tonic-gate 		if ((lpl->lpl_nrset == 1) && (lpl == lpl->lpl_rset[0])) {
21417c478bd9Sstevel@tonic-gate 
21427c478bd9Sstevel@tonic-gate 			/* verify that lgrp is also a leaf */
21437c478bd9Sstevel@tonic-gate 			ASSERT((lgrp->lgrp_childcnt == 0) &&
21447c478bd9Sstevel@tonic-gate 			    (klgrpset_ismember(lgrp->lgrp_leaves,
21457c478bd9Sstevel@tonic-gate 			    lpl->lpl_lgrpid)));
21467c478bd9Sstevel@tonic-gate 
21477c478bd9Sstevel@tonic-gate 			if ((lgrp->lgrp_childcnt > 0) ||
21487c478bd9Sstevel@tonic-gate 			    (!klgrpset_ismember(lgrp->lgrp_leaves,
21497c478bd9Sstevel@tonic-gate 			    lpl->lpl_lgrpid))) {
21507c478bd9Sstevel@tonic-gate 				return (LPL_TOPO_LGRP_NOT_LEAF);
21517c478bd9Sstevel@tonic-gate 			}
21527c478bd9Sstevel@tonic-gate 
21537c478bd9Sstevel@tonic-gate 			ASSERT((lgrp->lgrp_cpucnt >= lpl->lpl_ncpu) &&
21547c478bd9Sstevel@tonic-gate 			    (lpl->lpl_ncpu > 0));
21557c478bd9Sstevel@tonic-gate 			if ((lgrp->lgrp_cpucnt < lpl->lpl_ncpu) ||
21567c478bd9Sstevel@tonic-gate 				(lpl->lpl_ncpu <= 0)) {
21577c478bd9Sstevel@tonic-gate 				return (LPL_TOPO_BAD_CPUCNT);
21587c478bd9Sstevel@tonic-gate 			}
21597c478bd9Sstevel@tonic-gate 
21607c478bd9Sstevel@tonic-gate 			/*
21617c478bd9Sstevel@tonic-gate 			 * Check that lpl_ncpu also matches the number of
21627c478bd9Sstevel@tonic-gate 			 * cpus in the lpl's linked list.  This only exists in
21637c478bd9Sstevel@tonic-gate 			 * leaves, but they should always match.
21647c478bd9Sstevel@tonic-gate 			 */
21657c478bd9Sstevel@tonic-gate 			j = 0;
21667c478bd9Sstevel@tonic-gate 			cpu = cp_start = lpl->lpl_cpus;
21677c478bd9Sstevel@tonic-gate 			while (cpu != NULL) {
21687c478bd9Sstevel@tonic-gate 				j++;
21697c478bd9Sstevel@tonic-gate 
21707c478bd9Sstevel@tonic-gate 				/* check to make sure cpu's lpl is leaf lpl */
21717c478bd9Sstevel@tonic-gate 				ASSERT(cpu->cpu_lpl == lpl);
21727c478bd9Sstevel@tonic-gate 				if (cpu->cpu_lpl != lpl) {
21737c478bd9Sstevel@tonic-gate 					return (LPL_TOPO_CPU_HAS_BAD_LPL);
21747c478bd9Sstevel@tonic-gate 				}
21757c478bd9Sstevel@tonic-gate 
21767c478bd9Sstevel@tonic-gate 				/* check next cpu */
21777c478bd9Sstevel@tonic-gate 				if ((cpu = cpu->cpu_next_lpl) != cp_start) {
21787c478bd9Sstevel@tonic-gate 					continue;
21797c478bd9Sstevel@tonic-gate 				} else {
21807c478bd9Sstevel@tonic-gate 					cpu = NULL;
21817c478bd9Sstevel@tonic-gate 				}
21827c478bd9Sstevel@tonic-gate 			}
21837c478bd9Sstevel@tonic-gate 
21847c478bd9Sstevel@tonic-gate 			ASSERT(j == lpl->lpl_ncpu);
21857c478bd9Sstevel@tonic-gate 			if (j != lpl->lpl_ncpu) {
21867c478bd9Sstevel@tonic-gate 				return (LPL_TOPO_LPL_BAD_NCPU);
21877c478bd9Sstevel@tonic-gate 			}
21887c478bd9Sstevel@tonic-gate 
21897c478bd9Sstevel@tonic-gate 			/*
21907c478bd9Sstevel@tonic-gate 			 * Also, check that leaf lpl is contained in all
21917c478bd9Sstevel@tonic-gate 			 * intermediate lpls that name the leaf as a descendant
21927c478bd9Sstevel@tonic-gate 			 */
21937c478bd9Sstevel@tonic-gate 
21947c478bd9Sstevel@tonic-gate 			for (j = 0; j <= lgrp_alloc_max; j++) {
21957c478bd9Sstevel@tonic-gate 				klgrpset_t intersect;
21967c478bd9Sstevel@tonic-gate 				lgrp_t *lgrp_cand;
21977c478bd9Sstevel@tonic-gate 				lpl_t *lpl_cand;
21987c478bd9Sstevel@tonic-gate 
21997c478bd9Sstevel@tonic-gate 				lgrp_cand = lgrp_table[j];
22007c478bd9Sstevel@tonic-gate 				intersect = klgrpset_intersects(
22017c478bd9Sstevel@tonic-gate 				    lgrp_cand->lgrp_set[LGRP_RSRC_CPU],
22027c478bd9Sstevel@tonic-gate 				    cpupart->cp_lgrpset);
22037c478bd9Sstevel@tonic-gate 
22047c478bd9Sstevel@tonic-gate 				if (!LGRP_EXISTS(lgrp_cand) ||
22057c478bd9Sstevel@tonic-gate 				    !klgrpset_intersects(lgrp_cand->lgrp_leaves,
22067c478bd9Sstevel@tonic-gate 				    cpupart->cp_lgrpset) ||
22077c478bd9Sstevel@tonic-gate 				    (intersect == 0))
22087c478bd9Sstevel@tonic-gate 					continue;
22097c478bd9Sstevel@tonic-gate 
22107c478bd9Sstevel@tonic-gate 				lpl_cand =
22117c478bd9Sstevel@tonic-gate 				    &cpupart->cp_lgrploads[lgrp_cand->lgrp_id];
22127c478bd9Sstevel@tonic-gate 
22137c478bd9Sstevel@tonic-gate 				if (klgrpset_ismember(intersect,
22147c478bd9Sstevel@tonic-gate 				    lgrp->lgrp_id)) {
22157c478bd9Sstevel@tonic-gate 					ASSERT(lpl_rset_contains(lpl_cand,
22167c478bd9Sstevel@tonic-gate 					    lpl));
22177c478bd9Sstevel@tonic-gate 
22187c478bd9Sstevel@tonic-gate 					if (!lpl_rset_contains(lpl_cand, lpl)) {
22197c478bd9Sstevel@tonic-gate 						return (LPL_TOPO_RSET_MSSNG_LF);
22207c478bd9Sstevel@tonic-gate 					}
22217c478bd9Sstevel@tonic-gate 				}
22227c478bd9Sstevel@tonic-gate 			}
22237c478bd9Sstevel@tonic-gate 
22247c478bd9Sstevel@tonic-gate 		} else { /* non-leaf specific checks */
22257c478bd9Sstevel@tonic-gate 
22267c478bd9Sstevel@tonic-gate 			/*
22277c478bd9Sstevel@tonic-gate 			 * Non-leaf lpls should have lpl_cpus == NULL
22287c478bd9Sstevel@tonic-gate 			 * verify that this is so
22297c478bd9Sstevel@tonic-gate 			 */
22307c478bd9Sstevel@tonic-gate 			ASSERT(lpl->lpl_cpus == NULL);
22317c478bd9Sstevel@tonic-gate 			if (lpl->lpl_cpus != NULL) {
22327c478bd9Sstevel@tonic-gate 				return (LPL_TOPO_NONLEAF_HAS_CPUS);
22337c478bd9Sstevel@tonic-gate 			}
22347c478bd9Sstevel@tonic-gate 
22357c478bd9Sstevel@tonic-gate 			/*
22367c478bd9Sstevel@tonic-gate 			 * verify that the sum of the cpus in the leaf resources
22377c478bd9Sstevel@tonic-gate 			 * is equal to the total ncpu in the intermediate
22387c478bd9Sstevel@tonic-gate 			 */
22397c478bd9Sstevel@tonic-gate 			for (j = sum = 0; j < lpl->lpl_nrset; j++) {
22407c478bd9Sstevel@tonic-gate 				sum += lpl->lpl_rset[j]->lpl_ncpu;
22417c478bd9Sstevel@tonic-gate 			}
22427c478bd9Sstevel@tonic-gate 
22437c478bd9Sstevel@tonic-gate 			ASSERT(sum == lpl->lpl_ncpu);
22447c478bd9Sstevel@tonic-gate 			if (sum != lpl->lpl_ncpu) {
22457c478bd9Sstevel@tonic-gate 				return (LPL_TOPO_LPL_BAD_NCPU);
22467c478bd9Sstevel@tonic-gate 			}
22477c478bd9Sstevel@tonic-gate 		}
22487c478bd9Sstevel@tonic-gate 
22497c478bd9Sstevel@tonic-gate 		/*
22507c478bd9Sstevel@tonic-gate 		 * check on lpl_hint. Don't check root, since it has no parent.
22517c478bd9Sstevel@tonic-gate 		 */
22527c478bd9Sstevel@tonic-gate 		if (lpl->lpl_parent != NULL) {
22537c478bd9Sstevel@tonic-gate 			int hint;
22547c478bd9Sstevel@tonic-gate 			lpl_t *hint_lpl;
22557c478bd9Sstevel@tonic-gate 
22567c478bd9Sstevel@tonic-gate 			/* make sure hint is within limits of nrset */
22577c478bd9Sstevel@tonic-gate 			hint = lpl->lpl_hint;
22587c478bd9Sstevel@tonic-gate 			ASSERT(lpl->lpl_parent->lpl_nrset >= hint);
22597c478bd9Sstevel@tonic-gate 			if (lpl->lpl_parent->lpl_nrset < hint) {
22607c478bd9Sstevel@tonic-gate 				return (LPL_TOPO_BOGUS_HINT);
22617c478bd9Sstevel@tonic-gate 			}
22627c478bd9Sstevel@tonic-gate 
22637c478bd9Sstevel@tonic-gate 			/* make sure hint points to valid lpl */
22647c478bd9Sstevel@tonic-gate 			hint_lpl = lpl->lpl_parent->lpl_rset[hint];
22657c478bd9Sstevel@tonic-gate 			ASSERT(hint_lpl->lpl_ncpu > 0);
22667c478bd9Sstevel@tonic-gate 			if (hint_lpl->lpl_ncpu <= 0) {
22677c478bd9Sstevel@tonic-gate 				return (LPL_TOPO_BOGUS_HINT);
22687c478bd9Sstevel@tonic-gate 			}
22697c478bd9Sstevel@tonic-gate 		}
22707c478bd9Sstevel@tonic-gate 
22717c478bd9Sstevel@tonic-gate 		/*
22727c478bd9Sstevel@tonic-gate 		 * Check the rset of the lpl in question.  Make sure that each
22737c478bd9Sstevel@tonic-gate 		 * rset contains a subset of the resources in
22747c478bd9Sstevel@tonic-gate 		 * lgrp_set[LGRP_RSRC_CPU] and in cp_lgrpset.  This also makes
22757c478bd9Sstevel@tonic-gate 		 * sure that each rset doesn't include resources that are
22767c478bd9Sstevel@tonic-gate 		 * outside of that set.  (Which would be resources somehow not
22777c478bd9Sstevel@tonic-gate 		 * accounted for).
22787c478bd9Sstevel@tonic-gate 		 */
22797c478bd9Sstevel@tonic-gate 
22807c478bd9Sstevel@tonic-gate 		klgrpset_clear(rset);
22817c478bd9Sstevel@tonic-gate 		for (j = 0; j < lpl->lpl_nrset; j++) {
22827c478bd9Sstevel@tonic-gate 			klgrpset_add(rset, lpl->lpl_rset[j]->lpl_lgrpid);
22837c478bd9Sstevel@tonic-gate 		}
22847c478bd9Sstevel@tonic-gate 		klgrpset_copy(cset, rset);
22857c478bd9Sstevel@tonic-gate 		/* make sure lpl rset matches lgrp rset */
22867c478bd9Sstevel@tonic-gate 		klgrpset_diff(rset, lgrp->lgrp_set[LGRP_RSRC_CPU]);
22877c478bd9Sstevel@tonic-gate 		/* make sure rset is contained with in partition, too */
22887c478bd9Sstevel@tonic-gate 		klgrpset_diff(cset, cpupart->cp_lgrpset);
22897c478bd9Sstevel@tonic-gate 
22907c478bd9Sstevel@tonic-gate 		ASSERT(klgrpset_isempty(rset) &&
22917c478bd9Sstevel@tonic-gate 			    klgrpset_isempty(cset));
22927c478bd9Sstevel@tonic-gate 		if (!klgrpset_isempty(rset) ||
22937c478bd9Sstevel@tonic-gate 		    !klgrpset_isempty(cset)) {
22947c478bd9Sstevel@tonic-gate 			return (LPL_TOPO_RSET_MISMATCH);
22957c478bd9Sstevel@tonic-gate 		}
22967c478bd9Sstevel@tonic-gate 
22977c478bd9Sstevel@tonic-gate 		/*
22987c478bd9Sstevel@tonic-gate 		 * check to make sure lpl_nrset matches the number of rsets
22997c478bd9Sstevel@tonic-gate 		 * contained in the lpl
23007c478bd9Sstevel@tonic-gate 		 */
23017c478bd9Sstevel@tonic-gate 
23027c478bd9Sstevel@tonic-gate 		for (j = 0; (lpl->lpl_rset[j] != NULL) && (j < LPL_RSET_MAX);
23037c478bd9Sstevel@tonic-gate 		    j++);
23047c478bd9Sstevel@tonic-gate 
23057c478bd9Sstevel@tonic-gate 		ASSERT(j == lpl->lpl_nrset);
23067c478bd9Sstevel@tonic-gate 		if (j != lpl->lpl_nrset) {
23077c478bd9Sstevel@tonic-gate 			return (LPL_TOPO_BAD_RSETCNT);
23087c478bd9Sstevel@tonic-gate 		}
23097c478bd9Sstevel@tonic-gate 
23107c478bd9Sstevel@tonic-gate 	}
23117c478bd9Sstevel@tonic-gate 	return (LPL_TOPO_CORRECT);
23127c478bd9Sstevel@tonic-gate }
23137c478bd9Sstevel@tonic-gate 
23147c478bd9Sstevel@tonic-gate /*
23157c478bd9Sstevel@tonic-gate  * Flatten lpl topology to given number of levels.  This is presently only
23167c478bd9Sstevel@tonic-gate  * implemented for a flatten to 2 levels, which will prune out the intermediates
23177c478bd9Sstevel@tonic-gate  * and home the leaf lpls to the root lpl.
23187c478bd9Sstevel@tonic-gate  */
23197c478bd9Sstevel@tonic-gate int
23207c478bd9Sstevel@tonic-gate lpl_topo_flatten(int levels)
23217c478bd9Sstevel@tonic-gate {
23227c478bd9Sstevel@tonic-gate 	int		i;
23237c478bd9Sstevel@tonic-gate 	uint_t		sum;
23247c478bd9Sstevel@tonic-gate 	lgrp_t		*lgrp_cur;
23257c478bd9Sstevel@tonic-gate 	lpl_t		*lpl_cur;
23267c478bd9Sstevel@tonic-gate 	lpl_t		*lpl_root;
23277c478bd9Sstevel@tonic-gate 	cpupart_t	*cp;
23287c478bd9Sstevel@tonic-gate 
23297c478bd9Sstevel@tonic-gate 	if (levels != 2)
23307c478bd9Sstevel@tonic-gate 		return (0);
23317c478bd9Sstevel@tonic-gate 
23327c478bd9Sstevel@tonic-gate 	/* called w/ cpus paused - grab no locks! */
23337c478bd9Sstevel@tonic-gate 	ASSERT(MUTEX_HELD(&cpu_lock) || curthread->t_preempt > 0 ||
23347c478bd9Sstevel@tonic-gate 	    !lgrp_initialized);
23357c478bd9Sstevel@tonic-gate 
23367c478bd9Sstevel@tonic-gate 	cp = cp_list_head;
23377c478bd9Sstevel@tonic-gate 	do {
23387c478bd9Sstevel@tonic-gate 		lpl_root = &cp->cp_lgrploads[lgrp_root->lgrp_id];
23397c478bd9Sstevel@tonic-gate 		ASSERT(LGRP_EXISTS(lgrp_root) && (lpl_root->lpl_ncpu > 0));
23407c478bd9Sstevel@tonic-gate 
23417c478bd9Sstevel@tonic-gate 		for (i = 0; i <= lgrp_alloc_max; i++) {
23427c478bd9Sstevel@tonic-gate 			lgrp_cur = lgrp_table[i];
23437c478bd9Sstevel@tonic-gate 			lpl_cur = &cp->cp_lgrploads[i];
23447c478bd9Sstevel@tonic-gate 
23457c478bd9Sstevel@tonic-gate 			if ((lgrp_cur == lgrp_root) ||
23467c478bd9Sstevel@tonic-gate 			    (!LGRP_EXISTS(lgrp_cur) &&
23477c478bd9Sstevel@tonic-gate 			    (lpl_cur->lpl_ncpu == 0)))
23487c478bd9Sstevel@tonic-gate 				continue;
23497c478bd9Sstevel@tonic-gate 
23507c478bd9Sstevel@tonic-gate 			if (!LGRP_EXISTS(lgrp_cur) && (lpl_cur->lpl_ncpu > 0)) {
23517c478bd9Sstevel@tonic-gate 				/*
23527c478bd9Sstevel@tonic-gate 				 * this should be a deleted intermediate, so
23537c478bd9Sstevel@tonic-gate 				 * clear it
23547c478bd9Sstevel@tonic-gate 				 */
23557c478bd9Sstevel@tonic-gate 				lpl_clear(lpl_cur);
23567c478bd9Sstevel@tonic-gate 			} else if ((lpl_cur->lpl_nrset == 1) &&
23577c478bd9Sstevel@tonic-gate 			    (lpl_cur->lpl_rset[0] == lpl_cur) &&
23587c478bd9Sstevel@tonic-gate 			    ((lpl_cur->lpl_parent->lpl_ncpu == 0) ||
23597c478bd9Sstevel@tonic-gate 			    (!LGRP_EXISTS(lpl_cur->lpl_parent->lpl_lgrp)))) {
23607c478bd9Sstevel@tonic-gate 				/*
23617c478bd9Sstevel@tonic-gate 				 * this is a leaf whose parent was deleted, or
23627c478bd9Sstevel@tonic-gate 				 * whose parent had their lgrp deleted.  (And
23637c478bd9Sstevel@tonic-gate 				 * whose parent will soon be deleted).  Point
23647c478bd9Sstevel@tonic-gate 				 * this guy back to the root lpl.
23657c478bd9Sstevel@tonic-gate 				 */
23667c478bd9Sstevel@tonic-gate 				lpl_cur->lpl_parent = lpl_root;
23677c478bd9Sstevel@tonic-gate 				lpl_rset_add(lpl_root, lpl_cur);
23687c478bd9Sstevel@tonic-gate 			}
23697c478bd9Sstevel@tonic-gate 
23707c478bd9Sstevel@tonic-gate 		}
23717c478bd9Sstevel@tonic-gate 
23727c478bd9Sstevel@tonic-gate 		/*
23737c478bd9Sstevel@tonic-gate 		 * Now that we're done, make sure the count on the root lpl is
23747c478bd9Sstevel@tonic-gate 		 * correct, and update the hints of the children for the sake of
23757c478bd9Sstevel@tonic-gate 		 * thoroughness
23767c478bd9Sstevel@tonic-gate 		 */
23777c478bd9Sstevel@tonic-gate 		for (i = sum = 0; i < lpl_root->lpl_nrset; i++) {
23787c478bd9Sstevel@tonic-gate 			sum += lpl_root->lpl_rset[i]->lpl_ncpu;
23797c478bd9Sstevel@tonic-gate 		}
23807c478bd9Sstevel@tonic-gate 		lpl_root->lpl_ncpu = sum;
23817c478bd9Sstevel@tonic-gate 		lpl_child_update(lpl_root, cp);
23827c478bd9Sstevel@tonic-gate 
23837c478bd9Sstevel@tonic-gate 		cp = cp->cp_next;
23847c478bd9Sstevel@tonic-gate 	} while (cp != cp_list_head);
23857c478bd9Sstevel@tonic-gate 
23867c478bd9Sstevel@tonic-gate 	return (levels);
23877c478bd9Sstevel@tonic-gate }
23887c478bd9Sstevel@tonic-gate 
23897c478bd9Sstevel@tonic-gate /*
23907c478bd9Sstevel@tonic-gate  * Insert a lpl into the resource hierarchy and create any additional lpls that
23917c478bd9Sstevel@tonic-gate  * are necessary to represent the varying states of locality for the cpu
23927c478bd9Sstevel@tonic-gate  * resoruces newly added to the partition.
23937c478bd9Sstevel@tonic-gate  *
23947c478bd9Sstevel@tonic-gate  * This routine is clever enough that it can correctly add resources from the
23957c478bd9Sstevel@tonic-gate  * new leaf into both direct and indirect resource sets in the hierarchy.  (Ie,
23967c478bd9Sstevel@tonic-gate  * those for which the lpl is a leaf as opposed to simply a named equally local
23977c478bd9Sstevel@tonic-gate  * resource).  The one special case that needs additional processing is when a
23987c478bd9Sstevel@tonic-gate  * new intermediate lpl is introduced.  Since the main loop only traverses
23997c478bd9Sstevel@tonic-gate  * looking to add the leaf resource where it does not yet exist, additional work
24007c478bd9Sstevel@tonic-gate  * is necessary to add other leaf resources that may need to exist in the newly
24017c478bd9Sstevel@tonic-gate  * created intermediate.  This is performed by the second inner loop, and is
24027c478bd9Sstevel@tonic-gate  * only done when the check for more than one overlapping resource succeeds.
24037c478bd9Sstevel@tonic-gate  */
24047c478bd9Sstevel@tonic-gate 
24057c478bd9Sstevel@tonic-gate void
24067c478bd9Sstevel@tonic-gate lpl_leaf_insert(lpl_t *lpl_leaf, cpupart_t *cpupart)
24077c478bd9Sstevel@tonic-gate {
24087c478bd9Sstevel@tonic-gate 	int		i;
24097c478bd9Sstevel@tonic-gate 	int		j;
24107c478bd9Sstevel@tonic-gate 	int		hint;
24117c478bd9Sstevel@tonic-gate 	int		rset_num_intersect;
24127c478bd9Sstevel@tonic-gate 	lgrp_t		*lgrp_cur;
24137c478bd9Sstevel@tonic-gate 	lpl_t		*lpl_cur;
24147c478bd9Sstevel@tonic-gate 	lpl_t		*lpl_parent;
2415ab761399Sesaxe 	lgrp_id_t	parent_id;
24167c478bd9Sstevel@tonic-gate 	klgrpset_t	rset_intersect; /* resources in cpupart and lgrp */
24177c478bd9Sstevel@tonic-gate 
24187c478bd9Sstevel@tonic-gate 	for (i = 0; i <= lgrp_alloc_max; i++) {
24197c478bd9Sstevel@tonic-gate 		lgrp_cur = lgrp_table[i];
24207c478bd9Sstevel@tonic-gate 
24217c478bd9Sstevel@tonic-gate 		/*
24227c478bd9Sstevel@tonic-gate 		 * Don't insert if the lgrp isn't there, if the leaf isn't
24237c478bd9Sstevel@tonic-gate 		 * contained within the current lgrp, or if the current lgrp has
24247c478bd9Sstevel@tonic-gate 		 * no leaves in this partition
24257c478bd9Sstevel@tonic-gate 		 */
24267c478bd9Sstevel@tonic-gate 
24277c478bd9Sstevel@tonic-gate 		if (!LGRP_EXISTS(lgrp_cur) ||
24287c478bd9Sstevel@tonic-gate 		    !klgrpset_ismember(lgrp_cur->lgrp_set[LGRP_RSRC_CPU],
24297c478bd9Sstevel@tonic-gate 		    lpl_leaf->lpl_lgrpid) ||
24307c478bd9Sstevel@tonic-gate 		    !klgrpset_intersects(lgrp_cur->lgrp_leaves,
24317c478bd9Sstevel@tonic-gate 		    cpupart->cp_lgrpset))
24327c478bd9Sstevel@tonic-gate 			continue;
24337c478bd9Sstevel@tonic-gate 
24347c478bd9Sstevel@tonic-gate 		lpl_cur = &cpupart->cp_lgrploads[lgrp_cur->lgrp_id];
24357c478bd9Sstevel@tonic-gate 		if (lgrp_cur->lgrp_parent != NULL) {
24367c478bd9Sstevel@tonic-gate 			/* if lgrp has a parent, assign it properly */
24377c478bd9Sstevel@tonic-gate 			parent_id = lgrp_cur->lgrp_parent->lgrp_id;
24387c478bd9Sstevel@tonic-gate 			lpl_parent = &cpupart->cp_lgrploads[parent_id];
24397c478bd9Sstevel@tonic-gate 		} else {
24407c478bd9Sstevel@tonic-gate 			/* if not, make sure parent ptr gets set to null */
24417c478bd9Sstevel@tonic-gate 			lpl_parent = NULL;
24427c478bd9Sstevel@tonic-gate 		}
24437c478bd9Sstevel@tonic-gate 
24447c478bd9Sstevel@tonic-gate 		if (lpl_cur == lpl_leaf) {
24457c478bd9Sstevel@tonic-gate 			/*
24467c478bd9Sstevel@tonic-gate 			 * Almost all leaf state was initialized elsewhere.  The
24477c478bd9Sstevel@tonic-gate 			 * only thing left to do is to set the parent.
24487c478bd9Sstevel@tonic-gate 			 */
24497c478bd9Sstevel@tonic-gate 			lpl_cur->lpl_parent = lpl_parent;
24507c478bd9Sstevel@tonic-gate 			continue;
24517c478bd9Sstevel@tonic-gate 		}
24527c478bd9Sstevel@tonic-gate 
24537c478bd9Sstevel@tonic-gate 		/*
24547c478bd9Sstevel@tonic-gate 		 * Initialize intermediate lpl
24557c478bd9Sstevel@tonic-gate 		 * Save this lpl's hint though. Since we're changing this
24567c478bd9Sstevel@tonic-gate 		 * lpl's resources, we need to update the hint in this lpl's
24577c478bd9Sstevel@tonic-gate 		 * children, but the hint in this lpl is unaffected and
24587c478bd9Sstevel@tonic-gate 		 * should be preserved.
24597c478bd9Sstevel@tonic-gate 		 */
24607c478bd9Sstevel@tonic-gate 		hint = lpl_cur->lpl_hint;
24617c478bd9Sstevel@tonic-gate 
24627c478bd9Sstevel@tonic-gate 		lpl_clear(lpl_cur);
24637c478bd9Sstevel@tonic-gate 		lpl_init(lpl_cur, lpl_leaf, lgrp_cur);
24647c478bd9Sstevel@tonic-gate 
24657c478bd9Sstevel@tonic-gate 		lpl_cur->lpl_hint = hint;
24667c478bd9Sstevel@tonic-gate 		lpl_cur->lpl_parent = lpl_parent;
24677c478bd9Sstevel@tonic-gate 
24687c478bd9Sstevel@tonic-gate 		/* does new lpl need to be populated with other resources? */
24697c478bd9Sstevel@tonic-gate 		rset_intersect =
24707c478bd9Sstevel@tonic-gate 		    klgrpset_intersects(lgrp_cur->lgrp_set[LGRP_RSRC_CPU],
24717c478bd9Sstevel@tonic-gate 			cpupart->cp_lgrpset);
24727c478bd9Sstevel@tonic-gate 		klgrpset_nlgrps(rset_intersect, rset_num_intersect);
24737c478bd9Sstevel@tonic-gate 
24747c478bd9Sstevel@tonic-gate 		if (rset_num_intersect > 1) {
24757c478bd9Sstevel@tonic-gate 			/*
24767c478bd9Sstevel@tonic-gate 			 * If so, figure out what lpls have resources that
24777c478bd9Sstevel@tonic-gate 			 * intersect this one, and add them.
24787c478bd9Sstevel@tonic-gate 			 */
24797c478bd9Sstevel@tonic-gate 			for (j = 0; j <= lgrp_alloc_max; j++) {
24807c478bd9Sstevel@tonic-gate 				lgrp_t	*lgrp_cand;	/* candidate lgrp */
24817c478bd9Sstevel@tonic-gate 				lpl_t	*lpl_cand;	/* candidate lpl */
24827c478bd9Sstevel@tonic-gate 
24837c478bd9Sstevel@tonic-gate 				lgrp_cand = lgrp_table[j];
24847c478bd9Sstevel@tonic-gate 				if (!LGRP_EXISTS(lgrp_cand) ||
24857c478bd9Sstevel@tonic-gate 				    !klgrpset_ismember(rset_intersect,
24867c478bd9Sstevel@tonic-gate 					lgrp_cand->lgrp_id))
24877c478bd9Sstevel@tonic-gate 					continue;
24887c478bd9Sstevel@tonic-gate 				lpl_cand =
24897c478bd9Sstevel@tonic-gate 				    &cpupart->cp_lgrploads[lgrp_cand->lgrp_id];
24907c478bd9Sstevel@tonic-gate 				lpl_rset_add(lpl_cur, lpl_cand);
24917c478bd9Sstevel@tonic-gate 			}
24927c478bd9Sstevel@tonic-gate 		}
24937c478bd9Sstevel@tonic-gate 		/*
24947c478bd9Sstevel@tonic-gate 		 * This lpl's rset has changed. Update the hint in it's
24957c478bd9Sstevel@tonic-gate 		 * children.
24967c478bd9Sstevel@tonic-gate 		 */
24977c478bd9Sstevel@tonic-gate 		lpl_child_update(lpl_cur, cpupart);
24987c478bd9Sstevel@tonic-gate 	}
24997c478bd9Sstevel@tonic-gate }
25007c478bd9Sstevel@tonic-gate 
25017c478bd9Sstevel@tonic-gate /*
25027c478bd9Sstevel@tonic-gate  * remove a lpl from the hierarchy of resources, clearing its state when
25037c478bd9Sstevel@tonic-gate  * finished.  If the lpls at the intermediate levels of the hierarchy have no
25047c478bd9Sstevel@tonic-gate  * remaining resources, or no longer name a leaf resource in the cpu-partition,
25057c478bd9Sstevel@tonic-gate  * delete them as well.
25067c478bd9Sstevel@tonic-gate  */
25077c478bd9Sstevel@tonic-gate 
25087c478bd9Sstevel@tonic-gate void
25097c478bd9Sstevel@tonic-gate lpl_leaf_remove(lpl_t *lpl_leaf, cpupart_t *cpupart)
25107c478bd9Sstevel@tonic-gate {
25117c478bd9Sstevel@tonic-gate 	int		i;
25127c478bd9Sstevel@tonic-gate 	lgrp_t		*lgrp_cur;
25137c478bd9Sstevel@tonic-gate 	lpl_t		*lpl_cur;
25147c478bd9Sstevel@tonic-gate 	klgrpset_t	leaf_intersect;	/* intersection of leaves */
25157c478bd9Sstevel@tonic-gate 
25167c478bd9Sstevel@tonic-gate 	for (i = 0; i <= lgrp_alloc_max; i++) {
25177c478bd9Sstevel@tonic-gate 		lgrp_cur = lgrp_table[i];
25187c478bd9Sstevel@tonic-gate 
25197c478bd9Sstevel@tonic-gate 		/*
25207c478bd9Sstevel@tonic-gate 		 * Don't attempt to remove from lgrps that aren't there, that
25217c478bd9Sstevel@tonic-gate 		 * don't contain our leaf, or from the leaf itself. (We do that
25227c478bd9Sstevel@tonic-gate 		 * later)
25237c478bd9Sstevel@tonic-gate 		 */
25247c478bd9Sstevel@tonic-gate 
25257c478bd9Sstevel@tonic-gate 		if (!LGRP_EXISTS(lgrp_cur))
25267c478bd9Sstevel@tonic-gate 			continue;
25277c478bd9Sstevel@tonic-gate 
25287c478bd9Sstevel@tonic-gate 		lpl_cur = &cpupart->cp_lgrploads[lgrp_cur->lgrp_id];
25297c478bd9Sstevel@tonic-gate 
25307c478bd9Sstevel@tonic-gate 		if (!klgrpset_ismember(lgrp_cur->lgrp_set[LGRP_RSRC_CPU],
25317c478bd9Sstevel@tonic-gate 		    lpl_leaf->lpl_lgrpid) ||
25327c478bd9Sstevel@tonic-gate 		    (lpl_cur == lpl_leaf)) {
25337c478bd9Sstevel@tonic-gate 			continue;
25347c478bd9Sstevel@tonic-gate 		}
25357c478bd9Sstevel@tonic-gate 
25367c478bd9Sstevel@tonic-gate 		/*
25377c478bd9Sstevel@tonic-gate 		 * This is a slightly sleazy simplification in that we have
25387c478bd9Sstevel@tonic-gate 		 * already marked the cp_lgrpset as no longer containing the
25397c478bd9Sstevel@tonic-gate 		 * leaf we've deleted.  Any lpls that pass the above checks
25407c478bd9Sstevel@tonic-gate 		 * based upon lgrp membership but not necessarily cpu-part
25417c478bd9Sstevel@tonic-gate 		 * membership also get cleared by the checks below.  Currently
25427c478bd9Sstevel@tonic-gate 		 * this is harmless, as the lpls should be empty anyway.
25437c478bd9Sstevel@tonic-gate 		 *
25447c478bd9Sstevel@tonic-gate 		 * In particular, we want to preserve lpls that have additional
25457c478bd9Sstevel@tonic-gate 		 * leaf resources, even though we don't yet have a processor
25467c478bd9Sstevel@tonic-gate 		 * architecture that represents resources this way.
25477c478bd9Sstevel@tonic-gate 		 */
25487c478bd9Sstevel@tonic-gate 
25497c478bd9Sstevel@tonic-gate 		leaf_intersect = klgrpset_intersects(lgrp_cur->lgrp_leaves,
25507c478bd9Sstevel@tonic-gate 		    cpupart->cp_lgrpset);
25517c478bd9Sstevel@tonic-gate 
25527c478bd9Sstevel@tonic-gate 		lpl_rset_del(lpl_cur, lpl_leaf);
25537c478bd9Sstevel@tonic-gate 		if ((lpl_cur->lpl_nrset == 0) || (!leaf_intersect)) {
25547c478bd9Sstevel@tonic-gate 			lpl_clear(lpl_cur);
25557c478bd9Sstevel@tonic-gate 		} else {
25567c478bd9Sstevel@tonic-gate 			/*
25577c478bd9Sstevel@tonic-gate 			 * Update this lpl's children
25587c478bd9Sstevel@tonic-gate 			 */
25597c478bd9Sstevel@tonic-gate 			lpl_child_update(lpl_cur, cpupart);
25607c478bd9Sstevel@tonic-gate 		}
25617c478bd9Sstevel@tonic-gate 	}
25627c478bd9Sstevel@tonic-gate 	lpl_clear(lpl_leaf);
25637c478bd9Sstevel@tonic-gate }
25647c478bd9Sstevel@tonic-gate 
25657c478bd9Sstevel@tonic-gate /*
25667c478bd9Sstevel@tonic-gate  * add a cpu to a partition in terms of lgrp load avg bookeeping
25677c478bd9Sstevel@tonic-gate  *
25687c478bd9Sstevel@tonic-gate  * The lpl (cpu partition load average information) is now arranged in a
25697c478bd9Sstevel@tonic-gate  * hierarchical fashion whereby resources that are closest, ie. most local, to
25707c478bd9Sstevel@tonic-gate  * the cpu in question are considered to be leaves in a tree of resources.
25717c478bd9Sstevel@tonic-gate  * There are two general cases for cpu additon:
25727c478bd9Sstevel@tonic-gate  *
25737c478bd9Sstevel@tonic-gate  * 1. A lpl structure that contains resources already in the hierarchy tree.
25747c478bd9Sstevel@tonic-gate  * In this case, all of the associated lpl relationships have been defined, and
25757c478bd9Sstevel@tonic-gate  * all that is necessary is that we link the new cpu into the per-lpl list of
25767c478bd9Sstevel@tonic-gate  * cpus, and increment the ncpu count of all places where this cpu resource will
25777c478bd9Sstevel@tonic-gate  * be accounted for.  lpl_cpu_adjcnt updates the cpu count, and the cpu pointer
25787c478bd9Sstevel@tonic-gate  * pushing is accomplished by this routine.
25797c478bd9Sstevel@tonic-gate  *
25807c478bd9Sstevel@tonic-gate  * 2. The lpl to contain the resources in this cpu-partition for this lgrp does
25817c478bd9Sstevel@tonic-gate  * not exist yet.  In this case, it is necessary to build the leaf lpl, and
25827c478bd9Sstevel@tonic-gate  * construct the hierarchy of state necessary to name it's more distant
25837c478bd9Sstevel@tonic-gate  * resources, if they should exist.  The leaf structure is initialized by this
25847c478bd9Sstevel@tonic-gate  * routine, as is the cpu-partition state for the lgrp membership.  This routine
25857c478bd9Sstevel@tonic-gate  * also calls lpl_leaf_insert() which inserts the named lpl into the hierarchy
25867c478bd9Sstevel@tonic-gate  * and builds all of the "ancestoral" state necessary to identify resources at
25877c478bd9Sstevel@tonic-gate  * differing levels of locality.
25887c478bd9Sstevel@tonic-gate  */
25897c478bd9Sstevel@tonic-gate void
25907c478bd9Sstevel@tonic-gate lgrp_part_add_cpu(cpu_t *cp, lgrp_id_t lgrpid)
25917c478bd9Sstevel@tonic-gate {
25927c478bd9Sstevel@tonic-gate 	cpupart_t	*cpupart;
25937c478bd9Sstevel@tonic-gate 	lgrp_t		*lgrp_leaf;
25947c478bd9Sstevel@tonic-gate 	lpl_t		*lpl_leaf;
25957c478bd9Sstevel@tonic-gate 
25967c478bd9Sstevel@tonic-gate 	/* called sometimes w/ cpus paused - grab no locks */
25977c478bd9Sstevel@tonic-gate 	ASSERT(MUTEX_HELD(&cpu_lock) || !lgrp_initialized);
25987c478bd9Sstevel@tonic-gate 
25997c478bd9Sstevel@tonic-gate 	cpupart = cp->cpu_part;
26007c478bd9Sstevel@tonic-gate 	lgrp_leaf = lgrp_table[lgrpid];
26017c478bd9Sstevel@tonic-gate 
26027c478bd9Sstevel@tonic-gate 	/* don't add non-existent lgrp */
26037c478bd9Sstevel@tonic-gate 	ASSERT(LGRP_EXISTS(lgrp_leaf));
26047c478bd9Sstevel@tonic-gate 	lpl_leaf = &cpupart->cp_lgrploads[lgrpid];
26057c478bd9Sstevel@tonic-gate 	cp->cpu_lpl = lpl_leaf;
26067c478bd9Sstevel@tonic-gate 
26077c478bd9Sstevel@tonic-gate 	/* only leaf lpls contain cpus */
26087c478bd9Sstevel@tonic-gate 
26097c478bd9Sstevel@tonic-gate 	if (lpl_leaf->lpl_ncpu++ == 0) {
26107c478bd9Sstevel@tonic-gate 		lpl_init(lpl_leaf, lpl_leaf, lgrp_leaf);
26117c478bd9Sstevel@tonic-gate 		klgrpset_add(cpupart->cp_lgrpset, lgrpid);
26127c478bd9Sstevel@tonic-gate 		lpl_leaf_insert(lpl_leaf, cpupart);
26137c478bd9Sstevel@tonic-gate 	} else {
26147c478bd9Sstevel@tonic-gate 		/*
26157c478bd9Sstevel@tonic-gate 		 * the lpl should already exist in the parent, so just update
26167c478bd9Sstevel@tonic-gate 		 * the count of available CPUs
26177c478bd9Sstevel@tonic-gate 		 */
26187c478bd9Sstevel@tonic-gate 		lpl_cpu_adjcnt(LPL_INCREMENT, cp);
26197c478bd9Sstevel@tonic-gate 	}
26207c478bd9Sstevel@tonic-gate 
26217c478bd9Sstevel@tonic-gate 	/* link cpu into list of cpus in lpl */
26227c478bd9Sstevel@tonic-gate 
26237c478bd9Sstevel@tonic-gate 	if (lpl_leaf->lpl_cpus) {
26247c478bd9Sstevel@tonic-gate 		cp->cpu_next_lpl = lpl_leaf->lpl_cpus;
26257c478bd9Sstevel@tonic-gate 		cp->cpu_prev_lpl = lpl_leaf->lpl_cpus->cpu_prev_lpl;
26267c478bd9Sstevel@tonic-gate 		lpl_leaf->lpl_cpus->cpu_prev_lpl->cpu_next_lpl = cp;
26277c478bd9Sstevel@tonic-gate 		lpl_leaf->lpl_cpus->cpu_prev_lpl = cp;
26287c478bd9Sstevel@tonic-gate 	} else {
26297c478bd9Sstevel@tonic-gate 		/*
26307c478bd9Sstevel@tonic-gate 		 * We increment ncpu immediately after we create a new leaf
26317c478bd9Sstevel@tonic-gate 		 * lpl, so assert that ncpu == 1 for the case where we don't
26327c478bd9Sstevel@tonic-gate 		 * have any cpu pointers yet.
26337c478bd9Sstevel@tonic-gate 		 */
26347c478bd9Sstevel@tonic-gate 		ASSERT(lpl_leaf->lpl_ncpu == 1);
26357c478bd9Sstevel@tonic-gate 		lpl_leaf->lpl_cpus = cp->cpu_next_lpl = cp->cpu_prev_lpl = cp;
26367c478bd9Sstevel@tonic-gate 	}
26377c478bd9Sstevel@tonic-gate 
26387c478bd9Sstevel@tonic-gate }
26397c478bd9Sstevel@tonic-gate 
26407c478bd9Sstevel@tonic-gate 
26417c478bd9Sstevel@tonic-gate /*
26427c478bd9Sstevel@tonic-gate  * remove a cpu from a partition in terms of lgrp load avg bookeeping
26437c478bd9Sstevel@tonic-gate  *
26447c478bd9Sstevel@tonic-gate  * The lpl (cpu partition load average information) is now arranged in a
26457c478bd9Sstevel@tonic-gate  * hierarchical fashion whereby resources that are closest, ie. most local, to
26467c478bd9Sstevel@tonic-gate  * the cpu in question are considered to be leaves in a tree of resources.
26477c478bd9Sstevel@tonic-gate  * There are two removal cases in question:
26487c478bd9Sstevel@tonic-gate  *
26497c478bd9Sstevel@tonic-gate  * 1. Removal of the resource in the leaf leaves other resources remaining in
26507c478bd9Sstevel@tonic-gate  * that leaf.  (Another cpu still exists at this level of locality).  In this
26517c478bd9Sstevel@tonic-gate  * case, the count of available cpus is decremented in all assocated lpls by
26527c478bd9Sstevel@tonic-gate  * calling lpl_adj_cpucnt(), and the pointer to the removed cpu is pruned
26537c478bd9Sstevel@tonic-gate  * from the per-cpu lpl list.
26547c478bd9Sstevel@tonic-gate  *
26557c478bd9Sstevel@tonic-gate  * 2. Removal of the resource results in the lpl containing no resources.  (It's
26567c478bd9Sstevel@tonic-gate  * empty)  In this case, all of what has occurred for the first step must take
26577c478bd9Sstevel@tonic-gate  * place; however, additionally we must remove the lpl structure itself, prune
26587c478bd9Sstevel@tonic-gate  * out any stranded lpls that do not directly name a leaf resource, and mark the
26597c478bd9Sstevel@tonic-gate  * cpu partition in question as no longer containing resources from the lgrp of
26607c478bd9Sstevel@tonic-gate  * the lpl that has been delted.  Cpu-partition changes are handled by this
26617c478bd9Sstevel@tonic-gate  * method, but the lpl_leaf_remove function deals with the details of pruning
26627c478bd9Sstevel@tonic-gate  * out the empty lpl and any of its orphaned direct ancestors.
26637c478bd9Sstevel@tonic-gate  */
26647c478bd9Sstevel@tonic-gate void
26657c478bd9Sstevel@tonic-gate lgrp_part_del_cpu(cpu_t *cp)
26667c478bd9Sstevel@tonic-gate {
26677c478bd9Sstevel@tonic-gate 	lpl_t		*lpl;
26687c478bd9Sstevel@tonic-gate 	lpl_t		*leaf_lpl;
26697c478bd9Sstevel@tonic-gate 	lgrp_t		*lgrp_leaf;
26707c478bd9Sstevel@tonic-gate 
26717c478bd9Sstevel@tonic-gate 	/* called sometimes w/ cpus paused - grab no locks */
26727c478bd9Sstevel@tonic-gate 
26737c478bd9Sstevel@tonic-gate 	ASSERT(MUTEX_HELD(&cpu_lock) || !lgrp_initialized);
26747c478bd9Sstevel@tonic-gate 
26757c478bd9Sstevel@tonic-gate 	lpl = leaf_lpl = cp->cpu_lpl;
26767c478bd9Sstevel@tonic-gate 	lgrp_leaf = leaf_lpl->lpl_lgrp;
26777c478bd9Sstevel@tonic-gate 
26787c478bd9Sstevel@tonic-gate 	/* don't delete a leaf that isn't there */
26797c478bd9Sstevel@tonic-gate 	ASSERT(LGRP_EXISTS(lgrp_leaf));
26807c478bd9Sstevel@tonic-gate 
26817c478bd9Sstevel@tonic-gate 	/* no double-deletes */
26827c478bd9Sstevel@tonic-gate 	ASSERT(lpl->lpl_ncpu);
26837c478bd9Sstevel@tonic-gate 	if (--lpl->lpl_ncpu == 0) {
26847c478bd9Sstevel@tonic-gate 		/*
26857c478bd9Sstevel@tonic-gate 		 * This was the last cpu in this lgroup for this partition,
26867c478bd9Sstevel@tonic-gate 		 * clear its bit in the partition's lgroup bitmask
26877c478bd9Sstevel@tonic-gate 		 */
26887c478bd9Sstevel@tonic-gate 		klgrpset_del(cp->cpu_part->cp_lgrpset, lpl->lpl_lgrpid);
26897c478bd9Sstevel@tonic-gate 
26907c478bd9Sstevel@tonic-gate 		/* eliminate remaning lpl link pointers in cpu, lpl */
26917c478bd9Sstevel@tonic-gate 		lpl->lpl_cpus = cp->cpu_next_lpl = cp->cpu_prev_lpl = NULL;
26927c478bd9Sstevel@tonic-gate 
26937c478bd9Sstevel@tonic-gate 		lpl_leaf_remove(leaf_lpl, cp->cpu_part);
26947c478bd9Sstevel@tonic-gate 	} else {
26957c478bd9Sstevel@tonic-gate 
26967c478bd9Sstevel@tonic-gate 		/* unlink cpu from lists of cpus in lpl */
26977c478bd9Sstevel@tonic-gate 		cp->cpu_prev_lpl->cpu_next_lpl = cp->cpu_next_lpl;
26987c478bd9Sstevel@tonic-gate 		cp->cpu_next_lpl->cpu_prev_lpl = cp->cpu_prev_lpl;
26997c478bd9Sstevel@tonic-gate 		if (lpl->lpl_cpus == cp) {
27007c478bd9Sstevel@tonic-gate 			lpl->lpl_cpus = cp->cpu_next_lpl;
27017c478bd9Sstevel@tonic-gate 		}
27027c478bd9Sstevel@tonic-gate 
27037c478bd9Sstevel@tonic-gate 		/*
27047c478bd9Sstevel@tonic-gate 		 * Update the cpu count in the lpls associated with parent
27057c478bd9Sstevel@tonic-gate 		 * lgroups.
27067c478bd9Sstevel@tonic-gate 		 */
27077c478bd9Sstevel@tonic-gate 		lpl_cpu_adjcnt(LPL_DECREMENT, cp);
27087c478bd9Sstevel@tonic-gate 
27097c478bd9Sstevel@tonic-gate 	}
27107c478bd9Sstevel@tonic-gate 	/* clear cpu's lpl ptr when we're all done */
27117c478bd9Sstevel@tonic-gate 	cp->cpu_lpl = NULL;
27127c478bd9Sstevel@tonic-gate }
27137c478bd9Sstevel@tonic-gate 
27147c478bd9Sstevel@tonic-gate /*
27157c478bd9Sstevel@tonic-gate  * Recompute load average for the specified partition/lgrp fragment.
27167c478bd9Sstevel@tonic-gate  *
27177c478bd9Sstevel@tonic-gate  * We rely on the fact that this routine is called from the clock thread
27187c478bd9Sstevel@tonic-gate  * at a point before the clock thread can block (i.e. before its first
27197c478bd9Sstevel@tonic-gate  * lock request).  Since the clock thread can not be preempted (since it
27207c478bd9Sstevel@tonic-gate  * runs at highest priority), we know that cpu partitions can not change
27217c478bd9Sstevel@tonic-gate  * (since doing so would require either the repartition requester or the
27227c478bd9Sstevel@tonic-gate  * cpu_pause thread to run on this cpu), so we can update the cpu's load
27237c478bd9Sstevel@tonic-gate  * without grabbing cpu_lock.
27247c478bd9Sstevel@tonic-gate  */
27257c478bd9Sstevel@tonic-gate void
27267c478bd9Sstevel@tonic-gate lgrp_loadavg(lpl_t *lpl, uint_t nrcpus, int ageflag)
27277c478bd9Sstevel@tonic-gate {
27287c478bd9Sstevel@tonic-gate 	uint_t		ncpu;
27297c478bd9Sstevel@tonic-gate 	int64_t		old, new, f;
27307c478bd9Sstevel@tonic-gate 
27317c478bd9Sstevel@tonic-gate 	/*
27327c478bd9Sstevel@tonic-gate 	 * 1 - exp(-1/(20 * ncpu)) << 13 = 400 for 1 cpu...
27337c478bd9Sstevel@tonic-gate 	 */
27347c478bd9Sstevel@tonic-gate 	static short expval[] = {
27357c478bd9Sstevel@tonic-gate 	    0, 3196, 1618, 1083,
27367c478bd9Sstevel@tonic-gate 	    814, 652, 543, 466,
27377c478bd9Sstevel@tonic-gate 	    408, 363, 326, 297,
27387c478bd9Sstevel@tonic-gate 	    272, 251, 233, 218,
27397c478bd9Sstevel@tonic-gate 	    204, 192, 181, 172,
27407c478bd9Sstevel@tonic-gate 	    163, 155, 148, 142,
27417c478bd9Sstevel@tonic-gate 	    136, 130, 125, 121,
27427c478bd9Sstevel@tonic-gate 	    116, 112, 109, 105
27437c478bd9Sstevel@tonic-gate 	};
27447c478bd9Sstevel@tonic-gate 
27457c478bd9Sstevel@tonic-gate 	/* ASSERT (called from clock level) */
27467c478bd9Sstevel@tonic-gate 
27477c478bd9Sstevel@tonic-gate 	if ((lpl == NULL) ||	/* we're booting - this is easiest for now */
27487c478bd9Sstevel@tonic-gate 	    ((ncpu = lpl->lpl_ncpu) == 0)) {
27497c478bd9Sstevel@tonic-gate 		return;
27507c478bd9Sstevel@tonic-gate 	}
27517c478bd9Sstevel@tonic-gate 
27527c478bd9Sstevel@tonic-gate 	for (;;) {
27537c478bd9Sstevel@tonic-gate 
27547c478bd9Sstevel@tonic-gate 		if (ncpu >= sizeof (expval) / sizeof (expval[0]))
27557c478bd9Sstevel@tonic-gate 			f = expval[1]/ncpu; /* good approx. for large ncpu */
27567c478bd9Sstevel@tonic-gate 		else
27577c478bd9Sstevel@tonic-gate 			f = expval[ncpu];
27587c478bd9Sstevel@tonic-gate 
27597c478bd9Sstevel@tonic-gate 		/*
27607c478bd9Sstevel@tonic-gate 		 * Modify the load average atomically to avoid losing
27617c478bd9Sstevel@tonic-gate 		 * anticipatory load updates (see lgrp_move_thread()).
27627c478bd9Sstevel@tonic-gate 		 */
27637c478bd9Sstevel@tonic-gate 		if (ageflag) {
27647c478bd9Sstevel@tonic-gate 			/*
27657c478bd9Sstevel@tonic-gate 			 * We're supposed to both update and age the load.
27667c478bd9Sstevel@tonic-gate 			 * This happens 10 times/sec. per cpu.  We do a
27677c478bd9Sstevel@tonic-gate 			 * little hoop-jumping to avoid integer overflow.
27687c478bd9Sstevel@tonic-gate 			 */
27697c478bd9Sstevel@tonic-gate 			int64_t		q, r;
27707c478bd9Sstevel@tonic-gate 
27717c478bd9Sstevel@tonic-gate 			do {
27727c478bd9Sstevel@tonic-gate 				old = new = lpl->lpl_loadavg;
27737c478bd9Sstevel@tonic-gate 				q = (old  >> 16) << 7;
27747c478bd9Sstevel@tonic-gate 				r = (old  & 0xffff) << 7;
27757c478bd9Sstevel@tonic-gate 				new += ((long long)(nrcpus - q) * f -
27767c478bd9Sstevel@tonic-gate 				    ((r * f) >> 16)) >> 7;
27777c478bd9Sstevel@tonic-gate 
27787c478bd9Sstevel@tonic-gate 				/*
27797c478bd9Sstevel@tonic-gate 				 * Check for overflow
27807c478bd9Sstevel@tonic-gate 				 */
27817c478bd9Sstevel@tonic-gate 				if (new > LGRP_LOADAVG_MAX)
27827c478bd9Sstevel@tonic-gate 					new = LGRP_LOADAVG_MAX;
27837c478bd9Sstevel@tonic-gate 				else if (new < 0)
27847c478bd9Sstevel@tonic-gate 					new = 0;
27857c478bd9Sstevel@tonic-gate 			} while (cas32((lgrp_load_t *)&lpl->lpl_loadavg, old,
27867c478bd9Sstevel@tonic-gate 			    new) != old);
27877c478bd9Sstevel@tonic-gate 		} else {
27887c478bd9Sstevel@tonic-gate 			/*
27897c478bd9Sstevel@tonic-gate 			 * We're supposed to update the load, but not age it.
27907c478bd9Sstevel@tonic-gate 			 * This option is used to update the load (which either
27917c478bd9Sstevel@tonic-gate 			 * has already been aged in this 1/10 sec. interval or
27927c478bd9Sstevel@tonic-gate 			 * soon will be) to account for a remotely executing
27937c478bd9Sstevel@tonic-gate 			 * thread.
27947c478bd9Sstevel@tonic-gate 			 */
27957c478bd9Sstevel@tonic-gate 			do {
27967c478bd9Sstevel@tonic-gate 				old = new = lpl->lpl_loadavg;
27977c478bd9Sstevel@tonic-gate 				new += f;
27987c478bd9Sstevel@tonic-gate 				/*
27997c478bd9Sstevel@tonic-gate 				 * Check for overflow
28007c478bd9Sstevel@tonic-gate 				 * Underflow not possible here
28017c478bd9Sstevel@tonic-gate 				 */
28027c478bd9Sstevel@tonic-gate 				if (new < old)
28037c478bd9Sstevel@tonic-gate 					new = LGRP_LOADAVG_MAX;
28047c478bd9Sstevel@tonic-gate 			} while (cas32((lgrp_load_t *)&lpl->lpl_loadavg, old,
28057c478bd9Sstevel@tonic-gate 			    new) != old);
28067c478bd9Sstevel@tonic-gate 		}
28077c478bd9Sstevel@tonic-gate 
28087c478bd9Sstevel@tonic-gate 		/*
28097c478bd9Sstevel@tonic-gate 		 * Do the same for this lpl's parent
28107c478bd9Sstevel@tonic-gate 		 */
28117c478bd9Sstevel@tonic-gate 		if ((lpl = lpl->lpl_parent) == NULL)
28127c478bd9Sstevel@tonic-gate 			break;
28137c478bd9Sstevel@tonic-gate 		ncpu = lpl->lpl_ncpu;
28147c478bd9Sstevel@tonic-gate 	}
28157c478bd9Sstevel@tonic-gate }
28167c478bd9Sstevel@tonic-gate 
28177c478bd9Sstevel@tonic-gate /*
28187c478bd9Sstevel@tonic-gate  * Initialize lpl topology in the target based on topology currently present in
28197c478bd9Sstevel@tonic-gate  * lpl_bootstrap.
28207c478bd9Sstevel@tonic-gate  *
28217c478bd9Sstevel@tonic-gate  * lpl_topo_bootstrap is only called once from cpupart_initialize_default() to
28227c478bd9Sstevel@tonic-gate  * initialize cp_default list of lpls. Up to this point all topology operations
28237c478bd9Sstevel@tonic-gate  * were performed using lpl_bootstrap. Now cp_default has its own list of lpls
28247c478bd9Sstevel@tonic-gate  * and all subsequent lpl operations should use it instead of lpl_bootstrap. The
28257c478bd9Sstevel@tonic-gate  * `target' points to the list of lpls in cp_default and `size' is the size of
28267c478bd9Sstevel@tonic-gate  * this list.
28277c478bd9Sstevel@tonic-gate  *
28287c478bd9Sstevel@tonic-gate  * This function walks the lpl topology in lpl_bootstrap and does for things:
28297c478bd9Sstevel@tonic-gate  *
28307c478bd9Sstevel@tonic-gate  * 1) Copies all fields from lpl_bootstrap to the target.
28317c478bd9Sstevel@tonic-gate  *
28327c478bd9Sstevel@tonic-gate  * 2) Sets CPU0 lpl pointer to the correct element of the target list.
28337c478bd9Sstevel@tonic-gate  *
28347c478bd9Sstevel@tonic-gate  * 3) Updates lpl_parent pointers to point to the lpls in the target list
28357c478bd9Sstevel@tonic-gate  *    instead of lpl_bootstrap.
28367c478bd9Sstevel@tonic-gate  *
28377c478bd9Sstevel@tonic-gate  * 4) Updates pointers in the resource list of the target to point to the lpls
28387c478bd9Sstevel@tonic-gate  *    in the target list instead of lpl_bootstrap.
28397c478bd9Sstevel@tonic-gate  *
28407c478bd9Sstevel@tonic-gate  * After lpl_topo_bootstrap() completes, target contains the same information
28417c478bd9Sstevel@tonic-gate  * that would be present there if it were used during boot instead of
28427c478bd9Sstevel@tonic-gate  * lpl_bootstrap. There is no need in information in lpl_bootstrap after this
28437c478bd9Sstevel@tonic-gate  * and it is bzeroed.
28447c478bd9Sstevel@tonic-gate  */
28457c478bd9Sstevel@tonic-gate void
28467c478bd9Sstevel@tonic-gate lpl_topo_bootstrap(lpl_t *target, int size)
28477c478bd9Sstevel@tonic-gate {
28487c478bd9Sstevel@tonic-gate 	lpl_t	*lpl = lpl_bootstrap;
28497c478bd9Sstevel@tonic-gate 	lpl_t	*target_lpl = target;
28507c478bd9Sstevel@tonic-gate 	int	howmany;
28517c478bd9Sstevel@tonic-gate 	int	id;
28527c478bd9Sstevel@tonic-gate 	int	i;
28537c478bd9Sstevel@tonic-gate 
28547c478bd9Sstevel@tonic-gate 	/*
28557c478bd9Sstevel@tonic-gate 	 * The only target that should be passed here is cp_default lpl list.
28567c478bd9Sstevel@tonic-gate 	 */
28577c478bd9Sstevel@tonic-gate 	ASSERT(target == cp_default.cp_lgrploads);
28587c478bd9Sstevel@tonic-gate 	ASSERT(size == cp_default.cp_nlgrploads);
28597c478bd9Sstevel@tonic-gate 	ASSERT(!lgrp_topo_initialized);
28607c478bd9Sstevel@tonic-gate 	ASSERT(ncpus == 1);
28617c478bd9Sstevel@tonic-gate 
28627c478bd9Sstevel@tonic-gate 	howmany = MIN(LPL_BOOTSTRAP_SIZE, size);
28637c478bd9Sstevel@tonic-gate 	for (i = 0; i < howmany; i++, lpl++, target_lpl++) {
28647c478bd9Sstevel@tonic-gate 		/*
28657c478bd9Sstevel@tonic-gate 		 * Copy all fields from lpl.
28667c478bd9Sstevel@tonic-gate 		 */
28677c478bd9Sstevel@tonic-gate 
28687c478bd9Sstevel@tonic-gate 		*target_lpl = *lpl;
28697c478bd9Sstevel@tonic-gate 
28707c478bd9Sstevel@tonic-gate 		/*
28717c478bd9Sstevel@tonic-gate 		 * Substitute CPU0 lpl pointer with one relative to target.
28727c478bd9Sstevel@tonic-gate 		 */
28737c478bd9Sstevel@tonic-gate 		if (lpl->lpl_cpus == CPU) {
28747c478bd9Sstevel@tonic-gate 			ASSERT(CPU->cpu_lpl == lpl);
28757c478bd9Sstevel@tonic-gate 			CPU->cpu_lpl = target_lpl;
28767c478bd9Sstevel@tonic-gate 		}
28777c478bd9Sstevel@tonic-gate 
28787c478bd9Sstevel@tonic-gate 		/*
28797c478bd9Sstevel@tonic-gate 		 * Substitute parent information with parent relative to target.
28807c478bd9Sstevel@tonic-gate 		 */
28817c478bd9Sstevel@tonic-gate 		if (lpl->lpl_parent != NULL)
28827c478bd9Sstevel@tonic-gate 			target_lpl->lpl_parent = (lpl_t *)
28837c478bd9Sstevel@tonic-gate 			    (((uintptr_t)lpl->lpl_parent -
28847c478bd9Sstevel@tonic-gate 				(uintptr_t)lpl_bootstrap) +
28857c478bd9Sstevel@tonic-gate 				(uintptr_t)target);
28867c478bd9Sstevel@tonic-gate 
28877c478bd9Sstevel@tonic-gate 		/*
28887c478bd9Sstevel@tonic-gate 		 * Walk over resource set substituting pointers relative to
28897c478bd9Sstevel@tonic-gate 		 * lpl_bootstrap to pointers relative to target.
28907c478bd9Sstevel@tonic-gate 		 */
28917c478bd9Sstevel@tonic-gate 		ASSERT(lpl->lpl_nrset <= 1);
28927c478bd9Sstevel@tonic-gate 
28937c478bd9Sstevel@tonic-gate 		for (id = 0; id < lpl->lpl_nrset; id++) {
28947c478bd9Sstevel@tonic-gate 			if (lpl->lpl_rset[id] != NULL) {
28957c478bd9Sstevel@tonic-gate 				target_lpl->lpl_rset[id] =
28967c478bd9Sstevel@tonic-gate 				    (lpl_t *)
28977c478bd9Sstevel@tonic-gate 				    (((uintptr_t)lpl->lpl_rset[id] -
28987c478bd9Sstevel@tonic-gate 					(uintptr_t)lpl_bootstrap) +
28997c478bd9Sstevel@tonic-gate 					(uintptr_t)target);
29007c478bd9Sstevel@tonic-gate 			}
29017c478bd9Sstevel@tonic-gate 		}
29027c478bd9Sstevel@tonic-gate 	}
29037c478bd9Sstevel@tonic-gate 
29047c478bd9Sstevel@tonic-gate 	/*
29057c478bd9Sstevel@tonic-gate 	 * Topology information in lpl_bootstrap is no longer needed.
29067c478bd9Sstevel@tonic-gate 	 */
29077c478bd9Sstevel@tonic-gate 	bzero(lpl_bootstrap_list, sizeof (lpl_bootstrap_list));
29087c478bd9Sstevel@tonic-gate }
29097c478bd9Sstevel@tonic-gate 
29107c478bd9Sstevel@tonic-gate /*
29117c478bd9Sstevel@tonic-gate  * If the lowest load among the lgroups a process' threads are currently
29127c478bd9Sstevel@tonic-gate  * spread across is greater than lgrp_expand_proc_thresh, we'll consider
29137c478bd9Sstevel@tonic-gate  * expanding the process to a new lgroup.
29147c478bd9Sstevel@tonic-gate  */
29157c478bd9Sstevel@tonic-gate #define	LGRP_EXPAND_PROC_THRESH_DEFAULT 62250
29167c478bd9Sstevel@tonic-gate lgrp_load_t	lgrp_expand_proc_thresh = LGRP_EXPAND_PROC_THRESH_DEFAULT;
29177c478bd9Sstevel@tonic-gate 
29187c478bd9Sstevel@tonic-gate #define	LGRP_EXPAND_PROC_THRESH(ncpu) \
29197c478bd9Sstevel@tonic-gate 	((lgrp_expand_proc_thresh) / (ncpu))
29207c478bd9Sstevel@tonic-gate 
29217c478bd9Sstevel@tonic-gate /*
29227c478bd9Sstevel@tonic-gate  * A process will be expanded to a new lgroup only if the difference between
29237c478bd9Sstevel@tonic-gate  * the lowest load on the lgroups the process' thread's are currently spread
29247c478bd9Sstevel@tonic-gate  * across and the lowest load on the other lgroups in the process' partition
29257c478bd9Sstevel@tonic-gate  * is greater than lgrp_expand_proc_diff.
29267c478bd9Sstevel@tonic-gate  */
29277c478bd9Sstevel@tonic-gate #define	LGRP_EXPAND_PROC_DIFF_DEFAULT 60000
29287c478bd9Sstevel@tonic-gate lgrp_load_t	lgrp_expand_proc_diff = LGRP_EXPAND_PROC_DIFF_DEFAULT;
29297c478bd9Sstevel@tonic-gate 
29307c478bd9Sstevel@tonic-gate #define	LGRP_EXPAND_PROC_DIFF(ncpu) \
29317c478bd9Sstevel@tonic-gate 	((lgrp_expand_proc_diff) / (ncpu))
29327c478bd9Sstevel@tonic-gate 
29337c478bd9Sstevel@tonic-gate /*
29347c478bd9Sstevel@tonic-gate  * The loadavg tolerance accounts for "noise" inherent in the load, which may
29357c478bd9Sstevel@tonic-gate  * be present due to impreciseness of the load average decay algorithm.
29367c478bd9Sstevel@tonic-gate  *
29377c478bd9Sstevel@tonic-gate  * The default tolerance is lgrp_loadavg_max_effect. Note that the tunable
29387c478bd9Sstevel@tonic-gate  * tolerance is scaled by the number of cpus in the lgroup just like
29397c478bd9Sstevel@tonic-gate  * lgrp_loadavg_max_effect. For example, if lgrp_loadavg_tolerance = 0x10000,
29407c478bd9Sstevel@tonic-gate  * and ncpu = 4, then lgrp_choose will consider differences in lgroup loads
29417c478bd9Sstevel@tonic-gate  * of: 0x10000 / 4 => 0x4000 or greater to be significant.
29427c478bd9Sstevel@tonic-gate  */
29437c478bd9Sstevel@tonic-gate uint32_t	lgrp_loadavg_tolerance = LGRP_LOADAVG_THREAD_MAX;
29447c478bd9Sstevel@tonic-gate #define	LGRP_LOADAVG_TOLERANCE(ncpu)	\
29457c478bd9Sstevel@tonic-gate 	((lgrp_loadavg_tolerance) / ncpu)
29467c478bd9Sstevel@tonic-gate 
29477c478bd9Sstevel@tonic-gate /*
29487c478bd9Sstevel@tonic-gate  * lgrp_choose() will choose root lgroup as home when lowest lgroup load
29497c478bd9Sstevel@tonic-gate  * average is above this threshold
29507c478bd9Sstevel@tonic-gate  */
29517c478bd9Sstevel@tonic-gate uint32_t	lgrp_load_thresh = UINT32_MAX;
29527c478bd9Sstevel@tonic-gate 
29537c478bd9Sstevel@tonic-gate /*
29547c478bd9Sstevel@tonic-gate  * lgrp_choose() will try to skip any lgroups with less memory
29557c478bd9Sstevel@tonic-gate  * than this free when choosing a home lgroup
29567c478bd9Sstevel@tonic-gate  */
29577c478bd9Sstevel@tonic-gate pgcnt_t	lgrp_mem_free_thresh = 0;
29587c478bd9Sstevel@tonic-gate 
29597c478bd9Sstevel@tonic-gate /*
29607c478bd9Sstevel@tonic-gate  * When choosing between similarly loaded lgroups, lgrp_choose() will pick
29617c478bd9Sstevel@tonic-gate  * one based on one of the following policies:
29627c478bd9Sstevel@tonic-gate  * - Random selection
29637c478bd9Sstevel@tonic-gate  * - Pseudo round robin placement
29647c478bd9Sstevel@tonic-gate  * - Longest time since a thread was last placed
29657c478bd9Sstevel@tonic-gate  */
29667c478bd9Sstevel@tonic-gate #define	LGRP_CHOOSE_RANDOM	1
29677c478bd9Sstevel@tonic-gate #define	LGRP_CHOOSE_RR		2
29687c478bd9Sstevel@tonic-gate #define	LGRP_CHOOSE_TIME	3
29697c478bd9Sstevel@tonic-gate 
29707c478bd9Sstevel@tonic-gate int	lgrp_choose_policy = LGRP_CHOOSE_TIME;
29717c478bd9Sstevel@tonic-gate 
29727c478bd9Sstevel@tonic-gate /*
29737c478bd9Sstevel@tonic-gate  * Choose a suitable leaf lgroup for a kthread.  The kthread is assumed not to
29747c478bd9Sstevel@tonic-gate  * be bound to a CPU or processor set.
29757c478bd9Sstevel@tonic-gate  *
29767c478bd9Sstevel@tonic-gate  * Arguments:
29777c478bd9Sstevel@tonic-gate  *	t		The thread
29787c478bd9Sstevel@tonic-gate  *	cpupart		The partition the thread belongs to.
29797c478bd9Sstevel@tonic-gate  *
29807c478bd9Sstevel@tonic-gate  * NOTE: Should at least be called with the cpu_lock held, kernel preemption
29817c478bd9Sstevel@tonic-gate  *	 disabled, or thread_lock held (at splhigh) to protect against the CPU
29827c478bd9Sstevel@tonic-gate  *	 partitions changing out from under us and assumes that given thread is
29837c478bd9Sstevel@tonic-gate  *	 protected.  Also, called sometimes w/ cpus paused or kernel preemption
29847c478bd9Sstevel@tonic-gate  *	 disabled, so don't grab any locks because we should never block under
29857c478bd9Sstevel@tonic-gate  *	 those conditions.
29867c478bd9Sstevel@tonic-gate  */
29877c478bd9Sstevel@tonic-gate lpl_t *
29887c478bd9Sstevel@tonic-gate lgrp_choose(kthread_t *t, cpupart_t *cpupart)
29897c478bd9Sstevel@tonic-gate {
29907c478bd9Sstevel@tonic-gate 	lgrp_load_t	bestload, bestrload;
29917c478bd9Sstevel@tonic-gate 	int		lgrpid_offset, lgrp_count;
29927c478bd9Sstevel@tonic-gate 	lgrp_id_t	lgrpid, lgrpid_start;
29937c478bd9Sstevel@tonic-gate 	lpl_t		*lpl, *bestlpl, *bestrlpl;
29947c478bd9Sstevel@tonic-gate 	klgrpset_t	lgrpset;
29957c478bd9Sstevel@tonic-gate 	proc_t		*p;
29967c478bd9Sstevel@tonic-gate 
29977c478bd9Sstevel@tonic-gate 	ASSERT(t != NULL);
29987c478bd9Sstevel@tonic-gate 	ASSERT(MUTEX_HELD(&cpu_lock) || curthread->t_preempt > 0 ||
29997c478bd9Sstevel@tonic-gate 	    THREAD_LOCK_HELD(t));
30007c478bd9Sstevel@tonic-gate 	ASSERT(cpupart != NULL);
30017c478bd9Sstevel@tonic-gate 
30027c478bd9Sstevel@tonic-gate 	p = t->t_procp;
30037c478bd9Sstevel@tonic-gate 
30047c478bd9Sstevel@tonic-gate 	/* A process should always be in an active partition */
30057c478bd9Sstevel@tonic-gate 	ASSERT(!klgrpset_isempty(cpupart->cp_lgrpset));
30067c478bd9Sstevel@tonic-gate 
30077c478bd9Sstevel@tonic-gate 	bestlpl = bestrlpl = NULL;
30087c478bd9Sstevel@tonic-gate 	bestload = bestrload = LGRP_LOADAVG_MAX;
30097c478bd9Sstevel@tonic-gate 	lgrpset = cpupart->cp_lgrpset;
30107c478bd9Sstevel@tonic-gate 
30117c478bd9Sstevel@tonic-gate 	switch (lgrp_choose_policy) {
30127c478bd9Sstevel@tonic-gate 	case LGRP_CHOOSE_RR:
30137c478bd9Sstevel@tonic-gate 		lgrpid = cpupart->cp_lgrp_hint;
30147c478bd9Sstevel@tonic-gate 		do {
30157c478bd9Sstevel@tonic-gate 			if (++lgrpid > lgrp_alloc_max)
30167c478bd9Sstevel@tonic-gate 				lgrpid = 0;
30177c478bd9Sstevel@tonic-gate 		} while (!klgrpset_ismember(lgrpset, lgrpid));
30187c478bd9Sstevel@tonic-gate 
30197c478bd9Sstevel@tonic-gate 		break;
30207c478bd9Sstevel@tonic-gate 	default:
30217c478bd9Sstevel@tonic-gate 	case LGRP_CHOOSE_TIME:
30227c478bd9Sstevel@tonic-gate 	case LGRP_CHOOSE_RANDOM:
30237c478bd9Sstevel@tonic-gate 		klgrpset_nlgrps(lgrpset, lgrp_count);
30247c478bd9Sstevel@tonic-gate 		lgrpid_offset =
30257c478bd9Sstevel@tonic-gate 		    (((ushort_t)(gethrtime() >> 4)) % lgrp_count) + 1;
30267c478bd9Sstevel@tonic-gate 		for (lgrpid = 0; ; lgrpid++) {
30277c478bd9Sstevel@tonic-gate 			if (klgrpset_ismember(lgrpset, lgrpid)) {
30287c478bd9Sstevel@tonic-gate 				if (--lgrpid_offset == 0)
30297c478bd9Sstevel@tonic-gate 					break;
30307c478bd9Sstevel@tonic-gate 			}
30317c478bd9Sstevel@tonic-gate 		}
30327c478bd9Sstevel@tonic-gate 		break;
30337c478bd9Sstevel@tonic-gate 	}
30347c478bd9Sstevel@tonic-gate 
30357c478bd9Sstevel@tonic-gate 	lgrpid_start = lgrpid;
30367c478bd9Sstevel@tonic-gate 
30377c478bd9Sstevel@tonic-gate 	DTRACE_PROBE2(lgrp_choose_start, lgrp_id_t, lgrpid_start,
30387c478bd9Sstevel@tonic-gate 	    lgrp_id_t, cpupart->cp_lgrp_hint);
30397c478bd9Sstevel@tonic-gate 
30407c478bd9Sstevel@tonic-gate 	/*
30417c478bd9Sstevel@tonic-gate 	 * Use lgroup affinities (if any) to choose best lgroup
30427c478bd9Sstevel@tonic-gate 	 *
30437c478bd9Sstevel@tonic-gate 	 * NOTE: Assumes that thread is protected from going away and its
30447c478bd9Sstevel@tonic-gate 	 *	 lgroup affinities won't change (ie. p_lock, or
30457c478bd9Sstevel@tonic-gate 	 *	 thread_lock() being held and/or CPUs paused)
30467c478bd9Sstevel@tonic-gate 	 */
30477c478bd9Sstevel@tonic-gate 	if (t->t_lgrp_affinity) {
304803400a71Sjjc 		lpl = lgrp_affinity_best(t, cpupart, lgrpid_start, B_FALSE);
30497c478bd9Sstevel@tonic-gate 		if (lpl != NULL)
30507c478bd9Sstevel@tonic-gate 			return (lpl);
30517c478bd9Sstevel@tonic-gate 	}
30527c478bd9Sstevel@tonic-gate 
30537c478bd9Sstevel@tonic-gate 	ASSERT(klgrpset_ismember(lgrpset, lgrpid_start));
30547c478bd9Sstevel@tonic-gate 
30557c478bd9Sstevel@tonic-gate 	do {
30567c478bd9Sstevel@tonic-gate 		pgcnt_t	npgs;
30577c478bd9Sstevel@tonic-gate 
30587c478bd9Sstevel@tonic-gate 		/*
30597c478bd9Sstevel@tonic-gate 		 * Skip any lgroups outside of thread's pset
30607c478bd9Sstevel@tonic-gate 		 */
30617c478bd9Sstevel@tonic-gate 		if (!klgrpset_ismember(lgrpset, lgrpid)) {
30627c478bd9Sstevel@tonic-gate 			if (++lgrpid > lgrp_alloc_max)
30637c478bd9Sstevel@tonic-gate 				lgrpid = 0;	/* wrap the search */
30647c478bd9Sstevel@tonic-gate 			continue;
30657c478bd9Sstevel@tonic-gate 		}
30667c478bd9Sstevel@tonic-gate 
30677c478bd9Sstevel@tonic-gate 		/*
30687c478bd9Sstevel@tonic-gate 		 * Skip any non-leaf lgroups
30697c478bd9Sstevel@tonic-gate 		 */
30707c478bd9Sstevel@tonic-gate 		if (lgrp_table[lgrpid]->lgrp_childcnt != 0)
30717c478bd9Sstevel@tonic-gate 			continue;
30727c478bd9Sstevel@tonic-gate 
30737c478bd9Sstevel@tonic-gate 		/*
30747c478bd9Sstevel@tonic-gate 		 * Skip any lgroups without enough free memory
30757c478bd9Sstevel@tonic-gate 		 * (when threshold set to nonzero positive value)
30767c478bd9Sstevel@tonic-gate 		 */
30777c478bd9Sstevel@tonic-gate 		if (lgrp_mem_free_thresh > 0) {
30787c478bd9Sstevel@tonic-gate 			npgs = lgrp_mem_size(lgrpid, LGRP_MEM_SIZE_FREE);
30797c478bd9Sstevel@tonic-gate 			if (npgs < lgrp_mem_free_thresh) {
30807c478bd9Sstevel@tonic-gate 				if (++lgrpid > lgrp_alloc_max)
30817c478bd9Sstevel@tonic-gate 					lgrpid = 0;	/* wrap the search */
30827c478bd9Sstevel@tonic-gate 				continue;
30837c478bd9Sstevel@tonic-gate 			}
30847c478bd9Sstevel@tonic-gate 		}
30857c478bd9Sstevel@tonic-gate 
30867c478bd9Sstevel@tonic-gate 		lpl = &cpupart->cp_lgrploads[lgrpid];
30877c478bd9Sstevel@tonic-gate 		if (klgrpset_isempty(p->p_lgrpset) ||
30887c478bd9Sstevel@tonic-gate 		    klgrpset_ismember(p->p_lgrpset, lgrpid)) {
30897c478bd9Sstevel@tonic-gate 			/*
30907c478bd9Sstevel@tonic-gate 			 * Either this is a new process or the process already
30917c478bd9Sstevel@tonic-gate 			 * has threads on this lgrp, so this is a preferred
30927c478bd9Sstevel@tonic-gate 			 * lgroup for the thread.
30937c478bd9Sstevel@tonic-gate 			 */
3094ab761399Sesaxe 			if (bestlpl == NULL ||
3095ab761399Sesaxe 			    lpl_pick(lpl, bestlpl)) {
30967c478bd9Sstevel@tonic-gate 				bestload = lpl->lpl_loadavg;
30977c478bd9Sstevel@tonic-gate 				bestlpl = lpl;
30987c478bd9Sstevel@tonic-gate 			}
30997c478bd9Sstevel@tonic-gate 		} else {
31007c478bd9Sstevel@tonic-gate 			/*
31017c478bd9Sstevel@tonic-gate 			 * The process doesn't have any threads on this lgrp,
31027c478bd9Sstevel@tonic-gate 			 * but we're willing to consider this lgrp if the load
31037c478bd9Sstevel@tonic-gate 			 * difference is big enough to justify splitting up
31047c478bd9Sstevel@tonic-gate 			 * the process' threads.
31057c478bd9Sstevel@tonic-gate 			 */
3106ab761399Sesaxe 			if (bestrlpl == NULL ||
3107ab761399Sesaxe 			    lpl_pick(lpl, bestrlpl)) {
31087c478bd9Sstevel@tonic-gate 				bestrload = lpl->lpl_loadavg;
31097c478bd9Sstevel@tonic-gate 				bestrlpl = lpl;
31107c478bd9Sstevel@tonic-gate 			}
31117c478bd9Sstevel@tonic-gate 		}
31127c478bd9Sstevel@tonic-gate 		if (++lgrpid > lgrp_alloc_max)
31137c478bd9Sstevel@tonic-gate 			lgrpid = 0;	/* wrap the search */
31147c478bd9Sstevel@tonic-gate 	} while (lgrpid != lgrpid_start);
31157c478bd9Sstevel@tonic-gate 
31167c478bd9Sstevel@tonic-gate 	/*
31177c478bd9Sstevel@tonic-gate 	 * Return root lgroup if threshold isn't set to maximum value and
31187c478bd9Sstevel@tonic-gate 	 * lowest lgroup load average more than a certain threshold
31197c478bd9Sstevel@tonic-gate 	 */
31207c478bd9Sstevel@tonic-gate 	if (lgrp_load_thresh != UINT32_MAX &&
31217c478bd9Sstevel@tonic-gate 	    bestload >= lgrp_load_thresh && bestrload >= lgrp_load_thresh)
31227c478bd9Sstevel@tonic-gate 		return (&cpupart->cp_lgrploads[lgrp_root->lgrp_id]);
31237c478bd9Sstevel@tonic-gate 
31247c478bd9Sstevel@tonic-gate 	/*
31257c478bd9Sstevel@tonic-gate 	 * If all the lgroups over which the thread's process is spread are
3126ab761399Sesaxe 	 * heavily loaded, or otherwise undesirable, we'll consider placing
3127ab761399Sesaxe 	 * the thread on one of the other leaf lgroups in the thread's
3128ab761399Sesaxe 	 * partition.
31297c478bd9Sstevel@tonic-gate 	 */
3130ab761399Sesaxe 	if ((bestlpl == NULL) ||
3131ab761399Sesaxe 	    ((bestload > LGRP_EXPAND_PROC_THRESH(bestlpl->lpl_ncpu)) &&
31327c478bd9Sstevel@tonic-gate 	    (bestrload < bestload) &&	/* paranoid about wraparound */
31337c478bd9Sstevel@tonic-gate 	    (bestrload + LGRP_EXPAND_PROC_DIFF(bestrlpl->lpl_ncpu) <
3134ab761399Sesaxe 	    bestload))) {
31357c478bd9Sstevel@tonic-gate 		bestlpl = bestrlpl;
31367c478bd9Sstevel@tonic-gate 	}
31377c478bd9Sstevel@tonic-gate 
3138ab761399Sesaxe 	if (bestlpl == NULL) {
3139ab761399Sesaxe 		/*
3140ab761399Sesaxe 		 * No lgroup looked particularly good, but we still
3141ab761399Sesaxe 		 * have to pick something. Go with the randomly selected
3142ab761399Sesaxe 		 * legal lgroup we started with above.
3143ab761399Sesaxe 		 */
3144ab761399Sesaxe 		bestlpl = &cpupart->cp_lgrploads[lgrpid_start];
3145ab761399Sesaxe 	}
3146ab761399Sesaxe 
31477c478bd9Sstevel@tonic-gate 	cpupart->cp_lgrp_hint = bestlpl->lpl_lgrpid;
31487c478bd9Sstevel@tonic-gate 	bestlpl->lpl_homed_time = gethrtime_unscaled();
31497c478bd9Sstevel@tonic-gate 
31507c478bd9Sstevel@tonic-gate 	ASSERT(bestlpl->lpl_ncpu > 0);
31517c478bd9Sstevel@tonic-gate 	return (bestlpl);
31527c478bd9Sstevel@tonic-gate }
31537c478bd9Sstevel@tonic-gate 
31547c478bd9Sstevel@tonic-gate /*
3155ab761399Sesaxe  * Decide if lpl1 is a better candidate than lpl2 for lgrp homing.
3156ab761399Sesaxe  * Returns non-zero if lpl1 is a better candidate, and 0 otherwise.
31577c478bd9Sstevel@tonic-gate  */
31587c478bd9Sstevel@tonic-gate static int
31597c478bd9Sstevel@tonic-gate lpl_pick(lpl_t *lpl1, lpl_t *lpl2)
31607c478bd9Sstevel@tonic-gate {
31617c478bd9Sstevel@tonic-gate 	lgrp_load_t	l1, l2;
31627c478bd9Sstevel@tonic-gate 	lgrp_load_t	tolerance = LGRP_LOADAVG_TOLERANCE(lpl1->lpl_ncpu);
31637c478bd9Sstevel@tonic-gate 
31647c478bd9Sstevel@tonic-gate 	l1 = lpl1->lpl_loadavg;
31657c478bd9Sstevel@tonic-gate 	l2 = lpl2->lpl_loadavg;
31667c478bd9Sstevel@tonic-gate 
31677c478bd9Sstevel@tonic-gate 	if ((l1 + tolerance < l2) && (l1 < l2)) {
31687c478bd9Sstevel@tonic-gate 		/* lpl1 is significantly less loaded than lpl2 */
31697c478bd9Sstevel@tonic-gate 		return (1);
31707c478bd9Sstevel@tonic-gate 	}
31717c478bd9Sstevel@tonic-gate 
31727c478bd9Sstevel@tonic-gate 	if (lgrp_choose_policy == LGRP_CHOOSE_TIME &&
31737c478bd9Sstevel@tonic-gate 	    l1 + tolerance >= l2 && l1 < l2 &&
31747c478bd9Sstevel@tonic-gate 	    lpl1->lpl_homed_time < lpl2->lpl_homed_time) {
31757c478bd9Sstevel@tonic-gate 		/*
31767c478bd9Sstevel@tonic-gate 		 * lpl1's load is within the tolerance of lpl2. We're
31777c478bd9Sstevel@tonic-gate 		 * willing to consider it be to better however if
31787c478bd9Sstevel@tonic-gate 		 * it has been longer since we last homed a thread there
31797c478bd9Sstevel@tonic-gate 		 */
31807c478bd9Sstevel@tonic-gate 		return (1);
31817c478bd9Sstevel@tonic-gate 	}
31827c478bd9Sstevel@tonic-gate 
31837c478bd9Sstevel@tonic-gate 	return (0);
31847c478bd9Sstevel@tonic-gate }
31857c478bd9Sstevel@tonic-gate 
31867c478bd9Sstevel@tonic-gate /*
3187*2cb27123Saguzovsk  * lgrp_trthr_moves counts the number of times main thread (t_tid = 1) of a
3188*2cb27123Saguzovsk  * process that uses text replication changed home lgrp. This info is used by
3189*2cb27123Saguzovsk  * segvn asyncronous thread to detect if it needs to recheck what lgrps
3190*2cb27123Saguzovsk  * should be used for text replication.
3191*2cb27123Saguzovsk  */
3192*2cb27123Saguzovsk static uint64_t lgrp_trthr_moves = 0;
3193*2cb27123Saguzovsk 
3194*2cb27123Saguzovsk uint64_t
3195*2cb27123Saguzovsk lgrp_get_trthr_migrations(void)
3196*2cb27123Saguzovsk {
3197*2cb27123Saguzovsk 	return (lgrp_trthr_moves);
3198*2cb27123Saguzovsk }
3199*2cb27123Saguzovsk 
3200*2cb27123Saguzovsk void
3201*2cb27123Saguzovsk lgrp_update_trthr_migrations(uint64_t incr)
3202*2cb27123Saguzovsk {
3203*2cb27123Saguzovsk 	atomic_add_64(&lgrp_trthr_moves, incr);
3204*2cb27123Saguzovsk }
3205*2cb27123Saguzovsk 
3206*2cb27123Saguzovsk /*
32077c478bd9Sstevel@tonic-gate  * An LWP is expected to be assigned to an lgroup for at least this long
32087c478bd9Sstevel@tonic-gate  * for its anticipatory load to be justified.  NOTE that this value should
32097c478bd9Sstevel@tonic-gate  * not be set extremely huge (say, larger than 100 years), to avoid problems
32107c478bd9Sstevel@tonic-gate  * with overflow in the calculation that uses it.
32117c478bd9Sstevel@tonic-gate  */
32127c478bd9Sstevel@tonic-gate #define	LGRP_MIN_NSEC	(NANOSEC / 10)		/* 1/10 of a second */
32137c478bd9Sstevel@tonic-gate hrtime_t lgrp_min_nsec = LGRP_MIN_NSEC;
32147c478bd9Sstevel@tonic-gate 
32157c478bd9Sstevel@tonic-gate /*
32167c478bd9Sstevel@tonic-gate  * Routine to change a thread's lgroup affiliation.  This routine updates
32177c478bd9Sstevel@tonic-gate  * the thread's kthread_t struct and its process' proc_t struct to note the
32187c478bd9Sstevel@tonic-gate  * thread's new lgroup affiliation, and its lgroup affinities.
32197c478bd9Sstevel@tonic-gate  *
32207c478bd9Sstevel@tonic-gate  * Note that this is the only routine that modifies a thread's t_lpl field,
32217c478bd9Sstevel@tonic-gate  * and that adds in or removes anticipatory load.
32227c478bd9Sstevel@tonic-gate  *
32237c478bd9Sstevel@tonic-gate  * If the thread is exiting, newlpl is NULL.
32247c478bd9Sstevel@tonic-gate  *
32257c478bd9Sstevel@tonic-gate  * Locking:
32267c478bd9Sstevel@tonic-gate  * The following lock must be held on entry:
32277c478bd9Sstevel@tonic-gate  *	cpu_lock, kpreempt_disable(), or thread_lock -- to assure t's new lgrp
32287c478bd9Sstevel@tonic-gate  *		doesn't get removed from t's partition
32297c478bd9Sstevel@tonic-gate  *
32307c478bd9Sstevel@tonic-gate  * This routine is not allowed to grab any locks, since it may be called
32317c478bd9Sstevel@tonic-gate  * with cpus paused (such as from cpu_offline).
32327c478bd9Sstevel@tonic-gate  */
32337c478bd9Sstevel@tonic-gate void
32347c478bd9Sstevel@tonic-gate lgrp_move_thread(kthread_t *t, lpl_t *newlpl, int do_lgrpset_delete)
32357c478bd9Sstevel@tonic-gate {
32367c478bd9Sstevel@tonic-gate 	proc_t		*p;
32377c478bd9Sstevel@tonic-gate 	lpl_t		*lpl, *oldlpl;
32387c478bd9Sstevel@tonic-gate 	lgrp_id_t	oldid;
32397c478bd9Sstevel@tonic-gate 	kthread_t	*tp;
32407c478bd9Sstevel@tonic-gate 	uint_t		ncpu;
32417c478bd9Sstevel@tonic-gate 	lgrp_load_t	old, new;
32427c478bd9Sstevel@tonic-gate 
32437c478bd9Sstevel@tonic-gate 	ASSERT(t);
32447c478bd9Sstevel@tonic-gate 	ASSERT(MUTEX_HELD(&cpu_lock) || curthread->t_preempt > 0 ||
32457c478bd9Sstevel@tonic-gate 	    THREAD_LOCK_HELD(t));
32467c478bd9Sstevel@tonic-gate 
32477c478bd9Sstevel@tonic-gate 	/*
32487c478bd9Sstevel@tonic-gate 	 * If not changing lpls, just return
32497c478bd9Sstevel@tonic-gate 	 */
32507c478bd9Sstevel@tonic-gate 	if ((oldlpl = t->t_lpl) == newlpl)
32517c478bd9Sstevel@tonic-gate 		return;
32527c478bd9Sstevel@tonic-gate 
32537c478bd9Sstevel@tonic-gate 	/*
32547c478bd9Sstevel@tonic-gate 	 * Make sure the thread's lwp hasn't exited (if so, this thread is now
32557c478bd9Sstevel@tonic-gate 	 * associated with process 0 rather than with its original process).
32567c478bd9Sstevel@tonic-gate 	 */
32577c478bd9Sstevel@tonic-gate 	if (t->t_proc_flag & TP_LWPEXIT) {
32587c478bd9Sstevel@tonic-gate 		if (newlpl != NULL) {
32597c478bd9Sstevel@tonic-gate 			t->t_lpl = newlpl;
32607c478bd9Sstevel@tonic-gate 		}
32617c478bd9Sstevel@tonic-gate 		return;
32627c478bd9Sstevel@tonic-gate 	}
32637c478bd9Sstevel@tonic-gate 
32647c478bd9Sstevel@tonic-gate 	p = ttoproc(t);
32657c478bd9Sstevel@tonic-gate 
32667c478bd9Sstevel@tonic-gate 	/*
32677c478bd9Sstevel@tonic-gate 	 * If the thread had a previous lgroup, update its process' p_lgrpset
32687c478bd9Sstevel@tonic-gate 	 * to account for it being moved from its old lgroup.
32697c478bd9Sstevel@tonic-gate 	 */
32707c478bd9Sstevel@tonic-gate 	if ((oldlpl != NULL) &&	/* thread had a previous lgroup */
32717c478bd9Sstevel@tonic-gate 	    (p->p_tlist != NULL)) {
32727c478bd9Sstevel@tonic-gate 		oldid = oldlpl->lpl_lgrpid;
32737c478bd9Sstevel@tonic-gate 
32747c478bd9Sstevel@tonic-gate 		if (newlpl != NULL)
32757c478bd9Sstevel@tonic-gate 			lgrp_stat_add(oldid, LGRP_NUM_MIGR, 1);
32767c478bd9Sstevel@tonic-gate 
32777c478bd9Sstevel@tonic-gate 		if ((do_lgrpset_delete) &&
32787c478bd9Sstevel@tonic-gate 		    (klgrpset_ismember(p->p_lgrpset, oldid))) {
32797c478bd9Sstevel@tonic-gate 			for (tp = p->p_tlist->t_forw; ; tp = tp->t_forw) {
32807c478bd9Sstevel@tonic-gate 				/*
32817c478bd9Sstevel@tonic-gate 				 * Check if a thread other than the thread
32827c478bd9Sstevel@tonic-gate 				 * that's moving is assigned to the same
32837c478bd9Sstevel@tonic-gate 				 * lgroup as the thread that's moving.  Note
32847c478bd9Sstevel@tonic-gate 				 * that we have to compare lgroup IDs, rather
32857c478bd9Sstevel@tonic-gate 				 * than simply comparing t_lpl's, since the
32867c478bd9Sstevel@tonic-gate 				 * threads may belong to different partitions
32877c478bd9Sstevel@tonic-gate 				 * but be assigned to the same lgroup.
32887c478bd9Sstevel@tonic-gate 				 */
32897c478bd9Sstevel@tonic-gate 				ASSERT(tp->t_lpl != NULL);
32907c478bd9Sstevel@tonic-gate 
32917c478bd9Sstevel@tonic-gate 				if ((tp != t) &&
32927c478bd9Sstevel@tonic-gate 				    (tp->t_lpl->lpl_lgrpid == oldid)) {
32937c478bd9Sstevel@tonic-gate 					/*
32947c478bd9Sstevel@tonic-gate 					 * Another thread is assigned to the
32957c478bd9Sstevel@tonic-gate 					 * same lgroup as the thread that's
32967c478bd9Sstevel@tonic-gate 					 * moving, p_lgrpset doesn't change.
32977c478bd9Sstevel@tonic-gate 					 */
32987c478bd9Sstevel@tonic-gate 					break;
32997c478bd9Sstevel@tonic-gate 				} else if (tp == p->p_tlist) {
33007c478bd9Sstevel@tonic-gate 					/*
33017c478bd9Sstevel@tonic-gate 					 * No other thread is assigned to the
33027c478bd9Sstevel@tonic-gate 					 * same lgroup as the exiting thread,
33037c478bd9Sstevel@tonic-gate 					 * clear the lgroup's bit in p_lgrpset.
33047c478bd9Sstevel@tonic-gate 					 */
33057c478bd9Sstevel@tonic-gate 					klgrpset_del(p->p_lgrpset, oldid);
33067c478bd9Sstevel@tonic-gate 					break;
33077c478bd9Sstevel@tonic-gate 				}
33087c478bd9Sstevel@tonic-gate 			}
33097c478bd9Sstevel@tonic-gate 		}
33107c478bd9Sstevel@tonic-gate 
33117c478bd9Sstevel@tonic-gate 		/*
33127c478bd9Sstevel@tonic-gate 		 * If this thread was assigned to its old lgroup for such a
33137c478bd9Sstevel@tonic-gate 		 * short amount of time that the anticipatory load that was
33147c478bd9Sstevel@tonic-gate 		 * added on its behalf has aged very little, remove that
33157c478bd9Sstevel@tonic-gate 		 * anticipatory load.
33167c478bd9Sstevel@tonic-gate 		 */
33177c478bd9Sstevel@tonic-gate 		if ((t->t_anttime + lgrp_min_nsec > gethrtime()) &&
33187c478bd9Sstevel@tonic-gate 		    ((ncpu = oldlpl->lpl_ncpu) > 0)) {
33197c478bd9Sstevel@tonic-gate 			lpl = oldlpl;
33207c478bd9Sstevel@tonic-gate 			for (;;) {
33217c478bd9Sstevel@tonic-gate 				do {
33227c478bd9Sstevel@tonic-gate 					old = new = lpl->lpl_loadavg;
33237c478bd9Sstevel@tonic-gate 					new -= LGRP_LOADAVG_MAX_EFFECT(ncpu);
33247c478bd9Sstevel@tonic-gate 					if (new > old) {
33257c478bd9Sstevel@tonic-gate 						/*
33267c478bd9Sstevel@tonic-gate 						 * this can happen if the load
33277c478bd9Sstevel@tonic-gate 						 * average was aged since we
33287c478bd9Sstevel@tonic-gate 						 * added in the anticipatory
33297c478bd9Sstevel@tonic-gate 						 * load
33307c478bd9Sstevel@tonic-gate 						 */
33317c478bd9Sstevel@tonic-gate 						new = 0;
33327c478bd9Sstevel@tonic-gate 					}
33337c478bd9Sstevel@tonic-gate 				} while (cas32(
33347c478bd9Sstevel@tonic-gate 					(lgrp_load_t *)&lpl->lpl_loadavg, old,
33357c478bd9Sstevel@tonic-gate 					    new) != old);
33367c478bd9Sstevel@tonic-gate 
33377c478bd9Sstevel@tonic-gate 				lpl = lpl->lpl_parent;
33387c478bd9Sstevel@tonic-gate 				if (lpl == NULL)
33397c478bd9Sstevel@tonic-gate 					break;
33407c478bd9Sstevel@tonic-gate 
33417c478bd9Sstevel@tonic-gate 				ncpu = lpl->lpl_ncpu;
33427c478bd9Sstevel@tonic-gate 				ASSERT(ncpu > 0);
33437c478bd9Sstevel@tonic-gate 			}
33447c478bd9Sstevel@tonic-gate 		}
33457c478bd9Sstevel@tonic-gate 	}
33467c478bd9Sstevel@tonic-gate 	/*
33477c478bd9Sstevel@tonic-gate 	 * If the thread has a new lgroup (i.e. it's not exiting), update its
33487c478bd9Sstevel@tonic-gate 	 * t_lpl and its process' p_lgrpset, and apply an anticipatory load
33497c478bd9Sstevel@tonic-gate 	 * to its new lgroup to account for its move to its new lgroup.
33507c478bd9Sstevel@tonic-gate 	 */
33517c478bd9Sstevel@tonic-gate 	if (newlpl != NULL) {
33527c478bd9Sstevel@tonic-gate 		/*
33537c478bd9Sstevel@tonic-gate 		 * This thread is moving to a new lgroup
33547c478bd9Sstevel@tonic-gate 		 */
33557c478bd9Sstevel@tonic-gate 		t->t_lpl = newlpl;
3356*2cb27123Saguzovsk 		if (t->t_tid == 1 && p->p_t1_lgrpid != newlpl->lpl_lgrpid) {
3357*2cb27123Saguzovsk 			p->p_t1_lgrpid = newlpl->lpl_lgrpid;
3358*2cb27123Saguzovsk 			membar_producer();
3359*2cb27123Saguzovsk 			if (p->p_tr_lgrpid != LGRP_NONE &&
3360*2cb27123Saguzovsk 			    p->p_tr_lgrpid != p->p_t1_lgrpid) {
3361*2cb27123Saguzovsk 				lgrp_update_trthr_migrations(1);
3362*2cb27123Saguzovsk 			}
3363*2cb27123Saguzovsk 		}
33647c478bd9Sstevel@tonic-gate 
33657c478bd9Sstevel@tonic-gate 		/*
33667c478bd9Sstevel@tonic-gate 		 * Reflect move in load average of new lgroup
33677c478bd9Sstevel@tonic-gate 		 * unless it is root lgroup
33687c478bd9Sstevel@tonic-gate 		 */
33697c478bd9Sstevel@tonic-gate 		if (lgrp_table[newlpl->lpl_lgrpid] == lgrp_root)
33707c478bd9Sstevel@tonic-gate 			return;
33717c478bd9Sstevel@tonic-gate 
33727c478bd9Sstevel@tonic-gate 		if (!klgrpset_ismember(p->p_lgrpset, newlpl->lpl_lgrpid)) {
33737c478bd9Sstevel@tonic-gate 			klgrpset_add(p->p_lgrpset, newlpl->lpl_lgrpid);
33747c478bd9Sstevel@tonic-gate 		}
33757c478bd9Sstevel@tonic-gate 
33767c478bd9Sstevel@tonic-gate 		/*
33777c478bd9Sstevel@tonic-gate 		 * It'll take some time for the load on the new lgroup
33787c478bd9Sstevel@tonic-gate 		 * to reflect this thread's placement on it.  We'd
33797c478bd9Sstevel@tonic-gate 		 * like not, however, to have all threads between now
33807c478bd9Sstevel@tonic-gate 		 * and then also piling on to this lgroup.  To avoid
33817c478bd9Sstevel@tonic-gate 		 * this pileup, we anticipate the load this thread
33827c478bd9Sstevel@tonic-gate 		 * will generate on its new lgroup.  The goal is to
33837c478bd9Sstevel@tonic-gate 		 * make the lgroup's load appear as though the thread
33847c478bd9Sstevel@tonic-gate 		 * had been there all along.  We're very conservative
33857c478bd9Sstevel@tonic-gate 		 * in calculating this anticipatory load, we assume
33867c478bd9Sstevel@tonic-gate 		 * the worst case case (100% CPU-bound thread).  This
33877c478bd9Sstevel@tonic-gate 		 * may be modified in the future to be more accurate.
33887c478bd9Sstevel@tonic-gate 		 */
33897c478bd9Sstevel@tonic-gate 		lpl = newlpl;
33907c478bd9Sstevel@tonic-gate 		for (;;) {
33917c478bd9Sstevel@tonic-gate 			ncpu = lpl->lpl_ncpu;
33927c478bd9Sstevel@tonic-gate 			ASSERT(ncpu > 0);
33937c478bd9Sstevel@tonic-gate 			do {
33947c478bd9Sstevel@tonic-gate 				old = new = lpl->lpl_loadavg;
33957c478bd9Sstevel@tonic-gate 				new += LGRP_LOADAVG_MAX_EFFECT(ncpu);
33967c478bd9Sstevel@tonic-gate 				/*
33977c478bd9Sstevel@tonic-gate 				 * Check for overflow
33987c478bd9Sstevel@tonic-gate 				 * Underflow not possible here
33997c478bd9Sstevel@tonic-gate 				 */
34007c478bd9Sstevel@tonic-gate 				if (new < old)
34017c478bd9Sstevel@tonic-gate 					new = UINT32_MAX;
34027c478bd9Sstevel@tonic-gate 			} while (cas32((lgrp_load_t *)&lpl->lpl_loadavg, old,
34037c478bd9Sstevel@tonic-gate 			    new) != old);
34047c478bd9Sstevel@tonic-gate 
34057c478bd9Sstevel@tonic-gate 			lpl = lpl->lpl_parent;
34067c478bd9Sstevel@tonic-gate 			if (lpl == NULL)
34077c478bd9Sstevel@tonic-gate 				break;
34087c478bd9Sstevel@tonic-gate 		}
34097c478bd9Sstevel@tonic-gate 		t->t_anttime = gethrtime();
34107c478bd9Sstevel@tonic-gate 	}
34117c478bd9Sstevel@tonic-gate }
34127c478bd9Sstevel@tonic-gate 
34137c478bd9Sstevel@tonic-gate /*
34147c478bd9Sstevel@tonic-gate  * Return lgroup memory allocation policy given advice from madvise(3C)
34157c478bd9Sstevel@tonic-gate  */
34167c478bd9Sstevel@tonic-gate lgrp_mem_policy_t
34177c478bd9Sstevel@tonic-gate lgrp_madv_to_policy(uchar_t advice, size_t size, int type)
34187c478bd9Sstevel@tonic-gate {
34197c478bd9Sstevel@tonic-gate 	switch (advice) {
34207c478bd9Sstevel@tonic-gate 	case MADV_ACCESS_LWP:
34217c478bd9Sstevel@tonic-gate 		return (LGRP_MEM_POLICY_NEXT);
34227c478bd9Sstevel@tonic-gate 	case MADV_ACCESS_MANY:
34237c478bd9Sstevel@tonic-gate 		return (LGRP_MEM_POLICY_RANDOM);
34247c478bd9Sstevel@tonic-gate 	default:
34257c478bd9Sstevel@tonic-gate 		return (lgrp_mem_policy_default(size, type));
34267c478bd9Sstevel@tonic-gate 	}
34277c478bd9Sstevel@tonic-gate }
34287c478bd9Sstevel@tonic-gate 
34297c478bd9Sstevel@tonic-gate /*
34307c478bd9Sstevel@tonic-gate  * Figure out default policy
34317c478bd9Sstevel@tonic-gate  */
34327c478bd9Sstevel@tonic-gate lgrp_mem_policy_t
34337c478bd9Sstevel@tonic-gate lgrp_mem_policy_default(size_t size, int type)
34347c478bd9Sstevel@tonic-gate {
34357c478bd9Sstevel@tonic-gate 	cpupart_t		*cp;
34367c478bd9Sstevel@tonic-gate 	lgrp_mem_policy_t	policy;
34377c478bd9Sstevel@tonic-gate 	size_t			pset_mem_size;
34387c478bd9Sstevel@tonic-gate 
34397c478bd9Sstevel@tonic-gate 	/*
34407c478bd9Sstevel@tonic-gate 	 * Randomly allocate memory across lgroups for shared memory
34417c478bd9Sstevel@tonic-gate 	 * beyond a certain threshold
34427c478bd9Sstevel@tonic-gate 	 */
34437c478bd9Sstevel@tonic-gate 	if ((type != MAP_SHARED && size > lgrp_privm_random_thresh) ||
34447c478bd9Sstevel@tonic-gate 	    (type == MAP_SHARED && size > lgrp_shm_random_thresh)) {
34457c478bd9Sstevel@tonic-gate 		/*
34467c478bd9Sstevel@tonic-gate 		 * Get total memory size of current thread's pset
34477c478bd9Sstevel@tonic-gate 		 */
34487c478bd9Sstevel@tonic-gate 		kpreempt_disable();
34497c478bd9Sstevel@tonic-gate 		cp = curthread->t_cpupart;
34507c478bd9Sstevel@tonic-gate 		klgrpset_totalsize(cp->cp_lgrpset, pset_mem_size);
34517c478bd9Sstevel@tonic-gate 		kpreempt_enable();
34527c478bd9Sstevel@tonic-gate 
34537c478bd9Sstevel@tonic-gate 		/*
34547c478bd9Sstevel@tonic-gate 		 * Choose policy to randomly allocate memory across
34557c478bd9Sstevel@tonic-gate 		 * lgroups in pset if it will fit and is not default
34567c478bd9Sstevel@tonic-gate 		 * partition.  Otherwise, allocate memory randomly
34577c478bd9Sstevel@tonic-gate 		 * across machine.
34587c478bd9Sstevel@tonic-gate 		 */
34597c478bd9Sstevel@tonic-gate 		if (lgrp_mem_pset_aware && size < pset_mem_size)
34607c478bd9Sstevel@tonic-gate 			policy = LGRP_MEM_POLICY_RANDOM_PSET;
34617c478bd9Sstevel@tonic-gate 		else
34627c478bd9Sstevel@tonic-gate 			policy = LGRP_MEM_POLICY_RANDOM;
34637c478bd9Sstevel@tonic-gate 	} else
34647c478bd9Sstevel@tonic-gate 		/*
34657c478bd9Sstevel@tonic-gate 		 * Apply default policy for private memory and
34667c478bd9Sstevel@tonic-gate 		 * shared memory under the respective random
34677c478bd9Sstevel@tonic-gate 		 * threshold.
34687c478bd9Sstevel@tonic-gate 		 */
34697c478bd9Sstevel@tonic-gate 		policy = lgrp_mem_default_policy;
34707c478bd9Sstevel@tonic-gate 
34717c478bd9Sstevel@tonic-gate 	return (policy);
34727c478bd9Sstevel@tonic-gate }
34737c478bd9Sstevel@tonic-gate 
34747c478bd9Sstevel@tonic-gate /*
34757c478bd9Sstevel@tonic-gate  * Get memory allocation policy for this segment
34767c478bd9Sstevel@tonic-gate  */
34777c478bd9Sstevel@tonic-gate lgrp_mem_policy_info_t *
34787c478bd9Sstevel@tonic-gate lgrp_mem_policy_get(struct seg *seg, caddr_t vaddr)
34797c478bd9Sstevel@tonic-gate {
34807c478bd9Sstevel@tonic-gate 	lgrp_mem_policy_info_t	*policy_info;
34817c478bd9Sstevel@tonic-gate 	extern struct seg_ops	segspt_ops;
34827c478bd9Sstevel@tonic-gate 	extern struct seg_ops	segspt_shmops;
34837c478bd9Sstevel@tonic-gate 
34847c478bd9Sstevel@tonic-gate 	/*
34857c478bd9Sstevel@tonic-gate 	 * This is for binary compatibility to protect against third party
34867c478bd9Sstevel@tonic-gate 	 * segment drivers which haven't recompiled to allow for
34877c478bd9Sstevel@tonic-gate 	 * SEGOP_GETPOLICY()
34887c478bd9Sstevel@tonic-gate 	 */
34897c478bd9Sstevel@tonic-gate 	if (seg->s_ops != &segvn_ops && seg->s_ops != &segspt_ops &&
34907c478bd9Sstevel@tonic-gate 	    seg->s_ops != &segspt_shmops)
34917c478bd9Sstevel@tonic-gate 		return (NULL);
34927c478bd9Sstevel@tonic-gate 
34937c478bd9Sstevel@tonic-gate 	policy_info = NULL;
34947c478bd9Sstevel@tonic-gate 	if (seg->s_ops->getpolicy != NULL)
34957c478bd9Sstevel@tonic-gate 		policy_info = SEGOP_GETPOLICY(seg, vaddr);
34967c478bd9Sstevel@tonic-gate 
34977c478bd9Sstevel@tonic-gate 	return (policy_info);
34987c478bd9Sstevel@tonic-gate }
34997c478bd9Sstevel@tonic-gate 
35007c478bd9Sstevel@tonic-gate /*
35017c478bd9Sstevel@tonic-gate  * Set policy for allocating private memory given desired policy, policy info,
35027c478bd9Sstevel@tonic-gate  * size in bytes of memory that policy is being applied.
35037c478bd9Sstevel@tonic-gate  * Return 0 if policy wasn't set already and 1 if policy was set already
35047c478bd9Sstevel@tonic-gate  */
35057c478bd9Sstevel@tonic-gate int
35067c478bd9Sstevel@tonic-gate lgrp_privm_policy_set(lgrp_mem_policy_t policy,
35077c478bd9Sstevel@tonic-gate     lgrp_mem_policy_info_t *policy_info, size_t size)
35087c478bd9Sstevel@tonic-gate {
35097c478bd9Sstevel@tonic-gate 
35107c478bd9Sstevel@tonic-gate 	ASSERT(policy_info != NULL);
35117c478bd9Sstevel@tonic-gate 
35127c478bd9Sstevel@tonic-gate 	if (policy == LGRP_MEM_POLICY_DEFAULT)
35137c478bd9Sstevel@tonic-gate 		policy = lgrp_mem_policy_default(size, MAP_PRIVATE);
35147c478bd9Sstevel@tonic-gate 
35157c478bd9Sstevel@tonic-gate 	/*
35167c478bd9Sstevel@tonic-gate 	 * Policy set already?
35177c478bd9Sstevel@tonic-gate 	 */
35187c478bd9Sstevel@tonic-gate 	if (policy == policy_info->mem_policy)
35197c478bd9Sstevel@tonic-gate 		return (1);
35207c478bd9Sstevel@tonic-gate 
35217c478bd9Sstevel@tonic-gate 	/*
35227c478bd9Sstevel@tonic-gate 	 * Set policy
35237c478bd9Sstevel@tonic-gate 	 */
35247c478bd9Sstevel@tonic-gate 	policy_info->mem_policy = policy;
3525*2cb27123Saguzovsk 	policy_info->mem_lgrpid = LGRP_NONE;
35267c478bd9Sstevel@tonic-gate 
35277c478bd9Sstevel@tonic-gate 	return (0);
35287c478bd9Sstevel@tonic-gate }
35297c478bd9Sstevel@tonic-gate 
35307c478bd9Sstevel@tonic-gate 
35317c478bd9Sstevel@tonic-gate /*
35327c478bd9Sstevel@tonic-gate  * Get shared memory allocation policy with given tree and offset
35337c478bd9Sstevel@tonic-gate  */
35347c478bd9Sstevel@tonic-gate lgrp_mem_policy_info_t *
35357c478bd9Sstevel@tonic-gate lgrp_shm_policy_get(struct anon_map *amp, ulong_t anon_index, vnode_t *vp,
35367c478bd9Sstevel@tonic-gate     u_offset_t vn_off)
35377c478bd9Sstevel@tonic-gate {
35387c478bd9Sstevel@tonic-gate 	u_offset_t		off;
35397c478bd9Sstevel@tonic-gate 	lgrp_mem_policy_info_t	*policy_info;
35407c478bd9Sstevel@tonic-gate 	lgrp_shm_policy_seg_t	*policy_seg;
35417c478bd9Sstevel@tonic-gate 	lgrp_shm_locality_t	*shm_locality;
35427c478bd9Sstevel@tonic-gate 	avl_tree_t		*tree;
35437c478bd9Sstevel@tonic-gate 	avl_index_t		where;
35447c478bd9Sstevel@tonic-gate 
35457c478bd9Sstevel@tonic-gate 	/*
35467c478bd9Sstevel@tonic-gate 	 * Get policy segment tree from anon_map or vnode and use specified
35477c478bd9Sstevel@tonic-gate 	 * anon index or vnode offset as offset
35487c478bd9Sstevel@tonic-gate 	 *
35497c478bd9Sstevel@tonic-gate 	 * Assume that no lock needs to be held on anon_map or vnode, since
35507c478bd9Sstevel@tonic-gate 	 * they should be protected by their reference count which must be
35517c478bd9Sstevel@tonic-gate 	 * nonzero for an existing segment
35527c478bd9Sstevel@tonic-gate 	 */
35537c478bd9Sstevel@tonic-gate 	if (amp) {
35547c478bd9Sstevel@tonic-gate 		ASSERT(amp->refcnt != 0);
35557c478bd9Sstevel@tonic-gate 		shm_locality = amp->locality;
35567c478bd9Sstevel@tonic-gate 		if (shm_locality == NULL)
35577c478bd9Sstevel@tonic-gate 			return (NULL);
35587c478bd9Sstevel@tonic-gate 		tree = shm_locality->loc_tree;
35597c478bd9Sstevel@tonic-gate 		off = ptob(anon_index);
35607c478bd9Sstevel@tonic-gate 	} else if (vp) {
35617c478bd9Sstevel@tonic-gate 		shm_locality = vp->v_locality;
35627c478bd9Sstevel@tonic-gate 		if (shm_locality == NULL)
35637c478bd9Sstevel@tonic-gate 			return (NULL);
35647c478bd9Sstevel@tonic-gate 		ASSERT(shm_locality->loc_count != 0);
35657c478bd9Sstevel@tonic-gate 		tree = shm_locality->loc_tree;
35667c478bd9Sstevel@tonic-gate 		off = vn_off;
35677c478bd9Sstevel@tonic-gate 	}
35687c478bd9Sstevel@tonic-gate 
35697c478bd9Sstevel@tonic-gate 	if (tree == NULL)
35707c478bd9Sstevel@tonic-gate 		return (NULL);
35717c478bd9Sstevel@tonic-gate 
35727c478bd9Sstevel@tonic-gate 	/*
35737c478bd9Sstevel@tonic-gate 	 * Lookup policy segment for offset into shared object and return
35747c478bd9Sstevel@tonic-gate 	 * policy info
35757c478bd9Sstevel@tonic-gate 	 */
35767c478bd9Sstevel@tonic-gate 	rw_enter(&shm_locality->loc_lock, RW_READER);
35777c478bd9Sstevel@tonic-gate 	policy_info = NULL;
35787c478bd9Sstevel@tonic-gate 	policy_seg = avl_find(tree, &off, &where);
35797c478bd9Sstevel@tonic-gate 	if (policy_seg)
35807c478bd9Sstevel@tonic-gate 		policy_info = &policy_seg->shm_policy;
35817c478bd9Sstevel@tonic-gate 	rw_exit(&shm_locality->loc_lock);
35827c478bd9Sstevel@tonic-gate 
35837c478bd9Sstevel@tonic-gate 	return (policy_info);
35847c478bd9Sstevel@tonic-gate }
35857c478bd9Sstevel@tonic-gate 
35867c478bd9Sstevel@tonic-gate /*
3587611ffe8aSesaxe  * Default memory allocation policy for kernel segmap pages
3588611ffe8aSesaxe  */
3589611ffe8aSesaxe lgrp_mem_policy_t	lgrp_segmap_default_policy = LGRP_MEM_POLICY_RANDOM;
3590611ffe8aSesaxe 
3591611ffe8aSesaxe /*
35927c478bd9Sstevel@tonic-gate  * Return lgroup to use for allocating memory
35937c478bd9Sstevel@tonic-gate  * given the segment and address
35947c478bd9Sstevel@tonic-gate  *
35957c478bd9Sstevel@tonic-gate  * There isn't any mutual exclusion that exists between calls
35967c478bd9Sstevel@tonic-gate  * to this routine and DR, so this routine and whomever calls it
35977c478bd9Sstevel@tonic-gate  * should be mindful of the possibility that the lgrp returned
35987c478bd9Sstevel@tonic-gate  * may be deleted. If this happens, dereferences of the lgrp
35997c478bd9Sstevel@tonic-gate  * pointer will still be safe, but the resources in the lgrp will
36007c478bd9Sstevel@tonic-gate  * be gone, and LGRP_EXISTS() will no longer be true.
36017c478bd9Sstevel@tonic-gate  */
36027c478bd9Sstevel@tonic-gate lgrp_t *
36037c478bd9Sstevel@tonic-gate lgrp_mem_choose(struct seg *seg, caddr_t vaddr, size_t pgsz)
36047c478bd9Sstevel@tonic-gate {
36057c478bd9Sstevel@tonic-gate 	int			i;
36067c478bd9Sstevel@tonic-gate 	lgrp_t			*lgrp;
36077c478bd9Sstevel@tonic-gate 	klgrpset_t		lgrpset;
36087c478bd9Sstevel@tonic-gate 	int			lgrps_spanned;
36097c478bd9Sstevel@tonic-gate 	unsigned long		off;
36107c478bd9Sstevel@tonic-gate 	lgrp_mem_policy_t	policy;
36117c478bd9Sstevel@tonic-gate 	lgrp_mem_policy_info_t	*policy_info;
36127c478bd9Sstevel@tonic-gate 	ushort_t		random;
36137c478bd9Sstevel@tonic-gate 	int			stat = 0;
3614611ffe8aSesaxe 	extern struct seg	*segkmap;
36157c478bd9Sstevel@tonic-gate 
36167c478bd9Sstevel@tonic-gate 	/*
36177c478bd9Sstevel@tonic-gate 	 * Just return null if the lgrp framework hasn't finished
36187c478bd9Sstevel@tonic-gate 	 * initializing or if this is a UMA machine.
36197c478bd9Sstevel@tonic-gate 	 */
36207c478bd9Sstevel@tonic-gate 	if (nlgrps == 1 || !lgrp_initialized)
36217c478bd9Sstevel@tonic-gate 		return (lgrp_root);
36227c478bd9Sstevel@tonic-gate 
36237c478bd9Sstevel@tonic-gate 	/*
36247c478bd9Sstevel@tonic-gate 	 * Get memory allocation policy for this segment
36257c478bd9Sstevel@tonic-gate 	 */
36267c478bd9Sstevel@tonic-gate 	policy = lgrp_mem_default_policy;
36277c478bd9Sstevel@tonic-gate 	if (seg != NULL) {
36287c478bd9Sstevel@tonic-gate 		if (seg->s_as == &kas) {
3629611ffe8aSesaxe 			if (seg == segkmap)
3630611ffe8aSesaxe 				policy = lgrp_segmap_default_policy;
36317c478bd9Sstevel@tonic-gate 			if (policy == LGRP_MEM_POLICY_RANDOM_PROC ||
36327c478bd9Sstevel@tonic-gate 			    policy == LGRP_MEM_POLICY_RANDOM_PSET)
36337c478bd9Sstevel@tonic-gate 				policy = LGRP_MEM_POLICY_RANDOM;
36347c478bd9Sstevel@tonic-gate 		} else {
36357c478bd9Sstevel@tonic-gate 			policy_info = lgrp_mem_policy_get(seg, vaddr);
3636*2cb27123Saguzovsk 			if (policy_info != NULL) {
36377c478bd9Sstevel@tonic-gate 				policy = policy_info->mem_policy;
3638*2cb27123Saguzovsk 				if (policy == LGRP_MEM_POLICY_NEXT_SEG) {
3639*2cb27123Saguzovsk 					lgrp_id_t id = policy_info->mem_lgrpid;
3640*2cb27123Saguzovsk 					ASSERT(id != LGRP_NONE);
3641*2cb27123Saguzovsk 					ASSERT(id < NLGRPS_MAX);
3642*2cb27123Saguzovsk 					lgrp = lgrp_table[id];
3643*2cb27123Saguzovsk 					if (!LGRP_EXISTS(lgrp)) {
3644*2cb27123Saguzovsk 						policy = LGRP_MEM_POLICY_NEXT;
3645*2cb27123Saguzovsk 					} else {
3646*2cb27123Saguzovsk 						lgrp_stat_add(id,
3647*2cb27123Saguzovsk 						    LGRP_NUM_NEXT_SEG, 1);
3648*2cb27123Saguzovsk 						return (lgrp);
3649*2cb27123Saguzovsk 					}
3650*2cb27123Saguzovsk 				}
3651*2cb27123Saguzovsk 			}
36527c478bd9Sstevel@tonic-gate 		}
36537c478bd9Sstevel@tonic-gate 	}
36547c478bd9Sstevel@tonic-gate 	lgrpset = 0;
36557c478bd9Sstevel@tonic-gate 
36567c478bd9Sstevel@tonic-gate 	/*
36577c478bd9Sstevel@tonic-gate 	 * Initialize lgroup to home by default
36587c478bd9Sstevel@tonic-gate 	 */
36597c478bd9Sstevel@tonic-gate 	lgrp = lgrp_home_lgrp();
36607c478bd9Sstevel@tonic-gate 
36617c478bd9Sstevel@tonic-gate 	/*
36627c478bd9Sstevel@tonic-gate 	 * When homing threads on root lgrp, override default memory
36637c478bd9Sstevel@tonic-gate 	 * allocation policies with root lgroup memory allocation policy
36647c478bd9Sstevel@tonic-gate 	 */
36657c478bd9Sstevel@tonic-gate 	if (lgrp == lgrp_root)
36667c478bd9Sstevel@tonic-gate 		policy = lgrp_mem_policy_root;
36677c478bd9Sstevel@tonic-gate 
36687c478bd9Sstevel@tonic-gate 	/*
36697c478bd9Sstevel@tonic-gate 	 * Implement policy
36707c478bd9Sstevel@tonic-gate 	 */
36717c478bd9Sstevel@tonic-gate 	switch (policy) {
36727c478bd9Sstevel@tonic-gate 	case LGRP_MEM_POLICY_NEXT_CPU:
36737c478bd9Sstevel@tonic-gate 
36747c478bd9Sstevel@tonic-gate 		/*
36757c478bd9Sstevel@tonic-gate 		 * Return lgroup of current CPU which faulted on memory
3676394b433dSesaxe 		 * If the CPU isn't currently in an lgrp, then opt to
3677394b433dSesaxe 		 * allocate from the root.
3678394b433dSesaxe 		 *
3679394b433dSesaxe 		 * Kernel preemption needs to be disabled here to prevent
3680394b433dSesaxe 		 * the current CPU from going away before lgrp is found.
36817c478bd9Sstevel@tonic-gate 		 */
3682394b433dSesaxe 		if (LGRP_CPU_HAS_NO_LGRP(CPU)) {
3683394b433dSesaxe 			lgrp = lgrp_root;
3684394b433dSesaxe 		} else {
3685394b433dSesaxe 			kpreempt_disable();
36867c478bd9Sstevel@tonic-gate 			lgrp = lgrp_cpu_to_lgrp(CPU);
3687394b433dSesaxe 			kpreempt_enable();
3688394b433dSesaxe 		}
36897c478bd9Sstevel@tonic-gate 		break;
36907c478bd9Sstevel@tonic-gate 
36917c478bd9Sstevel@tonic-gate 	case LGRP_MEM_POLICY_NEXT:
36927c478bd9Sstevel@tonic-gate 	case LGRP_MEM_POLICY_DEFAULT:
36937c478bd9Sstevel@tonic-gate 	default:
36947c478bd9Sstevel@tonic-gate 
36957c478bd9Sstevel@tonic-gate 		/*
36967c478bd9Sstevel@tonic-gate 		 * Just return current thread's home lgroup
36977c478bd9Sstevel@tonic-gate 		 * for default policy (next touch)
36987c478bd9Sstevel@tonic-gate 		 * If the thread is homed to the root,
36997c478bd9Sstevel@tonic-gate 		 * then the default policy is random across lgroups.
37007c478bd9Sstevel@tonic-gate 		 * Fallthrough to the random case.
37017c478bd9Sstevel@tonic-gate 		 */
37027c478bd9Sstevel@tonic-gate 		if (lgrp != lgrp_root) {
37037c478bd9Sstevel@tonic-gate 			if (policy == LGRP_MEM_POLICY_NEXT)
37047c478bd9Sstevel@tonic-gate 				lgrp_stat_add(lgrp->lgrp_id, LGRP_NUM_NEXT, 1);
37057c478bd9Sstevel@tonic-gate 			else
37067c478bd9Sstevel@tonic-gate 				lgrp_stat_add(lgrp->lgrp_id,
37077c478bd9Sstevel@tonic-gate 				    LGRP_NUM_DEFAULT, 1);
37087c478bd9Sstevel@tonic-gate 			break;
37097c478bd9Sstevel@tonic-gate 		}
37107c478bd9Sstevel@tonic-gate 		/* LINTED fallthrough on case statement */
37117c478bd9Sstevel@tonic-gate 	case LGRP_MEM_POLICY_RANDOM:
37127c478bd9Sstevel@tonic-gate 
37137c478bd9Sstevel@tonic-gate 		/*
37147c478bd9Sstevel@tonic-gate 		 * Return a random leaf lgroup with memory
37157c478bd9Sstevel@tonic-gate 		 */
37167c478bd9Sstevel@tonic-gate 		lgrpset = lgrp_root->lgrp_set[LGRP_RSRC_MEM];
37177c478bd9Sstevel@tonic-gate 		/*
37187c478bd9Sstevel@tonic-gate 		 * Count how many lgroups are spanned
37197c478bd9Sstevel@tonic-gate 		 */
37207c478bd9Sstevel@tonic-gate 		klgrpset_nlgrps(lgrpset, lgrps_spanned);
37217c478bd9Sstevel@tonic-gate 
37227c478bd9Sstevel@tonic-gate 		/*
37237c478bd9Sstevel@tonic-gate 		 * There may be no memnodes in the root lgroup during DR copy
37247c478bd9Sstevel@tonic-gate 		 * rename on a system with only two boards (memnodes)
37257c478bd9Sstevel@tonic-gate 		 * configured. In this case just return the root lgrp.
37267c478bd9Sstevel@tonic-gate 		 */
37277c478bd9Sstevel@tonic-gate 		if (lgrps_spanned == 0) {
37287c478bd9Sstevel@tonic-gate 			lgrp = lgrp_root;
37297c478bd9Sstevel@tonic-gate 			break;
37307c478bd9Sstevel@tonic-gate 		}
37317c478bd9Sstevel@tonic-gate 
37327c478bd9Sstevel@tonic-gate 		/*
37337c478bd9Sstevel@tonic-gate 		 * Pick a random offset within lgroups spanned
37347c478bd9Sstevel@tonic-gate 		 * and return lgroup at that offset
37357c478bd9Sstevel@tonic-gate 		 */
37367c478bd9Sstevel@tonic-gate 		random = (ushort_t)gethrtime() >> 4;
37377c478bd9Sstevel@tonic-gate 		off = random % lgrps_spanned;
37387c478bd9Sstevel@tonic-gate 		ASSERT(off <= lgrp_alloc_max);
37397c478bd9Sstevel@tonic-gate 
37407c478bd9Sstevel@tonic-gate 		for (i = 0; i <= lgrp_alloc_max; i++) {
37417c478bd9Sstevel@tonic-gate 			if (!klgrpset_ismember(lgrpset, i))
37427c478bd9Sstevel@tonic-gate 				continue;
37437c478bd9Sstevel@tonic-gate 			if (off)
37447c478bd9Sstevel@tonic-gate 				off--;
37457c478bd9Sstevel@tonic-gate 			else {
37467c478bd9Sstevel@tonic-gate 				lgrp = lgrp_table[i];
37477c478bd9Sstevel@tonic-gate 				lgrp_stat_add(lgrp->lgrp_id, LGRP_NUM_RANDOM,
37487c478bd9Sstevel@tonic-gate 				    1);
37497c478bd9Sstevel@tonic-gate 				break;
37507c478bd9Sstevel@tonic-gate 			}
37517c478bd9Sstevel@tonic-gate 		}
37527c478bd9Sstevel@tonic-gate 		break;
37537c478bd9Sstevel@tonic-gate 
37547c478bd9Sstevel@tonic-gate 	case LGRP_MEM_POLICY_RANDOM_PROC:
37557c478bd9Sstevel@tonic-gate 
37567c478bd9Sstevel@tonic-gate 		/*
37577c478bd9Sstevel@tonic-gate 		 * Grab copy of bitmask of lgroups spanned by
37587c478bd9Sstevel@tonic-gate 		 * this process
37597c478bd9Sstevel@tonic-gate 		 */
37607c478bd9Sstevel@tonic-gate 		klgrpset_copy(lgrpset, curproc->p_lgrpset);
37617c478bd9Sstevel@tonic-gate 		stat = LGRP_NUM_RANDOM_PROC;
37627c478bd9Sstevel@tonic-gate 
37637c478bd9Sstevel@tonic-gate 		/* LINTED fallthrough on case statement */
37647c478bd9Sstevel@tonic-gate 	case LGRP_MEM_POLICY_RANDOM_PSET:
37657c478bd9Sstevel@tonic-gate 
37667c478bd9Sstevel@tonic-gate 		if (!stat)
37677c478bd9Sstevel@tonic-gate 			stat = LGRP_NUM_RANDOM_PSET;
37687c478bd9Sstevel@tonic-gate 
37697c478bd9Sstevel@tonic-gate 		if (klgrpset_isempty(lgrpset)) {
37707c478bd9Sstevel@tonic-gate 			/*
37717c478bd9Sstevel@tonic-gate 			 * Grab copy of bitmask of lgroups spanned by
37727c478bd9Sstevel@tonic-gate 			 * this processor set
37737c478bd9Sstevel@tonic-gate 			 */
37747c478bd9Sstevel@tonic-gate 			kpreempt_disable();
37757c478bd9Sstevel@tonic-gate 			klgrpset_copy(lgrpset,
37767c478bd9Sstevel@tonic-gate 			    curthread->t_cpupart->cp_lgrpset);
37777c478bd9Sstevel@tonic-gate 			kpreempt_enable();
37787c478bd9Sstevel@tonic-gate 		}
37797c478bd9Sstevel@tonic-gate 
37807c478bd9Sstevel@tonic-gate 		/*
37817c478bd9Sstevel@tonic-gate 		 * Count how many lgroups are spanned
37827c478bd9Sstevel@tonic-gate 		 */
37837c478bd9Sstevel@tonic-gate 		klgrpset_nlgrps(lgrpset, lgrps_spanned);
37847c478bd9Sstevel@tonic-gate 		ASSERT(lgrps_spanned <= nlgrps);
37857c478bd9Sstevel@tonic-gate 
37867c478bd9Sstevel@tonic-gate 		/*
37877c478bd9Sstevel@tonic-gate 		 * Probably lgrps_spanned should be always non-zero, but to be
37887c478bd9Sstevel@tonic-gate 		 * on the safe side we return lgrp_root if it is empty.
37897c478bd9Sstevel@tonic-gate 		 */
37907c478bd9Sstevel@tonic-gate 		if (lgrps_spanned == 0) {
37917c478bd9Sstevel@tonic-gate 			lgrp = lgrp_root;
37927c478bd9Sstevel@tonic-gate 			break;
37937c478bd9Sstevel@tonic-gate 		}
37947c478bd9Sstevel@tonic-gate 
37957c478bd9Sstevel@tonic-gate 		/*
37967c478bd9Sstevel@tonic-gate 		 * Pick a random offset within lgroups spanned
37977c478bd9Sstevel@tonic-gate 		 * and return lgroup at that offset
37987c478bd9Sstevel@tonic-gate 		 */
37997c478bd9Sstevel@tonic-gate 		random = (ushort_t)gethrtime() >> 4;
38007c478bd9Sstevel@tonic-gate 		off = random % lgrps_spanned;
38017c478bd9Sstevel@tonic-gate 		ASSERT(off <= lgrp_alloc_max);
38027c478bd9Sstevel@tonic-gate 
38037c478bd9Sstevel@tonic-gate 		for (i = 0; i <= lgrp_alloc_max; i++) {
38047c478bd9Sstevel@tonic-gate 			if (!klgrpset_ismember(lgrpset, i))
38057c478bd9Sstevel@tonic-gate 				continue;
38067c478bd9Sstevel@tonic-gate 			if (off)
38077c478bd9Sstevel@tonic-gate 				off--;
38087c478bd9Sstevel@tonic-gate 			else {
38097c478bd9Sstevel@tonic-gate 				lgrp = lgrp_table[i];
38107c478bd9Sstevel@tonic-gate 				lgrp_stat_add(lgrp->lgrp_id, LGRP_NUM_RANDOM,
38117c478bd9Sstevel@tonic-gate 				    1);
38127c478bd9Sstevel@tonic-gate 				break;
38137c478bd9Sstevel@tonic-gate 			}
38147c478bd9Sstevel@tonic-gate 		}
38157c478bd9Sstevel@tonic-gate 		break;
38167c478bd9Sstevel@tonic-gate 
38177c478bd9Sstevel@tonic-gate 	case LGRP_MEM_POLICY_ROUNDROBIN:
38187c478bd9Sstevel@tonic-gate 
38197c478bd9Sstevel@tonic-gate 		/*
38207c478bd9Sstevel@tonic-gate 		 * Use offset within segment to determine
38217c478bd9Sstevel@tonic-gate 		 * offset from home lgroup to choose for
38227c478bd9Sstevel@tonic-gate 		 * next lgroup to allocate memory from
38237c478bd9Sstevel@tonic-gate 		 */
38247c478bd9Sstevel@tonic-gate 		off = ((unsigned long)(vaddr - seg->s_base) / pgsz) %
38257c478bd9Sstevel@tonic-gate 		    (lgrp_alloc_max + 1);
38267c478bd9Sstevel@tonic-gate 
38277c478bd9Sstevel@tonic-gate 		kpreempt_disable();
38287c478bd9Sstevel@tonic-gate 		lgrpset = lgrp_root->lgrp_set[LGRP_RSRC_MEM];
38297c478bd9Sstevel@tonic-gate 		i = lgrp->lgrp_id;
38307c478bd9Sstevel@tonic-gate 		kpreempt_enable();
38317c478bd9Sstevel@tonic-gate 
38327c478bd9Sstevel@tonic-gate 		while (off > 0) {
38337c478bd9Sstevel@tonic-gate 			i = (i + 1) % (lgrp_alloc_max + 1);
38347c478bd9Sstevel@tonic-gate 			lgrp = lgrp_table[i];
38357c478bd9Sstevel@tonic-gate 			if (klgrpset_ismember(lgrpset, i))
38367c478bd9Sstevel@tonic-gate 				off--;
38377c478bd9Sstevel@tonic-gate 		}
38387c478bd9Sstevel@tonic-gate 		lgrp_stat_add(lgrp->lgrp_id, LGRP_NUM_ROUNDROBIN, 1);
38397c478bd9Sstevel@tonic-gate 
38407c478bd9Sstevel@tonic-gate 		break;
38417c478bd9Sstevel@tonic-gate 	}
38427c478bd9Sstevel@tonic-gate 
38437c478bd9Sstevel@tonic-gate 	ASSERT(lgrp != NULL);
38447c478bd9Sstevel@tonic-gate 	return (lgrp);
38457c478bd9Sstevel@tonic-gate }
38467c478bd9Sstevel@tonic-gate 
38477c478bd9Sstevel@tonic-gate /*
38487c478bd9Sstevel@tonic-gate  * Return the number of pages in an lgroup
38497c478bd9Sstevel@tonic-gate  *
38507c478bd9Sstevel@tonic-gate  * NOTE: NUMA test (numat) driver uses this, so changing arguments or semantics
38517c478bd9Sstevel@tonic-gate  *	 could cause tests that rely on the numat driver to fail....
38527c478bd9Sstevel@tonic-gate  */
38537c478bd9Sstevel@tonic-gate pgcnt_t
38547c478bd9Sstevel@tonic-gate lgrp_mem_size(lgrp_id_t lgrpid, lgrp_mem_query_t query)
38557c478bd9Sstevel@tonic-gate {
38567c478bd9Sstevel@tonic-gate 	lgrp_t *lgrp;
38577c478bd9Sstevel@tonic-gate 
38587c478bd9Sstevel@tonic-gate 	lgrp = lgrp_table[lgrpid];
38597c478bd9Sstevel@tonic-gate 	if (!LGRP_EXISTS(lgrp) ||
38607c478bd9Sstevel@tonic-gate 	    klgrpset_isempty(lgrp->lgrp_set[LGRP_RSRC_MEM]) ||
38617c478bd9Sstevel@tonic-gate 	    !klgrpset_ismember(lgrp->lgrp_set[LGRP_RSRC_MEM], lgrpid))
38627c478bd9Sstevel@tonic-gate 		return (0);
38637c478bd9Sstevel@tonic-gate 
38647c478bd9Sstevel@tonic-gate 	return (lgrp_plat_mem_size(lgrp->lgrp_plathand, query));
38657c478bd9Sstevel@tonic-gate }
38667c478bd9Sstevel@tonic-gate 
38677c478bd9Sstevel@tonic-gate /*
38687c478bd9Sstevel@tonic-gate  * Initialize lgroup shared memory allocation policy support
38697c478bd9Sstevel@tonic-gate  */
38707c478bd9Sstevel@tonic-gate void
38717c478bd9Sstevel@tonic-gate lgrp_shm_policy_init(struct anon_map *amp, vnode_t *vp)
38727c478bd9Sstevel@tonic-gate {
38737c478bd9Sstevel@tonic-gate 	lgrp_shm_locality_t	*shm_locality;
38747c478bd9Sstevel@tonic-gate 
38757c478bd9Sstevel@tonic-gate 	/*
38767c478bd9Sstevel@tonic-gate 	 * Initialize locality field in anon_map
38777c478bd9Sstevel@tonic-gate 	 * Don't need any locks because this is called when anon_map is
38787c478bd9Sstevel@tonic-gate 	 * allocated, but not used anywhere yet.
38797c478bd9Sstevel@tonic-gate 	 */
38807c478bd9Sstevel@tonic-gate 	if (amp) {
38817c478bd9Sstevel@tonic-gate 		ANON_LOCK_ENTER(&amp->a_rwlock, RW_WRITER);
38827c478bd9Sstevel@tonic-gate 		if (amp->locality == NULL) {
38837c478bd9Sstevel@tonic-gate 			/*
38847c478bd9Sstevel@tonic-gate 			 * Allocate and initialize shared memory locality info
38857c478bd9Sstevel@tonic-gate 			 * and set anon_map locality pointer to it
38867c478bd9Sstevel@tonic-gate 			 * Drop lock across kmem_alloc(KM_SLEEP)
38877c478bd9Sstevel@tonic-gate 			 */
38887c478bd9Sstevel@tonic-gate 			ANON_LOCK_EXIT(&amp->a_rwlock);
38897c478bd9Sstevel@tonic-gate 			shm_locality = kmem_alloc(sizeof (*shm_locality),
38907c478bd9Sstevel@tonic-gate 			    KM_SLEEP);
38917c478bd9Sstevel@tonic-gate 			rw_init(&shm_locality->loc_lock, NULL, RW_DEFAULT,
38927c478bd9Sstevel@tonic-gate 			    NULL);
38937c478bd9Sstevel@tonic-gate 			shm_locality->loc_count = 1;	/* not used for amp */
38947c478bd9Sstevel@tonic-gate 			shm_locality->loc_tree = NULL;
38957c478bd9Sstevel@tonic-gate 
38967c478bd9Sstevel@tonic-gate 			/*
38977c478bd9Sstevel@tonic-gate 			 * Reacquire lock and check to see whether anyone beat
38987c478bd9Sstevel@tonic-gate 			 * us to initializing the locality info
38997c478bd9Sstevel@tonic-gate 			 */
39007c478bd9Sstevel@tonic-gate 			ANON_LOCK_ENTER(&amp->a_rwlock, RW_WRITER);
39017c478bd9Sstevel@tonic-gate 			if (amp->locality != NULL) {
39027c478bd9Sstevel@tonic-gate 				rw_destroy(&shm_locality->loc_lock);
39037c478bd9Sstevel@tonic-gate 				kmem_free(shm_locality,
39047c478bd9Sstevel@tonic-gate 				    sizeof (*shm_locality));
39057c478bd9Sstevel@tonic-gate 			} else
39067c478bd9Sstevel@tonic-gate 				amp->locality = shm_locality;
39077c478bd9Sstevel@tonic-gate 		}
39087c478bd9Sstevel@tonic-gate 		ANON_LOCK_EXIT(&amp->a_rwlock);
39097c478bd9Sstevel@tonic-gate 		return;
39107c478bd9Sstevel@tonic-gate 	}
39117c478bd9Sstevel@tonic-gate 
39127c478bd9Sstevel@tonic-gate 	/*
39137c478bd9Sstevel@tonic-gate 	 * Allocate shared vnode policy info if vnode is not locality aware yet
39147c478bd9Sstevel@tonic-gate 	 */
39157c478bd9Sstevel@tonic-gate 	mutex_enter(&vp->v_lock);
39167c478bd9Sstevel@tonic-gate 	if ((vp->v_flag & V_LOCALITY) == 0) {
39177c478bd9Sstevel@tonic-gate 		/*
39187c478bd9Sstevel@tonic-gate 		 * Allocate and initialize shared memory locality info
39197c478bd9Sstevel@tonic-gate 		 */
39207c478bd9Sstevel@tonic-gate 		mutex_exit(&vp->v_lock);
39217c478bd9Sstevel@tonic-gate 		shm_locality = kmem_alloc(sizeof (*shm_locality), KM_SLEEP);
39227c478bd9Sstevel@tonic-gate 		rw_init(&shm_locality->loc_lock, NULL, RW_DEFAULT, NULL);
39237c478bd9Sstevel@tonic-gate 		shm_locality->loc_count = 1;
39247c478bd9Sstevel@tonic-gate 		shm_locality->loc_tree = NULL;
39257c478bd9Sstevel@tonic-gate 
39267c478bd9Sstevel@tonic-gate 		/*
39277c478bd9Sstevel@tonic-gate 		 * Point vnode locality field at shared vnode policy info
39287c478bd9Sstevel@tonic-gate 		 * and set locality aware flag in vnode
39297c478bd9Sstevel@tonic-gate 		 */
39307c478bd9Sstevel@tonic-gate 		mutex_enter(&vp->v_lock);
39317c478bd9Sstevel@tonic-gate 		if ((vp->v_flag & V_LOCALITY) == 0) {
39327c478bd9Sstevel@tonic-gate 			vp->v_locality = shm_locality;
39337c478bd9Sstevel@tonic-gate 			vp->v_flag |= V_LOCALITY;
39347c478bd9Sstevel@tonic-gate 		} else {
39357c478bd9Sstevel@tonic-gate 			/*
39367c478bd9Sstevel@tonic-gate 			 * Lost race so free locality info and increment count.
39377c478bd9Sstevel@tonic-gate 			 */
39387c478bd9Sstevel@tonic-gate 			rw_destroy(&shm_locality->loc_lock);
39397c478bd9Sstevel@tonic-gate 			kmem_free(shm_locality, sizeof (*shm_locality));
39407c478bd9Sstevel@tonic-gate 			shm_locality = vp->v_locality;
39417c478bd9Sstevel@tonic-gate 			shm_locality->loc_count++;
39427c478bd9Sstevel@tonic-gate 		}
39437c478bd9Sstevel@tonic-gate 		mutex_exit(&vp->v_lock);
39447c478bd9Sstevel@tonic-gate 
39457c478bd9Sstevel@tonic-gate 		return;
39467c478bd9Sstevel@tonic-gate 	}
39477c478bd9Sstevel@tonic-gate 
39487c478bd9Sstevel@tonic-gate 	/*
39497c478bd9Sstevel@tonic-gate 	 * Increment reference count of number of segments mapping this vnode
39507c478bd9Sstevel@tonic-gate 	 * shared
39517c478bd9Sstevel@tonic-gate 	 */
39527c478bd9Sstevel@tonic-gate 	shm_locality = vp->v_locality;
39537c478bd9Sstevel@tonic-gate 	shm_locality->loc_count++;
39547c478bd9Sstevel@tonic-gate 	mutex_exit(&vp->v_lock);
39557c478bd9Sstevel@tonic-gate }
39567c478bd9Sstevel@tonic-gate 
39577c478bd9Sstevel@tonic-gate /*
39587c478bd9Sstevel@tonic-gate  * Destroy the given shared memory policy segment tree
39597c478bd9Sstevel@tonic-gate  */
39607c478bd9Sstevel@tonic-gate void
39617c478bd9Sstevel@tonic-gate lgrp_shm_policy_tree_destroy(avl_tree_t *tree)
39627c478bd9Sstevel@tonic-gate {
39637c478bd9Sstevel@tonic-gate 	lgrp_shm_policy_seg_t	*cur;
39647c478bd9Sstevel@tonic-gate 	lgrp_shm_policy_seg_t	*next;
39657c478bd9Sstevel@tonic-gate 
39667c478bd9Sstevel@tonic-gate 	if (tree == NULL)
39677c478bd9Sstevel@tonic-gate 		return;
39687c478bd9Sstevel@tonic-gate 
39697c478bd9Sstevel@tonic-gate 	cur = (lgrp_shm_policy_seg_t *)avl_first(tree);
39707c478bd9Sstevel@tonic-gate 	while (cur != NULL) {
39717c478bd9Sstevel@tonic-gate 		next = AVL_NEXT(tree, cur);
39727c478bd9Sstevel@tonic-gate 		avl_remove(tree, cur);
39737c478bd9Sstevel@tonic-gate 		kmem_free(cur, sizeof (*cur));
39747c478bd9Sstevel@tonic-gate 		cur = next;
39757c478bd9Sstevel@tonic-gate 	}
39767c478bd9Sstevel@tonic-gate 	kmem_free(tree, sizeof (avl_tree_t));
39777c478bd9Sstevel@tonic-gate }
39787c478bd9Sstevel@tonic-gate 
39797c478bd9Sstevel@tonic-gate /*
39807c478bd9Sstevel@tonic-gate  * Uninitialize lgroup shared memory allocation policy support
39817c478bd9Sstevel@tonic-gate  */
39827c478bd9Sstevel@tonic-gate void
39837c478bd9Sstevel@tonic-gate lgrp_shm_policy_fini(struct anon_map *amp, vnode_t *vp)
39847c478bd9Sstevel@tonic-gate {
39857c478bd9Sstevel@tonic-gate 	lgrp_shm_locality_t	*shm_locality;
39867c478bd9Sstevel@tonic-gate 
39877c478bd9Sstevel@tonic-gate 	/*
39887c478bd9Sstevel@tonic-gate 	 * For anon_map, deallocate shared memory policy tree and
39897c478bd9Sstevel@tonic-gate 	 * zero locality field
39907c478bd9Sstevel@tonic-gate 	 * Don't need any locks because anon_map is being freed
39917c478bd9Sstevel@tonic-gate 	 */
39927c478bd9Sstevel@tonic-gate 	if (amp) {
39937c478bd9Sstevel@tonic-gate 		if (amp->locality == NULL)
39947c478bd9Sstevel@tonic-gate 			return;
39957c478bd9Sstevel@tonic-gate 		shm_locality = amp->locality;
39967c478bd9Sstevel@tonic-gate 		shm_locality->loc_count = 0;	/* not really used for amp */
39977c478bd9Sstevel@tonic-gate 		rw_destroy(&shm_locality->loc_lock);
39987c478bd9Sstevel@tonic-gate 		lgrp_shm_policy_tree_destroy(shm_locality->loc_tree);
39997c478bd9Sstevel@tonic-gate 		kmem_free(shm_locality, sizeof (*shm_locality));
40007c478bd9Sstevel@tonic-gate 		amp->locality = 0;
40017c478bd9Sstevel@tonic-gate 		return;
40027c478bd9Sstevel@tonic-gate 	}
40037c478bd9Sstevel@tonic-gate 
40047c478bd9Sstevel@tonic-gate 	/*
40057c478bd9Sstevel@tonic-gate 	 * For vnode, decrement reference count of segments mapping this vnode
40067c478bd9Sstevel@tonic-gate 	 * shared and delete locality info if reference count drops to 0
40077c478bd9Sstevel@tonic-gate 	 */
40087c478bd9Sstevel@tonic-gate 	mutex_enter(&vp->v_lock);
40097c478bd9Sstevel@tonic-gate 	shm_locality = vp->v_locality;
40107c478bd9Sstevel@tonic-gate 	shm_locality->loc_count--;
40117c478bd9Sstevel@tonic-gate 
40127c478bd9Sstevel@tonic-gate 	if (shm_locality->loc_count == 0) {
40137c478bd9Sstevel@tonic-gate 		rw_destroy(&shm_locality->loc_lock);
40147c478bd9Sstevel@tonic-gate 		lgrp_shm_policy_tree_destroy(shm_locality->loc_tree);
40157c478bd9Sstevel@tonic-gate 		kmem_free(shm_locality, sizeof (*shm_locality));
40167c478bd9Sstevel@tonic-gate 		vp->v_locality = 0;
40177c478bd9Sstevel@tonic-gate 		vp->v_flag &= ~V_LOCALITY;
40187c478bd9Sstevel@tonic-gate 	}
40197c478bd9Sstevel@tonic-gate 	mutex_exit(&vp->v_lock);
40207c478bd9Sstevel@tonic-gate }
40217c478bd9Sstevel@tonic-gate 
40227c478bd9Sstevel@tonic-gate /*
40237c478bd9Sstevel@tonic-gate  * Compare two shared memory policy segments
40247c478bd9Sstevel@tonic-gate  * Used by AVL tree code for searching
40257c478bd9Sstevel@tonic-gate  */
40267c478bd9Sstevel@tonic-gate int
40277c478bd9Sstevel@tonic-gate lgrp_shm_policy_compar(const void *x, const void *y)
40287c478bd9Sstevel@tonic-gate {
40297c478bd9Sstevel@tonic-gate 	lgrp_shm_policy_seg_t *a = (lgrp_shm_policy_seg_t *)x;
40307c478bd9Sstevel@tonic-gate 	lgrp_shm_policy_seg_t *b = (lgrp_shm_policy_seg_t *)y;
40317c478bd9Sstevel@tonic-gate 
40327c478bd9Sstevel@tonic-gate 	if (a->shm_off < b->shm_off)
40337c478bd9Sstevel@tonic-gate 		return (-1);
40347c478bd9Sstevel@tonic-gate 	if (a->shm_off >= b->shm_off + b->shm_size)
40357c478bd9Sstevel@tonic-gate 		return (1);
40367c478bd9Sstevel@tonic-gate 	return (0);
40377c478bd9Sstevel@tonic-gate }
40387c478bd9Sstevel@tonic-gate 
40397c478bd9Sstevel@tonic-gate /*
40407c478bd9Sstevel@tonic-gate  * Concatenate seg1 with seg2 and remove seg2
40417c478bd9Sstevel@tonic-gate  */
40427c478bd9Sstevel@tonic-gate static int
40437c478bd9Sstevel@tonic-gate lgrp_shm_policy_concat(avl_tree_t *tree, lgrp_shm_policy_seg_t *seg1,
40447c478bd9Sstevel@tonic-gate     lgrp_shm_policy_seg_t *seg2)
40457c478bd9Sstevel@tonic-gate {
40467c478bd9Sstevel@tonic-gate 	if (!seg1 || !seg2 ||
40477c478bd9Sstevel@tonic-gate 	    seg1->shm_off + seg1->shm_size != seg2->shm_off ||
40487c478bd9Sstevel@tonic-gate 	    seg1->shm_policy.mem_policy != seg2->shm_policy.mem_policy)
40497c478bd9Sstevel@tonic-gate 		return (-1);
40507c478bd9Sstevel@tonic-gate 
40517c478bd9Sstevel@tonic-gate 	seg1->shm_size += seg2->shm_size;
40527c478bd9Sstevel@tonic-gate 	avl_remove(tree, seg2);
40537c478bd9Sstevel@tonic-gate 	kmem_free(seg2, sizeof (*seg2));
40547c478bd9Sstevel@tonic-gate 	return (0);
40557c478bd9Sstevel@tonic-gate }
40567c478bd9Sstevel@tonic-gate 
40577c478bd9Sstevel@tonic-gate /*
40587c478bd9Sstevel@tonic-gate  * Split segment at given offset and return rightmost (uppermost) segment
40597c478bd9Sstevel@tonic-gate  * Assumes that there are no overlapping segments
40607c478bd9Sstevel@tonic-gate  */
40617c478bd9Sstevel@tonic-gate static lgrp_shm_policy_seg_t *
40627c478bd9Sstevel@tonic-gate lgrp_shm_policy_split(avl_tree_t *tree, lgrp_shm_policy_seg_t *seg,
40637c478bd9Sstevel@tonic-gate     u_offset_t off)
40647c478bd9Sstevel@tonic-gate {
40657c478bd9Sstevel@tonic-gate 	lgrp_shm_policy_seg_t	*newseg;
40667c478bd9Sstevel@tonic-gate 	avl_index_t		where;
40677c478bd9Sstevel@tonic-gate 
40687c478bd9Sstevel@tonic-gate 	ASSERT(seg != NULL);
40697c478bd9Sstevel@tonic-gate 	ASSERT(off >= seg->shm_off && off <= seg->shm_off + seg->shm_size);
40707c478bd9Sstevel@tonic-gate 
40717c478bd9Sstevel@tonic-gate 	if (!seg || off < seg->shm_off || off > seg->shm_off +
40727c478bd9Sstevel@tonic-gate 	    seg->shm_size)
40737c478bd9Sstevel@tonic-gate 		return (NULL);
40747c478bd9Sstevel@tonic-gate 
40757c478bd9Sstevel@tonic-gate 	if (off == seg->shm_off || off == seg->shm_off + seg->shm_size)
40767c478bd9Sstevel@tonic-gate 		return (seg);
40777c478bd9Sstevel@tonic-gate 
40787c478bd9Sstevel@tonic-gate 	/*
40797c478bd9Sstevel@tonic-gate 	 * Adjust size of left segment and allocate new (right) segment
40807c478bd9Sstevel@tonic-gate 	 */
40817c478bd9Sstevel@tonic-gate 	newseg = kmem_alloc(sizeof (lgrp_shm_policy_seg_t), KM_SLEEP);
40827c478bd9Sstevel@tonic-gate 	newseg->shm_policy = seg->shm_policy;
40837c478bd9Sstevel@tonic-gate 	newseg->shm_off = off;
40847c478bd9Sstevel@tonic-gate 	newseg->shm_size = seg->shm_size - (off - seg->shm_off);
40857c478bd9Sstevel@tonic-gate 	seg->shm_size = off - seg->shm_off;
40867c478bd9Sstevel@tonic-gate 
40877c478bd9Sstevel@tonic-gate 	/*
40887c478bd9Sstevel@tonic-gate 	 * Find where to insert new segment in AVL tree and insert it
40897c478bd9Sstevel@tonic-gate 	 */
40907c478bd9Sstevel@tonic-gate 	(void) avl_find(tree, &off, &where);
40917c478bd9Sstevel@tonic-gate 	avl_insert(tree, newseg, where);
40927c478bd9Sstevel@tonic-gate 
40937c478bd9Sstevel@tonic-gate 	return (newseg);
40947c478bd9Sstevel@tonic-gate }
40957c478bd9Sstevel@tonic-gate 
40967c478bd9Sstevel@tonic-gate /*
40977c478bd9Sstevel@tonic-gate  * Set shared memory allocation policy on specified shared object at given
40987c478bd9Sstevel@tonic-gate  * offset and length
40997c478bd9Sstevel@tonic-gate  *
41007c478bd9Sstevel@tonic-gate  * Return 0 if policy wasn't set already, 1 if policy was set already, and
41017c478bd9Sstevel@tonic-gate  * -1 if can't set policy.
41027c478bd9Sstevel@tonic-gate  */
41037c478bd9Sstevel@tonic-gate int
41047c478bd9Sstevel@tonic-gate lgrp_shm_policy_set(lgrp_mem_policy_t policy, struct anon_map *amp,
41057c478bd9Sstevel@tonic-gate     ulong_t anon_index, vnode_t *vp, u_offset_t vn_off, size_t len)
41067c478bd9Sstevel@tonic-gate {
41077c478bd9Sstevel@tonic-gate 	u_offset_t		eoff;
41087c478bd9Sstevel@tonic-gate 	lgrp_shm_policy_seg_t	*next;
41097c478bd9Sstevel@tonic-gate 	lgrp_shm_policy_seg_t	*newseg;
41107c478bd9Sstevel@tonic-gate 	u_offset_t		off;
41117c478bd9Sstevel@tonic-gate 	u_offset_t		oldeoff;
41127c478bd9Sstevel@tonic-gate 	lgrp_shm_policy_seg_t	*prev;
41137c478bd9Sstevel@tonic-gate 	int			retval;
41147c478bd9Sstevel@tonic-gate 	lgrp_shm_policy_seg_t	*seg;
41157c478bd9Sstevel@tonic-gate 	lgrp_shm_locality_t	*shm_locality;
41167c478bd9Sstevel@tonic-gate 	avl_tree_t		*tree;
41177c478bd9Sstevel@tonic-gate 	avl_index_t		where;
41187c478bd9Sstevel@tonic-gate 
41197c478bd9Sstevel@tonic-gate 	ASSERT(amp || vp);
41207c478bd9Sstevel@tonic-gate 	ASSERT((len & PAGEOFFSET) == 0);
41217c478bd9Sstevel@tonic-gate 
41227c478bd9Sstevel@tonic-gate 	if (len == 0)
41237c478bd9Sstevel@tonic-gate 		return (-1);
41247c478bd9Sstevel@tonic-gate 
41257c478bd9Sstevel@tonic-gate 	retval = 0;
41267c478bd9Sstevel@tonic-gate 
41277c478bd9Sstevel@tonic-gate 	/*
41287c478bd9Sstevel@tonic-gate 	 * Get locality info and starting offset into shared object
41297c478bd9Sstevel@tonic-gate 	 * Try anon map first and then vnode
41307c478bd9Sstevel@tonic-gate 	 * Assume that no locks need to be held on anon_map or vnode, since
41317c478bd9Sstevel@tonic-gate 	 * it should be protected by its reference count which must be nonzero
41327c478bd9Sstevel@tonic-gate 	 * for an existing segment.
41337c478bd9Sstevel@tonic-gate 	 */
41347c478bd9Sstevel@tonic-gate 	if (amp) {
41357c478bd9Sstevel@tonic-gate 		/*
41367c478bd9Sstevel@tonic-gate 		 * Get policy info from anon_map
41377c478bd9Sstevel@tonic-gate 		 *
41387c478bd9Sstevel@tonic-gate 		 */
41397c478bd9Sstevel@tonic-gate 		ASSERT(amp->refcnt != 0);
41407c478bd9Sstevel@tonic-gate 		if (amp->locality == NULL)
41417c478bd9Sstevel@tonic-gate 			lgrp_shm_policy_init(amp, NULL);
41427c478bd9Sstevel@tonic-gate 		shm_locality = amp->locality;
41437c478bd9Sstevel@tonic-gate 		off = ptob(anon_index);
41447c478bd9Sstevel@tonic-gate 	} else if (vp) {
41457c478bd9Sstevel@tonic-gate 		/*
41467c478bd9Sstevel@tonic-gate 		 * Get policy info from vnode
41477c478bd9Sstevel@tonic-gate 		 */
41487c478bd9Sstevel@tonic-gate 		if ((vp->v_flag & V_LOCALITY) == 0 || vp->v_locality == NULL)
41497c478bd9Sstevel@tonic-gate 			lgrp_shm_policy_init(NULL, vp);
41507c478bd9Sstevel@tonic-gate 		shm_locality = vp->v_locality;
41517c478bd9Sstevel@tonic-gate 		ASSERT(shm_locality->loc_count != 0);
41527c478bd9Sstevel@tonic-gate 		off = vn_off;
41537c478bd9Sstevel@tonic-gate 	} else
41547c478bd9Sstevel@tonic-gate 		return (-1);
41557c478bd9Sstevel@tonic-gate 
41567c478bd9Sstevel@tonic-gate 	ASSERT((off & PAGEOFFSET) == 0);
41577c478bd9Sstevel@tonic-gate 
41587c478bd9Sstevel@tonic-gate 	/*
41597c478bd9Sstevel@tonic-gate 	 * Figure out default policy
41607c478bd9Sstevel@tonic-gate 	 */
41617c478bd9Sstevel@tonic-gate 	if (policy == LGRP_MEM_POLICY_DEFAULT)
41627c478bd9Sstevel@tonic-gate 		policy = lgrp_mem_policy_default(len, MAP_SHARED);
41637c478bd9Sstevel@tonic-gate 
41647c478bd9Sstevel@tonic-gate 	/*
41657c478bd9Sstevel@tonic-gate 	 * Create AVL tree if there isn't one yet
41667c478bd9Sstevel@tonic-gate 	 * and set locality field to point at it
41677c478bd9Sstevel@tonic-gate 	 */
41687c478bd9Sstevel@tonic-gate 	rw_enter(&shm_locality->loc_lock, RW_WRITER);
41697c478bd9Sstevel@tonic-gate 	tree = shm_locality->loc_tree;
41707c478bd9Sstevel@tonic-gate 	if (!tree) {
41717c478bd9Sstevel@tonic-gate 		rw_exit(&shm_locality->loc_lock);
41727c478bd9Sstevel@tonic-gate 
41737c478bd9Sstevel@tonic-gate 		tree = kmem_alloc(sizeof (avl_tree_t), KM_SLEEP);
41747c478bd9Sstevel@tonic-gate 
41757c478bd9Sstevel@tonic-gate 		rw_enter(&shm_locality->loc_lock, RW_WRITER);
41767c478bd9Sstevel@tonic-gate 		if (shm_locality->loc_tree == NULL) {
41777c478bd9Sstevel@tonic-gate 			avl_create(tree, lgrp_shm_policy_compar,
41787c478bd9Sstevel@tonic-gate 			    sizeof (lgrp_shm_policy_seg_t),
41797c478bd9Sstevel@tonic-gate 			    offsetof(lgrp_shm_policy_seg_t, shm_tree));
41807c478bd9Sstevel@tonic-gate 			shm_locality->loc_tree = tree;
41817c478bd9Sstevel@tonic-gate 		} else {
41827c478bd9Sstevel@tonic-gate 			/*
41837c478bd9Sstevel@tonic-gate 			 * Another thread managed to set up the tree
41847c478bd9Sstevel@tonic-gate 			 * before we could. Free the tree we allocated
41857c478bd9Sstevel@tonic-gate 			 * and use the one that's already there.
41867c478bd9Sstevel@tonic-gate 			 */
41877c478bd9Sstevel@tonic-gate 			kmem_free(tree, sizeof (*tree));
41887c478bd9Sstevel@tonic-gate 			tree = shm_locality->loc_tree;
41897c478bd9Sstevel@tonic-gate 		}
41907c478bd9Sstevel@tonic-gate 	}
41917c478bd9Sstevel@tonic-gate 
41927c478bd9Sstevel@tonic-gate 	/*
41937c478bd9Sstevel@tonic-gate 	 * Set policy
41947c478bd9Sstevel@tonic-gate 	 *
41957c478bd9Sstevel@tonic-gate 	 * Need to maintain hold on writer's lock to keep tree from
41967c478bd9Sstevel@tonic-gate 	 * changing out from under us
41977c478bd9Sstevel@tonic-gate 	 */
41987c478bd9Sstevel@tonic-gate 	while (len != 0) {
41997c478bd9Sstevel@tonic-gate 		/*
42007c478bd9Sstevel@tonic-gate 		 * Find policy segment for specified offset into shared object
42017c478bd9Sstevel@tonic-gate 		 */
42027c478bd9Sstevel@tonic-gate 		seg = avl_find(tree, &off, &where);
42037c478bd9Sstevel@tonic-gate 
42047c478bd9Sstevel@tonic-gate 		/*
42057c478bd9Sstevel@tonic-gate 		 * Didn't find any existing segment that contains specified
42067c478bd9Sstevel@tonic-gate 		 * offset, so allocate new segment, insert it, and concatenate
42077c478bd9Sstevel@tonic-gate 		 * with adjacent segments if possible
42087c478bd9Sstevel@tonic-gate 		 */
42097c478bd9Sstevel@tonic-gate 		if (seg == NULL) {
42107c478bd9Sstevel@tonic-gate 			newseg = kmem_alloc(sizeof (lgrp_shm_policy_seg_t),
42117c478bd9Sstevel@tonic-gate 			    KM_SLEEP);
42127c478bd9Sstevel@tonic-gate 			newseg->shm_policy.mem_policy = policy;
4213*2cb27123Saguzovsk 			newseg->shm_policy.mem_lgrpid = LGRP_NONE;
42147c478bd9Sstevel@tonic-gate 			newseg->shm_off = off;
42157c478bd9Sstevel@tonic-gate 			avl_insert(tree, newseg, where);
42167c478bd9Sstevel@tonic-gate 
42177c478bd9Sstevel@tonic-gate 			/*
42187c478bd9Sstevel@tonic-gate 			 * Check to see whether new segment overlaps with next
42197c478bd9Sstevel@tonic-gate 			 * one, set length of new segment accordingly, and
42207c478bd9Sstevel@tonic-gate 			 * calculate remaining length and next offset
42217c478bd9Sstevel@tonic-gate 			 */
42227c478bd9Sstevel@tonic-gate 			seg = AVL_NEXT(tree, newseg);
42237c478bd9Sstevel@tonic-gate 			if (seg == NULL || off + len <= seg->shm_off) {
42247c478bd9Sstevel@tonic-gate 				newseg->shm_size = len;
42257c478bd9Sstevel@tonic-gate 				len = 0;
42267c478bd9Sstevel@tonic-gate 			} else {
42277c478bd9Sstevel@tonic-gate 				newseg->shm_size = seg->shm_off - off;
42287c478bd9Sstevel@tonic-gate 				off = seg->shm_off;
42297c478bd9Sstevel@tonic-gate 				len -= newseg->shm_size;
42307c478bd9Sstevel@tonic-gate 			}
42317c478bd9Sstevel@tonic-gate 
42327c478bd9Sstevel@tonic-gate 			/*
42337c478bd9Sstevel@tonic-gate 			 * Try to concatenate new segment with next and
42347c478bd9Sstevel@tonic-gate 			 * previous ones, since they might have the same policy
42357c478bd9Sstevel@tonic-gate 			 * now.  Grab previous and next segments first because
42367c478bd9Sstevel@tonic-gate 			 * they will change on concatenation.
42377c478bd9Sstevel@tonic-gate 			 */
42387c478bd9Sstevel@tonic-gate 			prev =  AVL_PREV(tree, newseg);
42397c478bd9Sstevel@tonic-gate 			next = AVL_NEXT(tree, newseg);
42407c478bd9Sstevel@tonic-gate 			(void) lgrp_shm_policy_concat(tree, newseg, next);
42417c478bd9Sstevel@tonic-gate 			(void) lgrp_shm_policy_concat(tree, prev, newseg);
42427c478bd9Sstevel@tonic-gate 
42437c478bd9Sstevel@tonic-gate 			continue;
42447c478bd9Sstevel@tonic-gate 		}
42457c478bd9Sstevel@tonic-gate 
42467c478bd9Sstevel@tonic-gate 		eoff = off + len;
42477c478bd9Sstevel@tonic-gate 		oldeoff = seg->shm_off + seg->shm_size;
42487c478bd9Sstevel@tonic-gate 
42497c478bd9Sstevel@tonic-gate 		/*
42507c478bd9Sstevel@tonic-gate 		 * Policy set already?
42517c478bd9Sstevel@tonic-gate 		 */
42527c478bd9Sstevel@tonic-gate 		if (policy == seg->shm_policy.mem_policy) {
42537c478bd9Sstevel@tonic-gate 			/*
42547c478bd9Sstevel@tonic-gate 			 * Nothing left to do if offset and length
42557c478bd9Sstevel@tonic-gate 			 * fall within this segment
42567c478bd9Sstevel@tonic-gate 			 */
42577c478bd9Sstevel@tonic-gate 			if (eoff <= oldeoff) {
42587c478bd9Sstevel@tonic-gate 				retval = 1;
42597c478bd9Sstevel@tonic-gate 				break;
42607c478bd9Sstevel@tonic-gate 			} else {
42617c478bd9Sstevel@tonic-gate 				len = eoff - oldeoff;
42627c478bd9Sstevel@tonic-gate 				off = oldeoff;
42637c478bd9Sstevel@tonic-gate 				continue;
42647c478bd9Sstevel@tonic-gate 			}
42657c478bd9Sstevel@tonic-gate 		}
42667c478bd9Sstevel@tonic-gate 
42677c478bd9Sstevel@tonic-gate 		/*
42687c478bd9Sstevel@tonic-gate 		 * Specified offset and length match existing segment exactly
42697c478bd9Sstevel@tonic-gate 		 */
42707c478bd9Sstevel@tonic-gate 		if (off == seg->shm_off && len == seg->shm_size) {
42717c478bd9Sstevel@tonic-gate 			/*
42727c478bd9Sstevel@tonic-gate 			 * Set policy and update current length
42737c478bd9Sstevel@tonic-gate 			 */
42747c478bd9Sstevel@tonic-gate 			seg->shm_policy.mem_policy = policy;
4275*2cb27123Saguzovsk 			seg->shm_policy.mem_lgrpid = LGRP_NONE;
42767c478bd9Sstevel@tonic-gate 			len = 0;
42777c478bd9Sstevel@tonic-gate 
42787c478bd9Sstevel@tonic-gate 			/*
42797c478bd9Sstevel@tonic-gate 			 * Try concatenating new segment with previous and next
42807c478bd9Sstevel@tonic-gate 			 * segments, since they might have the same policy now.
42817c478bd9Sstevel@tonic-gate 			 * Grab previous and next segments first because they
42827c478bd9Sstevel@tonic-gate 			 * will change on concatenation.
42837c478bd9Sstevel@tonic-gate 			 */
42847c478bd9Sstevel@tonic-gate 			prev =  AVL_PREV(tree, seg);
42857c478bd9Sstevel@tonic-gate 			next = AVL_NEXT(tree, seg);
42867c478bd9Sstevel@tonic-gate 			(void) lgrp_shm_policy_concat(tree, seg, next);
42877c478bd9Sstevel@tonic-gate 			(void) lgrp_shm_policy_concat(tree, prev, seg);
42887c478bd9Sstevel@tonic-gate 		} else {
42897c478bd9Sstevel@tonic-gate 			/*
42907c478bd9Sstevel@tonic-gate 			 * Specified offset and length only apply to part of
42917c478bd9Sstevel@tonic-gate 			 * existing segment
42927c478bd9Sstevel@tonic-gate 			 */
42937c478bd9Sstevel@tonic-gate 
42947c478bd9Sstevel@tonic-gate 			/*
42957c478bd9Sstevel@tonic-gate 			 * New segment starts in middle of old one, so split
42967c478bd9Sstevel@tonic-gate 			 * new one off near beginning of old one
42977c478bd9Sstevel@tonic-gate 			 */
42987c478bd9Sstevel@tonic-gate 			newseg = NULL;
42997c478bd9Sstevel@tonic-gate 			if (off > seg->shm_off) {
43007c478bd9Sstevel@tonic-gate 				newseg = lgrp_shm_policy_split(tree, seg, off);
43017c478bd9Sstevel@tonic-gate 
43027c478bd9Sstevel@tonic-gate 				/*
43037c478bd9Sstevel@tonic-gate 				 * New segment ends where old one did, so try
43047c478bd9Sstevel@tonic-gate 				 * to concatenate with next segment
43057c478bd9Sstevel@tonic-gate 				 */
43067c478bd9Sstevel@tonic-gate 				if (eoff == oldeoff) {
43077c478bd9Sstevel@tonic-gate 					newseg->shm_policy.mem_policy = policy;
4308*2cb27123Saguzovsk 					newseg->shm_policy.mem_lgrpid =
4309*2cb27123Saguzovsk 					    LGRP_NONE;
43107c478bd9Sstevel@tonic-gate 					(void) lgrp_shm_policy_concat(tree,
43117c478bd9Sstevel@tonic-gate 					    newseg, AVL_NEXT(tree, newseg));
43127c478bd9Sstevel@tonic-gate 					break;
43137c478bd9Sstevel@tonic-gate 				}
43147c478bd9Sstevel@tonic-gate 			}
43157c478bd9Sstevel@tonic-gate 
43167c478bd9Sstevel@tonic-gate 			/*
43177c478bd9Sstevel@tonic-gate 			 * New segment ends before old one, so split off end of
43187c478bd9Sstevel@tonic-gate 			 * old one
43197c478bd9Sstevel@tonic-gate 			 */
43207c478bd9Sstevel@tonic-gate 			if (eoff < oldeoff) {
43217c478bd9Sstevel@tonic-gate 				if (newseg) {
43227c478bd9Sstevel@tonic-gate 					(void) lgrp_shm_policy_split(tree,
43237c478bd9Sstevel@tonic-gate 					    newseg, eoff);
43247c478bd9Sstevel@tonic-gate 					newseg->shm_policy.mem_policy = policy;
4325*2cb27123Saguzovsk 					newseg->shm_policy.mem_lgrpid =
4326*2cb27123Saguzovsk 					    LGRP_NONE;
43277c478bd9Sstevel@tonic-gate 				} else {
43287c478bd9Sstevel@tonic-gate 					(void) lgrp_shm_policy_split(tree, seg,
43297c478bd9Sstevel@tonic-gate 					    eoff);
43307c478bd9Sstevel@tonic-gate 					seg->shm_policy.mem_policy = policy;
4331*2cb27123Saguzovsk 					seg->shm_policy.mem_lgrpid = LGRP_NONE;
43327c478bd9Sstevel@tonic-gate 				}
43337c478bd9Sstevel@tonic-gate 
43347c478bd9Sstevel@tonic-gate 				if (off == seg->shm_off)
43357c478bd9Sstevel@tonic-gate 					(void) lgrp_shm_policy_concat(tree,
43367c478bd9Sstevel@tonic-gate 					    AVL_PREV(tree, seg), seg);
43377c478bd9Sstevel@tonic-gate 				break;
43387c478bd9Sstevel@tonic-gate 			}
43397c478bd9Sstevel@tonic-gate 
43407c478bd9Sstevel@tonic-gate 			/*
43417c478bd9Sstevel@tonic-gate 			 * Calculate remaining length and next offset
43427c478bd9Sstevel@tonic-gate 			 */
43437c478bd9Sstevel@tonic-gate 			len = eoff - oldeoff;
43447c478bd9Sstevel@tonic-gate 			off = oldeoff;
43457c478bd9Sstevel@tonic-gate 		}
43467c478bd9Sstevel@tonic-gate 	}
43477c478bd9Sstevel@tonic-gate 
43487c478bd9Sstevel@tonic-gate 	rw_exit(&shm_locality->loc_lock);
43497c478bd9Sstevel@tonic-gate 	return (retval);
43507c478bd9Sstevel@tonic-gate }
43517c478bd9Sstevel@tonic-gate 
43527c478bd9Sstevel@tonic-gate /*
43537c478bd9Sstevel@tonic-gate  * Return the best memnode from which to allocate memory given
43547c478bd9Sstevel@tonic-gate  * an lgroup.
43557c478bd9Sstevel@tonic-gate  *
43567c478bd9Sstevel@tonic-gate  * "c" is for cookie, which is good enough for me.
43577c478bd9Sstevel@tonic-gate  * It references a cookie struct that should be zero'ed to initialize.
43587c478bd9Sstevel@tonic-gate  * The cookie should live on the caller's stack.
43597c478bd9Sstevel@tonic-gate  *
43607c478bd9Sstevel@tonic-gate  * The routine returns -1 when:
43617c478bd9Sstevel@tonic-gate  *	- traverse is 0, and all the memnodes in "lgrp" have been returned.
43627c478bd9Sstevel@tonic-gate  *	- traverse is 1, and all the memnodes in the system have been
43637c478bd9Sstevel@tonic-gate  *	  returned.
43647c478bd9Sstevel@tonic-gate  */
43657c478bd9Sstevel@tonic-gate int
43667c478bd9Sstevel@tonic-gate lgrp_memnode_choose(lgrp_mnode_cookie_t *c)
43677c478bd9Sstevel@tonic-gate {
43687c478bd9Sstevel@tonic-gate 	lgrp_t		*lp = c->lmc_lgrp;
43697c478bd9Sstevel@tonic-gate 	mnodeset_t	nodes = c->lmc_nodes;
43707c478bd9Sstevel@tonic-gate 	int		cnt = c->lmc_cnt;
43717c478bd9Sstevel@tonic-gate 	int		offset, mnode;
43727c478bd9Sstevel@tonic-gate 
43737c478bd9Sstevel@tonic-gate 	extern int	max_mem_nodes;
43747c478bd9Sstevel@tonic-gate 
43757c478bd9Sstevel@tonic-gate 	/*
43767c478bd9Sstevel@tonic-gate 	 * If the set is empty, and the caller is willing, traverse
43777c478bd9Sstevel@tonic-gate 	 * up the hierarchy until we find a non-empty set.
43787c478bd9Sstevel@tonic-gate 	 */
43797c478bd9Sstevel@tonic-gate 	while (nodes == (mnodeset_t)0 || cnt <= 0) {
43807c478bd9Sstevel@tonic-gate 		if (c->lmc_scope == LGRP_SRCH_LOCAL ||
43817c478bd9Sstevel@tonic-gate 		    ((lp = lp->lgrp_parent) == NULL))
43827c478bd9Sstevel@tonic-gate 			return (-1);
43837c478bd9Sstevel@tonic-gate 
43847c478bd9Sstevel@tonic-gate 		nodes = lp->lgrp_mnodes & ~(c->lmc_tried);
43857c478bd9Sstevel@tonic-gate 		cnt = lp->lgrp_nmnodes - c->lmc_ntried;
43867c478bd9Sstevel@tonic-gate 	}
43877c478bd9Sstevel@tonic-gate 
43887c478bd9Sstevel@tonic-gate 	/*
43897c478bd9Sstevel@tonic-gate 	 * Select a memnode by picking one at a "random" offset.
43907c478bd9Sstevel@tonic-gate 	 * Because of DR, memnodes can come and go at any time.
43917c478bd9Sstevel@tonic-gate 	 * This code must be able to cope with the possibility
43927c478bd9Sstevel@tonic-gate 	 * that the nodes count "cnt" is inconsistent with respect
43937c478bd9Sstevel@tonic-gate 	 * to the number of elements actually in "nodes", and
43947c478bd9Sstevel@tonic-gate 	 * therefore that the offset chosen could be greater than
43957c478bd9Sstevel@tonic-gate 	 * the number of elements in the set (some memnodes may
43967c478bd9Sstevel@tonic-gate 	 * have dissapeared just before cnt was read).
43977c478bd9Sstevel@tonic-gate 	 * If this happens, the search simply wraps back to the
43987c478bd9Sstevel@tonic-gate 	 * beginning of the set.
43997c478bd9Sstevel@tonic-gate 	 */
44007c478bd9Sstevel@tonic-gate 	ASSERT(nodes != (mnodeset_t)0 && cnt > 0);
44017c478bd9Sstevel@tonic-gate 	offset = c->lmc_rand % cnt;
44027c478bd9Sstevel@tonic-gate 	do {
44037c478bd9Sstevel@tonic-gate 		for (mnode = 0; mnode < max_mem_nodes; mnode++)
44047c478bd9Sstevel@tonic-gate 			if (nodes & ((mnodeset_t)1 << mnode))
44057c478bd9Sstevel@tonic-gate 				if (!offset--)
44067c478bd9Sstevel@tonic-gate 					break;
44077c478bd9Sstevel@tonic-gate 	} while (mnode >= max_mem_nodes);
44087c478bd9Sstevel@tonic-gate 
44097c478bd9Sstevel@tonic-gate 	/* Found a node. Store state before returning. */
44107c478bd9Sstevel@tonic-gate 	c->lmc_lgrp = lp;
44117c478bd9Sstevel@tonic-gate 	c->lmc_nodes = (nodes & ~((mnodeset_t)1 << mnode));
44127c478bd9Sstevel@tonic-gate 	c->lmc_cnt = cnt - 1;
44137c478bd9Sstevel@tonic-gate 	c->lmc_tried = (c->lmc_tried | ((mnodeset_t)1 << mnode));
44147c478bd9Sstevel@tonic-gate 	c->lmc_ntried++;
44157c478bd9Sstevel@tonic-gate 
44167c478bd9Sstevel@tonic-gate 	return (mnode);
44177c478bd9Sstevel@tonic-gate }
4418