xref: /titanic_50/usr/src/uts/common/disp/cmt.c (revision a66044504f7011cf96863fe62f0c6c692d5e357f)
1fb2f18f8Sesaxe /*
2fb2f18f8Sesaxe  * CDDL HEADER START
3fb2f18f8Sesaxe  *
4fb2f18f8Sesaxe  * The contents of this file are subject to the terms of the
5fb2f18f8Sesaxe  * Common Development and Distribution License (the "License").
6fb2f18f8Sesaxe  * You may not use this file except in compliance with the License.
7fb2f18f8Sesaxe  *
8fb2f18f8Sesaxe  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9fb2f18f8Sesaxe  * or http://www.opensolaris.org/os/licensing.
10fb2f18f8Sesaxe  * See the License for the specific language governing permissions
11fb2f18f8Sesaxe  * and limitations under the License.
12fb2f18f8Sesaxe  *
13fb2f18f8Sesaxe  * When distributing Covered Code, include this CDDL HEADER in each
14fb2f18f8Sesaxe  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15fb2f18f8Sesaxe  * If applicable, add the following below this CDDL HEADER, with the
16fb2f18f8Sesaxe  * fields enclosed by brackets "[]" replaced with your own identifying
17fb2f18f8Sesaxe  * information: Portions Copyright [yyyy] [name of copyright owner]
18fb2f18f8Sesaxe  *
19fb2f18f8Sesaxe  * CDDL HEADER END
20fb2f18f8Sesaxe  */
21fb2f18f8Sesaxe /*
22fb2f18f8Sesaxe  * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
23fb2f18f8Sesaxe  * Use is subject to license terms.
24fb2f18f8Sesaxe  */
25fb2f18f8Sesaxe 
26fb2f18f8Sesaxe #pragma ident	"%Z%%M%	%I%	%E% SMI"
27fb2f18f8Sesaxe 
28fb2f18f8Sesaxe #include <sys/systm.h>
29fb2f18f8Sesaxe #include <sys/types.h>
30fb2f18f8Sesaxe #include <sys/param.h>
31fb2f18f8Sesaxe #include <sys/thread.h>
32fb2f18f8Sesaxe #include <sys/cpuvar.h>
33fb2f18f8Sesaxe #include <sys/cpupart.h>
34fb2f18f8Sesaxe #include <sys/kmem.h>
35fb2f18f8Sesaxe #include <sys/cmn_err.h>
36fb2f18f8Sesaxe #include <sys/kstat.h>
37fb2f18f8Sesaxe #include <sys/processor.h>
38fb2f18f8Sesaxe #include <sys/disp.h>
39fb2f18f8Sesaxe #include <sys/group.h>
40fb2f18f8Sesaxe #include <sys/pghw.h>
41fb2f18f8Sesaxe #include <sys/bitset.h>
42fb2f18f8Sesaxe #include <sys/lgrp.h>
43fb2f18f8Sesaxe #include <sys/cmt.h>
44fb2f18f8Sesaxe 
45fb2f18f8Sesaxe /*
46fb2f18f8Sesaxe  * CMT scheduler / dispatcher support
47fb2f18f8Sesaxe  *
48fb2f18f8Sesaxe  * This file implements CMT scheduler support using Processor Groups.
49fb2f18f8Sesaxe  * The CMT processor group class creates and maintains the CMT class
50fb2f18f8Sesaxe  * specific processor group pg_cmt_t.
51fb2f18f8Sesaxe  *
52fb2f18f8Sesaxe  * ---------------------------- <-- pg_cmt_t *
53fb2f18f8Sesaxe  * | pghw_t                   |
54fb2f18f8Sesaxe  * ----------------------------
55fb2f18f8Sesaxe  * | CMT class specific data  |
56fb2f18f8Sesaxe  * | - hierarchy linkage      |
57fb2f18f8Sesaxe  * | - CMT load balancing data|
58fb2f18f8Sesaxe  * | - active CPU group/bitset|
59fb2f18f8Sesaxe  * ----------------------------
60fb2f18f8Sesaxe  *
61fb2f18f8Sesaxe  * The scheduler/dispatcher leverages knowledge of the performance
62fb2f18f8Sesaxe  * relevant CMT sharing relationships existing between cpus to implement
63fb2f18f8Sesaxe  * optimized affinity and load balancing policies.
64fb2f18f8Sesaxe  *
65fb2f18f8Sesaxe  * Load balancing policy seeks to improve performance by minimizing
66fb2f18f8Sesaxe  * contention over shared processor resources / facilities, while the
67fb2f18f8Sesaxe  * affinity policies seek to improve cache and TLB utilization.
68fb2f18f8Sesaxe  *
69fb2f18f8Sesaxe  * The CMT PGs created by this class are already arranged into a
70fb2f18f8Sesaxe  * hierarchy (which is done in the pghw layer). To implement the top-down
71fb2f18f8Sesaxe  * CMT load balancing algorithm, the CMT PGs additionally maintain
72fb2f18f8Sesaxe  * parent, child and sibling hierarchy relationships.
73fb2f18f8Sesaxe  * Parent PGs always contain a superset of their children(s) resources,
74fb2f18f8Sesaxe  * each PG can have at most one parent, and siblings are the group of PGs
75fb2f18f8Sesaxe  * sharing the same parent.
76fb2f18f8Sesaxe  *
77fb2f18f8Sesaxe  * On NUMA systems, the CMT load balancing algorithm balances across the
78fb2f18f8Sesaxe  * CMT PGs within their respective lgroups. On UMA based system, there
79fb2f18f8Sesaxe  * exists a top level group of PGs to balance across. On NUMA systems multiple
80fb2f18f8Sesaxe  * top level groups are instantiated, where the top level balancing begins by
81fb2f18f8Sesaxe  * balancng across the CMT PGs within their respective (per lgroup) top level
82fb2f18f8Sesaxe  * groups.
83fb2f18f8Sesaxe  */
84fb2f18f8Sesaxe 
85fb2f18f8Sesaxe typedef struct cmt_lgrp {
86fb2f18f8Sesaxe 	group_t		cl_pgs;		/* Top level group of active CMT PGs */
87fb2f18f8Sesaxe 	int		cl_npgs;	/* # of top level PGs in the lgroup */
88fb2f18f8Sesaxe 	lgrp_handle_t	cl_hand;	/* lgroup's platform handle */
89fb2f18f8Sesaxe 	struct cmt_lgrp *cl_next;	/* next cmt_lgrp */
90fb2f18f8Sesaxe } cmt_lgrp_t;
91fb2f18f8Sesaxe 
92*a6604450Sesaxe static cmt_lgrp_t	*cmt_lgrps = NULL;	/* cmt_lgrps list head */
93*a6604450Sesaxe static cmt_lgrp_t	*cpu0_lgrp = NULL;	/* boot CPU's initial lgrp */
94*a6604450Sesaxe 						/* used for null_proc_lpa */
95fb2f18f8Sesaxe 
96*a6604450Sesaxe static int		is_cpu0 = 1; /* true if this is boot CPU context */
97*a6604450Sesaxe 
98*a6604450Sesaxe /*
99*a6604450Sesaxe  * Set this to non-zero to disable CMT scheduling
100*a6604450Sesaxe  * This must be done via kmdb -d, as /etc/system will be too late
101*a6604450Sesaxe  */
102fb2f18f8Sesaxe static int		cmt_sched_disabled = 0;
103fb2f18f8Sesaxe 
104fb2f18f8Sesaxe static pg_cid_t		pg_cmt_class_id;		/* PG class id */
105fb2f18f8Sesaxe 
106fb2f18f8Sesaxe static pg_t		*pg_cmt_alloc();
107fb2f18f8Sesaxe static void		pg_cmt_free(pg_t *);
108fb2f18f8Sesaxe static void		pg_cmt_cpu_init(cpu_t *);
109fb2f18f8Sesaxe static void		pg_cmt_cpu_fini(cpu_t *);
110fb2f18f8Sesaxe static void		pg_cmt_cpu_active(cpu_t *);
111fb2f18f8Sesaxe static void		pg_cmt_cpu_inactive(cpu_t *);
112fb2f18f8Sesaxe static void		pg_cmt_cpupart_in(cpu_t *, cpupart_t *);
113fb2f18f8Sesaxe static void		pg_cmt_cpupart_move(cpu_t *, cpupart_t *, cpupart_t *);
114fb2f18f8Sesaxe static void		pg_cmt_hier_pack(pg_cmt_t **, int);
115fb2f18f8Sesaxe static int		pg_cmt_cpu_belongs(pg_t *, cpu_t *);
116fb2f18f8Sesaxe static int		pg_cmt_hw(pghw_type_t);
117fb2f18f8Sesaxe static cmt_lgrp_t	*pg_cmt_find_lgrp(lgrp_handle_t);
118*a6604450Sesaxe static cmt_lgrp_t	*pg_cmt_lgrp_create(lgrp_handle_t);
119fb2f18f8Sesaxe 
120fb2f18f8Sesaxe /*
121fb2f18f8Sesaxe  * Macro to test if PG is managed by the CMT PG class
122fb2f18f8Sesaxe  */
123fb2f18f8Sesaxe #define	IS_CMT_PG(pg)	(((pg_t *)(pg))->pg_class->pgc_id == pg_cmt_class_id)
124fb2f18f8Sesaxe 
125fb2f18f8Sesaxe /*
126fb2f18f8Sesaxe  * CMT PG ops
127fb2f18f8Sesaxe  */
128fb2f18f8Sesaxe struct pg_ops pg_ops_cmt = {
129fb2f18f8Sesaxe 	pg_cmt_alloc,
130fb2f18f8Sesaxe 	pg_cmt_free,
131fb2f18f8Sesaxe 	pg_cmt_cpu_init,
132fb2f18f8Sesaxe 	pg_cmt_cpu_fini,
133fb2f18f8Sesaxe 	pg_cmt_cpu_active,
134fb2f18f8Sesaxe 	pg_cmt_cpu_inactive,
135fb2f18f8Sesaxe 	pg_cmt_cpupart_in,
136fb2f18f8Sesaxe 	NULL,			/* cpupart_out */
137fb2f18f8Sesaxe 	pg_cmt_cpupart_move,
138fb2f18f8Sesaxe 	pg_cmt_cpu_belongs,
139fb2f18f8Sesaxe };
140fb2f18f8Sesaxe 
141fb2f18f8Sesaxe /*
142fb2f18f8Sesaxe  * Initialize the CMT PG class
143fb2f18f8Sesaxe  */
144fb2f18f8Sesaxe void
145fb2f18f8Sesaxe pg_cmt_class_init(void)
146fb2f18f8Sesaxe {
147fb2f18f8Sesaxe 	if (cmt_sched_disabled)
148fb2f18f8Sesaxe 		return;
149fb2f18f8Sesaxe 
150fb2f18f8Sesaxe 	pg_cmt_class_id = pg_class_register("cmt", &pg_ops_cmt, PGR_PHYSICAL);
151fb2f18f8Sesaxe }
152fb2f18f8Sesaxe 
153fb2f18f8Sesaxe /*
154fb2f18f8Sesaxe  * Called to indicate a new CPU has started up so
155fb2f18f8Sesaxe  * that either t0 or the slave startup thread can
156fb2f18f8Sesaxe  * be accounted for.
157fb2f18f8Sesaxe  */
158fb2f18f8Sesaxe void
159fb2f18f8Sesaxe pg_cmt_cpu_startup(cpu_t *cp)
160fb2f18f8Sesaxe {
161fb2f18f8Sesaxe 	PG_NRUN_UPDATE(cp, 1);
162fb2f18f8Sesaxe }
163fb2f18f8Sesaxe 
164fb2f18f8Sesaxe /*
165fb2f18f8Sesaxe  * Adjust the CMT load in the CMT PGs in which the CPU belongs
166fb2f18f8Sesaxe  * Note that "n" can be positive in the case of increasing
167fb2f18f8Sesaxe  * load, or negative in the case of decreasing load.
168fb2f18f8Sesaxe  */
169fb2f18f8Sesaxe void
170fb2f18f8Sesaxe pg_cmt_load(cpu_t *cp, int n)
171fb2f18f8Sesaxe {
172fb2f18f8Sesaxe 	pg_cmt_t	*pg;
173fb2f18f8Sesaxe 
174fb2f18f8Sesaxe 	pg = (pg_cmt_t *)cp->cpu_pg->cmt_lineage;
175fb2f18f8Sesaxe 	while (pg != NULL) {
176fb2f18f8Sesaxe 		ASSERT(IS_CMT_PG(pg));
177fb2f18f8Sesaxe 		atomic_add_32(&pg->cmt_nrunning, n);
178fb2f18f8Sesaxe 		pg = pg->cmt_parent;
179fb2f18f8Sesaxe 	}
180fb2f18f8Sesaxe }
181fb2f18f8Sesaxe 
182fb2f18f8Sesaxe /*
183fb2f18f8Sesaxe  * Return non-zero if thread can migrate between "from" and "to"
184fb2f18f8Sesaxe  * without a performance penalty
185fb2f18f8Sesaxe  */
186fb2f18f8Sesaxe int
187fb2f18f8Sesaxe pg_cmt_can_migrate(cpu_t *from, cpu_t *to)
188fb2f18f8Sesaxe {
189fb2f18f8Sesaxe 	if (from->cpu_physid->cpu_cacheid ==
190fb2f18f8Sesaxe 	    to->cpu_physid->cpu_cacheid)
191fb2f18f8Sesaxe 		return (1);
192fb2f18f8Sesaxe 	return (0);
193fb2f18f8Sesaxe }
194fb2f18f8Sesaxe 
195fb2f18f8Sesaxe /*
196fb2f18f8Sesaxe  * CMT class specific PG allocation
197fb2f18f8Sesaxe  */
198fb2f18f8Sesaxe static pg_t *
199fb2f18f8Sesaxe pg_cmt_alloc(void)
200fb2f18f8Sesaxe {
201fb2f18f8Sesaxe 	return (kmem_zalloc(sizeof (pg_cmt_t), KM_NOSLEEP));
202fb2f18f8Sesaxe }
203fb2f18f8Sesaxe 
204fb2f18f8Sesaxe /*
205fb2f18f8Sesaxe  * Class specific PG de-allocation
206fb2f18f8Sesaxe  */
207fb2f18f8Sesaxe static void
208fb2f18f8Sesaxe pg_cmt_free(pg_t *pg)
209fb2f18f8Sesaxe {
210fb2f18f8Sesaxe 	ASSERT(pg != NULL);
211fb2f18f8Sesaxe 	ASSERT(IS_CMT_PG(pg));
212fb2f18f8Sesaxe 
213fb2f18f8Sesaxe 	kmem_free((pg_cmt_t *)pg, sizeof (pg_cmt_t));
214fb2f18f8Sesaxe }
215fb2f18f8Sesaxe 
216fb2f18f8Sesaxe /*
217fb2f18f8Sesaxe  * Return 1 if CMT load balancing policies should be
218fb2f18f8Sesaxe  * implemented across instances of the specified hardware
219fb2f18f8Sesaxe  * sharing relationship.
220fb2f18f8Sesaxe  */
221fb2f18f8Sesaxe static int
222fb2f18f8Sesaxe pg_cmt_load_bal_hw(pghw_type_t hw)
223fb2f18f8Sesaxe {
224fb2f18f8Sesaxe 	if (hw == PGHW_IPIPE ||
225fb2f18f8Sesaxe 	    hw == PGHW_FPU ||
226fb2f18f8Sesaxe 	    hw == PGHW_CHIP)
227fb2f18f8Sesaxe 		return (1);
228fb2f18f8Sesaxe 	else
229fb2f18f8Sesaxe 		return (0);
230fb2f18f8Sesaxe }
231fb2f18f8Sesaxe 
232fb2f18f8Sesaxe /*
233fb2f18f8Sesaxe  * Return 1 if thread affinity polices should be implemented
234fb2f18f8Sesaxe  * for instances of the specifed hardware sharing relationship.
235fb2f18f8Sesaxe  */
236fb2f18f8Sesaxe static int
237fb2f18f8Sesaxe pg_cmt_affinity_hw(pghw_type_t hw)
238fb2f18f8Sesaxe {
239fb2f18f8Sesaxe 	if (hw == PGHW_CACHE)
240fb2f18f8Sesaxe 		return (1);
241fb2f18f8Sesaxe 	else
242fb2f18f8Sesaxe 		return (0);
243fb2f18f8Sesaxe }
244fb2f18f8Sesaxe 
245fb2f18f8Sesaxe /*
246fb2f18f8Sesaxe  * Return 1 if CMT scheduling policies should be impelmented
247fb2f18f8Sesaxe  * for the specified hardware sharing relationship.
248fb2f18f8Sesaxe  */
249fb2f18f8Sesaxe static int
250fb2f18f8Sesaxe pg_cmt_hw(pghw_type_t hw)
251fb2f18f8Sesaxe {
252fb2f18f8Sesaxe 	return (pg_cmt_load_bal_hw(hw) ||
253fb2f18f8Sesaxe 	    pg_cmt_affinity_hw(hw));
254fb2f18f8Sesaxe }
255fb2f18f8Sesaxe 
256fb2f18f8Sesaxe /*
257fb2f18f8Sesaxe  * CMT class callback for a new CPU entering the system
258fb2f18f8Sesaxe  */
259fb2f18f8Sesaxe static void
260fb2f18f8Sesaxe pg_cmt_cpu_init(cpu_t *cp)
261fb2f18f8Sesaxe {
262fb2f18f8Sesaxe 	pg_cmt_t	*pg;
263fb2f18f8Sesaxe 	group_t		*cmt_pgs;
264fb2f18f8Sesaxe 	int		level, max_level, nlevels;
265fb2f18f8Sesaxe 	pghw_type_t	hw;
266fb2f18f8Sesaxe 	pg_t		*pg_cache = NULL;
267fb2f18f8Sesaxe 	pg_cmt_t	*cpu_cmt_hier[PGHW_NUM_COMPONENTS];
268fb2f18f8Sesaxe 	lgrp_handle_t	lgrp_handle;
269fb2f18f8Sesaxe 	cmt_lgrp_t	*lgrp;
270fb2f18f8Sesaxe 
271fb2f18f8Sesaxe 	ASSERT(MUTEX_HELD(&cpu_lock));
272fb2f18f8Sesaxe 
273fb2f18f8Sesaxe 	/*
274fb2f18f8Sesaxe 	 * A new CPU is coming into the system.
275fb2f18f8Sesaxe 	 * Interrogate the platform to see if the CPU
276fb2f18f8Sesaxe 	 * has any performance relevant CMT sharing
277fb2f18f8Sesaxe 	 * relationships
278fb2f18f8Sesaxe 	 */
279fb2f18f8Sesaxe 	cmt_pgs = &cp->cpu_pg->cmt_pgs;
280fb2f18f8Sesaxe 	cp->cpu_pg->cmt_lineage = NULL;
281fb2f18f8Sesaxe 
282fb2f18f8Sesaxe 	bzero(cpu_cmt_hier, sizeof (cpu_cmt_hier));
283fb2f18f8Sesaxe 	max_level = nlevels = 0;
284fb2f18f8Sesaxe 	for (hw = PGHW_START; hw < PGHW_NUM_COMPONENTS; hw++) {
285fb2f18f8Sesaxe 
286fb2f18f8Sesaxe 		/*
287fb2f18f8Sesaxe 		 * We're only interested in CMT hw sharing relationships
288fb2f18f8Sesaxe 		 */
289fb2f18f8Sesaxe 		if (pg_cmt_hw(hw) == 0 || pg_plat_hw_shared(cp, hw) == 0)
290fb2f18f8Sesaxe 			continue;
291fb2f18f8Sesaxe 
292fb2f18f8Sesaxe 		/*
293fb2f18f8Sesaxe 		 * Find (or create) the PG associated with
294fb2f18f8Sesaxe 		 * the hw sharing relationship in which cp
295fb2f18f8Sesaxe 		 * belongs.
296fb2f18f8Sesaxe 		 *
297fb2f18f8Sesaxe 		 * Determine if a suitable PG already
298fb2f18f8Sesaxe 		 * exists, or if one needs to be created.
299fb2f18f8Sesaxe 		 */
300fb2f18f8Sesaxe 		pg = (pg_cmt_t *)pghw_place_cpu(cp, hw);
301fb2f18f8Sesaxe 		if (pg == NULL) {
302fb2f18f8Sesaxe 			/*
303fb2f18f8Sesaxe 			 * Create a new one.
304fb2f18f8Sesaxe 			 * Initialize the common...
305fb2f18f8Sesaxe 			 */
306fb2f18f8Sesaxe 			pg = (pg_cmt_t *)pg_create(pg_cmt_class_id);
307fb2f18f8Sesaxe 
308fb2f18f8Sesaxe 			/* ... physical ... */
309fb2f18f8Sesaxe 			pghw_init((pghw_t *)pg, cp, hw);
310fb2f18f8Sesaxe 
311fb2f18f8Sesaxe 			/*
312fb2f18f8Sesaxe 			 * ... and CMT specific portions of the
313fb2f18f8Sesaxe 			 * structure.
314fb2f18f8Sesaxe 			 */
315fb2f18f8Sesaxe 			bitset_init(&pg->cmt_cpus_actv_set);
316fb2f18f8Sesaxe 			group_create(&pg->cmt_cpus_actv);
317fb2f18f8Sesaxe 		} else {
318fb2f18f8Sesaxe 			ASSERT(IS_CMT_PG(pg));
319fb2f18f8Sesaxe 		}
320fb2f18f8Sesaxe 
321fb2f18f8Sesaxe 		/* Add the CPU to the PG */
322fb2f18f8Sesaxe 		pg_cpu_add((pg_t *)pg, cp);
323fb2f18f8Sesaxe 
324fb2f18f8Sesaxe 		/*
325fb2f18f8Sesaxe 		 * Ensure capacity of the active CPUs group/bitset
326fb2f18f8Sesaxe 		 */
327fb2f18f8Sesaxe 		group_expand(&pg->cmt_cpus_actv,
328fb2f18f8Sesaxe 		    GROUP_SIZE(&((pg_t *)pg)->pg_cpus));
329fb2f18f8Sesaxe 
330fb2f18f8Sesaxe 		if (cp->cpu_seqid >=
331fb2f18f8Sesaxe 		    bitset_capacity(&pg->cmt_cpus_actv_set)) {
332fb2f18f8Sesaxe 			bitset_resize(&pg->cmt_cpus_actv_set,
333fb2f18f8Sesaxe 			    cp->cpu_seqid + 1);
334fb2f18f8Sesaxe 		}
335fb2f18f8Sesaxe 
336fb2f18f8Sesaxe 		/*
337fb2f18f8Sesaxe 		 * Build a lineage of CMT PGs for load balancing
338fb2f18f8Sesaxe 		 */
339fb2f18f8Sesaxe 		if (pg_cmt_load_bal_hw(hw)) {
340fb2f18f8Sesaxe 			level = pghw_level(hw);
341fb2f18f8Sesaxe 			cpu_cmt_hier[level] = pg;
342fb2f18f8Sesaxe 			if (level > max_level)
343fb2f18f8Sesaxe 				max_level = level;
344fb2f18f8Sesaxe 			nlevels++;
345fb2f18f8Sesaxe 		}
346fb2f18f8Sesaxe 
347fb2f18f8Sesaxe 		/* Cache this for later */
348fb2f18f8Sesaxe 		if (hw == PGHW_CACHE)
349fb2f18f8Sesaxe 			pg_cache = (pg_t *)pg;
350fb2f18f8Sesaxe 	}
351fb2f18f8Sesaxe 
352fb2f18f8Sesaxe 	/*
353fb2f18f8Sesaxe 	 * Pack out any gaps in the constructed lineage.
354fb2f18f8Sesaxe 	 * Gaps may exist where the architecture knows
355fb2f18f8Sesaxe 	 * about a hardware sharing relationship, but such a
356fb2f18f8Sesaxe 	 * relationship either isn't relevant for load
357fb2f18f8Sesaxe 	 * balancing or doesn't exist between CPUs on the system.
358fb2f18f8Sesaxe 	 */
359fb2f18f8Sesaxe 	pg_cmt_hier_pack(cpu_cmt_hier, max_level + 1);
360fb2f18f8Sesaxe 
361fb2f18f8Sesaxe 	/*
362fb2f18f8Sesaxe 	 * For each of the PGs int the CPU's lineage:
363fb2f18f8Sesaxe 	 *	- Add an entry in the CPU sorted CMT PG group
364fb2f18f8Sesaxe 	 *	  which is used for top down CMT load balancing
365fb2f18f8Sesaxe 	 *	- Tie the PG into the CMT hierarchy by connecting
366fb2f18f8Sesaxe 	 *	  it to it's parent and siblings.
367fb2f18f8Sesaxe 	 */
368fb2f18f8Sesaxe 	group_expand(cmt_pgs, nlevels);
369fb2f18f8Sesaxe 
370fb2f18f8Sesaxe 	/*
371fb2f18f8Sesaxe 	 * Find the lgrp that encapsulates this CPU's CMT hierarchy
372fb2f18f8Sesaxe 	 */
373fb2f18f8Sesaxe 	lgrp_handle = lgrp_plat_cpu_to_hand(cp->cpu_id);
374fb2f18f8Sesaxe 	lgrp = pg_cmt_find_lgrp(lgrp_handle);
375*a6604450Sesaxe 	if (lgrp == NULL)
376*a6604450Sesaxe 		lgrp = pg_cmt_lgrp_create(lgrp_handle);
377fb2f18f8Sesaxe 
378fb2f18f8Sesaxe 	for (level = 0; level < nlevels; level++) {
379fb2f18f8Sesaxe 		uint_t		children;
380fb2f18f8Sesaxe 		int		err;
381fb2f18f8Sesaxe 
382fb2f18f8Sesaxe 		pg = cpu_cmt_hier[level];
383fb2f18f8Sesaxe 		err = group_add_at(cmt_pgs, pg, nlevels - level - 1);
384fb2f18f8Sesaxe 		ASSERT(err == 0);
385fb2f18f8Sesaxe 
386fb2f18f8Sesaxe 		if (level == 0)
387fb2f18f8Sesaxe 			cp->cpu_pg->cmt_lineage = (pg_t *)pg;
388fb2f18f8Sesaxe 
389fb2f18f8Sesaxe 		if (pg->cmt_siblings != NULL) {
390fb2f18f8Sesaxe 			/* Already initialized */
391fb2f18f8Sesaxe 			ASSERT(pg->cmt_parent == NULL ||
392fb2f18f8Sesaxe 			    pg->cmt_parent == cpu_cmt_hier[level + 1]);
393fb2f18f8Sesaxe 			ASSERT(pg->cmt_siblings == &lgrp->cl_pgs ||
394fb2f18f8Sesaxe 			    pg->cmt_siblings == pg->cmt_parent->cmt_children);
395fb2f18f8Sesaxe 			continue;
396fb2f18f8Sesaxe 		}
397fb2f18f8Sesaxe 
398fb2f18f8Sesaxe 		if ((level + 1) == nlevels) {
399fb2f18f8Sesaxe 			pg->cmt_parent = NULL;
400fb2f18f8Sesaxe 			pg->cmt_siblings = &lgrp->cl_pgs;
401fb2f18f8Sesaxe 			children = ++lgrp->cl_npgs;
402fb2f18f8Sesaxe 		} else {
403fb2f18f8Sesaxe 			pg->cmt_parent = cpu_cmt_hier[level + 1];
404fb2f18f8Sesaxe 
405fb2f18f8Sesaxe 			/*
406fb2f18f8Sesaxe 			 * A good parent keeps track of their children.
407fb2f18f8Sesaxe 			 * The parent's children group is also the PG's
408fb2f18f8Sesaxe 			 * siblings.
409fb2f18f8Sesaxe 			 */
410fb2f18f8Sesaxe 			if (pg->cmt_parent->cmt_children == NULL) {
411fb2f18f8Sesaxe 				pg->cmt_parent->cmt_children =
412fb2f18f8Sesaxe 				    kmem_zalloc(sizeof (group_t), KM_SLEEP);
413fb2f18f8Sesaxe 				group_create(pg->cmt_parent->cmt_children);
414fb2f18f8Sesaxe 			}
415fb2f18f8Sesaxe 			pg->cmt_siblings = pg->cmt_parent->cmt_children;
416fb2f18f8Sesaxe 			children = ++pg->cmt_parent->cmt_nchildren;
417fb2f18f8Sesaxe 		}
418fb2f18f8Sesaxe 		pg->cmt_hint = 0;
419fb2f18f8Sesaxe 		group_expand(pg->cmt_siblings, children);
420fb2f18f8Sesaxe 	}
421fb2f18f8Sesaxe 
422fb2f18f8Sesaxe 	/*
423fb2f18f8Sesaxe 	 * Cache the chip and core IDs in the cpu_t->cpu_physid structure
424fb2f18f8Sesaxe 	 * for fast lookups later.
425fb2f18f8Sesaxe 	 */
426fb2f18f8Sesaxe 	if (cp->cpu_physid) {
427fb2f18f8Sesaxe 		cp->cpu_physid->cpu_chipid =
428fb2f18f8Sesaxe 		    pg_plat_hw_instance_id(cp, PGHW_CHIP);
429fb2f18f8Sesaxe 		cp->cpu_physid->cpu_coreid = pg_plat_get_core_id(cp);
430fb2f18f8Sesaxe 
431fb2f18f8Sesaxe 		/*
432fb2f18f8Sesaxe 		 * If this cpu has a PG representing shared cache, then set
433fb2f18f8Sesaxe 		 * cpu_cacheid to that PG's logical id
434fb2f18f8Sesaxe 		 */
435fb2f18f8Sesaxe 		if (pg_cache)
436fb2f18f8Sesaxe 			cp->cpu_physid->cpu_cacheid = pg_cache->pg_id;
437fb2f18f8Sesaxe 	}
438fb2f18f8Sesaxe 
439fb2f18f8Sesaxe 	/* CPU0 only initialization */
440fb2f18f8Sesaxe 	if (is_cpu0) {
441fb2f18f8Sesaxe 		pg_cmt_cpu_startup(cp);
442fb2f18f8Sesaxe 		is_cpu0 = 0;
443*a6604450Sesaxe 		cpu0_lgrp = lgrp;
444fb2f18f8Sesaxe 	}
445fb2f18f8Sesaxe 
446fb2f18f8Sesaxe }
447fb2f18f8Sesaxe 
448fb2f18f8Sesaxe /*
449fb2f18f8Sesaxe  * Class callback when a CPU is leaving the system (deletion)
450fb2f18f8Sesaxe  */
451fb2f18f8Sesaxe static void
452fb2f18f8Sesaxe pg_cmt_cpu_fini(cpu_t *cp)
453fb2f18f8Sesaxe {
454fb2f18f8Sesaxe 	group_iter_t	i;
455fb2f18f8Sesaxe 	pg_cmt_t	*pg;
456fb2f18f8Sesaxe 	group_t		*pgs, *cmt_pgs;
457fb2f18f8Sesaxe 	lgrp_handle_t	lgrp_handle;
458fb2f18f8Sesaxe 	cmt_lgrp_t	*lgrp;
459fb2f18f8Sesaxe 
460fb2f18f8Sesaxe 	pgs = &cp->cpu_pg->pgs;
461fb2f18f8Sesaxe 	cmt_pgs = &cp->cpu_pg->cmt_pgs;
462fb2f18f8Sesaxe 
463fb2f18f8Sesaxe 	/*
464fb2f18f8Sesaxe 	 * Find the lgroup that encapsulates this CPU's CMT hierarchy
465fb2f18f8Sesaxe 	 */
466fb2f18f8Sesaxe 	lgrp_handle = lgrp_plat_cpu_to_hand(cp->cpu_id);
467*a6604450Sesaxe 
468fb2f18f8Sesaxe 	lgrp = pg_cmt_find_lgrp(lgrp_handle);
469*a6604450Sesaxe 	if (lgrp == NULL) {
470*a6604450Sesaxe 		/*
471*a6604450Sesaxe 		 * This is a bit of a special case.
472*a6604450Sesaxe 		 * The only way this can happen is if the CPU's lgrp
473*a6604450Sesaxe 		 * handle changed out from underneath us, which is what
474*a6604450Sesaxe 		 * happens with null_proc_lpa on starcat systems.
475*a6604450Sesaxe 		 *
476*a6604450Sesaxe 		 * Use the initial boot CPU lgrp, since this is what
477*a6604450Sesaxe 		 * we need to tear down.
478*a6604450Sesaxe 		 */
479*a6604450Sesaxe 		lgrp = cpu0_lgrp;
480*a6604450Sesaxe 	}
481fb2f18f8Sesaxe 
482fb2f18f8Sesaxe 	/*
483fb2f18f8Sesaxe 	 * First, clean up anything load balancing specific for each of
484fb2f18f8Sesaxe 	 * the CPU's PGs that participated in CMT load balancing
485fb2f18f8Sesaxe 	 */
486fb2f18f8Sesaxe 	pg = (pg_cmt_t *)cp->cpu_pg->cmt_lineage;
487fb2f18f8Sesaxe 	while (pg != NULL) {
488fb2f18f8Sesaxe 
489fb2f18f8Sesaxe 		/*
490fb2f18f8Sesaxe 		 * Remove the PG from the CPU's load balancing lineage
491fb2f18f8Sesaxe 		 */
492fb2f18f8Sesaxe 		(void) group_remove(cmt_pgs, pg, GRP_RESIZE);
493fb2f18f8Sesaxe 
494fb2f18f8Sesaxe 		/*
495fb2f18f8Sesaxe 		 * If it's about to become empty, destroy it's children
496fb2f18f8Sesaxe 		 * group, and remove it's reference from it's siblings.
497fb2f18f8Sesaxe 		 * This is done here (rather than below) to avoid removing
498fb2f18f8Sesaxe 		 * our reference from a PG that we just eliminated.
499fb2f18f8Sesaxe 		 */
500fb2f18f8Sesaxe 		if (GROUP_SIZE(&((pg_t *)pg)->pg_cpus) == 1) {
501fb2f18f8Sesaxe 			if (pg->cmt_children != NULL)
502fb2f18f8Sesaxe 				group_destroy(pg->cmt_children);
503fb2f18f8Sesaxe 			if (pg->cmt_siblings != NULL) {
504fb2f18f8Sesaxe 				if (pg->cmt_siblings == &lgrp->cl_pgs)
505fb2f18f8Sesaxe 					lgrp->cl_npgs--;
506fb2f18f8Sesaxe 				else
507fb2f18f8Sesaxe 					pg->cmt_parent->cmt_nchildren--;
508fb2f18f8Sesaxe 			}
509fb2f18f8Sesaxe 		}
510fb2f18f8Sesaxe 		pg = pg->cmt_parent;
511fb2f18f8Sesaxe 	}
512fb2f18f8Sesaxe 
513fb2f18f8Sesaxe 	ASSERT(GROUP_SIZE(cmt_pgs) == 0);
514fb2f18f8Sesaxe 
515fb2f18f8Sesaxe 	/*
516fb2f18f8Sesaxe 	 * Now that the load balancing lineage updates have happened,
517fb2f18f8Sesaxe 	 * remove the CPU from all it's PGs (destroying any that become
518fb2f18f8Sesaxe 	 * empty).
519fb2f18f8Sesaxe 	 */
520fb2f18f8Sesaxe 	group_iter_init(&i);
521fb2f18f8Sesaxe 	while ((pg = group_iterate(pgs, &i)) != NULL) {
522fb2f18f8Sesaxe 		if (IS_CMT_PG(pg) == 0)
523fb2f18f8Sesaxe 			continue;
524fb2f18f8Sesaxe 
525fb2f18f8Sesaxe 		pg_cpu_delete((pg_t *)pg, cp);
526fb2f18f8Sesaxe 		/*
527fb2f18f8Sesaxe 		 * Deleting the CPU from the PG changes the CPU's
528fb2f18f8Sesaxe 		 * PG group over which we are actively iterating
529fb2f18f8Sesaxe 		 * Re-initialize the iteration
530fb2f18f8Sesaxe 		 */
531fb2f18f8Sesaxe 		group_iter_init(&i);
532fb2f18f8Sesaxe 
533fb2f18f8Sesaxe 		if (GROUP_SIZE(&((pg_t *)pg)->pg_cpus) == 0) {
534fb2f18f8Sesaxe 
535fb2f18f8Sesaxe 			/*
536fb2f18f8Sesaxe 			 * The PG has become zero sized, so destroy it.
537fb2f18f8Sesaxe 			 */
538fb2f18f8Sesaxe 			group_destroy(&pg->cmt_cpus_actv);
539fb2f18f8Sesaxe 			bitset_fini(&pg->cmt_cpus_actv_set);
540fb2f18f8Sesaxe 			pghw_fini((pghw_t *)pg);
541fb2f18f8Sesaxe 
542fb2f18f8Sesaxe 			pg_destroy((pg_t *)pg);
543fb2f18f8Sesaxe 		}
544fb2f18f8Sesaxe 	}
545fb2f18f8Sesaxe }
546fb2f18f8Sesaxe 
547fb2f18f8Sesaxe /*
548fb2f18f8Sesaxe  * Class callback when a CPU is entering a cpu partition
549fb2f18f8Sesaxe  */
550fb2f18f8Sesaxe static void
551fb2f18f8Sesaxe pg_cmt_cpupart_in(cpu_t *cp, cpupart_t *pp)
552fb2f18f8Sesaxe {
553fb2f18f8Sesaxe 	group_t		*pgs;
554fb2f18f8Sesaxe 	pg_t		*pg;
555fb2f18f8Sesaxe 	group_iter_t	i;
556fb2f18f8Sesaxe 
557fb2f18f8Sesaxe 	ASSERT(MUTEX_HELD(&cpu_lock));
558fb2f18f8Sesaxe 
559fb2f18f8Sesaxe 	pgs = &cp->cpu_pg->pgs;
560fb2f18f8Sesaxe 
561fb2f18f8Sesaxe 	/*
562fb2f18f8Sesaxe 	 * Ensure that the new partition's PG bitset
563fb2f18f8Sesaxe 	 * is large enough for all CMT PG's to which cp
564fb2f18f8Sesaxe 	 * belongs
565fb2f18f8Sesaxe 	 */
566fb2f18f8Sesaxe 	group_iter_init(&i);
567fb2f18f8Sesaxe 	while ((pg = group_iterate(pgs, &i)) != NULL) {
568fb2f18f8Sesaxe 		if (IS_CMT_PG(pg) == 0)
569fb2f18f8Sesaxe 			continue;
570fb2f18f8Sesaxe 
571fb2f18f8Sesaxe 		if (bitset_capacity(&pp->cp_cmt_pgs) <= pg->pg_id)
572fb2f18f8Sesaxe 			bitset_resize(&pp->cp_cmt_pgs, pg->pg_id + 1);
573fb2f18f8Sesaxe 	}
574fb2f18f8Sesaxe }
575fb2f18f8Sesaxe 
576fb2f18f8Sesaxe /*
577fb2f18f8Sesaxe  * Class callback when a CPU is actually moving partitions
578fb2f18f8Sesaxe  */
579fb2f18f8Sesaxe static void
580fb2f18f8Sesaxe pg_cmt_cpupart_move(cpu_t *cp, cpupart_t *oldpp, cpupart_t *newpp)
581fb2f18f8Sesaxe {
582fb2f18f8Sesaxe 	cpu_t		*cpp;
583fb2f18f8Sesaxe 	group_t		*pgs;
584fb2f18f8Sesaxe 	pg_t		*pg;
585fb2f18f8Sesaxe 	group_iter_t	pg_iter;
586fb2f18f8Sesaxe 	pg_cpu_itr_t	cpu_iter;
587fb2f18f8Sesaxe 	boolean_t	found;
588fb2f18f8Sesaxe 
589fb2f18f8Sesaxe 	ASSERT(MUTEX_HELD(&cpu_lock));
590fb2f18f8Sesaxe 
591fb2f18f8Sesaxe 	pgs = &cp->cpu_pg->pgs;
592fb2f18f8Sesaxe 	group_iter_init(&pg_iter);
593fb2f18f8Sesaxe 
594fb2f18f8Sesaxe 	/*
595fb2f18f8Sesaxe 	 * Iterate over the CPUs CMT PGs
596fb2f18f8Sesaxe 	 */
597fb2f18f8Sesaxe 	while ((pg = group_iterate(pgs, &pg_iter)) != NULL) {
598fb2f18f8Sesaxe 
599fb2f18f8Sesaxe 		if (IS_CMT_PG(pg) == 0)
600fb2f18f8Sesaxe 			continue;
601fb2f18f8Sesaxe 
602fb2f18f8Sesaxe 		/*
603fb2f18f8Sesaxe 		 * Add the PG to the bitset in the new partition.
604fb2f18f8Sesaxe 		 */
605fb2f18f8Sesaxe 		bitset_add(&newpp->cp_cmt_pgs, pg->pg_id);
606fb2f18f8Sesaxe 
607fb2f18f8Sesaxe 		/*
608fb2f18f8Sesaxe 		 * Remove the PG from the bitset in the old partition
609fb2f18f8Sesaxe 		 * if the last of the PG's CPUs have left.
610fb2f18f8Sesaxe 		 */
611fb2f18f8Sesaxe 		found = B_FALSE;
612fb2f18f8Sesaxe 		PG_CPU_ITR_INIT(pg, cpu_iter);
613fb2f18f8Sesaxe 		while ((cpp = pg_cpu_next(&cpu_iter)) != NULL) {
614fb2f18f8Sesaxe 			if (cpp == cp)
615fb2f18f8Sesaxe 				continue;
616*a6604450Sesaxe 			if (CPU_ACTIVE(cpp) &&
617*a6604450Sesaxe 			    cpp->cpu_part->cp_id == oldpp->cp_id) {
618fb2f18f8Sesaxe 				found = B_TRUE;
619fb2f18f8Sesaxe 				break;
620fb2f18f8Sesaxe 			}
621fb2f18f8Sesaxe 		}
622fb2f18f8Sesaxe 		if (!found)
623fb2f18f8Sesaxe 			bitset_del(&cp->cpu_part->cp_cmt_pgs, pg->pg_id);
624fb2f18f8Sesaxe 	}
625fb2f18f8Sesaxe }
626fb2f18f8Sesaxe 
627fb2f18f8Sesaxe /*
628fb2f18f8Sesaxe  * Class callback when a CPU becomes active (online)
629fb2f18f8Sesaxe  *
630fb2f18f8Sesaxe  * This is called in a context where CPUs are paused
631fb2f18f8Sesaxe  */
632fb2f18f8Sesaxe static void
633fb2f18f8Sesaxe pg_cmt_cpu_active(cpu_t *cp)
634fb2f18f8Sesaxe {
635fb2f18f8Sesaxe 	int		err;
636fb2f18f8Sesaxe 	group_iter_t	i;
637fb2f18f8Sesaxe 	pg_cmt_t	*pg;
638fb2f18f8Sesaxe 	group_t		*pgs;
639fb2f18f8Sesaxe 
640fb2f18f8Sesaxe 	ASSERT(MUTEX_HELD(&cpu_lock));
641fb2f18f8Sesaxe 
642fb2f18f8Sesaxe 	pgs = &cp->cpu_pg->pgs;
643fb2f18f8Sesaxe 	group_iter_init(&i);
644fb2f18f8Sesaxe 
645fb2f18f8Sesaxe 	/*
646fb2f18f8Sesaxe 	 * Iterate over the CPU's PGs
647fb2f18f8Sesaxe 	 */
648fb2f18f8Sesaxe 	while ((pg = group_iterate(pgs, &i)) != NULL) {
649fb2f18f8Sesaxe 
650fb2f18f8Sesaxe 		if (IS_CMT_PG(pg) == 0)
651fb2f18f8Sesaxe 			continue;
652fb2f18f8Sesaxe 
653fb2f18f8Sesaxe 		err = group_add(&pg->cmt_cpus_actv, cp, GRP_NORESIZE);
654fb2f18f8Sesaxe 		ASSERT(err == 0);
655fb2f18f8Sesaxe 
656fb2f18f8Sesaxe 		/*
657fb2f18f8Sesaxe 		 * If this is the first active CPU in the PG, and it
658fb2f18f8Sesaxe 		 * represents a hardware sharing relationship over which
659fb2f18f8Sesaxe 		 * CMT load balancing is performed, add it as a candidate
660fb2f18f8Sesaxe 		 * for balancing with it's siblings.
661fb2f18f8Sesaxe 		 */
662fb2f18f8Sesaxe 		if (GROUP_SIZE(&pg->cmt_cpus_actv) == 1 &&
663fb2f18f8Sesaxe 		    pg_cmt_load_bal_hw(((pghw_t *)pg)->pghw_hw)) {
664fb2f18f8Sesaxe 			err = group_add(pg->cmt_siblings, pg, GRP_NORESIZE);
665fb2f18f8Sesaxe 			ASSERT(err == 0);
666fb2f18f8Sesaxe 		}
667fb2f18f8Sesaxe 
668fb2f18f8Sesaxe 		/*
669fb2f18f8Sesaxe 		 * Notate the CPU in the PGs active CPU bitset.
670fb2f18f8Sesaxe 		 * Also notate the PG as being active in it's associated
671fb2f18f8Sesaxe 		 * partition
672fb2f18f8Sesaxe 		 */
673fb2f18f8Sesaxe 		bitset_add(&pg->cmt_cpus_actv_set, cp->cpu_seqid);
674fb2f18f8Sesaxe 		bitset_add(&cp->cpu_part->cp_cmt_pgs, ((pg_t *)pg)->pg_id);
675fb2f18f8Sesaxe 	}
676fb2f18f8Sesaxe }
677fb2f18f8Sesaxe 
678fb2f18f8Sesaxe /*
679fb2f18f8Sesaxe  * Class callback when a CPU goes inactive (offline)
680fb2f18f8Sesaxe  *
681fb2f18f8Sesaxe  * This is called in a context where CPUs are paused
682fb2f18f8Sesaxe  */
683fb2f18f8Sesaxe static void
684fb2f18f8Sesaxe pg_cmt_cpu_inactive(cpu_t *cp)
685fb2f18f8Sesaxe {
686fb2f18f8Sesaxe 	int		err;
687fb2f18f8Sesaxe 	group_t		*pgs;
688fb2f18f8Sesaxe 	pg_cmt_t	*pg;
689fb2f18f8Sesaxe 	cpu_t		*cpp;
690fb2f18f8Sesaxe 	group_iter_t	i;
691fb2f18f8Sesaxe 	pg_cpu_itr_t	cpu_itr;
692fb2f18f8Sesaxe 	boolean_t	found;
693fb2f18f8Sesaxe 
694fb2f18f8Sesaxe 	ASSERT(MUTEX_HELD(&cpu_lock));
695fb2f18f8Sesaxe 
696fb2f18f8Sesaxe 	pgs = &cp->cpu_pg->pgs;
697fb2f18f8Sesaxe 	group_iter_init(&i);
698fb2f18f8Sesaxe 
699fb2f18f8Sesaxe 	while ((pg = group_iterate(pgs, &i)) != NULL) {
700fb2f18f8Sesaxe 
701fb2f18f8Sesaxe 		if (IS_CMT_PG(pg) == 0)
702fb2f18f8Sesaxe 			continue;
703fb2f18f8Sesaxe 
704fb2f18f8Sesaxe 		/*
705fb2f18f8Sesaxe 		 * Remove the CPU from the CMT PGs active CPU group
706fb2f18f8Sesaxe 		 * bitmap
707fb2f18f8Sesaxe 		 */
708fb2f18f8Sesaxe 		err = group_remove(&pg->cmt_cpus_actv, cp, GRP_NORESIZE);
709fb2f18f8Sesaxe 		ASSERT(err == 0);
710fb2f18f8Sesaxe 
711fb2f18f8Sesaxe 		bitset_del(&pg->cmt_cpus_actv_set, cp->cpu_seqid);
712fb2f18f8Sesaxe 
713fb2f18f8Sesaxe 		/*
714fb2f18f8Sesaxe 		 * If there are no more active CPUs in this PG over which
715fb2f18f8Sesaxe 		 * load was balanced, remove it as a balancing candidate.
716fb2f18f8Sesaxe 		 */
717fb2f18f8Sesaxe 		if (GROUP_SIZE(&pg->cmt_cpus_actv) == 0 &&
718fb2f18f8Sesaxe 		    pg_cmt_load_bal_hw(((pghw_t *)pg)->pghw_hw)) {
719fb2f18f8Sesaxe 			err = group_remove(pg->cmt_siblings, pg, GRP_NORESIZE);
720fb2f18f8Sesaxe 			ASSERT(err == 0);
721fb2f18f8Sesaxe 		}
722fb2f18f8Sesaxe 
723fb2f18f8Sesaxe 		/*
724fb2f18f8Sesaxe 		 * Assert the number of active CPUs does not exceed
725fb2f18f8Sesaxe 		 * the total number of CPUs in the PG
726fb2f18f8Sesaxe 		 */
727fb2f18f8Sesaxe 		ASSERT(GROUP_SIZE(&pg->cmt_cpus_actv) <=
728fb2f18f8Sesaxe 		    GROUP_SIZE(&((pg_t *)pg)->pg_cpus));
729fb2f18f8Sesaxe 
730fb2f18f8Sesaxe 		/*
731fb2f18f8Sesaxe 		 * Update the PG bitset in the CPU's old partition
732fb2f18f8Sesaxe 		 */
733fb2f18f8Sesaxe 		found = B_FALSE;
734fb2f18f8Sesaxe 		PG_CPU_ITR_INIT(pg, cpu_itr);
735fb2f18f8Sesaxe 		while ((cpp = pg_cpu_next(&cpu_itr)) != NULL) {
736fb2f18f8Sesaxe 			if (cpp == cp)
737fb2f18f8Sesaxe 				continue;
738*a6604450Sesaxe 			if (CPU_ACTIVE(cpp) &&
739*a6604450Sesaxe 			    cpp->cpu_part->cp_id == cp->cpu_part->cp_id) {
740fb2f18f8Sesaxe 				found = B_TRUE;
741fb2f18f8Sesaxe 				break;
742fb2f18f8Sesaxe 			}
743fb2f18f8Sesaxe 		}
744fb2f18f8Sesaxe 		if (!found) {
745fb2f18f8Sesaxe 			bitset_del(&cp->cpu_part->cp_cmt_pgs,
746fb2f18f8Sesaxe 			    ((pg_t *)pg)->pg_id);
747fb2f18f8Sesaxe 		}
748fb2f18f8Sesaxe 	}
749fb2f18f8Sesaxe }
750fb2f18f8Sesaxe 
751fb2f18f8Sesaxe /*
752fb2f18f8Sesaxe  * Return non-zero if the CPU belongs in the given PG
753fb2f18f8Sesaxe  */
754fb2f18f8Sesaxe static int
755fb2f18f8Sesaxe pg_cmt_cpu_belongs(pg_t *pg, cpu_t *cp)
756fb2f18f8Sesaxe {
757fb2f18f8Sesaxe 	cpu_t	*pg_cpu;
758fb2f18f8Sesaxe 
759fb2f18f8Sesaxe 	pg_cpu = GROUP_ACCESS(&pg->pg_cpus, 0);
760fb2f18f8Sesaxe 
761fb2f18f8Sesaxe 	ASSERT(pg_cpu != NULL);
762fb2f18f8Sesaxe 
763fb2f18f8Sesaxe 	/*
764fb2f18f8Sesaxe 	 * The CPU belongs if, given the nature of the hardware sharing
765fb2f18f8Sesaxe 	 * relationship represented by the PG, the CPU has that
766fb2f18f8Sesaxe 	 * relationship with some other CPU already in the PG
767fb2f18f8Sesaxe 	 */
768fb2f18f8Sesaxe 	if (pg_plat_cpus_share(cp, pg_cpu, ((pghw_t *)pg)->pghw_hw))
769fb2f18f8Sesaxe 		return (1);
770fb2f18f8Sesaxe 
771fb2f18f8Sesaxe 	return (0);
772fb2f18f8Sesaxe }
773fb2f18f8Sesaxe 
774fb2f18f8Sesaxe /*
775fb2f18f8Sesaxe  * Pack the CPUs CMT hierarchy
776fb2f18f8Sesaxe  * The hierarchy order is preserved
777fb2f18f8Sesaxe  */
778fb2f18f8Sesaxe static void
779fb2f18f8Sesaxe pg_cmt_hier_pack(pg_cmt_t *hier[], int sz)
780fb2f18f8Sesaxe {
781fb2f18f8Sesaxe 	int	i, j;
782fb2f18f8Sesaxe 
783fb2f18f8Sesaxe 	for (i = 0; i < sz; i++) {
784fb2f18f8Sesaxe 		if (hier[i] != NULL)
785fb2f18f8Sesaxe 			continue;
786fb2f18f8Sesaxe 
787fb2f18f8Sesaxe 		for (j = i; j < sz; j++) {
788fb2f18f8Sesaxe 			if (hier[j] != NULL) {
789fb2f18f8Sesaxe 				hier[i] = hier[j];
790fb2f18f8Sesaxe 				hier[j] = NULL;
791fb2f18f8Sesaxe 				break;
792fb2f18f8Sesaxe 			}
793fb2f18f8Sesaxe 		}
794fb2f18f8Sesaxe 		if (j == sz)
795fb2f18f8Sesaxe 			break;
796fb2f18f8Sesaxe 	}
797fb2f18f8Sesaxe }
798fb2f18f8Sesaxe 
799fb2f18f8Sesaxe /*
800fb2f18f8Sesaxe  * Return a cmt_lgrp_t * given an lgroup handle.
801fb2f18f8Sesaxe  */
802fb2f18f8Sesaxe static cmt_lgrp_t *
803fb2f18f8Sesaxe pg_cmt_find_lgrp(lgrp_handle_t hand)
804fb2f18f8Sesaxe {
805fb2f18f8Sesaxe 	cmt_lgrp_t	*lgrp;
806fb2f18f8Sesaxe 
807fb2f18f8Sesaxe 	ASSERT(MUTEX_HELD(&cpu_lock));
808fb2f18f8Sesaxe 
809fb2f18f8Sesaxe 	lgrp = cmt_lgrps;
810fb2f18f8Sesaxe 	while (lgrp != NULL) {
811fb2f18f8Sesaxe 		if (lgrp->cl_hand == hand)
812*a6604450Sesaxe 			break;
813fb2f18f8Sesaxe 		lgrp = lgrp->cl_next;
814fb2f18f8Sesaxe 	}
815*a6604450Sesaxe 	return (lgrp);
816*a6604450Sesaxe }
817fb2f18f8Sesaxe 
818fb2f18f8Sesaxe /*
819*a6604450Sesaxe  * Create a cmt_lgrp_t with the specified handle.
820fb2f18f8Sesaxe  */
821*a6604450Sesaxe static cmt_lgrp_t *
822*a6604450Sesaxe pg_cmt_lgrp_create(lgrp_handle_t hand)
823*a6604450Sesaxe {
824*a6604450Sesaxe 	cmt_lgrp_t	*lgrp;
825*a6604450Sesaxe 
826*a6604450Sesaxe 	ASSERT(MUTEX_HELD(&cpu_lock));
827*a6604450Sesaxe 
828fb2f18f8Sesaxe 	lgrp = kmem_zalloc(sizeof (cmt_lgrp_t), KM_SLEEP);
829fb2f18f8Sesaxe 
830fb2f18f8Sesaxe 	lgrp->cl_hand = hand;
831fb2f18f8Sesaxe 	lgrp->cl_npgs = 0;
832fb2f18f8Sesaxe 	lgrp->cl_next = cmt_lgrps;
833fb2f18f8Sesaxe 	cmt_lgrps = lgrp;
834fb2f18f8Sesaxe 	group_create(&lgrp->cl_pgs);
835fb2f18f8Sesaxe 
836fb2f18f8Sesaxe 	return (lgrp);
837fb2f18f8Sesaxe }
838