xref: /illumos-gate/usr/src/uts/common/disp/cmt.c (revision 437220cd296f6d8b6654d6d52508b40b1e2d1ac7)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 #pragma ident	"%Z%%M%	%I%	%E% SMI"
27 
28 #include <sys/systm.h>
29 #include <sys/types.h>
30 #include <sys/param.h>
31 #include <sys/thread.h>
32 #include <sys/cpuvar.h>
33 #include <sys/cpupart.h>
34 #include <sys/kmem.h>
35 #include <sys/cmn_err.h>
36 #include <sys/kstat.h>
37 #include <sys/processor.h>
38 #include <sys/disp.h>
39 #include <sys/group.h>
40 #include <sys/pghw.h>
41 #include <sys/bitset.h>
42 #include <sys/lgrp.h>
43 #include <sys/cmt.h>
44 
45 /*
46  * CMT scheduler / dispatcher support
47  *
48  * This file implements CMT scheduler support using Processor Groups.
49  * The CMT processor group class creates and maintains the CMT class
50  * specific processor group pg_cmt_t.
51  *
52  * ---------------------------- <-- pg_cmt_t *
53  * | pghw_t                   |
54  * ----------------------------
55  * | CMT class specific data  |
56  * | - hierarchy linkage      |
57  * | - CMT load balancing data|
58  * | - active CPU group/bitset|
59  * ----------------------------
60  *
61  * The scheduler/dispatcher leverages knowledge of the performance
62  * relevant CMT sharing relationships existing between cpus to implement
63  * optimized affinity and load balancing policies.
64  *
65  * Load balancing policy seeks to improve performance by minimizing
66  * contention over shared processor resources / facilities, while the
67  * affinity policies seek to improve cache and TLB utilization.
68  *
69  * The CMT PGs created by this class are already arranged into a
70  * hierarchy (which is done in the pghw layer). To implement the top-down
71  * CMT load balancing algorithm, the CMT PGs additionally maintain
72  * parent, child and sibling hierarchy relationships.
73  * Parent PGs always contain a superset of their children(s) resources,
74  * each PG can have at most one parent, and siblings are the group of PGs
75  * sharing the same parent.
76  *
77  * On NUMA systems, the CMT load balancing algorithm balances across the
78  * CMT PGs within their respective lgroups. On UMA based system, there
79  * exists a top level group of PGs to balance across. On NUMA systems multiple
80  * top level groups are instantiated, where the top level balancing begins by
81  * balancng across the CMT PGs within their respective (per lgroup) top level
82  * groups.
83  */
84 
85 typedef struct cmt_lgrp {
86 	group_t		cl_pgs;		/* Top level group of active CMT PGs */
87 	int		cl_npgs;	/* # of top level PGs in the lgroup */
88 	lgrp_handle_t	cl_hand;	/* lgroup's platform handle */
89 	struct cmt_lgrp *cl_next;	/* next cmt_lgrp */
90 } cmt_lgrp_t;
91 
92 static cmt_lgrp_t	*cmt_lgrps = NULL;	/* cmt_lgrps list head */
93 static cmt_lgrp_t	*cpu0_lgrp = NULL;	/* boot CPU's initial lgrp */
94 						/* used for null_proc_lpa */
95 
96 static int		is_cpu0 = 1; /* true if this is boot CPU context */
97 
98 /*
99  * Set this to non-zero to disable CMT scheduling
100  * This must be done via kmdb -d, as /etc/system will be too late
101  */
102 static int		cmt_sched_disabled = 0;
103 
104 static pg_cid_t		pg_cmt_class_id;		/* PG class id */
105 
106 static pg_t		*pg_cmt_alloc();
107 static void		pg_cmt_free(pg_t *);
108 static void		pg_cmt_cpu_init(cpu_t *);
109 static void		pg_cmt_cpu_fini(cpu_t *);
110 static void		pg_cmt_cpu_active(cpu_t *);
111 static void		pg_cmt_cpu_inactive(cpu_t *);
112 static void		pg_cmt_cpupart_in(cpu_t *, cpupart_t *);
113 static void		pg_cmt_cpupart_move(cpu_t *, cpupart_t *, cpupart_t *);
114 static void		pg_cmt_hier_pack(pg_cmt_t **, int);
115 static int		pg_cmt_cpu_belongs(pg_t *, cpu_t *);
116 static int		pg_cmt_hw(pghw_type_t);
117 static cmt_lgrp_t	*pg_cmt_find_lgrp(lgrp_handle_t);
118 static cmt_lgrp_t	*pg_cmt_lgrp_create(lgrp_handle_t);
119 
120 /*
121  * Macro to test if PG is managed by the CMT PG class
122  */
123 #define	IS_CMT_PG(pg)	(((pg_t *)(pg))->pg_class->pgc_id == pg_cmt_class_id)
124 
125 /*
126  * CMT PG ops
127  */
128 struct pg_ops pg_ops_cmt = {
129 	pg_cmt_alloc,
130 	pg_cmt_free,
131 	pg_cmt_cpu_init,
132 	pg_cmt_cpu_fini,
133 	pg_cmt_cpu_active,
134 	pg_cmt_cpu_inactive,
135 	pg_cmt_cpupart_in,
136 	NULL,			/* cpupart_out */
137 	pg_cmt_cpupart_move,
138 	pg_cmt_cpu_belongs,
139 };
140 
141 /*
142  * Initialize the CMT PG class
143  */
144 void
145 pg_cmt_class_init(void)
146 {
147 	if (cmt_sched_disabled)
148 		return;
149 
150 	pg_cmt_class_id = pg_class_register("cmt", &pg_ops_cmt, PGR_PHYSICAL);
151 }
152 
153 /*
154  * Called to indicate a new CPU has started up so
155  * that either t0 or the slave startup thread can
156  * be accounted for.
157  */
158 void
159 pg_cmt_cpu_startup(cpu_t *cp)
160 {
161 	PG_NRUN_UPDATE(cp, 1);
162 }
163 
164 /*
165  * Adjust the CMT load in the CMT PGs in which the CPU belongs
166  * Note that "n" can be positive in the case of increasing
167  * load, or negative in the case of decreasing load.
168  */
169 void
170 pg_cmt_load(cpu_t *cp, int n)
171 {
172 	pg_cmt_t	*pg;
173 
174 	pg = (pg_cmt_t *)cp->cpu_pg->cmt_lineage;
175 	while (pg != NULL) {
176 		ASSERT(IS_CMT_PG(pg));
177 		atomic_add_32(&pg->cmt_nrunning, n);
178 		pg = pg->cmt_parent;
179 	}
180 }
181 
182 /*
183  * Return non-zero if thread can migrate between "from" and "to"
184  * without a performance penalty
185  */
186 int
187 pg_cmt_can_migrate(cpu_t *from, cpu_t *to)
188 {
189 	if (from->cpu_physid->cpu_cacheid ==
190 	    to->cpu_physid->cpu_cacheid)
191 		return (1);
192 	return (0);
193 }
194 
195 /*
196  * CMT class specific PG allocation
197  */
198 static pg_t *
199 pg_cmt_alloc(void)
200 {
201 	return (kmem_zalloc(sizeof (pg_cmt_t), KM_NOSLEEP));
202 }
203 
204 /*
205  * Class specific PG de-allocation
206  */
207 static void
208 pg_cmt_free(pg_t *pg)
209 {
210 	ASSERT(pg != NULL);
211 	ASSERT(IS_CMT_PG(pg));
212 
213 	kmem_free((pg_cmt_t *)pg, sizeof (pg_cmt_t));
214 }
215 
216 /*
217  * Return 1 if CMT scheduling policies should be impelmented
218  * for the specified hardware sharing relationship.
219  */
220 static int
221 pg_cmt_hw(pghw_type_t hw)
222 {
223 	return (pg_plat_cmt_load_bal_hw(hw) ||
224 	    pg_plat_cmt_affinity_hw(hw));
225 }
226 
227 /*
228  * CMT class callback for a new CPU entering the system
229  */
230 static void
231 pg_cmt_cpu_init(cpu_t *cp)
232 {
233 	pg_cmt_t	*pg;
234 	group_t		*cmt_pgs;
235 	int		level, max_level, nlevels;
236 	pghw_type_t	hw;
237 	pg_t		*pg_cache = NULL;
238 	pg_cmt_t	*cpu_cmt_hier[PGHW_NUM_COMPONENTS];
239 	lgrp_handle_t	lgrp_handle;
240 	cmt_lgrp_t	*lgrp;
241 
242 	ASSERT(MUTEX_HELD(&cpu_lock));
243 
244 	/*
245 	 * A new CPU is coming into the system.
246 	 * Interrogate the platform to see if the CPU
247 	 * has any performance relevant CMT sharing
248 	 * relationships
249 	 */
250 	cmt_pgs = &cp->cpu_pg->cmt_pgs;
251 	cp->cpu_pg->cmt_lineage = NULL;
252 
253 	bzero(cpu_cmt_hier, sizeof (cpu_cmt_hier));
254 	max_level = nlevels = 0;
255 	for (hw = PGHW_START; hw < PGHW_NUM_COMPONENTS; hw++) {
256 
257 		/*
258 		 * We're only interested in CMT hw sharing relationships
259 		 */
260 		if (pg_cmt_hw(hw) == 0 || pg_plat_hw_shared(cp, hw) == 0)
261 			continue;
262 
263 		/*
264 		 * Find (or create) the PG associated with
265 		 * the hw sharing relationship in which cp
266 		 * belongs.
267 		 *
268 		 * Determine if a suitable PG already
269 		 * exists, or if one needs to be created.
270 		 */
271 		pg = (pg_cmt_t *)pghw_place_cpu(cp, hw);
272 		if (pg == NULL) {
273 			/*
274 			 * Create a new one.
275 			 * Initialize the common...
276 			 */
277 			pg = (pg_cmt_t *)pg_create(pg_cmt_class_id);
278 
279 			/* ... physical ... */
280 			pghw_init((pghw_t *)pg, cp, hw);
281 
282 			/*
283 			 * ... and CMT specific portions of the
284 			 * structure.
285 			 */
286 			bitset_init(&pg->cmt_cpus_actv_set);
287 			group_create(&pg->cmt_cpus_actv);
288 		} else {
289 			ASSERT(IS_CMT_PG(pg));
290 		}
291 
292 		/* Add the CPU to the PG */
293 		pg_cpu_add((pg_t *)pg, cp);
294 
295 		/*
296 		 * Ensure capacity of the active CPUs group/bitset
297 		 */
298 		group_expand(&pg->cmt_cpus_actv,
299 		    GROUP_SIZE(&((pg_t *)pg)->pg_cpus));
300 
301 		if (cp->cpu_seqid >=
302 		    bitset_capacity(&pg->cmt_cpus_actv_set)) {
303 			bitset_resize(&pg->cmt_cpus_actv_set,
304 			    cp->cpu_seqid + 1);
305 		}
306 
307 		/*
308 		 * Build a lineage of CMT PGs for load balancing
309 		 */
310 		if (pg_plat_cmt_load_bal_hw(hw)) {
311 			level = pghw_level(hw);
312 			cpu_cmt_hier[level] = pg;
313 			if (level > max_level)
314 				max_level = level;
315 			nlevels++;
316 		}
317 
318 		/* Cache this for later */
319 		if (hw == PGHW_CACHE)
320 			pg_cache = (pg_t *)pg;
321 	}
322 
323 	/*
324 	 * Pack out any gaps in the constructed lineage.
325 	 * Gaps may exist where the architecture knows
326 	 * about a hardware sharing relationship, but such a
327 	 * relationship either isn't relevant for load
328 	 * balancing or doesn't exist between CPUs on the system.
329 	 */
330 	pg_cmt_hier_pack(cpu_cmt_hier, max_level + 1);
331 
332 	/*
333 	 * For each of the PGs int the CPU's lineage:
334 	 *	- Add an entry in the CPU sorted CMT PG group
335 	 *	  which is used for top down CMT load balancing
336 	 *	- Tie the PG into the CMT hierarchy by connecting
337 	 *	  it to it's parent and siblings.
338 	 */
339 	group_expand(cmt_pgs, nlevels);
340 
341 	/*
342 	 * Find the lgrp that encapsulates this CPU's CMT hierarchy
343 	 */
344 	lgrp_handle = lgrp_plat_cpu_to_hand(cp->cpu_id);
345 	lgrp = pg_cmt_find_lgrp(lgrp_handle);
346 	if (lgrp == NULL)
347 		lgrp = pg_cmt_lgrp_create(lgrp_handle);
348 
349 	for (level = 0; level < nlevels; level++) {
350 		uint_t		children;
351 		int		err;
352 
353 		pg = cpu_cmt_hier[level];
354 		err = group_add_at(cmt_pgs, pg, nlevels - level - 1);
355 		ASSERT(err == 0);
356 
357 		if (level == 0)
358 			cp->cpu_pg->cmt_lineage = (pg_t *)pg;
359 
360 		if (pg->cmt_siblings != NULL) {
361 			/* Already initialized */
362 			ASSERT(pg->cmt_parent == NULL ||
363 			    pg->cmt_parent == cpu_cmt_hier[level + 1]);
364 			ASSERT(pg->cmt_siblings == &lgrp->cl_pgs ||
365 			    pg->cmt_siblings == pg->cmt_parent->cmt_children);
366 			continue;
367 		}
368 
369 		if ((level + 1) == nlevels) {
370 			pg->cmt_parent = NULL;
371 			pg->cmt_siblings = &lgrp->cl_pgs;
372 			children = ++lgrp->cl_npgs;
373 		} else {
374 			pg->cmt_parent = cpu_cmt_hier[level + 1];
375 
376 			/*
377 			 * A good parent keeps track of their children.
378 			 * The parent's children group is also the PG's
379 			 * siblings.
380 			 */
381 			if (pg->cmt_parent->cmt_children == NULL) {
382 				pg->cmt_parent->cmt_children =
383 				    kmem_zalloc(sizeof (group_t), KM_SLEEP);
384 				group_create(pg->cmt_parent->cmt_children);
385 			}
386 			pg->cmt_siblings = pg->cmt_parent->cmt_children;
387 			children = ++pg->cmt_parent->cmt_nchildren;
388 		}
389 		pg->cmt_hint = 0;
390 		group_expand(pg->cmt_siblings, children);
391 	}
392 
393 	/*
394 	 * Cache the chip and core IDs in the cpu_t->cpu_physid structure
395 	 * for fast lookups later.
396 	 */
397 	if (cp->cpu_physid) {
398 		cp->cpu_physid->cpu_chipid =
399 		    pg_plat_hw_instance_id(cp, PGHW_CHIP);
400 		cp->cpu_physid->cpu_coreid = pg_plat_get_core_id(cp);
401 
402 		/*
403 		 * If this cpu has a PG representing shared cache, then set
404 		 * cpu_cacheid to that PG's logical id
405 		 */
406 		if (pg_cache)
407 			cp->cpu_physid->cpu_cacheid = pg_cache->pg_id;
408 	}
409 
410 	/* CPU0 only initialization */
411 	if (is_cpu0) {
412 		pg_cmt_cpu_startup(cp);
413 		is_cpu0 = 0;
414 		cpu0_lgrp = lgrp;
415 	}
416 
417 }
418 
419 /*
420  * Class callback when a CPU is leaving the system (deletion)
421  */
422 static void
423 pg_cmt_cpu_fini(cpu_t *cp)
424 {
425 	group_iter_t	i;
426 	pg_cmt_t	*pg;
427 	group_t		*pgs, *cmt_pgs;
428 	lgrp_handle_t	lgrp_handle;
429 	cmt_lgrp_t	*lgrp;
430 
431 	pgs = &cp->cpu_pg->pgs;
432 	cmt_pgs = &cp->cpu_pg->cmt_pgs;
433 
434 	/*
435 	 * Find the lgroup that encapsulates this CPU's CMT hierarchy
436 	 */
437 	lgrp_handle = lgrp_plat_cpu_to_hand(cp->cpu_id);
438 
439 	lgrp = pg_cmt_find_lgrp(lgrp_handle);
440 	if (lgrp == NULL) {
441 		/*
442 		 * This is a bit of a special case.
443 		 * The only way this can happen is if the CPU's lgrp
444 		 * handle changed out from underneath us, which is what
445 		 * happens with null_proc_lpa on starcat systems.
446 		 *
447 		 * Use the initial boot CPU lgrp, since this is what
448 		 * we need to tear down.
449 		 */
450 		lgrp = cpu0_lgrp;
451 	}
452 
453 	/*
454 	 * First, clean up anything load balancing specific for each of
455 	 * the CPU's PGs that participated in CMT load balancing
456 	 */
457 	pg = (pg_cmt_t *)cp->cpu_pg->cmt_lineage;
458 	while (pg != NULL) {
459 
460 		/*
461 		 * Remove the PG from the CPU's load balancing lineage
462 		 */
463 		(void) group_remove(cmt_pgs, pg, GRP_RESIZE);
464 
465 		/*
466 		 * If it's about to become empty, destroy it's children
467 		 * group, and remove it's reference from it's siblings.
468 		 * This is done here (rather than below) to avoid removing
469 		 * our reference from a PG that we just eliminated.
470 		 */
471 		if (GROUP_SIZE(&((pg_t *)pg)->pg_cpus) == 1) {
472 			if (pg->cmt_children != NULL)
473 				group_destroy(pg->cmt_children);
474 			if (pg->cmt_siblings != NULL) {
475 				if (pg->cmt_siblings == &lgrp->cl_pgs)
476 					lgrp->cl_npgs--;
477 				else
478 					pg->cmt_parent->cmt_nchildren--;
479 			}
480 		}
481 		pg = pg->cmt_parent;
482 	}
483 
484 	ASSERT(GROUP_SIZE(cmt_pgs) == 0);
485 
486 	/*
487 	 * Now that the load balancing lineage updates have happened,
488 	 * remove the CPU from all it's PGs (destroying any that become
489 	 * empty).
490 	 */
491 	group_iter_init(&i);
492 	while ((pg = group_iterate(pgs, &i)) != NULL) {
493 		if (IS_CMT_PG(pg) == 0)
494 			continue;
495 
496 		pg_cpu_delete((pg_t *)pg, cp);
497 		/*
498 		 * Deleting the CPU from the PG changes the CPU's
499 		 * PG group over which we are actively iterating
500 		 * Re-initialize the iteration
501 		 */
502 		group_iter_init(&i);
503 
504 		if (GROUP_SIZE(&((pg_t *)pg)->pg_cpus) == 0) {
505 
506 			/*
507 			 * The PG has become zero sized, so destroy it.
508 			 */
509 			group_destroy(&pg->cmt_cpus_actv);
510 			bitset_fini(&pg->cmt_cpus_actv_set);
511 			pghw_fini((pghw_t *)pg);
512 
513 			pg_destroy((pg_t *)pg);
514 		}
515 	}
516 }
517 
518 /*
519  * Class callback when a CPU is entering a cpu partition
520  */
521 static void
522 pg_cmt_cpupart_in(cpu_t *cp, cpupart_t *pp)
523 {
524 	group_t		*pgs;
525 	pg_t		*pg;
526 	group_iter_t	i;
527 
528 	ASSERT(MUTEX_HELD(&cpu_lock));
529 
530 	pgs = &cp->cpu_pg->pgs;
531 
532 	/*
533 	 * Ensure that the new partition's PG bitset
534 	 * is large enough for all CMT PG's to which cp
535 	 * belongs
536 	 */
537 	group_iter_init(&i);
538 	while ((pg = group_iterate(pgs, &i)) != NULL) {
539 		if (IS_CMT_PG(pg) == 0)
540 			continue;
541 
542 		if (bitset_capacity(&pp->cp_cmt_pgs) <= pg->pg_id)
543 			bitset_resize(&pp->cp_cmt_pgs, pg->pg_id + 1);
544 	}
545 }
546 
547 /*
548  * Class callback when a CPU is actually moving partitions
549  */
550 static void
551 pg_cmt_cpupart_move(cpu_t *cp, cpupart_t *oldpp, cpupart_t *newpp)
552 {
553 	cpu_t		*cpp;
554 	group_t		*pgs;
555 	pg_t		*pg;
556 	group_iter_t	pg_iter;
557 	pg_cpu_itr_t	cpu_iter;
558 	boolean_t	found;
559 
560 	ASSERT(MUTEX_HELD(&cpu_lock));
561 
562 	pgs = &cp->cpu_pg->pgs;
563 	group_iter_init(&pg_iter);
564 
565 	/*
566 	 * Iterate over the CPUs CMT PGs
567 	 */
568 	while ((pg = group_iterate(pgs, &pg_iter)) != NULL) {
569 
570 		if (IS_CMT_PG(pg) == 0)
571 			continue;
572 
573 		/*
574 		 * Add the PG to the bitset in the new partition.
575 		 */
576 		bitset_add(&newpp->cp_cmt_pgs, pg->pg_id);
577 
578 		/*
579 		 * Remove the PG from the bitset in the old partition
580 		 * if the last of the PG's CPUs have left.
581 		 */
582 		found = B_FALSE;
583 		PG_CPU_ITR_INIT(pg, cpu_iter);
584 		while ((cpp = pg_cpu_next(&cpu_iter)) != NULL) {
585 			if (cpp == cp)
586 				continue;
587 			if (CPU_ACTIVE(cpp) &&
588 			    cpp->cpu_part->cp_id == oldpp->cp_id) {
589 				found = B_TRUE;
590 				break;
591 			}
592 		}
593 		if (!found)
594 			bitset_del(&cp->cpu_part->cp_cmt_pgs, pg->pg_id);
595 	}
596 }
597 
598 /*
599  * Class callback when a CPU becomes active (online)
600  *
601  * This is called in a context where CPUs are paused
602  */
603 static void
604 pg_cmt_cpu_active(cpu_t *cp)
605 {
606 	int		err;
607 	group_iter_t	i;
608 	pg_cmt_t	*pg;
609 	group_t		*pgs;
610 
611 	ASSERT(MUTEX_HELD(&cpu_lock));
612 
613 	pgs = &cp->cpu_pg->pgs;
614 	group_iter_init(&i);
615 
616 	/*
617 	 * Iterate over the CPU's PGs
618 	 */
619 	while ((pg = group_iterate(pgs, &i)) != NULL) {
620 
621 		if (IS_CMT_PG(pg) == 0)
622 			continue;
623 
624 		err = group_add(&pg->cmt_cpus_actv, cp, GRP_NORESIZE);
625 		ASSERT(err == 0);
626 
627 		/*
628 		 * If this is the first active CPU in the PG, and it
629 		 * represents a hardware sharing relationship over which
630 		 * CMT load balancing is performed, add it as a candidate
631 		 * for balancing with it's siblings.
632 		 */
633 		if (GROUP_SIZE(&pg->cmt_cpus_actv) == 1 &&
634 		    pg_plat_cmt_load_bal_hw(((pghw_t *)pg)->pghw_hw)) {
635 			err = group_add(pg->cmt_siblings, pg, GRP_NORESIZE);
636 			ASSERT(err == 0);
637 		}
638 
639 		/*
640 		 * Notate the CPU in the PGs active CPU bitset.
641 		 * Also notate the PG as being active in it's associated
642 		 * partition
643 		 */
644 		bitset_add(&pg->cmt_cpus_actv_set, cp->cpu_seqid);
645 		bitset_add(&cp->cpu_part->cp_cmt_pgs, ((pg_t *)pg)->pg_id);
646 	}
647 }
648 
649 /*
650  * Class callback when a CPU goes inactive (offline)
651  *
652  * This is called in a context where CPUs are paused
653  */
654 static void
655 pg_cmt_cpu_inactive(cpu_t *cp)
656 {
657 	int		err;
658 	group_t		*pgs;
659 	pg_cmt_t	*pg;
660 	cpu_t		*cpp;
661 	group_iter_t	i;
662 	pg_cpu_itr_t	cpu_itr;
663 	boolean_t	found;
664 
665 	ASSERT(MUTEX_HELD(&cpu_lock));
666 
667 	pgs = &cp->cpu_pg->pgs;
668 	group_iter_init(&i);
669 
670 	while ((pg = group_iterate(pgs, &i)) != NULL) {
671 
672 		if (IS_CMT_PG(pg) == 0)
673 			continue;
674 
675 		/*
676 		 * Remove the CPU from the CMT PGs active CPU group
677 		 * bitmap
678 		 */
679 		err = group_remove(&pg->cmt_cpus_actv, cp, GRP_NORESIZE);
680 		ASSERT(err == 0);
681 
682 		bitset_del(&pg->cmt_cpus_actv_set, cp->cpu_seqid);
683 
684 		/*
685 		 * If there are no more active CPUs in this PG over which
686 		 * load was balanced, remove it as a balancing candidate.
687 		 */
688 		if (GROUP_SIZE(&pg->cmt_cpus_actv) == 0 &&
689 		    pg_plat_cmt_load_bal_hw(((pghw_t *)pg)->pghw_hw)) {
690 			err = group_remove(pg->cmt_siblings, pg, GRP_NORESIZE);
691 			ASSERT(err == 0);
692 		}
693 
694 		/*
695 		 * Assert the number of active CPUs does not exceed
696 		 * the total number of CPUs in the PG
697 		 */
698 		ASSERT(GROUP_SIZE(&pg->cmt_cpus_actv) <=
699 		    GROUP_SIZE(&((pg_t *)pg)->pg_cpus));
700 
701 		/*
702 		 * Update the PG bitset in the CPU's old partition
703 		 */
704 		found = B_FALSE;
705 		PG_CPU_ITR_INIT(pg, cpu_itr);
706 		while ((cpp = pg_cpu_next(&cpu_itr)) != NULL) {
707 			if (cpp == cp)
708 				continue;
709 			if (CPU_ACTIVE(cpp) &&
710 			    cpp->cpu_part->cp_id == cp->cpu_part->cp_id) {
711 				found = B_TRUE;
712 				break;
713 			}
714 		}
715 		if (!found) {
716 			bitset_del(&cp->cpu_part->cp_cmt_pgs,
717 			    ((pg_t *)pg)->pg_id);
718 		}
719 	}
720 }
721 
722 /*
723  * Return non-zero if the CPU belongs in the given PG
724  */
725 static int
726 pg_cmt_cpu_belongs(pg_t *pg, cpu_t *cp)
727 {
728 	cpu_t	*pg_cpu;
729 
730 	pg_cpu = GROUP_ACCESS(&pg->pg_cpus, 0);
731 
732 	ASSERT(pg_cpu != NULL);
733 
734 	/*
735 	 * The CPU belongs if, given the nature of the hardware sharing
736 	 * relationship represented by the PG, the CPU has that
737 	 * relationship with some other CPU already in the PG
738 	 */
739 	if (pg_plat_cpus_share(cp, pg_cpu, ((pghw_t *)pg)->pghw_hw))
740 		return (1);
741 
742 	return (0);
743 }
744 
745 /*
746  * Pack the CPUs CMT hierarchy
747  * The hierarchy order is preserved
748  */
749 static void
750 pg_cmt_hier_pack(pg_cmt_t *hier[], int sz)
751 {
752 	int	i, j;
753 
754 	for (i = 0; i < sz; i++) {
755 		if (hier[i] != NULL)
756 			continue;
757 
758 		for (j = i; j < sz; j++) {
759 			if (hier[j] != NULL) {
760 				hier[i] = hier[j];
761 				hier[j] = NULL;
762 				break;
763 			}
764 		}
765 		if (j == sz)
766 			break;
767 	}
768 }
769 
770 /*
771  * Return a cmt_lgrp_t * given an lgroup handle.
772  */
773 static cmt_lgrp_t *
774 pg_cmt_find_lgrp(lgrp_handle_t hand)
775 {
776 	cmt_lgrp_t	*lgrp;
777 
778 	ASSERT(MUTEX_HELD(&cpu_lock));
779 
780 	lgrp = cmt_lgrps;
781 	while (lgrp != NULL) {
782 		if (lgrp->cl_hand == hand)
783 			break;
784 		lgrp = lgrp->cl_next;
785 	}
786 	return (lgrp);
787 }
788 
789 /*
790  * Create a cmt_lgrp_t with the specified handle.
791  */
792 static cmt_lgrp_t *
793 pg_cmt_lgrp_create(lgrp_handle_t hand)
794 {
795 	cmt_lgrp_t	*lgrp;
796 
797 	ASSERT(MUTEX_HELD(&cpu_lock));
798 
799 	lgrp = kmem_zalloc(sizeof (cmt_lgrp_t), KM_SLEEP);
800 
801 	lgrp->cl_hand = hand;
802 	lgrp->cl_npgs = 0;
803 	lgrp->cl_next = cmt_lgrps;
804 	cmt_lgrps = lgrp;
805 	group_create(&lgrp->cl_pgs);
806 
807 	return (lgrp);
808 }
809