1fb2f18f8Sesaxe /* 2fb2f18f8Sesaxe * CDDL HEADER START 3fb2f18f8Sesaxe * 4fb2f18f8Sesaxe * The contents of this file are subject to the terms of the 5fb2f18f8Sesaxe * Common Development and Distribution License (the "License"). 6fb2f18f8Sesaxe * You may not use this file except in compliance with the License. 7fb2f18f8Sesaxe * 8fb2f18f8Sesaxe * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9fb2f18f8Sesaxe * or http://www.opensolaris.org/os/licensing. 10fb2f18f8Sesaxe * See the License for the specific language governing permissions 11fb2f18f8Sesaxe * and limitations under the License. 12fb2f18f8Sesaxe * 13fb2f18f8Sesaxe * When distributing Covered Code, include this CDDL HEADER in each 14fb2f18f8Sesaxe * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15fb2f18f8Sesaxe * If applicable, add the following below this CDDL HEADER, with the 16fb2f18f8Sesaxe * fields enclosed by brackets "[]" replaced with your own identifying 17fb2f18f8Sesaxe * information: Portions Copyright [yyyy] [name of copyright owner] 18fb2f18f8Sesaxe * 19fb2f18f8Sesaxe * CDDL HEADER END 20fb2f18f8Sesaxe */ 21fb2f18f8Sesaxe /* 22fb2f18f8Sesaxe * Copyright 2007 Sun Microsystems, Inc. All rights reserved. 23fb2f18f8Sesaxe * Use is subject to license terms. 24fb2f18f8Sesaxe */ 25fb2f18f8Sesaxe 26fb2f18f8Sesaxe #pragma ident "%Z%%M% %I% %E% SMI" 27fb2f18f8Sesaxe 28fb2f18f8Sesaxe #include <sys/systm.h> 29fb2f18f8Sesaxe #include <sys/types.h> 30fb2f18f8Sesaxe #include <sys/param.h> 31fb2f18f8Sesaxe #include <sys/thread.h> 32fb2f18f8Sesaxe #include <sys/cpuvar.h> 33fb2f18f8Sesaxe #include <sys/cpupart.h> 34fb2f18f8Sesaxe #include <sys/kmem.h> 35fb2f18f8Sesaxe #include <sys/cmn_err.h> 36fb2f18f8Sesaxe #include <sys/kstat.h> 37fb2f18f8Sesaxe #include <sys/processor.h> 38fb2f18f8Sesaxe #include <sys/disp.h> 39fb2f18f8Sesaxe #include <sys/group.h> 40fb2f18f8Sesaxe #include <sys/pghw.h> 41fb2f18f8Sesaxe #include <sys/bitset.h> 42fb2f18f8Sesaxe #include <sys/lgrp.h> 43fb2f18f8Sesaxe #include <sys/cmt.h> 44fb2f18f8Sesaxe 45fb2f18f8Sesaxe /* 46fb2f18f8Sesaxe * CMT scheduler / dispatcher support 47fb2f18f8Sesaxe * 48fb2f18f8Sesaxe * This file implements CMT scheduler support using Processor Groups. 49fb2f18f8Sesaxe * The CMT processor group class creates and maintains the CMT class 50fb2f18f8Sesaxe * specific processor group pg_cmt_t. 51fb2f18f8Sesaxe * 52fb2f18f8Sesaxe * ---------------------------- <-- pg_cmt_t * 53fb2f18f8Sesaxe * | pghw_t | 54fb2f18f8Sesaxe * ---------------------------- 55fb2f18f8Sesaxe * | CMT class specific data | 56fb2f18f8Sesaxe * | - hierarchy linkage | 57fb2f18f8Sesaxe * | - CMT load balancing data| 58fb2f18f8Sesaxe * | - active CPU group/bitset| 59fb2f18f8Sesaxe * ---------------------------- 60fb2f18f8Sesaxe * 61fb2f18f8Sesaxe * The scheduler/dispatcher leverages knowledge of the performance 62fb2f18f8Sesaxe * relevant CMT sharing relationships existing between cpus to implement 63fb2f18f8Sesaxe * optimized affinity and load balancing policies. 64fb2f18f8Sesaxe * 65fb2f18f8Sesaxe * Load balancing policy seeks to improve performance by minimizing 66fb2f18f8Sesaxe * contention over shared processor resources / facilities, while the 67fb2f18f8Sesaxe * affinity policies seek to improve cache and TLB utilization. 68fb2f18f8Sesaxe * 69fb2f18f8Sesaxe * The CMT PGs created by this class are already arranged into a 70fb2f18f8Sesaxe * hierarchy (which is done in the pghw layer). To implement the top-down 71fb2f18f8Sesaxe * CMT load balancing algorithm, the CMT PGs additionally maintain 72fb2f18f8Sesaxe * parent, child and sibling hierarchy relationships. 73fb2f18f8Sesaxe * Parent PGs always contain a superset of their children(s) resources, 74fb2f18f8Sesaxe * each PG can have at most one parent, and siblings are the group of PGs 75fb2f18f8Sesaxe * sharing the same parent. 76fb2f18f8Sesaxe * 77fb2f18f8Sesaxe * On NUMA systems, the CMT load balancing algorithm balances across the 78fb2f18f8Sesaxe * CMT PGs within their respective lgroups. On UMA based system, there 79fb2f18f8Sesaxe * exists a top level group of PGs to balance across. On NUMA systems multiple 80fb2f18f8Sesaxe * top level groups are instantiated, where the top level balancing begins by 81fb2f18f8Sesaxe * balancng across the CMT PGs within their respective (per lgroup) top level 82fb2f18f8Sesaxe * groups. 83fb2f18f8Sesaxe */ 84fb2f18f8Sesaxe 85fb2f18f8Sesaxe typedef struct cmt_lgrp { 86fb2f18f8Sesaxe group_t cl_pgs; /* Top level group of active CMT PGs */ 87fb2f18f8Sesaxe int cl_npgs; /* # of top level PGs in the lgroup */ 88fb2f18f8Sesaxe lgrp_handle_t cl_hand; /* lgroup's platform handle */ 89fb2f18f8Sesaxe struct cmt_lgrp *cl_next; /* next cmt_lgrp */ 90fb2f18f8Sesaxe } cmt_lgrp_t; 91fb2f18f8Sesaxe 92*a6604450Sesaxe static cmt_lgrp_t *cmt_lgrps = NULL; /* cmt_lgrps list head */ 93*a6604450Sesaxe static cmt_lgrp_t *cpu0_lgrp = NULL; /* boot CPU's initial lgrp */ 94*a6604450Sesaxe /* used for null_proc_lpa */ 95fb2f18f8Sesaxe 96*a6604450Sesaxe static int is_cpu0 = 1; /* true if this is boot CPU context */ 97*a6604450Sesaxe 98*a6604450Sesaxe /* 99*a6604450Sesaxe * Set this to non-zero to disable CMT scheduling 100*a6604450Sesaxe * This must be done via kmdb -d, as /etc/system will be too late 101*a6604450Sesaxe */ 102fb2f18f8Sesaxe static int cmt_sched_disabled = 0; 103fb2f18f8Sesaxe 104fb2f18f8Sesaxe static pg_cid_t pg_cmt_class_id; /* PG class id */ 105fb2f18f8Sesaxe 106fb2f18f8Sesaxe static pg_t *pg_cmt_alloc(); 107fb2f18f8Sesaxe static void pg_cmt_free(pg_t *); 108fb2f18f8Sesaxe static void pg_cmt_cpu_init(cpu_t *); 109fb2f18f8Sesaxe static void pg_cmt_cpu_fini(cpu_t *); 110fb2f18f8Sesaxe static void pg_cmt_cpu_active(cpu_t *); 111fb2f18f8Sesaxe static void pg_cmt_cpu_inactive(cpu_t *); 112fb2f18f8Sesaxe static void pg_cmt_cpupart_in(cpu_t *, cpupart_t *); 113fb2f18f8Sesaxe static void pg_cmt_cpupart_move(cpu_t *, cpupart_t *, cpupart_t *); 114fb2f18f8Sesaxe static void pg_cmt_hier_pack(pg_cmt_t **, int); 115fb2f18f8Sesaxe static int pg_cmt_cpu_belongs(pg_t *, cpu_t *); 116fb2f18f8Sesaxe static int pg_cmt_hw(pghw_type_t); 117fb2f18f8Sesaxe static cmt_lgrp_t *pg_cmt_find_lgrp(lgrp_handle_t); 118*a6604450Sesaxe static cmt_lgrp_t *pg_cmt_lgrp_create(lgrp_handle_t); 119fb2f18f8Sesaxe 120fb2f18f8Sesaxe /* 121fb2f18f8Sesaxe * Macro to test if PG is managed by the CMT PG class 122fb2f18f8Sesaxe */ 123fb2f18f8Sesaxe #define IS_CMT_PG(pg) (((pg_t *)(pg))->pg_class->pgc_id == pg_cmt_class_id) 124fb2f18f8Sesaxe 125fb2f18f8Sesaxe /* 126fb2f18f8Sesaxe * CMT PG ops 127fb2f18f8Sesaxe */ 128fb2f18f8Sesaxe struct pg_ops pg_ops_cmt = { 129fb2f18f8Sesaxe pg_cmt_alloc, 130fb2f18f8Sesaxe pg_cmt_free, 131fb2f18f8Sesaxe pg_cmt_cpu_init, 132fb2f18f8Sesaxe pg_cmt_cpu_fini, 133fb2f18f8Sesaxe pg_cmt_cpu_active, 134fb2f18f8Sesaxe pg_cmt_cpu_inactive, 135fb2f18f8Sesaxe pg_cmt_cpupart_in, 136fb2f18f8Sesaxe NULL, /* cpupart_out */ 137fb2f18f8Sesaxe pg_cmt_cpupart_move, 138fb2f18f8Sesaxe pg_cmt_cpu_belongs, 139fb2f18f8Sesaxe }; 140fb2f18f8Sesaxe 141fb2f18f8Sesaxe /* 142fb2f18f8Sesaxe * Initialize the CMT PG class 143fb2f18f8Sesaxe */ 144fb2f18f8Sesaxe void 145fb2f18f8Sesaxe pg_cmt_class_init(void) 146fb2f18f8Sesaxe { 147fb2f18f8Sesaxe if (cmt_sched_disabled) 148fb2f18f8Sesaxe return; 149fb2f18f8Sesaxe 150fb2f18f8Sesaxe pg_cmt_class_id = pg_class_register("cmt", &pg_ops_cmt, PGR_PHYSICAL); 151fb2f18f8Sesaxe } 152fb2f18f8Sesaxe 153fb2f18f8Sesaxe /* 154fb2f18f8Sesaxe * Called to indicate a new CPU has started up so 155fb2f18f8Sesaxe * that either t0 or the slave startup thread can 156fb2f18f8Sesaxe * be accounted for. 157fb2f18f8Sesaxe */ 158fb2f18f8Sesaxe void 159fb2f18f8Sesaxe pg_cmt_cpu_startup(cpu_t *cp) 160fb2f18f8Sesaxe { 161fb2f18f8Sesaxe PG_NRUN_UPDATE(cp, 1); 162fb2f18f8Sesaxe } 163fb2f18f8Sesaxe 164fb2f18f8Sesaxe /* 165fb2f18f8Sesaxe * Adjust the CMT load in the CMT PGs in which the CPU belongs 166fb2f18f8Sesaxe * Note that "n" can be positive in the case of increasing 167fb2f18f8Sesaxe * load, or negative in the case of decreasing load. 168fb2f18f8Sesaxe */ 169fb2f18f8Sesaxe void 170fb2f18f8Sesaxe pg_cmt_load(cpu_t *cp, int n) 171fb2f18f8Sesaxe { 172fb2f18f8Sesaxe pg_cmt_t *pg; 173fb2f18f8Sesaxe 174fb2f18f8Sesaxe pg = (pg_cmt_t *)cp->cpu_pg->cmt_lineage; 175fb2f18f8Sesaxe while (pg != NULL) { 176fb2f18f8Sesaxe ASSERT(IS_CMT_PG(pg)); 177fb2f18f8Sesaxe atomic_add_32(&pg->cmt_nrunning, n); 178fb2f18f8Sesaxe pg = pg->cmt_parent; 179fb2f18f8Sesaxe } 180fb2f18f8Sesaxe } 181fb2f18f8Sesaxe 182fb2f18f8Sesaxe /* 183fb2f18f8Sesaxe * Return non-zero if thread can migrate between "from" and "to" 184fb2f18f8Sesaxe * without a performance penalty 185fb2f18f8Sesaxe */ 186fb2f18f8Sesaxe int 187fb2f18f8Sesaxe pg_cmt_can_migrate(cpu_t *from, cpu_t *to) 188fb2f18f8Sesaxe { 189fb2f18f8Sesaxe if (from->cpu_physid->cpu_cacheid == 190fb2f18f8Sesaxe to->cpu_physid->cpu_cacheid) 191fb2f18f8Sesaxe return (1); 192fb2f18f8Sesaxe return (0); 193fb2f18f8Sesaxe } 194fb2f18f8Sesaxe 195fb2f18f8Sesaxe /* 196fb2f18f8Sesaxe * CMT class specific PG allocation 197fb2f18f8Sesaxe */ 198fb2f18f8Sesaxe static pg_t * 199fb2f18f8Sesaxe pg_cmt_alloc(void) 200fb2f18f8Sesaxe { 201fb2f18f8Sesaxe return (kmem_zalloc(sizeof (pg_cmt_t), KM_NOSLEEP)); 202fb2f18f8Sesaxe } 203fb2f18f8Sesaxe 204fb2f18f8Sesaxe /* 205fb2f18f8Sesaxe * Class specific PG de-allocation 206fb2f18f8Sesaxe */ 207fb2f18f8Sesaxe static void 208fb2f18f8Sesaxe pg_cmt_free(pg_t *pg) 209fb2f18f8Sesaxe { 210fb2f18f8Sesaxe ASSERT(pg != NULL); 211fb2f18f8Sesaxe ASSERT(IS_CMT_PG(pg)); 212fb2f18f8Sesaxe 213fb2f18f8Sesaxe kmem_free((pg_cmt_t *)pg, sizeof (pg_cmt_t)); 214fb2f18f8Sesaxe } 215fb2f18f8Sesaxe 216fb2f18f8Sesaxe /* 217fb2f18f8Sesaxe * Return 1 if CMT load balancing policies should be 218fb2f18f8Sesaxe * implemented across instances of the specified hardware 219fb2f18f8Sesaxe * sharing relationship. 220fb2f18f8Sesaxe */ 221fb2f18f8Sesaxe static int 222fb2f18f8Sesaxe pg_cmt_load_bal_hw(pghw_type_t hw) 223fb2f18f8Sesaxe { 224fb2f18f8Sesaxe if (hw == PGHW_IPIPE || 225fb2f18f8Sesaxe hw == PGHW_FPU || 226fb2f18f8Sesaxe hw == PGHW_CHIP) 227fb2f18f8Sesaxe return (1); 228fb2f18f8Sesaxe else 229fb2f18f8Sesaxe return (0); 230fb2f18f8Sesaxe } 231fb2f18f8Sesaxe 232fb2f18f8Sesaxe /* 233fb2f18f8Sesaxe * Return 1 if thread affinity polices should be implemented 234fb2f18f8Sesaxe * for instances of the specifed hardware sharing relationship. 235fb2f18f8Sesaxe */ 236fb2f18f8Sesaxe static int 237fb2f18f8Sesaxe pg_cmt_affinity_hw(pghw_type_t hw) 238fb2f18f8Sesaxe { 239fb2f18f8Sesaxe if (hw == PGHW_CACHE) 240fb2f18f8Sesaxe return (1); 241fb2f18f8Sesaxe else 242fb2f18f8Sesaxe return (0); 243fb2f18f8Sesaxe } 244fb2f18f8Sesaxe 245fb2f18f8Sesaxe /* 246fb2f18f8Sesaxe * Return 1 if CMT scheduling policies should be impelmented 247fb2f18f8Sesaxe * for the specified hardware sharing relationship. 248fb2f18f8Sesaxe */ 249fb2f18f8Sesaxe static int 250fb2f18f8Sesaxe pg_cmt_hw(pghw_type_t hw) 251fb2f18f8Sesaxe { 252fb2f18f8Sesaxe return (pg_cmt_load_bal_hw(hw) || 253fb2f18f8Sesaxe pg_cmt_affinity_hw(hw)); 254fb2f18f8Sesaxe } 255fb2f18f8Sesaxe 256fb2f18f8Sesaxe /* 257fb2f18f8Sesaxe * CMT class callback for a new CPU entering the system 258fb2f18f8Sesaxe */ 259fb2f18f8Sesaxe static void 260fb2f18f8Sesaxe pg_cmt_cpu_init(cpu_t *cp) 261fb2f18f8Sesaxe { 262fb2f18f8Sesaxe pg_cmt_t *pg; 263fb2f18f8Sesaxe group_t *cmt_pgs; 264fb2f18f8Sesaxe int level, max_level, nlevels; 265fb2f18f8Sesaxe pghw_type_t hw; 266fb2f18f8Sesaxe pg_t *pg_cache = NULL; 267fb2f18f8Sesaxe pg_cmt_t *cpu_cmt_hier[PGHW_NUM_COMPONENTS]; 268fb2f18f8Sesaxe lgrp_handle_t lgrp_handle; 269fb2f18f8Sesaxe cmt_lgrp_t *lgrp; 270fb2f18f8Sesaxe 271fb2f18f8Sesaxe ASSERT(MUTEX_HELD(&cpu_lock)); 272fb2f18f8Sesaxe 273fb2f18f8Sesaxe /* 274fb2f18f8Sesaxe * A new CPU is coming into the system. 275fb2f18f8Sesaxe * Interrogate the platform to see if the CPU 276fb2f18f8Sesaxe * has any performance relevant CMT sharing 277fb2f18f8Sesaxe * relationships 278fb2f18f8Sesaxe */ 279fb2f18f8Sesaxe cmt_pgs = &cp->cpu_pg->cmt_pgs; 280fb2f18f8Sesaxe cp->cpu_pg->cmt_lineage = NULL; 281fb2f18f8Sesaxe 282fb2f18f8Sesaxe bzero(cpu_cmt_hier, sizeof (cpu_cmt_hier)); 283fb2f18f8Sesaxe max_level = nlevels = 0; 284fb2f18f8Sesaxe for (hw = PGHW_START; hw < PGHW_NUM_COMPONENTS; hw++) { 285fb2f18f8Sesaxe 286fb2f18f8Sesaxe /* 287fb2f18f8Sesaxe * We're only interested in CMT hw sharing relationships 288fb2f18f8Sesaxe */ 289fb2f18f8Sesaxe if (pg_cmt_hw(hw) == 0 || pg_plat_hw_shared(cp, hw) == 0) 290fb2f18f8Sesaxe continue; 291fb2f18f8Sesaxe 292fb2f18f8Sesaxe /* 293fb2f18f8Sesaxe * Find (or create) the PG associated with 294fb2f18f8Sesaxe * the hw sharing relationship in which cp 295fb2f18f8Sesaxe * belongs. 296fb2f18f8Sesaxe * 297fb2f18f8Sesaxe * Determine if a suitable PG already 298fb2f18f8Sesaxe * exists, or if one needs to be created. 299fb2f18f8Sesaxe */ 300fb2f18f8Sesaxe pg = (pg_cmt_t *)pghw_place_cpu(cp, hw); 301fb2f18f8Sesaxe if (pg == NULL) { 302fb2f18f8Sesaxe /* 303fb2f18f8Sesaxe * Create a new one. 304fb2f18f8Sesaxe * Initialize the common... 305fb2f18f8Sesaxe */ 306fb2f18f8Sesaxe pg = (pg_cmt_t *)pg_create(pg_cmt_class_id); 307fb2f18f8Sesaxe 308fb2f18f8Sesaxe /* ... physical ... */ 309fb2f18f8Sesaxe pghw_init((pghw_t *)pg, cp, hw); 310fb2f18f8Sesaxe 311fb2f18f8Sesaxe /* 312fb2f18f8Sesaxe * ... and CMT specific portions of the 313fb2f18f8Sesaxe * structure. 314fb2f18f8Sesaxe */ 315fb2f18f8Sesaxe bitset_init(&pg->cmt_cpus_actv_set); 316fb2f18f8Sesaxe group_create(&pg->cmt_cpus_actv); 317fb2f18f8Sesaxe } else { 318fb2f18f8Sesaxe ASSERT(IS_CMT_PG(pg)); 319fb2f18f8Sesaxe } 320fb2f18f8Sesaxe 321fb2f18f8Sesaxe /* Add the CPU to the PG */ 322fb2f18f8Sesaxe pg_cpu_add((pg_t *)pg, cp); 323fb2f18f8Sesaxe 324fb2f18f8Sesaxe /* 325fb2f18f8Sesaxe * Ensure capacity of the active CPUs group/bitset 326fb2f18f8Sesaxe */ 327fb2f18f8Sesaxe group_expand(&pg->cmt_cpus_actv, 328fb2f18f8Sesaxe GROUP_SIZE(&((pg_t *)pg)->pg_cpus)); 329fb2f18f8Sesaxe 330fb2f18f8Sesaxe if (cp->cpu_seqid >= 331fb2f18f8Sesaxe bitset_capacity(&pg->cmt_cpus_actv_set)) { 332fb2f18f8Sesaxe bitset_resize(&pg->cmt_cpus_actv_set, 333fb2f18f8Sesaxe cp->cpu_seqid + 1); 334fb2f18f8Sesaxe } 335fb2f18f8Sesaxe 336fb2f18f8Sesaxe /* 337fb2f18f8Sesaxe * Build a lineage of CMT PGs for load balancing 338fb2f18f8Sesaxe */ 339fb2f18f8Sesaxe if (pg_cmt_load_bal_hw(hw)) { 340fb2f18f8Sesaxe level = pghw_level(hw); 341fb2f18f8Sesaxe cpu_cmt_hier[level] = pg; 342fb2f18f8Sesaxe if (level > max_level) 343fb2f18f8Sesaxe max_level = level; 344fb2f18f8Sesaxe nlevels++; 345fb2f18f8Sesaxe } 346fb2f18f8Sesaxe 347fb2f18f8Sesaxe /* Cache this for later */ 348fb2f18f8Sesaxe if (hw == PGHW_CACHE) 349fb2f18f8Sesaxe pg_cache = (pg_t *)pg; 350fb2f18f8Sesaxe } 351fb2f18f8Sesaxe 352fb2f18f8Sesaxe /* 353fb2f18f8Sesaxe * Pack out any gaps in the constructed lineage. 354fb2f18f8Sesaxe * Gaps may exist where the architecture knows 355fb2f18f8Sesaxe * about a hardware sharing relationship, but such a 356fb2f18f8Sesaxe * relationship either isn't relevant for load 357fb2f18f8Sesaxe * balancing or doesn't exist between CPUs on the system. 358fb2f18f8Sesaxe */ 359fb2f18f8Sesaxe pg_cmt_hier_pack(cpu_cmt_hier, max_level + 1); 360fb2f18f8Sesaxe 361fb2f18f8Sesaxe /* 362fb2f18f8Sesaxe * For each of the PGs int the CPU's lineage: 363fb2f18f8Sesaxe * - Add an entry in the CPU sorted CMT PG group 364fb2f18f8Sesaxe * which is used for top down CMT load balancing 365fb2f18f8Sesaxe * - Tie the PG into the CMT hierarchy by connecting 366fb2f18f8Sesaxe * it to it's parent and siblings. 367fb2f18f8Sesaxe */ 368fb2f18f8Sesaxe group_expand(cmt_pgs, nlevels); 369fb2f18f8Sesaxe 370fb2f18f8Sesaxe /* 371fb2f18f8Sesaxe * Find the lgrp that encapsulates this CPU's CMT hierarchy 372fb2f18f8Sesaxe */ 373fb2f18f8Sesaxe lgrp_handle = lgrp_plat_cpu_to_hand(cp->cpu_id); 374fb2f18f8Sesaxe lgrp = pg_cmt_find_lgrp(lgrp_handle); 375*a6604450Sesaxe if (lgrp == NULL) 376*a6604450Sesaxe lgrp = pg_cmt_lgrp_create(lgrp_handle); 377fb2f18f8Sesaxe 378fb2f18f8Sesaxe for (level = 0; level < nlevels; level++) { 379fb2f18f8Sesaxe uint_t children; 380fb2f18f8Sesaxe int err; 381fb2f18f8Sesaxe 382fb2f18f8Sesaxe pg = cpu_cmt_hier[level]; 383fb2f18f8Sesaxe err = group_add_at(cmt_pgs, pg, nlevels - level - 1); 384fb2f18f8Sesaxe ASSERT(err == 0); 385fb2f18f8Sesaxe 386fb2f18f8Sesaxe if (level == 0) 387fb2f18f8Sesaxe cp->cpu_pg->cmt_lineage = (pg_t *)pg; 388fb2f18f8Sesaxe 389fb2f18f8Sesaxe if (pg->cmt_siblings != NULL) { 390fb2f18f8Sesaxe /* Already initialized */ 391fb2f18f8Sesaxe ASSERT(pg->cmt_parent == NULL || 392fb2f18f8Sesaxe pg->cmt_parent == cpu_cmt_hier[level + 1]); 393fb2f18f8Sesaxe ASSERT(pg->cmt_siblings == &lgrp->cl_pgs || 394fb2f18f8Sesaxe pg->cmt_siblings == pg->cmt_parent->cmt_children); 395fb2f18f8Sesaxe continue; 396fb2f18f8Sesaxe } 397fb2f18f8Sesaxe 398fb2f18f8Sesaxe if ((level + 1) == nlevels) { 399fb2f18f8Sesaxe pg->cmt_parent = NULL; 400fb2f18f8Sesaxe pg->cmt_siblings = &lgrp->cl_pgs; 401fb2f18f8Sesaxe children = ++lgrp->cl_npgs; 402fb2f18f8Sesaxe } else { 403fb2f18f8Sesaxe pg->cmt_parent = cpu_cmt_hier[level + 1]; 404fb2f18f8Sesaxe 405fb2f18f8Sesaxe /* 406fb2f18f8Sesaxe * A good parent keeps track of their children. 407fb2f18f8Sesaxe * The parent's children group is also the PG's 408fb2f18f8Sesaxe * siblings. 409fb2f18f8Sesaxe */ 410fb2f18f8Sesaxe if (pg->cmt_parent->cmt_children == NULL) { 411fb2f18f8Sesaxe pg->cmt_parent->cmt_children = 412fb2f18f8Sesaxe kmem_zalloc(sizeof (group_t), KM_SLEEP); 413fb2f18f8Sesaxe group_create(pg->cmt_parent->cmt_children); 414fb2f18f8Sesaxe } 415fb2f18f8Sesaxe pg->cmt_siblings = pg->cmt_parent->cmt_children; 416fb2f18f8Sesaxe children = ++pg->cmt_parent->cmt_nchildren; 417fb2f18f8Sesaxe } 418fb2f18f8Sesaxe pg->cmt_hint = 0; 419fb2f18f8Sesaxe group_expand(pg->cmt_siblings, children); 420fb2f18f8Sesaxe } 421fb2f18f8Sesaxe 422fb2f18f8Sesaxe /* 423fb2f18f8Sesaxe * Cache the chip and core IDs in the cpu_t->cpu_physid structure 424fb2f18f8Sesaxe * for fast lookups later. 425fb2f18f8Sesaxe */ 426fb2f18f8Sesaxe if (cp->cpu_physid) { 427fb2f18f8Sesaxe cp->cpu_physid->cpu_chipid = 428fb2f18f8Sesaxe pg_plat_hw_instance_id(cp, PGHW_CHIP); 429fb2f18f8Sesaxe cp->cpu_physid->cpu_coreid = pg_plat_get_core_id(cp); 430fb2f18f8Sesaxe 431fb2f18f8Sesaxe /* 432fb2f18f8Sesaxe * If this cpu has a PG representing shared cache, then set 433fb2f18f8Sesaxe * cpu_cacheid to that PG's logical id 434fb2f18f8Sesaxe */ 435fb2f18f8Sesaxe if (pg_cache) 436fb2f18f8Sesaxe cp->cpu_physid->cpu_cacheid = pg_cache->pg_id; 437fb2f18f8Sesaxe } 438fb2f18f8Sesaxe 439fb2f18f8Sesaxe /* CPU0 only initialization */ 440fb2f18f8Sesaxe if (is_cpu0) { 441fb2f18f8Sesaxe pg_cmt_cpu_startup(cp); 442fb2f18f8Sesaxe is_cpu0 = 0; 443*a6604450Sesaxe cpu0_lgrp = lgrp; 444fb2f18f8Sesaxe } 445fb2f18f8Sesaxe 446fb2f18f8Sesaxe } 447fb2f18f8Sesaxe 448fb2f18f8Sesaxe /* 449fb2f18f8Sesaxe * Class callback when a CPU is leaving the system (deletion) 450fb2f18f8Sesaxe */ 451fb2f18f8Sesaxe static void 452fb2f18f8Sesaxe pg_cmt_cpu_fini(cpu_t *cp) 453fb2f18f8Sesaxe { 454fb2f18f8Sesaxe group_iter_t i; 455fb2f18f8Sesaxe pg_cmt_t *pg; 456fb2f18f8Sesaxe group_t *pgs, *cmt_pgs; 457fb2f18f8Sesaxe lgrp_handle_t lgrp_handle; 458fb2f18f8Sesaxe cmt_lgrp_t *lgrp; 459fb2f18f8Sesaxe 460fb2f18f8Sesaxe pgs = &cp->cpu_pg->pgs; 461fb2f18f8Sesaxe cmt_pgs = &cp->cpu_pg->cmt_pgs; 462fb2f18f8Sesaxe 463fb2f18f8Sesaxe /* 464fb2f18f8Sesaxe * Find the lgroup that encapsulates this CPU's CMT hierarchy 465fb2f18f8Sesaxe */ 466fb2f18f8Sesaxe lgrp_handle = lgrp_plat_cpu_to_hand(cp->cpu_id); 467*a6604450Sesaxe 468fb2f18f8Sesaxe lgrp = pg_cmt_find_lgrp(lgrp_handle); 469*a6604450Sesaxe if (lgrp == NULL) { 470*a6604450Sesaxe /* 471*a6604450Sesaxe * This is a bit of a special case. 472*a6604450Sesaxe * The only way this can happen is if the CPU's lgrp 473*a6604450Sesaxe * handle changed out from underneath us, which is what 474*a6604450Sesaxe * happens with null_proc_lpa on starcat systems. 475*a6604450Sesaxe * 476*a6604450Sesaxe * Use the initial boot CPU lgrp, since this is what 477*a6604450Sesaxe * we need to tear down. 478*a6604450Sesaxe */ 479*a6604450Sesaxe lgrp = cpu0_lgrp; 480*a6604450Sesaxe } 481fb2f18f8Sesaxe 482fb2f18f8Sesaxe /* 483fb2f18f8Sesaxe * First, clean up anything load balancing specific for each of 484fb2f18f8Sesaxe * the CPU's PGs that participated in CMT load balancing 485fb2f18f8Sesaxe */ 486fb2f18f8Sesaxe pg = (pg_cmt_t *)cp->cpu_pg->cmt_lineage; 487fb2f18f8Sesaxe while (pg != NULL) { 488fb2f18f8Sesaxe 489fb2f18f8Sesaxe /* 490fb2f18f8Sesaxe * Remove the PG from the CPU's load balancing lineage 491fb2f18f8Sesaxe */ 492fb2f18f8Sesaxe (void) group_remove(cmt_pgs, pg, GRP_RESIZE); 493fb2f18f8Sesaxe 494fb2f18f8Sesaxe /* 495fb2f18f8Sesaxe * If it's about to become empty, destroy it's children 496fb2f18f8Sesaxe * group, and remove it's reference from it's siblings. 497fb2f18f8Sesaxe * This is done here (rather than below) to avoid removing 498fb2f18f8Sesaxe * our reference from a PG that we just eliminated. 499fb2f18f8Sesaxe */ 500fb2f18f8Sesaxe if (GROUP_SIZE(&((pg_t *)pg)->pg_cpus) == 1) { 501fb2f18f8Sesaxe if (pg->cmt_children != NULL) 502fb2f18f8Sesaxe group_destroy(pg->cmt_children); 503fb2f18f8Sesaxe if (pg->cmt_siblings != NULL) { 504fb2f18f8Sesaxe if (pg->cmt_siblings == &lgrp->cl_pgs) 505fb2f18f8Sesaxe lgrp->cl_npgs--; 506fb2f18f8Sesaxe else 507fb2f18f8Sesaxe pg->cmt_parent->cmt_nchildren--; 508fb2f18f8Sesaxe } 509fb2f18f8Sesaxe } 510fb2f18f8Sesaxe pg = pg->cmt_parent; 511fb2f18f8Sesaxe } 512fb2f18f8Sesaxe 513fb2f18f8Sesaxe ASSERT(GROUP_SIZE(cmt_pgs) == 0); 514fb2f18f8Sesaxe 515fb2f18f8Sesaxe /* 516fb2f18f8Sesaxe * Now that the load balancing lineage updates have happened, 517fb2f18f8Sesaxe * remove the CPU from all it's PGs (destroying any that become 518fb2f18f8Sesaxe * empty). 519fb2f18f8Sesaxe */ 520fb2f18f8Sesaxe group_iter_init(&i); 521fb2f18f8Sesaxe while ((pg = group_iterate(pgs, &i)) != NULL) { 522fb2f18f8Sesaxe if (IS_CMT_PG(pg) == 0) 523fb2f18f8Sesaxe continue; 524fb2f18f8Sesaxe 525fb2f18f8Sesaxe pg_cpu_delete((pg_t *)pg, cp); 526fb2f18f8Sesaxe /* 527fb2f18f8Sesaxe * Deleting the CPU from the PG changes the CPU's 528fb2f18f8Sesaxe * PG group over which we are actively iterating 529fb2f18f8Sesaxe * Re-initialize the iteration 530fb2f18f8Sesaxe */ 531fb2f18f8Sesaxe group_iter_init(&i); 532fb2f18f8Sesaxe 533fb2f18f8Sesaxe if (GROUP_SIZE(&((pg_t *)pg)->pg_cpus) == 0) { 534fb2f18f8Sesaxe 535fb2f18f8Sesaxe /* 536fb2f18f8Sesaxe * The PG has become zero sized, so destroy it. 537fb2f18f8Sesaxe */ 538fb2f18f8Sesaxe group_destroy(&pg->cmt_cpus_actv); 539fb2f18f8Sesaxe bitset_fini(&pg->cmt_cpus_actv_set); 540fb2f18f8Sesaxe pghw_fini((pghw_t *)pg); 541fb2f18f8Sesaxe 542fb2f18f8Sesaxe pg_destroy((pg_t *)pg); 543fb2f18f8Sesaxe } 544fb2f18f8Sesaxe } 545fb2f18f8Sesaxe } 546fb2f18f8Sesaxe 547fb2f18f8Sesaxe /* 548fb2f18f8Sesaxe * Class callback when a CPU is entering a cpu partition 549fb2f18f8Sesaxe */ 550fb2f18f8Sesaxe static void 551fb2f18f8Sesaxe pg_cmt_cpupart_in(cpu_t *cp, cpupart_t *pp) 552fb2f18f8Sesaxe { 553fb2f18f8Sesaxe group_t *pgs; 554fb2f18f8Sesaxe pg_t *pg; 555fb2f18f8Sesaxe group_iter_t i; 556fb2f18f8Sesaxe 557fb2f18f8Sesaxe ASSERT(MUTEX_HELD(&cpu_lock)); 558fb2f18f8Sesaxe 559fb2f18f8Sesaxe pgs = &cp->cpu_pg->pgs; 560fb2f18f8Sesaxe 561fb2f18f8Sesaxe /* 562fb2f18f8Sesaxe * Ensure that the new partition's PG bitset 563fb2f18f8Sesaxe * is large enough for all CMT PG's to which cp 564fb2f18f8Sesaxe * belongs 565fb2f18f8Sesaxe */ 566fb2f18f8Sesaxe group_iter_init(&i); 567fb2f18f8Sesaxe while ((pg = group_iterate(pgs, &i)) != NULL) { 568fb2f18f8Sesaxe if (IS_CMT_PG(pg) == 0) 569fb2f18f8Sesaxe continue; 570fb2f18f8Sesaxe 571fb2f18f8Sesaxe if (bitset_capacity(&pp->cp_cmt_pgs) <= pg->pg_id) 572fb2f18f8Sesaxe bitset_resize(&pp->cp_cmt_pgs, pg->pg_id + 1); 573fb2f18f8Sesaxe } 574fb2f18f8Sesaxe } 575fb2f18f8Sesaxe 576fb2f18f8Sesaxe /* 577fb2f18f8Sesaxe * Class callback when a CPU is actually moving partitions 578fb2f18f8Sesaxe */ 579fb2f18f8Sesaxe static void 580fb2f18f8Sesaxe pg_cmt_cpupart_move(cpu_t *cp, cpupart_t *oldpp, cpupart_t *newpp) 581fb2f18f8Sesaxe { 582fb2f18f8Sesaxe cpu_t *cpp; 583fb2f18f8Sesaxe group_t *pgs; 584fb2f18f8Sesaxe pg_t *pg; 585fb2f18f8Sesaxe group_iter_t pg_iter; 586fb2f18f8Sesaxe pg_cpu_itr_t cpu_iter; 587fb2f18f8Sesaxe boolean_t found; 588fb2f18f8Sesaxe 589fb2f18f8Sesaxe ASSERT(MUTEX_HELD(&cpu_lock)); 590fb2f18f8Sesaxe 591fb2f18f8Sesaxe pgs = &cp->cpu_pg->pgs; 592fb2f18f8Sesaxe group_iter_init(&pg_iter); 593fb2f18f8Sesaxe 594fb2f18f8Sesaxe /* 595fb2f18f8Sesaxe * Iterate over the CPUs CMT PGs 596fb2f18f8Sesaxe */ 597fb2f18f8Sesaxe while ((pg = group_iterate(pgs, &pg_iter)) != NULL) { 598fb2f18f8Sesaxe 599fb2f18f8Sesaxe if (IS_CMT_PG(pg) == 0) 600fb2f18f8Sesaxe continue; 601fb2f18f8Sesaxe 602fb2f18f8Sesaxe /* 603fb2f18f8Sesaxe * Add the PG to the bitset in the new partition. 604fb2f18f8Sesaxe */ 605fb2f18f8Sesaxe bitset_add(&newpp->cp_cmt_pgs, pg->pg_id); 606fb2f18f8Sesaxe 607fb2f18f8Sesaxe /* 608fb2f18f8Sesaxe * Remove the PG from the bitset in the old partition 609fb2f18f8Sesaxe * if the last of the PG's CPUs have left. 610fb2f18f8Sesaxe */ 611fb2f18f8Sesaxe found = B_FALSE; 612fb2f18f8Sesaxe PG_CPU_ITR_INIT(pg, cpu_iter); 613fb2f18f8Sesaxe while ((cpp = pg_cpu_next(&cpu_iter)) != NULL) { 614fb2f18f8Sesaxe if (cpp == cp) 615fb2f18f8Sesaxe continue; 616*a6604450Sesaxe if (CPU_ACTIVE(cpp) && 617*a6604450Sesaxe cpp->cpu_part->cp_id == oldpp->cp_id) { 618fb2f18f8Sesaxe found = B_TRUE; 619fb2f18f8Sesaxe break; 620fb2f18f8Sesaxe } 621fb2f18f8Sesaxe } 622fb2f18f8Sesaxe if (!found) 623fb2f18f8Sesaxe bitset_del(&cp->cpu_part->cp_cmt_pgs, pg->pg_id); 624fb2f18f8Sesaxe } 625fb2f18f8Sesaxe } 626fb2f18f8Sesaxe 627fb2f18f8Sesaxe /* 628fb2f18f8Sesaxe * Class callback when a CPU becomes active (online) 629fb2f18f8Sesaxe * 630fb2f18f8Sesaxe * This is called in a context where CPUs are paused 631fb2f18f8Sesaxe */ 632fb2f18f8Sesaxe static void 633fb2f18f8Sesaxe pg_cmt_cpu_active(cpu_t *cp) 634fb2f18f8Sesaxe { 635fb2f18f8Sesaxe int err; 636fb2f18f8Sesaxe group_iter_t i; 637fb2f18f8Sesaxe pg_cmt_t *pg; 638fb2f18f8Sesaxe group_t *pgs; 639fb2f18f8Sesaxe 640fb2f18f8Sesaxe ASSERT(MUTEX_HELD(&cpu_lock)); 641fb2f18f8Sesaxe 642fb2f18f8Sesaxe pgs = &cp->cpu_pg->pgs; 643fb2f18f8Sesaxe group_iter_init(&i); 644fb2f18f8Sesaxe 645fb2f18f8Sesaxe /* 646fb2f18f8Sesaxe * Iterate over the CPU's PGs 647fb2f18f8Sesaxe */ 648fb2f18f8Sesaxe while ((pg = group_iterate(pgs, &i)) != NULL) { 649fb2f18f8Sesaxe 650fb2f18f8Sesaxe if (IS_CMT_PG(pg) == 0) 651fb2f18f8Sesaxe continue; 652fb2f18f8Sesaxe 653fb2f18f8Sesaxe err = group_add(&pg->cmt_cpus_actv, cp, GRP_NORESIZE); 654fb2f18f8Sesaxe ASSERT(err == 0); 655fb2f18f8Sesaxe 656fb2f18f8Sesaxe /* 657fb2f18f8Sesaxe * If this is the first active CPU in the PG, and it 658fb2f18f8Sesaxe * represents a hardware sharing relationship over which 659fb2f18f8Sesaxe * CMT load balancing is performed, add it as a candidate 660fb2f18f8Sesaxe * for balancing with it's siblings. 661fb2f18f8Sesaxe */ 662fb2f18f8Sesaxe if (GROUP_SIZE(&pg->cmt_cpus_actv) == 1 && 663fb2f18f8Sesaxe pg_cmt_load_bal_hw(((pghw_t *)pg)->pghw_hw)) { 664fb2f18f8Sesaxe err = group_add(pg->cmt_siblings, pg, GRP_NORESIZE); 665fb2f18f8Sesaxe ASSERT(err == 0); 666fb2f18f8Sesaxe } 667fb2f18f8Sesaxe 668fb2f18f8Sesaxe /* 669fb2f18f8Sesaxe * Notate the CPU in the PGs active CPU bitset. 670fb2f18f8Sesaxe * Also notate the PG as being active in it's associated 671fb2f18f8Sesaxe * partition 672fb2f18f8Sesaxe */ 673fb2f18f8Sesaxe bitset_add(&pg->cmt_cpus_actv_set, cp->cpu_seqid); 674fb2f18f8Sesaxe bitset_add(&cp->cpu_part->cp_cmt_pgs, ((pg_t *)pg)->pg_id); 675fb2f18f8Sesaxe } 676fb2f18f8Sesaxe } 677fb2f18f8Sesaxe 678fb2f18f8Sesaxe /* 679fb2f18f8Sesaxe * Class callback when a CPU goes inactive (offline) 680fb2f18f8Sesaxe * 681fb2f18f8Sesaxe * This is called in a context where CPUs are paused 682fb2f18f8Sesaxe */ 683fb2f18f8Sesaxe static void 684fb2f18f8Sesaxe pg_cmt_cpu_inactive(cpu_t *cp) 685fb2f18f8Sesaxe { 686fb2f18f8Sesaxe int err; 687fb2f18f8Sesaxe group_t *pgs; 688fb2f18f8Sesaxe pg_cmt_t *pg; 689fb2f18f8Sesaxe cpu_t *cpp; 690fb2f18f8Sesaxe group_iter_t i; 691fb2f18f8Sesaxe pg_cpu_itr_t cpu_itr; 692fb2f18f8Sesaxe boolean_t found; 693fb2f18f8Sesaxe 694fb2f18f8Sesaxe ASSERT(MUTEX_HELD(&cpu_lock)); 695fb2f18f8Sesaxe 696fb2f18f8Sesaxe pgs = &cp->cpu_pg->pgs; 697fb2f18f8Sesaxe group_iter_init(&i); 698fb2f18f8Sesaxe 699fb2f18f8Sesaxe while ((pg = group_iterate(pgs, &i)) != NULL) { 700fb2f18f8Sesaxe 701fb2f18f8Sesaxe if (IS_CMT_PG(pg) == 0) 702fb2f18f8Sesaxe continue; 703fb2f18f8Sesaxe 704fb2f18f8Sesaxe /* 705fb2f18f8Sesaxe * Remove the CPU from the CMT PGs active CPU group 706fb2f18f8Sesaxe * bitmap 707fb2f18f8Sesaxe */ 708fb2f18f8Sesaxe err = group_remove(&pg->cmt_cpus_actv, cp, GRP_NORESIZE); 709fb2f18f8Sesaxe ASSERT(err == 0); 710fb2f18f8Sesaxe 711fb2f18f8Sesaxe bitset_del(&pg->cmt_cpus_actv_set, cp->cpu_seqid); 712fb2f18f8Sesaxe 713fb2f18f8Sesaxe /* 714fb2f18f8Sesaxe * If there are no more active CPUs in this PG over which 715fb2f18f8Sesaxe * load was balanced, remove it as a balancing candidate. 716fb2f18f8Sesaxe */ 717fb2f18f8Sesaxe if (GROUP_SIZE(&pg->cmt_cpus_actv) == 0 && 718fb2f18f8Sesaxe pg_cmt_load_bal_hw(((pghw_t *)pg)->pghw_hw)) { 719fb2f18f8Sesaxe err = group_remove(pg->cmt_siblings, pg, GRP_NORESIZE); 720fb2f18f8Sesaxe ASSERT(err == 0); 721fb2f18f8Sesaxe } 722fb2f18f8Sesaxe 723fb2f18f8Sesaxe /* 724fb2f18f8Sesaxe * Assert the number of active CPUs does not exceed 725fb2f18f8Sesaxe * the total number of CPUs in the PG 726fb2f18f8Sesaxe */ 727fb2f18f8Sesaxe ASSERT(GROUP_SIZE(&pg->cmt_cpus_actv) <= 728fb2f18f8Sesaxe GROUP_SIZE(&((pg_t *)pg)->pg_cpus)); 729fb2f18f8Sesaxe 730fb2f18f8Sesaxe /* 731fb2f18f8Sesaxe * Update the PG bitset in the CPU's old partition 732fb2f18f8Sesaxe */ 733fb2f18f8Sesaxe found = B_FALSE; 734fb2f18f8Sesaxe PG_CPU_ITR_INIT(pg, cpu_itr); 735fb2f18f8Sesaxe while ((cpp = pg_cpu_next(&cpu_itr)) != NULL) { 736fb2f18f8Sesaxe if (cpp == cp) 737fb2f18f8Sesaxe continue; 738*a6604450Sesaxe if (CPU_ACTIVE(cpp) && 739*a6604450Sesaxe cpp->cpu_part->cp_id == cp->cpu_part->cp_id) { 740fb2f18f8Sesaxe found = B_TRUE; 741fb2f18f8Sesaxe break; 742fb2f18f8Sesaxe } 743fb2f18f8Sesaxe } 744fb2f18f8Sesaxe if (!found) { 745fb2f18f8Sesaxe bitset_del(&cp->cpu_part->cp_cmt_pgs, 746fb2f18f8Sesaxe ((pg_t *)pg)->pg_id); 747fb2f18f8Sesaxe } 748fb2f18f8Sesaxe } 749fb2f18f8Sesaxe } 750fb2f18f8Sesaxe 751fb2f18f8Sesaxe /* 752fb2f18f8Sesaxe * Return non-zero if the CPU belongs in the given PG 753fb2f18f8Sesaxe */ 754fb2f18f8Sesaxe static int 755fb2f18f8Sesaxe pg_cmt_cpu_belongs(pg_t *pg, cpu_t *cp) 756fb2f18f8Sesaxe { 757fb2f18f8Sesaxe cpu_t *pg_cpu; 758fb2f18f8Sesaxe 759fb2f18f8Sesaxe pg_cpu = GROUP_ACCESS(&pg->pg_cpus, 0); 760fb2f18f8Sesaxe 761fb2f18f8Sesaxe ASSERT(pg_cpu != NULL); 762fb2f18f8Sesaxe 763fb2f18f8Sesaxe /* 764fb2f18f8Sesaxe * The CPU belongs if, given the nature of the hardware sharing 765fb2f18f8Sesaxe * relationship represented by the PG, the CPU has that 766fb2f18f8Sesaxe * relationship with some other CPU already in the PG 767fb2f18f8Sesaxe */ 768fb2f18f8Sesaxe if (pg_plat_cpus_share(cp, pg_cpu, ((pghw_t *)pg)->pghw_hw)) 769fb2f18f8Sesaxe return (1); 770fb2f18f8Sesaxe 771fb2f18f8Sesaxe return (0); 772fb2f18f8Sesaxe } 773fb2f18f8Sesaxe 774fb2f18f8Sesaxe /* 775fb2f18f8Sesaxe * Pack the CPUs CMT hierarchy 776fb2f18f8Sesaxe * The hierarchy order is preserved 777fb2f18f8Sesaxe */ 778fb2f18f8Sesaxe static void 779fb2f18f8Sesaxe pg_cmt_hier_pack(pg_cmt_t *hier[], int sz) 780fb2f18f8Sesaxe { 781fb2f18f8Sesaxe int i, j; 782fb2f18f8Sesaxe 783fb2f18f8Sesaxe for (i = 0; i < sz; i++) { 784fb2f18f8Sesaxe if (hier[i] != NULL) 785fb2f18f8Sesaxe continue; 786fb2f18f8Sesaxe 787fb2f18f8Sesaxe for (j = i; j < sz; j++) { 788fb2f18f8Sesaxe if (hier[j] != NULL) { 789fb2f18f8Sesaxe hier[i] = hier[j]; 790fb2f18f8Sesaxe hier[j] = NULL; 791fb2f18f8Sesaxe break; 792fb2f18f8Sesaxe } 793fb2f18f8Sesaxe } 794fb2f18f8Sesaxe if (j == sz) 795fb2f18f8Sesaxe break; 796fb2f18f8Sesaxe } 797fb2f18f8Sesaxe } 798fb2f18f8Sesaxe 799fb2f18f8Sesaxe /* 800fb2f18f8Sesaxe * Return a cmt_lgrp_t * given an lgroup handle. 801fb2f18f8Sesaxe */ 802fb2f18f8Sesaxe static cmt_lgrp_t * 803fb2f18f8Sesaxe pg_cmt_find_lgrp(lgrp_handle_t hand) 804fb2f18f8Sesaxe { 805fb2f18f8Sesaxe cmt_lgrp_t *lgrp; 806fb2f18f8Sesaxe 807fb2f18f8Sesaxe ASSERT(MUTEX_HELD(&cpu_lock)); 808fb2f18f8Sesaxe 809fb2f18f8Sesaxe lgrp = cmt_lgrps; 810fb2f18f8Sesaxe while (lgrp != NULL) { 811fb2f18f8Sesaxe if (lgrp->cl_hand == hand) 812*a6604450Sesaxe break; 813fb2f18f8Sesaxe lgrp = lgrp->cl_next; 814fb2f18f8Sesaxe } 815*a6604450Sesaxe return (lgrp); 816*a6604450Sesaxe } 817fb2f18f8Sesaxe 818fb2f18f8Sesaxe /* 819*a6604450Sesaxe * Create a cmt_lgrp_t with the specified handle. 820fb2f18f8Sesaxe */ 821*a6604450Sesaxe static cmt_lgrp_t * 822*a6604450Sesaxe pg_cmt_lgrp_create(lgrp_handle_t hand) 823*a6604450Sesaxe { 824*a6604450Sesaxe cmt_lgrp_t *lgrp; 825*a6604450Sesaxe 826*a6604450Sesaxe ASSERT(MUTEX_HELD(&cpu_lock)); 827*a6604450Sesaxe 828fb2f18f8Sesaxe lgrp = kmem_zalloc(sizeof (cmt_lgrp_t), KM_SLEEP); 829fb2f18f8Sesaxe 830fb2f18f8Sesaxe lgrp->cl_hand = hand; 831fb2f18f8Sesaxe lgrp->cl_npgs = 0; 832fb2f18f8Sesaxe lgrp->cl_next = cmt_lgrps; 833fb2f18f8Sesaxe cmt_lgrps = lgrp; 834fb2f18f8Sesaxe group_create(&lgrp->cl_pgs); 835fb2f18f8Sesaxe 836fb2f18f8Sesaxe return (lgrp); 837fb2f18f8Sesaxe } 838