1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 #ifndef _LGRP_H 28 #define _LGRP_H 29 30 /* 31 * locality group definitions for kernel 32 */ 33 34 #include <sys/types.h> 35 36 #ifdef __cplusplus 37 extern "C" { 38 #endif 39 40 #define LGRP_NONE (-1) /* non-existent lgroup ID */ 41 42 #if (!defined(_KERNEL) && !defined(_KMEMUSER)) 43 typedef struct lgrp_mem_policy_info { int opaque[2]; } lgrp_mem_policy_info_t; 44 #endif /* !_KERNEL && !_KMEMUSER */ 45 46 #if (defined(_KERNEL) || defined(_KMEMUSER)) 47 #include <sys/cpuvar.h> 48 #include <sys/bitmap.h> 49 #include <sys/vnode.h> 50 #include <vm/anon.h> 51 #include <vm/seg.h> 52 #include <sys/lgrp_user.h> 53 #include <sys/param.h> 54 55 typedef uint32_t lgrp_load_t; /* lgrp_loadavg type */ 56 typedef uintptr_t lgrp_handle_t; /* lgrp handle */ 57 58 #define LGRP_NONE_SUCH LGRP_NONE /* non-existent lgroup ID */ 59 /* null platform handle */ 60 #define LGRP_NULL_HANDLE ((lgrp_handle_t)0xbadbad) 61 #define LGRP_DEFAULT_HANDLE ((lgrp_handle_t)0xbabecafe) /* uma handle */ 62 #define LGRP_ROOTID (0) /* root lgroup ID */ 63 64 /* 65 * Maximum number of lgrps a platform may define. 66 */ 67 #define NLGRPS_MAX 64 68 #define LGRP_LOADAVG_MAX UINT32_MAX 69 70 /* 71 * The load-average we expect for one cpu-bound thread's worth of load 72 */ 73 #define LGRP_LOADAVG_THREAD_MAX 65516 74 75 /* 76 * The input to the load-average generating function for one cpu-bound thread's 77 * worth of load 78 */ 79 80 #define LGRP_LOADAVG_IN_THREAD_MAX 128 81 82 /* 83 * LPL actions 84 */ 85 86 typedef enum { 87 LPL_INCREMENT, 88 LPL_DECREMENT 89 } lpl_act_t; 90 91 /* 92 * lgroup statistics. Most of these are counters that are updated 93 * dynamically so they are hashed to CPU buckets to reduce cache 94 * interference. The remaining statistics are snapshots of kernel 95 * data, so they aren't stored in the array of counter stats. 96 * 97 * For the hashed stats to make sense, you have to sum all the buckets for 98 * that stat, hence macros are provided to read the stats. 99 */ 100 101 #define LGRP_NUM_CPU_BUCKETS 8 /* must be power of 2 */ 102 #define LGRP_CPU_BUCKET_MASK (LGRP_NUM_CPU_BUCKETS - 1) 103 104 /* 105 * Flags for what to do with lgroup memory policy 106 * Used for heap and stack where policy is extended to new segments added to 107 * the end 108 */ 109 #define LGRP_MP_FLAG_EXTEND_UP 0x1 /* policy should extend up */ 110 #define LGRP_MP_FLAG_EXTEND_DOWN 0x2 /* policy should extend down */ 111 112 #define LGRP_STAT(stats, bucket, whichstat) \ 113 ((stats)->ls_data[bucket][whichstat]) 114 115 /* Return a pointer suitable for an atomic 64-bit op on the bucket */ 116 #define LGRP_STAT_WRITE_PTR(stats, whichstat) \ 117 (&LGRP_STAT(stats, (CPU->cpu_id) & LGRP_CPU_BUCKET_MASK, \ 118 whichstat)) 119 120 /* Sum up all the buckets and return the value in 'val' */ 121 #define LGRP_STAT_READ(stats, whichstat, val) { \ 122 int bkt; \ 123 for (val = 0, bkt = 0; bkt < LGRP_NUM_CPU_BUCKETS; bkt++) \ 124 val += LGRP_STAT(stats, bkt, whichstat); \ 125 } 126 127 /* Reset all buckets for the stat to 0 */ 128 #define LGRP_STAT_RESET(stats, stat) { \ 129 int i; \ 130 for (i = 0; i < LGRP_NUM_CPU_BUCKETS; i++) \ 131 LGRP_STAT(stats, i, stat) = 0; \ 132 } 133 134 /* 135 * Define all of the statistics that are kept for lgrp kstats, 136 * and their corresponding text names. 137 */ 138 139 typedef enum lgrp_stat_types { 140 LGRP_NUM_MIGR, /* # migrations away from this lgrp */ 141 LGRP_NUM_ALLOC_FAIL, /* # times alloc fails for chosen lgrp */ 142 LGRP_PM_SRC_PGS, /* # pages migrated from this lgrp */ 143 LGRP_PM_DEST_PGS, /* # pages migrated to this lgrp */ 144 LGRP_PM_FAIL_ALLOC_PGS, /* # pages failed to migrate to this lgrp */ 145 LGRP_PM_FAIL_LOCK_PGS, /* # pages failed to migrate from this lgrp */ 146 LGRP_PMM_PGS, /* # pages marked to migrate from this lgrp */ 147 LGRP_PMM_FAIL_PGS, /* # pages marked to migrate from this lgrp */ 148 LGRP_NUM_DEFAULT, /* # of times default policy applied */ 149 LGRP_NUM_NEXT, /* # of times next touch policy applied */ 150 LGRP_NUM_RANDOM, /* # of times random policy applied */ 151 LGRP_NUM_RANDOM_PROC, /* # of times random proc policy applied */ 152 LGRP_NUM_RANDOM_PSET, /* # of times random pset policy applied */ 153 LGRP_NUM_ROUNDROBIN, /* # of times round robin policy applied */ 154 LGRP_NUM_NEXT_SEG, /* # of times next to seg policy applied */ 155 LGRP_NUM_COUNTER_STATS, /* always last */ 156 LGRP_CTR_STATS_ALLOC = 16 /* cache-align pad - multiple of 8 */ 157 /* always keep >= LGRP_NUM_COUNTER_STATS */ 158 } lgrp_stat_t; 159 160 typedef enum lgrp_snap_stat_types { 161 LGRP_NUM_CPUS, /* number of CPUs */ 162 LGRP_NUM_PG_FREE, /* # of free pages */ 163 LGRP_NUM_PG_AVAIL, /* # of allocatable physical pages */ 164 LGRP_NUM_PG_INSTALL, /* # of installed physical pages */ 165 LGRP_LOADAVG, /* unscaled load average of this lgrp */ 166 LGRP_LOADAVG_SCALE, /* load unit of one CPU bound thread */ 167 LGRP_NUM_SNAPSHOT_STATS /* always last */ 168 } lgrp_snap_stat_t; 169 170 #define LGRP_KSTAT_NAMES \ 171 static char *lgrp_kstat_names[] = { \ 172 \ 173 /* Counter stats */ \ 174 "lwp migrations", \ 175 "alloc fail", \ 176 "pages migrated from", \ 177 "pages migrated to", \ 178 "pages failed to migrate to", \ 179 "pages failed to migrate from", \ 180 "pages marked for migration", \ 181 "pages failed to mark", \ 182 "default policy", \ 183 "next-touch policy", \ 184 "random policy", \ 185 "span process policy", \ 186 "span psrset policy", \ 187 "round robin policy", \ 188 "next-seg policy", \ 189 \ 190 /* Snapshot stats */ \ 191 "cpus", \ 192 "pages free", \ 193 "pages avail", \ 194 "pages installed", \ 195 "load average", \ 196 "loadscale" \ 197 } 198 199 #define LGRP_NUM_STATS ((int)LGRP_NUM_COUNTER_STATS + \ 200 (int)LGRP_NUM_SNAPSHOT_STATS) 201 202 /* 203 * The contents of this structure are opaque and should only be 204 * accessed through the LGRP_STAT macro. 205 */ 206 struct lgrp_stats { 207 int64_t ls_data[LGRP_NUM_CPU_BUCKETS][LGRP_CTR_STATS_ALLOC]; 208 }; 209 210 /* The kernel's version of a bitmap of lgroups */ 211 typedef uint64_t klgrpset_t; 212 213 /* 214 * This really belongs in memnode.h, but it must be defined here to avoid 215 * recursive inclusion problems. Note that memnode.h includes this header. 216 */ 217 typedef uint64_t mnodeset_t; 218 219 /* 220 * lgroup structure 221 * 222 * Visible to generic code and contains the lgroup ID, CPUs in this lgroup, 223 * and a platform handle used to identify this lgroup to the lgroup platform 224 * support code 225 */ 226 typedef struct lgrp { 227 228 lgrp_id_t lgrp_id; /* which lgroup */ 229 int lgrp_latency; 230 lgrp_handle_t lgrp_plathand; /* handle for platform calls */ 231 struct lgrp *lgrp_parent; /* parent lgroup */ 232 uint_t lgrp_reserved1; /* filler */ 233 uint_t lgrp_childcnt; /* number of children lgroups */ 234 klgrpset_t lgrp_children; /* children lgroups */ 235 klgrpset_t lgrp_leaves; /* (direct decendant) leaf lgroups */ 236 237 /* 238 * set of lgroups containing a given type of resource 239 * at this level of locality 240 */ 241 klgrpset_t lgrp_set[LGRP_RSRC_COUNT]; 242 243 mnodeset_t lgrp_mnodes; /* set of memory nodes in this lgroup */ 244 uint_t lgrp_nmnodes; /* number of memnodes */ 245 uint_t lgrp_reserved2; /* filler */ 246 247 struct cpu *lgrp_cpu; /* pointer to a cpu may be null */ 248 uint_t lgrp_cpucnt; /* number of cpus in this lgrp */ 249 kstat_t *lgrp_kstat; /* per-lgrp kstats */ 250 } lgrp_t; 251 252 /* 253 * lgroup load average structure 254 */ 255 256 typedef struct lgrp_ld { 257 lgrp_load_t lpl_loadavg; /* load average */ 258 uint_t lpl_ncpu; /* how many cpus */ 259 lgrp_id_t lpl_lgrpid; /* which group this lpl part of */ 260 lgrp_t *lpl_lgrp; /* ptr to lpl's lgrp */ 261 struct lgrp_ld *lpl_parent; /* lpl of parent lgrp */ 262 struct cpu *lpl_cpus; /* list of cpus in lpl */ 263 /* NULL for non-leaf lgrps */ 264 uint_t lpl_nrset; /* no. of leaf lpls for lgrp */ 265 hrtime_t lpl_homed_time; /* time of last homing to this lpl */ 266 uint_t lpl_rset_sz; /* Resource set capacity */ 267 struct lgrp_ld **lpl_rset; /* leaf lpls for lgrp */ 268 /* contains ptr to self for leaf lgrp */ 269 int *lpl_id2rset; /* mapping of lgrpid to rset index */ 270 } lpl_t; 271 272 /* 273 * 1 << LGRP_MAX_EFFECT_SHFT == lgrp_loadavg_max_effect 274 */ 275 #define LGRP_MAX_EFFECT_SHFT 16 276 277 /* 278 * Operations handled by lgrp_config() 279 */ 280 typedef enum lgrp_config_flag { 281 LGRP_CONFIG_NOP, 282 LGRP_CONFIG_CPU_ADD, 283 LGRP_CONFIG_CPU_DEL, 284 LGRP_CONFIG_CPU_ONLINE, 285 LGRP_CONFIG_CPU_OFFLINE, 286 LGRP_CONFIG_CPUPART_ADD, 287 LGRP_CONFIG_CPUPART_DEL, 288 LGRP_CONFIG_MEM_ADD, 289 LGRP_CONFIG_MEM_DEL, 290 LGRP_CONFIG_MEM_RENAME, 291 LGRP_CONFIG_GEN_UPDATE, 292 LGRP_CONFIG_FLATTEN, 293 LGRP_CONFIG_LAT_CHANGE_ALL, 294 LGRP_CONFIG_LAT_CHANGE 295 } lgrp_config_flag_t; 296 297 /* 298 * Stages of lgroup framework initialization (done through lgrp_init()): 299 * 300 * 1) Initialize common and platform specific code (called in mlsetup()) 301 * 302 * 2) Setup root lgroup and add CPU 0 to lgroup(s) (called near beginning of 303 * main() before startup()) 304 * 305 * 3) Probe from CPU 0 and copy and release any BOP_ALLOC-ed memory temporarily 306 * allocated before kernel memory allocator is setup (called in main() 307 * after startup(), gethrtime() is setup, and before interrupts enabled) 308 * 309 * 4) Check for null proc LPA on Starcat, collapse lgroup topology (if 310 * necessary), setup lgroup kstats, etc. (called before start_other_cpus()) 311 * 312 * 5) Finish any lgroup initialization needed including updating lgroup 313 * topology after all CPUs started (called after start_other_cpus()) 314 */ 315 typedef enum lgrp_init_stages { 316 LGRP_INIT_STAGE1, 317 LGRP_INIT_STAGE2, 318 LGRP_INIT_STAGE3, 319 LGRP_INIT_STAGE4, 320 LGRP_INIT_STAGE5 321 } lgrp_init_stages_t; 322 323 /* 324 * Memory allocation policies 325 */ 326 typedef enum lgrp_mem_policy { 327 LGRP_MEM_POLICY_DEFAULT, 328 LGRP_MEM_POLICY_NEXT, /* near LWP to next touch */ 329 LGRP_MEM_POLICY_RANDOM_PROC, /* randomly across process */ 330 LGRP_MEM_POLICY_RANDOM_PSET, /* randomly across processor set */ 331 LGRP_MEM_POLICY_RANDOM, /* randomly across all lgroups */ 332 LGRP_MEM_POLICY_ROUNDROBIN, /* round robin across all lgroups */ 333 LGRP_MEM_POLICY_NEXT_CPU, /* Near next CPU to touch memory */ 334 LGRP_MEM_POLICY_NEXT_SEG, /* lgrp specified directly by seg */ 335 LGRP_NUM_MEM_POLICIES 336 } lgrp_mem_policy_t; 337 338 /* 339 * Search scopes for finding resouces 340 */ 341 typedef enum lgrp_res_ss { 342 LGRP_SRCH_LOCAL, /* Search local lgroup only */ 343 LGRP_SRCH_HIER /* Search entire hierarchy */ 344 } lgrp_res_ss_t; 345 346 /* 347 * Cookie used for lgrp mnode selection 348 */ 349 typedef struct lgrp_mnode_cookie { 350 lgrp_t *lmc_lgrp; /* lgrp under consideration */ 351 mnodeset_t lmc_nodes; /* nodes not yet tried in lgrp */ 352 int lmc_cnt; /* how many nodes in untried set */ 353 mnodeset_t lmc_tried; /* nodes already tried */ 354 int lmc_ntried; /* how many nodes in tried set */ 355 lgrp_res_ss_t lmc_scope; /* consider non-local nodes? */ 356 ushort_t lmc_rand; /* a "random" number */ 357 } lgrp_mnode_cookie_t; 358 359 /* 360 * Information needed to implement memory allocation policy 361 */ 362 typedef struct lgrp_mem_policy_info { 363 int mem_policy; /* memory allocation policy */ 364 lgrp_id_t mem_lgrpid; /* lgroup id */ 365 } lgrp_mem_policy_info_t; 366 367 /* 368 * Shared memory policy segment 369 */ 370 typedef struct lgrp_shm_policy_seg { 371 u_offset_t shm_off; /* offset into shared object */ 372 size_t shm_size; /* size of segment */ 373 lgrp_mem_policy_info_t shm_policy; /* memory allocation policy */ 374 avl_node_t shm_tree; /* AVL tree */ 375 } lgrp_shm_policy_seg_t; 376 377 /* 378 * Shared memory locality info 379 */ 380 typedef struct lgrp_shm_locality { 381 size_t loc_count; /* reference count */ 382 avl_tree_t *loc_tree; /* policy segment tree */ 383 krwlock_t loc_lock; /* protects tree */ 384 } lgrp_shm_locality_t; 385 386 /* 387 * Queries that may be made to determine lgroup memory size 388 */ 389 typedef enum { 390 LGRP_MEM_SIZE_FREE, /* number of free pages */ 391 LGRP_MEM_SIZE_AVAIL, /* number of pages in phys_avail */ 392 LGRP_MEM_SIZE_INSTALL /* number of pages in phys_install */ 393 } lgrp_mem_query_t; 394 395 /* 396 * Argument for the memory copy-rename operation, contains the source and the 397 * destination platform handles. 398 */ 399 typedef struct lgrp_config_mem_rename { 400 lgrp_handle_t lmem_rename_from; 401 lgrp_handle_t lmem_rename_to; 402 } lgrp_config_mem_rename_t; 403 404 /* Macro to clear an lgroup bitmap */ 405 #define klgrpset_clear(klgrpset) \ 406 (klgrpset) = (klgrpset_t)0 407 408 /* Macro to fill an lgroup bitmap */ 409 #define klgrpset_fill(klgrpset) \ 410 (klgrpset) = (klgrpset_t)(-1) 411 412 /* Macro to add an lgroup to an lgroup bitmap */ 413 #define klgrpset_add(klgrpset, lgrpid) \ 414 (klgrpset) |= ((klgrpset_t)1 << (lgrpid)) 415 416 /* Macro to delete an lgroup from an lgroup bitmap */ 417 #define klgrpset_del(klgrpset, lgrpid) \ 418 (klgrpset) &= ~((klgrpset_t)1 << (lgrpid)) 419 420 /* Macro to copy a klgrpset into another klgrpset */ 421 #define klgrpset_copy(klgrpset_to, klgrpset_from) \ 422 (klgrpset_to) = (klgrpset_from) 423 424 /* Macro to perform an 'and' operation on a pair of lgroup bitmaps */ 425 #define klgrpset_and(klgrpset_rslt, klgrpset_arg) \ 426 (klgrpset_rslt) &= (klgrpset_arg) 427 428 /* Macro to perform an 'or' operation on a pair of lgroup bitmaps */ 429 #define klgrpset_or(klgrpset_rslt, klgrpset_arg) \ 430 (klgrpset_rslt) |= (klgrpset_arg) 431 432 /* Macro to perform a 'diff' operation on a pair of lgroup bitmaps */ 433 #define klgrpset_diff(klgrpset_rslt, klgrpset_arg) \ 434 (klgrpset_rslt) &= ~(klgrpset_arg) 435 436 /* Macro to check if an lgroup is a member of an lgrpset */ 437 #define klgrpset_ismember(klgrpset, lgrpid) \ 438 ((klgrpset) & ((klgrpset_t)1 << (lgrpid))) 439 440 /* Macro to check if an lgroup bitmap is empty */ 441 #define klgrpset_isempty(klgrpset) \ 442 ((klgrpset) == (klgrpset_t)0) 443 444 /* Macro to check if two lgrpsets intersect */ 445 #define klgrpset_intersects(klgrpset1, klgrpset2) \ 446 ((klgrpset1) & (klgrpset2)) 447 448 /* Macro to count the number of members in an lgrpset */ 449 #define klgrpset_nlgrps(klgrpset, count) \ 450 { \ 451 lgrp_id_t lgrpid; \ 452 for (lgrpid = 0, count = 0; lgrpid <= lgrp_alloc_max; lgrpid++) {\ 453 if (klgrpset_ismember(klgrpset, lgrpid)) \ 454 count++; \ 455 } \ 456 } 457 458 /* Macro to get total memory size (in bytes) of a given set of lgroups */ 459 #define klgrpset_totalsize(klgrpset, size) \ 460 { \ 461 lgrp_handle_t hand; \ 462 lgrp_id_t lgrpid; \ 463 \ 464 for (lgrpid = 0, size = 0; lgrpid <= lgrp_alloc_max; lgrpid++) {\ 465 if (klgrpset_ismember(klgrpset, lgrpid) && \ 466 lgrp_table[lgrpid]) { \ 467 hand = lgrp_table[lgrpid]->lgrp_plathand; \ 468 size += lgrp_plat_mem_size(hand, \ 469 LGRP_MEM_SIZE_AVAIL) * PAGESIZE; \ 470 } \ 471 } \ 472 } 473 474 /* 475 * Does this lgroup exist? 476 */ 477 #define LGRP_EXISTS(lgrp) \ 478 (lgrp != NULL && lgrp->lgrp_id != LGRP_NONE) 479 480 /* 481 * Macro for testing if a CPU is contained in an lgrp. 482 */ 483 #define LGRP_CONTAINS_CPU(lgrp, cpu) \ 484 (klgrpset_ismember(lgrp->lgrp_set[LGRP_RSRC_CPU], \ 485 cpu->cpu_lpl->lpl_lgrpid)) 486 487 /* 488 * Initialize an lgrp_mnode_cookie 489 */ 490 #define LGRP_MNODE_COOKIE_INIT(c, lgrp, scope) \ 491 { \ 492 bzero(&(c), sizeof (lgrp_mnode_cookie_t)); \ 493 (&(c))->lmc_lgrp = lgrp; \ 494 (&(c))->lmc_nodes = lgrp->lgrp_mnodes; \ 495 (&(c))->lmc_cnt = lgrp->lgrp_nmnodes; \ 496 (&(c))->lmc_scope = scope; \ 497 (&(c))->lmc_rand = (ushort_t)gethrtime_unscaled() >> 4; \ 498 } 499 500 /* 501 * Upgrade cookie scope from LGRP_SRCH_LOCAL to LGRP_SRCH_HIER. 502 */ 503 #define LGRP_MNODE_COOKIE_UPGRADE(c) \ 504 { \ 505 ASSERT((&(c))->lmc_scope == LGRP_SRCH_LOCAL); \ 506 (&(c))->lmc_scope = LGRP_SRCH_HIER; \ 507 } 508 509 /* 510 * Macro to see whether memory allocation policy can be reapplied 511 */ 512 #define LGRP_MEM_POLICY_REAPPLICABLE(p) \ 513 (p == LGRP_MEM_POLICY_NEXT) 514 515 /* 516 * Return true if lgrp has CPU resources in the cpupart 517 */ 518 #define LGRP_CPUS_IN_PART(lgrpid, cpupart) \ 519 (cpupart->cp_lgrploads[lgrpid].lpl_ncpu > 0) 520 521 extern int lgrp_alloc_max; 522 extern lgrp_t *lgrp_table[NLGRPS_MAX]; /* indexed by lgrp_id */ 523 extern int nlgrps; /* number of lgroups in machine */ 524 extern int nlgrpsmax; /* max number of lgroups on platform */ 525 extern lgrp_gen_t lgrp_gen; /* generation of lgroup hierarchy */ 526 extern int lgrp_initialized; /* single-CPU initialization done */ 527 extern int lgrp_topo_initialized; /* lgrp topology constructed */ 528 extern lgrp_t *lgrp_root; /* root lgroup */ 529 extern unsigned int lgrp_topo_levels; 530 extern lpl_t *lpl_bootstrap; /* bootstrap lpl for non-active CPUs */ 531 532 533 /* generic interfaces */ 534 535 /* 536 * lgroup management 537 */ 538 int lgrp_optimizations(void); 539 void lgrp_init(lgrp_init_stages_t); 540 lgrp_t *lgrp_create(void); 541 void lgrp_destroy(lgrp_t *); 542 void lgrp_config(lgrp_config_flag_t, uintptr_t, uintptr_t); 543 lgrp_t *lgrp_hand_to_lgrp(lgrp_handle_t); 544 545 /* 546 * lgroup stats 547 */ 548 void lgrp_kstat_create(struct cpu *); 549 void lgrp_kstat_destroy(struct cpu *); 550 void lgrp_stat_add(lgrp_id_t, lgrp_stat_t, int64_t); 551 int64_t lgrp_stat_read(lgrp_id_t, lgrp_stat_t); 552 553 /* 554 * lgroup memory 555 */ 556 lgrp_mem_policy_t lgrp_madv_to_policy(uchar_t, size_t, int); 557 pgcnt_t lgrp_mem_size(lgrp_id_t, lgrp_mem_query_t); 558 lgrp_t *lgrp_mem_choose(struct seg *, caddr_t, size_t); 559 int lgrp_memnode_choose(lgrp_mnode_cookie_t *); 560 lgrp_mem_policy_t lgrp_mem_policy_default(size_t, int); 561 int lgrp_mnode_update(klgrpset_t, klgrpset_t *); 562 lgrp_t *lgrp_pfn_to_lgrp(pfn_t); 563 lgrp_t *lgrp_phys_to_lgrp(u_longlong_t); /* used by numat driver */ 564 int lgrp_privm_policy_set(lgrp_mem_policy_t, lgrp_mem_policy_info_t *, 565 size_t); 566 void lgrp_shm_policy_init(struct anon_map *, vnode_t *); 567 void lgrp_shm_policy_fini(struct anon_map *, vnode_t *); 568 lgrp_mem_policy_info_t *lgrp_shm_policy_get(struct anon_map *, ulong_t, 569 vnode_t *, u_offset_t); 570 int lgrp_shm_policy_set(lgrp_mem_policy_t, struct anon_map *, ulong_t, 571 vnode_t *, u_offset_t, size_t); 572 573 /* 574 * Used by numat driver 575 */ 576 int lgrp_query_cpu(processorid_t, lgrp_id_t *); 577 int lgrp_query_load(processorid_t, lgrp_load_t *); 578 579 /* 580 * lgroup thread placement 581 */ 582 lpl_t *lgrp_affinity_best(kthread_t *, struct cpupart *, lgrp_id_t, 583 boolean_t); 584 void lgrp_affinity_init(lgrp_affinity_t **); 585 void lgrp_affinity_free(lgrp_affinity_t **); 586 lpl_t *lgrp_choose(kthread_t *t, struct cpupart *); 587 lgrp_t *lgrp_home_lgrp(void); 588 lgrp_id_t lgrp_home_id(kthread_t *); 589 void lgrp_loadavg(lpl_t *, uint_t, int); 590 void lgrp_move_thread(kthread_t *, lpl_t *, int); 591 uint64_t lgrp_get_trthr_migrations(void); 592 void lgrp_update_trthr_migrations(uint64_t); 593 594 /* 595 * lgroup topology 596 */ 597 int lgrp_leaf_add(lgrp_t *, lgrp_t **, int, klgrpset_t *); 598 int lgrp_leaf_delete(lgrp_t *, lgrp_t **, int, klgrpset_t *); 599 int lgrp_rsets_empty(klgrpset_t *); 600 int lgrp_rsets_member(klgrpset_t *, lgrp_id_t); 601 int lgrp_topo_flatten(int, lgrp_t **, int, klgrpset_t *); 602 int lgrp_topo_ht_limit(void); 603 int lgrp_topo_ht_limit_default(void); 604 int lgrp_topo_ht_limit_set(int); 605 int lgrp_topo_update(lgrp_t **, int, klgrpset_t *); 606 607 /* 608 * lpl topology 609 */ 610 void lpl_topo_bootstrap(lpl_t *, int); 611 int lpl_topo_flatten(int); 612 int lpl_topo_verify(struct cpupart *); 613 614 615 /* platform interfaces */ 616 void lgrp_plat_init(lgrp_init_stages_t); 617 lgrp_t *lgrp_plat_alloc(lgrp_id_t lgrpid); 618 void lgrp_plat_config(lgrp_config_flag_t, uintptr_t); 619 lgrp_handle_t lgrp_plat_cpu_to_hand(processorid_t); 620 lgrp_handle_t lgrp_plat_pfn_to_hand(pfn_t); 621 int lgrp_plat_max_lgrps(void); 622 pgcnt_t lgrp_plat_mem_size(lgrp_handle_t, lgrp_mem_query_t); 623 int lgrp_plat_latency(lgrp_handle_t, lgrp_handle_t); 624 lgrp_handle_t lgrp_plat_root_hand(void); 625 626 extern uint32_t lgrp_expand_proc_thresh; 627 extern uint32_t lgrp_expand_proc_diff; 628 extern pgcnt_t lgrp_mem_free_thresh; 629 extern uint32_t lgrp_loadavg_tolerance; 630 extern uint32_t lgrp_loadavg_max_effect; 631 extern uint32_t lgrp_load_thresh; 632 extern lgrp_mem_policy_t lgrp_mem_policy_root; 633 634 #endif /* _KERNEL && _KMEMUSER */ 635 636 #ifdef __cplusplus 637 } 638 #endif 639 640 #endif /* _LGRP_H */ 641