17c478bd9Sstevel@tonic-gate /* 27c478bd9Sstevel@tonic-gate * CDDL HEADER START 37c478bd9Sstevel@tonic-gate * 47c478bd9Sstevel@tonic-gate * The contents of this file are subject to the terms of the 57c478bd9Sstevel@tonic-gate * Common Development and Distribution License, Version 1.0 only 67c478bd9Sstevel@tonic-gate * (the "License"). You may not use this file except in compliance 77c478bd9Sstevel@tonic-gate * with the License. 87c478bd9Sstevel@tonic-gate * 97c478bd9Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 107c478bd9Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing. 117c478bd9Sstevel@tonic-gate * See the License for the specific language governing permissions 127c478bd9Sstevel@tonic-gate * and limitations under the License. 137c478bd9Sstevel@tonic-gate * 147c478bd9Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each 157c478bd9Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 167c478bd9Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the 177c478bd9Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying 187c478bd9Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner] 197c478bd9Sstevel@tonic-gate * 207c478bd9Sstevel@tonic-gate * CDDL HEADER END 217c478bd9Sstevel@tonic-gate */ 227c478bd9Sstevel@tonic-gate /* 237c478bd9Sstevel@tonic-gate * Copyright 2005 Sun Microsystems, Inc. All rights reserved. 247c478bd9Sstevel@tonic-gate * Use is subject to license terms. 257c478bd9Sstevel@tonic-gate */ 267c478bd9Sstevel@tonic-gate 277c478bd9Sstevel@tonic-gate #pragma ident "%Z%%M% %I% %E% SMI" 287c478bd9Sstevel@tonic-gate 297c478bd9Sstevel@tonic-gate /* 307c478bd9Sstevel@tonic-gate * Basic NUMA support in terms of locality groups 317c478bd9Sstevel@tonic-gate * 327c478bd9Sstevel@tonic-gate * Solaris needs to know which CPUs, memory, etc. are near each other to 337c478bd9Sstevel@tonic-gate * provide good performance on NUMA machines by optimizing for locality. 347c478bd9Sstevel@tonic-gate * In order to do this, a new abstraction called a "locality group (lgroup)" 357c478bd9Sstevel@tonic-gate * has been introduced to keep track of which CPU-like and memory-like hardware 367c478bd9Sstevel@tonic-gate * resources are close to each other. Currently, latency is the only measure 377c478bd9Sstevel@tonic-gate * used to determine how to group hardware resources into lgroups, but this 387c478bd9Sstevel@tonic-gate * does not limit the groupings to be based solely on latency. Other factors 397c478bd9Sstevel@tonic-gate * may be used to determine the groupings in the future. 407c478bd9Sstevel@tonic-gate * 417c478bd9Sstevel@tonic-gate * Lgroups are organized into a hieararchy or topology that represents the 427c478bd9Sstevel@tonic-gate * latency topology of the machine. There is always at least a root lgroup in 437c478bd9Sstevel@tonic-gate * the system. It represents all the hardware resources in the machine at a 447c478bd9Sstevel@tonic-gate * latency big enough that any hardware resource can at least access any other 457c478bd9Sstevel@tonic-gate * hardware resource within that latency. A Uniform Memory Access (UMA) 467c478bd9Sstevel@tonic-gate * machine is represented with one lgroup (the root). In contrast, a NUMA 477c478bd9Sstevel@tonic-gate * machine is represented at least by the root lgroup and some number of leaf 487c478bd9Sstevel@tonic-gate * lgroups where the leaf lgroups contain the hardware resources within the 497c478bd9Sstevel@tonic-gate * least latency of each other and the root lgroup still contains all the 507c478bd9Sstevel@tonic-gate * resources in the machine. Some number of intermediate lgroups may exist 517c478bd9Sstevel@tonic-gate * which represent more levels of locality than just the local latency of the 527c478bd9Sstevel@tonic-gate * leaf lgroups and the system latency of the root lgroup. Non-leaf lgroups 537c478bd9Sstevel@tonic-gate * (eg. root and intermediate lgroups) contain the next nearest resources to 547c478bd9Sstevel@tonic-gate * its children lgroups. Thus, the lgroup hierarchy from a given leaf lgroup 557c478bd9Sstevel@tonic-gate * to the root lgroup shows the hardware resources from closest to farthest 567c478bd9Sstevel@tonic-gate * from the leaf lgroup such that each successive ancestor lgroup contains 577c478bd9Sstevel@tonic-gate * the next nearest resources at the next level of locality from the previous. 587c478bd9Sstevel@tonic-gate * 597c478bd9Sstevel@tonic-gate * The kernel uses the lgroup abstraction to know how to allocate resources 607c478bd9Sstevel@tonic-gate * near a given process/thread. At fork() and lwp/thread_create() time, a 617c478bd9Sstevel@tonic-gate * "home" lgroup is chosen for a thread. This is done by picking the lgroup 627c478bd9Sstevel@tonic-gate * with the lowest load average. Binding to a processor or processor set will 637c478bd9Sstevel@tonic-gate * change the home lgroup for a thread. The scheduler has been modified to try 647c478bd9Sstevel@tonic-gate * to dispatch a thread on a CPU in its home lgroup. Physical memory 657c478bd9Sstevel@tonic-gate * allocation is lgroup aware too, so memory will be allocated from the current 667c478bd9Sstevel@tonic-gate * thread's home lgroup if possible. If the desired resources are not 677c478bd9Sstevel@tonic-gate * available, the kernel traverses the lgroup hierarchy going to the parent 687c478bd9Sstevel@tonic-gate * lgroup to find resources at the next level of locality until it reaches the 697c478bd9Sstevel@tonic-gate * root lgroup. 707c478bd9Sstevel@tonic-gate */ 717c478bd9Sstevel@tonic-gate 727c478bd9Sstevel@tonic-gate #include <sys/lgrp.h> 737c478bd9Sstevel@tonic-gate #include <sys/lgrp_user.h> 747c478bd9Sstevel@tonic-gate #include <sys/types.h> 757c478bd9Sstevel@tonic-gate #include <sys/mman.h> 767c478bd9Sstevel@tonic-gate #include <sys/param.h> 777c478bd9Sstevel@tonic-gate #include <sys/var.h> 787c478bd9Sstevel@tonic-gate #include <sys/thread.h> 797c478bd9Sstevel@tonic-gate #include <sys/cpuvar.h> 807c478bd9Sstevel@tonic-gate #include <sys/cpupart.h> 817c478bd9Sstevel@tonic-gate #include <sys/kmem.h> 827c478bd9Sstevel@tonic-gate #include <vm/seg.h> 837c478bd9Sstevel@tonic-gate #include <vm/seg_kmem.h> 847c478bd9Sstevel@tonic-gate #include <vm/seg_spt.h> 857c478bd9Sstevel@tonic-gate #include <vm/seg_vn.h> 867c478bd9Sstevel@tonic-gate #include <vm/as.h> 877c478bd9Sstevel@tonic-gate #include <sys/atomic.h> 887c478bd9Sstevel@tonic-gate #include <sys/systm.h> 897c478bd9Sstevel@tonic-gate #include <sys/errno.h> 907c478bd9Sstevel@tonic-gate #include <sys/cmn_err.h> 917c478bd9Sstevel@tonic-gate #include <sys/kstat.h> 927c478bd9Sstevel@tonic-gate #include <sys/sysmacros.h> 937c478bd9Sstevel@tonic-gate #include <sys/chip.h> 947c478bd9Sstevel@tonic-gate #include <sys/promif.h> 957c478bd9Sstevel@tonic-gate #include <sys/sdt.h> 967c478bd9Sstevel@tonic-gate 977c478bd9Sstevel@tonic-gate lgrp_gen_t lgrp_gen = 0; /* generation of lgroup hierarchy */ 987c478bd9Sstevel@tonic-gate lgrp_t *lgrp_table[NLGRPS_MAX]; /* table of all initialized lgrp_t structs */ 997c478bd9Sstevel@tonic-gate /* indexed by lgrp_id */ 1007c478bd9Sstevel@tonic-gate int nlgrps; /* number of lgroups in machine */ 1017c478bd9Sstevel@tonic-gate int lgrp_alloc_hint = -1; /* hint for where to try to allocate next */ 1027c478bd9Sstevel@tonic-gate int lgrp_alloc_max = 0; /* max lgroup ID allocated so far */ 1037c478bd9Sstevel@tonic-gate 1047c478bd9Sstevel@tonic-gate /* 1057c478bd9Sstevel@tonic-gate * Kstat data for lgroups. 1067c478bd9Sstevel@tonic-gate * 1077c478bd9Sstevel@tonic-gate * Actual kstat data is collected in lgrp_stats array. 1087c478bd9Sstevel@tonic-gate * The lgrp_kstat_data array of named kstats is used to extract data from 1097c478bd9Sstevel@tonic-gate * lgrp_stats and present it to kstat framework. It is protected from partallel 1107c478bd9Sstevel@tonic-gate * modifications by lgrp_kstat_mutex. This may cause some contention when 1117c478bd9Sstevel@tonic-gate * several kstat commands run in parallel but this is not the 1127c478bd9Sstevel@tonic-gate * performance-critical path. 1137c478bd9Sstevel@tonic-gate */ 1147c478bd9Sstevel@tonic-gate extern struct lgrp_stats lgrp_stats[]; /* table of per-lgrp stats */ 1157c478bd9Sstevel@tonic-gate 1167c478bd9Sstevel@tonic-gate /* 1177c478bd9Sstevel@tonic-gate * Declare kstat names statically for enums as defined in the header file. 1187c478bd9Sstevel@tonic-gate */ 1197c478bd9Sstevel@tonic-gate LGRP_KSTAT_NAMES; 1207c478bd9Sstevel@tonic-gate 1217c478bd9Sstevel@tonic-gate static void lgrp_kstat_init(void); 1227c478bd9Sstevel@tonic-gate static int lgrp_kstat_extract(kstat_t *, int); 1237c478bd9Sstevel@tonic-gate static void lgrp_kstat_reset(lgrp_id_t); 1247c478bd9Sstevel@tonic-gate 1257c478bd9Sstevel@tonic-gate static struct kstat_named lgrp_kstat_data[LGRP_NUM_STATS]; 1267c478bd9Sstevel@tonic-gate static kmutex_t lgrp_kstat_mutex; 1277c478bd9Sstevel@tonic-gate 1287c478bd9Sstevel@tonic-gate 1297c478bd9Sstevel@tonic-gate /* 1307c478bd9Sstevel@tonic-gate * max number of lgroups supported by the platform 1317c478bd9Sstevel@tonic-gate */ 1327c478bd9Sstevel@tonic-gate int nlgrpsmax = 0; 1337c478bd9Sstevel@tonic-gate 1347c478bd9Sstevel@tonic-gate /* 1357c478bd9Sstevel@tonic-gate * The root lgroup. Represents the set of resources at the system wide 1367c478bd9Sstevel@tonic-gate * level of locality. 1377c478bd9Sstevel@tonic-gate */ 1387c478bd9Sstevel@tonic-gate lgrp_t *lgrp_root = NULL; 1397c478bd9Sstevel@tonic-gate 1407c478bd9Sstevel@tonic-gate /* 1417c478bd9Sstevel@tonic-gate * During system bootstrap cp_default does not contain the list of lgrp load 1427c478bd9Sstevel@tonic-gate * averages (cp_lgrploads). The list is allocated after the first CPU is brought 1437c478bd9Sstevel@tonic-gate * on-line when cp_default is initialized by cpupart_initialize_default(). 1447c478bd9Sstevel@tonic-gate * Configuring CPU0 may create a two-level topology with root and one leaf node 1457c478bd9Sstevel@tonic-gate * containing CPU0. This topology is initially constructed in a special 1467c478bd9Sstevel@tonic-gate * statically allocated 2-element lpl list lpl_bootstrap_list and later cloned 1477c478bd9Sstevel@tonic-gate * to cp_default when cp_default is initialized. The lpl_bootstrap_list is used 1487c478bd9Sstevel@tonic-gate * for all lpl operations until cp_default is fully constructed. 1497c478bd9Sstevel@tonic-gate * 1507c478bd9Sstevel@tonic-gate * The lpl_bootstrap_list is maintained by the code in lgrp.c. Every other 1517c478bd9Sstevel@tonic-gate * consumer who needs default lpl should use lpl_bootstrap which is a pointer to 1527c478bd9Sstevel@tonic-gate * the first element of lpl_bootstrap_list. 1537c478bd9Sstevel@tonic-gate */ 1547c478bd9Sstevel@tonic-gate #define LPL_BOOTSTRAP_SIZE 2 1557c478bd9Sstevel@tonic-gate static lpl_t lpl_bootstrap_list[LPL_BOOTSTRAP_SIZE]; 1567c478bd9Sstevel@tonic-gate lpl_t *lpl_bootstrap; 1577c478bd9Sstevel@tonic-gate 1587c478bd9Sstevel@tonic-gate static lgrp_t lroot; 1597c478bd9Sstevel@tonic-gate 1607c478bd9Sstevel@tonic-gate 1617c478bd9Sstevel@tonic-gate /* 1627c478bd9Sstevel@tonic-gate * Size, in bytes, beyond which random memory allocation policy is applied 1637c478bd9Sstevel@tonic-gate * to non-shared memory. Default is the maximum size, so random memory 1647c478bd9Sstevel@tonic-gate * allocation won't be used for non-shared memory by default. 1657c478bd9Sstevel@tonic-gate */ 1667c478bd9Sstevel@tonic-gate size_t lgrp_privm_random_thresh = (size_t)(-1); 1677c478bd9Sstevel@tonic-gate 1687c478bd9Sstevel@tonic-gate /* 1697c478bd9Sstevel@tonic-gate * Size, in bytes, beyond which random memory allocation policy is applied to 1707c478bd9Sstevel@tonic-gate * shared memory. Default is 8MB (2 ISM pages). 1717c478bd9Sstevel@tonic-gate */ 1727c478bd9Sstevel@tonic-gate size_t lgrp_shm_random_thresh = 8*1024*1024; 1737c478bd9Sstevel@tonic-gate 1747c478bd9Sstevel@tonic-gate /* 1757c478bd9Sstevel@tonic-gate * Whether to do processor set aware memory allocation by default 1767c478bd9Sstevel@tonic-gate */ 1777c478bd9Sstevel@tonic-gate int lgrp_mem_pset_aware = 0; 1787c478bd9Sstevel@tonic-gate 1797c478bd9Sstevel@tonic-gate /* 1807c478bd9Sstevel@tonic-gate * Set the default memory allocation policy for root lgroup 1817c478bd9Sstevel@tonic-gate */ 1827c478bd9Sstevel@tonic-gate lgrp_mem_policy_t lgrp_mem_policy_root = LGRP_MEM_POLICY_RANDOM; 1837c478bd9Sstevel@tonic-gate 1847c478bd9Sstevel@tonic-gate /* 1857c478bd9Sstevel@tonic-gate * Set the default memory allocation policy. For most platforms, 1867c478bd9Sstevel@tonic-gate * next touch is sufficient, but some platforms may wish to override 1877c478bd9Sstevel@tonic-gate * this. 1887c478bd9Sstevel@tonic-gate */ 1897c478bd9Sstevel@tonic-gate lgrp_mem_policy_t lgrp_mem_default_policy = LGRP_MEM_POLICY_NEXT; 1907c478bd9Sstevel@tonic-gate 1917c478bd9Sstevel@tonic-gate 1927c478bd9Sstevel@tonic-gate /* 1937c478bd9Sstevel@tonic-gate * lgroup CPU event handlers 1947c478bd9Sstevel@tonic-gate */ 1957c478bd9Sstevel@tonic-gate static void lgrp_cpu_init(struct cpu *); 1967c478bd9Sstevel@tonic-gate static void lgrp_cpu_fini(struct cpu *, lgrp_id_t); 1977c478bd9Sstevel@tonic-gate static lgrp_t *lgrp_cpu_to_lgrp(struct cpu *); 1987c478bd9Sstevel@tonic-gate 1997c478bd9Sstevel@tonic-gate static void lgrp_latency_change(u_longlong_t, u_longlong_t); 2007c478bd9Sstevel@tonic-gate 2017c478bd9Sstevel@tonic-gate /* 2027c478bd9Sstevel@tonic-gate * lgroup memory event handlers 2037c478bd9Sstevel@tonic-gate */ 2047c478bd9Sstevel@tonic-gate static void lgrp_mem_init(int, lgrp_handle_t, boolean_t); 2057c478bd9Sstevel@tonic-gate static void lgrp_mem_fini(int, lgrp_handle_t, boolean_t); 2067c478bd9Sstevel@tonic-gate static void lgrp_mem_rename(int, lgrp_handle_t, lgrp_handle_t); 2077c478bd9Sstevel@tonic-gate 2087c478bd9Sstevel@tonic-gate /* 2097c478bd9Sstevel@tonic-gate * lgroup CPU partition event handlers 2107c478bd9Sstevel@tonic-gate */ 2117c478bd9Sstevel@tonic-gate static void lgrp_part_add_cpu(struct cpu *, lgrp_id_t); 2127c478bd9Sstevel@tonic-gate static void lgrp_part_del_cpu(struct cpu *); 2137c478bd9Sstevel@tonic-gate 2147c478bd9Sstevel@tonic-gate static void lgrp_root_init(void); 2157c478bd9Sstevel@tonic-gate 2167c478bd9Sstevel@tonic-gate /* 2177c478bd9Sstevel@tonic-gate * lpl topology 2187c478bd9Sstevel@tonic-gate */ 2197c478bd9Sstevel@tonic-gate static void lpl_init(lpl_t *, lpl_t *, lgrp_t *); 2207c478bd9Sstevel@tonic-gate static void lpl_clear(lpl_t *); 2217c478bd9Sstevel@tonic-gate static void lpl_leaf_insert(lpl_t *, struct cpupart *); 2227c478bd9Sstevel@tonic-gate static void lpl_leaf_remove(lpl_t *, struct cpupart *); 2237c478bd9Sstevel@tonic-gate static void lpl_rset_add(lpl_t *, lpl_t *); 2247c478bd9Sstevel@tonic-gate static void lpl_rset_del(lpl_t *, lpl_t *); 2257c478bd9Sstevel@tonic-gate static int lpl_rset_contains(lpl_t *, lpl_t *); 2267c478bd9Sstevel@tonic-gate static void lpl_cpu_adjcnt(lpl_act_t, struct cpu *); 2277c478bd9Sstevel@tonic-gate static void lpl_child_update(lpl_t *, struct cpupart *); 2287c478bd9Sstevel@tonic-gate static int lpl_pick(lpl_t *, lpl_t *); 2297c478bd9Sstevel@tonic-gate static void lpl_verify_wrapper(struct cpupart *); 2307c478bd9Sstevel@tonic-gate 2317c478bd9Sstevel@tonic-gate /* 2327c478bd9Sstevel@tonic-gate * defines for lpl topology verifier return codes 2337c478bd9Sstevel@tonic-gate */ 2347c478bd9Sstevel@tonic-gate 2357c478bd9Sstevel@tonic-gate #define LPL_TOPO_CORRECT 0 2367c478bd9Sstevel@tonic-gate #define LPL_TOPO_PART_HAS_NO_LPL -1 2377c478bd9Sstevel@tonic-gate #define LPL_TOPO_CPUS_NOT_EMPTY -2 2387c478bd9Sstevel@tonic-gate #define LPL_TOPO_LGRP_MISMATCH -3 2397c478bd9Sstevel@tonic-gate #define LPL_TOPO_MISSING_PARENT -4 2407c478bd9Sstevel@tonic-gate #define LPL_TOPO_PARENT_MISMATCH -5 2417c478bd9Sstevel@tonic-gate #define LPL_TOPO_BAD_CPUCNT -6 2427c478bd9Sstevel@tonic-gate #define LPL_TOPO_RSET_MISMATCH -7 2437c478bd9Sstevel@tonic-gate #define LPL_TOPO_LPL_ORPHANED -8 2447c478bd9Sstevel@tonic-gate #define LPL_TOPO_LPL_BAD_NCPU -9 2457c478bd9Sstevel@tonic-gate #define LPL_TOPO_RSET_MSSNG_LF -10 2467c478bd9Sstevel@tonic-gate #define LPL_TOPO_CPU_HAS_BAD_LPL -11 2477c478bd9Sstevel@tonic-gate #define LPL_TOPO_BOGUS_HINT -12 2487c478bd9Sstevel@tonic-gate #define LPL_TOPO_NONLEAF_HAS_CPUS -13 2497c478bd9Sstevel@tonic-gate #define LPL_TOPO_LGRP_NOT_LEAF -14 2507c478bd9Sstevel@tonic-gate #define LPL_TOPO_BAD_RSETCNT -15 2517c478bd9Sstevel@tonic-gate 2527c478bd9Sstevel@tonic-gate /* 2537c478bd9Sstevel@tonic-gate * Return whether lgroup optimizations should be enabled on this system 2547c478bd9Sstevel@tonic-gate */ 2557c478bd9Sstevel@tonic-gate int 2567c478bd9Sstevel@tonic-gate lgrp_optimizations(void) 2577c478bd9Sstevel@tonic-gate { 2587c478bd9Sstevel@tonic-gate /* 2597c478bd9Sstevel@tonic-gate * System must have more than 2 lgroups to enable lgroup optimizations 2607c478bd9Sstevel@tonic-gate * 2617c478bd9Sstevel@tonic-gate * XXX This assumes that a 2 lgroup system has an empty root lgroup 2627c478bd9Sstevel@tonic-gate * with one child lgroup containing all the resources. A 2 lgroup 2637c478bd9Sstevel@tonic-gate * system with a root lgroup directly containing CPUs or memory might 2647c478bd9Sstevel@tonic-gate * need lgroup optimizations with its child lgroup, but there 2657c478bd9Sstevel@tonic-gate * isn't such a machine for now.... 2667c478bd9Sstevel@tonic-gate */ 2677c478bd9Sstevel@tonic-gate if (nlgrps > 2) 2687c478bd9Sstevel@tonic-gate return (1); 2697c478bd9Sstevel@tonic-gate 2707c478bd9Sstevel@tonic-gate return (0); 2717c478bd9Sstevel@tonic-gate } 2727c478bd9Sstevel@tonic-gate 2737c478bd9Sstevel@tonic-gate /* 2747c478bd9Sstevel@tonic-gate * Build full lgroup topology 2757c478bd9Sstevel@tonic-gate */ 2767c478bd9Sstevel@tonic-gate static void 2777c478bd9Sstevel@tonic-gate lgrp_root_init(void) 2787c478bd9Sstevel@tonic-gate { 2797c478bd9Sstevel@tonic-gate lgrp_handle_t hand; 2807c478bd9Sstevel@tonic-gate int i; 2817c478bd9Sstevel@tonic-gate lgrp_id_t id; 2827c478bd9Sstevel@tonic-gate 2837c478bd9Sstevel@tonic-gate /* 2847c478bd9Sstevel@tonic-gate * Create the "root" lgroup 2857c478bd9Sstevel@tonic-gate */ 2867c478bd9Sstevel@tonic-gate ASSERT(nlgrps == 0); 2877c478bd9Sstevel@tonic-gate id = nlgrps++; 2887c478bd9Sstevel@tonic-gate 2897c478bd9Sstevel@tonic-gate lgrp_root = &lroot; 2907c478bd9Sstevel@tonic-gate 2917c478bd9Sstevel@tonic-gate lgrp_root->lgrp_cpu = NULL; 2927c478bd9Sstevel@tonic-gate lgrp_root->lgrp_mnodes = 0; 2937c478bd9Sstevel@tonic-gate lgrp_root->lgrp_nmnodes = 0; 2947c478bd9Sstevel@tonic-gate hand = lgrp_plat_root_hand(); 2957c478bd9Sstevel@tonic-gate lgrp_root->lgrp_plathand = hand; 2967c478bd9Sstevel@tonic-gate 2977c478bd9Sstevel@tonic-gate lgrp_root->lgrp_id = id; 2987c478bd9Sstevel@tonic-gate lgrp_root->lgrp_cpucnt = 0; 2997c478bd9Sstevel@tonic-gate lgrp_root->lgrp_childcnt = 0; 3007c478bd9Sstevel@tonic-gate klgrpset_clear(lgrp_root->lgrp_children); 3017c478bd9Sstevel@tonic-gate klgrpset_clear(lgrp_root->lgrp_leaves); 3027c478bd9Sstevel@tonic-gate lgrp_root->lgrp_parent = NULL; 3037c478bd9Sstevel@tonic-gate lgrp_root->lgrp_chips = NULL; 3047c478bd9Sstevel@tonic-gate lgrp_root->lgrp_chipcnt = 0; 3057c478bd9Sstevel@tonic-gate lgrp_root->lgrp_latency = lgrp_plat_latency(hand, hand); 3067c478bd9Sstevel@tonic-gate 3077c478bd9Sstevel@tonic-gate for (i = 0; i < LGRP_RSRC_COUNT; i++) 3087c478bd9Sstevel@tonic-gate klgrpset_clear(lgrp_root->lgrp_set[i]); 3097c478bd9Sstevel@tonic-gate 3107c478bd9Sstevel@tonic-gate lgrp_root->lgrp_kstat = NULL; 3117c478bd9Sstevel@tonic-gate 3127c478bd9Sstevel@tonic-gate lgrp_table[id] = lgrp_root; 3137c478bd9Sstevel@tonic-gate 3147c478bd9Sstevel@tonic-gate /* 3157c478bd9Sstevel@tonic-gate * Setup initial lpl list for CPU0 and initial t0 home. 3167c478bd9Sstevel@tonic-gate * The only lpl space we have so far is lpl_bootstrap. It is used for 3177c478bd9Sstevel@tonic-gate * all topology operations untill cp_default until cp_default is 3187c478bd9Sstevel@tonic-gate * initialized at which point t0.t_lpl will be updated. 3197c478bd9Sstevel@tonic-gate */ 3207c478bd9Sstevel@tonic-gate lpl_bootstrap = lpl_bootstrap_list; 3217c478bd9Sstevel@tonic-gate t0.t_lpl = lpl_bootstrap; 3227c478bd9Sstevel@tonic-gate cp_default.cp_nlgrploads = LPL_BOOTSTRAP_SIZE; 3237c478bd9Sstevel@tonic-gate lpl_bootstrap_list[1].lpl_lgrpid = 1; 3247c478bd9Sstevel@tonic-gate cp_default.cp_lgrploads = lpl_bootstrap; 3257c478bd9Sstevel@tonic-gate } 3267c478bd9Sstevel@tonic-gate 3277c478bd9Sstevel@tonic-gate /* 3287c478bd9Sstevel@tonic-gate * Initialize the lgroup framework and allow the platform to do the same 3297c478bd9Sstevel@tonic-gate */ 3307c478bd9Sstevel@tonic-gate void 3317c478bd9Sstevel@tonic-gate lgrp_init(void) 3327c478bd9Sstevel@tonic-gate { 3337c478bd9Sstevel@tonic-gate /* 3347c478bd9Sstevel@tonic-gate * Initialize the platform 3357c478bd9Sstevel@tonic-gate */ 3367c478bd9Sstevel@tonic-gate lgrp_plat_init(); 3377c478bd9Sstevel@tonic-gate 3387c478bd9Sstevel@tonic-gate /* 3397c478bd9Sstevel@tonic-gate * Set max number of lgroups supported on this platform which must be 3407c478bd9Sstevel@tonic-gate * less than the max number of lgroups supported by the common lgroup 3417c478bd9Sstevel@tonic-gate * framework (eg. NLGRPS_MAX is max elements in lgrp_table[], etc.) 3427c478bd9Sstevel@tonic-gate */ 3437c478bd9Sstevel@tonic-gate nlgrpsmax = lgrp_plat_max_lgrps(); 3447c478bd9Sstevel@tonic-gate ASSERT(nlgrpsmax <= NLGRPS_MAX); 3457c478bd9Sstevel@tonic-gate } 3467c478bd9Sstevel@tonic-gate 3477c478bd9Sstevel@tonic-gate /* 3487c478bd9Sstevel@tonic-gate * Create the root and cpu0's lgroup, and set t0's home. 3497c478bd9Sstevel@tonic-gate */ 3507c478bd9Sstevel@tonic-gate void 3517c478bd9Sstevel@tonic-gate lgrp_setup(void) 3527c478bd9Sstevel@tonic-gate { 3537c478bd9Sstevel@tonic-gate /* 3547c478bd9Sstevel@tonic-gate * Setup the root lgroup 3557c478bd9Sstevel@tonic-gate */ 3567c478bd9Sstevel@tonic-gate lgrp_root_init(); 3577c478bd9Sstevel@tonic-gate 3587c478bd9Sstevel@tonic-gate /* 3597c478bd9Sstevel@tonic-gate * Add cpu0 to an lgroup 3607c478bd9Sstevel@tonic-gate */ 3617c478bd9Sstevel@tonic-gate lgrp_config(LGRP_CONFIG_CPU_ADD, (uintptr_t)CPU, 0); 3627c478bd9Sstevel@tonic-gate lgrp_config(LGRP_CONFIG_CPU_ONLINE, (uintptr_t)CPU, 0); 3637c478bd9Sstevel@tonic-gate } 3647c478bd9Sstevel@tonic-gate 3657c478bd9Sstevel@tonic-gate /* 3667c478bd9Sstevel@tonic-gate * Lgroup initialization is split in two parts. The first part 3677c478bd9Sstevel@tonic-gate * (lgrp_main_init()) is called right before start_other_cpus() in main. The 3687c478bd9Sstevel@tonic-gate * second part (lgrp_main_mp_init()) is called right after start_other_cpus() 3697c478bd9Sstevel@tonic-gate * when all CPUs are brought online and all distance information is available. 3707c478bd9Sstevel@tonic-gate * 3717c478bd9Sstevel@tonic-gate * When lgrp_main_init() is complete it sets lgrp_initialized. The 3727c478bd9Sstevel@tonic-gate * lgrp_main_mp_init() sets lgrp_topo_initialized. 3737c478bd9Sstevel@tonic-gate */ 3747c478bd9Sstevel@tonic-gate 3757c478bd9Sstevel@tonic-gate /* 3767c478bd9Sstevel@tonic-gate * true when lgrp initialization has been completed. 3777c478bd9Sstevel@tonic-gate */ 3787c478bd9Sstevel@tonic-gate int lgrp_initialized = 0; 3797c478bd9Sstevel@tonic-gate 3807c478bd9Sstevel@tonic-gate /* 3817c478bd9Sstevel@tonic-gate * True when lgrp topology is constructed. 3827c478bd9Sstevel@tonic-gate */ 3837c478bd9Sstevel@tonic-gate int lgrp_topo_initialized = 0; 3847c478bd9Sstevel@tonic-gate 3857c478bd9Sstevel@tonic-gate /* 3867c478bd9Sstevel@tonic-gate * Init routine called after startup(), /etc/system has been processed, 3877c478bd9Sstevel@tonic-gate * and cpu0 has been added to an lgroup. 3887c478bd9Sstevel@tonic-gate */ 3897c478bd9Sstevel@tonic-gate void 3907c478bd9Sstevel@tonic-gate lgrp_main_init(void) 3917c478bd9Sstevel@tonic-gate { 3927c478bd9Sstevel@tonic-gate cpu_t *cp = CPU; 3937c478bd9Sstevel@tonic-gate lgrp_id_t lgrpid; 3947c478bd9Sstevel@tonic-gate int i; 3957c478bd9Sstevel@tonic-gate /* 3967c478bd9Sstevel@tonic-gate * Enforce a valid lgrp_mem_default_policy 3977c478bd9Sstevel@tonic-gate */ 3987c478bd9Sstevel@tonic-gate if ((lgrp_mem_default_policy <= LGRP_MEM_POLICY_DEFAULT) || 3997c478bd9Sstevel@tonic-gate (lgrp_mem_default_policy >= LGRP_NUM_MEM_POLICIES)) 4007c478bd9Sstevel@tonic-gate lgrp_mem_default_policy = LGRP_MEM_POLICY_NEXT; 4017c478bd9Sstevel@tonic-gate 4027c478bd9Sstevel@tonic-gate /* 4037c478bd9Sstevel@tonic-gate * See if mpo should be disabled. 4047c478bd9Sstevel@tonic-gate * This may happen in the case of null proc LPA on Starcat. 4057c478bd9Sstevel@tonic-gate * The platform won't be able to detect null proc LPA until after 4067c478bd9Sstevel@tonic-gate * cpu0 and memory have already been added to lgroups. 4077c478bd9Sstevel@tonic-gate * When and if it is detected, the Starcat platform will return 4087c478bd9Sstevel@tonic-gate * a different platform handle for cpu0 which is what we check for 4097c478bd9Sstevel@tonic-gate * here. If mpo should be disabled move cpu0 to it's rightful place 4107c478bd9Sstevel@tonic-gate * (the root), and destroy the remaining lgroups. This effectively 4117c478bd9Sstevel@tonic-gate * provides an UMA lgroup topology. 4127c478bd9Sstevel@tonic-gate */ 4137c478bd9Sstevel@tonic-gate lgrpid = cp->cpu_lpl->lpl_lgrpid; 4147c478bd9Sstevel@tonic-gate if (lgrp_table[lgrpid]->lgrp_plathand != 4157c478bd9Sstevel@tonic-gate lgrp_plat_cpu_to_hand(cp->cpu_id)) { 4167c478bd9Sstevel@tonic-gate lgrp_part_del_cpu(cp); 4177c478bd9Sstevel@tonic-gate lgrp_cpu_fini(cp, lgrpid); 4187c478bd9Sstevel@tonic-gate 4197c478bd9Sstevel@tonic-gate lgrp_cpu_init(cp); 4207c478bd9Sstevel@tonic-gate lgrp_part_add_cpu(cp, cp->cpu_lpl->lpl_lgrpid); 4217c478bd9Sstevel@tonic-gate 4227c478bd9Sstevel@tonic-gate ASSERT(cp->cpu_lpl->lpl_lgrpid == LGRP_ROOTID); 4237c478bd9Sstevel@tonic-gate 4247c478bd9Sstevel@tonic-gate for (i = 0; i <= lgrp_alloc_max; i++) { 4257c478bd9Sstevel@tonic-gate if (LGRP_EXISTS(lgrp_table[i]) && 4267c478bd9Sstevel@tonic-gate lgrp_table[i] != lgrp_root) 4277c478bd9Sstevel@tonic-gate lgrp_destroy(lgrp_table[i]); 4287c478bd9Sstevel@tonic-gate } 4297c478bd9Sstevel@tonic-gate klgrpset_clear(lgrp_root->lgrp_set[LGRP_RSRC_MEM]); 4307c478bd9Sstevel@tonic-gate klgrpset_add(lgrp_root->lgrp_set[LGRP_RSRC_MEM], LGRP_ROOTID); 4317c478bd9Sstevel@tonic-gate } 4327c478bd9Sstevel@tonic-gate 4337c478bd9Sstevel@tonic-gate /* 4347c478bd9Sstevel@tonic-gate * Initialize kstats framework. 4357c478bd9Sstevel@tonic-gate */ 4367c478bd9Sstevel@tonic-gate lgrp_kstat_init(); 4377c478bd9Sstevel@tonic-gate /* 4387c478bd9Sstevel@tonic-gate * cpu0 is finally where it should be, so create it's lgroup's kstats 4397c478bd9Sstevel@tonic-gate */ 4407c478bd9Sstevel@tonic-gate mutex_enter(&cpu_lock); 4417c478bd9Sstevel@tonic-gate lgrp_kstat_create(cp); 4427c478bd9Sstevel@tonic-gate mutex_exit(&cpu_lock); 4437c478bd9Sstevel@tonic-gate 4447c478bd9Sstevel@tonic-gate lgrp_plat_main_init(); 4457c478bd9Sstevel@tonic-gate lgrp_initialized = 1; 4467c478bd9Sstevel@tonic-gate } 4477c478bd9Sstevel@tonic-gate 4487c478bd9Sstevel@tonic-gate /* 4497c478bd9Sstevel@tonic-gate * Finish lgrp initialization after all CPUS are brought on-line. 4507c478bd9Sstevel@tonic-gate * This routine is called after start_other_cpus(). 4517c478bd9Sstevel@tonic-gate */ 4527c478bd9Sstevel@tonic-gate void 4537c478bd9Sstevel@tonic-gate lgrp_main_mp_init(void) 4547c478bd9Sstevel@tonic-gate { 4557c478bd9Sstevel@tonic-gate klgrpset_t changed; 4567c478bd9Sstevel@tonic-gate 4577c478bd9Sstevel@tonic-gate /* 4587c478bd9Sstevel@tonic-gate * Update lgroup topology (if necessary) 4597c478bd9Sstevel@tonic-gate */ 4607c478bd9Sstevel@tonic-gate klgrpset_clear(changed); 4617c478bd9Sstevel@tonic-gate (void) lgrp_topo_update(lgrp_table, lgrp_alloc_max + 1, &changed); 4627c478bd9Sstevel@tonic-gate lgrp_topo_initialized = 1; 4637c478bd9Sstevel@tonic-gate } 4647c478bd9Sstevel@tonic-gate 4657c478bd9Sstevel@tonic-gate /* 4667c478bd9Sstevel@tonic-gate * Handle lgroup (re)configuration events (eg. addition of CPU, etc.) 4677c478bd9Sstevel@tonic-gate */ 4687c478bd9Sstevel@tonic-gate void 4697c478bd9Sstevel@tonic-gate lgrp_config(lgrp_config_flag_t event, uintptr_t resource, uintptr_t where) 4707c478bd9Sstevel@tonic-gate { 4717c478bd9Sstevel@tonic-gate klgrpset_t changed; 4727c478bd9Sstevel@tonic-gate cpu_t *cp; 4737c478bd9Sstevel@tonic-gate lgrp_id_t id; 4747c478bd9Sstevel@tonic-gate int rc; 4757c478bd9Sstevel@tonic-gate 4767c478bd9Sstevel@tonic-gate switch (event) { 4777c478bd9Sstevel@tonic-gate /* 4787c478bd9Sstevel@tonic-gate * The following (re)configuration events are common code 4797c478bd9Sstevel@tonic-gate * initiated. lgrp_plat_config() is called here to inform the 4807c478bd9Sstevel@tonic-gate * platform of the reconfiguration event. 4817c478bd9Sstevel@tonic-gate */ 4827c478bd9Sstevel@tonic-gate case LGRP_CONFIG_CPU_ADD: 4837c478bd9Sstevel@tonic-gate lgrp_plat_config(event, resource); 4847c478bd9Sstevel@tonic-gate atomic_add_32(&lgrp_gen, 1); 4857c478bd9Sstevel@tonic-gate 4867c478bd9Sstevel@tonic-gate break; 4877c478bd9Sstevel@tonic-gate case LGRP_CONFIG_CPU_DEL: 4887c478bd9Sstevel@tonic-gate lgrp_plat_config(event, resource); 4897c478bd9Sstevel@tonic-gate atomic_add_32(&lgrp_gen, 1); 4907c478bd9Sstevel@tonic-gate 4917c478bd9Sstevel@tonic-gate break; 4927c478bd9Sstevel@tonic-gate case LGRP_CONFIG_CPU_ONLINE: 4937c478bd9Sstevel@tonic-gate cp = (cpu_t *)resource; 4947c478bd9Sstevel@tonic-gate lgrp_cpu_init(cp); 4957c478bd9Sstevel@tonic-gate lgrp_part_add_cpu(cp, cp->cpu_lpl->lpl_lgrpid); 4967c478bd9Sstevel@tonic-gate rc = lpl_topo_verify(cp->cpu_part); 4977c478bd9Sstevel@tonic-gate if (rc != LPL_TOPO_CORRECT) { 4987c478bd9Sstevel@tonic-gate panic("lpl_topo_verify failed: %d", rc); 4997c478bd9Sstevel@tonic-gate } 5007c478bd9Sstevel@tonic-gate lgrp_plat_config(event, resource); 5017c478bd9Sstevel@tonic-gate atomic_add_32(&lgrp_gen, 1); 5027c478bd9Sstevel@tonic-gate 5037c478bd9Sstevel@tonic-gate break; 5047c478bd9Sstevel@tonic-gate case LGRP_CONFIG_CPU_OFFLINE: 5057c478bd9Sstevel@tonic-gate cp = (cpu_t *)resource; 5067c478bd9Sstevel@tonic-gate id = cp->cpu_lpl->lpl_lgrpid; 5077c478bd9Sstevel@tonic-gate lgrp_part_del_cpu(cp); 5087c478bd9Sstevel@tonic-gate lgrp_cpu_fini(cp, id); 5097c478bd9Sstevel@tonic-gate rc = lpl_topo_verify(cp->cpu_part); 5107c478bd9Sstevel@tonic-gate if (rc != LPL_TOPO_CORRECT) { 5117c478bd9Sstevel@tonic-gate panic("lpl_topo_verify failed: %d", rc); 5127c478bd9Sstevel@tonic-gate } 5137c478bd9Sstevel@tonic-gate lgrp_plat_config(event, resource); 5147c478bd9Sstevel@tonic-gate atomic_add_32(&lgrp_gen, 1); 5157c478bd9Sstevel@tonic-gate 5167c478bd9Sstevel@tonic-gate break; 5177c478bd9Sstevel@tonic-gate case LGRP_CONFIG_CPUPART_ADD: 5187c478bd9Sstevel@tonic-gate cp = (cpu_t *)resource; 5197c478bd9Sstevel@tonic-gate lgrp_part_add_cpu((cpu_t *)resource, (lgrp_id_t)where); 5207c478bd9Sstevel@tonic-gate rc = lpl_topo_verify(cp->cpu_part); 5217c478bd9Sstevel@tonic-gate if (rc != LPL_TOPO_CORRECT) { 5227c478bd9Sstevel@tonic-gate panic("lpl_topo_verify failed: %d", rc); 5237c478bd9Sstevel@tonic-gate } 5247c478bd9Sstevel@tonic-gate lgrp_plat_config(event, resource); 5257c478bd9Sstevel@tonic-gate 5267c478bd9Sstevel@tonic-gate break; 5277c478bd9Sstevel@tonic-gate case LGRP_CONFIG_CPUPART_DEL: 5287c478bd9Sstevel@tonic-gate cp = (cpu_t *)resource; 5297c478bd9Sstevel@tonic-gate lgrp_part_del_cpu((cpu_t *)resource); 5307c478bd9Sstevel@tonic-gate rc = lpl_topo_verify(cp->cpu_part); 5317c478bd9Sstevel@tonic-gate if (rc != LPL_TOPO_CORRECT) { 5327c478bd9Sstevel@tonic-gate panic("lpl_topo_verify failed: %d", rc); 5337c478bd9Sstevel@tonic-gate } 5347c478bd9Sstevel@tonic-gate lgrp_plat_config(event, resource); 5357c478bd9Sstevel@tonic-gate 5367c478bd9Sstevel@tonic-gate break; 5377c478bd9Sstevel@tonic-gate /* 5387c478bd9Sstevel@tonic-gate * The following events are initiated by the memnode 5397c478bd9Sstevel@tonic-gate * subsystem. 5407c478bd9Sstevel@tonic-gate */ 5417c478bd9Sstevel@tonic-gate case LGRP_CONFIG_MEM_ADD: 5427c478bd9Sstevel@tonic-gate lgrp_mem_init((int)resource, where, B_FALSE); 5437c478bd9Sstevel@tonic-gate atomic_add_32(&lgrp_gen, 1); 5447c478bd9Sstevel@tonic-gate 5457c478bd9Sstevel@tonic-gate break; 5467c478bd9Sstevel@tonic-gate case LGRP_CONFIG_MEM_DEL: 5477c478bd9Sstevel@tonic-gate lgrp_mem_fini((int)resource, where, B_FALSE); 5487c478bd9Sstevel@tonic-gate atomic_add_32(&lgrp_gen, 1); 5497c478bd9Sstevel@tonic-gate 5507c478bd9Sstevel@tonic-gate break; 5517c478bd9Sstevel@tonic-gate case LGRP_CONFIG_MEM_RENAME: { 5527c478bd9Sstevel@tonic-gate lgrp_config_mem_rename_t *ren_arg = 5537c478bd9Sstevel@tonic-gate (lgrp_config_mem_rename_t *)where; 5547c478bd9Sstevel@tonic-gate 5557c478bd9Sstevel@tonic-gate lgrp_mem_rename((int)resource, 5567c478bd9Sstevel@tonic-gate ren_arg->lmem_rename_from, 5577c478bd9Sstevel@tonic-gate ren_arg->lmem_rename_to); 5587c478bd9Sstevel@tonic-gate atomic_add_32(&lgrp_gen, 1); 5597c478bd9Sstevel@tonic-gate 5607c478bd9Sstevel@tonic-gate break; 5617c478bd9Sstevel@tonic-gate } 5627c478bd9Sstevel@tonic-gate case LGRP_CONFIG_GEN_UPDATE: 5637c478bd9Sstevel@tonic-gate atomic_add_32(&lgrp_gen, 1); 5647c478bd9Sstevel@tonic-gate 5657c478bd9Sstevel@tonic-gate break; 5667c478bd9Sstevel@tonic-gate case LGRP_CONFIG_FLATTEN: 5677c478bd9Sstevel@tonic-gate if (where == 0) 5687c478bd9Sstevel@tonic-gate lgrp_topo_levels = (int)resource; 5697c478bd9Sstevel@tonic-gate else 5707c478bd9Sstevel@tonic-gate (void) lgrp_topo_flatten(resource, 5717c478bd9Sstevel@tonic-gate lgrp_table, lgrp_alloc_max, &changed); 5727c478bd9Sstevel@tonic-gate 5737c478bd9Sstevel@tonic-gate break; 5747c478bd9Sstevel@tonic-gate /* 5757c478bd9Sstevel@tonic-gate * Initiated by platform latency probing code 5767c478bd9Sstevel@tonic-gate */ 5777c478bd9Sstevel@tonic-gate case LGRP_CONFIG_LATENCY_CHANGE: 5787c478bd9Sstevel@tonic-gate lgrp_latency_change((u_longlong_t)resource, 5797c478bd9Sstevel@tonic-gate (u_longlong_t)where); 5807c478bd9Sstevel@tonic-gate 5817c478bd9Sstevel@tonic-gate break; 5827c478bd9Sstevel@tonic-gate case LGRP_CONFIG_NOP: 5837c478bd9Sstevel@tonic-gate 5847c478bd9Sstevel@tonic-gate break; 5857c478bd9Sstevel@tonic-gate default: 5867c478bd9Sstevel@tonic-gate break; 5877c478bd9Sstevel@tonic-gate } 5887c478bd9Sstevel@tonic-gate 5897c478bd9Sstevel@tonic-gate } 5907c478bd9Sstevel@tonic-gate 5917c478bd9Sstevel@tonic-gate /* 5927c478bd9Sstevel@tonic-gate * Called to add lgrp info into cpu structure from cpu_add_unit; 5937c478bd9Sstevel@tonic-gate * do not assume cpu is in cpu[] yet! 5947c478bd9Sstevel@tonic-gate * 5957c478bd9Sstevel@tonic-gate * CPUs are brought online with all other CPUs paused so we can't 5967c478bd9Sstevel@tonic-gate * allocate memory or we could deadlock the system, so we rely on 5977c478bd9Sstevel@tonic-gate * the platform to statically allocate as much space as we need 5987c478bd9Sstevel@tonic-gate * for the lgrp structs and stats. 5997c478bd9Sstevel@tonic-gate */ 6007c478bd9Sstevel@tonic-gate static void 6017c478bd9Sstevel@tonic-gate lgrp_cpu_init(struct cpu *cp) 6027c478bd9Sstevel@tonic-gate { 6037c478bd9Sstevel@tonic-gate klgrpset_t changed; 6047c478bd9Sstevel@tonic-gate int count; 6057c478bd9Sstevel@tonic-gate lgrp_handle_t hand; 6067c478bd9Sstevel@tonic-gate int first_cpu; 6077c478bd9Sstevel@tonic-gate lgrp_t *my_lgrp; 6087c478bd9Sstevel@tonic-gate lgrp_id_t lgrpid; 6097c478bd9Sstevel@tonic-gate struct cpu *cptr; 6107c478bd9Sstevel@tonic-gate struct chip *chp; 6117c478bd9Sstevel@tonic-gate 6127c478bd9Sstevel@tonic-gate /* 6137c478bd9Sstevel@tonic-gate * This is the first time through if the resource set 6147c478bd9Sstevel@tonic-gate * for the root lgroup is empty. After cpu0 has been 6157c478bd9Sstevel@tonic-gate * initially added to an lgroup, the root's CPU resource 6167c478bd9Sstevel@tonic-gate * set can never be empty, since the system's last CPU 6177c478bd9Sstevel@tonic-gate * cannot be offlined. 6187c478bd9Sstevel@tonic-gate */ 6197c478bd9Sstevel@tonic-gate if (klgrpset_isempty(lgrp_root->lgrp_set[LGRP_RSRC_CPU])) { 6207c478bd9Sstevel@tonic-gate /* 6217c478bd9Sstevel@tonic-gate * First time through. 6227c478bd9Sstevel@tonic-gate */ 6237c478bd9Sstevel@tonic-gate first_cpu = 1; 6247c478bd9Sstevel@tonic-gate } else { 6257c478bd9Sstevel@tonic-gate /* 6267c478bd9Sstevel@tonic-gate * If cpu0 needs to move lgroups, we may come 6277c478bd9Sstevel@tonic-gate * through here again, at which time cpu_lock won't 6287c478bd9Sstevel@tonic-gate * be held, and lgrp_initialized will be false. 6297c478bd9Sstevel@tonic-gate */ 6307c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&cpu_lock) || !lgrp_initialized); 6317c478bd9Sstevel@tonic-gate ASSERT(cp->cpu_part != NULL); 6327c478bd9Sstevel@tonic-gate first_cpu = 0; 6337c478bd9Sstevel@tonic-gate } 6347c478bd9Sstevel@tonic-gate 6357c478bd9Sstevel@tonic-gate hand = lgrp_plat_cpu_to_hand(cp->cpu_id); 6367c478bd9Sstevel@tonic-gate my_lgrp = lgrp_hand_to_lgrp(hand); 6377c478bd9Sstevel@tonic-gate 6387c478bd9Sstevel@tonic-gate if (my_lgrp == NULL) { 6397c478bd9Sstevel@tonic-gate /* 6407c478bd9Sstevel@tonic-gate * Create new lgrp and add it to lgroup topology 6417c478bd9Sstevel@tonic-gate */ 6427c478bd9Sstevel@tonic-gate my_lgrp = lgrp_create(); 6437c478bd9Sstevel@tonic-gate my_lgrp->lgrp_plathand = hand; 6447c478bd9Sstevel@tonic-gate my_lgrp->lgrp_latency = lgrp_plat_latency(hand, hand); 6457c478bd9Sstevel@tonic-gate lgrpid = my_lgrp->lgrp_id; 6467c478bd9Sstevel@tonic-gate klgrpset_add(my_lgrp->lgrp_leaves, lgrpid); 6477c478bd9Sstevel@tonic-gate klgrpset_add(my_lgrp->lgrp_set[LGRP_RSRC_CPU], lgrpid); 6487c478bd9Sstevel@tonic-gate 6497c478bd9Sstevel@tonic-gate count = 0; 6507c478bd9Sstevel@tonic-gate klgrpset_clear(changed); 6517c478bd9Sstevel@tonic-gate count += lgrp_leaf_add(my_lgrp, lgrp_table, lgrp_alloc_max + 1, 6527c478bd9Sstevel@tonic-gate &changed); 653*2dae3fb5Sjjc /* 654*2dae3fb5Sjjc * May have added new intermediate lgroups, so need to add 655*2dae3fb5Sjjc * resources other than CPUs which are added below 656*2dae3fb5Sjjc */ 657*2dae3fb5Sjjc (void) lgrp_mnode_update(changed, NULL); 6587c478bd9Sstevel@tonic-gate } else if (my_lgrp->lgrp_latency == 0 && lgrp_plat_latency(hand, hand) 6597c478bd9Sstevel@tonic-gate > 0) { 6607c478bd9Sstevel@tonic-gate /* 6617c478bd9Sstevel@tonic-gate * Leaf lgroup was created, but latency wasn't available 6627c478bd9Sstevel@tonic-gate * then. So, set latency for it and fill in rest of lgroup 6637c478bd9Sstevel@tonic-gate * topology now that we know how far it is from other leaf 6647c478bd9Sstevel@tonic-gate * lgroups. 6657c478bd9Sstevel@tonic-gate */ 6667c478bd9Sstevel@tonic-gate lgrpid = my_lgrp->lgrp_id; 6677c478bd9Sstevel@tonic-gate klgrpset_clear(changed); 6687c478bd9Sstevel@tonic-gate if (!klgrpset_ismember(my_lgrp->lgrp_set[LGRP_RSRC_CPU], 6697c478bd9Sstevel@tonic-gate lgrpid)) 6707c478bd9Sstevel@tonic-gate klgrpset_add(my_lgrp->lgrp_set[LGRP_RSRC_CPU], lgrpid); 6717c478bd9Sstevel@tonic-gate count = lgrp_leaf_add(my_lgrp, lgrp_table, lgrp_alloc_max + 1, 6727c478bd9Sstevel@tonic-gate &changed); 6737c478bd9Sstevel@tonic-gate 6747c478bd9Sstevel@tonic-gate /* 6757c478bd9Sstevel@tonic-gate * May have added new intermediate lgroups, so need to add 6767c478bd9Sstevel@tonic-gate * resources other than CPUs which are added below 6777c478bd9Sstevel@tonic-gate */ 6787c478bd9Sstevel@tonic-gate (void) lgrp_mnode_update(changed, NULL); 6797c478bd9Sstevel@tonic-gate } else if (!klgrpset_ismember(my_lgrp->lgrp_set[LGRP_RSRC_CPU], 6807c478bd9Sstevel@tonic-gate my_lgrp->lgrp_id)) { 6817c478bd9Sstevel@tonic-gate int i; 6827c478bd9Sstevel@tonic-gate 6837c478bd9Sstevel@tonic-gate /* 6847c478bd9Sstevel@tonic-gate * Update existing lgroup and lgroups containing it with CPU 6857c478bd9Sstevel@tonic-gate * resource 6867c478bd9Sstevel@tonic-gate */ 6877c478bd9Sstevel@tonic-gate lgrpid = my_lgrp->lgrp_id; 6887c478bd9Sstevel@tonic-gate klgrpset_add(my_lgrp->lgrp_set[LGRP_RSRC_CPU], lgrpid); 6897c478bd9Sstevel@tonic-gate for (i = 0; i <= lgrp_alloc_max; i++) { 6907c478bd9Sstevel@tonic-gate lgrp_t *lgrp; 6917c478bd9Sstevel@tonic-gate 6927c478bd9Sstevel@tonic-gate lgrp = lgrp_table[i]; 6937c478bd9Sstevel@tonic-gate if (!LGRP_EXISTS(lgrp) || 6947c478bd9Sstevel@tonic-gate !lgrp_rsets_member(lgrp->lgrp_set, lgrpid)) 6957c478bd9Sstevel@tonic-gate continue; 6967c478bd9Sstevel@tonic-gate 6977c478bd9Sstevel@tonic-gate klgrpset_add(lgrp->lgrp_set[LGRP_RSRC_CPU], lgrpid); 6987c478bd9Sstevel@tonic-gate } 6997c478bd9Sstevel@tonic-gate } 7007c478bd9Sstevel@tonic-gate 7017c478bd9Sstevel@tonic-gate lgrpid = my_lgrp->lgrp_id; 7027c478bd9Sstevel@tonic-gate cp->cpu_lpl = &cp->cpu_part->cp_lgrploads[lgrpid]; 7037c478bd9Sstevel@tonic-gate 7047c478bd9Sstevel@tonic-gate /* 7057c478bd9Sstevel@tonic-gate * For multi-lgroup systems, need to setup lpl for CPU0 or CPU0 will 7067c478bd9Sstevel@tonic-gate * end up in lpl for lgroup 0 whether it is supposed to be in there or 7077c478bd9Sstevel@tonic-gate * not since none of lgroup IDs in the lpl's have been set yet. 7087c478bd9Sstevel@tonic-gate */ 7097c478bd9Sstevel@tonic-gate if (first_cpu && nlgrpsmax > 1 && lgrpid != cp->cpu_lpl->lpl_lgrpid) 7107c478bd9Sstevel@tonic-gate cp->cpu_lpl->lpl_lgrpid = lgrpid; 7117c478bd9Sstevel@tonic-gate 7127c478bd9Sstevel@tonic-gate /* 7137c478bd9Sstevel@tonic-gate * link the CPU into the lgrp's CPU list 7147c478bd9Sstevel@tonic-gate */ 7157c478bd9Sstevel@tonic-gate if (my_lgrp->lgrp_cpucnt == 0) { 7167c478bd9Sstevel@tonic-gate my_lgrp->lgrp_cpu = cp; 7177c478bd9Sstevel@tonic-gate cp->cpu_next_lgrp = cp->cpu_prev_lgrp = cp; 7187c478bd9Sstevel@tonic-gate } else { 7197c478bd9Sstevel@tonic-gate cptr = my_lgrp->lgrp_cpu; 7207c478bd9Sstevel@tonic-gate cp->cpu_next_lgrp = cptr; 7217c478bd9Sstevel@tonic-gate cp->cpu_prev_lgrp = cptr->cpu_prev_lgrp; 7227c478bd9Sstevel@tonic-gate cptr->cpu_prev_lgrp->cpu_next_lgrp = cp; 7237c478bd9Sstevel@tonic-gate cptr->cpu_prev_lgrp = cp; 7247c478bd9Sstevel@tonic-gate } 7257c478bd9Sstevel@tonic-gate my_lgrp->lgrp_cpucnt++; 7267c478bd9Sstevel@tonic-gate 7277c478bd9Sstevel@tonic-gate /* 7287c478bd9Sstevel@tonic-gate * Add this cpu's chip to the per lgroup list 7297c478bd9Sstevel@tonic-gate * if necessary 7307c478bd9Sstevel@tonic-gate */ 7317c478bd9Sstevel@tonic-gate if (cp->cpu_chip->chip_lgrp == NULL) { 7327c478bd9Sstevel@tonic-gate struct chip *lcpr; 7337c478bd9Sstevel@tonic-gate 7347c478bd9Sstevel@tonic-gate chp = cp->cpu_chip; 7357c478bd9Sstevel@tonic-gate 7367c478bd9Sstevel@tonic-gate if (my_lgrp->lgrp_chipcnt == 0) { 7377c478bd9Sstevel@tonic-gate my_lgrp->lgrp_chips = chp; 7387c478bd9Sstevel@tonic-gate chp->chip_next_lgrp = 7397c478bd9Sstevel@tonic-gate chp->chip_prev_lgrp = chp; 7407c478bd9Sstevel@tonic-gate } else { 7417c478bd9Sstevel@tonic-gate lcpr = my_lgrp->lgrp_chips; 7427c478bd9Sstevel@tonic-gate chp->chip_next_lgrp = lcpr; 7437c478bd9Sstevel@tonic-gate chp->chip_prev_lgrp = 7447c478bd9Sstevel@tonic-gate lcpr->chip_prev_lgrp; 7457c478bd9Sstevel@tonic-gate lcpr->chip_prev_lgrp->chip_next_lgrp = 7467c478bd9Sstevel@tonic-gate chp; 7477c478bd9Sstevel@tonic-gate lcpr->chip_prev_lgrp = chp; 7487c478bd9Sstevel@tonic-gate } 7497c478bd9Sstevel@tonic-gate chp->chip_lgrp = my_lgrp; 7507c478bd9Sstevel@tonic-gate chp->chip_balance = chp->chip_next_lgrp; 7517c478bd9Sstevel@tonic-gate my_lgrp->lgrp_chipcnt++; 7527c478bd9Sstevel@tonic-gate } 7537c478bd9Sstevel@tonic-gate } 7547c478bd9Sstevel@tonic-gate 7557c478bd9Sstevel@tonic-gate lgrp_t * 7567c478bd9Sstevel@tonic-gate lgrp_create(void) 7577c478bd9Sstevel@tonic-gate { 7587c478bd9Sstevel@tonic-gate lgrp_t *my_lgrp; 7597c478bd9Sstevel@tonic-gate lgrp_id_t lgrpid; 7607c478bd9Sstevel@tonic-gate int i; 7617c478bd9Sstevel@tonic-gate 7627c478bd9Sstevel@tonic-gate ASSERT(!lgrp_initialized || MUTEX_HELD(&cpu_lock)); 7637c478bd9Sstevel@tonic-gate 7647c478bd9Sstevel@tonic-gate /* 7657c478bd9Sstevel@tonic-gate * Find an open slot in the lgroup table and recycle unused lgroup 7667c478bd9Sstevel@tonic-gate * left there if any 7677c478bd9Sstevel@tonic-gate */ 7687c478bd9Sstevel@tonic-gate my_lgrp = NULL; 7697c478bd9Sstevel@tonic-gate if (lgrp_alloc_hint == -1) 7707c478bd9Sstevel@tonic-gate /* 7717c478bd9Sstevel@tonic-gate * Allocate from end when hint not set yet because no lgroups 7727c478bd9Sstevel@tonic-gate * have been deleted yet 7737c478bd9Sstevel@tonic-gate */ 7747c478bd9Sstevel@tonic-gate lgrpid = nlgrps++; 7757c478bd9Sstevel@tonic-gate else { 7767c478bd9Sstevel@tonic-gate /* 7777c478bd9Sstevel@tonic-gate * Start looking for next open slot from hint and leave hint 7787c478bd9Sstevel@tonic-gate * at slot allocated 7797c478bd9Sstevel@tonic-gate */ 7807c478bd9Sstevel@tonic-gate for (i = lgrp_alloc_hint; i < nlgrpsmax; i++) { 7817c478bd9Sstevel@tonic-gate my_lgrp = lgrp_table[i]; 7827c478bd9Sstevel@tonic-gate if (!LGRP_EXISTS(my_lgrp)) { 7837c478bd9Sstevel@tonic-gate lgrpid = i; 7847c478bd9Sstevel@tonic-gate nlgrps++; 7857c478bd9Sstevel@tonic-gate break; 7867c478bd9Sstevel@tonic-gate } 7877c478bd9Sstevel@tonic-gate } 7887c478bd9Sstevel@tonic-gate lgrp_alloc_hint = lgrpid; 7897c478bd9Sstevel@tonic-gate } 7907c478bd9Sstevel@tonic-gate 7917c478bd9Sstevel@tonic-gate /* 7927c478bd9Sstevel@tonic-gate * Keep track of max lgroup ID allocated so far to cut down on searches 7937c478bd9Sstevel@tonic-gate */ 7947c478bd9Sstevel@tonic-gate if (lgrpid > lgrp_alloc_max) 7957c478bd9Sstevel@tonic-gate lgrp_alloc_max = lgrpid; 7967c478bd9Sstevel@tonic-gate 7977c478bd9Sstevel@tonic-gate /* 7987c478bd9Sstevel@tonic-gate * Need to allocate new lgroup if next open slot didn't have one 7997c478bd9Sstevel@tonic-gate * for recycling 8007c478bd9Sstevel@tonic-gate */ 8017c478bd9Sstevel@tonic-gate if (my_lgrp == NULL) 8027c478bd9Sstevel@tonic-gate my_lgrp = lgrp_plat_alloc(lgrpid); 8037c478bd9Sstevel@tonic-gate 8047c478bd9Sstevel@tonic-gate if (nlgrps > nlgrpsmax || my_lgrp == NULL) 8057c478bd9Sstevel@tonic-gate panic("Too many lgrps for platform (%d)", nlgrps); 8067c478bd9Sstevel@tonic-gate 8077c478bd9Sstevel@tonic-gate my_lgrp->lgrp_id = lgrpid; 8087c478bd9Sstevel@tonic-gate my_lgrp->lgrp_latency = 0; 8097c478bd9Sstevel@tonic-gate my_lgrp->lgrp_plathand = LGRP_NULL_HANDLE; 8107c478bd9Sstevel@tonic-gate my_lgrp->lgrp_parent = NULL; 8117c478bd9Sstevel@tonic-gate my_lgrp->lgrp_childcnt = 0; 8127c478bd9Sstevel@tonic-gate my_lgrp->lgrp_mnodes = (mnodeset_t)0; 8137c478bd9Sstevel@tonic-gate my_lgrp->lgrp_nmnodes = 0; 8147c478bd9Sstevel@tonic-gate klgrpset_clear(my_lgrp->lgrp_children); 8157c478bd9Sstevel@tonic-gate klgrpset_clear(my_lgrp->lgrp_leaves); 8167c478bd9Sstevel@tonic-gate for (i = 0; i < LGRP_RSRC_COUNT; i++) 8177c478bd9Sstevel@tonic-gate klgrpset_clear(my_lgrp->lgrp_set[i]); 8187c478bd9Sstevel@tonic-gate 8197c478bd9Sstevel@tonic-gate my_lgrp->lgrp_cpu = NULL; 8207c478bd9Sstevel@tonic-gate my_lgrp->lgrp_cpucnt = 0; 8217c478bd9Sstevel@tonic-gate my_lgrp->lgrp_chips = NULL; 8227c478bd9Sstevel@tonic-gate my_lgrp->lgrp_chipcnt = 0; 8237c478bd9Sstevel@tonic-gate 8247c478bd9Sstevel@tonic-gate if (my_lgrp->lgrp_kstat != NULL) 8257c478bd9Sstevel@tonic-gate lgrp_kstat_reset(lgrpid); 8267c478bd9Sstevel@tonic-gate 8277c478bd9Sstevel@tonic-gate lgrp_table[my_lgrp->lgrp_id] = my_lgrp; 8287c478bd9Sstevel@tonic-gate 8297c478bd9Sstevel@tonic-gate return (my_lgrp); 8307c478bd9Sstevel@tonic-gate } 8317c478bd9Sstevel@tonic-gate 8327c478bd9Sstevel@tonic-gate void 8337c478bd9Sstevel@tonic-gate lgrp_destroy(lgrp_t *lgrp) 8347c478bd9Sstevel@tonic-gate { 8357c478bd9Sstevel@tonic-gate int i; 8367c478bd9Sstevel@tonic-gate 8377c478bd9Sstevel@tonic-gate /* 8387c478bd9Sstevel@tonic-gate * Unless this lgroup is being destroyed on behalf of 8397c478bd9Sstevel@tonic-gate * the boot CPU, cpu_lock must be held 8407c478bd9Sstevel@tonic-gate */ 8417c478bd9Sstevel@tonic-gate ASSERT(!lgrp_initialized || MUTEX_HELD(&cpu_lock)); 8427c478bd9Sstevel@tonic-gate 8437c478bd9Sstevel@tonic-gate if (nlgrps == 1) 8447c478bd9Sstevel@tonic-gate cmn_err(CE_PANIC, "Can't destroy only lgroup!"); 8457c478bd9Sstevel@tonic-gate 8467c478bd9Sstevel@tonic-gate if (!LGRP_EXISTS(lgrp)) 8477c478bd9Sstevel@tonic-gate return; 8487c478bd9Sstevel@tonic-gate 8497c478bd9Sstevel@tonic-gate /* 8507c478bd9Sstevel@tonic-gate * Set hint to lgroup being deleted and try to keep lower numbered 8517c478bd9Sstevel@tonic-gate * hints to facilitate finding empty slots 8527c478bd9Sstevel@tonic-gate */ 8537c478bd9Sstevel@tonic-gate if (lgrp_alloc_hint == -1 || lgrp->lgrp_id < lgrp_alloc_hint) 8547c478bd9Sstevel@tonic-gate lgrp_alloc_hint = lgrp->lgrp_id; 8557c478bd9Sstevel@tonic-gate 8567c478bd9Sstevel@tonic-gate /* 8577c478bd9Sstevel@tonic-gate * Mark this lgroup to be recycled by setting its lgroup ID to 8587c478bd9Sstevel@tonic-gate * LGRP_NONE and clear relevant fields 8597c478bd9Sstevel@tonic-gate */ 8607c478bd9Sstevel@tonic-gate lgrp->lgrp_id = LGRP_NONE; 8617c478bd9Sstevel@tonic-gate lgrp->lgrp_latency = 0; 8627c478bd9Sstevel@tonic-gate lgrp->lgrp_plathand = LGRP_NULL_HANDLE; 8637c478bd9Sstevel@tonic-gate lgrp->lgrp_parent = NULL; 8647c478bd9Sstevel@tonic-gate lgrp->lgrp_childcnt = 0; 8657c478bd9Sstevel@tonic-gate 8667c478bd9Sstevel@tonic-gate klgrpset_clear(lgrp->lgrp_children); 8677c478bd9Sstevel@tonic-gate klgrpset_clear(lgrp->lgrp_leaves); 8687c478bd9Sstevel@tonic-gate for (i = 0; i < LGRP_RSRC_COUNT; i++) 8697c478bd9Sstevel@tonic-gate klgrpset_clear(lgrp->lgrp_set[i]); 8707c478bd9Sstevel@tonic-gate 8717c478bd9Sstevel@tonic-gate lgrp->lgrp_mnodes = (mnodeset_t)0; 8727c478bd9Sstevel@tonic-gate lgrp->lgrp_nmnodes = 0; 8737c478bd9Sstevel@tonic-gate 8747c478bd9Sstevel@tonic-gate lgrp->lgrp_cpu = NULL; 8757c478bd9Sstevel@tonic-gate lgrp->lgrp_cpucnt = 0; 8767c478bd9Sstevel@tonic-gate lgrp->lgrp_chipcnt = 0; 8777c478bd9Sstevel@tonic-gate lgrp->lgrp_chips = NULL; 8787c478bd9Sstevel@tonic-gate 8797c478bd9Sstevel@tonic-gate nlgrps--; 8807c478bd9Sstevel@tonic-gate } 8817c478bd9Sstevel@tonic-gate 8827c478bd9Sstevel@tonic-gate /* 8837c478bd9Sstevel@tonic-gate * Initialize kstat data. Called from lgrp intialization code. 8847c478bd9Sstevel@tonic-gate */ 8857c478bd9Sstevel@tonic-gate static void 8867c478bd9Sstevel@tonic-gate lgrp_kstat_init(void) 8877c478bd9Sstevel@tonic-gate { 8887c478bd9Sstevel@tonic-gate lgrp_stat_t stat; 8897c478bd9Sstevel@tonic-gate 8907c478bd9Sstevel@tonic-gate mutex_init(&lgrp_kstat_mutex, NULL, MUTEX_DEFAULT, NULL); 8917c478bd9Sstevel@tonic-gate 8927c478bd9Sstevel@tonic-gate for (stat = 0; stat < LGRP_NUM_STATS; stat++) 8937c478bd9Sstevel@tonic-gate kstat_named_init(&lgrp_kstat_data[stat], 8947c478bd9Sstevel@tonic-gate lgrp_kstat_names[stat], KSTAT_DATA_INT64); 8957c478bd9Sstevel@tonic-gate } 8967c478bd9Sstevel@tonic-gate 8977c478bd9Sstevel@tonic-gate /* 8987c478bd9Sstevel@tonic-gate * initialize an lgrp's kstats if needed 8997c478bd9Sstevel@tonic-gate * called with cpu_lock held but not with cpus paused. 9007c478bd9Sstevel@tonic-gate * we don't tear these down now because we don't know about 9017c478bd9Sstevel@tonic-gate * memory leaving the lgrp yet... 9027c478bd9Sstevel@tonic-gate */ 9037c478bd9Sstevel@tonic-gate 9047c478bd9Sstevel@tonic-gate void 9057c478bd9Sstevel@tonic-gate lgrp_kstat_create(cpu_t *cp) 9067c478bd9Sstevel@tonic-gate { 9077c478bd9Sstevel@tonic-gate kstat_t *lgrp_kstat; 9087c478bd9Sstevel@tonic-gate lgrp_id_t lgrpid; 9097c478bd9Sstevel@tonic-gate lgrp_t *my_lgrp; 9107c478bd9Sstevel@tonic-gate 9117c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&cpu_lock)); 9127c478bd9Sstevel@tonic-gate 9137c478bd9Sstevel@tonic-gate lgrpid = cp->cpu_lpl->lpl_lgrpid; 9147c478bd9Sstevel@tonic-gate my_lgrp = lgrp_table[lgrpid]; 9157c478bd9Sstevel@tonic-gate 9167c478bd9Sstevel@tonic-gate if (my_lgrp->lgrp_kstat != NULL) 9177c478bd9Sstevel@tonic-gate return; /* already initialized */ 9187c478bd9Sstevel@tonic-gate 9197c478bd9Sstevel@tonic-gate lgrp_kstat = kstat_create("lgrp", lgrpid, NULL, "misc", 9207c478bd9Sstevel@tonic-gate KSTAT_TYPE_NAMED, LGRP_NUM_STATS, 9217c478bd9Sstevel@tonic-gate KSTAT_FLAG_VIRTUAL | KSTAT_FLAG_WRITABLE); 9227c478bd9Sstevel@tonic-gate 9237c478bd9Sstevel@tonic-gate if (lgrp_kstat != NULL) { 9247c478bd9Sstevel@tonic-gate lgrp_kstat->ks_lock = &lgrp_kstat_mutex; 9257c478bd9Sstevel@tonic-gate lgrp_kstat->ks_private = my_lgrp; 9267c478bd9Sstevel@tonic-gate lgrp_kstat->ks_data = &lgrp_kstat_data; 9277c478bd9Sstevel@tonic-gate lgrp_kstat->ks_update = lgrp_kstat_extract; 9287c478bd9Sstevel@tonic-gate my_lgrp->lgrp_kstat = lgrp_kstat; 9297c478bd9Sstevel@tonic-gate kstat_install(lgrp_kstat); 9307c478bd9Sstevel@tonic-gate } 9317c478bd9Sstevel@tonic-gate } 9327c478bd9Sstevel@tonic-gate 9337c478bd9Sstevel@tonic-gate /* 9347c478bd9Sstevel@tonic-gate * this will do something when we manage to remove now unused lgrps 9357c478bd9Sstevel@tonic-gate */ 9367c478bd9Sstevel@tonic-gate 9377c478bd9Sstevel@tonic-gate /* ARGSUSED */ 9387c478bd9Sstevel@tonic-gate void 9397c478bd9Sstevel@tonic-gate lgrp_kstat_destroy(cpu_t *cp) 9407c478bd9Sstevel@tonic-gate { 9417c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&cpu_lock)); 9427c478bd9Sstevel@tonic-gate } 9437c478bd9Sstevel@tonic-gate 9447c478bd9Sstevel@tonic-gate /* 9457c478bd9Sstevel@tonic-gate * Called when a CPU is off-lined. 9467c478bd9Sstevel@tonic-gate */ 9477c478bd9Sstevel@tonic-gate static void 9487c478bd9Sstevel@tonic-gate lgrp_cpu_fini(struct cpu *cp, lgrp_id_t lgrpid) 9497c478bd9Sstevel@tonic-gate { 9507c478bd9Sstevel@tonic-gate lgrp_t *my_lgrp; 9517c478bd9Sstevel@tonic-gate struct cpu *prev; 9527c478bd9Sstevel@tonic-gate struct cpu *next; 9537c478bd9Sstevel@tonic-gate chip_t *chp; 9547c478bd9Sstevel@tonic-gate 9557c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&cpu_lock) || !lgrp_initialized); 9567c478bd9Sstevel@tonic-gate 9577c478bd9Sstevel@tonic-gate prev = cp->cpu_prev_lgrp; 9587c478bd9Sstevel@tonic-gate next = cp->cpu_next_lgrp; 9597c478bd9Sstevel@tonic-gate 9607c478bd9Sstevel@tonic-gate prev->cpu_next_lgrp = next; 9617c478bd9Sstevel@tonic-gate next->cpu_prev_lgrp = prev; 9627c478bd9Sstevel@tonic-gate 9637c478bd9Sstevel@tonic-gate /* 9647c478bd9Sstevel@tonic-gate * just because I'm paranoid doesn't mean... 9657c478bd9Sstevel@tonic-gate */ 9667c478bd9Sstevel@tonic-gate 9677c478bd9Sstevel@tonic-gate cp->cpu_next_lgrp = cp->cpu_prev_lgrp = NULL; 9687c478bd9Sstevel@tonic-gate 9697c478bd9Sstevel@tonic-gate my_lgrp = lgrp_table[lgrpid]; 9707c478bd9Sstevel@tonic-gate my_lgrp->lgrp_cpucnt--; 9717c478bd9Sstevel@tonic-gate 9727c478bd9Sstevel@tonic-gate /* 9737c478bd9Sstevel@tonic-gate * If the last CPU on it's chip is being offlined 9747c478bd9Sstevel@tonic-gate * then remove this chip from the per lgroup list. 9757c478bd9Sstevel@tonic-gate * 9767c478bd9Sstevel@tonic-gate * This is also done for the boot CPU when it needs 9777c478bd9Sstevel@tonic-gate * to move between lgroups as a consequence of 9787c478bd9Sstevel@tonic-gate * null proc lpa. 9797c478bd9Sstevel@tonic-gate */ 9807c478bd9Sstevel@tonic-gate chp = cp->cpu_chip; 9817c478bd9Sstevel@tonic-gate if (chp->chip_ncpu == 0 || !lgrp_initialized) { 9827c478bd9Sstevel@tonic-gate 9837c478bd9Sstevel@tonic-gate chip_t *chpp; 9847c478bd9Sstevel@tonic-gate 9857c478bd9Sstevel@tonic-gate if (--my_lgrp->lgrp_chipcnt == 0) 9867c478bd9Sstevel@tonic-gate my_lgrp->lgrp_chips = NULL; 9877c478bd9Sstevel@tonic-gate else if (my_lgrp->lgrp_chips == chp) 9887c478bd9Sstevel@tonic-gate my_lgrp->lgrp_chips = chp->chip_next_lgrp; 9897c478bd9Sstevel@tonic-gate 9907c478bd9Sstevel@tonic-gate /* 9917c478bd9Sstevel@tonic-gate * Walk this lgroup's chip list looking for chips that 9927c478bd9Sstevel@tonic-gate * may try to balance against the one that's leaving 9937c478bd9Sstevel@tonic-gate */ 9947c478bd9Sstevel@tonic-gate for (chpp = chp->chip_next_lgrp; chpp != chp; 9957c478bd9Sstevel@tonic-gate chpp = chpp->chip_next_lgrp) { 9967c478bd9Sstevel@tonic-gate if (chpp->chip_balance == chp) 9977c478bd9Sstevel@tonic-gate chpp->chip_balance = chp->chip_next_lgrp; 9987c478bd9Sstevel@tonic-gate } 9997c478bd9Sstevel@tonic-gate 10007c478bd9Sstevel@tonic-gate chp->chip_prev_lgrp->chip_next_lgrp = chp->chip_next_lgrp; 10017c478bd9Sstevel@tonic-gate chp->chip_next_lgrp->chip_prev_lgrp = chp->chip_prev_lgrp; 10027c478bd9Sstevel@tonic-gate 10037c478bd9Sstevel@tonic-gate chp->chip_next_lgrp = chp->chip_prev_lgrp = NULL; 10047c478bd9Sstevel@tonic-gate chp->chip_lgrp = NULL; 10057c478bd9Sstevel@tonic-gate chp->chip_balance = NULL; 10067c478bd9Sstevel@tonic-gate } 10077c478bd9Sstevel@tonic-gate 10087c478bd9Sstevel@tonic-gate /* 10097c478bd9Sstevel@tonic-gate * Removing last CPU in lgroup, so update lgroup topology 10107c478bd9Sstevel@tonic-gate */ 10117c478bd9Sstevel@tonic-gate if (my_lgrp->lgrp_cpucnt == 0) { 10127c478bd9Sstevel@tonic-gate klgrpset_t changed; 10137c478bd9Sstevel@tonic-gate int count; 10147c478bd9Sstevel@tonic-gate int i; 10157c478bd9Sstevel@tonic-gate 10167c478bd9Sstevel@tonic-gate my_lgrp->lgrp_cpu = NULL; 10177c478bd9Sstevel@tonic-gate 10187c478bd9Sstevel@tonic-gate /* 10197c478bd9Sstevel@tonic-gate * Remove this lgroup from its lgroup CPU resources and remove 10207c478bd9Sstevel@tonic-gate * lgroup from lgroup topology if it doesn't have any more 10217c478bd9Sstevel@tonic-gate * resources in it now 10227c478bd9Sstevel@tonic-gate */ 10237c478bd9Sstevel@tonic-gate klgrpset_del(my_lgrp->lgrp_set[LGRP_RSRC_CPU], lgrpid); 10247c478bd9Sstevel@tonic-gate if (lgrp_rsets_empty(my_lgrp->lgrp_set)) { 10257c478bd9Sstevel@tonic-gate count = 0; 10267c478bd9Sstevel@tonic-gate klgrpset_clear(changed); 10277c478bd9Sstevel@tonic-gate count += lgrp_leaf_delete(my_lgrp, lgrp_table, 10287c478bd9Sstevel@tonic-gate lgrp_alloc_max + 1, &changed); 10297c478bd9Sstevel@tonic-gate return; 10307c478bd9Sstevel@tonic-gate } 10317c478bd9Sstevel@tonic-gate 10327c478bd9Sstevel@tonic-gate /* 10337c478bd9Sstevel@tonic-gate * This lgroup isn't empty, so just remove it from CPU 10347c478bd9Sstevel@tonic-gate * resources of any lgroups that contain it as such 10357c478bd9Sstevel@tonic-gate */ 10367c478bd9Sstevel@tonic-gate for (i = 0; i <= lgrp_alloc_max; i++) { 10377c478bd9Sstevel@tonic-gate lgrp_t *lgrp; 10387c478bd9Sstevel@tonic-gate 10397c478bd9Sstevel@tonic-gate lgrp = lgrp_table[i]; 10407c478bd9Sstevel@tonic-gate if (!LGRP_EXISTS(lgrp) || 10417c478bd9Sstevel@tonic-gate !klgrpset_ismember(lgrp->lgrp_set[LGRP_RSRC_CPU], 10427c478bd9Sstevel@tonic-gate lgrpid)) 10437c478bd9Sstevel@tonic-gate continue; 10447c478bd9Sstevel@tonic-gate 10457c478bd9Sstevel@tonic-gate klgrpset_del(lgrp->lgrp_set[LGRP_RSRC_CPU], lgrpid); 10467c478bd9Sstevel@tonic-gate } 10477c478bd9Sstevel@tonic-gate return; 10487c478bd9Sstevel@tonic-gate } 10497c478bd9Sstevel@tonic-gate 10507c478bd9Sstevel@tonic-gate if (my_lgrp->lgrp_cpu == cp) 10517c478bd9Sstevel@tonic-gate my_lgrp->lgrp_cpu = next; 10527c478bd9Sstevel@tonic-gate 10537c478bd9Sstevel@tonic-gate } 10547c478bd9Sstevel@tonic-gate 10557c478bd9Sstevel@tonic-gate /* 10567c478bd9Sstevel@tonic-gate * Update memory nodes in target lgroups and return ones that get changed 10577c478bd9Sstevel@tonic-gate */ 10587c478bd9Sstevel@tonic-gate int 10597c478bd9Sstevel@tonic-gate lgrp_mnode_update(klgrpset_t target, klgrpset_t *changed) 10607c478bd9Sstevel@tonic-gate { 10617c478bd9Sstevel@tonic-gate int count; 10627c478bd9Sstevel@tonic-gate int i; 10637c478bd9Sstevel@tonic-gate int j; 10647c478bd9Sstevel@tonic-gate lgrp_t *lgrp; 10657c478bd9Sstevel@tonic-gate lgrp_t *lgrp_rsrc; 10667c478bd9Sstevel@tonic-gate 10677c478bd9Sstevel@tonic-gate count = 0; 10687c478bd9Sstevel@tonic-gate if (changed) 10697c478bd9Sstevel@tonic-gate klgrpset_clear(*changed); 10707c478bd9Sstevel@tonic-gate 10717c478bd9Sstevel@tonic-gate if (klgrpset_isempty(target)) 10727c478bd9Sstevel@tonic-gate return (0); 10737c478bd9Sstevel@tonic-gate 10747c478bd9Sstevel@tonic-gate /* 10757c478bd9Sstevel@tonic-gate * Find each lgroup in target lgroups 10767c478bd9Sstevel@tonic-gate */ 10777c478bd9Sstevel@tonic-gate for (i = 0; i <= lgrp_alloc_max; i++) { 10787c478bd9Sstevel@tonic-gate /* 10797c478bd9Sstevel@tonic-gate * Skip any lgroups that don't exist or aren't in target group 10807c478bd9Sstevel@tonic-gate */ 10817c478bd9Sstevel@tonic-gate lgrp = lgrp_table[i]; 10827c478bd9Sstevel@tonic-gate if (!klgrpset_ismember(target, i) || !LGRP_EXISTS(lgrp)) { 10837c478bd9Sstevel@tonic-gate continue; 10847c478bd9Sstevel@tonic-gate } 10857c478bd9Sstevel@tonic-gate 10867c478bd9Sstevel@tonic-gate /* 10877c478bd9Sstevel@tonic-gate * Initialize memnodes for intermediate lgroups to 0 10887c478bd9Sstevel@tonic-gate * and update them from scratch since they may have completely 10897c478bd9Sstevel@tonic-gate * changed 10907c478bd9Sstevel@tonic-gate */ 10917c478bd9Sstevel@tonic-gate if (lgrp->lgrp_childcnt && lgrp != lgrp_root) { 10927c478bd9Sstevel@tonic-gate lgrp->lgrp_mnodes = (mnodeset_t)0; 10937c478bd9Sstevel@tonic-gate lgrp->lgrp_nmnodes = 0; 10947c478bd9Sstevel@tonic-gate } 10957c478bd9Sstevel@tonic-gate 10967c478bd9Sstevel@tonic-gate /* 10977c478bd9Sstevel@tonic-gate * Update memory nodes of of target lgroup with memory nodes 10987c478bd9Sstevel@tonic-gate * from each lgroup in its lgroup memory resource set 10997c478bd9Sstevel@tonic-gate */ 11007c478bd9Sstevel@tonic-gate for (j = 0; j <= lgrp_alloc_max; j++) { 11017c478bd9Sstevel@tonic-gate int k; 11027c478bd9Sstevel@tonic-gate 11037c478bd9Sstevel@tonic-gate /* 11047c478bd9Sstevel@tonic-gate * Skip any lgroups that don't exist or aren't in 11057c478bd9Sstevel@tonic-gate * memory resources of target lgroup 11067c478bd9Sstevel@tonic-gate */ 11077c478bd9Sstevel@tonic-gate lgrp_rsrc = lgrp_table[j]; 11087c478bd9Sstevel@tonic-gate if (!LGRP_EXISTS(lgrp_rsrc) || 11097c478bd9Sstevel@tonic-gate !klgrpset_ismember(lgrp->lgrp_set[LGRP_RSRC_MEM], 11107c478bd9Sstevel@tonic-gate j)) 11117c478bd9Sstevel@tonic-gate continue; 11127c478bd9Sstevel@tonic-gate 11137c478bd9Sstevel@tonic-gate /* 11147c478bd9Sstevel@tonic-gate * Update target lgroup's memnodes to include memnodes 11157c478bd9Sstevel@tonic-gate * of this lgroup 11167c478bd9Sstevel@tonic-gate */ 11177c478bd9Sstevel@tonic-gate for (k = 0; k < sizeof (mnodeset_t) * NBBY; k++) { 11187c478bd9Sstevel@tonic-gate mnodeset_t mnode_mask; 11197c478bd9Sstevel@tonic-gate 11207c478bd9Sstevel@tonic-gate mnode_mask = (mnodeset_t)1 << k; 11217c478bd9Sstevel@tonic-gate if ((lgrp_rsrc->lgrp_mnodes & mnode_mask) && 11227c478bd9Sstevel@tonic-gate !(lgrp->lgrp_mnodes & mnode_mask)) { 11237c478bd9Sstevel@tonic-gate lgrp->lgrp_mnodes |= mnode_mask; 11247c478bd9Sstevel@tonic-gate lgrp->lgrp_nmnodes++; 11257c478bd9Sstevel@tonic-gate } 11267c478bd9Sstevel@tonic-gate } 11277c478bd9Sstevel@tonic-gate count++; 11287c478bd9Sstevel@tonic-gate if (changed) 11297c478bd9Sstevel@tonic-gate klgrpset_add(*changed, lgrp->lgrp_id); 11307c478bd9Sstevel@tonic-gate } 11317c478bd9Sstevel@tonic-gate } 11327c478bd9Sstevel@tonic-gate 11337c478bd9Sstevel@tonic-gate return (count); 11347c478bd9Sstevel@tonic-gate } 11357c478bd9Sstevel@tonic-gate 11367c478bd9Sstevel@tonic-gate /* 11377c478bd9Sstevel@tonic-gate * Memory copy-rename. Called when the "mnode" containing the kernel cage memory 11387c478bd9Sstevel@tonic-gate * is moved from one board to another. The "from" and "to" arguments specify the 11397c478bd9Sstevel@tonic-gate * source and the destination of the move. 11407c478bd9Sstevel@tonic-gate * 11417c478bd9Sstevel@tonic-gate * See plat_lgrp_config() for a detailed description of the copy-rename 11427c478bd9Sstevel@tonic-gate * semantics. 11437c478bd9Sstevel@tonic-gate * 11447c478bd9Sstevel@tonic-gate * The lgrp_mem_rename() is called by the platform copy-rename code to update 11457c478bd9Sstevel@tonic-gate * the lgroup topology which is changing as memory moves from one lgroup to 11467c478bd9Sstevel@tonic-gate * another. It removes the mnode from the source lgroup and re-inserts it in the 11477c478bd9Sstevel@tonic-gate * target lgroup. 11487c478bd9Sstevel@tonic-gate * 11497c478bd9Sstevel@tonic-gate * The lgrp_mem_rename() function passes a flag to lgrp_mem_init() and 11507c478bd9Sstevel@tonic-gate * lgrp_mem_fini() telling that the insertion and deleteion are part of a DR 11517c478bd9Sstevel@tonic-gate * copy-rename operation. 11527c478bd9Sstevel@tonic-gate * 11537c478bd9Sstevel@tonic-gate * There is one case which requires special handling. If the system contains 11547c478bd9Sstevel@tonic-gate * only two boards (mnodes), the lgrp_mem_fini() removes the only mnode from the 11557c478bd9Sstevel@tonic-gate * lgroup hierarchy. This mnode is soon re-inserted back in the hierarchy by 11567c478bd9Sstevel@tonic-gate * lgrp_mem_init), but there is a window when the system has no memory in the 11577c478bd9Sstevel@tonic-gate * lgroup hierarchy. If another thread tries to allocate memory during this 11587c478bd9Sstevel@tonic-gate * window, the allocation will fail, although the system has physical memory. 11597c478bd9Sstevel@tonic-gate * This may cause a system panic or a deadlock (some sleeping memory allocations 11607c478bd9Sstevel@tonic-gate * happen with cpu_lock held which prevents lgrp_mem_init() from re-inserting 11617c478bd9Sstevel@tonic-gate * the mnode back). 11627c478bd9Sstevel@tonic-gate * 11637c478bd9Sstevel@tonic-gate * The lgrp_memnode_choose() function walks the lgroup hierarchy looking for the 11647c478bd9Sstevel@tonic-gate * lgrp with non-empty lgrp_mnodes. To deal with the special case above, 11657c478bd9Sstevel@tonic-gate * lgrp_mem_fini() does not remove the last mnode from the lroot->lgrp_mnodes, 11667c478bd9Sstevel@tonic-gate * but it updates the rest of the lgroup topology as if the mnode was actually 11677c478bd9Sstevel@tonic-gate * removed. The lgrp_mem_init() function recognizes that the mnode being 11687c478bd9Sstevel@tonic-gate * inserted represents such a special case and updates the topology 11697c478bd9Sstevel@tonic-gate * appropriately. 11707c478bd9Sstevel@tonic-gate */ 11717c478bd9Sstevel@tonic-gate void 11727c478bd9Sstevel@tonic-gate lgrp_mem_rename(int mnode, lgrp_handle_t from, lgrp_handle_t to) 11737c478bd9Sstevel@tonic-gate { 11747c478bd9Sstevel@tonic-gate /* 11757c478bd9Sstevel@tonic-gate * Remove the memory from the source node and add it to the destination 11767c478bd9Sstevel@tonic-gate * node. 11777c478bd9Sstevel@tonic-gate */ 11787c478bd9Sstevel@tonic-gate lgrp_mem_fini(mnode, from, B_TRUE); 11797c478bd9Sstevel@tonic-gate lgrp_mem_init(mnode, to, B_TRUE); 11807c478bd9Sstevel@tonic-gate } 11817c478bd9Sstevel@tonic-gate 11827c478bd9Sstevel@tonic-gate /* 11837c478bd9Sstevel@tonic-gate * Called to indicate that the lgrp with platform handle "hand" now 11847c478bd9Sstevel@tonic-gate * contains the memory identified by "mnode". 11857c478bd9Sstevel@tonic-gate * 11867c478bd9Sstevel@tonic-gate * LOCKING for this routine is a bit tricky. Usually it is called without 11877c478bd9Sstevel@tonic-gate * cpu_lock and it must must grab cpu_lock here to prevent racing with other 11887c478bd9Sstevel@tonic-gate * callers. During DR of the board containing the caged memory it may be called 11897c478bd9Sstevel@tonic-gate * with cpu_lock already held and CPUs paused. 11907c478bd9Sstevel@tonic-gate * 11917c478bd9Sstevel@tonic-gate * If the insertion is part of the DR copy-rename and the inserted mnode (and 11927c478bd9Sstevel@tonic-gate * only this mnode) is already present in the lgrp_root->lgrp_mnodes set, we are 11937c478bd9Sstevel@tonic-gate * dealing with the special case of DR copy-rename described in 11947c478bd9Sstevel@tonic-gate * lgrp_mem_rename(). 11957c478bd9Sstevel@tonic-gate */ 11967c478bd9Sstevel@tonic-gate void 11977c478bd9Sstevel@tonic-gate lgrp_mem_init(int mnode, lgrp_handle_t hand, boolean_t is_copy_rename) 11987c478bd9Sstevel@tonic-gate { 11997c478bd9Sstevel@tonic-gate klgrpset_t changed; 12007c478bd9Sstevel@tonic-gate int count; 12017c478bd9Sstevel@tonic-gate int i; 12027c478bd9Sstevel@tonic-gate lgrp_t *my_lgrp; 12037c478bd9Sstevel@tonic-gate lgrp_id_t lgrpid; 12047c478bd9Sstevel@tonic-gate mnodeset_t mnodes_mask = ((mnodeset_t)1 << mnode); 12057c478bd9Sstevel@tonic-gate boolean_t drop_lock = B_FALSE; 12067c478bd9Sstevel@tonic-gate boolean_t need_synch = B_FALSE; 12077c478bd9Sstevel@tonic-gate 12087c478bd9Sstevel@tonic-gate /* 12097c478bd9Sstevel@tonic-gate * Grab CPU lock (if we haven't already) 12107c478bd9Sstevel@tonic-gate */ 12117c478bd9Sstevel@tonic-gate if (!MUTEX_HELD(&cpu_lock)) { 12127c478bd9Sstevel@tonic-gate mutex_enter(&cpu_lock); 12137c478bd9Sstevel@tonic-gate drop_lock = B_TRUE; 12147c478bd9Sstevel@tonic-gate } 12157c478bd9Sstevel@tonic-gate 12167c478bd9Sstevel@tonic-gate /* 12177c478bd9Sstevel@tonic-gate * This routine may be called from a context where we already 12187c478bd9Sstevel@tonic-gate * hold cpu_lock, and have already paused cpus. 12197c478bd9Sstevel@tonic-gate */ 12207c478bd9Sstevel@tonic-gate if (!cpus_paused()) 12217c478bd9Sstevel@tonic-gate need_synch = B_TRUE; 12227c478bd9Sstevel@tonic-gate 12237c478bd9Sstevel@tonic-gate /* 12247c478bd9Sstevel@tonic-gate * Check if this mnode is already configured and return immediately if 12257c478bd9Sstevel@tonic-gate * it is. 12267c478bd9Sstevel@tonic-gate * 12277c478bd9Sstevel@tonic-gate * NOTE: in special case of copy-rename of the only remaining mnode, 12287c478bd9Sstevel@tonic-gate * lgrp_mem_fini() refuses to remove the last mnode from the root, so we 12297c478bd9Sstevel@tonic-gate * recognize this case and continue as usual, but skip the update to 12307c478bd9Sstevel@tonic-gate * the lgrp_mnodes and the lgrp_nmnodes. This restores the inconsistency 12317c478bd9Sstevel@tonic-gate * in topology, temporarily introduced by lgrp_mem_fini(). 12327c478bd9Sstevel@tonic-gate */ 12337c478bd9Sstevel@tonic-gate if (! (is_copy_rename && (lgrp_root->lgrp_mnodes == mnodes_mask)) && 12347c478bd9Sstevel@tonic-gate lgrp_root->lgrp_mnodes & mnodes_mask) { 12357c478bd9Sstevel@tonic-gate if (drop_lock) 12367c478bd9Sstevel@tonic-gate mutex_exit(&cpu_lock); 12377c478bd9Sstevel@tonic-gate return; 12387c478bd9Sstevel@tonic-gate } 12397c478bd9Sstevel@tonic-gate 12407c478bd9Sstevel@tonic-gate /* 12417c478bd9Sstevel@tonic-gate * Update lgroup topology with new memory resources, keeping track of 12427c478bd9Sstevel@tonic-gate * which lgroups change 12437c478bd9Sstevel@tonic-gate */ 12447c478bd9Sstevel@tonic-gate count = 0; 12457c478bd9Sstevel@tonic-gate klgrpset_clear(changed); 12467c478bd9Sstevel@tonic-gate my_lgrp = lgrp_hand_to_lgrp(hand); 12477c478bd9Sstevel@tonic-gate if (my_lgrp == NULL) { 12487c478bd9Sstevel@tonic-gate /* new lgrp */ 12497c478bd9Sstevel@tonic-gate my_lgrp = lgrp_create(); 12507c478bd9Sstevel@tonic-gate lgrpid = my_lgrp->lgrp_id; 12517c478bd9Sstevel@tonic-gate my_lgrp->lgrp_plathand = hand; 12527c478bd9Sstevel@tonic-gate my_lgrp->lgrp_latency = lgrp_plat_latency(hand, hand); 12537c478bd9Sstevel@tonic-gate klgrpset_add(my_lgrp->lgrp_leaves, lgrpid); 12547c478bd9Sstevel@tonic-gate klgrpset_add(my_lgrp->lgrp_set[LGRP_RSRC_MEM], lgrpid); 12557c478bd9Sstevel@tonic-gate 12567c478bd9Sstevel@tonic-gate if (need_synch) 12577c478bd9Sstevel@tonic-gate pause_cpus(NULL); 12587c478bd9Sstevel@tonic-gate count = lgrp_leaf_add(my_lgrp, lgrp_table, lgrp_alloc_max + 1, 12597c478bd9Sstevel@tonic-gate &changed); 12607c478bd9Sstevel@tonic-gate if (need_synch) 12617c478bd9Sstevel@tonic-gate start_cpus(); 12627c478bd9Sstevel@tonic-gate } else if (my_lgrp->lgrp_latency == 0 && lgrp_plat_latency(hand, hand) 12637c478bd9Sstevel@tonic-gate > 0) { 12647c478bd9Sstevel@tonic-gate /* 12657c478bd9Sstevel@tonic-gate * Leaf lgroup was created, but latency wasn't available 12667c478bd9Sstevel@tonic-gate * then. So, set latency for it and fill in rest of lgroup 12677c478bd9Sstevel@tonic-gate * topology now that we know how far it is from other leaf 12687c478bd9Sstevel@tonic-gate * lgroups. 12697c478bd9Sstevel@tonic-gate */ 12707c478bd9Sstevel@tonic-gate klgrpset_clear(changed); 12717c478bd9Sstevel@tonic-gate lgrpid = my_lgrp->lgrp_id; 12727c478bd9Sstevel@tonic-gate if (!klgrpset_ismember(my_lgrp->lgrp_set[LGRP_RSRC_MEM], 12737c478bd9Sstevel@tonic-gate lgrpid)) 12747c478bd9Sstevel@tonic-gate klgrpset_add(my_lgrp->lgrp_set[LGRP_RSRC_MEM], lgrpid); 12757c478bd9Sstevel@tonic-gate if (need_synch) 12767c478bd9Sstevel@tonic-gate pause_cpus(NULL); 12777c478bd9Sstevel@tonic-gate count = lgrp_leaf_add(my_lgrp, lgrp_table, lgrp_alloc_max + 1, 12787c478bd9Sstevel@tonic-gate &changed); 12797c478bd9Sstevel@tonic-gate if (need_synch) 12807c478bd9Sstevel@tonic-gate start_cpus(); 12817c478bd9Sstevel@tonic-gate } else if (!klgrpset_ismember(my_lgrp->lgrp_set[LGRP_RSRC_MEM], 12827c478bd9Sstevel@tonic-gate my_lgrp->lgrp_id)) { 1283*2dae3fb5Sjjc /* 1284*2dae3fb5Sjjc * Add new lgroup memory resource to existing lgroup 1285*2dae3fb5Sjjc */ 12867c478bd9Sstevel@tonic-gate lgrpid = my_lgrp->lgrp_id; 12877c478bd9Sstevel@tonic-gate klgrpset_add(my_lgrp->lgrp_set[LGRP_RSRC_MEM], lgrpid); 12887c478bd9Sstevel@tonic-gate klgrpset_add(changed, lgrpid); 12897c478bd9Sstevel@tonic-gate count++; 12907c478bd9Sstevel@tonic-gate for (i = 0; i <= lgrp_alloc_max; i++) { 12917c478bd9Sstevel@tonic-gate lgrp_t *lgrp; 12927c478bd9Sstevel@tonic-gate 12937c478bd9Sstevel@tonic-gate lgrp = lgrp_table[i]; 12947c478bd9Sstevel@tonic-gate if (!LGRP_EXISTS(lgrp) || 12957c478bd9Sstevel@tonic-gate !lgrp_rsets_member(lgrp->lgrp_set, lgrpid)) 12967c478bd9Sstevel@tonic-gate continue; 12977c478bd9Sstevel@tonic-gate 12987c478bd9Sstevel@tonic-gate klgrpset_add(lgrp->lgrp_set[LGRP_RSRC_MEM], lgrpid); 12997c478bd9Sstevel@tonic-gate klgrpset_add(changed, lgrp->lgrp_id); 13007c478bd9Sstevel@tonic-gate count++; 13017c478bd9Sstevel@tonic-gate } 13027c478bd9Sstevel@tonic-gate } 13037c478bd9Sstevel@tonic-gate 13047c478bd9Sstevel@tonic-gate /* 13057c478bd9Sstevel@tonic-gate * Add memory node to lgroup and remove lgroup from ones that need 13067c478bd9Sstevel@tonic-gate * to be updated 13077c478bd9Sstevel@tonic-gate */ 13087c478bd9Sstevel@tonic-gate if (!(my_lgrp->lgrp_mnodes & mnodes_mask)) { 13097c478bd9Sstevel@tonic-gate my_lgrp->lgrp_mnodes |= mnodes_mask; 13107c478bd9Sstevel@tonic-gate my_lgrp->lgrp_nmnodes++; 13117c478bd9Sstevel@tonic-gate } 13127c478bd9Sstevel@tonic-gate klgrpset_del(changed, lgrpid); 13137c478bd9Sstevel@tonic-gate 13147c478bd9Sstevel@tonic-gate /* 13157c478bd9Sstevel@tonic-gate * Update memory node information for all lgroups that changed and 13167c478bd9Sstevel@tonic-gate * contain new memory node as a resource 13177c478bd9Sstevel@tonic-gate */ 13187c478bd9Sstevel@tonic-gate if (count) 13197c478bd9Sstevel@tonic-gate (void) lgrp_mnode_update(changed, NULL); 13207c478bd9Sstevel@tonic-gate 13217c478bd9Sstevel@tonic-gate if (drop_lock) 13227c478bd9Sstevel@tonic-gate mutex_exit(&cpu_lock); 13237c478bd9Sstevel@tonic-gate } 13247c478bd9Sstevel@tonic-gate 13257c478bd9Sstevel@tonic-gate /* 13267c478bd9Sstevel@tonic-gate * Called to indicate that the lgroup associated with the platform 13277c478bd9Sstevel@tonic-gate * handle "hand" no longer contains given memory node 13287c478bd9Sstevel@tonic-gate * 13297c478bd9Sstevel@tonic-gate * LOCKING for this routine is a bit tricky. Usually it is called without 13307c478bd9Sstevel@tonic-gate * cpu_lock and it must must grab cpu_lock here to prevent racing with other 13317c478bd9Sstevel@tonic-gate * callers. During DR of the board containing the caged memory it may be called 13327c478bd9Sstevel@tonic-gate * with cpu_lock already held and CPUs paused. 13337c478bd9Sstevel@tonic-gate * 13347c478bd9Sstevel@tonic-gate * If the deletion is part of the DR copy-rename and the deleted mnode is the 13357c478bd9Sstevel@tonic-gate * only one present in the lgrp_root->lgrp_mnodes, all the topology is updated, 13367c478bd9Sstevel@tonic-gate * but lgrp_root->lgrp_mnodes is left intact. Later, lgrp_mem_init() will insert 13377c478bd9Sstevel@tonic-gate * the same mnode back into the topology. See lgrp_mem_rename() and 13387c478bd9Sstevel@tonic-gate * lgrp_mem_init() for additional details. 13397c478bd9Sstevel@tonic-gate */ 13407c478bd9Sstevel@tonic-gate void 13417c478bd9Sstevel@tonic-gate lgrp_mem_fini(int mnode, lgrp_handle_t hand, boolean_t is_copy_rename) 13427c478bd9Sstevel@tonic-gate { 13437c478bd9Sstevel@tonic-gate klgrpset_t changed; 13447c478bd9Sstevel@tonic-gate int count; 13457c478bd9Sstevel@tonic-gate int i; 13467c478bd9Sstevel@tonic-gate lgrp_t *my_lgrp; 13477c478bd9Sstevel@tonic-gate lgrp_id_t lgrpid; 13487c478bd9Sstevel@tonic-gate mnodeset_t mnodes_mask; 13497c478bd9Sstevel@tonic-gate boolean_t drop_lock = B_FALSE; 13507c478bd9Sstevel@tonic-gate boolean_t need_synch = B_FALSE; 13517c478bd9Sstevel@tonic-gate 13527c478bd9Sstevel@tonic-gate /* 13537c478bd9Sstevel@tonic-gate * Grab CPU lock (if we haven't already) 13547c478bd9Sstevel@tonic-gate */ 13557c478bd9Sstevel@tonic-gate if (!MUTEX_HELD(&cpu_lock)) { 13567c478bd9Sstevel@tonic-gate mutex_enter(&cpu_lock); 13577c478bd9Sstevel@tonic-gate drop_lock = B_TRUE; 13587c478bd9Sstevel@tonic-gate } 13597c478bd9Sstevel@tonic-gate 13607c478bd9Sstevel@tonic-gate /* 13617c478bd9Sstevel@tonic-gate * This routine may be called from a context where we already 13627c478bd9Sstevel@tonic-gate * hold cpu_lock and have already paused cpus. 13637c478bd9Sstevel@tonic-gate */ 13647c478bd9Sstevel@tonic-gate if (!cpus_paused()) 13657c478bd9Sstevel@tonic-gate need_synch = B_TRUE; 13667c478bd9Sstevel@tonic-gate 13677c478bd9Sstevel@tonic-gate my_lgrp = lgrp_hand_to_lgrp(hand); 13687c478bd9Sstevel@tonic-gate 13697c478bd9Sstevel@tonic-gate /* 13707c478bd9Sstevel@tonic-gate * The lgrp *must* be pre-existing 13717c478bd9Sstevel@tonic-gate */ 13727c478bd9Sstevel@tonic-gate ASSERT(my_lgrp != NULL); 13737c478bd9Sstevel@tonic-gate 13747c478bd9Sstevel@tonic-gate /* 13757c478bd9Sstevel@tonic-gate * Delete memory node from lgroups which contain it 13767c478bd9Sstevel@tonic-gate */ 13777c478bd9Sstevel@tonic-gate mnodes_mask = ((mnodeset_t)1 << mnode); 13787c478bd9Sstevel@tonic-gate for (i = 0; i <= lgrp_alloc_max; i++) { 13797c478bd9Sstevel@tonic-gate lgrp_t *lgrp = lgrp_table[i]; 13807c478bd9Sstevel@tonic-gate /* 13817c478bd9Sstevel@tonic-gate * Skip any non-existent lgroups and any lgroups that don't 13827c478bd9Sstevel@tonic-gate * contain leaf lgroup of memory as a memory resource 13837c478bd9Sstevel@tonic-gate */ 13847c478bd9Sstevel@tonic-gate if (!LGRP_EXISTS(lgrp) || 13857c478bd9Sstevel@tonic-gate !(lgrp->lgrp_mnodes & mnodes_mask)) 13867c478bd9Sstevel@tonic-gate continue; 13877c478bd9Sstevel@tonic-gate 13887c478bd9Sstevel@tonic-gate /* 13897c478bd9Sstevel@tonic-gate * Avoid removing the last mnode from the root in the DR 13907c478bd9Sstevel@tonic-gate * copy-rename case. See lgrp_mem_rename() for details. 13917c478bd9Sstevel@tonic-gate */ 13927c478bd9Sstevel@tonic-gate if (is_copy_rename && 13937c478bd9Sstevel@tonic-gate (lgrp == lgrp_root) && (lgrp->lgrp_mnodes == mnodes_mask)) 13947c478bd9Sstevel@tonic-gate continue; 13957c478bd9Sstevel@tonic-gate 13967c478bd9Sstevel@tonic-gate /* 13977c478bd9Sstevel@tonic-gate * Remove memory node from lgroup. 13987c478bd9Sstevel@tonic-gate */ 13997c478bd9Sstevel@tonic-gate lgrp->lgrp_mnodes &= ~mnodes_mask; 14007c478bd9Sstevel@tonic-gate lgrp->lgrp_nmnodes--; 14017c478bd9Sstevel@tonic-gate ASSERT(lgrp->lgrp_nmnodes >= 0); 14027c478bd9Sstevel@tonic-gate } 14037c478bd9Sstevel@tonic-gate ASSERT(lgrp_root->lgrp_nmnodes > 0); 14047c478bd9Sstevel@tonic-gate 14057c478bd9Sstevel@tonic-gate /* 14067c478bd9Sstevel@tonic-gate * Don't need to update lgroup topology if this lgroup still has memory. 14077c478bd9Sstevel@tonic-gate * 14087c478bd9Sstevel@tonic-gate * In the special case of DR copy-rename with the only mnode being 14097c478bd9Sstevel@tonic-gate * removed, the lgrp_mnodes for the root is always non-zero, but we 14107c478bd9Sstevel@tonic-gate * still need to update the lgroup topology. 14117c478bd9Sstevel@tonic-gate */ 14127c478bd9Sstevel@tonic-gate if ((my_lgrp->lgrp_nmnodes > 0) && 14137c478bd9Sstevel@tonic-gate !(is_copy_rename && 14147c478bd9Sstevel@tonic-gate (my_lgrp == lgrp_root) && 14157c478bd9Sstevel@tonic-gate (my_lgrp->lgrp_mnodes == mnodes_mask))) { 14167c478bd9Sstevel@tonic-gate if (drop_lock) 14177c478bd9Sstevel@tonic-gate mutex_exit(&cpu_lock); 14187c478bd9Sstevel@tonic-gate return; 14197c478bd9Sstevel@tonic-gate } 14207c478bd9Sstevel@tonic-gate 14217c478bd9Sstevel@tonic-gate /* 14227c478bd9Sstevel@tonic-gate * This lgroup does not contain any memory now 14237c478bd9Sstevel@tonic-gate */ 14247c478bd9Sstevel@tonic-gate klgrpset_clear(my_lgrp->lgrp_set[LGRP_RSRC_MEM]); 14257c478bd9Sstevel@tonic-gate 14267c478bd9Sstevel@tonic-gate /* 14277c478bd9Sstevel@tonic-gate * Remove this lgroup from lgroup topology if it does not contain any 14287c478bd9Sstevel@tonic-gate * resources now 14297c478bd9Sstevel@tonic-gate */ 14307c478bd9Sstevel@tonic-gate lgrpid = my_lgrp->lgrp_id; 14317c478bd9Sstevel@tonic-gate count = 0; 14327c478bd9Sstevel@tonic-gate klgrpset_clear(changed); 14337c478bd9Sstevel@tonic-gate if (lgrp_rsets_empty(my_lgrp->lgrp_set)) { 14347c478bd9Sstevel@tonic-gate /* 14357c478bd9Sstevel@tonic-gate * Delete lgroup when no more resources 14367c478bd9Sstevel@tonic-gate */ 14377c478bd9Sstevel@tonic-gate if (need_synch) 14387c478bd9Sstevel@tonic-gate pause_cpus(NULL); 14397c478bd9Sstevel@tonic-gate count = lgrp_leaf_delete(my_lgrp, lgrp_table, 14407c478bd9Sstevel@tonic-gate lgrp_alloc_max + 1, &changed); 14417c478bd9Sstevel@tonic-gate ASSERT(count > 0); 14427c478bd9Sstevel@tonic-gate if (need_synch) 14437c478bd9Sstevel@tonic-gate start_cpus(); 14447c478bd9Sstevel@tonic-gate } else { 14457c478bd9Sstevel@tonic-gate /* 14467c478bd9Sstevel@tonic-gate * Remove lgroup from memory resources of any lgroups that 14477c478bd9Sstevel@tonic-gate * contain it as such 14487c478bd9Sstevel@tonic-gate */ 14497c478bd9Sstevel@tonic-gate for (i = 0; i <= lgrp_alloc_max; i++) { 14507c478bd9Sstevel@tonic-gate lgrp_t *lgrp; 14517c478bd9Sstevel@tonic-gate 14527c478bd9Sstevel@tonic-gate lgrp = lgrp_table[i]; 14537c478bd9Sstevel@tonic-gate if (!LGRP_EXISTS(lgrp) || 14547c478bd9Sstevel@tonic-gate !klgrpset_ismember(lgrp->lgrp_set[LGRP_RSRC_MEM], 14557c478bd9Sstevel@tonic-gate lgrpid)) 14567c478bd9Sstevel@tonic-gate continue; 14577c478bd9Sstevel@tonic-gate 14587c478bd9Sstevel@tonic-gate klgrpset_del(lgrp->lgrp_set[LGRP_RSRC_MEM], lgrpid); 14597c478bd9Sstevel@tonic-gate } 14607c478bd9Sstevel@tonic-gate } 14617c478bd9Sstevel@tonic-gate if (drop_lock) 14627c478bd9Sstevel@tonic-gate mutex_exit(&cpu_lock); 14637c478bd9Sstevel@tonic-gate } 14647c478bd9Sstevel@tonic-gate 14657c478bd9Sstevel@tonic-gate /* 14667c478bd9Sstevel@tonic-gate * Return lgroup with given platform handle 14677c478bd9Sstevel@tonic-gate */ 14687c478bd9Sstevel@tonic-gate lgrp_t * 14697c478bd9Sstevel@tonic-gate lgrp_hand_to_lgrp(lgrp_handle_t hand) 14707c478bd9Sstevel@tonic-gate { 14717c478bd9Sstevel@tonic-gate int i; 14727c478bd9Sstevel@tonic-gate lgrp_t *lgrp; 14737c478bd9Sstevel@tonic-gate 14747c478bd9Sstevel@tonic-gate if (hand == LGRP_NULL_HANDLE) 14757c478bd9Sstevel@tonic-gate return (NULL); 14767c478bd9Sstevel@tonic-gate 14777c478bd9Sstevel@tonic-gate for (i = 0; i <= lgrp_alloc_max; i++) { 14787c478bd9Sstevel@tonic-gate lgrp = lgrp_table[i]; 14797c478bd9Sstevel@tonic-gate if (LGRP_EXISTS(lgrp) && lgrp->lgrp_plathand == hand) 14807c478bd9Sstevel@tonic-gate return (lgrp); 14817c478bd9Sstevel@tonic-gate } 14827c478bd9Sstevel@tonic-gate return (NULL); 14837c478bd9Sstevel@tonic-gate } 14847c478bd9Sstevel@tonic-gate 14857c478bd9Sstevel@tonic-gate /* 14867c478bd9Sstevel@tonic-gate * Return the home lgroup of the current thread. 14877c478bd9Sstevel@tonic-gate * We must do this with kernel preemption disabled, since we don't want our 14887c478bd9Sstevel@tonic-gate * thread to be re-homed while we're poking around with its lpl, and the lpl 14897c478bd9Sstevel@tonic-gate * should never be NULL. 14907c478bd9Sstevel@tonic-gate * 14917c478bd9Sstevel@tonic-gate * NOTE: Can't guarantee that lgroup will be valid once kernel preemption 14927c478bd9Sstevel@tonic-gate * is enabled because of DR. Callers can use disable kernel preemption 14937c478bd9Sstevel@tonic-gate * around this call to guarantee that the lgroup will be valid beyond this 14947c478bd9Sstevel@tonic-gate * routine, since kernel preemption can be recursive. 14957c478bd9Sstevel@tonic-gate */ 14967c478bd9Sstevel@tonic-gate lgrp_t * 14977c478bd9Sstevel@tonic-gate lgrp_home_lgrp(void) 14987c478bd9Sstevel@tonic-gate { 14997c478bd9Sstevel@tonic-gate lgrp_t *lgrp; 15007c478bd9Sstevel@tonic-gate lpl_t *lpl; 15017c478bd9Sstevel@tonic-gate 15027c478bd9Sstevel@tonic-gate kpreempt_disable(); 15037c478bd9Sstevel@tonic-gate 15047c478bd9Sstevel@tonic-gate lpl = curthread->t_lpl; 15057c478bd9Sstevel@tonic-gate ASSERT(lpl != NULL); 15067c478bd9Sstevel@tonic-gate ASSERT(lpl->lpl_lgrpid >= 0 && lpl->lpl_lgrpid <= lgrp_alloc_max); 15077c478bd9Sstevel@tonic-gate ASSERT(LGRP_EXISTS(lgrp_table[lpl->lpl_lgrpid])); 15087c478bd9Sstevel@tonic-gate lgrp = lgrp_table[lpl->lpl_lgrpid]; 15097c478bd9Sstevel@tonic-gate 15107c478bd9Sstevel@tonic-gate kpreempt_enable(); 15117c478bd9Sstevel@tonic-gate 15127c478bd9Sstevel@tonic-gate return (lgrp); 15137c478bd9Sstevel@tonic-gate } 15147c478bd9Sstevel@tonic-gate 15157c478bd9Sstevel@tonic-gate /* 15167c478bd9Sstevel@tonic-gate * Return ID of home lgroup for given thread 15177c478bd9Sstevel@tonic-gate * (See comments for lgrp_home_lgrp() for special care and handling 15187c478bd9Sstevel@tonic-gate * instructions) 15197c478bd9Sstevel@tonic-gate */ 15207c478bd9Sstevel@tonic-gate lgrp_id_t 15217c478bd9Sstevel@tonic-gate lgrp_home_id(kthread_t *t) 15227c478bd9Sstevel@tonic-gate { 15237c478bd9Sstevel@tonic-gate lgrp_id_t lgrp; 15247c478bd9Sstevel@tonic-gate lpl_t *lpl; 15257c478bd9Sstevel@tonic-gate 15267c478bd9Sstevel@tonic-gate ASSERT(t != NULL); 15277c478bd9Sstevel@tonic-gate /* 15287c478bd9Sstevel@tonic-gate * We'd like to ASSERT(MUTEX_HELD(&ttoproc(t)->p_lock)), but we 15297c478bd9Sstevel@tonic-gate * cannot since the HAT layer can call into this routine to 15307c478bd9Sstevel@tonic-gate * determine the locality for its data structures in the context 15317c478bd9Sstevel@tonic-gate * of a page fault. 15327c478bd9Sstevel@tonic-gate */ 15337c478bd9Sstevel@tonic-gate 15347c478bd9Sstevel@tonic-gate kpreempt_disable(); 15357c478bd9Sstevel@tonic-gate 15367c478bd9Sstevel@tonic-gate lpl = t->t_lpl; 15377c478bd9Sstevel@tonic-gate ASSERT(lpl != NULL); 15387c478bd9Sstevel@tonic-gate ASSERT(lpl->lpl_lgrpid >= 0 && lpl->lpl_lgrpid <= lgrp_alloc_max); 15397c478bd9Sstevel@tonic-gate lgrp = lpl->lpl_lgrpid; 15407c478bd9Sstevel@tonic-gate 15417c478bd9Sstevel@tonic-gate kpreempt_enable(); 15427c478bd9Sstevel@tonic-gate 15437c478bd9Sstevel@tonic-gate return (lgrp); 15447c478bd9Sstevel@tonic-gate } 15457c478bd9Sstevel@tonic-gate 15467c478bd9Sstevel@tonic-gate /* 15477c478bd9Sstevel@tonic-gate * Return lgroup containing the physical memory for the given page frame number 15487c478bd9Sstevel@tonic-gate */ 15497c478bd9Sstevel@tonic-gate lgrp_t * 15507c478bd9Sstevel@tonic-gate lgrp_pfn_to_lgrp(pfn_t pfn) 15517c478bd9Sstevel@tonic-gate { 15527c478bd9Sstevel@tonic-gate lgrp_handle_t hand; 15537c478bd9Sstevel@tonic-gate int i; 15547c478bd9Sstevel@tonic-gate lgrp_t *lgrp; 15557c478bd9Sstevel@tonic-gate 15567c478bd9Sstevel@tonic-gate hand = lgrp_plat_pfn_to_hand(pfn); 15577c478bd9Sstevel@tonic-gate if (hand != LGRP_NULL_HANDLE) 15587c478bd9Sstevel@tonic-gate for (i = 0; i <= lgrp_alloc_max; i++) { 15597c478bd9Sstevel@tonic-gate lgrp = lgrp_table[i]; 15607c478bd9Sstevel@tonic-gate if (LGRP_EXISTS(lgrp) && lgrp->lgrp_plathand == hand) 15617c478bd9Sstevel@tonic-gate return (lgrp); 15627c478bd9Sstevel@tonic-gate } 15637c478bd9Sstevel@tonic-gate return (NULL); 15647c478bd9Sstevel@tonic-gate } 15657c478bd9Sstevel@tonic-gate 15667c478bd9Sstevel@tonic-gate /* 15677c478bd9Sstevel@tonic-gate * Return lgroup containing the physical memory for the given page frame number 15687c478bd9Sstevel@tonic-gate */ 15697c478bd9Sstevel@tonic-gate lgrp_t * 15707c478bd9Sstevel@tonic-gate lgrp_phys_to_lgrp(u_longlong_t physaddr) 15717c478bd9Sstevel@tonic-gate { 15727c478bd9Sstevel@tonic-gate lgrp_handle_t hand; 15737c478bd9Sstevel@tonic-gate int i; 15747c478bd9Sstevel@tonic-gate lgrp_t *lgrp; 15757c478bd9Sstevel@tonic-gate pfn_t pfn; 15767c478bd9Sstevel@tonic-gate 15777c478bd9Sstevel@tonic-gate pfn = btop(physaddr); 15787c478bd9Sstevel@tonic-gate hand = lgrp_plat_pfn_to_hand(pfn); 15797c478bd9Sstevel@tonic-gate if (hand != LGRP_NULL_HANDLE) 15807c478bd9Sstevel@tonic-gate for (i = 0; i <= lgrp_alloc_max; i++) { 15817c478bd9Sstevel@tonic-gate lgrp = lgrp_table[i]; 15827c478bd9Sstevel@tonic-gate if (LGRP_EXISTS(lgrp) && lgrp->lgrp_plathand == hand) 15837c478bd9Sstevel@tonic-gate return (lgrp); 15847c478bd9Sstevel@tonic-gate } 15857c478bd9Sstevel@tonic-gate return (NULL); 15867c478bd9Sstevel@tonic-gate } 15877c478bd9Sstevel@tonic-gate 15887c478bd9Sstevel@tonic-gate /* 15897c478bd9Sstevel@tonic-gate * Return the leaf lgroup containing the given CPU 15907c478bd9Sstevel@tonic-gate */ 15917c478bd9Sstevel@tonic-gate static lgrp_t * 15927c478bd9Sstevel@tonic-gate lgrp_cpu_to_lgrp(cpu_t *cpu) 15937c478bd9Sstevel@tonic-gate { 15947c478bd9Sstevel@tonic-gate return (cpu->cpu_chip->chip_lgrp); 15957c478bd9Sstevel@tonic-gate } 15967c478bd9Sstevel@tonic-gate 15977c478bd9Sstevel@tonic-gate /* 15987c478bd9Sstevel@tonic-gate * Return the sum of the partition loads in an lgrp divided by 15997c478bd9Sstevel@tonic-gate * the number of CPUs in the lgrp. This is our best approximation 16007c478bd9Sstevel@tonic-gate * of an 'lgroup load average' for a useful per-lgroup kstat. 16017c478bd9Sstevel@tonic-gate */ 16027c478bd9Sstevel@tonic-gate static uint64_t 16037c478bd9Sstevel@tonic-gate lgrp_sum_loadavgs(lgrp_t *lgrp) 16047c478bd9Sstevel@tonic-gate { 16057c478bd9Sstevel@tonic-gate cpu_t *cpu; 16067c478bd9Sstevel@tonic-gate int ncpu; 16077c478bd9Sstevel@tonic-gate uint64_t loads = 0; 16087c478bd9Sstevel@tonic-gate 16097c478bd9Sstevel@tonic-gate mutex_enter(&cpu_lock); 16107c478bd9Sstevel@tonic-gate 16117c478bd9Sstevel@tonic-gate cpu = lgrp->lgrp_cpu; 16127c478bd9Sstevel@tonic-gate ncpu = lgrp->lgrp_cpucnt; 16137c478bd9Sstevel@tonic-gate 16147c478bd9Sstevel@tonic-gate if (cpu == NULL || ncpu == 0) { 16157c478bd9Sstevel@tonic-gate mutex_exit(&cpu_lock); 16167c478bd9Sstevel@tonic-gate return (0ull); 16177c478bd9Sstevel@tonic-gate } 16187c478bd9Sstevel@tonic-gate 16197c478bd9Sstevel@tonic-gate do { 16207c478bd9Sstevel@tonic-gate loads += cpu->cpu_lpl->lpl_loadavg; 16217c478bd9Sstevel@tonic-gate cpu = cpu->cpu_next_lgrp; 16227c478bd9Sstevel@tonic-gate } while (cpu != lgrp->lgrp_cpu); 16237c478bd9Sstevel@tonic-gate 16247c478bd9Sstevel@tonic-gate mutex_exit(&cpu_lock); 16257c478bd9Sstevel@tonic-gate 16267c478bd9Sstevel@tonic-gate return (loads / ncpu); 16277c478bd9Sstevel@tonic-gate } 16287c478bd9Sstevel@tonic-gate 16297c478bd9Sstevel@tonic-gate void 16307c478bd9Sstevel@tonic-gate lgrp_stat_add(lgrp_id_t lgrpid, lgrp_stat_t stat, int64_t val) 16317c478bd9Sstevel@tonic-gate { 16327c478bd9Sstevel@tonic-gate struct lgrp_stats *pstats; 16337c478bd9Sstevel@tonic-gate 16347c478bd9Sstevel@tonic-gate /* 16357c478bd9Sstevel@tonic-gate * Verify that the caller isn't trying to add to 16367c478bd9Sstevel@tonic-gate * a statistic for an lgroup that has gone away 16377c478bd9Sstevel@tonic-gate */ 16387c478bd9Sstevel@tonic-gate if (lgrpid < 0 || lgrpid > lgrp_alloc_max) 16397c478bd9Sstevel@tonic-gate return; 16407c478bd9Sstevel@tonic-gate 16417c478bd9Sstevel@tonic-gate pstats = &lgrp_stats[lgrpid]; 16427c478bd9Sstevel@tonic-gate atomic_add_64((uint64_t *)LGRP_STAT_WRITE_PTR(pstats, stat), val); 16437c478bd9Sstevel@tonic-gate } 16447c478bd9Sstevel@tonic-gate 16457c478bd9Sstevel@tonic-gate int64_t 16467c478bd9Sstevel@tonic-gate lgrp_stat_read(lgrp_id_t lgrpid, lgrp_stat_t stat) 16477c478bd9Sstevel@tonic-gate { 16487c478bd9Sstevel@tonic-gate uint64_t val; 16497c478bd9Sstevel@tonic-gate struct lgrp_stats *pstats; 16507c478bd9Sstevel@tonic-gate 16517c478bd9Sstevel@tonic-gate if (lgrpid < 0 || lgrpid > lgrp_alloc_max) 16527c478bd9Sstevel@tonic-gate return ((int64_t)0); 16537c478bd9Sstevel@tonic-gate 16547c478bd9Sstevel@tonic-gate pstats = &lgrp_stats[lgrpid]; 16557c478bd9Sstevel@tonic-gate LGRP_STAT_READ(pstats, stat, val); 16567c478bd9Sstevel@tonic-gate return (val); 16577c478bd9Sstevel@tonic-gate } 16587c478bd9Sstevel@tonic-gate 16597c478bd9Sstevel@tonic-gate /* 16607c478bd9Sstevel@tonic-gate * Reset all kstats for lgrp specified by its lgrpid. 16617c478bd9Sstevel@tonic-gate */ 16627c478bd9Sstevel@tonic-gate static void 16637c478bd9Sstevel@tonic-gate lgrp_kstat_reset(lgrp_id_t lgrpid) 16647c478bd9Sstevel@tonic-gate { 16657c478bd9Sstevel@tonic-gate lgrp_stat_t stat; 16667c478bd9Sstevel@tonic-gate 16677c478bd9Sstevel@tonic-gate if (lgrpid < 0 || lgrpid > lgrp_alloc_max) 16687c478bd9Sstevel@tonic-gate return; 16697c478bd9Sstevel@tonic-gate 16707c478bd9Sstevel@tonic-gate for (stat = 0; stat < LGRP_NUM_COUNTER_STATS; stat++) { 16717c478bd9Sstevel@tonic-gate LGRP_STAT_RESET(&lgrp_stats[lgrpid], stat); 16727c478bd9Sstevel@tonic-gate } 16737c478bd9Sstevel@tonic-gate } 16747c478bd9Sstevel@tonic-gate 16757c478bd9Sstevel@tonic-gate /* 16767c478bd9Sstevel@tonic-gate * Collect all per-lgrp statistics for the lgrp associated with this 16777c478bd9Sstevel@tonic-gate * kstat, and store them in the ks_data array. 16787c478bd9Sstevel@tonic-gate * 16797c478bd9Sstevel@tonic-gate * The superuser can reset all the running counter statistics for an 16807c478bd9Sstevel@tonic-gate * lgrp by writing to any of the lgrp's stats. 16817c478bd9Sstevel@tonic-gate */ 16827c478bd9Sstevel@tonic-gate static int 16837c478bd9Sstevel@tonic-gate lgrp_kstat_extract(kstat_t *ksp, int rw) 16847c478bd9Sstevel@tonic-gate { 16857c478bd9Sstevel@tonic-gate lgrp_stat_t stat; 16867c478bd9Sstevel@tonic-gate struct kstat_named *ksd; 16877c478bd9Sstevel@tonic-gate lgrp_t *lgrp; 16887c478bd9Sstevel@tonic-gate lgrp_id_t lgrpid; 16897c478bd9Sstevel@tonic-gate 16907c478bd9Sstevel@tonic-gate lgrp = (lgrp_t *)ksp->ks_private; 16917c478bd9Sstevel@tonic-gate 16927c478bd9Sstevel@tonic-gate ksd = (struct kstat_named *)ksp->ks_data; 16937c478bd9Sstevel@tonic-gate ASSERT(ksd == (struct kstat_named *)&lgrp_kstat_data); 16947c478bd9Sstevel@tonic-gate 16957c478bd9Sstevel@tonic-gate lgrpid = lgrp->lgrp_id; 16967c478bd9Sstevel@tonic-gate 16977c478bd9Sstevel@tonic-gate if (lgrpid == LGRP_NONE) { 16987c478bd9Sstevel@tonic-gate /* 16997c478bd9Sstevel@tonic-gate * Return all zeroes as stats for freed lgrp. 17007c478bd9Sstevel@tonic-gate */ 17017c478bd9Sstevel@tonic-gate for (stat = 0; stat < LGRP_NUM_COUNTER_STATS; stat++) { 17027c478bd9Sstevel@tonic-gate ksd[stat].value.i64 = 0; 17037c478bd9Sstevel@tonic-gate } 17047c478bd9Sstevel@tonic-gate ksd[stat + LGRP_NUM_CPUS].value.i64 = 0; 17057c478bd9Sstevel@tonic-gate ksd[stat + LGRP_NUM_PG_INSTALL].value.i64 = 0; 17067c478bd9Sstevel@tonic-gate ksd[stat + LGRP_NUM_PG_AVAIL].value.i64 = 0; 17077c478bd9Sstevel@tonic-gate ksd[stat + LGRP_NUM_PG_FREE].value.i64 = 0; 17087c478bd9Sstevel@tonic-gate ksd[stat + LGRP_LOADAVG].value.i64 = 0; 17097c478bd9Sstevel@tonic-gate } else if (rw != KSTAT_WRITE) { 17107c478bd9Sstevel@tonic-gate /* 17117c478bd9Sstevel@tonic-gate * Handle counter stats 17127c478bd9Sstevel@tonic-gate */ 17137c478bd9Sstevel@tonic-gate for (stat = 0; stat < LGRP_NUM_COUNTER_STATS; stat++) { 17147c478bd9Sstevel@tonic-gate ksd[stat].value.i64 = lgrp_stat_read(lgrpid, stat); 17157c478bd9Sstevel@tonic-gate } 17167c478bd9Sstevel@tonic-gate 17177c478bd9Sstevel@tonic-gate /* 17187c478bd9Sstevel@tonic-gate * Handle kernel data snapshot stats 17197c478bd9Sstevel@tonic-gate */ 17207c478bd9Sstevel@tonic-gate ksd[stat + LGRP_NUM_CPUS].value.i64 = lgrp->lgrp_cpucnt; 17217c478bd9Sstevel@tonic-gate ksd[stat + LGRP_NUM_PG_INSTALL].value.i64 = 17227c478bd9Sstevel@tonic-gate lgrp_mem_size(lgrpid, LGRP_MEM_SIZE_INSTALL); 17237c478bd9Sstevel@tonic-gate ksd[stat + LGRP_NUM_PG_AVAIL].value.i64 = 17247c478bd9Sstevel@tonic-gate lgrp_mem_size(lgrpid, LGRP_MEM_SIZE_AVAIL); 17257c478bd9Sstevel@tonic-gate ksd[stat + LGRP_NUM_PG_FREE].value.i64 = 17267c478bd9Sstevel@tonic-gate lgrp_mem_size(lgrpid, LGRP_MEM_SIZE_FREE); 17277c478bd9Sstevel@tonic-gate ksd[stat + LGRP_LOADAVG].value.i64 = lgrp_sum_loadavgs(lgrp); 17287c478bd9Sstevel@tonic-gate } else { 17297c478bd9Sstevel@tonic-gate lgrp_kstat_reset(lgrpid); 17307c478bd9Sstevel@tonic-gate } 17317c478bd9Sstevel@tonic-gate 17327c478bd9Sstevel@tonic-gate return (0); 17337c478bd9Sstevel@tonic-gate } 17347c478bd9Sstevel@tonic-gate 17357c478bd9Sstevel@tonic-gate int 17367c478bd9Sstevel@tonic-gate lgrp_query_cpu(processorid_t id, lgrp_id_t *lp) 17377c478bd9Sstevel@tonic-gate { 17387c478bd9Sstevel@tonic-gate cpu_t *cp; 17397c478bd9Sstevel@tonic-gate 17407c478bd9Sstevel@tonic-gate mutex_enter(&cpu_lock); 17417c478bd9Sstevel@tonic-gate 17427c478bd9Sstevel@tonic-gate if ((cp = cpu_get(id)) == NULL) { 17437c478bd9Sstevel@tonic-gate mutex_exit(&cpu_lock); 17447c478bd9Sstevel@tonic-gate return (EINVAL); 17457c478bd9Sstevel@tonic-gate } 17467c478bd9Sstevel@tonic-gate 17477c478bd9Sstevel@tonic-gate if (cpu_is_offline(cp) || cpu_is_poweredoff(cp)) { 17487c478bd9Sstevel@tonic-gate mutex_exit(&cpu_lock); 17497c478bd9Sstevel@tonic-gate return (EINVAL); 17507c478bd9Sstevel@tonic-gate } 17517c478bd9Sstevel@tonic-gate 17527c478bd9Sstevel@tonic-gate ASSERT(cp->cpu_lpl != NULL); 17537c478bd9Sstevel@tonic-gate 17547c478bd9Sstevel@tonic-gate *lp = cp->cpu_lpl->lpl_lgrpid; 17557c478bd9Sstevel@tonic-gate 17567c478bd9Sstevel@tonic-gate mutex_exit(&cpu_lock); 17577c478bd9Sstevel@tonic-gate 17587c478bd9Sstevel@tonic-gate return (0); 17597c478bd9Sstevel@tonic-gate } 17607c478bd9Sstevel@tonic-gate 17617c478bd9Sstevel@tonic-gate int 17627c478bd9Sstevel@tonic-gate lgrp_query_load(processorid_t id, lgrp_load_t *lp) 17637c478bd9Sstevel@tonic-gate { 17647c478bd9Sstevel@tonic-gate cpu_t *cp; 17657c478bd9Sstevel@tonic-gate 17667c478bd9Sstevel@tonic-gate mutex_enter(&cpu_lock); 17677c478bd9Sstevel@tonic-gate 17687c478bd9Sstevel@tonic-gate if ((cp = cpu_get(id)) == NULL) { 17697c478bd9Sstevel@tonic-gate mutex_exit(&cpu_lock); 17707c478bd9Sstevel@tonic-gate return (EINVAL); 17717c478bd9Sstevel@tonic-gate } 17727c478bd9Sstevel@tonic-gate 17737c478bd9Sstevel@tonic-gate ASSERT(cp->cpu_lpl != NULL); 17747c478bd9Sstevel@tonic-gate 17757c478bd9Sstevel@tonic-gate *lp = cp->cpu_lpl->lpl_loadavg; 17767c478bd9Sstevel@tonic-gate 17777c478bd9Sstevel@tonic-gate mutex_exit(&cpu_lock); 17787c478bd9Sstevel@tonic-gate 17797c478bd9Sstevel@tonic-gate return (0); 17807c478bd9Sstevel@tonic-gate } 17817c478bd9Sstevel@tonic-gate 17827c478bd9Sstevel@tonic-gate void 17837c478bd9Sstevel@tonic-gate lgrp_latency_change(u_longlong_t oldtime, u_longlong_t newtime) 17847c478bd9Sstevel@tonic-gate { 17857c478bd9Sstevel@tonic-gate lgrp_t *lgrp; 17867c478bd9Sstevel@tonic-gate int i; 17877c478bd9Sstevel@tonic-gate 17887c478bd9Sstevel@tonic-gate for (i = 0; i <= lgrp_alloc_max; i++) { 17897c478bd9Sstevel@tonic-gate lgrp = lgrp_table[i]; 17907c478bd9Sstevel@tonic-gate 17917c478bd9Sstevel@tonic-gate if (LGRP_EXISTS(lgrp) && (lgrp->lgrp_latency == oldtime)) 17927c478bd9Sstevel@tonic-gate lgrp->lgrp_latency = (int)newtime; 17937c478bd9Sstevel@tonic-gate } 17947c478bd9Sstevel@tonic-gate } 17957c478bd9Sstevel@tonic-gate 17967c478bd9Sstevel@tonic-gate /* 17977c478bd9Sstevel@tonic-gate * Add a resource named by lpl_leaf to rset of lpl_target 17987c478bd9Sstevel@tonic-gate * 17997c478bd9Sstevel@tonic-gate * This routine also adjusts ncpu and nrset if the call succeeds in adding a 18007c478bd9Sstevel@tonic-gate * resource. It is adjusted here, as this is presently the only place that we 18017c478bd9Sstevel@tonic-gate * can be certain a resource addition has succeeded. 18027c478bd9Sstevel@tonic-gate * 18037c478bd9Sstevel@tonic-gate * We keep the list of rsets sorted so that the dispatcher can quickly walk the 18047c478bd9Sstevel@tonic-gate * list in order until it reaches a NULL. (This list is required to be NULL 18057c478bd9Sstevel@tonic-gate * terminated, too). This is done so that we can mark start pos + 1, so that 18067c478bd9Sstevel@tonic-gate * each lpl is traversed sequentially, but in a different order. We hope this 18077c478bd9Sstevel@tonic-gate * will improve performance a bit. (Hopefully, less read-to-own traffic...) 18087c478bd9Sstevel@tonic-gate */ 18097c478bd9Sstevel@tonic-gate 18107c478bd9Sstevel@tonic-gate void 18117c478bd9Sstevel@tonic-gate lpl_rset_add(lpl_t *lpl_target, lpl_t *lpl_leaf) 18127c478bd9Sstevel@tonic-gate { 18137c478bd9Sstevel@tonic-gate int i; 18147c478bd9Sstevel@tonic-gate int entry_slot = 0; 18157c478bd9Sstevel@tonic-gate 18167c478bd9Sstevel@tonic-gate /* return if leaf is already present */ 18177c478bd9Sstevel@tonic-gate for (i = 0; i < lpl_target->lpl_nrset; i++) { 18187c478bd9Sstevel@tonic-gate if (lpl_target->lpl_rset[i] == lpl_leaf) { 18197c478bd9Sstevel@tonic-gate return; 18207c478bd9Sstevel@tonic-gate } 18217c478bd9Sstevel@tonic-gate 18227c478bd9Sstevel@tonic-gate if (lpl_target->lpl_rset[i]->lpl_lgrpid > 18237c478bd9Sstevel@tonic-gate lpl_leaf->lpl_lgrpid) { 18247c478bd9Sstevel@tonic-gate break; 18257c478bd9Sstevel@tonic-gate } 18267c478bd9Sstevel@tonic-gate } 18277c478bd9Sstevel@tonic-gate 18287c478bd9Sstevel@tonic-gate /* insert leaf, update counts */ 18297c478bd9Sstevel@tonic-gate entry_slot = i; 18307c478bd9Sstevel@tonic-gate i = lpl_target->lpl_nrset++; 18317c478bd9Sstevel@tonic-gate if (lpl_target->lpl_nrset >= LPL_RSET_MAX) { 18327c478bd9Sstevel@tonic-gate panic("More leaf lgrps in system than are supported!\n"); 18337c478bd9Sstevel@tonic-gate } 18347c478bd9Sstevel@tonic-gate 18357c478bd9Sstevel@tonic-gate /* 18367c478bd9Sstevel@tonic-gate * Start at the end of the rset array and work backwards towards the 18377c478bd9Sstevel@tonic-gate * slot into which the new lpl will be inserted. This effectively 18387c478bd9Sstevel@tonic-gate * preserves the current ordering by scooting everybody over one entry, 18397c478bd9Sstevel@tonic-gate * and placing the new entry into the space created. 18407c478bd9Sstevel@tonic-gate */ 18417c478bd9Sstevel@tonic-gate 18427c478bd9Sstevel@tonic-gate while (i-- > entry_slot) { 18437c478bd9Sstevel@tonic-gate lpl_target->lpl_rset[i + 1] = lpl_target->lpl_rset[i]; 18447c478bd9Sstevel@tonic-gate } 18457c478bd9Sstevel@tonic-gate 18467c478bd9Sstevel@tonic-gate lpl_target->lpl_rset[entry_slot] = lpl_leaf; 18477c478bd9Sstevel@tonic-gate lpl_target->lpl_ncpu += lpl_leaf->lpl_ncpu; 18487c478bd9Sstevel@tonic-gate } 18497c478bd9Sstevel@tonic-gate 18507c478bd9Sstevel@tonic-gate /* 18517c478bd9Sstevel@tonic-gate * Update each of lpl_parent's children with a proper hint and 18527c478bd9Sstevel@tonic-gate * a reference to their parent. 18537c478bd9Sstevel@tonic-gate * The lgrp topology is used as the reference since it is fully 18547c478bd9Sstevel@tonic-gate * consistent and correct at this point. 18557c478bd9Sstevel@tonic-gate * 18567c478bd9Sstevel@tonic-gate * Each child's hint will reference an element in lpl_parent's 18577c478bd9Sstevel@tonic-gate * rset that designates where the child should start searching 18587c478bd9Sstevel@tonic-gate * for CPU resources. The hint selected is the highest order leaf present 18597c478bd9Sstevel@tonic-gate * in the child's lineage. 18607c478bd9Sstevel@tonic-gate * 18617c478bd9Sstevel@tonic-gate * This should be called after any potential change in lpl_parent's 18627c478bd9Sstevel@tonic-gate * rset. 18637c478bd9Sstevel@tonic-gate */ 18647c478bd9Sstevel@tonic-gate static void 18657c478bd9Sstevel@tonic-gate lpl_child_update(lpl_t *lpl_parent, struct cpupart *cp) 18667c478bd9Sstevel@tonic-gate { 18677c478bd9Sstevel@tonic-gate klgrpset_t children, leaves; 18687c478bd9Sstevel@tonic-gate lpl_t *lpl; 18697c478bd9Sstevel@tonic-gate int hint; 18707c478bd9Sstevel@tonic-gate int i, j; 18717c478bd9Sstevel@tonic-gate 18727c478bd9Sstevel@tonic-gate children = lgrp_table[lpl_parent->lpl_lgrpid]->lgrp_children; 18737c478bd9Sstevel@tonic-gate if (klgrpset_isempty(children)) 18747c478bd9Sstevel@tonic-gate return; /* nothing to do */ 18757c478bd9Sstevel@tonic-gate 18767c478bd9Sstevel@tonic-gate for (i = 0; i <= lgrp_alloc_max; i++) { 18777c478bd9Sstevel@tonic-gate if (klgrpset_ismember(children, i)) { 18787c478bd9Sstevel@tonic-gate 18797c478bd9Sstevel@tonic-gate /* 18807c478bd9Sstevel@tonic-gate * Given the set of leaves in this child's lineage, 18817c478bd9Sstevel@tonic-gate * find the highest order leaf present in the parent's 18827c478bd9Sstevel@tonic-gate * rset. Select this as the hint for the child. 18837c478bd9Sstevel@tonic-gate */ 18847c478bd9Sstevel@tonic-gate leaves = lgrp_table[i]->lgrp_leaves; 18857c478bd9Sstevel@tonic-gate hint = 0; 18867c478bd9Sstevel@tonic-gate for (j = 0; j < lpl_parent->lpl_nrset; j++) { 18877c478bd9Sstevel@tonic-gate lpl = lpl_parent->lpl_rset[j]; 18887c478bd9Sstevel@tonic-gate if (klgrpset_ismember(leaves, lpl->lpl_lgrpid)) 18897c478bd9Sstevel@tonic-gate hint = j; 18907c478bd9Sstevel@tonic-gate } 18917c478bd9Sstevel@tonic-gate cp->cp_lgrploads[i].lpl_hint = hint; 18927c478bd9Sstevel@tonic-gate 18937c478bd9Sstevel@tonic-gate /* 18947c478bd9Sstevel@tonic-gate * (Re)set the parent. It may be incorrect if 18957c478bd9Sstevel@tonic-gate * lpl_parent is new in the topology. 18967c478bd9Sstevel@tonic-gate */ 18977c478bd9Sstevel@tonic-gate cp->cp_lgrploads[i].lpl_parent = lpl_parent; 18987c478bd9Sstevel@tonic-gate } 18997c478bd9Sstevel@tonic-gate } 19007c478bd9Sstevel@tonic-gate } 19017c478bd9Sstevel@tonic-gate 19027c478bd9Sstevel@tonic-gate /* 19037c478bd9Sstevel@tonic-gate * Delete resource lpl_leaf from rset of lpl_target, assuming it's there. 19047c478bd9Sstevel@tonic-gate * 19057c478bd9Sstevel@tonic-gate * This routine also adjusts ncpu and nrset if the call succeeds in deleting a 19067c478bd9Sstevel@tonic-gate * resource. The values are adjusted here, as this is the only place that we can 19077c478bd9Sstevel@tonic-gate * be certain a resource was successfully deleted. 19087c478bd9Sstevel@tonic-gate */ 19097c478bd9Sstevel@tonic-gate void 19107c478bd9Sstevel@tonic-gate lpl_rset_del(lpl_t *lpl_target, lpl_t *lpl_leaf) 19117c478bd9Sstevel@tonic-gate { 19127c478bd9Sstevel@tonic-gate int i; 19137c478bd9Sstevel@tonic-gate 19147c478bd9Sstevel@tonic-gate /* find leaf in intermediate node */ 19157c478bd9Sstevel@tonic-gate for (i = 0; i < lpl_target->lpl_nrset; i++) { 19167c478bd9Sstevel@tonic-gate if (lpl_target->lpl_rset[i] == lpl_leaf) 19177c478bd9Sstevel@tonic-gate break; 19187c478bd9Sstevel@tonic-gate } 19197c478bd9Sstevel@tonic-gate 19207c478bd9Sstevel@tonic-gate /* return if leaf not found */ 19217c478bd9Sstevel@tonic-gate if (lpl_target->lpl_rset[i] != lpl_leaf) 19227c478bd9Sstevel@tonic-gate return; 19237c478bd9Sstevel@tonic-gate 19247c478bd9Sstevel@tonic-gate /* prune leaf, compress array */ 19257c478bd9Sstevel@tonic-gate ASSERT(lpl_target->lpl_nrset < LPL_RSET_MAX); 19267c478bd9Sstevel@tonic-gate lpl_target->lpl_rset[lpl_target->lpl_nrset--] = NULL; 19277c478bd9Sstevel@tonic-gate lpl_target->lpl_ncpu--; 19287c478bd9Sstevel@tonic-gate do { 19297c478bd9Sstevel@tonic-gate lpl_target->lpl_rset[i] = lpl_target->lpl_rset[i + 1]; 19307c478bd9Sstevel@tonic-gate } while (i++ < lpl_target->lpl_nrset); 19317c478bd9Sstevel@tonic-gate } 19327c478bd9Sstevel@tonic-gate 19337c478bd9Sstevel@tonic-gate /* 19347c478bd9Sstevel@tonic-gate * Check to see if the resource set of the target lpl contains the 19357c478bd9Sstevel@tonic-gate * supplied leaf lpl. This returns 1 if the lpl is found, 0 if it is not. 19367c478bd9Sstevel@tonic-gate */ 19377c478bd9Sstevel@tonic-gate 19387c478bd9Sstevel@tonic-gate int 19397c478bd9Sstevel@tonic-gate lpl_rset_contains(lpl_t *lpl_target, lpl_t *lpl_leaf) 19407c478bd9Sstevel@tonic-gate { 19417c478bd9Sstevel@tonic-gate int i; 19427c478bd9Sstevel@tonic-gate 19437c478bd9Sstevel@tonic-gate for (i = 0; i < lpl_target->lpl_nrset; i++) { 19447c478bd9Sstevel@tonic-gate if (lpl_target->lpl_rset[i] == lpl_leaf) 19457c478bd9Sstevel@tonic-gate return (1); 19467c478bd9Sstevel@tonic-gate } 19477c478bd9Sstevel@tonic-gate 19487c478bd9Sstevel@tonic-gate return (0); 19497c478bd9Sstevel@tonic-gate } 19507c478bd9Sstevel@tonic-gate 19517c478bd9Sstevel@tonic-gate /* 19527c478bd9Sstevel@tonic-gate * Called when we change cpu lpl membership. This increments or decrements the 19537c478bd9Sstevel@tonic-gate * per-cpu counter in every lpl in which our leaf appears. 19547c478bd9Sstevel@tonic-gate */ 19557c478bd9Sstevel@tonic-gate void 19567c478bd9Sstevel@tonic-gate lpl_cpu_adjcnt(lpl_act_t act, cpu_t *cp) 19577c478bd9Sstevel@tonic-gate { 19587c478bd9Sstevel@tonic-gate cpupart_t *cpupart; 19597c478bd9Sstevel@tonic-gate lgrp_t *lgrp_leaf; 19607c478bd9Sstevel@tonic-gate lgrp_t *lgrp_cur; 19617c478bd9Sstevel@tonic-gate lpl_t *lpl_leaf; 19627c478bd9Sstevel@tonic-gate lpl_t *lpl_cur; 19637c478bd9Sstevel@tonic-gate int i; 19647c478bd9Sstevel@tonic-gate 19657c478bd9Sstevel@tonic-gate ASSERT(act == LPL_DECREMENT || act == LPL_INCREMENT); 19667c478bd9Sstevel@tonic-gate 19677c478bd9Sstevel@tonic-gate cpupart = cp->cpu_part; 19687c478bd9Sstevel@tonic-gate lpl_leaf = cp->cpu_lpl; 19697c478bd9Sstevel@tonic-gate lgrp_leaf = lgrp_table[lpl_leaf->lpl_lgrpid]; 19707c478bd9Sstevel@tonic-gate 19717c478bd9Sstevel@tonic-gate for (i = 0; i <= lgrp_alloc_max; i++) { 19727c478bd9Sstevel@tonic-gate lgrp_cur = lgrp_table[i]; 19737c478bd9Sstevel@tonic-gate 19747c478bd9Sstevel@tonic-gate /* 19757c478bd9Sstevel@tonic-gate * Don't adjust if the lgrp isn't there, if we're the leaf lpl 19767c478bd9Sstevel@tonic-gate * for the cpu in question, or if the current lgrp and leaf 19777c478bd9Sstevel@tonic-gate * don't share the same resources. 19787c478bd9Sstevel@tonic-gate */ 19797c478bd9Sstevel@tonic-gate 19807c478bd9Sstevel@tonic-gate if (!LGRP_EXISTS(lgrp_cur) || (lgrp_cur == lgrp_leaf) || 19817c478bd9Sstevel@tonic-gate !klgrpset_intersects(lgrp_leaf->lgrp_set[LGRP_RSRC_CPU], 19827c478bd9Sstevel@tonic-gate lgrp_cur->lgrp_set[LGRP_RSRC_CPU])) 19837c478bd9Sstevel@tonic-gate continue; 19847c478bd9Sstevel@tonic-gate 19857c478bd9Sstevel@tonic-gate 19867c478bd9Sstevel@tonic-gate lpl_cur = &cpupart->cp_lgrploads[lgrp_cur->lgrp_id]; 19877c478bd9Sstevel@tonic-gate 19887c478bd9Sstevel@tonic-gate if (lpl_cur->lpl_nrset > 0) { 19897c478bd9Sstevel@tonic-gate if (act == LPL_INCREMENT) { 19907c478bd9Sstevel@tonic-gate lpl_cur->lpl_ncpu++; 19917c478bd9Sstevel@tonic-gate } else if (act == LPL_DECREMENT) { 19927c478bd9Sstevel@tonic-gate lpl_cur->lpl_ncpu--; 19937c478bd9Sstevel@tonic-gate } 19947c478bd9Sstevel@tonic-gate } 19957c478bd9Sstevel@tonic-gate } 19967c478bd9Sstevel@tonic-gate } 19977c478bd9Sstevel@tonic-gate 19987c478bd9Sstevel@tonic-gate /* 19997c478bd9Sstevel@tonic-gate * Initialize lpl with given resources and specified lgrp 20007c478bd9Sstevel@tonic-gate */ 20017c478bd9Sstevel@tonic-gate 20027c478bd9Sstevel@tonic-gate void 20037c478bd9Sstevel@tonic-gate lpl_init(lpl_t *lpl, lpl_t *lpl_leaf, lgrp_t *lgrp) 20047c478bd9Sstevel@tonic-gate { 20057c478bd9Sstevel@tonic-gate lpl->lpl_lgrpid = lgrp->lgrp_id; 20067c478bd9Sstevel@tonic-gate lpl->lpl_loadavg = 0; 20077c478bd9Sstevel@tonic-gate if (lpl == lpl_leaf) 20087c478bd9Sstevel@tonic-gate lpl->lpl_ncpu = 1; 20097c478bd9Sstevel@tonic-gate else 20107c478bd9Sstevel@tonic-gate lpl->lpl_ncpu = lpl_leaf->lpl_ncpu; 20117c478bd9Sstevel@tonic-gate lpl->lpl_nrset = 1; 20127c478bd9Sstevel@tonic-gate lpl->lpl_rset[0] = lpl_leaf; 20137c478bd9Sstevel@tonic-gate lpl->lpl_lgrp = lgrp; 20147c478bd9Sstevel@tonic-gate lpl->lpl_parent = NULL; /* set by lpl_leaf_insert() */ 20157c478bd9Sstevel@tonic-gate lpl->lpl_cpus = NULL; /* set by lgrp_part_add_cpu() */ 20167c478bd9Sstevel@tonic-gate } 20177c478bd9Sstevel@tonic-gate 20187c478bd9Sstevel@tonic-gate /* 20197c478bd9Sstevel@tonic-gate * Clear an unused lpl 20207c478bd9Sstevel@tonic-gate */ 20217c478bd9Sstevel@tonic-gate 20227c478bd9Sstevel@tonic-gate void 20237c478bd9Sstevel@tonic-gate lpl_clear(lpl_t *lpl) 20247c478bd9Sstevel@tonic-gate { 20257c478bd9Sstevel@tonic-gate lgrpid_t lid; 20267c478bd9Sstevel@tonic-gate 20277c478bd9Sstevel@tonic-gate /* save lid for debugging purposes */ 20287c478bd9Sstevel@tonic-gate lid = lpl->lpl_lgrpid; 20297c478bd9Sstevel@tonic-gate bzero(lpl, sizeof (lpl_t)); 20307c478bd9Sstevel@tonic-gate lpl->lpl_lgrpid = lid; 20317c478bd9Sstevel@tonic-gate } 20327c478bd9Sstevel@tonic-gate 20337c478bd9Sstevel@tonic-gate /* 20347c478bd9Sstevel@tonic-gate * Given a CPU-partition, verify that the lpl topology in the CPU-partition 20357c478bd9Sstevel@tonic-gate * is in sync with the lgroup toplogy in the system. The lpl topology may not 20367c478bd9Sstevel@tonic-gate * make full use of all of the lgroup topology, but this checks to make sure 20377c478bd9Sstevel@tonic-gate * that for the parts that it does use, it has correctly understood the 20387c478bd9Sstevel@tonic-gate * relationships that exist. This function returns 20397c478bd9Sstevel@tonic-gate * 0 if the topology is correct, and a non-zero error code, for non-debug 20407c478bd9Sstevel@tonic-gate * kernels if incorrect. Asserts are spread throughout the code to aid in 20417c478bd9Sstevel@tonic-gate * debugging on a DEBUG kernel. 20427c478bd9Sstevel@tonic-gate */ 20437c478bd9Sstevel@tonic-gate int 20447c478bd9Sstevel@tonic-gate lpl_topo_verify(cpupart_t *cpupart) 20457c478bd9Sstevel@tonic-gate { 20467c478bd9Sstevel@tonic-gate lgrp_t *lgrp; 20477c478bd9Sstevel@tonic-gate lpl_t *lpl; 20487c478bd9Sstevel@tonic-gate klgrpset_t rset; 20497c478bd9Sstevel@tonic-gate klgrpset_t cset; 20507c478bd9Sstevel@tonic-gate cpu_t *cpu; 20517c478bd9Sstevel@tonic-gate cpu_t *cp_start; 20527c478bd9Sstevel@tonic-gate int i; 20537c478bd9Sstevel@tonic-gate int j; 20547c478bd9Sstevel@tonic-gate int sum; 20557c478bd9Sstevel@tonic-gate 20567c478bd9Sstevel@tonic-gate /* topology can't be incorrect if it doesn't exist */ 20577c478bd9Sstevel@tonic-gate if (!lgrp_topo_initialized || !lgrp_initialized) 20587c478bd9Sstevel@tonic-gate return (LPL_TOPO_CORRECT); 20597c478bd9Sstevel@tonic-gate 20607c478bd9Sstevel@tonic-gate ASSERT(cpupart != NULL); 20617c478bd9Sstevel@tonic-gate 20627c478bd9Sstevel@tonic-gate for (i = 0; i <= lgrp_alloc_max; i++) { 20637c478bd9Sstevel@tonic-gate lgrp = lgrp_table[i]; 20647c478bd9Sstevel@tonic-gate lpl = NULL; 20657c478bd9Sstevel@tonic-gate /* make sure lpls are allocated */ 20667c478bd9Sstevel@tonic-gate ASSERT(cpupart->cp_lgrploads); 20677c478bd9Sstevel@tonic-gate if (!cpupart->cp_lgrploads) 20687c478bd9Sstevel@tonic-gate return (LPL_TOPO_PART_HAS_NO_LPL); 20697c478bd9Sstevel@tonic-gate 20707c478bd9Sstevel@tonic-gate lpl = &cpupart->cp_lgrploads[i]; 20717c478bd9Sstevel@tonic-gate /* make sure our index is good */ 20727c478bd9Sstevel@tonic-gate ASSERT(i < cpupart->cp_nlgrploads); 20737c478bd9Sstevel@tonic-gate 20747c478bd9Sstevel@tonic-gate /* if lgroup doesn't exist, make sure lpl is empty */ 20757c478bd9Sstevel@tonic-gate if (!LGRP_EXISTS(lgrp)) { 20767c478bd9Sstevel@tonic-gate ASSERT(lpl->lpl_ncpu == 0); 20777c478bd9Sstevel@tonic-gate if (lpl->lpl_ncpu > 0) { 20787c478bd9Sstevel@tonic-gate return (LPL_TOPO_CPUS_NOT_EMPTY); 20797c478bd9Sstevel@tonic-gate } else { 20807c478bd9Sstevel@tonic-gate continue; 20817c478bd9Sstevel@tonic-gate } 20827c478bd9Sstevel@tonic-gate } 20837c478bd9Sstevel@tonic-gate 20847c478bd9Sstevel@tonic-gate /* verify that lgroup and lpl are identically numbered */ 20857c478bd9Sstevel@tonic-gate ASSERT(lgrp->lgrp_id == lpl->lpl_lgrpid); 20867c478bd9Sstevel@tonic-gate 20877c478bd9Sstevel@tonic-gate /* if lgroup isn't in our partition, make sure lpl is empty */ 20887c478bd9Sstevel@tonic-gate if (!klgrpset_intersects(lgrp->lgrp_leaves, 20897c478bd9Sstevel@tonic-gate cpupart->cp_lgrpset)) { 20907c478bd9Sstevel@tonic-gate ASSERT(lpl->lpl_ncpu == 0); 20917c478bd9Sstevel@tonic-gate if (lpl->lpl_ncpu > 0) { 20927c478bd9Sstevel@tonic-gate return (LPL_TOPO_CPUS_NOT_EMPTY); 20937c478bd9Sstevel@tonic-gate } 20947c478bd9Sstevel@tonic-gate /* 20957c478bd9Sstevel@tonic-gate * lpl is empty, and lgroup isn't in partition. verify 20967c478bd9Sstevel@tonic-gate * that lpl doesn't show up in anyone else's rsets (in 20977c478bd9Sstevel@tonic-gate * this partition, anyway) 20987c478bd9Sstevel@tonic-gate */ 20997c478bd9Sstevel@tonic-gate 21007c478bd9Sstevel@tonic-gate for (j = 0; j < cpupart->cp_nlgrploads; j++) { 21017c478bd9Sstevel@tonic-gate lpl_t *i_lpl; /* lpl we're iterating over */ 21027c478bd9Sstevel@tonic-gate 21037c478bd9Sstevel@tonic-gate i_lpl = &cpupart->cp_lgrploads[j]; 21047c478bd9Sstevel@tonic-gate 21057c478bd9Sstevel@tonic-gate ASSERT(!lpl_rset_contains(i_lpl, lpl)); 21067c478bd9Sstevel@tonic-gate if (lpl_rset_contains(i_lpl, lpl)) { 21077c478bd9Sstevel@tonic-gate return (LPL_TOPO_LPL_ORPHANED); 21087c478bd9Sstevel@tonic-gate } 21097c478bd9Sstevel@tonic-gate } 21107c478bd9Sstevel@tonic-gate /* lgroup is empty, and everything is ok. continue */ 21117c478bd9Sstevel@tonic-gate continue; 21127c478bd9Sstevel@tonic-gate } 21137c478bd9Sstevel@tonic-gate 21147c478bd9Sstevel@tonic-gate 21157c478bd9Sstevel@tonic-gate /* lgroup is in this partition, now check it against lpl */ 21167c478bd9Sstevel@tonic-gate 21177c478bd9Sstevel@tonic-gate /* do both have matching lgrps? */ 21187c478bd9Sstevel@tonic-gate ASSERT(lgrp == lpl->lpl_lgrp); 21197c478bd9Sstevel@tonic-gate if (lgrp != lpl->lpl_lgrp) { 21207c478bd9Sstevel@tonic-gate return (LPL_TOPO_LGRP_MISMATCH); 21217c478bd9Sstevel@tonic-gate } 21227c478bd9Sstevel@tonic-gate 21237c478bd9Sstevel@tonic-gate /* do the parent lgroups exist and do they match? */ 21247c478bd9Sstevel@tonic-gate if (lgrp->lgrp_parent) { 21257c478bd9Sstevel@tonic-gate ASSERT(lpl->lpl_parent); 21267c478bd9Sstevel@tonic-gate ASSERT(lgrp->lgrp_parent->lgrp_id == 21277c478bd9Sstevel@tonic-gate lpl->lpl_parent->lpl_lgrpid); 21287c478bd9Sstevel@tonic-gate 21297c478bd9Sstevel@tonic-gate if (!lpl->lpl_parent) { 21307c478bd9Sstevel@tonic-gate return (LPL_TOPO_MISSING_PARENT); 21317c478bd9Sstevel@tonic-gate } else if (lgrp->lgrp_parent->lgrp_id != 21327c478bd9Sstevel@tonic-gate lpl->lpl_parent->lpl_lgrpid) { 21337c478bd9Sstevel@tonic-gate return (LPL_TOPO_PARENT_MISMATCH); 21347c478bd9Sstevel@tonic-gate } 21357c478bd9Sstevel@tonic-gate } 21367c478bd9Sstevel@tonic-gate 21377c478bd9Sstevel@tonic-gate /* only leaf lgroups keep a cpucnt, only check leaves */ 21387c478bd9Sstevel@tonic-gate if ((lpl->lpl_nrset == 1) && (lpl == lpl->lpl_rset[0])) { 21397c478bd9Sstevel@tonic-gate 21407c478bd9Sstevel@tonic-gate /* verify that lgrp is also a leaf */ 21417c478bd9Sstevel@tonic-gate ASSERT((lgrp->lgrp_childcnt == 0) && 21427c478bd9Sstevel@tonic-gate (klgrpset_ismember(lgrp->lgrp_leaves, 21437c478bd9Sstevel@tonic-gate lpl->lpl_lgrpid))); 21447c478bd9Sstevel@tonic-gate 21457c478bd9Sstevel@tonic-gate if ((lgrp->lgrp_childcnt > 0) || 21467c478bd9Sstevel@tonic-gate (!klgrpset_ismember(lgrp->lgrp_leaves, 21477c478bd9Sstevel@tonic-gate lpl->lpl_lgrpid))) { 21487c478bd9Sstevel@tonic-gate return (LPL_TOPO_LGRP_NOT_LEAF); 21497c478bd9Sstevel@tonic-gate } 21507c478bd9Sstevel@tonic-gate 21517c478bd9Sstevel@tonic-gate ASSERT((lgrp->lgrp_cpucnt >= lpl->lpl_ncpu) && 21527c478bd9Sstevel@tonic-gate (lpl->lpl_ncpu > 0)); 21537c478bd9Sstevel@tonic-gate if ((lgrp->lgrp_cpucnt < lpl->lpl_ncpu) || 21547c478bd9Sstevel@tonic-gate (lpl->lpl_ncpu <= 0)) { 21557c478bd9Sstevel@tonic-gate return (LPL_TOPO_BAD_CPUCNT); 21567c478bd9Sstevel@tonic-gate } 21577c478bd9Sstevel@tonic-gate 21587c478bd9Sstevel@tonic-gate /* 21597c478bd9Sstevel@tonic-gate * Check that lpl_ncpu also matches the number of 21607c478bd9Sstevel@tonic-gate * cpus in the lpl's linked list. This only exists in 21617c478bd9Sstevel@tonic-gate * leaves, but they should always match. 21627c478bd9Sstevel@tonic-gate */ 21637c478bd9Sstevel@tonic-gate j = 0; 21647c478bd9Sstevel@tonic-gate cpu = cp_start = lpl->lpl_cpus; 21657c478bd9Sstevel@tonic-gate while (cpu != NULL) { 21667c478bd9Sstevel@tonic-gate j++; 21677c478bd9Sstevel@tonic-gate 21687c478bd9Sstevel@tonic-gate /* check to make sure cpu's lpl is leaf lpl */ 21697c478bd9Sstevel@tonic-gate ASSERT(cpu->cpu_lpl == lpl); 21707c478bd9Sstevel@tonic-gate if (cpu->cpu_lpl != lpl) { 21717c478bd9Sstevel@tonic-gate return (LPL_TOPO_CPU_HAS_BAD_LPL); 21727c478bd9Sstevel@tonic-gate } 21737c478bd9Sstevel@tonic-gate 21747c478bd9Sstevel@tonic-gate /* check next cpu */ 21757c478bd9Sstevel@tonic-gate if ((cpu = cpu->cpu_next_lpl) != cp_start) { 21767c478bd9Sstevel@tonic-gate continue; 21777c478bd9Sstevel@tonic-gate } else { 21787c478bd9Sstevel@tonic-gate cpu = NULL; 21797c478bd9Sstevel@tonic-gate } 21807c478bd9Sstevel@tonic-gate } 21817c478bd9Sstevel@tonic-gate 21827c478bd9Sstevel@tonic-gate ASSERT(j == lpl->lpl_ncpu); 21837c478bd9Sstevel@tonic-gate if (j != lpl->lpl_ncpu) { 21847c478bd9Sstevel@tonic-gate return (LPL_TOPO_LPL_BAD_NCPU); 21857c478bd9Sstevel@tonic-gate } 21867c478bd9Sstevel@tonic-gate 21877c478bd9Sstevel@tonic-gate /* 21887c478bd9Sstevel@tonic-gate * Also, check that leaf lpl is contained in all 21897c478bd9Sstevel@tonic-gate * intermediate lpls that name the leaf as a descendant 21907c478bd9Sstevel@tonic-gate */ 21917c478bd9Sstevel@tonic-gate 21927c478bd9Sstevel@tonic-gate for (j = 0; j <= lgrp_alloc_max; j++) { 21937c478bd9Sstevel@tonic-gate klgrpset_t intersect; 21947c478bd9Sstevel@tonic-gate lgrp_t *lgrp_cand; 21957c478bd9Sstevel@tonic-gate lpl_t *lpl_cand; 21967c478bd9Sstevel@tonic-gate 21977c478bd9Sstevel@tonic-gate lgrp_cand = lgrp_table[j]; 21987c478bd9Sstevel@tonic-gate intersect = klgrpset_intersects( 21997c478bd9Sstevel@tonic-gate lgrp_cand->lgrp_set[LGRP_RSRC_CPU], 22007c478bd9Sstevel@tonic-gate cpupart->cp_lgrpset); 22017c478bd9Sstevel@tonic-gate 22027c478bd9Sstevel@tonic-gate if (!LGRP_EXISTS(lgrp_cand) || 22037c478bd9Sstevel@tonic-gate !klgrpset_intersects(lgrp_cand->lgrp_leaves, 22047c478bd9Sstevel@tonic-gate cpupart->cp_lgrpset) || 22057c478bd9Sstevel@tonic-gate (intersect == 0)) 22067c478bd9Sstevel@tonic-gate continue; 22077c478bd9Sstevel@tonic-gate 22087c478bd9Sstevel@tonic-gate lpl_cand = 22097c478bd9Sstevel@tonic-gate &cpupart->cp_lgrploads[lgrp_cand->lgrp_id]; 22107c478bd9Sstevel@tonic-gate 22117c478bd9Sstevel@tonic-gate if (klgrpset_ismember(intersect, 22127c478bd9Sstevel@tonic-gate lgrp->lgrp_id)) { 22137c478bd9Sstevel@tonic-gate ASSERT(lpl_rset_contains(lpl_cand, 22147c478bd9Sstevel@tonic-gate lpl)); 22157c478bd9Sstevel@tonic-gate 22167c478bd9Sstevel@tonic-gate if (!lpl_rset_contains(lpl_cand, lpl)) { 22177c478bd9Sstevel@tonic-gate return (LPL_TOPO_RSET_MSSNG_LF); 22187c478bd9Sstevel@tonic-gate } 22197c478bd9Sstevel@tonic-gate } 22207c478bd9Sstevel@tonic-gate } 22217c478bd9Sstevel@tonic-gate 22227c478bd9Sstevel@tonic-gate } else { /* non-leaf specific checks */ 22237c478bd9Sstevel@tonic-gate 22247c478bd9Sstevel@tonic-gate /* 22257c478bd9Sstevel@tonic-gate * Non-leaf lpls should have lpl_cpus == NULL 22267c478bd9Sstevel@tonic-gate * verify that this is so 22277c478bd9Sstevel@tonic-gate */ 22287c478bd9Sstevel@tonic-gate ASSERT(lpl->lpl_cpus == NULL); 22297c478bd9Sstevel@tonic-gate if (lpl->lpl_cpus != NULL) { 22307c478bd9Sstevel@tonic-gate return (LPL_TOPO_NONLEAF_HAS_CPUS); 22317c478bd9Sstevel@tonic-gate } 22327c478bd9Sstevel@tonic-gate 22337c478bd9Sstevel@tonic-gate /* 22347c478bd9Sstevel@tonic-gate * verify that the sum of the cpus in the leaf resources 22357c478bd9Sstevel@tonic-gate * is equal to the total ncpu in the intermediate 22367c478bd9Sstevel@tonic-gate */ 22377c478bd9Sstevel@tonic-gate for (j = sum = 0; j < lpl->lpl_nrset; j++) { 22387c478bd9Sstevel@tonic-gate sum += lpl->lpl_rset[j]->lpl_ncpu; 22397c478bd9Sstevel@tonic-gate } 22407c478bd9Sstevel@tonic-gate 22417c478bd9Sstevel@tonic-gate ASSERT(sum == lpl->lpl_ncpu); 22427c478bd9Sstevel@tonic-gate if (sum != lpl->lpl_ncpu) { 22437c478bd9Sstevel@tonic-gate return (LPL_TOPO_LPL_BAD_NCPU); 22447c478bd9Sstevel@tonic-gate } 22457c478bd9Sstevel@tonic-gate } 22467c478bd9Sstevel@tonic-gate 22477c478bd9Sstevel@tonic-gate /* 22487c478bd9Sstevel@tonic-gate * check on lpl_hint. Don't check root, since it has no parent. 22497c478bd9Sstevel@tonic-gate */ 22507c478bd9Sstevel@tonic-gate if (lpl->lpl_parent != NULL) { 22517c478bd9Sstevel@tonic-gate int hint; 22527c478bd9Sstevel@tonic-gate lpl_t *hint_lpl; 22537c478bd9Sstevel@tonic-gate 22547c478bd9Sstevel@tonic-gate /* make sure hint is within limits of nrset */ 22557c478bd9Sstevel@tonic-gate hint = lpl->lpl_hint; 22567c478bd9Sstevel@tonic-gate ASSERT(lpl->lpl_parent->lpl_nrset >= hint); 22577c478bd9Sstevel@tonic-gate if (lpl->lpl_parent->lpl_nrset < hint) { 22587c478bd9Sstevel@tonic-gate return (LPL_TOPO_BOGUS_HINT); 22597c478bd9Sstevel@tonic-gate } 22607c478bd9Sstevel@tonic-gate 22617c478bd9Sstevel@tonic-gate /* make sure hint points to valid lpl */ 22627c478bd9Sstevel@tonic-gate hint_lpl = lpl->lpl_parent->lpl_rset[hint]; 22637c478bd9Sstevel@tonic-gate ASSERT(hint_lpl->lpl_ncpu > 0); 22647c478bd9Sstevel@tonic-gate if (hint_lpl->lpl_ncpu <= 0) { 22657c478bd9Sstevel@tonic-gate return (LPL_TOPO_BOGUS_HINT); 22667c478bd9Sstevel@tonic-gate } 22677c478bd9Sstevel@tonic-gate } 22687c478bd9Sstevel@tonic-gate 22697c478bd9Sstevel@tonic-gate /* 22707c478bd9Sstevel@tonic-gate * Check the rset of the lpl in question. Make sure that each 22717c478bd9Sstevel@tonic-gate * rset contains a subset of the resources in 22727c478bd9Sstevel@tonic-gate * lgrp_set[LGRP_RSRC_CPU] and in cp_lgrpset. This also makes 22737c478bd9Sstevel@tonic-gate * sure that each rset doesn't include resources that are 22747c478bd9Sstevel@tonic-gate * outside of that set. (Which would be resources somehow not 22757c478bd9Sstevel@tonic-gate * accounted for). 22767c478bd9Sstevel@tonic-gate */ 22777c478bd9Sstevel@tonic-gate 22787c478bd9Sstevel@tonic-gate klgrpset_clear(rset); 22797c478bd9Sstevel@tonic-gate for (j = 0; j < lpl->lpl_nrset; j++) { 22807c478bd9Sstevel@tonic-gate klgrpset_add(rset, lpl->lpl_rset[j]->lpl_lgrpid); 22817c478bd9Sstevel@tonic-gate } 22827c478bd9Sstevel@tonic-gate klgrpset_copy(cset, rset); 22837c478bd9Sstevel@tonic-gate /* make sure lpl rset matches lgrp rset */ 22847c478bd9Sstevel@tonic-gate klgrpset_diff(rset, lgrp->lgrp_set[LGRP_RSRC_CPU]); 22857c478bd9Sstevel@tonic-gate /* make sure rset is contained with in partition, too */ 22867c478bd9Sstevel@tonic-gate klgrpset_diff(cset, cpupart->cp_lgrpset); 22877c478bd9Sstevel@tonic-gate 22887c478bd9Sstevel@tonic-gate ASSERT(klgrpset_isempty(rset) && 22897c478bd9Sstevel@tonic-gate klgrpset_isempty(cset)); 22907c478bd9Sstevel@tonic-gate if (!klgrpset_isempty(rset) || 22917c478bd9Sstevel@tonic-gate !klgrpset_isempty(cset)) { 22927c478bd9Sstevel@tonic-gate return (LPL_TOPO_RSET_MISMATCH); 22937c478bd9Sstevel@tonic-gate } 22947c478bd9Sstevel@tonic-gate 22957c478bd9Sstevel@tonic-gate /* 22967c478bd9Sstevel@tonic-gate * check to make sure lpl_nrset matches the number of rsets 22977c478bd9Sstevel@tonic-gate * contained in the lpl 22987c478bd9Sstevel@tonic-gate */ 22997c478bd9Sstevel@tonic-gate 23007c478bd9Sstevel@tonic-gate for (j = 0; (lpl->lpl_rset[j] != NULL) && (j < LPL_RSET_MAX); 23017c478bd9Sstevel@tonic-gate j++); 23027c478bd9Sstevel@tonic-gate 23037c478bd9Sstevel@tonic-gate ASSERT(j == lpl->lpl_nrset); 23047c478bd9Sstevel@tonic-gate if (j != lpl->lpl_nrset) { 23057c478bd9Sstevel@tonic-gate return (LPL_TOPO_BAD_RSETCNT); 23067c478bd9Sstevel@tonic-gate } 23077c478bd9Sstevel@tonic-gate 23087c478bd9Sstevel@tonic-gate } 23097c478bd9Sstevel@tonic-gate return (LPL_TOPO_CORRECT); 23107c478bd9Sstevel@tonic-gate } 23117c478bd9Sstevel@tonic-gate 23127c478bd9Sstevel@tonic-gate /* 23137c478bd9Sstevel@tonic-gate * Flatten lpl topology to given number of levels. This is presently only 23147c478bd9Sstevel@tonic-gate * implemented for a flatten to 2 levels, which will prune out the intermediates 23157c478bd9Sstevel@tonic-gate * and home the leaf lpls to the root lpl. 23167c478bd9Sstevel@tonic-gate */ 23177c478bd9Sstevel@tonic-gate int 23187c478bd9Sstevel@tonic-gate lpl_topo_flatten(int levels) 23197c478bd9Sstevel@tonic-gate { 23207c478bd9Sstevel@tonic-gate int i; 23217c478bd9Sstevel@tonic-gate uint_t sum; 23227c478bd9Sstevel@tonic-gate lgrp_t *lgrp_cur; 23237c478bd9Sstevel@tonic-gate lpl_t *lpl_cur; 23247c478bd9Sstevel@tonic-gate lpl_t *lpl_root; 23257c478bd9Sstevel@tonic-gate cpupart_t *cp; 23267c478bd9Sstevel@tonic-gate 23277c478bd9Sstevel@tonic-gate if (levels != 2) 23287c478bd9Sstevel@tonic-gate return (0); 23297c478bd9Sstevel@tonic-gate 23307c478bd9Sstevel@tonic-gate /* called w/ cpus paused - grab no locks! */ 23317c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&cpu_lock) || curthread->t_preempt > 0 || 23327c478bd9Sstevel@tonic-gate !lgrp_initialized); 23337c478bd9Sstevel@tonic-gate 23347c478bd9Sstevel@tonic-gate cp = cp_list_head; 23357c478bd9Sstevel@tonic-gate do { 23367c478bd9Sstevel@tonic-gate lpl_root = &cp->cp_lgrploads[lgrp_root->lgrp_id]; 23377c478bd9Sstevel@tonic-gate ASSERT(LGRP_EXISTS(lgrp_root) && (lpl_root->lpl_ncpu > 0)); 23387c478bd9Sstevel@tonic-gate 23397c478bd9Sstevel@tonic-gate for (i = 0; i <= lgrp_alloc_max; i++) { 23407c478bd9Sstevel@tonic-gate lgrp_cur = lgrp_table[i]; 23417c478bd9Sstevel@tonic-gate lpl_cur = &cp->cp_lgrploads[i]; 23427c478bd9Sstevel@tonic-gate 23437c478bd9Sstevel@tonic-gate if ((lgrp_cur == lgrp_root) || 23447c478bd9Sstevel@tonic-gate (!LGRP_EXISTS(lgrp_cur) && 23457c478bd9Sstevel@tonic-gate (lpl_cur->lpl_ncpu == 0))) 23467c478bd9Sstevel@tonic-gate continue; 23477c478bd9Sstevel@tonic-gate 23487c478bd9Sstevel@tonic-gate if (!LGRP_EXISTS(lgrp_cur) && (lpl_cur->lpl_ncpu > 0)) { 23497c478bd9Sstevel@tonic-gate /* 23507c478bd9Sstevel@tonic-gate * this should be a deleted intermediate, so 23517c478bd9Sstevel@tonic-gate * clear it 23527c478bd9Sstevel@tonic-gate */ 23537c478bd9Sstevel@tonic-gate lpl_clear(lpl_cur); 23547c478bd9Sstevel@tonic-gate } else if ((lpl_cur->lpl_nrset == 1) && 23557c478bd9Sstevel@tonic-gate (lpl_cur->lpl_rset[0] == lpl_cur) && 23567c478bd9Sstevel@tonic-gate ((lpl_cur->lpl_parent->lpl_ncpu == 0) || 23577c478bd9Sstevel@tonic-gate (!LGRP_EXISTS(lpl_cur->lpl_parent->lpl_lgrp)))) { 23587c478bd9Sstevel@tonic-gate /* 23597c478bd9Sstevel@tonic-gate * this is a leaf whose parent was deleted, or 23607c478bd9Sstevel@tonic-gate * whose parent had their lgrp deleted. (And 23617c478bd9Sstevel@tonic-gate * whose parent will soon be deleted). Point 23627c478bd9Sstevel@tonic-gate * this guy back to the root lpl. 23637c478bd9Sstevel@tonic-gate */ 23647c478bd9Sstevel@tonic-gate lpl_cur->lpl_parent = lpl_root; 23657c478bd9Sstevel@tonic-gate lpl_rset_add(lpl_root, lpl_cur); 23667c478bd9Sstevel@tonic-gate } 23677c478bd9Sstevel@tonic-gate 23687c478bd9Sstevel@tonic-gate } 23697c478bd9Sstevel@tonic-gate 23707c478bd9Sstevel@tonic-gate /* 23717c478bd9Sstevel@tonic-gate * Now that we're done, make sure the count on the root lpl is 23727c478bd9Sstevel@tonic-gate * correct, and update the hints of the children for the sake of 23737c478bd9Sstevel@tonic-gate * thoroughness 23747c478bd9Sstevel@tonic-gate */ 23757c478bd9Sstevel@tonic-gate for (i = sum = 0; i < lpl_root->lpl_nrset; i++) { 23767c478bd9Sstevel@tonic-gate sum += lpl_root->lpl_rset[i]->lpl_ncpu; 23777c478bd9Sstevel@tonic-gate } 23787c478bd9Sstevel@tonic-gate lpl_root->lpl_ncpu = sum; 23797c478bd9Sstevel@tonic-gate lpl_child_update(lpl_root, cp); 23807c478bd9Sstevel@tonic-gate 23817c478bd9Sstevel@tonic-gate cp = cp->cp_next; 23827c478bd9Sstevel@tonic-gate } while (cp != cp_list_head); 23837c478bd9Sstevel@tonic-gate 23847c478bd9Sstevel@tonic-gate return (levels); 23857c478bd9Sstevel@tonic-gate } 23867c478bd9Sstevel@tonic-gate 23877c478bd9Sstevel@tonic-gate /* 23887c478bd9Sstevel@tonic-gate * Insert a lpl into the resource hierarchy and create any additional lpls that 23897c478bd9Sstevel@tonic-gate * are necessary to represent the varying states of locality for the cpu 23907c478bd9Sstevel@tonic-gate * resoruces newly added to the partition. 23917c478bd9Sstevel@tonic-gate * 23927c478bd9Sstevel@tonic-gate * This routine is clever enough that it can correctly add resources from the 23937c478bd9Sstevel@tonic-gate * new leaf into both direct and indirect resource sets in the hierarchy. (Ie, 23947c478bd9Sstevel@tonic-gate * those for which the lpl is a leaf as opposed to simply a named equally local 23957c478bd9Sstevel@tonic-gate * resource). The one special case that needs additional processing is when a 23967c478bd9Sstevel@tonic-gate * new intermediate lpl is introduced. Since the main loop only traverses 23977c478bd9Sstevel@tonic-gate * looking to add the leaf resource where it does not yet exist, additional work 23987c478bd9Sstevel@tonic-gate * is necessary to add other leaf resources that may need to exist in the newly 23997c478bd9Sstevel@tonic-gate * created intermediate. This is performed by the second inner loop, and is 24007c478bd9Sstevel@tonic-gate * only done when the check for more than one overlapping resource succeeds. 24017c478bd9Sstevel@tonic-gate */ 24027c478bd9Sstevel@tonic-gate 24037c478bd9Sstevel@tonic-gate void 24047c478bd9Sstevel@tonic-gate lpl_leaf_insert(lpl_t *lpl_leaf, cpupart_t *cpupart) 24057c478bd9Sstevel@tonic-gate { 24067c478bd9Sstevel@tonic-gate int i; 24077c478bd9Sstevel@tonic-gate int j; 24087c478bd9Sstevel@tonic-gate int hint; 24097c478bd9Sstevel@tonic-gate int rset_num_intersect; 24107c478bd9Sstevel@tonic-gate lgrp_t *lgrp_cur; 24117c478bd9Sstevel@tonic-gate lpl_t *lpl_cur; 24127c478bd9Sstevel@tonic-gate lpl_t *lpl_parent; 24137c478bd9Sstevel@tonic-gate lgrpid_t parent_id; 24147c478bd9Sstevel@tonic-gate klgrpset_t rset_intersect; /* resources in cpupart and lgrp */ 24157c478bd9Sstevel@tonic-gate 24167c478bd9Sstevel@tonic-gate for (i = 0; i <= lgrp_alloc_max; i++) { 24177c478bd9Sstevel@tonic-gate lgrp_cur = lgrp_table[i]; 24187c478bd9Sstevel@tonic-gate 24197c478bd9Sstevel@tonic-gate /* 24207c478bd9Sstevel@tonic-gate * Don't insert if the lgrp isn't there, if the leaf isn't 24217c478bd9Sstevel@tonic-gate * contained within the current lgrp, or if the current lgrp has 24227c478bd9Sstevel@tonic-gate * no leaves in this partition 24237c478bd9Sstevel@tonic-gate */ 24247c478bd9Sstevel@tonic-gate 24257c478bd9Sstevel@tonic-gate if (!LGRP_EXISTS(lgrp_cur) || 24267c478bd9Sstevel@tonic-gate !klgrpset_ismember(lgrp_cur->lgrp_set[LGRP_RSRC_CPU], 24277c478bd9Sstevel@tonic-gate lpl_leaf->lpl_lgrpid) || 24287c478bd9Sstevel@tonic-gate !klgrpset_intersects(lgrp_cur->lgrp_leaves, 24297c478bd9Sstevel@tonic-gate cpupart->cp_lgrpset)) 24307c478bd9Sstevel@tonic-gate continue; 24317c478bd9Sstevel@tonic-gate 24327c478bd9Sstevel@tonic-gate lpl_cur = &cpupart->cp_lgrploads[lgrp_cur->lgrp_id]; 24337c478bd9Sstevel@tonic-gate if (lgrp_cur->lgrp_parent != NULL) { 24347c478bd9Sstevel@tonic-gate /* if lgrp has a parent, assign it properly */ 24357c478bd9Sstevel@tonic-gate parent_id = lgrp_cur->lgrp_parent->lgrp_id; 24367c478bd9Sstevel@tonic-gate lpl_parent = &cpupart->cp_lgrploads[parent_id]; 24377c478bd9Sstevel@tonic-gate } else { 24387c478bd9Sstevel@tonic-gate /* if not, make sure parent ptr gets set to null */ 24397c478bd9Sstevel@tonic-gate lpl_parent = NULL; 24407c478bd9Sstevel@tonic-gate } 24417c478bd9Sstevel@tonic-gate 24427c478bd9Sstevel@tonic-gate if (lpl_cur == lpl_leaf) { 24437c478bd9Sstevel@tonic-gate /* 24447c478bd9Sstevel@tonic-gate * Almost all leaf state was initialized elsewhere. The 24457c478bd9Sstevel@tonic-gate * only thing left to do is to set the parent. 24467c478bd9Sstevel@tonic-gate */ 24477c478bd9Sstevel@tonic-gate lpl_cur->lpl_parent = lpl_parent; 24487c478bd9Sstevel@tonic-gate continue; 24497c478bd9Sstevel@tonic-gate } 24507c478bd9Sstevel@tonic-gate 24517c478bd9Sstevel@tonic-gate /* 24527c478bd9Sstevel@tonic-gate * Initialize intermediate lpl 24537c478bd9Sstevel@tonic-gate * Save this lpl's hint though. Since we're changing this 24547c478bd9Sstevel@tonic-gate * lpl's resources, we need to update the hint in this lpl's 24557c478bd9Sstevel@tonic-gate * children, but the hint in this lpl is unaffected and 24567c478bd9Sstevel@tonic-gate * should be preserved. 24577c478bd9Sstevel@tonic-gate */ 24587c478bd9Sstevel@tonic-gate hint = lpl_cur->lpl_hint; 24597c478bd9Sstevel@tonic-gate 24607c478bd9Sstevel@tonic-gate lpl_clear(lpl_cur); 24617c478bd9Sstevel@tonic-gate lpl_init(lpl_cur, lpl_leaf, lgrp_cur); 24627c478bd9Sstevel@tonic-gate 24637c478bd9Sstevel@tonic-gate lpl_cur->lpl_hint = hint; 24647c478bd9Sstevel@tonic-gate lpl_cur->lpl_parent = lpl_parent; 24657c478bd9Sstevel@tonic-gate 24667c478bd9Sstevel@tonic-gate /* does new lpl need to be populated with other resources? */ 24677c478bd9Sstevel@tonic-gate rset_intersect = 24687c478bd9Sstevel@tonic-gate klgrpset_intersects(lgrp_cur->lgrp_set[LGRP_RSRC_CPU], 24697c478bd9Sstevel@tonic-gate cpupart->cp_lgrpset); 24707c478bd9Sstevel@tonic-gate klgrpset_nlgrps(rset_intersect, rset_num_intersect); 24717c478bd9Sstevel@tonic-gate 24727c478bd9Sstevel@tonic-gate if (rset_num_intersect > 1) { 24737c478bd9Sstevel@tonic-gate /* 24747c478bd9Sstevel@tonic-gate * If so, figure out what lpls have resources that 24757c478bd9Sstevel@tonic-gate * intersect this one, and add them. 24767c478bd9Sstevel@tonic-gate */ 24777c478bd9Sstevel@tonic-gate for (j = 0; j <= lgrp_alloc_max; j++) { 24787c478bd9Sstevel@tonic-gate lgrp_t *lgrp_cand; /* candidate lgrp */ 24797c478bd9Sstevel@tonic-gate lpl_t *lpl_cand; /* candidate lpl */ 24807c478bd9Sstevel@tonic-gate 24817c478bd9Sstevel@tonic-gate lgrp_cand = lgrp_table[j]; 24827c478bd9Sstevel@tonic-gate if (!LGRP_EXISTS(lgrp_cand) || 24837c478bd9Sstevel@tonic-gate !klgrpset_ismember(rset_intersect, 24847c478bd9Sstevel@tonic-gate lgrp_cand->lgrp_id)) 24857c478bd9Sstevel@tonic-gate continue; 24867c478bd9Sstevel@tonic-gate lpl_cand = 24877c478bd9Sstevel@tonic-gate &cpupart->cp_lgrploads[lgrp_cand->lgrp_id]; 24887c478bd9Sstevel@tonic-gate lpl_rset_add(lpl_cur, lpl_cand); 24897c478bd9Sstevel@tonic-gate } 24907c478bd9Sstevel@tonic-gate } 24917c478bd9Sstevel@tonic-gate /* 24927c478bd9Sstevel@tonic-gate * This lpl's rset has changed. Update the hint in it's 24937c478bd9Sstevel@tonic-gate * children. 24947c478bd9Sstevel@tonic-gate */ 24957c478bd9Sstevel@tonic-gate lpl_child_update(lpl_cur, cpupart); 24967c478bd9Sstevel@tonic-gate } 24977c478bd9Sstevel@tonic-gate } 24987c478bd9Sstevel@tonic-gate 24997c478bd9Sstevel@tonic-gate /* 25007c478bd9Sstevel@tonic-gate * remove a lpl from the hierarchy of resources, clearing its state when 25017c478bd9Sstevel@tonic-gate * finished. If the lpls at the intermediate levels of the hierarchy have no 25027c478bd9Sstevel@tonic-gate * remaining resources, or no longer name a leaf resource in the cpu-partition, 25037c478bd9Sstevel@tonic-gate * delete them as well. 25047c478bd9Sstevel@tonic-gate */ 25057c478bd9Sstevel@tonic-gate 25067c478bd9Sstevel@tonic-gate void 25077c478bd9Sstevel@tonic-gate lpl_leaf_remove(lpl_t *lpl_leaf, cpupart_t *cpupart) 25087c478bd9Sstevel@tonic-gate { 25097c478bd9Sstevel@tonic-gate int i; 25107c478bd9Sstevel@tonic-gate lgrp_t *lgrp_cur; 25117c478bd9Sstevel@tonic-gate lpl_t *lpl_cur; 25127c478bd9Sstevel@tonic-gate klgrpset_t leaf_intersect; /* intersection of leaves */ 25137c478bd9Sstevel@tonic-gate 25147c478bd9Sstevel@tonic-gate for (i = 0; i <= lgrp_alloc_max; i++) { 25157c478bd9Sstevel@tonic-gate lgrp_cur = lgrp_table[i]; 25167c478bd9Sstevel@tonic-gate 25177c478bd9Sstevel@tonic-gate /* 25187c478bd9Sstevel@tonic-gate * Don't attempt to remove from lgrps that aren't there, that 25197c478bd9Sstevel@tonic-gate * don't contain our leaf, or from the leaf itself. (We do that 25207c478bd9Sstevel@tonic-gate * later) 25217c478bd9Sstevel@tonic-gate */ 25227c478bd9Sstevel@tonic-gate 25237c478bd9Sstevel@tonic-gate if (!LGRP_EXISTS(lgrp_cur)) 25247c478bd9Sstevel@tonic-gate continue; 25257c478bd9Sstevel@tonic-gate 25267c478bd9Sstevel@tonic-gate lpl_cur = &cpupart->cp_lgrploads[lgrp_cur->lgrp_id]; 25277c478bd9Sstevel@tonic-gate 25287c478bd9Sstevel@tonic-gate if (!klgrpset_ismember(lgrp_cur->lgrp_set[LGRP_RSRC_CPU], 25297c478bd9Sstevel@tonic-gate lpl_leaf->lpl_lgrpid) || 25307c478bd9Sstevel@tonic-gate (lpl_cur == lpl_leaf)) { 25317c478bd9Sstevel@tonic-gate continue; 25327c478bd9Sstevel@tonic-gate } 25337c478bd9Sstevel@tonic-gate 25347c478bd9Sstevel@tonic-gate /* 25357c478bd9Sstevel@tonic-gate * This is a slightly sleazy simplification in that we have 25367c478bd9Sstevel@tonic-gate * already marked the cp_lgrpset as no longer containing the 25377c478bd9Sstevel@tonic-gate * leaf we've deleted. Any lpls that pass the above checks 25387c478bd9Sstevel@tonic-gate * based upon lgrp membership but not necessarily cpu-part 25397c478bd9Sstevel@tonic-gate * membership also get cleared by the checks below. Currently 25407c478bd9Sstevel@tonic-gate * this is harmless, as the lpls should be empty anyway. 25417c478bd9Sstevel@tonic-gate * 25427c478bd9Sstevel@tonic-gate * In particular, we want to preserve lpls that have additional 25437c478bd9Sstevel@tonic-gate * leaf resources, even though we don't yet have a processor 25447c478bd9Sstevel@tonic-gate * architecture that represents resources this way. 25457c478bd9Sstevel@tonic-gate */ 25467c478bd9Sstevel@tonic-gate 25477c478bd9Sstevel@tonic-gate leaf_intersect = klgrpset_intersects(lgrp_cur->lgrp_leaves, 25487c478bd9Sstevel@tonic-gate cpupart->cp_lgrpset); 25497c478bd9Sstevel@tonic-gate 25507c478bd9Sstevel@tonic-gate lpl_rset_del(lpl_cur, lpl_leaf); 25517c478bd9Sstevel@tonic-gate if ((lpl_cur->lpl_nrset == 0) || (!leaf_intersect)) { 25527c478bd9Sstevel@tonic-gate lpl_clear(lpl_cur); 25537c478bd9Sstevel@tonic-gate } else { 25547c478bd9Sstevel@tonic-gate /* 25557c478bd9Sstevel@tonic-gate * Update this lpl's children 25567c478bd9Sstevel@tonic-gate */ 25577c478bd9Sstevel@tonic-gate lpl_child_update(lpl_cur, cpupart); 25587c478bd9Sstevel@tonic-gate } 25597c478bd9Sstevel@tonic-gate } 25607c478bd9Sstevel@tonic-gate lpl_clear(lpl_leaf); 25617c478bd9Sstevel@tonic-gate } 25627c478bd9Sstevel@tonic-gate 25637c478bd9Sstevel@tonic-gate /* 25647c478bd9Sstevel@tonic-gate * add a cpu to a partition in terms of lgrp load avg bookeeping 25657c478bd9Sstevel@tonic-gate * 25667c478bd9Sstevel@tonic-gate * The lpl (cpu partition load average information) is now arranged in a 25677c478bd9Sstevel@tonic-gate * hierarchical fashion whereby resources that are closest, ie. most local, to 25687c478bd9Sstevel@tonic-gate * the cpu in question are considered to be leaves in a tree of resources. 25697c478bd9Sstevel@tonic-gate * There are two general cases for cpu additon: 25707c478bd9Sstevel@tonic-gate * 25717c478bd9Sstevel@tonic-gate * 1. A lpl structure that contains resources already in the hierarchy tree. 25727c478bd9Sstevel@tonic-gate * In this case, all of the associated lpl relationships have been defined, and 25737c478bd9Sstevel@tonic-gate * all that is necessary is that we link the new cpu into the per-lpl list of 25747c478bd9Sstevel@tonic-gate * cpus, and increment the ncpu count of all places where this cpu resource will 25757c478bd9Sstevel@tonic-gate * be accounted for. lpl_cpu_adjcnt updates the cpu count, and the cpu pointer 25767c478bd9Sstevel@tonic-gate * pushing is accomplished by this routine. 25777c478bd9Sstevel@tonic-gate * 25787c478bd9Sstevel@tonic-gate * 2. The lpl to contain the resources in this cpu-partition for this lgrp does 25797c478bd9Sstevel@tonic-gate * not exist yet. In this case, it is necessary to build the leaf lpl, and 25807c478bd9Sstevel@tonic-gate * construct the hierarchy of state necessary to name it's more distant 25817c478bd9Sstevel@tonic-gate * resources, if they should exist. The leaf structure is initialized by this 25827c478bd9Sstevel@tonic-gate * routine, as is the cpu-partition state for the lgrp membership. This routine 25837c478bd9Sstevel@tonic-gate * also calls lpl_leaf_insert() which inserts the named lpl into the hierarchy 25847c478bd9Sstevel@tonic-gate * and builds all of the "ancestoral" state necessary to identify resources at 25857c478bd9Sstevel@tonic-gate * differing levels of locality. 25867c478bd9Sstevel@tonic-gate */ 25877c478bd9Sstevel@tonic-gate void 25887c478bd9Sstevel@tonic-gate lgrp_part_add_cpu(cpu_t *cp, lgrp_id_t lgrpid) 25897c478bd9Sstevel@tonic-gate { 25907c478bd9Sstevel@tonic-gate cpupart_t *cpupart; 25917c478bd9Sstevel@tonic-gate lgrp_t *lgrp_leaf; 25927c478bd9Sstevel@tonic-gate lpl_t *lpl_leaf; 25937c478bd9Sstevel@tonic-gate 25947c478bd9Sstevel@tonic-gate /* called sometimes w/ cpus paused - grab no locks */ 25957c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&cpu_lock) || !lgrp_initialized); 25967c478bd9Sstevel@tonic-gate 25977c478bd9Sstevel@tonic-gate cpupart = cp->cpu_part; 25987c478bd9Sstevel@tonic-gate lgrp_leaf = lgrp_table[lgrpid]; 25997c478bd9Sstevel@tonic-gate 26007c478bd9Sstevel@tonic-gate /* don't add non-existent lgrp */ 26017c478bd9Sstevel@tonic-gate ASSERT(LGRP_EXISTS(lgrp_leaf)); 26027c478bd9Sstevel@tonic-gate lpl_leaf = &cpupart->cp_lgrploads[lgrpid]; 26037c478bd9Sstevel@tonic-gate cp->cpu_lpl = lpl_leaf; 26047c478bd9Sstevel@tonic-gate 26057c478bd9Sstevel@tonic-gate /* only leaf lpls contain cpus */ 26067c478bd9Sstevel@tonic-gate 26077c478bd9Sstevel@tonic-gate if (lpl_leaf->lpl_ncpu++ == 0) { 26087c478bd9Sstevel@tonic-gate lpl_init(lpl_leaf, lpl_leaf, lgrp_leaf); 26097c478bd9Sstevel@tonic-gate klgrpset_add(cpupart->cp_lgrpset, lgrpid); 26107c478bd9Sstevel@tonic-gate lpl_leaf_insert(lpl_leaf, cpupart); 26117c478bd9Sstevel@tonic-gate } else { 26127c478bd9Sstevel@tonic-gate /* 26137c478bd9Sstevel@tonic-gate * the lpl should already exist in the parent, so just update 26147c478bd9Sstevel@tonic-gate * the count of available CPUs 26157c478bd9Sstevel@tonic-gate */ 26167c478bd9Sstevel@tonic-gate lpl_cpu_adjcnt(LPL_INCREMENT, cp); 26177c478bd9Sstevel@tonic-gate } 26187c478bd9Sstevel@tonic-gate 26197c478bd9Sstevel@tonic-gate /* link cpu into list of cpus in lpl */ 26207c478bd9Sstevel@tonic-gate 26217c478bd9Sstevel@tonic-gate if (lpl_leaf->lpl_cpus) { 26227c478bd9Sstevel@tonic-gate cp->cpu_next_lpl = lpl_leaf->lpl_cpus; 26237c478bd9Sstevel@tonic-gate cp->cpu_prev_lpl = lpl_leaf->lpl_cpus->cpu_prev_lpl; 26247c478bd9Sstevel@tonic-gate lpl_leaf->lpl_cpus->cpu_prev_lpl->cpu_next_lpl = cp; 26257c478bd9Sstevel@tonic-gate lpl_leaf->lpl_cpus->cpu_prev_lpl = cp; 26267c478bd9Sstevel@tonic-gate } else { 26277c478bd9Sstevel@tonic-gate /* 26287c478bd9Sstevel@tonic-gate * We increment ncpu immediately after we create a new leaf 26297c478bd9Sstevel@tonic-gate * lpl, so assert that ncpu == 1 for the case where we don't 26307c478bd9Sstevel@tonic-gate * have any cpu pointers yet. 26317c478bd9Sstevel@tonic-gate */ 26327c478bd9Sstevel@tonic-gate ASSERT(lpl_leaf->lpl_ncpu == 1); 26337c478bd9Sstevel@tonic-gate lpl_leaf->lpl_cpus = cp->cpu_next_lpl = cp->cpu_prev_lpl = cp; 26347c478bd9Sstevel@tonic-gate } 26357c478bd9Sstevel@tonic-gate 26367c478bd9Sstevel@tonic-gate } 26377c478bd9Sstevel@tonic-gate 26387c478bd9Sstevel@tonic-gate 26397c478bd9Sstevel@tonic-gate /* 26407c478bd9Sstevel@tonic-gate * remove a cpu from a partition in terms of lgrp load avg bookeeping 26417c478bd9Sstevel@tonic-gate * 26427c478bd9Sstevel@tonic-gate * The lpl (cpu partition load average information) is now arranged in a 26437c478bd9Sstevel@tonic-gate * hierarchical fashion whereby resources that are closest, ie. most local, to 26447c478bd9Sstevel@tonic-gate * the cpu in question are considered to be leaves in a tree of resources. 26457c478bd9Sstevel@tonic-gate * There are two removal cases in question: 26467c478bd9Sstevel@tonic-gate * 26477c478bd9Sstevel@tonic-gate * 1. Removal of the resource in the leaf leaves other resources remaining in 26487c478bd9Sstevel@tonic-gate * that leaf. (Another cpu still exists at this level of locality). In this 26497c478bd9Sstevel@tonic-gate * case, the count of available cpus is decremented in all assocated lpls by 26507c478bd9Sstevel@tonic-gate * calling lpl_adj_cpucnt(), and the pointer to the removed cpu is pruned 26517c478bd9Sstevel@tonic-gate * from the per-cpu lpl list. 26527c478bd9Sstevel@tonic-gate * 26537c478bd9Sstevel@tonic-gate * 2. Removal of the resource results in the lpl containing no resources. (It's 26547c478bd9Sstevel@tonic-gate * empty) In this case, all of what has occurred for the first step must take 26557c478bd9Sstevel@tonic-gate * place; however, additionally we must remove the lpl structure itself, prune 26567c478bd9Sstevel@tonic-gate * out any stranded lpls that do not directly name a leaf resource, and mark the 26577c478bd9Sstevel@tonic-gate * cpu partition in question as no longer containing resources from the lgrp of 26587c478bd9Sstevel@tonic-gate * the lpl that has been delted. Cpu-partition changes are handled by this 26597c478bd9Sstevel@tonic-gate * method, but the lpl_leaf_remove function deals with the details of pruning 26607c478bd9Sstevel@tonic-gate * out the empty lpl and any of its orphaned direct ancestors. 26617c478bd9Sstevel@tonic-gate */ 26627c478bd9Sstevel@tonic-gate void 26637c478bd9Sstevel@tonic-gate lgrp_part_del_cpu(cpu_t *cp) 26647c478bd9Sstevel@tonic-gate { 26657c478bd9Sstevel@tonic-gate lpl_t *lpl; 26667c478bd9Sstevel@tonic-gate lpl_t *leaf_lpl; 26677c478bd9Sstevel@tonic-gate lgrp_t *lgrp_leaf; 26687c478bd9Sstevel@tonic-gate 26697c478bd9Sstevel@tonic-gate /* called sometimes w/ cpus paused - grab no locks */ 26707c478bd9Sstevel@tonic-gate 26717c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&cpu_lock) || !lgrp_initialized); 26727c478bd9Sstevel@tonic-gate 26737c478bd9Sstevel@tonic-gate lpl = leaf_lpl = cp->cpu_lpl; 26747c478bd9Sstevel@tonic-gate lgrp_leaf = leaf_lpl->lpl_lgrp; 26757c478bd9Sstevel@tonic-gate 26767c478bd9Sstevel@tonic-gate /* don't delete a leaf that isn't there */ 26777c478bd9Sstevel@tonic-gate ASSERT(LGRP_EXISTS(lgrp_leaf)); 26787c478bd9Sstevel@tonic-gate 26797c478bd9Sstevel@tonic-gate /* no double-deletes */ 26807c478bd9Sstevel@tonic-gate ASSERT(lpl->lpl_ncpu); 26817c478bd9Sstevel@tonic-gate if (--lpl->lpl_ncpu == 0) { 26827c478bd9Sstevel@tonic-gate /* 26837c478bd9Sstevel@tonic-gate * This was the last cpu in this lgroup for this partition, 26847c478bd9Sstevel@tonic-gate * clear its bit in the partition's lgroup bitmask 26857c478bd9Sstevel@tonic-gate */ 26867c478bd9Sstevel@tonic-gate klgrpset_del(cp->cpu_part->cp_lgrpset, lpl->lpl_lgrpid); 26877c478bd9Sstevel@tonic-gate 26887c478bd9Sstevel@tonic-gate /* eliminate remaning lpl link pointers in cpu, lpl */ 26897c478bd9Sstevel@tonic-gate lpl->lpl_cpus = cp->cpu_next_lpl = cp->cpu_prev_lpl = NULL; 26907c478bd9Sstevel@tonic-gate 26917c478bd9Sstevel@tonic-gate lpl_leaf_remove(leaf_lpl, cp->cpu_part); 26927c478bd9Sstevel@tonic-gate } else { 26937c478bd9Sstevel@tonic-gate 26947c478bd9Sstevel@tonic-gate /* unlink cpu from lists of cpus in lpl */ 26957c478bd9Sstevel@tonic-gate cp->cpu_prev_lpl->cpu_next_lpl = cp->cpu_next_lpl; 26967c478bd9Sstevel@tonic-gate cp->cpu_next_lpl->cpu_prev_lpl = cp->cpu_prev_lpl; 26977c478bd9Sstevel@tonic-gate if (lpl->lpl_cpus == cp) { 26987c478bd9Sstevel@tonic-gate lpl->lpl_cpus = cp->cpu_next_lpl; 26997c478bd9Sstevel@tonic-gate } 27007c478bd9Sstevel@tonic-gate 27017c478bd9Sstevel@tonic-gate /* 27027c478bd9Sstevel@tonic-gate * Update the cpu count in the lpls associated with parent 27037c478bd9Sstevel@tonic-gate * lgroups. 27047c478bd9Sstevel@tonic-gate */ 27057c478bd9Sstevel@tonic-gate lpl_cpu_adjcnt(LPL_DECREMENT, cp); 27067c478bd9Sstevel@tonic-gate 27077c478bd9Sstevel@tonic-gate } 27087c478bd9Sstevel@tonic-gate /* clear cpu's lpl ptr when we're all done */ 27097c478bd9Sstevel@tonic-gate cp->cpu_lpl = NULL; 27107c478bd9Sstevel@tonic-gate } 27117c478bd9Sstevel@tonic-gate 27127c478bd9Sstevel@tonic-gate /* 27137c478bd9Sstevel@tonic-gate * Recompute load average for the specified partition/lgrp fragment. 27147c478bd9Sstevel@tonic-gate * 27157c478bd9Sstevel@tonic-gate * We rely on the fact that this routine is called from the clock thread 27167c478bd9Sstevel@tonic-gate * at a point before the clock thread can block (i.e. before its first 27177c478bd9Sstevel@tonic-gate * lock request). Since the clock thread can not be preempted (since it 27187c478bd9Sstevel@tonic-gate * runs at highest priority), we know that cpu partitions can not change 27197c478bd9Sstevel@tonic-gate * (since doing so would require either the repartition requester or the 27207c478bd9Sstevel@tonic-gate * cpu_pause thread to run on this cpu), so we can update the cpu's load 27217c478bd9Sstevel@tonic-gate * without grabbing cpu_lock. 27227c478bd9Sstevel@tonic-gate */ 27237c478bd9Sstevel@tonic-gate void 27247c478bd9Sstevel@tonic-gate lgrp_loadavg(lpl_t *lpl, uint_t nrcpus, int ageflag) 27257c478bd9Sstevel@tonic-gate { 27267c478bd9Sstevel@tonic-gate uint_t ncpu; 27277c478bd9Sstevel@tonic-gate int64_t old, new, f; 27287c478bd9Sstevel@tonic-gate 27297c478bd9Sstevel@tonic-gate /* 27307c478bd9Sstevel@tonic-gate * 1 - exp(-1/(20 * ncpu)) << 13 = 400 for 1 cpu... 27317c478bd9Sstevel@tonic-gate */ 27327c478bd9Sstevel@tonic-gate static short expval[] = { 27337c478bd9Sstevel@tonic-gate 0, 3196, 1618, 1083, 27347c478bd9Sstevel@tonic-gate 814, 652, 543, 466, 27357c478bd9Sstevel@tonic-gate 408, 363, 326, 297, 27367c478bd9Sstevel@tonic-gate 272, 251, 233, 218, 27377c478bd9Sstevel@tonic-gate 204, 192, 181, 172, 27387c478bd9Sstevel@tonic-gate 163, 155, 148, 142, 27397c478bd9Sstevel@tonic-gate 136, 130, 125, 121, 27407c478bd9Sstevel@tonic-gate 116, 112, 109, 105 27417c478bd9Sstevel@tonic-gate }; 27427c478bd9Sstevel@tonic-gate 27437c478bd9Sstevel@tonic-gate /* ASSERT (called from clock level) */ 27447c478bd9Sstevel@tonic-gate 27457c478bd9Sstevel@tonic-gate if ((lpl == NULL) || /* we're booting - this is easiest for now */ 27467c478bd9Sstevel@tonic-gate ((ncpu = lpl->lpl_ncpu) == 0)) { 27477c478bd9Sstevel@tonic-gate return; 27487c478bd9Sstevel@tonic-gate } 27497c478bd9Sstevel@tonic-gate 27507c478bd9Sstevel@tonic-gate for (;;) { 27517c478bd9Sstevel@tonic-gate 27527c478bd9Sstevel@tonic-gate if (ncpu >= sizeof (expval) / sizeof (expval[0])) 27537c478bd9Sstevel@tonic-gate f = expval[1]/ncpu; /* good approx. for large ncpu */ 27547c478bd9Sstevel@tonic-gate else 27557c478bd9Sstevel@tonic-gate f = expval[ncpu]; 27567c478bd9Sstevel@tonic-gate 27577c478bd9Sstevel@tonic-gate /* 27587c478bd9Sstevel@tonic-gate * Modify the load average atomically to avoid losing 27597c478bd9Sstevel@tonic-gate * anticipatory load updates (see lgrp_move_thread()). 27607c478bd9Sstevel@tonic-gate */ 27617c478bd9Sstevel@tonic-gate if (ageflag) { 27627c478bd9Sstevel@tonic-gate /* 27637c478bd9Sstevel@tonic-gate * We're supposed to both update and age the load. 27647c478bd9Sstevel@tonic-gate * This happens 10 times/sec. per cpu. We do a 27657c478bd9Sstevel@tonic-gate * little hoop-jumping to avoid integer overflow. 27667c478bd9Sstevel@tonic-gate */ 27677c478bd9Sstevel@tonic-gate int64_t q, r; 27687c478bd9Sstevel@tonic-gate 27697c478bd9Sstevel@tonic-gate do { 27707c478bd9Sstevel@tonic-gate old = new = lpl->lpl_loadavg; 27717c478bd9Sstevel@tonic-gate q = (old >> 16) << 7; 27727c478bd9Sstevel@tonic-gate r = (old & 0xffff) << 7; 27737c478bd9Sstevel@tonic-gate new += ((long long)(nrcpus - q) * f - 27747c478bd9Sstevel@tonic-gate ((r * f) >> 16)) >> 7; 27757c478bd9Sstevel@tonic-gate 27767c478bd9Sstevel@tonic-gate /* 27777c478bd9Sstevel@tonic-gate * Check for overflow 27787c478bd9Sstevel@tonic-gate */ 27797c478bd9Sstevel@tonic-gate if (new > LGRP_LOADAVG_MAX) 27807c478bd9Sstevel@tonic-gate new = LGRP_LOADAVG_MAX; 27817c478bd9Sstevel@tonic-gate else if (new < 0) 27827c478bd9Sstevel@tonic-gate new = 0; 27837c478bd9Sstevel@tonic-gate } while (cas32((lgrp_load_t *)&lpl->lpl_loadavg, old, 27847c478bd9Sstevel@tonic-gate new) != old); 27857c478bd9Sstevel@tonic-gate } else { 27867c478bd9Sstevel@tonic-gate /* 27877c478bd9Sstevel@tonic-gate * We're supposed to update the load, but not age it. 27887c478bd9Sstevel@tonic-gate * This option is used to update the load (which either 27897c478bd9Sstevel@tonic-gate * has already been aged in this 1/10 sec. interval or 27907c478bd9Sstevel@tonic-gate * soon will be) to account for a remotely executing 27917c478bd9Sstevel@tonic-gate * thread. 27927c478bd9Sstevel@tonic-gate */ 27937c478bd9Sstevel@tonic-gate do { 27947c478bd9Sstevel@tonic-gate old = new = lpl->lpl_loadavg; 27957c478bd9Sstevel@tonic-gate new += f; 27967c478bd9Sstevel@tonic-gate /* 27977c478bd9Sstevel@tonic-gate * Check for overflow 27987c478bd9Sstevel@tonic-gate * Underflow not possible here 27997c478bd9Sstevel@tonic-gate */ 28007c478bd9Sstevel@tonic-gate if (new < old) 28017c478bd9Sstevel@tonic-gate new = LGRP_LOADAVG_MAX; 28027c478bd9Sstevel@tonic-gate } while (cas32((lgrp_load_t *)&lpl->lpl_loadavg, old, 28037c478bd9Sstevel@tonic-gate new) != old); 28047c478bd9Sstevel@tonic-gate } 28057c478bd9Sstevel@tonic-gate 28067c478bd9Sstevel@tonic-gate /* 28077c478bd9Sstevel@tonic-gate * Do the same for this lpl's parent 28087c478bd9Sstevel@tonic-gate */ 28097c478bd9Sstevel@tonic-gate if ((lpl = lpl->lpl_parent) == NULL) 28107c478bd9Sstevel@tonic-gate break; 28117c478bd9Sstevel@tonic-gate ncpu = lpl->lpl_ncpu; 28127c478bd9Sstevel@tonic-gate } 28137c478bd9Sstevel@tonic-gate } 28147c478bd9Sstevel@tonic-gate 28157c478bd9Sstevel@tonic-gate /* 28167c478bd9Sstevel@tonic-gate * Initialize lpl topology in the target based on topology currently present in 28177c478bd9Sstevel@tonic-gate * lpl_bootstrap. 28187c478bd9Sstevel@tonic-gate * 28197c478bd9Sstevel@tonic-gate * lpl_topo_bootstrap is only called once from cpupart_initialize_default() to 28207c478bd9Sstevel@tonic-gate * initialize cp_default list of lpls. Up to this point all topology operations 28217c478bd9Sstevel@tonic-gate * were performed using lpl_bootstrap. Now cp_default has its own list of lpls 28227c478bd9Sstevel@tonic-gate * and all subsequent lpl operations should use it instead of lpl_bootstrap. The 28237c478bd9Sstevel@tonic-gate * `target' points to the list of lpls in cp_default and `size' is the size of 28247c478bd9Sstevel@tonic-gate * this list. 28257c478bd9Sstevel@tonic-gate * 28267c478bd9Sstevel@tonic-gate * This function walks the lpl topology in lpl_bootstrap and does for things: 28277c478bd9Sstevel@tonic-gate * 28287c478bd9Sstevel@tonic-gate * 1) Copies all fields from lpl_bootstrap to the target. 28297c478bd9Sstevel@tonic-gate * 28307c478bd9Sstevel@tonic-gate * 2) Sets CPU0 lpl pointer to the correct element of the target list. 28317c478bd9Sstevel@tonic-gate * 28327c478bd9Sstevel@tonic-gate * 3) Updates lpl_parent pointers to point to the lpls in the target list 28337c478bd9Sstevel@tonic-gate * instead of lpl_bootstrap. 28347c478bd9Sstevel@tonic-gate * 28357c478bd9Sstevel@tonic-gate * 4) Updates pointers in the resource list of the target to point to the lpls 28367c478bd9Sstevel@tonic-gate * in the target list instead of lpl_bootstrap. 28377c478bd9Sstevel@tonic-gate * 28387c478bd9Sstevel@tonic-gate * After lpl_topo_bootstrap() completes, target contains the same information 28397c478bd9Sstevel@tonic-gate * that would be present there if it were used during boot instead of 28407c478bd9Sstevel@tonic-gate * lpl_bootstrap. There is no need in information in lpl_bootstrap after this 28417c478bd9Sstevel@tonic-gate * and it is bzeroed. 28427c478bd9Sstevel@tonic-gate */ 28437c478bd9Sstevel@tonic-gate void 28447c478bd9Sstevel@tonic-gate lpl_topo_bootstrap(lpl_t *target, int size) 28457c478bd9Sstevel@tonic-gate { 28467c478bd9Sstevel@tonic-gate lpl_t *lpl = lpl_bootstrap; 28477c478bd9Sstevel@tonic-gate lpl_t *target_lpl = target; 28487c478bd9Sstevel@tonic-gate int howmany; 28497c478bd9Sstevel@tonic-gate int id; 28507c478bd9Sstevel@tonic-gate int i; 28517c478bd9Sstevel@tonic-gate 28527c478bd9Sstevel@tonic-gate /* 28537c478bd9Sstevel@tonic-gate * The only target that should be passed here is cp_default lpl list. 28547c478bd9Sstevel@tonic-gate */ 28557c478bd9Sstevel@tonic-gate ASSERT(target == cp_default.cp_lgrploads); 28567c478bd9Sstevel@tonic-gate ASSERT(size == cp_default.cp_nlgrploads); 28577c478bd9Sstevel@tonic-gate ASSERT(!lgrp_topo_initialized); 28587c478bd9Sstevel@tonic-gate ASSERT(ncpus == 1); 28597c478bd9Sstevel@tonic-gate 28607c478bd9Sstevel@tonic-gate howmany = MIN(LPL_BOOTSTRAP_SIZE, size); 28617c478bd9Sstevel@tonic-gate for (i = 0; i < howmany; i++, lpl++, target_lpl++) { 28627c478bd9Sstevel@tonic-gate /* 28637c478bd9Sstevel@tonic-gate * Copy all fields from lpl. 28647c478bd9Sstevel@tonic-gate */ 28657c478bd9Sstevel@tonic-gate 28667c478bd9Sstevel@tonic-gate *target_lpl = *lpl; 28677c478bd9Sstevel@tonic-gate 28687c478bd9Sstevel@tonic-gate /* 28697c478bd9Sstevel@tonic-gate * Substitute CPU0 lpl pointer with one relative to target. 28707c478bd9Sstevel@tonic-gate */ 28717c478bd9Sstevel@tonic-gate if (lpl->lpl_cpus == CPU) { 28727c478bd9Sstevel@tonic-gate ASSERT(CPU->cpu_lpl == lpl); 28737c478bd9Sstevel@tonic-gate CPU->cpu_lpl = target_lpl; 28747c478bd9Sstevel@tonic-gate } 28757c478bd9Sstevel@tonic-gate 28767c478bd9Sstevel@tonic-gate /* 28777c478bd9Sstevel@tonic-gate * Substitute parent information with parent relative to target. 28787c478bd9Sstevel@tonic-gate */ 28797c478bd9Sstevel@tonic-gate if (lpl->lpl_parent != NULL) 28807c478bd9Sstevel@tonic-gate target_lpl->lpl_parent = (lpl_t *) 28817c478bd9Sstevel@tonic-gate (((uintptr_t)lpl->lpl_parent - 28827c478bd9Sstevel@tonic-gate (uintptr_t)lpl_bootstrap) + 28837c478bd9Sstevel@tonic-gate (uintptr_t)target); 28847c478bd9Sstevel@tonic-gate 28857c478bd9Sstevel@tonic-gate /* 28867c478bd9Sstevel@tonic-gate * Walk over resource set substituting pointers relative to 28877c478bd9Sstevel@tonic-gate * lpl_bootstrap to pointers relative to target. 28887c478bd9Sstevel@tonic-gate */ 28897c478bd9Sstevel@tonic-gate ASSERT(lpl->lpl_nrset <= 1); 28907c478bd9Sstevel@tonic-gate 28917c478bd9Sstevel@tonic-gate for (id = 0; id < lpl->lpl_nrset; id++) { 28927c478bd9Sstevel@tonic-gate if (lpl->lpl_rset[id] != NULL) { 28937c478bd9Sstevel@tonic-gate target_lpl->lpl_rset[id] = 28947c478bd9Sstevel@tonic-gate (lpl_t *) 28957c478bd9Sstevel@tonic-gate (((uintptr_t)lpl->lpl_rset[id] - 28967c478bd9Sstevel@tonic-gate (uintptr_t)lpl_bootstrap) + 28977c478bd9Sstevel@tonic-gate (uintptr_t)target); 28987c478bd9Sstevel@tonic-gate } 28997c478bd9Sstevel@tonic-gate } 29007c478bd9Sstevel@tonic-gate } 29017c478bd9Sstevel@tonic-gate 29027c478bd9Sstevel@tonic-gate /* 29037c478bd9Sstevel@tonic-gate * Topology information in lpl_bootstrap is no longer needed. 29047c478bd9Sstevel@tonic-gate */ 29057c478bd9Sstevel@tonic-gate bzero(lpl_bootstrap_list, sizeof (lpl_bootstrap_list)); 29067c478bd9Sstevel@tonic-gate } 29077c478bd9Sstevel@tonic-gate 29087c478bd9Sstevel@tonic-gate /* the maximum effect that a single thread can have on it's lgroup's load */ 29097c478bd9Sstevel@tonic-gate #define LGRP_LOADAVG_MAX_EFFECT(ncpu) \ 29107c478bd9Sstevel@tonic-gate ((lgrp_loadavg_max_effect) / (ncpu)) 29117c478bd9Sstevel@tonic-gate uint32_t lgrp_loadavg_max_effect = LGRP_LOADAVG_THREAD_MAX; 29127c478bd9Sstevel@tonic-gate 29137c478bd9Sstevel@tonic-gate /* 29147c478bd9Sstevel@tonic-gate * If the lowest load among the lgroups a process' threads are currently 29157c478bd9Sstevel@tonic-gate * spread across is greater than lgrp_expand_proc_thresh, we'll consider 29167c478bd9Sstevel@tonic-gate * expanding the process to a new lgroup. 29177c478bd9Sstevel@tonic-gate */ 29187c478bd9Sstevel@tonic-gate #define LGRP_EXPAND_PROC_THRESH_DEFAULT 62250 29197c478bd9Sstevel@tonic-gate lgrp_load_t lgrp_expand_proc_thresh = LGRP_EXPAND_PROC_THRESH_DEFAULT; 29207c478bd9Sstevel@tonic-gate 29217c478bd9Sstevel@tonic-gate #define LGRP_EXPAND_PROC_THRESH(ncpu) \ 29227c478bd9Sstevel@tonic-gate ((lgrp_expand_proc_thresh) / (ncpu)) 29237c478bd9Sstevel@tonic-gate 29247c478bd9Sstevel@tonic-gate /* 29257c478bd9Sstevel@tonic-gate * A process will be expanded to a new lgroup only if the difference between 29267c478bd9Sstevel@tonic-gate * the lowest load on the lgroups the process' thread's are currently spread 29277c478bd9Sstevel@tonic-gate * across and the lowest load on the other lgroups in the process' partition 29287c478bd9Sstevel@tonic-gate * is greater than lgrp_expand_proc_diff. 29297c478bd9Sstevel@tonic-gate */ 29307c478bd9Sstevel@tonic-gate #define LGRP_EXPAND_PROC_DIFF_DEFAULT 60000 29317c478bd9Sstevel@tonic-gate lgrp_load_t lgrp_expand_proc_diff = LGRP_EXPAND_PROC_DIFF_DEFAULT; 29327c478bd9Sstevel@tonic-gate 29337c478bd9Sstevel@tonic-gate #define LGRP_EXPAND_PROC_DIFF(ncpu) \ 29347c478bd9Sstevel@tonic-gate ((lgrp_expand_proc_diff) / (ncpu)) 29357c478bd9Sstevel@tonic-gate 29367c478bd9Sstevel@tonic-gate /* 29377c478bd9Sstevel@tonic-gate * The loadavg tolerance accounts for "noise" inherent in the load, which may 29387c478bd9Sstevel@tonic-gate * be present due to impreciseness of the load average decay algorithm. 29397c478bd9Sstevel@tonic-gate * 29407c478bd9Sstevel@tonic-gate * The default tolerance is lgrp_loadavg_max_effect. Note that the tunable 29417c478bd9Sstevel@tonic-gate * tolerance is scaled by the number of cpus in the lgroup just like 29427c478bd9Sstevel@tonic-gate * lgrp_loadavg_max_effect. For example, if lgrp_loadavg_tolerance = 0x10000, 29437c478bd9Sstevel@tonic-gate * and ncpu = 4, then lgrp_choose will consider differences in lgroup loads 29447c478bd9Sstevel@tonic-gate * of: 0x10000 / 4 => 0x4000 or greater to be significant. 29457c478bd9Sstevel@tonic-gate */ 29467c478bd9Sstevel@tonic-gate uint32_t lgrp_loadavg_tolerance = LGRP_LOADAVG_THREAD_MAX; 29477c478bd9Sstevel@tonic-gate #define LGRP_LOADAVG_TOLERANCE(ncpu) \ 29487c478bd9Sstevel@tonic-gate ((lgrp_loadavg_tolerance) / ncpu) 29497c478bd9Sstevel@tonic-gate 29507c478bd9Sstevel@tonic-gate /* 29517c478bd9Sstevel@tonic-gate * lgrp_choose() will choose root lgroup as home when lowest lgroup load 29527c478bd9Sstevel@tonic-gate * average is above this threshold 29537c478bd9Sstevel@tonic-gate */ 29547c478bd9Sstevel@tonic-gate uint32_t lgrp_load_thresh = UINT32_MAX; 29557c478bd9Sstevel@tonic-gate 29567c478bd9Sstevel@tonic-gate /* 29577c478bd9Sstevel@tonic-gate * lgrp_choose() will try to skip any lgroups with less memory 29587c478bd9Sstevel@tonic-gate * than this free when choosing a home lgroup 29597c478bd9Sstevel@tonic-gate */ 29607c478bd9Sstevel@tonic-gate pgcnt_t lgrp_mem_free_thresh = 0; 29617c478bd9Sstevel@tonic-gate 29627c478bd9Sstevel@tonic-gate /* 29637c478bd9Sstevel@tonic-gate * When choosing between similarly loaded lgroups, lgrp_choose() will pick 29647c478bd9Sstevel@tonic-gate * one based on one of the following policies: 29657c478bd9Sstevel@tonic-gate * - Random selection 29667c478bd9Sstevel@tonic-gate * - Pseudo round robin placement 29677c478bd9Sstevel@tonic-gate * - Longest time since a thread was last placed 29687c478bd9Sstevel@tonic-gate */ 29697c478bd9Sstevel@tonic-gate #define LGRP_CHOOSE_RANDOM 1 29707c478bd9Sstevel@tonic-gate #define LGRP_CHOOSE_RR 2 29717c478bd9Sstevel@tonic-gate #define LGRP_CHOOSE_TIME 3 29727c478bd9Sstevel@tonic-gate 29737c478bd9Sstevel@tonic-gate int lgrp_choose_policy = LGRP_CHOOSE_TIME; 29747c478bd9Sstevel@tonic-gate 29757c478bd9Sstevel@tonic-gate /* 29767c478bd9Sstevel@tonic-gate * Choose a suitable leaf lgroup for a kthread. The kthread is assumed not to 29777c478bd9Sstevel@tonic-gate * be bound to a CPU or processor set. 29787c478bd9Sstevel@tonic-gate * 29797c478bd9Sstevel@tonic-gate * Arguments: 29807c478bd9Sstevel@tonic-gate * t The thread 29817c478bd9Sstevel@tonic-gate * cpupart The partition the thread belongs to. 29827c478bd9Sstevel@tonic-gate * 29837c478bd9Sstevel@tonic-gate * NOTE: Should at least be called with the cpu_lock held, kernel preemption 29847c478bd9Sstevel@tonic-gate * disabled, or thread_lock held (at splhigh) to protect against the CPU 29857c478bd9Sstevel@tonic-gate * partitions changing out from under us and assumes that given thread is 29867c478bd9Sstevel@tonic-gate * protected. Also, called sometimes w/ cpus paused or kernel preemption 29877c478bd9Sstevel@tonic-gate * disabled, so don't grab any locks because we should never block under 29887c478bd9Sstevel@tonic-gate * those conditions. 29897c478bd9Sstevel@tonic-gate */ 29907c478bd9Sstevel@tonic-gate lpl_t * 29917c478bd9Sstevel@tonic-gate lgrp_choose(kthread_t *t, cpupart_t *cpupart) 29927c478bd9Sstevel@tonic-gate { 29937c478bd9Sstevel@tonic-gate lgrp_load_t bestload, bestrload; 29947c478bd9Sstevel@tonic-gate int lgrpid_offset, lgrp_count; 29957c478bd9Sstevel@tonic-gate lgrp_id_t lgrpid, lgrpid_start; 29967c478bd9Sstevel@tonic-gate lpl_t *lpl, *bestlpl, *bestrlpl; 29977c478bd9Sstevel@tonic-gate klgrpset_t lgrpset; 29987c478bd9Sstevel@tonic-gate proc_t *p; 29997c478bd9Sstevel@tonic-gate 30007c478bd9Sstevel@tonic-gate ASSERT(t != NULL); 30017c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&cpu_lock) || curthread->t_preempt > 0 || 30027c478bd9Sstevel@tonic-gate THREAD_LOCK_HELD(t)); 30037c478bd9Sstevel@tonic-gate ASSERT(cpupart != NULL); 30047c478bd9Sstevel@tonic-gate 30057c478bd9Sstevel@tonic-gate p = t->t_procp; 30067c478bd9Sstevel@tonic-gate 30077c478bd9Sstevel@tonic-gate /* A process should always be in an active partition */ 30087c478bd9Sstevel@tonic-gate ASSERT(!klgrpset_isempty(cpupart->cp_lgrpset)); 30097c478bd9Sstevel@tonic-gate 30107c478bd9Sstevel@tonic-gate bestlpl = bestrlpl = NULL; 30117c478bd9Sstevel@tonic-gate bestload = bestrload = LGRP_LOADAVG_MAX; 30127c478bd9Sstevel@tonic-gate lgrpset = cpupart->cp_lgrpset; 30137c478bd9Sstevel@tonic-gate 30147c478bd9Sstevel@tonic-gate switch (lgrp_choose_policy) { 30157c478bd9Sstevel@tonic-gate case LGRP_CHOOSE_RR: 30167c478bd9Sstevel@tonic-gate lgrpid = cpupart->cp_lgrp_hint; 30177c478bd9Sstevel@tonic-gate do { 30187c478bd9Sstevel@tonic-gate if (++lgrpid > lgrp_alloc_max) 30197c478bd9Sstevel@tonic-gate lgrpid = 0; 30207c478bd9Sstevel@tonic-gate } while (!klgrpset_ismember(lgrpset, lgrpid)); 30217c478bd9Sstevel@tonic-gate 30227c478bd9Sstevel@tonic-gate break; 30237c478bd9Sstevel@tonic-gate default: 30247c478bd9Sstevel@tonic-gate case LGRP_CHOOSE_TIME: 30257c478bd9Sstevel@tonic-gate case LGRP_CHOOSE_RANDOM: 30267c478bd9Sstevel@tonic-gate klgrpset_nlgrps(lgrpset, lgrp_count); 30277c478bd9Sstevel@tonic-gate lgrpid_offset = 30287c478bd9Sstevel@tonic-gate (((ushort_t)(gethrtime() >> 4)) % lgrp_count) + 1; 30297c478bd9Sstevel@tonic-gate for (lgrpid = 0; ; lgrpid++) { 30307c478bd9Sstevel@tonic-gate if (klgrpset_ismember(lgrpset, lgrpid)) { 30317c478bd9Sstevel@tonic-gate if (--lgrpid_offset == 0) 30327c478bd9Sstevel@tonic-gate break; 30337c478bd9Sstevel@tonic-gate } 30347c478bd9Sstevel@tonic-gate } 30357c478bd9Sstevel@tonic-gate break; 30367c478bd9Sstevel@tonic-gate } 30377c478bd9Sstevel@tonic-gate 30387c478bd9Sstevel@tonic-gate lgrpid_start = lgrpid; 30397c478bd9Sstevel@tonic-gate 30407c478bd9Sstevel@tonic-gate DTRACE_PROBE2(lgrp_choose_start, lgrp_id_t, lgrpid_start, 30417c478bd9Sstevel@tonic-gate lgrp_id_t, cpupart->cp_lgrp_hint); 30427c478bd9Sstevel@tonic-gate 30437c478bd9Sstevel@tonic-gate /* 30447c478bd9Sstevel@tonic-gate * Use lgroup affinities (if any) to choose best lgroup 30457c478bd9Sstevel@tonic-gate * 30467c478bd9Sstevel@tonic-gate * NOTE: Assumes that thread is protected from going away and its 30477c478bd9Sstevel@tonic-gate * lgroup affinities won't change (ie. p_lock, or 30487c478bd9Sstevel@tonic-gate * thread_lock() being held and/or CPUs paused) 30497c478bd9Sstevel@tonic-gate */ 30507c478bd9Sstevel@tonic-gate if (t->t_lgrp_affinity) { 30517c478bd9Sstevel@tonic-gate lpl = lgrp_affinity_best(t, cpupart, lgrpid_start); 30527c478bd9Sstevel@tonic-gate if (lpl != NULL) 30537c478bd9Sstevel@tonic-gate return (lpl); 30547c478bd9Sstevel@tonic-gate } 30557c478bd9Sstevel@tonic-gate 30567c478bd9Sstevel@tonic-gate ASSERT(klgrpset_ismember(lgrpset, lgrpid_start)); 30577c478bd9Sstevel@tonic-gate bestlpl = &cpupart->cp_lgrploads[lgrpid_start]; 30587c478bd9Sstevel@tonic-gate 30597c478bd9Sstevel@tonic-gate do { 30607c478bd9Sstevel@tonic-gate pgcnt_t npgs; 30617c478bd9Sstevel@tonic-gate 30627c478bd9Sstevel@tonic-gate /* 30637c478bd9Sstevel@tonic-gate * Skip any lgroups outside of thread's pset 30647c478bd9Sstevel@tonic-gate */ 30657c478bd9Sstevel@tonic-gate if (!klgrpset_ismember(lgrpset, lgrpid)) { 30667c478bd9Sstevel@tonic-gate if (++lgrpid > lgrp_alloc_max) 30677c478bd9Sstevel@tonic-gate lgrpid = 0; /* wrap the search */ 30687c478bd9Sstevel@tonic-gate continue; 30697c478bd9Sstevel@tonic-gate } 30707c478bd9Sstevel@tonic-gate 30717c478bd9Sstevel@tonic-gate /* 30727c478bd9Sstevel@tonic-gate * Skip any non-leaf lgroups 30737c478bd9Sstevel@tonic-gate */ 30747c478bd9Sstevel@tonic-gate if (lgrp_table[lgrpid]->lgrp_childcnt != 0) 30757c478bd9Sstevel@tonic-gate continue; 30767c478bd9Sstevel@tonic-gate 30777c478bd9Sstevel@tonic-gate /* 30787c478bd9Sstevel@tonic-gate * Skip any lgroups without enough free memory 30797c478bd9Sstevel@tonic-gate * (when threshold set to nonzero positive value) 30807c478bd9Sstevel@tonic-gate */ 30817c478bd9Sstevel@tonic-gate if (lgrp_mem_free_thresh > 0) { 30827c478bd9Sstevel@tonic-gate npgs = lgrp_mem_size(lgrpid, LGRP_MEM_SIZE_FREE); 30837c478bd9Sstevel@tonic-gate if (npgs < lgrp_mem_free_thresh) { 30847c478bd9Sstevel@tonic-gate if (++lgrpid > lgrp_alloc_max) 30857c478bd9Sstevel@tonic-gate lgrpid = 0; /* wrap the search */ 30867c478bd9Sstevel@tonic-gate continue; 30877c478bd9Sstevel@tonic-gate } 30887c478bd9Sstevel@tonic-gate } 30897c478bd9Sstevel@tonic-gate 30907c478bd9Sstevel@tonic-gate lpl = &cpupart->cp_lgrploads[lgrpid]; 30917c478bd9Sstevel@tonic-gate if (klgrpset_isempty(p->p_lgrpset) || 30927c478bd9Sstevel@tonic-gate klgrpset_ismember(p->p_lgrpset, lgrpid)) { 30937c478bd9Sstevel@tonic-gate /* 30947c478bd9Sstevel@tonic-gate * Either this is a new process or the process already 30957c478bd9Sstevel@tonic-gate * has threads on this lgrp, so this is a preferred 30967c478bd9Sstevel@tonic-gate * lgroup for the thread. 30977c478bd9Sstevel@tonic-gate */ 30987c478bd9Sstevel@tonic-gate if (lpl_pick(lpl, bestlpl)) { 30997c478bd9Sstevel@tonic-gate bestload = lpl->lpl_loadavg; 31007c478bd9Sstevel@tonic-gate bestlpl = lpl; 31017c478bd9Sstevel@tonic-gate } 31027c478bd9Sstevel@tonic-gate } else { 31037c478bd9Sstevel@tonic-gate /* 31047c478bd9Sstevel@tonic-gate * The process doesn't have any threads on this lgrp, 31057c478bd9Sstevel@tonic-gate * but we're willing to consider this lgrp if the load 31067c478bd9Sstevel@tonic-gate * difference is big enough to justify splitting up 31077c478bd9Sstevel@tonic-gate * the process' threads. 31087c478bd9Sstevel@tonic-gate */ 31097c478bd9Sstevel@tonic-gate if (lpl_pick(lpl, bestrlpl)) { 31107c478bd9Sstevel@tonic-gate bestrload = lpl->lpl_loadavg; 31117c478bd9Sstevel@tonic-gate bestrlpl = lpl; 31127c478bd9Sstevel@tonic-gate } 31137c478bd9Sstevel@tonic-gate } 31147c478bd9Sstevel@tonic-gate if (++lgrpid > lgrp_alloc_max) 31157c478bd9Sstevel@tonic-gate lgrpid = 0; /* wrap the search */ 31167c478bd9Sstevel@tonic-gate } while (lgrpid != lgrpid_start); 31177c478bd9Sstevel@tonic-gate 31187c478bd9Sstevel@tonic-gate /* 31197c478bd9Sstevel@tonic-gate * Return root lgroup if threshold isn't set to maximum value and 31207c478bd9Sstevel@tonic-gate * lowest lgroup load average more than a certain threshold 31217c478bd9Sstevel@tonic-gate */ 31227c478bd9Sstevel@tonic-gate if (lgrp_load_thresh != UINT32_MAX && 31237c478bd9Sstevel@tonic-gate bestload >= lgrp_load_thresh && bestrload >= lgrp_load_thresh) 31247c478bd9Sstevel@tonic-gate return (&cpupart->cp_lgrploads[lgrp_root->lgrp_id]); 31257c478bd9Sstevel@tonic-gate 31267c478bd9Sstevel@tonic-gate /* 31277c478bd9Sstevel@tonic-gate * If all the lgroups over which the thread's process is spread are 31287c478bd9Sstevel@tonic-gate * heavily loaded, we'll consider placing the thread on one of the 31297c478bd9Sstevel@tonic-gate * other leaf lgroups in the thread's partition. 31307c478bd9Sstevel@tonic-gate */ 31317c478bd9Sstevel@tonic-gate if ((bestload > LGRP_EXPAND_PROC_THRESH(bestlpl->lpl_ncpu)) && 31327c478bd9Sstevel@tonic-gate (bestrload < bestload) && /* paranoid about wraparound */ 31337c478bd9Sstevel@tonic-gate (bestrload + LGRP_EXPAND_PROC_DIFF(bestrlpl->lpl_ncpu) < 31347c478bd9Sstevel@tonic-gate bestload)) { 31357c478bd9Sstevel@tonic-gate bestlpl = bestrlpl; 31367c478bd9Sstevel@tonic-gate } 31377c478bd9Sstevel@tonic-gate 31387c478bd9Sstevel@tonic-gate cpupart->cp_lgrp_hint = bestlpl->lpl_lgrpid; 31397c478bd9Sstevel@tonic-gate bestlpl->lpl_homed_time = gethrtime_unscaled(); 31407c478bd9Sstevel@tonic-gate 31417c478bd9Sstevel@tonic-gate ASSERT(bestlpl->lpl_ncpu > 0); 31427c478bd9Sstevel@tonic-gate return (bestlpl); 31437c478bd9Sstevel@tonic-gate } 31447c478bd9Sstevel@tonic-gate 31457c478bd9Sstevel@tonic-gate /* 31467c478bd9Sstevel@tonic-gate * Return 1 if lpl1 is a better candidate than lpl2 for lgrp homing. 31477c478bd9Sstevel@tonic-gate */ 31487c478bd9Sstevel@tonic-gate static int 31497c478bd9Sstevel@tonic-gate lpl_pick(lpl_t *lpl1, lpl_t *lpl2) 31507c478bd9Sstevel@tonic-gate { 31517c478bd9Sstevel@tonic-gate lgrp_load_t l1, l2; 31527c478bd9Sstevel@tonic-gate lgrp_load_t tolerance = LGRP_LOADAVG_TOLERANCE(lpl1->lpl_ncpu); 31537c478bd9Sstevel@tonic-gate 31547c478bd9Sstevel@tonic-gate 31557c478bd9Sstevel@tonic-gate if (lpl2 == NULL) 31567c478bd9Sstevel@tonic-gate return (1); 31577c478bd9Sstevel@tonic-gate 31587c478bd9Sstevel@tonic-gate l1 = lpl1->lpl_loadavg; 31597c478bd9Sstevel@tonic-gate l2 = lpl2->lpl_loadavg; 31607c478bd9Sstevel@tonic-gate 31617c478bd9Sstevel@tonic-gate if ((l1 + tolerance < l2) && (l1 < l2)) { 31627c478bd9Sstevel@tonic-gate /* lpl1 is significantly less loaded than lpl2 */ 31637c478bd9Sstevel@tonic-gate return (1); 31647c478bd9Sstevel@tonic-gate } 31657c478bd9Sstevel@tonic-gate 31667c478bd9Sstevel@tonic-gate if (lgrp_choose_policy == LGRP_CHOOSE_TIME && 31677c478bd9Sstevel@tonic-gate l1 + tolerance >= l2 && l1 < l2 && 31687c478bd9Sstevel@tonic-gate lpl1->lpl_homed_time < lpl2->lpl_homed_time) { 31697c478bd9Sstevel@tonic-gate /* 31707c478bd9Sstevel@tonic-gate * lpl1's load is within the tolerance of lpl2. We're 31717c478bd9Sstevel@tonic-gate * willing to consider it be to better however if 31727c478bd9Sstevel@tonic-gate * it has been longer since we last homed a thread there 31737c478bd9Sstevel@tonic-gate */ 31747c478bd9Sstevel@tonic-gate return (1); 31757c478bd9Sstevel@tonic-gate } 31767c478bd9Sstevel@tonic-gate 31777c478bd9Sstevel@tonic-gate return (0); 31787c478bd9Sstevel@tonic-gate } 31797c478bd9Sstevel@tonic-gate 31807c478bd9Sstevel@tonic-gate /* 31817c478bd9Sstevel@tonic-gate * An LWP is expected to be assigned to an lgroup for at least this long 31827c478bd9Sstevel@tonic-gate * for its anticipatory load to be justified. NOTE that this value should 31837c478bd9Sstevel@tonic-gate * not be set extremely huge (say, larger than 100 years), to avoid problems 31847c478bd9Sstevel@tonic-gate * with overflow in the calculation that uses it. 31857c478bd9Sstevel@tonic-gate */ 31867c478bd9Sstevel@tonic-gate #define LGRP_MIN_NSEC (NANOSEC / 10) /* 1/10 of a second */ 31877c478bd9Sstevel@tonic-gate hrtime_t lgrp_min_nsec = LGRP_MIN_NSEC; 31887c478bd9Sstevel@tonic-gate 31897c478bd9Sstevel@tonic-gate /* 31907c478bd9Sstevel@tonic-gate * Routine to change a thread's lgroup affiliation. This routine updates 31917c478bd9Sstevel@tonic-gate * the thread's kthread_t struct and its process' proc_t struct to note the 31927c478bd9Sstevel@tonic-gate * thread's new lgroup affiliation, and its lgroup affinities. 31937c478bd9Sstevel@tonic-gate * 31947c478bd9Sstevel@tonic-gate * Note that this is the only routine that modifies a thread's t_lpl field, 31957c478bd9Sstevel@tonic-gate * and that adds in or removes anticipatory load. 31967c478bd9Sstevel@tonic-gate * 31977c478bd9Sstevel@tonic-gate * If the thread is exiting, newlpl is NULL. 31987c478bd9Sstevel@tonic-gate * 31997c478bd9Sstevel@tonic-gate * Locking: 32007c478bd9Sstevel@tonic-gate * The following lock must be held on entry: 32017c478bd9Sstevel@tonic-gate * cpu_lock, kpreempt_disable(), or thread_lock -- to assure t's new lgrp 32027c478bd9Sstevel@tonic-gate * doesn't get removed from t's partition 32037c478bd9Sstevel@tonic-gate * 32047c478bd9Sstevel@tonic-gate * This routine is not allowed to grab any locks, since it may be called 32057c478bd9Sstevel@tonic-gate * with cpus paused (such as from cpu_offline). 32067c478bd9Sstevel@tonic-gate */ 32077c478bd9Sstevel@tonic-gate void 32087c478bd9Sstevel@tonic-gate lgrp_move_thread(kthread_t *t, lpl_t *newlpl, int do_lgrpset_delete) 32097c478bd9Sstevel@tonic-gate { 32107c478bd9Sstevel@tonic-gate proc_t *p; 32117c478bd9Sstevel@tonic-gate lpl_t *lpl, *oldlpl; 32127c478bd9Sstevel@tonic-gate lgrp_id_t oldid; 32137c478bd9Sstevel@tonic-gate kthread_t *tp; 32147c478bd9Sstevel@tonic-gate uint_t ncpu; 32157c478bd9Sstevel@tonic-gate lgrp_load_t old, new; 32167c478bd9Sstevel@tonic-gate 32177c478bd9Sstevel@tonic-gate ASSERT(t); 32187c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&cpu_lock) || curthread->t_preempt > 0 || 32197c478bd9Sstevel@tonic-gate THREAD_LOCK_HELD(t)); 32207c478bd9Sstevel@tonic-gate 32217c478bd9Sstevel@tonic-gate /* 32227c478bd9Sstevel@tonic-gate * If not changing lpls, just return 32237c478bd9Sstevel@tonic-gate */ 32247c478bd9Sstevel@tonic-gate if ((oldlpl = t->t_lpl) == newlpl) 32257c478bd9Sstevel@tonic-gate return; 32267c478bd9Sstevel@tonic-gate 32277c478bd9Sstevel@tonic-gate /* 32287c478bd9Sstevel@tonic-gate * Make sure the thread's lwp hasn't exited (if so, this thread is now 32297c478bd9Sstevel@tonic-gate * associated with process 0 rather than with its original process). 32307c478bd9Sstevel@tonic-gate */ 32317c478bd9Sstevel@tonic-gate if (t->t_proc_flag & TP_LWPEXIT) { 32327c478bd9Sstevel@tonic-gate if (newlpl != NULL) { 32337c478bd9Sstevel@tonic-gate t->t_lpl = newlpl; 32347c478bd9Sstevel@tonic-gate } 32357c478bd9Sstevel@tonic-gate return; 32367c478bd9Sstevel@tonic-gate } 32377c478bd9Sstevel@tonic-gate 32387c478bd9Sstevel@tonic-gate p = ttoproc(t); 32397c478bd9Sstevel@tonic-gate 32407c478bd9Sstevel@tonic-gate /* 32417c478bd9Sstevel@tonic-gate * If the thread had a previous lgroup, update its process' p_lgrpset 32427c478bd9Sstevel@tonic-gate * to account for it being moved from its old lgroup. 32437c478bd9Sstevel@tonic-gate */ 32447c478bd9Sstevel@tonic-gate if ((oldlpl != NULL) && /* thread had a previous lgroup */ 32457c478bd9Sstevel@tonic-gate (p->p_tlist != NULL)) { 32467c478bd9Sstevel@tonic-gate oldid = oldlpl->lpl_lgrpid; 32477c478bd9Sstevel@tonic-gate 32487c478bd9Sstevel@tonic-gate if (newlpl != NULL) 32497c478bd9Sstevel@tonic-gate lgrp_stat_add(oldid, LGRP_NUM_MIGR, 1); 32507c478bd9Sstevel@tonic-gate 32517c478bd9Sstevel@tonic-gate if ((do_lgrpset_delete) && 32527c478bd9Sstevel@tonic-gate (klgrpset_ismember(p->p_lgrpset, oldid))) { 32537c478bd9Sstevel@tonic-gate for (tp = p->p_tlist->t_forw; ; tp = tp->t_forw) { 32547c478bd9Sstevel@tonic-gate /* 32557c478bd9Sstevel@tonic-gate * Check if a thread other than the thread 32567c478bd9Sstevel@tonic-gate * that's moving is assigned to the same 32577c478bd9Sstevel@tonic-gate * lgroup as the thread that's moving. Note 32587c478bd9Sstevel@tonic-gate * that we have to compare lgroup IDs, rather 32597c478bd9Sstevel@tonic-gate * than simply comparing t_lpl's, since the 32607c478bd9Sstevel@tonic-gate * threads may belong to different partitions 32617c478bd9Sstevel@tonic-gate * but be assigned to the same lgroup. 32627c478bd9Sstevel@tonic-gate */ 32637c478bd9Sstevel@tonic-gate ASSERT(tp->t_lpl != NULL); 32647c478bd9Sstevel@tonic-gate 32657c478bd9Sstevel@tonic-gate if ((tp != t) && 32667c478bd9Sstevel@tonic-gate (tp->t_lpl->lpl_lgrpid == oldid)) { 32677c478bd9Sstevel@tonic-gate /* 32687c478bd9Sstevel@tonic-gate * Another thread is assigned to the 32697c478bd9Sstevel@tonic-gate * same lgroup as the thread that's 32707c478bd9Sstevel@tonic-gate * moving, p_lgrpset doesn't change. 32717c478bd9Sstevel@tonic-gate */ 32727c478bd9Sstevel@tonic-gate break; 32737c478bd9Sstevel@tonic-gate } else if (tp == p->p_tlist) { 32747c478bd9Sstevel@tonic-gate /* 32757c478bd9Sstevel@tonic-gate * No other thread is assigned to the 32767c478bd9Sstevel@tonic-gate * same lgroup as the exiting thread, 32777c478bd9Sstevel@tonic-gate * clear the lgroup's bit in p_lgrpset. 32787c478bd9Sstevel@tonic-gate */ 32797c478bd9Sstevel@tonic-gate klgrpset_del(p->p_lgrpset, oldid); 32807c478bd9Sstevel@tonic-gate break; 32817c478bd9Sstevel@tonic-gate } 32827c478bd9Sstevel@tonic-gate } 32837c478bd9Sstevel@tonic-gate } 32847c478bd9Sstevel@tonic-gate 32857c478bd9Sstevel@tonic-gate /* 32867c478bd9Sstevel@tonic-gate * If this thread was assigned to its old lgroup for such a 32877c478bd9Sstevel@tonic-gate * short amount of time that the anticipatory load that was 32887c478bd9Sstevel@tonic-gate * added on its behalf has aged very little, remove that 32897c478bd9Sstevel@tonic-gate * anticipatory load. 32907c478bd9Sstevel@tonic-gate */ 32917c478bd9Sstevel@tonic-gate if ((t->t_anttime + lgrp_min_nsec > gethrtime()) && 32927c478bd9Sstevel@tonic-gate ((ncpu = oldlpl->lpl_ncpu) > 0)) { 32937c478bd9Sstevel@tonic-gate lpl = oldlpl; 32947c478bd9Sstevel@tonic-gate for (;;) { 32957c478bd9Sstevel@tonic-gate do { 32967c478bd9Sstevel@tonic-gate old = new = lpl->lpl_loadavg; 32977c478bd9Sstevel@tonic-gate new -= LGRP_LOADAVG_MAX_EFFECT(ncpu); 32987c478bd9Sstevel@tonic-gate if (new > old) { 32997c478bd9Sstevel@tonic-gate /* 33007c478bd9Sstevel@tonic-gate * this can happen if the load 33017c478bd9Sstevel@tonic-gate * average was aged since we 33027c478bd9Sstevel@tonic-gate * added in the anticipatory 33037c478bd9Sstevel@tonic-gate * load 33047c478bd9Sstevel@tonic-gate */ 33057c478bd9Sstevel@tonic-gate new = 0; 33067c478bd9Sstevel@tonic-gate } 33077c478bd9Sstevel@tonic-gate } while (cas32( 33087c478bd9Sstevel@tonic-gate (lgrp_load_t *)&lpl->lpl_loadavg, old, 33097c478bd9Sstevel@tonic-gate new) != old); 33107c478bd9Sstevel@tonic-gate 33117c478bd9Sstevel@tonic-gate lpl = lpl->lpl_parent; 33127c478bd9Sstevel@tonic-gate if (lpl == NULL) 33137c478bd9Sstevel@tonic-gate break; 33147c478bd9Sstevel@tonic-gate 33157c478bd9Sstevel@tonic-gate ncpu = lpl->lpl_ncpu; 33167c478bd9Sstevel@tonic-gate ASSERT(ncpu > 0); 33177c478bd9Sstevel@tonic-gate } 33187c478bd9Sstevel@tonic-gate } 33197c478bd9Sstevel@tonic-gate } 33207c478bd9Sstevel@tonic-gate /* 33217c478bd9Sstevel@tonic-gate * If the thread has a new lgroup (i.e. it's not exiting), update its 33227c478bd9Sstevel@tonic-gate * t_lpl and its process' p_lgrpset, and apply an anticipatory load 33237c478bd9Sstevel@tonic-gate * to its new lgroup to account for its move to its new lgroup. 33247c478bd9Sstevel@tonic-gate */ 33257c478bd9Sstevel@tonic-gate if (newlpl != NULL) { 33267c478bd9Sstevel@tonic-gate /* 33277c478bd9Sstevel@tonic-gate * This thread is moving to a new lgroup 33287c478bd9Sstevel@tonic-gate */ 33297c478bd9Sstevel@tonic-gate t->t_lpl = newlpl; 33307c478bd9Sstevel@tonic-gate 33317c478bd9Sstevel@tonic-gate /* 33327c478bd9Sstevel@tonic-gate * Reflect move in load average of new lgroup 33337c478bd9Sstevel@tonic-gate * unless it is root lgroup 33347c478bd9Sstevel@tonic-gate */ 33357c478bd9Sstevel@tonic-gate if (lgrp_table[newlpl->lpl_lgrpid] == lgrp_root) 33367c478bd9Sstevel@tonic-gate return; 33377c478bd9Sstevel@tonic-gate 33387c478bd9Sstevel@tonic-gate if (!klgrpset_ismember(p->p_lgrpset, newlpl->lpl_lgrpid)) { 33397c478bd9Sstevel@tonic-gate klgrpset_add(p->p_lgrpset, newlpl->lpl_lgrpid); 33407c478bd9Sstevel@tonic-gate } 33417c478bd9Sstevel@tonic-gate 33427c478bd9Sstevel@tonic-gate /* 33437c478bd9Sstevel@tonic-gate * It'll take some time for the load on the new lgroup 33447c478bd9Sstevel@tonic-gate * to reflect this thread's placement on it. We'd 33457c478bd9Sstevel@tonic-gate * like not, however, to have all threads between now 33467c478bd9Sstevel@tonic-gate * and then also piling on to this lgroup. To avoid 33477c478bd9Sstevel@tonic-gate * this pileup, we anticipate the load this thread 33487c478bd9Sstevel@tonic-gate * will generate on its new lgroup. The goal is to 33497c478bd9Sstevel@tonic-gate * make the lgroup's load appear as though the thread 33507c478bd9Sstevel@tonic-gate * had been there all along. We're very conservative 33517c478bd9Sstevel@tonic-gate * in calculating this anticipatory load, we assume 33527c478bd9Sstevel@tonic-gate * the worst case case (100% CPU-bound thread). This 33537c478bd9Sstevel@tonic-gate * may be modified in the future to be more accurate. 33547c478bd9Sstevel@tonic-gate */ 33557c478bd9Sstevel@tonic-gate lpl = newlpl; 33567c478bd9Sstevel@tonic-gate for (;;) { 33577c478bd9Sstevel@tonic-gate ncpu = lpl->lpl_ncpu; 33587c478bd9Sstevel@tonic-gate ASSERT(ncpu > 0); 33597c478bd9Sstevel@tonic-gate do { 33607c478bd9Sstevel@tonic-gate old = new = lpl->lpl_loadavg; 33617c478bd9Sstevel@tonic-gate new += LGRP_LOADAVG_MAX_EFFECT(ncpu); 33627c478bd9Sstevel@tonic-gate /* 33637c478bd9Sstevel@tonic-gate * Check for overflow 33647c478bd9Sstevel@tonic-gate * Underflow not possible here 33657c478bd9Sstevel@tonic-gate */ 33667c478bd9Sstevel@tonic-gate if (new < old) 33677c478bd9Sstevel@tonic-gate new = UINT32_MAX; 33687c478bd9Sstevel@tonic-gate } while (cas32((lgrp_load_t *)&lpl->lpl_loadavg, old, 33697c478bd9Sstevel@tonic-gate new) != old); 33707c478bd9Sstevel@tonic-gate 33717c478bd9Sstevel@tonic-gate lpl = lpl->lpl_parent; 33727c478bd9Sstevel@tonic-gate if (lpl == NULL) 33737c478bd9Sstevel@tonic-gate break; 33747c478bd9Sstevel@tonic-gate } 33757c478bd9Sstevel@tonic-gate t->t_anttime = gethrtime(); 33767c478bd9Sstevel@tonic-gate } 33777c478bd9Sstevel@tonic-gate } 33787c478bd9Sstevel@tonic-gate 33797c478bd9Sstevel@tonic-gate /* 33807c478bd9Sstevel@tonic-gate * Return lgroup memory allocation policy given advice from madvise(3C) 33817c478bd9Sstevel@tonic-gate */ 33827c478bd9Sstevel@tonic-gate lgrp_mem_policy_t 33837c478bd9Sstevel@tonic-gate lgrp_madv_to_policy(uchar_t advice, size_t size, int type) 33847c478bd9Sstevel@tonic-gate { 33857c478bd9Sstevel@tonic-gate switch (advice) { 33867c478bd9Sstevel@tonic-gate case MADV_ACCESS_LWP: 33877c478bd9Sstevel@tonic-gate return (LGRP_MEM_POLICY_NEXT); 33887c478bd9Sstevel@tonic-gate case MADV_ACCESS_MANY: 33897c478bd9Sstevel@tonic-gate return (LGRP_MEM_POLICY_RANDOM); 33907c478bd9Sstevel@tonic-gate default: 33917c478bd9Sstevel@tonic-gate return (lgrp_mem_policy_default(size, type)); 33927c478bd9Sstevel@tonic-gate } 33937c478bd9Sstevel@tonic-gate } 33947c478bd9Sstevel@tonic-gate 33957c478bd9Sstevel@tonic-gate /* 33967c478bd9Sstevel@tonic-gate * Figure out default policy 33977c478bd9Sstevel@tonic-gate */ 33987c478bd9Sstevel@tonic-gate lgrp_mem_policy_t 33997c478bd9Sstevel@tonic-gate lgrp_mem_policy_default(size_t size, int type) 34007c478bd9Sstevel@tonic-gate { 34017c478bd9Sstevel@tonic-gate cpupart_t *cp; 34027c478bd9Sstevel@tonic-gate lgrp_mem_policy_t policy; 34037c478bd9Sstevel@tonic-gate size_t pset_mem_size; 34047c478bd9Sstevel@tonic-gate 34057c478bd9Sstevel@tonic-gate /* 34067c478bd9Sstevel@tonic-gate * Randomly allocate memory across lgroups for shared memory 34077c478bd9Sstevel@tonic-gate * beyond a certain threshold 34087c478bd9Sstevel@tonic-gate */ 34097c478bd9Sstevel@tonic-gate if ((type != MAP_SHARED && size > lgrp_privm_random_thresh) || 34107c478bd9Sstevel@tonic-gate (type == MAP_SHARED && size > lgrp_shm_random_thresh)) { 34117c478bd9Sstevel@tonic-gate /* 34127c478bd9Sstevel@tonic-gate * Get total memory size of current thread's pset 34137c478bd9Sstevel@tonic-gate */ 34147c478bd9Sstevel@tonic-gate kpreempt_disable(); 34157c478bd9Sstevel@tonic-gate cp = curthread->t_cpupart; 34167c478bd9Sstevel@tonic-gate klgrpset_totalsize(cp->cp_lgrpset, pset_mem_size); 34177c478bd9Sstevel@tonic-gate kpreempt_enable(); 34187c478bd9Sstevel@tonic-gate 34197c478bd9Sstevel@tonic-gate /* 34207c478bd9Sstevel@tonic-gate * Choose policy to randomly allocate memory across 34217c478bd9Sstevel@tonic-gate * lgroups in pset if it will fit and is not default 34227c478bd9Sstevel@tonic-gate * partition. Otherwise, allocate memory randomly 34237c478bd9Sstevel@tonic-gate * across machine. 34247c478bd9Sstevel@tonic-gate */ 34257c478bd9Sstevel@tonic-gate if (lgrp_mem_pset_aware && size < pset_mem_size) 34267c478bd9Sstevel@tonic-gate policy = LGRP_MEM_POLICY_RANDOM_PSET; 34277c478bd9Sstevel@tonic-gate else 34287c478bd9Sstevel@tonic-gate policy = LGRP_MEM_POLICY_RANDOM; 34297c478bd9Sstevel@tonic-gate } else 34307c478bd9Sstevel@tonic-gate /* 34317c478bd9Sstevel@tonic-gate * Apply default policy for private memory and 34327c478bd9Sstevel@tonic-gate * shared memory under the respective random 34337c478bd9Sstevel@tonic-gate * threshold. 34347c478bd9Sstevel@tonic-gate */ 34357c478bd9Sstevel@tonic-gate policy = lgrp_mem_default_policy; 34367c478bd9Sstevel@tonic-gate 34377c478bd9Sstevel@tonic-gate return (policy); 34387c478bd9Sstevel@tonic-gate } 34397c478bd9Sstevel@tonic-gate 34407c478bd9Sstevel@tonic-gate /* 34417c478bd9Sstevel@tonic-gate * Get memory allocation policy for this segment 34427c478bd9Sstevel@tonic-gate */ 34437c478bd9Sstevel@tonic-gate lgrp_mem_policy_info_t * 34447c478bd9Sstevel@tonic-gate lgrp_mem_policy_get(struct seg *seg, caddr_t vaddr) 34457c478bd9Sstevel@tonic-gate { 34467c478bd9Sstevel@tonic-gate lgrp_mem_policy_info_t *policy_info; 34477c478bd9Sstevel@tonic-gate extern struct seg_ops segspt_ops; 34487c478bd9Sstevel@tonic-gate extern struct seg_ops segspt_shmops; 34497c478bd9Sstevel@tonic-gate 34507c478bd9Sstevel@tonic-gate /* 34517c478bd9Sstevel@tonic-gate * This is for binary compatibility to protect against third party 34527c478bd9Sstevel@tonic-gate * segment drivers which haven't recompiled to allow for 34537c478bd9Sstevel@tonic-gate * SEGOP_GETPOLICY() 34547c478bd9Sstevel@tonic-gate */ 34557c478bd9Sstevel@tonic-gate if (seg->s_ops != &segvn_ops && seg->s_ops != &segspt_ops && 34567c478bd9Sstevel@tonic-gate seg->s_ops != &segspt_shmops) 34577c478bd9Sstevel@tonic-gate return (NULL); 34587c478bd9Sstevel@tonic-gate 34597c478bd9Sstevel@tonic-gate policy_info = NULL; 34607c478bd9Sstevel@tonic-gate if (seg->s_ops->getpolicy != NULL) 34617c478bd9Sstevel@tonic-gate policy_info = SEGOP_GETPOLICY(seg, vaddr); 34627c478bd9Sstevel@tonic-gate 34637c478bd9Sstevel@tonic-gate return (policy_info); 34647c478bd9Sstevel@tonic-gate } 34657c478bd9Sstevel@tonic-gate 34667c478bd9Sstevel@tonic-gate /* 34677c478bd9Sstevel@tonic-gate * Set policy for allocating private memory given desired policy, policy info, 34687c478bd9Sstevel@tonic-gate * size in bytes of memory that policy is being applied. 34697c478bd9Sstevel@tonic-gate * Return 0 if policy wasn't set already and 1 if policy was set already 34707c478bd9Sstevel@tonic-gate */ 34717c478bd9Sstevel@tonic-gate int 34727c478bd9Sstevel@tonic-gate lgrp_privm_policy_set(lgrp_mem_policy_t policy, 34737c478bd9Sstevel@tonic-gate lgrp_mem_policy_info_t *policy_info, size_t size) 34747c478bd9Sstevel@tonic-gate { 34757c478bd9Sstevel@tonic-gate 34767c478bd9Sstevel@tonic-gate ASSERT(policy_info != NULL); 34777c478bd9Sstevel@tonic-gate 34787c478bd9Sstevel@tonic-gate if (policy == LGRP_MEM_POLICY_DEFAULT) 34797c478bd9Sstevel@tonic-gate policy = lgrp_mem_policy_default(size, MAP_PRIVATE); 34807c478bd9Sstevel@tonic-gate 34817c478bd9Sstevel@tonic-gate /* 34827c478bd9Sstevel@tonic-gate * Policy set already? 34837c478bd9Sstevel@tonic-gate */ 34847c478bd9Sstevel@tonic-gate if (policy == policy_info->mem_policy) 34857c478bd9Sstevel@tonic-gate return (1); 34867c478bd9Sstevel@tonic-gate 34877c478bd9Sstevel@tonic-gate /* 34887c478bd9Sstevel@tonic-gate * Set policy 34897c478bd9Sstevel@tonic-gate */ 34907c478bd9Sstevel@tonic-gate policy_info->mem_policy = policy; 34917c478bd9Sstevel@tonic-gate policy_info->mem_reserved = 0; 34927c478bd9Sstevel@tonic-gate 34937c478bd9Sstevel@tonic-gate return (0); 34947c478bd9Sstevel@tonic-gate } 34957c478bd9Sstevel@tonic-gate 34967c478bd9Sstevel@tonic-gate 34977c478bd9Sstevel@tonic-gate /* 34987c478bd9Sstevel@tonic-gate * Get shared memory allocation policy with given tree and offset 34997c478bd9Sstevel@tonic-gate */ 35007c478bd9Sstevel@tonic-gate lgrp_mem_policy_info_t * 35017c478bd9Sstevel@tonic-gate lgrp_shm_policy_get(struct anon_map *amp, ulong_t anon_index, vnode_t *vp, 35027c478bd9Sstevel@tonic-gate u_offset_t vn_off) 35037c478bd9Sstevel@tonic-gate { 35047c478bd9Sstevel@tonic-gate u_offset_t off; 35057c478bd9Sstevel@tonic-gate lgrp_mem_policy_info_t *policy_info; 35067c478bd9Sstevel@tonic-gate lgrp_shm_policy_seg_t *policy_seg; 35077c478bd9Sstevel@tonic-gate lgrp_shm_locality_t *shm_locality; 35087c478bd9Sstevel@tonic-gate avl_tree_t *tree; 35097c478bd9Sstevel@tonic-gate avl_index_t where; 35107c478bd9Sstevel@tonic-gate 35117c478bd9Sstevel@tonic-gate /* 35127c478bd9Sstevel@tonic-gate * Get policy segment tree from anon_map or vnode and use specified 35137c478bd9Sstevel@tonic-gate * anon index or vnode offset as offset 35147c478bd9Sstevel@tonic-gate * 35157c478bd9Sstevel@tonic-gate * Assume that no lock needs to be held on anon_map or vnode, since 35167c478bd9Sstevel@tonic-gate * they should be protected by their reference count which must be 35177c478bd9Sstevel@tonic-gate * nonzero for an existing segment 35187c478bd9Sstevel@tonic-gate */ 35197c478bd9Sstevel@tonic-gate if (amp) { 35207c478bd9Sstevel@tonic-gate ASSERT(amp->refcnt != 0); 35217c478bd9Sstevel@tonic-gate shm_locality = amp->locality; 35227c478bd9Sstevel@tonic-gate if (shm_locality == NULL) 35237c478bd9Sstevel@tonic-gate return (NULL); 35247c478bd9Sstevel@tonic-gate tree = shm_locality->loc_tree; 35257c478bd9Sstevel@tonic-gate off = ptob(anon_index); 35267c478bd9Sstevel@tonic-gate } else if (vp) { 35277c478bd9Sstevel@tonic-gate shm_locality = vp->v_locality; 35287c478bd9Sstevel@tonic-gate if (shm_locality == NULL) 35297c478bd9Sstevel@tonic-gate return (NULL); 35307c478bd9Sstevel@tonic-gate ASSERT(shm_locality->loc_count != 0); 35317c478bd9Sstevel@tonic-gate tree = shm_locality->loc_tree; 35327c478bd9Sstevel@tonic-gate off = vn_off; 35337c478bd9Sstevel@tonic-gate } 35347c478bd9Sstevel@tonic-gate 35357c478bd9Sstevel@tonic-gate if (tree == NULL) 35367c478bd9Sstevel@tonic-gate return (NULL); 35377c478bd9Sstevel@tonic-gate 35387c478bd9Sstevel@tonic-gate /* 35397c478bd9Sstevel@tonic-gate * Lookup policy segment for offset into shared object and return 35407c478bd9Sstevel@tonic-gate * policy info 35417c478bd9Sstevel@tonic-gate */ 35427c478bd9Sstevel@tonic-gate rw_enter(&shm_locality->loc_lock, RW_READER); 35437c478bd9Sstevel@tonic-gate policy_info = NULL; 35447c478bd9Sstevel@tonic-gate policy_seg = avl_find(tree, &off, &where); 35457c478bd9Sstevel@tonic-gate if (policy_seg) 35467c478bd9Sstevel@tonic-gate policy_info = &policy_seg->shm_policy; 35477c478bd9Sstevel@tonic-gate rw_exit(&shm_locality->loc_lock); 35487c478bd9Sstevel@tonic-gate 35497c478bd9Sstevel@tonic-gate return (policy_info); 35507c478bd9Sstevel@tonic-gate } 35517c478bd9Sstevel@tonic-gate 35527c478bd9Sstevel@tonic-gate /* 35537c478bd9Sstevel@tonic-gate * Return lgroup to use for allocating memory 35547c478bd9Sstevel@tonic-gate * given the segment and address 35557c478bd9Sstevel@tonic-gate * 35567c478bd9Sstevel@tonic-gate * There isn't any mutual exclusion that exists between calls 35577c478bd9Sstevel@tonic-gate * to this routine and DR, so this routine and whomever calls it 35587c478bd9Sstevel@tonic-gate * should be mindful of the possibility that the lgrp returned 35597c478bd9Sstevel@tonic-gate * may be deleted. If this happens, dereferences of the lgrp 35607c478bd9Sstevel@tonic-gate * pointer will still be safe, but the resources in the lgrp will 35617c478bd9Sstevel@tonic-gate * be gone, and LGRP_EXISTS() will no longer be true. 35627c478bd9Sstevel@tonic-gate */ 35637c478bd9Sstevel@tonic-gate lgrp_t * 35647c478bd9Sstevel@tonic-gate lgrp_mem_choose(struct seg *seg, caddr_t vaddr, size_t pgsz) 35657c478bd9Sstevel@tonic-gate { 35667c478bd9Sstevel@tonic-gate int i; 35677c478bd9Sstevel@tonic-gate lgrp_t *lgrp; 35687c478bd9Sstevel@tonic-gate klgrpset_t lgrpset; 35697c478bd9Sstevel@tonic-gate int lgrps_spanned; 35707c478bd9Sstevel@tonic-gate unsigned long off; 35717c478bd9Sstevel@tonic-gate lgrp_mem_policy_t policy; 35727c478bd9Sstevel@tonic-gate lgrp_mem_policy_info_t *policy_info; 35737c478bd9Sstevel@tonic-gate ushort_t random; 35747c478bd9Sstevel@tonic-gate int stat = 0; 35757c478bd9Sstevel@tonic-gate 35767c478bd9Sstevel@tonic-gate /* 35777c478bd9Sstevel@tonic-gate * Just return null if the lgrp framework hasn't finished 35787c478bd9Sstevel@tonic-gate * initializing or if this is a UMA machine. 35797c478bd9Sstevel@tonic-gate */ 35807c478bd9Sstevel@tonic-gate if (nlgrps == 1 || !lgrp_initialized) 35817c478bd9Sstevel@tonic-gate return (lgrp_root); 35827c478bd9Sstevel@tonic-gate 35837c478bd9Sstevel@tonic-gate /* 35847c478bd9Sstevel@tonic-gate * Get memory allocation policy for this segment 35857c478bd9Sstevel@tonic-gate */ 35867c478bd9Sstevel@tonic-gate policy = lgrp_mem_default_policy; 35877c478bd9Sstevel@tonic-gate if (seg != NULL) { 35887c478bd9Sstevel@tonic-gate if (seg->s_as == &kas) { 35897c478bd9Sstevel@tonic-gate if (policy == LGRP_MEM_POLICY_RANDOM_PROC || 35907c478bd9Sstevel@tonic-gate policy == LGRP_MEM_POLICY_RANDOM_PSET) 35917c478bd9Sstevel@tonic-gate policy = LGRP_MEM_POLICY_RANDOM; 35927c478bd9Sstevel@tonic-gate } else { 35937c478bd9Sstevel@tonic-gate policy_info = lgrp_mem_policy_get(seg, vaddr); 35947c478bd9Sstevel@tonic-gate if (policy_info != NULL) 35957c478bd9Sstevel@tonic-gate policy = policy_info->mem_policy; 35967c478bd9Sstevel@tonic-gate } 35977c478bd9Sstevel@tonic-gate } 35987c478bd9Sstevel@tonic-gate lgrpset = 0; 35997c478bd9Sstevel@tonic-gate 36007c478bd9Sstevel@tonic-gate /* 36017c478bd9Sstevel@tonic-gate * Initialize lgroup to home by default 36027c478bd9Sstevel@tonic-gate */ 36037c478bd9Sstevel@tonic-gate lgrp = lgrp_home_lgrp(); 36047c478bd9Sstevel@tonic-gate 36057c478bd9Sstevel@tonic-gate /* 36067c478bd9Sstevel@tonic-gate * When homing threads on root lgrp, override default memory 36077c478bd9Sstevel@tonic-gate * allocation policies with root lgroup memory allocation policy 36087c478bd9Sstevel@tonic-gate */ 36097c478bd9Sstevel@tonic-gate if (lgrp == lgrp_root) 36107c478bd9Sstevel@tonic-gate policy = lgrp_mem_policy_root; 36117c478bd9Sstevel@tonic-gate 36127c478bd9Sstevel@tonic-gate /* 36137c478bd9Sstevel@tonic-gate * Implement policy 36147c478bd9Sstevel@tonic-gate */ 36157c478bd9Sstevel@tonic-gate switch (policy) { 36167c478bd9Sstevel@tonic-gate case LGRP_MEM_POLICY_NEXT_CPU: 36177c478bd9Sstevel@tonic-gate 36187c478bd9Sstevel@tonic-gate /* 36197c478bd9Sstevel@tonic-gate * Return lgroup of current CPU which faulted on memory 36207c478bd9Sstevel@tonic-gate */ 36217c478bd9Sstevel@tonic-gate lgrp = lgrp_cpu_to_lgrp(CPU); 36227c478bd9Sstevel@tonic-gate break; 36237c478bd9Sstevel@tonic-gate 36247c478bd9Sstevel@tonic-gate case LGRP_MEM_POLICY_NEXT: 36257c478bd9Sstevel@tonic-gate case LGRP_MEM_POLICY_DEFAULT: 36267c478bd9Sstevel@tonic-gate default: 36277c478bd9Sstevel@tonic-gate 36287c478bd9Sstevel@tonic-gate /* 36297c478bd9Sstevel@tonic-gate * Just return current thread's home lgroup 36307c478bd9Sstevel@tonic-gate * for default policy (next touch) 36317c478bd9Sstevel@tonic-gate * If the thread is homed to the root, 36327c478bd9Sstevel@tonic-gate * then the default policy is random across lgroups. 36337c478bd9Sstevel@tonic-gate * Fallthrough to the random case. 36347c478bd9Sstevel@tonic-gate */ 36357c478bd9Sstevel@tonic-gate if (lgrp != lgrp_root) { 36367c478bd9Sstevel@tonic-gate if (policy == LGRP_MEM_POLICY_NEXT) 36377c478bd9Sstevel@tonic-gate lgrp_stat_add(lgrp->lgrp_id, LGRP_NUM_NEXT, 1); 36387c478bd9Sstevel@tonic-gate else 36397c478bd9Sstevel@tonic-gate lgrp_stat_add(lgrp->lgrp_id, 36407c478bd9Sstevel@tonic-gate LGRP_NUM_DEFAULT, 1); 36417c478bd9Sstevel@tonic-gate break; 36427c478bd9Sstevel@tonic-gate } 36437c478bd9Sstevel@tonic-gate /* LINTED fallthrough on case statement */ 36447c478bd9Sstevel@tonic-gate case LGRP_MEM_POLICY_RANDOM: 36457c478bd9Sstevel@tonic-gate 36467c478bd9Sstevel@tonic-gate /* 36477c478bd9Sstevel@tonic-gate * Return a random leaf lgroup with memory 36487c478bd9Sstevel@tonic-gate */ 36497c478bd9Sstevel@tonic-gate lgrpset = lgrp_root->lgrp_set[LGRP_RSRC_MEM]; 36507c478bd9Sstevel@tonic-gate /* 36517c478bd9Sstevel@tonic-gate * Count how many lgroups are spanned 36527c478bd9Sstevel@tonic-gate */ 36537c478bd9Sstevel@tonic-gate klgrpset_nlgrps(lgrpset, lgrps_spanned); 36547c478bd9Sstevel@tonic-gate 36557c478bd9Sstevel@tonic-gate /* 36567c478bd9Sstevel@tonic-gate * There may be no memnodes in the root lgroup during DR copy 36577c478bd9Sstevel@tonic-gate * rename on a system with only two boards (memnodes) 36587c478bd9Sstevel@tonic-gate * configured. In this case just return the root lgrp. 36597c478bd9Sstevel@tonic-gate */ 36607c478bd9Sstevel@tonic-gate if (lgrps_spanned == 0) { 36617c478bd9Sstevel@tonic-gate lgrp = lgrp_root; 36627c478bd9Sstevel@tonic-gate break; 36637c478bd9Sstevel@tonic-gate } 36647c478bd9Sstevel@tonic-gate 36657c478bd9Sstevel@tonic-gate /* 36667c478bd9Sstevel@tonic-gate * Pick a random offset within lgroups spanned 36677c478bd9Sstevel@tonic-gate * and return lgroup at that offset 36687c478bd9Sstevel@tonic-gate */ 36697c478bd9Sstevel@tonic-gate random = (ushort_t)gethrtime() >> 4; 36707c478bd9Sstevel@tonic-gate off = random % lgrps_spanned; 36717c478bd9Sstevel@tonic-gate ASSERT(off <= lgrp_alloc_max); 36727c478bd9Sstevel@tonic-gate 36737c478bd9Sstevel@tonic-gate for (i = 0; i <= lgrp_alloc_max; i++) { 36747c478bd9Sstevel@tonic-gate if (!klgrpset_ismember(lgrpset, i)) 36757c478bd9Sstevel@tonic-gate continue; 36767c478bd9Sstevel@tonic-gate if (off) 36777c478bd9Sstevel@tonic-gate off--; 36787c478bd9Sstevel@tonic-gate else { 36797c478bd9Sstevel@tonic-gate lgrp = lgrp_table[i]; 36807c478bd9Sstevel@tonic-gate lgrp_stat_add(lgrp->lgrp_id, LGRP_NUM_RANDOM, 36817c478bd9Sstevel@tonic-gate 1); 36827c478bd9Sstevel@tonic-gate break; 36837c478bd9Sstevel@tonic-gate } 36847c478bd9Sstevel@tonic-gate } 36857c478bd9Sstevel@tonic-gate break; 36867c478bd9Sstevel@tonic-gate 36877c478bd9Sstevel@tonic-gate case LGRP_MEM_POLICY_RANDOM_PROC: 36887c478bd9Sstevel@tonic-gate 36897c478bd9Sstevel@tonic-gate /* 36907c478bd9Sstevel@tonic-gate * Grab copy of bitmask of lgroups spanned by 36917c478bd9Sstevel@tonic-gate * this process 36927c478bd9Sstevel@tonic-gate */ 36937c478bd9Sstevel@tonic-gate klgrpset_copy(lgrpset, curproc->p_lgrpset); 36947c478bd9Sstevel@tonic-gate stat = LGRP_NUM_RANDOM_PROC; 36957c478bd9Sstevel@tonic-gate 36967c478bd9Sstevel@tonic-gate /* LINTED fallthrough on case statement */ 36977c478bd9Sstevel@tonic-gate case LGRP_MEM_POLICY_RANDOM_PSET: 36987c478bd9Sstevel@tonic-gate 36997c478bd9Sstevel@tonic-gate if (!stat) 37007c478bd9Sstevel@tonic-gate stat = LGRP_NUM_RANDOM_PSET; 37017c478bd9Sstevel@tonic-gate 37027c478bd9Sstevel@tonic-gate if (klgrpset_isempty(lgrpset)) { 37037c478bd9Sstevel@tonic-gate /* 37047c478bd9Sstevel@tonic-gate * Grab copy of bitmask of lgroups spanned by 37057c478bd9Sstevel@tonic-gate * this processor set 37067c478bd9Sstevel@tonic-gate */ 37077c478bd9Sstevel@tonic-gate kpreempt_disable(); 37087c478bd9Sstevel@tonic-gate klgrpset_copy(lgrpset, 37097c478bd9Sstevel@tonic-gate curthread->t_cpupart->cp_lgrpset); 37107c478bd9Sstevel@tonic-gate kpreempt_enable(); 37117c478bd9Sstevel@tonic-gate } 37127c478bd9Sstevel@tonic-gate 37137c478bd9Sstevel@tonic-gate /* 37147c478bd9Sstevel@tonic-gate * Count how many lgroups are spanned 37157c478bd9Sstevel@tonic-gate */ 37167c478bd9Sstevel@tonic-gate klgrpset_nlgrps(lgrpset, lgrps_spanned); 37177c478bd9Sstevel@tonic-gate ASSERT(lgrps_spanned <= nlgrps); 37187c478bd9Sstevel@tonic-gate 37197c478bd9Sstevel@tonic-gate /* 37207c478bd9Sstevel@tonic-gate * Probably lgrps_spanned should be always non-zero, but to be 37217c478bd9Sstevel@tonic-gate * on the safe side we return lgrp_root if it is empty. 37227c478bd9Sstevel@tonic-gate */ 37237c478bd9Sstevel@tonic-gate if (lgrps_spanned == 0) { 37247c478bd9Sstevel@tonic-gate lgrp = lgrp_root; 37257c478bd9Sstevel@tonic-gate break; 37267c478bd9Sstevel@tonic-gate } 37277c478bd9Sstevel@tonic-gate 37287c478bd9Sstevel@tonic-gate /* 37297c478bd9Sstevel@tonic-gate * Pick a random offset within lgroups spanned 37307c478bd9Sstevel@tonic-gate * and return lgroup at that offset 37317c478bd9Sstevel@tonic-gate */ 37327c478bd9Sstevel@tonic-gate random = (ushort_t)gethrtime() >> 4; 37337c478bd9Sstevel@tonic-gate off = random % lgrps_spanned; 37347c478bd9Sstevel@tonic-gate ASSERT(off <= lgrp_alloc_max); 37357c478bd9Sstevel@tonic-gate 37367c478bd9Sstevel@tonic-gate for (i = 0; i <= lgrp_alloc_max; i++) { 37377c478bd9Sstevel@tonic-gate if (!klgrpset_ismember(lgrpset, i)) 37387c478bd9Sstevel@tonic-gate continue; 37397c478bd9Sstevel@tonic-gate if (off) 37407c478bd9Sstevel@tonic-gate off--; 37417c478bd9Sstevel@tonic-gate else { 37427c478bd9Sstevel@tonic-gate lgrp = lgrp_table[i]; 37437c478bd9Sstevel@tonic-gate lgrp_stat_add(lgrp->lgrp_id, LGRP_NUM_RANDOM, 37447c478bd9Sstevel@tonic-gate 1); 37457c478bd9Sstevel@tonic-gate break; 37467c478bd9Sstevel@tonic-gate } 37477c478bd9Sstevel@tonic-gate } 37487c478bd9Sstevel@tonic-gate break; 37497c478bd9Sstevel@tonic-gate 37507c478bd9Sstevel@tonic-gate case LGRP_MEM_POLICY_ROUNDROBIN: 37517c478bd9Sstevel@tonic-gate 37527c478bd9Sstevel@tonic-gate /* 37537c478bd9Sstevel@tonic-gate * Use offset within segment to determine 37547c478bd9Sstevel@tonic-gate * offset from home lgroup to choose for 37557c478bd9Sstevel@tonic-gate * next lgroup to allocate memory from 37567c478bd9Sstevel@tonic-gate */ 37577c478bd9Sstevel@tonic-gate off = ((unsigned long)(vaddr - seg->s_base) / pgsz) % 37587c478bd9Sstevel@tonic-gate (lgrp_alloc_max + 1); 37597c478bd9Sstevel@tonic-gate 37607c478bd9Sstevel@tonic-gate kpreempt_disable(); 37617c478bd9Sstevel@tonic-gate lgrpset = lgrp_root->lgrp_set[LGRP_RSRC_MEM]; 37627c478bd9Sstevel@tonic-gate i = lgrp->lgrp_id; 37637c478bd9Sstevel@tonic-gate kpreempt_enable(); 37647c478bd9Sstevel@tonic-gate 37657c478bd9Sstevel@tonic-gate while (off > 0) { 37667c478bd9Sstevel@tonic-gate i = (i + 1) % (lgrp_alloc_max + 1); 37677c478bd9Sstevel@tonic-gate lgrp = lgrp_table[i]; 37687c478bd9Sstevel@tonic-gate if (klgrpset_ismember(lgrpset, i)) 37697c478bd9Sstevel@tonic-gate off--; 37707c478bd9Sstevel@tonic-gate } 37717c478bd9Sstevel@tonic-gate lgrp_stat_add(lgrp->lgrp_id, LGRP_NUM_ROUNDROBIN, 1); 37727c478bd9Sstevel@tonic-gate 37737c478bd9Sstevel@tonic-gate break; 37747c478bd9Sstevel@tonic-gate } 37757c478bd9Sstevel@tonic-gate 37767c478bd9Sstevel@tonic-gate ASSERT(lgrp != NULL); 37777c478bd9Sstevel@tonic-gate return (lgrp); 37787c478bd9Sstevel@tonic-gate } 37797c478bd9Sstevel@tonic-gate 37807c478bd9Sstevel@tonic-gate /* 37817c478bd9Sstevel@tonic-gate * Return the number of pages in an lgroup 37827c478bd9Sstevel@tonic-gate * 37837c478bd9Sstevel@tonic-gate * NOTE: NUMA test (numat) driver uses this, so changing arguments or semantics 37847c478bd9Sstevel@tonic-gate * could cause tests that rely on the numat driver to fail.... 37857c478bd9Sstevel@tonic-gate */ 37867c478bd9Sstevel@tonic-gate pgcnt_t 37877c478bd9Sstevel@tonic-gate lgrp_mem_size(lgrp_id_t lgrpid, lgrp_mem_query_t query) 37887c478bd9Sstevel@tonic-gate { 37897c478bd9Sstevel@tonic-gate lgrp_t *lgrp; 37907c478bd9Sstevel@tonic-gate 37917c478bd9Sstevel@tonic-gate lgrp = lgrp_table[lgrpid]; 37927c478bd9Sstevel@tonic-gate if (!LGRP_EXISTS(lgrp) || 37937c478bd9Sstevel@tonic-gate klgrpset_isempty(lgrp->lgrp_set[LGRP_RSRC_MEM]) || 37947c478bd9Sstevel@tonic-gate !klgrpset_ismember(lgrp->lgrp_set[LGRP_RSRC_MEM], lgrpid)) 37957c478bd9Sstevel@tonic-gate return (0); 37967c478bd9Sstevel@tonic-gate 37977c478bd9Sstevel@tonic-gate return (lgrp_plat_mem_size(lgrp->lgrp_plathand, query)); 37987c478bd9Sstevel@tonic-gate } 37997c478bd9Sstevel@tonic-gate 38007c478bd9Sstevel@tonic-gate /* 38017c478bd9Sstevel@tonic-gate * Initialize lgroup shared memory allocation policy support 38027c478bd9Sstevel@tonic-gate */ 38037c478bd9Sstevel@tonic-gate void 38047c478bd9Sstevel@tonic-gate lgrp_shm_policy_init(struct anon_map *amp, vnode_t *vp) 38057c478bd9Sstevel@tonic-gate { 38067c478bd9Sstevel@tonic-gate lgrp_shm_locality_t *shm_locality; 38077c478bd9Sstevel@tonic-gate 38087c478bd9Sstevel@tonic-gate /* 38097c478bd9Sstevel@tonic-gate * Initialize locality field in anon_map 38107c478bd9Sstevel@tonic-gate * Don't need any locks because this is called when anon_map is 38117c478bd9Sstevel@tonic-gate * allocated, but not used anywhere yet. 38127c478bd9Sstevel@tonic-gate */ 38137c478bd9Sstevel@tonic-gate if (amp) { 38147c478bd9Sstevel@tonic-gate ANON_LOCK_ENTER(&->a_rwlock, RW_WRITER); 38157c478bd9Sstevel@tonic-gate if (amp->locality == NULL) { 38167c478bd9Sstevel@tonic-gate /* 38177c478bd9Sstevel@tonic-gate * Allocate and initialize shared memory locality info 38187c478bd9Sstevel@tonic-gate * and set anon_map locality pointer to it 38197c478bd9Sstevel@tonic-gate * Drop lock across kmem_alloc(KM_SLEEP) 38207c478bd9Sstevel@tonic-gate */ 38217c478bd9Sstevel@tonic-gate ANON_LOCK_EXIT(&->a_rwlock); 38227c478bd9Sstevel@tonic-gate shm_locality = kmem_alloc(sizeof (*shm_locality), 38237c478bd9Sstevel@tonic-gate KM_SLEEP); 38247c478bd9Sstevel@tonic-gate rw_init(&shm_locality->loc_lock, NULL, RW_DEFAULT, 38257c478bd9Sstevel@tonic-gate NULL); 38267c478bd9Sstevel@tonic-gate shm_locality->loc_count = 1; /* not used for amp */ 38277c478bd9Sstevel@tonic-gate shm_locality->loc_tree = NULL; 38287c478bd9Sstevel@tonic-gate 38297c478bd9Sstevel@tonic-gate /* 38307c478bd9Sstevel@tonic-gate * Reacquire lock and check to see whether anyone beat 38317c478bd9Sstevel@tonic-gate * us to initializing the locality info 38327c478bd9Sstevel@tonic-gate */ 38337c478bd9Sstevel@tonic-gate ANON_LOCK_ENTER(&->a_rwlock, RW_WRITER); 38347c478bd9Sstevel@tonic-gate if (amp->locality != NULL) { 38357c478bd9Sstevel@tonic-gate rw_destroy(&shm_locality->loc_lock); 38367c478bd9Sstevel@tonic-gate kmem_free(shm_locality, 38377c478bd9Sstevel@tonic-gate sizeof (*shm_locality)); 38387c478bd9Sstevel@tonic-gate } else 38397c478bd9Sstevel@tonic-gate amp->locality = shm_locality; 38407c478bd9Sstevel@tonic-gate } 38417c478bd9Sstevel@tonic-gate ANON_LOCK_EXIT(&->a_rwlock); 38427c478bd9Sstevel@tonic-gate return; 38437c478bd9Sstevel@tonic-gate } 38447c478bd9Sstevel@tonic-gate 38457c478bd9Sstevel@tonic-gate /* 38467c478bd9Sstevel@tonic-gate * Allocate shared vnode policy info if vnode is not locality aware yet 38477c478bd9Sstevel@tonic-gate */ 38487c478bd9Sstevel@tonic-gate mutex_enter(&vp->v_lock); 38497c478bd9Sstevel@tonic-gate if ((vp->v_flag & V_LOCALITY) == 0) { 38507c478bd9Sstevel@tonic-gate /* 38517c478bd9Sstevel@tonic-gate * Allocate and initialize shared memory locality info 38527c478bd9Sstevel@tonic-gate */ 38537c478bd9Sstevel@tonic-gate mutex_exit(&vp->v_lock); 38547c478bd9Sstevel@tonic-gate shm_locality = kmem_alloc(sizeof (*shm_locality), KM_SLEEP); 38557c478bd9Sstevel@tonic-gate rw_init(&shm_locality->loc_lock, NULL, RW_DEFAULT, NULL); 38567c478bd9Sstevel@tonic-gate shm_locality->loc_count = 1; 38577c478bd9Sstevel@tonic-gate shm_locality->loc_tree = NULL; 38587c478bd9Sstevel@tonic-gate 38597c478bd9Sstevel@tonic-gate /* 38607c478bd9Sstevel@tonic-gate * Point vnode locality field at shared vnode policy info 38617c478bd9Sstevel@tonic-gate * and set locality aware flag in vnode 38627c478bd9Sstevel@tonic-gate */ 38637c478bd9Sstevel@tonic-gate mutex_enter(&vp->v_lock); 38647c478bd9Sstevel@tonic-gate if ((vp->v_flag & V_LOCALITY) == 0) { 38657c478bd9Sstevel@tonic-gate vp->v_locality = shm_locality; 38667c478bd9Sstevel@tonic-gate vp->v_flag |= V_LOCALITY; 38677c478bd9Sstevel@tonic-gate } else { 38687c478bd9Sstevel@tonic-gate /* 38697c478bd9Sstevel@tonic-gate * Lost race so free locality info and increment count. 38707c478bd9Sstevel@tonic-gate */ 38717c478bd9Sstevel@tonic-gate rw_destroy(&shm_locality->loc_lock); 38727c478bd9Sstevel@tonic-gate kmem_free(shm_locality, sizeof (*shm_locality)); 38737c478bd9Sstevel@tonic-gate shm_locality = vp->v_locality; 38747c478bd9Sstevel@tonic-gate shm_locality->loc_count++; 38757c478bd9Sstevel@tonic-gate } 38767c478bd9Sstevel@tonic-gate mutex_exit(&vp->v_lock); 38777c478bd9Sstevel@tonic-gate 38787c478bd9Sstevel@tonic-gate return; 38797c478bd9Sstevel@tonic-gate } 38807c478bd9Sstevel@tonic-gate 38817c478bd9Sstevel@tonic-gate /* 38827c478bd9Sstevel@tonic-gate * Increment reference count of number of segments mapping this vnode 38837c478bd9Sstevel@tonic-gate * shared 38847c478bd9Sstevel@tonic-gate */ 38857c478bd9Sstevel@tonic-gate shm_locality = vp->v_locality; 38867c478bd9Sstevel@tonic-gate shm_locality->loc_count++; 38877c478bd9Sstevel@tonic-gate mutex_exit(&vp->v_lock); 38887c478bd9Sstevel@tonic-gate } 38897c478bd9Sstevel@tonic-gate 38907c478bd9Sstevel@tonic-gate /* 38917c478bd9Sstevel@tonic-gate * Destroy the given shared memory policy segment tree 38927c478bd9Sstevel@tonic-gate */ 38937c478bd9Sstevel@tonic-gate void 38947c478bd9Sstevel@tonic-gate lgrp_shm_policy_tree_destroy(avl_tree_t *tree) 38957c478bd9Sstevel@tonic-gate { 38967c478bd9Sstevel@tonic-gate lgrp_shm_policy_seg_t *cur; 38977c478bd9Sstevel@tonic-gate lgrp_shm_policy_seg_t *next; 38987c478bd9Sstevel@tonic-gate 38997c478bd9Sstevel@tonic-gate if (tree == NULL) 39007c478bd9Sstevel@tonic-gate return; 39017c478bd9Sstevel@tonic-gate 39027c478bd9Sstevel@tonic-gate cur = (lgrp_shm_policy_seg_t *)avl_first(tree); 39037c478bd9Sstevel@tonic-gate while (cur != NULL) { 39047c478bd9Sstevel@tonic-gate next = AVL_NEXT(tree, cur); 39057c478bd9Sstevel@tonic-gate avl_remove(tree, cur); 39067c478bd9Sstevel@tonic-gate kmem_free(cur, sizeof (*cur)); 39077c478bd9Sstevel@tonic-gate cur = next; 39087c478bd9Sstevel@tonic-gate } 39097c478bd9Sstevel@tonic-gate kmem_free(tree, sizeof (avl_tree_t)); 39107c478bd9Sstevel@tonic-gate } 39117c478bd9Sstevel@tonic-gate 39127c478bd9Sstevel@tonic-gate /* 39137c478bd9Sstevel@tonic-gate * Uninitialize lgroup shared memory allocation policy support 39147c478bd9Sstevel@tonic-gate */ 39157c478bd9Sstevel@tonic-gate void 39167c478bd9Sstevel@tonic-gate lgrp_shm_policy_fini(struct anon_map *amp, vnode_t *vp) 39177c478bd9Sstevel@tonic-gate { 39187c478bd9Sstevel@tonic-gate lgrp_shm_locality_t *shm_locality; 39197c478bd9Sstevel@tonic-gate 39207c478bd9Sstevel@tonic-gate /* 39217c478bd9Sstevel@tonic-gate * For anon_map, deallocate shared memory policy tree and 39227c478bd9Sstevel@tonic-gate * zero locality field 39237c478bd9Sstevel@tonic-gate * Don't need any locks because anon_map is being freed 39247c478bd9Sstevel@tonic-gate */ 39257c478bd9Sstevel@tonic-gate if (amp) { 39267c478bd9Sstevel@tonic-gate if (amp->locality == NULL) 39277c478bd9Sstevel@tonic-gate return; 39287c478bd9Sstevel@tonic-gate shm_locality = amp->locality; 39297c478bd9Sstevel@tonic-gate shm_locality->loc_count = 0; /* not really used for amp */ 39307c478bd9Sstevel@tonic-gate rw_destroy(&shm_locality->loc_lock); 39317c478bd9Sstevel@tonic-gate lgrp_shm_policy_tree_destroy(shm_locality->loc_tree); 39327c478bd9Sstevel@tonic-gate kmem_free(shm_locality, sizeof (*shm_locality)); 39337c478bd9Sstevel@tonic-gate amp->locality = 0; 39347c478bd9Sstevel@tonic-gate return; 39357c478bd9Sstevel@tonic-gate } 39367c478bd9Sstevel@tonic-gate 39377c478bd9Sstevel@tonic-gate /* 39387c478bd9Sstevel@tonic-gate * For vnode, decrement reference count of segments mapping this vnode 39397c478bd9Sstevel@tonic-gate * shared and delete locality info if reference count drops to 0 39407c478bd9Sstevel@tonic-gate */ 39417c478bd9Sstevel@tonic-gate mutex_enter(&vp->v_lock); 39427c478bd9Sstevel@tonic-gate shm_locality = vp->v_locality; 39437c478bd9Sstevel@tonic-gate shm_locality->loc_count--; 39447c478bd9Sstevel@tonic-gate 39457c478bd9Sstevel@tonic-gate if (shm_locality->loc_count == 0) { 39467c478bd9Sstevel@tonic-gate rw_destroy(&shm_locality->loc_lock); 39477c478bd9Sstevel@tonic-gate lgrp_shm_policy_tree_destroy(shm_locality->loc_tree); 39487c478bd9Sstevel@tonic-gate kmem_free(shm_locality, sizeof (*shm_locality)); 39497c478bd9Sstevel@tonic-gate vp->v_locality = 0; 39507c478bd9Sstevel@tonic-gate vp->v_flag &= ~V_LOCALITY; 39517c478bd9Sstevel@tonic-gate } 39527c478bd9Sstevel@tonic-gate mutex_exit(&vp->v_lock); 39537c478bd9Sstevel@tonic-gate } 39547c478bd9Sstevel@tonic-gate 39557c478bd9Sstevel@tonic-gate /* 39567c478bd9Sstevel@tonic-gate * Compare two shared memory policy segments 39577c478bd9Sstevel@tonic-gate * Used by AVL tree code for searching 39587c478bd9Sstevel@tonic-gate */ 39597c478bd9Sstevel@tonic-gate int 39607c478bd9Sstevel@tonic-gate lgrp_shm_policy_compar(const void *x, const void *y) 39617c478bd9Sstevel@tonic-gate { 39627c478bd9Sstevel@tonic-gate lgrp_shm_policy_seg_t *a = (lgrp_shm_policy_seg_t *)x; 39637c478bd9Sstevel@tonic-gate lgrp_shm_policy_seg_t *b = (lgrp_shm_policy_seg_t *)y; 39647c478bd9Sstevel@tonic-gate 39657c478bd9Sstevel@tonic-gate if (a->shm_off < b->shm_off) 39667c478bd9Sstevel@tonic-gate return (-1); 39677c478bd9Sstevel@tonic-gate if (a->shm_off >= b->shm_off + b->shm_size) 39687c478bd9Sstevel@tonic-gate return (1); 39697c478bd9Sstevel@tonic-gate return (0); 39707c478bd9Sstevel@tonic-gate } 39717c478bd9Sstevel@tonic-gate 39727c478bd9Sstevel@tonic-gate /* 39737c478bd9Sstevel@tonic-gate * Concatenate seg1 with seg2 and remove seg2 39747c478bd9Sstevel@tonic-gate */ 39757c478bd9Sstevel@tonic-gate static int 39767c478bd9Sstevel@tonic-gate lgrp_shm_policy_concat(avl_tree_t *tree, lgrp_shm_policy_seg_t *seg1, 39777c478bd9Sstevel@tonic-gate lgrp_shm_policy_seg_t *seg2) 39787c478bd9Sstevel@tonic-gate { 39797c478bd9Sstevel@tonic-gate if (!seg1 || !seg2 || 39807c478bd9Sstevel@tonic-gate seg1->shm_off + seg1->shm_size != seg2->shm_off || 39817c478bd9Sstevel@tonic-gate seg1->shm_policy.mem_policy != seg2->shm_policy.mem_policy) 39827c478bd9Sstevel@tonic-gate return (-1); 39837c478bd9Sstevel@tonic-gate 39847c478bd9Sstevel@tonic-gate seg1->shm_size += seg2->shm_size; 39857c478bd9Sstevel@tonic-gate avl_remove(tree, seg2); 39867c478bd9Sstevel@tonic-gate kmem_free(seg2, sizeof (*seg2)); 39877c478bd9Sstevel@tonic-gate return (0); 39887c478bd9Sstevel@tonic-gate } 39897c478bd9Sstevel@tonic-gate 39907c478bd9Sstevel@tonic-gate /* 39917c478bd9Sstevel@tonic-gate * Split segment at given offset and return rightmost (uppermost) segment 39927c478bd9Sstevel@tonic-gate * Assumes that there are no overlapping segments 39937c478bd9Sstevel@tonic-gate */ 39947c478bd9Sstevel@tonic-gate static lgrp_shm_policy_seg_t * 39957c478bd9Sstevel@tonic-gate lgrp_shm_policy_split(avl_tree_t *tree, lgrp_shm_policy_seg_t *seg, 39967c478bd9Sstevel@tonic-gate u_offset_t off) 39977c478bd9Sstevel@tonic-gate { 39987c478bd9Sstevel@tonic-gate lgrp_shm_policy_seg_t *newseg; 39997c478bd9Sstevel@tonic-gate avl_index_t where; 40007c478bd9Sstevel@tonic-gate 40017c478bd9Sstevel@tonic-gate ASSERT(seg != NULL); 40027c478bd9Sstevel@tonic-gate ASSERT(off >= seg->shm_off && off <= seg->shm_off + seg->shm_size); 40037c478bd9Sstevel@tonic-gate 40047c478bd9Sstevel@tonic-gate if (!seg || off < seg->shm_off || off > seg->shm_off + 40057c478bd9Sstevel@tonic-gate seg->shm_size) 40067c478bd9Sstevel@tonic-gate return (NULL); 40077c478bd9Sstevel@tonic-gate 40087c478bd9Sstevel@tonic-gate if (off == seg->shm_off || off == seg->shm_off + seg->shm_size) 40097c478bd9Sstevel@tonic-gate return (seg); 40107c478bd9Sstevel@tonic-gate 40117c478bd9Sstevel@tonic-gate /* 40127c478bd9Sstevel@tonic-gate * Adjust size of left segment and allocate new (right) segment 40137c478bd9Sstevel@tonic-gate */ 40147c478bd9Sstevel@tonic-gate newseg = kmem_alloc(sizeof (lgrp_shm_policy_seg_t), KM_SLEEP); 40157c478bd9Sstevel@tonic-gate newseg->shm_policy = seg->shm_policy; 40167c478bd9Sstevel@tonic-gate newseg->shm_off = off; 40177c478bd9Sstevel@tonic-gate newseg->shm_size = seg->shm_size - (off - seg->shm_off); 40187c478bd9Sstevel@tonic-gate seg->shm_size = off - seg->shm_off; 40197c478bd9Sstevel@tonic-gate 40207c478bd9Sstevel@tonic-gate /* 40217c478bd9Sstevel@tonic-gate * Find where to insert new segment in AVL tree and insert it 40227c478bd9Sstevel@tonic-gate */ 40237c478bd9Sstevel@tonic-gate (void) avl_find(tree, &off, &where); 40247c478bd9Sstevel@tonic-gate avl_insert(tree, newseg, where); 40257c478bd9Sstevel@tonic-gate 40267c478bd9Sstevel@tonic-gate return (newseg); 40277c478bd9Sstevel@tonic-gate } 40287c478bd9Sstevel@tonic-gate 40297c478bd9Sstevel@tonic-gate /* 40307c478bd9Sstevel@tonic-gate * Set shared memory allocation policy on specified shared object at given 40317c478bd9Sstevel@tonic-gate * offset and length 40327c478bd9Sstevel@tonic-gate * 40337c478bd9Sstevel@tonic-gate * Return 0 if policy wasn't set already, 1 if policy was set already, and 40347c478bd9Sstevel@tonic-gate * -1 if can't set policy. 40357c478bd9Sstevel@tonic-gate */ 40367c478bd9Sstevel@tonic-gate int 40377c478bd9Sstevel@tonic-gate lgrp_shm_policy_set(lgrp_mem_policy_t policy, struct anon_map *amp, 40387c478bd9Sstevel@tonic-gate ulong_t anon_index, vnode_t *vp, u_offset_t vn_off, size_t len) 40397c478bd9Sstevel@tonic-gate { 40407c478bd9Sstevel@tonic-gate u_offset_t eoff; 40417c478bd9Sstevel@tonic-gate lgrp_shm_policy_seg_t *next; 40427c478bd9Sstevel@tonic-gate lgrp_shm_policy_seg_t *newseg; 40437c478bd9Sstevel@tonic-gate u_offset_t off; 40447c478bd9Sstevel@tonic-gate u_offset_t oldeoff; 40457c478bd9Sstevel@tonic-gate lgrp_shm_policy_seg_t *prev; 40467c478bd9Sstevel@tonic-gate int retval; 40477c478bd9Sstevel@tonic-gate lgrp_shm_policy_seg_t *seg; 40487c478bd9Sstevel@tonic-gate lgrp_shm_locality_t *shm_locality; 40497c478bd9Sstevel@tonic-gate avl_tree_t *tree; 40507c478bd9Sstevel@tonic-gate avl_index_t where; 40517c478bd9Sstevel@tonic-gate 40527c478bd9Sstevel@tonic-gate ASSERT(amp || vp); 40537c478bd9Sstevel@tonic-gate ASSERT((len & PAGEOFFSET) == 0); 40547c478bd9Sstevel@tonic-gate 40557c478bd9Sstevel@tonic-gate if (len == 0) 40567c478bd9Sstevel@tonic-gate return (-1); 40577c478bd9Sstevel@tonic-gate 40587c478bd9Sstevel@tonic-gate retval = 0; 40597c478bd9Sstevel@tonic-gate 40607c478bd9Sstevel@tonic-gate /* 40617c478bd9Sstevel@tonic-gate * Get locality info and starting offset into shared object 40627c478bd9Sstevel@tonic-gate * Try anon map first and then vnode 40637c478bd9Sstevel@tonic-gate * Assume that no locks need to be held on anon_map or vnode, since 40647c478bd9Sstevel@tonic-gate * it should be protected by its reference count which must be nonzero 40657c478bd9Sstevel@tonic-gate * for an existing segment. 40667c478bd9Sstevel@tonic-gate */ 40677c478bd9Sstevel@tonic-gate if (amp) { 40687c478bd9Sstevel@tonic-gate /* 40697c478bd9Sstevel@tonic-gate * Get policy info from anon_map 40707c478bd9Sstevel@tonic-gate * 40717c478bd9Sstevel@tonic-gate */ 40727c478bd9Sstevel@tonic-gate ASSERT(amp->refcnt != 0); 40737c478bd9Sstevel@tonic-gate if (amp->locality == NULL) 40747c478bd9Sstevel@tonic-gate lgrp_shm_policy_init(amp, NULL); 40757c478bd9Sstevel@tonic-gate shm_locality = amp->locality; 40767c478bd9Sstevel@tonic-gate off = ptob(anon_index); 40777c478bd9Sstevel@tonic-gate } else if (vp) { 40787c478bd9Sstevel@tonic-gate /* 40797c478bd9Sstevel@tonic-gate * Get policy info from vnode 40807c478bd9Sstevel@tonic-gate */ 40817c478bd9Sstevel@tonic-gate if ((vp->v_flag & V_LOCALITY) == 0 || vp->v_locality == NULL) 40827c478bd9Sstevel@tonic-gate lgrp_shm_policy_init(NULL, vp); 40837c478bd9Sstevel@tonic-gate shm_locality = vp->v_locality; 40847c478bd9Sstevel@tonic-gate ASSERT(shm_locality->loc_count != 0); 40857c478bd9Sstevel@tonic-gate off = vn_off; 40867c478bd9Sstevel@tonic-gate } else 40877c478bd9Sstevel@tonic-gate return (-1); 40887c478bd9Sstevel@tonic-gate 40897c478bd9Sstevel@tonic-gate ASSERT((off & PAGEOFFSET) == 0); 40907c478bd9Sstevel@tonic-gate 40917c478bd9Sstevel@tonic-gate /* 40927c478bd9Sstevel@tonic-gate * Figure out default policy 40937c478bd9Sstevel@tonic-gate */ 40947c478bd9Sstevel@tonic-gate if (policy == LGRP_MEM_POLICY_DEFAULT) 40957c478bd9Sstevel@tonic-gate policy = lgrp_mem_policy_default(len, MAP_SHARED); 40967c478bd9Sstevel@tonic-gate 40977c478bd9Sstevel@tonic-gate /* 40987c478bd9Sstevel@tonic-gate * Create AVL tree if there isn't one yet 40997c478bd9Sstevel@tonic-gate * and set locality field to point at it 41007c478bd9Sstevel@tonic-gate */ 41017c478bd9Sstevel@tonic-gate rw_enter(&shm_locality->loc_lock, RW_WRITER); 41027c478bd9Sstevel@tonic-gate tree = shm_locality->loc_tree; 41037c478bd9Sstevel@tonic-gate if (!tree) { 41047c478bd9Sstevel@tonic-gate rw_exit(&shm_locality->loc_lock); 41057c478bd9Sstevel@tonic-gate 41067c478bd9Sstevel@tonic-gate tree = kmem_alloc(sizeof (avl_tree_t), KM_SLEEP); 41077c478bd9Sstevel@tonic-gate 41087c478bd9Sstevel@tonic-gate rw_enter(&shm_locality->loc_lock, RW_WRITER); 41097c478bd9Sstevel@tonic-gate if (shm_locality->loc_tree == NULL) { 41107c478bd9Sstevel@tonic-gate avl_create(tree, lgrp_shm_policy_compar, 41117c478bd9Sstevel@tonic-gate sizeof (lgrp_shm_policy_seg_t), 41127c478bd9Sstevel@tonic-gate offsetof(lgrp_shm_policy_seg_t, shm_tree)); 41137c478bd9Sstevel@tonic-gate shm_locality->loc_tree = tree; 41147c478bd9Sstevel@tonic-gate } else { 41157c478bd9Sstevel@tonic-gate /* 41167c478bd9Sstevel@tonic-gate * Another thread managed to set up the tree 41177c478bd9Sstevel@tonic-gate * before we could. Free the tree we allocated 41187c478bd9Sstevel@tonic-gate * and use the one that's already there. 41197c478bd9Sstevel@tonic-gate */ 41207c478bd9Sstevel@tonic-gate kmem_free(tree, sizeof (*tree)); 41217c478bd9Sstevel@tonic-gate tree = shm_locality->loc_tree; 41227c478bd9Sstevel@tonic-gate } 41237c478bd9Sstevel@tonic-gate } 41247c478bd9Sstevel@tonic-gate 41257c478bd9Sstevel@tonic-gate /* 41267c478bd9Sstevel@tonic-gate * Set policy 41277c478bd9Sstevel@tonic-gate * 41287c478bd9Sstevel@tonic-gate * Need to maintain hold on writer's lock to keep tree from 41297c478bd9Sstevel@tonic-gate * changing out from under us 41307c478bd9Sstevel@tonic-gate */ 41317c478bd9Sstevel@tonic-gate while (len != 0) { 41327c478bd9Sstevel@tonic-gate /* 41337c478bd9Sstevel@tonic-gate * Find policy segment for specified offset into shared object 41347c478bd9Sstevel@tonic-gate */ 41357c478bd9Sstevel@tonic-gate seg = avl_find(tree, &off, &where); 41367c478bd9Sstevel@tonic-gate 41377c478bd9Sstevel@tonic-gate /* 41387c478bd9Sstevel@tonic-gate * Didn't find any existing segment that contains specified 41397c478bd9Sstevel@tonic-gate * offset, so allocate new segment, insert it, and concatenate 41407c478bd9Sstevel@tonic-gate * with adjacent segments if possible 41417c478bd9Sstevel@tonic-gate */ 41427c478bd9Sstevel@tonic-gate if (seg == NULL) { 41437c478bd9Sstevel@tonic-gate newseg = kmem_alloc(sizeof (lgrp_shm_policy_seg_t), 41447c478bd9Sstevel@tonic-gate KM_SLEEP); 41457c478bd9Sstevel@tonic-gate newseg->shm_policy.mem_policy = policy; 41467c478bd9Sstevel@tonic-gate newseg->shm_policy.mem_reserved = 0; 41477c478bd9Sstevel@tonic-gate newseg->shm_off = off; 41487c478bd9Sstevel@tonic-gate avl_insert(tree, newseg, where); 41497c478bd9Sstevel@tonic-gate 41507c478bd9Sstevel@tonic-gate /* 41517c478bd9Sstevel@tonic-gate * Check to see whether new segment overlaps with next 41527c478bd9Sstevel@tonic-gate * one, set length of new segment accordingly, and 41537c478bd9Sstevel@tonic-gate * calculate remaining length and next offset 41547c478bd9Sstevel@tonic-gate */ 41557c478bd9Sstevel@tonic-gate seg = AVL_NEXT(tree, newseg); 41567c478bd9Sstevel@tonic-gate if (seg == NULL || off + len <= seg->shm_off) { 41577c478bd9Sstevel@tonic-gate newseg->shm_size = len; 41587c478bd9Sstevel@tonic-gate len = 0; 41597c478bd9Sstevel@tonic-gate } else { 41607c478bd9Sstevel@tonic-gate newseg->shm_size = seg->shm_off - off; 41617c478bd9Sstevel@tonic-gate off = seg->shm_off; 41627c478bd9Sstevel@tonic-gate len -= newseg->shm_size; 41637c478bd9Sstevel@tonic-gate } 41647c478bd9Sstevel@tonic-gate 41657c478bd9Sstevel@tonic-gate /* 41667c478bd9Sstevel@tonic-gate * Try to concatenate new segment with next and 41677c478bd9Sstevel@tonic-gate * previous ones, since they might have the same policy 41687c478bd9Sstevel@tonic-gate * now. Grab previous and next segments first because 41697c478bd9Sstevel@tonic-gate * they will change on concatenation. 41707c478bd9Sstevel@tonic-gate */ 41717c478bd9Sstevel@tonic-gate prev = AVL_PREV(tree, newseg); 41727c478bd9Sstevel@tonic-gate next = AVL_NEXT(tree, newseg); 41737c478bd9Sstevel@tonic-gate (void) lgrp_shm_policy_concat(tree, newseg, next); 41747c478bd9Sstevel@tonic-gate (void) lgrp_shm_policy_concat(tree, prev, newseg); 41757c478bd9Sstevel@tonic-gate 41767c478bd9Sstevel@tonic-gate continue; 41777c478bd9Sstevel@tonic-gate } 41787c478bd9Sstevel@tonic-gate 41797c478bd9Sstevel@tonic-gate eoff = off + len; 41807c478bd9Sstevel@tonic-gate oldeoff = seg->shm_off + seg->shm_size; 41817c478bd9Sstevel@tonic-gate 41827c478bd9Sstevel@tonic-gate /* 41837c478bd9Sstevel@tonic-gate * Policy set already? 41847c478bd9Sstevel@tonic-gate */ 41857c478bd9Sstevel@tonic-gate if (policy == seg->shm_policy.mem_policy) { 41867c478bd9Sstevel@tonic-gate /* 41877c478bd9Sstevel@tonic-gate * Nothing left to do if offset and length 41887c478bd9Sstevel@tonic-gate * fall within this segment 41897c478bd9Sstevel@tonic-gate */ 41907c478bd9Sstevel@tonic-gate if (eoff <= oldeoff) { 41917c478bd9Sstevel@tonic-gate retval = 1; 41927c478bd9Sstevel@tonic-gate break; 41937c478bd9Sstevel@tonic-gate } else { 41947c478bd9Sstevel@tonic-gate len = eoff - oldeoff; 41957c478bd9Sstevel@tonic-gate off = oldeoff; 41967c478bd9Sstevel@tonic-gate continue; 41977c478bd9Sstevel@tonic-gate } 41987c478bd9Sstevel@tonic-gate } 41997c478bd9Sstevel@tonic-gate 42007c478bd9Sstevel@tonic-gate /* 42017c478bd9Sstevel@tonic-gate * Specified offset and length match existing segment exactly 42027c478bd9Sstevel@tonic-gate */ 42037c478bd9Sstevel@tonic-gate if (off == seg->shm_off && len == seg->shm_size) { 42047c478bd9Sstevel@tonic-gate /* 42057c478bd9Sstevel@tonic-gate * Set policy and update current length 42067c478bd9Sstevel@tonic-gate */ 42077c478bd9Sstevel@tonic-gate seg->shm_policy.mem_policy = policy; 42087c478bd9Sstevel@tonic-gate seg->shm_policy.mem_reserved = 0; 42097c478bd9Sstevel@tonic-gate len = 0; 42107c478bd9Sstevel@tonic-gate 42117c478bd9Sstevel@tonic-gate /* 42127c478bd9Sstevel@tonic-gate * Try concatenating new segment with previous and next 42137c478bd9Sstevel@tonic-gate * segments, since they might have the same policy now. 42147c478bd9Sstevel@tonic-gate * Grab previous and next segments first because they 42157c478bd9Sstevel@tonic-gate * will change on concatenation. 42167c478bd9Sstevel@tonic-gate */ 42177c478bd9Sstevel@tonic-gate prev = AVL_PREV(tree, seg); 42187c478bd9Sstevel@tonic-gate next = AVL_NEXT(tree, seg); 42197c478bd9Sstevel@tonic-gate (void) lgrp_shm_policy_concat(tree, seg, next); 42207c478bd9Sstevel@tonic-gate (void) lgrp_shm_policy_concat(tree, prev, seg); 42217c478bd9Sstevel@tonic-gate } else { 42227c478bd9Sstevel@tonic-gate /* 42237c478bd9Sstevel@tonic-gate * Specified offset and length only apply to part of 42247c478bd9Sstevel@tonic-gate * existing segment 42257c478bd9Sstevel@tonic-gate */ 42267c478bd9Sstevel@tonic-gate 42277c478bd9Sstevel@tonic-gate /* 42287c478bd9Sstevel@tonic-gate * New segment starts in middle of old one, so split 42297c478bd9Sstevel@tonic-gate * new one off near beginning of old one 42307c478bd9Sstevel@tonic-gate */ 42317c478bd9Sstevel@tonic-gate newseg = NULL; 42327c478bd9Sstevel@tonic-gate if (off > seg->shm_off) { 42337c478bd9Sstevel@tonic-gate newseg = lgrp_shm_policy_split(tree, seg, off); 42347c478bd9Sstevel@tonic-gate 42357c478bd9Sstevel@tonic-gate /* 42367c478bd9Sstevel@tonic-gate * New segment ends where old one did, so try 42377c478bd9Sstevel@tonic-gate * to concatenate with next segment 42387c478bd9Sstevel@tonic-gate */ 42397c478bd9Sstevel@tonic-gate if (eoff == oldeoff) { 42407c478bd9Sstevel@tonic-gate newseg->shm_policy.mem_policy = policy; 42417c478bd9Sstevel@tonic-gate newseg->shm_policy.mem_reserved = 0; 42427c478bd9Sstevel@tonic-gate (void) lgrp_shm_policy_concat(tree, 42437c478bd9Sstevel@tonic-gate newseg, AVL_NEXT(tree, newseg)); 42447c478bd9Sstevel@tonic-gate break; 42457c478bd9Sstevel@tonic-gate } 42467c478bd9Sstevel@tonic-gate } 42477c478bd9Sstevel@tonic-gate 42487c478bd9Sstevel@tonic-gate /* 42497c478bd9Sstevel@tonic-gate * New segment ends before old one, so split off end of 42507c478bd9Sstevel@tonic-gate * old one 42517c478bd9Sstevel@tonic-gate */ 42527c478bd9Sstevel@tonic-gate if (eoff < oldeoff) { 42537c478bd9Sstevel@tonic-gate if (newseg) { 42547c478bd9Sstevel@tonic-gate (void) lgrp_shm_policy_split(tree, 42557c478bd9Sstevel@tonic-gate newseg, eoff); 42567c478bd9Sstevel@tonic-gate newseg->shm_policy.mem_policy = policy; 42577c478bd9Sstevel@tonic-gate newseg->shm_policy.mem_reserved = 0; 42587c478bd9Sstevel@tonic-gate } else { 42597c478bd9Sstevel@tonic-gate (void) lgrp_shm_policy_split(tree, seg, 42607c478bd9Sstevel@tonic-gate eoff); 42617c478bd9Sstevel@tonic-gate seg->shm_policy.mem_policy = policy; 42627c478bd9Sstevel@tonic-gate seg->shm_policy.mem_reserved = 0; 42637c478bd9Sstevel@tonic-gate } 42647c478bd9Sstevel@tonic-gate 42657c478bd9Sstevel@tonic-gate if (off == seg->shm_off) 42667c478bd9Sstevel@tonic-gate (void) lgrp_shm_policy_concat(tree, 42677c478bd9Sstevel@tonic-gate AVL_PREV(tree, seg), seg); 42687c478bd9Sstevel@tonic-gate break; 42697c478bd9Sstevel@tonic-gate } 42707c478bd9Sstevel@tonic-gate 42717c478bd9Sstevel@tonic-gate /* 42727c478bd9Sstevel@tonic-gate * Calculate remaining length and next offset 42737c478bd9Sstevel@tonic-gate */ 42747c478bd9Sstevel@tonic-gate len = eoff - oldeoff; 42757c478bd9Sstevel@tonic-gate off = oldeoff; 42767c478bd9Sstevel@tonic-gate } 42777c478bd9Sstevel@tonic-gate } 42787c478bd9Sstevel@tonic-gate 42797c478bd9Sstevel@tonic-gate rw_exit(&shm_locality->loc_lock); 42807c478bd9Sstevel@tonic-gate return (retval); 42817c478bd9Sstevel@tonic-gate } 42827c478bd9Sstevel@tonic-gate 42837c478bd9Sstevel@tonic-gate /* 42847c478bd9Sstevel@tonic-gate * Return the best memnode from which to allocate memory given 42857c478bd9Sstevel@tonic-gate * an lgroup. 42867c478bd9Sstevel@tonic-gate * 42877c478bd9Sstevel@tonic-gate * "c" is for cookie, which is good enough for me. 42887c478bd9Sstevel@tonic-gate * It references a cookie struct that should be zero'ed to initialize. 42897c478bd9Sstevel@tonic-gate * The cookie should live on the caller's stack. 42907c478bd9Sstevel@tonic-gate * 42917c478bd9Sstevel@tonic-gate * The routine returns -1 when: 42927c478bd9Sstevel@tonic-gate * - traverse is 0, and all the memnodes in "lgrp" have been returned. 42937c478bd9Sstevel@tonic-gate * - traverse is 1, and all the memnodes in the system have been 42947c478bd9Sstevel@tonic-gate * returned. 42957c478bd9Sstevel@tonic-gate */ 42967c478bd9Sstevel@tonic-gate int 42977c478bd9Sstevel@tonic-gate lgrp_memnode_choose(lgrp_mnode_cookie_t *c) 42987c478bd9Sstevel@tonic-gate { 42997c478bd9Sstevel@tonic-gate lgrp_t *lp = c->lmc_lgrp; 43007c478bd9Sstevel@tonic-gate mnodeset_t nodes = c->lmc_nodes; 43017c478bd9Sstevel@tonic-gate int cnt = c->lmc_cnt; 43027c478bd9Sstevel@tonic-gate int offset, mnode; 43037c478bd9Sstevel@tonic-gate 43047c478bd9Sstevel@tonic-gate extern int max_mem_nodes; 43057c478bd9Sstevel@tonic-gate 43067c478bd9Sstevel@tonic-gate /* 43077c478bd9Sstevel@tonic-gate * If the set is empty, and the caller is willing, traverse 43087c478bd9Sstevel@tonic-gate * up the hierarchy until we find a non-empty set. 43097c478bd9Sstevel@tonic-gate */ 43107c478bd9Sstevel@tonic-gate while (nodes == (mnodeset_t)0 || cnt <= 0) { 43117c478bd9Sstevel@tonic-gate if (c->lmc_scope == LGRP_SRCH_LOCAL || 43127c478bd9Sstevel@tonic-gate ((lp = lp->lgrp_parent) == NULL)) 43137c478bd9Sstevel@tonic-gate return (-1); 43147c478bd9Sstevel@tonic-gate 43157c478bd9Sstevel@tonic-gate nodes = lp->lgrp_mnodes & ~(c->lmc_tried); 43167c478bd9Sstevel@tonic-gate cnt = lp->lgrp_nmnodes - c->lmc_ntried; 43177c478bd9Sstevel@tonic-gate } 43187c478bd9Sstevel@tonic-gate 43197c478bd9Sstevel@tonic-gate /* 43207c478bd9Sstevel@tonic-gate * Select a memnode by picking one at a "random" offset. 43217c478bd9Sstevel@tonic-gate * Because of DR, memnodes can come and go at any time. 43227c478bd9Sstevel@tonic-gate * This code must be able to cope with the possibility 43237c478bd9Sstevel@tonic-gate * that the nodes count "cnt" is inconsistent with respect 43247c478bd9Sstevel@tonic-gate * to the number of elements actually in "nodes", and 43257c478bd9Sstevel@tonic-gate * therefore that the offset chosen could be greater than 43267c478bd9Sstevel@tonic-gate * the number of elements in the set (some memnodes may 43277c478bd9Sstevel@tonic-gate * have dissapeared just before cnt was read). 43287c478bd9Sstevel@tonic-gate * If this happens, the search simply wraps back to the 43297c478bd9Sstevel@tonic-gate * beginning of the set. 43307c478bd9Sstevel@tonic-gate */ 43317c478bd9Sstevel@tonic-gate ASSERT(nodes != (mnodeset_t)0 && cnt > 0); 43327c478bd9Sstevel@tonic-gate offset = c->lmc_rand % cnt; 43337c478bd9Sstevel@tonic-gate do { 43347c478bd9Sstevel@tonic-gate for (mnode = 0; mnode < max_mem_nodes; mnode++) 43357c478bd9Sstevel@tonic-gate if (nodes & ((mnodeset_t)1 << mnode)) 43367c478bd9Sstevel@tonic-gate if (!offset--) 43377c478bd9Sstevel@tonic-gate break; 43387c478bd9Sstevel@tonic-gate } while (mnode >= max_mem_nodes); 43397c478bd9Sstevel@tonic-gate 43407c478bd9Sstevel@tonic-gate /* Found a node. Store state before returning. */ 43417c478bd9Sstevel@tonic-gate c->lmc_lgrp = lp; 43427c478bd9Sstevel@tonic-gate c->lmc_nodes = (nodes & ~((mnodeset_t)1 << mnode)); 43437c478bd9Sstevel@tonic-gate c->lmc_cnt = cnt - 1; 43447c478bd9Sstevel@tonic-gate c->lmc_tried = (c->lmc_tried | ((mnodeset_t)1 << mnode)); 43457c478bd9Sstevel@tonic-gate c->lmc_ntried++; 43467c478bd9Sstevel@tonic-gate 43477c478bd9Sstevel@tonic-gate return (mnode); 43487c478bd9Sstevel@tonic-gate } 4349