17c478bd9Sstevel@tonic-gate /* 27c478bd9Sstevel@tonic-gate * CDDL HEADER START 37c478bd9Sstevel@tonic-gate * 47c478bd9Sstevel@tonic-gate * The contents of this file are subject to the terms of the 544961713Sgirish * Common Development and Distribution License (the "License"). 644961713Sgirish * You may not use this file except in compliance with the License. 77c478bd9Sstevel@tonic-gate * 87c478bd9Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 97c478bd9Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing. 107c478bd9Sstevel@tonic-gate * See the License for the specific language governing permissions 117c478bd9Sstevel@tonic-gate * and limitations under the License. 127c478bd9Sstevel@tonic-gate * 137c478bd9Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each 147c478bd9Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 157c478bd9Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the 167c478bd9Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying 177c478bd9Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner] 187c478bd9Sstevel@tonic-gate * 197c478bd9Sstevel@tonic-gate * CDDL HEADER END 207c478bd9Sstevel@tonic-gate */ 217aec1d6eScindi 227c478bd9Sstevel@tonic-gate /* 238d4e547dSae112802 * Copyright 2007 Sun Microsystems, Inc. All rights reserved. 247c478bd9Sstevel@tonic-gate * Use is subject to license terms. 257c478bd9Sstevel@tonic-gate */ 267c478bd9Sstevel@tonic-gate 277c478bd9Sstevel@tonic-gate #pragma ident "%Z%%M% %I% %E% SMI" 287c478bd9Sstevel@tonic-gate 297c478bd9Sstevel@tonic-gate #include <sys/param.h> 307c478bd9Sstevel@tonic-gate #include <sys/thread.h> 317c478bd9Sstevel@tonic-gate #include <sys/cpuvar.h> 327c478bd9Sstevel@tonic-gate #include <sys/inttypes.h> 337c478bd9Sstevel@tonic-gate #include <sys/cmn_err.h> 347c478bd9Sstevel@tonic-gate #include <sys/time.h> 357c478bd9Sstevel@tonic-gate #include <sys/mutex.h> 367c478bd9Sstevel@tonic-gate #include <sys/systm.h> 377c478bd9Sstevel@tonic-gate #include <sys/kcpc.h> 387c478bd9Sstevel@tonic-gate #include <sys/cpc_impl.h> 397c478bd9Sstevel@tonic-gate #include <sys/cpc_pcbe.h> 407c478bd9Sstevel@tonic-gate #include <sys/atomic.h> 417c478bd9Sstevel@tonic-gate #include <sys/sunddi.h> 427c478bd9Sstevel@tonic-gate #include <sys/modctl.h> 437c478bd9Sstevel@tonic-gate #include <sys/sdt.h> 447c478bd9Sstevel@tonic-gate #if defined(__x86) 457c478bd9Sstevel@tonic-gate #include <asm/clock.h> 467c478bd9Sstevel@tonic-gate #endif 477c478bd9Sstevel@tonic-gate 487c478bd9Sstevel@tonic-gate kmutex_t kcpc_ctx_llock[CPC_HASH_BUCKETS]; /* protects ctx_list */ 497c478bd9Sstevel@tonic-gate kcpc_ctx_t *kcpc_ctx_list[CPC_HASH_BUCKETS]; /* head of list */ 507c478bd9Sstevel@tonic-gate 517c478bd9Sstevel@tonic-gate 527c478bd9Sstevel@tonic-gate krwlock_t kcpc_cpuctx_lock; /* lock for 'kcpc_cpuctx' below */ 537c478bd9Sstevel@tonic-gate int kcpc_cpuctx; /* number of cpu-specific contexts */ 547c478bd9Sstevel@tonic-gate 557c478bd9Sstevel@tonic-gate int kcpc_counts_include_idle = 1; /* Project Private /etc/system variable */ 567c478bd9Sstevel@tonic-gate 577c478bd9Sstevel@tonic-gate /* 587c478bd9Sstevel@tonic-gate * These are set when a PCBE module is loaded. 597c478bd9Sstevel@tonic-gate */ 607c478bd9Sstevel@tonic-gate uint_t cpc_ncounters = 0; 617c478bd9Sstevel@tonic-gate pcbe_ops_t *pcbe_ops = NULL; 627c478bd9Sstevel@tonic-gate 637c478bd9Sstevel@tonic-gate /* 647c478bd9Sstevel@tonic-gate * Statistics on (mis)behavior 657c478bd9Sstevel@tonic-gate */ 667c478bd9Sstevel@tonic-gate static uint32_t kcpc_intrctx_count; /* # overflows in an interrupt handler */ 677c478bd9Sstevel@tonic-gate static uint32_t kcpc_nullctx_count; /* # overflows in a thread with no ctx */ 687c478bd9Sstevel@tonic-gate 697c478bd9Sstevel@tonic-gate /* 707c478bd9Sstevel@tonic-gate * Is misbehaviour (overflow in a thread with no context) fatal? 717c478bd9Sstevel@tonic-gate */ 727c478bd9Sstevel@tonic-gate #ifdef DEBUG 737c478bd9Sstevel@tonic-gate static int kcpc_nullctx_panic = 1; 747c478bd9Sstevel@tonic-gate #else 757c478bd9Sstevel@tonic-gate static int kcpc_nullctx_panic = 0; 767c478bd9Sstevel@tonic-gate #endif 777c478bd9Sstevel@tonic-gate 787c478bd9Sstevel@tonic-gate static void kcpc_lwp_create(kthread_t *t, kthread_t *ct); 797c478bd9Sstevel@tonic-gate static void kcpc_restore(kcpc_ctx_t *ctx); 807c478bd9Sstevel@tonic-gate static void kcpc_save(kcpc_ctx_t *ctx); 817c478bd9Sstevel@tonic-gate static void kcpc_free(kcpc_ctx_t *ctx, int isexec); 827c478bd9Sstevel@tonic-gate static int kcpc_configure_reqs(kcpc_ctx_t *ctx, kcpc_set_t *set, int *subcode); 837c478bd9Sstevel@tonic-gate static void kcpc_free_configs(kcpc_set_t *set); 847c478bd9Sstevel@tonic-gate static kcpc_ctx_t *kcpc_ctx_alloc(void); 857c478bd9Sstevel@tonic-gate static void kcpc_ctx_clone(kcpc_ctx_t *ctx, kcpc_ctx_t *cctx); 867c478bd9Sstevel@tonic-gate static void kcpc_ctx_free(kcpc_ctx_t *ctx); 877c478bd9Sstevel@tonic-gate static int kcpc_assign_reqs(kcpc_set_t *set, kcpc_ctx_t *ctx); 887c478bd9Sstevel@tonic-gate static int kcpc_tryassign(kcpc_set_t *set, int starting_req, int *scratch); 897c478bd9Sstevel@tonic-gate static kcpc_set_t *kcpc_dup_set(kcpc_set_t *set); 907c478bd9Sstevel@tonic-gate 917c478bd9Sstevel@tonic-gate void 927c478bd9Sstevel@tonic-gate kcpc_register_pcbe(pcbe_ops_t *ops) 937c478bd9Sstevel@tonic-gate { 947c478bd9Sstevel@tonic-gate pcbe_ops = ops; 957c478bd9Sstevel@tonic-gate cpc_ncounters = pcbe_ops->pcbe_ncounters(); 967c478bd9Sstevel@tonic-gate } 977c478bd9Sstevel@tonic-gate 987c478bd9Sstevel@tonic-gate int 997c478bd9Sstevel@tonic-gate kcpc_bind_cpu(kcpc_set_t *set, processorid_t cpuid, int *subcode) 1007c478bd9Sstevel@tonic-gate { 1017c478bd9Sstevel@tonic-gate cpu_t *cp; 1027c478bd9Sstevel@tonic-gate kcpc_ctx_t *ctx; 1037c478bd9Sstevel@tonic-gate int error; 1047c478bd9Sstevel@tonic-gate 1057c478bd9Sstevel@tonic-gate ctx = kcpc_ctx_alloc(); 1067c478bd9Sstevel@tonic-gate 1077c478bd9Sstevel@tonic-gate if (kcpc_assign_reqs(set, ctx) != 0) { 1087c478bd9Sstevel@tonic-gate kcpc_ctx_free(ctx); 1097c478bd9Sstevel@tonic-gate *subcode = CPC_RESOURCE_UNAVAIL; 1107c478bd9Sstevel@tonic-gate return (EINVAL); 1117c478bd9Sstevel@tonic-gate } 1127c478bd9Sstevel@tonic-gate 1137c478bd9Sstevel@tonic-gate ctx->kc_cpuid = cpuid; 1147c478bd9Sstevel@tonic-gate ctx->kc_thread = curthread; 1157c478bd9Sstevel@tonic-gate 1167c478bd9Sstevel@tonic-gate set->ks_data = kmem_zalloc(set->ks_nreqs * sizeof (uint64_t), KM_SLEEP); 1177c478bd9Sstevel@tonic-gate 1187c478bd9Sstevel@tonic-gate if ((error = kcpc_configure_reqs(ctx, set, subcode)) != 0) { 1197c478bd9Sstevel@tonic-gate kmem_free(set->ks_data, set->ks_nreqs * sizeof (uint64_t)); 1207c478bd9Sstevel@tonic-gate kcpc_ctx_free(ctx); 1217c478bd9Sstevel@tonic-gate return (error); 1227c478bd9Sstevel@tonic-gate } 1237c478bd9Sstevel@tonic-gate 1247c478bd9Sstevel@tonic-gate set->ks_ctx = ctx; 1257c478bd9Sstevel@tonic-gate ctx->kc_set = set; 1267c478bd9Sstevel@tonic-gate 1277c478bd9Sstevel@tonic-gate /* 1287c478bd9Sstevel@tonic-gate * We must hold cpu_lock to prevent DR, offlining, or unbinding while 1297c478bd9Sstevel@tonic-gate * we are manipulating the cpu_t and programming the hardware, else the 1307c478bd9Sstevel@tonic-gate * the cpu_t could go away while we're looking at it. 1317c478bd9Sstevel@tonic-gate */ 1327c478bd9Sstevel@tonic-gate mutex_enter(&cpu_lock); 1337c478bd9Sstevel@tonic-gate cp = cpu_get(cpuid); 1347c478bd9Sstevel@tonic-gate 1357c478bd9Sstevel@tonic-gate if (cp == NULL) 1367c478bd9Sstevel@tonic-gate /* 1377c478bd9Sstevel@tonic-gate * The CPU could have been DRd out while we were getting set up. 1387c478bd9Sstevel@tonic-gate */ 1397c478bd9Sstevel@tonic-gate goto unbound; 1407c478bd9Sstevel@tonic-gate 1417c478bd9Sstevel@tonic-gate mutex_enter(&cp->cpu_cpc_ctxlock); 1427c478bd9Sstevel@tonic-gate 1437c478bd9Sstevel@tonic-gate if (cp->cpu_cpc_ctx != NULL) { 1447c478bd9Sstevel@tonic-gate /* 1457c478bd9Sstevel@tonic-gate * If this CPU already has a bound set, return an error. 1467c478bd9Sstevel@tonic-gate */ 1477c478bd9Sstevel@tonic-gate mutex_exit(&cp->cpu_cpc_ctxlock); 1487c478bd9Sstevel@tonic-gate goto unbound; 1497c478bd9Sstevel@tonic-gate } 1507c478bd9Sstevel@tonic-gate 1517c478bd9Sstevel@tonic-gate if (curthread->t_bind_cpu != cpuid) { 1527c478bd9Sstevel@tonic-gate mutex_exit(&cp->cpu_cpc_ctxlock); 1537c478bd9Sstevel@tonic-gate goto unbound; 1547c478bd9Sstevel@tonic-gate } 1557c478bd9Sstevel@tonic-gate cp->cpu_cpc_ctx = ctx; 1567c478bd9Sstevel@tonic-gate 1577c478bd9Sstevel@tonic-gate /* 1587c478bd9Sstevel@tonic-gate * Kernel preemption must be disabled while fiddling with the hardware 1597c478bd9Sstevel@tonic-gate * registers to prevent partial updates. 1607c478bd9Sstevel@tonic-gate */ 1617c478bd9Sstevel@tonic-gate kpreempt_disable(); 1627c478bd9Sstevel@tonic-gate ctx->kc_rawtick = KCPC_GET_TICK(); 1637c478bd9Sstevel@tonic-gate pcbe_ops->pcbe_program(ctx); 1647c478bd9Sstevel@tonic-gate kpreempt_enable(); 1657c478bd9Sstevel@tonic-gate 1667c478bd9Sstevel@tonic-gate mutex_exit(&cp->cpu_cpc_ctxlock); 1677c478bd9Sstevel@tonic-gate mutex_exit(&cpu_lock); 1687c478bd9Sstevel@tonic-gate 1697c478bd9Sstevel@tonic-gate return (0); 1707c478bd9Sstevel@tonic-gate 1717c478bd9Sstevel@tonic-gate unbound: 1727c478bd9Sstevel@tonic-gate mutex_exit(&cpu_lock); 1737c478bd9Sstevel@tonic-gate set->ks_ctx = NULL; 1747c478bd9Sstevel@tonic-gate kmem_free(set->ks_data, set->ks_nreqs * sizeof (uint64_t)); 1757c478bd9Sstevel@tonic-gate kcpc_ctx_free(ctx); 1767c478bd9Sstevel@tonic-gate return (EAGAIN); 1777c478bd9Sstevel@tonic-gate } 1787c478bd9Sstevel@tonic-gate 1797c478bd9Sstevel@tonic-gate int 1807c478bd9Sstevel@tonic-gate kcpc_bind_thread(kcpc_set_t *set, kthread_t *t, int *subcode) 1817c478bd9Sstevel@tonic-gate { 1827c478bd9Sstevel@tonic-gate kcpc_ctx_t *ctx; 1837c478bd9Sstevel@tonic-gate int error; 1847c478bd9Sstevel@tonic-gate 1857c478bd9Sstevel@tonic-gate /* 1867c478bd9Sstevel@tonic-gate * Only one set is allowed per context, so ensure there is no 1877c478bd9Sstevel@tonic-gate * existing context. 1887c478bd9Sstevel@tonic-gate */ 1897c478bd9Sstevel@tonic-gate 1907c478bd9Sstevel@tonic-gate if (t->t_cpc_ctx != NULL) 1917c478bd9Sstevel@tonic-gate return (EEXIST); 1927c478bd9Sstevel@tonic-gate 1937c478bd9Sstevel@tonic-gate ctx = kcpc_ctx_alloc(); 1947c478bd9Sstevel@tonic-gate 1957c478bd9Sstevel@tonic-gate /* 1967c478bd9Sstevel@tonic-gate * The context must begin life frozen until it has been properly 1977c478bd9Sstevel@tonic-gate * programmed onto the hardware. This prevents the context ops from 1987c478bd9Sstevel@tonic-gate * worrying about it until we're ready. 1997c478bd9Sstevel@tonic-gate */ 2007c478bd9Sstevel@tonic-gate ctx->kc_flags |= KCPC_CTX_FREEZE; 2017c478bd9Sstevel@tonic-gate ctx->kc_hrtime = gethrtime(); 2027c478bd9Sstevel@tonic-gate 2037c478bd9Sstevel@tonic-gate if (kcpc_assign_reqs(set, ctx) != 0) { 2047c478bd9Sstevel@tonic-gate kcpc_ctx_free(ctx); 2057c478bd9Sstevel@tonic-gate *subcode = CPC_RESOURCE_UNAVAIL; 2067c478bd9Sstevel@tonic-gate return (EINVAL); 2077c478bd9Sstevel@tonic-gate } 2087c478bd9Sstevel@tonic-gate 2097c478bd9Sstevel@tonic-gate ctx->kc_cpuid = -1; 2107c478bd9Sstevel@tonic-gate if (set->ks_flags & CPC_BIND_LWP_INHERIT) 2117c478bd9Sstevel@tonic-gate ctx->kc_flags |= KCPC_CTX_LWPINHERIT; 2127c478bd9Sstevel@tonic-gate ctx->kc_thread = t; 2137c478bd9Sstevel@tonic-gate t->t_cpc_ctx = ctx; 2147c478bd9Sstevel@tonic-gate /* 2157c478bd9Sstevel@tonic-gate * Permit threads to look at their own hardware counters from userland. 2167c478bd9Sstevel@tonic-gate */ 2177c478bd9Sstevel@tonic-gate ctx->kc_flags |= KCPC_CTX_NONPRIV; 2187c478bd9Sstevel@tonic-gate 2197c478bd9Sstevel@tonic-gate /* 2207c478bd9Sstevel@tonic-gate * Create the data store for this set. 2217c478bd9Sstevel@tonic-gate */ 2227c478bd9Sstevel@tonic-gate set->ks_data = kmem_alloc(set->ks_nreqs * sizeof (uint64_t), KM_SLEEP); 2237c478bd9Sstevel@tonic-gate 2247c478bd9Sstevel@tonic-gate if ((error = kcpc_configure_reqs(ctx, set, subcode)) != 0) { 2257c478bd9Sstevel@tonic-gate kmem_free(set->ks_data, set->ks_nreqs * sizeof (uint64_t)); 2267c478bd9Sstevel@tonic-gate kcpc_ctx_free(ctx); 2277c478bd9Sstevel@tonic-gate t->t_cpc_ctx = NULL; 2287c478bd9Sstevel@tonic-gate return (error); 2297c478bd9Sstevel@tonic-gate } 2307c478bd9Sstevel@tonic-gate 2317c478bd9Sstevel@tonic-gate set->ks_ctx = ctx; 2327c478bd9Sstevel@tonic-gate ctx->kc_set = set; 2337c478bd9Sstevel@tonic-gate 2347c478bd9Sstevel@tonic-gate /* 2357c478bd9Sstevel@tonic-gate * Add a device context to the subject thread. 2367c478bd9Sstevel@tonic-gate */ 2377c478bd9Sstevel@tonic-gate installctx(t, ctx, kcpc_save, kcpc_restore, NULL, 2387c478bd9Sstevel@tonic-gate kcpc_lwp_create, NULL, kcpc_free); 2397c478bd9Sstevel@tonic-gate 2407c478bd9Sstevel@tonic-gate /* 2417c478bd9Sstevel@tonic-gate * Ask the backend to program the hardware. 2427c478bd9Sstevel@tonic-gate */ 2437c478bd9Sstevel@tonic-gate if (t == curthread) { 2447c478bd9Sstevel@tonic-gate kpreempt_disable(); 2457c478bd9Sstevel@tonic-gate ctx->kc_rawtick = KCPC_GET_TICK(); 2467c478bd9Sstevel@tonic-gate atomic_and_uint(&ctx->kc_flags, ~KCPC_CTX_FREEZE); 2477c478bd9Sstevel@tonic-gate pcbe_ops->pcbe_program(ctx); 2487c478bd9Sstevel@tonic-gate kpreempt_enable(); 2497c478bd9Sstevel@tonic-gate } else 2507c478bd9Sstevel@tonic-gate /* 2517c478bd9Sstevel@tonic-gate * Since we are the agent LWP, we know the victim LWP is stopped 2527c478bd9Sstevel@tonic-gate * until we're done here; no need to worry about preemption or 2537c478bd9Sstevel@tonic-gate * migration here. We still use an atomic op to clear the flag 2547c478bd9Sstevel@tonic-gate * to ensure the flags are always self-consistent; they can 2557c478bd9Sstevel@tonic-gate * still be accessed from, for instance, another CPU doing a 2567c478bd9Sstevel@tonic-gate * kcpc_invalidate_all(). 2577c478bd9Sstevel@tonic-gate */ 2587c478bd9Sstevel@tonic-gate atomic_and_uint(&ctx->kc_flags, ~KCPC_CTX_FREEZE); 2597c478bd9Sstevel@tonic-gate 2607c478bd9Sstevel@tonic-gate 2617c478bd9Sstevel@tonic-gate return (0); 2627c478bd9Sstevel@tonic-gate } 2637c478bd9Sstevel@tonic-gate 2647c478bd9Sstevel@tonic-gate /* 2657c478bd9Sstevel@tonic-gate * Walk through each request in the set and ask the PCBE to configure a 2667c478bd9Sstevel@tonic-gate * corresponding counter. 2677c478bd9Sstevel@tonic-gate */ 2687c478bd9Sstevel@tonic-gate static int 2697c478bd9Sstevel@tonic-gate kcpc_configure_reqs(kcpc_ctx_t *ctx, kcpc_set_t *set, int *subcode) 2707c478bd9Sstevel@tonic-gate { 2717c478bd9Sstevel@tonic-gate int i; 2727c478bd9Sstevel@tonic-gate int ret; 2737c478bd9Sstevel@tonic-gate kcpc_request_t *rp; 2747c478bd9Sstevel@tonic-gate 2757c478bd9Sstevel@tonic-gate for (i = 0; i < set->ks_nreqs; i++) { 2767c478bd9Sstevel@tonic-gate int n; 2777c478bd9Sstevel@tonic-gate rp = &set->ks_req[i]; 2787c478bd9Sstevel@tonic-gate 2797c478bd9Sstevel@tonic-gate n = rp->kr_picnum; 2807c478bd9Sstevel@tonic-gate 2817c478bd9Sstevel@tonic-gate ASSERT(n >= 0 && n < cpc_ncounters); 2827c478bd9Sstevel@tonic-gate 2837c478bd9Sstevel@tonic-gate ASSERT(ctx->kc_pics[n].kp_req == NULL); 2847c478bd9Sstevel@tonic-gate 2857c478bd9Sstevel@tonic-gate if (rp->kr_flags & CPC_OVF_NOTIFY_EMT) { 2867c478bd9Sstevel@tonic-gate if ((pcbe_ops->pcbe_caps & CPC_CAP_OVERFLOW_INTERRUPT) 2877c478bd9Sstevel@tonic-gate == 0) { 2887c478bd9Sstevel@tonic-gate *subcode = -1; 2897c478bd9Sstevel@tonic-gate return (ENOTSUP); 2907c478bd9Sstevel@tonic-gate } 2917c478bd9Sstevel@tonic-gate /* 2927c478bd9Sstevel@tonic-gate * If any of the counters have requested overflow 2937c478bd9Sstevel@tonic-gate * notification, we flag the context as being one that 2947c478bd9Sstevel@tonic-gate * cares about overflow. 2957c478bd9Sstevel@tonic-gate */ 2967c478bd9Sstevel@tonic-gate ctx->kc_flags |= KCPC_CTX_SIGOVF; 2977c478bd9Sstevel@tonic-gate } 2987c478bd9Sstevel@tonic-gate 2997c478bd9Sstevel@tonic-gate rp->kr_config = NULL; 3007c478bd9Sstevel@tonic-gate if ((ret = pcbe_ops->pcbe_configure(n, rp->kr_event, 3017c478bd9Sstevel@tonic-gate rp->kr_preset, rp->kr_flags, rp->kr_nattrs, rp->kr_attr, 3027c478bd9Sstevel@tonic-gate &(rp->kr_config), (void *)ctx)) != 0) { 3037c478bd9Sstevel@tonic-gate kcpc_free_configs(set); 3047c478bd9Sstevel@tonic-gate *subcode = ret; 3058d4e547dSae112802 switch (ret) { 3068d4e547dSae112802 case CPC_ATTR_REQUIRES_PRIVILEGE: 3078d4e547dSae112802 case CPC_HV_NO_ACCESS: 3087c478bd9Sstevel@tonic-gate return (EACCES); 3098d4e547dSae112802 default: 3107c478bd9Sstevel@tonic-gate return (EINVAL); 3117c478bd9Sstevel@tonic-gate } 3128d4e547dSae112802 } 3137c478bd9Sstevel@tonic-gate 3147c478bd9Sstevel@tonic-gate ctx->kc_pics[n].kp_req = rp; 3157c478bd9Sstevel@tonic-gate rp->kr_picp = &ctx->kc_pics[n]; 3167c478bd9Sstevel@tonic-gate rp->kr_data = set->ks_data + rp->kr_index; 3177c478bd9Sstevel@tonic-gate *rp->kr_data = rp->kr_preset; 3187c478bd9Sstevel@tonic-gate } 3197c478bd9Sstevel@tonic-gate 3207c478bd9Sstevel@tonic-gate return (0); 3217c478bd9Sstevel@tonic-gate } 3227c478bd9Sstevel@tonic-gate 3237c478bd9Sstevel@tonic-gate static void 3247c478bd9Sstevel@tonic-gate kcpc_free_configs(kcpc_set_t *set) 3257c478bd9Sstevel@tonic-gate { 3267c478bd9Sstevel@tonic-gate int i; 3277c478bd9Sstevel@tonic-gate 3287c478bd9Sstevel@tonic-gate for (i = 0; i < set->ks_nreqs; i++) 3297c478bd9Sstevel@tonic-gate if (set->ks_req[i].kr_config != NULL) 3307c478bd9Sstevel@tonic-gate pcbe_ops->pcbe_free(set->ks_req[i].kr_config); 3317c478bd9Sstevel@tonic-gate } 3327c478bd9Sstevel@tonic-gate 3337c478bd9Sstevel@tonic-gate /* 3347c478bd9Sstevel@tonic-gate * buf points to a user address and the data should be copied out to that 3357c478bd9Sstevel@tonic-gate * address in the current process. 3367c478bd9Sstevel@tonic-gate */ 3377c478bd9Sstevel@tonic-gate int 3387c478bd9Sstevel@tonic-gate kcpc_sample(kcpc_set_t *set, uint64_t *buf, hrtime_t *hrtime, uint64_t *tick) 3397c478bd9Sstevel@tonic-gate { 3407c478bd9Sstevel@tonic-gate kcpc_ctx_t *ctx = set->ks_ctx; 3417c478bd9Sstevel@tonic-gate uint64_t curtick = KCPC_GET_TICK(); 3427c478bd9Sstevel@tonic-gate 3437c478bd9Sstevel@tonic-gate if (ctx == NULL) 3447c478bd9Sstevel@tonic-gate return (EINVAL); 3457c478bd9Sstevel@tonic-gate else if (ctx->kc_flags & KCPC_CTX_INVALID) 3467c478bd9Sstevel@tonic-gate return (EAGAIN); 3477c478bd9Sstevel@tonic-gate 3487c478bd9Sstevel@tonic-gate if ((ctx->kc_flags & KCPC_CTX_FREEZE) == 0) { 3497c478bd9Sstevel@tonic-gate /* 3507c478bd9Sstevel@tonic-gate * Kernel preemption must be disabled while reading the 3517c478bd9Sstevel@tonic-gate * hardware regs, and if this is a CPU-bound context, while 3527c478bd9Sstevel@tonic-gate * checking the CPU binding of the current thread. 3537c478bd9Sstevel@tonic-gate */ 3547c478bd9Sstevel@tonic-gate kpreempt_disable(); 3557c478bd9Sstevel@tonic-gate 3567c478bd9Sstevel@tonic-gate if (ctx->kc_cpuid != -1) { 3577c478bd9Sstevel@tonic-gate if (curthread->t_bind_cpu != ctx->kc_cpuid) { 3587c478bd9Sstevel@tonic-gate kpreempt_enable(); 3597c478bd9Sstevel@tonic-gate return (EAGAIN); 3607c478bd9Sstevel@tonic-gate } 3617c478bd9Sstevel@tonic-gate } 3627c478bd9Sstevel@tonic-gate 3637c478bd9Sstevel@tonic-gate if (ctx->kc_thread == curthread) { 3647c478bd9Sstevel@tonic-gate ctx->kc_hrtime = gethrtime(); 3657c478bd9Sstevel@tonic-gate pcbe_ops->pcbe_sample(ctx); 3667c478bd9Sstevel@tonic-gate ctx->kc_vtick += curtick - ctx->kc_rawtick; 3677c478bd9Sstevel@tonic-gate ctx->kc_rawtick = curtick; 3687c478bd9Sstevel@tonic-gate } 3697c478bd9Sstevel@tonic-gate 3707c478bd9Sstevel@tonic-gate kpreempt_enable(); 3718d4e547dSae112802 3728d4e547dSae112802 /* 3738d4e547dSae112802 * The config may have been invalidated by 3748d4e547dSae112802 * the pcbe_sample op. 3758d4e547dSae112802 */ 3768d4e547dSae112802 if (ctx->kc_flags & KCPC_CTX_INVALID) 3778d4e547dSae112802 return (EAGAIN); 3787c478bd9Sstevel@tonic-gate } 3797c478bd9Sstevel@tonic-gate 3807c478bd9Sstevel@tonic-gate if (copyout(set->ks_data, buf, 3817c478bd9Sstevel@tonic-gate set->ks_nreqs * sizeof (uint64_t)) == -1) 3827c478bd9Sstevel@tonic-gate return (EFAULT); 3837c478bd9Sstevel@tonic-gate if (copyout(&ctx->kc_hrtime, hrtime, sizeof (uint64_t)) == -1) 3847c478bd9Sstevel@tonic-gate return (EFAULT); 3857c478bd9Sstevel@tonic-gate if (copyout(&ctx->kc_vtick, tick, sizeof (uint64_t)) == -1) 3867c478bd9Sstevel@tonic-gate return (EFAULT); 3877c478bd9Sstevel@tonic-gate 3887c478bd9Sstevel@tonic-gate return (0); 3897c478bd9Sstevel@tonic-gate } 3907c478bd9Sstevel@tonic-gate 3917c478bd9Sstevel@tonic-gate /* 3927c478bd9Sstevel@tonic-gate * Stop the counters on the CPU this context is bound to. 3937c478bd9Sstevel@tonic-gate */ 3947c478bd9Sstevel@tonic-gate static void 3957c478bd9Sstevel@tonic-gate kcpc_stop_hw(kcpc_ctx_t *ctx) 3967c478bd9Sstevel@tonic-gate { 3977c478bd9Sstevel@tonic-gate cpu_t *cp; 3987c478bd9Sstevel@tonic-gate 3997c478bd9Sstevel@tonic-gate ASSERT((ctx->kc_flags & (KCPC_CTX_INVALID | KCPC_CTX_INVALID_STOPPED)) 4007c478bd9Sstevel@tonic-gate == KCPC_CTX_INVALID); 4017c478bd9Sstevel@tonic-gate 4027c478bd9Sstevel@tonic-gate kpreempt_disable(); 4037c478bd9Sstevel@tonic-gate 4047c478bd9Sstevel@tonic-gate cp = cpu_get(ctx->kc_cpuid); 4057c478bd9Sstevel@tonic-gate ASSERT(cp != NULL); 4067c478bd9Sstevel@tonic-gate 4077c478bd9Sstevel@tonic-gate if (cp == CPU) { 4087c478bd9Sstevel@tonic-gate pcbe_ops->pcbe_allstop(); 4097c478bd9Sstevel@tonic-gate atomic_or_uint(&ctx->kc_flags, 4107c478bd9Sstevel@tonic-gate KCPC_CTX_INVALID_STOPPED); 4117c478bd9Sstevel@tonic-gate } else 4127c478bd9Sstevel@tonic-gate kcpc_remote_stop(cp); 4137c478bd9Sstevel@tonic-gate kpreempt_enable(); 4147c478bd9Sstevel@tonic-gate } 4157c478bd9Sstevel@tonic-gate 4167c478bd9Sstevel@tonic-gate int 4177c478bd9Sstevel@tonic-gate kcpc_unbind(kcpc_set_t *set) 4187c478bd9Sstevel@tonic-gate { 4197c478bd9Sstevel@tonic-gate kcpc_ctx_t *ctx = set->ks_ctx; 4207c478bd9Sstevel@tonic-gate kthread_t *t; 4217c478bd9Sstevel@tonic-gate 4227c478bd9Sstevel@tonic-gate if (ctx == NULL) 4237c478bd9Sstevel@tonic-gate return (EINVAL); 4247c478bd9Sstevel@tonic-gate 4257c478bd9Sstevel@tonic-gate atomic_or_uint(&ctx->kc_flags, KCPC_CTX_INVALID); 4267c478bd9Sstevel@tonic-gate 4277c478bd9Sstevel@tonic-gate if (ctx->kc_cpuid == -1) { 4287c478bd9Sstevel@tonic-gate t = ctx->kc_thread; 4297c478bd9Sstevel@tonic-gate /* 4307c478bd9Sstevel@tonic-gate * The context is thread-bound and therefore has a device 4317c478bd9Sstevel@tonic-gate * context. It will be freed via removectx() calling 4327c478bd9Sstevel@tonic-gate * freectx() calling kcpc_free(). 4337c478bd9Sstevel@tonic-gate */ 4347c478bd9Sstevel@tonic-gate if (t == curthread && 4357c478bd9Sstevel@tonic-gate (ctx->kc_flags & KCPC_CTX_INVALID_STOPPED) == 0) { 4367c478bd9Sstevel@tonic-gate kpreempt_disable(); 4377c478bd9Sstevel@tonic-gate pcbe_ops->pcbe_allstop(); 4387c478bd9Sstevel@tonic-gate atomic_or_uint(&ctx->kc_flags, 4397c478bd9Sstevel@tonic-gate KCPC_CTX_INVALID_STOPPED); 4407c478bd9Sstevel@tonic-gate kpreempt_enable(); 4417c478bd9Sstevel@tonic-gate } 4427c478bd9Sstevel@tonic-gate #ifdef DEBUG 4437c478bd9Sstevel@tonic-gate if (removectx(t, ctx, kcpc_save, kcpc_restore, NULL, 4447c478bd9Sstevel@tonic-gate kcpc_lwp_create, NULL, kcpc_free) == 0) 4457c478bd9Sstevel@tonic-gate panic("kcpc_unbind: context %p not preset on thread %p", 4467c478bd9Sstevel@tonic-gate ctx, t); 4477c478bd9Sstevel@tonic-gate #else 4487c478bd9Sstevel@tonic-gate (void) removectx(t, ctx, kcpc_save, kcpc_restore, NULL, 4497c478bd9Sstevel@tonic-gate kcpc_lwp_create, NULL, kcpc_free); 4507c478bd9Sstevel@tonic-gate #endif /* DEBUG */ 4517c478bd9Sstevel@tonic-gate t->t_cpc_set = NULL; 4527c478bd9Sstevel@tonic-gate t->t_cpc_ctx = NULL; 4537c478bd9Sstevel@tonic-gate } else { 4547c478bd9Sstevel@tonic-gate /* 4557c478bd9Sstevel@tonic-gate * If we are unbinding a CPU-bound set from a remote CPU, the 4567c478bd9Sstevel@tonic-gate * native CPU's idle thread could be in the midst of programming 4577c478bd9Sstevel@tonic-gate * this context onto the CPU. We grab the context's lock here to 4587c478bd9Sstevel@tonic-gate * ensure that the idle thread is done with it. When we release 4597c478bd9Sstevel@tonic-gate * the lock, the CPU no longer has a context and the idle thread 4607c478bd9Sstevel@tonic-gate * will move on. 4617c478bd9Sstevel@tonic-gate * 4627c478bd9Sstevel@tonic-gate * cpu_lock must be held to prevent the CPU from being DR'd out 4637c478bd9Sstevel@tonic-gate * while we disassociate the context from the cpu_t. 4647c478bd9Sstevel@tonic-gate */ 4657c478bd9Sstevel@tonic-gate cpu_t *cp; 4667c478bd9Sstevel@tonic-gate mutex_enter(&cpu_lock); 4677c478bd9Sstevel@tonic-gate cp = cpu_get(ctx->kc_cpuid); 4687c478bd9Sstevel@tonic-gate if (cp != NULL) { 4697c478bd9Sstevel@tonic-gate /* 4707c478bd9Sstevel@tonic-gate * The CPU may have been DR'd out of the system. 4717c478bd9Sstevel@tonic-gate */ 4727c478bd9Sstevel@tonic-gate mutex_enter(&cp->cpu_cpc_ctxlock); 4737c478bd9Sstevel@tonic-gate if ((ctx->kc_flags & KCPC_CTX_INVALID_STOPPED) == 0) 4747c478bd9Sstevel@tonic-gate kcpc_stop_hw(ctx); 4757c478bd9Sstevel@tonic-gate ASSERT(ctx->kc_flags & KCPC_CTX_INVALID_STOPPED); 4767c478bd9Sstevel@tonic-gate cp->cpu_cpc_ctx = NULL; 4777c478bd9Sstevel@tonic-gate mutex_exit(&cp->cpu_cpc_ctxlock); 4787c478bd9Sstevel@tonic-gate } 4797c478bd9Sstevel@tonic-gate mutex_exit(&cpu_lock); 4807c478bd9Sstevel@tonic-gate if (ctx->kc_thread == curthread) { 4817c478bd9Sstevel@tonic-gate kcpc_free(ctx, 0); 4827c478bd9Sstevel@tonic-gate curthread->t_cpc_set = NULL; 4837c478bd9Sstevel@tonic-gate } 4847c478bd9Sstevel@tonic-gate } 4857c478bd9Sstevel@tonic-gate 4867c478bd9Sstevel@tonic-gate return (0); 4877c478bd9Sstevel@tonic-gate } 4887c478bd9Sstevel@tonic-gate 4897c478bd9Sstevel@tonic-gate int 4907c478bd9Sstevel@tonic-gate kcpc_preset(kcpc_set_t *set, int index, uint64_t preset) 4917c478bd9Sstevel@tonic-gate { 4927c478bd9Sstevel@tonic-gate int i; 4937c478bd9Sstevel@tonic-gate 4947c478bd9Sstevel@tonic-gate ASSERT(set != NULL); 4957c478bd9Sstevel@tonic-gate ASSERT(set->ks_ctx != NULL); 4967c478bd9Sstevel@tonic-gate ASSERT(set->ks_ctx->kc_thread == curthread); 4977c478bd9Sstevel@tonic-gate ASSERT(set->ks_ctx->kc_cpuid == -1); 4987c478bd9Sstevel@tonic-gate 4997c478bd9Sstevel@tonic-gate if (index < 0 || index >= set->ks_nreqs) 5007c478bd9Sstevel@tonic-gate return (EINVAL); 5017c478bd9Sstevel@tonic-gate 5027c478bd9Sstevel@tonic-gate for (i = 0; i < set->ks_nreqs; i++) 5037c478bd9Sstevel@tonic-gate if (set->ks_req[i].kr_index == index) 5047c478bd9Sstevel@tonic-gate break; 5057c478bd9Sstevel@tonic-gate ASSERT(i != set->ks_nreqs); 5067c478bd9Sstevel@tonic-gate 5077c478bd9Sstevel@tonic-gate set->ks_req[i].kr_preset = preset; 5087c478bd9Sstevel@tonic-gate return (0); 5097c478bd9Sstevel@tonic-gate } 5107c478bd9Sstevel@tonic-gate 5117c478bd9Sstevel@tonic-gate int 5127c478bd9Sstevel@tonic-gate kcpc_restart(kcpc_set_t *set) 5137c478bd9Sstevel@tonic-gate { 5147c478bd9Sstevel@tonic-gate kcpc_ctx_t *ctx = set->ks_ctx; 5157c478bd9Sstevel@tonic-gate int i; 5167c478bd9Sstevel@tonic-gate 5177c478bd9Sstevel@tonic-gate ASSERT(ctx != NULL); 5187c478bd9Sstevel@tonic-gate ASSERT(ctx->kc_thread == curthread); 5197c478bd9Sstevel@tonic-gate ASSERT(ctx->kc_cpuid == -1); 5207c478bd9Sstevel@tonic-gate 5217c478bd9Sstevel@tonic-gate kpreempt_disable(); 5227c478bd9Sstevel@tonic-gate 5237c478bd9Sstevel@tonic-gate /* 5247c478bd9Sstevel@tonic-gate * If the user is doing this on a running set, make sure the counters 5257c478bd9Sstevel@tonic-gate * are stopped first. 5267c478bd9Sstevel@tonic-gate */ 5277c478bd9Sstevel@tonic-gate if ((ctx->kc_flags & KCPC_CTX_FREEZE) == 0) 5287c478bd9Sstevel@tonic-gate pcbe_ops->pcbe_allstop(); 5297c478bd9Sstevel@tonic-gate 5307c478bd9Sstevel@tonic-gate for (i = 0; i < set->ks_nreqs; i++) { 5317c478bd9Sstevel@tonic-gate *(set->ks_req[i].kr_data) = set->ks_req[i].kr_preset; 5327c478bd9Sstevel@tonic-gate pcbe_ops->pcbe_configure(0, NULL, set->ks_req[i].kr_preset, 5337c478bd9Sstevel@tonic-gate 0, 0, NULL, &set->ks_req[i].kr_config, NULL); 5347c478bd9Sstevel@tonic-gate } 5357c478bd9Sstevel@tonic-gate 5367c478bd9Sstevel@tonic-gate /* 5377c478bd9Sstevel@tonic-gate * Ask the backend to program the hardware. 5387c478bd9Sstevel@tonic-gate */ 5397c478bd9Sstevel@tonic-gate ctx->kc_rawtick = KCPC_GET_TICK(); 5407c478bd9Sstevel@tonic-gate atomic_and_uint(&ctx->kc_flags, ~KCPC_CTX_FREEZE); 5417c478bd9Sstevel@tonic-gate pcbe_ops->pcbe_program(ctx); 5427c478bd9Sstevel@tonic-gate kpreempt_enable(); 5437c478bd9Sstevel@tonic-gate 5447c478bd9Sstevel@tonic-gate return (0); 5457c478bd9Sstevel@tonic-gate } 5467c478bd9Sstevel@tonic-gate 5477c478bd9Sstevel@tonic-gate /* 5487c478bd9Sstevel@tonic-gate * Caller must hold kcpc_cpuctx_lock. 5497c478bd9Sstevel@tonic-gate */ 5507c478bd9Sstevel@tonic-gate int 5517c478bd9Sstevel@tonic-gate kcpc_enable(kthread_t *t, int cmd, int enable) 5527c478bd9Sstevel@tonic-gate { 5537c478bd9Sstevel@tonic-gate kcpc_ctx_t *ctx = t->t_cpc_ctx; 5547c478bd9Sstevel@tonic-gate kcpc_set_t *set = t->t_cpc_set; 5557c478bd9Sstevel@tonic-gate kcpc_set_t *newset; 5567c478bd9Sstevel@tonic-gate int i; 5577c478bd9Sstevel@tonic-gate int flag; 5587c478bd9Sstevel@tonic-gate int err; 5597c478bd9Sstevel@tonic-gate 5607c478bd9Sstevel@tonic-gate ASSERT(RW_READ_HELD(&kcpc_cpuctx_lock)); 5617c478bd9Sstevel@tonic-gate 5627c478bd9Sstevel@tonic-gate if (ctx == NULL) { 5637c478bd9Sstevel@tonic-gate /* 5647c478bd9Sstevel@tonic-gate * This thread has a set but no context; it must be a 5657c478bd9Sstevel@tonic-gate * CPU-bound set. 5667c478bd9Sstevel@tonic-gate */ 5677c478bd9Sstevel@tonic-gate ASSERT(t->t_cpc_set != NULL); 5687c478bd9Sstevel@tonic-gate ASSERT(t->t_cpc_set->ks_ctx->kc_cpuid != -1); 5697c478bd9Sstevel@tonic-gate return (EINVAL); 5707c478bd9Sstevel@tonic-gate } else if (ctx->kc_flags & KCPC_CTX_INVALID) 5717c478bd9Sstevel@tonic-gate return (EAGAIN); 5727c478bd9Sstevel@tonic-gate 5737c478bd9Sstevel@tonic-gate if (cmd == CPC_ENABLE) { 5747c478bd9Sstevel@tonic-gate if ((ctx->kc_flags & KCPC_CTX_FREEZE) == 0) 5757c478bd9Sstevel@tonic-gate return (EINVAL); 5767c478bd9Sstevel@tonic-gate kpreempt_disable(); 5777c478bd9Sstevel@tonic-gate atomic_and_uint(&ctx->kc_flags, ~KCPC_CTX_FREEZE); 5787c478bd9Sstevel@tonic-gate kcpc_restore(ctx); 5797c478bd9Sstevel@tonic-gate kpreempt_enable(); 5807c478bd9Sstevel@tonic-gate } else if (cmd == CPC_DISABLE) { 5817c478bd9Sstevel@tonic-gate if (ctx->kc_flags & KCPC_CTX_FREEZE) 5827c478bd9Sstevel@tonic-gate return (EINVAL); 5837c478bd9Sstevel@tonic-gate kpreempt_disable(); 5847c478bd9Sstevel@tonic-gate kcpc_save(ctx); 5857c478bd9Sstevel@tonic-gate atomic_or_uint(&ctx->kc_flags, KCPC_CTX_FREEZE); 5867c478bd9Sstevel@tonic-gate kpreempt_enable(); 5877c478bd9Sstevel@tonic-gate } else if (cmd == CPC_USR_EVENTS || cmd == CPC_SYS_EVENTS) { 5887c478bd9Sstevel@tonic-gate /* 5897c478bd9Sstevel@tonic-gate * Strategy for usr/sys: stop counters and update set's presets 5907c478bd9Sstevel@tonic-gate * with current counter values, unbind, update requests with 5917c478bd9Sstevel@tonic-gate * new config, then re-bind. 5927c478bd9Sstevel@tonic-gate */ 5937c478bd9Sstevel@tonic-gate flag = (cmd == CPC_USR_EVENTS) ? 5947c478bd9Sstevel@tonic-gate CPC_COUNT_USER: CPC_COUNT_SYSTEM; 5957c478bd9Sstevel@tonic-gate 5967c478bd9Sstevel@tonic-gate kpreempt_disable(); 5977c478bd9Sstevel@tonic-gate atomic_or_uint(&ctx->kc_flags, 5987c478bd9Sstevel@tonic-gate KCPC_CTX_INVALID | KCPC_CTX_INVALID_STOPPED); 5997c478bd9Sstevel@tonic-gate pcbe_ops->pcbe_allstop(); 6007c478bd9Sstevel@tonic-gate kpreempt_enable(); 6017c478bd9Sstevel@tonic-gate for (i = 0; i < set->ks_nreqs; i++) { 6027c478bd9Sstevel@tonic-gate set->ks_req[i].kr_preset = *(set->ks_req[i].kr_data); 6037c478bd9Sstevel@tonic-gate if (enable) 6047c478bd9Sstevel@tonic-gate set->ks_req[i].kr_flags |= flag; 6057c478bd9Sstevel@tonic-gate else 6067c478bd9Sstevel@tonic-gate set->ks_req[i].kr_flags &= ~flag; 6077c478bd9Sstevel@tonic-gate } 6087c478bd9Sstevel@tonic-gate newset = kcpc_dup_set(set); 6097c478bd9Sstevel@tonic-gate if (kcpc_unbind(set) != 0) 6107c478bd9Sstevel@tonic-gate return (EINVAL); 6117c478bd9Sstevel@tonic-gate t->t_cpc_set = newset; 6127c478bd9Sstevel@tonic-gate if (kcpc_bind_thread(newset, t, &err) != 0) { 6137c478bd9Sstevel@tonic-gate t->t_cpc_set = NULL; 6147c478bd9Sstevel@tonic-gate kcpc_free_set(newset); 6157c478bd9Sstevel@tonic-gate return (EINVAL); 6167c478bd9Sstevel@tonic-gate } 6177c478bd9Sstevel@tonic-gate } else 6187c478bd9Sstevel@tonic-gate return (EINVAL); 6197c478bd9Sstevel@tonic-gate 6207c478bd9Sstevel@tonic-gate return (0); 6217c478bd9Sstevel@tonic-gate } 6227c478bd9Sstevel@tonic-gate 6237c478bd9Sstevel@tonic-gate /* 6247c478bd9Sstevel@tonic-gate * Provide PCBEs with a way of obtaining the configs of every counter which will 6257c478bd9Sstevel@tonic-gate * be programmed together. 6267c478bd9Sstevel@tonic-gate * 6277c478bd9Sstevel@tonic-gate * If current is NULL, provide the first config. 6287c478bd9Sstevel@tonic-gate * 6297c478bd9Sstevel@tonic-gate * If data != NULL, caller wants to know where the data store associated with 6307c478bd9Sstevel@tonic-gate * the config we return is located. 6317c478bd9Sstevel@tonic-gate */ 6327c478bd9Sstevel@tonic-gate void * 6337c478bd9Sstevel@tonic-gate kcpc_next_config(void *token, void *current, uint64_t **data) 6347c478bd9Sstevel@tonic-gate { 6357c478bd9Sstevel@tonic-gate int i; 6367c478bd9Sstevel@tonic-gate kcpc_pic_t *pic; 6377c478bd9Sstevel@tonic-gate kcpc_ctx_t *ctx = (kcpc_ctx_t *)token; 6387c478bd9Sstevel@tonic-gate 6397c478bd9Sstevel@tonic-gate if (current == NULL) { 6407c478bd9Sstevel@tonic-gate /* 6417c478bd9Sstevel@tonic-gate * Client would like the first config, which may not be in 6427c478bd9Sstevel@tonic-gate * counter 0; we need to search through the counters for the 6437c478bd9Sstevel@tonic-gate * first config. 6447c478bd9Sstevel@tonic-gate */ 6457c478bd9Sstevel@tonic-gate for (i = 0; i < cpc_ncounters; i++) 6467c478bd9Sstevel@tonic-gate if (ctx->kc_pics[i].kp_req != NULL) 6477c478bd9Sstevel@tonic-gate break; 6487c478bd9Sstevel@tonic-gate /* 6497c478bd9Sstevel@tonic-gate * There are no counters configured for the given context. 6507c478bd9Sstevel@tonic-gate */ 6517c478bd9Sstevel@tonic-gate if (i == cpc_ncounters) 6527c478bd9Sstevel@tonic-gate return (NULL); 6537c478bd9Sstevel@tonic-gate } else { 6547c478bd9Sstevel@tonic-gate /* 6557c478bd9Sstevel@tonic-gate * There surely is a faster way to do this. 6567c478bd9Sstevel@tonic-gate */ 6577c478bd9Sstevel@tonic-gate for (i = 0; i < cpc_ncounters; i++) { 6587c478bd9Sstevel@tonic-gate pic = &ctx->kc_pics[i]; 6597c478bd9Sstevel@tonic-gate 6607c478bd9Sstevel@tonic-gate if (pic->kp_req != NULL && 6617c478bd9Sstevel@tonic-gate current == pic->kp_req->kr_config) 6627c478bd9Sstevel@tonic-gate break; 6637c478bd9Sstevel@tonic-gate } 6647c478bd9Sstevel@tonic-gate 6657c478bd9Sstevel@tonic-gate /* 6667c478bd9Sstevel@tonic-gate * We found the current config at picnum i. Now search for the 6677c478bd9Sstevel@tonic-gate * next configured PIC. 6687c478bd9Sstevel@tonic-gate */ 6697c478bd9Sstevel@tonic-gate for (i++; i < cpc_ncounters; i++) { 6707c478bd9Sstevel@tonic-gate pic = &ctx->kc_pics[i]; 6717c478bd9Sstevel@tonic-gate if (pic->kp_req != NULL) 6727c478bd9Sstevel@tonic-gate break; 6737c478bd9Sstevel@tonic-gate } 6747c478bd9Sstevel@tonic-gate 6757c478bd9Sstevel@tonic-gate if (i == cpc_ncounters) 6767c478bd9Sstevel@tonic-gate return (NULL); 6777c478bd9Sstevel@tonic-gate } 6787c478bd9Sstevel@tonic-gate 6797c478bd9Sstevel@tonic-gate if (data != NULL) { 6807c478bd9Sstevel@tonic-gate *data = ctx->kc_pics[i].kp_req->kr_data; 6817c478bd9Sstevel@tonic-gate } 6827c478bd9Sstevel@tonic-gate 6837c478bd9Sstevel@tonic-gate return (ctx->kc_pics[i].kp_req->kr_config); 6847c478bd9Sstevel@tonic-gate } 6857c478bd9Sstevel@tonic-gate 6867c478bd9Sstevel@tonic-gate 6877c478bd9Sstevel@tonic-gate static kcpc_ctx_t * 6887c478bd9Sstevel@tonic-gate kcpc_ctx_alloc(void) 6897c478bd9Sstevel@tonic-gate { 6907c478bd9Sstevel@tonic-gate kcpc_ctx_t *ctx; 6917c478bd9Sstevel@tonic-gate long hash; 6927c478bd9Sstevel@tonic-gate 6937c478bd9Sstevel@tonic-gate ctx = (kcpc_ctx_t *)kmem_alloc(sizeof (kcpc_ctx_t), KM_SLEEP); 6947c478bd9Sstevel@tonic-gate 6957c478bd9Sstevel@tonic-gate hash = CPC_HASH_CTX(ctx); 6967c478bd9Sstevel@tonic-gate mutex_enter(&kcpc_ctx_llock[hash]); 6977c478bd9Sstevel@tonic-gate ctx->kc_next = kcpc_ctx_list[hash]; 6987c478bd9Sstevel@tonic-gate kcpc_ctx_list[hash] = ctx; 6997c478bd9Sstevel@tonic-gate mutex_exit(&kcpc_ctx_llock[hash]); 7007c478bd9Sstevel@tonic-gate 7017c478bd9Sstevel@tonic-gate ctx->kc_pics = (kcpc_pic_t *)kmem_zalloc(sizeof (kcpc_pic_t) * 7027c478bd9Sstevel@tonic-gate cpc_ncounters, KM_SLEEP); 7037c478bd9Sstevel@tonic-gate 7047c478bd9Sstevel@tonic-gate ctx->kc_flags = 0; 7057c478bd9Sstevel@tonic-gate ctx->kc_vtick = 0; 7067c478bd9Sstevel@tonic-gate ctx->kc_rawtick = 0; 7077c478bd9Sstevel@tonic-gate ctx->kc_cpuid = -1; 7087c478bd9Sstevel@tonic-gate 7097c478bd9Sstevel@tonic-gate return (ctx); 7107c478bd9Sstevel@tonic-gate } 7117c478bd9Sstevel@tonic-gate 7127c478bd9Sstevel@tonic-gate /* 7137c478bd9Sstevel@tonic-gate * Copy set from ctx to the child context, cctx, if it has CPC_BIND_LWP_INHERIT 7147c478bd9Sstevel@tonic-gate * in the flags. 7157c478bd9Sstevel@tonic-gate */ 7167c478bd9Sstevel@tonic-gate static void 7177c478bd9Sstevel@tonic-gate kcpc_ctx_clone(kcpc_ctx_t *ctx, kcpc_ctx_t *cctx) 7187c478bd9Sstevel@tonic-gate { 7197c478bd9Sstevel@tonic-gate kcpc_set_t *ks = ctx->kc_set, *cks; 7207c478bd9Sstevel@tonic-gate int i, j; 7217c478bd9Sstevel@tonic-gate int code; 7227c478bd9Sstevel@tonic-gate 7237c478bd9Sstevel@tonic-gate ASSERT(ks != NULL); 7247c478bd9Sstevel@tonic-gate 7257c478bd9Sstevel@tonic-gate if ((ks->ks_flags & CPC_BIND_LWP_INHERIT) == 0) 7267c478bd9Sstevel@tonic-gate return; 7277c478bd9Sstevel@tonic-gate 7287c478bd9Sstevel@tonic-gate cks = kmem_alloc(sizeof (*cks), KM_SLEEP); 7297c478bd9Sstevel@tonic-gate cctx->kc_set = cks; 7307c478bd9Sstevel@tonic-gate cks->ks_flags = ks->ks_flags; 7317c478bd9Sstevel@tonic-gate cks->ks_nreqs = ks->ks_nreqs; 7327c478bd9Sstevel@tonic-gate cks->ks_req = kmem_alloc(cks->ks_nreqs * 7337c478bd9Sstevel@tonic-gate sizeof (kcpc_request_t), KM_SLEEP); 7347c478bd9Sstevel@tonic-gate cks->ks_data = kmem_alloc(cks->ks_nreqs * sizeof (uint64_t), 7357c478bd9Sstevel@tonic-gate KM_SLEEP); 7367c478bd9Sstevel@tonic-gate cks->ks_ctx = cctx; 7377c478bd9Sstevel@tonic-gate 7387c478bd9Sstevel@tonic-gate for (i = 0; i < cks->ks_nreqs; i++) { 7397c478bd9Sstevel@tonic-gate cks->ks_req[i].kr_index = ks->ks_req[i].kr_index; 7407c478bd9Sstevel@tonic-gate cks->ks_req[i].kr_picnum = ks->ks_req[i].kr_picnum; 7417c478bd9Sstevel@tonic-gate (void) strncpy(cks->ks_req[i].kr_event, 7427c478bd9Sstevel@tonic-gate ks->ks_req[i].kr_event, CPC_MAX_EVENT_LEN); 7437c478bd9Sstevel@tonic-gate cks->ks_req[i].kr_preset = ks->ks_req[i].kr_preset; 7447c478bd9Sstevel@tonic-gate cks->ks_req[i].kr_flags = ks->ks_req[i].kr_flags; 7457c478bd9Sstevel@tonic-gate cks->ks_req[i].kr_nattrs = ks->ks_req[i].kr_nattrs; 7467c478bd9Sstevel@tonic-gate if (ks->ks_req[i].kr_nattrs > 0) { 7477c478bd9Sstevel@tonic-gate cks->ks_req[i].kr_attr = 7487c478bd9Sstevel@tonic-gate kmem_alloc(ks->ks_req[i].kr_nattrs * 7497c478bd9Sstevel@tonic-gate sizeof (kcpc_attr_t), KM_SLEEP); 7507c478bd9Sstevel@tonic-gate } 7517c478bd9Sstevel@tonic-gate for (j = 0; j < ks->ks_req[i].kr_nattrs; j++) { 7527c478bd9Sstevel@tonic-gate (void) strncpy(cks->ks_req[i].kr_attr[j].ka_name, 7537c478bd9Sstevel@tonic-gate ks->ks_req[i].kr_attr[j].ka_name, 7547c478bd9Sstevel@tonic-gate CPC_MAX_ATTR_LEN); 7557c478bd9Sstevel@tonic-gate cks->ks_req[i].kr_attr[j].ka_val = 7567c478bd9Sstevel@tonic-gate ks->ks_req[i].kr_attr[j].ka_val; 7577c478bd9Sstevel@tonic-gate } 7587c478bd9Sstevel@tonic-gate } 7597c478bd9Sstevel@tonic-gate if (kcpc_configure_reqs(cctx, cks, &code) != 0) 7608d4e547dSae112802 kcpc_invalidate_config(cctx); 7617c478bd9Sstevel@tonic-gate } 7627c478bd9Sstevel@tonic-gate 7637c478bd9Sstevel@tonic-gate 7647c478bd9Sstevel@tonic-gate static void 7657c478bd9Sstevel@tonic-gate kcpc_ctx_free(kcpc_ctx_t *ctx) 7667c478bd9Sstevel@tonic-gate { 7677c478bd9Sstevel@tonic-gate kcpc_ctx_t **loc; 7687c478bd9Sstevel@tonic-gate long hash = CPC_HASH_CTX(ctx); 7697c478bd9Sstevel@tonic-gate 7707c478bd9Sstevel@tonic-gate mutex_enter(&kcpc_ctx_llock[hash]); 7717c478bd9Sstevel@tonic-gate loc = &kcpc_ctx_list[hash]; 7727c478bd9Sstevel@tonic-gate ASSERT(*loc != NULL); 7737c478bd9Sstevel@tonic-gate while (*loc != ctx) 7747c478bd9Sstevel@tonic-gate loc = &(*loc)->kc_next; 7757c478bd9Sstevel@tonic-gate *loc = ctx->kc_next; 7767c478bd9Sstevel@tonic-gate mutex_exit(&kcpc_ctx_llock[hash]); 7777c478bd9Sstevel@tonic-gate 7787c478bd9Sstevel@tonic-gate kmem_free(ctx->kc_pics, cpc_ncounters * sizeof (kcpc_pic_t)); 7797c478bd9Sstevel@tonic-gate kmem_free(ctx, sizeof (*ctx)); 7807c478bd9Sstevel@tonic-gate } 7817c478bd9Sstevel@tonic-gate 7827c478bd9Sstevel@tonic-gate /* 7837c478bd9Sstevel@tonic-gate * Generic interrupt handler used on hardware that generates 7847c478bd9Sstevel@tonic-gate * overflow interrupts. 7857c478bd9Sstevel@tonic-gate * 7867c478bd9Sstevel@tonic-gate * Note: executed at high-level interrupt context! 7877c478bd9Sstevel@tonic-gate */ 7887c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 7897c478bd9Sstevel@tonic-gate kcpc_ctx_t * 7907c478bd9Sstevel@tonic-gate kcpc_overflow_intr(caddr_t arg, uint64_t bitmap) 7917c478bd9Sstevel@tonic-gate { 7927c478bd9Sstevel@tonic-gate kcpc_ctx_t *ctx; 7937c478bd9Sstevel@tonic-gate kthread_t *t = curthread; 7947c478bd9Sstevel@tonic-gate int i; 7957c478bd9Sstevel@tonic-gate 7967c478bd9Sstevel@tonic-gate /* 7977c478bd9Sstevel@tonic-gate * On both x86 and UltraSPARC, we may deliver the high-level 7987c478bd9Sstevel@tonic-gate * interrupt in kernel mode, just after we've started to run an 7997c478bd9Sstevel@tonic-gate * interrupt thread. (That's because the hardware helpfully 8007c478bd9Sstevel@tonic-gate * delivers the overflow interrupt some random number of cycles 8017c478bd9Sstevel@tonic-gate * after the instruction that caused the overflow by which time 8027c478bd9Sstevel@tonic-gate * we're in some part of the kernel, not necessarily running on 8037c478bd9Sstevel@tonic-gate * the right thread). 8047c478bd9Sstevel@tonic-gate * 8057c478bd9Sstevel@tonic-gate * Check for this case here -- find the pinned thread 8067c478bd9Sstevel@tonic-gate * that was running when the interrupt went off. 8077c478bd9Sstevel@tonic-gate */ 8087c478bd9Sstevel@tonic-gate if (t->t_flag & T_INTR_THREAD) { 8097c478bd9Sstevel@tonic-gate klwp_t *lwp; 8107c478bd9Sstevel@tonic-gate 8117c478bd9Sstevel@tonic-gate atomic_add_32(&kcpc_intrctx_count, 1); 8127c478bd9Sstevel@tonic-gate 8137c478bd9Sstevel@tonic-gate /* 8147c478bd9Sstevel@tonic-gate * Note that t_lwp is always set to point at the underlying 8157c478bd9Sstevel@tonic-gate * thread, thus this will work in the presence of nested 8167c478bd9Sstevel@tonic-gate * interrupts. 8177c478bd9Sstevel@tonic-gate */ 8187c478bd9Sstevel@tonic-gate ctx = NULL; 8197c478bd9Sstevel@tonic-gate if ((lwp = t->t_lwp) != NULL) { 8207c478bd9Sstevel@tonic-gate t = lwptot(lwp); 8217c478bd9Sstevel@tonic-gate ctx = t->t_cpc_ctx; 8227c478bd9Sstevel@tonic-gate } 8237c478bd9Sstevel@tonic-gate } else 8247c478bd9Sstevel@tonic-gate ctx = t->t_cpc_ctx; 8257c478bd9Sstevel@tonic-gate 8267c478bd9Sstevel@tonic-gate if (ctx == NULL) { 8277c478bd9Sstevel@tonic-gate /* 8287c478bd9Sstevel@tonic-gate * This can easily happen if we're using the counters in 8297c478bd9Sstevel@tonic-gate * "shared" mode, for example, and an overflow interrupt 8307c478bd9Sstevel@tonic-gate * occurs while we are running cpustat. In that case, the 8317c478bd9Sstevel@tonic-gate * bound thread that has the context that belongs to this 8327c478bd9Sstevel@tonic-gate * CPU is almost certainly sleeping (if it was running on 8337c478bd9Sstevel@tonic-gate * the CPU we'd have found it above), and the actual 8347c478bd9Sstevel@tonic-gate * interrupted thread has no knowledge of performance counters! 8357c478bd9Sstevel@tonic-gate */ 8367c478bd9Sstevel@tonic-gate ctx = curthread->t_cpu->cpu_cpc_ctx; 8377c478bd9Sstevel@tonic-gate if (ctx != NULL) { 8387c478bd9Sstevel@tonic-gate /* 8397c478bd9Sstevel@tonic-gate * Return the bound context for this CPU to 8407c478bd9Sstevel@tonic-gate * the interrupt handler so that it can synchronously 8417c478bd9Sstevel@tonic-gate * sample the hardware counters and restart them. 8427c478bd9Sstevel@tonic-gate */ 8437c478bd9Sstevel@tonic-gate return (ctx); 8447c478bd9Sstevel@tonic-gate } 8457c478bd9Sstevel@tonic-gate 8467c478bd9Sstevel@tonic-gate /* 8477c478bd9Sstevel@tonic-gate * As long as the overflow interrupt really is delivered early 8487c478bd9Sstevel@tonic-gate * enough after trapping into the kernel to avoid switching 8497c478bd9Sstevel@tonic-gate * threads, we must always be able to find the cpc context, 8507c478bd9Sstevel@tonic-gate * or something went terribly wrong i.e. we ended up 8517c478bd9Sstevel@tonic-gate * running a passivated interrupt thread, a kernel 8527c478bd9Sstevel@tonic-gate * thread or we interrupted idle, all of which are Very Bad. 8537c478bd9Sstevel@tonic-gate */ 8547c478bd9Sstevel@tonic-gate if (kcpc_nullctx_panic) 8557c478bd9Sstevel@tonic-gate panic("null cpc context, thread %p", (void *)t); 8567c478bd9Sstevel@tonic-gate atomic_add_32(&kcpc_nullctx_count, 1); 8577c478bd9Sstevel@tonic-gate } else if ((ctx->kc_flags & KCPC_CTX_INVALID) == 0) { 8587c478bd9Sstevel@tonic-gate /* 8597c478bd9Sstevel@tonic-gate * Schedule an ast to sample the counters, which will 8607c478bd9Sstevel@tonic-gate * propagate any overflow into the virtualized performance 8617c478bd9Sstevel@tonic-gate * counter(s), and may deliver a signal. 8627c478bd9Sstevel@tonic-gate */ 8637c478bd9Sstevel@tonic-gate ttolwp(t)->lwp_pcb.pcb_flags |= CPC_OVERFLOW; 8647c478bd9Sstevel@tonic-gate /* 8657c478bd9Sstevel@tonic-gate * If a counter has overflowed which was counting on behalf of 8667c478bd9Sstevel@tonic-gate * a request which specified CPC_OVF_NOTIFY_EMT, send the 8677c478bd9Sstevel@tonic-gate * process a signal. 8687c478bd9Sstevel@tonic-gate */ 8697c478bd9Sstevel@tonic-gate for (i = 0; i < cpc_ncounters; i++) { 8707c478bd9Sstevel@tonic-gate if (ctx->kc_pics[i].kp_req != NULL && 8717c478bd9Sstevel@tonic-gate bitmap & (1 << i) && 8727c478bd9Sstevel@tonic-gate ctx->kc_pics[i].kp_req->kr_flags & 8737c478bd9Sstevel@tonic-gate CPC_OVF_NOTIFY_EMT) { 8747c478bd9Sstevel@tonic-gate /* 8757c478bd9Sstevel@tonic-gate * A signal has been requested for this PIC, so 8767c478bd9Sstevel@tonic-gate * so freeze the context. The interrupt handler 8777c478bd9Sstevel@tonic-gate * has already stopped the counter hardware. 8787c478bd9Sstevel@tonic-gate */ 8797c478bd9Sstevel@tonic-gate atomic_or_uint(&ctx->kc_flags, KCPC_CTX_FREEZE); 8807c478bd9Sstevel@tonic-gate atomic_or_uint(&ctx->kc_pics[i].kp_flags, 8817c478bd9Sstevel@tonic-gate KCPC_PIC_OVERFLOWED); 8827c478bd9Sstevel@tonic-gate } 8837c478bd9Sstevel@tonic-gate } 8847c478bd9Sstevel@tonic-gate aston(t); 8857c478bd9Sstevel@tonic-gate } 8867c478bd9Sstevel@tonic-gate return (NULL); 8877c478bd9Sstevel@tonic-gate } 8887c478bd9Sstevel@tonic-gate 8897c478bd9Sstevel@tonic-gate /* 8907c478bd9Sstevel@tonic-gate * The current thread context had an overflow interrupt; we're 8917c478bd9Sstevel@tonic-gate * executing here in high-level interrupt context. 8927c478bd9Sstevel@tonic-gate */ 8937c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 8947c478bd9Sstevel@tonic-gate uint_t 8957c478bd9Sstevel@tonic-gate kcpc_hw_overflow_intr(caddr_t arg1, caddr_t arg2) 8967c478bd9Sstevel@tonic-gate { 8977c478bd9Sstevel@tonic-gate kcpc_ctx_t *ctx; 8987c478bd9Sstevel@tonic-gate uint64_t bitmap; 8997c478bd9Sstevel@tonic-gate 9007c478bd9Sstevel@tonic-gate if (pcbe_ops == NULL || 9017c478bd9Sstevel@tonic-gate (bitmap = pcbe_ops->pcbe_overflow_bitmap()) == 0) 9027c478bd9Sstevel@tonic-gate return (DDI_INTR_UNCLAIMED); 903bb4f5042Sha137994 9047c478bd9Sstevel@tonic-gate /* 9057c478bd9Sstevel@tonic-gate * Prevent any further interrupts. 9067c478bd9Sstevel@tonic-gate */ 9077c478bd9Sstevel@tonic-gate pcbe_ops->pcbe_allstop(); 9087c478bd9Sstevel@tonic-gate 9097c478bd9Sstevel@tonic-gate /* 9107c478bd9Sstevel@tonic-gate * Invoke the "generic" handler. 9117c478bd9Sstevel@tonic-gate * 9127c478bd9Sstevel@tonic-gate * If the interrupt has occurred in the context of an lwp owning 9137c478bd9Sstevel@tonic-gate * the counters, then the handler posts an AST to the lwp to 9147c478bd9Sstevel@tonic-gate * trigger the actual sampling, and optionally deliver a signal or 9157c478bd9Sstevel@tonic-gate * restart the counters, on the way out of the kernel using 9167c478bd9Sstevel@tonic-gate * kcpc_hw_overflow_ast() (see below). 9177c478bd9Sstevel@tonic-gate * 9187c478bd9Sstevel@tonic-gate * On the other hand, if the handler returns the context to us 9197c478bd9Sstevel@tonic-gate * directly, then it means that there are no other threads in 9207c478bd9Sstevel@tonic-gate * the middle of updating it, no AST has been posted, and so we 9217c478bd9Sstevel@tonic-gate * should sample the counters here, and restart them with no 9227c478bd9Sstevel@tonic-gate * further fuss. 9237c478bd9Sstevel@tonic-gate */ 9247c478bd9Sstevel@tonic-gate if ((ctx = kcpc_overflow_intr(arg1, bitmap)) != NULL) { 9257c478bd9Sstevel@tonic-gate uint64_t curtick = KCPC_GET_TICK(); 9267c478bd9Sstevel@tonic-gate 9277c478bd9Sstevel@tonic-gate ctx->kc_hrtime = gethrtime_waitfree(); 9287c478bd9Sstevel@tonic-gate ctx->kc_vtick += curtick - ctx->kc_rawtick; 9297c478bd9Sstevel@tonic-gate ctx->kc_rawtick = curtick; 9307c478bd9Sstevel@tonic-gate pcbe_ops->pcbe_sample(ctx); 9317c478bd9Sstevel@tonic-gate pcbe_ops->pcbe_program(ctx); 9327c478bd9Sstevel@tonic-gate } 9337c478bd9Sstevel@tonic-gate 9347c478bd9Sstevel@tonic-gate return (DDI_INTR_CLAIMED); 9357c478bd9Sstevel@tonic-gate } 9367c478bd9Sstevel@tonic-gate 9377c478bd9Sstevel@tonic-gate /* 9387c478bd9Sstevel@tonic-gate * Called from trap() when processing the ast posted by the high-level 9397c478bd9Sstevel@tonic-gate * interrupt handler. 9407c478bd9Sstevel@tonic-gate */ 9417c478bd9Sstevel@tonic-gate int 9427c478bd9Sstevel@tonic-gate kcpc_overflow_ast() 9437c478bd9Sstevel@tonic-gate { 9447c478bd9Sstevel@tonic-gate kcpc_ctx_t *ctx = curthread->t_cpc_ctx; 9457c478bd9Sstevel@tonic-gate int i; 9467c478bd9Sstevel@tonic-gate int found = 0; 9477c478bd9Sstevel@tonic-gate uint64_t curtick = KCPC_GET_TICK(); 9487c478bd9Sstevel@tonic-gate 9497c478bd9Sstevel@tonic-gate ASSERT(ctx != NULL); /* Beware of interrupt skid. */ 9507c478bd9Sstevel@tonic-gate 9517c478bd9Sstevel@tonic-gate /* 9527c478bd9Sstevel@tonic-gate * An overflow happened: sample the context to ensure that 9537c478bd9Sstevel@tonic-gate * the overflow is propagated into the upper bits of the 9547c478bd9Sstevel@tonic-gate * virtualized 64-bit counter(s). 9557c478bd9Sstevel@tonic-gate */ 9567c478bd9Sstevel@tonic-gate kpreempt_disable(); 9577c478bd9Sstevel@tonic-gate ctx->kc_hrtime = gethrtime_waitfree(); 9587c478bd9Sstevel@tonic-gate pcbe_ops->pcbe_sample(ctx); 9597c478bd9Sstevel@tonic-gate kpreempt_enable(); 9607c478bd9Sstevel@tonic-gate 9617c478bd9Sstevel@tonic-gate ctx->kc_vtick += curtick - ctx->kc_rawtick; 9627c478bd9Sstevel@tonic-gate 9637c478bd9Sstevel@tonic-gate /* 9647c478bd9Sstevel@tonic-gate * The interrupt handler has marked any pics with KCPC_PIC_OVERFLOWED 9657c478bd9Sstevel@tonic-gate * if that pic generated an overflow and if the request it was counting 9667c478bd9Sstevel@tonic-gate * on behalf of had CPC_OVERFLOW_REQUEST specified. We go through all 9677c478bd9Sstevel@tonic-gate * pics in the context and clear the KCPC_PIC_OVERFLOWED flags. If we 9687c478bd9Sstevel@tonic-gate * found any overflowed pics, keep the context frozen and return true 9697c478bd9Sstevel@tonic-gate * (thus causing a signal to be sent). 9707c478bd9Sstevel@tonic-gate */ 9717c478bd9Sstevel@tonic-gate for (i = 0; i < cpc_ncounters; i++) { 9727c478bd9Sstevel@tonic-gate if (ctx->kc_pics[i].kp_flags & KCPC_PIC_OVERFLOWED) { 9737c478bd9Sstevel@tonic-gate atomic_and_uint(&ctx->kc_pics[i].kp_flags, 9747c478bd9Sstevel@tonic-gate ~KCPC_PIC_OVERFLOWED); 9757c478bd9Sstevel@tonic-gate found = 1; 9767c478bd9Sstevel@tonic-gate } 9777c478bd9Sstevel@tonic-gate } 9787c478bd9Sstevel@tonic-gate if (found) 9797c478bd9Sstevel@tonic-gate return (1); 9807c478bd9Sstevel@tonic-gate 9817c478bd9Sstevel@tonic-gate /* 9827c478bd9Sstevel@tonic-gate * Otherwise, re-enable the counters and continue life as before. 9837c478bd9Sstevel@tonic-gate */ 9847c478bd9Sstevel@tonic-gate kpreempt_disable(); 9857c478bd9Sstevel@tonic-gate atomic_and_uint(&ctx->kc_flags, ~KCPC_CTX_FREEZE); 9867c478bd9Sstevel@tonic-gate pcbe_ops->pcbe_program(ctx); 9877c478bd9Sstevel@tonic-gate kpreempt_enable(); 9887c478bd9Sstevel@tonic-gate return (0); 9897c478bd9Sstevel@tonic-gate } 9907c478bd9Sstevel@tonic-gate 9917c478bd9Sstevel@tonic-gate /* 9927c478bd9Sstevel@tonic-gate * Called when switching away from current thread. 9937c478bd9Sstevel@tonic-gate */ 9947c478bd9Sstevel@tonic-gate static void 9957c478bd9Sstevel@tonic-gate kcpc_save(kcpc_ctx_t *ctx) 9967c478bd9Sstevel@tonic-gate { 9977c478bd9Sstevel@tonic-gate if (ctx->kc_flags & KCPC_CTX_INVALID) { 9987c478bd9Sstevel@tonic-gate if (ctx->kc_flags & KCPC_CTX_INVALID_STOPPED) 9997c478bd9Sstevel@tonic-gate return; 10007c478bd9Sstevel@tonic-gate /* 10017c478bd9Sstevel@tonic-gate * This context has been invalidated but the counters have not 10027c478bd9Sstevel@tonic-gate * been stopped. Stop them here and mark the context stopped. 10037c478bd9Sstevel@tonic-gate */ 10047c478bd9Sstevel@tonic-gate pcbe_ops->pcbe_allstop(); 10057c478bd9Sstevel@tonic-gate atomic_or_uint(&ctx->kc_flags, KCPC_CTX_INVALID_STOPPED); 10067c478bd9Sstevel@tonic-gate return; 10077c478bd9Sstevel@tonic-gate } 10087c478bd9Sstevel@tonic-gate 10097c478bd9Sstevel@tonic-gate pcbe_ops->pcbe_allstop(); 10107c478bd9Sstevel@tonic-gate if (ctx->kc_flags & KCPC_CTX_FREEZE) 10117c478bd9Sstevel@tonic-gate return; 10127c478bd9Sstevel@tonic-gate 10137c478bd9Sstevel@tonic-gate /* 10147c478bd9Sstevel@tonic-gate * Need to sample for all reqs into each req's current mpic. 10157c478bd9Sstevel@tonic-gate */ 10167c478bd9Sstevel@tonic-gate ctx->kc_hrtime = gethrtime(); 10177c478bd9Sstevel@tonic-gate ctx->kc_vtick += KCPC_GET_TICK() - ctx->kc_rawtick; 10187c478bd9Sstevel@tonic-gate pcbe_ops->pcbe_sample(ctx); 10197c478bd9Sstevel@tonic-gate } 10207c478bd9Sstevel@tonic-gate 10217c478bd9Sstevel@tonic-gate static void 10227c478bd9Sstevel@tonic-gate kcpc_restore(kcpc_ctx_t *ctx) 10237c478bd9Sstevel@tonic-gate { 10247c478bd9Sstevel@tonic-gate if ((ctx->kc_flags & (KCPC_CTX_INVALID | KCPC_CTX_INVALID_STOPPED)) == 10257c478bd9Sstevel@tonic-gate KCPC_CTX_INVALID) 10267c478bd9Sstevel@tonic-gate /* 10277c478bd9Sstevel@tonic-gate * The context is invalidated but has not been marked stopped. 10287c478bd9Sstevel@tonic-gate * We mark it as such here because we will not start the 10297c478bd9Sstevel@tonic-gate * counters during this context switch. 10307c478bd9Sstevel@tonic-gate */ 10317c478bd9Sstevel@tonic-gate atomic_or_uint(&ctx->kc_flags, KCPC_CTX_INVALID_STOPPED); 10327c478bd9Sstevel@tonic-gate 10337c478bd9Sstevel@tonic-gate 10347c478bd9Sstevel@tonic-gate if (ctx->kc_flags & (KCPC_CTX_INVALID | KCPC_CTX_FREEZE)) 10357c478bd9Sstevel@tonic-gate return; 10367c478bd9Sstevel@tonic-gate 10377c478bd9Sstevel@tonic-gate /* 10387c478bd9Sstevel@tonic-gate * While programming the hardware, the counters should be stopped. We 10397c478bd9Sstevel@tonic-gate * don't do an explicit pcbe_allstop() here because they should have 10407c478bd9Sstevel@tonic-gate * been stopped already by the last consumer. 10417c478bd9Sstevel@tonic-gate */ 10427c478bd9Sstevel@tonic-gate ctx->kc_rawtick = KCPC_GET_TICK(); 10437c478bd9Sstevel@tonic-gate pcbe_ops->pcbe_program(ctx); 10447c478bd9Sstevel@tonic-gate } 10457c478bd9Sstevel@tonic-gate 10467c478bd9Sstevel@tonic-gate /* 10477c478bd9Sstevel@tonic-gate * If kcpc_counts_include_idle is set to 0 by the sys admin, we add the the 10487c478bd9Sstevel@tonic-gate * following context operators to the idle thread on each CPU. They stop the 10497c478bd9Sstevel@tonic-gate * counters when the idle thread is switched on, and they start them again when 10507c478bd9Sstevel@tonic-gate * it is switched off. 10517c478bd9Sstevel@tonic-gate */ 10527c478bd9Sstevel@tonic-gate 10537c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 10547c478bd9Sstevel@tonic-gate void 10557c478bd9Sstevel@tonic-gate kcpc_idle_save(struct cpu *cp) 10567c478bd9Sstevel@tonic-gate { 10577c478bd9Sstevel@tonic-gate /* 10587c478bd9Sstevel@tonic-gate * The idle thread shouldn't be run anywhere else. 10597c478bd9Sstevel@tonic-gate */ 10607c478bd9Sstevel@tonic-gate ASSERT(CPU == cp); 10617c478bd9Sstevel@tonic-gate 10627c478bd9Sstevel@tonic-gate /* 10637c478bd9Sstevel@tonic-gate * We must hold the CPU's context lock to ensure the context isn't freed 10647c478bd9Sstevel@tonic-gate * while we're looking at it. 10657c478bd9Sstevel@tonic-gate */ 10667c478bd9Sstevel@tonic-gate mutex_enter(&cp->cpu_cpc_ctxlock); 10677c478bd9Sstevel@tonic-gate 10687c478bd9Sstevel@tonic-gate if ((cp->cpu_cpc_ctx == NULL) || 10697c478bd9Sstevel@tonic-gate (cp->cpu_cpc_ctx->kc_flags & KCPC_CTX_INVALID)) { 10707c478bd9Sstevel@tonic-gate mutex_exit(&cp->cpu_cpc_ctxlock); 10717c478bd9Sstevel@tonic-gate return; 10727c478bd9Sstevel@tonic-gate } 10737c478bd9Sstevel@tonic-gate 10747c478bd9Sstevel@tonic-gate pcbe_ops->pcbe_program(cp->cpu_cpc_ctx); 10757c478bd9Sstevel@tonic-gate mutex_exit(&cp->cpu_cpc_ctxlock); 10767c478bd9Sstevel@tonic-gate } 10777c478bd9Sstevel@tonic-gate 10787c478bd9Sstevel@tonic-gate void 10797c478bd9Sstevel@tonic-gate kcpc_idle_restore(struct cpu *cp) 10807c478bd9Sstevel@tonic-gate { 10817c478bd9Sstevel@tonic-gate /* 10827c478bd9Sstevel@tonic-gate * The idle thread shouldn't be run anywhere else. 10837c478bd9Sstevel@tonic-gate */ 10847c478bd9Sstevel@tonic-gate ASSERT(CPU == cp); 10857c478bd9Sstevel@tonic-gate 10867c478bd9Sstevel@tonic-gate /* 10877c478bd9Sstevel@tonic-gate * We must hold the CPU's context lock to ensure the context isn't freed 10887c478bd9Sstevel@tonic-gate * while we're looking at it. 10897c478bd9Sstevel@tonic-gate */ 10907c478bd9Sstevel@tonic-gate mutex_enter(&cp->cpu_cpc_ctxlock); 10917c478bd9Sstevel@tonic-gate 10927c478bd9Sstevel@tonic-gate if ((cp->cpu_cpc_ctx == NULL) || 10937c478bd9Sstevel@tonic-gate (cp->cpu_cpc_ctx->kc_flags & KCPC_CTX_INVALID)) { 10947c478bd9Sstevel@tonic-gate mutex_exit(&cp->cpu_cpc_ctxlock); 10957c478bd9Sstevel@tonic-gate return; 10967c478bd9Sstevel@tonic-gate } 10977c478bd9Sstevel@tonic-gate 10987c478bd9Sstevel@tonic-gate pcbe_ops->pcbe_allstop(); 10997c478bd9Sstevel@tonic-gate mutex_exit(&cp->cpu_cpc_ctxlock); 11007c478bd9Sstevel@tonic-gate } 11017c478bd9Sstevel@tonic-gate 11027c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 11037c478bd9Sstevel@tonic-gate static void 11047c478bd9Sstevel@tonic-gate kcpc_lwp_create(kthread_t *t, kthread_t *ct) 11057c478bd9Sstevel@tonic-gate { 11067c478bd9Sstevel@tonic-gate kcpc_ctx_t *ctx = t->t_cpc_ctx, *cctx; 11077c478bd9Sstevel@tonic-gate int i; 11087c478bd9Sstevel@tonic-gate 11097c478bd9Sstevel@tonic-gate if (ctx == NULL || (ctx->kc_flags & KCPC_CTX_LWPINHERIT) == 0) 11107c478bd9Sstevel@tonic-gate return; 11117c478bd9Sstevel@tonic-gate 11127c478bd9Sstevel@tonic-gate rw_enter(&kcpc_cpuctx_lock, RW_READER); 11137c478bd9Sstevel@tonic-gate if (ctx->kc_flags & KCPC_CTX_INVALID) { 11147c478bd9Sstevel@tonic-gate rw_exit(&kcpc_cpuctx_lock); 11157c478bd9Sstevel@tonic-gate return; 11167c478bd9Sstevel@tonic-gate } 11177c478bd9Sstevel@tonic-gate cctx = kcpc_ctx_alloc(); 11187c478bd9Sstevel@tonic-gate kcpc_ctx_clone(ctx, cctx); 11197c478bd9Sstevel@tonic-gate rw_exit(&kcpc_cpuctx_lock); 11207c478bd9Sstevel@tonic-gate 11218d4e547dSae112802 /* 11228d4e547dSae112802 * Copy the parent context's kc_flags field, but don't overwrite 11238d4e547dSae112802 * the child's in case it was modified during kcpc_ctx_clone. 11248d4e547dSae112802 */ 11258d4e547dSae112802 cctx->kc_flags |= ctx->kc_flags; 11267c478bd9Sstevel@tonic-gate cctx->kc_thread = ct; 11277c478bd9Sstevel@tonic-gate cctx->kc_cpuid = -1; 11287c478bd9Sstevel@tonic-gate ct->t_cpc_set = cctx->kc_set; 11297c478bd9Sstevel@tonic-gate ct->t_cpc_ctx = cctx; 11307c478bd9Sstevel@tonic-gate 11317c478bd9Sstevel@tonic-gate if (cctx->kc_flags & KCPC_CTX_SIGOVF) { 11327c478bd9Sstevel@tonic-gate kcpc_set_t *ks = cctx->kc_set; 11337c478bd9Sstevel@tonic-gate /* 11347c478bd9Sstevel@tonic-gate * Our contract with the user requires us to immediately send an 11357c478bd9Sstevel@tonic-gate * overflow signal to all children if we have the LWPINHERIT 11367c478bd9Sstevel@tonic-gate * and SIGOVF flags set. In addition, all counters should be 11377c478bd9Sstevel@tonic-gate * set to UINT64_MAX, and their pic's overflow flag turned on 11387c478bd9Sstevel@tonic-gate * so that our trap() processing knows to send a signal. 11397c478bd9Sstevel@tonic-gate */ 11407c478bd9Sstevel@tonic-gate atomic_or_uint(&cctx->kc_flags, KCPC_CTX_FREEZE); 11417c478bd9Sstevel@tonic-gate for (i = 0; i < ks->ks_nreqs; i++) { 11427c478bd9Sstevel@tonic-gate kcpc_request_t *kr = &ks->ks_req[i]; 11437c478bd9Sstevel@tonic-gate 11447c478bd9Sstevel@tonic-gate if (kr->kr_flags & CPC_OVF_NOTIFY_EMT) { 11457c478bd9Sstevel@tonic-gate *(kr->kr_data) = UINT64_MAX; 11467c478bd9Sstevel@tonic-gate kr->kr_picp->kp_flags |= KCPC_PIC_OVERFLOWED; 11477c478bd9Sstevel@tonic-gate } 11487c478bd9Sstevel@tonic-gate } 11497c478bd9Sstevel@tonic-gate ttolwp(ct)->lwp_pcb.pcb_flags |= CPC_OVERFLOW; 11507c478bd9Sstevel@tonic-gate aston(ct); 11517c478bd9Sstevel@tonic-gate } 11527c478bd9Sstevel@tonic-gate 11537c478bd9Sstevel@tonic-gate installctx(ct, cctx, kcpc_save, kcpc_restore, 11547c478bd9Sstevel@tonic-gate NULL, kcpc_lwp_create, NULL, kcpc_free); 11557c478bd9Sstevel@tonic-gate } 11567c478bd9Sstevel@tonic-gate 11577c478bd9Sstevel@tonic-gate /* 11587c478bd9Sstevel@tonic-gate * Counter Stoppage Theory 11597c478bd9Sstevel@tonic-gate * 11607c478bd9Sstevel@tonic-gate * The counters may need to be stopped properly at the following occasions: 11617c478bd9Sstevel@tonic-gate * 11627c478bd9Sstevel@tonic-gate * 1) An LWP exits. 11637c478bd9Sstevel@tonic-gate * 2) A thread exits. 11647c478bd9Sstevel@tonic-gate * 3) An LWP performs an exec(). 11657c478bd9Sstevel@tonic-gate * 4) A bound set is unbound. 11667c478bd9Sstevel@tonic-gate * 11677c478bd9Sstevel@tonic-gate * In addition to stopping the counters, the CPC context (a kcpc_ctx_t) may need 11687c478bd9Sstevel@tonic-gate * to be freed as well. 11697c478bd9Sstevel@tonic-gate * 11707c478bd9Sstevel@tonic-gate * Case 1: kcpc_passivate(), called via lwp_exit(), stops the counters. Later on 11717c478bd9Sstevel@tonic-gate * when the thread is freed, kcpc_free(), called by freectx(), frees the 11727c478bd9Sstevel@tonic-gate * context. 11737c478bd9Sstevel@tonic-gate * 11747c478bd9Sstevel@tonic-gate * Case 2: same as case 1 except kcpc_passivate is called from thread_exit(). 11757c478bd9Sstevel@tonic-gate * 11767c478bd9Sstevel@tonic-gate * Case 3: kcpc_free(), called via freectx() via exec(), recognizes that it has 11777c478bd9Sstevel@tonic-gate * been called from exec. It stops the counters _and_ frees the context. 11787c478bd9Sstevel@tonic-gate * 11797c478bd9Sstevel@tonic-gate * Case 4: kcpc_unbind() stops the hardware _and_ frees the context. 11807c478bd9Sstevel@tonic-gate * 11817c478bd9Sstevel@tonic-gate * CPU-bound counters are always stopped via kcpc_unbind(). 11827c478bd9Sstevel@tonic-gate */ 11837c478bd9Sstevel@tonic-gate 11847c478bd9Sstevel@tonic-gate /* 11857c478bd9Sstevel@tonic-gate * We're being called to delete the context; we ensure that all associated data 11867c478bd9Sstevel@tonic-gate * structures are freed, and that the hardware is passivated if this is an exec. 11877c478bd9Sstevel@tonic-gate */ 11887c478bd9Sstevel@tonic-gate 11897c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 11907c478bd9Sstevel@tonic-gate static void 11917c478bd9Sstevel@tonic-gate kcpc_free(kcpc_ctx_t *ctx, int isexec) 11927c478bd9Sstevel@tonic-gate { 11937c478bd9Sstevel@tonic-gate int i; 11947c478bd9Sstevel@tonic-gate kcpc_set_t *set = ctx->kc_set; 11957c478bd9Sstevel@tonic-gate 11967c478bd9Sstevel@tonic-gate ASSERT(set != NULL); 11977c478bd9Sstevel@tonic-gate 11987c478bd9Sstevel@tonic-gate atomic_or_uint(&ctx->kc_flags, KCPC_CTX_INVALID); 11997c478bd9Sstevel@tonic-gate 12007c478bd9Sstevel@tonic-gate if (isexec) { 12017c478bd9Sstevel@tonic-gate /* 12027c478bd9Sstevel@tonic-gate * This thread is execing, and after the exec it should not have 12037c478bd9Sstevel@tonic-gate * any performance counter context. Stop the counters properly 12047c478bd9Sstevel@tonic-gate * here so the system isn't surprised by an overflow interrupt 12057c478bd9Sstevel@tonic-gate * later. 12067c478bd9Sstevel@tonic-gate */ 12077c478bd9Sstevel@tonic-gate if (ctx->kc_cpuid != -1) { 12087c478bd9Sstevel@tonic-gate cpu_t *cp; 12097c478bd9Sstevel@tonic-gate /* 12107c478bd9Sstevel@tonic-gate * CPU-bound context; stop the appropriate CPU's ctrs. 12117c478bd9Sstevel@tonic-gate * Hold cpu_lock while examining the CPU to ensure it 12127c478bd9Sstevel@tonic-gate * doesn't go away. 12137c478bd9Sstevel@tonic-gate */ 12147c478bd9Sstevel@tonic-gate mutex_enter(&cpu_lock); 12157c478bd9Sstevel@tonic-gate cp = cpu_get(ctx->kc_cpuid); 12167c478bd9Sstevel@tonic-gate /* 12177c478bd9Sstevel@tonic-gate * The CPU could have been DR'd out, so only stop the 12187c478bd9Sstevel@tonic-gate * CPU and clear its context pointer if the CPU still 12197c478bd9Sstevel@tonic-gate * exists. 12207c478bd9Sstevel@tonic-gate */ 12217c478bd9Sstevel@tonic-gate if (cp != NULL) { 12227c478bd9Sstevel@tonic-gate mutex_enter(&cp->cpu_cpc_ctxlock); 12237c478bd9Sstevel@tonic-gate kcpc_stop_hw(ctx); 12247c478bd9Sstevel@tonic-gate cp->cpu_cpc_ctx = NULL; 12257c478bd9Sstevel@tonic-gate mutex_exit(&cp->cpu_cpc_ctxlock); 12267c478bd9Sstevel@tonic-gate } 12277c478bd9Sstevel@tonic-gate mutex_exit(&cpu_lock); 12287c478bd9Sstevel@tonic-gate ASSERT(curthread->t_cpc_ctx == NULL); 12297c478bd9Sstevel@tonic-gate } else { 12307c478bd9Sstevel@tonic-gate /* 12317c478bd9Sstevel@tonic-gate * Thread-bound context; stop _this_ CPU's counters. 12327c478bd9Sstevel@tonic-gate */ 12337c478bd9Sstevel@tonic-gate kpreempt_disable(); 12347c478bd9Sstevel@tonic-gate pcbe_ops->pcbe_allstop(); 12357c478bd9Sstevel@tonic-gate atomic_or_uint(&ctx->kc_flags, 12367c478bd9Sstevel@tonic-gate KCPC_CTX_INVALID_STOPPED); 12377c478bd9Sstevel@tonic-gate kpreempt_enable(); 12387c478bd9Sstevel@tonic-gate curthread->t_cpc_ctx = NULL; 12397c478bd9Sstevel@tonic-gate } 12407c478bd9Sstevel@tonic-gate 12417c478bd9Sstevel@tonic-gate /* 12427c478bd9Sstevel@tonic-gate * Since we are being called from an exec and we know that 12437c478bd9Sstevel@tonic-gate * exec is not permitted via the agent thread, we should clean 12447c478bd9Sstevel@tonic-gate * up this thread's CPC state completely, and not leave dangling 12457c478bd9Sstevel@tonic-gate * CPC pointers behind. 12467c478bd9Sstevel@tonic-gate */ 12477c478bd9Sstevel@tonic-gate ASSERT(ctx->kc_thread == curthread); 12487c478bd9Sstevel@tonic-gate curthread->t_cpc_set = NULL; 12497c478bd9Sstevel@tonic-gate } 12507c478bd9Sstevel@tonic-gate 12517c478bd9Sstevel@tonic-gate /* 12527c478bd9Sstevel@tonic-gate * Walk through each request in this context's set and free the PCBE's 12537c478bd9Sstevel@tonic-gate * configuration if it exists. 12547c478bd9Sstevel@tonic-gate */ 12557c478bd9Sstevel@tonic-gate for (i = 0; i < set->ks_nreqs; i++) { 12567c478bd9Sstevel@tonic-gate if (set->ks_req[i].kr_config != NULL) 12577c478bd9Sstevel@tonic-gate pcbe_ops->pcbe_free(set->ks_req[i].kr_config); 12587c478bd9Sstevel@tonic-gate } 12597c478bd9Sstevel@tonic-gate 12607c478bd9Sstevel@tonic-gate kmem_free(set->ks_data, set->ks_nreqs * sizeof (uint64_t)); 12617c478bd9Sstevel@tonic-gate kcpc_ctx_free(ctx); 12627c478bd9Sstevel@tonic-gate kcpc_free_set(set); 12637c478bd9Sstevel@tonic-gate } 12647c478bd9Sstevel@tonic-gate 12657c478bd9Sstevel@tonic-gate /* 12667c478bd9Sstevel@tonic-gate * Free the memory associated with a request set. 12677c478bd9Sstevel@tonic-gate */ 12687c478bd9Sstevel@tonic-gate void 12697c478bd9Sstevel@tonic-gate kcpc_free_set(kcpc_set_t *set) 12707c478bd9Sstevel@tonic-gate { 12717c478bd9Sstevel@tonic-gate int i; 12727c478bd9Sstevel@tonic-gate kcpc_request_t *req; 12737c478bd9Sstevel@tonic-gate 12747c478bd9Sstevel@tonic-gate ASSERT(set->ks_req != NULL); 12757c478bd9Sstevel@tonic-gate 12767c478bd9Sstevel@tonic-gate for (i = 0; i < set->ks_nreqs; i++) { 12777c478bd9Sstevel@tonic-gate req = &set->ks_req[i]; 12787c478bd9Sstevel@tonic-gate 12797c478bd9Sstevel@tonic-gate if (req->kr_nattrs != 0) { 12807c478bd9Sstevel@tonic-gate kmem_free(req->kr_attr, 12817c478bd9Sstevel@tonic-gate req->kr_nattrs * sizeof (kcpc_attr_t)); 12827c478bd9Sstevel@tonic-gate } 12837c478bd9Sstevel@tonic-gate } 12847c478bd9Sstevel@tonic-gate 12857c478bd9Sstevel@tonic-gate kmem_free(set->ks_req, sizeof (kcpc_request_t) * set->ks_nreqs); 12867c478bd9Sstevel@tonic-gate kmem_free(set, sizeof (kcpc_set_t)); 12877c478bd9Sstevel@tonic-gate } 12887c478bd9Sstevel@tonic-gate 12897c478bd9Sstevel@tonic-gate /* 12907c478bd9Sstevel@tonic-gate * Grab every existing context and mark it as invalid. 12917c478bd9Sstevel@tonic-gate */ 12927c478bd9Sstevel@tonic-gate void 12937c478bd9Sstevel@tonic-gate kcpc_invalidate_all(void) 12947c478bd9Sstevel@tonic-gate { 12957c478bd9Sstevel@tonic-gate kcpc_ctx_t *ctx; 12967c478bd9Sstevel@tonic-gate long hash; 12977c478bd9Sstevel@tonic-gate 12987c478bd9Sstevel@tonic-gate for (hash = 0; hash < CPC_HASH_BUCKETS; hash++) { 12997c478bd9Sstevel@tonic-gate mutex_enter(&kcpc_ctx_llock[hash]); 13007c478bd9Sstevel@tonic-gate for (ctx = kcpc_ctx_list[hash]; ctx; ctx = ctx->kc_next) 13017c478bd9Sstevel@tonic-gate atomic_or_uint(&ctx->kc_flags, KCPC_CTX_INVALID); 13027c478bd9Sstevel@tonic-gate mutex_exit(&kcpc_ctx_llock[hash]); 13037c478bd9Sstevel@tonic-gate } 13047c478bd9Sstevel@tonic-gate } 13057c478bd9Sstevel@tonic-gate 13067c478bd9Sstevel@tonic-gate /* 13078d4e547dSae112802 * Interface for PCBEs to signal that an existing configuration has suddenly 13088d4e547dSae112802 * become invalid. 13098d4e547dSae112802 */ 13108d4e547dSae112802 void 13118d4e547dSae112802 kcpc_invalidate_config(void *token) 13128d4e547dSae112802 { 13138d4e547dSae112802 kcpc_ctx_t *ctx = token; 13148d4e547dSae112802 13158d4e547dSae112802 ASSERT(ctx != NULL); 13168d4e547dSae112802 13178d4e547dSae112802 atomic_or_uint(&ctx->kc_flags, KCPC_CTX_INVALID); 13188d4e547dSae112802 } 13198d4e547dSae112802 13208d4e547dSae112802 /* 13217c478bd9Sstevel@tonic-gate * Called from lwp_exit() and thread_exit() 13227c478bd9Sstevel@tonic-gate */ 13237c478bd9Sstevel@tonic-gate void 13247c478bd9Sstevel@tonic-gate kcpc_passivate(void) 13257c478bd9Sstevel@tonic-gate { 13267c478bd9Sstevel@tonic-gate kcpc_ctx_t *ctx = curthread->t_cpc_ctx; 13277c478bd9Sstevel@tonic-gate kcpc_set_t *set = curthread->t_cpc_set; 13287c478bd9Sstevel@tonic-gate 13297c478bd9Sstevel@tonic-gate if (set == NULL) 13307c478bd9Sstevel@tonic-gate return; 13317c478bd9Sstevel@tonic-gate 13327c478bd9Sstevel@tonic-gate /* 13337c478bd9Sstevel@tonic-gate * We're cleaning up after this thread; ensure there are no dangling 13347c478bd9Sstevel@tonic-gate * CPC pointers left behind. The context and set will be freed by 13357c478bd9Sstevel@tonic-gate * freectx() in the case of an LWP-bound set, and by kcpc_unbind() in 13367c478bd9Sstevel@tonic-gate * the case of a CPU-bound set. 13377c478bd9Sstevel@tonic-gate */ 13387c478bd9Sstevel@tonic-gate curthread->t_cpc_ctx = NULL; 13397c478bd9Sstevel@tonic-gate 13407c478bd9Sstevel@tonic-gate if (ctx == NULL) { 13417c478bd9Sstevel@tonic-gate /* 13427c478bd9Sstevel@tonic-gate * This thread has a set but no context; it must be a CPU-bound 13437c478bd9Sstevel@tonic-gate * set. The hardware will be stopped via kcpc_unbind() when the 13447c478bd9Sstevel@tonic-gate * process exits and closes its file descriptors with 13457c478bd9Sstevel@tonic-gate * kcpc_close(). Our only job here is to clean up this thread's 13467c478bd9Sstevel@tonic-gate * state; the set will be freed with the unbind(). 13477c478bd9Sstevel@tonic-gate */ 13487c478bd9Sstevel@tonic-gate (void) kcpc_unbind(set); 13497c478bd9Sstevel@tonic-gate /* 13507c478bd9Sstevel@tonic-gate * Unbinding a set belonging to the current thread should clear 13517c478bd9Sstevel@tonic-gate * its set pointer. 13527c478bd9Sstevel@tonic-gate */ 13537c478bd9Sstevel@tonic-gate ASSERT(curthread->t_cpc_set == NULL); 13547c478bd9Sstevel@tonic-gate return; 13557c478bd9Sstevel@tonic-gate } 13567c478bd9Sstevel@tonic-gate 13577c478bd9Sstevel@tonic-gate curthread->t_cpc_set = NULL; 13587c478bd9Sstevel@tonic-gate 13597c478bd9Sstevel@tonic-gate /* 13607c478bd9Sstevel@tonic-gate * This thread/LWP is exiting but context switches will continue to 13617c478bd9Sstevel@tonic-gate * happen for a bit as the exit proceeds. Kernel preemption must be 13627c478bd9Sstevel@tonic-gate * disabled here to prevent a race between checking or setting the 13637c478bd9Sstevel@tonic-gate * INVALID_STOPPED flag here and kcpc_restore() setting the flag during 13647c478bd9Sstevel@tonic-gate * a context switch. 13657c478bd9Sstevel@tonic-gate */ 13667c478bd9Sstevel@tonic-gate 13677c478bd9Sstevel@tonic-gate kpreempt_disable(); 13687c478bd9Sstevel@tonic-gate if ((ctx->kc_flags & KCPC_CTX_INVALID_STOPPED) == 0) { 13697c478bd9Sstevel@tonic-gate pcbe_ops->pcbe_allstop(); 13707c478bd9Sstevel@tonic-gate atomic_or_uint(&ctx->kc_flags, 13717c478bd9Sstevel@tonic-gate KCPC_CTX_INVALID | KCPC_CTX_INVALID_STOPPED); 13727c478bd9Sstevel@tonic-gate } 13737c478bd9Sstevel@tonic-gate kpreempt_enable(); 13747c478bd9Sstevel@tonic-gate } 13757c478bd9Sstevel@tonic-gate 13767c478bd9Sstevel@tonic-gate /* 13777c478bd9Sstevel@tonic-gate * Assign the requests in the given set to the PICs in the context. 13787c478bd9Sstevel@tonic-gate * Returns 0 if successful, -1 on failure. 13797c478bd9Sstevel@tonic-gate */ 13807c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 13817c478bd9Sstevel@tonic-gate static int 13827c478bd9Sstevel@tonic-gate kcpc_assign_reqs(kcpc_set_t *set, kcpc_ctx_t *ctx) 13837c478bd9Sstevel@tonic-gate { 13847c478bd9Sstevel@tonic-gate int i; 13857c478bd9Sstevel@tonic-gate int *picnum_save; 13867c478bd9Sstevel@tonic-gate 13877c478bd9Sstevel@tonic-gate ASSERT(set->ks_nreqs <= cpc_ncounters); 13887c478bd9Sstevel@tonic-gate 13897c478bd9Sstevel@tonic-gate /* 13907c478bd9Sstevel@tonic-gate * Provide kcpc_tryassign() with scratch space to avoid doing an 13917c478bd9Sstevel@tonic-gate * alloc/free with every invocation. 13927c478bd9Sstevel@tonic-gate */ 13937c478bd9Sstevel@tonic-gate picnum_save = kmem_alloc(set->ks_nreqs * sizeof (int), KM_SLEEP); 13947c478bd9Sstevel@tonic-gate /* 13957c478bd9Sstevel@tonic-gate * kcpc_tryassign() blindly walks through each request in the set, 13967c478bd9Sstevel@tonic-gate * seeing if a counter can count its event. If yes, it assigns that 13977c478bd9Sstevel@tonic-gate * counter. However, that counter may have been the only capable counter 13987c478bd9Sstevel@tonic-gate * for _another_ request's event. The solution is to try every possible 13997c478bd9Sstevel@tonic-gate * request first. Note that this does not cover all solutions, as 14007c478bd9Sstevel@tonic-gate * that would require all unique orderings of requests, an n^n operation 14017c478bd9Sstevel@tonic-gate * which would be unacceptable for architectures with many counters. 14027c478bd9Sstevel@tonic-gate */ 14037c478bd9Sstevel@tonic-gate for (i = 0; i < set->ks_nreqs; i++) 14047c478bd9Sstevel@tonic-gate if (kcpc_tryassign(set, i, picnum_save) == 0) 14057c478bd9Sstevel@tonic-gate break; 14067c478bd9Sstevel@tonic-gate 14077c478bd9Sstevel@tonic-gate kmem_free(picnum_save, set->ks_nreqs * sizeof (int)); 14087c478bd9Sstevel@tonic-gate if (i == set->ks_nreqs) 14097c478bd9Sstevel@tonic-gate return (-1); 14107c478bd9Sstevel@tonic-gate return (0); 14117c478bd9Sstevel@tonic-gate } 14127c478bd9Sstevel@tonic-gate 14137c478bd9Sstevel@tonic-gate static int 14147c478bd9Sstevel@tonic-gate kcpc_tryassign(kcpc_set_t *set, int starting_req, int *scratch) 14157c478bd9Sstevel@tonic-gate { 14167c478bd9Sstevel@tonic-gate int i; 14177c478bd9Sstevel@tonic-gate int j; 14187c478bd9Sstevel@tonic-gate uint64_t bitmap = 0, resmap = 0; 14197c478bd9Sstevel@tonic-gate uint64_t ctrmap; 14207c478bd9Sstevel@tonic-gate 14217c478bd9Sstevel@tonic-gate /* 14227c478bd9Sstevel@tonic-gate * We are attempting to assign the reqs to pics, but we may fail. If we 14237c478bd9Sstevel@tonic-gate * fail, we need to restore the state of the requests to what it was 14247c478bd9Sstevel@tonic-gate * when we found it, as some reqs may have been explicitly assigned to 14257c478bd9Sstevel@tonic-gate * a specific PIC beforehand. We do this by snapshotting the assignments 14267c478bd9Sstevel@tonic-gate * now and restoring from it later if we fail. 14277c478bd9Sstevel@tonic-gate * 14287c478bd9Sstevel@tonic-gate * Also we note here which counters have already been claimed by 14297c478bd9Sstevel@tonic-gate * requests with explicit counter assignments. 14307c478bd9Sstevel@tonic-gate */ 14317c478bd9Sstevel@tonic-gate for (i = 0; i < set->ks_nreqs; i++) { 14327c478bd9Sstevel@tonic-gate scratch[i] = set->ks_req[i].kr_picnum; 14337c478bd9Sstevel@tonic-gate if (set->ks_req[i].kr_picnum != -1) 14347c478bd9Sstevel@tonic-gate resmap |= (1 << set->ks_req[i].kr_picnum); 14357c478bd9Sstevel@tonic-gate } 14367c478bd9Sstevel@tonic-gate 14377c478bd9Sstevel@tonic-gate /* 14387c478bd9Sstevel@tonic-gate * Walk through requests assigning them to the first PIC that is 14397c478bd9Sstevel@tonic-gate * capable. 14407c478bd9Sstevel@tonic-gate */ 14417c478bd9Sstevel@tonic-gate i = starting_req; 14427c478bd9Sstevel@tonic-gate do { 14437c478bd9Sstevel@tonic-gate if (set->ks_req[i].kr_picnum != -1) { 14447c478bd9Sstevel@tonic-gate ASSERT((bitmap & (1 << set->ks_req[i].kr_picnum)) == 0); 14457c478bd9Sstevel@tonic-gate bitmap |= (1 << set->ks_req[i].kr_picnum); 14467c478bd9Sstevel@tonic-gate if (++i == set->ks_nreqs) 14477c478bd9Sstevel@tonic-gate i = 0; 14487c478bd9Sstevel@tonic-gate continue; 14497c478bd9Sstevel@tonic-gate } 14507c478bd9Sstevel@tonic-gate 14517c478bd9Sstevel@tonic-gate ctrmap = pcbe_ops->pcbe_event_coverage(set->ks_req[i].kr_event); 14527c478bd9Sstevel@tonic-gate for (j = 0; j < cpc_ncounters; j++) { 14537c478bd9Sstevel@tonic-gate if (ctrmap & (1 << j) && (bitmap & (1 << j)) == 0 && 14547c478bd9Sstevel@tonic-gate (resmap & (1 << j)) == 0) { 14557c478bd9Sstevel@tonic-gate /* 14567c478bd9Sstevel@tonic-gate * We can assign this counter because: 14577c478bd9Sstevel@tonic-gate * 14587c478bd9Sstevel@tonic-gate * 1. It can count the event (ctrmap) 14597c478bd9Sstevel@tonic-gate * 2. It hasn't been assigned yet (bitmap) 14607c478bd9Sstevel@tonic-gate * 3. It wasn't reserved by a request (resmap) 14617c478bd9Sstevel@tonic-gate */ 14627c478bd9Sstevel@tonic-gate bitmap |= (1 << j); 14637c478bd9Sstevel@tonic-gate break; 14647c478bd9Sstevel@tonic-gate } 14657c478bd9Sstevel@tonic-gate } 14667c478bd9Sstevel@tonic-gate if (j == cpc_ncounters) { 14677c478bd9Sstevel@tonic-gate for (i = 0; i < set->ks_nreqs; i++) 14687c478bd9Sstevel@tonic-gate set->ks_req[i].kr_picnum = scratch[i]; 14697c478bd9Sstevel@tonic-gate return (-1); 14707c478bd9Sstevel@tonic-gate } 14717c478bd9Sstevel@tonic-gate set->ks_req[i].kr_picnum = j; 14727c478bd9Sstevel@tonic-gate 14737c478bd9Sstevel@tonic-gate if (++i == set->ks_nreqs) 14747c478bd9Sstevel@tonic-gate i = 0; 14757c478bd9Sstevel@tonic-gate } while (i != starting_req); 14767c478bd9Sstevel@tonic-gate 14777c478bd9Sstevel@tonic-gate return (0); 14787c478bd9Sstevel@tonic-gate } 14797c478bd9Sstevel@tonic-gate 14807c478bd9Sstevel@tonic-gate kcpc_set_t * 14817c478bd9Sstevel@tonic-gate kcpc_dup_set(kcpc_set_t *set) 14827c478bd9Sstevel@tonic-gate { 14837c478bd9Sstevel@tonic-gate kcpc_set_t *new; 14847c478bd9Sstevel@tonic-gate int i; 14857c478bd9Sstevel@tonic-gate int j; 14867c478bd9Sstevel@tonic-gate 14877c478bd9Sstevel@tonic-gate new = kmem_alloc(sizeof (*new), KM_SLEEP); 14887c478bd9Sstevel@tonic-gate new->ks_flags = set->ks_flags; 14897c478bd9Sstevel@tonic-gate new->ks_nreqs = set->ks_nreqs; 14907c478bd9Sstevel@tonic-gate new->ks_req = kmem_alloc(set->ks_nreqs * sizeof (kcpc_request_t), 14917c478bd9Sstevel@tonic-gate KM_SLEEP); 14927c478bd9Sstevel@tonic-gate new->ks_data = NULL; 14937c478bd9Sstevel@tonic-gate new->ks_ctx = NULL; 14947c478bd9Sstevel@tonic-gate 14957c478bd9Sstevel@tonic-gate for (i = 0; i < new->ks_nreqs; i++) { 14967c478bd9Sstevel@tonic-gate new->ks_req[i].kr_config = NULL; 14977c478bd9Sstevel@tonic-gate new->ks_req[i].kr_index = set->ks_req[i].kr_index; 14987c478bd9Sstevel@tonic-gate new->ks_req[i].kr_picnum = set->ks_req[i].kr_picnum; 14997c478bd9Sstevel@tonic-gate new->ks_req[i].kr_picp = NULL; 15007c478bd9Sstevel@tonic-gate new->ks_req[i].kr_data = NULL; 15017c478bd9Sstevel@tonic-gate (void) strncpy(new->ks_req[i].kr_event, set->ks_req[i].kr_event, 15027c478bd9Sstevel@tonic-gate CPC_MAX_EVENT_LEN); 15037c478bd9Sstevel@tonic-gate new->ks_req[i].kr_preset = set->ks_req[i].kr_preset; 15047c478bd9Sstevel@tonic-gate new->ks_req[i].kr_flags = set->ks_req[i].kr_flags; 15057c478bd9Sstevel@tonic-gate new->ks_req[i].kr_nattrs = set->ks_req[i].kr_nattrs; 15067c478bd9Sstevel@tonic-gate new->ks_req[i].kr_attr = kmem_alloc(new->ks_req[i].kr_nattrs * 15077c478bd9Sstevel@tonic-gate sizeof (kcpc_attr_t), KM_SLEEP); 15087c478bd9Sstevel@tonic-gate for (j = 0; j < new->ks_req[i].kr_nattrs; j++) { 15097c478bd9Sstevel@tonic-gate new->ks_req[i].kr_attr[j].ka_val = 15107c478bd9Sstevel@tonic-gate set->ks_req[i].kr_attr[j].ka_val; 15117c478bd9Sstevel@tonic-gate (void) strncpy(new->ks_req[i].kr_attr[j].ka_name, 15127c478bd9Sstevel@tonic-gate set->ks_req[i].kr_attr[j].ka_name, 15137c478bd9Sstevel@tonic-gate CPC_MAX_ATTR_LEN); 15147c478bd9Sstevel@tonic-gate } 15157c478bd9Sstevel@tonic-gate } 15167c478bd9Sstevel@tonic-gate 15177c478bd9Sstevel@tonic-gate return (new); 15187c478bd9Sstevel@tonic-gate } 15197c478bd9Sstevel@tonic-gate 15207c478bd9Sstevel@tonic-gate int 15217c478bd9Sstevel@tonic-gate kcpc_allow_nonpriv(void *token) 15227c478bd9Sstevel@tonic-gate { 15237c478bd9Sstevel@tonic-gate return (((kcpc_ctx_t *)token)->kc_flags & KCPC_CTX_NONPRIV); 15247c478bd9Sstevel@tonic-gate } 15257c478bd9Sstevel@tonic-gate 15267c478bd9Sstevel@tonic-gate void 15277c478bd9Sstevel@tonic-gate kcpc_invalidate(kthread_t *t) 15287c478bd9Sstevel@tonic-gate { 15297c478bd9Sstevel@tonic-gate kcpc_ctx_t *ctx = t->t_cpc_ctx; 15307c478bd9Sstevel@tonic-gate 15317c478bd9Sstevel@tonic-gate if (ctx != NULL) 15327c478bd9Sstevel@tonic-gate atomic_or_uint(&ctx->kc_flags, KCPC_CTX_INVALID); 15337c478bd9Sstevel@tonic-gate } 15347c478bd9Sstevel@tonic-gate 15357c478bd9Sstevel@tonic-gate /* 15367c478bd9Sstevel@tonic-gate * Given a PCBE ID, attempt to load a matching PCBE module. The strings given 15377c478bd9Sstevel@tonic-gate * are used to construct PCBE names, starting with the most specific, 15387c478bd9Sstevel@tonic-gate * "pcbe.first.second.third.fourth" and ending with the least specific, 15397c478bd9Sstevel@tonic-gate * "pcbe.first". 15407c478bd9Sstevel@tonic-gate * 15417c478bd9Sstevel@tonic-gate * Returns 0 if a PCBE was successfully loaded and -1 upon error. 15427c478bd9Sstevel@tonic-gate */ 15437c478bd9Sstevel@tonic-gate int 15447c478bd9Sstevel@tonic-gate kcpc_pcbe_tryload(const char *prefix, uint_t first, uint_t second, uint_t third) 15457c478bd9Sstevel@tonic-gate { 15467aec1d6eScindi uint_t s[3]; 15477c478bd9Sstevel@tonic-gate 15487aec1d6eScindi s[0] = first; 15497aec1d6eScindi s[1] = second; 15507aec1d6eScindi s[2] = third; 15517c478bd9Sstevel@tonic-gate 15527aec1d6eScindi return (modload_qualified("pcbe", 1553*20c794b3Sgavinm "pcbe", prefix, ".", s, 3, NULL) < 0 ? -1 : 0); 15547c478bd9Sstevel@tonic-gate } 1555