17c478bd9Sstevel@tonic-gate /* 27c478bd9Sstevel@tonic-gate * CDDL HEADER START 37c478bd9Sstevel@tonic-gate * 47c478bd9Sstevel@tonic-gate * The contents of this file are subject to the terms of the 5fb2f18f8Sesaxe * Common Development and Distribution License (the "License"). 6fb2f18f8Sesaxe * You may not use this file except in compliance with the License. 77c478bd9Sstevel@tonic-gate * 87c478bd9Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 97c478bd9Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing. 107c478bd9Sstevel@tonic-gate * See the License for the specific language governing permissions 117c478bd9Sstevel@tonic-gate * and limitations under the License. 127c478bd9Sstevel@tonic-gate * 137c478bd9Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each 147c478bd9Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 157c478bd9Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the 167c478bd9Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying 177c478bd9Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner] 187c478bd9Sstevel@tonic-gate * 197c478bd9Sstevel@tonic-gate * CDDL HEADER END 207c478bd9Sstevel@tonic-gate */ 217c478bd9Sstevel@tonic-gate /* 227417cfdeSKuriakose Kuruvilla * Copyright (c) 1999, 2010, Oracle and/or its affiliates. All rights reserved. 23*3db3a4acSRobert Mustacchi * Copyright 2019 Joyent, Inc. 247c478bd9Sstevel@tonic-gate */ 257c478bd9Sstevel@tonic-gate 267c478bd9Sstevel@tonic-gate /* 277c478bd9Sstevel@tonic-gate * x86-specific routines used by the CPU Performance counter driver. 287c478bd9Sstevel@tonic-gate */ 297c478bd9Sstevel@tonic-gate 307c478bd9Sstevel@tonic-gate #include <sys/types.h> 317c478bd9Sstevel@tonic-gate #include <sys/time.h> 327c478bd9Sstevel@tonic-gate #include <sys/atomic.h> 337c478bd9Sstevel@tonic-gate #include <sys/regset.h> 347c478bd9Sstevel@tonic-gate #include <sys/privregs.h> 357c478bd9Sstevel@tonic-gate #include <sys/x86_archext.h> 367c478bd9Sstevel@tonic-gate #include <sys/cpuvar.h> 377c478bd9Sstevel@tonic-gate #include <sys/machcpuvar.h> 387c478bd9Sstevel@tonic-gate #include <sys/archsystm.h> 397c478bd9Sstevel@tonic-gate #include <sys/cpc_pcbe.h> 407c478bd9Sstevel@tonic-gate #include <sys/cpc_impl.h> 417c478bd9Sstevel@tonic-gate #include <sys/x_call.h> 427c478bd9Sstevel@tonic-gate #include <sys/cmn_err.h> 43fb2f18f8Sesaxe #include <sys/cmt.h> 447c478bd9Sstevel@tonic-gate #include <sys/spl.h> 45ae115bc7Smrj #include <sys/apic.h> 467c478bd9Sstevel@tonic-gate 477c478bd9Sstevel@tonic-gate static const uint64_t allstopped = 0; 487c478bd9Sstevel@tonic-gate static kcpc_ctx_t *(*overflow_intr_handler)(caddr_t); 497c478bd9Sstevel@tonic-gate 50a18ddb3cSKuriakose Kuruvilla /* Do threads share performance monitoring hardware? */ 51a18ddb3cSKuriakose Kuruvilla static int strands_perfmon_shared = 0; 52a18ddb3cSKuriakose Kuruvilla 537c478bd9Sstevel@tonic-gate int kcpc_hw_overflow_intr_installed; /* set by APIC code */ 547c478bd9Sstevel@tonic-gate extern kcpc_ctx_t *kcpc_overflow_intr(caddr_t arg, uint64_t bitmap); 557c478bd9Sstevel@tonic-gate 567c478bd9Sstevel@tonic-gate extern int kcpc_counts_include_idle; /* Project Private /etc/system variable */ 577c478bd9Sstevel@tonic-gate 587c478bd9Sstevel@tonic-gate void (*kcpc_hw_enable_cpc_intr)(void); /* set by APIC code */ 597c478bd9Sstevel@tonic-gate 607c478bd9Sstevel@tonic-gate int 617c478bd9Sstevel@tonic-gate kcpc_hw_add_ovf_intr(kcpc_ctx_t *(*handler)(caddr_t)) 627c478bd9Sstevel@tonic-gate { 637c478bd9Sstevel@tonic-gate if (x86_type != X86_TYPE_P6) 647c478bd9Sstevel@tonic-gate return (0); 657c478bd9Sstevel@tonic-gate overflow_intr_handler = handler; 667c478bd9Sstevel@tonic-gate return (ipltospl(APIC_PCINT_IPL)); 677c478bd9Sstevel@tonic-gate } 687c478bd9Sstevel@tonic-gate 697c478bd9Sstevel@tonic-gate void 707c478bd9Sstevel@tonic-gate kcpc_hw_rem_ovf_intr(void) 717c478bd9Sstevel@tonic-gate { 727c478bd9Sstevel@tonic-gate overflow_intr_handler = NULL; 737c478bd9Sstevel@tonic-gate } 747c478bd9Sstevel@tonic-gate 757c478bd9Sstevel@tonic-gate /* 767c478bd9Sstevel@tonic-gate * Hook used on P4 systems to catch online/offline events. 777c478bd9Sstevel@tonic-gate */ 787c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 797c478bd9Sstevel@tonic-gate static int 807c478bd9Sstevel@tonic-gate kcpc_cpu_setup(cpu_setup_t what, int cpuid, void *arg) 817c478bd9Sstevel@tonic-gate { 82fb2f18f8Sesaxe pg_cmt_t *chip_pg; 83fb2f18f8Sesaxe int active_cpus_cnt; 847c478bd9Sstevel@tonic-gate 857c478bd9Sstevel@tonic-gate if (what != CPU_ON) 867c478bd9Sstevel@tonic-gate return (0); 877c478bd9Sstevel@tonic-gate 887c478bd9Sstevel@tonic-gate /* 897c478bd9Sstevel@tonic-gate * If any CPU-bound contexts exist, we don't need to invalidate 907c478bd9Sstevel@tonic-gate * anything, as no per-LWP contexts can coexist. 917c478bd9Sstevel@tonic-gate */ 92af4595edSJonathan Haslam if (kcpc_cpuctx || dtrace_cpc_in_use) 937c478bd9Sstevel@tonic-gate return (0); 947c478bd9Sstevel@tonic-gate 957c478bd9Sstevel@tonic-gate /* 967c478bd9Sstevel@tonic-gate * If this chip now has more than 1 active cpu, we must invalidate all 977c478bd9Sstevel@tonic-gate * contexts in the system. 987c478bd9Sstevel@tonic-gate */ 99fb2f18f8Sesaxe chip_pg = (pg_cmt_t *)pghw_find_pg(cpu[cpuid], PGHW_CHIP); 100fb2f18f8Sesaxe if (chip_pg != NULL) { 101fb2f18f8Sesaxe active_cpus_cnt = GROUP_SIZE(&chip_pg->cmt_cpus_actv); 102fb2f18f8Sesaxe if (active_cpus_cnt > 1) 1037c478bd9Sstevel@tonic-gate kcpc_invalidate_all(); 104fb2f18f8Sesaxe } 1057c478bd9Sstevel@tonic-gate 1067c478bd9Sstevel@tonic-gate return (0); 1077c478bd9Sstevel@tonic-gate } 1087c478bd9Sstevel@tonic-gate 1097c478bd9Sstevel@tonic-gate static kmutex_t cpu_setup_lock; /* protects setup_registered */ 1107c478bd9Sstevel@tonic-gate static int setup_registered; 1117c478bd9Sstevel@tonic-gate 112a18ddb3cSKuriakose Kuruvilla 1137c478bd9Sstevel@tonic-gate void 1147c478bd9Sstevel@tonic-gate kcpc_hw_init(cpu_t *cp) 1157c478bd9Sstevel@tonic-gate { 1167c478bd9Sstevel@tonic-gate kthread_t *t = cp->cpu_idle_thread; 117a18ddb3cSKuriakose Kuruvilla uint32_t versionid; 118a18ddb3cSKuriakose Kuruvilla struct cpuid_regs cpuid; 1197c478bd9Sstevel@tonic-gate 120a18ddb3cSKuriakose Kuruvilla strands_perfmon_shared = 0; 1217417cfdeSKuriakose Kuruvilla if (is_x86_feature(x86_featureset, X86FSET_HTT)) { 122a18ddb3cSKuriakose Kuruvilla if (cpuid_getvendor(cpu[0]) == X86_VENDOR_Intel) { 123a18ddb3cSKuriakose Kuruvilla /* 124a18ddb3cSKuriakose Kuruvilla * Intel processors that support Architectural 125a18ddb3cSKuriakose Kuruvilla * Performance Monitoring Version 3 have per strand 126a18ddb3cSKuriakose Kuruvilla * performance monitoring hardware. 127a18ddb3cSKuriakose Kuruvilla * Hence we can allow use of performance counters on 128a18ddb3cSKuriakose Kuruvilla * multiple strands on the same core simultaneously. 129a18ddb3cSKuriakose Kuruvilla */ 130a18ddb3cSKuriakose Kuruvilla cpuid.cp_eax = 0x0; 131a18ddb3cSKuriakose Kuruvilla (void) __cpuid_insn(&cpuid); 132a18ddb3cSKuriakose Kuruvilla if (cpuid.cp_eax < 0xa) { 133a18ddb3cSKuriakose Kuruvilla strands_perfmon_shared = 1; 134a18ddb3cSKuriakose Kuruvilla } else { 135a18ddb3cSKuriakose Kuruvilla cpuid.cp_eax = 0xa; 136a18ddb3cSKuriakose Kuruvilla (void) __cpuid_insn(&cpuid); 137a18ddb3cSKuriakose Kuruvilla 138a18ddb3cSKuriakose Kuruvilla versionid = cpuid.cp_eax & 0xFF; 139a18ddb3cSKuriakose Kuruvilla if (versionid < 3) { 140a18ddb3cSKuriakose Kuruvilla strands_perfmon_shared = 1; 141a18ddb3cSKuriakose Kuruvilla } 142a18ddb3cSKuriakose Kuruvilla } 143*3db3a4acSRobert Mustacchi } else if (cpuid_getvendor(cpu[0]) == X86_VENDOR_AMD) { 144*3db3a4acSRobert Mustacchi /* 145*3db3a4acSRobert Mustacchi * On AMD systems with HT, all of the performance 146*3db3a4acSRobert Mustacchi * monitors exist on a per-logical CPU basis. 147*3db3a4acSRobert Mustacchi */ 148*3db3a4acSRobert Mustacchi strands_perfmon_shared = 0; 149a18ddb3cSKuriakose Kuruvilla } else { 150a18ddb3cSKuriakose Kuruvilla strands_perfmon_shared = 1; 151a18ddb3cSKuriakose Kuruvilla } 152a18ddb3cSKuriakose Kuruvilla } 153a18ddb3cSKuriakose Kuruvilla 154a18ddb3cSKuriakose Kuruvilla if (strands_perfmon_shared) { 1557c478bd9Sstevel@tonic-gate mutex_enter(&cpu_setup_lock); 1567c478bd9Sstevel@tonic-gate if (setup_registered == 0) { 1577c478bd9Sstevel@tonic-gate mutex_enter(&cpu_lock); 1587c478bd9Sstevel@tonic-gate register_cpu_setup_func(kcpc_cpu_setup, NULL); 1597c478bd9Sstevel@tonic-gate mutex_exit(&cpu_lock); 1607c478bd9Sstevel@tonic-gate setup_registered = 1; 1617c478bd9Sstevel@tonic-gate } 1627c478bd9Sstevel@tonic-gate mutex_exit(&cpu_setup_lock); 1637c478bd9Sstevel@tonic-gate } 1647c478bd9Sstevel@tonic-gate 1657c478bd9Sstevel@tonic-gate mutex_init(&cp->cpu_cpc_ctxlock, "cpu_cpc_ctxlock", MUTEX_DEFAULT, 0); 1667c478bd9Sstevel@tonic-gate 1677c478bd9Sstevel@tonic-gate if (kcpc_counts_include_idle) 1687c478bd9Sstevel@tonic-gate return; 1697c478bd9Sstevel@tonic-gate 1707c478bd9Sstevel@tonic-gate installctx(t, cp, kcpc_idle_save, kcpc_idle_restore, 1717c478bd9Sstevel@tonic-gate NULL, NULL, NULL, NULL); 1727c478bd9Sstevel@tonic-gate } 1737c478bd9Sstevel@tonic-gate 174ae115bc7Smrj void 175ae115bc7Smrj kcpc_hw_fini(cpu_t *cp) 176ae115bc7Smrj { 177ae115bc7Smrj ASSERT(cp->cpu_idle_thread == NULL); 178ae115bc7Smrj 179ae115bc7Smrj mutex_destroy(&cp->cpu_cpc_ctxlock); 180ae115bc7Smrj } 181ae115bc7Smrj 1827c478bd9Sstevel@tonic-gate #define BITS(v, u, l) \ 1837c478bd9Sstevel@tonic-gate (((v) >> (l)) & ((1 << (1 + (u) - (l))) - 1)) 1847c478bd9Sstevel@tonic-gate 1857c478bd9Sstevel@tonic-gate #define PCBE_NAMELEN 30 /* Enough Room for pcbe.manuf.model.family.stepping */ 1867c478bd9Sstevel@tonic-gate 1877c478bd9Sstevel@tonic-gate /* 1887c478bd9Sstevel@tonic-gate * Examine the processor and load an appropriate PCBE. 1897c478bd9Sstevel@tonic-gate */ 1907c478bd9Sstevel@tonic-gate int 1917c478bd9Sstevel@tonic-gate kcpc_hw_load_pcbe(void) 1927c478bd9Sstevel@tonic-gate { 1937c478bd9Sstevel@tonic-gate return (kcpc_pcbe_tryload(cpuid_getvendorstr(CPU), cpuid_getfamily(CPU), 1947c478bd9Sstevel@tonic-gate cpuid_getmodel(CPU), cpuid_getstep(CPU))); 1957c478bd9Sstevel@tonic-gate } 1967c478bd9Sstevel@tonic-gate 1977c478bd9Sstevel@tonic-gate /* 1987c478bd9Sstevel@tonic-gate * Called by the generic framework to check if it's OK to bind a set to a CPU. 1997c478bd9Sstevel@tonic-gate */ 2007c478bd9Sstevel@tonic-gate int 2017c478bd9Sstevel@tonic-gate kcpc_hw_cpu_hook(processorid_t cpuid, ulong_t *kcpc_cpumap) 2027c478bd9Sstevel@tonic-gate { 203fb2f18f8Sesaxe cpu_t *cpu, *p; 204fb2f18f8Sesaxe pg_t *chip_pg; 205fb2f18f8Sesaxe pg_cpu_itr_t itr; 2067c478bd9Sstevel@tonic-gate 207a18ddb3cSKuriakose Kuruvilla if (!strands_perfmon_shared) 2087c478bd9Sstevel@tonic-gate return (0); 2097c478bd9Sstevel@tonic-gate 2107c478bd9Sstevel@tonic-gate /* 2117c478bd9Sstevel@tonic-gate * Only one logical CPU on each Pentium 4 HT CPU may be bound to at 2127c478bd9Sstevel@tonic-gate * once. 2137c478bd9Sstevel@tonic-gate * 2147c478bd9Sstevel@tonic-gate * This loop is protected by holding cpu_lock, in order to properly 215fb2f18f8Sesaxe * access the cpu_t of the desired cpu. 2167c478bd9Sstevel@tonic-gate */ 2177c478bd9Sstevel@tonic-gate mutex_enter(&cpu_lock); 2187c478bd9Sstevel@tonic-gate if ((cpu = cpu_get(cpuid)) == NULL) { 2197c478bd9Sstevel@tonic-gate mutex_exit(&cpu_lock); 2207c478bd9Sstevel@tonic-gate return (-1); 2217c478bd9Sstevel@tonic-gate } 2227c478bd9Sstevel@tonic-gate 223fb2f18f8Sesaxe chip_pg = (pg_t *)pghw_find_pg(cpu, PGHW_CHIP); 224fb2f18f8Sesaxe 225fb2f18f8Sesaxe PG_CPU_ITR_INIT(chip_pg, itr); 226fb2f18f8Sesaxe while ((p = pg_cpu_next(&itr)) != NULL) { 227fb2f18f8Sesaxe if (p == cpu) 228fb2f18f8Sesaxe continue; 2297c478bd9Sstevel@tonic-gate if (BT_TEST(kcpc_cpumap, p->cpu_id)) { 2307c478bd9Sstevel@tonic-gate mutex_exit(&cpu_lock); 2317c478bd9Sstevel@tonic-gate return (-1); 2327c478bd9Sstevel@tonic-gate } 2337c478bd9Sstevel@tonic-gate } 2347c478bd9Sstevel@tonic-gate 2357c478bd9Sstevel@tonic-gate mutex_exit(&cpu_lock); 2367c478bd9Sstevel@tonic-gate return (0); 2377c478bd9Sstevel@tonic-gate } 2387c478bd9Sstevel@tonic-gate 2397c478bd9Sstevel@tonic-gate /* 2407c478bd9Sstevel@tonic-gate * Called by the generic framework to check if it's OK to bind a set to an LWP. 2417c478bd9Sstevel@tonic-gate */ 2427c478bd9Sstevel@tonic-gate int 2437c478bd9Sstevel@tonic-gate kcpc_hw_lwp_hook(void) 2447c478bd9Sstevel@tonic-gate { 245fb2f18f8Sesaxe pg_cmt_t *chip; 246fb2f18f8Sesaxe group_t *chips; 247fb2f18f8Sesaxe group_iter_t i; 2487c478bd9Sstevel@tonic-gate 249a18ddb3cSKuriakose Kuruvilla if (!strands_perfmon_shared) 2507c478bd9Sstevel@tonic-gate return (0); 2517c478bd9Sstevel@tonic-gate 2527c478bd9Sstevel@tonic-gate /* 2537c478bd9Sstevel@tonic-gate * Only one CPU per chip may be online. 2547c478bd9Sstevel@tonic-gate */ 2557c478bd9Sstevel@tonic-gate mutex_enter(&cpu_lock); 256fb2f18f8Sesaxe 257fb2f18f8Sesaxe chips = pghw_set_lookup(PGHW_CHIP); 258fb2f18f8Sesaxe if (chips == NULL) { 259fb2f18f8Sesaxe mutex_exit(&cpu_lock); 260fb2f18f8Sesaxe return (0); 261fb2f18f8Sesaxe } 262fb2f18f8Sesaxe 263fb2f18f8Sesaxe group_iter_init(&i); 264fb2f18f8Sesaxe while ((chip = group_iterate(chips, &i)) != NULL) { 265fb2f18f8Sesaxe if (GROUP_SIZE(&chip->cmt_cpus_actv) > 1) { 2667c478bd9Sstevel@tonic-gate mutex_exit(&cpu_lock); 2677c478bd9Sstevel@tonic-gate return (-1); 2687c478bd9Sstevel@tonic-gate } 269fb2f18f8Sesaxe } 270fb2f18f8Sesaxe 2717c478bd9Sstevel@tonic-gate mutex_exit(&cpu_lock); 2727c478bd9Sstevel@tonic-gate return (0); 2737c478bd9Sstevel@tonic-gate } 274