1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2003-2008 Joseph Koshy 5 * Copyright (c) 2007 The FreeBSD Foundation 6 * All rights reserved. 7 * 8 * Portions of this software were developed by A. Joseph Koshy under 9 * sponsorship from the FreeBSD Foundation and Google, Inc. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND 21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE 24 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 26 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 27 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 28 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 29 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 30 * SUCH DAMAGE. 31 */ 32 33 #include <sys/cdefs.h> 34 __FBSDID("$FreeBSD$"); 35 36 #include "opt_hwpmc_hooks.h" 37 38 #include <sys/types.h> 39 #include <sys/ctype.h> 40 #include <sys/param.h> 41 #include <sys/malloc.h> 42 #include <sys/kernel.h> 43 #include <sys/lock.h> 44 #include <sys/mutex.h> 45 #include <sys/pmc.h> 46 #include <sys/pmckern.h> 47 #include <sys/smp.h> 48 #include <sys/sysctl.h> 49 #include <sys/systm.h> 50 51 #ifdef HWPMC_HOOKS 52 FEATURE(hwpmc_hooks, "Kernel support for HW PMC"); 53 #define PMC_KERNEL_VERSION PMC_VERSION 54 #else 55 #define PMC_KERNEL_VERSION 0 56 #endif 57 58 MALLOC_DECLARE(M_PMCHOOKS); 59 MALLOC_DEFINE(M_PMCHOOKS, "pmchooks", "Memory space for PMC hooks"); 60 61 const int pmc_kernel_version = PMC_KERNEL_VERSION; 62 63 /* Hook variable. */ 64 int __read_mostly (*pmc_hook)(struct thread *td, int function, void *arg) = NULL; 65 66 /* Interrupt handler */ 67 int __read_mostly (*pmc_intr)(int cpu, struct trapframe *tf) = NULL; 68 69 /* Bitmask of CPUs requiring servicing at hardclock time */ 70 volatile cpuset_t pmc_cpumask; 71 72 /* 73 * A global count of SS mode PMCs. When non-zero, this means that 74 * we have processes that are sampling the system as a whole. 75 */ 76 volatile int pmc_ss_count; 77 78 /* 79 * Since PMC(4) may not be loaded in the current kernel, the 80 * convention followed is that a non-NULL value of 'pmc_hook' implies 81 * the presence of this kernel module. 82 * 83 * This requires us to protect 'pmc_hook' with a 84 * shared (sx) lock -- thus making the process of calling into PMC(4) 85 * somewhat more expensive than a simple 'if' check and indirect call. 86 */ 87 struct sx pmc_sx; 88 89 /* 90 * PMC Soft per cpu trapframe. 91 */ 92 struct trapframe pmc_tf[MAXCPU]; 93 94 /* 95 * PMC Soft use a global table to store registered events. 96 */ 97 98 SYSCTL_NODE(_kern, OID_AUTO, hwpmc, CTLFLAG_RW, 0, "HWPMC parameters"); 99 100 static int pmc_softevents = 16; 101 SYSCTL_INT(_kern_hwpmc, OID_AUTO, softevents, CTLFLAG_RDTUN, 102 &pmc_softevents, 0, "maximum number of soft events"); 103 104 struct mtx pmc_softs_mtx; 105 int pmc_softs_count; 106 struct pmc_soft **pmc_softs; 107 108 MTX_SYSINIT(pmc_soft_mtx, &pmc_softs_mtx, "pmc-softs", MTX_SPIN); 109 110 static void 111 pmc_init_sx(void) 112 { 113 sx_init_flags(&pmc_sx, "pmc-sx", SX_NOWITNESS); 114 } 115 116 SYSINIT(pmcsx, SI_SUB_LOCK, SI_ORDER_MIDDLE, pmc_init_sx, NULL); 117 118 /* 119 * Helper functions. 120 */ 121 122 /* 123 * A note on the CPU numbering scheme used by the hwpmc(4) driver. 124 * 125 * CPUs are denoted using numbers in the range 0..[pmc_cpu_max()-1]. 126 * CPUs could be numbered "sparsely" in this range; the predicate 127 * `pmc_cpu_is_present()' is used to test whether a given CPU is 128 * physically present. 129 * 130 * Further, a CPU that is physically present may be administratively 131 * disabled or otherwise unavailable for use by hwpmc(4). The 132 * `pmc_cpu_is_active()' predicate tests for CPU usability. An 133 * "active" CPU participates in thread scheduling and can field 134 * interrupts raised by PMC hardware. 135 * 136 * On systems with hyperthreaded CPUs, multiple logical CPUs may share 137 * PMC hardware resources. For such processors one logical CPU is 138 * denoted as the primary owner of the in-CPU PMC resources. The 139 * pmc_cpu_is_primary() predicate is used to distinguish this primary 140 * CPU from the others. 141 */ 142 143 int 144 pmc_cpu_is_active(int cpu) 145 { 146 #ifdef SMP 147 return (pmc_cpu_is_present(cpu) && 148 !CPU_ISSET(cpu, &hlt_cpus_mask)); 149 #else 150 return (1); 151 #endif 152 } 153 154 /* Deprecated. */ 155 int 156 pmc_cpu_is_disabled(int cpu) 157 { 158 return (!pmc_cpu_is_active(cpu)); 159 } 160 161 int 162 pmc_cpu_is_present(int cpu) 163 { 164 #ifdef SMP 165 return (!CPU_ABSENT(cpu)); 166 #else 167 return (1); 168 #endif 169 } 170 171 int 172 pmc_cpu_is_primary(int cpu) 173 { 174 #ifdef SMP 175 return (!CPU_ISSET(cpu, &logical_cpus_mask)); 176 #else 177 return (1); 178 #endif 179 } 180 181 182 /* 183 * Return the maximum CPU number supported by the system. The return 184 * value is used for scaling internal data structures and for runtime 185 * checks. 186 */ 187 unsigned int 188 pmc_cpu_max(void) 189 { 190 #ifdef SMP 191 return (mp_maxid+1); 192 #else 193 return (1); 194 #endif 195 } 196 197 #ifdef INVARIANTS 198 199 /* 200 * Return the count of CPUs in the `active' state in the system. 201 */ 202 int 203 pmc_cpu_max_active(void) 204 { 205 #ifdef SMP 206 /* 207 * When support for CPU hot-plugging is added to the kernel, 208 * this function would change to return the current number 209 * of "active" CPUs. 210 */ 211 return (mp_ncpus); 212 #else 213 return (1); 214 #endif 215 } 216 217 #endif 218 219 /* 220 * Cleanup event name: 221 * - remove duplicate '_' 222 * - all uppercase 223 */ 224 static void 225 pmc_soft_namecleanup(char *name) 226 { 227 char *p, *q; 228 229 p = q = name; 230 231 for ( ; *p == '_' ; p++) 232 ; 233 for ( ; *p ; p++) { 234 if (*p == '_' && (*(p + 1) == '_' || *(p + 1) == '\0')) 235 continue; 236 else 237 *q++ = toupper(*p); 238 } 239 *q = '\0'; 240 } 241 242 void 243 pmc_soft_ev_register(struct pmc_soft *ps) 244 { 245 static int warned = 0; 246 int n; 247 248 ps->ps_running = 0; 249 ps->ps_ev.pm_ev_code = 0; /* invalid */ 250 pmc_soft_namecleanup(ps->ps_ev.pm_ev_name); 251 252 mtx_lock_spin(&pmc_softs_mtx); 253 254 if (pmc_softs_count >= pmc_softevents) { 255 /* 256 * XXX Reusing events can enter a race condition where 257 * new allocated event will be used as an old one. 258 */ 259 for (n = 0; n < pmc_softevents; n++) 260 if (pmc_softs[n] == NULL) 261 break; 262 if (n == pmc_softevents) { 263 mtx_unlock_spin(&pmc_softs_mtx); 264 if (!warned) { 265 printf("hwpmc: too many soft events, " 266 "increase kern.hwpmc.softevents tunable\n"); 267 warned = 1; 268 } 269 return; 270 } 271 272 ps->ps_ev.pm_ev_code = PMC_EV_SOFT_FIRST + n; 273 pmc_softs[n] = ps; 274 } else { 275 ps->ps_ev.pm_ev_code = PMC_EV_SOFT_FIRST + pmc_softs_count; 276 pmc_softs[pmc_softs_count++] = ps; 277 } 278 279 mtx_unlock_spin(&pmc_softs_mtx); 280 } 281 282 void 283 pmc_soft_ev_deregister(struct pmc_soft *ps) 284 { 285 286 KASSERT(ps != NULL, ("pmc_soft_deregister: called with NULL")); 287 288 mtx_lock_spin(&pmc_softs_mtx); 289 290 if (ps->ps_ev.pm_ev_code != 0 && 291 (ps->ps_ev.pm_ev_code - PMC_EV_SOFT_FIRST) < pmc_softevents) { 292 KASSERT((int)ps->ps_ev.pm_ev_code >= PMC_EV_SOFT_FIRST && 293 (int)ps->ps_ev.pm_ev_code <= PMC_EV_SOFT_LAST, 294 ("pmc_soft_deregister: invalid event value")); 295 pmc_softs[ps->ps_ev.pm_ev_code - PMC_EV_SOFT_FIRST] = NULL; 296 } 297 298 mtx_unlock_spin(&pmc_softs_mtx); 299 } 300 301 struct pmc_soft * 302 pmc_soft_ev_acquire(enum pmc_event ev) 303 { 304 struct pmc_soft *ps; 305 306 if (ev == 0 || (ev - PMC_EV_SOFT_FIRST) >= pmc_softevents) 307 return NULL; 308 309 KASSERT((int)ev >= PMC_EV_SOFT_FIRST && 310 (int)ev <= PMC_EV_SOFT_LAST, 311 ("event out of range")); 312 313 mtx_lock_spin(&pmc_softs_mtx); 314 315 ps = pmc_softs[ev - PMC_EV_SOFT_FIRST]; 316 if (ps == NULL) 317 mtx_unlock_spin(&pmc_softs_mtx); 318 319 return ps; 320 } 321 322 void 323 pmc_soft_ev_release(struct pmc_soft *ps) 324 { 325 326 mtx_unlock_spin(&pmc_softs_mtx); 327 } 328 329 /* 330 * Initialise hwpmc. 331 */ 332 static void 333 init_hwpmc(void *dummy __unused) 334 { 335 if (pmc_softevents <= 0 || 336 pmc_softevents > PMC_EV_DYN_COUNT) { 337 (void) printf("hwpmc: tunable \"softevents\"=%d out of " 338 "range.\n", pmc_softevents); 339 pmc_softevents = PMC_EV_DYN_COUNT; 340 } 341 pmc_softs = malloc(pmc_softevents * sizeof(struct pmc_soft *), M_PMCHOOKS, M_NOWAIT|M_ZERO); 342 KASSERT(pmc_softs != NULL, ("cannot allocate soft events table")); 343 } 344 345 SYSINIT(hwpmc, SI_SUB_KDTRACE, SI_ORDER_FIRST, init_hwpmc, NULL); 346 347