1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2003-2008 Joseph Koshy 5 * Copyright (c) 2007 The FreeBSD Foundation 6 * All rights reserved. 7 * 8 * Portions of this software were developed by A. Joseph Koshy under 9 * sponsorship from the FreeBSD Foundation and Google, Inc. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND 21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE 24 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 26 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 27 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 28 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 29 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 30 * SUCH DAMAGE. 31 */ 32 33 #include <sys/cdefs.h> 34 #include "opt_hwpmc_hooks.h" 35 36 #include <sys/param.h> 37 #include <sys/ctype.h> 38 #include <sys/domainset.h> 39 #include <sys/param.h> 40 #include <sys/malloc.h> 41 #include <sys/kernel.h> 42 #include <sys/lock.h> 43 #include <sys/mutex.h> 44 #include <sys/pmc.h> 45 #include <sys/pmckern.h> 46 #include <sys/smp.h> 47 #include <sys/sysctl.h> 48 #include <sys/systm.h> 49 50 #include <vm/vm.h> 51 #include <vm/vm_extern.h> 52 #include <vm/vm_kern.h> 53 54 #ifdef HWPMC_HOOKS 55 FEATURE(hwpmc_hooks, "Kernel support for HW PMC"); 56 #define PMC_KERNEL_VERSION PMC_VERSION 57 #else 58 #define PMC_KERNEL_VERSION 0 59 #endif 60 61 MALLOC_DECLARE(M_PMCHOOKS); 62 MALLOC_DEFINE(M_PMCHOOKS, "pmchooks", "Memory space for PMC hooks"); 63 64 /* memory pool */ 65 MALLOC_DEFINE(M_PMC, "pmc", "Memory space for the PMC module"); 66 67 const int pmc_kernel_version = PMC_KERNEL_VERSION; 68 69 /* Hook variable. */ 70 int __read_mostly (*pmc_hook)(struct thread *td, int function, void *arg) = NULL; 71 72 /* Interrupt handler */ 73 int __read_mostly (*pmc_intr)(struct trapframe *tf) = NULL; 74 75 /* HWT hooks */ 76 void __read_mostly (*hwt_hook)(struct thread *td, int func, void *arg) = NULL; 77 int __read_mostly (*hwt_intr)(struct trapframe *tf) = NULL; 78 79 DPCPU_DEFINE(uint8_t, pmc_sampled); 80 81 /* 82 * A global count of SS mode PMCs. When non-zero, this means that 83 * we have processes that are sampling the system as a whole. 84 */ 85 volatile int pmc_ss_count; 86 87 /* 88 * Since PMC(4) may not be loaded in the current kernel, the 89 * convention followed is that a non-NULL value of 'pmc_hook' implies 90 * the presence of this kernel module. 91 * 92 * This requires us to protect 'pmc_hook' with a 93 * shared (sx) lock -- thus making the process of calling into PMC(4) 94 * somewhat more expensive than a simple 'if' check and indirect call. 95 */ 96 struct sx pmc_sx; 97 SX_SYSINIT(pmcsx, &pmc_sx, "pmc-sx"); 98 99 /* 100 * PMC Soft per cpu trapframe. 101 */ 102 struct trapframe pmc_tf[MAXCPU]; 103 104 /* 105 * Per domain list of buffer headers 106 */ 107 __read_mostly struct pmc_domain_buffer_header *pmc_dom_hdrs[MAXMEMDOM]; 108 109 /* 110 * PMC Soft use a global table to store registered events. 111 */ 112 113 SYSCTL_NODE(_kern, OID_AUTO, hwpmc, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 114 "HWPMC parameters"); 115 116 static int pmc_softevents = 16; 117 SYSCTL_INT(_kern_hwpmc, OID_AUTO, softevents, CTLFLAG_RDTUN, 118 &pmc_softevents, 0, "maximum number of soft events"); 119 120 int pmc_softs_count; 121 struct pmc_soft **pmc_softs; 122 123 struct mtx pmc_softs_mtx; 124 MTX_SYSINIT(pmc_soft_mtx, &pmc_softs_mtx, "pmc-softs", MTX_SPIN); 125 126 /* 127 * Helper functions. 128 */ 129 130 /* 131 * A note on the CPU numbering scheme used by the hwpmc(4) driver. 132 * 133 * CPUs are denoted using numbers in the range 0..[pmc_cpu_max()-1]. 134 * CPUs could be numbered "sparsely" in this range; the predicate 135 * `pmc_cpu_is_present()' is used to test whether a given CPU is 136 * physically present. 137 * 138 * Further, a CPU that is physically present may be administratively 139 * disabled or otherwise unavailable for use by hwpmc(4). The 140 * `pmc_cpu_is_active()' predicate tests for CPU usability. An 141 * "active" CPU participates in thread scheduling and can field 142 * interrupts raised by PMC hardware. 143 * 144 * On systems with hyperthreaded CPUs, multiple logical CPUs may share 145 * PMC hardware resources. For such processors one logical CPU is 146 * denoted as the primary owner of the in-CPU PMC resources. The 147 * pmc_cpu_is_primary() predicate is used to distinguish this primary 148 * CPU from the others. 149 */ 150 151 int 152 pmc_cpu_is_active(int cpu) 153 { 154 #ifdef SMP 155 return (pmc_cpu_is_present(cpu) && 156 !CPU_ISSET(cpu, &hlt_cpus_mask)); 157 #else 158 return (1); 159 #endif 160 } 161 162 /* Deprecated. */ 163 int 164 pmc_cpu_is_disabled(int cpu) 165 { 166 return (!pmc_cpu_is_active(cpu)); 167 } 168 169 int 170 pmc_cpu_is_present(int cpu) 171 { 172 #ifdef SMP 173 return (!CPU_ABSENT(cpu)); 174 #else 175 return (1); 176 #endif 177 } 178 179 int 180 pmc_cpu_is_primary(int cpu) 181 { 182 #ifdef SMP 183 return (!CPU_ISSET(cpu, &logical_cpus_mask)); 184 #else 185 return (1); 186 #endif 187 } 188 189 /* 190 * Return the maximum CPU number supported by the system. The return 191 * value is used for scaling internal data structures and for runtime 192 * checks. 193 */ 194 unsigned int 195 pmc_cpu_max(void) 196 { 197 #ifdef SMP 198 return (mp_maxid+1); 199 #else 200 return (1); 201 #endif 202 } 203 204 #ifdef INVARIANTS 205 206 /* 207 * Return the count of CPUs in the `active' state in the system. 208 */ 209 int 210 pmc_cpu_max_active(void) 211 { 212 #ifdef SMP 213 /* 214 * When support for CPU hot-plugging is added to the kernel, 215 * this function would change to return the current number 216 * of "active" CPUs. 217 */ 218 return (mp_ncpus); 219 #else 220 return (1); 221 #endif 222 } 223 224 #endif 225 226 /* 227 * Cleanup event name: 228 * - remove duplicate '_' 229 * - all uppercase 230 */ 231 static void 232 pmc_soft_namecleanup(char *name) 233 { 234 char *p, *q; 235 236 p = q = name; 237 238 for ( ; *p == '_' ; p++) 239 ; 240 for ( ; *p ; p++) { 241 if (*p == '_' && (*(p + 1) == '_' || *(p + 1) == '\0')) 242 continue; 243 else 244 *q++ = toupper(*p); 245 } 246 *q = '\0'; 247 } 248 249 void 250 pmc_soft_ev_register(struct pmc_soft *ps) 251 { 252 static int warned = 0; 253 int n; 254 255 ps->ps_running = 0; 256 ps->ps_ev.pm_ev_code = 0; /* invalid */ 257 pmc_soft_namecleanup(ps->ps_ev.pm_ev_name); 258 259 mtx_lock_spin(&pmc_softs_mtx); 260 261 if (pmc_softs_count >= pmc_softevents) { 262 /* 263 * XXX Reusing events can enter a race condition where 264 * new allocated event will be used as an old one. 265 */ 266 for (n = 0; n < pmc_softevents; n++) 267 if (pmc_softs[n] == NULL) 268 break; 269 if (n == pmc_softevents) { 270 mtx_unlock_spin(&pmc_softs_mtx); 271 if (!warned) { 272 printf("hwpmc: too many soft events, " 273 "increase kern.hwpmc.softevents tunable\n"); 274 warned = 1; 275 } 276 return; 277 } 278 279 ps->ps_ev.pm_ev_code = PMC_EV_SOFT_FIRST + n; 280 pmc_softs[n] = ps; 281 } else { 282 ps->ps_ev.pm_ev_code = PMC_EV_SOFT_FIRST + pmc_softs_count; 283 pmc_softs[pmc_softs_count++] = ps; 284 } 285 286 mtx_unlock_spin(&pmc_softs_mtx); 287 } 288 289 void 290 pmc_soft_ev_deregister(struct pmc_soft *ps) 291 { 292 293 KASSERT(ps != NULL, ("pmc_soft_deregister: called with NULL")); 294 295 mtx_lock_spin(&pmc_softs_mtx); 296 297 if (ps->ps_ev.pm_ev_code != 0 && 298 (ps->ps_ev.pm_ev_code - PMC_EV_SOFT_FIRST) < pmc_softevents) { 299 KASSERT((int)ps->ps_ev.pm_ev_code >= PMC_EV_SOFT_FIRST && 300 (int)ps->ps_ev.pm_ev_code <= PMC_EV_SOFT_LAST, 301 ("pmc_soft_deregister: invalid event value")); 302 pmc_softs[ps->ps_ev.pm_ev_code - PMC_EV_SOFT_FIRST] = NULL; 303 } 304 305 mtx_unlock_spin(&pmc_softs_mtx); 306 } 307 308 struct pmc_soft * 309 pmc_soft_ev_acquire(enum pmc_event ev) 310 { 311 struct pmc_soft *ps; 312 313 if (ev == 0 || (ev - PMC_EV_SOFT_FIRST) >= pmc_softevents) 314 return NULL; 315 316 KASSERT((int)ev >= PMC_EV_SOFT_FIRST && 317 (int)ev <= PMC_EV_SOFT_LAST, 318 ("event out of range")); 319 320 mtx_lock_spin(&pmc_softs_mtx); 321 322 ps = pmc_softs[ev - PMC_EV_SOFT_FIRST]; 323 if (ps == NULL) 324 mtx_unlock_spin(&pmc_softs_mtx); 325 326 return ps; 327 } 328 329 void 330 pmc_soft_ev_release(struct pmc_soft *ps) 331 { 332 333 mtx_unlock_spin(&pmc_softs_mtx); 334 } 335 336 /* 337 * Initialise hwpmc. 338 */ 339 static void 340 init_hwpmc(void *dummy __unused) 341 { 342 int domain, cpu; 343 344 if (pmc_softevents <= 0 || 345 pmc_softevents > PMC_EV_DYN_COUNT) { 346 (void) printf("hwpmc: tunable \"softevents\"=%d out of " 347 "range.\n", pmc_softevents); 348 pmc_softevents = PMC_EV_DYN_COUNT; 349 } 350 pmc_softs = malloc(pmc_softevents * sizeof(*pmc_softs), M_PMCHOOKS, 351 M_WAITOK | M_ZERO); 352 353 for (domain = 0; domain < vm_ndomains; domain++) { 354 pmc_dom_hdrs[domain] = malloc_domainset( 355 sizeof(struct pmc_domain_buffer_header), M_PMC, 356 DOMAINSET_PREF(domain), M_WAITOK | M_ZERO); 357 mtx_init(&pmc_dom_hdrs[domain]->pdbh_mtx, "pmc_bufferlist_mtx", "pmc-leaf", MTX_SPIN); 358 TAILQ_INIT(&pmc_dom_hdrs[domain]->pdbh_head); 359 } 360 CPU_FOREACH(cpu) { 361 domain = pcpu_find(cpu)->pc_domain; 362 KASSERT(pmc_dom_hdrs[domain] != NULL, ("no mem allocated for domain: %d", domain)); 363 pmc_dom_hdrs[domain]->pdbh_ncpus++; 364 } 365 366 } 367 368 SYSINIT(hwpmc, SI_SUB_KDTRACE, SI_ORDER_FIRST, init_hwpmc, NULL); 369