1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2003-2008 Joseph Koshy 5 * Copyright (c) 2007 The FreeBSD Foundation 6 * All rights reserved. 7 * 8 * Portions of this software were developed by A. Joseph Koshy under 9 * sponsorship from the FreeBSD Foundation and Google, Inc. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND 21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE 24 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 26 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 27 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 28 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 29 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 30 * SUCH DAMAGE. 31 */ 32 33 #include <sys/cdefs.h> 34 __FBSDID("$FreeBSD$"); 35 36 #include "opt_hwpmc_hooks.h" 37 38 #include <sys/param.h> 39 #include <sys/ctype.h> 40 #include <sys/domainset.h> 41 #include <sys/param.h> 42 #include <sys/malloc.h> 43 #include <sys/kernel.h> 44 #include <sys/lock.h> 45 #include <sys/mutex.h> 46 #include <sys/pmc.h> 47 #include <sys/pmckern.h> 48 #include <sys/smp.h> 49 #include <sys/sysctl.h> 50 #include <sys/systm.h> 51 52 #include <vm/vm.h> 53 #include <vm/vm_extern.h> 54 #include <vm/vm_kern.h> 55 56 #ifdef HWPMC_HOOKS 57 FEATURE(hwpmc_hooks, "Kernel support for HW PMC"); 58 #define PMC_KERNEL_VERSION PMC_VERSION 59 #else 60 #define PMC_KERNEL_VERSION 0 61 #endif 62 63 MALLOC_DECLARE(M_PMCHOOKS); 64 MALLOC_DEFINE(M_PMCHOOKS, "pmchooks", "Memory space for PMC hooks"); 65 66 /* memory pool */ 67 MALLOC_DEFINE(M_PMC, "pmc", "Memory space for the PMC module"); 68 69 const int pmc_kernel_version = PMC_KERNEL_VERSION; 70 71 /* Hook variable. */ 72 int __read_mostly (*pmc_hook)(struct thread *td, int function, void *arg) = NULL; 73 74 /* Interrupt handler */ 75 int __read_mostly (*pmc_intr)(struct trapframe *tf) = NULL; 76 77 DPCPU_DEFINE(uint8_t, pmc_sampled); 78 79 /* 80 * A global count of SS mode PMCs. When non-zero, this means that 81 * we have processes that are sampling the system as a whole. 82 */ 83 volatile int pmc_ss_count; 84 85 /* 86 * Since PMC(4) may not be loaded in the current kernel, the 87 * convention followed is that a non-NULL value of 'pmc_hook' implies 88 * the presence of this kernel module. 89 * 90 * This requires us to protect 'pmc_hook' with a 91 * shared (sx) lock -- thus making the process of calling into PMC(4) 92 * somewhat more expensive than a simple 'if' check and indirect call. 93 */ 94 struct sx pmc_sx; 95 SX_SYSINIT(pmcsx, &pmc_sx, "pmc-sx"); 96 97 /* 98 * PMC Soft per cpu trapframe. 99 */ 100 struct trapframe pmc_tf[MAXCPU]; 101 102 /* 103 * Per domain list of buffer headers 104 */ 105 __read_mostly struct pmc_domain_buffer_header *pmc_dom_hdrs[MAXMEMDOM]; 106 107 /* 108 * PMC Soft use a global table to store registered events. 109 */ 110 111 SYSCTL_NODE(_kern, OID_AUTO, hwpmc, CTLFLAG_RW, 0, "HWPMC parameters"); 112 113 static int pmc_softevents = 16; 114 SYSCTL_INT(_kern_hwpmc, OID_AUTO, softevents, CTLFLAG_RDTUN, 115 &pmc_softevents, 0, "maximum number of soft events"); 116 117 int pmc_softs_count; 118 struct pmc_soft **pmc_softs; 119 120 struct mtx pmc_softs_mtx; 121 MTX_SYSINIT(pmc_soft_mtx, &pmc_softs_mtx, "pmc-softs", MTX_SPIN); 122 123 /* 124 * Helper functions. 125 */ 126 127 /* 128 * A note on the CPU numbering scheme used by the hwpmc(4) driver. 129 * 130 * CPUs are denoted using numbers in the range 0..[pmc_cpu_max()-1]. 131 * CPUs could be numbered "sparsely" in this range; the predicate 132 * `pmc_cpu_is_present()' is used to test whether a given CPU is 133 * physically present. 134 * 135 * Further, a CPU that is physically present may be administratively 136 * disabled or otherwise unavailable for use by hwpmc(4). The 137 * `pmc_cpu_is_active()' predicate tests for CPU usability. An 138 * "active" CPU participates in thread scheduling and can field 139 * interrupts raised by PMC hardware. 140 * 141 * On systems with hyperthreaded CPUs, multiple logical CPUs may share 142 * PMC hardware resources. For such processors one logical CPU is 143 * denoted as the primary owner of the in-CPU PMC resources. The 144 * pmc_cpu_is_primary() predicate is used to distinguish this primary 145 * CPU from the others. 146 */ 147 148 int 149 pmc_cpu_is_active(int cpu) 150 { 151 #ifdef SMP 152 return (pmc_cpu_is_present(cpu) && 153 !CPU_ISSET(cpu, &hlt_cpus_mask)); 154 #else 155 return (1); 156 #endif 157 } 158 159 /* Deprecated. */ 160 int 161 pmc_cpu_is_disabled(int cpu) 162 { 163 return (!pmc_cpu_is_active(cpu)); 164 } 165 166 int 167 pmc_cpu_is_present(int cpu) 168 { 169 #ifdef SMP 170 return (!CPU_ABSENT(cpu)); 171 #else 172 return (1); 173 #endif 174 } 175 176 int 177 pmc_cpu_is_primary(int cpu) 178 { 179 #ifdef SMP 180 return (!CPU_ISSET(cpu, &logical_cpus_mask)); 181 #else 182 return (1); 183 #endif 184 } 185 186 /* 187 * Return the maximum CPU number supported by the system. The return 188 * value is used for scaling internal data structures and for runtime 189 * checks. 190 */ 191 unsigned int 192 pmc_cpu_max(void) 193 { 194 #ifdef SMP 195 return (mp_maxid+1); 196 #else 197 return (1); 198 #endif 199 } 200 201 #ifdef INVARIANTS 202 203 /* 204 * Return the count of CPUs in the `active' state in the system. 205 */ 206 int 207 pmc_cpu_max_active(void) 208 { 209 #ifdef SMP 210 /* 211 * When support for CPU hot-plugging is added to the kernel, 212 * this function would change to return the current number 213 * of "active" CPUs. 214 */ 215 return (mp_ncpus); 216 #else 217 return (1); 218 #endif 219 } 220 221 #endif 222 223 /* 224 * Cleanup event name: 225 * - remove duplicate '_' 226 * - all uppercase 227 */ 228 static void 229 pmc_soft_namecleanup(char *name) 230 { 231 char *p, *q; 232 233 p = q = name; 234 235 for ( ; *p == '_' ; p++) 236 ; 237 for ( ; *p ; p++) { 238 if (*p == '_' && (*(p + 1) == '_' || *(p + 1) == '\0')) 239 continue; 240 else 241 *q++ = toupper(*p); 242 } 243 *q = '\0'; 244 } 245 246 void 247 pmc_soft_ev_register(struct pmc_soft *ps) 248 { 249 static int warned = 0; 250 int n; 251 252 ps->ps_running = 0; 253 ps->ps_ev.pm_ev_code = 0; /* invalid */ 254 pmc_soft_namecleanup(ps->ps_ev.pm_ev_name); 255 256 mtx_lock_spin(&pmc_softs_mtx); 257 258 if (pmc_softs_count >= pmc_softevents) { 259 /* 260 * XXX Reusing events can enter a race condition where 261 * new allocated event will be used as an old one. 262 */ 263 for (n = 0; n < pmc_softevents; n++) 264 if (pmc_softs[n] == NULL) 265 break; 266 if (n == pmc_softevents) { 267 mtx_unlock_spin(&pmc_softs_mtx); 268 if (!warned) { 269 printf("hwpmc: too many soft events, " 270 "increase kern.hwpmc.softevents tunable\n"); 271 warned = 1; 272 } 273 return; 274 } 275 276 ps->ps_ev.pm_ev_code = PMC_EV_SOFT_FIRST + n; 277 pmc_softs[n] = ps; 278 } else { 279 ps->ps_ev.pm_ev_code = PMC_EV_SOFT_FIRST + pmc_softs_count; 280 pmc_softs[pmc_softs_count++] = ps; 281 } 282 283 mtx_unlock_spin(&pmc_softs_mtx); 284 } 285 286 void 287 pmc_soft_ev_deregister(struct pmc_soft *ps) 288 { 289 290 KASSERT(ps != NULL, ("pmc_soft_deregister: called with NULL")); 291 292 mtx_lock_spin(&pmc_softs_mtx); 293 294 if (ps->ps_ev.pm_ev_code != 0 && 295 (ps->ps_ev.pm_ev_code - PMC_EV_SOFT_FIRST) < pmc_softevents) { 296 KASSERT((int)ps->ps_ev.pm_ev_code >= PMC_EV_SOFT_FIRST && 297 (int)ps->ps_ev.pm_ev_code <= PMC_EV_SOFT_LAST, 298 ("pmc_soft_deregister: invalid event value")); 299 pmc_softs[ps->ps_ev.pm_ev_code - PMC_EV_SOFT_FIRST] = NULL; 300 } 301 302 mtx_unlock_spin(&pmc_softs_mtx); 303 } 304 305 struct pmc_soft * 306 pmc_soft_ev_acquire(enum pmc_event ev) 307 { 308 struct pmc_soft *ps; 309 310 if (ev == 0 || (ev - PMC_EV_SOFT_FIRST) >= pmc_softevents) 311 return NULL; 312 313 KASSERT((int)ev >= PMC_EV_SOFT_FIRST && 314 (int)ev <= PMC_EV_SOFT_LAST, 315 ("event out of range")); 316 317 mtx_lock_spin(&pmc_softs_mtx); 318 319 ps = pmc_softs[ev - PMC_EV_SOFT_FIRST]; 320 if (ps == NULL) 321 mtx_unlock_spin(&pmc_softs_mtx); 322 323 return ps; 324 } 325 326 void 327 pmc_soft_ev_release(struct pmc_soft *ps) 328 { 329 330 mtx_unlock_spin(&pmc_softs_mtx); 331 } 332 333 /* 334 * Initialise hwpmc. 335 */ 336 static void 337 init_hwpmc(void *dummy __unused) 338 { 339 int domain, cpu; 340 341 if (pmc_softevents <= 0 || 342 pmc_softevents > PMC_EV_DYN_COUNT) { 343 (void) printf("hwpmc: tunable \"softevents\"=%d out of " 344 "range.\n", pmc_softevents); 345 pmc_softevents = PMC_EV_DYN_COUNT; 346 } 347 pmc_softs = malloc(pmc_softevents * sizeof(*pmc_softs), M_PMCHOOKS, 348 M_WAITOK | M_ZERO); 349 350 for (domain = 0; domain < vm_ndomains; domain++) { 351 pmc_dom_hdrs[domain] = malloc_domainset( 352 sizeof(struct pmc_domain_buffer_header), M_PMC, 353 DOMAINSET_PREF(domain), M_WAITOK | M_ZERO); 354 mtx_init(&pmc_dom_hdrs[domain]->pdbh_mtx, "pmc_bufferlist_mtx", "pmc-leaf", MTX_SPIN); 355 TAILQ_INIT(&pmc_dom_hdrs[domain]->pdbh_head); 356 } 357 CPU_FOREACH(cpu) { 358 domain = pcpu_find(cpu)->pc_domain; 359 KASSERT(pmc_dom_hdrs[domain] != NULL, ("no mem allocated for domain: %d", domain)); 360 pmc_dom_hdrs[domain]->pdbh_ncpus++; 361 } 362 363 } 364 365 SYSINIT(hwpmc, SI_SUB_KDTRACE, SI_ORDER_FIRST, init_hwpmc, NULL); 366 367