1 /*- 2 * Copyright (c) 2003-2008 Joseph Koshy 3 * Copyright (c) 2007 The FreeBSD Foundation 4 * All rights reserved. 5 * 6 * Portions of this software were developed by A. Joseph Koshy under 7 * sponsorship from the FreeBSD Foundation and Google, Inc. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 * 30 */ 31 32 #include <sys/cdefs.h> 33 __FBSDID("$FreeBSD$"); 34 35 #include <sys/param.h> 36 #include <sys/eventhandler.h> 37 #include <sys/jail.h> 38 #include <sys/kernel.h> 39 #include <sys/kthread.h> 40 #include <sys/limits.h> 41 #include <sys/lock.h> 42 #include <sys/malloc.h> 43 #include <sys/module.h> 44 #include <sys/mount.h> 45 #include <sys/mutex.h> 46 #include <sys/pmc.h> 47 #include <sys/pmckern.h> 48 #include <sys/pmclog.h> 49 #include <sys/priv.h> 50 #include <sys/proc.h> 51 #include <sys/queue.h> 52 #include <sys/resourcevar.h> 53 #include <sys/rwlock.h> 54 #include <sys/sched.h> 55 #include <sys/signalvar.h> 56 #include <sys/smp.h> 57 #include <sys/sx.h> 58 #include <sys/sysctl.h> 59 #include <sys/sysent.h> 60 #include <sys/systm.h> 61 #include <sys/vnode.h> 62 63 #include <sys/linker.h> /* needs to be after <sys/malloc.h> */ 64 65 #include <machine/atomic.h> 66 #include <machine/md_var.h> 67 68 #include <vm/vm.h> 69 #include <vm/vm_extern.h> 70 #include <vm/pmap.h> 71 #include <vm/vm_map.h> 72 #include <vm/vm_object.h> 73 74 #include "hwpmc_soft.h" 75 76 /* 77 * Types 78 */ 79 80 enum pmc_flags { 81 PMC_FLAG_NONE = 0x00, /* do nothing */ 82 PMC_FLAG_REMOVE = 0x01, /* atomically remove entry from hash */ 83 PMC_FLAG_ALLOCATE = 0x02, /* add entry to hash if not found */ 84 }; 85 86 /* 87 * The offset in sysent where the syscall is allocated. 88 */ 89 90 static int pmc_syscall_num = NO_SYSCALL; 91 struct pmc_cpu **pmc_pcpu; /* per-cpu state */ 92 pmc_value_t *pmc_pcpu_saved; /* saved PMC values: CSW handling */ 93 94 #define PMC_PCPU_SAVED(C,R) pmc_pcpu_saved[(R) + md->pmd_npmc*(C)] 95 96 struct mtx_pool *pmc_mtxpool; 97 static int *pmc_pmcdisp; /* PMC row dispositions */ 98 99 #define PMC_ROW_DISP_IS_FREE(R) (pmc_pmcdisp[(R)] == 0) 100 #define PMC_ROW_DISP_IS_THREAD(R) (pmc_pmcdisp[(R)] > 0) 101 #define PMC_ROW_DISP_IS_STANDALONE(R) (pmc_pmcdisp[(R)] < 0) 102 103 #define PMC_MARK_ROW_FREE(R) do { \ 104 pmc_pmcdisp[(R)] = 0; \ 105 } while (0) 106 107 #define PMC_MARK_ROW_STANDALONE(R) do { \ 108 KASSERT(pmc_pmcdisp[(R)] <= 0, ("[pmc,%d] row disposition error", \ 109 __LINE__)); \ 110 atomic_add_int(&pmc_pmcdisp[(R)], -1); \ 111 KASSERT(pmc_pmcdisp[(R)] >= (-pmc_cpu_max_active()), \ 112 ("[pmc,%d] row disposition error", __LINE__)); \ 113 } while (0) 114 115 #define PMC_UNMARK_ROW_STANDALONE(R) do { \ 116 atomic_add_int(&pmc_pmcdisp[(R)], 1); \ 117 KASSERT(pmc_pmcdisp[(R)] <= 0, ("[pmc,%d] row disposition error", \ 118 __LINE__)); \ 119 } while (0) 120 121 #define PMC_MARK_ROW_THREAD(R) do { \ 122 KASSERT(pmc_pmcdisp[(R)] >= 0, ("[pmc,%d] row disposition error", \ 123 __LINE__)); \ 124 atomic_add_int(&pmc_pmcdisp[(R)], 1); \ 125 } while (0) 126 127 #define PMC_UNMARK_ROW_THREAD(R) do { \ 128 atomic_add_int(&pmc_pmcdisp[(R)], -1); \ 129 KASSERT(pmc_pmcdisp[(R)] >= 0, ("[pmc,%d] row disposition error", \ 130 __LINE__)); \ 131 } while (0) 132 133 134 /* various event handlers */ 135 static eventhandler_tag pmc_exit_tag, pmc_fork_tag, pmc_kld_load_tag, 136 pmc_kld_unload_tag; 137 138 /* Module statistics */ 139 struct pmc_op_getdriverstats pmc_stats; 140 141 /* Machine/processor dependent operations */ 142 static struct pmc_mdep *md; 143 144 /* 145 * Hash tables mapping owner processes and target threads to PMCs. 146 */ 147 148 struct mtx pmc_processhash_mtx; /* spin mutex */ 149 static u_long pmc_processhashmask; 150 static LIST_HEAD(pmc_processhash, pmc_process) *pmc_processhash; 151 152 /* 153 * Hash table of PMC owner descriptors. This table is protected by 154 * the shared PMC "sx" lock. 155 */ 156 157 static u_long pmc_ownerhashmask; 158 static LIST_HEAD(pmc_ownerhash, pmc_owner) *pmc_ownerhash; 159 160 /* 161 * List of PMC owners with system-wide sampling PMCs. 162 */ 163 164 static LIST_HEAD(, pmc_owner) pmc_ss_owners; 165 166 167 /* 168 * A map of row indices to classdep structures. 169 */ 170 static struct pmc_classdep **pmc_rowindex_to_classdep; 171 172 /* 173 * Prototypes 174 */ 175 176 #ifdef DEBUG 177 static int pmc_debugflags_sysctl_handler(SYSCTL_HANDLER_ARGS); 178 static int pmc_debugflags_parse(char *newstr, char *fence); 179 #endif 180 181 static int load(struct module *module, int cmd, void *arg); 182 static int pmc_attach_process(struct proc *p, struct pmc *pm); 183 static struct pmc *pmc_allocate_pmc_descriptor(void); 184 static struct pmc_owner *pmc_allocate_owner_descriptor(struct proc *p); 185 static int pmc_attach_one_process(struct proc *p, struct pmc *pm); 186 static int pmc_can_allocate_rowindex(struct proc *p, unsigned int ri, 187 int cpu); 188 static int pmc_can_attach(struct pmc *pm, struct proc *p); 189 static void pmc_capture_user_callchain(int cpu, int soft, struct trapframe *tf); 190 static void pmc_cleanup(void); 191 static int pmc_detach_process(struct proc *p, struct pmc *pm); 192 static int pmc_detach_one_process(struct proc *p, struct pmc *pm, 193 int flags); 194 static void pmc_destroy_owner_descriptor(struct pmc_owner *po); 195 static void pmc_destroy_pmc_descriptor(struct pmc *pm); 196 static struct pmc_owner *pmc_find_owner_descriptor(struct proc *p); 197 static int pmc_find_pmc(pmc_id_t pmcid, struct pmc **pm); 198 static struct pmc *pmc_find_pmc_descriptor_in_process(struct pmc_owner *po, 199 pmc_id_t pmc); 200 static struct pmc_process *pmc_find_process_descriptor(struct proc *p, 201 uint32_t mode); 202 static void pmc_force_context_switch(void); 203 static void pmc_link_target_process(struct pmc *pm, 204 struct pmc_process *pp); 205 static void pmc_log_all_process_mappings(struct pmc_owner *po); 206 static void pmc_log_kernel_mappings(struct pmc *pm); 207 static void pmc_log_process_mappings(struct pmc_owner *po, struct proc *p); 208 static void pmc_maybe_remove_owner(struct pmc_owner *po); 209 static void pmc_process_csw_in(struct thread *td); 210 static void pmc_process_csw_out(struct thread *td); 211 static void pmc_process_exit(void *arg, struct proc *p); 212 static void pmc_process_fork(void *arg, struct proc *p1, 213 struct proc *p2, int n); 214 static void pmc_process_samples(int cpu, int soft); 215 static void pmc_release_pmc_descriptor(struct pmc *pmc); 216 static void pmc_remove_owner(struct pmc_owner *po); 217 static void pmc_remove_process_descriptor(struct pmc_process *pp); 218 static void pmc_restore_cpu_binding(struct pmc_binding *pb); 219 static void pmc_save_cpu_binding(struct pmc_binding *pb); 220 static void pmc_select_cpu(int cpu); 221 static int pmc_start(struct pmc *pm); 222 static int pmc_stop(struct pmc *pm); 223 static int pmc_syscall_handler(struct thread *td, void *syscall_args); 224 static void pmc_unlink_target_process(struct pmc *pmc, 225 struct pmc_process *pp); 226 static int generic_switch_in(struct pmc_cpu *pc, struct pmc_process *pp); 227 static int generic_switch_out(struct pmc_cpu *pc, struct pmc_process *pp); 228 static struct pmc_mdep *pmc_generic_cpu_initialize(void); 229 static void pmc_generic_cpu_finalize(struct pmc_mdep *md); 230 231 /* 232 * Kernel tunables and sysctl(8) interface. 233 */ 234 235 SYSCTL_DECL(_kern_hwpmc); 236 237 static int pmc_callchaindepth = PMC_CALLCHAIN_DEPTH; 238 SYSCTL_INT(_kern_hwpmc, OID_AUTO, callchaindepth, CTLFLAG_RDTUN, 239 &pmc_callchaindepth, 0, "depth of call chain records"); 240 241 #ifdef DEBUG 242 struct pmc_debugflags pmc_debugflags = PMC_DEBUG_DEFAULT_FLAGS; 243 char pmc_debugstr[PMC_DEBUG_STRSIZE]; 244 TUNABLE_STR(PMC_SYSCTL_NAME_PREFIX "debugflags", pmc_debugstr, 245 sizeof(pmc_debugstr)); 246 SYSCTL_PROC(_kern_hwpmc, OID_AUTO, debugflags, 247 CTLTYPE_STRING | CTLFLAG_RWTUN | CTLFLAG_NOFETCH, 248 0, 0, pmc_debugflags_sysctl_handler, "A", "debug flags"); 249 #endif 250 251 /* 252 * kern.hwpmc.hashrows -- determines the number of rows in the 253 * of the hash table used to look up threads 254 */ 255 256 static int pmc_hashsize = PMC_HASH_SIZE; 257 SYSCTL_INT(_kern_hwpmc, OID_AUTO, hashsize, CTLFLAG_RDTUN, 258 &pmc_hashsize, 0, "rows in hash tables"); 259 260 /* 261 * kern.hwpmc.nsamples --- number of PC samples/callchain stacks per CPU 262 */ 263 264 static int pmc_nsamples = PMC_NSAMPLES; 265 SYSCTL_INT(_kern_hwpmc, OID_AUTO, nsamples, CTLFLAG_RDTUN, 266 &pmc_nsamples, 0, "number of PC samples per CPU"); 267 268 269 /* 270 * kern.hwpmc.mtxpoolsize -- number of mutexes in the mutex pool. 271 */ 272 273 static int pmc_mtxpool_size = PMC_MTXPOOL_SIZE; 274 SYSCTL_INT(_kern_hwpmc, OID_AUTO, mtxpoolsize, CTLFLAG_RDTUN, 275 &pmc_mtxpool_size, 0, "size of spin mutex pool"); 276 277 278 /* 279 * security.bsd.unprivileged_syspmcs -- allow non-root processes to 280 * allocate system-wide PMCs. 281 * 282 * Allowing unprivileged processes to allocate system PMCs is convenient 283 * if system-wide measurements need to be taken concurrently with other 284 * per-process measurements. This feature is turned off by default. 285 */ 286 287 static int pmc_unprivileged_syspmcs = 0; 288 SYSCTL_INT(_security_bsd, OID_AUTO, unprivileged_syspmcs, CTLFLAG_RWTUN, 289 &pmc_unprivileged_syspmcs, 0, 290 "allow unprivileged process to allocate system PMCs"); 291 292 /* 293 * Hash function. Discard the lower 2 bits of the pointer since 294 * these are always zero for our uses. The hash multiplier is 295 * round((2^LONG_BIT) * ((sqrt(5)-1)/2)). 296 */ 297 298 #if LONG_BIT == 64 299 #define _PMC_HM 11400714819323198486u 300 #elif LONG_BIT == 32 301 #define _PMC_HM 2654435769u 302 #else 303 #error Must know the size of 'long' to compile 304 #endif 305 306 #define PMC_HASH_PTR(P,M) ((((unsigned long) (P) >> 2) * _PMC_HM) & (M)) 307 308 /* 309 * Syscall structures 310 */ 311 312 /* The `sysent' for the new syscall */ 313 static struct sysent pmc_sysent = { 314 2, /* sy_narg */ 315 pmc_syscall_handler /* sy_call */ 316 }; 317 318 static struct syscall_module_data pmc_syscall_mod = { 319 load, 320 NULL, 321 &pmc_syscall_num, 322 &pmc_sysent, 323 { 0, NULL }, 324 SY_THR_STATIC_KLD, 325 }; 326 327 static moduledata_t pmc_mod = { 328 PMC_MODULE_NAME, 329 syscall_module_handler, 330 &pmc_syscall_mod 331 }; 332 333 DECLARE_MODULE(pmc, pmc_mod, SI_SUB_SMP, SI_ORDER_ANY); 334 MODULE_VERSION(pmc, PMC_VERSION); 335 336 #ifdef DEBUG 337 enum pmc_dbgparse_state { 338 PMCDS_WS, /* in whitespace */ 339 PMCDS_MAJOR, /* seen a major keyword */ 340 PMCDS_MINOR 341 }; 342 343 static int 344 pmc_debugflags_parse(char *newstr, char *fence) 345 { 346 char c, *p, *q; 347 struct pmc_debugflags *tmpflags; 348 int error, found, *newbits, tmp; 349 size_t kwlen; 350 351 tmpflags = malloc(sizeof(*tmpflags), M_PMC, M_WAITOK|M_ZERO); 352 353 p = newstr; 354 error = 0; 355 356 for (; p < fence && (c = *p); p++) { 357 358 /* skip white space */ 359 if (c == ' ' || c == '\t') 360 continue; 361 362 /* look for a keyword followed by "=" */ 363 for (q = p; p < fence && (c = *p) && c != '='; p++) 364 ; 365 if (c != '=') { 366 error = EINVAL; 367 goto done; 368 } 369 370 kwlen = p - q; 371 newbits = NULL; 372 373 /* lookup flag group name */ 374 #define DBG_SET_FLAG_MAJ(S,F) \ 375 if (kwlen == sizeof(S)-1 && strncmp(q, S, kwlen) == 0) \ 376 newbits = &tmpflags->pdb_ ## F; 377 378 DBG_SET_FLAG_MAJ("cpu", CPU); 379 DBG_SET_FLAG_MAJ("csw", CSW); 380 DBG_SET_FLAG_MAJ("logging", LOG); 381 DBG_SET_FLAG_MAJ("module", MOD); 382 DBG_SET_FLAG_MAJ("md", MDP); 383 DBG_SET_FLAG_MAJ("owner", OWN); 384 DBG_SET_FLAG_MAJ("pmc", PMC); 385 DBG_SET_FLAG_MAJ("process", PRC); 386 DBG_SET_FLAG_MAJ("sampling", SAM); 387 388 if (newbits == NULL) { 389 error = EINVAL; 390 goto done; 391 } 392 393 p++; /* skip the '=' */ 394 395 /* Now parse the individual flags */ 396 tmp = 0; 397 newflag: 398 for (q = p; p < fence && (c = *p); p++) 399 if (c == ' ' || c == '\t' || c == ',') 400 break; 401 402 /* p == fence or c == ws or c == "," or c == 0 */ 403 404 if ((kwlen = p - q) == 0) { 405 *newbits = tmp; 406 continue; 407 } 408 409 found = 0; 410 #define DBG_SET_FLAG_MIN(S,F) \ 411 if (kwlen == sizeof(S)-1 && strncmp(q, S, kwlen) == 0) \ 412 tmp |= found = (1 << PMC_DEBUG_MIN_ ## F) 413 414 /* a '*' denotes all possible flags in the group */ 415 if (kwlen == 1 && *q == '*') 416 tmp = found = ~0; 417 /* look for individual flag names */ 418 DBG_SET_FLAG_MIN("allocaterow", ALR); 419 DBG_SET_FLAG_MIN("allocate", ALL); 420 DBG_SET_FLAG_MIN("attach", ATT); 421 DBG_SET_FLAG_MIN("bind", BND); 422 DBG_SET_FLAG_MIN("config", CFG); 423 DBG_SET_FLAG_MIN("exec", EXC); 424 DBG_SET_FLAG_MIN("exit", EXT); 425 DBG_SET_FLAG_MIN("find", FND); 426 DBG_SET_FLAG_MIN("flush", FLS); 427 DBG_SET_FLAG_MIN("fork", FRK); 428 DBG_SET_FLAG_MIN("getbuf", GTB); 429 DBG_SET_FLAG_MIN("hook", PMH); 430 DBG_SET_FLAG_MIN("init", INI); 431 DBG_SET_FLAG_MIN("intr", INT); 432 DBG_SET_FLAG_MIN("linktarget", TLK); 433 DBG_SET_FLAG_MIN("mayberemove", OMR); 434 DBG_SET_FLAG_MIN("ops", OPS); 435 DBG_SET_FLAG_MIN("read", REA); 436 DBG_SET_FLAG_MIN("register", REG); 437 DBG_SET_FLAG_MIN("release", REL); 438 DBG_SET_FLAG_MIN("remove", ORM); 439 DBG_SET_FLAG_MIN("sample", SAM); 440 DBG_SET_FLAG_MIN("scheduleio", SIO); 441 DBG_SET_FLAG_MIN("select", SEL); 442 DBG_SET_FLAG_MIN("signal", SIG); 443 DBG_SET_FLAG_MIN("swi", SWI); 444 DBG_SET_FLAG_MIN("swo", SWO); 445 DBG_SET_FLAG_MIN("start", STA); 446 DBG_SET_FLAG_MIN("stop", STO); 447 DBG_SET_FLAG_MIN("syscall", PMS); 448 DBG_SET_FLAG_MIN("unlinktarget", TUL); 449 DBG_SET_FLAG_MIN("write", WRI); 450 if (found == 0) { 451 /* unrecognized flag name */ 452 error = EINVAL; 453 goto done; 454 } 455 456 if (c == 0 || c == ' ' || c == '\t') { /* end of flag group */ 457 *newbits = tmp; 458 continue; 459 } 460 461 p++; 462 goto newflag; 463 } 464 465 /* save the new flag set */ 466 bcopy(tmpflags, &pmc_debugflags, sizeof(pmc_debugflags)); 467 468 done: 469 free(tmpflags, M_PMC); 470 return error; 471 } 472 473 static int 474 pmc_debugflags_sysctl_handler(SYSCTL_HANDLER_ARGS) 475 { 476 char *fence, *newstr; 477 int error; 478 unsigned int n; 479 480 (void) arg1; (void) arg2; /* unused parameters */ 481 482 n = sizeof(pmc_debugstr); 483 newstr = malloc(n, M_PMC, M_WAITOK|M_ZERO); 484 (void) strlcpy(newstr, pmc_debugstr, n); 485 486 error = sysctl_handle_string(oidp, newstr, n, req); 487 488 /* if there is a new string, parse and copy it */ 489 if (error == 0 && req->newptr != NULL) { 490 fence = newstr + (n < req->newlen ? n : req->newlen + 1); 491 if ((error = pmc_debugflags_parse(newstr, fence)) == 0) 492 (void) strlcpy(pmc_debugstr, newstr, 493 sizeof(pmc_debugstr)); 494 } 495 496 free(newstr, M_PMC); 497 498 return error; 499 } 500 #endif 501 502 /* 503 * Map a row index to a classdep structure and return the adjusted row 504 * index for the PMC class index. 505 */ 506 static struct pmc_classdep * 507 pmc_ri_to_classdep(struct pmc_mdep *md, int ri, int *adjri) 508 { 509 struct pmc_classdep *pcd; 510 511 (void) md; 512 513 KASSERT(ri >= 0 && ri < md->pmd_npmc, 514 ("[pmc,%d] illegal row-index %d", __LINE__, ri)); 515 516 pcd = pmc_rowindex_to_classdep[ri]; 517 518 KASSERT(pcd != NULL, 519 ("[pmc,%d] ri %d null pcd", __LINE__, ri)); 520 521 *adjri = ri - pcd->pcd_ri; 522 523 KASSERT(*adjri >= 0 && *adjri < pcd->pcd_num, 524 ("[pmc,%d] adjusted row-index %d", __LINE__, *adjri)); 525 526 return (pcd); 527 } 528 529 /* 530 * Concurrency Control 531 * 532 * The driver manages the following data structures: 533 * 534 * - target process descriptors, one per target process 535 * - owner process descriptors (and attached lists), one per owner process 536 * - lookup hash tables for owner and target processes 537 * - PMC descriptors (and attached lists) 538 * - per-cpu hardware state 539 * - the 'hook' variable through which the kernel calls into 540 * this module 541 * - the machine hardware state (managed by the MD layer) 542 * 543 * These data structures are accessed from: 544 * 545 * - thread context-switch code 546 * - interrupt handlers (possibly on multiple cpus) 547 * - kernel threads on multiple cpus running on behalf of user 548 * processes doing system calls 549 * - this driver's private kernel threads 550 * 551 * = Locks and Locking strategy = 552 * 553 * The driver uses four locking strategies for its operation: 554 * 555 * - The global SX lock "pmc_sx" is used to protect internal 556 * data structures. 557 * 558 * Calls into the module by syscall() start with this lock being 559 * held in exclusive mode. Depending on the requested operation, 560 * the lock may be downgraded to 'shared' mode to allow more 561 * concurrent readers into the module. Calls into the module from 562 * other parts of the kernel acquire the lock in shared mode. 563 * 564 * This SX lock is held in exclusive mode for any operations that 565 * modify the linkages between the driver's internal data structures. 566 * 567 * The 'pmc_hook' function pointer is also protected by this lock. 568 * It is only examined with the sx lock held in exclusive mode. The 569 * kernel module is allowed to be unloaded only with the sx lock held 570 * in exclusive mode. In normal syscall handling, after acquiring the 571 * pmc_sx lock we first check that 'pmc_hook' is non-null before 572 * proceeding. This prevents races between the thread unloading the module 573 * and other threads seeking to use the module. 574 * 575 * - Lookups of target process structures and owner process structures 576 * cannot use the global "pmc_sx" SX lock because these lookups need 577 * to happen during context switches and in other critical sections 578 * where sleeping is not allowed. We protect these lookup tables 579 * with their own private spin-mutexes, "pmc_processhash_mtx" and 580 * "pmc_ownerhash_mtx". 581 * 582 * - Interrupt handlers work in a lock free manner. At interrupt 583 * time, handlers look at the PMC pointer (phw->phw_pmc) configured 584 * when the PMC was started. If this pointer is NULL, the interrupt 585 * is ignored after updating driver statistics. We ensure that this 586 * pointer is set (using an atomic operation if necessary) before the 587 * PMC hardware is started. Conversely, this pointer is unset atomically 588 * only after the PMC hardware is stopped. 589 * 590 * We ensure that everything needed for the operation of an 591 * interrupt handler is available without it needing to acquire any 592 * locks. We also ensure that a PMC's software state is destroyed only 593 * after the PMC is taken off hardware (on all CPUs). 594 * 595 * - Context-switch handling with process-private PMCs needs more 596 * care. 597 * 598 * A given process may be the target of multiple PMCs. For example, 599 * PMCATTACH and PMCDETACH may be requested by a process on one CPU 600 * while the target process is running on another. A PMC could also 601 * be getting released because its owner is exiting. We tackle 602 * these situations in the following manner: 603 * 604 * - each target process structure 'pmc_process' has an array 605 * of 'struct pmc *' pointers, one for each hardware PMC. 606 * 607 * - At context switch IN time, each "target" PMC in RUNNING state 608 * gets started on hardware and a pointer to each PMC is copied into 609 * the per-cpu phw array. The 'runcount' for the PMC is 610 * incremented. 611 * 612 * - At context switch OUT time, all process-virtual PMCs are stopped 613 * on hardware. The saved value is added to the PMCs value field 614 * only if the PMC is in a non-deleted state (the PMCs state could 615 * have changed during the current time slice). 616 * 617 * Note that since in-between a switch IN on a processor and a switch 618 * OUT, the PMC could have been released on another CPU. Therefore 619 * context switch OUT always looks at the hardware state to turn 620 * OFF PMCs and will update a PMC's saved value only if reachable 621 * from the target process record. 622 * 623 * - OP PMCRELEASE could be called on a PMC at any time (the PMC could 624 * be attached to many processes at the time of the call and could 625 * be active on multiple CPUs). 626 * 627 * We prevent further scheduling of the PMC by marking it as in 628 * state 'DELETED'. If the runcount of the PMC is non-zero then 629 * this PMC is currently running on a CPU somewhere. The thread 630 * doing the PMCRELEASE operation waits by repeatedly doing a 631 * pause() till the runcount comes to zero. 632 * 633 * The contents of a PMC descriptor (struct pmc) are protected using 634 * a spin-mutex. In order to save space, we use a mutex pool. 635 * 636 * In terms of lock types used by witness(4), we use: 637 * - Type "pmc-sx", used by the global SX lock. 638 * - Type "pmc-sleep", for sleep mutexes used by logger threads. 639 * - Type "pmc-per-proc", for protecting PMC owner descriptors. 640 * - Type "pmc-leaf", used for all other spin mutexes. 641 */ 642 643 /* 644 * save the cpu binding of the current kthread 645 */ 646 647 static void 648 pmc_save_cpu_binding(struct pmc_binding *pb) 649 { 650 PMCDBG(CPU,BND,2, "%s", "save-cpu"); 651 thread_lock(curthread); 652 pb->pb_bound = sched_is_bound(curthread); 653 pb->pb_cpu = curthread->td_oncpu; 654 thread_unlock(curthread); 655 PMCDBG(CPU,BND,2, "save-cpu cpu=%d", pb->pb_cpu); 656 } 657 658 /* 659 * restore the cpu binding of the current thread 660 */ 661 662 static void 663 pmc_restore_cpu_binding(struct pmc_binding *pb) 664 { 665 PMCDBG(CPU,BND,2, "restore-cpu curcpu=%d restore=%d", 666 curthread->td_oncpu, pb->pb_cpu); 667 thread_lock(curthread); 668 if (pb->pb_bound) 669 sched_bind(curthread, pb->pb_cpu); 670 else 671 sched_unbind(curthread); 672 thread_unlock(curthread); 673 PMCDBG(CPU,BND,2, "%s", "restore-cpu done"); 674 } 675 676 /* 677 * move execution over the specified cpu and bind it there. 678 */ 679 680 static void 681 pmc_select_cpu(int cpu) 682 { 683 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 684 ("[pmc,%d] bad cpu number %d", __LINE__, cpu)); 685 686 /* Never move to an inactive CPU. */ 687 KASSERT(pmc_cpu_is_active(cpu), ("[pmc,%d] selecting inactive " 688 "CPU %d", __LINE__, cpu)); 689 690 PMCDBG(CPU,SEL,2, "select-cpu cpu=%d", cpu); 691 thread_lock(curthread); 692 sched_bind(curthread, cpu); 693 thread_unlock(curthread); 694 695 KASSERT(curthread->td_oncpu == cpu, 696 ("[pmc,%d] CPU not bound [cpu=%d, curr=%d]", __LINE__, 697 cpu, curthread->td_oncpu)); 698 699 PMCDBG(CPU,SEL,2, "select-cpu cpu=%d ok", cpu); 700 } 701 702 /* 703 * Force a context switch. 704 * 705 * We do this by pause'ing for 1 tick -- invoking mi_switch() is not 706 * guaranteed to force a context switch. 707 */ 708 709 static void 710 pmc_force_context_switch(void) 711 { 712 713 pause("pmcctx", 1); 714 } 715 716 /* 717 * Get the file name for an executable. This is a simple wrapper 718 * around vn_fullpath(9). 719 */ 720 721 static void 722 pmc_getfilename(struct vnode *v, char **fullpath, char **freepath) 723 { 724 725 *fullpath = "unknown"; 726 *freepath = NULL; 727 vn_fullpath(curthread, v, fullpath, freepath); 728 } 729 730 /* 731 * remove an process owning PMCs 732 */ 733 734 void 735 pmc_remove_owner(struct pmc_owner *po) 736 { 737 struct pmc *pm, *tmp; 738 739 sx_assert(&pmc_sx, SX_XLOCKED); 740 741 PMCDBG(OWN,ORM,1, "remove-owner po=%p", po); 742 743 /* Remove descriptor from the owner hash table */ 744 LIST_REMOVE(po, po_next); 745 746 /* release all owned PMC descriptors */ 747 LIST_FOREACH_SAFE(pm, &po->po_pmcs, pm_next, tmp) { 748 PMCDBG(OWN,ORM,2, "pmc=%p", pm); 749 KASSERT(pm->pm_owner == po, 750 ("[pmc,%d] owner %p != po %p", __LINE__, pm->pm_owner, po)); 751 752 pmc_release_pmc_descriptor(pm); /* will unlink from the list */ 753 pmc_destroy_pmc_descriptor(pm); 754 } 755 756 KASSERT(po->po_sscount == 0, 757 ("[pmc,%d] SS count not zero", __LINE__)); 758 KASSERT(LIST_EMPTY(&po->po_pmcs), 759 ("[pmc,%d] PMC list not empty", __LINE__)); 760 761 /* de-configure the log file if present */ 762 if (po->po_flags & PMC_PO_OWNS_LOGFILE) 763 pmclog_deconfigure_log(po); 764 } 765 766 /* 767 * remove an owner process record if all conditions are met. 768 */ 769 770 static void 771 pmc_maybe_remove_owner(struct pmc_owner *po) 772 { 773 774 PMCDBG(OWN,OMR,1, "maybe-remove-owner po=%p", po); 775 776 /* 777 * Remove owner record if 778 * - this process does not own any PMCs 779 * - this process has not allocated a system-wide sampling buffer 780 */ 781 782 if (LIST_EMPTY(&po->po_pmcs) && 783 ((po->po_flags & PMC_PO_OWNS_LOGFILE) == 0)) { 784 pmc_remove_owner(po); 785 pmc_destroy_owner_descriptor(po); 786 } 787 } 788 789 /* 790 * Add an association between a target process and a PMC. 791 */ 792 793 static void 794 pmc_link_target_process(struct pmc *pm, struct pmc_process *pp) 795 { 796 int ri; 797 struct pmc_target *pt; 798 799 sx_assert(&pmc_sx, SX_XLOCKED); 800 801 KASSERT(pm != NULL && pp != NULL, 802 ("[pmc,%d] Null pm %p or pp %p", __LINE__, pm, pp)); 803 KASSERT(PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)), 804 ("[pmc,%d] Attaching a non-process-virtual pmc=%p to pid=%d", 805 __LINE__, pm, pp->pp_proc->p_pid)); 806 KASSERT(pp->pp_refcnt >= 0 && pp->pp_refcnt <= ((int) md->pmd_npmc - 1), 807 ("[pmc,%d] Illegal reference count %d for process record %p", 808 __LINE__, pp->pp_refcnt, (void *) pp)); 809 810 ri = PMC_TO_ROWINDEX(pm); 811 812 PMCDBG(PRC,TLK,1, "link-target pmc=%p ri=%d pmc-process=%p", 813 pm, ri, pp); 814 815 #ifdef DEBUG 816 LIST_FOREACH(pt, &pm->pm_targets, pt_next) 817 if (pt->pt_process == pp) 818 KASSERT(0, ("[pmc,%d] pp %p already in pmc %p targets", 819 __LINE__, pp, pm)); 820 #endif 821 822 pt = malloc(sizeof(struct pmc_target), M_PMC, M_WAITOK|M_ZERO); 823 pt->pt_process = pp; 824 825 LIST_INSERT_HEAD(&pm->pm_targets, pt, pt_next); 826 827 atomic_store_rel_ptr((uintptr_t *)&pp->pp_pmcs[ri].pp_pmc, 828 (uintptr_t)pm); 829 830 if (pm->pm_owner->po_owner == pp->pp_proc) 831 pm->pm_flags |= PMC_F_ATTACHED_TO_OWNER; 832 833 /* 834 * Initialize the per-process values at this row index. 835 */ 836 pp->pp_pmcs[ri].pp_pmcval = PMC_TO_MODE(pm) == PMC_MODE_TS ? 837 pm->pm_sc.pm_reloadcount : 0; 838 839 pp->pp_refcnt++; 840 841 } 842 843 /* 844 * Removes the association between a target process and a PMC. 845 */ 846 847 static void 848 pmc_unlink_target_process(struct pmc *pm, struct pmc_process *pp) 849 { 850 int ri; 851 struct proc *p; 852 struct pmc_target *ptgt; 853 854 sx_assert(&pmc_sx, SX_XLOCKED); 855 856 KASSERT(pm != NULL && pp != NULL, 857 ("[pmc,%d] Null pm %p or pp %p", __LINE__, pm, pp)); 858 859 KASSERT(pp->pp_refcnt >= 1 && pp->pp_refcnt <= (int) md->pmd_npmc, 860 ("[pmc,%d] Illegal ref count %d on process record %p", 861 __LINE__, pp->pp_refcnt, (void *) pp)); 862 863 ri = PMC_TO_ROWINDEX(pm); 864 865 PMCDBG(PRC,TUL,1, "unlink-target pmc=%p ri=%d pmc-process=%p", 866 pm, ri, pp); 867 868 KASSERT(pp->pp_pmcs[ri].pp_pmc == pm, 869 ("[pmc,%d] PMC ri %d mismatch pmc %p pp->[ri] %p", __LINE__, 870 ri, pm, pp->pp_pmcs[ri].pp_pmc)); 871 872 pp->pp_pmcs[ri].pp_pmc = NULL; 873 pp->pp_pmcs[ri].pp_pmcval = (pmc_value_t) 0; 874 875 /* Remove owner-specific flags */ 876 if (pm->pm_owner->po_owner == pp->pp_proc) { 877 pp->pp_flags &= ~PMC_PP_ENABLE_MSR_ACCESS; 878 pm->pm_flags &= ~PMC_F_ATTACHED_TO_OWNER; 879 } 880 881 pp->pp_refcnt--; 882 883 /* Remove the target process from the PMC structure */ 884 LIST_FOREACH(ptgt, &pm->pm_targets, pt_next) 885 if (ptgt->pt_process == pp) 886 break; 887 888 KASSERT(ptgt != NULL, ("[pmc,%d] process %p (pp: %p) not found " 889 "in pmc %p", __LINE__, pp->pp_proc, pp, pm)); 890 891 LIST_REMOVE(ptgt, pt_next); 892 free(ptgt, M_PMC); 893 894 /* if the PMC now lacks targets, send the owner a SIGIO */ 895 if (LIST_EMPTY(&pm->pm_targets)) { 896 p = pm->pm_owner->po_owner; 897 PROC_LOCK(p); 898 kern_psignal(p, SIGIO); 899 PROC_UNLOCK(p); 900 901 PMCDBG(PRC,SIG,2, "signalling proc=%p signal=%d", p, 902 SIGIO); 903 } 904 } 905 906 /* 907 * Check if PMC 'pm' may be attached to target process 't'. 908 */ 909 910 static int 911 pmc_can_attach(struct pmc *pm, struct proc *t) 912 { 913 struct proc *o; /* pmc owner */ 914 struct ucred *oc, *tc; /* owner, target credentials */ 915 int decline_attach, i; 916 917 /* 918 * A PMC's owner can always attach that PMC to itself. 919 */ 920 921 if ((o = pm->pm_owner->po_owner) == t) 922 return 0; 923 924 PROC_LOCK(o); 925 oc = o->p_ucred; 926 crhold(oc); 927 PROC_UNLOCK(o); 928 929 PROC_LOCK(t); 930 tc = t->p_ucred; 931 crhold(tc); 932 PROC_UNLOCK(t); 933 934 /* 935 * The effective uid of the PMC owner should match at least one 936 * of the {effective,real,saved} uids of the target process. 937 */ 938 939 decline_attach = oc->cr_uid != tc->cr_uid && 940 oc->cr_uid != tc->cr_svuid && 941 oc->cr_uid != tc->cr_ruid; 942 943 /* 944 * Every one of the target's group ids, must be in the owner's 945 * group list. 946 */ 947 for (i = 0; !decline_attach && i < tc->cr_ngroups; i++) 948 decline_attach = !groupmember(tc->cr_groups[i], oc); 949 950 /* check the read and saved gids too */ 951 if (decline_attach == 0) 952 decline_attach = !groupmember(tc->cr_rgid, oc) || 953 !groupmember(tc->cr_svgid, oc); 954 955 crfree(tc); 956 crfree(oc); 957 958 return !decline_attach; 959 } 960 961 /* 962 * Attach a process to a PMC. 963 */ 964 965 static int 966 pmc_attach_one_process(struct proc *p, struct pmc *pm) 967 { 968 int ri; 969 char *fullpath, *freepath; 970 struct pmc_process *pp; 971 972 sx_assert(&pmc_sx, SX_XLOCKED); 973 974 PMCDBG(PRC,ATT,2, "attach-one pm=%p ri=%d proc=%p (%d, %s)", pm, 975 PMC_TO_ROWINDEX(pm), p, p->p_pid, p->p_comm); 976 977 /* 978 * Locate the process descriptor corresponding to process 'p', 979 * allocating space as needed. 980 * 981 * Verify that rowindex 'pm_rowindex' is free in the process 982 * descriptor. 983 * 984 * If not, allocate space for a descriptor and link the 985 * process descriptor and PMC. 986 */ 987 ri = PMC_TO_ROWINDEX(pm); 988 989 if ((pp = pmc_find_process_descriptor(p, PMC_FLAG_ALLOCATE)) == NULL) 990 return ENOMEM; 991 992 if (pp->pp_pmcs[ri].pp_pmc == pm) /* already present at slot [ri] */ 993 return EEXIST; 994 995 if (pp->pp_pmcs[ri].pp_pmc != NULL) 996 return EBUSY; 997 998 pmc_link_target_process(pm, pp); 999 1000 if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)) && 1001 (pm->pm_flags & PMC_F_ATTACHED_TO_OWNER) == 0) 1002 pm->pm_flags |= PMC_F_NEEDS_LOGFILE; 1003 1004 pm->pm_flags |= PMC_F_ATTACH_DONE; /* mark as attached */ 1005 1006 /* issue an attach event to a configured log file */ 1007 if (pm->pm_owner->po_flags & PMC_PO_OWNS_LOGFILE) { 1008 pmc_getfilename(p->p_textvp, &fullpath, &freepath); 1009 if (p->p_flag & P_KTHREAD) { 1010 fullpath = kernelname; 1011 freepath = NULL; 1012 } else 1013 pmclog_process_pmcattach(pm, p->p_pid, fullpath); 1014 if (freepath) 1015 free(freepath, M_TEMP); 1016 if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) 1017 pmc_log_process_mappings(pm->pm_owner, p); 1018 } 1019 /* mark process as using HWPMCs */ 1020 PROC_LOCK(p); 1021 p->p_flag |= P_HWPMC; 1022 PROC_UNLOCK(p); 1023 1024 return 0; 1025 } 1026 1027 /* 1028 * Attach a process and optionally its children 1029 */ 1030 1031 static int 1032 pmc_attach_process(struct proc *p, struct pmc *pm) 1033 { 1034 int error; 1035 struct proc *top; 1036 1037 sx_assert(&pmc_sx, SX_XLOCKED); 1038 1039 PMCDBG(PRC,ATT,1, "attach pm=%p ri=%d proc=%p (%d, %s)", pm, 1040 PMC_TO_ROWINDEX(pm), p, p->p_pid, p->p_comm); 1041 1042 1043 /* 1044 * If this PMC successfully allowed a GETMSR operation 1045 * in the past, disallow further ATTACHes. 1046 */ 1047 1048 if ((pm->pm_flags & PMC_PP_ENABLE_MSR_ACCESS) != 0) 1049 return EPERM; 1050 1051 if ((pm->pm_flags & PMC_F_DESCENDANTS) == 0) 1052 return pmc_attach_one_process(p, pm); 1053 1054 /* 1055 * Traverse all child processes, attaching them to 1056 * this PMC. 1057 */ 1058 1059 sx_slock(&proctree_lock); 1060 1061 top = p; 1062 1063 for (;;) { 1064 if ((error = pmc_attach_one_process(p, pm)) != 0) 1065 break; 1066 if (!LIST_EMPTY(&p->p_children)) 1067 p = LIST_FIRST(&p->p_children); 1068 else for (;;) { 1069 if (p == top) 1070 goto done; 1071 if (LIST_NEXT(p, p_sibling)) { 1072 p = LIST_NEXT(p, p_sibling); 1073 break; 1074 } 1075 p = p->p_pptr; 1076 } 1077 } 1078 1079 if (error) 1080 (void) pmc_detach_process(top, pm); 1081 1082 done: 1083 sx_sunlock(&proctree_lock); 1084 return error; 1085 } 1086 1087 /* 1088 * Detach a process from a PMC. If there are no other PMCs tracking 1089 * this process, remove the process structure from its hash table. If 1090 * 'flags' contains PMC_FLAG_REMOVE, then free the process structure. 1091 */ 1092 1093 static int 1094 pmc_detach_one_process(struct proc *p, struct pmc *pm, int flags) 1095 { 1096 int ri; 1097 struct pmc_process *pp; 1098 1099 sx_assert(&pmc_sx, SX_XLOCKED); 1100 1101 KASSERT(pm != NULL, 1102 ("[pmc,%d] null pm pointer", __LINE__)); 1103 1104 ri = PMC_TO_ROWINDEX(pm); 1105 1106 PMCDBG(PRC,ATT,2, "detach-one pm=%p ri=%d proc=%p (%d, %s) flags=0x%x", 1107 pm, ri, p, p->p_pid, p->p_comm, flags); 1108 1109 if ((pp = pmc_find_process_descriptor(p, 0)) == NULL) 1110 return ESRCH; 1111 1112 if (pp->pp_pmcs[ri].pp_pmc != pm) 1113 return EINVAL; 1114 1115 pmc_unlink_target_process(pm, pp); 1116 1117 /* Issue a detach entry if a log file is configured */ 1118 if (pm->pm_owner->po_flags & PMC_PO_OWNS_LOGFILE) 1119 pmclog_process_pmcdetach(pm, p->p_pid); 1120 1121 /* 1122 * If there are no PMCs targetting this process, we remove its 1123 * descriptor from the target hash table and unset the P_HWPMC 1124 * flag in the struct proc. 1125 */ 1126 KASSERT(pp->pp_refcnt >= 0 && pp->pp_refcnt <= (int) md->pmd_npmc, 1127 ("[pmc,%d] Illegal refcnt %d for process struct %p", 1128 __LINE__, pp->pp_refcnt, pp)); 1129 1130 if (pp->pp_refcnt != 0) /* still a target of some PMC */ 1131 return 0; 1132 1133 pmc_remove_process_descriptor(pp); 1134 1135 if (flags & PMC_FLAG_REMOVE) 1136 free(pp, M_PMC); 1137 1138 PROC_LOCK(p); 1139 p->p_flag &= ~P_HWPMC; 1140 PROC_UNLOCK(p); 1141 1142 return 0; 1143 } 1144 1145 /* 1146 * Detach a process and optionally its descendants from a PMC. 1147 */ 1148 1149 static int 1150 pmc_detach_process(struct proc *p, struct pmc *pm) 1151 { 1152 struct proc *top; 1153 1154 sx_assert(&pmc_sx, SX_XLOCKED); 1155 1156 PMCDBG(PRC,ATT,1, "detach pm=%p ri=%d proc=%p (%d, %s)", pm, 1157 PMC_TO_ROWINDEX(pm), p, p->p_pid, p->p_comm); 1158 1159 if ((pm->pm_flags & PMC_F_DESCENDANTS) == 0) 1160 return pmc_detach_one_process(p, pm, PMC_FLAG_REMOVE); 1161 1162 /* 1163 * Traverse all children, detaching them from this PMC. We 1164 * ignore errors since we could be detaching a PMC from a 1165 * partially attached proc tree. 1166 */ 1167 1168 sx_slock(&proctree_lock); 1169 1170 top = p; 1171 1172 for (;;) { 1173 (void) pmc_detach_one_process(p, pm, PMC_FLAG_REMOVE); 1174 1175 if (!LIST_EMPTY(&p->p_children)) 1176 p = LIST_FIRST(&p->p_children); 1177 else for (;;) { 1178 if (p == top) 1179 goto done; 1180 if (LIST_NEXT(p, p_sibling)) { 1181 p = LIST_NEXT(p, p_sibling); 1182 break; 1183 } 1184 p = p->p_pptr; 1185 } 1186 } 1187 1188 done: 1189 sx_sunlock(&proctree_lock); 1190 1191 if (LIST_EMPTY(&pm->pm_targets)) 1192 pm->pm_flags &= ~PMC_F_ATTACH_DONE; 1193 1194 return 0; 1195 } 1196 1197 1198 /* 1199 * Thread context switch IN 1200 */ 1201 1202 static void 1203 pmc_process_csw_in(struct thread *td) 1204 { 1205 int cpu; 1206 unsigned int adjri, ri; 1207 struct pmc *pm; 1208 struct proc *p; 1209 struct pmc_cpu *pc; 1210 struct pmc_hw *phw; 1211 pmc_value_t newvalue; 1212 struct pmc_process *pp; 1213 struct pmc_classdep *pcd; 1214 1215 p = td->td_proc; 1216 1217 if ((pp = pmc_find_process_descriptor(p, PMC_FLAG_NONE)) == NULL) 1218 return; 1219 1220 KASSERT(pp->pp_proc == td->td_proc, 1221 ("[pmc,%d] not my thread state", __LINE__)); 1222 1223 critical_enter(); /* no preemption from this point */ 1224 1225 cpu = PCPU_GET(cpuid); /* td->td_oncpu is invalid */ 1226 1227 PMCDBG(CSW,SWI,1, "cpu=%d proc=%p (%d, %s) pp=%p", cpu, p, 1228 p->p_pid, p->p_comm, pp); 1229 1230 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 1231 ("[pmc,%d] wierd CPU id %d", __LINE__, cpu)); 1232 1233 pc = pmc_pcpu[cpu]; 1234 1235 for (ri = 0; ri < md->pmd_npmc; ri++) { 1236 1237 if ((pm = pp->pp_pmcs[ri].pp_pmc) == NULL) 1238 continue; 1239 1240 KASSERT(PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)), 1241 ("[pmc,%d] Target PMC in non-virtual mode (%d)", 1242 __LINE__, PMC_TO_MODE(pm))); 1243 1244 KASSERT(PMC_TO_ROWINDEX(pm) == ri, 1245 ("[pmc,%d] Row index mismatch pmc %d != ri %d", 1246 __LINE__, PMC_TO_ROWINDEX(pm), ri)); 1247 1248 /* 1249 * Only PMCs that are marked as 'RUNNING' need 1250 * be placed on hardware. 1251 */ 1252 1253 if (pm->pm_state != PMC_STATE_RUNNING) 1254 continue; 1255 1256 /* increment PMC runcount */ 1257 atomic_add_rel_int(&pm->pm_runcount, 1); 1258 1259 /* configure the HWPMC we are going to use. */ 1260 pcd = pmc_ri_to_classdep(md, ri, &adjri); 1261 pcd->pcd_config_pmc(cpu, adjri, pm); 1262 1263 phw = pc->pc_hwpmcs[ri]; 1264 1265 KASSERT(phw != NULL, 1266 ("[pmc,%d] null hw pointer", __LINE__)); 1267 1268 KASSERT(phw->phw_pmc == pm, 1269 ("[pmc,%d] hw->pmc %p != pmc %p", __LINE__, 1270 phw->phw_pmc, pm)); 1271 1272 /* 1273 * Write out saved value and start the PMC. 1274 * 1275 * Sampling PMCs use a per-process value, while 1276 * counting mode PMCs use a per-pmc value that is 1277 * inherited across descendants. 1278 */ 1279 if (PMC_TO_MODE(pm) == PMC_MODE_TS) { 1280 mtx_pool_lock_spin(pmc_mtxpool, pm); 1281 newvalue = PMC_PCPU_SAVED(cpu,ri) = 1282 pp->pp_pmcs[ri].pp_pmcval; 1283 mtx_pool_unlock_spin(pmc_mtxpool, pm); 1284 } else { 1285 KASSERT(PMC_TO_MODE(pm) == PMC_MODE_TC, 1286 ("[pmc,%d] illegal mode=%d", __LINE__, 1287 PMC_TO_MODE(pm))); 1288 mtx_pool_lock_spin(pmc_mtxpool, pm); 1289 newvalue = PMC_PCPU_SAVED(cpu, ri) = 1290 pm->pm_gv.pm_savedvalue; 1291 mtx_pool_unlock_spin(pmc_mtxpool, pm); 1292 } 1293 1294 PMCDBG(CSW,SWI,1,"cpu=%d ri=%d new=%jd", cpu, ri, newvalue); 1295 1296 pcd->pcd_write_pmc(cpu, adjri, newvalue); 1297 pcd->pcd_start_pmc(cpu, adjri); 1298 } 1299 1300 /* 1301 * perform any other architecture/cpu dependent thread 1302 * switch-in actions. 1303 */ 1304 1305 (void) (*md->pmd_switch_in)(pc, pp); 1306 1307 critical_exit(); 1308 1309 } 1310 1311 /* 1312 * Thread context switch OUT. 1313 */ 1314 1315 static void 1316 pmc_process_csw_out(struct thread *td) 1317 { 1318 int cpu; 1319 int64_t tmp; 1320 struct pmc *pm; 1321 struct proc *p; 1322 enum pmc_mode mode; 1323 struct pmc_cpu *pc; 1324 pmc_value_t newvalue; 1325 unsigned int adjri, ri; 1326 struct pmc_process *pp; 1327 struct pmc_classdep *pcd; 1328 1329 1330 /* 1331 * Locate our process descriptor; this may be NULL if 1332 * this process is exiting and we have already removed 1333 * the process from the target process table. 1334 * 1335 * Note that due to kernel preemption, multiple 1336 * context switches may happen while the process is 1337 * exiting. 1338 * 1339 * Note also that if the target process cannot be 1340 * found we still need to deconfigure any PMCs that 1341 * are currently running on hardware. 1342 */ 1343 1344 p = td->td_proc; 1345 pp = pmc_find_process_descriptor(p, PMC_FLAG_NONE); 1346 1347 /* 1348 * save PMCs 1349 */ 1350 1351 critical_enter(); 1352 1353 cpu = PCPU_GET(cpuid); /* td->td_oncpu is invalid */ 1354 1355 PMCDBG(CSW,SWO,1, "cpu=%d proc=%p (%d, %s) pp=%p", cpu, p, 1356 p->p_pid, p->p_comm, pp); 1357 1358 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 1359 ("[pmc,%d wierd CPU id %d", __LINE__, cpu)); 1360 1361 pc = pmc_pcpu[cpu]; 1362 1363 /* 1364 * When a PMC gets unlinked from a target PMC, it will 1365 * be removed from the target's pp_pmc[] array. 1366 * 1367 * However, on a MP system, the target could have been 1368 * executing on another CPU at the time of the unlink. 1369 * So, at context switch OUT time, we need to look at 1370 * the hardware to determine if a PMC is scheduled on 1371 * it. 1372 */ 1373 1374 for (ri = 0; ri < md->pmd_npmc; ri++) { 1375 1376 pcd = pmc_ri_to_classdep(md, ri, &adjri); 1377 pm = NULL; 1378 (void) (*pcd->pcd_get_config)(cpu, adjri, &pm); 1379 1380 if (pm == NULL) /* nothing at this row index */ 1381 continue; 1382 1383 mode = PMC_TO_MODE(pm); 1384 if (!PMC_IS_VIRTUAL_MODE(mode)) 1385 continue; /* not a process virtual PMC */ 1386 1387 KASSERT(PMC_TO_ROWINDEX(pm) == ri, 1388 ("[pmc,%d] ri mismatch pmc(%d) ri(%d)", 1389 __LINE__, PMC_TO_ROWINDEX(pm), ri)); 1390 1391 /* Stop hardware if not already stopped */ 1392 if (pm->pm_stalled == 0) 1393 pcd->pcd_stop_pmc(cpu, adjri); 1394 1395 /* reduce this PMC's runcount */ 1396 atomic_subtract_rel_int(&pm->pm_runcount, 1); 1397 1398 /* 1399 * If this PMC is associated with this process, 1400 * save the reading. 1401 */ 1402 1403 if (pp != NULL && pp->pp_pmcs[ri].pp_pmc != NULL) { 1404 1405 KASSERT(pm == pp->pp_pmcs[ri].pp_pmc, 1406 ("[pmc,%d] pm %p != pp_pmcs[%d] %p", __LINE__, 1407 pm, ri, pp->pp_pmcs[ri].pp_pmc)); 1408 1409 KASSERT(pp->pp_refcnt > 0, 1410 ("[pmc,%d] pp refcnt = %d", __LINE__, 1411 pp->pp_refcnt)); 1412 1413 pcd->pcd_read_pmc(cpu, adjri, &newvalue); 1414 1415 tmp = newvalue - PMC_PCPU_SAVED(cpu,ri); 1416 1417 PMCDBG(CSW,SWO,1,"cpu=%d ri=%d tmp=%jd", cpu, ri, 1418 tmp); 1419 1420 if (mode == PMC_MODE_TS) { 1421 1422 /* 1423 * For sampling process-virtual PMCs, 1424 * we expect the count to be 1425 * decreasing as the 'value' 1426 * programmed into the PMC is the 1427 * number of events to be seen till 1428 * the next sampling interrupt. 1429 */ 1430 if (tmp < 0) 1431 tmp += pm->pm_sc.pm_reloadcount; 1432 mtx_pool_lock_spin(pmc_mtxpool, pm); 1433 pp->pp_pmcs[ri].pp_pmcval -= tmp; 1434 if ((int64_t) pp->pp_pmcs[ri].pp_pmcval < 0) 1435 pp->pp_pmcs[ri].pp_pmcval += 1436 pm->pm_sc.pm_reloadcount; 1437 mtx_pool_unlock_spin(pmc_mtxpool, pm); 1438 1439 } else { 1440 1441 /* 1442 * For counting process-virtual PMCs, 1443 * we expect the count to be 1444 * increasing monotonically, modulo a 64 1445 * bit wraparound. 1446 */ 1447 KASSERT((int64_t) tmp >= 0, 1448 ("[pmc,%d] negative increment cpu=%d " 1449 "ri=%d newvalue=%jx saved=%jx " 1450 "incr=%jx", __LINE__, cpu, ri, 1451 newvalue, PMC_PCPU_SAVED(cpu,ri), tmp)); 1452 1453 mtx_pool_lock_spin(pmc_mtxpool, pm); 1454 pm->pm_gv.pm_savedvalue += tmp; 1455 pp->pp_pmcs[ri].pp_pmcval += tmp; 1456 mtx_pool_unlock_spin(pmc_mtxpool, pm); 1457 1458 if (pm->pm_flags & PMC_F_LOG_PROCCSW) 1459 pmclog_process_proccsw(pm, pp, tmp); 1460 } 1461 } 1462 1463 /* mark hardware as free */ 1464 pcd->pcd_config_pmc(cpu, adjri, NULL); 1465 } 1466 1467 /* 1468 * perform any other architecture/cpu dependent thread 1469 * switch out functions. 1470 */ 1471 1472 (void) (*md->pmd_switch_out)(pc, pp); 1473 1474 critical_exit(); 1475 } 1476 1477 /* 1478 * A mapping change for a process. 1479 */ 1480 1481 static void 1482 pmc_process_mmap(struct thread *td, struct pmckern_map_in *pkm) 1483 { 1484 int ri; 1485 pid_t pid; 1486 char *fullpath, *freepath; 1487 const struct pmc *pm; 1488 struct pmc_owner *po; 1489 const struct pmc_process *pp; 1490 1491 freepath = fullpath = NULL; 1492 pmc_getfilename((struct vnode *) pkm->pm_file, &fullpath, &freepath); 1493 1494 pid = td->td_proc->p_pid; 1495 1496 /* Inform owners of all system-wide sampling PMCs. */ 1497 LIST_FOREACH(po, &pmc_ss_owners, po_ssnext) 1498 if (po->po_flags & PMC_PO_OWNS_LOGFILE) 1499 pmclog_process_map_in(po, pid, pkm->pm_address, fullpath); 1500 1501 if ((pp = pmc_find_process_descriptor(td->td_proc, 0)) == NULL) 1502 goto done; 1503 1504 /* 1505 * Inform sampling PMC owners tracking this process. 1506 */ 1507 for (ri = 0; ri < md->pmd_npmc; ri++) 1508 if ((pm = pp->pp_pmcs[ri].pp_pmc) != NULL && 1509 PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) 1510 pmclog_process_map_in(pm->pm_owner, 1511 pid, pkm->pm_address, fullpath); 1512 1513 done: 1514 if (freepath) 1515 free(freepath, M_TEMP); 1516 } 1517 1518 1519 /* 1520 * Log an munmap request. 1521 */ 1522 1523 static void 1524 pmc_process_munmap(struct thread *td, struct pmckern_map_out *pkm) 1525 { 1526 int ri; 1527 pid_t pid; 1528 struct pmc_owner *po; 1529 const struct pmc *pm; 1530 const struct pmc_process *pp; 1531 1532 pid = td->td_proc->p_pid; 1533 1534 LIST_FOREACH(po, &pmc_ss_owners, po_ssnext) 1535 if (po->po_flags & PMC_PO_OWNS_LOGFILE) 1536 pmclog_process_map_out(po, pid, pkm->pm_address, 1537 pkm->pm_address + pkm->pm_size); 1538 1539 if ((pp = pmc_find_process_descriptor(td->td_proc, 0)) == NULL) 1540 return; 1541 1542 for (ri = 0; ri < md->pmd_npmc; ri++) 1543 if ((pm = pp->pp_pmcs[ri].pp_pmc) != NULL && 1544 PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) 1545 pmclog_process_map_out(pm->pm_owner, pid, 1546 pkm->pm_address, pkm->pm_address + pkm->pm_size); 1547 } 1548 1549 /* 1550 * Log mapping information about the kernel. 1551 */ 1552 1553 static void 1554 pmc_log_kernel_mappings(struct pmc *pm) 1555 { 1556 struct pmc_owner *po; 1557 struct pmckern_map_in *km, *kmbase; 1558 1559 sx_assert(&pmc_sx, SX_LOCKED); 1560 KASSERT(PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)), 1561 ("[pmc,%d] non-sampling PMC (%p) desires mapping information", 1562 __LINE__, (void *) pm)); 1563 1564 po = pm->pm_owner; 1565 1566 if (po->po_flags & PMC_PO_INITIAL_MAPPINGS_DONE) 1567 return; 1568 1569 /* 1570 * Log the current set of kernel modules. 1571 */ 1572 kmbase = linker_hwpmc_list_objects(); 1573 for (km = kmbase; km->pm_file != NULL; km++) { 1574 PMCDBG(LOG,REG,1,"%s %p", (char *) km->pm_file, 1575 (void *) km->pm_address); 1576 pmclog_process_map_in(po, (pid_t) -1, km->pm_address, 1577 km->pm_file); 1578 } 1579 free(kmbase, M_LINKER); 1580 1581 po->po_flags |= PMC_PO_INITIAL_MAPPINGS_DONE; 1582 } 1583 1584 /* 1585 * Log the mappings for a single process. 1586 */ 1587 1588 static void 1589 pmc_log_process_mappings(struct pmc_owner *po, struct proc *p) 1590 { 1591 vm_map_t map; 1592 struct vnode *vp; 1593 struct vmspace *vm; 1594 vm_map_entry_t entry; 1595 vm_offset_t last_end; 1596 u_int last_timestamp; 1597 struct vnode *last_vp; 1598 vm_offset_t start_addr; 1599 vm_object_t obj, lobj, tobj; 1600 char *fullpath, *freepath; 1601 1602 last_vp = NULL; 1603 last_end = (vm_offset_t) 0; 1604 fullpath = freepath = NULL; 1605 1606 if ((vm = vmspace_acquire_ref(p)) == NULL) 1607 return; 1608 1609 map = &vm->vm_map; 1610 vm_map_lock_read(map); 1611 1612 for (entry = map->header.next; entry != &map->header; entry = entry->next) { 1613 1614 if (entry == NULL) { 1615 PMCDBG(LOG,OPS,2, "hwpmc: vm_map entry unexpectedly " 1616 "NULL! pid=%d vm_map=%p\n", p->p_pid, map); 1617 break; 1618 } 1619 1620 /* 1621 * We only care about executable map entries. 1622 */ 1623 if ((entry->eflags & MAP_ENTRY_IS_SUB_MAP) || 1624 !(entry->protection & VM_PROT_EXECUTE) || 1625 (entry->object.vm_object == NULL)) { 1626 continue; 1627 } 1628 1629 obj = entry->object.vm_object; 1630 VM_OBJECT_RLOCK(obj); 1631 1632 /* 1633 * Walk the backing_object list to find the base 1634 * (non-shadowed) vm_object. 1635 */ 1636 for (lobj = tobj = obj; tobj != NULL; tobj = tobj->backing_object) { 1637 if (tobj != obj) 1638 VM_OBJECT_RLOCK(tobj); 1639 if (lobj != obj) 1640 VM_OBJECT_RUNLOCK(lobj); 1641 lobj = tobj; 1642 } 1643 1644 /* 1645 * At this point lobj is the base vm_object and it is locked. 1646 */ 1647 if (lobj == NULL) { 1648 PMCDBG(LOG,OPS,2, "hwpmc: lobj unexpectedly NULL! pid=%d " 1649 "vm_map=%p vm_obj=%p\n", p->p_pid, map, obj); 1650 VM_OBJECT_RUNLOCK(obj); 1651 continue; 1652 } 1653 1654 if (lobj->type != OBJT_VNODE || lobj->handle == NULL) { 1655 if (lobj != obj) 1656 VM_OBJECT_RUNLOCK(lobj); 1657 VM_OBJECT_RUNLOCK(obj); 1658 continue; 1659 } 1660 1661 /* 1662 * Skip contiguous regions that point to the same 1663 * vnode, so we don't emit redundant MAP-IN 1664 * directives. 1665 */ 1666 if (entry->start == last_end && lobj->handle == last_vp) { 1667 last_end = entry->end; 1668 if (lobj != obj) 1669 VM_OBJECT_RUNLOCK(lobj); 1670 VM_OBJECT_RUNLOCK(obj); 1671 continue; 1672 } 1673 1674 /* 1675 * We don't want to keep the proc's vm_map or this 1676 * vm_object locked while we walk the pathname, since 1677 * vn_fullpath() can sleep. However, if we drop the 1678 * lock, it's possible for concurrent activity to 1679 * modify the vm_map list. To protect against this, 1680 * we save the vm_map timestamp before we release the 1681 * lock, and check it after we reacquire the lock 1682 * below. 1683 */ 1684 start_addr = entry->start; 1685 last_end = entry->end; 1686 last_timestamp = map->timestamp; 1687 vm_map_unlock_read(map); 1688 1689 vp = lobj->handle; 1690 vref(vp); 1691 if (lobj != obj) 1692 VM_OBJECT_RUNLOCK(lobj); 1693 1694 VM_OBJECT_RUNLOCK(obj); 1695 1696 freepath = NULL; 1697 pmc_getfilename(vp, &fullpath, &freepath); 1698 last_vp = vp; 1699 1700 vrele(vp); 1701 1702 vp = NULL; 1703 pmclog_process_map_in(po, p->p_pid, start_addr, fullpath); 1704 if (freepath) 1705 free(freepath, M_TEMP); 1706 1707 vm_map_lock_read(map); 1708 1709 /* 1710 * If our saved timestamp doesn't match, this means 1711 * that the vm_map was modified out from under us and 1712 * we can't trust our current "entry" pointer. Do a 1713 * new lookup for this entry. If there is no entry 1714 * for this address range, vm_map_lookup_entry() will 1715 * return the previous one, so we always want to go to 1716 * entry->next on the next loop iteration. 1717 * 1718 * There is an edge condition here that can occur if 1719 * there is no entry at or before this address. In 1720 * this situation, vm_map_lookup_entry returns 1721 * &map->header, which would cause our loop to abort 1722 * without processing the rest of the map. However, 1723 * in practice this will never happen for process 1724 * vm_map. This is because the executable's text 1725 * segment is the first mapping in the proc's address 1726 * space, and this mapping is never removed until the 1727 * process exits, so there will always be a non-header 1728 * entry at or before the requested address for 1729 * vm_map_lookup_entry to return. 1730 */ 1731 if (map->timestamp != last_timestamp) 1732 vm_map_lookup_entry(map, last_end - 1, &entry); 1733 } 1734 1735 vm_map_unlock_read(map); 1736 vmspace_free(vm); 1737 return; 1738 } 1739 1740 /* 1741 * Log mappings for all processes in the system. 1742 */ 1743 1744 static void 1745 pmc_log_all_process_mappings(struct pmc_owner *po) 1746 { 1747 struct proc *p, *top; 1748 1749 sx_assert(&pmc_sx, SX_XLOCKED); 1750 1751 if ((p = pfind(1)) == NULL) 1752 panic("[pmc,%d] Cannot find init", __LINE__); 1753 1754 PROC_UNLOCK(p); 1755 1756 sx_slock(&proctree_lock); 1757 1758 top = p; 1759 1760 for (;;) { 1761 pmc_log_process_mappings(po, p); 1762 if (!LIST_EMPTY(&p->p_children)) 1763 p = LIST_FIRST(&p->p_children); 1764 else for (;;) { 1765 if (p == top) 1766 goto done; 1767 if (LIST_NEXT(p, p_sibling)) { 1768 p = LIST_NEXT(p, p_sibling); 1769 break; 1770 } 1771 p = p->p_pptr; 1772 } 1773 } 1774 done: 1775 sx_sunlock(&proctree_lock); 1776 } 1777 1778 /* 1779 * The 'hook' invoked from the kernel proper 1780 */ 1781 1782 1783 #ifdef DEBUG 1784 const char *pmc_hooknames[] = { 1785 /* these strings correspond to PMC_FN_* in <sys/pmckern.h> */ 1786 "", 1787 "EXEC", 1788 "CSW-IN", 1789 "CSW-OUT", 1790 "SAMPLE", 1791 "UNUSED1", 1792 "UNUSED2", 1793 "MMAP", 1794 "MUNMAP", 1795 "CALLCHAIN-NMI", 1796 "CALLCHAIN-SOFT", 1797 "SOFTSAMPLING" 1798 }; 1799 #endif 1800 1801 static int 1802 pmc_hook_handler(struct thread *td, int function, void *arg) 1803 { 1804 1805 PMCDBG(MOD,PMH,1, "hook td=%p func=%d \"%s\" arg=%p", td, function, 1806 pmc_hooknames[function], arg); 1807 1808 switch (function) 1809 { 1810 1811 /* 1812 * Process exec() 1813 */ 1814 1815 case PMC_FN_PROCESS_EXEC: 1816 { 1817 char *fullpath, *freepath; 1818 unsigned int ri; 1819 int is_using_hwpmcs; 1820 struct pmc *pm; 1821 struct proc *p; 1822 struct pmc_owner *po; 1823 struct pmc_process *pp; 1824 struct pmckern_procexec *pk; 1825 1826 sx_assert(&pmc_sx, SX_XLOCKED); 1827 1828 p = td->td_proc; 1829 pmc_getfilename(p->p_textvp, &fullpath, &freepath); 1830 1831 pk = (struct pmckern_procexec *) arg; 1832 1833 /* Inform owners of SS mode PMCs of the exec event. */ 1834 LIST_FOREACH(po, &pmc_ss_owners, po_ssnext) 1835 if (po->po_flags & PMC_PO_OWNS_LOGFILE) 1836 pmclog_process_procexec(po, PMC_ID_INVALID, 1837 p->p_pid, pk->pm_entryaddr, fullpath); 1838 1839 PROC_LOCK(p); 1840 is_using_hwpmcs = p->p_flag & P_HWPMC; 1841 PROC_UNLOCK(p); 1842 1843 if (!is_using_hwpmcs) { 1844 if (freepath) 1845 free(freepath, M_TEMP); 1846 break; 1847 } 1848 1849 /* 1850 * PMCs are not inherited across an exec(): remove any 1851 * PMCs that this process is the owner of. 1852 */ 1853 1854 if ((po = pmc_find_owner_descriptor(p)) != NULL) { 1855 pmc_remove_owner(po); 1856 pmc_destroy_owner_descriptor(po); 1857 } 1858 1859 /* 1860 * If the process being exec'ed is not the target of any 1861 * PMC, we are done. 1862 */ 1863 if ((pp = pmc_find_process_descriptor(p, 0)) == NULL) { 1864 if (freepath) 1865 free(freepath, M_TEMP); 1866 break; 1867 } 1868 1869 /* 1870 * Log the exec event to all monitoring owners. Skip 1871 * owners who have already recieved the event because 1872 * they had system sampling PMCs active. 1873 */ 1874 for (ri = 0; ri < md->pmd_npmc; ri++) 1875 if ((pm = pp->pp_pmcs[ri].pp_pmc) != NULL) { 1876 po = pm->pm_owner; 1877 if (po->po_sscount == 0 && 1878 po->po_flags & PMC_PO_OWNS_LOGFILE) 1879 pmclog_process_procexec(po, pm->pm_id, 1880 p->p_pid, pk->pm_entryaddr, 1881 fullpath); 1882 } 1883 1884 if (freepath) 1885 free(freepath, M_TEMP); 1886 1887 1888 PMCDBG(PRC,EXC,1, "exec proc=%p (%d, %s) cred-changed=%d", 1889 p, p->p_pid, p->p_comm, pk->pm_credentialschanged); 1890 1891 if (pk->pm_credentialschanged == 0) /* no change */ 1892 break; 1893 1894 /* 1895 * If the newly exec()'ed process has a different credential 1896 * than before, allow it to be the target of a PMC only if 1897 * the PMC's owner has sufficient priviledge. 1898 */ 1899 1900 for (ri = 0; ri < md->pmd_npmc; ri++) 1901 if ((pm = pp->pp_pmcs[ri].pp_pmc) != NULL) 1902 if (pmc_can_attach(pm, td->td_proc) != 0) 1903 pmc_detach_one_process(td->td_proc, 1904 pm, PMC_FLAG_NONE); 1905 1906 KASSERT(pp->pp_refcnt >= 0 && pp->pp_refcnt <= (int) md->pmd_npmc, 1907 ("[pmc,%d] Illegal ref count %d on pp %p", __LINE__, 1908 pp->pp_refcnt, pp)); 1909 1910 /* 1911 * If this process is no longer the target of any 1912 * PMCs, we can remove the process entry and free 1913 * up space. 1914 */ 1915 1916 if (pp->pp_refcnt == 0) { 1917 pmc_remove_process_descriptor(pp); 1918 free(pp, M_PMC); 1919 break; 1920 } 1921 1922 } 1923 break; 1924 1925 case PMC_FN_CSW_IN: 1926 pmc_process_csw_in(td); 1927 break; 1928 1929 case PMC_FN_CSW_OUT: 1930 pmc_process_csw_out(td); 1931 break; 1932 1933 /* 1934 * Process accumulated PC samples. 1935 * 1936 * This function is expected to be called by hardclock() for 1937 * each CPU that has accumulated PC samples. 1938 * 1939 * This function is to be executed on the CPU whose samples 1940 * are being processed. 1941 */ 1942 case PMC_FN_DO_SAMPLES: 1943 1944 /* 1945 * Clear the cpu specific bit in the CPU mask before 1946 * do the rest of the processing. If the NMI handler 1947 * gets invoked after the "atomic_clear_int()" call 1948 * below but before "pmc_process_samples()" gets 1949 * around to processing the interrupt, then we will 1950 * come back here at the next hardclock() tick (and 1951 * may find nothing to do if "pmc_process_samples()" 1952 * had already processed the interrupt). We don't 1953 * lose the interrupt sample. 1954 */ 1955 CPU_CLR_ATOMIC(PCPU_GET(cpuid), &pmc_cpumask); 1956 pmc_process_samples(PCPU_GET(cpuid), PMC_HR); 1957 pmc_process_samples(PCPU_GET(cpuid), PMC_SR); 1958 break; 1959 1960 case PMC_FN_MMAP: 1961 sx_assert(&pmc_sx, SX_LOCKED); 1962 pmc_process_mmap(td, (struct pmckern_map_in *) arg); 1963 break; 1964 1965 case PMC_FN_MUNMAP: 1966 sx_assert(&pmc_sx, SX_LOCKED); 1967 pmc_process_munmap(td, (struct pmckern_map_out *) arg); 1968 break; 1969 1970 case PMC_FN_USER_CALLCHAIN: 1971 /* 1972 * Record a call chain. 1973 */ 1974 KASSERT(td == curthread, ("[pmc,%d] td != curthread", 1975 __LINE__)); 1976 1977 pmc_capture_user_callchain(PCPU_GET(cpuid), PMC_HR, 1978 (struct trapframe *) arg); 1979 td->td_pflags &= ~TDP_CALLCHAIN; 1980 break; 1981 1982 case PMC_FN_USER_CALLCHAIN_SOFT: 1983 /* 1984 * Record a call chain. 1985 */ 1986 KASSERT(td == curthread, ("[pmc,%d] td != curthread", 1987 __LINE__)); 1988 pmc_capture_user_callchain(PCPU_GET(cpuid), PMC_SR, 1989 (struct trapframe *) arg); 1990 td->td_pflags &= ~TDP_CALLCHAIN; 1991 break; 1992 1993 case PMC_FN_SOFT_SAMPLING: 1994 /* 1995 * Call soft PMC sampling intr. 1996 */ 1997 pmc_soft_intr((struct pmckern_soft *) arg); 1998 break; 1999 2000 default: 2001 #ifdef DEBUG 2002 KASSERT(0, ("[pmc,%d] unknown hook %d\n", __LINE__, function)); 2003 #endif 2004 break; 2005 2006 } 2007 2008 return 0; 2009 } 2010 2011 /* 2012 * allocate a 'struct pmc_owner' descriptor in the owner hash table. 2013 */ 2014 2015 static struct pmc_owner * 2016 pmc_allocate_owner_descriptor(struct proc *p) 2017 { 2018 uint32_t hindex; 2019 struct pmc_owner *po; 2020 struct pmc_ownerhash *poh; 2021 2022 hindex = PMC_HASH_PTR(p, pmc_ownerhashmask); 2023 poh = &pmc_ownerhash[hindex]; 2024 2025 /* allocate space for N pointers and one descriptor struct */ 2026 po = malloc(sizeof(struct pmc_owner), M_PMC, M_WAITOK|M_ZERO); 2027 po->po_owner = p; 2028 LIST_INSERT_HEAD(poh, po, po_next); /* insert into hash table */ 2029 2030 TAILQ_INIT(&po->po_logbuffers); 2031 mtx_init(&po->po_mtx, "pmc-owner-mtx", "pmc-per-proc", MTX_SPIN); 2032 2033 PMCDBG(OWN,ALL,1, "allocate-owner proc=%p (%d, %s) pmc-owner=%p", 2034 p, p->p_pid, p->p_comm, po); 2035 2036 return po; 2037 } 2038 2039 static void 2040 pmc_destroy_owner_descriptor(struct pmc_owner *po) 2041 { 2042 2043 PMCDBG(OWN,REL,1, "destroy-owner po=%p proc=%p (%d, %s)", 2044 po, po->po_owner, po->po_owner->p_pid, po->po_owner->p_comm); 2045 2046 mtx_destroy(&po->po_mtx); 2047 free(po, M_PMC); 2048 } 2049 2050 /* 2051 * find the descriptor corresponding to process 'p', adding or removing it 2052 * as specified by 'mode'. 2053 */ 2054 2055 static struct pmc_process * 2056 pmc_find_process_descriptor(struct proc *p, uint32_t mode) 2057 { 2058 uint32_t hindex; 2059 struct pmc_process *pp, *ppnew; 2060 struct pmc_processhash *pph; 2061 2062 hindex = PMC_HASH_PTR(p, pmc_processhashmask); 2063 pph = &pmc_processhash[hindex]; 2064 2065 ppnew = NULL; 2066 2067 /* 2068 * Pre-allocate memory in the FIND_ALLOCATE case since we 2069 * cannot call malloc(9) once we hold a spin lock. 2070 */ 2071 if (mode & PMC_FLAG_ALLOCATE) 2072 ppnew = malloc(sizeof(struct pmc_process) + md->pmd_npmc * 2073 sizeof(struct pmc_targetstate), M_PMC, M_WAITOK|M_ZERO); 2074 2075 mtx_lock_spin(&pmc_processhash_mtx); 2076 LIST_FOREACH(pp, pph, pp_next) 2077 if (pp->pp_proc == p) 2078 break; 2079 2080 if ((mode & PMC_FLAG_REMOVE) && pp != NULL) 2081 LIST_REMOVE(pp, pp_next); 2082 2083 if ((mode & PMC_FLAG_ALLOCATE) && pp == NULL && 2084 ppnew != NULL) { 2085 ppnew->pp_proc = p; 2086 LIST_INSERT_HEAD(pph, ppnew, pp_next); 2087 pp = ppnew; 2088 ppnew = NULL; 2089 } 2090 mtx_unlock_spin(&pmc_processhash_mtx); 2091 2092 if (pp != NULL && ppnew != NULL) 2093 free(ppnew, M_PMC); 2094 2095 return pp; 2096 } 2097 2098 /* 2099 * remove a process descriptor from the process hash table. 2100 */ 2101 2102 static void 2103 pmc_remove_process_descriptor(struct pmc_process *pp) 2104 { 2105 KASSERT(pp->pp_refcnt == 0, 2106 ("[pmc,%d] Removing process descriptor %p with count %d", 2107 __LINE__, pp, pp->pp_refcnt)); 2108 2109 mtx_lock_spin(&pmc_processhash_mtx); 2110 LIST_REMOVE(pp, pp_next); 2111 mtx_unlock_spin(&pmc_processhash_mtx); 2112 } 2113 2114 2115 /* 2116 * find an owner descriptor corresponding to proc 'p' 2117 */ 2118 2119 static struct pmc_owner * 2120 pmc_find_owner_descriptor(struct proc *p) 2121 { 2122 uint32_t hindex; 2123 struct pmc_owner *po; 2124 struct pmc_ownerhash *poh; 2125 2126 hindex = PMC_HASH_PTR(p, pmc_ownerhashmask); 2127 poh = &pmc_ownerhash[hindex]; 2128 2129 po = NULL; 2130 LIST_FOREACH(po, poh, po_next) 2131 if (po->po_owner == p) 2132 break; 2133 2134 PMCDBG(OWN,FND,1, "find-owner proc=%p (%d, %s) hindex=0x%x -> " 2135 "pmc-owner=%p", p, p->p_pid, p->p_comm, hindex, po); 2136 2137 return po; 2138 } 2139 2140 /* 2141 * pmc_allocate_pmc_descriptor 2142 * 2143 * Allocate a pmc descriptor and initialize its 2144 * fields. 2145 */ 2146 2147 static struct pmc * 2148 pmc_allocate_pmc_descriptor(void) 2149 { 2150 struct pmc *pmc; 2151 2152 pmc = malloc(sizeof(struct pmc), M_PMC, M_WAITOK|M_ZERO); 2153 2154 PMCDBG(PMC,ALL,1, "allocate-pmc -> pmc=%p", pmc); 2155 2156 return pmc; 2157 } 2158 2159 /* 2160 * Destroy a pmc descriptor. 2161 */ 2162 2163 static void 2164 pmc_destroy_pmc_descriptor(struct pmc *pm) 2165 { 2166 2167 KASSERT(pm->pm_state == PMC_STATE_DELETED || 2168 pm->pm_state == PMC_STATE_FREE, 2169 ("[pmc,%d] destroying non-deleted PMC", __LINE__)); 2170 KASSERT(LIST_EMPTY(&pm->pm_targets), 2171 ("[pmc,%d] destroying pmc with targets", __LINE__)); 2172 KASSERT(pm->pm_owner == NULL, 2173 ("[pmc,%d] destroying pmc attached to an owner", __LINE__)); 2174 KASSERT(pm->pm_runcount == 0, 2175 ("[pmc,%d] pmc has non-zero run count %d", __LINE__, 2176 pm->pm_runcount)); 2177 2178 free(pm, M_PMC); 2179 } 2180 2181 static void 2182 pmc_wait_for_pmc_idle(struct pmc *pm) 2183 { 2184 #ifdef DEBUG 2185 volatile int maxloop; 2186 2187 maxloop = 100 * pmc_cpu_max(); 2188 #endif 2189 /* 2190 * Loop (with a forced context switch) till the PMC's runcount 2191 * comes down to zero. 2192 */ 2193 while (atomic_load_acq_32(&pm->pm_runcount) > 0) { 2194 #ifdef DEBUG 2195 maxloop--; 2196 KASSERT(maxloop > 0, 2197 ("[pmc,%d] (ri%d, rc%d) waiting too long for " 2198 "pmc to be free", __LINE__, 2199 PMC_TO_ROWINDEX(pm), pm->pm_runcount)); 2200 #endif 2201 pmc_force_context_switch(); 2202 } 2203 } 2204 2205 /* 2206 * This function does the following things: 2207 * 2208 * - detaches the PMC from hardware 2209 * - unlinks all target threads that were attached to it 2210 * - removes the PMC from its owner's list 2211 * - destroys the PMC private mutex 2212 * 2213 * Once this function completes, the given pmc pointer can be freed by 2214 * calling pmc_destroy_pmc_descriptor(). 2215 */ 2216 2217 static void 2218 pmc_release_pmc_descriptor(struct pmc *pm) 2219 { 2220 enum pmc_mode mode; 2221 struct pmc_hw *phw; 2222 u_int adjri, ri, cpu; 2223 struct pmc_owner *po; 2224 struct pmc_binding pb; 2225 struct pmc_process *pp; 2226 struct pmc_classdep *pcd; 2227 struct pmc_target *ptgt, *tmp; 2228 2229 sx_assert(&pmc_sx, SX_XLOCKED); 2230 2231 KASSERT(pm, ("[pmc,%d] null pmc", __LINE__)); 2232 2233 ri = PMC_TO_ROWINDEX(pm); 2234 pcd = pmc_ri_to_classdep(md, ri, &adjri); 2235 mode = PMC_TO_MODE(pm); 2236 2237 PMCDBG(PMC,REL,1, "release-pmc pmc=%p ri=%d mode=%d", pm, ri, 2238 mode); 2239 2240 /* 2241 * First, we take the PMC off hardware. 2242 */ 2243 cpu = 0; 2244 if (PMC_IS_SYSTEM_MODE(mode)) { 2245 2246 /* 2247 * A system mode PMC runs on a specific CPU. Switch 2248 * to this CPU and turn hardware off. 2249 */ 2250 pmc_save_cpu_binding(&pb); 2251 2252 cpu = PMC_TO_CPU(pm); 2253 2254 pmc_select_cpu(cpu); 2255 2256 /* switch off non-stalled CPUs */ 2257 if (pm->pm_state == PMC_STATE_RUNNING && 2258 pm->pm_stalled == 0) { 2259 2260 phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; 2261 2262 KASSERT(phw->phw_pmc == pm, 2263 ("[pmc, %d] pmc ptr ri(%d) hw(%p) pm(%p)", 2264 __LINE__, ri, phw->phw_pmc, pm)); 2265 PMCDBG(PMC,REL,2, "stopping cpu=%d ri=%d", cpu, ri); 2266 2267 critical_enter(); 2268 pcd->pcd_stop_pmc(cpu, adjri); 2269 critical_exit(); 2270 } 2271 2272 PMCDBG(PMC,REL,2, "decfg cpu=%d ri=%d", cpu, ri); 2273 2274 critical_enter(); 2275 pcd->pcd_config_pmc(cpu, adjri, NULL); 2276 critical_exit(); 2277 2278 /* adjust the global and process count of SS mode PMCs */ 2279 if (mode == PMC_MODE_SS && pm->pm_state == PMC_STATE_RUNNING) { 2280 po = pm->pm_owner; 2281 po->po_sscount--; 2282 if (po->po_sscount == 0) { 2283 atomic_subtract_rel_int(&pmc_ss_count, 1); 2284 LIST_REMOVE(po, po_ssnext); 2285 } 2286 } 2287 2288 pm->pm_state = PMC_STATE_DELETED; 2289 2290 pmc_restore_cpu_binding(&pb); 2291 2292 /* 2293 * We could have references to this PMC structure in 2294 * the per-cpu sample queues. Wait for the queue to 2295 * drain. 2296 */ 2297 pmc_wait_for_pmc_idle(pm); 2298 2299 } else if (PMC_IS_VIRTUAL_MODE(mode)) { 2300 2301 /* 2302 * A virtual PMC could be running on multiple CPUs at 2303 * a given instant. 2304 * 2305 * By marking its state as DELETED, we ensure that 2306 * this PMC is never further scheduled on hardware. 2307 * 2308 * Then we wait till all CPUs are done with this PMC. 2309 */ 2310 pm->pm_state = PMC_STATE_DELETED; 2311 2312 2313 /* Wait for the PMCs runcount to come to zero. */ 2314 pmc_wait_for_pmc_idle(pm); 2315 2316 /* 2317 * At this point the PMC is off all CPUs and cannot be 2318 * freshly scheduled onto a CPU. It is now safe to 2319 * unlink all targets from this PMC. If a 2320 * process-record's refcount falls to zero, we remove 2321 * it from the hash table. The module-wide SX lock 2322 * protects us from races. 2323 */ 2324 LIST_FOREACH_SAFE(ptgt, &pm->pm_targets, pt_next, tmp) { 2325 pp = ptgt->pt_process; 2326 pmc_unlink_target_process(pm, pp); /* frees 'ptgt' */ 2327 2328 PMCDBG(PMC,REL,3, "pp->refcnt=%d", pp->pp_refcnt); 2329 2330 /* 2331 * If the target process record shows that no 2332 * PMCs are attached to it, reclaim its space. 2333 */ 2334 2335 if (pp->pp_refcnt == 0) { 2336 pmc_remove_process_descriptor(pp); 2337 free(pp, M_PMC); 2338 } 2339 } 2340 2341 cpu = curthread->td_oncpu; /* setup cpu for pmd_release() */ 2342 2343 } 2344 2345 /* 2346 * Release any MD resources 2347 */ 2348 (void) pcd->pcd_release_pmc(cpu, adjri, pm); 2349 2350 /* 2351 * Update row disposition 2352 */ 2353 2354 if (PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pm))) 2355 PMC_UNMARK_ROW_STANDALONE(ri); 2356 else 2357 PMC_UNMARK_ROW_THREAD(ri); 2358 2359 /* unlink from the owner's list */ 2360 if (pm->pm_owner) { 2361 LIST_REMOVE(pm, pm_next); 2362 pm->pm_owner = NULL; 2363 } 2364 } 2365 2366 /* 2367 * Register an owner and a pmc. 2368 */ 2369 2370 static int 2371 pmc_register_owner(struct proc *p, struct pmc *pmc) 2372 { 2373 struct pmc_owner *po; 2374 2375 sx_assert(&pmc_sx, SX_XLOCKED); 2376 2377 if ((po = pmc_find_owner_descriptor(p)) == NULL) 2378 if ((po = pmc_allocate_owner_descriptor(p)) == NULL) 2379 return ENOMEM; 2380 2381 KASSERT(pmc->pm_owner == NULL, 2382 ("[pmc,%d] attempting to own an initialized PMC", __LINE__)); 2383 pmc->pm_owner = po; 2384 2385 LIST_INSERT_HEAD(&po->po_pmcs, pmc, pm_next); 2386 2387 PROC_LOCK(p); 2388 p->p_flag |= P_HWPMC; 2389 PROC_UNLOCK(p); 2390 2391 if (po->po_flags & PMC_PO_OWNS_LOGFILE) 2392 pmclog_process_pmcallocate(pmc); 2393 2394 PMCDBG(PMC,REG,1, "register-owner pmc-owner=%p pmc=%p", 2395 po, pmc); 2396 2397 return 0; 2398 } 2399 2400 /* 2401 * Return the current row disposition: 2402 * == 0 => FREE 2403 * > 0 => PROCESS MODE 2404 * < 0 => SYSTEM MODE 2405 */ 2406 2407 int 2408 pmc_getrowdisp(int ri) 2409 { 2410 return pmc_pmcdisp[ri]; 2411 } 2412 2413 /* 2414 * Check if a PMC at row index 'ri' can be allocated to the current 2415 * process. 2416 * 2417 * Allocation can fail if: 2418 * - the current process is already being profiled by a PMC at index 'ri', 2419 * attached to it via OP_PMCATTACH. 2420 * - the current process has already allocated a PMC at index 'ri' 2421 * via OP_ALLOCATE. 2422 */ 2423 2424 static int 2425 pmc_can_allocate_rowindex(struct proc *p, unsigned int ri, int cpu) 2426 { 2427 enum pmc_mode mode; 2428 struct pmc *pm; 2429 struct pmc_owner *po; 2430 struct pmc_process *pp; 2431 2432 PMCDBG(PMC,ALR,1, "can-allocate-rowindex proc=%p (%d, %s) ri=%d " 2433 "cpu=%d", p, p->p_pid, p->p_comm, ri, cpu); 2434 2435 /* 2436 * We shouldn't have already allocated a process-mode PMC at 2437 * row index 'ri'. 2438 * 2439 * We shouldn't have allocated a system-wide PMC on the same 2440 * CPU and same RI. 2441 */ 2442 if ((po = pmc_find_owner_descriptor(p)) != NULL) 2443 LIST_FOREACH(pm, &po->po_pmcs, pm_next) { 2444 if (PMC_TO_ROWINDEX(pm) == ri) { 2445 mode = PMC_TO_MODE(pm); 2446 if (PMC_IS_VIRTUAL_MODE(mode)) 2447 return EEXIST; 2448 if (PMC_IS_SYSTEM_MODE(mode) && 2449 (int) PMC_TO_CPU(pm) == cpu) 2450 return EEXIST; 2451 } 2452 } 2453 2454 /* 2455 * We also shouldn't be the target of any PMC at this index 2456 * since otherwise a PMC_ATTACH to ourselves will fail. 2457 */ 2458 if ((pp = pmc_find_process_descriptor(p, 0)) != NULL) 2459 if (pp->pp_pmcs[ri].pp_pmc) 2460 return EEXIST; 2461 2462 PMCDBG(PMC,ALR,2, "can-allocate-rowindex proc=%p (%d, %s) ri=%d ok", 2463 p, p->p_pid, p->p_comm, ri); 2464 2465 return 0; 2466 } 2467 2468 /* 2469 * Check if a given PMC at row index 'ri' can be currently used in 2470 * mode 'mode'. 2471 */ 2472 2473 static int 2474 pmc_can_allocate_row(int ri, enum pmc_mode mode) 2475 { 2476 enum pmc_disp disp; 2477 2478 sx_assert(&pmc_sx, SX_XLOCKED); 2479 2480 PMCDBG(PMC,ALR,1, "can-allocate-row ri=%d mode=%d", ri, mode); 2481 2482 if (PMC_IS_SYSTEM_MODE(mode)) 2483 disp = PMC_DISP_STANDALONE; 2484 else 2485 disp = PMC_DISP_THREAD; 2486 2487 /* 2488 * check disposition for PMC row 'ri': 2489 * 2490 * Expected disposition Row-disposition Result 2491 * 2492 * STANDALONE STANDALONE or FREE proceed 2493 * STANDALONE THREAD fail 2494 * THREAD THREAD or FREE proceed 2495 * THREAD STANDALONE fail 2496 */ 2497 2498 if (!PMC_ROW_DISP_IS_FREE(ri) && 2499 !(disp == PMC_DISP_THREAD && PMC_ROW_DISP_IS_THREAD(ri)) && 2500 !(disp == PMC_DISP_STANDALONE && PMC_ROW_DISP_IS_STANDALONE(ri))) 2501 return EBUSY; 2502 2503 /* 2504 * All OK 2505 */ 2506 2507 PMCDBG(PMC,ALR,2, "can-allocate-row ri=%d mode=%d ok", ri, mode); 2508 2509 return 0; 2510 2511 } 2512 2513 /* 2514 * Find a PMC descriptor with user handle 'pmcid' for thread 'td'. 2515 */ 2516 2517 static struct pmc * 2518 pmc_find_pmc_descriptor_in_process(struct pmc_owner *po, pmc_id_t pmcid) 2519 { 2520 struct pmc *pm; 2521 2522 KASSERT(PMC_ID_TO_ROWINDEX(pmcid) < md->pmd_npmc, 2523 ("[pmc,%d] Illegal pmc index %d (max %d)", __LINE__, 2524 PMC_ID_TO_ROWINDEX(pmcid), md->pmd_npmc)); 2525 2526 LIST_FOREACH(pm, &po->po_pmcs, pm_next) 2527 if (pm->pm_id == pmcid) 2528 return pm; 2529 2530 return NULL; 2531 } 2532 2533 static int 2534 pmc_find_pmc(pmc_id_t pmcid, struct pmc **pmc) 2535 { 2536 2537 struct pmc *pm; 2538 struct pmc_owner *po; 2539 2540 PMCDBG(PMC,FND,1, "find-pmc id=%d", pmcid); 2541 2542 if ((po = pmc_find_owner_descriptor(curthread->td_proc)) == NULL) 2543 return ESRCH; 2544 2545 if ((pm = pmc_find_pmc_descriptor_in_process(po, pmcid)) == NULL) 2546 return EINVAL; 2547 2548 PMCDBG(PMC,FND,2, "find-pmc id=%d -> pmc=%p", pmcid, pm); 2549 2550 *pmc = pm; 2551 return 0; 2552 } 2553 2554 /* 2555 * Start a PMC. 2556 */ 2557 2558 static int 2559 pmc_start(struct pmc *pm) 2560 { 2561 enum pmc_mode mode; 2562 struct pmc_owner *po; 2563 struct pmc_binding pb; 2564 struct pmc_classdep *pcd; 2565 int adjri, error, cpu, ri; 2566 2567 KASSERT(pm != NULL, 2568 ("[pmc,%d] null pm", __LINE__)); 2569 2570 mode = PMC_TO_MODE(pm); 2571 ri = PMC_TO_ROWINDEX(pm); 2572 pcd = pmc_ri_to_classdep(md, ri, &adjri); 2573 2574 error = 0; 2575 2576 PMCDBG(PMC,OPS,1, "start pmc=%p mode=%d ri=%d", pm, mode, ri); 2577 2578 po = pm->pm_owner; 2579 2580 /* 2581 * Disallow PMCSTART if a logfile is required but has not been 2582 * configured yet. 2583 */ 2584 if ((pm->pm_flags & PMC_F_NEEDS_LOGFILE) && 2585 (po->po_flags & PMC_PO_OWNS_LOGFILE) == 0) 2586 return (EDOOFUS); /* programming error */ 2587 2588 /* 2589 * If this is a sampling mode PMC, log mapping information for 2590 * the kernel modules that are currently loaded. 2591 */ 2592 if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) 2593 pmc_log_kernel_mappings(pm); 2594 2595 if (PMC_IS_VIRTUAL_MODE(mode)) { 2596 2597 /* 2598 * If a PMCATTACH has never been done on this PMC, 2599 * attach it to its owner process. 2600 */ 2601 2602 if (LIST_EMPTY(&pm->pm_targets)) 2603 error = (pm->pm_flags & PMC_F_ATTACH_DONE) ? ESRCH : 2604 pmc_attach_process(po->po_owner, pm); 2605 2606 /* 2607 * If the PMC is attached to its owner, then force a context 2608 * switch to ensure that the MD state gets set correctly. 2609 */ 2610 2611 if (error == 0) { 2612 pm->pm_state = PMC_STATE_RUNNING; 2613 if (pm->pm_flags & PMC_F_ATTACHED_TO_OWNER) 2614 pmc_force_context_switch(); 2615 } 2616 2617 return (error); 2618 } 2619 2620 2621 /* 2622 * A system-wide PMC. 2623 * 2624 * Add the owner to the global list if this is a system-wide 2625 * sampling PMC. 2626 */ 2627 2628 if (mode == PMC_MODE_SS) { 2629 if (po->po_sscount == 0) { 2630 LIST_INSERT_HEAD(&pmc_ss_owners, po, po_ssnext); 2631 atomic_add_rel_int(&pmc_ss_count, 1); 2632 PMCDBG(PMC,OPS,1, "po=%p in global list", po); 2633 } 2634 po->po_sscount++; 2635 2636 /* 2637 * Log mapping information for all existing processes in the 2638 * system. Subsequent mappings are logged as they happen; 2639 * see pmc_process_mmap(). 2640 */ 2641 if (po->po_logprocmaps == 0) { 2642 pmc_log_all_process_mappings(po); 2643 po->po_logprocmaps = 1; 2644 } 2645 } 2646 2647 /* 2648 * Move to the CPU associated with this 2649 * PMC, and start the hardware. 2650 */ 2651 2652 pmc_save_cpu_binding(&pb); 2653 2654 cpu = PMC_TO_CPU(pm); 2655 2656 if (!pmc_cpu_is_active(cpu)) 2657 return (ENXIO); 2658 2659 pmc_select_cpu(cpu); 2660 2661 /* 2662 * global PMCs are configured at allocation time 2663 * so write out the initial value and start the PMC. 2664 */ 2665 2666 pm->pm_state = PMC_STATE_RUNNING; 2667 2668 critical_enter(); 2669 if ((error = pcd->pcd_write_pmc(cpu, adjri, 2670 PMC_IS_SAMPLING_MODE(mode) ? 2671 pm->pm_sc.pm_reloadcount : 2672 pm->pm_sc.pm_initial)) == 0) 2673 error = pcd->pcd_start_pmc(cpu, adjri); 2674 critical_exit(); 2675 2676 pmc_restore_cpu_binding(&pb); 2677 2678 return (error); 2679 } 2680 2681 /* 2682 * Stop a PMC. 2683 */ 2684 2685 static int 2686 pmc_stop(struct pmc *pm) 2687 { 2688 struct pmc_owner *po; 2689 struct pmc_binding pb; 2690 struct pmc_classdep *pcd; 2691 int adjri, cpu, error, ri; 2692 2693 KASSERT(pm != NULL, ("[pmc,%d] null pmc", __LINE__)); 2694 2695 PMCDBG(PMC,OPS,1, "stop pmc=%p mode=%d ri=%d", pm, 2696 PMC_TO_MODE(pm), PMC_TO_ROWINDEX(pm)); 2697 2698 pm->pm_state = PMC_STATE_STOPPED; 2699 2700 /* 2701 * If the PMC is a virtual mode one, changing the state to 2702 * non-RUNNING is enough to ensure that the PMC never gets 2703 * scheduled. 2704 * 2705 * If this PMC is current running on a CPU, then it will 2706 * handled correctly at the time its target process is context 2707 * switched out. 2708 */ 2709 2710 if (PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm))) 2711 return 0; 2712 2713 /* 2714 * A system-mode PMC. Move to the CPU associated with 2715 * this PMC, and stop the hardware. We update the 2716 * 'initial count' so that a subsequent PMCSTART will 2717 * resume counting from the current hardware count. 2718 */ 2719 2720 pmc_save_cpu_binding(&pb); 2721 2722 cpu = PMC_TO_CPU(pm); 2723 2724 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 2725 ("[pmc,%d] illegal cpu=%d", __LINE__, cpu)); 2726 2727 if (!pmc_cpu_is_active(cpu)) 2728 return ENXIO; 2729 2730 pmc_select_cpu(cpu); 2731 2732 ri = PMC_TO_ROWINDEX(pm); 2733 pcd = pmc_ri_to_classdep(md, ri, &adjri); 2734 2735 critical_enter(); 2736 if ((error = pcd->pcd_stop_pmc(cpu, adjri)) == 0) 2737 error = pcd->pcd_read_pmc(cpu, adjri, &pm->pm_sc.pm_initial); 2738 critical_exit(); 2739 2740 pmc_restore_cpu_binding(&pb); 2741 2742 po = pm->pm_owner; 2743 2744 /* remove this owner from the global list of SS PMC owners */ 2745 if (PMC_TO_MODE(pm) == PMC_MODE_SS) { 2746 po->po_sscount--; 2747 if (po->po_sscount == 0) { 2748 atomic_subtract_rel_int(&pmc_ss_count, 1); 2749 LIST_REMOVE(po, po_ssnext); 2750 PMCDBG(PMC,OPS,2,"po=%p removed from global list", po); 2751 } 2752 } 2753 2754 return (error); 2755 } 2756 2757 2758 #ifdef DEBUG 2759 static const char *pmc_op_to_name[] = { 2760 #undef __PMC_OP 2761 #define __PMC_OP(N, D) #N , 2762 __PMC_OPS() 2763 NULL 2764 }; 2765 #endif 2766 2767 /* 2768 * The syscall interface 2769 */ 2770 2771 #define PMC_GET_SX_XLOCK(...) do { \ 2772 sx_xlock(&pmc_sx); \ 2773 if (pmc_hook == NULL) { \ 2774 sx_xunlock(&pmc_sx); \ 2775 return __VA_ARGS__; \ 2776 } \ 2777 } while (0) 2778 2779 #define PMC_DOWNGRADE_SX() do { \ 2780 sx_downgrade(&pmc_sx); \ 2781 is_sx_downgraded = 1; \ 2782 } while (0) 2783 2784 static int 2785 pmc_syscall_handler(struct thread *td, void *syscall_args) 2786 { 2787 int error, is_sx_downgraded, is_sx_locked, op; 2788 struct pmc_syscall_args *c; 2789 void *arg; 2790 2791 PMC_GET_SX_XLOCK(ENOSYS); 2792 2793 DROP_GIANT(); 2794 2795 is_sx_downgraded = 0; 2796 is_sx_locked = 1; 2797 2798 c = (struct pmc_syscall_args *) syscall_args; 2799 2800 op = c->pmop_code; 2801 arg = c->pmop_data; 2802 2803 PMCDBG(MOD,PMS,1, "syscall op=%d \"%s\" arg=%p", op, 2804 pmc_op_to_name[op], arg); 2805 2806 error = 0; 2807 atomic_add_int(&pmc_stats.pm_syscalls, 1); 2808 2809 switch(op) 2810 { 2811 2812 2813 /* 2814 * Configure a log file. 2815 * 2816 * XXX This OP will be reworked. 2817 */ 2818 2819 case PMC_OP_CONFIGURELOG: 2820 { 2821 struct proc *p; 2822 struct pmc *pm; 2823 struct pmc_owner *po; 2824 struct pmc_op_configurelog cl; 2825 2826 sx_assert(&pmc_sx, SX_XLOCKED); 2827 2828 if ((error = copyin(arg, &cl, sizeof(cl))) != 0) 2829 break; 2830 2831 /* mark this process as owning a log file */ 2832 p = td->td_proc; 2833 if ((po = pmc_find_owner_descriptor(p)) == NULL) 2834 if ((po = pmc_allocate_owner_descriptor(p)) == NULL) { 2835 error = ENOMEM; 2836 break; 2837 } 2838 2839 /* 2840 * If a valid fd was passed in, try to configure that, 2841 * otherwise if 'fd' was less than zero and there was 2842 * a log file configured, flush its buffers and 2843 * de-configure it. 2844 */ 2845 if (cl.pm_logfd >= 0) { 2846 sx_xunlock(&pmc_sx); 2847 is_sx_locked = 0; 2848 error = pmclog_configure_log(md, po, cl.pm_logfd); 2849 } else if (po->po_flags & PMC_PO_OWNS_LOGFILE) { 2850 pmclog_process_closelog(po); 2851 error = pmclog_close(po); 2852 if (error == 0) { 2853 LIST_FOREACH(pm, &po->po_pmcs, pm_next) 2854 if (pm->pm_flags & PMC_F_NEEDS_LOGFILE && 2855 pm->pm_state == PMC_STATE_RUNNING) 2856 pmc_stop(pm); 2857 error = pmclog_deconfigure_log(po); 2858 } 2859 } else 2860 error = EINVAL; 2861 2862 if (error) 2863 break; 2864 } 2865 break; 2866 2867 /* 2868 * Flush a log file. 2869 */ 2870 2871 case PMC_OP_FLUSHLOG: 2872 { 2873 struct pmc_owner *po; 2874 2875 sx_assert(&pmc_sx, SX_XLOCKED); 2876 2877 if ((po = pmc_find_owner_descriptor(td->td_proc)) == NULL) { 2878 error = EINVAL; 2879 break; 2880 } 2881 2882 error = pmclog_flush(po); 2883 } 2884 break; 2885 2886 /* 2887 * Close a log file. 2888 */ 2889 2890 case PMC_OP_CLOSELOG: 2891 { 2892 struct pmc_owner *po; 2893 2894 sx_assert(&pmc_sx, SX_XLOCKED); 2895 2896 if ((po = pmc_find_owner_descriptor(td->td_proc)) == NULL) { 2897 error = EINVAL; 2898 break; 2899 } 2900 2901 error = pmclog_close(po); 2902 } 2903 break; 2904 2905 /* 2906 * Retrieve hardware configuration. 2907 */ 2908 2909 case PMC_OP_GETCPUINFO: /* CPU information */ 2910 { 2911 struct pmc_op_getcpuinfo gci; 2912 struct pmc_classinfo *pci; 2913 struct pmc_classdep *pcd; 2914 int cl; 2915 2916 gci.pm_cputype = md->pmd_cputype; 2917 gci.pm_ncpu = pmc_cpu_max(); 2918 gci.pm_npmc = md->pmd_npmc; 2919 gci.pm_nclass = md->pmd_nclass; 2920 pci = gci.pm_classes; 2921 pcd = md->pmd_classdep; 2922 for (cl = 0; cl < md->pmd_nclass; cl++, pci++, pcd++) { 2923 pci->pm_caps = pcd->pcd_caps; 2924 pci->pm_class = pcd->pcd_class; 2925 pci->pm_width = pcd->pcd_width; 2926 pci->pm_num = pcd->pcd_num; 2927 } 2928 error = copyout(&gci, arg, sizeof(gci)); 2929 } 2930 break; 2931 2932 /* 2933 * Retrieve soft events list. 2934 */ 2935 case PMC_OP_GETDYNEVENTINFO: 2936 { 2937 enum pmc_class cl; 2938 enum pmc_event ev; 2939 struct pmc_op_getdyneventinfo *gei; 2940 struct pmc_dyn_event_descr dev; 2941 struct pmc_soft *ps; 2942 uint32_t nevent; 2943 2944 sx_assert(&pmc_sx, SX_LOCKED); 2945 2946 gei = (struct pmc_op_getdyneventinfo *) arg; 2947 2948 if ((error = copyin(&gei->pm_class, &cl, sizeof(cl))) != 0) 2949 break; 2950 2951 /* Only SOFT class is dynamic. */ 2952 if (cl != PMC_CLASS_SOFT) { 2953 error = EINVAL; 2954 break; 2955 } 2956 2957 nevent = 0; 2958 for (ev = PMC_EV_SOFT_FIRST; (int)ev <= PMC_EV_SOFT_LAST; ev++) { 2959 ps = pmc_soft_ev_acquire(ev); 2960 if (ps == NULL) 2961 continue; 2962 bcopy(&ps->ps_ev, &dev, sizeof(dev)); 2963 pmc_soft_ev_release(ps); 2964 2965 error = copyout(&dev, 2966 &gei->pm_events[nevent], 2967 sizeof(struct pmc_dyn_event_descr)); 2968 if (error != 0) 2969 break; 2970 nevent++; 2971 } 2972 if (error != 0) 2973 break; 2974 2975 error = copyout(&nevent, &gei->pm_nevent, 2976 sizeof(nevent)); 2977 } 2978 break; 2979 2980 /* 2981 * Get module statistics 2982 */ 2983 2984 case PMC_OP_GETDRIVERSTATS: 2985 { 2986 struct pmc_op_getdriverstats gms; 2987 2988 bcopy(&pmc_stats, &gms, sizeof(gms)); 2989 error = copyout(&gms, arg, sizeof(gms)); 2990 } 2991 break; 2992 2993 2994 /* 2995 * Retrieve module version number 2996 */ 2997 2998 case PMC_OP_GETMODULEVERSION: 2999 { 3000 uint32_t cv, modv; 3001 3002 /* retrieve the client's idea of the ABI version */ 3003 if ((error = copyin(arg, &cv, sizeof(uint32_t))) != 0) 3004 break; 3005 /* don't service clients newer than our driver */ 3006 modv = PMC_VERSION; 3007 if ((cv & 0xFFFF0000) > (modv & 0xFFFF0000)) { 3008 error = EPROGMISMATCH; 3009 break; 3010 } 3011 error = copyout(&modv, arg, sizeof(int)); 3012 } 3013 break; 3014 3015 3016 /* 3017 * Retrieve the state of all the PMCs on a given 3018 * CPU. 3019 */ 3020 3021 case PMC_OP_GETPMCINFO: 3022 { 3023 int ari; 3024 struct pmc *pm; 3025 size_t pmcinfo_size; 3026 uint32_t cpu, n, npmc; 3027 struct pmc_owner *po; 3028 struct pmc_binding pb; 3029 struct pmc_classdep *pcd; 3030 struct pmc_info *p, *pmcinfo; 3031 struct pmc_op_getpmcinfo *gpi; 3032 3033 PMC_DOWNGRADE_SX(); 3034 3035 gpi = (struct pmc_op_getpmcinfo *) arg; 3036 3037 if ((error = copyin(&gpi->pm_cpu, &cpu, sizeof(cpu))) != 0) 3038 break; 3039 3040 if (cpu >= pmc_cpu_max()) { 3041 error = EINVAL; 3042 break; 3043 } 3044 3045 if (!pmc_cpu_is_active(cpu)) { 3046 error = ENXIO; 3047 break; 3048 } 3049 3050 /* switch to CPU 'cpu' */ 3051 pmc_save_cpu_binding(&pb); 3052 pmc_select_cpu(cpu); 3053 3054 npmc = md->pmd_npmc; 3055 3056 pmcinfo_size = npmc * sizeof(struct pmc_info); 3057 pmcinfo = malloc(pmcinfo_size, M_PMC, M_WAITOK); 3058 3059 p = pmcinfo; 3060 3061 for (n = 0; n < md->pmd_npmc; n++, p++) { 3062 3063 pcd = pmc_ri_to_classdep(md, n, &ari); 3064 3065 KASSERT(pcd != NULL, 3066 ("[pmc,%d] null pcd ri=%d", __LINE__, n)); 3067 3068 if ((error = pcd->pcd_describe(cpu, ari, p, &pm)) != 0) 3069 break; 3070 3071 if (PMC_ROW_DISP_IS_STANDALONE(n)) 3072 p->pm_rowdisp = PMC_DISP_STANDALONE; 3073 else if (PMC_ROW_DISP_IS_THREAD(n)) 3074 p->pm_rowdisp = PMC_DISP_THREAD; 3075 else 3076 p->pm_rowdisp = PMC_DISP_FREE; 3077 3078 p->pm_ownerpid = -1; 3079 3080 if (pm == NULL) /* no PMC associated */ 3081 continue; 3082 3083 po = pm->pm_owner; 3084 3085 KASSERT(po->po_owner != NULL, 3086 ("[pmc,%d] pmc_owner had a null proc pointer", 3087 __LINE__)); 3088 3089 p->pm_ownerpid = po->po_owner->p_pid; 3090 p->pm_mode = PMC_TO_MODE(pm); 3091 p->pm_event = pm->pm_event; 3092 p->pm_flags = pm->pm_flags; 3093 3094 if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) 3095 p->pm_reloadcount = 3096 pm->pm_sc.pm_reloadcount; 3097 } 3098 3099 pmc_restore_cpu_binding(&pb); 3100 3101 /* now copy out the PMC info collected */ 3102 if (error == 0) 3103 error = copyout(pmcinfo, &gpi->pm_pmcs, pmcinfo_size); 3104 3105 free(pmcinfo, M_PMC); 3106 } 3107 break; 3108 3109 3110 /* 3111 * Set the administrative state of a PMC. I.e. whether 3112 * the PMC is to be used or not. 3113 */ 3114 3115 case PMC_OP_PMCADMIN: 3116 { 3117 int cpu, ri; 3118 enum pmc_state request; 3119 struct pmc_cpu *pc; 3120 struct pmc_hw *phw; 3121 struct pmc_op_pmcadmin pma; 3122 struct pmc_binding pb; 3123 3124 sx_assert(&pmc_sx, SX_XLOCKED); 3125 3126 KASSERT(td == curthread, 3127 ("[pmc,%d] td != curthread", __LINE__)); 3128 3129 error = priv_check(td, PRIV_PMC_MANAGE); 3130 if (error) 3131 break; 3132 3133 if ((error = copyin(arg, &pma, sizeof(pma))) != 0) 3134 break; 3135 3136 cpu = pma.pm_cpu; 3137 3138 if (cpu < 0 || cpu >= (int) pmc_cpu_max()) { 3139 error = EINVAL; 3140 break; 3141 } 3142 3143 if (!pmc_cpu_is_active(cpu)) { 3144 error = ENXIO; 3145 break; 3146 } 3147 3148 request = pma.pm_state; 3149 3150 if (request != PMC_STATE_DISABLED && 3151 request != PMC_STATE_FREE) { 3152 error = EINVAL; 3153 break; 3154 } 3155 3156 ri = pma.pm_pmc; /* pmc id == row index */ 3157 if (ri < 0 || ri >= (int) md->pmd_npmc) { 3158 error = EINVAL; 3159 break; 3160 } 3161 3162 /* 3163 * We can't disable a PMC with a row-index allocated 3164 * for process virtual PMCs. 3165 */ 3166 3167 if (PMC_ROW_DISP_IS_THREAD(ri) && 3168 request == PMC_STATE_DISABLED) { 3169 error = EBUSY; 3170 break; 3171 } 3172 3173 /* 3174 * otherwise, this PMC on this CPU is either free or 3175 * in system-wide mode. 3176 */ 3177 3178 pmc_save_cpu_binding(&pb); 3179 pmc_select_cpu(cpu); 3180 3181 pc = pmc_pcpu[cpu]; 3182 phw = pc->pc_hwpmcs[ri]; 3183 3184 /* 3185 * XXX do we need some kind of 'forced' disable? 3186 */ 3187 3188 if (phw->phw_pmc == NULL) { 3189 if (request == PMC_STATE_DISABLED && 3190 (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED)) { 3191 phw->phw_state &= ~PMC_PHW_FLAG_IS_ENABLED; 3192 PMC_MARK_ROW_STANDALONE(ri); 3193 } else if (request == PMC_STATE_FREE && 3194 (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) == 0) { 3195 phw->phw_state |= PMC_PHW_FLAG_IS_ENABLED; 3196 PMC_UNMARK_ROW_STANDALONE(ri); 3197 } 3198 /* other cases are a no-op */ 3199 } else 3200 error = EBUSY; 3201 3202 pmc_restore_cpu_binding(&pb); 3203 } 3204 break; 3205 3206 3207 /* 3208 * Allocate a PMC. 3209 */ 3210 3211 case PMC_OP_PMCALLOCATE: 3212 { 3213 int adjri, n; 3214 u_int cpu; 3215 uint32_t caps; 3216 struct pmc *pmc; 3217 enum pmc_mode mode; 3218 struct pmc_hw *phw; 3219 struct pmc_binding pb; 3220 struct pmc_classdep *pcd; 3221 struct pmc_op_pmcallocate pa; 3222 3223 if ((error = copyin(arg, &pa, sizeof(pa))) != 0) 3224 break; 3225 3226 caps = pa.pm_caps; 3227 mode = pa.pm_mode; 3228 cpu = pa.pm_cpu; 3229 3230 if ((mode != PMC_MODE_SS && mode != PMC_MODE_SC && 3231 mode != PMC_MODE_TS && mode != PMC_MODE_TC) || 3232 (cpu != (u_int) PMC_CPU_ANY && cpu >= pmc_cpu_max())) { 3233 error = EINVAL; 3234 break; 3235 } 3236 3237 /* 3238 * Virtual PMCs should only ask for a default CPU. 3239 * System mode PMCs need to specify a non-default CPU. 3240 */ 3241 3242 if ((PMC_IS_VIRTUAL_MODE(mode) && cpu != (u_int) PMC_CPU_ANY) || 3243 (PMC_IS_SYSTEM_MODE(mode) && cpu == (u_int) PMC_CPU_ANY)) { 3244 error = EINVAL; 3245 break; 3246 } 3247 3248 /* 3249 * Check that an inactive CPU is not being asked for. 3250 */ 3251 3252 if (PMC_IS_SYSTEM_MODE(mode) && !pmc_cpu_is_active(cpu)) { 3253 error = ENXIO; 3254 break; 3255 } 3256 3257 /* 3258 * Refuse an allocation for a system-wide PMC if this 3259 * process has been jailed, or if this process lacks 3260 * super-user credentials and the sysctl tunable 3261 * 'security.bsd.unprivileged_syspmcs' is zero. 3262 */ 3263 3264 if (PMC_IS_SYSTEM_MODE(mode)) { 3265 if (jailed(curthread->td_ucred)) { 3266 error = EPERM; 3267 break; 3268 } 3269 if (!pmc_unprivileged_syspmcs) { 3270 error = priv_check(curthread, 3271 PRIV_PMC_SYSTEM); 3272 if (error) 3273 break; 3274 } 3275 } 3276 3277 /* 3278 * Look for valid values for 'pm_flags' 3279 */ 3280 3281 if ((pa.pm_flags & ~(PMC_F_DESCENDANTS | PMC_F_LOG_PROCCSW | 3282 PMC_F_LOG_PROCEXIT | PMC_F_CALLCHAIN)) != 0) { 3283 error = EINVAL; 3284 break; 3285 } 3286 3287 /* process logging options are not allowed for system PMCs */ 3288 if (PMC_IS_SYSTEM_MODE(mode) && (pa.pm_flags & 3289 (PMC_F_LOG_PROCCSW | PMC_F_LOG_PROCEXIT))) { 3290 error = EINVAL; 3291 break; 3292 } 3293 3294 /* 3295 * All sampling mode PMCs need to be able to interrupt the 3296 * CPU. 3297 */ 3298 if (PMC_IS_SAMPLING_MODE(mode)) 3299 caps |= PMC_CAP_INTERRUPT; 3300 3301 /* A valid class specifier should have been passed in. */ 3302 for (n = 0; n < md->pmd_nclass; n++) 3303 if (md->pmd_classdep[n].pcd_class == pa.pm_class) 3304 break; 3305 if (n == md->pmd_nclass) { 3306 error = EINVAL; 3307 break; 3308 } 3309 3310 /* The requested PMC capabilities should be feasible. */ 3311 if ((md->pmd_classdep[n].pcd_caps & caps) != caps) { 3312 error = EOPNOTSUPP; 3313 break; 3314 } 3315 3316 PMCDBG(PMC,ALL,2, "event=%d caps=0x%x mode=%d cpu=%d", 3317 pa.pm_ev, caps, mode, cpu); 3318 3319 pmc = pmc_allocate_pmc_descriptor(); 3320 pmc->pm_id = PMC_ID_MAKE_ID(cpu,pa.pm_mode,pa.pm_class, 3321 PMC_ID_INVALID); 3322 pmc->pm_event = pa.pm_ev; 3323 pmc->pm_state = PMC_STATE_FREE; 3324 pmc->pm_caps = caps; 3325 pmc->pm_flags = pa.pm_flags; 3326 3327 /* switch thread to CPU 'cpu' */ 3328 pmc_save_cpu_binding(&pb); 3329 3330 #define PMC_IS_SHAREABLE_PMC(cpu, n) \ 3331 (pmc_pcpu[(cpu)]->pc_hwpmcs[(n)]->phw_state & \ 3332 PMC_PHW_FLAG_IS_SHAREABLE) 3333 #define PMC_IS_UNALLOCATED(cpu, n) \ 3334 (pmc_pcpu[(cpu)]->pc_hwpmcs[(n)]->phw_pmc == NULL) 3335 3336 if (PMC_IS_SYSTEM_MODE(mode)) { 3337 pmc_select_cpu(cpu); 3338 for (n = 0; n < (int) md->pmd_npmc; n++) { 3339 pcd = pmc_ri_to_classdep(md, n, &adjri); 3340 if (pmc_can_allocate_row(n, mode) == 0 && 3341 pmc_can_allocate_rowindex( 3342 curthread->td_proc, n, cpu) == 0 && 3343 (PMC_IS_UNALLOCATED(cpu, n) || 3344 PMC_IS_SHAREABLE_PMC(cpu, n)) && 3345 pcd->pcd_allocate_pmc(cpu, adjri, pmc, 3346 &pa) == 0) 3347 break; 3348 } 3349 } else { 3350 /* Process virtual mode */ 3351 for (n = 0; n < (int) md->pmd_npmc; n++) { 3352 pcd = pmc_ri_to_classdep(md, n, &adjri); 3353 if (pmc_can_allocate_row(n, mode) == 0 && 3354 pmc_can_allocate_rowindex( 3355 curthread->td_proc, n, 3356 PMC_CPU_ANY) == 0 && 3357 pcd->pcd_allocate_pmc(curthread->td_oncpu, 3358 adjri, pmc, &pa) == 0) 3359 break; 3360 } 3361 } 3362 3363 #undef PMC_IS_UNALLOCATED 3364 #undef PMC_IS_SHAREABLE_PMC 3365 3366 pmc_restore_cpu_binding(&pb); 3367 3368 if (n == (int) md->pmd_npmc) { 3369 pmc_destroy_pmc_descriptor(pmc); 3370 pmc = NULL; 3371 error = EINVAL; 3372 break; 3373 } 3374 3375 /* Fill in the correct value in the ID field */ 3376 pmc->pm_id = PMC_ID_MAKE_ID(cpu,mode,pa.pm_class,n); 3377 3378 PMCDBG(PMC,ALL,2, "ev=%d class=%d mode=%d n=%d -> pmcid=%x", 3379 pmc->pm_event, pa.pm_class, mode, n, pmc->pm_id); 3380 3381 /* Process mode PMCs with logging enabled need log files */ 3382 if (pmc->pm_flags & (PMC_F_LOG_PROCEXIT | PMC_F_LOG_PROCCSW)) 3383 pmc->pm_flags |= PMC_F_NEEDS_LOGFILE; 3384 3385 /* All system mode sampling PMCs require a log file */ 3386 if (PMC_IS_SAMPLING_MODE(mode) && PMC_IS_SYSTEM_MODE(mode)) 3387 pmc->pm_flags |= PMC_F_NEEDS_LOGFILE; 3388 3389 /* 3390 * Configure global pmc's immediately 3391 */ 3392 3393 if (PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pmc))) { 3394 3395 pmc_save_cpu_binding(&pb); 3396 pmc_select_cpu(cpu); 3397 3398 phw = pmc_pcpu[cpu]->pc_hwpmcs[n]; 3399 pcd = pmc_ri_to_classdep(md, n, &adjri); 3400 3401 if ((phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) == 0 || 3402 (error = pcd->pcd_config_pmc(cpu, adjri, pmc)) != 0) { 3403 (void) pcd->pcd_release_pmc(cpu, adjri, pmc); 3404 pmc_destroy_pmc_descriptor(pmc); 3405 pmc = NULL; 3406 pmc_restore_cpu_binding(&pb); 3407 error = EPERM; 3408 break; 3409 } 3410 3411 pmc_restore_cpu_binding(&pb); 3412 } 3413 3414 pmc->pm_state = PMC_STATE_ALLOCATED; 3415 3416 /* 3417 * mark row disposition 3418 */ 3419 3420 if (PMC_IS_SYSTEM_MODE(mode)) 3421 PMC_MARK_ROW_STANDALONE(n); 3422 else 3423 PMC_MARK_ROW_THREAD(n); 3424 3425 /* 3426 * Register this PMC with the current thread as its owner. 3427 */ 3428 3429 if ((error = 3430 pmc_register_owner(curthread->td_proc, pmc)) != 0) { 3431 pmc_release_pmc_descriptor(pmc); 3432 pmc_destroy_pmc_descriptor(pmc); 3433 pmc = NULL; 3434 break; 3435 } 3436 3437 /* 3438 * Return the allocated index. 3439 */ 3440 3441 pa.pm_pmcid = pmc->pm_id; 3442 3443 error = copyout(&pa, arg, sizeof(pa)); 3444 } 3445 break; 3446 3447 3448 /* 3449 * Attach a PMC to a process. 3450 */ 3451 3452 case PMC_OP_PMCATTACH: 3453 { 3454 struct pmc *pm; 3455 struct proc *p; 3456 struct pmc_op_pmcattach a; 3457 3458 sx_assert(&pmc_sx, SX_XLOCKED); 3459 3460 if ((error = copyin(arg, &a, sizeof(a))) != 0) 3461 break; 3462 3463 if (a.pm_pid < 0) { 3464 error = EINVAL; 3465 break; 3466 } else if (a.pm_pid == 0) 3467 a.pm_pid = td->td_proc->p_pid; 3468 3469 if ((error = pmc_find_pmc(a.pm_pmc, &pm)) != 0) 3470 break; 3471 3472 if (PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pm))) { 3473 error = EINVAL; 3474 break; 3475 } 3476 3477 /* PMCs may be (re)attached only when allocated or stopped */ 3478 if (pm->pm_state == PMC_STATE_RUNNING) { 3479 error = EBUSY; 3480 break; 3481 } else if (pm->pm_state != PMC_STATE_ALLOCATED && 3482 pm->pm_state != PMC_STATE_STOPPED) { 3483 error = EINVAL; 3484 break; 3485 } 3486 3487 /* lookup pid */ 3488 if ((p = pfind(a.pm_pid)) == NULL) { 3489 error = ESRCH; 3490 break; 3491 } 3492 3493 /* 3494 * Ignore processes that are working on exiting. 3495 */ 3496 if (p->p_flag & P_WEXIT) { 3497 error = ESRCH; 3498 PROC_UNLOCK(p); /* pfind() returns a locked process */ 3499 break; 3500 } 3501 3502 /* 3503 * we are allowed to attach a PMC to a process if 3504 * we can debug it. 3505 */ 3506 error = p_candebug(curthread, p); 3507 3508 PROC_UNLOCK(p); 3509 3510 if (error == 0) 3511 error = pmc_attach_process(p, pm); 3512 } 3513 break; 3514 3515 3516 /* 3517 * Detach an attached PMC from a process. 3518 */ 3519 3520 case PMC_OP_PMCDETACH: 3521 { 3522 struct pmc *pm; 3523 struct proc *p; 3524 struct pmc_op_pmcattach a; 3525 3526 if ((error = copyin(arg, &a, sizeof(a))) != 0) 3527 break; 3528 3529 if (a.pm_pid < 0) { 3530 error = EINVAL; 3531 break; 3532 } else if (a.pm_pid == 0) 3533 a.pm_pid = td->td_proc->p_pid; 3534 3535 if ((error = pmc_find_pmc(a.pm_pmc, &pm)) != 0) 3536 break; 3537 3538 if ((p = pfind(a.pm_pid)) == NULL) { 3539 error = ESRCH; 3540 break; 3541 } 3542 3543 /* 3544 * Treat processes that are in the process of exiting 3545 * as if they were not present. 3546 */ 3547 3548 if (p->p_flag & P_WEXIT) 3549 error = ESRCH; 3550 3551 PROC_UNLOCK(p); /* pfind() returns a locked process */ 3552 3553 if (error == 0) 3554 error = pmc_detach_process(p, pm); 3555 } 3556 break; 3557 3558 3559 /* 3560 * Retrieve the MSR number associated with the counter 3561 * 'pmc_id'. This allows processes to directly use RDPMC 3562 * instructions to read their PMCs, without the overhead of a 3563 * system call. 3564 */ 3565 3566 case PMC_OP_PMCGETMSR: 3567 { 3568 int adjri, ri; 3569 struct pmc *pm; 3570 struct pmc_target *pt; 3571 struct pmc_op_getmsr gm; 3572 struct pmc_classdep *pcd; 3573 3574 PMC_DOWNGRADE_SX(); 3575 3576 if ((error = copyin(arg, &gm, sizeof(gm))) != 0) 3577 break; 3578 3579 if ((error = pmc_find_pmc(gm.pm_pmcid, &pm)) != 0) 3580 break; 3581 3582 /* 3583 * The allocated PMC has to be a process virtual PMC, 3584 * i.e., of type MODE_T[CS]. Global PMCs can only be 3585 * read using the PMCREAD operation since they may be 3586 * allocated on a different CPU than the one we could 3587 * be running on at the time of the RDPMC instruction. 3588 * 3589 * The GETMSR operation is not allowed for PMCs that 3590 * are inherited across processes. 3591 */ 3592 3593 if (!PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)) || 3594 (pm->pm_flags & PMC_F_DESCENDANTS)) { 3595 error = EINVAL; 3596 break; 3597 } 3598 3599 /* 3600 * It only makes sense to use a RDPMC (or its 3601 * equivalent instruction on non-x86 architectures) on 3602 * a process that has allocated and attached a PMC to 3603 * itself. Conversely the PMC is only allowed to have 3604 * one process attached to it -- its owner. 3605 */ 3606 3607 if ((pt = LIST_FIRST(&pm->pm_targets)) == NULL || 3608 LIST_NEXT(pt, pt_next) != NULL || 3609 pt->pt_process->pp_proc != pm->pm_owner->po_owner) { 3610 error = EINVAL; 3611 break; 3612 } 3613 3614 ri = PMC_TO_ROWINDEX(pm); 3615 pcd = pmc_ri_to_classdep(md, ri, &adjri); 3616 3617 /* PMC class has no 'GETMSR' support */ 3618 if (pcd->pcd_get_msr == NULL) { 3619 error = ENOSYS; 3620 break; 3621 } 3622 3623 if ((error = (*pcd->pcd_get_msr)(adjri, &gm.pm_msr)) < 0) 3624 break; 3625 3626 if ((error = copyout(&gm, arg, sizeof(gm))) < 0) 3627 break; 3628 3629 /* 3630 * Mark our process as using MSRs. Update machine 3631 * state using a forced context switch. 3632 */ 3633 3634 pt->pt_process->pp_flags |= PMC_PP_ENABLE_MSR_ACCESS; 3635 pmc_force_context_switch(); 3636 3637 } 3638 break; 3639 3640 /* 3641 * Release an allocated PMC 3642 */ 3643 3644 case PMC_OP_PMCRELEASE: 3645 { 3646 pmc_id_t pmcid; 3647 struct pmc *pm; 3648 struct pmc_owner *po; 3649 struct pmc_op_simple sp; 3650 3651 /* 3652 * Find PMC pointer for the named PMC. 3653 * 3654 * Use pmc_release_pmc_descriptor() to switch off the 3655 * PMC, remove all its target threads, and remove the 3656 * PMC from its owner's list. 3657 * 3658 * Remove the owner record if this is the last PMC 3659 * owned. 3660 * 3661 * Free up space. 3662 */ 3663 3664 if ((error = copyin(arg, &sp, sizeof(sp))) != 0) 3665 break; 3666 3667 pmcid = sp.pm_pmcid; 3668 3669 if ((error = pmc_find_pmc(pmcid, &pm)) != 0) 3670 break; 3671 3672 po = pm->pm_owner; 3673 pmc_release_pmc_descriptor(pm); 3674 pmc_maybe_remove_owner(po); 3675 pmc_destroy_pmc_descriptor(pm); 3676 } 3677 break; 3678 3679 3680 /* 3681 * Read and/or write a PMC. 3682 */ 3683 3684 case PMC_OP_PMCRW: 3685 { 3686 int adjri; 3687 struct pmc *pm; 3688 uint32_t cpu, ri; 3689 pmc_value_t oldvalue; 3690 struct pmc_binding pb; 3691 struct pmc_op_pmcrw prw; 3692 struct pmc_classdep *pcd; 3693 struct pmc_op_pmcrw *pprw; 3694 3695 PMC_DOWNGRADE_SX(); 3696 3697 if ((error = copyin(arg, &prw, sizeof(prw))) != 0) 3698 break; 3699 3700 ri = 0; 3701 PMCDBG(PMC,OPS,1, "rw id=%d flags=0x%x", prw.pm_pmcid, 3702 prw.pm_flags); 3703 3704 /* must have at least one flag set */ 3705 if ((prw.pm_flags & (PMC_F_OLDVALUE|PMC_F_NEWVALUE)) == 0) { 3706 error = EINVAL; 3707 break; 3708 } 3709 3710 /* locate pmc descriptor */ 3711 if ((error = pmc_find_pmc(prw.pm_pmcid, &pm)) != 0) 3712 break; 3713 3714 /* Can't read a PMC that hasn't been started. */ 3715 if (pm->pm_state != PMC_STATE_ALLOCATED && 3716 pm->pm_state != PMC_STATE_STOPPED && 3717 pm->pm_state != PMC_STATE_RUNNING) { 3718 error = EINVAL; 3719 break; 3720 } 3721 3722 /* writing a new value is allowed only for 'STOPPED' pmcs */ 3723 if (pm->pm_state == PMC_STATE_RUNNING && 3724 (prw.pm_flags & PMC_F_NEWVALUE)) { 3725 error = EBUSY; 3726 break; 3727 } 3728 3729 if (PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm))) { 3730 3731 /* 3732 * If this PMC is attached to its owner (i.e., 3733 * the process requesting this operation) and 3734 * is running, then attempt to get an 3735 * upto-date reading from hardware for a READ. 3736 * Writes are only allowed when the PMC is 3737 * stopped, so only update the saved value 3738 * field. 3739 * 3740 * If the PMC is not running, or is not 3741 * attached to its owner, read/write to the 3742 * savedvalue field. 3743 */ 3744 3745 ri = PMC_TO_ROWINDEX(pm); 3746 pcd = pmc_ri_to_classdep(md, ri, &adjri); 3747 3748 mtx_pool_lock_spin(pmc_mtxpool, pm); 3749 cpu = curthread->td_oncpu; 3750 3751 if (prw.pm_flags & PMC_F_OLDVALUE) { 3752 if ((pm->pm_flags & PMC_F_ATTACHED_TO_OWNER) && 3753 (pm->pm_state == PMC_STATE_RUNNING)) 3754 error = (*pcd->pcd_read_pmc)(cpu, adjri, 3755 &oldvalue); 3756 else 3757 oldvalue = pm->pm_gv.pm_savedvalue; 3758 } 3759 if (prw.pm_flags & PMC_F_NEWVALUE) 3760 pm->pm_gv.pm_savedvalue = prw.pm_value; 3761 3762 mtx_pool_unlock_spin(pmc_mtxpool, pm); 3763 3764 } else { /* System mode PMCs */ 3765 cpu = PMC_TO_CPU(pm); 3766 ri = PMC_TO_ROWINDEX(pm); 3767 pcd = pmc_ri_to_classdep(md, ri, &adjri); 3768 3769 if (!pmc_cpu_is_active(cpu)) { 3770 error = ENXIO; 3771 break; 3772 } 3773 3774 /* move this thread to CPU 'cpu' */ 3775 pmc_save_cpu_binding(&pb); 3776 pmc_select_cpu(cpu); 3777 3778 critical_enter(); 3779 /* save old value */ 3780 if (prw.pm_flags & PMC_F_OLDVALUE) 3781 if ((error = (*pcd->pcd_read_pmc)(cpu, adjri, 3782 &oldvalue))) 3783 goto error; 3784 /* write out new value */ 3785 if (prw.pm_flags & PMC_F_NEWVALUE) 3786 error = (*pcd->pcd_write_pmc)(cpu, adjri, 3787 prw.pm_value); 3788 error: 3789 critical_exit(); 3790 pmc_restore_cpu_binding(&pb); 3791 if (error) 3792 break; 3793 } 3794 3795 pprw = (struct pmc_op_pmcrw *) arg; 3796 3797 #ifdef DEBUG 3798 if (prw.pm_flags & PMC_F_NEWVALUE) 3799 PMCDBG(PMC,OPS,2, "rw id=%d new %jx -> old %jx", 3800 ri, prw.pm_value, oldvalue); 3801 else if (prw.pm_flags & PMC_F_OLDVALUE) 3802 PMCDBG(PMC,OPS,2, "rw id=%d -> old %jx", ri, oldvalue); 3803 #endif 3804 3805 /* return old value if requested */ 3806 if (prw.pm_flags & PMC_F_OLDVALUE) 3807 if ((error = copyout(&oldvalue, &pprw->pm_value, 3808 sizeof(prw.pm_value)))) 3809 break; 3810 3811 } 3812 break; 3813 3814 3815 /* 3816 * Set the sampling rate for a sampling mode PMC and the 3817 * initial count for a counting mode PMC. 3818 */ 3819 3820 case PMC_OP_PMCSETCOUNT: 3821 { 3822 struct pmc *pm; 3823 struct pmc_op_pmcsetcount sc; 3824 3825 PMC_DOWNGRADE_SX(); 3826 3827 if ((error = copyin(arg, &sc, sizeof(sc))) != 0) 3828 break; 3829 3830 if ((error = pmc_find_pmc(sc.pm_pmcid, &pm)) != 0) 3831 break; 3832 3833 if (pm->pm_state == PMC_STATE_RUNNING) { 3834 error = EBUSY; 3835 break; 3836 } 3837 3838 if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) 3839 pm->pm_sc.pm_reloadcount = sc.pm_count; 3840 else 3841 pm->pm_sc.pm_initial = sc.pm_count; 3842 } 3843 break; 3844 3845 3846 /* 3847 * Start a PMC. 3848 */ 3849 3850 case PMC_OP_PMCSTART: 3851 { 3852 pmc_id_t pmcid; 3853 struct pmc *pm; 3854 struct pmc_op_simple sp; 3855 3856 sx_assert(&pmc_sx, SX_XLOCKED); 3857 3858 if ((error = copyin(arg, &sp, sizeof(sp))) != 0) 3859 break; 3860 3861 pmcid = sp.pm_pmcid; 3862 3863 if ((error = pmc_find_pmc(pmcid, &pm)) != 0) 3864 break; 3865 3866 KASSERT(pmcid == pm->pm_id, 3867 ("[pmc,%d] pmcid %x != id %x", __LINE__, 3868 pm->pm_id, pmcid)); 3869 3870 if (pm->pm_state == PMC_STATE_RUNNING) /* already running */ 3871 break; 3872 else if (pm->pm_state != PMC_STATE_STOPPED && 3873 pm->pm_state != PMC_STATE_ALLOCATED) { 3874 error = EINVAL; 3875 break; 3876 } 3877 3878 error = pmc_start(pm); 3879 } 3880 break; 3881 3882 3883 /* 3884 * Stop a PMC. 3885 */ 3886 3887 case PMC_OP_PMCSTOP: 3888 { 3889 pmc_id_t pmcid; 3890 struct pmc *pm; 3891 struct pmc_op_simple sp; 3892 3893 PMC_DOWNGRADE_SX(); 3894 3895 if ((error = copyin(arg, &sp, sizeof(sp))) != 0) 3896 break; 3897 3898 pmcid = sp.pm_pmcid; 3899 3900 /* 3901 * Mark the PMC as inactive and invoke the MD stop 3902 * routines if needed. 3903 */ 3904 3905 if ((error = pmc_find_pmc(pmcid, &pm)) != 0) 3906 break; 3907 3908 KASSERT(pmcid == pm->pm_id, 3909 ("[pmc,%d] pmc id %x != pmcid %x", __LINE__, 3910 pm->pm_id, pmcid)); 3911 3912 if (pm->pm_state == PMC_STATE_STOPPED) /* already stopped */ 3913 break; 3914 else if (pm->pm_state != PMC_STATE_RUNNING) { 3915 error = EINVAL; 3916 break; 3917 } 3918 3919 error = pmc_stop(pm); 3920 } 3921 break; 3922 3923 3924 /* 3925 * Write a user supplied value to the log file. 3926 */ 3927 3928 case PMC_OP_WRITELOG: 3929 { 3930 struct pmc_op_writelog wl; 3931 struct pmc_owner *po; 3932 3933 PMC_DOWNGRADE_SX(); 3934 3935 if ((error = copyin(arg, &wl, sizeof(wl))) != 0) 3936 break; 3937 3938 if ((po = pmc_find_owner_descriptor(td->td_proc)) == NULL) { 3939 error = EINVAL; 3940 break; 3941 } 3942 3943 if ((po->po_flags & PMC_PO_OWNS_LOGFILE) == 0) { 3944 error = EINVAL; 3945 break; 3946 } 3947 3948 error = pmclog_process_userlog(po, &wl); 3949 } 3950 break; 3951 3952 3953 default: 3954 error = EINVAL; 3955 break; 3956 } 3957 3958 if (is_sx_locked != 0) { 3959 if (is_sx_downgraded) 3960 sx_sunlock(&pmc_sx); 3961 else 3962 sx_xunlock(&pmc_sx); 3963 } 3964 3965 if (error) 3966 atomic_add_int(&pmc_stats.pm_syscall_errors, 1); 3967 3968 PICKUP_GIANT(); 3969 3970 return error; 3971 } 3972 3973 /* 3974 * Helper functions 3975 */ 3976 3977 3978 /* 3979 * Mark the thread as needing callchain capture and post an AST. The 3980 * actual callchain capture will be done in a context where it is safe 3981 * to take page faults. 3982 */ 3983 3984 static void 3985 pmc_post_callchain_callback(void) 3986 { 3987 struct thread *td; 3988 3989 td = curthread; 3990 3991 /* 3992 * If there is multiple PMCs for the same interrupt ignore new post 3993 */ 3994 if (td->td_pflags & TDP_CALLCHAIN) 3995 return; 3996 3997 /* 3998 * Mark this thread as needing callchain capture. 3999 * `td->td_pflags' will be safe to touch because this thread 4000 * was in user space when it was interrupted. 4001 */ 4002 td->td_pflags |= TDP_CALLCHAIN; 4003 4004 /* 4005 * Don't let this thread migrate between CPUs until callchain 4006 * capture completes. 4007 */ 4008 sched_pin(); 4009 4010 return; 4011 } 4012 4013 /* 4014 * Interrupt processing. 4015 * 4016 * Find a free slot in the per-cpu array of samples and capture the 4017 * current callchain there. If a sample was successfully added, a bit 4018 * is set in mask 'pmc_cpumask' denoting that the DO_SAMPLES hook 4019 * needs to be invoked from the clock handler. 4020 * 4021 * This function is meant to be called from an NMI handler. It cannot 4022 * use any of the locking primitives supplied by the OS. 4023 */ 4024 4025 int 4026 pmc_process_interrupt(int cpu, int ring, struct pmc *pm, struct trapframe *tf, 4027 int inuserspace) 4028 { 4029 int error, callchaindepth; 4030 struct thread *td; 4031 struct pmc_sample *ps; 4032 struct pmc_samplebuffer *psb; 4033 4034 error = 0; 4035 4036 /* 4037 * Allocate space for a sample buffer. 4038 */ 4039 psb = pmc_pcpu[cpu]->pc_sb[ring]; 4040 4041 ps = psb->ps_write; 4042 if (ps->ps_nsamples) { /* in use, reader hasn't caught up */ 4043 pm->pm_stalled = 1; 4044 atomic_add_int(&pmc_stats.pm_intr_bufferfull, 1); 4045 PMCDBG(SAM,INT,1,"(spc) cpu=%d pm=%p tf=%p um=%d wr=%d rd=%d", 4046 cpu, pm, (void *) tf, inuserspace, 4047 (int) (psb->ps_write - psb->ps_samples), 4048 (int) (psb->ps_read - psb->ps_samples)); 4049 error = ENOMEM; 4050 goto done; 4051 } 4052 4053 4054 /* Fill in entry. */ 4055 PMCDBG(SAM,INT,1,"cpu=%d pm=%p tf=%p um=%d wr=%d rd=%d", cpu, pm, 4056 (void *) tf, inuserspace, 4057 (int) (psb->ps_write - psb->ps_samples), 4058 (int) (psb->ps_read - psb->ps_samples)); 4059 4060 KASSERT(pm->pm_runcount >= 0, 4061 ("[pmc,%d] pm=%p runcount %d", __LINE__, (void *) pm, 4062 pm->pm_runcount)); 4063 4064 atomic_add_rel_int(&pm->pm_runcount, 1); /* hold onto PMC */ 4065 4066 ps->ps_pmc = pm; 4067 if ((td = curthread) && td->td_proc) 4068 ps->ps_pid = td->td_proc->p_pid; 4069 else 4070 ps->ps_pid = -1; 4071 ps->ps_cpu = cpu; 4072 ps->ps_td = td; 4073 ps->ps_flags = inuserspace ? PMC_CC_F_USERSPACE : 0; 4074 4075 callchaindepth = (pm->pm_flags & PMC_F_CALLCHAIN) ? 4076 pmc_callchaindepth : 1; 4077 4078 if (callchaindepth == 1) 4079 ps->ps_pc[0] = PMC_TRAPFRAME_TO_PC(tf); 4080 else { 4081 /* 4082 * Kernel stack traversals can be done immediately, 4083 * while we defer to an AST for user space traversals. 4084 */ 4085 if (!inuserspace) { 4086 callchaindepth = 4087 pmc_save_kernel_callchain(ps->ps_pc, 4088 callchaindepth, tf); 4089 } else { 4090 pmc_post_callchain_callback(); 4091 callchaindepth = PMC_SAMPLE_INUSE; 4092 } 4093 } 4094 4095 ps->ps_nsamples = callchaindepth; /* mark entry as in use */ 4096 4097 /* increment write pointer, modulo ring buffer size */ 4098 ps++; 4099 if (ps == psb->ps_fence) 4100 psb->ps_write = psb->ps_samples; 4101 else 4102 psb->ps_write = ps; 4103 4104 done: 4105 /* mark CPU as needing processing */ 4106 CPU_SET_ATOMIC(cpu, &pmc_cpumask); 4107 4108 return (error); 4109 } 4110 4111 /* 4112 * Capture a user call chain. This function will be called from ast() 4113 * before control returns to userland and before the process gets 4114 * rescheduled. 4115 */ 4116 4117 static void 4118 pmc_capture_user_callchain(int cpu, int ring, struct trapframe *tf) 4119 { 4120 int i; 4121 struct pmc *pm; 4122 struct thread *td; 4123 struct pmc_sample *ps; 4124 struct pmc_samplebuffer *psb; 4125 #ifdef INVARIANTS 4126 int ncallchains; 4127 #endif 4128 4129 psb = pmc_pcpu[cpu]->pc_sb[ring]; 4130 td = curthread; 4131 4132 KASSERT(td->td_pflags & TDP_CALLCHAIN, 4133 ("[pmc,%d] Retrieving callchain for thread that doesn't want it", 4134 __LINE__)); 4135 4136 #ifdef INVARIANTS 4137 ncallchains = 0; 4138 #endif 4139 4140 /* 4141 * Iterate through all deferred callchain requests. 4142 */ 4143 4144 ps = psb->ps_samples; 4145 for (i = 0; i < pmc_nsamples; i++, ps++) { 4146 4147 if (ps->ps_nsamples != PMC_SAMPLE_INUSE) 4148 continue; 4149 if (ps->ps_td != td) 4150 continue; 4151 4152 KASSERT(ps->ps_cpu == cpu, 4153 ("[pmc,%d] cpu mismatch ps_cpu=%d pcpu=%d", __LINE__, 4154 ps->ps_cpu, PCPU_GET(cpuid))); 4155 4156 pm = ps->ps_pmc; 4157 4158 KASSERT(pm->pm_flags & PMC_F_CALLCHAIN, 4159 ("[pmc,%d] Retrieving callchain for PMC that doesn't " 4160 "want it", __LINE__)); 4161 4162 KASSERT(pm->pm_runcount > 0, 4163 ("[pmc,%d] runcount %d", __LINE__, pm->pm_runcount)); 4164 4165 /* 4166 * Retrieve the callchain and mark the sample buffer 4167 * as 'processable' by the timer tick sweep code. 4168 */ 4169 ps->ps_nsamples = pmc_save_user_callchain(ps->ps_pc, 4170 pmc_callchaindepth, tf); 4171 4172 #ifdef INVARIANTS 4173 ncallchains++; 4174 #endif 4175 } 4176 4177 KASSERT(ncallchains > 0, 4178 ("[pmc,%d] cpu %d didn't find a sample to collect", __LINE__, 4179 cpu)); 4180 4181 KASSERT(td->td_pinned == 1, 4182 ("[pmc,%d] invalid td_pinned value", __LINE__)); 4183 sched_unpin(); /* Can migrate safely now. */ 4184 4185 return; 4186 } 4187 4188 /* 4189 * Process saved PC samples. 4190 */ 4191 4192 static void 4193 pmc_process_samples(int cpu, int ring) 4194 { 4195 struct pmc *pm; 4196 int adjri, n; 4197 struct thread *td; 4198 struct pmc_owner *po; 4199 struct pmc_sample *ps; 4200 struct pmc_classdep *pcd; 4201 struct pmc_samplebuffer *psb; 4202 4203 KASSERT(PCPU_GET(cpuid) == cpu, 4204 ("[pmc,%d] not on the correct CPU pcpu=%d cpu=%d", __LINE__, 4205 PCPU_GET(cpuid), cpu)); 4206 4207 psb = pmc_pcpu[cpu]->pc_sb[ring]; 4208 4209 for (n = 0; n < pmc_nsamples; n++) { /* bound on #iterations */ 4210 4211 ps = psb->ps_read; 4212 if (ps->ps_nsamples == PMC_SAMPLE_FREE) 4213 break; 4214 4215 pm = ps->ps_pmc; 4216 4217 KASSERT(pm->pm_runcount > 0, 4218 ("[pmc,%d] pm=%p runcount %d", __LINE__, (void *) pm, 4219 pm->pm_runcount)); 4220 4221 po = pm->pm_owner; 4222 4223 KASSERT(PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)), 4224 ("[pmc,%d] pmc=%p non-sampling mode=%d", __LINE__, 4225 pm, PMC_TO_MODE(pm))); 4226 4227 /* Ignore PMCs that have been switched off */ 4228 if (pm->pm_state != PMC_STATE_RUNNING) 4229 goto entrydone; 4230 4231 /* If there is a pending AST wait for completion */ 4232 if (ps->ps_nsamples == PMC_SAMPLE_INUSE) { 4233 /* Need a rescan at a later time. */ 4234 CPU_SET_ATOMIC(cpu, &pmc_cpumask); 4235 break; 4236 } 4237 4238 PMCDBG(SAM,OPS,1,"cpu=%d pm=%p n=%d fl=%x wr=%d rd=%d", cpu, 4239 pm, ps->ps_nsamples, ps->ps_flags, 4240 (int) (psb->ps_write - psb->ps_samples), 4241 (int) (psb->ps_read - psb->ps_samples)); 4242 4243 /* 4244 * If this is a process-mode PMC that is attached to 4245 * its owner, and if the PC is in user mode, update 4246 * profiling statistics like timer-based profiling 4247 * would have done. 4248 */ 4249 if (pm->pm_flags & PMC_F_ATTACHED_TO_OWNER) { 4250 if (ps->ps_flags & PMC_CC_F_USERSPACE) { 4251 td = FIRST_THREAD_IN_PROC(po->po_owner); 4252 addupc_intr(td, ps->ps_pc[0], 1); 4253 } 4254 goto entrydone; 4255 } 4256 4257 /* 4258 * Otherwise, this is either a sampling mode PMC that 4259 * is attached to a different process than its owner, 4260 * or a system-wide sampling PMC. Dispatch a log 4261 * entry to the PMC's owner process. 4262 */ 4263 pmclog_process_callchain(pm, ps); 4264 4265 entrydone: 4266 ps->ps_nsamples = 0; /* mark entry as free */ 4267 atomic_subtract_rel_int(&pm->pm_runcount, 1); 4268 4269 /* increment read pointer, modulo sample size */ 4270 if (++ps == psb->ps_fence) 4271 psb->ps_read = psb->ps_samples; 4272 else 4273 psb->ps_read = ps; 4274 } 4275 4276 atomic_add_int(&pmc_stats.pm_log_sweeps, 1); 4277 4278 /* Do not re-enable stalled PMCs if we failed to process any samples */ 4279 if (n == 0) 4280 return; 4281 4282 /* 4283 * Restart any stalled sampling PMCs on this CPU. 4284 * 4285 * If the NMI handler sets the pm_stalled field of a PMC after 4286 * the check below, we'll end up processing the stalled PMC at 4287 * the next hardclock tick. 4288 */ 4289 for (n = 0; n < md->pmd_npmc; n++) { 4290 pcd = pmc_ri_to_classdep(md, n, &adjri); 4291 KASSERT(pcd != NULL, 4292 ("[pmc,%d] null pcd ri=%d", __LINE__, n)); 4293 (void) (*pcd->pcd_get_config)(cpu,adjri,&pm); 4294 4295 if (pm == NULL || /* !cfg'ed */ 4296 pm->pm_state != PMC_STATE_RUNNING || /* !active */ 4297 !PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)) || /* !sampling */ 4298 pm->pm_stalled == 0) /* !stalled */ 4299 continue; 4300 4301 pm->pm_stalled = 0; 4302 (*pcd->pcd_start_pmc)(cpu, adjri); 4303 } 4304 } 4305 4306 /* 4307 * Event handlers. 4308 */ 4309 4310 /* 4311 * Handle a process exit. 4312 * 4313 * Remove this process from all hash tables. If this process 4314 * owned any PMCs, turn off those PMCs and deallocate them, 4315 * removing any associations with target processes. 4316 * 4317 * This function will be called by the last 'thread' of a 4318 * process. 4319 * 4320 * XXX This eventhandler gets called early in the exit process. 4321 * Consider using a 'hook' invocation from thread_exit() or equivalent 4322 * spot. Another negative is that kse_exit doesn't seem to call 4323 * exit1() [??]. 4324 * 4325 */ 4326 4327 static void 4328 pmc_process_exit(void *arg __unused, struct proc *p) 4329 { 4330 struct pmc *pm; 4331 int adjri, cpu; 4332 unsigned int ri; 4333 int is_using_hwpmcs; 4334 struct pmc_owner *po; 4335 struct pmc_process *pp; 4336 struct pmc_classdep *pcd; 4337 pmc_value_t newvalue, tmp; 4338 4339 PROC_LOCK(p); 4340 is_using_hwpmcs = p->p_flag & P_HWPMC; 4341 PROC_UNLOCK(p); 4342 4343 /* 4344 * Log a sysexit event to all SS PMC owners. 4345 */ 4346 LIST_FOREACH(po, &pmc_ss_owners, po_ssnext) 4347 if (po->po_flags & PMC_PO_OWNS_LOGFILE) 4348 pmclog_process_sysexit(po, p->p_pid); 4349 4350 if (!is_using_hwpmcs) 4351 return; 4352 4353 PMC_GET_SX_XLOCK(); 4354 PMCDBG(PRC,EXT,1,"process-exit proc=%p (%d, %s)", p, p->p_pid, 4355 p->p_comm); 4356 4357 /* 4358 * Since this code is invoked by the last thread in an exiting 4359 * process, we would have context switched IN at some prior 4360 * point. However, with PREEMPTION, kernel mode context 4361 * switches may happen any time, so we want to disable a 4362 * context switch OUT till we get any PMCs targetting this 4363 * process off the hardware. 4364 * 4365 * We also need to atomically remove this process' 4366 * entry from our target process hash table, using 4367 * PMC_FLAG_REMOVE. 4368 */ 4369 PMCDBG(PRC,EXT,1, "process-exit proc=%p (%d, %s)", p, p->p_pid, 4370 p->p_comm); 4371 4372 critical_enter(); /* no preemption */ 4373 4374 cpu = curthread->td_oncpu; 4375 4376 if ((pp = pmc_find_process_descriptor(p, 4377 PMC_FLAG_REMOVE)) != NULL) { 4378 4379 PMCDBG(PRC,EXT,2, 4380 "process-exit proc=%p pmc-process=%p", p, pp); 4381 4382 /* 4383 * The exiting process could the target of 4384 * some PMCs which will be running on 4385 * currently executing CPU. 4386 * 4387 * We need to turn these PMCs off like we 4388 * would do at context switch OUT time. 4389 */ 4390 for (ri = 0; ri < md->pmd_npmc; ri++) { 4391 4392 /* 4393 * Pick up the pmc pointer from hardware 4394 * state similar to the CSW_OUT code. 4395 */ 4396 pm = NULL; 4397 4398 pcd = pmc_ri_to_classdep(md, ri, &adjri); 4399 4400 (void) (*pcd->pcd_get_config)(cpu, adjri, &pm); 4401 4402 PMCDBG(PRC,EXT,2, "ri=%d pm=%p", ri, pm); 4403 4404 if (pm == NULL || 4405 !PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm))) 4406 continue; 4407 4408 PMCDBG(PRC,EXT,2, "ppmcs[%d]=%p pm=%p " 4409 "state=%d", ri, pp->pp_pmcs[ri].pp_pmc, 4410 pm, pm->pm_state); 4411 4412 KASSERT(PMC_TO_ROWINDEX(pm) == ri, 4413 ("[pmc,%d] ri mismatch pmc(%d) ri(%d)", 4414 __LINE__, PMC_TO_ROWINDEX(pm), ri)); 4415 4416 KASSERT(pm == pp->pp_pmcs[ri].pp_pmc, 4417 ("[pmc,%d] pm %p != pp_pmcs[%d] %p", 4418 __LINE__, pm, ri, pp->pp_pmcs[ri].pp_pmc)); 4419 4420 (void) pcd->pcd_stop_pmc(cpu, adjri); 4421 4422 KASSERT(pm->pm_runcount > 0, 4423 ("[pmc,%d] bad runcount ri %d rc %d", 4424 __LINE__, ri, pm->pm_runcount)); 4425 4426 /* Stop hardware only if it is actually running */ 4427 if (pm->pm_state == PMC_STATE_RUNNING && 4428 pm->pm_stalled == 0) { 4429 pcd->pcd_read_pmc(cpu, adjri, &newvalue); 4430 tmp = newvalue - 4431 PMC_PCPU_SAVED(cpu,ri); 4432 4433 mtx_pool_lock_spin(pmc_mtxpool, pm); 4434 pm->pm_gv.pm_savedvalue += tmp; 4435 pp->pp_pmcs[ri].pp_pmcval += tmp; 4436 mtx_pool_unlock_spin(pmc_mtxpool, pm); 4437 } 4438 4439 atomic_subtract_rel_int(&pm->pm_runcount,1); 4440 4441 KASSERT((int) pm->pm_runcount >= 0, 4442 ("[pmc,%d] runcount is %d", __LINE__, ri)); 4443 4444 (void) pcd->pcd_config_pmc(cpu, adjri, NULL); 4445 } 4446 4447 /* 4448 * Inform the MD layer of this pseudo "context switch 4449 * out" 4450 */ 4451 (void) md->pmd_switch_out(pmc_pcpu[cpu], pp); 4452 4453 critical_exit(); /* ok to be pre-empted now */ 4454 4455 /* 4456 * Unlink this process from the PMCs that are 4457 * targetting it. This will send a signal to 4458 * all PMC owner's whose PMCs are orphaned. 4459 * 4460 * Log PMC value at exit time if requested. 4461 */ 4462 for (ri = 0; ri < md->pmd_npmc; ri++) 4463 if ((pm = pp->pp_pmcs[ri].pp_pmc) != NULL) { 4464 if (pm->pm_flags & PMC_F_NEEDS_LOGFILE && 4465 PMC_IS_COUNTING_MODE(PMC_TO_MODE(pm))) 4466 pmclog_process_procexit(pm, pp); 4467 pmc_unlink_target_process(pm, pp); 4468 } 4469 free(pp, M_PMC); 4470 4471 } else 4472 critical_exit(); /* pp == NULL */ 4473 4474 4475 /* 4476 * If the process owned PMCs, free them up and free up 4477 * memory. 4478 */ 4479 if ((po = pmc_find_owner_descriptor(p)) != NULL) { 4480 pmc_remove_owner(po); 4481 pmc_destroy_owner_descriptor(po); 4482 } 4483 4484 sx_xunlock(&pmc_sx); 4485 } 4486 4487 /* 4488 * Handle a process fork. 4489 * 4490 * If the parent process 'p1' is under HWPMC monitoring, then copy 4491 * over any attached PMCs that have 'do_descendants' semantics. 4492 */ 4493 4494 static void 4495 pmc_process_fork(void *arg __unused, struct proc *p1, struct proc *newproc, 4496 int flags) 4497 { 4498 int is_using_hwpmcs; 4499 unsigned int ri; 4500 uint32_t do_descendants; 4501 struct pmc *pm; 4502 struct pmc_owner *po; 4503 struct pmc_process *ppnew, *ppold; 4504 4505 (void) flags; /* unused parameter */ 4506 4507 PROC_LOCK(p1); 4508 is_using_hwpmcs = p1->p_flag & P_HWPMC; 4509 PROC_UNLOCK(p1); 4510 4511 /* 4512 * If there are system-wide sampling PMCs active, we need to 4513 * log all fork events to their owner's logs. 4514 */ 4515 4516 LIST_FOREACH(po, &pmc_ss_owners, po_ssnext) 4517 if (po->po_flags & PMC_PO_OWNS_LOGFILE) 4518 pmclog_process_procfork(po, p1->p_pid, newproc->p_pid); 4519 4520 if (!is_using_hwpmcs) 4521 return; 4522 4523 PMC_GET_SX_XLOCK(); 4524 PMCDBG(PMC,FRK,1, "process-fork proc=%p (%d, %s) -> %p", p1, 4525 p1->p_pid, p1->p_comm, newproc); 4526 4527 /* 4528 * If the parent process (curthread->td_proc) is a 4529 * target of any PMCs, look for PMCs that are to be 4530 * inherited, and link these into the new process 4531 * descriptor. 4532 */ 4533 if ((ppold = pmc_find_process_descriptor(curthread->td_proc, 4534 PMC_FLAG_NONE)) == NULL) 4535 goto done; /* nothing to do */ 4536 4537 do_descendants = 0; 4538 for (ri = 0; ri < md->pmd_npmc; ri++) 4539 if ((pm = ppold->pp_pmcs[ri].pp_pmc) != NULL) 4540 do_descendants |= pm->pm_flags & PMC_F_DESCENDANTS; 4541 if (do_descendants == 0) /* nothing to do */ 4542 goto done; 4543 4544 /* allocate a descriptor for the new process */ 4545 if ((ppnew = pmc_find_process_descriptor(newproc, 4546 PMC_FLAG_ALLOCATE)) == NULL) 4547 goto done; 4548 4549 /* 4550 * Run through all PMCs that were targeting the old process 4551 * and which specified F_DESCENDANTS and attach them to the 4552 * new process. 4553 * 4554 * Log the fork event to all owners of PMCs attached to this 4555 * process, if not already logged. 4556 */ 4557 for (ri = 0; ri < md->pmd_npmc; ri++) 4558 if ((pm = ppold->pp_pmcs[ri].pp_pmc) != NULL && 4559 (pm->pm_flags & PMC_F_DESCENDANTS)) { 4560 pmc_link_target_process(pm, ppnew); 4561 po = pm->pm_owner; 4562 if (po->po_sscount == 0 && 4563 po->po_flags & PMC_PO_OWNS_LOGFILE) 4564 pmclog_process_procfork(po, p1->p_pid, 4565 newproc->p_pid); 4566 } 4567 4568 /* 4569 * Now mark the new process as being tracked by this driver. 4570 */ 4571 PROC_LOCK(newproc); 4572 newproc->p_flag |= P_HWPMC; 4573 PROC_UNLOCK(newproc); 4574 4575 done: 4576 sx_xunlock(&pmc_sx); 4577 } 4578 4579 static void 4580 pmc_kld_load(void *arg __unused, linker_file_t lf) 4581 { 4582 struct pmc_owner *po; 4583 4584 sx_slock(&pmc_sx); 4585 4586 /* 4587 * Notify owners of system sampling PMCs about KLD operations. 4588 */ 4589 LIST_FOREACH(po, &pmc_ss_owners, po_ssnext) 4590 if (po->po_flags & PMC_PO_OWNS_LOGFILE) 4591 pmclog_process_map_in(po, (pid_t) -1, 4592 (uintfptr_t) lf->address, lf->filename); 4593 4594 /* 4595 * TODO: Notify owners of (all) process-sampling PMCs too. 4596 */ 4597 4598 sx_sunlock(&pmc_sx); 4599 } 4600 4601 static void 4602 pmc_kld_unload(void *arg __unused, const char *filename __unused, 4603 caddr_t address, size_t size) 4604 { 4605 struct pmc_owner *po; 4606 4607 sx_slock(&pmc_sx); 4608 4609 LIST_FOREACH(po, &pmc_ss_owners, po_ssnext) 4610 if (po->po_flags & PMC_PO_OWNS_LOGFILE) 4611 pmclog_process_map_out(po, (pid_t) -1, 4612 (uintfptr_t) address, (uintfptr_t) address + size); 4613 4614 /* 4615 * TODO: Notify owners of process-sampling PMCs. 4616 */ 4617 4618 sx_sunlock(&pmc_sx); 4619 } 4620 4621 /* 4622 * initialization 4623 */ 4624 4625 static const char *pmc_name_of_pmcclass[] = { 4626 #undef __PMC_CLASS 4627 #define __PMC_CLASS(N) #N , 4628 __PMC_CLASSES() 4629 }; 4630 4631 /* 4632 * Base class initializer: allocate structure and set default classes. 4633 */ 4634 struct pmc_mdep * 4635 pmc_mdep_alloc(int nclasses) 4636 { 4637 struct pmc_mdep *md; 4638 int n; 4639 4640 /* SOFT + md classes */ 4641 n = 1 + nclasses; 4642 md = malloc(sizeof(struct pmc_mdep) + n * 4643 sizeof(struct pmc_classdep), M_PMC, M_WAITOK|M_ZERO); 4644 md->pmd_nclass = n; 4645 4646 /* Add base class. */ 4647 pmc_soft_initialize(md); 4648 return md; 4649 } 4650 4651 void 4652 pmc_mdep_free(struct pmc_mdep *md) 4653 { 4654 pmc_soft_finalize(md); 4655 free(md, M_PMC); 4656 } 4657 4658 static int 4659 generic_switch_in(struct pmc_cpu *pc, struct pmc_process *pp) 4660 { 4661 (void) pc; (void) pp; 4662 4663 return (0); 4664 } 4665 4666 static int 4667 generic_switch_out(struct pmc_cpu *pc, struct pmc_process *pp) 4668 { 4669 (void) pc; (void) pp; 4670 4671 return (0); 4672 } 4673 4674 static struct pmc_mdep * 4675 pmc_generic_cpu_initialize(void) 4676 { 4677 struct pmc_mdep *md; 4678 4679 md = pmc_mdep_alloc(0); 4680 4681 md->pmd_cputype = PMC_CPU_GENERIC; 4682 4683 md->pmd_pcpu_init = NULL; 4684 md->pmd_pcpu_fini = NULL; 4685 md->pmd_switch_in = generic_switch_in; 4686 md->pmd_switch_out = generic_switch_out; 4687 4688 return (md); 4689 } 4690 4691 static void 4692 pmc_generic_cpu_finalize(struct pmc_mdep *md) 4693 { 4694 (void) md; 4695 } 4696 4697 4698 static int 4699 pmc_initialize(void) 4700 { 4701 int c, cpu, error, n, ri; 4702 unsigned int maxcpu; 4703 struct pmc_binding pb; 4704 struct pmc_sample *ps; 4705 struct pmc_classdep *pcd; 4706 struct pmc_samplebuffer *sb; 4707 4708 md = NULL; 4709 error = 0; 4710 4711 #ifdef DEBUG 4712 /* parse debug flags first */ 4713 if (TUNABLE_STR_FETCH(PMC_SYSCTL_NAME_PREFIX "debugflags", 4714 pmc_debugstr, sizeof(pmc_debugstr))) 4715 pmc_debugflags_parse(pmc_debugstr, 4716 pmc_debugstr+strlen(pmc_debugstr)); 4717 #endif 4718 4719 PMCDBG(MOD,INI,0, "PMC Initialize (version %x)", PMC_VERSION); 4720 4721 /* check kernel version */ 4722 if (pmc_kernel_version != PMC_VERSION) { 4723 if (pmc_kernel_version == 0) 4724 printf("hwpmc: this kernel has not been compiled with " 4725 "'options HWPMC_HOOKS'.\n"); 4726 else 4727 printf("hwpmc: kernel version (0x%x) does not match " 4728 "module version (0x%x).\n", pmc_kernel_version, 4729 PMC_VERSION); 4730 return EPROGMISMATCH; 4731 } 4732 4733 /* 4734 * check sysctl parameters 4735 */ 4736 4737 if (pmc_hashsize <= 0) { 4738 (void) printf("hwpmc: tunable \"hashsize\"=%d must be " 4739 "greater than zero.\n", pmc_hashsize); 4740 pmc_hashsize = PMC_HASH_SIZE; 4741 } 4742 4743 if (pmc_nsamples <= 0 || pmc_nsamples > 65535) { 4744 (void) printf("hwpmc: tunable \"nsamples\"=%d out of " 4745 "range.\n", pmc_nsamples); 4746 pmc_nsamples = PMC_NSAMPLES; 4747 } 4748 4749 if (pmc_callchaindepth <= 0 || 4750 pmc_callchaindepth > PMC_CALLCHAIN_DEPTH_MAX) { 4751 (void) printf("hwpmc: tunable \"callchaindepth\"=%d out of " 4752 "range.\n", pmc_callchaindepth); 4753 pmc_callchaindepth = PMC_CALLCHAIN_DEPTH; 4754 } 4755 4756 md = pmc_md_initialize(); 4757 if (md == NULL) { 4758 /* Default to generic CPU. */ 4759 md = pmc_generic_cpu_initialize(); 4760 if (md == NULL) 4761 return (ENOSYS); 4762 } 4763 4764 KASSERT(md->pmd_nclass >= 1 && md->pmd_npmc >= 1, 4765 ("[pmc,%d] no classes or pmcs", __LINE__)); 4766 4767 /* Compute the map from row-indices to classdep pointers. */ 4768 pmc_rowindex_to_classdep = malloc(sizeof(struct pmc_classdep *) * 4769 md->pmd_npmc, M_PMC, M_WAITOK|M_ZERO); 4770 4771 for (n = 0; n < md->pmd_npmc; n++) 4772 pmc_rowindex_to_classdep[n] = NULL; 4773 for (ri = c = 0; c < md->pmd_nclass; c++) { 4774 pcd = &md->pmd_classdep[c]; 4775 for (n = 0; n < pcd->pcd_num; n++, ri++) 4776 pmc_rowindex_to_classdep[ri] = pcd; 4777 } 4778 4779 KASSERT(ri == md->pmd_npmc, 4780 ("[pmc,%d] npmc miscomputed: ri=%d, md->npmc=%d", __LINE__, 4781 ri, md->pmd_npmc)); 4782 4783 maxcpu = pmc_cpu_max(); 4784 4785 /* allocate space for the per-cpu array */ 4786 pmc_pcpu = malloc(maxcpu * sizeof(struct pmc_cpu *), M_PMC, 4787 M_WAITOK|M_ZERO); 4788 4789 /* per-cpu 'saved values' for managing process-mode PMCs */ 4790 pmc_pcpu_saved = malloc(sizeof(pmc_value_t) * maxcpu * md->pmd_npmc, 4791 M_PMC, M_WAITOK); 4792 4793 /* Perform CPU-dependent initialization. */ 4794 pmc_save_cpu_binding(&pb); 4795 error = 0; 4796 for (cpu = 0; error == 0 && cpu < maxcpu; cpu++) { 4797 if (!pmc_cpu_is_active(cpu)) 4798 continue; 4799 pmc_select_cpu(cpu); 4800 pmc_pcpu[cpu] = malloc(sizeof(struct pmc_cpu) + 4801 md->pmd_npmc * sizeof(struct pmc_hw *), M_PMC, 4802 M_WAITOK|M_ZERO); 4803 if (md->pmd_pcpu_init) 4804 error = md->pmd_pcpu_init(md, cpu); 4805 for (n = 0; error == 0 && n < md->pmd_nclass; n++) 4806 error = md->pmd_classdep[n].pcd_pcpu_init(md, cpu); 4807 } 4808 pmc_restore_cpu_binding(&pb); 4809 4810 if (error) 4811 return (error); 4812 4813 /* allocate space for the sample array */ 4814 for (cpu = 0; cpu < maxcpu; cpu++) { 4815 if (!pmc_cpu_is_active(cpu)) 4816 continue; 4817 4818 sb = malloc(sizeof(struct pmc_samplebuffer) + 4819 pmc_nsamples * sizeof(struct pmc_sample), M_PMC, 4820 M_WAITOK|M_ZERO); 4821 sb->ps_read = sb->ps_write = sb->ps_samples; 4822 sb->ps_fence = sb->ps_samples + pmc_nsamples; 4823 4824 KASSERT(pmc_pcpu[cpu] != NULL, 4825 ("[pmc,%d] cpu=%d Null per-cpu data", __LINE__, cpu)); 4826 4827 sb->ps_callchains = malloc(pmc_callchaindepth * pmc_nsamples * 4828 sizeof(uintptr_t), M_PMC, M_WAITOK|M_ZERO); 4829 4830 for (n = 0, ps = sb->ps_samples; n < pmc_nsamples; n++, ps++) 4831 ps->ps_pc = sb->ps_callchains + 4832 (n * pmc_callchaindepth); 4833 4834 pmc_pcpu[cpu]->pc_sb[PMC_HR] = sb; 4835 4836 sb = malloc(sizeof(struct pmc_samplebuffer) + 4837 pmc_nsamples * sizeof(struct pmc_sample), M_PMC, 4838 M_WAITOK|M_ZERO); 4839 sb->ps_read = sb->ps_write = sb->ps_samples; 4840 sb->ps_fence = sb->ps_samples + pmc_nsamples; 4841 4842 KASSERT(pmc_pcpu[cpu] != NULL, 4843 ("[pmc,%d] cpu=%d Null per-cpu data", __LINE__, cpu)); 4844 4845 sb->ps_callchains = malloc(pmc_callchaindepth * pmc_nsamples * 4846 sizeof(uintptr_t), M_PMC, M_WAITOK|M_ZERO); 4847 4848 for (n = 0, ps = sb->ps_samples; n < pmc_nsamples; n++, ps++) 4849 ps->ps_pc = sb->ps_callchains + 4850 (n * pmc_callchaindepth); 4851 4852 pmc_pcpu[cpu]->pc_sb[PMC_SR] = sb; 4853 } 4854 4855 /* allocate space for the row disposition array */ 4856 pmc_pmcdisp = malloc(sizeof(enum pmc_mode) * md->pmd_npmc, 4857 M_PMC, M_WAITOK|M_ZERO); 4858 4859 /* mark all PMCs as available */ 4860 for (n = 0; n < (int) md->pmd_npmc; n++) 4861 PMC_MARK_ROW_FREE(n); 4862 4863 /* allocate thread hash tables */ 4864 pmc_ownerhash = hashinit(pmc_hashsize, M_PMC, 4865 &pmc_ownerhashmask); 4866 4867 pmc_processhash = hashinit(pmc_hashsize, M_PMC, 4868 &pmc_processhashmask); 4869 mtx_init(&pmc_processhash_mtx, "pmc-process-hash", "pmc-leaf", 4870 MTX_SPIN); 4871 4872 LIST_INIT(&pmc_ss_owners); 4873 pmc_ss_count = 0; 4874 4875 /* allocate a pool of spin mutexes */ 4876 pmc_mtxpool = mtx_pool_create("pmc-leaf", pmc_mtxpool_size, 4877 MTX_SPIN); 4878 4879 PMCDBG(MOD,INI,1, "pmc_ownerhash=%p, mask=0x%lx " 4880 "targethash=%p mask=0x%lx", pmc_ownerhash, pmc_ownerhashmask, 4881 pmc_processhash, pmc_processhashmask); 4882 4883 /* register process {exit,fork,exec} handlers */ 4884 pmc_exit_tag = EVENTHANDLER_REGISTER(process_exit, 4885 pmc_process_exit, NULL, EVENTHANDLER_PRI_ANY); 4886 pmc_fork_tag = EVENTHANDLER_REGISTER(process_fork, 4887 pmc_process_fork, NULL, EVENTHANDLER_PRI_ANY); 4888 4889 /* register kld event handlers */ 4890 pmc_kld_load_tag = EVENTHANDLER_REGISTER(kld_load, pmc_kld_load, 4891 NULL, EVENTHANDLER_PRI_ANY); 4892 pmc_kld_unload_tag = EVENTHANDLER_REGISTER(kld_unload, pmc_kld_unload, 4893 NULL, EVENTHANDLER_PRI_ANY); 4894 4895 /* initialize logging */ 4896 pmclog_initialize(); 4897 4898 /* set hook functions */ 4899 pmc_intr = md->pmd_intr; 4900 pmc_hook = pmc_hook_handler; 4901 4902 if (error == 0) { 4903 printf(PMC_MODULE_NAME ":"); 4904 for (n = 0; n < (int) md->pmd_nclass; n++) { 4905 pcd = &md->pmd_classdep[n]; 4906 printf(" %s/%d/%d/0x%b", 4907 pmc_name_of_pmcclass[pcd->pcd_class], 4908 pcd->pcd_num, 4909 pcd->pcd_width, 4910 pcd->pcd_caps, 4911 "\20" 4912 "\1INT\2USR\3SYS\4EDG\5THR" 4913 "\6REA\7WRI\10INV\11QUA\12PRC" 4914 "\13TAG\14CSC"); 4915 } 4916 printf("\n"); 4917 } 4918 4919 return (error); 4920 } 4921 4922 /* prepare to be unloaded */ 4923 static void 4924 pmc_cleanup(void) 4925 { 4926 int c, cpu; 4927 unsigned int maxcpu; 4928 struct pmc_ownerhash *ph; 4929 struct pmc_owner *po, *tmp; 4930 struct pmc_binding pb; 4931 #ifdef DEBUG 4932 struct pmc_processhash *prh; 4933 #endif 4934 4935 PMCDBG(MOD,INI,0, "%s", "cleanup"); 4936 4937 /* switch off sampling */ 4938 CPU_ZERO(&pmc_cpumask); 4939 pmc_intr = NULL; 4940 4941 sx_xlock(&pmc_sx); 4942 if (pmc_hook == NULL) { /* being unloaded already */ 4943 sx_xunlock(&pmc_sx); 4944 return; 4945 } 4946 4947 pmc_hook = NULL; /* prevent new threads from entering module */ 4948 4949 /* deregister event handlers */ 4950 EVENTHANDLER_DEREGISTER(process_fork, pmc_fork_tag); 4951 EVENTHANDLER_DEREGISTER(process_exit, pmc_exit_tag); 4952 EVENTHANDLER_DEREGISTER(kld_load, pmc_kld_load_tag); 4953 EVENTHANDLER_DEREGISTER(kld_unload, pmc_kld_unload_tag); 4954 4955 /* send SIGBUS to all owner threads, free up allocations */ 4956 if (pmc_ownerhash) 4957 for (ph = pmc_ownerhash; 4958 ph <= &pmc_ownerhash[pmc_ownerhashmask]; 4959 ph++) { 4960 LIST_FOREACH_SAFE(po, ph, po_next, tmp) { 4961 pmc_remove_owner(po); 4962 4963 /* send SIGBUS to owner processes */ 4964 PMCDBG(MOD,INI,2, "cleanup signal proc=%p " 4965 "(%d, %s)", po->po_owner, 4966 po->po_owner->p_pid, 4967 po->po_owner->p_comm); 4968 4969 PROC_LOCK(po->po_owner); 4970 kern_psignal(po->po_owner, SIGBUS); 4971 PROC_UNLOCK(po->po_owner); 4972 4973 pmc_destroy_owner_descriptor(po); 4974 } 4975 } 4976 4977 /* reclaim allocated data structures */ 4978 if (pmc_mtxpool) 4979 mtx_pool_destroy(&pmc_mtxpool); 4980 4981 mtx_destroy(&pmc_processhash_mtx); 4982 if (pmc_processhash) { 4983 #ifdef DEBUG 4984 struct pmc_process *pp; 4985 4986 PMCDBG(MOD,INI,3, "%s", "destroy process hash"); 4987 for (prh = pmc_processhash; 4988 prh <= &pmc_processhash[pmc_processhashmask]; 4989 prh++) 4990 LIST_FOREACH(pp, prh, pp_next) 4991 PMCDBG(MOD,INI,3, "pid=%d", pp->pp_proc->p_pid); 4992 #endif 4993 4994 hashdestroy(pmc_processhash, M_PMC, pmc_processhashmask); 4995 pmc_processhash = NULL; 4996 } 4997 4998 if (pmc_ownerhash) { 4999 PMCDBG(MOD,INI,3, "%s", "destroy owner hash"); 5000 hashdestroy(pmc_ownerhash, M_PMC, pmc_ownerhashmask); 5001 pmc_ownerhash = NULL; 5002 } 5003 5004 KASSERT(LIST_EMPTY(&pmc_ss_owners), 5005 ("[pmc,%d] Global SS owner list not empty", __LINE__)); 5006 KASSERT(pmc_ss_count == 0, 5007 ("[pmc,%d] Global SS count not empty", __LINE__)); 5008 5009 /* do processor and pmc-class dependent cleanup */ 5010 maxcpu = pmc_cpu_max(); 5011 5012 PMCDBG(MOD,INI,3, "%s", "md cleanup"); 5013 if (md) { 5014 pmc_save_cpu_binding(&pb); 5015 for (cpu = 0; cpu < maxcpu; cpu++) { 5016 PMCDBG(MOD,INI,1,"pmc-cleanup cpu=%d pcs=%p", 5017 cpu, pmc_pcpu[cpu]); 5018 if (!pmc_cpu_is_active(cpu) || pmc_pcpu[cpu] == NULL) 5019 continue; 5020 pmc_select_cpu(cpu); 5021 for (c = 0; c < md->pmd_nclass; c++) 5022 md->pmd_classdep[c].pcd_pcpu_fini(md, cpu); 5023 if (md->pmd_pcpu_fini) 5024 md->pmd_pcpu_fini(md, cpu); 5025 } 5026 5027 if (md->pmd_cputype == PMC_CPU_GENERIC) 5028 pmc_generic_cpu_finalize(md); 5029 else 5030 pmc_md_finalize(md); 5031 5032 pmc_mdep_free(md); 5033 md = NULL; 5034 pmc_restore_cpu_binding(&pb); 5035 } 5036 5037 /* Free per-cpu descriptors. */ 5038 for (cpu = 0; cpu < maxcpu; cpu++) { 5039 if (!pmc_cpu_is_active(cpu)) 5040 continue; 5041 KASSERT(pmc_pcpu[cpu]->pc_sb[PMC_HR] != NULL, 5042 ("[pmc,%d] Null hw cpu sample buffer cpu=%d", __LINE__, 5043 cpu)); 5044 KASSERT(pmc_pcpu[cpu]->pc_sb[PMC_SR] != NULL, 5045 ("[pmc,%d] Null sw cpu sample buffer cpu=%d", __LINE__, 5046 cpu)); 5047 free(pmc_pcpu[cpu]->pc_sb[PMC_HR]->ps_callchains, M_PMC); 5048 free(pmc_pcpu[cpu]->pc_sb[PMC_HR], M_PMC); 5049 free(pmc_pcpu[cpu]->pc_sb[PMC_SR]->ps_callchains, M_PMC); 5050 free(pmc_pcpu[cpu]->pc_sb[PMC_SR], M_PMC); 5051 free(pmc_pcpu[cpu], M_PMC); 5052 } 5053 5054 free(pmc_pcpu, M_PMC); 5055 pmc_pcpu = NULL; 5056 5057 free(pmc_pcpu_saved, M_PMC); 5058 pmc_pcpu_saved = NULL; 5059 5060 if (pmc_pmcdisp) { 5061 free(pmc_pmcdisp, M_PMC); 5062 pmc_pmcdisp = NULL; 5063 } 5064 5065 if (pmc_rowindex_to_classdep) { 5066 free(pmc_rowindex_to_classdep, M_PMC); 5067 pmc_rowindex_to_classdep = NULL; 5068 } 5069 5070 pmclog_shutdown(); 5071 5072 sx_xunlock(&pmc_sx); /* we are done */ 5073 } 5074 5075 /* 5076 * The function called at load/unload. 5077 */ 5078 5079 static int 5080 load (struct module *module __unused, int cmd, void *arg __unused) 5081 { 5082 int error; 5083 5084 error = 0; 5085 5086 switch (cmd) { 5087 case MOD_LOAD : 5088 /* initialize the subsystem */ 5089 error = pmc_initialize(); 5090 if (error != 0) 5091 break; 5092 PMCDBG(MOD,INI,1, "syscall=%d maxcpu=%d", 5093 pmc_syscall_num, pmc_cpu_max()); 5094 break; 5095 5096 5097 case MOD_UNLOAD : 5098 case MOD_SHUTDOWN: 5099 pmc_cleanup(); 5100 PMCDBG(MOD,INI,1, "%s", "unloaded"); 5101 break; 5102 5103 default : 5104 error = EINVAL; /* XXX should panic(9) */ 5105 break; 5106 } 5107 5108 return error; 5109 } 5110 5111 /* memory pool */ 5112 MALLOC_DEFINE(M_PMC, "pmc", "Memory space for the PMC module"); 5113