1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2012 Fabien Thomas 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 29 #include <sys/cdefs.h> 30 #include <sys/param.h> 31 #include <sys/pmc.h> 32 #include <sys/pmckern.h> 33 #include <sys/systm.h> 34 #include <sys/mutex.h> 35 36 #include <machine/cpu.h> 37 #include <machine/cpufunc.h> 38 39 #include "hwpmc_soft.h" 40 41 /* 42 * Software PMC support. 43 */ 44 45 #define SOFT_CAPS (PMC_CAP_READ | PMC_CAP_WRITE | PMC_CAP_INTERRUPT | \ 46 PMC_CAP_USER | PMC_CAP_SYSTEM) 47 48 struct soft_descr { 49 struct pmc_descr pm_descr; /* "base class" */ 50 }; 51 52 static struct soft_descr soft_pmcdesc[SOFT_NPMCS] = 53 { 54 #define SOFT_PMCDESCR(N) \ 55 { \ 56 .pm_descr = \ 57 { \ 58 .pd_name = #N, \ 59 .pd_class = PMC_CLASS_SOFT, \ 60 .pd_caps = SOFT_CAPS, \ 61 .pd_width = 64 \ 62 }, \ 63 } 64 65 SOFT_PMCDESCR(SOFT0), 66 SOFT_PMCDESCR(SOFT1), 67 SOFT_PMCDESCR(SOFT2), 68 SOFT_PMCDESCR(SOFT3), 69 SOFT_PMCDESCR(SOFT4), 70 SOFT_PMCDESCR(SOFT5), 71 SOFT_PMCDESCR(SOFT6), 72 SOFT_PMCDESCR(SOFT7), 73 SOFT_PMCDESCR(SOFT8), 74 SOFT_PMCDESCR(SOFT9), 75 SOFT_PMCDESCR(SOFT10), 76 SOFT_PMCDESCR(SOFT11), 77 SOFT_PMCDESCR(SOFT12), 78 SOFT_PMCDESCR(SOFT13), 79 SOFT_PMCDESCR(SOFT14), 80 SOFT_PMCDESCR(SOFT15) 81 }; 82 83 /* 84 * Per-CPU data structure. 85 */ 86 87 struct soft_cpu { 88 struct pmc_hw soft_hw[SOFT_NPMCS]; 89 pmc_value_t soft_values[SOFT_NPMCS]; 90 }; 91 92 93 static struct soft_cpu **soft_pcpu; 94 95 static int 96 soft_allocate_pmc(int cpu, int ri, struct pmc *pm, 97 const struct pmc_op_pmcallocate *a) 98 { 99 enum pmc_event ev; 100 struct pmc_soft *ps; 101 102 (void) cpu; 103 104 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 105 ("[soft,%d] illegal CPU value %d", __LINE__, cpu)); 106 KASSERT(ri >= 0 && ri < SOFT_NPMCS, 107 ("[soft,%d] illegal row-index %d", __LINE__, ri)); 108 109 if (a->pm_class != PMC_CLASS_SOFT) 110 return (EINVAL); 111 112 if ((pm->pm_caps & SOFT_CAPS) == 0) 113 return (EINVAL); 114 115 if ((pm->pm_caps & ~SOFT_CAPS) != 0) 116 return (EPERM); 117 118 ev = pm->pm_event; 119 if ((int)ev < PMC_EV_SOFT_FIRST || (int)ev > PMC_EV_SOFT_LAST) 120 return (EINVAL); 121 122 /* Check if event is registered. */ 123 ps = pmc_soft_ev_acquire(ev); 124 if (ps == NULL) 125 return (EINVAL); 126 pmc_soft_ev_release(ps); 127 /* Module unload is protected by pmc SX lock. */ 128 if (ps->ps_alloc != NULL) 129 ps->ps_alloc(); 130 131 return (0); 132 } 133 134 static int 135 soft_config_pmc(int cpu, int ri, struct pmc *pm) 136 { 137 struct pmc_hw *phw; 138 139 PMCDBG3(MDP,CFG,1, "cpu=%d ri=%d pm=%p", cpu, ri, pm); 140 141 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 142 ("[soft,%d] illegal CPU value %d", __LINE__, cpu)); 143 KASSERT(ri >= 0 && ri < SOFT_NPMCS, 144 ("[soft,%d] illegal row-index %d", __LINE__, ri)); 145 146 phw = &soft_pcpu[cpu]->soft_hw[ri]; 147 148 KASSERT(pm == NULL || phw->phw_pmc == NULL, 149 ("[soft,%d] pm=%p phw->pm=%p hwpmc not unconfigured", __LINE__, 150 pm, phw->phw_pmc)); 151 152 phw->phw_pmc = pm; 153 154 return (0); 155 } 156 157 static int 158 soft_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc) 159 { 160 const struct soft_descr *pd; 161 struct pmc_hw *phw; 162 163 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 164 ("[soft,%d] illegal CPU %d", __LINE__, cpu)); 165 KASSERT(ri >= 0 && ri < SOFT_NPMCS, 166 ("[soft,%d] illegal row-index %d", __LINE__, ri)); 167 168 phw = &soft_pcpu[cpu]->soft_hw[ri]; 169 pd = &soft_pmcdesc[ri]; 170 171 strlcpy(pi->pm_name, pd->pm_descr.pd_name, sizeof(pi->pm_name)); 172 pi->pm_class = pd->pm_descr.pd_class; 173 174 if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) { 175 pi->pm_enabled = TRUE; 176 *ppmc = phw->phw_pmc; 177 } else { 178 pi->pm_enabled = FALSE; 179 *ppmc = NULL; 180 } 181 182 return (0); 183 } 184 185 static int 186 soft_get_config(int cpu, int ri, struct pmc **ppm) 187 { 188 (void) ri; 189 190 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 191 ("[soft,%d] illegal CPU %d", __LINE__, cpu)); 192 KASSERT(ri >= 0 && ri < SOFT_NPMCS, 193 ("[soft,%d] illegal row-index %d", __LINE__, ri)); 194 195 *ppm = soft_pcpu[cpu]->soft_hw[ri].phw_pmc; 196 return (0); 197 } 198 199 static int 200 soft_pcpu_fini(struct pmc_mdep *md, int cpu) 201 { 202 int ri; 203 struct pmc_cpu *pc; 204 205 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 206 ("[soft,%d] illegal cpu %d", __LINE__, cpu)); 207 KASSERT(soft_pcpu[cpu] != NULL, ("[soft,%d] null pcpu", __LINE__)); 208 209 free(soft_pcpu[cpu], M_PMC); 210 soft_pcpu[cpu] = NULL; 211 212 ri = md->pmd_classdep[PMC_CLASS_INDEX_SOFT].pcd_ri; 213 214 KASSERT(ri >= 0 && ri < SOFT_NPMCS, 215 ("[soft,%d] ri=%d", __LINE__, ri)); 216 217 pc = pmc_pcpu[cpu]; 218 pc->pc_hwpmcs[ri] = NULL; 219 220 return (0); 221 } 222 223 static int 224 soft_pcpu_init(struct pmc_mdep *md, int cpu) 225 { 226 int first_ri, n; 227 struct pmc_cpu *pc; 228 struct soft_cpu *soft_pc; 229 struct pmc_hw *phw; 230 231 232 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 233 ("[soft,%d] illegal cpu %d", __LINE__, cpu)); 234 KASSERT(soft_pcpu, ("[soft,%d] null pcpu", __LINE__)); 235 KASSERT(soft_pcpu[cpu] == NULL, ("[soft,%d] non-null per-cpu", 236 __LINE__)); 237 238 soft_pc = malloc(sizeof(struct soft_cpu), M_PMC, M_WAITOK|M_ZERO); 239 pc = pmc_pcpu[cpu]; 240 241 KASSERT(pc != NULL, ("[soft,%d] cpu %d null per-cpu", __LINE__, cpu)); 242 243 soft_pcpu[cpu] = soft_pc; 244 phw = soft_pc->soft_hw; 245 first_ri = md->pmd_classdep[PMC_CLASS_INDEX_SOFT].pcd_ri; 246 247 for (n = 0; n < SOFT_NPMCS; n++, phw++) { 248 phw->phw_state = PMC_PHW_FLAG_IS_ENABLED | 249 PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(n); 250 phw->phw_pmc = NULL; 251 pc->pc_hwpmcs[n + first_ri] = phw; 252 } 253 254 return (0); 255 } 256 257 static int 258 soft_read_pmc(int cpu, int ri, struct pmc *pm __unused, pmc_value_t *v) 259 { 260 261 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 262 ("[soft,%d] illegal CPU value %d", __LINE__, cpu)); 263 KASSERT(ri >= 0 && ri < SOFT_NPMCS, 264 ("[soft,%d] illegal row-index %d", __LINE__, ri)); 265 266 PMCDBG1(MDP,REA,1,"soft-read id=%d", ri); 267 268 *v = soft_pcpu[cpu]->soft_values[ri]; 269 270 return (0); 271 } 272 273 static int 274 soft_write_pmc(int cpu, int ri, struct pmc *pm __unused, pmc_value_t v) 275 { 276 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 277 ("[soft,%d] illegal cpu value %d", __LINE__, cpu)); 278 KASSERT(ri >= 0 && ri < SOFT_NPMCS, 279 ("[soft,%d] illegal row-index %d", __LINE__, ri)); 280 281 PMCDBG3(MDP,WRI,1, "soft-write cpu=%d ri=%d v=%jx", cpu, ri, v); 282 283 soft_pcpu[cpu]->soft_values[ri] = v; 284 285 return (0); 286 } 287 288 static int 289 soft_release_pmc(int cpu, int ri, struct pmc *pmc) 290 { 291 struct pmc_hw *phw __diagused; 292 enum pmc_event ev; 293 struct pmc_soft *ps; 294 295 (void) pmc; 296 297 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 298 ("[soft,%d] illegal CPU value %d", __LINE__, cpu)); 299 KASSERT(ri >= 0 && ri < SOFT_NPMCS, 300 ("[soft,%d] illegal row-index %d", __LINE__, ri)); 301 302 phw = &soft_pcpu[cpu]->soft_hw[ri]; 303 304 KASSERT(phw->phw_pmc == NULL, 305 ("[soft,%d] PHW pmc %p non-NULL", __LINE__, phw->phw_pmc)); 306 307 ev = pmc->pm_event; 308 309 /* Check if event is registered. */ 310 ps = pmc_soft_ev_acquire(ev); 311 KASSERT(ps != NULL, 312 ("[soft,%d] unregistered event %d", __LINE__, ev)); 313 pmc_soft_ev_release(ps); 314 /* Module unload is protected by pmc SX lock. */ 315 if (ps->ps_release != NULL) 316 ps->ps_release(); 317 return (0); 318 } 319 320 static int 321 soft_start_pmc(int cpu, int ri, struct pmc *pm) 322 { 323 struct pmc_soft *ps; 324 325 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 326 ("[soft,%d] illegal CPU value %d", __LINE__, cpu)); 327 KASSERT(ri >= 0 && ri < SOFT_NPMCS, 328 ("[soft,%d] illegal row-index %d", __LINE__, ri)); 329 330 ps = pmc_soft_ev_acquire(pm->pm_event); 331 if (ps == NULL) 332 return (EINVAL); 333 atomic_add_int(&ps->ps_running, 1); 334 pmc_soft_ev_release(ps); 335 336 return (0); 337 } 338 339 static int 340 soft_stop_pmc(int cpu, int ri, struct pmc *pm) 341 { 342 struct pmc_soft *ps; 343 344 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 345 ("[soft,%d] illegal CPU value %d", __LINE__, cpu)); 346 KASSERT(ri >= 0 && ri < SOFT_NPMCS, 347 ("[soft,%d] illegal row-index %d", __LINE__, ri)); 348 349 ps = pmc_soft_ev_acquire(pm->pm_event); 350 /* event unregistered ? */ 351 if (ps != NULL) { 352 atomic_subtract_int(&ps->ps_running, 1); 353 pmc_soft_ev_release(ps); 354 } 355 356 return (0); 357 } 358 359 int 360 pmc_soft_intr(struct pmckern_soft *ks) 361 { 362 struct pmc *pm; 363 struct soft_cpu *pc; 364 int ri, processed, error, user_mode; 365 366 KASSERT(ks->pm_cpu >= 0 && ks->pm_cpu < pmc_cpu_max(), 367 ("[soft,%d] CPU %d out of range", __LINE__, ks->pm_cpu)); 368 369 processed = 0; 370 pc = soft_pcpu[ks->pm_cpu]; 371 372 for (ri = 0; ri < SOFT_NPMCS; ri++) { 373 374 pm = pc->soft_hw[ri].phw_pmc; 375 if (pm == NULL || 376 pm->pm_state != PMC_STATE_RUNNING || 377 pm->pm_event != ks->pm_ev) { 378 continue; 379 } 380 381 processed = 1; 382 if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) { 383 if ((pc->soft_values[ri]--) <= 0) 384 pc->soft_values[ri] += pm->pm_sc.pm_reloadcount; 385 else 386 continue; 387 user_mode = TRAPF_USERMODE(ks->pm_tf); 388 error = pmc_process_interrupt(PMC_SR, pm, ks->pm_tf); 389 if (error) { 390 soft_stop_pmc(ks->pm_cpu, ri, pm); 391 continue; 392 } 393 394 if (user_mode) { 395 /* 396 * If in user mode setup AST to process 397 * callchain out of interrupt context. 398 */ 399 ast_sched(curthread, TDA_HWPMC); 400 } 401 } else 402 pc->soft_values[ri]++; 403 } 404 if (processed) 405 counter_u64_add(pmc_stats.pm_intr_processed, 1); 406 else 407 counter_u64_add(pmc_stats.pm_intr_ignored, 1); 408 409 return (processed); 410 } 411 412 static void 413 ast_hwpmc(struct thread *td, int tda __unused) 414 { 415 /* Handle Software PMC callchain capture. */ 416 if (PMC_IS_PENDING_CALLCHAIN(td)) 417 PMC_CALL_HOOK_UNLOCKED(td, PMC_FN_USER_CALLCHAIN_SOFT, 418 (void *)td->td_frame); 419 } 420 421 void 422 pmc_soft_initialize(struct pmc_mdep *md) 423 { 424 struct pmc_classdep *pcd; 425 426 /* Add SOFT PMCs. */ 427 soft_pcpu = malloc(sizeof(struct soft_cpu *) * pmc_cpu_max(), M_PMC, 428 M_ZERO|M_WAITOK); 429 430 pcd = &md->pmd_classdep[PMC_CLASS_INDEX_SOFT]; 431 432 pcd->pcd_caps = SOFT_CAPS; 433 pcd->pcd_class = PMC_CLASS_SOFT; 434 pcd->pcd_num = SOFT_NPMCS; 435 pcd->pcd_ri = md->pmd_npmc; 436 pcd->pcd_width = 64; 437 438 pcd->pcd_allocate_pmc = soft_allocate_pmc; 439 pcd->pcd_config_pmc = soft_config_pmc; 440 pcd->pcd_describe = soft_describe; 441 pcd->pcd_get_config = soft_get_config; 442 pcd->pcd_get_msr = NULL; 443 pcd->pcd_pcpu_init = soft_pcpu_init; 444 pcd->pcd_pcpu_fini = soft_pcpu_fini; 445 pcd->pcd_read_pmc = soft_read_pmc; 446 pcd->pcd_write_pmc = soft_write_pmc; 447 pcd->pcd_release_pmc = soft_release_pmc; 448 pcd->pcd_start_pmc = soft_start_pmc; 449 pcd->pcd_stop_pmc = soft_stop_pmc; 450 451 md->pmd_npmc += SOFT_NPMCS; 452 453 ast_register(TDA_HWPMC, ASTR_UNCOND, 0, ast_hwpmc); 454 } 455 456 void 457 pmc_soft_finalize(struct pmc_mdep *md) 458 { 459 #ifdef INVARIANTS 460 int i, ncpus; 461 462 ncpus = pmc_cpu_max(); 463 for (i = 0; i < ncpus; i++) 464 KASSERT(soft_pcpu[i] == NULL, ("[soft,%d] non-null pcpu cpu %d", 465 __LINE__, i)); 466 467 KASSERT(md->pmd_classdep[PMC_CLASS_INDEX_SOFT].pcd_class == 468 PMC_CLASS_SOFT, ("[soft,%d] class mismatch", __LINE__)); 469 #endif 470 ast_deregister(TDA_HWPMC); 471 free(soft_pcpu, M_PMC); 472 soft_pcpu = NULL; 473 } 474