1 /*- 2 * Copyright (c) 2012 Fabien Thomas 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 27 #include <sys/cdefs.h> 28 __FBSDID("$FreeBSD$"); 29 30 #include <sys/param.h> 31 #include <sys/pmc.h> 32 #include <sys/pmckern.h> 33 #include <sys/systm.h> 34 #include <sys/mutex.h> 35 36 #include <machine/cpu.h> 37 #include <machine/cpufunc.h> 38 39 #include "hwpmc_soft.h" 40 41 /* 42 * Software PMC support. 43 */ 44 45 #define SOFT_CAPS (PMC_CAP_READ | PMC_CAP_WRITE | PMC_CAP_INTERRUPT | \ 46 PMC_CAP_USER | PMC_CAP_SYSTEM) 47 48 PMC_SOFT_DECLARE( , , clock, prof); 49 50 struct soft_descr { 51 struct pmc_descr pm_descr; /* "base class" */ 52 }; 53 54 static struct soft_descr soft_pmcdesc[SOFT_NPMCS] = 55 { 56 #define SOFT_PMCDESCR(N) \ 57 { \ 58 .pm_descr = \ 59 { \ 60 .pd_name = #N, \ 61 .pd_class = PMC_CLASS_SOFT, \ 62 .pd_caps = SOFT_CAPS, \ 63 .pd_width = 64 \ 64 }, \ 65 } 66 67 SOFT_PMCDESCR(SOFT0), 68 SOFT_PMCDESCR(SOFT1), 69 SOFT_PMCDESCR(SOFT2), 70 SOFT_PMCDESCR(SOFT3), 71 SOFT_PMCDESCR(SOFT4), 72 SOFT_PMCDESCR(SOFT5), 73 SOFT_PMCDESCR(SOFT6), 74 SOFT_PMCDESCR(SOFT7), 75 SOFT_PMCDESCR(SOFT8), 76 SOFT_PMCDESCR(SOFT9), 77 SOFT_PMCDESCR(SOFT10), 78 SOFT_PMCDESCR(SOFT11), 79 SOFT_PMCDESCR(SOFT12), 80 SOFT_PMCDESCR(SOFT13), 81 SOFT_PMCDESCR(SOFT14), 82 SOFT_PMCDESCR(SOFT15) 83 }; 84 85 /* 86 * Per-CPU data structure. 87 */ 88 89 struct soft_cpu { 90 struct pmc_hw soft_hw[SOFT_NPMCS]; 91 pmc_value_t soft_values[SOFT_NPMCS]; 92 }; 93 94 95 static struct soft_cpu **soft_pcpu; 96 97 static int 98 soft_allocate_pmc(int cpu, int ri, struct pmc *pm, 99 const struct pmc_op_pmcallocate *a) 100 { 101 enum pmc_event ev; 102 struct pmc_soft *ps; 103 104 (void) cpu; 105 106 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 107 ("[soft,%d] illegal CPU value %d", __LINE__, cpu)); 108 KASSERT(ri >= 0 && ri < SOFT_NPMCS, 109 ("[soft,%d] illegal row-index %d", __LINE__, ri)); 110 111 if (a->pm_class != PMC_CLASS_SOFT) 112 return (EINVAL); 113 114 if ((pm->pm_caps & SOFT_CAPS) == 0) 115 return (EINVAL); 116 117 if ((pm->pm_caps & ~SOFT_CAPS) != 0) 118 return (EPERM); 119 120 ev = pm->pm_event; 121 if ((int)ev < PMC_EV_SOFT_FIRST || (int)ev > PMC_EV_SOFT_LAST) 122 return (EINVAL); 123 124 /* Check if event is registered. */ 125 ps = pmc_soft_ev_acquire(ev); 126 if (ps == NULL) 127 return (EINVAL); 128 pmc_soft_ev_release(ps); 129 130 if (ev == pmc___clock_prof.ps_ev.pm_ev_code) 131 cpu_startprofclock(); 132 return (0); 133 } 134 135 static int 136 soft_config_pmc(int cpu, int ri, struct pmc *pm) 137 { 138 struct pmc_hw *phw; 139 140 PMCDBG(MDP,CFG,1, "cpu=%d ri=%d pm=%p", cpu, ri, pm); 141 142 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 143 ("[soft,%d] illegal CPU value %d", __LINE__, cpu)); 144 KASSERT(ri >= 0 && ri < SOFT_NPMCS, 145 ("[soft,%d] illegal row-index %d", __LINE__, ri)); 146 147 phw = &soft_pcpu[cpu]->soft_hw[ri]; 148 149 KASSERT(pm == NULL || phw->phw_pmc == NULL, 150 ("[soft,%d] pm=%p phw->pm=%p hwpmc not unconfigured", __LINE__, 151 pm, phw->phw_pmc)); 152 153 phw->phw_pmc = pm; 154 155 return (0); 156 } 157 158 static int 159 soft_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc) 160 { 161 int error; 162 size_t copied; 163 const struct soft_descr *pd; 164 struct pmc_hw *phw; 165 166 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 167 ("[soft,%d] illegal CPU %d", __LINE__, cpu)); 168 KASSERT(ri >= 0 && ri < SOFT_NPMCS, 169 ("[soft,%d] illegal row-index %d", __LINE__, ri)); 170 171 phw = &soft_pcpu[cpu]->soft_hw[ri]; 172 pd = &soft_pmcdesc[ri]; 173 174 if ((error = copystr(pd->pm_descr.pd_name, pi->pm_name, 175 PMC_NAME_MAX, &copied)) != 0) 176 return (error); 177 178 pi->pm_class = pd->pm_descr.pd_class; 179 180 if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) { 181 pi->pm_enabled = TRUE; 182 *ppmc = phw->phw_pmc; 183 } else { 184 pi->pm_enabled = FALSE; 185 *ppmc = NULL; 186 } 187 188 return (0); 189 } 190 191 static int 192 soft_get_config(int cpu, int ri, struct pmc **ppm) 193 { 194 (void) ri; 195 196 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 197 ("[soft,%d] illegal CPU %d", __LINE__, cpu)); 198 KASSERT(ri >= 0 && ri < SOFT_NPMCS, 199 ("[soft,%d] illegal row-index %d", __LINE__, ri)); 200 201 *ppm = soft_pcpu[cpu]->soft_hw[ri].phw_pmc; 202 return (0); 203 } 204 205 static int 206 soft_pcpu_fini(struct pmc_mdep *md, int cpu) 207 { 208 int ri; 209 struct pmc_cpu *pc; 210 211 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 212 ("[soft,%d] illegal cpu %d", __LINE__, cpu)); 213 KASSERT(soft_pcpu[cpu] != NULL, ("[soft,%d] null pcpu", __LINE__)); 214 215 free(soft_pcpu[cpu], M_PMC); 216 soft_pcpu[cpu] = NULL; 217 218 ri = md->pmd_classdep[PMC_CLASS_INDEX_SOFT].pcd_ri; 219 220 KASSERT(ri >= 0 && ri < SOFT_NPMCS, 221 ("[soft,%d] ri=%d", __LINE__, ri)); 222 223 pc = pmc_pcpu[cpu]; 224 pc->pc_hwpmcs[ri] = NULL; 225 226 return (0); 227 } 228 229 static int 230 soft_pcpu_init(struct pmc_mdep *md, int cpu) 231 { 232 int first_ri, n; 233 struct pmc_cpu *pc; 234 struct soft_cpu *soft_pc; 235 struct pmc_hw *phw; 236 237 238 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 239 ("[soft,%d] illegal cpu %d", __LINE__, cpu)); 240 KASSERT(soft_pcpu, ("[soft,%d] null pcpu", __LINE__)); 241 KASSERT(soft_pcpu[cpu] == NULL, ("[soft,%d] non-null per-cpu", 242 __LINE__)); 243 244 soft_pc = malloc(sizeof(struct soft_cpu), M_PMC, M_WAITOK|M_ZERO); 245 if (soft_pc == NULL) 246 return (ENOMEM); 247 248 pc = pmc_pcpu[cpu]; 249 250 KASSERT(pc != NULL, ("[soft,%d] cpu %d null per-cpu", __LINE__, cpu)); 251 252 soft_pcpu[cpu] = soft_pc; 253 phw = soft_pc->soft_hw; 254 first_ri = md->pmd_classdep[PMC_CLASS_INDEX_SOFT].pcd_ri; 255 256 for (n = 0; n < SOFT_NPMCS; n++, phw++) { 257 phw->phw_state = PMC_PHW_FLAG_IS_ENABLED | 258 PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(n); 259 phw->phw_pmc = NULL; 260 pc->pc_hwpmcs[n + first_ri] = phw; 261 } 262 263 return (0); 264 } 265 266 static int 267 soft_read_pmc(int cpu, int ri, pmc_value_t *v) 268 { 269 struct pmc *pm; 270 const struct pmc_hw *phw; 271 272 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 273 ("[soft,%d] illegal CPU value %d", __LINE__, cpu)); 274 KASSERT(ri >= 0 && ri < SOFT_NPMCS, 275 ("[soft,%d] illegal row-index %d", __LINE__, ri)); 276 277 phw = &soft_pcpu[cpu]->soft_hw[ri]; 278 pm = phw->phw_pmc; 279 280 KASSERT(pm != NULL, 281 ("[soft,%d] no owner for PHW [cpu%d,pmc%d]", __LINE__, cpu, ri)); 282 283 PMCDBG(MDP,REA,1,"soft-read id=%d", ri); 284 285 *v = soft_pcpu[cpu]->soft_values[ri]; 286 287 return (0); 288 } 289 290 static int 291 soft_write_pmc(int cpu, int ri, pmc_value_t v) 292 { 293 struct pmc *pm; 294 const struct soft_descr *pd; 295 296 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 297 ("[soft,%d] illegal cpu value %d", __LINE__, cpu)); 298 KASSERT(ri >= 0 && ri < SOFT_NPMCS, 299 ("[soft,%d] illegal row-index %d", __LINE__, ri)); 300 301 pm = soft_pcpu[cpu]->soft_hw[ri].phw_pmc; 302 pd = &soft_pmcdesc[ri]; 303 304 KASSERT(pm, 305 ("[soft,%d] cpu %d ri %d pmc not configured", __LINE__, cpu, ri)); 306 307 PMCDBG(MDP,WRI,1, "soft-write cpu=%d ri=%d v=%jx", cpu, ri, v); 308 309 soft_pcpu[cpu]->soft_values[ri] = v; 310 311 return (0); 312 } 313 314 static int 315 soft_release_pmc(int cpu, int ri, struct pmc *pmc) 316 { 317 struct pmc_hw *phw; 318 319 (void) pmc; 320 321 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 322 ("[soft,%d] illegal CPU value %d", __LINE__, cpu)); 323 KASSERT(ri >= 0 && ri < SOFT_NPMCS, 324 ("[soft,%d] illegal row-index %d", __LINE__, ri)); 325 326 phw = &soft_pcpu[cpu]->soft_hw[ri]; 327 328 KASSERT(phw->phw_pmc == NULL, 329 ("[soft,%d] PHW pmc %p non-NULL", __LINE__, phw->phw_pmc)); 330 331 if (pmc->pm_event == pmc___clock_prof.ps_ev.pm_ev_code) 332 cpu_stopprofclock(); 333 return (0); 334 } 335 336 static int 337 soft_start_pmc(int cpu, int ri) 338 { 339 struct pmc *pm; 340 struct soft_cpu *pc; 341 struct pmc_soft *ps; 342 343 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 344 ("[soft,%d] illegal CPU value %d", __LINE__, cpu)); 345 KASSERT(ri >= 0 && ri < SOFT_NPMCS, 346 ("[soft,%d] illegal row-index %d", __LINE__, ri)); 347 348 pc = soft_pcpu[cpu]; 349 pm = pc->soft_hw[ri].phw_pmc; 350 351 KASSERT(pm, 352 ("[soft,%d] cpu %d ri %d pmc not configured", __LINE__, cpu, ri)); 353 354 ps = pmc_soft_ev_acquire(pm->pm_event); 355 if (ps == NULL) 356 return (EINVAL); 357 atomic_add_int(&ps->ps_running, 1); 358 pmc_soft_ev_release(ps); 359 360 return (0); 361 } 362 363 static int 364 soft_stop_pmc(int cpu, int ri) 365 { 366 struct pmc *pm; 367 struct soft_cpu *pc; 368 struct pmc_soft *ps; 369 370 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 371 ("[soft,%d] illegal CPU value %d", __LINE__, cpu)); 372 KASSERT(ri >= 0 && ri < SOFT_NPMCS, 373 ("[soft,%d] illegal row-index %d", __LINE__, ri)); 374 375 pc = soft_pcpu[cpu]; 376 pm = pc->soft_hw[ri].phw_pmc; 377 378 KASSERT(pm, 379 ("[soft,%d] cpu %d ri %d pmc not configured", __LINE__, cpu, ri)); 380 381 ps = pmc_soft_ev_acquire(pm->pm_event); 382 /* event unregistered ? */ 383 if (ps != NULL) { 384 atomic_subtract_int(&ps->ps_running, 1); 385 pmc_soft_ev_release(ps); 386 } 387 388 return (0); 389 } 390 391 int 392 pmc_soft_intr(struct pmckern_soft *ks) 393 { 394 struct pmc *pm; 395 struct soft_cpu *pc; 396 int ri, processed, error, user_mode; 397 398 KASSERT(ks->pm_cpu >= 0 && ks->pm_cpu < pmc_cpu_max(), 399 ("[soft,%d] CPU %d out of range", __LINE__, ks->pm_cpu)); 400 401 processed = 0; 402 pc = soft_pcpu[ks->pm_cpu]; 403 404 for (ri = 0; ri < SOFT_NPMCS; ri++) { 405 406 pm = pc->soft_hw[ri].phw_pmc; 407 if (pm == NULL || 408 pm->pm_state != PMC_STATE_RUNNING || 409 pm->pm_event != ks->pm_ev) { 410 continue; 411 } 412 413 processed = 1; 414 if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) { 415 if ((pc->soft_values[ri]--) <= 0) 416 pc->soft_values[ri] += pm->pm_sc.pm_reloadcount; 417 else 418 continue; 419 user_mode = TRAPF_USERMODE(ks->pm_tf); 420 error = pmc_process_interrupt(ks->pm_cpu, PMC_SR, pm, 421 ks->pm_tf, user_mode); 422 if (error) { 423 soft_stop_pmc(ks->pm_cpu, ri); 424 continue; 425 } 426 427 if (user_mode) { 428 /* If in user mode setup AST to process 429 * callchain out of interrupt context. 430 */ 431 curthread->td_flags |= TDF_ASTPENDING; 432 } 433 } else 434 pc->soft_values[ri]++; 435 } 436 437 atomic_add_int(processed ? &pmc_stats.pm_intr_processed : 438 &pmc_stats.pm_intr_ignored, 1); 439 440 return (processed); 441 } 442 443 void 444 pmc_soft_initialize(struct pmc_mdep *md) 445 { 446 struct pmc_classdep *pcd; 447 448 /* Add SOFT PMCs. */ 449 soft_pcpu = malloc(sizeof(struct soft_cpu *) * pmc_cpu_max(), M_PMC, 450 M_ZERO|M_WAITOK); 451 452 pcd = &md->pmd_classdep[PMC_CLASS_INDEX_SOFT]; 453 454 pcd->pcd_caps = SOFT_CAPS; 455 pcd->pcd_class = PMC_CLASS_SOFT; 456 pcd->pcd_num = SOFT_NPMCS; 457 pcd->pcd_ri = md->pmd_npmc; 458 pcd->pcd_width = 64; 459 460 pcd->pcd_allocate_pmc = soft_allocate_pmc; 461 pcd->pcd_config_pmc = soft_config_pmc; 462 pcd->pcd_describe = soft_describe; 463 pcd->pcd_get_config = soft_get_config; 464 pcd->pcd_get_msr = NULL; 465 pcd->pcd_pcpu_init = soft_pcpu_init; 466 pcd->pcd_pcpu_fini = soft_pcpu_fini; 467 pcd->pcd_read_pmc = soft_read_pmc; 468 pcd->pcd_write_pmc = soft_write_pmc; 469 pcd->pcd_release_pmc = soft_release_pmc; 470 pcd->pcd_start_pmc = soft_start_pmc; 471 pcd->pcd_stop_pmc = soft_stop_pmc; 472 473 md->pmd_npmc += SOFT_NPMCS; 474 } 475 476 void 477 pmc_soft_finalize(struct pmc_mdep *md) 478 { 479 #ifdef INVARIANTS 480 int i, ncpus; 481 482 ncpus = pmc_cpu_max(); 483 for (i = 0; i < ncpus; i++) 484 KASSERT(soft_pcpu[i] == NULL, ("[soft,%d] non-null pcpu cpu %d", 485 __LINE__, i)); 486 487 KASSERT(md->pmd_classdep[PMC_CLASS_INDEX_SOFT].pcd_class == 488 PMC_CLASS_SOFT, ("[soft,%d] class mismatch", __LINE__)); 489 #endif 490 free(soft_pcpu, M_PMC); 491 soft_pcpu = NULL; 492 } 493