1 /*- 2 * Copyright (c) 2012 Fabien Thomas 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 27 #include <sys/cdefs.h> 28 __FBSDID("$FreeBSD$"); 29 30 #include <sys/param.h> 31 #include <sys/pmc.h> 32 #include <sys/pmckern.h> 33 #include <sys/systm.h> 34 #include <sys/mutex.h> 35 36 #include <machine/cpu.h> 37 #include <machine/cpufunc.h> 38 39 #include "hwpmc_soft.h" 40 41 /* 42 * Software PMC support. 43 */ 44 45 #define SOFT_CAPS (PMC_CAP_READ | PMC_CAP_WRITE | PMC_CAP_INTERRUPT | \ 46 PMC_CAP_USER | PMC_CAP_SYSTEM) 47 48 struct soft_descr { 49 struct pmc_descr pm_descr; /* "base class" */ 50 }; 51 52 static struct soft_descr soft_pmcdesc[SOFT_NPMCS] = 53 { 54 #define SOFT_PMCDESCR(N) \ 55 { \ 56 .pm_descr = \ 57 { \ 58 .pd_name = #N, \ 59 .pd_class = PMC_CLASS_SOFT, \ 60 .pd_caps = SOFT_CAPS, \ 61 .pd_width = 64 \ 62 }, \ 63 } 64 65 SOFT_PMCDESCR(SOFT0), 66 SOFT_PMCDESCR(SOFT1), 67 SOFT_PMCDESCR(SOFT2), 68 SOFT_PMCDESCR(SOFT3), 69 SOFT_PMCDESCR(SOFT4), 70 SOFT_PMCDESCR(SOFT5), 71 SOFT_PMCDESCR(SOFT6), 72 SOFT_PMCDESCR(SOFT7), 73 SOFT_PMCDESCR(SOFT8), 74 SOFT_PMCDESCR(SOFT9), 75 SOFT_PMCDESCR(SOFT10), 76 SOFT_PMCDESCR(SOFT11), 77 SOFT_PMCDESCR(SOFT12), 78 SOFT_PMCDESCR(SOFT13), 79 SOFT_PMCDESCR(SOFT14), 80 SOFT_PMCDESCR(SOFT15) 81 }; 82 83 /* 84 * Per-CPU data structure. 85 */ 86 87 struct soft_cpu { 88 struct pmc_hw soft_hw[SOFT_NPMCS]; 89 pmc_value_t soft_values[SOFT_NPMCS]; 90 }; 91 92 93 static struct soft_cpu **soft_pcpu; 94 95 static int 96 soft_allocate_pmc(int cpu, int ri, struct pmc *pm, 97 const struct pmc_op_pmcallocate *a) 98 { 99 enum pmc_event ev; 100 struct pmc_soft *ps; 101 102 (void) cpu; 103 104 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 105 ("[soft,%d] illegal CPU value %d", __LINE__, cpu)); 106 KASSERT(ri >= 0 && ri < SOFT_NPMCS, 107 ("[soft,%d] illegal row-index %d", __LINE__, ri)); 108 109 if (a->pm_class != PMC_CLASS_SOFT) 110 return (EINVAL); 111 112 if ((pm->pm_caps & SOFT_CAPS) == 0) 113 return (EINVAL); 114 115 if ((pm->pm_caps & ~SOFT_CAPS) != 0) 116 return (EPERM); 117 118 ev = pm->pm_event; 119 if ((int)ev < PMC_EV_SOFT_FIRST || (int)ev > PMC_EV_SOFT_LAST) 120 return (EINVAL); 121 122 /* Check if event is registered. */ 123 ps = pmc_soft_ev_acquire(ev); 124 if (ps == NULL) 125 return (EINVAL); 126 pmc_soft_ev_release(ps); 127 128 return (0); 129 } 130 131 static int 132 soft_config_pmc(int cpu, int ri, struct pmc *pm) 133 { 134 struct pmc_hw *phw; 135 136 PMCDBG(MDP,CFG,1, "cpu=%d ri=%d pm=%p", cpu, ri, pm); 137 138 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 139 ("[soft,%d] illegal CPU value %d", __LINE__, cpu)); 140 KASSERT(ri >= 0 && ri < SOFT_NPMCS, 141 ("[soft,%d] illegal row-index %d", __LINE__, ri)); 142 143 phw = &soft_pcpu[cpu]->soft_hw[ri]; 144 145 KASSERT(pm == NULL || phw->phw_pmc == NULL, 146 ("[soft,%d] pm=%p phw->pm=%p hwpmc not unconfigured", __LINE__, 147 pm, phw->phw_pmc)); 148 149 phw->phw_pmc = pm; 150 151 return (0); 152 } 153 154 static int 155 soft_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc) 156 { 157 int error; 158 size_t copied; 159 const struct soft_descr *pd; 160 struct pmc_hw *phw; 161 162 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 163 ("[soft,%d] illegal CPU %d", __LINE__, cpu)); 164 KASSERT(ri >= 0 && ri < SOFT_NPMCS, 165 ("[soft,%d] illegal row-index %d", __LINE__, ri)); 166 167 phw = &soft_pcpu[cpu]->soft_hw[ri]; 168 pd = &soft_pmcdesc[ri]; 169 170 if ((error = copystr(pd->pm_descr.pd_name, pi->pm_name, 171 PMC_NAME_MAX, &copied)) != 0) 172 return (error); 173 174 pi->pm_class = pd->pm_descr.pd_class; 175 176 if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) { 177 pi->pm_enabled = TRUE; 178 *ppmc = phw->phw_pmc; 179 } else { 180 pi->pm_enabled = FALSE; 181 *ppmc = NULL; 182 } 183 184 return (0); 185 } 186 187 static int 188 soft_get_config(int cpu, int ri, struct pmc **ppm) 189 { 190 (void) ri; 191 192 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 193 ("[soft,%d] illegal CPU %d", __LINE__, cpu)); 194 KASSERT(ri >= 0 && ri < SOFT_NPMCS, 195 ("[soft,%d] illegal row-index %d", __LINE__, ri)); 196 197 *ppm = soft_pcpu[cpu]->soft_hw[ri].phw_pmc; 198 return (0); 199 } 200 201 static int 202 soft_pcpu_fini(struct pmc_mdep *md, int cpu) 203 { 204 int ri; 205 struct pmc_cpu *pc; 206 207 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 208 ("[soft,%d] illegal cpu %d", __LINE__, cpu)); 209 KASSERT(soft_pcpu[cpu] != NULL, ("[soft,%d] null pcpu", __LINE__)); 210 211 free(soft_pcpu[cpu], M_PMC); 212 soft_pcpu[cpu] = NULL; 213 214 ri = md->pmd_classdep[PMC_CLASS_INDEX_SOFT].pcd_ri; 215 216 KASSERT(ri >= 0 && ri < SOFT_NPMCS, 217 ("[soft,%d] ri=%d", __LINE__, ri)); 218 219 pc = pmc_pcpu[cpu]; 220 pc->pc_hwpmcs[ri] = NULL; 221 222 return (0); 223 } 224 225 static int 226 soft_pcpu_init(struct pmc_mdep *md, int cpu) 227 { 228 int first_ri, n; 229 struct pmc_cpu *pc; 230 struct soft_cpu *soft_pc; 231 struct pmc_hw *phw; 232 233 234 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 235 ("[soft,%d] illegal cpu %d", __LINE__, cpu)); 236 KASSERT(soft_pcpu, ("[soft,%d] null pcpu", __LINE__)); 237 KASSERT(soft_pcpu[cpu] == NULL, ("[soft,%d] non-null per-cpu", 238 __LINE__)); 239 240 soft_pc = malloc(sizeof(struct soft_cpu), M_PMC, M_WAITOK|M_ZERO); 241 if (soft_pc == NULL) 242 return (ENOMEM); 243 244 pc = pmc_pcpu[cpu]; 245 246 KASSERT(pc != NULL, ("[soft,%d] cpu %d null per-cpu", __LINE__, cpu)); 247 248 soft_pcpu[cpu] = soft_pc; 249 phw = soft_pc->soft_hw; 250 first_ri = md->pmd_classdep[PMC_CLASS_INDEX_SOFT].pcd_ri; 251 252 for (n = 0; n < SOFT_NPMCS; n++, phw++) { 253 phw->phw_state = PMC_PHW_FLAG_IS_ENABLED | 254 PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(n); 255 phw->phw_pmc = NULL; 256 pc->pc_hwpmcs[n + first_ri] = phw; 257 } 258 259 return (0); 260 } 261 262 static int 263 soft_read_pmc(int cpu, int ri, pmc_value_t *v) 264 { 265 struct pmc *pm; 266 const struct pmc_hw *phw; 267 268 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 269 ("[soft,%d] illegal CPU value %d", __LINE__, cpu)); 270 KASSERT(ri >= 0 && ri < SOFT_NPMCS, 271 ("[soft,%d] illegal row-index %d", __LINE__, ri)); 272 273 phw = &soft_pcpu[cpu]->soft_hw[ri]; 274 pm = phw->phw_pmc; 275 276 KASSERT(pm != NULL, 277 ("[soft,%d] no owner for PHW [cpu%d,pmc%d]", __LINE__, cpu, ri)); 278 279 PMCDBG(MDP,REA,1,"soft-read id=%d", ri); 280 281 *v = soft_pcpu[cpu]->soft_values[ri]; 282 283 return (0); 284 } 285 286 static int 287 soft_write_pmc(int cpu, int ri, pmc_value_t v) 288 { 289 struct pmc *pm; 290 const struct soft_descr *pd; 291 292 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 293 ("[soft,%d] illegal cpu value %d", __LINE__, cpu)); 294 KASSERT(ri >= 0 && ri < SOFT_NPMCS, 295 ("[soft,%d] illegal row-index %d", __LINE__, ri)); 296 297 pm = soft_pcpu[cpu]->soft_hw[ri].phw_pmc; 298 pd = &soft_pmcdesc[ri]; 299 300 KASSERT(pm, 301 ("[soft,%d] cpu %d ri %d pmc not configured", __LINE__, cpu, ri)); 302 303 PMCDBG(MDP,WRI,1, "soft-write cpu=%d ri=%d v=%jx", cpu, ri, v); 304 305 soft_pcpu[cpu]->soft_values[ri] = v; 306 307 return (0); 308 } 309 310 static int 311 soft_release_pmc(int cpu, int ri, struct pmc *pmc) 312 { 313 struct pmc_hw *phw; 314 315 (void) pmc; 316 317 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 318 ("[soft,%d] illegal CPU value %d", __LINE__, cpu)); 319 KASSERT(ri >= 0 && ri < SOFT_NPMCS, 320 ("[soft,%d] illegal row-index %d", __LINE__, ri)); 321 322 phw = &soft_pcpu[cpu]->soft_hw[ri]; 323 324 KASSERT(phw->phw_pmc == NULL, 325 ("[soft,%d] PHW pmc %p non-NULL", __LINE__, phw->phw_pmc)); 326 327 /* 328 * Nothing to do. 329 */ 330 return (0); 331 } 332 333 static int 334 soft_start_pmc(int cpu, int ri) 335 { 336 struct pmc *pm; 337 struct soft_cpu *pc; 338 struct pmc_soft *ps; 339 340 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 341 ("[soft,%d] illegal CPU value %d", __LINE__, cpu)); 342 KASSERT(ri >= 0 && ri < SOFT_NPMCS, 343 ("[soft,%d] illegal row-index %d", __LINE__, ri)); 344 345 pc = soft_pcpu[cpu]; 346 pm = pc->soft_hw[ri].phw_pmc; 347 348 KASSERT(pm, 349 ("[soft,%d] cpu %d ri %d pmc not configured", __LINE__, cpu, ri)); 350 351 ps = pmc_soft_ev_acquire(pm->pm_event); 352 if (ps == NULL) 353 return (EINVAL); 354 atomic_add_int(&ps->ps_running, 1); 355 pmc_soft_ev_release(ps); 356 357 return (0); 358 } 359 360 static int 361 soft_stop_pmc(int cpu, int ri) 362 { 363 struct pmc *pm; 364 struct soft_cpu *pc; 365 struct pmc_soft *ps; 366 367 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 368 ("[soft,%d] illegal CPU value %d", __LINE__, cpu)); 369 KASSERT(ri >= 0 && ri < SOFT_NPMCS, 370 ("[soft,%d] illegal row-index %d", __LINE__, ri)); 371 372 pc = soft_pcpu[cpu]; 373 pm = pc->soft_hw[ri].phw_pmc; 374 375 KASSERT(pm, 376 ("[soft,%d] cpu %d ri %d pmc not configured", __LINE__, cpu, ri)); 377 378 ps = pmc_soft_ev_acquire(pm->pm_event); 379 /* event unregistered ? */ 380 if (ps != NULL) { 381 atomic_subtract_int(&ps->ps_running, 1); 382 pmc_soft_ev_release(ps); 383 } 384 385 return (0); 386 } 387 388 int 389 pmc_soft_intr(struct pmckern_soft *ks) 390 { 391 struct pmc *pm; 392 struct soft_cpu *pc; 393 int ri, processed, error, user_mode; 394 395 KASSERT(ks->pm_cpu >= 0 && ks->pm_cpu < pmc_cpu_max(), 396 ("[soft,%d] CPU %d out of range", __LINE__, ks->pm_cpu)); 397 398 processed = 0; 399 pc = soft_pcpu[ks->pm_cpu]; 400 401 for (ri = 0; ri < SOFT_NPMCS; ri++) { 402 403 pm = pc->soft_hw[ri].phw_pmc; 404 if (pm == NULL || 405 pm->pm_state != PMC_STATE_RUNNING || 406 pm->pm_event != ks->pm_ev) { 407 continue; 408 } 409 410 processed = 1; 411 pc->soft_values[ri]++; 412 if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) { 413 user_mode = TRAPF_USERMODE(ks->pm_tf); 414 error = pmc_process_interrupt(ks->pm_cpu, PMC_SR, pm, 415 ks->pm_tf, user_mode); 416 if (error) { 417 soft_stop_pmc(ks->pm_cpu, ri); 418 continue; 419 } 420 421 if (user_mode) { 422 /* If in user mode setup AST to process 423 * callchain out of interrupt context. 424 */ 425 curthread->td_flags |= TDF_ASTPENDING; 426 } 427 } 428 } 429 430 atomic_add_int(processed ? &pmc_stats.pm_intr_processed : 431 &pmc_stats.pm_intr_ignored, 1); 432 433 return (processed); 434 } 435 436 void 437 pmc_soft_initialize(struct pmc_mdep *md) 438 { 439 struct pmc_classdep *pcd; 440 441 /* Add SOFT PMCs. */ 442 soft_pcpu = malloc(sizeof(struct soft_cpu *) * pmc_cpu_max(), M_PMC, 443 M_ZERO|M_WAITOK); 444 445 pcd = &md->pmd_classdep[PMC_CLASS_INDEX_SOFT]; 446 447 pcd->pcd_caps = SOFT_CAPS; 448 pcd->pcd_class = PMC_CLASS_SOFT; 449 pcd->pcd_num = SOFT_NPMCS; 450 pcd->pcd_ri = md->pmd_npmc; 451 pcd->pcd_width = 64; 452 453 pcd->pcd_allocate_pmc = soft_allocate_pmc; 454 pcd->pcd_config_pmc = soft_config_pmc; 455 pcd->pcd_describe = soft_describe; 456 pcd->pcd_get_config = soft_get_config; 457 pcd->pcd_get_msr = NULL; 458 pcd->pcd_pcpu_init = soft_pcpu_init; 459 pcd->pcd_pcpu_fini = soft_pcpu_fini; 460 pcd->pcd_read_pmc = soft_read_pmc; 461 pcd->pcd_write_pmc = soft_write_pmc; 462 pcd->pcd_release_pmc = soft_release_pmc; 463 pcd->pcd_start_pmc = soft_start_pmc; 464 pcd->pcd_stop_pmc = soft_stop_pmc; 465 466 md->pmd_npmc += SOFT_NPMCS; 467 } 468 469 void 470 pmc_soft_finalize(struct pmc_mdep *md) 471 { 472 #ifdef INVARIANTS 473 int i, ncpus; 474 475 ncpus = pmc_cpu_max(); 476 for (i = 0; i < ncpus; i++) 477 KASSERT(soft_pcpu[i] == NULL, ("[soft,%d] non-null pcpu cpu %d", 478 __LINE__, i)); 479 480 KASSERT(md->pmd_classdep[PMC_CLASS_INDEX_SOFT].pcd_class == 481 PMC_CLASS_SOFT, ("[soft,%d] class mismatch", __LINE__)); 482 #endif 483 free(soft_pcpu, M_PMC); 484 soft_pcpu = NULL; 485 } 486