1 /* 2 * Performance events - AMD IBS 3 * 4 * Copyright (C) 2011 Advanced Micro Devices, Inc., Robert Richter 5 * 6 * For licencing details see kernel-base/COPYING 7 */ 8 9 #include <linux/perf_event.h> 10 #include <linux/init.h> 11 #include <linux/export.h> 12 #include <linux/pci.h> 13 #include <linux/ptrace.h> 14 #include <linux/syscore_ops.h> 15 #include <linux/sched/clock.h> 16 17 #include <asm/apic.h> 18 #include <asm/cpuid/api.h> 19 #include <asm/msr.h> 20 21 #include "../perf_event.h" 22 23 static u32 ibs_caps; 24 25 #if defined(CONFIG_PERF_EVENTS) && defined(CONFIG_CPU_SUP_AMD) 26 27 #include <linux/kprobes.h> 28 #include <linux/hardirq.h> 29 30 #include <asm/nmi.h> 31 #include <asm/amd/ibs.h> 32 33 /* attr.config2 */ 34 #define IBS_SW_FILTER_MASK 1 35 36 /* attr.config1 */ 37 #define IBS_OP_CONFIG1_LDLAT_MASK (0xFFFULL << 0) 38 #define IBS_OP_CONFIG1_STRMST_MASK (1ULL << 12) 39 #define IBS_OP_CONFIG1_STRMST_SHIFT (12) 40 41 #define IBS_FETCH_CONFIG1_FETCHLAT_MASK (0x7FFULL << 0) 42 43 /* 44 * IBS states: 45 * 46 * ENABLED; tracks the pmu::add(), pmu::del() state, when set the counter is taken 47 * and any further add()s must fail. 48 * 49 * STARTED/STOPPING/STOPPED; deal with pmu::start(), pmu::stop() state but are 50 * complicated by the fact that the IBS hardware can send late NMIs (ie. after 51 * we've cleared the EN bit). 52 * 53 * In order to consume these late NMIs we have the STOPPED state, any NMI that 54 * happens after we've cleared the EN state will clear this bit and report the 55 * NMI handled (this is fundamentally racy in the face or multiple NMI sources, 56 * someone else can consume our BIT and our NMI will go unhandled). 57 * 58 * And since we cannot set/clear this separate bit together with the EN bit, 59 * there are races; if we cleared STARTED early, an NMI could land in 60 * between clearing STARTED and clearing the EN bit (in fact multiple NMIs 61 * could happen if the period is small enough), and consume our STOPPED bit 62 * and trigger streams of unhandled NMIs. 63 * 64 * If, however, we clear STARTED late, an NMI can hit between clearing the 65 * EN bit and clearing STARTED, still see STARTED set and process the event. 66 * If this event will have the VALID bit clear, we bail properly, but this 67 * is not a given. With VALID set we can end up calling pmu::stop() again 68 * (the throttle logic) and trigger the WARNs in there. 69 * 70 * So what we do is set STOPPING before clearing EN to avoid the pmu::stop() 71 * nesting, and clear STARTED late, so that we have a well defined state over 72 * the clearing of the EN bit. 73 * 74 * XXX: we could probably be using !atomic bitops for all this. 75 */ 76 77 enum ibs_states { 78 IBS_ENABLED = 0, 79 IBS_STARTED = 1, 80 IBS_STOPPING = 2, 81 IBS_STOPPED = 3, 82 83 IBS_MAX_STATES, 84 }; 85 86 struct cpu_perf_ibs { 87 struct perf_event *event; 88 unsigned long state[BITS_TO_LONGS(IBS_MAX_STATES)]; 89 }; 90 91 struct perf_ibs { 92 struct pmu pmu; 93 unsigned int msr; 94 unsigned int msr2; 95 u64 config_mask; 96 u64 cnt_mask; 97 u64 enable_mask; 98 u64 disable_mask; 99 u64 valid_mask; 100 u16 min_period; 101 u64 max_period; 102 unsigned long offset_mask[1]; 103 int offset_max; 104 unsigned int fetch_count_reset_broken : 1; 105 unsigned int fetch_ignore_if_zero_rip : 1; 106 struct cpu_perf_ibs __percpu *pcpu; 107 108 u64 (*get_count)(u64 config); 109 }; 110 111 static int 112 perf_event_set_period(struct hw_perf_event *hwc, u64 min, u64 max, u64 *hw_period) 113 { 114 s64 left = local64_read(&hwc->period_left); 115 s64 period = hwc->sample_period; 116 int overflow = 0; 117 118 /* 119 * If we are way outside a reasonable range then just skip forward: 120 */ 121 if (unlikely(left <= -period)) { 122 left = period; 123 local64_set(&hwc->period_left, left); 124 hwc->last_period = period; 125 overflow = 1; 126 } 127 128 if (unlikely(left < (s64)min)) { 129 left += period; 130 local64_set(&hwc->period_left, left); 131 hwc->last_period = period; 132 overflow = 1; 133 } 134 135 /* 136 * If the hw period that triggers the sw overflow is too short 137 * we might hit the irq handler. This biases the results. 138 * Thus we shorten the next-to-last period and set the last 139 * period to the max period. 140 */ 141 if (left > max) { 142 left -= max; 143 if (left > max) 144 left = max; 145 else if (left < min) 146 left = min; 147 } 148 149 *hw_period = (u64)left; 150 151 return overflow; 152 } 153 154 static int 155 perf_event_try_update(struct perf_event *event, u64 new_raw_count, int width) 156 { 157 struct hw_perf_event *hwc = &event->hw; 158 int shift = 64 - width; 159 u64 prev_raw_count; 160 u64 delta; 161 162 /* 163 * Careful: an NMI might modify the previous event value. 164 * 165 * Our tactic to handle this is to first atomically read and 166 * exchange a new raw count - then add that new-prev delta 167 * count to the generic event atomically: 168 */ 169 prev_raw_count = local64_read(&hwc->prev_count); 170 if (!local64_try_cmpxchg(&hwc->prev_count, 171 &prev_raw_count, new_raw_count)) 172 return 0; 173 174 /* 175 * Now we have the new raw value and have updated the prev 176 * timestamp already. We can now calculate the elapsed delta 177 * (event-)time and add that to the generic event. 178 * 179 * Careful, not all hw sign-extends above the physical width 180 * of the count. 181 */ 182 delta = (new_raw_count << shift) - (prev_raw_count << shift); 183 delta >>= shift; 184 185 local64_add(delta, &event->count); 186 local64_sub(delta, &hwc->period_left); 187 188 return 1; 189 } 190 191 static struct perf_ibs perf_ibs_fetch; 192 static struct perf_ibs perf_ibs_op; 193 194 static struct perf_ibs *get_ibs_pmu(int type) 195 { 196 if (perf_ibs_fetch.pmu.type == type) 197 return &perf_ibs_fetch; 198 if (perf_ibs_op.pmu.type == type) 199 return &perf_ibs_op; 200 return NULL; 201 } 202 203 /* 204 * core pmu config -> IBS config 205 * 206 * perf record -a -e cpu-cycles:p ... # use ibs op counting cycle count 207 * perf record -a -e r076:p ... # same as -e cpu-cycles:p 208 * perf record -a -e r0C1:p ... # use ibs op counting micro-ops 209 * 210 * IbsOpCntCtl (bit 19) of IBS Execution Control Register (IbsOpCtl, 211 * MSRC001_1033) is used to select either cycle or micro-ops counting 212 * mode. 213 */ 214 static int core_pmu_ibs_config(struct perf_event *event, u64 *config) 215 { 216 switch (event->attr.type) { 217 case PERF_TYPE_HARDWARE: 218 switch (event->attr.config) { 219 case PERF_COUNT_HW_CPU_CYCLES: 220 *config = 0; 221 return 0; 222 } 223 break; 224 case PERF_TYPE_RAW: 225 switch (event->attr.config) { 226 case 0x0076: 227 *config = 0; 228 return 0; 229 case 0x00C1: 230 *config = IBS_OP_CNT_CTL; 231 return 0; 232 } 233 break; 234 default: 235 return -ENOENT; 236 } 237 238 return -EOPNOTSUPP; 239 } 240 241 /* 242 * The rip of IBS samples has skid 0. Thus, IBS supports precise 243 * levels 1 and 2 and the PERF_EFLAGS_EXACT is set. In rare cases the 244 * rip is invalid when IBS was not able to record the rip correctly. 245 * We clear PERF_EFLAGS_EXACT and take the rip from pt_regs then. 246 */ 247 int forward_event_to_ibs(struct perf_event *event) 248 { 249 u64 config = 0; 250 251 if (!event->attr.precise_ip || event->attr.precise_ip > 2) 252 return -EOPNOTSUPP; 253 254 if (!core_pmu_ibs_config(event, &config)) { 255 event->attr.type = perf_ibs_op.pmu.type; 256 event->attr.config = config; 257 } 258 return -ENOENT; 259 } 260 261 /* 262 * Grouping of IBS events is not possible since IBS can have only 263 * one event active at any point in time. 264 */ 265 static int validate_group(struct perf_event *event) 266 { 267 struct perf_event *sibling; 268 269 if (event->group_leader == event) 270 return 0; 271 272 if (event->group_leader->pmu == event->pmu) 273 return -EINVAL; 274 275 for_each_sibling_event(sibling, event->group_leader) { 276 if (sibling->pmu == event->pmu) 277 return -EINVAL; 278 } 279 return 0; 280 } 281 282 static bool perf_ibs_ldlat_event(struct perf_ibs *perf_ibs, 283 struct perf_event *event) 284 { 285 return perf_ibs == &perf_ibs_op && 286 (ibs_caps & IBS_CAPS_OPLDLAT) && 287 (event->attr.config1 & IBS_OP_CONFIG1_LDLAT_MASK); 288 } 289 290 static bool perf_ibs_fetch_lat_event(struct perf_ibs *perf_ibs, 291 struct perf_event *event) 292 { 293 return perf_ibs == &perf_ibs_fetch && 294 (ibs_caps & IBS_CAPS_FETCHLAT) && 295 (event->attr.config1 & IBS_FETCH_CONFIG1_FETCHLAT_MASK); 296 } 297 298 static bool perf_ibs_strmst_event(struct perf_ibs *perf_ibs, 299 struct perf_event *event) 300 { 301 return perf_ibs == &perf_ibs_op && 302 (ibs_caps & IBS_CAPS_STRMST_RMTSOCKET) && 303 (event->attr.config1 & IBS_OP_CONFIG1_STRMST_MASK); 304 } 305 306 static int perf_ibs_init(struct perf_event *event) 307 { 308 struct hw_perf_event *hwc = &event->hw; 309 struct perf_ibs *perf_ibs; 310 u64 config; 311 int ret; 312 313 perf_ibs = get_ibs_pmu(event->attr.type); 314 if (!perf_ibs) 315 return -ENOENT; 316 317 config = event->attr.config; 318 hwc->extra_reg.config = 0; 319 hwc->extra_reg.reg = 0; 320 321 if (event->pmu != &perf_ibs->pmu) 322 return -ENOENT; 323 324 if (config & ~perf_ibs->config_mask) 325 return -EINVAL; 326 327 if (has_branch_stack(event)) 328 return -EOPNOTSUPP; 329 330 /* handle exclude_{user,kernel} in the IRQ handler */ 331 if (event->attr.exclude_host || event->attr.exclude_guest || 332 event->attr.exclude_idle) 333 return -EINVAL; 334 335 ret = validate_group(event); 336 if (ret) 337 return ret; 338 339 if (perf_allow_kernel()) 340 hwc->flags |= PERF_X86_EVENT_UNPRIVILEGED; 341 342 if (ibs_caps & IBS_CAPS_DIS) { 343 hwc->extra_reg.config &= ~perf_ibs->disable_mask; 344 hwc->extra_reg.reg = perf_ibs->msr2; 345 } 346 347 if (ibs_caps & IBS_CAPS_BIT63_FILTER) { 348 if (perf_ibs == &perf_ibs_fetch) { 349 if (event->attr.exclude_kernel) { 350 hwc->extra_reg.config |= IBS_FETCH_2_EXCL_RIP_63_EQ_1; 351 hwc->extra_reg.reg = perf_ibs->msr2; 352 } 353 if (event->attr.exclude_user) { 354 hwc->extra_reg.config |= IBS_FETCH_2_EXCL_RIP_63_EQ_0; 355 hwc->extra_reg.reg = perf_ibs->msr2; 356 } 357 } else { 358 if (event->attr.exclude_kernel) { 359 hwc->extra_reg.config |= IBS_OP_2_EXCL_RIP_63_EQ_1; 360 hwc->extra_reg.reg = perf_ibs->msr2; 361 } 362 if (event->attr.exclude_user) { 363 hwc->extra_reg.config |= IBS_OP_2_EXCL_RIP_63_EQ_0; 364 hwc->extra_reg.reg = perf_ibs->msr2; 365 } 366 } 367 } else if (!(event->attr.config2 & IBS_SW_FILTER_MASK) && 368 (event->attr.exclude_kernel || event->attr.exclude_user || 369 event->attr.exclude_hv)) { 370 return -EINVAL; 371 } 372 373 if (hwc->sample_period) { 374 if (config & perf_ibs->cnt_mask) 375 /* raw max_cnt may not be set */ 376 return -EINVAL; 377 378 if (event->attr.freq) { 379 hwc->sample_period = perf_ibs->min_period; 380 } else { 381 /* Silently mask off lower nibble. IBS hw mandates it. */ 382 hwc->sample_period &= ~0x0FULL; 383 if (hwc->sample_period < perf_ibs->min_period) 384 return -EINVAL; 385 } 386 } else { 387 u64 period = 0; 388 389 if (event->attr.freq) 390 return -EINVAL; 391 392 if (perf_ibs == &perf_ibs_op) { 393 period = (config & IBS_OP_MAX_CNT) << 4; 394 if (ibs_caps & IBS_CAPS_OPCNTEXT) 395 period |= config & IBS_OP_MAX_CNT_EXT_MASK; 396 } else { 397 period = (config & IBS_FETCH_MAX_CNT) << 4; 398 } 399 400 config &= ~perf_ibs->cnt_mask; 401 event->attr.sample_period = period; 402 hwc->sample_period = period; 403 404 if (hwc->sample_period < perf_ibs->min_period) 405 return -EINVAL; 406 } 407 408 if (perf_ibs_ldlat_event(perf_ibs, event)) { 409 u64 ldlat = event->attr.config1 & IBS_OP_CONFIG1_LDLAT_MASK; 410 411 if (ldlat < 128 || ldlat > 2048) 412 return -EINVAL; 413 ldlat >>= 7; 414 415 config |= (ldlat - 1) << IBS_OP_LDLAT_THRSH_SHIFT; 416 417 config |= IBS_OP_LDLAT_EN; 418 if (cpu_feature_enabled(X86_FEATURE_ZEN5)) 419 config |= IBS_OP_L3MISSONLY; 420 } 421 422 if (perf_ibs_fetch_lat_event(perf_ibs, event)) { 423 u64 fetchlat = event->attr.config1 & IBS_FETCH_CONFIG1_FETCHLAT_MASK; 424 425 if (fetchlat < 128 || fetchlat > 1920) 426 return -EINVAL; 427 fetchlat >>= 7; 428 429 hwc->extra_reg.reg = perf_ibs->msr2; 430 hwc->extra_reg.config |= fetchlat << IBS_FETCH_2_FETCHLAT_FILTER_SHIFT; 431 } 432 433 if (perf_ibs_strmst_event(perf_ibs, event)) { 434 u64 strmst = event->attr.config1 & IBS_OP_CONFIG1_STRMST_MASK; 435 436 strmst >>= IBS_OP_CONFIG1_STRMST_SHIFT; 437 438 hwc->extra_reg.reg = perf_ibs->msr2; 439 hwc->extra_reg.config |= strmst << IBS_OP_2_STRM_ST_FILTER_SHIFT; 440 } 441 442 /* 443 * If we modify hwc->sample_period, we also need to update 444 * hwc->last_period and hwc->period_left. 445 */ 446 hwc->last_period = hwc->sample_period; 447 local64_set(&hwc->period_left, hwc->sample_period); 448 449 hwc->config_base = perf_ibs->msr; 450 hwc->config = config; 451 452 return 0; 453 } 454 455 static int perf_ibs_set_period(struct perf_ibs *perf_ibs, 456 struct hw_perf_event *hwc, u64 *period) 457 { 458 int overflow; 459 460 /* ignore lower 4 bits in min count: */ 461 overflow = perf_event_set_period(hwc, perf_ibs->min_period, 462 perf_ibs->max_period, period); 463 local64_set(&hwc->prev_count, 0); 464 465 return overflow; 466 } 467 468 static u64 get_ibs_fetch_count(u64 config) 469 { 470 union ibs_fetch_ctl fetch_ctl = (union ibs_fetch_ctl)config; 471 472 return fetch_ctl.fetch_cnt << 4; 473 } 474 475 static u64 get_ibs_op_count(u64 config) 476 { 477 union ibs_op_ctl op_ctl = (union ibs_op_ctl)config; 478 u64 count = 0; 479 480 /* 481 * If the internal 27-bit counter rolled over, the count is MaxCnt 482 * and the lower 7 bits of CurCnt are randomized. 483 * Otherwise CurCnt has the full 27-bit current counter value. 484 */ 485 if (op_ctl.op_val) { 486 count = op_ctl.opmaxcnt << 4; 487 if (ibs_caps & IBS_CAPS_OPCNTEXT) 488 count += op_ctl.opmaxcnt_ext << 20; 489 } else if (ibs_caps & IBS_CAPS_RDWROPCNT) { 490 count = op_ctl.opcurcnt; 491 } 492 493 return count; 494 } 495 496 static void 497 perf_ibs_event_update(struct perf_ibs *perf_ibs, struct perf_event *event, 498 u64 *config) 499 { 500 u64 count = perf_ibs->get_count(*config); 501 502 /* 503 * Set width to 64 since we do not overflow on max width but 504 * instead on max count. In perf_ibs_set_period() we clear 505 * prev count manually on overflow. 506 */ 507 while (!perf_event_try_update(event, count, 64)) { 508 rdmsrq(event->hw.config_base, *config); 509 count = perf_ibs->get_count(*config); 510 } 511 } 512 513 static inline void perf_ibs_enable_event(struct perf_ibs *perf_ibs, 514 struct hw_perf_event *hwc, u64 config) 515 { 516 u64 tmp = hwc->config | config; 517 518 if (perf_ibs->fetch_count_reset_broken) 519 wrmsrq(hwc->config_base, tmp & ~perf_ibs->enable_mask); 520 521 wrmsrq(hwc->config_base, tmp | perf_ibs->enable_mask); 522 523 if (hwc->extra_reg.reg) 524 wrmsrq(hwc->extra_reg.reg, hwc->extra_reg.config); 525 } 526 527 /* 528 * Erratum #420 Instruction-Based Sampling Engine May Generate 529 * Interrupt that Cannot Be Cleared: 530 * 531 * Must clear counter mask first, then clear the enable bit. See 532 * Revision Guide for AMD Family 10h Processors, Publication #41322. 533 */ 534 static inline void perf_ibs_disable_event(struct perf_ibs *perf_ibs, 535 struct hw_perf_event *hwc, u64 config) 536 { 537 if (ibs_caps & IBS_CAPS_DIS) { 538 wrmsrq(hwc->extra_reg.reg, perf_ibs->disable_mask); 539 return; 540 } 541 542 config &= ~perf_ibs->cnt_mask; 543 if (boot_cpu_data.x86 == 0x10) 544 wrmsrq(hwc->config_base, config); 545 config &= ~perf_ibs->enable_mask; 546 wrmsrq(hwc->config_base, config); 547 } 548 549 /* 550 * We cannot restore the ibs pmu state, so we always needs to update 551 * the event while stopping it and then reset the state when starting 552 * again. Thus, ignoring PERF_EF_RELOAD and PERF_EF_UPDATE flags in 553 * perf_ibs_start()/perf_ibs_stop() and instead always do it. 554 */ 555 static void perf_ibs_start(struct perf_event *event, int flags) 556 { 557 struct hw_perf_event *hwc = &event->hw; 558 struct perf_ibs *perf_ibs = container_of(event->pmu, struct perf_ibs, pmu); 559 struct cpu_perf_ibs *pcpu = this_cpu_ptr(perf_ibs->pcpu); 560 u64 period, config = 0; 561 562 if (WARN_ON_ONCE(!(hwc->state & PERF_HES_STOPPED))) 563 return; 564 565 WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE)); 566 hwc->state = 0; 567 568 if (event->attr.freq && hwc->sample_period < perf_ibs->min_period) 569 hwc->sample_period = perf_ibs->min_period; 570 571 perf_ibs_set_period(perf_ibs, hwc, &period); 572 if (perf_ibs == &perf_ibs_op && (ibs_caps & IBS_CAPS_OPCNTEXT)) { 573 config |= period & IBS_OP_MAX_CNT_EXT_MASK; 574 period &= ~IBS_OP_MAX_CNT_EXT_MASK; 575 } 576 config |= period >> 4; 577 578 /* 579 * Reset the IBS_{FETCH|OP}_CTL MSR before updating pcpu->state. 580 * Doing so prevents a race condition in which an NMI due to other 581 * source might accidentally activate the event before we enable 582 * it ourselves. 583 */ 584 perf_ibs_disable_event(perf_ibs, hwc, 0); 585 586 /* 587 * Set STARTED before enabling the hardware, such that a subsequent NMI 588 * must observe it. 589 */ 590 set_bit(IBS_STARTED, pcpu->state); 591 clear_bit(IBS_STOPPING, pcpu->state); 592 perf_ibs_enable_event(perf_ibs, hwc, config); 593 594 perf_event_update_userpage(event); 595 } 596 597 static void perf_ibs_stop(struct perf_event *event, int flags) 598 { 599 struct hw_perf_event *hwc = &event->hw; 600 struct perf_ibs *perf_ibs = container_of(event->pmu, struct perf_ibs, pmu); 601 struct cpu_perf_ibs *pcpu = this_cpu_ptr(perf_ibs->pcpu); 602 u64 config; 603 int stopping; 604 605 if (test_and_set_bit(IBS_STOPPING, pcpu->state)) 606 return; 607 608 stopping = test_bit(IBS_STARTED, pcpu->state); 609 610 if (!stopping && (hwc->state & PERF_HES_UPTODATE)) 611 return; 612 613 rdmsrq(hwc->config_base, config); 614 615 if (stopping) { 616 /* 617 * Set STOPPED before disabling the hardware, such that it 618 * must be visible to NMIs the moment we clear the EN bit, 619 * at which point we can generate an !VALID sample which 620 * we need to consume. 621 */ 622 set_bit(IBS_STOPPED, pcpu->state); 623 perf_ibs_disable_event(perf_ibs, hwc, config); 624 /* 625 * Clear STARTED after disabling the hardware; if it were 626 * cleared before an NMI hitting after the clear but before 627 * clearing the EN bit might think it a spurious NMI and not 628 * handle it. 629 * 630 * Clearing it after, however, creates the problem of the NMI 631 * handler seeing STARTED but not having a valid sample. 632 */ 633 clear_bit(IBS_STARTED, pcpu->state); 634 WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED); 635 hwc->state |= PERF_HES_STOPPED; 636 } 637 638 if (hwc->state & PERF_HES_UPTODATE) 639 return; 640 641 /* 642 * Clear valid bit to not count rollovers on update, rollovers 643 * are only updated in the irq handler. 644 */ 645 config &= ~perf_ibs->valid_mask; 646 647 perf_ibs_event_update(perf_ibs, event, &config); 648 hwc->state |= PERF_HES_UPTODATE; 649 } 650 651 static int perf_ibs_add(struct perf_event *event, int flags) 652 { 653 struct perf_ibs *perf_ibs = container_of(event->pmu, struct perf_ibs, pmu); 654 struct cpu_perf_ibs *pcpu = this_cpu_ptr(perf_ibs->pcpu); 655 656 if (test_and_set_bit(IBS_ENABLED, pcpu->state)) 657 return -ENOSPC; 658 659 event->hw.state = PERF_HES_UPTODATE | PERF_HES_STOPPED; 660 661 pcpu->event = event; 662 663 if (flags & PERF_EF_START) 664 perf_ibs_start(event, PERF_EF_RELOAD); 665 666 return 0; 667 } 668 669 static void perf_ibs_del(struct perf_event *event, int flags) 670 { 671 struct perf_ibs *perf_ibs = container_of(event->pmu, struct perf_ibs, pmu); 672 struct cpu_perf_ibs *pcpu = this_cpu_ptr(perf_ibs->pcpu); 673 674 if (!test_and_clear_bit(IBS_ENABLED, pcpu->state)) 675 return; 676 677 perf_ibs_stop(event, PERF_EF_UPDATE); 678 679 pcpu->event = NULL; 680 681 perf_event_update_userpage(event); 682 } 683 684 static void perf_ibs_read(struct perf_event *event) { } 685 686 static int perf_ibs_check_period(struct perf_event *event, u64 value) 687 { 688 struct perf_ibs *perf_ibs; 689 u64 low_nibble; 690 691 if (event->attr.freq) 692 return 0; 693 694 perf_ibs = container_of(event->pmu, struct perf_ibs, pmu); 695 low_nibble = value & 0xFULL; 696 697 /* 698 * This contradicts with perf_ibs_init() which allows sample period 699 * with lower nibble bits set but silently masks them off. Whereas 700 * this returns error. 701 */ 702 if (low_nibble || value < perf_ibs->min_period) 703 return -EINVAL; 704 705 return 0; 706 } 707 708 /* 709 * We need to initialize with empty group if all attributes in the 710 * group are dynamic. 711 */ 712 static struct attribute *attrs_empty[] = { 713 NULL, 714 }; 715 716 static struct attribute_group empty_caps_group = { 717 .name = "caps", 718 .attrs = attrs_empty, 719 }; 720 721 PMU_FORMAT_ATTR(rand_en, "config:57"); 722 PMU_FORMAT_ATTR(cnt_ctl, "config:19"); 723 PMU_FORMAT_ATTR(swfilt, "config2:0"); 724 PMU_EVENT_ATTR_STRING(l3missonly, fetch_l3missonly, "config:59"); 725 PMU_EVENT_ATTR_STRING(l3missonly, op_l3missonly, "config:16"); 726 PMU_EVENT_ATTR_STRING(ldlat, ibs_op_ldlat_format, "config1:0-11"); 727 PMU_EVENT_ATTR_STRING(zen4_ibs_extensions, zen4_ibs_extensions, "1"); 728 PMU_EVENT_ATTR_STRING(ldlat, ibs_op_ldlat_cap, "1"); 729 PMU_EVENT_ATTR_STRING(dtlb_pgsize, ibs_op_dtlb_pgsize_cap, "1"); 730 PMU_EVENT_ATTR_STRING(fetchlat, ibs_fetch_lat_format, "config1:0-10"); 731 PMU_EVENT_ATTR_STRING(fetchlat, ibs_fetch_lat_cap, "1"); 732 PMU_EVENT_ATTR_STRING(strmst, ibs_op_strmst_format, "config1:12"); 733 PMU_EVENT_ATTR_STRING(strmst, ibs_op_strmst_cap, "1"); 734 PMU_EVENT_ATTR_STRING(rmtsocket, ibs_op_rmtsocket_cap, "1"); 735 736 static umode_t 737 zen4_ibs_extensions_is_visible(struct kobject *kobj, struct attribute *attr, int i) 738 { 739 return ibs_caps & IBS_CAPS_ZEN4 ? attr->mode : 0; 740 } 741 742 static umode_t 743 ibs_fetch_lat_is_visible(struct kobject *kobj, struct attribute *attr, int i) 744 { 745 return ibs_caps & IBS_CAPS_FETCHLAT ? attr->mode : 0; 746 } 747 748 static umode_t 749 ibs_op_strmst_is_visible(struct kobject *kobj, struct attribute *attr, int i) 750 { 751 return ibs_caps & IBS_CAPS_STRMST_RMTSOCKET ? attr->mode : 0; 752 } 753 754 static umode_t 755 ibs_op_rmtsocket_is_visible(struct kobject *kobj, struct attribute *attr, int i) 756 { 757 return ibs_caps & IBS_CAPS_STRMST_RMTSOCKET ? attr->mode : 0; 758 } 759 760 static umode_t 761 ibs_op_ldlat_is_visible(struct kobject *kobj, struct attribute *attr, int i) 762 { 763 return ibs_caps & IBS_CAPS_OPLDLAT ? attr->mode : 0; 764 } 765 766 static umode_t 767 ibs_op_dtlb_pgsize_is_visible(struct kobject *kobj, struct attribute *attr, int i) 768 { 769 return ibs_caps & IBS_CAPS_OPDTLBPGSIZE ? attr->mode : 0; 770 } 771 772 static struct attribute *fetch_attrs[] = { 773 &format_attr_rand_en.attr, 774 &format_attr_swfilt.attr, 775 NULL, 776 }; 777 778 static struct attribute *fetch_l3missonly_attrs[] = { 779 &fetch_l3missonly.attr.attr, 780 NULL, 781 }; 782 783 static struct attribute *zen4_ibs_extensions_attrs[] = { 784 &zen4_ibs_extensions.attr.attr, 785 NULL, 786 }; 787 788 static struct attribute *ibs_fetch_lat_format_attrs[] = { 789 &ibs_fetch_lat_format.attr.attr, 790 NULL, 791 }; 792 793 static struct attribute *ibs_fetch_lat_cap_attrs[] = { 794 &ibs_fetch_lat_cap.attr.attr, 795 NULL, 796 }; 797 798 static struct attribute *ibs_op_ldlat_cap_attrs[] = { 799 &ibs_op_ldlat_cap.attr.attr, 800 NULL, 801 }; 802 803 static struct attribute *ibs_op_dtlb_pgsize_cap_attrs[] = { 804 &ibs_op_dtlb_pgsize_cap.attr.attr, 805 NULL, 806 }; 807 808 static struct attribute *ibs_op_strmst_cap_attrs[] = { 809 &ibs_op_strmst_cap.attr.attr, 810 NULL, 811 }; 812 813 static struct attribute *ibs_op_rmtsocket_cap_attrs[] = { 814 &ibs_op_rmtsocket_cap.attr.attr, 815 NULL, 816 }; 817 818 static struct attribute_group group_fetch_formats = { 819 .name = "format", 820 .attrs = fetch_attrs, 821 }; 822 823 static struct attribute_group group_fetch_l3missonly = { 824 .name = "format", 825 .attrs = fetch_l3missonly_attrs, 826 .is_visible = zen4_ibs_extensions_is_visible, 827 }; 828 829 static struct attribute_group group_zen4_ibs_extensions = { 830 .name = "caps", 831 .attrs = zen4_ibs_extensions_attrs, 832 .is_visible = zen4_ibs_extensions_is_visible, 833 }; 834 835 static struct attribute_group group_ibs_fetch_lat_cap = { 836 .name = "caps", 837 .attrs = ibs_fetch_lat_cap_attrs, 838 .is_visible = ibs_fetch_lat_is_visible, 839 }; 840 841 static struct attribute_group group_ibs_fetch_lat_format = { 842 .name = "format", 843 .attrs = ibs_fetch_lat_format_attrs, 844 .is_visible = ibs_fetch_lat_is_visible, 845 }; 846 847 static struct attribute_group group_ibs_op_ldlat_cap = { 848 .name = "caps", 849 .attrs = ibs_op_ldlat_cap_attrs, 850 .is_visible = ibs_op_ldlat_is_visible, 851 }; 852 853 static struct attribute_group group_ibs_op_dtlb_pgsize_cap = { 854 .name = "caps", 855 .attrs = ibs_op_dtlb_pgsize_cap_attrs, 856 .is_visible = ibs_op_dtlb_pgsize_is_visible, 857 }; 858 859 static struct attribute_group group_ibs_op_strmst_cap = { 860 .name = "caps", 861 .attrs = ibs_op_strmst_cap_attrs, 862 .is_visible = ibs_op_strmst_is_visible, 863 }; 864 865 static struct attribute_group group_ibs_op_rmtsocket_cap = { 866 .name = "caps", 867 .attrs = ibs_op_rmtsocket_cap_attrs, 868 .is_visible = ibs_op_rmtsocket_is_visible, 869 }; 870 871 static const struct attribute_group *fetch_attr_groups[] = { 872 &group_fetch_formats, 873 &empty_caps_group, 874 NULL, 875 }; 876 877 static const struct attribute_group *fetch_attr_update[] = { 878 &group_fetch_l3missonly, 879 &group_zen4_ibs_extensions, 880 &group_ibs_fetch_lat_cap, 881 &group_ibs_fetch_lat_format, 882 NULL, 883 }; 884 885 static umode_t 886 cnt_ctl_is_visible(struct kobject *kobj, struct attribute *attr, int i) 887 { 888 return ibs_caps & IBS_CAPS_OPCNT ? attr->mode : 0; 889 } 890 891 static struct attribute *op_attrs[] = { 892 &format_attr_swfilt.attr, 893 NULL, 894 }; 895 896 static struct attribute *cnt_ctl_attrs[] = { 897 &format_attr_cnt_ctl.attr, 898 NULL, 899 }; 900 901 static struct attribute *op_l3missonly_attrs[] = { 902 &op_l3missonly.attr.attr, 903 NULL, 904 }; 905 906 static struct attribute_group group_op_formats = { 907 .name = "format", 908 .attrs = op_attrs, 909 }; 910 911 static struct attribute *ibs_op_ldlat_format_attrs[] = { 912 &ibs_op_ldlat_format.attr.attr, 913 NULL, 914 }; 915 916 static struct attribute *ibs_op_strmst_format_attrs[] = { 917 &ibs_op_strmst_format.attr.attr, 918 NULL, 919 }; 920 921 static struct attribute_group group_cnt_ctl = { 922 .name = "format", 923 .attrs = cnt_ctl_attrs, 924 .is_visible = cnt_ctl_is_visible, 925 }; 926 927 static struct attribute_group group_op_l3missonly = { 928 .name = "format", 929 .attrs = op_l3missonly_attrs, 930 .is_visible = zen4_ibs_extensions_is_visible, 931 }; 932 933 static const struct attribute_group *op_attr_groups[] = { 934 &group_op_formats, 935 &empty_caps_group, 936 NULL, 937 }; 938 939 static struct attribute_group group_ibs_op_ldlat_format = { 940 .name = "format", 941 .attrs = ibs_op_ldlat_format_attrs, 942 .is_visible = ibs_op_ldlat_is_visible, 943 }; 944 945 static struct attribute_group group_ibs_op_strmst_format = { 946 .name = "format", 947 .attrs = ibs_op_strmst_format_attrs, 948 .is_visible = ibs_op_strmst_is_visible, 949 }; 950 951 static const struct attribute_group *op_attr_update[] = { 952 &group_cnt_ctl, 953 &group_op_l3missonly, 954 &group_zen4_ibs_extensions, 955 &group_ibs_op_ldlat_cap, 956 &group_ibs_op_ldlat_format, 957 &group_ibs_op_dtlb_pgsize_cap, 958 &group_ibs_op_strmst_cap, 959 &group_ibs_op_strmst_format, 960 &group_ibs_op_rmtsocket_cap, 961 NULL, 962 }; 963 964 static struct perf_ibs perf_ibs_fetch = { 965 .pmu = { 966 .task_ctx_nr = perf_hw_context, 967 968 .event_init = perf_ibs_init, 969 .add = perf_ibs_add, 970 .del = perf_ibs_del, 971 .start = perf_ibs_start, 972 .stop = perf_ibs_stop, 973 .read = perf_ibs_read, 974 .check_period = perf_ibs_check_period, 975 }, 976 .msr = MSR_AMD64_IBSFETCHCTL, 977 .msr2 = MSR_AMD64_IBSFETCHCTL2, 978 .config_mask = IBS_FETCH_MAX_CNT | IBS_FETCH_RAND_EN, 979 .cnt_mask = IBS_FETCH_MAX_CNT, 980 .enable_mask = IBS_FETCH_ENABLE, 981 .valid_mask = IBS_FETCH_VAL, 982 .min_period = 0x10, 983 .max_period = IBS_FETCH_MAX_CNT << 4, 984 .offset_mask = { MSR_AMD64_IBSFETCH_REG_MASK }, 985 .offset_max = MSR_AMD64_IBSFETCH_REG_COUNT, 986 987 .get_count = get_ibs_fetch_count, 988 }; 989 990 static struct perf_ibs perf_ibs_op = { 991 .pmu = { 992 .task_ctx_nr = perf_hw_context, 993 994 .event_init = perf_ibs_init, 995 .add = perf_ibs_add, 996 .del = perf_ibs_del, 997 .start = perf_ibs_start, 998 .stop = perf_ibs_stop, 999 .read = perf_ibs_read, 1000 .check_period = perf_ibs_check_period, 1001 }, 1002 .msr = MSR_AMD64_IBSOPCTL, 1003 .msr2 = MSR_AMD64_IBSOPCTL2, 1004 .config_mask = IBS_OP_MAX_CNT, 1005 .cnt_mask = IBS_OP_MAX_CNT | IBS_OP_CUR_CNT | 1006 IBS_OP_CUR_CNT_RAND, 1007 .enable_mask = IBS_OP_ENABLE, 1008 .valid_mask = IBS_OP_VAL, 1009 .min_period = 0x90, 1010 .max_period = IBS_OP_MAX_CNT << 4, 1011 .offset_mask = { MSR_AMD64_IBSOP_REG_MASK }, 1012 .offset_max = MSR_AMD64_IBSOP_REG_COUNT, 1013 1014 .get_count = get_ibs_op_count, 1015 }; 1016 1017 static void perf_ibs_get_mem_op(union ibs_op_data3 *op_data3, 1018 struct perf_sample_data *data) 1019 { 1020 union perf_mem_data_src *data_src = &data->data_src; 1021 1022 data_src->mem_op = PERF_MEM_OP_NA; 1023 1024 if (op_data3->ld_op) 1025 data_src->mem_op = PERF_MEM_OP_LOAD; 1026 else if (op_data3->st_op) 1027 data_src->mem_op = PERF_MEM_OP_STORE; 1028 } 1029 1030 /* 1031 * Processors having CPUID_Fn8000001B_EAX[11] aka IBS_CAPS_ZEN4 has 1032 * more fine granular DataSrc encodings. Others have coarse. 1033 */ 1034 static u8 perf_ibs_data_src(union ibs_op_data2 *op_data2) 1035 { 1036 if (ibs_caps & IBS_CAPS_ZEN4) 1037 return (op_data2->data_src_hi << 3) | op_data2->data_src_lo; 1038 1039 return op_data2->data_src_lo; 1040 } 1041 1042 #define L(x) (PERF_MEM_S(LVL, x) | PERF_MEM_S(LVL, HIT)) 1043 #define LN(x) PERF_MEM_S(LVLNUM, x) 1044 #define REM PERF_MEM_S(REMOTE, REMOTE) 1045 #define HOPS(x) PERF_MEM_S(HOPS, x) 1046 1047 static u64 g_data_src[8] = { 1048 [IBS_DATA_SRC_LOC_CACHE] = L(L3) | L(REM_CCE1) | LN(ANY_CACHE) | HOPS(0), 1049 [IBS_DATA_SRC_DRAM] = L(LOC_RAM) | LN(RAM), 1050 [IBS_DATA_SRC_REM_CACHE] = L(REM_CCE2) | LN(ANY_CACHE) | REM | HOPS(1), 1051 [IBS_DATA_SRC_IO] = L(IO) | LN(IO), 1052 }; 1053 1054 #define RMT_NODE_BITS (1 << IBS_DATA_SRC_DRAM) 1055 #define RMT_NODE_APPLICABLE(x) (RMT_NODE_BITS & (1 << x)) 1056 1057 static u64 g_zen4_data_src[32] = { 1058 [IBS_DATA_SRC_EXT_LOC_CACHE] = L(L3) | LN(L3), 1059 [IBS_DATA_SRC_EXT_NEAR_CCX_CACHE] = L(REM_CCE1) | LN(ANY_CACHE) | REM | HOPS(0), 1060 [IBS_DATA_SRC_EXT_DRAM] = L(LOC_RAM) | LN(RAM), 1061 [IBS_DATA_SRC_EXT_FAR_CCX_CACHE] = L(REM_CCE2) | LN(ANY_CACHE) | REM | HOPS(1), 1062 [IBS_DATA_SRC_EXT_PMEM] = LN(PMEM), 1063 [IBS_DATA_SRC_EXT_IO] = L(IO) | LN(IO), 1064 [IBS_DATA_SRC_EXT_EXT_MEM] = LN(CXL), 1065 }; 1066 1067 #define ZEN4_RMT_NODE_BITS ((1 << IBS_DATA_SRC_EXT_DRAM) | \ 1068 (1 << IBS_DATA_SRC_EXT_PMEM) | \ 1069 (1 << IBS_DATA_SRC_EXT_EXT_MEM)) 1070 #define ZEN4_RMT_NODE_APPLICABLE(x) (ZEN4_RMT_NODE_BITS & (1 << x)) 1071 1072 static __u64 perf_ibs_get_mem_lvl(union ibs_op_data2 *op_data2, 1073 union ibs_op_data3 *op_data3, 1074 struct perf_sample_data *data) 1075 { 1076 union perf_mem_data_src *data_src = &data->data_src; 1077 u8 ibs_data_src = perf_ibs_data_src(op_data2); 1078 1079 data_src->mem_lvl = 0; 1080 data_src->mem_lvl_num = 0; 1081 1082 /* 1083 * DcMiss, L2Miss, DataSrc, DcMissLat etc. are all invalid for Uncached 1084 * memory accesses. So, check DcUcMemAcc bit early. 1085 */ 1086 if (op_data3->dc_uc_mem_acc && ibs_data_src != IBS_DATA_SRC_EXT_IO) 1087 return L(UNC) | LN(UNC); 1088 1089 /* L1 Hit */ 1090 if (op_data3->dc_miss == 0) 1091 return L(L1) | LN(L1); 1092 1093 /* L2 Hit */ 1094 if (op_data3->l2_miss == 0) { 1095 /* Erratum #1293 */ 1096 if (boot_cpu_data.x86 != 0x19 || boot_cpu_data.x86_model > 0xF || 1097 !(op_data3->sw_pf || op_data3->dc_miss_no_mab_alloc)) 1098 return L(L2) | LN(L2); 1099 } 1100 1101 /* 1102 * OP_DATA2 is valid only for load ops. Skip all checks which 1103 * uses OP_DATA2[DataSrc]. 1104 */ 1105 if (data_src->mem_op != PERF_MEM_OP_LOAD) 1106 goto check_mab; 1107 1108 if (ibs_caps & IBS_CAPS_ZEN4) { 1109 u64 val = g_zen4_data_src[ibs_data_src]; 1110 1111 if (!val) 1112 goto check_mab; 1113 1114 /* HOPS_1 because IBS doesn't provide remote socket detail */ 1115 if (op_data2->rmt_node && ZEN4_RMT_NODE_APPLICABLE(ibs_data_src)) { 1116 if (ibs_data_src == IBS_DATA_SRC_EXT_DRAM) 1117 val = L(REM_RAM1) | LN(RAM) | REM | HOPS(1); 1118 else 1119 val |= REM | HOPS(1); 1120 } 1121 1122 return val; 1123 } else { 1124 u64 val = g_data_src[ibs_data_src]; 1125 1126 if (!val) 1127 goto check_mab; 1128 1129 /* HOPS_1 because IBS doesn't provide remote socket detail */ 1130 if (op_data2->rmt_node && RMT_NODE_APPLICABLE(ibs_data_src)) { 1131 if (ibs_data_src == IBS_DATA_SRC_DRAM) 1132 val = L(REM_RAM1) | LN(RAM) | REM | HOPS(1); 1133 else 1134 val |= REM | HOPS(1); 1135 } 1136 1137 return val; 1138 } 1139 1140 check_mab: 1141 /* 1142 * MAB (Miss Address Buffer) Hit. MAB keeps track of outstanding 1143 * DC misses. However, such data may come from any level in mem 1144 * hierarchy. IBS provides detail about both MAB as well as actual 1145 * DataSrc simultaneously. Prioritize DataSrc over MAB, i.e. set 1146 * MAB only when IBS fails to provide DataSrc. 1147 */ 1148 if (op_data3->dc_miss_no_mab_alloc) 1149 return L(LFB) | LN(LFB); 1150 1151 /* Don't set HIT with NA */ 1152 return PERF_MEM_S(LVL, NA) | LN(NA); 1153 } 1154 1155 static bool perf_ibs_cache_hit_st_valid(void) 1156 { 1157 /* 0: Uninitialized, 1: Valid, -1: Invalid */ 1158 static int cache_hit_st_valid; 1159 1160 if (unlikely(!cache_hit_st_valid)) { 1161 if (boot_cpu_data.x86 == 0x19 && 1162 (boot_cpu_data.x86_model <= 0xF || 1163 (boot_cpu_data.x86_model >= 0x20 && 1164 boot_cpu_data.x86_model <= 0x5F))) { 1165 cache_hit_st_valid = -1; 1166 } else { 1167 cache_hit_st_valid = 1; 1168 } 1169 } 1170 1171 return cache_hit_st_valid == 1; 1172 } 1173 1174 static void perf_ibs_get_mem_snoop(union ibs_op_data2 *op_data2, 1175 struct perf_sample_data *data) 1176 { 1177 union perf_mem_data_src *data_src = &data->data_src; 1178 u8 ibs_data_src; 1179 1180 data_src->mem_snoop = PERF_MEM_SNOOP_NA; 1181 1182 if (!perf_ibs_cache_hit_st_valid() || 1183 data_src->mem_op != PERF_MEM_OP_LOAD || 1184 data_src->mem_lvl & PERF_MEM_LVL_L1 || 1185 data_src->mem_lvl & PERF_MEM_LVL_L2 || 1186 op_data2->cache_hit_st) 1187 return; 1188 1189 ibs_data_src = perf_ibs_data_src(op_data2); 1190 1191 if (ibs_caps & IBS_CAPS_ZEN4) { 1192 if (ibs_data_src == IBS_DATA_SRC_EXT_LOC_CACHE || 1193 ibs_data_src == IBS_DATA_SRC_EXT_NEAR_CCX_CACHE || 1194 ibs_data_src == IBS_DATA_SRC_EXT_FAR_CCX_CACHE) 1195 data_src->mem_snoop = PERF_MEM_SNOOP_HITM; 1196 } else if (ibs_data_src == IBS_DATA_SRC_LOC_CACHE) { 1197 data_src->mem_snoop = PERF_MEM_SNOOP_HITM; 1198 } 1199 } 1200 1201 static void perf_ibs_get_tlb_lvl(union ibs_op_data3 *op_data3, 1202 struct perf_sample_data *data) 1203 { 1204 union perf_mem_data_src *data_src = &data->data_src; 1205 1206 data_src->mem_dtlb = PERF_MEM_TLB_NA; 1207 1208 if (!op_data3->dc_lin_addr_valid) 1209 return; 1210 1211 if ((ibs_caps & IBS_CAPS_OPDTLBPGSIZE) && 1212 !op_data3->dc_phy_addr_valid) 1213 return; 1214 1215 if (!op_data3->dc_l1tlb_miss) { 1216 data_src->mem_dtlb = PERF_MEM_TLB_L1 | PERF_MEM_TLB_HIT; 1217 return; 1218 } 1219 1220 if (!op_data3->dc_l2tlb_miss) { 1221 data_src->mem_dtlb = PERF_MEM_TLB_L2 | PERF_MEM_TLB_HIT; 1222 return; 1223 } 1224 1225 data_src->mem_dtlb = PERF_MEM_TLB_L2 | PERF_MEM_TLB_MISS; 1226 } 1227 1228 static void perf_ibs_get_mem_lock(union ibs_op_data3 *op_data3, 1229 struct perf_sample_data *data) 1230 { 1231 union perf_mem_data_src *data_src = &data->data_src; 1232 1233 data_src->mem_lock = PERF_MEM_LOCK_NA; 1234 1235 if (op_data3->dc_locked_op) 1236 data_src->mem_lock = PERF_MEM_LOCK_LOCKED; 1237 } 1238 1239 /* Be careful. Works only for contiguous MSRs. */ 1240 #define ibs_fetch_msr_idx(msr) (msr - MSR_AMD64_IBSFETCHCTL) 1241 #define ibs_op_msr_idx(msr) (msr - MSR_AMD64_IBSOPCTL) 1242 1243 static void perf_ibs_get_data_src(struct perf_ibs_data *ibs_data, 1244 struct perf_sample_data *data, 1245 union ibs_op_data2 *op_data2, 1246 union ibs_op_data3 *op_data3) 1247 { 1248 union perf_mem_data_src *data_src = &data->data_src; 1249 1250 data_src->val |= perf_ibs_get_mem_lvl(op_data2, op_data3, data); 1251 perf_ibs_get_mem_snoop(op_data2, data); 1252 perf_ibs_get_tlb_lvl(op_data3, data); 1253 perf_ibs_get_mem_lock(op_data3, data); 1254 } 1255 1256 static __u64 perf_ibs_get_op_data2(struct perf_ibs_data *ibs_data, 1257 union ibs_op_data3 *op_data3) 1258 { 1259 __u64 val = ibs_data->regs[ibs_op_msr_idx(MSR_AMD64_IBSOPDATA2)]; 1260 1261 /* Erratum #1293 */ 1262 if (boot_cpu_data.x86 == 0x19 && boot_cpu_data.x86_model <= 0xF && 1263 (op_data3->sw_pf || op_data3->dc_miss_no_mab_alloc)) { 1264 /* 1265 * OP_DATA2 has only two fields on Zen3: DataSrc and RmtNode. 1266 * DataSrc=0 is 'No valid status' and RmtNode is invalid when 1267 * DataSrc=0. 1268 */ 1269 val = 0; 1270 } 1271 return val; 1272 } 1273 1274 static void perf_ibs_parse_ld_st_data(__u64 sample_type, 1275 struct perf_ibs_data *ibs_data, 1276 struct perf_sample_data *data) 1277 { 1278 union ibs_op_data3 op_data3; 1279 union ibs_op_data2 op_data2; 1280 union ibs_op_data op_data; 1281 1282 data->data_src.val = PERF_MEM_NA; 1283 op_data3.val = ibs_data->regs[ibs_op_msr_idx(MSR_AMD64_IBSOPDATA3)]; 1284 1285 perf_ibs_get_mem_op(&op_data3, data); 1286 if (data->data_src.mem_op != PERF_MEM_OP_LOAD && 1287 data->data_src.mem_op != PERF_MEM_OP_STORE) 1288 return; 1289 1290 op_data2.val = perf_ibs_get_op_data2(ibs_data, &op_data3); 1291 1292 if (sample_type & PERF_SAMPLE_DATA_SRC) { 1293 perf_ibs_get_data_src(ibs_data, data, &op_data2, &op_data3); 1294 data->sample_flags |= PERF_SAMPLE_DATA_SRC; 1295 } 1296 1297 if (sample_type & PERF_SAMPLE_WEIGHT_TYPE && op_data3.dc_miss && 1298 data->data_src.mem_op == PERF_MEM_OP_LOAD) { 1299 op_data.val = ibs_data->regs[ibs_op_msr_idx(MSR_AMD64_IBSOPDATA)]; 1300 1301 if (sample_type & PERF_SAMPLE_WEIGHT_STRUCT) { 1302 data->weight.var1_dw = op_data3.dc_miss_lat; 1303 data->weight.var2_w = op_data.tag_to_ret_ctr; 1304 } else if (sample_type & PERF_SAMPLE_WEIGHT) { 1305 data->weight.full = op_data3.dc_miss_lat; 1306 } 1307 data->sample_flags |= PERF_SAMPLE_WEIGHT_TYPE; 1308 } 1309 1310 if (sample_type & PERF_SAMPLE_ADDR && op_data3.dc_lin_addr_valid) { 1311 data->addr = ibs_data->regs[ibs_op_msr_idx(MSR_AMD64_IBSDCLINAD)]; 1312 data->sample_flags |= PERF_SAMPLE_ADDR; 1313 } 1314 1315 if (sample_type & PERF_SAMPLE_PHYS_ADDR && op_data3.dc_phy_addr_valid) { 1316 data->phys_addr = ibs_data->regs[ibs_op_msr_idx(MSR_AMD64_IBSDCPHYSAD)]; 1317 data->sample_flags |= PERF_SAMPLE_PHYS_ADDR; 1318 } 1319 } 1320 1321 static bool perf_ibs_is_mem_sample_type(struct perf_ibs *perf_ibs, 1322 struct perf_event *event) 1323 { 1324 u64 sample_type = event->attr.sample_type; 1325 1326 return perf_ibs == &perf_ibs_op && 1327 sample_type & (PERF_SAMPLE_DATA_SRC | 1328 PERF_SAMPLE_WEIGHT_TYPE | 1329 PERF_SAMPLE_ADDR | 1330 PERF_SAMPLE_PHYS_ADDR); 1331 } 1332 1333 static int perf_ibs_get_offset_max(struct perf_ibs *perf_ibs, 1334 struct perf_event *event, 1335 int check_rip) 1336 { 1337 if (event->attr.sample_type & PERF_SAMPLE_RAW || 1338 perf_ibs_is_mem_sample_type(perf_ibs, event) || 1339 perf_ibs_ldlat_event(perf_ibs, event) || 1340 perf_ibs_fetch_lat_event(perf_ibs, event)) 1341 return perf_ibs->offset_max; 1342 else if (check_rip) 1343 return 3; 1344 return 1; 1345 } 1346 1347 static bool perf_ibs_is_kernel_data_addr(struct perf_event *event, 1348 struct perf_ibs_data *ibs_data) 1349 { 1350 u64 sample_type_mask = PERF_SAMPLE_ADDR | PERF_SAMPLE_RAW; 1351 union ibs_op_data3 op_data3; 1352 u64 dc_lin_addr; 1353 1354 op_data3.val = ibs_data->regs[ibs_op_msr_idx(MSR_AMD64_IBSOPDATA3)]; 1355 dc_lin_addr = ibs_data->regs[ibs_op_msr_idx(MSR_AMD64_IBSDCLINAD)]; 1356 1357 return unlikely((event->attr.sample_type & sample_type_mask) && 1358 op_data3.dc_lin_addr_valid && kernel_ip(dc_lin_addr)); 1359 } 1360 1361 static bool perf_ibs_is_kernel_br_target(struct perf_event *event, 1362 struct perf_ibs_data *ibs_data, 1363 int br_target_idx) 1364 { 1365 union ibs_op_data op_data; 1366 u64 br_target; 1367 1368 op_data.val = ibs_data->regs[ibs_op_msr_idx(MSR_AMD64_IBSOPDATA)]; 1369 br_target = ibs_data->regs[br_target_idx]; 1370 1371 return unlikely((event->attr.sample_type & PERF_SAMPLE_RAW) && 1372 op_data.op_brn_ret && kernel_ip(br_target)); 1373 } 1374 1375 static bool perf_ibs_discard_sample(struct perf_ibs *perf_ibs, struct perf_event *event, 1376 struct pt_regs *regs, struct perf_ibs_data *ibs_data, 1377 int br_target_idx) 1378 { 1379 if (perf_exclude_event(event, regs)) 1380 return true; 1381 1382 if (perf_ibs != &perf_ibs_op || !event->attr.exclude_kernel) 1383 return false; 1384 1385 if (perf_ibs_is_kernel_data_addr(event, ibs_data)) 1386 return true; 1387 1388 if (br_target_idx != -1 && 1389 perf_ibs_is_kernel_br_target(event, ibs_data, br_target_idx)) 1390 return true; 1391 1392 return false; 1393 } 1394 1395 static void perf_ibs_phyaddr_clear(struct perf_ibs *perf_ibs, 1396 struct perf_ibs_data *ibs_data) 1397 { 1398 if (perf_ibs == &perf_ibs_op) { 1399 ibs_data->regs[ibs_op_msr_idx(MSR_AMD64_IBSDCPHYSAD)] = 0; 1400 return; 1401 } 1402 1403 ibs_data->regs[ibs_fetch_msr_idx(MSR_AMD64_IBSFETCHPHYSAD)] = 0; 1404 } 1405 1406 static int perf_ibs_handle_irq(struct perf_ibs *perf_ibs, struct pt_regs *iregs) 1407 { 1408 struct cpu_perf_ibs *pcpu = this_cpu_ptr(perf_ibs->pcpu); 1409 struct perf_event *event = pcpu->event; 1410 struct hw_perf_event *hwc; 1411 struct perf_sample_data data; 1412 struct perf_raw_record raw; 1413 struct pt_regs regs; 1414 struct perf_ibs_data ibs_data; 1415 int offset, size, check_rip, offset_max, throttle = 0; 1416 unsigned int msr; 1417 u64 *buf, *config, period, new_config = 0; 1418 int br_target_idx = -1; 1419 1420 if (!test_bit(IBS_STARTED, pcpu->state)) { 1421 fail: 1422 /* 1423 * Catch spurious interrupts after stopping IBS: After 1424 * disabling IBS there could be still incoming NMIs 1425 * with samples that even have the valid bit cleared. 1426 * Mark all this NMIs as handled. 1427 */ 1428 if (test_and_clear_bit(IBS_STOPPED, pcpu->state)) 1429 return 1; 1430 1431 return 0; 1432 } 1433 1434 if (WARN_ON_ONCE(!event)) 1435 goto fail; 1436 1437 hwc = &event->hw; 1438 msr = hwc->config_base; 1439 buf = ibs_data.regs; 1440 rdmsrq(msr, *buf); 1441 if (!(*buf++ & perf_ibs->valid_mask)) 1442 goto fail; 1443 1444 config = &ibs_data.regs[0]; 1445 perf_ibs_event_update(perf_ibs, event, config); 1446 perf_sample_data_init(&data, 0, hwc->last_period); 1447 if (!perf_ibs_set_period(perf_ibs, hwc, &period)) 1448 goto out; /* no sw counter overflow */ 1449 1450 ibs_data.caps = ibs_caps; 1451 size = 1; 1452 offset = 1; 1453 check_rip = (perf_ibs == &perf_ibs_op && (ibs_caps & IBS_CAPS_RIPINVALIDCHK)); 1454 1455 offset_max = perf_ibs_get_offset_max(perf_ibs, event, check_rip); 1456 1457 do { 1458 rdmsrq(msr + offset, *buf++); 1459 size++; 1460 offset = find_next_bit(perf_ibs->offset_mask, 1461 perf_ibs->offset_max, 1462 offset + 1); 1463 } while (offset < offset_max); 1464 1465 if (perf_ibs_ldlat_event(perf_ibs, event)) { 1466 union ibs_op_data3 op_data3; 1467 1468 op_data3.val = ibs_data.regs[ibs_op_msr_idx(MSR_AMD64_IBSOPDATA3)]; 1469 /* 1470 * Opening event is errored out if load latency threshold is 1471 * outside of [128, 2048] range. Since the event has reached 1472 * interrupt handler, we can safely assume the threshold is 1473 * within [128, 2048] range. 1474 */ 1475 if (!op_data3.ld_op || !op_data3.dc_miss || 1476 op_data3.dc_miss_lat <= (event->attr.config1 & IBS_OP_CONFIG1_LDLAT_MASK)) { 1477 throttle = perf_event_account_interrupt(event); 1478 goto out; 1479 } 1480 } 1481 1482 if (perf_ibs_fetch_lat_event(perf_ibs, event)) { 1483 union ibs_fetch_ctl fetch_ctl; 1484 1485 fetch_ctl.val = ibs_data.regs[ibs_fetch_msr_idx(MSR_AMD64_IBSFETCHCTL)]; 1486 if (fetch_ctl.fetch_lat < (event->attr.config1 & IBS_FETCH_CONFIG1_FETCHLAT_MASK)) { 1487 throttle = perf_event_account_interrupt(event); 1488 goto out; 1489 } 1490 } 1491 1492 /* 1493 * Read IbsBrTarget, IbsOpData4, and IbsExtdCtl separately 1494 * depending on their availability. 1495 * Can't add to offset_max as they are staggered 1496 */ 1497 if (event->attr.sample_type & PERF_SAMPLE_RAW) { 1498 if (perf_ibs == &perf_ibs_op) { 1499 if (ibs_caps & IBS_CAPS_BRNTRGT) { 1500 rdmsrq(MSR_AMD64_IBSBRTARGET, *buf++); 1501 br_target_idx = size; 1502 size++; 1503 } 1504 if (ibs_caps & IBS_CAPS_OPDATA4) { 1505 rdmsrq(MSR_AMD64_IBSOPDATA4, *buf++); 1506 size++; 1507 } 1508 } 1509 if (perf_ibs == &perf_ibs_fetch && (ibs_caps & IBS_CAPS_FETCHCTLEXTD)) { 1510 rdmsrq(MSR_AMD64_ICIBSEXTDCTL, *buf++); 1511 size++; 1512 } 1513 } 1514 ibs_data.size = sizeof(u64) * size; 1515 1516 regs = *iregs; 1517 if (check_rip && (ibs_data.regs[2] & IBS_RIP_INVALID)) { 1518 regs.flags &= ~PERF_EFLAGS_EXACT; 1519 } else { 1520 /* Workaround for erratum #1197 */ 1521 if (perf_ibs->fetch_ignore_if_zero_rip && !(ibs_data.regs[1])) { 1522 throttle = perf_event_account_interrupt(event); 1523 goto out; 1524 } 1525 1526 set_linear_ip(®s, ibs_data.regs[1]); 1527 regs.flags |= PERF_EFLAGS_EXACT; 1528 } 1529 1530 if (((ibs_caps & IBS_CAPS_BIT63_FILTER) || 1531 (event->attr.config2 & IBS_SW_FILTER_MASK)) && 1532 perf_ibs_discard_sample(perf_ibs, event, ®s, &ibs_data, br_target_idx)) { 1533 throttle = perf_event_account_interrupt(event); 1534 goto out; 1535 } 1536 /* 1537 * Prevent leaking physical addresses to unprivileged users. Skip 1538 * PERF_SAMPLE_PHYS_ADDR check since generic code prevents it for 1539 * unprivileged users. 1540 */ 1541 if ((event->attr.sample_type & PERF_SAMPLE_RAW) && 1542 (hwc->flags & PERF_X86_EVENT_UNPRIVILEGED)) { 1543 perf_ibs_phyaddr_clear(perf_ibs, &ibs_data); 1544 } 1545 1546 if (event->attr.sample_type & PERF_SAMPLE_RAW) { 1547 raw = (struct perf_raw_record){ 1548 .frag = { 1549 .size = sizeof(u32) + ibs_data.size, 1550 .data = ibs_data.data, 1551 }, 1552 }; 1553 perf_sample_save_raw_data(&data, event, &raw); 1554 } 1555 1556 if (perf_ibs == &perf_ibs_op) 1557 perf_ibs_parse_ld_st_data(event->attr.sample_type, &ibs_data, &data); 1558 1559 /* 1560 * rip recorded by IbsOpRip will not be consistent with rsp and rbp 1561 * recorded as part of interrupt regs. Thus we need to use rip from 1562 * interrupt regs while unwinding call stack. 1563 */ 1564 perf_sample_save_callchain(&data, event, iregs); 1565 1566 throttle = perf_event_overflow(event, &data, ®s); 1567 1568 if (event->attr.freq && hwc->sample_period < perf_ibs->min_period) 1569 hwc->sample_period = perf_ibs->min_period; 1570 1571 out: 1572 if (!throttle) { 1573 if (ibs_caps & IBS_CAPS_DIS) 1574 wrmsrq(hwc->extra_reg.reg, perf_ibs->disable_mask); 1575 1576 if (perf_ibs == &perf_ibs_op) { 1577 if (ibs_caps & IBS_CAPS_OPCNTEXT) { 1578 new_config = period & IBS_OP_MAX_CNT_EXT_MASK; 1579 period &= ~IBS_OP_MAX_CNT_EXT_MASK; 1580 } 1581 if ((ibs_caps & IBS_CAPS_RDWROPCNT) && (*config & IBS_OP_CNT_CTL)) 1582 new_config |= *config & IBS_OP_CUR_CNT_RAND; 1583 } 1584 new_config |= period >> 4; 1585 1586 perf_ibs_enable_event(perf_ibs, hwc, new_config); 1587 } 1588 1589 perf_event_update_userpage(event); 1590 1591 return 1; 1592 } 1593 1594 static int 1595 perf_ibs_nmi_handler(unsigned int cmd, struct pt_regs *regs) 1596 { 1597 u64 stamp = sched_clock(); 1598 int handled = 0; 1599 1600 handled += perf_ibs_handle_irq(&perf_ibs_fetch, regs); 1601 handled += perf_ibs_handle_irq(&perf_ibs_op, regs); 1602 1603 if (handled) 1604 inc_perf_irq_stat(); 1605 1606 perf_sample_event_took(sched_clock() - stamp); 1607 1608 return handled; 1609 } 1610 NOKPROBE_SYMBOL(perf_ibs_nmi_handler); 1611 1612 static __init int perf_ibs_pmu_init(struct perf_ibs *perf_ibs, char *name) 1613 { 1614 struct cpu_perf_ibs __percpu *pcpu; 1615 int ret; 1616 1617 pcpu = alloc_percpu(struct cpu_perf_ibs); 1618 if (!pcpu) 1619 return -ENOMEM; 1620 1621 perf_ibs->pcpu = pcpu; 1622 1623 ret = perf_pmu_register(&perf_ibs->pmu, name, -1); 1624 if (ret) { 1625 perf_ibs->pcpu = NULL; 1626 free_percpu(pcpu); 1627 } 1628 1629 return ret; 1630 } 1631 1632 static __init int perf_ibs_fetch_init(void) 1633 { 1634 /* 1635 * Some chips fail to reset the fetch count when it is written; instead 1636 * they need a 0-1 transition of IbsFetchEn. 1637 */ 1638 if (boot_cpu_data.x86 >= 0x16 && boot_cpu_data.x86 <= 0x18) 1639 perf_ibs_fetch.fetch_count_reset_broken = 1; 1640 1641 if (boot_cpu_data.x86 == 0x19 && boot_cpu_data.x86_model < 0x10) 1642 perf_ibs_fetch.fetch_ignore_if_zero_rip = 1; 1643 1644 if (ibs_caps & IBS_CAPS_ZEN4) 1645 perf_ibs_fetch.config_mask |= IBS_FETCH_L3MISSONLY; 1646 1647 if (ibs_caps & IBS_CAPS_DIS) 1648 perf_ibs_fetch.disable_mask = IBS_FETCH_2_DIS; 1649 1650 perf_ibs_fetch.pmu.attr_groups = fetch_attr_groups; 1651 perf_ibs_fetch.pmu.attr_update = fetch_attr_update; 1652 1653 return perf_ibs_pmu_init(&perf_ibs_fetch, "ibs_fetch"); 1654 } 1655 1656 static __init int perf_ibs_op_init(void) 1657 { 1658 if (ibs_caps & IBS_CAPS_OPCNT) 1659 perf_ibs_op.config_mask |= IBS_OP_CNT_CTL; 1660 1661 if (ibs_caps & IBS_CAPS_OPCNTEXT) { 1662 perf_ibs_op.max_period |= IBS_OP_MAX_CNT_EXT_MASK; 1663 perf_ibs_op.config_mask |= IBS_OP_MAX_CNT_EXT_MASK; 1664 perf_ibs_op.cnt_mask |= (IBS_OP_MAX_CNT_EXT_MASK | 1665 IBS_OP_CUR_CNT_EXT_MASK); 1666 } 1667 1668 if (ibs_caps & IBS_CAPS_ZEN4) 1669 perf_ibs_op.config_mask |= IBS_OP_L3MISSONLY; 1670 1671 if (ibs_caps & IBS_CAPS_DIS) 1672 perf_ibs_op.disable_mask = IBS_OP_2_DIS; 1673 1674 perf_ibs_op.pmu.attr_groups = op_attr_groups; 1675 perf_ibs_op.pmu.attr_update = op_attr_update; 1676 1677 return perf_ibs_pmu_init(&perf_ibs_op, "ibs_op"); 1678 } 1679 1680 static __init int perf_event_ibs_init(void) 1681 { 1682 int ret; 1683 1684 ret = perf_ibs_fetch_init(); 1685 if (ret) 1686 return ret; 1687 1688 ret = perf_ibs_op_init(); 1689 if (ret) 1690 goto err_op; 1691 1692 ret = register_nmi_handler(NMI_LOCAL, perf_ibs_nmi_handler, 0, "perf_ibs"); 1693 if (ret) 1694 goto err_nmi; 1695 1696 pr_info("perf: AMD IBS detected (0x%08x)\n", ibs_caps); 1697 return 0; 1698 1699 err_nmi: 1700 perf_pmu_unregister(&perf_ibs_op.pmu); 1701 free_percpu(perf_ibs_op.pcpu); 1702 perf_ibs_op.pcpu = NULL; 1703 err_op: 1704 perf_pmu_unregister(&perf_ibs_fetch.pmu); 1705 free_percpu(perf_ibs_fetch.pcpu); 1706 perf_ibs_fetch.pcpu = NULL; 1707 1708 return ret; 1709 } 1710 1711 #else /* defined(CONFIG_PERF_EVENTS) && defined(CONFIG_CPU_SUP_AMD) */ 1712 1713 static __init int perf_event_ibs_init(void) 1714 { 1715 return 0; 1716 } 1717 1718 #endif 1719 1720 /* IBS - apic initialization, for perf and oprofile */ 1721 1722 static __init u32 __get_ibs_caps(void) 1723 { 1724 u32 caps; 1725 unsigned int max_level; 1726 1727 if (!boot_cpu_has(X86_FEATURE_IBS)) 1728 return 0; 1729 1730 /* check IBS cpuid feature flags */ 1731 max_level = cpuid_eax(0x80000000); 1732 if (max_level < IBS_CPUID_FEATURES) 1733 return IBS_CAPS_DEFAULT; 1734 1735 caps = cpuid_eax(IBS_CPUID_FEATURES); 1736 if (!(caps & IBS_CAPS_AVAIL)) 1737 /* cpuid flags not valid */ 1738 return IBS_CAPS_DEFAULT; 1739 1740 return caps; 1741 } 1742 1743 u32 get_ibs_caps(void) 1744 { 1745 return ibs_caps; 1746 } 1747 1748 EXPORT_SYMBOL(get_ibs_caps); 1749 1750 static inline int get_eilvt(int offset) 1751 { 1752 return !setup_APIC_eilvt(offset, 0, APIC_DELIVERY_MODE_NMI, 1); 1753 } 1754 1755 static inline int put_eilvt(int offset) 1756 { 1757 return !setup_APIC_eilvt(offset, 0, 0, 1); 1758 } 1759 1760 /* 1761 * Check and reserve APIC extended interrupt LVT offset for IBS if available. 1762 */ 1763 static inline int ibs_eilvt_valid(void) 1764 { 1765 int offset; 1766 u64 val; 1767 int valid = 0; 1768 1769 preempt_disable(); 1770 1771 rdmsrq(MSR_AMD64_IBSCTL, val); 1772 offset = val & IBSCTL_LVT_OFFSET_MASK; 1773 1774 if (!(val & IBSCTL_LVT_OFFSET_VALID)) { 1775 pr_err(FW_BUG "cpu %d, invalid IBS interrupt offset %d (MSR%08X=0x%016llx)\n", 1776 smp_processor_id(), offset, MSR_AMD64_IBSCTL, val); 1777 goto out; 1778 } 1779 1780 if (!get_eilvt(offset)) { 1781 pr_err(FW_BUG "cpu %d, IBS interrupt offset %d not available (MSR%08X=0x%016llx)\n", 1782 smp_processor_id(), offset, MSR_AMD64_IBSCTL, val); 1783 goto out; 1784 } 1785 1786 valid = 1; 1787 out: 1788 preempt_enable(); 1789 1790 return valid; 1791 } 1792 1793 static int setup_ibs_ctl(int ibs_eilvt_off) 1794 { 1795 struct pci_dev *cpu_cfg; 1796 int nodes; 1797 u32 value = 0; 1798 1799 nodes = 0; 1800 cpu_cfg = NULL; 1801 do { 1802 cpu_cfg = pci_get_device(PCI_VENDOR_ID_AMD, 1803 PCI_DEVICE_ID_AMD_10H_NB_MISC, 1804 cpu_cfg); 1805 if (!cpu_cfg) 1806 break; 1807 ++nodes; 1808 pci_write_config_dword(cpu_cfg, IBSCTL, ibs_eilvt_off 1809 | IBSCTL_LVT_OFFSET_VALID); 1810 pci_read_config_dword(cpu_cfg, IBSCTL, &value); 1811 if (value != (ibs_eilvt_off | IBSCTL_LVT_OFFSET_VALID)) { 1812 pci_dev_put(cpu_cfg); 1813 pr_debug("Failed to setup IBS LVT offset, IBSCTL = 0x%08x\n", 1814 value); 1815 return -EINVAL; 1816 } 1817 } while (1); 1818 1819 if (!nodes) { 1820 pr_debug("No CPU node configured for IBS\n"); 1821 return -ENODEV; 1822 } 1823 1824 return 0; 1825 } 1826 1827 /* 1828 * This runs only on the current cpu. We try to find an LVT offset and 1829 * setup the local APIC. For this we must disable preemption. On 1830 * success we initialize all nodes with this offset. This updates then 1831 * the offset in the IBS_CTL per-node msr. The per-core APIC setup of 1832 * the IBS interrupt vector is handled by perf_ibs_cpu_notifier that 1833 * is using the new offset. 1834 */ 1835 static void force_ibs_eilvt_setup(void) 1836 { 1837 int offset; 1838 int ret; 1839 1840 preempt_disable(); 1841 /* find the next free available EILVT entry, skip offset 0 */ 1842 for (offset = 1; offset < APIC_EILVT_NR_MAX; offset++) { 1843 if (get_eilvt(offset)) 1844 break; 1845 } 1846 preempt_enable(); 1847 1848 if (offset == APIC_EILVT_NR_MAX) { 1849 pr_debug("No EILVT entry available\n"); 1850 return; 1851 } 1852 1853 ret = setup_ibs_ctl(offset); 1854 if (ret) 1855 goto out; 1856 1857 if (!ibs_eilvt_valid()) 1858 goto out; 1859 1860 pr_info("LVT offset %d assigned\n", offset); 1861 1862 return; 1863 out: 1864 preempt_disable(); 1865 put_eilvt(offset); 1866 preempt_enable(); 1867 return; 1868 } 1869 1870 static void ibs_eilvt_setup(void) 1871 { 1872 /* 1873 * Force LVT offset assignment for family 10h: The offsets are 1874 * not assigned by the BIOS for this family, so the OS is 1875 * responsible for doing it. If the OS assignment fails, fall 1876 * back to BIOS settings and try to setup this. 1877 */ 1878 if (boot_cpu_data.x86 == 0x10) 1879 force_ibs_eilvt_setup(); 1880 } 1881 1882 static inline int get_ibs_lvt_offset(void) 1883 { 1884 u64 val; 1885 1886 rdmsrq(MSR_AMD64_IBSCTL, val); 1887 if (!(val & IBSCTL_LVT_OFFSET_VALID)) 1888 return -EINVAL; 1889 1890 return val & IBSCTL_LVT_OFFSET_MASK; 1891 } 1892 1893 static void setup_APIC_ibs(void) 1894 { 1895 int offset; 1896 1897 offset = get_ibs_lvt_offset(); 1898 if (offset < 0) 1899 goto failed; 1900 1901 if (!setup_APIC_eilvt(offset, 0, APIC_DELIVERY_MODE_NMI, 0)) 1902 return; 1903 failed: 1904 pr_warn("perf: IBS APIC setup failed on cpu #%d\n", 1905 smp_processor_id()); 1906 } 1907 1908 static void clear_APIC_ibs(void) 1909 { 1910 int offset; 1911 1912 offset = get_ibs_lvt_offset(); 1913 if (offset >= 0) 1914 setup_APIC_eilvt(offset, 0, APIC_DELIVERY_MODE_FIXED, 1); 1915 } 1916 1917 static int x86_pmu_amd_ibs_starting_cpu(unsigned int cpu) 1918 { 1919 setup_APIC_ibs(); 1920 1921 if (ibs_caps & IBS_CAPS_DIS) { 1922 /* 1923 * IBS enable sequence: 1924 * CTL[En] = 1; 1925 * CTL2[Dis] = 0; 1926 * 1927 * IBS disable sequence: 1928 * CTL2[Dis] = 1; 1929 * 1930 * Set CTL2[Dis] when CPU comes up. This is needed to make 1931 * enable sequence effective. 1932 */ 1933 wrmsrq(MSR_AMD64_IBSFETCHCTL2, IBS_FETCH_2_DIS); 1934 wrmsrq(MSR_AMD64_IBSOPCTL2, IBS_OP_2_DIS); 1935 } 1936 1937 return 0; 1938 } 1939 1940 #ifdef CONFIG_PM 1941 1942 static int perf_ibs_suspend(void *data) 1943 { 1944 clear_APIC_ibs(); 1945 return 0; 1946 } 1947 1948 static void perf_ibs_resume(void *data) 1949 { 1950 ibs_eilvt_setup(); 1951 setup_APIC_ibs(); 1952 } 1953 1954 static const struct syscore_ops perf_ibs_syscore_ops = { 1955 .resume = perf_ibs_resume, 1956 .suspend = perf_ibs_suspend, 1957 }; 1958 1959 static struct syscore perf_ibs_syscore = { 1960 .ops = &perf_ibs_syscore_ops, 1961 }; 1962 1963 static void perf_ibs_pm_init(void) 1964 { 1965 register_syscore(&perf_ibs_syscore); 1966 } 1967 1968 #else 1969 1970 static inline void perf_ibs_pm_init(void) { } 1971 1972 #endif 1973 1974 static int x86_pmu_amd_ibs_dying_cpu(unsigned int cpu) 1975 { 1976 clear_APIC_ibs(); 1977 return 0; 1978 } 1979 1980 static __init int amd_ibs_init(void) 1981 { 1982 u32 caps; 1983 1984 caps = __get_ibs_caps(); 1985 if (!caps) 1986 return -ENODEV; /* ibs not supported by the cpu */ 1987 1988 ibs_eilvt_setup(); 1989 1990 if (!ibs_eilvt_valid()) 1991 return -EINVAL; 1992 1993 perf_ibs_pm_init(); 1994 1995 #ifdef CONFIG_X86_32 1996 /* 1997 * IBS_CAPS_BIT63_FILTER is used for exclude_kernel/user filtering, 1998 * which obviously won't work for 32 bit kernel. 1999 */ 2000 caps &= ~IBS_CAPS_BIT63_FILTER; 2001 #endif 2002 2003 ibs_caps = caps; 2004 /* make ibs_caps visible to other cpus: */ 2005 smp_mb(); 2006 /* 2007 * x86_pmu_amd_ibs_starting_cpu will be called from core on 2008 * all online cpus. 2009 */ 2010 cpuhp_setup_state(CPUHP_AP_PERF_X86_AMD_IBS_STARTING, 2011 "perf/x86/amd/ibs:starting", 2012 x86_pmu_amd_ibs_starting_cpu, 2013 x86_pmu_amd_ibs_dying_cpu); 2014 2015 return perf_event_ibs_init(); 2016 } 2017 2018 /* Since we need the pci subsystem to init ibs we can't do this earlier: */ 2019 device_initcall(amd_ibs_init); 2020