1 // SPDX-License-Identifier: GPL-2.0 2 #include <linux/kvm_types.h> 3 #include <linux/perf_event.h> 4 #include <linux/types.h> 5 6 #include <asm/cpu_device_id.h> 7 #include <asm/cpuid/api.h> 8 #include <asm/perf_event.h> 9 #include <asm/msr.h> 10 11 #include "../perf_event.h" 12 13 /* 14 * Intel LBR_SELECT bits 15 * Intel Vol3a, April 2011, Section 16.7 Table 16-10 16 * 17 * Hardware branch filter (not available on all CPUs) 18 */ 19 #define LBR_KERNEL_BIT 0 /* do not capture at ring0 */ 20 #define LBR_USER_BIT 1 /* do not capture at ring > 0 */ 21 #define LBR_JCC_BIT 2 /* do not capture conditional branches */ 22 #define LBR_REL_CALL_BIT 3 /* do not capture relative calls */ 23 #define LBR_IND_CALL_BIT 4 /* do not capture indirect calls */ 24 #define LBR_RETURN_BIT 5 /* do not capture near returns */ 25 #define LBR_IND_JMP_BIT 6 /* do not capture indirect jumps */ 26 #define LBR_REL_JMP_BIT 7 /* do not capture relative jumps */ 27 #define LBR_FAR_BIT 8 /* do not capture far branches */ 28 #define LBR_CALL_STACK_BIT 9 /* enable call stack */ 29 30 /* 31 * Following bit only exists in Linux; we mask it out before writing it to 32 * the actual MSR. But it helps the constraint perf code to understand 33 * that this is a separate configuration. 34 */ 35 #define LBR_NO_INFO_BIT 63 /* don't read LBR_INFO. */ 36 37 #define LBR_KERNEL (1 << LBR_KERNEL_BIT) 38 #define LBR_USER (1 << LBR_USER_BIT) 39 #define LBR_JCC (1 << LBR_JCC_BIT) 40 #define LBR_REL_CALL (1 << LBR_REL_CALL_BIT) 41 #define LBR_IND_CALL (1 << LBR_IND_CALL_BIT) 42 #define LBR_RETURN (1 << LBR_RETURN_BIT) 43 #define LBR_REL_JMP (1 << LBR_REL_JMP_BIT) 44 #define LBR_IND_JMP (1 << LBR_IND_JMP_BIT) 45 #define LBR_FAR (1 << LBR_FAR_BIT) 46 #define LBR_CALL_STACK (1 << LBR_CALL_STACK_BIT) 47 #define LBR_NO_INFO (1ULL << LBR_NO_INFO_BIT) 48 49 #define LBR_PLM (LBR_KERNEL | LBR_USER) 50 51 #define LBR_SEL_MASK 0x3ff /* valid bits in LBR_SELECT */ 52 #define LBR_NOT_SUPP -1 /* LBR filter not supported */ 53 #define LBR_IGN 0 /* ignored */ 54 55 #define LBR_ANY \ 56 (LBR_JCC |\ 57 LBR_REL_CALL |\ 58 LBR_IND_CALL |\ 59 LBR_RETURN |\ 60 LBR_REL_JMP |\ 61 LBR_IND_JMP |\ 62 LBR_FAR) 63 64 #define LBR_FROM_FLAG_MISPRED BIT_ULL(63) 65 #define LBR_FROM_FLAG_IN_TX BIT_ULL(62) 66 #define LBR_FROM_FLAG_ABORT BIT_ULL(61) 67 68 #define LBR_FROM_SIGNEXT_2MSB (BIT_ULL(60) | BIT_ULL(59)) 69 70 /* 71 * Intel LBR_CTL bits 72 * 73 * Hardware branch filter for Arch LBR 74 */ 75 #define ARCH_LBR_KERNEL_BIT 1 /* capture at ring0 */ 76 #define ARCH_LBR_USER_BIT 2 /* capture at ring > 0 */ 77 #define ARCH_LBR_CALL_STACK_BIT 3 /* enable call stack */ 78 #define ARCH_LBR_JCC_BIT 16 /* capture conditional branches */ 79 #define ARCH_LBR_REL_JMP_BIT 17 /* capture relative jumps */ 80 #define ARCH_LBR_IND_JMP_BIT 18 /* capture indirect jumps */ 81 #define ARCH_LBR_REL_CALL_BIT 19 /* capture relative calls */ 82 #define ARCH_LBR_IND_CALL_BIT 20 /* capture indirect calls */ 83 #define ARCH_LBR_RETURN_BIT 21 /* capture near returns */ 84 #define ARCH_LBR_OTHER_BRANCH_BIT 22 /* capture other branches */ 85 86 #define ARCH_LBR_KERNEL (1ULL << ARCH_LBR_KERNEL_BIT) 87 #define ARCH_LBR_USER (1ULL << ARCH_LBR_USER_BIT) 88 #define ARCH_LBR_CALL_STACK (1ULL << ARCH_LBR_CALL_STACK_BIT) 89 #define ARCH_LBR_JCC (1ULL << ARCH_LBR_JCC_BIT) 90 #define ARCH_LBR_REL_JMP (1ULL << ARCH_LBR_REL_JMP_BIT) 91 #define ARCH_LBR_IND_JMP (1ULL << ARCH_LBR_IND_JMP_BIT) 92 #define ARCH_LBR_REL_CALL (1ULL << ARCH_LBR_REL_CALL_BIT) 93 #define ARCH_LBR_IND_CALL (1ULL << ARCH_LBR_IND_CALL_BIT) 94 #define ARCH_LBR_RETURN (1ULL << ARCH_LBR_RETURN_BIT) 95 #define ARCH_LBR_OTHER_BRANCH (1ULL << ARCH_LBR_OTHER_BRANCH_BIT) 96 97 #define ARCH_LBR_ANY \ 98 (ARCH_LBR_JCC |\ 99 ARCH_LBR_REL_JMP |\ 100 ARCH_LBR_IND_JMP |\ 101 ARCH_LBR_REL_CALL |\ 102 ARCH_LBR_IND_CALL |\ 103 ARCH_LBR_RETURN |\ 104 ARCH_LBR_OTHER_BRANCH) 105 106 #define ARCH_LBR_CTL_MASK 0x7f000e 107 108 static void intel_pmu_lbr_filter(struct cpu_hw_events *cpuc); 109 110 static __always_inline bool is_lbr_call_stack_bit_set(u64 config) 111 { 112 if (static_cpu_has(X86_FEATURE_ARCH_LBR)) 113 return !!(config & ARCH_LBR_CALL_STACK); 114 115 return !!(config & LBR_CALL_STACK); 116 } 117 118 /* 119 * We only support LBR implementations that have FREEZE_LBRS_ON_PMI 120 * otherwise it becomes near impossible to get a reliable stack. 121 */ 122 123 static void __intel_pmu_lbr_enable(bool pmi) 124 { 125 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 126 u64 debugctl, lbr_select = 0, orig_debugctl; 127 128 /* 129 * No need to unfreeze manually, as v4 can do that as part 130 * of the GLOBAL_STATUS ack. 131 */ 132 if (pmi && x86_pmu.version >= 4) 133 return; 134 135 /* 136 * No need to reprogram LBR_SELECT in a PMI, as it 137 * did not change. 138 */ 139 if (cpuc->lbr_sel) 140 lbr_select = cpuc->lbr_sel->config & x86_pmu.lbr_sel_mask; 141 if (!static_cpu_has(X86_FEATURE_ARCH_LBR) && !pmi && cpuc->lbr_sel) 142 wrmsrq(MSR_LBR_SELECT, lbr_select); 143 144 rdmsrq(MSR_IA32_DEBUGCTLMSR, debugctl); 145 orig_debugctl = debugctl; 146 147 if (!static_cpu_has(X86_FEATURE_ARCH_LBR)) 148 debugctl |= DEBUGCTLMSR_LBR; 149 /* 150 * LBR callstack does not work well with FREEZE_LBRS_ON_PMI. 151 * If FREEZE_LBRS_ON_PMI is set, PMI near call/return instructions 152 * may cause superfluous increase/decrease of LBR_TOS. 153 */ 154 if (is_lbr_call_stack_bit_set(lbr_select)) 155 debugctl &= ~DEBUGCTLMSR_FREEZE_LBRS_ON_PMI; 156 else 157 debugctl |= DEBUGCTLMSR_FREEZE_LBRS_ON_PMI; 158 159 if (orig_debugctl != debugctl) 160 wrmsrq(MSR_IA32_DEBUGCTLMSR, debugctl); 161 162 if (static_cpu_has(X86_FEATURE_ARCH_LBR)) 163 wrmsrq(MSR_ARCH_LBR_CTL, lbr_select | ARCH_LBR_CTL_LBREN); 164 } 165 166 void intel_pmu_lbr_reset_32(void) 167 { 168 int i; 169 170 for (i = 0; i < x86_pmu.lbr_nr; i++) 171 wrmsrq(x86_pmu.lbr_from + i, 0); 172 } 173 174 void intel_pmu_lbr_reset_64(void) 175 { 176 int i; 177 178 for (i = 0; i < x86_pmu.lbr_nr; i++) { 179 wrmsrq(x86_pmu.lbr_from + i, 0); 180 wrmsrq(x86_pmu.lbr_to + i, 0); 181 if (x86_pmu.lbr_has_info) 182 wrmsrq(x86_pmu.lbr_info + i, 0); 183 } 184 } 185 186 static void intel_pmu_arch_lbr_reset(void) 187 { 188 /* Write to ARCH_LBR_DEPTH MSR, all LBR entries are reset to 0 */ 189 wrmsrq(MSR_ARCH_LBR_DEPTH, x86_pmu.lbr_nr); 190 } 191 192 void intel_pmu_lbr_reset(void) 193 { 194 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 195 196 if (!x86_pmu.lbr_nr) 197 return; 198 199 x86_pmu.lbr_reset(); 200 201 cpuc->last_task_ctx = NULL; 202 cpuc->last_log_id = 0; 203 if (!static_cpu_has(X86_FEATURE_ARCH_LBR) && cpuc->lbr_select) 204 wrmsrq(MSR_LBR_SELECT, 0); 205 } 206 207 /* 208 * TOS = most recently recorded branch 209 */ 210 static inline u64 intel_pmu_lbr_tos(void) 211 { 212 u64 tos; 213 214 rdmsrq(x86_pmu.lbr_tos, tos); 215 return tos; 216 } 217 218 enum { 219 LBR_NONE, 220 LBR_VALID, 221 }; 222 223 /* 224 * For format LBR_FORMAT_EIP_FLAGS2, bits 61:62 in MSR_LAST_BRANCH_FROM_x 225 * are the TSX flags when TSX is supported, but when TSX is not supported 226 * they have no consistent behavior: 227 * 228 * - For wrmsr(), bits 61:62 are considered part of the sign extension. 229 * - For HW updates (branch captures) bits 61:62 are always OFF and are not 230 * part of the sign extension. 231 * 232 * Therefore, if: 233 * 234 * 1) LBR format LBR_FORMAT_EIP_FLAGS2 235 * 2) CPU has no TSX support enabled 236 * 237 * ... then any value passed to wrmsr() must be sign extended to 63 bits and any 238 * value from rdmsr() must be converted to have a 61 bits sign extension, 239 * ignoring the TSX flags. 240 */ 241 static inline bool lbr_from_signext_quirk_needed(void) 242 { 243 bool tsx_support = boot_cpu_has(X86_FEATURE_HLE) || 244 boot_cpu_has(X86_FEATURE_RTM); 245 246 return !tsx_support; 247 } 248 249 static DEFINE_STATIC_KEY_FALSE(lbr_from_quirk_key); 250 251 /* If quirk is enabled, ensure sign extension is 63 bits: */ 252 inline u64 lbr_from_signext_quirk_wr(u64 val) 253 { 254 if (static_branch_unlikely(&lbr_from_quirk_key)) { 255 /* 256 * Sign extend into bits 61:62 while preserving bit 63. 257 * 258 * Quirk is enabled when TSX is disabled. Therefore TSX bits 259 * in val are always OFF and must be changed to be sign 260 * extension bits. Since bits 59:60 are guaranteed to be 261 * part of the sign extension bits, we can just copy them 262 * to 61:62. 263 */ 264 val |= (LBR_FROM_SIGNEXT_2MSB & val) << 2; 265 } 266 return val; 267 } 268 269 /* 270 * If quirk is needed, ensure sign extension is 61 bits: 271 */ 272 static u64 lbr_from_signext_quirk_rd(u64 val) 273 { 274 if (static_branch_unlikely(&lbr_from_quirk_key)) { 275 /* 276 * Quirk is on when TSX is not enabled. Therefore TSX 277 * flags must be read as OFF. 278 */ 279 val &= ~(LBR_FROM_FLAG_IN_TX | LBR_FROM_FLAG_ABORT); 280 } 281 return val; 282 } 283 284 static __always_inline void wrlbr_from(unsigned int idx, u64 val) 285 { 286 val = lbr_from_signext_quirk_wr(val); 287 wrmsrq(x86_pmu.lbr_from + idx, val); 288 } 289 290 static __always_inline void wrlbr_to(unsigned int idx, u64 val) 291 { 292 wrmsrq(x86_pmu.lbr_to + idx, val); 293 } 294 295 static __always_inline void wrlbr_info(unsigned int idx, u64 val) 296 { 297 wrmsrq(x86_pmu.lbr_info + idx, val); 298 } 299 300 static __always_inline u64 rdlbr_from(unsigned int idx, struct lbr_entry *lbr) 301 { 302 u64 val; 303 304 if (lbr) 305 return lbr->from; 306 307 rdmsrq(x86_pmu.lbr_from + idx, val); 308 309 return lbr_from_signext_quirk_rd(val); 310 } 311 312 static __always_inline u64 rdlbr_to(unsigned int idx, struct lbr_entry *lbr) 313 { 314 u64 val; 315 316 if (lbr) 317 return lbr->to; 318 319 rdmsrq(x86_pmu.lbr_to + idx, val); 320 321 return val; 322 } 323 324 static __always_inline u64 rdlbr_info(unsigned int idx, struct lbr_entry *lbr) 325 { 326 u64 val; 327 328 if (lbr) 329 return lbr->info; 330 331 rdmsrq(x86_pmu.lbr_info + idx, val); 332 333 return val; 334 } 335 336 static inline void 337 wrlbr_all(struct lbr_entry *lbr, unsigned int idx, bool need_info) 338 { 339 wrlbr_from(idx, lbr->from); 340 wrlbr_to(idx, lbr->to); 341 if (need_info) 342 wrlbr_info(idx, lbr->info); 343 } 344 345 static inline bool 346 rdlbr_all(struct lbr_entry *lbr, unsigned int idx, bool need_info) 347 { 348 u64 from = rdlbr_from(idx, NULL); 349 350 /* Don't read invalid entry */ 351 if (!from) 352 return false; 353 354 lbr->from = from; 355 lbr->to = rdlbr_to(idx, NULL); 356 if (need_info) 357 lbr->info = rdlbr_info(idx, NULL); 358 359 return true; 360 } 361 362 void intel_pmu_lbr_restore(void *ctx) 363 { 364 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 365 struct x86_perf_task_context *task_ctx = ctx; 366 bool need_info = x86_pmu.lbr_has_info; 367 u64 tos = task_ctx->tos; 368 unsigned lbr_idx, mask; 369 int i; 370 371 mask = x86_pmu.lbr_nr - 1; 372 for (i = 0; i < task_ctx->valid_lbrs; i++) { 373 lbr_idx = (tos - i) & mask; 374 wrlbr_all(&task_ctx->lbr[i], lbr_idx, need_info); 375 } 376 377 for (; i < x86_pmu.lbr_nr; i++) { 378 lbr_idx = (tos - i) & mask; 379 wrlbr_from(lbr_idx, 0); 380 wrlbr_to(lbr_idx, 0); 381 if (need_info) 382 wrlbr_info(lbr_idx, 0); 383 } 384 385 wrmsrq(x86_pmu.lbr_tos, tos); 386 387 if (cpuc->lbr_select) 388 wrmsrq(MSR_LBR_SELECT, task_ctx->lbr_sel); 389 } 390 391 static void intel_pmu_arch_lbr_restore(void *ctx) 392 { 393 struct x86_perf_task_context_arch_lbr *task_ctx = ctx; 394 struct lbr_entry *entries = task_ctx->entries; 395 int i; 396 397 /* Fast reset the LBRs before restore if the call stack is not full. */ 398 if (!entries[x86_pmu.lbr_nr - 1].from) 399 intel_pmu_arch_lbr_reset(); 400 401 for (i = 0; i < x86_pmu.lbr_nr; i++) { 402 if (!entries[i].from) 403 break; 404 wrlbr_all(&entries[i], i, true); 405 } 406 } 407 408 /* 409 * Restore the Architecture LBR state from the xsave area in the perf 410 * context data for the task via the XRSTORS instruction. 411 */ 412 static void intel_pmu_arch_lbr_xrstors(void *ctx) 413 { 414 struct x86_perf_task_context_arch_lbr_xsave *task_ctx = ctx; 415 416 xrstors(&task_ctx->xsave, XFEATURE_MASK_LBR); 417 } 418 419 static __always_inline bool lbr_is_reset_in_cstate(void *ctx) 420 { 421 if (static_cpu_has(X86_FEATURE_ARCH_LBR)) 422 return x86_pmu.lbr_deep_c_reset && !rdlbr_from(0, NULL); 423 424 return !rdlbr_from(((struct x86_perf_task_context *)ctx)->tos, NULL); 425 } 426 427 static inline bool has_lbr_callstack_users(void *ctx) 428 { 429 return task_context_opt(ctx)->lbr_callstack_users || 430 x86_pmu.lbr_callstack_users; 431 } 432 433 static void __intel_pmu_lbr_restore(void *ctx) 434 { 435 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 436 437 if (!has_lbr_callstack_users(ctx) || 438 task_context_opt(ctx)->lbr_stack_state == LBR_NONE) { 439 intel_pmu_lbr_reset(); 440 return; 441 } 442 443 /* 444 * Does not restore the LBR registers, if 445 * - No one else touched them, and 446 * - Was not cleared in Cstate 447 */ 448 if ((ctx == cpuc->last_task_ctx) && 449 (task_context_opt(ctx)->log_id == cpuc->last_log_id) && 450 !lbr_is_reset_in_cstate(ctx)) { 451 task_context_opt(ctx)->lbr_stack_state = LBR_NONE; 452 return; 453 } 454 455 x86_pmu.lbr_restore(ctx); 456 457 task_context_opt(ctx)->lbr_stack_state = LBR_NONE; 458 } 459 460 void intel_pmu_lbr_save(void *ctx) 461 { 462 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 463 struct x86_perf_task_context *task_ctx = ctx; 464 bool need_info = x86_pmu.lbr_has_info; 465 unsigned lbr_idx, mask; 466 u64 tos; 467 int i; 468 469 mask = x86_pmu.lbr_nr - 1; 470 tos = intel_pmu_lbr_tos(); 471 for (i = 0; i < x86_pmu.lbr_nr; i++) { 472 lbr_idx = (tos - i) & mask; 473 if (!rdlbr_all(&task_ctx->lbr[i], lbr_idx, need_info)) 474 break; 475 } 476 task_ctx->valid_lbrs = i; 477 task_ctx->tos = tos; 478 479 if (cpuc->lbr_select) 480 rdmsrq(MSR_LBR_SELECT, task_ctx->lbr_sel); 481 } 482 483 static void intel_pmu_arch_lbr_save(void *ctx) 484 { 485 struct x86_perf_task_context_arch_lbr *task_ctx = ctx; 486 struct lbr_entry *entries = task_ctx->entries; 487 int i; 488 489 for (i = 0; i < x86_pmu.lbr_nr; i++) { 490 if (!rdlbr_all(&entries[i], i, true)) 491 break; 492 } 493 494 /* LBR call stack is not full. Reset is required in restore. */ 495 if (i < x86_pmu.lbr_nr) 496 entries[x86_pmu.lbr_nr - 1].from = 0; 497 } 498 499 /* 500 * Save the Architecture LBR state to the xsave area in the perf 501 * context data for the task via the XSAVES instruction. 502 */ 503 static void intel_pmu_arch_lbr_xsaves(void *ctx) 504 { 505 struct x86_perf_task_context_arch_lbr_xsave *task_ctx = ctx; 506 507 xsaves(&task_ctx->xsave, XFEATURE_MASK_LBR); 508 } 509 510 static void __intel_pmu_lbr_save(void *ctx) 511 { 512 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 513 514 if (!has_lbr_callstack_users(ctx)) { 515 task_context_opt(ctx)->lbr_stack_state = LBR_NONE; 516 return; 517 } 518 519 x86_pmu.lbr_save(ctx); 520 521 task_context_opt(ctx)->lbr_stack_state = LBR_VALID; 522 523 cpuc->last_task_ctx = ctx; 524 cpuc->last_log_id = ++task_context_opt(ctx)->log_id; 525 } 526 527 void intel_pmu_lbr_sched_task(struct perf_event_pmu_context *pmu_ctx, 528 struct task_struct *task, bool sched_in) 529 { 530 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 531 struct perf_ctx_data *ctx_data; 532 void *task_ctx; 533 534 if (!cpuc->lbr_users) 535 return; 536 537 /* 538 * If LBR callstack feature is enabled and the stack was saved when 539 * the task was scheduled out, restore the stack. Otherwise flush 540 * the LBR stack. 541 */ 542 rcu_read_lock(); 543 ctx_data = rcu_dereference(task->perf_ctx_data); 544 task_ctx = ctx_data ? ctx_data->data : NULL; 545 if (task_ctx) { 546 if (sched_in) 547 __intel_pmu_lbr_restore(task_ctx); 548 else 549 __intel_pmu_lbr_save(task_ctx); 550 rcu_read_unlock(); 551 return; 552 } 553 rcu_read_unlock(); 554 555 /* 556 * Since a context switch can flip the address space and LBR entries 557 * are not tagged with an identifier, we need to wipe the LBR, even for 558 * per-cpu events. You simply cannot resolve the branches from the old 559 * address space. 560 */ 561 if (sched_in) 562 intel_pmu_lbr_reset(); 563 } 564 565 static inline bool branch_user_callstack(unsigned br_sel) 566 { 567 return (br_sel & X86_BR_USER) && (br_sel & X86_BR_CALL_STACK); 568 } 569 570 void intel_pmu_lbr_add(struct perf_event *event) 571 { 572 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 573 574 if (!x86_pmu.lbr_nr) 575 return; 576 577 if (event->hw.flags & PERF_X86_EVENT_LBR_SELECT) 578 cpuc->lbr_select = 1; 579 580 cpuc->br_sel = event->hw.branch_reg.reg; 581 582 if (branch_user_callstack(cpuc->br_sel)) { 583 if (event->attach_state & PERF_ATTACH_TASK) { 584 struct task_struct *task = event->hw.target; 585 struct perf_ctx_data *ctx_data; 586 587 rcu_read_lock(); 588 ctx_data = rcu_dereference(task->perf_ctx_data); 589 if (ctx_data) 590 task_context_opt(ctx_data->data)->lbr_callstack_users++; 591 rcu_read_unlock(); 592 } else 593 x86_pmu.lbr_callstack_users++; 594 } 595 /* 596 * Request pmu::sched_task() callback, which will fire inside the 597 * regular perf event scheduling, so that call will: 598 * 599 * - restore or wipe; when LBR-callstack, 600 * - wipe; otherwise, 601 * 602 * when this is from __perf_event_task_sched_in(). 603 * 604 * However, if this is from perf_install_in_context(), no such callback 605 * will follow and we'll need to reset the LBR here if this is the 606 * first LBR event. 607 * 608 * The problem is, we cannot tell these cases apart... but we can 609 * exclude the biggest chunk of cases by looking at 610 * event->total_time_running. An event that has accrued runtime cannot 611 * be 'new'. Conversely, a new event can get installed through the 612 * context switch path for the first time. 613 */ 614 if (x86_pmu.intel_cap.pebs_baseline && event->attr.precise_ip > 0) 615 cpuc->lbr_pebs_users++; 616 perf_sched_cb_inc(event->pmu); 617 if (!cpuc->lbr_users++ && !event->total_time_running) 618 intel_pmu_lbr_reset(); 619 } 620 621 void release_lbr_buffers(void) 622 { 623 struct kmem_cache *kmem_cache; 624 struct cpu_hw_events *cpuc; 625 int cpu; 626 627 if (!static_cpu_has(X86_FEATURE_ARCH_LBR)) 628 return; 629 630 for_each_possible_cpu(cpu) { 631 cpuc = per_cpu_ptr(&cpu_hw_events, cpu); 632 kmem_cache = x86_get_pmu(cpu)->task_ctx_cache; 633 if (kmem_cache && cpuc->lbr_xsave) { 634 kmem_cache_free(kmem_cache, cpuc->lbr_xsave); 635 cpuc->lbr_xsave = NULL; 636 } 637 } 638 } 639 640 void reserve_lbr_buffers(void) 641 { 642 struct kmem_cache *kmem_cache; 643 struct cpu_hw_events *cpuc; 644 int cpu; 645 646 if (!static_cpu_has(X86_FEATURE_ARCH_LBR)) 647 return; 648 649 for_each_possible_cpu(cpu) { 650 cpuc = per_cpu_ptr(&cpu_hw_events, cpu); 651 kmem_cache = x86_get_pmu(cpu)->task_ctx_cache; 652 if (!kmem_cache || cpuc->lbr_xsave) 653 continue; 654 655 cpuc->lbr_xsave = kmem_cache_alloc_node(kmem_cache, 656 GFP_KERNEL | __GFP_ZERO, 657 cpu_to_node(cpu)); 658 } 659 } 660 661 void intel_pmu_lbr_del(struct perf_event *event) 662 { 663 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 664 665 if (!x86_pmu.lbr_nr) 666 return; 667 668 if (branch_user_callstack(cpuc->br_sel)) { 669 if (event->attach_state & PERF_ATTACH_TASK) { 670 struct task_struct *task = event->hw.target; 671 struct perf_ctx_data *ctx_data; 672 673 rcu_read_lock(); 674 ctx_data = rcu_dereference(task->perf_ctx_data); 675 if (ctx_data) 676 task_context_opt(ctx_data->data)->lbr_callstack_users--; 677 rcu_read_unlock(); 678 } else 679 x86_pmu.lbr_callstack_users--; 680 } 681 682 if (event->hw.flags & PERF_X86_EVENT_LBR_SELECT) 683 cpuc->lbr_select = 0; 684 685 if (x86_pmu.intel_cap.pebs_baseline && event->attr.precise_ip > 0) 686 cpuc->lbr_pebs_users--; 687 cpuc->lbr_users--; 688 WARN_ON_ONCE(cpuc->lbr_users < 0); 689 WARN_ON_ONCE(cpuc->lbr_pebs_users < 0); 690 perf_sched_cb_dec(event->pmu); 691 692 /* 693 * The logged occurrences information is only valid for the 694 * current LBR group. If another LBR group is scheduled in 695 * later, the information from the stale LBRs will be wrongly 696 * interpreted. Reset the LBRs here. 697 * 698 * Only clear once for a branch counter group with the leader 699 * event. Because 700 * - Cannot simply reset the LBRs with the !cpuc->lbr_users. 701 * Because it's possible that the last LBR user is not in a 702 * branch counter group, e.g., a branch_counters group + 703 * several normal LBR events. 704 * - The LBR reset can be done with any one of the events in a 705 * branch counter group, since they are always scheduled together. 706 * It's easy to force the leader event an LBR event. 707 */ 708 if (is_branch_counters_group(event) && event == event->group_leader) 709 intel_pmu_lbr_reset(); 710 } 711 712 static inline bool vlbr_exclude_host(void) 713 { 714 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 715 716 return test_bit(INTEL_PMC_IDX_FIXED_VLBR, 717 (unsigned long *)&cpuc->intel_ctrl_guest_mask); 718 } 719 720 void intel_pmu_lbr_enable_all(bool pmi) 721 { 722 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 723 724 if (cpuc->lbr_users && !vlbr_exclude_host()) 725 __intel_pmu_lbr_enable(pmi); 726 } 727 728 void intel_pmu_lbr_disable_all(void) 729 { 730 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 731 732 if (cpuc->lbr_users && !vlbr_exclude_host()) { 733 if (static_cpu_has(X86_FEATURE_ARCH_LBR)) 734 return __intel_pmu_arch_lbr_disable(); 735 736 __intel_pmu_lbr_disable(); 737 } 738 } 739 740 void intel_pmu_lbr_read_32(struct cpu_hw_events *cpuc) 741 { 742 unsigned long mask = x86_pmu.lbr_nr - 1; 743 struct perf_branch_entry *br = cpuc->lbr_entries; 744 u64 tos = intel_pmu_lbr_tos(); 745 int i; 746 747 for (i = 0; i < x86_pmu.lbr_nr; i++) { 748 unsigned long lbr_idx = (tos - i) & mask; 749 union { 750 struct { 751 u32 from; 752 u32 to; 753 }; 754 u64 lbr; 755 } msr_lastbranch; 756 757 rdmsrq(x86_pmu.lbr_from + lbr_idx, msr_lastbranch.lbr); 758 759 perf_clear_branch_entry_bitfields(br); 760 761 br->from = msr_lastbranch.from; 762 br->to = msr_lastbranch.to; 763 br++; 764 } 765 cpuc->lbr_stack.nr = i; 766 cpuc->lbr_stack.hw_idx = tos; 767 } 768 769 /* 770 * Due to lack of segmentation in Linux the effective address (offset) 771 * is the same as the linear address, allowing us to merge the LIP and EIP 772 * LBR formats. 773 */ 774 void intel_pmu_lbr_read_64(struct cpu_hw_events *cpuc) 775 { 776 bool need_info = false, call_stack = false; 777 unsigned long mask = x86_pmu.lbr_nr - 1; 778 struct perf_branch_entry *br = cpuc->lbr_entries; 779 u64 tos = intel_pmu_lbr_tos(); 780 int i; 781 int out = 0; 782 int num = x86_pmu.lbr_nr; 783 784 if (cpuc->lbr_sel) { 785 need_info = !(cpuc->lbr_sel->config & LBR_NO_INFO); 786 if (cpuc->lbr_sel->config & LBR_CALL_STACK) 787 call_stack = true; 788 } 789 790 for (i = 0; i < num; i++) { 791 unsigned long lbr_idx = (tos - i) & mask; 792 u64 from, to, mis = 0, pred = 0, in_tx = 0, abort = 0; 793 u16 cycles = 0; 794 795 from = rdlbr_from(lbr_idx, NULL); 796 to = rdlbr_to(lbr_idx, NULL); 797 798 /* 799 * Read LBR call stack entries 800 * until invalid entry (0s) is detected. 801 */ 802 if (call_stack && !from) 803 break; 804 805 if (x86_pmu.lbr_has_info) { 806 if (need_info) { 807 u64 info; 808 809 info = rdlbr_info(lbr_idx, NULL); 810 mis = !!(info & LBR_INFO_MISPRED); 811 pred = !mis; 812 cycles = (info & LBR_INFO_CYCLES); 813 if (x86_pmu.lbr_has_tsx) { 814 in_tx = !!(info & LBR_INFO_IN_TX); 815 abort = !!(info & LBR_INFO_ABORT); 816 } 817 } 818 } else { 819 int skip = 0; 820 821 if (x86_pmu.lbr_from_flags) { 822 mis = !!(from & LBR_FROM_FLAG_MISPRED); 823 pred = !mis; 824 skip = 1; 825 } 826 if (x86_pmu.lbr_has_tsx) { 827 in_tx = !!(from & LBR_FROM_FLAG_IN_TX); 828 abort = !!(from & LBR_FROM_FLAG_ABORT); 829 skip = 3; 830 } 831 from = (u64)((((s64)from) << skip) >> skip); 832 833 if (x86_pmu.lbr_to_cycles) { 834 cycles = ((to >> 48) & LBR_INFO_CYCLES); 835 to = (u64)((((s64)to) << 16) >> 16); 836 } 837 } 838 839 /* 840 * Some CPUs report duplicated abort records, 841 * with the second entry not having an abort bit set. 842 * Skip them here. This loop runs backwards, 843 * so we need to undo the previous record. 844 * If the abort just happened outside the window 845 * the extra entry cannot be removed. 846 */ 847 if (abort && x86_pmu.lbr_double_abort && out > 0) 848 out--; 849 850 perf_clear_branch_entry_bitfields(br+out); 851 br[out].from = from; 852 br[out].to = to; 853 br[out].mispred = mis; 854 br[out].predicted = pred; 855 br[out].in_tx = in_tx; 856 br[out].abort = abort; 857 br[out].cycles = cycles; 858 out++; 859 } 860 cpuc->lbr_stack.nr = out; 861 cpuc->lbr_stack.hw_idx = tos; 862 } 863 864 static DEFINE_STATIC_KEY_FALSE(x86_lbr_mispred); 865 static DEFINE_STATIC_KEY_FALSE(x86_lbr_cycles); 866 static DEFINE_STATIC_KEY_FALSE(x86_lbr_type); 867 868 static __always_inline int get_lbr_br_type(u64 info) 869 { 870 int type = 0; 871 872 if (static_branch_likely(&x86_lbr_type)) 873 type = (info & LBR_INFO_BR_TYPE) >> LBR_INFO_BR_TYPE_OFFSET; 874 875 return type; 876 } 877 878 static __always_inline bool get_lbr_mispred(u64 info) 879 { 880 bool mispred = 0; 881 882 if (static_branch_likely(&x86_lbr_mispred)) 883 mispred = !!(info & LBR_INFO_MISPRED); 884 885 return mispred; 886 } 887 888 static __always_inline u16 get_lbr_cycles(u64 info) 889 { 890 u16 cycles = info & LBR_INFO_CYCLES; 891 892 if (static_cpu_has(X86_FEATURE_ARCH_LBR) && 893 (!static_branch_likely(&x86_lbr_cycles) || 894 !(info & LBR_INFO_CYC_CNT_VALID))) 895 cycles = 0; 896 897 return cycles; 898 } 899 900 static_assert((64 - PERF_BRANCH_ENTRY_INFO_BITS_MAX) > LBR_INFO_BR_CNTR_NUM * LBR_INFO_BR_CNTR_BITS); 901 902 static void intel_pmu_store_lbr(struct cpu_hw_events *cpuc, 903 struct lbr_entry *entries) 904 { 905 struct perf_branch_entry *e; 906 struct lbr_entry *lbr; 907 u64 from, to, info; 908 int i; 909 910 for (i = 0; i < x86_pmu.lbr_nr; i++) { 911 lbr = entries ? &entries[i] : NULL; 912 e = &cpuc->lbr_entries[i]; 913 914 from = rdlbr_from(i, lbr); 915 /* 916 * Read LBR entries until invalid entry (0s) is detected. 917 */ 918 if (!from) 919 break; 920 921 to = rdlbr_to(i, lbr); 922 info = rdlbr_info(i, lbr); 923 924 perf_clear_branch_entry_bitfields(e); 925 926 e->from = from; 927 e->to = to; 928 e->mispred = get_lbr_mispred(info); 929 e->predicted = !e->mispred; 930 e->in_tx = !!(info & LBR_INFO_IN_TX); 931 e->abort = !!(info & LBR_INFO_ABORT); 932 e->cycles = get_lbr_cycles(info); 933 e->type = get_lbr_br_type(info); 934 935 /* 936 * Leverage the reserved field of cpuc->lbr_entries[i] to 937 * temporarily store the branch counters information. 938 * The later code will decide what content can be disclosed 939 * to the perf tool. Pleae see intel_pmu_lbr_counters_reorder(). 940 */ 941 e->reserved = (info >> LBR_INFO_BR_CNTR_OFFSET) & LBR_INFO_BR_CNTR_FULL_MASK; 942 } 943 944 cpuc->lbr_stack.nr = i; 945 } 946 947 /* 948 * The enabled order may be different from the counter order. 949 * Update the lbr_counters with the enabled order. 950 */ 951 static void intel_pmu_lbr_counters_reorder(struct cpu_hw_events *cpuc, 952 struct perf_event *event) 953 { 954 int i, j, pos = 0, order[X86_PMC_IDX_MAX]; 955 struct perf_event *leader, *sibling; 956 u64 src, dst, cnt; 957 958 leader = event->group_leader; 959 if (branch_sample_counters(leader)) 960 order[pos++] = leader->hw.idx; 961 962 for_each_sibling_event(sibling, leader) { 963 if (!branch_sample_counters(sibling)) 964 continue; 965 order[pos++] = sibling->hw.idx; 966 } 967 968 WARN_ON_ONCE(!pos); 969 970 for (i = 0; i < cpuc->lbr_stack.nr; i++) { 971 src = cpuc->lbr_entries[i].reserved; 972 dst = 0; 973 for (j = 0; j < pos; j++) { 974 cnt = (src >> (order[j] * LBR_INFO_BR_CNTR_BITS)) & LBR_INFO_BR_CNTR_MASK; 975 dst |= cnt << j * LBR_INFO_BR_CNTR_BITS; 976 } 977 cpuc->lbr_counters[i] = dst; 978 cpuc->lbr_entries[i].reserved = 0; 979 } 980 } 981 982 void intel_pmu_lbr_save_brstack(struct perf_sample_data *data, 983 struct cpu_hw_events *cpuc, 984 struct perf_event *event) 985 { 986 if (is_branch_counters_group(event)) { 987 intel_pmu_lbr_counters_reorder(cpuc, event); 988 perf_sample_save_brstack(data, event, &cpuc->lbr_stack, cpuc->lbr_counters); 989 return; 990 } 991 992 perf_sample_save_brstack(data, event, &cpuc->lbr_stack, NULL); 993 } 994 995 static void intel_pmu_arch_lbr_read(struct cpu_hw_events *cpuc) 996 { 997 intel_pmu_store_lbr(cpuc, NULL); 998 } 999 1000 static void intel_pmu_arch_lbr_read_xsave(struct cpu_hw_events *cpuc) 1001 { 1002 struct x86_perf_task_context_arch_lbr_xsave *xsave = cpuc->lbr_xsave; 1003 1004 if (!xsave) { 1005 intel_pmu_store_lbr(cpuc, NULL); 1006 return; 1007 } 1008 xsaves(&xsave->xsave, XFEATURE_MASK_LBR); 1009 1010 intel_pmu_store_lbr(cpuc, xsave->lbr.entries); 1011 } 1012 1013 void intel_pmu_lbr_read(void) 1014 { 1015 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 1016 1017 /* 1018 * Don't read when all LBRs users are using adaptive PEBS. 1019 * 1020 * This could be smarter and actually check the event, 1021 * but this simple approach seems to work for now. 1022 */ 1023 if (!cpuc->lbr_users || vlbr_exclude_host() || 1024 cpuc->lbr_users == cpuc->lbr_pebs_users) 1025 return; 1026 1027 x86_pmu.lbr_read(cpuc); 1028 1029 intel_pmu_lbr_filter(cpuc); 1030 } 1031 1032 /* 1033 * SW filter is used: 1034 * - in case there is no HW filter 1035 * - in case the HW filter has errata or limitations 1036 */ 1037 static int intel_pmu_setup_sw_lbr_filter(struct perf_event *event) 1038 { 1039 u64 br_type = event->attr.branch_sample_type; 1040 int mask = 0; 1041 1042 if (br_type & PERF_SAMPLE_BRANCH_USER) 1043 mask |= X86_BR_USER; 1044 1045 if (br_type & PERF_SAMPLE_BRANCH_KERNEL) 1046 mask |= X86_BR_KERNEL; 1047 1048 /* we ignore BRANCH_HV here */ 1049 1050 if (br_type & PERF_SAMPLE_BRANCH_ANY) 1051 mask |= X86_BR_ANY; 1052 1053 if (br_type & PERF_SAMPLE_BRANCH_ANY_CALL) 1054 mask |= X86_BR_ANY_CALL; 1055 1056 if (br_type & PERF_SAMPLE_BRANCH_ANY_RETURN) 1057 mask |= X86_BR_RET | X86_BR_IRET | X86_BR_SYSRET; 1058 1059 if (br_type & PERF_SAMPLE_BRANCH_IND_CALL) 1060 mask |= X86_BR_IND_CALL; 1061 1062 if (br_type & PERF_SAMPLE_BRANCH_ABORT_TX) 1063 mask |= X86_BR_ABORT; 1064 1065 if (br_type & PERF_SAMPLE_BRANCH_IN_TX) 1066 mask |= X86_BR_IN_TX; 1067 1068 if (br_type & PERF_SAMPLE_BRANCH_NO_TX) 1069 mask |= X86_BR_NO_TX; 1070 1071 if (br_type & PERF_SAMPLE_BRANCH_COND) 1072 mask |= X86_BR_JCC; 1073 1074 if (br_type & PERF_SAMPLE_BRANCH_CALL_STACK) { 1075 if (!x86_pmu_has_lbr_callstack()) 1076 return -EOPNOTSUPP; 1077 if (mask & ~(X86_BR_USER | X86_BR_KERNEL)) 1078 return -EINVAL; 1079 mask |= X86_BR_CALL | X86_BR_IND_CALL | X86_BR_RET | 1080 X86_BR_CALL_STACK; 1081 } 1082 1083 if (br_type & PERF_SAMPLE_BRANCH_IND_JUMP) 1084 mask |= X86_BR_IND_JMP; 1085 1086 if (br_type & PERF_SAMPLE_BRANCH_CALL) 1087 mask |= X86_BR_CALL | X86_BR_ZERO_CALL; 1088 1089 if (br_type & PERF_SAMPLE_BRANCH_TYPE_SAVE) 1090 mask |= X86_BR_TYPE_SAVE; 1091 1092 /* 1093 * stash actual user request into reg, it may 1094 * be used by fixup code for some CPU 1095 */ 1096 event->hw.branch_reg.reg = mask; 1097 return 0; 1098 } 1099 1100 /* 1101 * setup the HW LBR filter 1102 * Used only when available, may not be enough to disambiguate 1103 * all branches, may need the help of the SW filter 1104 */ 1105 static int intel_pmu_setup_hw_lbr_filter(struct perf_event *event) 1106 { 1107 struct hw_perf_event_extra *reg; 1108 u64 br_type = event->attr.branch_sample_type; 1109 u64 mask = 0, v; 1110 int i; 1111 1112 for (i = 0; i < PERF_SAMPLE_BRANCH_MAX_SHIFT; i++) { 1113 if (!(br_type & (1ULL << i))) 1114 continue; 1115 1116 v = x86_pmu.lbr_sel_map[i]; 1117 if (v == LBR_NOT_SUPP) 1118 return -EOPNOTSUPP; 1119 1120 if (v != LBR_IGN) 1121 mask |= v; 1122 } 1123 1124 reg = &event->hw.branch_reg; 1125 reg->idx = EXTRA_REG_LBR; 1126 1127 if (static_cpu_has(X86_FEATURE_ARCH_LBR)) { 1128 reg->config = mask; 1129 1130 /* 1131 * The Arch LBR HW can retrieve the common branch types 1132 * from the LBR_INFO. It doesn't require the high overhead 1133 * SW disassemble. 1134 * Enable the branch type by default for the Arch LBR. 1135 */ 1136 reg->reg |= X86_BR_TYPE_SAVE; 1137 return 0; 1138 } 1139 1140 /* 1141 * The first 9 bits (LBR_SEL_MASK) in LBR_SELECT operate 1142 * in suppress mode. So LBR_SELECT should be set to 1143 * (~mask & LBR_SEL_MASK) | (mask & ~LBR_SEL_MASK) 1144 * But the 10th bit LBR_CALL_STACK does not operate 1145 * in suppress mode. 1146 */ 1147 reg->config = mask ^ (x86_pmu.lbr_sel_mask & ~LBR_CALL_STACK); 1148 1149 if ((br_type & PERF_SAMPLE_BRANCH_NO_CYCLES) && 1150 (br_type & PERF_SAMPLE_BRANCH_NO_FLAGS) && 1151 x86_pmu.lbr_has_info) 1152 reg->config |= LBR_NO_INFO; 1153 1154 return 0; 1155 } 1156 1157 int intel_pmu_setup_lbr_filter(struct perf_event *event) 1158 { 1159 int ret = 0; 1160 1161 /* 1162 * no LBR on this PMU 1163 */ 1164 if (!x86_pmu.lbr_nr) 1165 return -EOPNOTSUPP; 1166 1167 /* 1168 * setup SW LBR filter 1169 */ 1170 ret = intel_pmu_setup_sw_lbr_filter(event); 1171 if (ret) 1172 return ret; 1173 1174 /* 1175 * setup HW LBR filter, if any 1176 */ 1177 if (x86_pmu.lbr_sel_map) 1178 ret = intel_pmu_setup_hw_lbr_filter(event); 1179 1180 return ret; 1181 } 1182 1183 enum { 1184 ARCH_LBR_BR_TYPE_JCC = 0, 1185 ARCH_LBR_BR_TYPE_NEAR_IND_JMP = 1, 1186 ARCH_LBR_BR_TYPE_NEAR_REL_JMP = 2, 1187 ARCH_LBR_BR_TYPE_NEAR_IND_CALL = 3, 1188 ARCH_LBR_BR_TYPE_NEAR_REL_CALL = 4, 1189 ARCH_LBR_BR_TYPE_NEAR_RET = 5, 1190 ARCH_LBR_BR_TYPE_KNOWN_MAX = ARCH_LBR_BR_TYPE_NEAR_RET, 1191 1192 ARCH_LBR_BR_TYPE_MAP_MAX = 16, 1193 }; 1194 1195 static const int arch_lbr_br_type_map[ARCH_LBR_BR_TYPE_MAP_MAX] = { 1196 [ARCH_LBR_BR_TYPE_JCC] = X86_BR_JCC, 1197 [ARCH_LBR_BR_TYPE_NEAR_IND_JMP] = X86_BR_IND_JMP, 1198 [ARCH_LBR_BR_TYPE_NEAR_REL_JMP] = X86_BR_JMP, 1199 [ARCH_LBR_BR_TYPE_NEAR_IND_CALL] = X86_BR_IND_CALL, 1200 [ARCH_LBR_BR_TYPE_NEAR_REL_CALL] = X86_BR_CALL, 1201 [ARCH_LBR_BR_TYPE_NEAR_RET] = X86_BR_RET, 1202 }; 1203 1204 /* 1205 * implement actual branch filter based on user demand. 1206 * Hardware may not exactly satisfy that request, thus 1207 * we need to inspect opcodes. Mismatched branches are 1208 * discarded. Therefore, the number of branches returned 1209 * in PERF_SAMPLE_BRANCH_STACK sample may vary. 1210 */ 1211 static void 1212 intel_pmu_lbr_filter(struct cpu_hw_events *cpuc) 1213 { 1214 u64 from, to; 1215 int br_sel = cpuc->br_sel; 1216 int i, j, type, to_plm; 1217 bool compress = false; 1218 1219 /* if sampling all branches, then nothing to filter */ 1220 if (((br_sel & X86_BR_ALL) == X86_BR_ALL) && 1221 ((br_sel & X86_BR_TYPE_SAVE) != X86_BR_TYPE_SAVE)) 1222 return; 1223 1224 for (i = 0; i < cpuc->lbr_stack.nr; i++) { 1225 1226 from = cpuc->lbr_entries[i].from; 1227 to = cpuc->lbr_entries[i].to; 1228 type = cpuc->lbr_entries[i].type; 1229 1230 /* 1231 * Parse the branch type recorded in LBR_x_INFO MSR. 1232 * Doesn't support OTHER_BRANCH decoding for now. 1233 * OTHER_BRANCH branch type still rely on software decoding. 1234 */ 1235 if (static_cpu_has(X86_FEATURE_ARCH_LBR) && 1236 type <= ARCH_LBR_BR_TYPE_KNOWN_MAX) { 1237 to_plm = kernel_ip(to) ? X86_BR_KERNEL : X86_BR_USER; 1238 type = arch_lbr_br_type_map[type] | to_plm; 1239 } else 1240 type = branch_type(from, to, cpuc->lbr_entries[i].abort); 1241 if (type != X86_BR_NONE && (br_sel & X86_BR_ANYTX)) { 1242 if (cpuc->lbr_entries[i].in_tx) 1243 type |= X86_BR_IN_TX; 1244 else 1245 type |= X86_BR_NO_TX; 1246 } 1247 1248 /* if type does not correspond, then discard */ 1249 if (type == X86_BR_NONE || (br_sel & type) != type) { 1250 cpuc->lbr_entries[i].from = 0; 1251 compress = true; 1252 } 1253 1254 if ((br_sel & X86_BR_TYPE_SAVE) == X86_BR_TYPE_SAVE) 1255 cpuc->lbr_entries[i].type = common_branch_type(type); 1256 } 1257 1258 if (!compress) 1259 return; 1260 1261 /* remove all entries with from=0 */ 1262 for (i = 0; i < cpuc->lbr_stack.nr; ) { 1263 if (!cpuc->lbr_entries[i].from) { 1264 j = i; 1265 while (++j < cpuc->lbr_stack.nr) { 1266 cpuc->lbr_entries[j-1] = cpuc->lbr_entries[j]; 1267 cpuc->lbr_counters[j-1] = cpuc->lbr_counters[j]; 1268 } 1269 cpuc->lbr_stack.nr--; 1270 if (!cpuc->lbr_entries[i].from) 1271 continue; 1272 } 1273 i++; 1274 } 1275 } 1276 1277 void intel_pmu_store_pebs_lbrs(struct lbr_entry *lbr) 1278 { 1279 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 1280 1281 /* Cannot get TOS for large PEBS and Arch LBR */ 1282 if (static_cpu_has(X86_FEATURE_ARCH_LBR) || 1283 (cpuc->n_pebs == cpuc->n_large_pebs)) 1284 cpuc->lbr_stack.hw_idx = -1ULL; 1285 else 1286 cpuc->lbr_stack.hw_idx = intel_pmu_lbr_tos(); 1287 1288 intel_pmu_store_lbr(cpuc, lbr); 1289 intel_pmu_lbr_filter(cpuc); 1290 } 1291 1292 /* 1293 * Map interface branch filters onto LBR filters 1294 */ 1295 static const int nhm_lbr_sel_map[PERF_SAMPLE_BRANCH_MAX_SHIFT] = { 1296 [PERF_SAMPLE_BRANCH_ANY_SHIFT] = LBR_ANY, 1297 [PERF_SAMPLE_BRANCH_USER_SHIFT] = LBR_USER, 1298 [PERF_SAMPLE_BRANCH_KERNEL_SHIFT] = LBR_KERNEL, 1299 [PERF_SAMPLE_BRANCH_HV_SHIFT] = LBR_IGN, 1300 [PERF_SAMPLE_BRANCH_ANY_RETURN_SHIFT] = LBR_RETURN | LBR_REL_JMP 1301 | LBR_IND_JMP | LBR_FAR, 1302 /* 1303 * NHM/WSM erratum: must include REL_JMP+IND_JMP to get CALL branches 1304 */ 1305 [PERF_SAMPLE_BRANCH_ANY_CALL_SHIFT] = 1306 LBR_REL_CALL | LBR_IND_CALL | LBR_REL_JMP | LBR_IND_JMP | LBR_FAR, 1307 /* 1308 * NHM/WSM erratum: must include IND_JMP to capture IND_CALL 1309 */ 1310 [PERF_SAMPLE_BRANCH_IND_CALL_SHIFT] = LBR_IND_CALL | LBR_IND_JMP, 1311 [PERF_SAMPLE_BRANCH_COND_SHIFT] = LBR_JCC, 1312 [PERF_SAMPLE_BRANCH_IND_JUMP_SHIFT] = LBR_IND_JMP, 1313 }; 1314 1315 static const int snb_lbr_sel_map[PERF_SAMPLE_BRANCH_MAX_SHIFT] = { 1316 [PERF_SAMPLE_BRANCH_ANY_SHIFT] = LBR_ANY, 1317 [PERF_SAMPLE_BRANCH_USER_SHIFT] = LBR_USER, 1318 [PERF_SAMPLE_BRANCH_KERNEL_SHIFT] = LBR_KERNEL, 1319 [PERF_SAMPLE_BRANCH_HV_SHIFT] = LBR_IGN, 1320 [PERF_SAMPLE_BRANCH_ANY_RETURN_SHIFT] = LBR_RETURN | LBR_FAR, 1321 [PERF_SAMPLE_BRANCH_ANY_CALL_SHIFT] = LBR_REL_CALL | LBR_IND_CALL 1322 | LBR_FAR, 1323 [PERF_SAMPLE_BRANCH_IND_CALL_SHIFT] = LBR_IND_CALL, 1324 [PERF_SAMPLE_BRANCH_COND_SHIFT] = LBR_JCC, 1325 [PERF_SAMPLE_BRANCH_IND_JUMP_SHIFT] = LBR_IND_JMP, 1326 [PERF_SAMPLE_BRANCH_CALL_SHIFT] = LBR_REL_CALL, 1327 }; 1328 1329 static const int hsw_lbr_sel_map[PERF_SAMPLE_BRANCH_MAX_SHIFT] = { 1330 [PERF_SAMPLE_BRANCH_ANY_SHIFT] = LBR_ANY, 1331 [PERF_SAMPLE_BRANCH_USER_SHIFT] = LBR_USER, 1332 [PERF_SAMPLE_BRANCH_KERNEL_SHIFT] = LBR_KERNEL, 1333 [PERF_SAMPLE_BRANCH_HV_SHIFT] = LBR_IGN, 1334 [PERF_SAMPLE_BRANCH_ANY_RETURN_SHIFT] = LBR_RETURN | LBR_FAR, 1335 [PERF_SAMPLE_BRANCH_ANY_CALL_SHIFT] = LBR_REL_CALL | LBR_IND_CALL 1336 | LBR_FAR, 1337 [PERF_SAMPLE_BRANCH_IND_CALL_SHIFT] = LBR_IND_CALL, 1338 [PERF_SAMPLE_BRANCH_COND_SHIFT] = LBR_JCC, 1339 [PERF_SAMPLE_BRANCH_CALL_STACK_SHIFT] = LBR_REL_CALL | LBR_IND_CALL 1340 | LBR_RETURN | LBR_CALL_STACK, 1341 [PERF_SAMPLE_BRANCH_IND_JUMP_SHIFT] = LBR_IND_JMP, 1342 [PERF_SAMPLE_BRANCH_CALL_SHIFT] = LBR_REL_CALL, 1343 }; 1344 1345 static int arch_lbr_ctl_map[PERF_SAMPLE_BRANCH_MAX_SHIFT] = { 1346 [PERF_SAMPLE_BRANCH_ANY_SHIFT] = ARCH_LBR_ANY, 1347 [PERF_SAMPLE_BRANCH_USER_SHIFT] = ARCH_LBR_USER, 1348 [PERF_SAMPLE_BRANCH_KERNEL_SHIFT] = ARCH_LBR_KERNEL, 1349 [PERF_SAMPLE_BRANCH_HV_SHIFT] = LBR_IGN, 1350 [PERF_SAMPLE_BRANCH_ANY_RETURN_SHIFT] = ARCH_LBR_RETURN | 1351 ARCH_LBR_OTHER_BRANCH, 1352 [PERF_SAMPLE_BRANCH_ANY_CALL_SHIFT] = ARCH_LBR_REL_CALL | 1353 ARCH_LBR_IND_CALL | 1354 ARCH_LBR_OTHER_BRANCH, 1355 [PERF_SAMPLE_BRANCH_IND_CALL_SHIFT] = ARCH_LBR_IND_CALL, 1356 [PERF_SAMPLE_BRANCH_COND_SHIFT] = ARCH_LBR_JCC, 1357 [PERF_SAMPLE_BRANCH_CALL_STACK_SHIFT] = ARCH_LBR_REL_CALL | 1358 ARCH_LBR_IND_CALL | 1359 ARCH_LBR_RETURN | 1360 ARCH_LBR_CALL_STACK, 1361 [PERF_SAMPLE_BRANCH_IND_JUMP_SHIFT] = ARCH_LBR_IND_JMP, 1362 [PERF_SAMPLE_BRANCH_CALL_SHIFT] = ARCH_LBR_REL_CALL, 1363 }; 1364 1365 /* core */ 1366 void __init intel_pmu_lbr_init_core(void) 1367 { 1368 x86_pmu.lbr_nr = 4; 1369 x86_pmu.lbr_tos = MSR_LBR_TOS; 1370 x86_pmu.lbr_from = MSR_LBR_CORE_FROM; 1371 x86_pmu.lbr_to = MSR_LBR_CORE_TO; 1372 1373 /* 1374 * SW branch filter usage: 1375 * - compensate for lack of HW filter 1376 */ 1377 } 1378 1379 /* nehalem/westmere */ 1380 void __init intel_pmu_lbr_init_nhm(void) 1381 { 1382 x86_pmu.lbr_nr = 16; 1383 x86_pmu.lbr_tos = MSR_LBR_TOS; 1384 x86_pmu.lbr_from = MSR_LBR_NHM_FROM; 1385 x86_pmu.lbr_to = MSR_LBR_NHM_TO; 1386 1387 x86_pmu.lbr_sel_mask = LBR_SEL_MASK; 1388 x86_pmu.lbr_sel_map = nhm_lbr_sel_map; 1389 1390 /* 1391 * SW branch filter usage: 1392 * - workaround LBR_SEL errata (see above) 1393 * - support syscall, sysret capture. 1394 * That requires LBR_FAR but that means far 1395 * jmp need to be filtered out 1396 */ 1397 } 1398 1399 /* sandy bridge */ 1400 void __init intel_pmu_lbr_init_snb(void) 1401 { 1402 x86_pmu.lbr_nr = 16; 1403 x86_pmu.lbr_tos = MSR_LBR_TOS; 1404 x86_pmu.lbr_from = MSR_LBR_NHM_FROM; 1405 x86_pmu.lbr_to = MSR_LBR_NHM_TO; 1406 1407 x86_pmu.lbr_sel_mask = LBR_SEL_MASK; 1408 x86_pmu.lbr_sel_map = snb_lbr_sel_map; 1409 1410 /* 1411 * SW branch filter usage: 1412 * - support syscall, sysret capture. 1413 * That requires LBR_FAR but that means far 1414 * jmp need to be filtered out 1415 */ 1416 } 1417 1418 static inline struct kmem_cache * 1419 create_lbr_kmem_cache(size_t size, size_t align) 1420 { 1421 return kmem_cache_create("x86_lbr", size, align, 0, NULL); 1422 } 1423 1424 /* haswell */ 1425 void intel_pmu_lbr_init_hsw(void) 1426 { 1427 size_t size = sizeof(struct x86_perf_task_context); 1428 1429 x86_pmu.lbr_nr = 16; 1430 x86_pmu.lbr_tos = MSR_LBR_TOS; 1431 x86_pmu.lbr_from = MSR_LBR_NHM_FROM; 1432 x86_pmu.lbr_to = MSR_LBR_NHM_TO; 1433 1434 x86_pmu.lbr_sel_mask = LBR_SEL_MASK; 1435 x86_pmu.lbr_sel_map = hsw_lbr_sel_map; 1436 1437 x86_get_pmu(smp_processor_id())->task_ctx_cache = create_lbr_kmem_cache(size, 0); 1438 } 1439 1440 /* skylake */ 1441 __init void intel_pmu_lbr_init_skl(void) 1442 { 1443 size_t size = sizeof(struct x86_perf_task_context); 1444 1445 x86_pmu.lbr_nr = 32; 1446 x86_pmu.lbr_tos = MSR_LBR_TOS; 1447 x86_pmu.lbr_from = MSR_LBR_NHM_FROM; 1448 x86_pmu.lbr_to = MSR_LBR_NHM_TO; 1449 x86_pmu.lbr_info = MSR_LBR_INFO_0; 1450 1451 x86_pmu.lbr_sel_mask = LBR_SEL_MASK; 1452 x86_pmu.lbr_sel_map = hsw_lbr_sel_map; 1453 1454 x86_get_pmu(smp_processor_id())->task_ctx_cache = create_lbr_kmem_cache(size, 0); 1455 1456 /* 1457 * SW branch filter usage: 1458 * - support syscall, sysret capture. 1459 * That requires LBR_FAR but that means far 1460 * jmp need to be filtered out 1461 */ 1462 } 1463 1464 /* atom */ 1465 void __init intel_pmu_lbr_init_atom(void) 1466 { 1467 /* 1468 * only models starting at stepping 10 seems 1469 * to have an operational LBR which can freeze 1470 * on PMU interrupt 1471 */ 1472 if (boot_cpu_data.x86_vfm == INTEL_ATOM_BONNELL 1473 && boot_cpu_data.x86_stepping < 10) { 1474 pr_cont("LBR disabled due to erratum"); 1475 return; 1476 } 1477 1478 x86_pmu.lbr_nr = 8; 1479 x86_pmu.lbr_tos = MSR_LBR_TOS; 1480 x86_pmu.lbr_from = MSR_LBR_CORE_FROM; 1481 x86_pmu.lbr_to = MSR_LBR_CORE_TO; 1482 1483 /* 1484 * SW branch filter usage: 1485 * - compensate for lack of HW filter 1486 */ 1487 } 1488 1489 /* slm */ 1490 void __init intel_pmu_lbr_init_slm(void) 1491 { 1492 x86_pmu.lbr_nr = 8; 1493 x86_pmu.lbr_tos = MSR_LBR_TOS; 1494 x86_pmu.lbr_from = MSR_LBR_CORE_FROM; 1495 x86_pmu.lbr_to = MSR_LBR_CORE_TO; 1496 1497 x86_pmu.lbr_sel_mask = LBR_SEL_MASK; 1498 x86_pmu.lbr_sel_map = nhm_lbr_sel_map; 1499 1500 /* 1501 * SW branch filter usage: 1502 * - compensate for lack of HW filter 1503 */ 1504 pr_cont("8-deep LBR, "); 1505 } 1506 1507 /* Knights Landing */ 1508 void intel_pmu_lbr_init_knl(void) 1509 { 1510 x86_pmu.lbr_nr = 8; 1511 x86_pmu.lbr_tos = MSR_LBR_TOS; 1512 x86_pmu.lbr_from = MSR_LBR_NHM_FROM; 1513 x86_pmu.lbr_to = MSR_LBR_NHM_TO; 1514 1515 x86_pmu.lbr_sel_mask = LBR_SEL_MASK; 1516 x86_pmu.lbr_sel_map = snb_lbr_sel_map; 1517 1518 /* Knights Landing does have MISPREDICT bit */ 1519 if (x86_pmu.intel_cap.lbr_format == LBR_FORMAT_LIP) 1520 x86_pmu.intel_cap.lbr_format = LBR_FORMAT_EIP_FLAGS; 1521 } 1522 1523 void intel_pmu_lbr_init(void) 1524 { 1525 switch (x86_pmu.intel_cap.lbr_format) { 1526 case LBR_FORMAT_EIP_FLAGS2: 1527 x86_pmu.lbr_has_tsx = 1; 1528 x86_pmu.lbr_from_flags = 1; 1529 if (lbr_from_signext_quirk_needed()) 1530 static_branch_enable(&lbr_from_quirk_key); 1531 break; 1532 1533 case LBR_FORMAT_EIP_FLAGS: 1534 x86_pmu.lbr_from_flags = 1; 1535 break; 1536 1537 case LBR_FORMAT_INFO: 1538 x86_pmu.lbr_has_tsx = 1; 1539 fallthrough; 1540 case LBR_FORMAT_INFO2: 1541 x86_pmu.lbr_has_info = 1; 1542 break; 1543 1544 case LBR_FORMAT_TIME: 1545 x86_pmu.lbr_from_flags = 1; 1546 x86_pmu.lbr_to_cycles = 1; 1547 break; 1548 } 1549 1550 if (x86_pmu.lbr_has_info) { 1551 /* 1552 * Only used in combination with baseline pebs. 1553 */ 1554 static_branch_enable(&x86_lbr_mispred); 1555 static_branch_enable(&x86_lbr_cycles); 1556 } 1557 } 1558 1559 /* 1560 * LBR state size is variable based on the max number of registers. 1561 * This calculates the expected state size, which should match 1562 * what the hardware enumerates for the size of XFEATURE_LBR. 1563 */ 1564 static inline unsigned int get_lbr_state_size(void) 1565 { 1566 return sizeof(struct arch_lbr_state) + 1567 x86_pmu.lbr_nr * sizeof(struct lbr_entry); 1568 } 1569 1570 static bool is_arch_lbr_xsave_available(void) 1571 { 1572 if (!boot_cpu_has(X86_FEATURE_XSAVES)) 1573 return false; 1574 1575 /* 1576 * Check the LBR state with the corresponding software structure. 1577 * Disable LBR XSAVES support if the size doesn't match. 1578 */ 1579 if (xfeature_size(XFEATURE_LBR) == 0) 1580 return false; 1581 1582 if (WARN_ON(xfeature_size(XFEATURE_LBR) != get_lbr_state_size())) 1583 return false; 1584 1585 return true; 1586 } 1587 1588 void __init intel_pmu_arch_lbr_init(void) 1589 { 1590 struct pmu *pmu = x86_get_pmu(smp_processor_id()); 1591 union cpuid28_eax eax; 1592 union cpuid28_ebx ebx; 1593 union cpuid28_ecx ecx; 1594 unsigned int unused_edx; 1595 bool arch_lbr_xsave; 1596 size_t size; 1597 u64 lbr_nr; 1598 1599 /* Arch LBR Capabilities */ 1600 cpuid(28, &eax.full, &ebx.full, &ecx.full, &unused_edx); 1601 1602 lbr_nr = fls(eax.split.lbr_depth_mask) * 8; 1603 if (!lbr_nr) 1604 goto clear_arch_lbr; 1605 1606 /* Apply the max depth of Arch LBR */ 1607 if (wrmsrq_safe(MSR_ARCH_LBR_DEPTH, lbr_nr)) 1608 goto clear_arch_lbr; 1609 1610 x86_pmu.lbr_depth_mask = eax.split.lbr_depth_mask; 1611 x86_pmu.lbr_deep_c_reset = eax.split.lbr_deep_c_reset; 1612 x86_pmu.lbr_lip = eax.split.lbr_lip; 1613 x86_pmu.lbr_cpl = ebx.split.lbr_cpl; 1614 x86_pmu.lbr_filter = ebx.split.lbr_filter; 1615 x86_pmu.lbr_call_stack = ebx.split.lbr_call_stack; 1616 x86_pmu.lbr_mispred = ecx.split.lbr_mispred; 1617 x86_pmu.lbr_timed_lbr = ecx.split.lbr_timed_lbr; 1618 x86_pmu.lbr_br_type = ecx.split.lbr_br_type; 1619 x86_pmu.lbr_counters = ecx.split.lbr_counters; 1620 x86_pmu.lbr_nr = lbr_nr; 1621 1622 if (!!x86_pmu.lbr_counters) 1623 x86_pmu.flags |= PMU_FL_BR_CNTR | PMU_FL_DYN_CONSTRAINT; 1624 1625 if (x86_pmu.lbr_mispred) 1626 static_branch_enable(&x86_lbr_mispred); 1627 if (x86_pmu.lbr_timed_lbr) 1628 static_branch_enable(&x86_lbr_cycles); 1629 if (x86_pmu.lbr_br_type) 1630 static_branch_enable(&x86_lbr_type); 1631 1632 arch_lbr_xsave = is_arch_lbr_xsave_available(); 1633 if (arch_lbr_xsave) { 1634 size = sizeof(struct x86_perf_task_context_arch_lbr_xsave) + 1635 get_lbr_state_size(); 1636 pmu->task_ctx_cache = create_lbr_kmem_cache(size, 1637 XSAVE_ALIGNMENT); 1638 } 1639 1640 if (!pmu->task_ctx_cache) { 1641 arch_lbr_xsave = false; 1642 1643 size = sizeof(struct x86_perf_task_context_arch_lbr) + 1644 lbr_nr * sizeof(struct lbr_entry); 1645 pmu->task_ctx_cache = create_lbr_kmem_cache(size, 0); 1646 } 1647 1648 x86_pmu.lbr_from = MSR_ARCH_LBR_FROM_0; 1649 x86_pmu.lbr_to = MSR_ARCH_LBR_TO_0; 1650 x86_pmu.lbr_info = MSR_ARCH_LBR_INFO_0; 1651 1652 /* LBR callstack requires both CPL and Branch Filtering support */ 1653 if (!x86_pmu.lbr_cpl || 1654 !x86_pmu.lbr_filter || 1655 !x86_pmu.lbr_call_stack) 1656 arch_lbr_ctl_map[PERF_SAMPLE_BRANCH_CALL_STACK_SHIFT] = LBR_NOT_SUPP; 1657 1658 if (!x86_pmu.lbr_cpl) { 1659 arch_lbr_ctl_map[PERF_SAMPLE_BRANCH_USER_SHIFT] = LBR_NOT_SUPP; 1660 arch_lbr_ctl_map[PERF_SAMPLE_BRANCH_KERNEL_SHIFT] = LBR_NOT_SUPP; 1661 } else if (!x86_pmu.lbr_filter) { 1662 arch_lbr_ctl_map[PERF_SAMPLE_BRANCH_ANY_SHIFT] = LBR_NOT_SUPP; 1663 arch_lbr_ctl_map[PERF_SAMPLE_BRANCH_ANY_RETURN_SHIFT] = LBR_NOT_SUPP; 1664 arch_lbr_ctl_map[PERF_SAMPLE_BRANCH_ANY_CALL_SHIFT] = LBR_NOT_SUPP; 1665 arch_lbr_ctl_map[PERF_SAMPLE_BRANCH_IND_CALL_SHIFT] = LBR_NOT_SUPP; 1666 arch_lbr_ctl_map[PERF_SAMPLE_BRANCH_COND_SHIFT] = LBR_NOT_SUPP; 1667 arch_lbr_ctl_map[PERF_SAMPLE_BRANCH_IND_JUMP_SHIFT] = LBR_NOT_SUPP; 1668 arch_lbr_ctl_map[PERF_SAMPLE_BRANCH_CALL_SHIFT] = LBR_NOT_SUPP; 1669 } 1670 1671 x86_pmu.lbr_ctl_mask = ARCH_LBR_CTL_MASK; 1672 x86_pmu.lbr_ctl_map = arch_lbr_ctl_map; 1673 1674 if (!x86_pmu.lbr_cpl && !x86_pmu.lbr_filter) 1675 x86_pmu.lbr_ctl_map = NULL; 1676 1677 x86_pmu.lbr_reset = intel_pmu_arch_lbr_reset; 1678 if (arch_lbr_xsave) { 1679 x86_pmu.lbr_save = intel_pmu_arch_lbr_xsaves; 1680 x86_pmu.lbr_restore = intel_pmu_arch_lbr_xrstors; 1681 x86_pmu.lbr_read = intel_pmu_arch_lbr_read_xsave; 1682 pr_cont("XSAVE "); 1683 } else { 1684 x86_pmu.lbr_save = intel_pmu_arch_lbr_save; 1685 x86_pmu.lbr_restore = intel_pmu_arch_lbr_restore; 1686 x86_pmu.lbr_read = intel_pmu_arch_lbr_read; 1687 } 1688 1689 pr_cont("Architectural LBR, "); 1690 1691 return; 1692 1693 clear_arch_lbr: 1694 setup_clear_cpu_cap(X86_FEATURE_ARCH_LBR); 1695 } 1696 1697 /** 1698 * x86_perf_get_lbr - get the LBR records information 1699 * 1700 * @lbr: the caller's memory to store the LBR records information 1701 */ 1702 void x86_perf_get_lbr(struct x86_pmu_lbr *lbr) 1703 { 1704 lbr->nr = x86_pmu.lbr_nr; 1705 lbr->from = x86_pmu.lbr_from; 1706 lbr->to = x86_pmu.lbr_to; 1707 lbr->info = x86_pmu.lbr_info; 1708 lbr->has_callstack = x86_pmu_has_lbr_callstack(); 1709 } 1710 EXPORT_SYMBOL_FOR_KVM(x86_perf_get_lbr); 1711 1712 struct event_constraint vlbr_constraint = 1713 __EVENT_CONSTRAINT(INTEL_FIXED_VLBR_EVENT, (1ULL << INTEL_PMC_IDX_FIXED_VLBR), 1714 FIXED_EVENT_FLAGS, 1, 0, PERF_X86_EVENT_LBR_SELECT); 1715