1 // SPDX-License-Identifier: GPL-2.0 2 #include <linux/kvm_types.h> 3 #include <linux/perf_event.h> 4 #include <linux/types.h> 5 6 #include <asm/cpu_device_id.h> 7 #include <asm/cpuid/api.h> 8 #include <asm/perf_event.h> 9 #include <asm/msr.h> 10 11 #include "../perf_event.h" 12 13 /* 14 * Intel LBR_SELECT bits 15 * Intel Vol3a, April 2011, Section 16.7 Table 16-10 16 * 17 * Hardware branch filter (not available on all CPUs) 18 */ 19 #define LBR_KERNEL_BIT 0 /* do not capture at ring0 */ 20 #define LBR_USER_BIT 1 /* do not capture at ring > 0 */ 21 #define LBR_JCC_BIT 2 /* do not capture conditional branches */ 22 #define LBR_REL_CALL_BIT 3 /* do not capture relative calls */ 23 #define LBR_IND_CALL_BIT 4 /* do not capture indirect calls */ 24 #define LBR_RETURN_BIT 5 /* do not capture near returns */ 25 #define LBR_IND_JMP_BIT 6 /* do not capture indirect jumps */ 26 #define LBR_REL_JMP_BIT 7 /* do not capture relative jumps */ 27 #define LBR_FAR_BIT 8 /* do not capture far branches */ 28 #define LBR_CALL_STACK_BIT 9 /* enable call stack */ 29 30 /* 31 * Following bit only exists in Linux; we mask it out before writing it to 32 * the actual MSR. But it helps the constraint perf code to understand 33 * that this is a separate configuration. 34 */ 35 #define LBR_NO_INFO_BIT 63 /* don't read LBR_INFO. */ 36 37 #define LBR_KERNEL (1 << LBR_KERNEL_BIT) 38 #define LBR_USER (1 << LBR_USER_BIT) 39 #define LBR_JCC (1 << LBR_JCC_BIT) 40 #define LBR_REL_CALL (1 << LBR_REL_CALL_BIT) 41 #define LBR_IND_CALL (1 << LBR_IND_CALL_BIT) 42 #define LBR_RETURN (1 << LBR_RETURN_BIT) 43 #define LBR_REL_JMP (1 << LBR_REL_JMP_BIT) 44 #define LBR_IND_JMP (1 << LBR_IND_JMP_BIT) 45 #define LBR_FAR (1 << LBR_FAR_BIT) 46 #define LBR_CALL_STACK (1 << LBR_CALL_STACK_BIT) 47 #define LBR_NO_INFO (1ULL << LBR_NO_INFO_BIT) 48 49 #define LBR_PLM (LBR_KERNEL | LBR_USER) 50 51 #define LBR_SEL_MASK 0x3ff /* valid bits in LBR_SELECT */ 52 #define LBR_NOT_SUPP -1 /* LBR filter not supported */ 53 #define LBR_IGN 0 /* ignored */ 54 55 #define LBR_ANY \ 56 (LBR_JCC |\ 57 LBR_REL_CALL |\ 58 LBR_IND_CALL |\ 59 LBR_RETURN |\ 60 LBR_REL_JMP |\ 61 LBR_IND_JMP |\ 62 LBR_FAR) 63 64 #define LBR_FROM_FLAG_MISPRED BIT_ULL(63) 65 #define LBR_FROM_FLAG_IN_TX BIT_ULL(62) 66 #define LBR_FROM_FLAG_ABORT BIT_ULL(61) 67 68 #define LBR_FROM_SIGNEXT_2MSB (BIT_ULL(60) | BIT_ULL(59)) 69 70 /* 71 * Intel LBR_CTL bits 72 * 73 * Hardware branch filter for Arch LBR 74 */ 75 #define ARCH_LBR_KERNEL_BIT 1 /* capture at ring0 */ 76 #define ARCH_LBR_USER_BIT 2 /* capture at ring > 0 */ 77 #define ARCH_LBR_CALL_STACK_BIT 3 /* enable call stack */ 78 #define ARCH_LBR_JCC_BIT 16 /* capture conditional branches */ 79 #define ARCH_LBR_REL_JMP_BIT 17 /* capture relative jumps */ 80 #define ARCH_LBR_IND_JMP_BIT 18 /* capture indirect jumps */ 81 #define ARCH_LBR_REL_CALL_BIT 19 /* capture relative calls */ 82 #define ARCH_LBR_IND_CALL_BIT 20 /* capture indirect calls */ 83 #define ARCH_LBR_RETURN_BIT 21 /* capture near returns */ 84 #define ARCH_LBR_OTHER_BRANCH_BIT 22 /* capture other branches */ 85 86 #define ARCH_LBR_KERNEL (1ULL << ARCH_LBR_KERNEL_BIT) 87 #define ARCH_LBR_USER (1ULL << ARCH_LBR_USER_BIT) 88 #define ARCH_LBR_CALL_STACK (1ULL << ARCH_LBR_CALL_STACK_BIT) 89 #define ARCH_LBR_JCC (1ULL << ARCH_LBR_JCC_BIT) 90 #define ARCH_LBR_REL_JMP (1ULL << ARCH_LBR_REL_JMP_BIT) 91 #define ARCH_LBR_IND_JMP (1ULL << ARCH_LBR_IND_JMP_BIT) 92 #define ARCH_LBR_REL_CALL (1ULL << ARCH_LBR_REL_CALL_BIT) 93 #define ARCH_LBR_IND_CALL (1ULL << ARCH_LBR_IND_CALL_BIT) 94 #define ARCH_LBR_RETURN (1ULL << ARCH_LBR_RETURN_BIT) 95 #define ARCH_LBR_OTHER_BRANCH (1ULL << ARCH_LBR_OTHER_BRANCH_BIT) 96 97 #define ARCH_LBR_ANY \ 98 (ARCH_LBR_JCC |\ 99 ARCH_LBR_REL_JMP |\ 100 ARCH_LBR_IND_JMP |\ 101 ARCH_LBR_REL_CALL |\ 102 ARCH_LBR_IND_CALL |\ 103 ARCH_LBR_RETURN |\ 104 ARCH_LBR_OTHER_BRANCH) 105 106 #define ARCH_LBR_CTL_MASK 0x7f000e 107 108 static void intel_pmu_lbr_filter(struct cpu_hw_events *cpuc); 109 110 static __always_inline bool is_lbr_call_stack_bit_set(u64 config) 111 { 112 if (cpu_feature_enabled(X86_FEATURE_ARCH_LBR)) 113 return !!(config & ARCH_LBR_CALL_STACK); 114 115 return !!(config & LBR_CALL_STACK); 116 } 117 118 /* 119 * We only support LBR implementations that have FREEZE_LBRS_ON_PMI 120 * otherwise it becomes near impossible to get a reliable stack. 121 */ 122 123 static void __intel_pmu_lbr_enable(bool pmi) 124 { 125 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 126 u64 debugctl, lbr_select = 0, orig_debugctl; 127 128 /* 129 * No need to unfreeze manually, as v4 can do that as part 130 * of the GLOBAL_STATUS ack. 131 */ 132 if (pmi && x86_pmu.version >= 4) 133 return; 134 135 /* 136 * No need to reprogram LBR_SELECT in a PMI, as it 137 * did not change. 138 */ 139 if (cpuc->lbr_sel) 140 lbr_select = cpuc->lbr_sel->config & x86_pmu.lbr_sel_mask; 141 if (!cpu_feature_enabled(X86_FEATURE_ARCH_LBR) && !pmi && cpuc->lbr_sel) 142 wrmsrq(MSR_LBR_SELECT, lbr_select); 143 144 rdmsrq(MSR_IA32_DEBUGCTLMSR, debugctl); 145 orig_debugctl = debugctl; 146 147 if (!cpu_feature_enabled(X86_FEATURE_ARCH_LBR)) 148 debugctl |= DEBUGCTLMSR_LBR; 149 /* 150 * LBR callstack does not work well with FREEZE_LBRS_ON_PMI. 151 * If FREEZE_LBRS_ON_PMI is set, PMI near call/return instructions 152 * may cause superfluous increase/decrease of LBR_TOS. 153 */ 154 if (is_lbr_call_stack_bit_set(lbr_select)) 155 debugctl &= ~DEBUGCTLMSR_FREEZE_LBRS_ON_PMI; 156 else 157 debugctl |= DEBUGCTLMSR_FREEZE_LBRS_ON_PMI; 158 159 if (orig_debugctl != debugctl) 160 wrmsrq(MSR_IA32_DEBUGCTLMSR, debugctl); 161 162 if (cpu_feature_enabled(X86_FEATURE_ARCH_LBR)) 163 wrmsrq(MSR_ARCH_LBR_CTL, lbr_select | ARCH_LBR_CTL_LBREN); 164 } 165 166 void intel_pmu_lbr_reset_32(void) 167 { 168 int i; 169 170 for (i = 0; i < x86_pmu.lbr_nr; i++) 171 wrmsrq(x86_pmu.lbr_from + i, 0); 172 } 173 174 void intel_pmu_lbr_reset_64(void) 175 { 176 int i; 177 178 for (i = 0; i < x86_pmu.lbr_nr; i++) { 179 wrmsrq(x86_pmu.lbr_from + i, 0); 180 wrmsrq(x86_pmu.lbr_to + i, 0); 181 if (x86_pmu.lbr_has_info) 182 wrmsrq(x86_pmu.lbr_info + i, 0); 183 } 184 } 185 186 static void intel_pmu_arch_lbr_reset(void) 187 { 188 /* Write to ARCH_LBR_DEPTH MSR, all LBR entries are reset to 0 */ 189 wrmsrq(MSR_ARCH_LBR_DEPTH, x86_pmu.lbr_nr); 190 } 191 192 void intel_pmu_lbr_reset(void) 193 { 194 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 195 196 if (!x86_pmu.lbr_nr) 197 return; 198 199 x86_pmu.lbr_reset(); 200 201 cpuc->last_task_ctx = NULL; 202 cpuc->last_log_id = 0; 203 if (!cpu_feature_enabled(X86_FEATURE_ARCH_LBR) && cpuc->lbr_select) 204 wrmsrq(MSR_LBR_SELECT, 0); 205 } 206 207 /* 208 * TOS = most recently recorded branch 209 */ 210 static inline u64 intel_pmu_lbr_tos(void) 211 { 212 u64 tos; 213 214 rdmsrq(x86_pmu.lbr_tos, tos); 215 return tos; 216 } 217 218 enum { 219 LBR_NONE, 220 LBR_VALID, 221 }; 222 223 /* 224 * For format LBR_FORMAT_EIP_FLAGS2, bits 61:62 in MSR_LAST_BRANCH_FROM_x 225 * are the TSX flags when TSX is supported, but when TSX is not supported 226 * they have no consistent behavior: 227 * 228 * - For wrmsr(), bits 61:62 are considered part of the sign extension. 229 * - For HW updates (branch captures) bits 61:62 are always OFF and are not 230 * part of the sign extension. 231 * 232 * Therefore, if: 233 * 234 * 1) LBR format LBR_FORMAT_EIP_FLAGS2 235 * 2) CPU has no TSX support enabled 236 * 237 * ... then any value passed to wrmsr() must be sign extended to 63 bits and any 238 * value from rdmsr() must be converted to have a 61 bits sign extension, 239 * ignoring the TSX flags. 240 */ 241 static inline bool lbr_from_signext_quirk_needed(void) 242 { 243 bool tsx_support = boot_cpu_has(X86_FEATURE_HLE) || 244 boot_cpu_has(X86_FEATURE_RTM); 245 246 return !tsx_support; 247 } 248 249 static DEFINE_STATIC_KEY_FALSE(lbr_from_quirk_key); 250 251 /* If quirk is enabled, ensure sign extension is 63 bits: */ 252 inline u64 lbr_from_signext_quirk_wr(u64 val) 253 { 254 if (static_branch_unlikely(&lbr_from_quirk_key)) { 255 /* 256 * Sign extend into bits 61:62 while preserving bit 63. 257 * 258 * Quirk is enabled when TSX is disabled. Therefore TSX bits 259 * in val are always OFF and must be changed to be sign 260 * extension bits. Since bits 59:60 are guaranteed to be 261 * part of the sign extension bits, we can just copy them 262 * to 61:62. 263 */ 264 val |= (LBR_FROM_SIGNEXT_2MSB & val) << 2; 265 } 266 return val; 267 } 268 269 /* 270 * If quirk is needed, ensure sign extension is 61 bits: 271 */ 272 static u64 lbr_from_signext_quirk_rd(u64 val) 273 { 274 if (static_branch_unlikely(&lbr_from_quirk_key)) { 275 /* 276 * Quirk is on when TSX is not enabled. Therefore TSX 277 * flags must be read as OFF. 278 */ 279 val &= ~(LBR_FROM_FLAG_IN_TX | LBR_FROM_FLAG_ABORT); 280 } 281 return val; 282 } 283 284 static __always_inline void wrlbr_from(unsigned int idx, u64 val) 285 { 286 val = lbr_from_signext_quirk_wr(val); 287 wrmsrq(x86_pmu.lbr_from + idx, val); 288 } 289 290 static __always_inline void wrlbr_to(unsigned int idx, u64 val) 291 { 292 wrmsrq(x86_pmu.lbr_to + idx, val); 293 } 294 295 static __always_inline void wrlbr_info(unsigned int idx, u64 val) 296 { 297 wrmsrq(x86_pmu.lbr_info + idx, val); 298 } 299 300 static __always_inline u64 rdlbr_from(unsigned int idx, struct lbr_entry *lbr) 301 { 302 u64 val; 303 304 if (lbr) 305 return lbr->from; 306 307 rdmsrq(x86_pmu.lbr_from + idx, val); 308 309 return lbr_from_signext_quirk_rd(val); 310 } 311 312 static __always_inline u64 rdlbr_to(unsigned int idx, struct lbr_entry *lbr) 313 { 314 u64 val; 315 316 if (lbr) 317 return lbr->to; 318 319 rdmsrq(x86_pmu.lbr_to + idx, val); 320 321 return val; 322 } 323 324 static __always_inline u64 rdlbr_info(unsigned int idx, struct lbr_entry *lbr) 325 { 326 u64 val; 327 328 if (lbr) 329 return lbr->info; 330 331 rdmsrq(x86_pmu.lbr_info + idx, val); 332 333 return val; 334 } 335 336 static inline void 337 wrlbr_all(struct lbr_entry *lbr, unsigned int idx, bool need_info) 338 { 339 wrlbr_from(idx, lbr->from); 340 wrlbr_to(idx, lbr->to); 341 if (need_info) 342 wrlbr_info(idx, lbr->info); 343 } 344 345 static inline bool 346 rdlbr_all(struct lbr_entry *lbr, unsigned int idx, bool need_info) 347 { 348 u64 from = rdlbr_from(idx, NULL); 349 350 /* Don't read invalid entry */ 351 if (!from) 352 return false; 353 354 lbr->from = from; 355 lbr->to = rdlbr_to(idx, NULL); 356 if (need_info) 357 lbr->info = rdlbr_info(idx, NULL); 358 359 return true; 360 } 361 362 void intel_pmu_lbr_restore(void *ctx) 363 { 364 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 365 struct x86_perf_task_context *task_ctx = ctx; 366 bool need_info = x86_pmu.lbr_has_info; 367 u64 tos = task_ctx->tos; 368 unsigned lbr_idx, mask; 369 int i; 370 371 mask = x86_pmu.lbr_nr - 1; 372 for (i = 0; i < task_ctx->valid_lbrs; i++) { 373 lbr_idx = (tos - i) & mask; 374 wrlbr_all(&task_ctx->lbr[i], lbr_idx, need_info); 375 } 376 377 for (; i < x86_pmu.lbr_nr; i++) { 378 lbr_idx = (tos - i) & mask; 379 wrlbr_from(lbr_idx, 0); 380 wrlbr_to(lbr_idx, 0); 381 if (need_info) 382 wrlbr_info(lbr_idx, 0); 383 } 384 385 wrmsrq(x86_pmu.lbr_tos, tos); 386 387 if (cpuc->lbr_select) 388 wrmsrq(MSR_LBR_SELECT, task_ctx->lbr_sel); 389 } 390 391 static void intel_pmu_arch_lbr_restore(void *ctx) 392 { 393 struct x86_perf_task_context_arch_lbr *task_ctx = ctx; 394 struct lbr_entry *entries = task_ctx->entries; 395 int i; 396 397 /* Fast reset the LBRs before restore if the call stack is not full. */ 398 if (!entries[x86_pmu.lbr_nr - 1].from) 399 intel_pmu_arch_lbr_reset(); 400 401 for (i = 0; i < x86_pmu.lbr_nr; i++) { 402 if (!entries[i].from) 403 break; 404 wrlbr_all(&entries[i], i, true); 405 } 406 } 407 408 /* 409 * Restore the Architecture LBR state from the xsave area in the perf 410 * context data for the task via the XRSTORS instruction. 411 */ 412 static void intel_pmu_arch_lbr_xrstors(void *ctx) 413 { 414 struct x86_perf_task_context_arch_lbr_xsave *task_ctx = ctx; 415 416 xrstors(&task_ctx->xsave, XFEATURE_MASK_LBR); 417 } 418 419 static __always_inline bool lbr_is_reset_in_cstate(void *ctx) 420 { 421 if (cpu_feature_enabled(X86_FEATURE_ARCH_LBR)) 422 return x86_pmu.lbr_deep_c_reset && !rdlbr_from(0, NULL); 423 424 return !rdlbr_from(((struct x86_perf_task_context *)ctx)->tos, NULL); 425 } 426 427 static inline bool has_lbr_callstack_users(void *ctx) 428 { 429 return task_context_opt(ctx)->lbr_callstack_users || 430 x86_pmu.lbr_callstack_users; 431 } 432 433 static void __intel_pmu_lbr_restore(void *ctx) 434 { 435 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 436 437 if (!has_lbr_callstack_users(ctx) || 438 task_context_opt(ctx)->lbr_stack_state == LBR_NONE) { 439 intel_pmu_lbr_reset(); 440 return; 441 } 442 443 /* 444 * Does not restore the LBR registers, if 445 * - No one else touched them, and 446 * - Was not cleared in Cstate 447 */ 448 if ((ctx == cpuc->last_task_ctx) && 449 (task_context_opt(ctx)->log_id == cpuc->last_log_id) && 450 !lbr_is_reset_in_cstate(ctx)) { 451 task_context_opt(ctx)->lbr_stack_state = LBR_NONE; 452 return; 453 } 454 455 x86_pmu.lbr_restore(ctx); 456 457 task_context_opt(ctx)->lbr_stack_state = LBR_NONE; 458 } 459 460 void intel_pmu_lbr_save(void *ctx) 461 { 462 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 463 struct x86_perf_task_context *task_ctx = ctx; 464 bool need_info = x86_pmu.lbr_has_info; 465 unsigned lbr_idx, mask; 466 u64 tos; 467 int i; 468 469 mask = x86_pmu.lbr_nr - 1; 470 tos = intel_pmu_lbr_tos(); 471 for (i = 0; i < x86_pmu.lbr_nr; i++) { 472 lbr_idx = (tos - i) & mask; 473 if (!rdlbr_all(&task_ctx->lbr[i], lbr_idx, need_info)) 474 break; 475 } 476 task_ctx->valid_lbrs = i; 477 task_ctx->tos = tos; 478 479 if (cpuc->lbr_select) 480 rdmsrq(MSR_LBR_SELECT, task_ctx->lbr_sel); 481 } 482 483 static void intel_pmu_arch_lbr_save(void *ctx) 484 { 485 struct x86_perf_task_context_arch_lbr *task_ctx = ctx; 486 struct lbr_entry *entries = task_ctx->entries; 487 int i; 488 489 for (i = 0; i < x86_pmu.lbr_nr; i++) { 490 if (!rdlbr_all(&entries[i], i, true)) 491 break; 492 } 493 494 /* LBR call stack is not full. Reset is required in restore. */ 495 if (i < x86_pmu.lbr_nr) 496 entries[x86_pmu.lbr_nr - 1].from = 0; 497 } 498 499 /* 500 * Save the Architecture LBR state to the xsave area in the perf 501 * context data for the task via the XSAVES instruction. 502 */ 503 static void intel_pmu_arch_lbr_xsaves(void *ctx) 504 { 505 struct x86_perf_task_context_arch_lbr_xsave *task_ctx = ctx; 506 507 xsaves(&task_ctx->xsave, XFEATURE_MASK_LBR); 508 } 509 510 static void __intel_pmu_lbr_save(void *ctx) 511 { 512 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 513 514 if (!has_lbr_callstack_users(ctx)) { 515 task_context_opt(ctx)->lbr_stack_state = LBR_NONE; 516 return; 517 } 518 519 x86_pmu.lbr_save(ctx); 520 521 task_context_opt(ctx)->lbr_stack_state = LBR_VALID; 522 523 cpuc->last_task_ctx = ctx; 524 cpuc->last_log_id = ++task_context_opt(ctx)->log_id; 525 } 526 527 void intel_pmu_lbr_sched_task(struct perf_event_pmu_context *pmu_ctx, 528 struct task_struct *task, bool sched_in) 529 { 530 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 531 struct perf_ctx_data *ctx_data; 532 void *task_ctx; 533 534 if (!cpuc->lbr_users) 535 return; 536 537 /* 538 * If LBR callstack feature is enabled and the stack was saved when 539 * the task was scheduled out, restore the stack. Otherwise flush 540 * the LBR stack. 541 */ 542 rcu_read_lock(); 543 ctx_data = rcu_dereference(task->perf_ctx_data); 544 task_ctx = ctx_data ? ctx_data->data : NULL; 545 if (task_ctx) { 546 if (sched_in) 547 __intel_pmu_lbr_restore(task_ctx); 548 else 549 __intel_pmu_lbr_save(task_ctx); 550 rcu_read_unlock(); 551 return; 552 } 553 rcu_read_unlock(); 554 555 /* 556 * Since a context switch can flip the address space and LBR entries 557 * are not tagged with an identifier, we need to wipe the LBR, even for 558 * per-cpu events. You simply cannot resolve the branches from the old 559 * address space. 560 */ 561 if (sched_in) 562 intel_pmu_lbr_reset(); 563 } 564 565 static inline bool branch_user_callstack(unsigned br_sel) 566 { 567 return (br_sel & X86_BR_USER) && (br_sel & X86_BR_CALL_STACK); 568 } 569 570 void intel_pmu_lbr_add(struct perf_event *event) 571 { 572 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 573 574 if (!x86_pmu.lbr_nr) 575 return; 576 577 if (event->hw.flags & PERF_X86_EVENT_LBR_SELECT) 578 cpuc->lbr_select = 1; 579 580 cpuc->br_sel = event->hw.branch_reg.reg; 581 582 if (branch_user_callstack(cpuc->br_sel)) { 583 if (event->attach_state & PERF_ATTACH_TASK) { 584 struct task_struct *task = event->hw.target; 585 struct perf_ctx_data *ctx_data; 586 587 rcu_read_lock(); 588 ctx_data = rcu_dereference(task->perf_ctx_data); 589 if (ctx_data) 590 task_context_opt(ctx_data->data)->lbr_callstack_users++; 591 rcu_read_unlock(); 592 } else 593 x86_pmu.lbr_callstack_users++; 594 } 595 /* 596 * Request pmu::sched_task() callback, which will fire inside the 597 * regular perf event scheduling, so that call will: 598 * 599 * - restore or wipe; when LBR-callstack, 600 * - wipe; otherwise, 601 * 602 * when this is from __perf_event_task_sched_in(). 603 * 604 * However, if this is from perf_install_in_context(), no such callback 605 * will follow and we'll need to reset the LBR here if this is the 606 * first LBR event. 607 * 608 * The problem is, we cannot tell these cases apart... but we can 609 * exclude the biggest chunk of cases by looking at 610 * event->total_time_running. An event that has accrued runtime cannot 611 * be 'new'. Conversely, a new event can get installed through the 612 * context switch path for the first time. 613 */ 614 if (x86_pmu.intel_cap.pebs_baseline && event->attr.precise_ip > 0) 615 cpuc->lbr_pebs_users++; 616 perf_sched_cb_inc(event->pmu); 617 if (!cpuc->lbr_users++ && !event->total_time_running) 618 intel_pmu_lbr_reset(); 619 } 620 621 void release_lbr_buffers(void) 622 { 623 struct kmem_cache *kmem_cache; 624 struct cpu_hw_events *cpuc; 625 int cpu; 626 627 if (!cpu_feature_enabled(X86_FEATURE_ARCH_LBR)) 628 return; 629 630 for_each_possible_cpu(cpu) { 631 cpuc = per_cpu_ptr(&cpu_hw_events, cpu); 632 kmem_cache = x86_get_pmu(cpu)->task_ctx_cache; 633 if (kmem_cache && cpuc->lbr_xsave) { 634 kmem_cache_free(kmem_cache, cpuc->lbr_xsave); 635 cpuc->lbr_xsave = NULL; 636 } 637 } 638 } 639 640 void reserve_lbr_buffers(void) 641 { 642 struct kmem_cache *kmem_cache; 643 struct cpu_hw_events *cpuc; 644 int cpu; 645 646 if (!cpu_feature_enabled(X86_FEATURE_ARCH_LBR)) 647 return; 648 649 for_each_possible_cpu(cpu) { 650 cpuc = per_cpu_ptr(&cpu_hw_events, cpu); 651 kmem_cache = x86_get_pmu(cpu)->task_ctx_cache; 652 if (!kmem_cache || cpuc->lbr_xsave) 653 continue; 654 655 cpuc->lbr_xsave = kmem_cache_alloc_node(kmem_cache, 656 GFP_KERNEL | __GFP_ZERO, 657 cpu_to_node(cpu)); 658 } 659 } 660 661 void intel_pmu_lbr_del(struct perf_event *event) 662 { 663 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 664 665 if (!x86_pmu.lbr_nr) 666 return; 667 668 if (branch_user_callstack(cpuc->br_sel)) { 669 if (event->attach_state & PERF_ATTACH_TASK) { 670 struct task_struct *task = event->hw.target; 671 struct perf_ctx_data *ctx_data; 672 673 rcu_read_lock(); 674 ctx_data = rcu_dereference(task->perf_ctx_data); 675 if (ctx_data) 676 task_context_opt(ctx_data->data)->lbr_callstack_users--; 677 rcu_read_unlock(); 678 } else 679 x86_pmu.lbr_callstack_users--; 680 } 681 682 if (event->hw.flags & PERF_X86_EVENT_LBR_SELECT) 683 cpuc->lbr_select = 0; 684 685 if (x86_pmu.intel_cap.pebs_baseline && event->attr.precise_ip > 0) 686 cpuc->lbr_pebs_users--; 687 cpuc->lbr_users--; 688 WARN_ON_ONCE(cpuc->lbr_users < 0); 689 WARN_ON_ONCE(cpuc->lbr_pebs_users < 0); 690 perf_sched_cb_dec(event->pmu); 691 692 /* 693 * The logged occurrences information is only valid for the 694 * current LBR group. If another LBR group is scheduled in 695 * later, the information from the stale LBRs will be wrongly 696 * interpreted. Reset the LBRs here. 697 * 698 * Only clear once for a branch counter group with the leader 699 * event. Because 700 * - Cannot simply reset the LBRs with the !cpuc->lbr_users. 701 * Because it's possible that the last LBR user is not in a 702 * branch counter group, e.g., a branch_counters group + 703 * several normal LBR events. 704 * - The LBR reset can be done with any one of the events in a 705 * branch counter group, since they are always scheduled together. 706 * It's easy to force the leader event an LBR event. 707 */ 708 if (is_branch_counters_group(event) && event == event->group_leader) 709 intel_pmu_lbr_reset(); 710 } 711 712 static inline bool vlbr_exclude_host(void) 713 { 714 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 715 716 return test_bit(INTEL_PMC_IDX_FIXED_VLBR, 717 (unsigned long *)&cpuc->intel_ctrl_guest_mask); 718 } 719 720 void intel_pmu_lbr_enable_all(bool pmi) 721 { 722 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 723 724 if (cpuc->lbr_users && !vlbr_exclude_host()) 725 __intel_pmu_lbr_enable(pmi); 726 } 727 728 void intel_pmu_lbr_disable_all(void) 729 { 730 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 731 732 if (cpuc->lbr_users && !vlbr_exclude_host()) { 733 if (cpu_feature_enabled(X86_FEATURE_ARCH_LBR)) 734 return __intel_pmu_arch_lbr_disable(); 735 736 __intel_pmu_lbr_disable(); 737 } 738 } 739 740 void intel_pmu_lbr_read_32(struct cpu_hw_events *cpuc) 741 { 742 unsigned long mask = x86_pmu.lbr_nr - 1; 743 struct perf_branch_entry *br = cpuc->lbr_entries; 744 u64 tos = intel_pmu_lbr_tos(); 745 int i; 746 747 for (i = 0; i < x86_pmu.lbr_nr; i++) { 748 unsigned long lbr_idx = (tos - i) & mask; 749 union { 750 struct { 751 u32 from; 752 u32 to; 753 }; 754 u64 lbr; 755 } msr_lastbranch; 756 757 rdmsrq(x86_pmu.lbr_from + lbr_idx, msr_lastbranch.lbr); 758 759 perf_clear_branch_entry_bitfields(br); 760 761 br->from = msr_lastbranch.from; 762 br->to = msr_lastbranch.to; 763 br++; 764 } 765 cpuc->lbr_stack.nr = i; 766 cpuc->lbr_stack.hw_idx = tos; 767 } 768 769 /* 770 * Due to lack of segmentation in Linux the effective address (offset) 771 * is the same as the linear address, allowing us to merge the LIP and EIP 772 * LBR formats. 773 */ 774 void intel_pmu_lbr_read_64(struct cpu_hw_events *cpuc) 775 { 776 bool need_info = false, call_stack = false; 777 unsigned long mask = x86_pmu.lbr_nr - 1; 778 struct perf_branch_entry *br = cpuc->lbr_entries; 779 u64 tos = intel_pmu_lbr_tos(); 780 int i; 781 int out = 0; 782 int num = x86_pmu.lbr_nr; 783 784 if (cpuc->lbr_sel) { 785 need_info = !(cpuc->lbr_sel->config & LBR_NO_INFO); 786 if (cpuc->lbr_sel->config & LBR_CALL_STACK) 787 call_stack = true; 788 } 789 790 for (i = 0; i < num; i++) { 791 unsigned long lbr_idx = (tos - i) & mask; 792 u64 from, to, mis = 0, pred = 0, in_tx = 0, abort = 0; 793 u16 cycles = 0; 794 795 from = rdlbr_from(lbr_idx, NULL); 796 to = rdlbr_to(lbr_idx, NULL); 797 798 /* 799 * Read LBR call stack entries 800 * until invalid entry (0s) is detected. 801 */ 802 if (call_stack && !from) 803 break; 804 805 if (x86_pmu.lbr_has_info) { 806 if (need_info) { 807 u64 info; 808 809 info = rdlbr_info(lbr_idx, NULL); 810 mis = !!(info & LBR_INFO_MISPRED); 811 pred = !mis; 812 cycles = (info & LBR_INFO_CYCLES); 813 if (x86_pmu.lbr_has_tsx) { 814 in_tx = !!(info & LBR_INFO_IN_TX); 815 abort = !!(info & LBR_INFO_ABORT); 816 } 817 } 818 } else { 819 int skip = 0; 820 821 if (x86_pmu.lbr_from_flags) { 822 mis = !!(from & LBR_FROM_FLAG_MISPRED); 823 pred = !mis; 824 skip = 1; 825 } 826 if (x86_pmu.lbr_has_tsx) { 827 in_tx = !!(from & LBR_FROM_FLAG_IN_TX); 828 abort = !!(from & LBR_FROM_FLAG_ABORT); 829 skip = 3; 830 } 831 from = (u64)((((s64)from) << skip) >> skip); 832 833 if (x86_pmu.lbr_to_cycles) { 834 cycles = ((to >> 48) & LBR_INFO_CYCLES); 835 to = (u64)((((s64)to) << 16) >> 16); 836 } 837 } 838 839 /* 840 * Some CPUs report duplicated abort records, 841 * with the second entry not having an abort bit set. 842 * Skip them here. This loop runs backwards, 843 * so we need to undo the previous record. 844 * If the abort just happened outside the window 845 * the extra entry cannot be removed. 846 */ 847 if (abort && x86_pmu.lbr_double_abort && out > 0) 848 out--; 849 850 perf_clear_branch_entry_bitfields(br+out); 851 br[out].from = from; 852 br[out].to = to; 853 br[out].mispred = mis; 854 br[out].predicted = pred; 855 br[out].in_tx = in_tx; 856 br[out].abort = abort; 857 br[out].cycles = cycles; 858 out++; 859 } 860 cpuc->lbr_stack.nr = out; 861 cpuc->lbr_stack.hw_idx = tos; 862 } 863 864 static DEFINE_STATIC_KEY_FALSE(x86_lbr_mispred); 865 static DEFINE_STATIC_KEY_FALSE(x86_lbr_cycles); 866 static DEFINE_STATIC_KEY_FALSE(x86_lbr_type); 867 868 static __always_inline int get_lbr_br_type(u64 info) 869 { 870 int type = 0; 871 872 if (static_branch_likely(&x86_lbr_type)) 873 type = (info & LBR_INFO_BR_TYPE) >> LBR_INFO_BR_TYPE_OFFSET; 874 875 return type; 876 } 877 878 static __always_inline bool get_lbr_mispred(u64 info) 879 { 880 bool mispred = 0; 881 882 if (static_branch_likely(&x86_lbr_mispred)) 883 mispred = !!(info & LBR_INFO_MISPRED); 884 885 return mispred; 886 } 887 888 static __always_inline u16 get_lbr_cycles(u64 info) 889 { 890 u16 cycles = info & LBR_INFO_CYCLES; 891 892 if (cpu_feature_enabled(X86_FEATURE_ARCH_LBR) && 893 (!static_branch_likely(&x86_lbr_cycles) || 894 !(info & LBR_INFO_CYC_CNT_VALID))) 895 cycles = 0; 896 897 return cycles; 898 } 899 900 static_assert((64 - PERF_BRANCH_ENTRY_INFO_BITS_MAX) > LBR_INFO_BR_CNTR_NUM * LBR_INFO_BR_CNTR_BITS); 901 902 static void intel_pmu_store_lbr(struct cpu_hw_events *cpuc, 903 struct lbr_entry *entries) 904 { 905 struct perf_branch_entry *e; 906 struct lbr_entry *lbr; 907 u64 from, to, info; 908 int i; 909 910 for (i = 0; i < x86_pmu.lbr_nr; i++) { 911 lbr = entries ? &entries[i] : NULL; 912 e = &cpuc->lbr_entries[i]; 913 914 from = rdlbr_from(i, lbr); 915 /* 916 * Read LBR entries until invalid entry (0s) is detected. 917 */ 918 if (!from) 919 break; 920 921 to = rdlbr_to(i, lbr); 922 info = rdlbr_info(i, lbr); 923 924 perf_clear_branch_entry_bitfields(e); 925 926 e->from = from; 927 e->to = to; 928 e->mispred = get_lbr_mispred(info); 929 e->predicted = !e->mispred; 930 e->in_tx = !!(info & LBR_INFO_IN_TX); 931 e->abort = !!(info & LBR_INFO_ABORT); 932 e->cycles = get_lbr_cycles(info); 933 e->type = get_lbr_br_type(info); 934 935 /* 936 * Leverage the reserved field of cpuc->lbr_entries[i] to 937 * temporarily store the branch counters information. 938 * The later code will decide what content can be disclosed 939 * to the perf tool. Pleae see intel_pmu_lbr_counters_reorder(). 940 */ 941 e->reserved = (info >> LBR_INFO_BR_CNTR_OFFSET) & LBR_INFO_BR_CNTR_FULL_MASK; 942 } 943 944 cpuc->lbr_stack.nr = i; 945 } 946 947 /* 948 * The enabled order may be different from the counter order. 949 * Update the lbr_counters with the enabled order. 950 */ 951 static void intel_pmu_lbr_counters_reorder(struct cpu_hw_events *cpuc, 952 struct perf_event *event) 953 { 954 int i, j, pos = 0, order[X86_PMC_IDX_MAX]; 955 struct perf_event *leader, *sibling; 956 u64 src, dst, cnt; 957 958 leader = event->group_leader; 959 if (branch_sample_counters(leader)) 960 order[pos++] = leader->hw.idx; 961 962 for_each_sibling_event(sibling, leader) { 963 if (!branch_sample_counters(sibling)) 964 continue; 965 order[pos++] = sibling->hw.idx; 966 } 967 968 WARN_ON_ONCE(!pos); 969 970 for (i = 0; i < cpuc->lbr_stack.nr; i++) { 971 src = cpuc->lbr_entries[i].reserved; 972 dst = 0; 973 for (j = 0; j < pos; j++) { 974 cnt = (src >> (order[j] * LBR_INFO_BR_CNTR_BITS)) & LBR_INFO_BR_CNTR_MASK; 975 dst |= cnt << j * LBR_INFO_BR_CNTR_BITS; 976 } 977 cpuc->lbr_counters[i] = dst; 978 cpuc->lbr_entries[i].reserved = 0; 979 } 980 } 981 982 void intel_pmu_lbr_save_brstack(struct perf_sample_data *data, 983 struct cpu_hw_events *cpuc, 984 struct perf_event *event) 985 { 986 if (is_branch_counters_group(event)) { 987 intel_pmu_lbr_counters_reorder(cpuc, event); 988 perf_sample_save_brstack(data, event, &cpuc->lbr_stack, cpuc->lbr_counters); 989 return; 990 } 991 992 perf_sample_save_brstack(data, event, &cpuc->lbr_stack, NULL); 993 } 994 995 static void intel_pmu_arch_lbr_read(struct cpu_hw_events *cpuc) 996 { 997 intel_pmu_store_lbr(cpuc, NULL); 998 } 999 1000 static void intel_pmu_arch_lbr_read_xsave(struct cpu_hw_events *cpuc) 1001 { 1002 struct x86_perf_task_context_arch_lbr_xsave *xsave = cpuc->lbr_xsave; 1003 1004 if (!xsave) { 1005 intel_pmu_store_lbr(cpuc, NULL); 1006 return; 1007 } 1008 xsaves(&xsave->xsave, XFEATURE_MASK_LBR); 1009 1010 intel_pmu_store_lbr(cpuc, xsave->lbr.entries); 1011 } 1012 1013 void intel_pmu_lbr_read(void) 1014 { 1015 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 1016 1017 /* 1018 * Don't read when all LBRs users are using adaptive PEBS. 1019 * 1020 * This could be smarter and actually check the event, 1021 * but this simple approach seems to work for now. 1022 */ 1023 if (!cpuc->lbr_users || vlbr_exclude_host() || 1024 cpuc->lbr_users == cpuc->lbr_pebs_users) 1025 return; 1026 1027 x86_pmu.lbr_read(cpuc); 1028 1029 intel_pmu_lbr_filter(cpuc); 1030 } 1031 1032 /* 1033 * SW filter is used: 1034 * - in case there is no HW filter 1035 * - in case the HW filter has errata or limitations 1036 */ 1037 static int intel_pmu_setup_sw_lbr_filter(struct perf_event *event) 1038 { 1039 u64 br_type = event->attr.branch_sample_type; 1040 int mask = 0; 1041 1042 if (br_type & PERF_SAMPLE_BRANCH_USER) 1043 mask |= X86_BR_USER; 1044 1045 if (br_type & PERF_SAMPLE_BRANCH_KERNEL) 1046 mask |= X86_BR_KERNEL; 1047 1048 /* we ignore BRANCH_HV here */ 1049 1050 if (br_type & PERF_SAMPLE_BRANCH_ANY) 1051 mask |= X86_BR_ANY; 1052 1053 if (br_type & PERF_SAMPLE_BRANCH_ANY_CALL) 1054 mask |= X86_BR_ANY_CALL; 1055 1056 if (br_type & PERF_SAMPLE_BRANCH_ANY_RETURN) 1057 mask |= X86_BR_RET | X86_BR_IRET | X86_BR_SYSRET; 1058 1059 if (br_type & PERF_SAMPLE_BRANCH_IND_CALL) 1060 mask |= X86_BR_IND_CALL; 1061 1062 if (br_type & PERF_SAMPLE_BRANCH_ABORT_TX) 1063 mask |= X86_BR_ABORT; 1064 1065 if (br_type & PERF_SAMPLE_BRANCH_IN_TX) 1066 mask |= X86_BR_IN_TX; 1067 1068 if (br_type & PERF_SAMPLE_BRANCH_NO_TX) 1069 mask |= X86_BR_NO_TX; 1070 1071 if (br_type & PERF_SAMPLE_BRANCH_COND) 1072 mask |= X86_BR_JCC; 1073 1074 if (br_type & PERF_SAMPLE_BRANCH_CALL_STACK) { 1075 if (!x86_pmu_has_lbr_callstack()) 1076 return -EOPNOTSUPP; 1077 if (mask & ~(X86_BR_USER | X86_BR_KERNEL)) 1078 return -EINVAL; 1079 mask |= X86_BR_CALL | X86_BR_IND_CALL | X86_BR_RET | 1080 X86_BR_CALL_STACK; 1081 } 1082 1083 if (br_type & PERF_SAMPLE_BRANCH_IND_JUMP) 1084 mask |= X86_BR_IND_JMP; 1085 1086 if (br_type & PERF_SAMPLE_BRANCH_CALL) 1087 mask |= X86_BR_CALL | X86_BR_ZERO_CALL; 1088 1089 if (br_type & PERF_SAMPLE_BRANCH_TYPE_SAVE) 1090 mask |= X86_BR_TYPE_SAVE; 1091 1092 /* 1093 * stash actual user request into reg, it may 1094 * be used by fixup code for some CPU 1095 */ 1096 event->hw.branch_reg.reg = mask; 1097 return 0; 1098 } 1099 1100 /* 1101 * setup the HW LBR filter 1102 * Used only when available, may not be enough to disambiguate 1103 * all branches, may need the help of the SW filter 1104 */ 1105 static int intel_pmu_setup_hw_lbr_filter(struct perf_event *event) 1106 { 1107 struct hw_perf_event_extra *reg; 1108 u64 br_type = event->attr.branch_sample_type; 1109 u64 mask = 0, v; 1110 int i; 1111 1112 for (i = 0; i < PERF_SAMPLE_BRANCH_MAX_SHIFT; i++) { 1113 if (!(br_type & (1ULL << i))) 1114 continue; 1115 1116 v = x86_pmu.lbr_sel_map[i]; 1117 if (v == LBR_NOT_SUPP) 1118 return -EOPNOTSUPP; 1119 1120 if (v != LBR_IGN) 1121 mask |= v; 1122 } 1123 1124 reg = &event->hw.branch_reg; 1125 reg->idx = EXTRA_REG_LBR; 1126 1127 if (cpu_feature_enabled(X86_FEATURE_ARCH_LBR)) { 1128 reg->config = mask; 1129 1130 /* 1131 * The Arch LBR HW can retrieve the common branch types 1132 * from the LBR_INFO. It doesn't require the high overhead 1133 * SW disassemble. 1134 * Enable the branch type by default for the Arch LBR. 1135 */ 1136 reg->reg |= X86_BR_TYPE_SAVE; 1137 return 0; 1138 } 1139 1140 /* 1141 * The first 9 bits (LBR_SEL_MASK) in LBR_SELECT operate 1142 * in suppress mode. So LBR_SELECT should be set to 1143 * (~mask & LBR_SEL_MASK) | (mask & ~LBR_SEL_MASK) 1144 * But the 10th bit LBR_CALL_STACK does not operate 1145 * in suppress mode. 1146 */ 1147 reg->config = mask ^ (x86_pmu.lbr_sel_mask & ~LBR_CALL_STACK); 1148 1149 if ((br_type & PERF_SAMPLE_BRANCH_NO_CYCLES) && 1150 (br_type & PERF_SAMPLE_BRANCH_NO_FLAGS) && 1151 x86_pmu.lbr_has_info) 1152 reg->config |= LBR_NO_INFO; 1153 1154 return 0; 1155 } 1156 1157 int intel_pmu_setup_lbr_filter(struct perf_event *event) 1158 { 1159 int ret = 0; 1160 1161 /* 1162 * no LBR on this PMU 1163 */ 1164 if (!x86_pmu.lbr_nr) 1165 return -EOPNOTSUPP; 1166 1167 /* 1168 * setup SW LBR filter 1169 */ 1170 ret = intel_pmu_setup_sw_lbr_filter(event); 1171 if (ret) 1172 return ret; 1173 1174 /* 1175 * setup HW LBR filter, if any 1176 */ 1177 if (x86_pmu.lbr_sel_map) 1178 ret = intel_pmu_setup_hw_lbr_filter(event); 1179 1180 return ret; 1181 } 1182 1183 enum { 1184 ARCH_LBR_BR_TYPE_JCC = 0, 1185 ARCH_LBR_BR_TYPE_NEAR_IND_JMP = 1, 1186 ARCH_LBR_BR_TYPE_NEAR_REL_JMP = 2, 1187 ARCH_LBR_BR_TYPE_NEAR_IND_CALL = 3, 1188 ARCH_LBR_BR_TYPE_NEAR_REL_CALL = 4, 1189 ARCH_LBR_BR_TYPE_NEAR_RET = 5, 1190 ARCH_LBR_BR_TYPE_KNOWN_MAX = ARCH_LBR_BR_TYPE_NEAR_RET, 1191 1192 ARCH_LBR_BR_TYPE_MAP_MAX = 16, 1193 }; 1194 1195 static const int arch_lbr_br_type_map[ARCH_LBR_BR_TYPE_MAP_MAX] = { 1196 [ARCH_LBR_BR_TYPE_JCC] = X86_BR_JCC, 1197 [ARCH_LBR_BR_TYPE_NEAR_IND_JMP] = X86_BR_IND_JMP, 1198 [ARCH_LBR_BR_TYPE_NEAR_REL_JMP] = X86_BR_JMP, 1199 [ARCH_LBR_BR_TYPE_NEAR_IND_CALL] = X86_BR_IND_CALL, 1200 [ARCH_LBR_BR_TYPE_NEAR_REL_CALL] = X86_BR_CALL, 1201 [ARCH_LBR_BR_TYPE_NEAR_RET] = X86_BR_RET, 1202 }; 1203 1204 /* 1205 * implement actual branch filter based on user demand. 1206 * Hardware may not exactly satisfy that request, thus 1207 * we need to inspect opcodes. Mismatched branches are 1208 * discarded. Therefore, the number of branches returned 1209 * in PERF_SAMPLE_BRANCH_STACK sample may vary. 1210 */ 1211 static void 1212 intel_pmu_lbr_filter(struct cpu_hw_events *cpuc) 1213 { 1214 u64 from, to; 1215 int br_sel = cpuc->br_sel; 1216 int i, j, type, from_plm, to_plm; 1217 bool compress = false; 1218 1219 /* if sampling all branches, then nothing to filter */ 1220 if (((br_sel & X86_BR_ALL) == X86_BR_ALL) && 1221 ((br_sel & X86_BR_TYPE_SAVE) != X86_BR_TYPE_SAVE)) 1222 return; 1223 1224 for (i = 0; i < cpuc->lbr_stack.nr; i++) { 1225 1226 from = cpuc->lbr_entries[i].from; 1227 to = cpuc->lbr_entries[i].to; 1228 type = cpuc->lbr_entries[i].type; 1229 1230 /* 1231 * Parse the branch type recorded in LBR_x_INFO MSR. 1232 * Doesn't support OTHER_BRANCH decoding for now. 1233 * OTHER_BRANCH branch type still rely on software decoding. 1234 */ 1235 if (static_branch_likely(&x86_lbr_type) && 1236 type <= ARCH_LBR_BR_TYPE_KNOWN_MAX) { 1237 to_plm = kernel_ip(to) ? X86_BR_KERNEL : X86_BR_USER; 1238 type = arch_lbr_br_type_map[type] | to_plm; 1239 } else 1240 type = branch_type(from, to, cpuc->lbr_entries[i].abort); 1241 if (type != X86_BR_NONE && (br_sel & X86_BR_ANYTX)) { 1242 if (cpuc->lbr_entries[i].in_tx) 1243 type |= X86_BR_IN_TX; 1244 else 1245 type |= X86_BR_NO_TX; 1246 } 1247 1248 from_plm = kernel_ip(from) ? X86_BR_KERNEL : X86_BR_USER; 1249 /* 1250 * If type does not correspond, then discard. 1251 * Specifically reject entries whose from address is in 1252 * kernel space when only X86_BR_USER is requested. 1253 */ 1254 if (type == X86_BR_NONE || (br_sel & type) != type || 1255 (!(br_sel & X86_BR_KERNEL) && (from_plm & X86_BR_KERNEL))) { 1256 cpuc->lbr_entries[i].from = 0; 1257 compress = true; 1258 } 1259 1260 if ((br_sel & X86_BR_TYPE_SAVE) == X86_BR_TYPE_SAVE) 1261 cpuc->lbr_entries[i].type = common_branch_type(type); 1262 } 1263 1264 if (!compress) 1265 return; 1266 1267 /* remove all entries with from=0 */ 1268 for (i = 0; i < cpuc->lbr_stack.nr; ) { 1269 if (!cpuc->lbr_entries[i].from) { 1270 j = i; 1271 while (++j < cpuc->lbr_stack.nr) { 1272 cpuc->lbr_entries[j-1] = cpuc->lbr_entries[j]; 1273 cpuc->lbr_counters[j-1] = cpuc->lbr_counters[j]; 1274 } 1275 cpuc->lbr_stack.nr--; 1276 if (!cpuc->lbr_entries[i].from) 1277 continue; 1278 } 1279 i++; 1280 } 1281 } 1282 1283 void intel_pmu_store_pebs_lbrs(struct lbr_entry *lbr) 1284 { 1285 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 1286 1287 /* Cannot get TOS for large PEBS and Arch LBR */ 1288 if (cpu_feature_enabled(X86_FEATURE_ARCH_LBR) || 1289 (cpuc->n_pebs == cpuc->n_large_pebs)) 1290 cpuc->lbr_stack.hw_idx = -1ULL; 1291 else 1292 cpuc->lbr_stack.hw_idx = intel_pmu_lbr_tos(); 1293 1294 intel_pmu_store_lbr(cpuc, lbr); 1295 intel_pmu_lbr_filter(cpuc); 1296 } 1297 1298 /* 1299 * Map interface branch filters onto LBR filters 1300 */ 1301 static const int nhm_lbr_sel_map[PERF_SAMPLE_BRANCH_MAX_SHIFT] = { 1302 [PERF_SAMPLE_BRANCH_ANY_SHIFT] = LBR_ANY, 1303 [PERF_SAMPLE_BRANCH_USER_SHIFT] = LBR_USER, 1304 [PERF_SAMPLE_BRANCH_KERNEL_SHIFT] = LBR_KERNEL, 1305 [PERF_SAMPLE_BRANCH_HV_SHIFT] = LBR_IGN, 1306 [PERF_SAMPLE_BRANCH_ANY_RETURN_SHIFT] = LBR_RETURN | LBR_REL_JMP 1307 | LBR_IND_JMP | LBR_FAR, 1308 /* 1309 * NHM/WSM erratum: must include REL_JMP+IND_JMP to get CALL branches 1310 */ 1311 [PERF_SAMPLE_BRANCH_ANY_CALL_SHIFT] = 1312 LBR_REL_CALL | LBR_IND_CALL | LBR_REL_JMP | LBR_IND_JMP | LBR_FAR, 1313 /* 1314 * NHM/WSM erratum: must include IND_JMP to capture IND_CALL 1315 */ 1316 [PERF_SAMPLE_BRANCH_IND_CALL_SHIFT] = LBR_IND_CALL | LBR_IND_JMP, 1317 [PERF_SAMPLE_BRANCH_COND_SHIFT] = LBR_JCC, 1318 [PERF_SAMPLE_BRANCH_IND_JUMP_SHIFT] = LBR_IND_JMP, 1319 }; 1320 1321 static const int snb_lbr_sel_map[PERF_SAMPLE_BRANCH_MAX_SHIFT] = { 1322 [PERF_SAMPLE_BRANCH_ANY_SHIFT] = LBR_ANY, 1323 [PERF_SAMPLE_BRANCH_USER_SHIFT] = LBR_USER, 1324 [PERF_SAMPLE_BRANCH_KERNEL_SHIFT] = LBR_KERNEL, 1325 [PERF_SAMPLE_BRANCH_HV_SHIFT] = LBR_IGN, 1326 [PERF_SAMPLE_BRANCH_ANY_RETURN_SHIFT] = LBR_RETURN | LBR_FAR, 1327 [PERF_SAMPLE_BRANCH_ANY_CALL_SHIFT] = LBR_REL_CALL | LBR_IND_CALL 1328 | LBR_FAR, 1329 [PERF_SAMPLE_BRANCH_IND_CALL_SHIFT] = LBR_IND_CALL, 1330 [PERF_SAMPLE_BRANCH_COND_SHIFT] = LBR_JCC, 1331 [PERF_SAMPLE_BRANCH_IND_JUMP_SHIFT] = LBR_IND_JMP, 1332 [PERF_SAMPLE_BRANCH_CALL_SHIFT] = LBR_REL_CALL, 1333 }; 1334 1335 static const int hsw_lbr_sel_map[PERF_SAMPLE_BRANCH_MAX_SHIFT] = { 1336 [PERF_SAMPLE_BRANCH_ANY_SHIFT] = LBR_ANY, 1337 [PERF_SAMPLE_BRANCH_USER_SHIFT] = LBR_USER, 1338 [PERF_SAMPLE_BRANCH_KERNEL_SHIFT] = LBR_KERNEL, 1339 [PERF_SAMPLE_BRANCH_HV_SHIFT] = LBR_IGN, 1340 [PERF_SAMPLE_BRANCH_ANY_RETURN_SHIFT] = LBR_RETURN | LBR_FAR, 1341 [PERF_SAMPLE_BRANCH_ANY_CALL_SHIFT] = LBR_REL_CALL | LBR_IND_CALL 1342 | LBR_FAR, 1343 [PERF_SAMPLE_BRANCH_IND_CALL_SHIFT] = LBR_IND_CALL, 1344 [PERF_SAMPLE_BRANCH_COND_SHIFT] = LBR_JCC, 1345 [PERF_SAMPLE_BRANCH_CALL_STACK_SHIFT] = LBR_REL_CALL | LBR_IND_CALL 1346 | LBR_RETURN | LBR_CALL_STACK, 1347 [PERF_SAMPLE_BRANCH_IND_JUMP_SHIFT] = LBR_IND_JMP, 1348 [PERF_SAMPLE_BRANCH_CALL_SHIFT] = LBR_REL_CALL, 1349 }; 1350 1351 static int arch_lbr_ctl_map[PERF_SAMPLE_BRANCH_MAX_SHIFT] = { 1352 [PERF_SAMPLE_BRANCH_ANY_SHIFT] = ARCH_LBR_ANY, 1353 [PERF_SAMPLE_BRANCH_USER_SHIFT] = ARCH_LBR_USER, 1354 [PERF_SAMPLE_BRANCH_KERNEL_SHIFT] = ARCH_LBR_KERNEL, 1355 [PERF_SAMPLE_BRANCH_HV_SHIFT] = LBR_IGN, 1356 [PERF_SAMPLE_BRANCH_ANY_RETURN_SHIFT] = ARCH_LBR_RETURN | 1357 ARCH_LBR_OTHER_BRANCH, 1358 [PERF_SAMPLE_BRANCH_ANY_CALL_SHIFT] = ARCH_LBR_REL_CALL | 1359 ARCH_LBR_IND_CALL | 1360 ARCH_LBR_OTHER_BRANCH, 1361 [PERF_SAMPLE_BRANCH_IND_CALL_SHIFT] = ARCH_LBR_IND_CALL, 1362 [PERF_SAMPLE_BRANCH_COND_SHIFT] = ARCH_LBR_JCC, 1363 [PERF_SAMPLE_BRANCH_CALL_STACK_SHIFT] = ARCH_LBR_REL_CALL | 1364 ARCH_LBR_IND_CALL | 1365 ARCH_LBR_RETURN | 1366 ARCH_LBR_CALL_STACK, 1367 [PERF_SAMPLE_BRANCH_IND_JUMP_SHIFT] = ARCH_LBR_IND_JMP, 1368 [PERF_SAMPLE_BRANCH_CALL_SHIFT] = ARCH_LBR_REL_CALL, 1369 }; 1370 1371 /* core */ 1372 void __init intel_pmu_lbr_init_core(void) 1373 { 1374 x86_pmu.lbr_nr = 4; 1375 x86_pmu.lbr_tos = MSR_LBR_TOS; 1376 x86_pmu.lbr_from = MSR_LBR_CORE_FROM; 1377 x86_pmu.lbr_to = MSR_LBR_CORE_TO; 1378 1379 /* 1380 * SW branch filter usage: 1381 * - compensate for lack of HW filter 1382 */ 1383 } 1384 1385 /* nehalem/westmere */ 1386 void __init intel_pmu_lbr_init_nhm(void) 1387 { 1388 x86_pmu.lbr_nr = 16; 1389 x86_pmu.lbr_tos = MSR_LBR_TOS; 1390 x86_pmu.lbr_from = MSR_LBR_NHM_FROM; 1391 x86_pmu.lbr_to = MSR_LBR_NHM_TO; 1392 1393 x86_pmu.lbr_sel_mask = LBR_SEL_MASK; 1394 x86_pmu.lbr_sel_map = nhm_lbr_sel_map; 1395 1396 /* 1397 * SW branch filter usage: 1398 * - workaround LBR_SEL errata (see above) 1399 * - support syscall, sysret capture. 1400 * That requires LBR_FAR but that means far 1401 * jmp need to be filtered out 1402 */ 1403 } 1404 1405 /* sandy bridge */ 1406 void __init intel_pmu_lbr_init_snb(void) 1407 { 1408 x86_pmu.lbr_nr = 16; 1409 x86_pmu.lbr_tos = MSR_LBR_TOS; 1410 x86_pmu.lbr_from = MSR_LBR_NHM_FROM; 1411 x86_pmu.lbr_to = MSR_LBR_NHM_TO; 1412 1413 x86_pmu.lbr_sel_mask = LBR_SEL_MASK; 1414 x86_pmu.lbr_sel_map = snb_lbr_sel_map; 1415 1416 /* 1417 * SW branch filter usage: 1418 * - support syscall, sysret capture. 1419 * That requires LBR_FAR but that means far 1420 * jmp need to be filtered out 1421 */ 1422 } 1423 1424 static inline struct kmem_cache * 1425 create_lbr_kmem_cache(size_t size, size_t align) 1426 { 1427 return kmem_cache_create("x86_lbr", size, align, 0, NULL); 1428 } 1429 1430 /* haswell */ 1431 void intel_pmu_lbr_init_hsw(void) 1432 { 1433 size_t size = sizeof(struct x86_perf_task_context); 1434 1435 x86_pmu.lbr_nr = 16; 1436 x86_pmu.lbr_tos = MSR_LBR_TOS; 1437 x86_pmu.lbr_from = MSR_LBR_NHM_FROM; 1438 x86_pmu.lbr_to = MSR_LBR_NHM_TO; 1439 1440 x86_pmu.lbr_sel_mask = LBR_SEL_MASK; 1441 x86_pmu.lbr_sel_map = hsw_lbr_sel_map; 1442 1443 x86_get_pmu(smp_processor_id())->task_ctx_cache = create_lbr_kmem_cache(size, 0); 1444 } 1445 1446 /* skylake */ 1447 __init void intel_pmu_lbr_init_skl(void) 1448 { 1449 size_t size = sizeof(struct x86_perf_task_context); 1450 1451 x86_pmu.lbr_nr = 32; 1452 x86_pmu.lbr_tos = MSR_LBR_TOS; 1453 x86_pmu.lbr_from = MSR_LBR_NHM_FROM; 1454 x86_pmu.lbr_to = MSR_LBR_NHM_TO; 1455 x86_pmu.lbr_info = MSR_LBR_INFO_0; 1456 1457 x86_pmu.lbr_sel_mask = LBR_SEL_MASK; 1458 x86_pmu.lbr_sel_map = hsw_lbr_sel_map; 1459 1460 x86_get_pmu(smp_processor_id())->task_ctx_cache = create_lbr_kmem_cache(size, 0); 1461 1462 /* 1463 * SW branch filter usage: 1464 * - support syscall, sysret capture. 1465 * That requires LBR_FAR but that means far 1466 * jmp need to be filtered out 1467 */ 1468 } 1469 1470 /* atom */ 1471 void __init intel_pmu_lbr_init_atom(void) 1472 { 1473 /* 1474 * only models starting at stepping 10 seems 1475 * to have an operational LBR which can freeze 1476 * on PMU interrupt 1477 */ 1478 if (boot_cpu_data.x86_vfm == INTEL_ATOM_BONNELL 1479 && boot_cpu_data.x86_stepping < 10) { 1480 pr_cont("LBR disabled due to erratum"); 1481 return; 1482 } 1483 1484 x86_pmu.lbr_nr = 8; 1485 x86_pmu.lbr_tos = MSR_LBR_TOS; 1486 x86_pmu.lbr_from = MSR_LBR_CORE_FROM; 1487 x86_pmu.lbr_to = MSR_LBR_CORE_TO; 1488 1489 /* 1490 * SW branch filter usage: 1491 * - compensate for lack of HW filter 1492 */ 1493 } 1494 1495 /* slm */ 1496 void __init intel_pmu_lbr_init_slm(void) 1497 { 1498 x86_pmu.lbr_nr = 8; 1499 x86_pmu.lbr_tos = MSR_LBR_TOS; 1500 x86_pmu.lbr_from = MSR_LBR_CORE_FROM; 1501 x86_pmu.lbr_to = MSR_LBR_CORE_TO; 1502 1503 x86_pmu.lbr_sel_mask = LBR_SEL_MASK; 1504 x86_pmu.lbr_sel_map = nhm_lbr_sel_map; 1505 1506 /* 1507 * SW branch filter usage: 1508 * - compensate for lack of HW filter 1509 */ 1510 pr_cont("8-deep LBR, "); 1511 } 1512 1513 /* Knights Landing */ 1514 void intel_pmu_lbr_init_knl(void) 1515 { 1516 x86_pmu.lbr_nr = 8; 1517 x86_pmu.lbr_tos = MSR_LBR_TOS; 1518 x86_pmu.lbr_from = MSR_LBR_NHM_FROM; 1519 x86_pmu.lbr_to = MSR_LBR_NHM_TO; 1520 1521 x86_pmu.lbr_sel_mask = LBR_SEL_MASK; 1522 x86_pmu.lbr_sel_map = snb_lbr_sel_map; 1523 1524 /* Knights Landing does have MISPREDICT bit */ 1525 if (x86_pmu.intel_cap.lbr_format == LBR_FORMAT_LIP) 1526 x86_pmu.intel_cap.lbr_format = LBR_FORMAT_EIP_FLAGS; 1527 } 1528 1529 void intel_pmu_lbr_init(void) 1530 { 1531 switch (x86_pmu.intel_cap.lbr_format) { 1532 case LBR_FORMAT_EIP_FLAGS2: 1533 x86_pmu.lbr_has_tsx = 1; 1534 x86_pmu.lbr_from_flags = 1; 1535 if (lbr_from_signext_quirk_needed()) 1536 static_branch_enable(&lbr_from_quirk_key); 1537 break; 1538 1539 case LBR_FORMAT_EIP_FLAGS: 1540 x86_pmu.lbr_from_flags = 1; 1541 break; 1542 1543 case LBR_FORMAT_INFO: 1544 x86_pmu.lbr_has_tsx = 1; 1545 fallthrough; 1546 case LBR_FORMAT_INFO2: 1547 x86_pmu.lbr_has_info = 1; 1548 break; 1549 1550 case LBR_FORMAT_TIME: 1551 x86_pmu.lbr_from_flags = 1; 1552 x86_pmu.lbr_to_cycles = 1; 1553 break; 1554 } 1555 1556 if (x86_pmu.lbr_has_info) { 1557 /* 1558 * Only used in combination with baseline pebs. 1559 */ 1560 static_branch_enable(&x86_lbr_mispred); 1561 static_branch_enable(&x86_lbr_cycles); 1562 } 1563 } 1564 1565 /* 1566 * LBR state size is variable based on the max number of registers. 1567 * This calculates the expected state size, which should match 1568 * what the hardware enumerates for the size of XFEATURE_LBR. 1569 */ 1570 static inline unsigned int get_lbr_state_size(void) 1571 { 1572 return sizeof(struct arch_lbr_state) + 1573 x86_pmu.lbr_nr * sizeof(struct lbr_entry); 1574 } 1575 1576 static bool is_arch_lbr_xsave_available(void) 1577 { 1578 if (!boot_cpu_has(X86_FEATURE_XSAVES)) 1579 return false; 1580 1581 /* 1582 * Check the LBR state with the corresponding software structure. 1583 * Disable LBR XSAVES support if the size doesn't match. 1584 */ 1585 if (xfeature_size(XFEATURE_LBR) == 0) 1586 return false; 1587 1588 if (WARN_ON(xfeature_size(XFEATURE_LBR) != get_lbr_state_size())) 1589 return false; 1590 1591 return true; 1592 } 1593 1594 void __init intel_pmu_arch_lbr_init(void) 1595 { 1596 struct pmu *pmu = x86_get_pmu(smp_processor_id()); 1597 union cpuid28_eax eax; 1598 union cpuid28_ebx ebx; 1599 union cpuid28_ecx ecx; 1600 unsigned int unused_edx; 1601 bool arch_lbr_xsave; 1602 size_t size; 1603 u64 lbr_nr; 1604 1605 /* Arch LBR Capabilities */ 1606 cpuid(28, &eax.full, &ebx.full, &ecx.full, &unused_edx); 1607 1608 lbr_nr = fls(eax.split.lbr_depth_mask) * 8; 1609 if (!lbr_nr) 1610 goto clear_arch_lbr; 1611 1612 /* Apply the max depth of Arch LBR */ 1613 if (wrmsrq_safe(MSR_ARCH_LBR_DEPTH, lbr_nr)) 1614 goto clear_arch_lbr; 1615 1616 x86_pmu.lbr_depth_mask = eax.split.lbr_depth_mask; 1617 x86_pmu.lbr_deep_c_reset = eax.split.lbr_deep_c_reset; 1618 x86_pmu.lbr_lip = eax.split.lbr_lip; 1619 x86_pmu.lbr_cpl = ebx.split.lbr_cpl; 1620 x86_pmu.lbr_filter = ebx.split.lbr_filter; 1621 x86_pmu.lbr_call_stack = ebx.split.lbr_call_stack; 1622 x86_pmu.lbr_mispred = ecx.split.lbr_mispred; 1623 x86_pmu.lbr_timed_lbr = ecx.split.lbr_timed_lbr; 1624 x86_pmu.lbr_br_type = ecx.split.lbr_br_type; 1625 x86_pmu.lbr_counters = ecx.split.lbr_counters; 1626 x86_pmu.lbr_nr = lbr_nr; 1627 1628 if (!!x86_pmu.lbr_counters) 1629 x86_pmu.flags |= PMU_FL_BR_CNTR | PMU_FL_DYN_CONSTRAINT; 1630 1631 if (x86_pmu.lbr_mispred) 1632 static_branch_enable(&x86_lbr_mispred); 1633 if (x86_pmu.lbr_timed_lbr) 1634 static_branch_enable(&x86_lbr_cycles); 1635 if (x86_pmu.lbr_br_type) 1636 static_branch_enable(&x86_lbr_type); 1637 1638 arch_lbr_xsave = is_arch_lbr_xsave_available(); 1639 if (arch_lbr_xsave) { 1640 size = sizeof(struct x86_perf_task_context_arch_lbr_xsave) + 1641 get_lbr_state_size(); 1642 pmu->task_ctx_cache = create_lbr_kmem_cache(size, 1643 XSAVE_ALIGNMENT); 1644 } 1645 1646 if (!pmu->task_ctx_cache) { 1647 arch_lbr_xsave = false; 1648 1649 size = sizeof(struct x86_perf_task_context_arch_lbr) + 1650 lbr_nr * sizeof(struct lbr_entry); 1651 pmu->task_ctx_cache = create_lbr_kmem_cache(size, 0); 1652 } 1653 1654 x86_pmu.lbr_from = MSR_ARCH_LBR_FROM_0; 1655 x86_pmu.lbr_to = MSR_ARCH_LBR_TO_0; 1656 x86_pmu.lbr_info = MSR_ARCH_LBR_INFO_0; 1657 1658 /* LBR callstack requires both CPL and Branch Filtering support */ 1659 if (!x86_pmu.lbr_cpl || 1660 !x86_pmu.lbr_filter || 1661 !x86_pmu.lbr_call_stack) 1662 arch_lbr_ctl_map[PERF_SAMPLE_BRANCH_CALL_STACK_SHIFT] = LBR_NOT_SUPP; 1663 1664 if (!x86_pmu.lbr_cpl) { 1665 arch_lbr_ctl_map[PERF_SAMPLE_BRANCH_USER_SHIFT] = LBR_NOT_SUPP; 1666 arch_lbr_ctl_map[PERF_SAMPLE_BRANCH_KERNEL_SHIFT] = LBR_NOT_SUPP; 1667 } else if (!x86_pmu.lbr_filter) { 1668 arch_lbr_ctl_map[PERF_SAMPLE_BRANCH_ANY_SHIFT] = LBR_NOT_SUPP; 1669 arch_lbr_ctl_map[PERF_SAMPLE_BRANCH_ANY_RETURN_SHIFT] = LBR_NOT_SUPP; 1670 arch_lbr_ctl_map[PERF_SAMPLE_BRANCH_ANY_CALL_SHIFT] = LBR_NOT_SUPP; 1671 arch_lbr_ctl_map[PERF_SAMPLE_BRANCH_IND_CALL_SHIFT] = LBR_NOT_SUPP; 1672 arch_lbr_ctl_map[PERF_SAMPLE_BRANCH_COND_SHIFT] = LBR_NOT_SUPP; 1673 arch_lbr_ctl_map[PERF_SAMPLE_BRANCH_IND_JUMP_SHIFT] = LBR_NOT_SUPP; 1674 arch_lbr_ctl_map[PERF_SAMPLE_BRANCH_CALL_SHIFT] = LBR_NOT_SUPP; 1675 } 1676 1677 x86_pmu.lbr_ctl_mask = ARCH_LBR_CTL_MASK; 1678 x86_pmu.lbr_ctl_map = arch_lbr_ctl_map; 1679 1680 if (!x86_pmu.lbr_cpl && !x86_pmu.lbr_filter) 1681 x86_pmu.lbr_ctl_map = NULL; 1682 1683 x86_pmu.lbr_reset = intel_pmu_arch_lbr_reset; 1684 if (arch_lbr_xsave) { 1685 x86_pmu.lbr_save = intel_pmu_arch_lbr_xsaves; 1686 x86_pmu.lbr_restore = intel_pmu_arch_lbr_xrstors; 1687 x86_pmu.lbr_read = intel_pmu_arch_lbr_read_xsave; 1688 pr_cont("XSAVE "); 1689 } else { 1690 x86_pmu.lbr_save = intel_pmu_arch_lbr_save; 1691 x86_pmu.lbr_restore = intel_pmu_arch_lbr_restore; 1692 x86_pmu.lbr_read = intel_pmu_arch_lbr_read; 1693 } 1694 1695 pr_cont("Architectural LBR, "); 1696 1697 return; 1698 1699 clear_arch_lbr: 1700 setup_clear_cpu_cap(X86_FEATURE_ARCH_LBR); 1701 } 1702 1703 /** 1704 * x86_perf_get_lbr - get the LBR records information 1705 * 1706 * @lbr: the caller's memory to store the LBR records information 1707 */ 1708 void x86_perf_get_lbr(struct x86_pmu_lbr *lbr) 1709 { 1710 lbr->nr = x86_pmu.lbr_nr; 1711 lbr->from = x86_pmu.lbr_from; 1712 lbr->to = x86_pmu.lbr_to; 1713 lbr->info = x86_pmu.lbr_info; 1714 lbr->has_callstack = x86_pmu_has_lbr_callstack(); 1715 } 1716 EXPORT_SYMBOL_FOR_KVM(x86_perf_get_lbr); 1717 1718 struct event_constraint vlbr_constraint = 1719 __EVENT_CONSTRAINT(INTEL_FIXED_VLBR_EVENT, (1ULL << INTEL_PMC_IDX_FIXED_VLBR), 1720 FIXED_EVENT_FLAGS, 1, 0, PERF_X86_EVENT_LBR_SELECT); 1721