1 // SPDX-License-Identifier: GPL-2.0 2 #include <linux/perf_event.h> 3 #include <asm/perf_event.h> 4 5 #include "../perf_event.h" 6 7 /* LBR Branch Select valid bits */ 8 #define LBR_SELECT_MASK 0x1ff 9 10 /* 11 * LBR Branch Select filter bits which when set, ensures that the 12 * corresponding type of branches are not recorded 13 */ 14 #define LBR_SELECT_KERNEL 0 /* Branches ending in CPL = 0 */ 15 #define LBR_SELECT_USER 1 /* Branches ending in CPL > 0 */ 16 #define LBR_SELECT_JCC 2 /* Conditional branches */ 17 #define LBR_SELECT_CALL_NEAR_REL 3 /* Near relative calls */ 18 #define LBR_SELECT_CALL_NEAR_IND 4 /* Indirect relative calls */ 19 #define LBR_SELECT_RET_NEAR 5 /* Near returns */ 20 #define LBR_SELECT_JMP_NEAR_IND 6 /* Near indirect jumps (excl. calls and returns) */ 21 #define LBR_SELECT_JMP_NEAR_REL 7 /* Near relative jumps (excl. calls) */ 22 #define LBR_SELECT_FAR_BRANCH 8 /* Far branches */ 23 24 #define LBR_KERNEL BIT(LBR_SELECT_KERNEL) 25 #define LBR_USER BIT(LBR_SELECT_USER) 26 #define LBR_JCC BIT(LBR_SELECT_JCC) 27 #define LBR_REL_CALL BIT(LBR_SELECT_CALL_NEAR_REL) 28 #define LBR_IND_CALL BIT(LBR_SELECT_CALL_NEAR_IND) 29 #define LBR_RETURN BIT(LBR_SELECT_RET_NEAR) 30 #define LBR_REL_JMP BIT(LBR_SELECT_JMP_NEAR_REL) 31 #define LBR_IND_JMP BIT(LBR_SELECT_JMP_NEAR_IND) 32 #define LBR_FAR BIT(LBR_SELECT_FAR_BRANCH) 33 #define LBR_NOT_SUPP -1 /* unsupported filter */ 34 #define LBR_IGNORE 0 35 36 #define LBR_ANY \ 37 (LBR_JCC | LBR_REL_CALL | LBR_IND_CALL | LBR_RETURN | \ 38 LBR_REL_JMP | LBR_IND_JMP | LBR_FAR) 39 40 struct branch_entry { 41 union { 42 struct { 43 u64 ip:58; 44 u64 ip_sign_ext:5; 45 u64 mispredict:1; 46 } split; 47 u64 full; 48 } from; 49 50 union { 51 struct { 52 u64 ip:58; 53 u64 ip_sign_ext:3; 54 u64 reserved:1; 55 u64 spec:1; 56 u64 valid:1; 57 } split; 58 u64 full; 59 } to; 60 }; 61 62 static __always_inline void amd_pmu_lbr_set_from(unsigned int idx, u64 val) 63 { 64 wrmsrl(MSR_AMD_SAMP_BR_FROM + idx * 2, val); 65 } 66 67 static __always_inline void amd_pmu_lbr_set_to(unsigned int idx, u64 val) 68 { 69 wrmsrl(MSR_AMD_SAMP_BR_FROM + idx * 2 + 1, val); 70 } 71 72 static __always_inline u64 amd_pmu_lbr_get_from(unsigned int idx) 73 { 74 u64 val; 75 76 rdmsrl(MSR_AMD_SAMP_BR_FROM + idx * 2, val); 77 78 return val; 79 } 80 81 static __always_inline u64 amd_pmu_lbr_get_to(unsigned int idx) 82 { 83 u64 val; 84 85 rdmsrl(MSR_AMD_SAMP_BR_FROM + idx * 2 + 1, val); 86 87 return val; 88 } 89 90 static __always_inline u64 sign_ext_branch_ip(u64 ip) 91 { 92 u32 shift = 64 - boot_cpu_data.x86_virt_bits; 93 94 return (u64)(((s64)ip << shift) >> shift); 95 } 96 97 static void amd_pmu_lbr_filter(void) 98 { 99 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 100 int br_sel = cpuc->br_sel, offset, type, i, j; 101 bool compress = false; 102 bool fused_only = false; 103 u64 from, to; 104 105 /* If sampling all branches, there is nothing to filter */ 106 if (((br_sel & X86_BR_ALL) == X86_BR_ALL) && 107 ((br_sel & X86_BR_TYPE_SAVE) != X86_BR_TYPE_SAVE)) 108 fused_only = true; 109 110 for (i = 0; i < cpuc->lbr_stack.nr; i++) { 111 from = cpuc->lbr_entries[i].from; 112 to = cpuc->lbr_entries[i].to; 113 type = branch_type_fused(from, to, 0, &offset); 114 115 /* 116 * Adjust the branch from address in case of instruction 117 * fusion where it points to an instruction preceding the 118 * actual branch 119 */ 120 if (offset) { 121 cpuc->lbr_entries[i].from += offset; 122 if (fused_only) 123 continue; 124 } 125 126 /* If type does not correspond, then discard */ 127 if (type == X86_BR_NONE || (br_sel & type) != type) { 128 cpuc->lbr_entries[i].from = 0; /* mark invalid */ 129 compress = true; 130 } 131 132 if ((br_sel & X86_BR_TYPE_SAVE) == X86_BR_TYPE_SAVE) 133 cpuc->lbr_entries[i].type = common_branch_type(type); 134 } 135 136 if (!compress) 137 return; 138 139 /* Remove all invalid entries */ 140 for (i = 0; i < cpuc->lbr_stack.nr; ) { 141 if (!cpuc->lbr_entries[i].from) { 142 j = i; 143 while (++j < cpuc->lbr_stack.nr) 144 cpuc->lbr_entries[j - 1] = cpuc->lbr_entries[j]; 145 cpuc->lbr_stack.nr--; 146 if (!cpuc->lbr_entries[i].from) 147 continue; 148 } 149 i++; 150 } 151 } 152 153 static const int lbr_spec_map[PERF_BR_SPEC_MAX] = { 154 PERF_BR_SPEC_NA, 155 PERF_BR_SPEC_WRONG_PATH, 156 PERF_BR_NON_SPEC_CORRECT_PATH, 157 PERF_BR_SPEC_CORRECT_PATH, 158 }; 159 160 void amd_pmu_lbr_read(void) 161 { 162 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 163 struct perf_branch_entry *br = cpuc->lbr_entries; 164 struct branch_entry entry; 165 int out = 0, idx, i; 166 167 if (!cpuc->lbr_users) 168 return; 169 170 for (i = 0; i < x86_pmu.lbr_nr; i++) { 171 entry.from.full = amd_pmu_lbr_get_from(i); 172 entry.to.full = amd_pmu_lbr_get_to(i); 173 174 /* 175 * Check if a branch has been logged; if valid = 0, spec = 0 176 * then no branch was recorded 177 */ 178 if (!entry.to.split.valid && !entry.to.split.spec) 179 continue; 180 181 perf_clear_branch_entry_bitfields(br + out); 182 183 br[out].from = sign_ext_branch_ip(entry.from.split.ip); 184 br[out].to = sign_ext_branch_ip(entry.to.split.ip); 185 br[out].mispred = entry.from.split.mispredict; 186 br[out].predicted = !br[out].mispred; 187 188 /* 189 * Set branch speculation information using the status of 190 * the valid and spec bits. 191 * 192 * When valid = 0, spec = 0, no branch was recorded and the 193 * entry is discarded as seen above. 194 * 195 * When valid = 0, spec = 1, the recorded branch was 196 * speculative but took the wrong path. 197 * 198 * When valid = 1, spec = 0, the recorded branch was 199 * non-speculative but took the correct path. 200 * 201 * When valid = 1, spec = 1, the recorded branch was 202 * speculative and took the correct path 203 */ 204 idx = (entry.to.split.valid << 1) | entry.to.split.spec; 205 br[out].spec = lbr_spec_map[idx]; 206 out++; 207 } 208 209 cpuc->lbr_stack.nr = out; 210 211 /* 212 * Internal register renaming always ensures that LBR From[0] and 213 * LBR To[0] always represent the TOS 214 */ 215 cpuc->lbr_stack.hw_idx = 0; 216 217 /* Perform further software filtering */ 218 amd_pmu_lbr_filter(); 219 } 220 221 static const int lbr_select_map[PERF_SAMPLE_BRANCH_MAX_SHIFT] = { 222 [PERF_SAMPLE_BRANCH_USER_SHIFT] = LBR_USER, 223 [PERF_SAMPLE_BRANCH_KERNEL_SHIFT] = LBR_KERNEL, 224 [PERF_SAMPLE_BRANCH_HV_SHIFT] = LBR_IGNORE, 225 226 [PERF_SAMPLE_BRANCH_ANY_SHIFT] = LBR_ANY, 227 [PERF_SAMPLE_BRANCH_ANY_CALL_SHIFT] = LBR_REL_CALL | LBR_IND_CALL | LBR_FAR, 228 [PERF_SAMPLE_BRANCH_ANY_RETURN_SHIFT] = LBR_RETURN | LBR_FAR, 229 [PERF_SAMPLE_BRANCH_IND_CALL_SHIFT] = LBR_IND_CALL, 230 [PERF_SAMPLE_BRANCH_ABORT_TX_SHIFT] = LBR_NOT_SUPP, 231 [PERF_SAMPLE_BRANCH_IN_TX_SHIFT] = LBR_NOT_SUPP, 232 [PERF_SAMPLE_BRANCH_NO_TX_SHIFT] = LBR_NOT_SUPP, 233 [PERF_SAMPLE_BRANCH_COND_SHIFT] = LBR_JCC, 234 235 [PERF_SAMPLE_BRANCH_CALL_STACK_SHIFT] = LBR_NOT_SUPP, 236 [PERF_SAMPLE_BRANCH_IND_JUMP_SHIFT] = LBR_IND_JMP, 237 [PERF_SAMPLE_BRANCH_CALL_SHIFT] = LBR_REL_CALL, 238 239 [PERF_SAMPLE_BRANCH_NO_FLAGS_SHIFT] = LBR_NOT_SUPP, 240 [PERF_SAMPLE_BRANCH_NO_CYCLES_SHIFT] = LBR_NOT_SUPP, 241 }; 242 243 static int amd_pmu_lbr_setup_filter(struct perf_event *event) 244 { 245 struct hw_perf_event_extra *reg = &event->hw.branch_reg; 246 u64 br_type = event->attr.branch_sample_type; 247 u64 mask = 0, v; 248 int i; 249 250 /* No LBR support */ 251 if (!x86_pmu.lbr_nr) 252 return -EOPNOTSUPP; 253 254 if (br_type & PERF_SAMPLE_BRANCH_USER) 255 mask |= X86_BR_USER; 256 257 if (br_type & PERF_SAMPLE_BRANCH_KERNEL) 258 mask |= X86_BR_KERNEL; 259 260 /* Ignore BRANCH_HV here */ 261 262 if (br_type & PERF_SAMPLE_BRANCH_ANY) 263 mask |= X86_BR_ANY; 264 265 if (br_type & PERF_SAMPLE_BRANCH_ANY_CALL) 266 mask |= X86_BR_ANY_CALL; 267 268 if (br_type & PERF_SAMPLE_BRANCH_ANY_RETURN) 269 mask |= X86_BR_RET | X86_BR_IRET | X86_BR_SYSRET; 270 271 if (br_type & PERF_SAMPLE_BRANCH_IND_CALL) 272 mask |= X86_BR_IND_CALL; 273 274 if (br_type & PERF_SAMPLE_BRANCH_COND) 275 mask |= X86_BR_JCC; 276 277 if (br_type & PERF_SAMPLE_BRANCH_IND_JUMP) 278 mask |= X86_BR_IND_JMP; 279 280 if (br_type & PERF_SAMPLE_BRANCH_CALL) 281 mask |= X86_BR_CALL | X86_BR_ZERO_CALL; 282 283 if (br_type & PERF_SAMPLE_BRANCH_TYPE_SAVE) 284 mask |= X86_BR_TYPE_SAVE; 285 286 reg->reg = mask; 287 mask = 0; 288 289 for (i = 0; i < PERF_SAMPLE_BRANCH_MAX_SHIFT; i++) { 290 if (!(br_type & BIT_ULL(i))) 291 continue; 292 293 v = lbr_select_map[i]; 294 if (v == LBR_NOT_SUPP) 295 return -EOPNOTSUPP; 296 297 if (v != LBR_IGNORE) 298 mask |= v; 299 } 300 301 /* Filter bits operate in suppress mode */ 302 reg->config = mask ^ LBR_SELECT_MASK; 303 304 return 0; 305 } 306 307 int amd_pmu_lbr_hw_config(struct perf_event *event) 308 { 309 int ret = 0; 310 311 /* LBR is not recommended in counting mode */ 312 if (!is_sampling_event(event)) 313 return -EINVAL; 314 315 ret = amd_pmu_lbr_setup_filter(event); 316 if (!ret) 317 event->attach_state |= PERF_ATTACH_SCHED_CB; 318 319 return ret; 320 } 321 322 void amd_pmu_lbr_reset(void) 323 { 324 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 325 int i; 326 327 if (!x86_pmu.lbr_nr) 328 return; 329 330 /* Reset all branch records individually */ 331 for (i = 0; i < x86_pmu.lbr_nr; i++) { 332 amd_pmu_lbr_set_from(i, 0); 333 amd_pmu_lbr_set_to(i, 0); 334 } 335 336 cpuc->last_task_ctx = NULL; 337 cpuc->last_log_id = 0; 338 wrmsrl(MSR_AMD64_LBR_SELECT, 0); 339 } 340 341 void amd_pmu_lbr_add(struct perf_event *event) 342 { 343 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 344 struct hw_perf_event_extra *reg = &event->hw.branch_reg; 345 346 if (!x86_pmu.lbr_nr) 347 return; 348 349 if (has_branch_stack(event)) { 350 cpuc->lbr_select = 1; 351 cpuc->lbr_sel->config = reg->config; 352 cpuc->br_sel = reg->reg; 353 } 354 355 perf_sched_cb_inc(event->pmu); 356 357 if (!cpuc->lbr_users++ && !event->total_time_running) 358 amd_pmu_lbr_reset(); 359 } 360 361 void amd_pmu_lbr_del(struct perf_event *event) 362 { 363 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 364 365 if (!x86_pmu.lbr_nr) 366 return; 367 368 if (has_branch_stack(event)) 369 cpuc->lbr_select = 0; 370 371 cpuc->lbr_users--; 372 WARN_ON_ONCE(cpuc->lbr_users < 0); 373 perf_sched_cb_dec(event->pmu); 374 } 375 376 void amd_pmu_lbr_sched_task(struct perf_event_pmu_context *pmu_ctx, bool sched_in) 377 { 378 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 379 380 /* 381 * A context switch can flip the address space and LBR entries are 382 * not tagged with an identifier. Hence, branches cannot be resolved 383 * from the old address space and the LBR records should be wiped. 384 */ 385 if (cpuc->lbr_users && sched_in) 386 amd_pmu_lbr_reset(); 387 } 388 389 void amd_pmu_lbr_enable_all(void) 390 { 391 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 392 u64 lbr_select, dbg_ctl, dbg_extn_cfg; 393 394 if (!cpuc->lbr_users || !x86_pmu.lbr_nr) 395 return; 396 397 /* Set hardware branch filter */ 398 if (cpuc->lbr_select) { 399 lbr_select = cpuc->lbr_sel->config & LBR_SELECT_MASK; 400 wrmsrl(MSR_AMD64_LBR_SELECT, lbr_select); 401 } 402 403 rdmsrl(MSR_IA32_DEBUGCTLMSR, dbg_ctl); 404 rdmsrl(MSR_AMD_DBG_EXTN_CFG, dbg_extn_cfg); 405 406 wrmsrl(MSR_IA32_DEBUGCTLMSR, dbg_ctl | DEBUGCTLMSR_FREEZE_LBRS_ON_PMI); 407 wrmsrl(MSR_AMD_DBG_EXTN_CFG, dbg_extn_cfg | DBG_EXTN_CFG_LBRV2EN); 408 } 409 410 void amd_pmu_lbr_disable_all(void) 411 { 412 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 413 u64 dbg_ctl, dbg_extn_cfg; 414 415 if (!cpuc->lbr_users || !x86_pmu.lbr_nr) 416 return; 417 418 rdmsrl(MSR_AMD_DBG_EXTN_CFG, dbg_extn_cfg); 419 rdmsrl(MSR_IA32_DEBUGCTLMSR, dbg_ctl); 420 421 wrmsrl(MSR_AMD_DBG_EXTN_CFG, dbg_extn_cfg & ~DBG_EXTN_CFG_LBRV2EN); 422 wrmsrl(MSR_IA32_DEBUGCTLMSR, dbg_ctl & ~DEBUGCTLMSR_FREEZE_LBRS_ON_PMI); 423 } 424 425 __init int amd_pmu_lbr_init(void) 426 { 427 union cpuid_0x80000022_ebx ebx; 428 429 if (x86_pmu.version < 2 || !boot_cpu_has(X86_FEATURE_AMD_LBR_V2)) 430 return -EOPNOTSUPP; 431 432 /* Set number of entries */ 433 ebx.full = cpuid_ebx(EXT_PERFMON_DEBUG_FEATURES); 434 x86_pmu.lbr_nr = ebx.split.lbr_v2_stack_sz; 435 436 pr_cont("%d-deep LBR, ", x86_pmu.lbr_nr); 437 438 return 0; 439 } 440