1 // SPDX-License-Identifier: GPL-2.0 2 #include <linux/perf_event.h> 3 4 #include <asm/cpuid/api.h> 5 #include <asm/msr.h> 6 #include <asm/perf_event.h> 7 8 #include "../perf_event.h" 9 10 /* LBR Branch Select valid bits */ 11 #define LBR_SELECT_MASK 0x1ff 12 13 /* 14 * LBR Branch Select filter bits which when set, ensures that the 15 * corresponding type of branches are not recorded 16 */ 17 #define LBR_SELECT_KERNEL 0 /* Branches ending in CPL = 0 */ 18 #define LBR_SELECT_USER 1 /* Branches ending in CPL > 0 */ 19 #define LBR_SELECT_JCC 2 /* Conditional branches */ 20 #define LBR_SELECT_CALL_NEAR_REL 3 /* Near relative calls */ 21 #define LBR_SELECT_CALL_NEAR_IND 4 /* Indirect relative calls */ 22 #define LBR_SELECT_RET_NEAR 5 /* Near returns */ 23 #define LBR_SELECT_JMP_NEAR_IND 6 /* Near indirect jumps (excl. calls and returns) */ 24 #define LBR_SELECT_JMP_NEAR_REL 7 /* Near relative jumps (excl. calls) */ 25 #define LBR_SELECT_FAR_BRANCH 8 /* Far branches */ 26 27 #define LBR_KERNEL BIT(LBR_SELECT_KERNEL) 28 #define LBR_USER BIT(LBR_SELECT_USER) 29 #define LBR_JCC BIT(LBR_SELECT_JCC) 30 #define LBR_REL_CALL BIT(LBR_SELECT_CALL_NEAR_REL) 31 #define LBR_IND_CALL BIT(LBR_SELECT_CALL_NEAR_IND) 32 #define LBR_RETURN BIT(LBR_SELECT_RET_NEAR) 33 #define LBR_REL_JMP BIT(LBR_SELECT_JMP_NEAR_REL) 34 #define LBR_IND_JMP BIT(LBR_SELECT_JMP_NEAR_IND) 35 #define LBR_FAR BIT(LBR_SELECT_FAR_BRANCH) 36 #define LBR_NOT_SUPP -1 /* unsupported filter */ 37 #define LBR_IGNORE 0 38 39 #define LBR_ANY \ 40 (LBR_JCC | LBR_REL_CALL | LBR_IND_CALL | LBR_RETURN | \ 41 LBR_REL_JMP | LBR_IND_JMP | LBR_FAR) 42 43 struct branch_entry { 44 union { 45 struct { 46 u64 ip:58; 47 u64 ip_sign_ext:5; 48 u64 mispredict:1; 49 } split; 50 u64 full; 51 } from; 52 53 union { 54 struct { 55 u64 ip:58; 56 u64 ip_sign_ext:3; 57 u64 reserved:1; 58 u64 spec:1; 59 u64 valid:1; 60 } split; 61 u64 full; 62 } to; 63 }; 64 65 static __always_inline void amd_pmu_lbr_set_from(unsigned int idx, u64 val) 66 { 67 wrmsrq(MSR_AMD_SAMP_BR_FROM + idx * 2, val); 68 } 69 70 static __always_inline void amd_pmu_lbr_set_to(unsigned int idx, u64 val) 71 { 72 wrmsrq(MSR_AMD_SAMP_BR_FROM + idx * 2 + 1, val); 73 } 74 75 static __always_inline u64 amd_pmu_lbr_get_from(unsigned int idx) 76 { 77 u64 val; 78 79 rdmsrq(MSR_AMD_SAMP_BR_FROM + idx * 2, val); 80 81 return val; 82 } 83 84 static __always_inline u64 amd_pmu_lbr_get_to(unsigned int idx) 85 { 86 u64 val; 87 88 rdmsrq(MSR_AMD_SAMP_BR_FROM + idx * 2 + 1, val); 89 90 return val; 91 } 92 93 static __always_inline u64 sign_ext_branch_ip(u64 ip) 94 { 95 u32 shift = 64 - boot_cpu_data.x86_virt_bits; 96 97 return (u64)(((s64)ip << shift) >> shift); 98 } 99 100 static void amd_pmu_lbr_filter(void) 101 { 102 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 103 int br_sel = cpuc->br_sel, offset, type, i, j; 104 bool compress = false; 105 bool fused_only = false; 106 u64 from, to; 107 108 /* If sampling all branches, there is nothing to filter */ 109 if (((br_sel & X86_BR_ALL) == X86_BR_ALL) && 110 ((br_sel & X86_BR_TYPE_SAVE) != X86_BR_TYPE_SAVE)) 111 fused_only = true; 112 113 for (i = 0; i < cpuc->lbr_stack.nr; i++) { 114 from = cpuc->lbr_entries[i].from; 115 to = cpuc->lbr_entries[i].to; 116 type = branch_type_fused(from, to, 0, &offset); 117 118 /* 119 * Adjust the branch from address in case of instruction 120 * fusion where it points to an instruction preceding the 121 * actual branch 122 */ 123 if (offset) { 124 cpuc->lbr_entries[i].from += offset; 125 if (fused_only) 126 continue; 127 } 128 129 /* If type does not correspond, then discard */ 130 if (type == X86_BR_NONE || (br_sel & type) != type || 131 (!(br_sel & X86_BR_KERNEL) && kernel_ip(cpuc->lbr_entries[i].from))) { 132 cpuc->lbr_entries[i].from = 0; /* mark invalid */ 133 compress = true; 134 } 135 136 if ((br_sel & X86_BR_TYPE_SAVE) == X86_BR_TYPE_SAVE) 137 cpuc->lbr_entries[i].type = common_branch_type(type); 138 } 139 140 if (!compress) 141 return; 142 143 /* Remove all invalid entries */ 144 for (i = 0; i < cpuc->lbr_stack.nr; ) { 145 if (!cpuc->lbr_entries[i].from) { 146 j = i; 147 while (++j < cpuc->lbr_stack.nr) 148 cpuc->lbr_entries[j - 1] = cpuc->lbr_entries[j]; 149 cpuc->lbr_stack.nr--; 150 if (!cpuc->lbr_entries[i].from) 151 continue; 152 } 153 i++; 154 } 155 } 156 157 static const int lbr_spec_map[PERF_BR_SPEC_MAX] = { 158 PERF_BR_SPEC_NA, 159 PERF_BR_SPEC_WRONG_PATH, 160 PERF_BR_NON_SPEC_CORRECT_PATH, 161 PERF_BR_SPEC_CORRECT_PATH, 162 }; 163 164 void amd_pmu_lbr_read(void) 165 { 166 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 167 struct perf_branch_entry *br = cpuc->lbr_entries; 168 struct branch_entry entry; 169 int out = 0, idx, i; 170 171 if (!cpuc->lbr_users) 172 return; 173 174 for (i = 0; i < x86_pmu.lbr_nr; i++) { 175 entry.from.full = amd_pmu_lbr_get_from(i); 176 entry.to.full = amd_pmu_lbr_get_to(i); 177 178 /* 179 * Check if a branch has been logged; if valid = 0, spec = 0 180 * then no branch was recorded; if reserved = 1 then an 181 * erroneous branch was recorded (see Erratum 1452) 182 */ 183 if ((!entry.to.split.valid && !entry.to.split.spec) || 184 entry.to.split.reserved) 185 continue; 186 187 perf_clear_branch_entry_bitfields(br + out); 188 189 br[out].from = sign_ext_branch_ip(entry.from.split.ip); 190 br[out].to = sign_ext_branch_ip(entry.to.split.ip); 191 br[out].mispred = entry.from.split.mispredict; 192 br[out].predicted = !br[out].mispred; 193 194 /* 195 * Set branch speculation information using the status of 196 * the valid and spec bits. 197 * 198 * When valid = 0, spec = 0, no branch was recorded and the 199 * entry is discarded as seen above. 200 * 201 * When valid = 0, spec = 1, the recorded branch was 202 * speculative but took the wrong path. 203 * 204 * When valid = 1, spec = 0, the recorded branch was 205 * non-speculative but took the correct path. 206 * 207 * When valid = 1, spec = 1, the recorded branch was 208 * speculative and took the correct path 209 */ 210 idx = (entry.to.split.valid << 1) | entry.to.split.spec; 211 br[out].spec = lbr_spec_map[idx]; 212 out++; 213 } 214 215 cpuc->lbr_stack.nr = out; 216 217 /* 218 * Internal register renaming always ensures that LBR From[0] and 219 * LBR To[0] always represent the TOS 220 */ 221 cpuc->lbr_stack.hw_idx = 0; 222 223 /* Perform further software filtering */ 224 amd_pmu_lbr_filter(); 225 } 226 227 static const int lbr_select_map[PERF_SAMPLE_BRANCH_MAX_SHIFT] = { 228 [PERF_SAMPLE_BRANCH_USER_SHIFT] = LBR_USER, 229 [PERF_SAMPLE_BRANCH_KERNEL_SHIFT] = LBR_KERNEL, 230 [PERF_SAMPLE_BRANCH_HV_SHIFT] = LBR_IGNORE, 231 232 [PERF_SAMPLE_BRANCH_ANY_SHIFT] = LBR_ANY, 233 [PERF_SAMPLE_BRANCH_ANY_CALL_SHIFT] = LBR_REL_CALL | LBR_IND_CALL | LBR_FAR, 234 [PERF_SAMPLE_BRANCH_ANY_RETURN_SHIFT] = LBR_RETURN | LBR_FAR, 235 [PERF_SAMPLE_BRANCH_IND_CALL_SHIFT] = LBR_IND_CALL, 236 [PERF_SAMPLE_BRANCH_ABORT_TX_SHIFT] = LBR_NOT_SUPP, 237 [PERF_SAMPLE_BRANCH_IN_TX_SHIFT] = LBR_NOT_SUPP, 238 [PERF_SAMPLE_BRANCH_NO_TX_SHIFT] = LBR_NOT_SUPP, 239 [PERF_SAMPLE_BRANCH_COND_SHIFT] = LBR_JCC, 240 241 [PERF_SAMPLE_BRANCH_CALL_STACK_SHIFT] = LBR_NOT_SUPP, 242 [PERF_SAMPLE_BRANCH_IND_JUMP_SHIFT] = LBR_IND_JMP, 243 [PERF_SAMPLE_BRANCH_CALL_SHIFT] = LBR_REL_CALL, 244 245 [PERF_SAMPLE_BRANCH_NO_FLAGS_SHIFT] = LBR_NOT_SUPP, 246 [PERF_SAMPLE_BRANCH_NO_CYCLES_SHIFT] = LBR_NOT_SUPP, 247 }; 248 249 static int amd_pmu_lbr_setup_filter(struct perf_event *event) 250 { 251 struct hw_perf_event_extra *reg = &event->hw.branch_reg; 252 u64 br_type = event->attr.branch_sample_type; 253 u64 mask = 0, v; 254 int i; 255 256 /* No LBR support */ 257 if (!x86_pmu.lbr_nr) 258 return -EOPNOTSUPP; 259 260 if (br_type & PERF_SAMPLE_BRANCH_USER) 261 mask |= X86_BR_USER; 262 263 if (br_type & PERF_SAMPLE_BRANCH_KERNEL) 264 mask |= X86_BR_KERNEL; 265 266 /* Ignore BRANCH_HV here */ 267 268 if (br_type & PERF_SAMPLE_BRANCH_ANY) 269 mask |= X86_BR_ANY; 270 271 if (br_type & PERF_SAMPLE_BRANCH_ANY_CALL) 272 mask |= X86_BR_ANY_CALL; 273 274 if (br_type & PERF_SAMPLE_BRANCH_ANY_RETURN) 275 mask |= X86_BR_RET | X86_BR_IRET | X86_BR_SYSRET; 276 277 if (br_type & PERF_SAMPLE_BRANCH_IND_CALL) 278 mask |= X86_BR_IND_CALL; 279 280 if (br_type & PERF_SAMPLE_BRANCH_COND) 281 mask |= X86_BR_JCC; 282 283 if (br_type & PERF_SAMPLE_BRANCH_IND_JUMP) 284 mask |= X86_BR_IND_JMP; 285 286 if (br_type & PERF_SAMPLE_BRANCH_CALL) 287 mask |= X86_BR_CALL | X86_BR_ZERO_CALL; 288 289 if (br_type & PERF_SAMPLE_BRANCH_TYPE_SAVE) 290 mask |= X86_BR_TYPE_SAVE; 291 292 reg->reg = mask; 293 mask = 0; 294 295 for (i = 0; i < PERF_SAMPLE_BRANCH_MAX_SHIFT; i++) { 296 if (!(br_type & BIT_ULL(i))) 297 continue; 298 299 v = lbr_select_map[i]; 300 if (v == LBR_NOT_SUPP) 301 return -EOPNOTSUPP; 302 303 if (v != LBR_IGNORE) 304 mask |= v; 305 } 306 307 /* Filter bits operate in suppress mode */ 308 reg->config = mask ^ LBR_SELECT_MASK; 309 310 return 0; 311 } 312 313 int amd_pmu_lbr_hw_config(struct perf_event *event) 314 { 315 int ret = 0; 316 317 ret = amd_pmu_lbr_setup_filter(event); 318 if (!ret) 319 event->attach_state |= PERF_ATTACH_SCHED_CB; 320 321 return ret; 322 } 323 324 void amd_pmu_lbr_reset(void) 325 { 326 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 327 int i; 328 329 if (!x86_pmu.lbr_nr) 330 return; 331 332 /* Reset all branch records individually */ 333 for (i = 0; i < x86_pmu.lbr_nr; i++) { 334 amd_pmu_lbr_set_from(i, 0); 335 amd_pmu_lbr_set_to(i, 0); 336 } 337 338 cpuc->last_task_ctx = NULL; 339 cpuc->last_log_id = 0; 340 wrmsrq(MSR_AMD64_LBR_SELECT, 0); 341 } 342 343 void amd_pmu_lbr_add(struct perf_event *event) 344 { 345 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 346 struct hw_perf_event_extra *reg = &event->hw.branch_reg; 347 348 if (!x86_pmu.lbr_nr) 349 return; 350 351 if (has_branch_stack(event)) { 352 cpuc->lbr_select = 1; 353 cpuc->lbr_sel->config = reg->config; 354 cpuc->br_sel = reg->reg; 355 } 356 357 perf_sched_cb_inc(event->pmu); 358 359 if (!cpuc->lbr_users++ && !event->total_time_running) 360 amd_pmu_lbr_reset(); 361 } 362 363 void amd_pmu_lbr_del(struct perf_event *event) 364 { 365 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 366 367 if (!x86_pmu.lbr_nr) 368 return; 369 370 if (has_branch_stack(event)) 371 cpuc->lbr_select = 0; 372 373 cpuc->lbr_users--; 374 WARN_ON_ONCE(cpuc->lbr_users < 0); 375 perf_sched_cb_dec(event->pmu); 376 } 377 378 void amd_pmu_lbr_sched_task(struct perf_event_pmu_context *pmu_ctx, 379 struct task_struct *task, bool sched_in) 380 { 381 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 382 383 /* 384 * A context switch can flip the address space and LBR entries are 385 * not tagged with an identifier. Hence, branches cannot be resolved 386 * from the old address space and the LBR records should be wiped. 387 */ 388 if (cpuc->lbr_users && sched_in) 389 amd_pmu_lbr_reset(); 390 } 391 392 void amd_pmu_lbr_enable_all(void) 393 { 394 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 395 u64 lbr_select, dbg_ctl, dbg_extn_cfg; 396 397 if (!cpuc->lbr_users || !x86_pmu.lbr_nr) 398 return; 399 400 /* Set hardware branch filter */ 401 if (cpuc->lbr_select) { 402 lbr_select = cpuc->lbr_sel->config & LBR_SELECT_MASK; 403 wrmsrq(MSR_AMD64_LBR_SELECT, lbr_select); 404 } 405 406 if (cpu_feature_enabled(X86_FEATURE_AMD_LBR_PMC_FREEZE)) { 407 rdmsrq(MSR_IA32_DEBUGCTLMSR, dbg_ctl); 408 wrmsrq(MSR_IA32_DEBUGCTLMSR, dbg_ctl | DEBUGCTLMSR_FREEZE_LBRS_ON_PMI); 409 } 410 411 rdmsrq(MSR_AMD_DBG_EXTN_CFG, dbg_extn_cfg); 412 wrmsrq(MSR_AMD_DBG_EXTN_CFG, dbg_extn_cfg | DBG_EXTN_CFG_LBRV2EN); 413 } 414 415 void amd_pmu_lbr_disable_all(void) 416 { 417 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 418 419 if (!cpuc->lbr_users || !x86_pmu.lbr_nr) 420 return; 421 422 __amd_pmu_lbr_disable(); 423 } 424 425 __init int amd_pmu_lbr_init(void) 426 { 427 union cpuid_0x80000022_ebx ebx; 428 429 if (x86_pmu.version < 2 || !boot_cpu_has(X86_FEATURE_AMD_LBR_V2)) 430 return -EOPNOTSUPP; 431 432 /* Set number of entries */ 433 ebx.full = cpuid_ebx(EXT_PERFMON_DEBUG_FEATURES); 434 x86_pmu.lbr_nr = ebx.split.lbr_v2_stack_sz; 435 436 pr_cont("%d-deep LBR, ", x86_pmu.lbr_nr); 437 438 return 0; 439 } 440