1 // SPDX-License-Identifier: GPL-2.0
2 #include <linux/perf_event.h>
3
4 #include <asm/cpuid/api.h>
5 #include <asm/msr.h>
6 #include <asm/perf_event.h>
7
8 #include "../perf_event.h"
9
10 /* LBR Branch Select valid bits */
11 #define LBR_SELECT_MASK 0x1ff
12
13 /*
14 * LBR Branch Select filter bits which when set, ensures that the
15 * corresponding type of branches are not recorded
16 */
17 #define LBR_SELECT_KERNEL 0 /* Branches ending in CPL = 0 */
18 #define LBR_SELECT_USER 1 /* Branches ending in CPL > 0 */
19 #define LBR_SELECT_JCC 2 /* Conditional branches */
20 #define LBR_SELECT_CALL_NEAR_REL 3 /* Near relative calls */
21 #define LBR_SELECT_CALL_NEAR_IND 4 /* Indirect relative calls */
22 #define LBR_SELECT_RET_NEAR 5 /* Near returns */
23 #define LBR_SELECT_JMP_NEAR_IND 6 /* Near indirect jumps (excl. calls and returns) */
24 #define LBR_SELECT_JMP_NEAR_REL 7 /* Near relative jumps (excl. calls) */
25 #define LBR_SELECT_FAR_BRANCH 8 /* Far branches */
26
27 #define LBR_KERNEL BIT(LBR_SELECT_KERNEL)
28 #define LBR_USER BIT(LBR_SELECT_USER)
29 #define LBR_JCC BIT(LBR_SELECT_JCC)
30 #define LBR_REL_CALL BIT(LBR_SELECT_CALL_NEAR_REL)
31 #define LBR_IND_CALL BIT(LBR_SELECT_CALL_NEAR_IND)
32 #define LBR_RETURN BIT(LBR_SELECT_RET_NEAR)
33 #define LBR_REL_JMP BIT(LBR_SELECT_JMP_NEAR_REL)
34 #define LBR_IND_JMP BIT(LBR_SELECT_JMP_NEAR_IND)
35 #define LBR_FAR BIT(LBR_SELECT_FAR_BRANCH)
36 #define LBR_NOT_SUPP -1 /* unsupported filter */
37 #define LBR_IGNORE 0
38
39 #define LBR_ANY \
40 (LBR_JCC | LBR_REL_CALL | LBR_IND_CALL | LBR_RETURN | \
41 LBR_REL_JMP | LBR_IND_JMP | LBR_FAR)
42
43 struct branch_entry {
44 union {
45 struct {
46 u64 ip:58;
47 u64 ip_sign_ext:5;
48 u64 mispredict:1;
49 } split;
50 u64 full;
51 } from;
52
53 union {
54 struct {
55 u64 ip:58;
56 u64 ip_sign_ext:3;
57 u64 reserved:1;
58 u64 spec:1;
59 u64 valid:1;
60 } split;
61 u64 full;
62 } to;
63 };
64
amd_pmu_lbr_set_from(unsigned int idx,u64 val)65 static __always_inline void amd_pmu_lbr_set_from(unsigned int idx, u64 val)
66 {
67 wrmsrq(MSR_AMD_SAMP_BR_FROM + idx * 2, val);
68 }
69
amd_pmu_lbr_set_to(unsigned int idx,u64 val)70 static __always_inline void amd_pmu_lbr_set_to(unsigned int idx, u64 val)
71 {
72 wrmsrq(MSR_AMD_SAMP_BR_FROM + idx * 2 + 1, val);
73 }
74
amd_pmu_lbr_get_from(unsigned int idx)75 static __always_inline u64 amd_pmu_lbr_get_from(unsigned int idx)
76 {
77 u64 val;
78
79 rdmsrq(MSR_AMD_SAMP_BR_FROM + idx * 2, val);
80
81 return val;
82 }
83
amd_pmu_lbr_get_to(unsigned int idx)84 static __always_inline u64 amd_pmu_lbr_get_to(unsigned int idx)
85 {
86 u64 val;
87
88 rdmsrq(MSR_AMD_SAMP_BR_FROM + idx * 2 + 1, val);
89
90 return val;
91 }
92
sign_ext_branch_ip(u64 ip)93 static __always_inline u64 sign_ext_branch_ip(u64 ip)
94 {
95 u32 shift = 64 - boot_cpu_data.x86_virt_bits;
96
97 return (u64)(((s64)ip << shift) >> shift);
98 }
99
amd_pmu_lbr_filter(void)100 static void amd_pmu_lbr_filter(void)
101 {
102 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
103 int br_sel = cpuc->br_sel, offset, type, i, j;
104 bool compress = false;
105 bool fused_only = false;
106 u64 from, to;
107
108 /* If sampling all branches, there is nothing to filter */
109 if (((br_sel & X86_BR_ALL) == X86_BR_ALL) &&
110 ((br_sel & X86_BR_TYPE_SAVE) != X86_BR_TYPE_SAVE))
111 fused_only = true;
112
113 for (i = 0; i < cpuc->lbr_stack.nr; i++) {
114 from = cpuc->lbr_entries[i].from;
115 to = cpuc->lbr_entries[i].to;
116 type = branch_type_fused(from, to, 0, &offset);
117
118 /*
119 * Adjust the branch from address in case of instruction
120 * fusion where it points to an instruction preceding the
121 * actual branch
122 */
123 if (offset) {
124 cpuc->lbr_entries[i].from += offset;
125 if (fused_only)
126 continue;
127 }
128
129 /* If type does not correspond, then discard */
130 if (type == X86_BR_NONE || (br_sel & type) != type ||
131 (!(br_sel & X86_BR_KERNEL) && kernel_ip(cpuc->lbr_entries[i].from))) {
132 cpuc->lbr_entries[i].from = 0; /* mark invalid */
133 compress = true;
134 }
135
136 if ((br_sel & X86_BR_TYPE_SAVE) == X86_BR_TYPE_SAVE)
137 cpuc->lbr_entries[i].type = common_branch_type(type);
138 }
139
140 if (!compress)
141 return;
142
143 /* Remove all invalid entries */
144 for (i = 0; i < cpuc->lbr_stack.nr; ) {
145 if (!cpuc->lbr_entries[i].from) {
146 j = i;
147 while (++j < cpuc->lbr_stack.nr)
148 cpuc->lbr_entries[j - 1] = cpuc->lbr_entries[j];
149 cpuc->lbr_stack.nr--;
150 if (!cpuc->lbr_entries[i].from)
151 continue;
152 }
153 i++;
154 }
155 }
156
157 static const int lbr_spec_map[PERF_BR_SPEC_MAX] = {
158 PERF_BR_SPEC_NA,
159 PERF_BR_SPEC_WRONG_PATH,
160 PERF_BR_NON_SPEC_CORRECT_PATH,
161 PERF_BR_SPEC_CORRECT_PATH,
162 };
163
amd_pmu_lbr_read(void)164 void amd_pmu_lbr_read(void)
165 {
166 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
167 struct perf_branch_entry *br = cpuc->lbr_entries;
168 struct branch_entry entry;
169 int out = 0, idx, i;
170
171 if (!cpuc->lbr_users)
172 return;
173
174 for (i = 0; i < x86_pmu.lbr_nr; i++) {
175 entry.from.full = amd_pmu_lbr_get_from(i);
176 entry.to.full = amd_pmu_lbr_get_to(i);
177
178 /*
179 * Check if a branch has been logged; if valid = 0, spec = 0
180 * then no branch was recorded; if reserved = 1 then an
181 * erroneous branch was recorded (see Erratum 1452)
182 */
183 if ((!entry.to.split.valid && !entry.to.split.spec) ||
184 entry.to.split.reserved)
185 continue;
186
187 perf_clear_branch_entry_bitfields(br + out);
188
189 br[out].from = sign_ext_branch_ip(entry.from.split.ip);
190 br[out].to = sign_ext_branch_ip(entry.to.split.ip);
191 br[out].mispred = entry.from.split.mispredict;
192 br[out].predicted = !br[out].mispred;
193
194 /*
195 * Set branch speculation information using the status of
196 * the valid and spec bits.
197 *
198 * When valid = 0, spec = 0, no branch was recorded and the
199 * entry is discarded as seen above.
200 *
201 * When valid = 0, spec = 1, the recorded branch was
202 * speculative but took the wrong path.
203 *
204 * When valid = 1, spec = 0, the recorded branch was
205 * non-speculative but took the correct path.
206 *
207 * When valid = 1, spec = 1, the recorded branch was
208 * speculative and took the correct path
209 */
210 idx = (entry.to.split.valid << 1) | entry.to.split.spec;
211 br[out].spec = lbr_spec_map[idx];
212 out++;
213 }
214
215 cpuc->lbr_stack.nr = out;
216
217 /*
218 * Internal register renaming always ensures that LBR From[0] and
219 * LBR To[0] always represent the TOS
220 */
221 cpuc->lbr_stack.hw_idx = 0;
222
223 /* Perform further software filtering */
224 amd_pmu_lbr_filter();
225 }
226
227 static const int lbr_select_map[PERF_SAMPLE_BRANCH_MAX_SHIFT] = {
228 [PERF_SAMPLE_BRANCH_USER_SHIFT] = LBR_USER,
229 [PERF_SAMPLE_BRANCH_KERNEL_SHIFT] = LBR_KERNEL,
230 [PERF_SAMPLE_BRANCH_HV_SHIFT] = LBR_IGNORE,
231
232 [PERF_SAMPLE_BRANCH_ANY_SHIFT] = LBR_ANY,
233 [PERF_SAMPLE_BRANCH_ANY_CALL_SHIFT] = LBR_REL_CALL | LBR_IND_CALL | LBR_FAR,
234 [PERF_SAMPLE_BRANCH_ANY_RETURN_SHIFT] = LBR_RETURN | LBR_FAR,
235 [PERF_SAMPLE_BRANCH_IND_CALL_SHIFT] = LBR_IND_CALL,
236 [PERF_SAMPLE_BRANCH_ABORT_TX_SHIFT] = LBR_NOT_SUPP,
237 [PERF_SAMPLE_BRANCH_IN_TX_SHIFT] = LBR_NOT_SUPP,
238 [PERF_SAMPLE_BRANCH_NO_TX_SHIFT] = LBR_NOT_SUPP,
239 [PERF_SAMPLE_BRANCH_COND_SHIFT] = LBR_JCC,
240
241 [PERF_SAMPLE_BRANCH_CALL_STACK_SHIFT] = LBR_NOT_SUPP,
242 [PERF_SAMPLE_BRANCH_IND_JUMP_SHIFT] = LBR_IND_JMP,
243 [PERF_SAMPLE_BRANCH_CALL_SHIFT] = LBR_REL_CALL,
244
245 [PERF_SAMPLE_BRANCH_NO_FLAGS_SHIFT] = LBR_NOT_SUPP,
246 [PERF_SAMPLE_BRANCH_NO_CYCLES_SHIFT] = LBR_NOT_SUPP,
247 };
248
amd_pmu_lbr_setup_filter(struct perf_event * event)249 static int amd_pmu_lbr_setup_filter(struct perf_event *event)
250 {
251 struct hw_perf_event_extra *reg = &event->hw.branch_reg;
252 u64 br_type = event->attr.branch_sample_type;
253 u64 mask = 0, v;
254 int i;
255
256 /* No LBR support */
257 if (!x86_pmu.lbr_nr)
258 return -EOPNOTSUPP;
259
260 if (br_type & PERF_SAMPLE_BRANCH_USER)
261 mask |= X86_BR_USER;
262
263 if (br_type & PERF_SAMPLE_BRANCH_KERNEL)
264 mask |= X86_BR_KERNEL;
265
266 /* Ignore BRANCH_HV here */
267
268 if (br_type & PERF_SAMPLE_BRANCH_ANY)
269 mask |= X86_BR_ANY;
270
271 if (br_type & PERF_SAMPLE_BRANCH_ANY_CALL)
272 mask |= X86_BR_ANY_CALL;
273
274 if (br_type & PERF_SAMPLE_BRANCH_ANY_RETURN)
275 mask |= X86_BR_RET | X86_BR_IRET | X86_BR_SYSRET;
276
277 if (br_type & PERF_SAMPLE_BRANCH_IND_CALL)
278 mask |= X86_BR_IND_CALL;
279
280 if (br_type & PERF_SAMPLE_BRANCH_COND)
281 mask |= X86_BR_JCC;
282
283 if (br_type & PERF_SAMPLE_BRANCH_IND_JUMP)
284 mask |= X86_BR_IND_JMP;
285
286 if (br_type & PERF_SAMPLE_BRANCH_CALL)
287 mask |= X86_BR_CALL | X86_BR_ZERO_CALL;
288
289 if (br_type & PERF_SAMPLE_BRANCH_TYPE_SAVE)
290 mask |= X86_BR_TYPE_SAVE;
291
292 reg->reg = mask;
293 mask = 0;
294
295 for (i = 0; i < PERF_SAMPLE_BRANCH_MAX_SHIFT; i++) {
296 if (!(br_type & BIT_ULL(i)))
297 continue;
298
299 v = lbr_select_map[i];
300 if (v == LBR_NOT_SUPP)
301 return -EOPNOTSUPP;
302
303 if (v != LBR_IGNORE)
304 mask |= v;
305 }
306
307 /* Filter bits operate in suppress mode */
308 reg->config = mask ^ LBR_SELECT_MASK;
309
310 return 0;
311 }
312
amd_pmu_lbr_hw_config(struct perf_event * event)313 int amd_pmu_lbr_hw_config(struct perf_event *event)
314 {
315 int ret = 0;
316
317 ret = amd_pmu_lbr_setup_filter(event);
318 if (!ret)
319 event->attach_state |= PERF_ATTACH_SCHED_CB;
320
321 return ret;
322 }
323
amd_pmu_lbr_reset(void)324 void amd_pmu_lbr_reset(void)
325 {
326 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
327 int i;
328
329 if (!x86_pmu.lbr_nr)
330 return;
331
332 /* Reset all branch records individually */
333 for (i = 0; i < x86_pmu.lbr_nr; i++) {
334 amd_pmu_lbr_set_from(i, 0);
335 amd_pmu_lbr_set_to(i, 0);
336 }
337
338 cpuc->last_task_ctx = NULL;
339 cpuc->last_log_id = 0;
340 wrmsrq(MSR_AMD64_LBR_SELECT, 0);
341 }
342
amd_pmu_lbr_add(struct perf_event * event)343 void amd_pmu_lbr_add(struct perf_event *event)
344 {
345 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
346 struct hw_perf_event_extra *reg = &event->hw.branch_reg;
347
348 if (!x86_pmu.lbr_nr)
349 return;
350
351 if (has_branch_stack(event)) {
352 cpuc->lbr_select = 1;
353 cpuc->lbr_sel->config = reg->config;
354 cpuc->br_sel = reg->reg;
355 }
356
357 perf_sched_cb_inc(event->pmu);
358
359 if (!cpuc->lbr_users++ && !event->total_time_running)
360 amd_pmu_lbr_reset();
361 }
362
amd_pmu_lbr_del(struct perf_event * event)363 void amd_pmu_lbr_del(struct perf_event *event)
364 {
365 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
366
367 if (!x86_pmu.lbr_nr)
368 return;
369
370 if (has_branch_stack(event))
371 cpuc->lbr_select = 0;
372
373 cpuc->lbr_users--;
374 WARN_ON_ONCE(cpuc->lbr_users < 0);
375 perf_sched_cb_dec(event->pmu);
376 }
377
amd_pmu_lbr_sched_task(struct perf_event_pmu_context * pmu_ctx,struct task_struct * task,bool sched_in)378 void amd_pmu_lbr_sched_task(struct perf_event_pmu_context *pmu_ctx,
379 struct task_struct *task, bool sched_in)
380 {
381 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
382
383 /*
384 * A context switch can flip the address space and LBR entries are
385 * not tagged with an identifier. Hence, branches cannot be resolved
386 * from the old address space and the LBR records should be wiped.
387 */
388 if (cpuc->lbr_users && sched_in)
389 amd_pmu_lbr_reset();
390 }
391
amd_pmu_lbr_enable_all(void)392 void amd_pmu_lbr_enable_all(void)
393 {
394 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
395 u64 lbr_select, dbg_ctl, dbg_extn_cfg;
396
397 if (!cpuc->lbr_users || !x86_pmu.lbr_nr)
398 return;
399
400 /* Set hardware branch filter */
401 if (cpuc->lbr_select) {
402 lbr_select = cpuc->lbr_sel->config & LBR_SELECT_MASK;
403 wrmsrq(MSR_AMD64_LBR_SELECT, lbr_select);
404 }
405
406 if (cpu_feature_enabled(X86_FEATURE_AMD_LBR_PMC_FREEZE)) {
407 rdmsrq(MSR_IA32_DEBUGCTLMSR, dbg_ctl);
408 wrmsrq(MSR_IA32_DEBUGCTLMSR, dbg_ctl | DEBUGCTLMSR_FREEZE_LBRS_ON_PMI);
409 }
410
411 rdmsrq(MSR_AMD_DBG_EXTN_CFG, dbg_extn_cfg);
412 wrmsrq(MSR_AMD_DBG_EXTN_CFG, dbg_extn_cfg | DBG_EXTN_CFG_LBRV2EN);
413 }
414
amd_pmu_lbr_disable_all(void)415 void amd_pmu_lbr_disable_all(void)
416 {
417 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
418
419 if (!cpuc->lbr_users || !x86_pmu.lbr_nr)
420 return;
421
422 __amd_pmu_lbr_disable();
423 }
424
amd_pmu_lbr_init(void)425 __init int amd_pmu_lbr_init(void)
426 {
427 union cpuid_0x80000022_ebx ebx;
428
429 if (x86_pmu.version < 2 || !boot_cpu_has(X86_FEATURE_AMD_LBR_V2))
430 return -EOPNOTSUPP;
431
432 /* Set number of entries */
433 ebx.full = cpuid_ebx(EXT_PERFMON_DEBUG_FEATURES);
434 x86_pmu.lbr_nr = ebx.split.lbr_v2_stack_sz;
435
436 pr_cont("%d-deep LBR, ", x86_pmu.lbr_nr);
437
438 return 0;
439 }
440