1 // SPDX-License-Identifier: GPL-2.0
2 #include <linux/bitops.h>
3 #include <linux/types.h>
4 #include <linux/slab.h>
5 #include <linux/sched/clock.h>
6
7 #include <asm/cpu_entry_area.h>
8 #include <asm/debugreg.h>
9 #include <asm/perf_event.h>
10 #include <asm/tlbflush.h>
11 #include <asm/insn.h>
12 #include <asm/io.h>
13 #include <asm/msr.h>
14 #include <asm/timer.h>
15
16 #include "../perf_event.h"
17
18 /* Waste a full page so it can be mapped into the cpu_entry_area */
19 DEFINE_PER_CPU_PAGE_ALIGNED(struct debug_store, cpu_debug_store);
20
21 /* The size of a BTS record in bytes: */
22 #define BTS_RECORD_SIZE 24
23
24 #define PEBS_FIXUP_SIZE PAGE_SIZE
25
26 /*
27 * pebs_record_32 for p4 and core not supported
28
29 struct pebs_record_32 {
30 u32 flags, ip;
31 u32 ax, bc, cx, dx;
32 u32 si, di, bp, sp;
33 };
34
35 */
36
37 union intel_x86_pebs_dse {
38 u64 val;
39 struct {
40 unsigned int ld_dse:4;
41 unsigned int ld_stlb_miss:1;
42 unsigned int ld_locked:1;
43 unsigned int ld_data_blk:1;
44 unsigned int ld_addr_blk:1;
45 unsigned int ld_reserved:24;
46 };
47 struct {
48 unsigned int st_l1d_hit:1;
49 unsigned int st_reserved1:3;
50 unsigned int st_stlb_miss:1;
51 unsigned int st_locked:1;
52 unsigned int st_reserved2:26;
53 };
54 struct {
55 unsigned int st_lat_dse:4;
56 unsigned int st_lat_stlb_miss:1;
57 unsigned int st_lat_locked:1;
58 unsigned int ld_reserved3:26;
59 };
60 struct {
61 unsigned int mtl_dse:5;
62 unsigned int mtl_locked:1;
63 unsigned int mtl_stlb_miss:1;
64 unsigned int mtl_fwd_blk:1;
65 unsigned int ld_reserved4:24;
66 };
67 struct {
68 unsigned int lnc_dse:8;
69 unsigned int ld_reserved5:2;
70 unsigned int lnc_stlb_miss:1;
71 unsigned int lnc_locked:1;
72 unsigned int lnc_data_blk:1;
73 unsigned int lnc_addr_blk:1;
74 unsigned int ld_reserved6:18;
75 };
76 };
77
78
79 /*
80 * Map PEBS Load Latency Data Source encodings to generic
81 * memory data source information
82 */
83 #define P(a, b) PERF_MEM_S(a, b)
84 #define OP_LH (P(OP, LOAD) | P(LVL, HIT))
85 #define LEVEL(x) P(LVLNUM, x)
86 #define REM P(REMOTE, REMOTE)
87 #define SNOOP_NONE_MISS (P(SNOOP, NONE) | P(SNOOP, MISS))
88
89 /* Version for Sandy Bridge and later */
90 static u64 pebs_data_source[PERF_PEBS_DATA_SOURCE_MAX] = {
91 P(OP, LOAD) | P(LVL, MISS) | LEVEL(L3) | P(SNOOP, NA),/* 0x00:ukn L3 */
92 OP_LH | P(LVL, L1) | LEVEL(L1) | P(SNOOP, NONE), /* 0x01: L1 local */
93 OP_LH | P(LVL, LFB) | LEVEL(LFB) | P(SNOOP, NONE), /* 0x02: LFB hit */
94 OP_LH | P(LVL, L2) | LEVEL(L2) | P(SNOOP, NONE), /* 0x03: L2 hit */
95 OP_LH | P(LVL, L3) | LEVEL(L3) | P(SNOOP, NONE), /* 0x04: L3 hit */
96 OP_LH | P(LVL, L3) | LEVEL(L3) | P(SNOOP, MISS), /* 0x05: L3 hit, snoop miss */
97 OP_LH | P(LVL, L3) | LEVEL(L3) | P(SNOOP, HIT), /* 0x06: L3 hit, snoop hit */
98 OP_LH | P(LVL, L3) | LEVEL(L3) | P(SNOOP, HITM), /* 0x07: L3 hit, snoop hitm */
99 OP_LH | P(LVL, REM_CCE1) | REM | LEVEL(L3) | P(SNOOP, HIT), /* 0x08: L3 miss snoop hit */
100 OP_LH | P(LVL, REM_CCE1) | REM | LEVEL(L3) | P(SNOOP, HITM), /* 0x09: L3 miss snoop hitm*/
101 OP_LH | P(LVL, LOC_RAM) | LEVEL(RAM) | P(SNOOP, HIT), /* 0x0a: L3 miss, shared */
102 OP_LH | P(LVL, REM_RAM1) | REM | LEVEL(L3) | P(SNOOP, HIT), /* 0x0b: L3 miss, shared */
103 OP_LH | P(LVL, LOC_RAM) | LEVEL(RAM) | SNOOP_NONE_MISS, /* 0x0c: L3 miss, excl */
104 OP_LH | P(LVL, REM_RAM1) | LEVEL(RAM) | REM | SNOOP_NONE_MISS, /* 0x0d: L3 miss, excl */
105 OP_LH | P(LVL, IO) | LEVEL(NA) | P(SNOOP, NONE), /* 0x0e: I/O */
106 OP_LH | P(LVL, UNC) | LEVEL(NA) | P(SNOOP, NONE), /* 0x0f: uncached */
107 };
108
109 /* Patch up minor differences in the bits */
intel_pmu_pebs_data_source_nhm(void)110 void __init intel_pmu_pebs_data_source_nhm(void)
111 {
112 pebs_data_source[0x05] = OP_LH | P(LVL, L3) | LEVEL(L3) | P(SNOOP, HIT);
113 pebs_data_source[0x06] = OP_LH | P(LVL, L3) | LEVEL(L3) | P(SNOOP, HITM);
114 pebs_data_source[0x07] = OP_LH | P(LVL, L3) | LEVEL(L3) | P(SNOOP, HITM);
115 }
116
__intel_pmu_pebs_data_source_skl(bool pmem,u64 * data_source)117 static void __init __intel_pmu_pebs_data_source_skl(bool pmem, u64 *data_source)
118 {
119 u64 pmem_or_l4 = pmem ? LEVEL(PMEM) : LEVEL(L4);
120
121 data_source[0x08] = OP_LH | pmem_or_l4 | P(SNOOP, HIT);
122 data_source[0x09] = OP_LH | pmem_or_l4 | REM | P(SNOOP, HIT);
123 data_source[0x0b] = OP_LH | LEVEL(RAM) | REM | P(SNOOP, NONE);
124 data_source[0x0c] = OP_LH | LEVEL(ANY_CACHE) | REM | P(SNOOPX, FWD);
125 data_source[0x0d] = OP_LH | LEVEL(ANY_CACHE) | REM | P(SNOOP, HITM);
126 }
127
intel_pmu_pebs_data_source_skl(bool pmem)128 void __init intel_pmu_pebs_data_source_skl(bool pmem)
129 {
130 __intel_pmu_pebs_data_source_skl(pmem, pebs_data_source);
131 }
132
__intel_pmu_pebs_data_source_grt(u64 * data_source)133 static void __init __intel_pmu_pebs_data_source_grt(u64 *data_source)
134 {
135 data_source[0x05] = OP_LH | P(LVL, L3) | LEVEL(L3) | P(SNOOP, HIT);
136 data_source[0x06] = OP_LH | P(LVL, L3) | LEVEL(L3) | P(SNOOP, HITM);
137 data_source[0x08] = OP_LH | P(LVL, L3) | LEVEL(L3) | P(SNOOPX, FWD);
138 }
139
intel_pmu_pebs_data_source_grt(void)140 void __init intel_pmu_pebs_data_source_grt(void)
141 {
142 __intel_pmu_pebs_data_source_grt(pebs_data_source);
143 }
144
intel_pmu_pebs_data_source_adl(void)145 void __init intel_pmu_pebs_data_source_adl(void)
146 {
147 u64 *data_source;
148
149 data_source = x86_pmu.hybrid_pmu[X86_HYBRID_PMU_CORE_IDX].pebs_data_source;
150 memcpy(data_source, pebs_data_source, sizeof(pebs_data_source));
151 __intel_pmu_pebs_data_source_skl(false, data_source);
152
153 data_source = x86_pmu.hybrid_pmu[X86_HYBRID_PMU_ATOM_IDX].pebs_data_source;
154 memcpy(data_source, pebs_data_source, sizeof(pebs_data_source));
155 __intel_pmu_pebs_data_source_grt(data_source);
156 }
157
__intel_pmu_pebs_data_source_cmt(u64 * data_source)158 static void __init __intel_pmu_pebs_data_source_cmt(u64 *data_source)
159 {
160 data_source[0x07] = OP_LH | P(LVL, L3) | LEVEL(L3) | P(SNOOPX, FWD);
161 data_source[0x08] = OP_LH | P(LVL, L3) | LEVEL(L3) | P(SNOOP, HITM);
162 data_source[0x0a] = OP_LH | P(LVL, LOC_RAM) | LEVEL(RAM) | P(SNOOP, NONE);
163 data_source[0x0b] = OP_LH | LEVEL(RAM) | REM | P(SNOOP, NONE);
164 data_source[0x0c] = OP_LH | LEVEL(RAM) | REM | P(SNOOPX, FWD);
165 data_source[0x0d] = OP_LH | LEVEL(RAM) | REM | P(SNOOP, HITM);
166 }
167
intel_pmu_pebs_data_source_mtl(void)168 void __init intel_pmu_pebs_data_source_mtl(void)
169 {
170 u64 *data_source;
171
172 data_source = x86_pmu.hybrid_pmu[X86_HYBRID_PMU_CORE_IDX].pebs_data_source;
173 memcpy(data_source, pebs_data_source, sizeof(pebs_data_source));
174 __intel_pmu_pebs_data_source_skl(false, data_source);
175
176 data_source = x86_pmu.hybrid_pmu[X86_HYBRID_PMU_ATOM_IDX].pebs_data_source;
177 memcpy(data_source, pebs_data_source, sizeof(pebs_data_source));
178 __intel_pmu_pebs_data_source_cmt(data_source);
179 }
180
intel_pmu_pebs_data_source_arl_h(void)181 void __init intel_pmu_pebs_data_source_arl_h(void)
182 {
183 u64 *data_source;
184
185 intel_pmu_pebs_data_source_lnl();
186
187 data_source = x86_pmu.hybrid_pmu[X86_HYBRID_PMU_TINY_IDX].pebs_data_source;
188 memcpy(data_source, pebs_data_source, sizeof(pebs_data_source));
189 __intel_pmu_pebs_data_source_cmt(data_source);
190 }
191
intel_pmu_pebs_data_source_cmt(void)192 void __init intel_pmu_pebs_data_source_cmt(void)
193 {
194 __intel_pmu_pebs_data_source_cmt(pebs_data_source);
195 }
196
197 /* Version for Lion Cove and later */
198 static u64 lnc_pebs_data_source[PERF_PEBS_DATA_SOURCE_MAX] = {
199 P(OP, LOAD) | P(LVL, MISS) | LEVEL(L3) | P(SNOOP, NA), /* 0x00: ukn L3 */
200 OP_LH | P(LVL, L1) | LEVEL(L1) | P(SNOOP, NONE), /* 0x01: L1 hit */
201 OP_LH | P(LVL, L1) | LEVEL(L1) | P(SNOOP, NONE), /* 0x02: L1 hit */
202 OP_LH | P(LVL, LFB) | LEVEL(LFB) | P(SNOOP, NONE), /* 0x03: LFB/L1 Miss Handling Buffer hit */
203 0, /* 0x04: Reserved */
204 OP_LH | P(LVL, L2) | LEVEL(L2) | P(SNOOP, NONE), /* 0x05: L2 Hit */
205 OP_LH | LEVEL(L2_MHB) | P(SNOOP, NONE), /* 0x06: L2 Miss Handling Buffer Hit */
206 0, /* 0x07: Reserved */
207 OP_LH | P(LVL, L3) | LEVEL(L3) | P(SNOOP, NONE), /* 0x08: L3 Hit */
208 0, /* 0x09: Reserved */
209 0, /* 0x0a: Reserved */
210 0, /* 0x0b: Reserved */
211 OP_LH | P(LVL, L3) | LEVEL(L3) | P(SNOOPX, FWD), /* 0x0c: L3 Hit Snoop Fwd */
212 OP_LH | P(LVL, L3) | LEVEL(L3) | P(SNOOP, HITM), /* 0x0d: L3 Hit Snoop HitM */
213 0, /* 0x0e: Reserved */
214 P(OP, LOAD) | P(LVL, MISS) | P(LVL, L3) | LEVEL(L3) | P(SNOOP, HITM), /* 0x0f: L3 Miss Snoop HitM */
215 OP_LH | LEVEL(MSC) | P(SNOOP, NONE), /* 0x10: Memory-side Cache Hit */
216 OP_LH | P(LVL, LOC_RAM) | LEVEL(RAM) | P(SNOOP, NONE), /* 0x11: Local Memory Hit */
217 };
218
intel_pmu_pebs_data_source_lnl(void)219 void __init intel_pmu_pebs_data_source_lnl(void)
220 {
221 u64 *data_source;
222
223 data_source = x86_pmu.hybrid_pmu[X86_HYBRID_PMU_CORE_IDX].pebs_data_source;
224 memcpy(data_source, lnc_pebs_data_source, sizeof(lnc_pebs_data_source));
225
226 data_source = x86_pmu.hybrid_pmu[X86_HYBRID_PMU_ATOM_IDX].pebs_data_source;
227 memcpy(data_source, pebs_data_source, sizeof(pebs_data_source));
228 __intel_pmu_pebs_data_source_cmt(data_source);
229 }
230
precise_store_data(u64 status)231 static u64 precise_store_data(u64 status)
232 {
233 union intel_x86_pebs_dse dse;
234 u64 val = P(OP, STORE) | P(SNOOP, NA) | P(LVL, L1) | P(TLB, L2);
235
236 dse.val = status;
237
238 /*
239 * bit 4: TLB access
240 * 1 = stored missed 2nd level TLB
241 *
242 * so it either hit the walker or the OS
243 * otherwise hit 2nd level TLB
244 */
245 if (dse.st_stlb_miss)
246 val |= P(TLB, MISS);
247 else
248 val |= P(TLB, HIT);
249
250 /*
251 * bit 0: hit L1 data cache
252 * if not set, then all we know is that
253 * it missed L1D
254 */
255 if (dse.st_l1d_hit)
256 val |= P(LVL, HIT);
257 else
258 val |= P(LVL, MISS);
259
260 /*
261 * bit 5: Locked prefix
262 */
263 if (dse.st_locked)
264 val |= P(LOCK, LOCKED);
265
266 return val;
267 }
268
precise_datala_hsw(struct perf_event * event,u64 status)269 static u64 precise_datala_hsw(struct perf_event *event, u64 status)
270 {
271 union perf_mem_data_src dse;
272
273 dse.val = PERF_MEM_NA;
274
275 if (event->hw.flags & PERF_X86_EVENT_PEBS_ST_HSW)
276 dse.mem_op = PERF_MEM_OP_STORE;
277 else if (event->hw.flags & PERF_X86_EVENT_PEBS_LD_HSW)
278 dse.mem_op = PERF_MEM_OP_LOAD;
279
280 /*
281 * L1 info only valid for following events:
282 *
283 * MEM_UOPS_RETIRED.STLB_MISS_STORES
284 * MEM_UOPS_RETIRED.LOCK_STORES
285 * MEM_UOPS_RETIRED.SPLIT_STORES
286 * MEM_UOPS_RETIRED.ALL_STORES
287 */
288 if (event->hw.flags & PERF_X86_EVENT_PEBS_ST_HSW) {
289 if (status & 1)
290 dse.mem_lvl = PERF_MEM_LVL_L1 | PERF_MEM_LVL_HIT;
291 else
292 dse.mem_lvl = PERF_MEM_LVL_L1 | PERF_MEM_LVL_MISS;
293 }
294 return dse.val;
295 }
296
pebs_set_tlb_lock(u64 * val,bool tlb,bool lock)297 static inline void pebs_set_tlb_lock(u64 *val, bool tlb, bool lock)
298 {
299 /*
300 * TLB access
301 * 0 = did not miss 2nd level TLB
302 * 1 = missed 2nd level TLB
303 */
304 if (tlb)
305 *val |= P(TLB, MISS) | P(TLB, L2);
306 else
307 *val |= P(TLB, HIT) | P(TLB, L1) | P(TLB, L2);
308
309 /* locked prefix */
310 if (lock)
311 *val |= P(LOCK, LOCKED);
312 }
313
314 /* Retrieve the latency data for e-core of ADL */
__grt_latency_data(struct perf_event * event,u64 status,u8 dse,bool tlb,bool lock,bool blk)315 static u64 __grt_latency_data(struct perf_event *event, u64 status,
316 u8 dse, bool tlb, bool lock, bool blk)
317 {
318 u64 val;
319
320 WARN_ON_ONCE(is_hybrid() &&
321 hybrid_pmu(event->pmu)->pmu_type == hybrid_big);
322
323 dse &= PERF_PEBS_DATA_SOURCE_GRT_MASK;
324 val = hybrid_var(event->pmu, pebs_data_source)[dse];
325
326 pebs_set_tlb_lock(&val, tlb, lock);
327
328 if (blk)
329 val |= P(BLK, DATA);
330 else
331 val |= P(BLK, NA);
332
333 return val;
334 }
335
grt_latency_data(struct perf_event * event,u64 status)336 u64 grt_latency_data(struct perf_event *event, u64 status)
337 {
338 union intel_x86_pebs_dse dse;
339
340 dse.val = status;
341
342 return __grt_latency_data(event, status, dse.ld_dse,
343 dse.ld_locked, dse.ld_stlb_miss,
344 dse.ld_data_blk);
345 }
346
347 /* Retrieve the latency data for e-core of MTL */
cmt_latency_data(struct perf_event * event,u64 status)348 u64 cmt_latency_data(struct perf_event *event, u64 status)
349 {
350 union intel_x86_pebs_dse dse;
351
352 dse.val = status;
353
354 return __grt_latency_data(event, status, dse.mtl_dse,
355 dse.mtl_stlb_miss, dse.mtl_locked,
356 dse.mtl_fwd_blk);
357 }
358
lnc_latency_data(struct perf_event * event,u64 status)359 static u64 lnc_latency_data(struct perf_event *event, u64 status)
360 {
361 union intel_x86_pebs_dse dse;
362 union perf_mem_data_src src;
363 u64 val;
364
365 dse.val = status;
366
367 /* LNC core latency data */
368 val = hybrid_var(event->pmu, pebs_data_source)[status & PERF_PEBS_DATA_SOURCE_MASK];
369 if (!val)
370 val = P(OP, LOAD) | LEVEL(NA) | P(SNOOP, NA);
371
372 if (dse.lnc_stlb_miss)
373 val |= P(TLB, MISS) | P(TLB, L2);
374 else
375 val |= P(TLB, HIT) | P(TLB, L1) | P(TLB, L2);
376
377 if (dse.lnc_locked)
378 val |= P(LOCK, LOCKED);
379
380 if (dse.lnc_data_blk)
381 val |= P(BLK, DATA);
382 if (dse.lnc_addr_blk)
383 val |= P(BLK, ADDR);
384 if (!dse.lnc_data_blk && !dse.lnc_addr_blk)
385 val |= P(BLK, NA);
386
387 src.val = val;
388 if (event->hw.flags & PERF_X86_EVENT_PEBS_ST_HSW)
389 src.mem_op = P(OP, STORE);
390
391 return src.val;
392 }
393
lnl_latency_data(struct perf_event * event,u64 status)394 u64 lnl_latency_data(struct perf_event *event, u64 status)
395 {
396 struct x86_hybrid_pmu *pmu = hybrid_pmu(event->pmu);
397
398 if (pmu->pmu_type == hybrid_small)
399 return cmt_latency_data(event, status);
400
401 return lnc_latency_data(event, status);
402 }
403
arl_h_latency_data(struct perf_event * event,u64 status)404 u64 arl_h_latency_data(struct perf_event *event, u64 status)
405 {
406 struct x86_hybrid_pmu *pmu = hybrid_pmu(event->pmu);
407
408 if (pmu->pmu_type == hybrid_tiny)
409 return cmt_latency_data(event, status);
410
411 return lnl_latency_data(event, status);
412 }
413
load_latency_data(struct perf_event * event,u64 status)414 static u64 load_latency_data(struct perf_event *event, u64 status)
415 {
416 union intel_x86_pebs_dse dse;
417 u64 val;
418
419 dse.val = status;
420
421 /*
422 * use the mapping table for bit 0-3
423 */
424 val = hybrid_var(event->pmu, pebs_data_source)[dse.ld_dse];
425
426 /*
427 * Nehalem models do not support TLB, Lock infos
428 */
429 if (x86_pmu.pebs_no_tlb) {
430 val |= P(TLB, NA) | P(LOCK, NA);
431 return val;
432 }
433
434 pebs_set_tlb_lock(&val, dse.ld_stlb_miss, dse.ld_locked);
435
436 /*
437 * Ice Lake and earlier models do not support block infos.
438 */
439 if (!x86_pmu.pebs_block) {
440 val |= P(BLK, NA);
441 return val;
442 }
443 /*
444 * bit 6: load was blocked since its data could not be forwarded
445 * from a preceding store
446 */
447 if (dse.ld_data_blk)
448 val |= P(BLK, DATA);
449
450 /*
451 * bit 7: load was blocked due to potential address conflict with
452 * a preceding store
453 */
454 if (dse.ld_addr_blk)
455 val |= P(BLK, ADDR);
456
457 if (!dse.ld_data_blk && !dse.ld_addr_blk)
458 val |= P(BLK, NA);
459
460 return val;
461 }
462
store_latency_data(struct perf_event * event,u64 status)463 static u64 store_latency_data(struct perf_event *event, u64 status)
464 {
465 union intel_x86_pebs_dse dse;
466 union perf_mem_data_src src;
467 u64 val;
468
469 dse.val = status;
470
471 /*
472 * use the mapping table for bit 0-3
473 */
474 val = hybrid_var(event->pmu, pebs_data_source)[dse.st_lat_dse];
475
476 pebs_set_tlb_lock(&val, dse.st_lat_stlb_miss, dse.st_lat_locked);
477
478 val |= P(BLK, NA);
479
480 /*
481 * the pebs_data_source table is only for loads
482 * so override the mem_op to say STORE instead
483 */
484 src.val = val;
485 src.mem_op = P(OP,STORE);
486
487 return src.val;
488 }
489
490 struct pebs_record_core {
491 u64 flags, ip;
492 u64 ax, bx, cx, dx;
493 u64 si, di, bp, sp;
494 u64 r8, r9, r10, r11;
495 u64 r12, r13, r14, r15;
496 };
497
498 struct pebs_record_nhm {
499 u64 flags, ip;
500 u64 ax, bx, cx, dx;
501 u64 si, di, bp, sp;
502 u64 r8, r9, r10, r11;
503 u64 r12, r13, r14, r15;
504 u64 status, dla, dse, lat;
505 };
506
507 /*
508 * Same as pebs_record_nhm, with two additional fields.
509 */
510 struct pebs_record_hsw {
511 u64 flags, ip;
512 u64 ax, bx, cx, dx;
513 u64 si, di, bp, sp;
514 u64 r8, r9, r10, r11;
515 u64 r12, r13, r14, r15;
516 u64 status, dla, dse, lat;
517 u64 real_ip, tsx_tuning;
518 };
519
520 union hsw_tsx_tuning {
521 struct {
522 u32 cycles_last_block : 32,
523 hle_abort : 1,
524 rtm_abort : 1,
525 instruction_abort : 1,
526 non_instruction_abort : 1,
527 retry : 1,
528 data_conflict : 1,
529 capacity_writes : 1,
530 capacity_reads : 1;
531 };
532 u64 value;
533 };
534
535 #define PEBS_HSW_TSX_FLAGS 0xff00000000ULL
536
537 /* Same as HSW, plus TSC */
538
539 struct pebs_record_skl {
540 u64 flags, ip;
541 u64 ax, bx, cx, dx;
542 u64 si, di, bp, sp;
543 u64 r8, r9, r10, r11;
544 u64 r12, r13, r14, r15;
545 u64 status, dla, dse, lat;
546 u64 real_ip, tsx_tuning;
547 u64 tsc;
548 };
549
init_debug_store_on_cpu(int cpu)550 void init_debug_store_on_cpu(int cpu)
551 {
552 struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;
553
554 if (!ds)
555 return;
556
557 wrmsr_on_cpu(cpu, MSR_IA32_DS_AREA,
558 (u32)((u64)(unsigned long)ds),
559 (u32)((u64)(unsigned long)ds >> 32));
560 }
561
fini_debug_store_on_cpu(int cpu)562 void fini_debug_store_on_cpu(int cpu)
563 {
564 if (!per_cpu(cpu_hw_events, cpu).ds)
565 return;
566
567 wrmsr_on_cpu(cpu, MSR_IA32_DS_AREA, 0, 0);
568 }
569
570 static DEFINE_PER_CPU(void *, insn_buffer);
571
ds_update_cea(void * cea,void * addr,size_t size,pgprot_t prot)572 static void ds_update_cea(void *cea, void *addr, size_t size, pgprot_t prot)
573 {
574 unsigned long start = (unsigned long)cea;
575 phys_addr_t pa;
576 size_t msz = 0;
577
578 pa = virt_to_phys(addr);
579
580 preempt_disable();
581 for (; msz < size; msz += PAGE_SIZE, pa += PAGE_SIZE, cea += PAGE_SIZE)
582 cea_set_pte(cea, pa, prot);
583
584 /*
585 * This is a cross-CPU update of the cpu_entry_area, we must shoot down
586 * all TLB entries for it.
587 */
588 flush_tlb_kernel_range(start, start + size);
589 preempt_enable();
590 }
591
ds_clear_cea(void * cea,size_t size)592 static void ds_clear_cea(void *cea, size_t size)
593 {
594 unsigned long start = (unsigned long)cea;
595 size_t msz = 0;
596
597 preempt_disable();
598 for (; msz < size; msz += PAGE_SIZE, cea += PAGE_SIZE)
599 cea_set_pte(cea, 0, PAGE_NONE);
600
601 flush_tlb_kernel_range(start, start + size);
602 preempt_enable();
603 }
604
dsalloc_pages(size_t size,gfp_t flags,int cpu)605 static void *dsalloc_pages(size_t size, gfp_t flags, int cpu)
606 {
607 unsigned int order = get_order(size);
608 int node = cpu_to_node(cpu);
609 struct page *page;
610
611 page = __alloc_pages_node(node, flags | __GFP_ZERO, order);
612 return page ? page_address(page) : NULL;
613 }
614
dsfree_pages(const void * buffer,size_t size)615 static void dsfree_pages(const void *buffer, size_t size)
616 {
617 if (buffer)
618 free_pages((unsigned long)buffer, get_order(size));
619 }
620
alloc_pebs_buffer(int cpu)621 static int alloc_pebs_buffer(int cpu)
622 {
623 struct cpu_hw_events *hwev = per_cpu_ptr(&cpu_hw_events, cpu);
624 struct debug_store *ds = hwev->ds;
625 size_t bsiz = x86_pmu.pebs_buffer_size;
626 int max, node = cpu_to_node(cpu);
627 void *buffer, *insn_buff, *cea;
628
629 if (!intel_pmu_has_pebs())
630 return 0;
631
632 buffer = dsalloc_pages(bsiz, GFP_KERNEL, cpu);
633 if (unlikely(!buffer))
634 return -ENOMEM;
635
636 if (x86_pmu.arch_pebs) {
637 hwev->pebs_vaddr = buffer;
638 return 0;
639 }
640
641 /*
642 * HSW+ already provides us the eventing ip; no need to allocate this
643 * buffer then.
644 */
645 if (x86_pmu.intel_cap.pebs_format < 2) {
646 insn_buff = kzalloc_node(PEBS_FIXUP_SIZE, GFP_KERNEL, node);
647 if (!insn_buff) {
648 dsfree_pages(buffer, bsiz);
649 return -ENOMEM;
650 }
651 per_cpu(insn_buffer, cpu) = insn_buff;
652 }
653 hwev->pebs_vaddr = buffer;
654 /* Update the cpu entry area mapping */
655 cea = &get_cpu_entry_area(cpu)->cpu_debug_buffers.pebs_buffer;
656 ds->pebs_buffer_base = (unsigned long) cea;
657 ds_update_cea(cea, buffer, bsiz, PAGE_KERNEL);
658 ds->pebs_index = ds->pebs_buffer_base;
659 max = x86_pmu.pebs_record_size * (bsiz / x86_pmu.pebs_record_size);
660 ds->pebs_absolute_maximum = ds->pebs_buffer_base + max;
661 return 0;
662 }
663
release_pebs_buffer(int cpu)664 static void release_pebs_buffer(int cpu)
665 {
666 struct cpu_hw_events *hwev = per_cpu_ptr(&cpu_hw_events, cpu);
667 void *cea;
668
669 if (!intel_pmu_has_pebs())
670 return;
671
672 if (x86_pmu.ds_pebs) {
673 kfree(per_cpu(insn_buffer, cpu));
674 per_cpu(insn_buffer, cpu) = NULL;
675
676 /* Clear the fixmap */
677 cea = &get_cpu_entry_area(cpu)->cpu_debug_buffers.pebs_buffer;
678 ds_clear_cea(cea, x86_pmu.pebs_buffer_size);
679 }
680
681 dsfree_pages(hwev->pebs_vaddr, x86_pmu.pebs_buffer_size);
682 hwev->pebs_vaddr = NULL;
683 }
684
alloc_bts_buffer(int cpu)685 static int alloc_bts_buffer(int cpu)
686 {
687 struct cpu_hw_events *hwev = per_cpu_ptr(&cpu_hw_events, cpu);
688 struct debug_store *ds = hwev->ds;
689 void *buffer, *cea;
690 int max;
691
692 if (!x86_pmu.bts)
693 return 0;
694
695 buffer = dsalloc_pages(BTS_BUFFER_SIZE, GFP_KERNEL | __GFP_NOWARN, cpu);
696 if (unlikely(!buffer)) {
697 WARN_ONCE(1, "%s: BTS buffer allocation failure\n", __func__);
698 return -ENOMEM;
699 }
700 hwev->ds_bts_vaddr = buffer;
701 /* Update the fixmap */
702 cea = &get_cpu_entry_area(cpu)->cpu_debug_buffers.bts_buffer;
703 ds->bts_buffer_base = (unsigned long) cea;
704 ds_update_cea(cea, buffer, BTS_BUFFER_SIZE, PAGE_KERNEL);
705 ds->bts_index = ds->bts_buffer_base;
706 max = BTS_BUFFER_SIZE / BTS_RECORD_SIZE;
707 ds->bts_absolute_maximum = ds->bts_buffer_base +
708 max * BTS_RECORD_SIZE;
709 ds->bts_interrupt_threshold = ds->bts_absolute_maximum -
710 (max / 16) * BTS_RECORD_SIZE;
711 return 0;
712 }
713
release_bts_buffer(int cpu)714 static void release_bts_buffer(int cpu)
715 {
716 struct cpu_hw_events *hwev = per_cpu_ptr(&cpu_hw_events, cpu);
717 void *cea;
718
719 if (!x86_pmu.bts)
720 return;
721
722 /* Clear the fixmap */
723 cea = &get_cpu_entry_area(cpu)->cpu_debug_buffers.bts_buffer;
724 ds_clear_cea(cea, BTS_BUFFER_SIZE);
725 dsfree_pages(hwev->ds_bts_vaddr, BTS_BUFFER_SIZE);
726 hwev->ds_bts_vaddr = NULL;
727 }
728
alloc_ds_buffer(int cpu)729 static int alloc_ds_buffer(int cpu)
730 {
731 struct debug_store *ds = &get_cpu_entry_area(cpu)->cpu_debug_store;
732
733 memset(ds, 0, sizeof(*ds));
734 per_cpu(cpu_hw_events, cpu).ds = ds;
735 return 0;
736 }
737
release_ds_buffer(int cpu)738 static void release_ds_buffer(int cpu)
739 {
740 per_cpu(cpu_hw_events, cpu).ds = NULL;
741 }
742
release_ds_buffers(void)743 void release_ds_buffers(void)
744 {
745 int cpu;
746
747 if (!x86_pmu.bts && !x86_pmu.ds_pebs)
748 return;
749
750 for_each_possible_cpu(cpu)
751 release_ds_buffer(cpu);
752
753 for_each_possible_cpu(cpu) {
754 /*
755 * Again, ignore errors from offline CPUs, they will no longer
756 * observe cpu_hw_events.ds and not program the DS_AREA when
757 * they come up.
758 */
759 fini_debug_store_on_cpu(cpu);
760 }
761
762 for_each_possible_cpu(cpu) {
763 if (x86_pmu.ds_pebs)
764 release_pebs_buffer(cpu);
765 release_bts_buffer(cpu);
766 }
767 }
768
reserve_ds_buffers(void)769 void reserve_ds_buffers(void)
770 {
771 int bts_err = 0, pebs_err = 0;
772 int cpu;
773
774 x86_pmu.bts_active = 0;
775
776 if (x86_pmu.ds_pebs)
777 x86_pmu.pebs_active = 0;
778
779 if (!x86_pmu.bts && !x86_pmu.ds_pebs)
780 return;
781
782 if (!x86_pmu.bts)
783 bts_err = 1;
784
785 if (!x86_pmu.ds_pebs)
786 pebs_err = 1;
787
788 for_each_possible_cpu(cpu) {
789 if (alloc_ds_buffer(cpu)) {
790 bts_err = 1;
791 pebs_err = 1;
792 }
793
794 if (!bts_err && alloc_bts_buffer(cpu))
795 bts_err = 1;
796
797 if (x86_pmu.ds_pebs && !pebs_err &&
798 alloc_pebs_buffer(cpu))
799 pebs_err = 1;
800
801 if (bts_err && pebs_err)
802 break;
803 }
804
805 if (bts_err) {
806 for_each_possible_cpu(cpu)
807 release_bts_buffer(cpu);
808 }
809
810 if (x86_pmu.ds_pebs && pebs_err) {
811 for_each_possible_cpu(cpu)
812 release_pebs_buffer(cpu);
813 }
814
815 if (bts_err && pebs_err) {
816 for_each_possible_cpu(cpu)
817 release_ds_buffer(cpu);
818 } else {
819 if (x86_pmu.bts && !bts_err)
820 x86_pmu.bts_active = 1;
821
822 if (x86_pmu.ds_pebs && !pebs_err)
823 x86_pmu.pebs_active = 1;
824
825 for_each_possible_cpu(cpu) {
826 /*
827 * Ignores wrmsr_on_cpu() errors for offline CPUs they
828 * will get this call through intel_pmu_cpu_starting().
829 */
830 init_debug_store_on_cpu(cpu);
831 }
832 }
833 }
834
alloc_arch_pebs_buf_on_cpu(int cpu)835 inline int alloc_arch_pebs_buf_on_cpu(int cpu)
836 {
837 if (!x86_pmu.arch_pebs)
838 return 0;
839
840 return alloc_pebs_buffer(cpu);
841 }
842
release_arch_pebs_buf_on_cpu(int cpu)843 inline void release_arch_pebs_buf_on_cpu(int cpu)
844 {
845 if (!x86_pmu.arch_pebs)
846 return;
847
848 release_pebs_buffer(cpu);
849 }
850
init_arch_pebs_on_cpu(int cpu)851 void init_arch_pebs_on_cpu(int cpu)
852 {
853 struct cpu_hw_events *cpuc = per_cpu_ptr(&cpu_hw_events, cpu);
854 u64 arch_pebs_base;
855
856 if (!x86_pmu.arch_pebs)
857 return;
858
859 if (!cpuc->pebs_vaddr) {
860 WARN(1, "Fail to allocate PEBS buffer on CPU %d\n", cpu);
861 x86_pmu.pebs_active = 0;
862 return;
863 }
864
865 /*
866 * 4KB-aligned pointer of the output buffer
867 * (__alloc_pages_node() return page aligned address)
868 * Buffer Size = 4KB * 2^SIZE
869 * contiguous physical buffer (__alloc_pages_node() with order)
870 */
871 arch_pebs_base = virt_to_phys(cpuc->pebs_vaddr) | PEBS_BUFFER_SHIFT;
872 wrmsr_on_cpu(cpu, MSR_IA32_PEBS_BASE, (u32)arch_pebs_base,
873 (u32)(arch_pebs_base >> 32));
874 x86_pmu.pebs_active = 1;
875 }
876
fini_arch_pebs_on_cpu(int cpu)877 inline void fini_arch_pebs_on_cpu(int cpu)
878 {
879 if (!x86_pmu.arch_pebs)
880 return;
881
882 wrmsr_on_cpu(cpu, MSR_IA32_PEBS_BASE, 0, 0);
883 }
884
885 /*
886 * BTS
887 */
888
889 struct event_constraint bts_constraint =
890 EVENT_CONSTRAINT(0, 1ULL << INTEL_PMC_IDX_FIXED_BTS, 0);
891
intel_pmu_enable_bts(u64 config)892 void intel_pmu_enable_bts(u64 config)
893 {
894 unsigned long debugctlmsr;
895
896 debugctlmsr = get_debugctlmsr();
897
898 debugctlmsr |= DEBUGCTLMSR_TR;
899 debugctlmsr |= DEBUGCTLMSR_BTS;
900 if (config & ARCH_PERFMON_EVENTSEL_INT)
901 debugctlmsr |= DEBUGCTLMSR_BTINT;
902
903 if (!(config & ARCH_PERFMON_EVENTSEL_OS))
904 debugctlmsr |= DEBUGCTLMSR_BTS_OFF_OS;
905
906 if (!(config & ARCH_PERFMON_EVENTSEL_USR))
907 debugctlmsr |= DEBUGCTLMSR_BTS_OFF_USR;
908
909 update_debugctlmsr(debugctlmsr);
910 }
911
intel_pmu_disable_bts(void)912 void intel_pmu_disable_bts(void)
913 {
914 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
915 unsigned long debugctlmsr;
916
917 if (!cpuc->ds)
918 return;
919
920 debugctlmsr = get_debugctlmsr();
921
922 debugctlmsr &=
923 ~(DEBUGCTLMSR_TR | DEBUGCTLMSR_BTS | DEBUGCTLMSR_BTINT |
924 DEBUGCTLMSR_BTS_OFF_OS | DEBUGCTLMSR_BTS_OFF_USR);
925
926 update_debugctlmsr(debugctlmsr);
927 }
928
intel_pmu_drain_bts_buffer(void)929 int intel_pmu_drain_bts_buffer(void)
930 {
931 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
932 struct debug_store *ds = cpuc->ds;
933 struct bts_record {
934 u64 from;
935 u64 to;
936 u64 flags;
937 };
938 struct perf_event *event = cpuc->events[INTEL_PMC_IDX_FIXED_BTS];
939 struct bts_record *at, *base, *top;
940 struct perf_output_handle handle;
941 struct perf_event_header header;
942 struct perf_sample_data data;
943 unsigned long skip = 0;
944 struct pt_regs regs;
945
946 if (!event)
947 return 0;
948
949 if (!x86_pmu.bts_active)
950 return 0;
951
952 base = (struct bts_record *)(unsigned long)ds->bts_buffer_base;
953 top = (struct bts_record *)(unsigned long)ds->bts_index;
954
955 if (top <= base)
956 return 0;
957
958 memset(®s, 0, sizeof(regs));
959
960 ds->bts_index = ds->bts_buffer_base;
961
962 perf_sample_data_init(&data, 0, event->hw.last_period);
963
964 /*
965 * BTS leaks kernel addresses in branches across the cpl boundary,
966 * such as traps or system calls, so unless the user is asking for
967 * kernel tracing (and right now it's not possible), we'd need to
968 * filter them out. But first we need to count how many of those we
969 * have in the current batch. This is an extra O(n) pass, however,
970 * it's much faster than the other one especially considering that
971 * n <= 2560 (BTS_BUFFER_SIZE / BTS_RECORD_SIZE * 15/16; see the
972 * alloc_bts_buffer()).
973 */
974 for (at = base; at < top; at++) {
975 /*
976 * Note that right now *this* BTS code only works if
977 * attr::exclude_kernel is set, but let's keep this extra
978 * check here in case that changes.
979 */
980 if (event->attr.exclude_kernel &&
981 (kernel_ip(at->from) || kernel_ip(at->to)))
982 skip++;
983 }
984
985 /*
986 * Prepare a generic sample, i.e. fill in the invariant fields.
987 * We will overwrite the from and to address before we output
988 * the sample.
989 */
990 rcu_read_lock();
991 perf_prepare_sample(&data, event, ®s);
992 perf_prepare_header(&header, &data, event, ®s);
993
994 if (perf_output_begin(&handle, &data, event,
995 header.size * (top - base - skip)))
996 goto unlock;
997
998 for (at = base; at < top; at++) {
999 /* Filter out any records that contain kernel addresses. */
1000 if (event->attr.exclude_kernel &&
1001 (kernel_ip(at->from) || kernel_ip(at->to)))
1002 continue;
1003
1004 data.ip = at->from;
1005 data.addr = at->to;
1006
1007 perf_output_sample(&handle, &header, &data, event);
1008 }
1009
1010 perf_output_end(&handle);
1011
1012 /* There's new data available. */
1013 event->hw.interrupts++;
1014 event->pending_kill = POLL_IN;
1015 unlock:
1016 rcu_read_unlock();
1017 return 1;
1018 }
1019
intel_pmu_drain_pebs_buffer(void)1020 void intel_pmu_drain_pebs_buffer(void)
1021 {
1022 struct perf_sample_data data;
1023
1024 static_call(x86_pmu_drain_pebs)(NULL, &data);
1025 }
1026
1027 /*
1028 * PEBS
1029 */
1030 struct event_constraint intel_core2_pebs_event_constraints[] = {
1031 INTEL_FLAGS_UEVENT_CONSTRAINT(0x00c0, 0x1), /* INST_RETIRED.ANY */
1032 INTEL_FLAGS_UEVENT_CONSTRAINT(0xfec1, 0x1), /* X87_OPS_RETIRED.ANY */
1033 INTEL_FLAGS_UEVENT_CONSTRAINT(0x00c5, 0x1), /* BR_INST_RETIRED.MISPRED */
1034 INTEL_FLAGS_UEVENT_CONSTRAINT(0x1fc7, 0x1), /* SIMD_INST_RETURED.ANY */
1035 INTEL_FLAGS_EVENT_CONSTRAINT(0xcb, 0x1), /* MEM_LOAD_RETIRED.* */
1036 /* INST_RETIRED.ANY_P, inv=1, cmask=16 (cycles:p). */
1037 INTEL_FLAGS_UEVENT_CONSTRAINT(0x108000c0, 0x01),
1038 EVENT_CONSTRAINT_END
1039 };
1040
1041 struct event_constraint intel_atom_pebs_event_constraints[] = {
1042 INTEL_FLAGS_UEVENT_CONSTRAINT(0x00c0, 0x1), /* INST_RETIRED.ANY */
1043 INTEL_FLAGS_UEVENT_CONSTRAINT(0x00c5, 0x1), /* MISPREDICTED_BRANCH_RETIRED */
1044 INTEL_FLAGS_EVENT_CONSTRAINT(0xcb, 0x1), /* MEM_LOAD_RETIRED.* */
1045 /* INST_RETIRED.ANY_P, inv=1, cmask=16 (cycles:p). */
1046 INTEL_FLAGS_UEVENT_CONSTRAINT(0x108000c0, 0x01),
1047 /* Allow all events as PEBS with no flags */
1048 INTEL_ALL_EVENT_CONSTRAINT(0, 0x1),
1049 EVENT_CONSTRAINT_END
1050 };
1051
1052 struct event_constraint intel_slm_pebs_event_constraints[] = {
1053 /* INST_RETIRED.ANY_P, inv=1, cmask=16 (cycles:p). */
1054 INTEL_FLAGS_UEVENT_CONSTRAINT(0x108000c0, 0x1),
1055 /* Allow all events as PEBS with no flags */
1056 INTEL_ALL_EVENT_CONSTRAINT(0, 0x1),
1057 EVENT_CONSTRAINT_END
1058 };
1059
1060 struct event_constraint intel_glm_pebs_event_constraints[] = {
1061 /* Allow all events as PEBS with no flags */
1062 INTEL_ALL_EVENT_CONSTRAINT(0, 0x1),
1063 EVENT_CONSTRAINT_END
1064 };
1065
1066 struct event_constraint intel_grt_pebs_event_constraints[] = {
1067 /* Allow all events as PEBS with no flags */
1068 INTEL_HYBRID_LAT_CONSTRAINT(0x5d0, 0x3),
1069 INTEL_HYBRID_LAT_CONSTRAINT(0x6d0, 0xf),
1070 EVENT_CONSTRAINT_END
1071 };
1072
1073 struct event_constraint intel_nehalem_pebs_event_constraints[] = {
1074 INTEL_PLD_CONSTRAINT(0x100b, 0xf), /* MEM_INST_RETIRED.* */
1075 INTEL_FLAGS_EVENT_CONSTRAINT(0x0f, 0xf), /* MEM_UNCORE_RETIRED.* */
1076 INTEL_FLAGS_UEVENT_CONSTRAINT(0x010c, 0xf), /* MEM_STORE_RETIRED.DTLB_MISS */
1077 INTEL_FLAGS_EVENT_CONSTRAINT(0xc0, 0xf), /* INST_RETIRED.ANY */
1078 INTEL_EVENT_CONSTRAINT(0xc2, 0xf), /* UOPS_RETIRED.* */
1079 INTEL_FLAGS_EVENT_CONSTRAINT(0xc4, 0xf), /* BR_INST_RETIRED.* */
1080 INTEL_FLAGS_UEVENT_CONSTRAINT(0x02c5, 0xf), /* BR_MISP_RETIRED.NEAR_CALL */
1081 INTEL_FLAGS_EVENT_CONSTRAINT(0xc7, 0xf), /* SSEX_UOPS_RETIRED.* */
1082 INTEL_FLAGS_UEVENT_CONSTRAINT(0x20c8, 0xf), /* ITLB_MISS_RETIRED */
1083 INTEL_FLAGS_EVENT_CONSTRAINT(0xcb, 0xf), /* MEM_LOAD_RETIRED.* */
1084 INTEL_FLAGS_EVENT_CONSTRAINT(0xf7, 0xf), /* FP_ASSIST.* */
1085 /* INST_RETIRED.ANY_P, inv=1, cmask=16 (cycles:p). */
1086 INTEL_FLAGS_UEVENT_CONSTRAINT(0x108000c0, 0x0f),
1087 EVENT_CONSTRAINT_END
1088 };
1089
1090 struct event_constraint intel_westmere_pebs_event_constraints[] = {
1091 INTEL_PLD_CONSTRAINT(0x100b, 0xf), /* MEM_INST_RETIRED.* */
1092 INTEL_FLAGS_EVENT_CONSTRAINT(0x0f, 0xf), /* MEM_UNCORE_RETIRED.* */
1093 INTEL_FLAGS_UEVENT_CONSTRAINT(0x010c, 0xf), /* MEM_STORE_RETIRED.DTLB_MISS */
1094 INTEL_FLAGS_EVENT_CONSTRAINT(0xc0, 0xf), /* INSTR_RETIRED.* */
1095 INTEL_EVENT_CONSTRAINT(0xc2, 0xf), /* UOPS_RETIRED.* */
1096 INTEL_FLAGS_EVENT_CONSTRAINT(0xc4, 0xf), /* BR_INST_RETIRED.* */
1097 INTEL_FLAGS_EVENT_CONSTRAINT(0xc5, 0xf), /* BR_MISP_RETIRED.* */
1098 INTEL_FLAGS_EVENT_CONSTRAINT(0xc7, 0xf), /* SSEX_UOPS_RETIRED.* */
1099 INTEL_FLAGS_UEVENT_CONSTRAINT(0x20c8, 0xf), /* ITLB_MISS_RETIRED */
1100 INTEL_FLAGS_EVENT_CONSTRAINT(0xcb, 0xf), /* MEM_LOAD_RETIRED.* */
1101 INTEL_FLAGS_EVENT_CONSTRAINT(0xf7, 0xf), /* FP_ASSIST.* */
1102 /* INST_RETIRED.ANY_P, inv=1, cmask=16 (cycles:p). */
1103 INTEL_FLAGS_UEVENT_CONSTRAINT(0x108000c0, 0x0f),
1104 EVENT_CONSTRAINT_END
1105 };
1106
1107 struct event_constraint intel_snb_pebs_event_constraints[] = {
1108 INTEL_FLAGS_UEVENT_CONSTRAINT(0x01c0, 0x2), /* INST_RETIRED.PRECDIST */
1109 INTEL_PLD_CONSTRAINT(0x01cd, 0x8), /* MEM_TRANS_RETIRED.LAT_ABOVE_THR */
1110 INTEL_PST_CONSTRAINT(0x02cd, 0x8), /* MEM_TRANS_RETIRED.PRECISE_STORES */
1111 /* UOPS_RETIRED.ALL, inv=1, cmask=16 (cycles:p). */
1112 INTEL_FLAGS_UEVENT_CONSTRAINT(0x108001c2, 0xf),
1113 INTEL_EXCLEVT_CONSTRAINT(0xd0, 0xf), /* MEM_UOP_RETIRED.* */
1114 INTEL_EXCLEVT_CONSTRAINT(0xd1, 0xf), /* MEM_LOAD_UOPS_RETIRED.* */
1115 INTEL_EXCLEVT_CONSTRAINT(0xd2, 0xf), /* MEM_LOAD_UOPS_LLC_HIT_RETIRED.* */
1116 INTEL_EXCLEVT_CONSTRAINT(0xd3, 0xf), /* MEM_LOAD_UOPS_LLC_MISS_RETIRED.* */
1117 /* Allow all events as PEBS with no flags */
1118 INTEL_ALL_EVENT_CONSTRAINT(0, 0xf),
1119 EVENT_CONSTRAINT_END
1120 };
1121
1122 struct event_constraint intel_ivb_pebs_event_constraints[] = {
1123 INTEL_FLAGS_UEVENT_CONSTRAINT(0x01c0, 0x2), /* INST_RETIRED.PRECDIST */
1124 INTEL_PLD_CONSTRAINT(0x01cd, 0x8), /* MEM_TRANS_RETIRED.LAT_ABOVE_THR */
1125 INTEL_PST_CONSTRAINT(0x02cd, 0x8), /* MEM_TRANS_RETIRED.PRECISE_STORES */
1126 /* UOPS_RETIRED.ALL, inv=1, cmask=16 (cycles:p). */
1127 INTEL_FLAGS_UEVENT_CONSTRAINT(0x108001c2, 0xf),
1128 /* INST_RETIRED.PREC_DIST, inv=1, cmask=16 (cycles:ppp). */
1129 INTEL_FLAGS_UEVENT_CONSTRAINT(0x108001c0, 0x2),
1130 INTEL_EXCLEVT_CONSTRAINT(0xd0, 0xf), /* MEM_UOP_RETIRED.* */
1131 INTEL_EXCLEVT_CONSTRAINT(0xd1, 0xf), /* MEM_LOAD_UOPS_RETIRED.* */
1132 INTEL_EXCLEVT_CONSTRAINT(0xd2, 0xf), /* MEM_LOAD_UOPS_LLC_HIT_RETIRED.* */
1133 INTEL_EXCLEVT_CONSTRAINT(0xd3, 0xf), /* MEM_LOAD_UOPS_LLC_MISS_RETIRED.* */
1134 /* Allow all events as PEBS with no flags */
1135 INTEL_ALL_EVENT_CONSTRAINT(0, 0xf),
1136 EVENT_CONSTRAINT_END
1137 };
1138
1139 struct event_constraint intel_hsw_pebs_event_constraints[] = {
1140 INTEL_FLAGS_UEVENT_CONSTRAINT(0x01c0, 0x2), /* INST_RETIRED.PRECDIST */
1141 INTEL_PLD_CONSTRAINT(0x01cd, 0xf), /* MEM_TRANS_RETIRED.* */
1142 /* UOPS_RETIRED.ALL, inv=1, cmask=16 (cycles:p). */
1143 INTEL_FLAGS_UEVENT_CONSTRAINT(0x108001c2, 0xf),
1144 /* INST_RETIRED.PREC_DIST, inv=1, cmask=16 (cycles:ppp). */
1145 INTEL_FLAGS_UEVENT_CONSTRAINT(0x108001c0, 0x2),
1146 INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_NA(0x01c2, 0xf), /* UOPS_RETIRED.ALL */
1147 INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_XLD(0x11d0, 0xf), /* MEM_UOPS_RETIRED.STLB_MISS_LOADS */
1148 INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_XLD(0x21d0, 0xf), /* MEM_UOPS_RETIRED.LOCK_LOADS */
1149 INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_XLD(0x41d0, 0xf), /* MEM_UOPS_RETIRED.SPLIT_LOADS */
1150 INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_XLD(0x81d0, 0xf), /* MEM_UOPS_RETIRED.ALL_LOADS */
1151 INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_XST(0x12d0, 0xf), /* MEM_UOPS_RETIRED.STLB_MISS_STORES */
1152 INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_XST(0x42d0, 0xf), /* MEM_UOPS_RETIRED.SPLIT_STORES */
1153 INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_XST(0x82d0, 0xf), /* MEM_UOPS_RETIRED.ALL_STORES */
1154 INTEL_FLAGS_EVENT_CONSTRAINT_DATALA_XLD(0xd1, 0xf), /* MEM_LOAD_UOPS_RETIRED.* */
1155 INTEL_FLAGS_EVENT_CONSTRAINT_DATALA_XLD(0xd2, 0xf), /* MEM_LOAD_UOPS_L3_HIT_RETIRED.* */
1156 INTEL_FLAGS_EVENT_CONSTRAINT_DATALA_XLD(0xd3, 0xf), /* MEM_LOAD_UOPS_L3_MISS_RETIRED.* */
1157 /* Allow all events as PEBS with no flags */
1158 INTEL_ALL_EVENT_CONSTRAINT(0, 0xf),
1159 EVENT_CONSTRAINT_END
1160 };
1161
1162 struct event_constraint intel_bdw_pebs_event_constraints[] = {
1163 INTEL_FLAGS_UEVENT_CONSTRAINT(0x01c0, 0x2), /* INST_RETIRED.PRECDIST */
1164 INTEL_PLD_CONSTRAINT(0x01cd, 0xf), /* MEM_TRANS_RETIRED.* */
1165 /* UOPS_RETIRED.ALL, inv=1, cmask=16 (cycles:p). */
1166 INTEL_FLAGS_UEVENT_CONSTRAINT(0x108001c2, 0xf),
1167 /* INST_RETIRED.PREC_DIST, inv=1, cmask=16 (cycles:ppp). */
1168 INTEL_FLAGS_UEVENT_CONSTRAINT(0x108001c0, 0x2),
1169 INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_NA(0x01c2, 0xf), /* UOPS_RETIRED.ALL */
1170 INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x11d0, 0xf), /* MEM_UOPS_RETIRED.STLB_MISS_LOADS */
1171 INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x21d0, 0xf), /* MEM_UOPS_RETIRED.LOCK_LOADS */
1172 INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x41d0, 0xf), /* MEM_UOPS_RETIRED.SPLIT_LOADS */
1173 INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x81d0, 0xf), /* MEM_UOPS_RETIRED.ALL_LOADS */
1174 INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_ST(0x12d0, 0xf), /* MEM_UOPS_RETIRED.STLB_MISS_STORES */
1175 INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_ST(0x42d0, 0xf), /* MEM_UOPS_RETIRED.SPLIT_STORES */
1176 INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_ST(0x82d0, 0xf), /* MEM_UOPS_RETIRED.ALL_STORES */
1177 INTEL_FLAGS_EVENT_CONSTRAINT_DATALA_LD(0xd1, 0xf), /* MEM_LOAD_UOPS_RETIRED.* */
1178 INTEL_FLAGS_EVENT_CONSTRAINT_DATALA_LD(0xd2, 0xf), /* MEM_LOAD_UOPS_L3_HIT_RETIRED.* */
1179 INTEL_FLAGS_EVENT_CONSTRAINT_DATALA_LD(0xd3, 0xf), /* MEM_LOAD_UOPS_L3_MISS_RETIRED.* */
1180 /* Allow all events as PEBS with no flags */
1181 INTEL_ALL_EVENT_CONSTRAINT(0, 0xf),
1182 EVENT_CONSTRAINT_END
1183 };
1184
1185
1186 struct event_constraint intel_skl_pebs_event_constraints[] = {
1187 INTEL_FLAGS_UEVENT_CONSTRAINT(0x1c0, 0x2), /* INST_RETIRED.PREC_DIST */
1188 /* INST_RETIRED.PREC_DIST, inv=1, cmask=16 (cycles:ppp). */
1189 INTEL_FLAGS_UEVENT_CONSTRAINT(0x108001c0, 0x2),
1190 /* INST_RETIRED.TOTAL_CYCLES_PS (inv=1, cmask=16) (cycles:p). */
1191 INTEL_FLAGS_UEVENT_CONSTRAINT(0x108000c0, 0x0f),
1192 INTEL_PLD_CONSTRAINT(0x1cd, 0xf), /* MEM_TRANS_RETIRED.* */
1193 INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x11d0, 0xf), /* MEM_INST_RETIRED.STLB_MISS_LOADS */
1194 INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_ST(0x12d0, 0xf), /* MEM_INST_RETIRED.STLB_MISS_STORES */
1195 INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x21d0, 0xf), /* MEM_INST_RETIRED.LOCK_LOADS */
1196 INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_ST(0x22d0, 0xf), /* MEM_INST_RETIRED.LOCK_STORES */
1197 INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x41d0, 0xf), /* MEM_INST_RETIRED.SPLIT_LOADS */
1198 INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_ST(0x42d0, 0xf), /* MEM_INST_RETIRED.SPLIT_STORES */
1199 INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x81d0, 0xf), /* MEM_INST_RETIRED.ALL_LOADS */
1200 INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_ST(0x82d0, 0xf), /* MEM_INST_RETIRED.ALL_STORES */
1201 INTEL_FLAGS_EVENT_CONSTRAINT_DATALA_LD(0xd1, 0xf), /* MEM_LOAD_RETIRED.* */
1202 INTEL_FLAGS_EVENT_CONSTRAINT_DATALA_LD(0xd2, 0xf), /* MEM_LOAD_L3_HIT_RETIRED.* */
1203 INTEL_FLAGS_EVENT_CONSTRAINT_DATALA_LD(0xd3, 0xf), /* MEM_LOAD_L3_MISS_RETIRED.* */
1204 /* Allow all events as PEBS with no flags */
1205 INTEL_ALL_EVENT_CONSTRAINT(0, 0xf),
1206 EVENT_CONSTRAINT_END
1207 };
1208
1209 struct event_constraint intel_icl_pebs_event_constraints[] = {
1210 INTEL_FLAGS_UEVENT_CONSTRAINT(0x01c0, 0x100000000ULL), /* old INST_RETIRED.PREC_DIST */
1211 INTEL_FLAGS_UEVENT_CONSTRAINT(0x0100, 0x100000000ULL), /* INST_RETIRED.PREC_DIST */
1212 INTEL_FLAGS_UEVENT_CONSTRAINT(0x0400, 0x800000000ULL), /* SLOTS */
1213
1214 INTEL_PLD_CONSTRAINT(0x1cd, 0xff), /* MEM_TRANS_RETIRED.LOAD_LATENCY */
1215 INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x11d0, 0xf), /* MEM_INST_RETIRED.STLB_MISS_LOADS */
1216 INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_ST(0x12d0, 0xf), /* MEM_INST_RETIRED.STLB_MISS_STORES */
1217 INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x21d0, 0xf), /* MEM_INST_RETIRED.LOCK_LOADS */
1218 INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x41d0, 0xf), /* MEM_INST_RETIRED.SPLIT_LOADS */
1219 INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_ST(0x42d0, 0xf), /* MEM_INST_RETIRED.SPLIT_STORES */
1220 INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x81d0, 0xf), /* MEM_INST_RETIRED.ALL_LOADS */
1221 INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_ST(0x82d0, 0xf), /* MEM_INST_RETIRED.ALL_STORES */
1222
1223 INTEL_FLAGS_EVENT_CONSTRAINT_DATALA_LD_RANGE(0xd1, 0xd4, 0xf), /* MEM_LOAD_*_RETIRED.* */
1224
1225 INTEL_FLAGS_EVENT_CONSTRAINT(0xd0, 0xf), /* MEM_INST_RETIRED.* */
1226
1227 /*
1228 * Everything else is handled by PMU_FL_PEBS_ALL, because we
1229 * need the full constraints from the main table.
1230 */
1231
1232 EVENT_CONSTRAINT_END
1233 };
1234
1235 struct event_constraint intel_glc_pebs_event_constraints[] = {
1236 INTEL_FLAGS_UEVENT_CONSTRAINT(0x100, 0x100000000ULL), /* INST_RETIRED.PREC_DIST */
1237 INTEL_FLAGS_UEVENT_CONSTRAINT(0x0400, 0x800000000ULL),
1238
1239 INTEL_FLAGS_EVENT_CONSTRAINT(0xc0, 0xfe),
1240 INTEL_PLD_CONSTRAINT(0x1cd, 0xfe),
1241 INTEL_PSD_CONSTRAINT(0x2cd, 0x1),
1242 INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x11d0, 0xf), /* MEM_INST_RETIRED.STLB_MISS_LOADS */
1243 INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_ST(0x12d0, 0xf), /* MEM_INST_RETIRED.STLB_MISS_STORES */
1244 INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x21d0, 0xf), /* MEM_INST_RETIRED.LOCK_LOADS */
1245 INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x41d0, 0xf), /* MEM_INST_RETIRED.SPLIT_LOADS */
1246 INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_ST(0x42d0, 0xf), /* MEM_INST_RETIRED.SPLIT_STORES */
1247 INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x81d0, 0xf), /* MEM_INST_RETIRED.ALL_LOADS */
1248 INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_ST(0x82d0, 0xf), /* MEM_INST_RETIRED.ALL_STORES */
1249
1250 INTEL_FLAGS_EVENT_CONSTRAINT_DATALA_LD_RANGE(0xd1, 0xd4, 0xf),
1251
1252 INTEL_FLAGS_EVENT_CONSTRAINT(0xd0, 0xf),
1253
1254 /*
1255 * Everything else is handled by PMU_FL_PEBS_ALL, because we
1256 * need the full constraints from the main table.
1257 */
1258
1259 EVENT_CONSTRAINT_END
1260 };
1261
1262 struct event_constraint intel_lnc_pebs_event_constraints[] = {
1263 INTEL_FLAGS_UEVENT_CONSTRAINT(0x100, 0x100000000ULL), /* INST_RETIRED.PREC_DIST */
1264 INTEL_FLAGS_UEVENT_CONSTRAINT(0x0400, 0x800000000ULL),
1265
1266 INTEL_HYBRID_LDLAT_CONSTRAINT(0x1cd, 0x3fc),
1267 INTEL_HYBRID_STLAT_CONSTRAINT(0x2cd, 0x3),
1268 INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x11d0, 0xf), /* MEM_INST_RETIRED.STLB_MISS_LOADS */
1269 INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_ST(0x12d0, 0xf), /* MEM_INST_RETIRED.STLB_MISS_STORES */
1270 INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x21d0, 0xf), /* MEM_INST_RETIRED.LOCK_LOADS */
1271 INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x41d0, 0xf), /* MEM_INST_RETIRED.SPLIT_LOADS */
1272 INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_ST(0x42d0, 0xf), /* MEM_INST_RETIRED.SPLIT_STORES */
1273 INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x81d0, 0xf), /* MEM_INST_RETIRED.ALL_LOADS */
1274 INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_ST(0x82d0, 0xf), /* MEM_INST_RETIRED.ALL_STORES */
1275
1276 INTEL_FLAGS_EVENT_CONSTRAINT_DATALA_LD_RANGE(0xd1, 0xd4, 0xf),
1277
1278 INTEL_FLAGS_EVENT_CONSTRAINT(0xd0, 0xf),
1279
1280 /*
1281 * Everything else is handled by PMU_FL_PEBS_ALL, because we
1282 * need the full constraints from the main table.
1283 */
1284
1285 EVENT_CONSTRAINT_END
1286 };
1287
intel_pebs_constraints(struct perf_event * event)1288 struct event_constraint *intel_pebs_constraints(struct perf_event *event)
1289 {
1290 struct event_constraint *pebs_constraints = hybrid(event->pmu, pebs_constraints);
1291 struct event_constraint *c;
1292
1293 if (!event->attr.precise_ip)
1294 return NULL;
1295
1296 if (pebs_constraints) {
1297 for_each_event_constraint(c, pebs_constraints) {
1298 if (constraint_match(c, event->hw.config)) {
1299 event->hw.flags |= c->flags;
1300 return c;
1301 }
1302 }
1303 }
1304
1305 /*
1306 * Extended PEBS support
1307 * Makes the PEBS code search the normal constraints.
1308 */
1309 if (x86_pmu.flags & PMU_FL_PEBS_ALL)
1310 return NULL;
1311
1312 return &emptyconstraint;
1313 }
1314
1315 /*
1316 * We need the sched_task callback even for per-cpu events when we use
1317 * the large interrupt threshold, such that we can provide PID and TID
1318 * to PEBS samples.
1319 */
pebs_needs_sched_cb(struct cpu_hw_events * cpuc)1320 static inline bool pebs_needs_sched_cb(struct cpu_hw_events *cpuc)
1321 {
1322 if (cpuc->n_pebs == cpuc->n_pebs_via_pt)
1323 return false;
1324
1325 return cpuc->n_pebs && (cpuc->n_pebs == cpuc->n_large_pebs);
1326 }
1327
intel_pmu_pebs_sched_task(struct perf_event_pmu_context * pmu_ctx,bool sched_in)1328 void intel_pmu_pebs_sched_task(struct perf_event_pmu_context *pmu_ctx, bool sched_in)
1329 {
1330 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1331
1332 if (!sched_in && pebs_needs_sched_cb(cpuc))
1333 intel_pmu_drain_pebs_buffer();
1334 }
1335
pebs_update_threshold(struct cpu_hw_events * cpuc)1336 static inline void pebs_update_threshold(struct cpu_hw_events *cpuc)
1337 {
1338 struct debug_store *ds = cpuc->ds;
1339 int max_pebs_events = intel_pmu_max_num_pebs(cpuc->pmu);
1340 u64 threshold;
1341 int reserved;
1342
1343 if (cpuc->n_pebs_via_pt)
1344 return;
1345
1346 if (x86_pmu.flags & PMU_FL_PEBS_ALL)
1347 reserved = max_pebs_events + x86_pmu_max_num_counters_fixed(cpuc->pmu);
1348 else
1349 reserved = max_pebs_events;
1350
1351 if (cpuc->n_pebs == cpuc->n_large_pebs) {
1352 threshold = ds->pebs_absolute_maximum -
1353 reserved * cpuc->pebs_record_size;
1354 } else {
1355 threshold = ds->pebs_buffer_base + cpuc->pebs_record_size;
1356 }
1357
1358 ds->pebs_interrupt_threshold = threshold;
1359 }
1360
1361 #define PEBS_DATACFG_CNTRS(x) \
1362 ((x >> PEBS_DATACFG_CNTR_SHIFT) & PEBS_DATACFG_CNTR_MASK)
1363
1364 #define PEBS_DATACFG_CNTR_BIT(x) \
1365 (((1ULL << x) & PEBS_DATACFG_CNTR_MASK) << PEBS_DATACFG_CNTR_SHIFT)
1366
1367 #define PEBS_DATACFG_FIX(x) \
1368 ((x >> PEBS_DATACFG_FIX_SHIFT) & PEBS_DATACFG_FIX_MASK)
1369
1370 #define PEBS_DATACFG_FIX_BIT(x) \
1371 (((1ULL << (x)) & PEBS_DATACFG_FIX_MASK) \
1372 << PEBS_DATACFG_FIX_SHIFT)
1373
adaptive_pebs_record_size_update(void)1374 static void adaptive_pebs_record_size_update(void)
1375 {
1376 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1377 u64 pebs_data_cfg = cpuc->pebs_data_cfg;
1378 int sz = sizeof(struct pebs_basic);
1379
1380 if (pebs_data_cfg & PEBS_DATACFG_MEMINFO)
1381 sz += sizeof(struct pebs_meminfo);
1382 if (pebs_data_cfg & PEBS_DATACFG_GP)
1383 sz += sizeof(struct pebs_gprs);
1384 if (pebs_data_cfg & PEBS_DATACFG_XMMS)
1385 sz += sizeof(struct pebs_xmm);
1386 if (pebs_data_cfg & PEBS_DATACFG_LBRS)
1387 sz += x86_pmu.lbr_nr * sizeof(struct lbr_entry);
1388 if (pebs_data_cfg & (PEBS_DATACFG_METRICS | PEBS_DATACFG_CNTR)) {
1389 sz += sizeof(struct pebs_cntr_header);
1390
1391 /* Metrics base and Metrics Data */
1392 if (pebs_data_cfg & PEBS_DATACFG_METRICS)
1393 sz += 2 * sizeof(u64);
1394
1395 if (pebs_data_cfg & PEBS_DATACFG_CNTR) {
1396 sz += (hweight64(PEBS_DATACFG_CNTRS(pebs_data_cfg)) +
1397 hweight64(PEBS_DATACFG_FIX(pebs_data_cfg))) *
1398 sizeof(u64);
1399 }
1400 }
1401
1402 cpuc->pebs_record_size = sz;
1403 }
1404
__intel_pmu_pebs_update_cfg(struct perf_event * event,int idx,u64 * pebs_data_cfg)1405 static void __intel_pmu_pebs_update_cfg(struct perf_event *event,
1406 int idx, u64 *pebs_data_cfg)
1407 {
1408 if (is_metric_event(event)) {
1409 *pebs_data_cfg |= PEBS_DATACFG_METRICS;
1410 return;
1411 }
1412
1413 *pebs_data_cfg |= PEBS_DATACFG_CNTR;
1414
1415 if (idx >= INTEL_PMC_IDX_FIXED)
1416 *pebs_data_cfg |= PEBS_DATACFG_FIX_BIT(idx - INTEL_PMC_IDX_FIXED);
1417 else
1418 *pebs_data_cfg |= PEBS_DATACFG_CNTR_BIT(idx);
1419 }
1420
1421
intel_pmu_pebs_late_setup(struct cpu_hw_events * cpuc)1422 void intel_pmu_pebs_late_setup(struct cpu_hw_events *cpuc)
1423 {
1424 struct perf_event *event;
1425 u64 pebs_data_cfg = 0;
1426 int i;
1427
1428 for (i = 0; i < cpuc->n_events; i++) {
1429 event = cpuc->event_list[i];
1430 if (!is_pebs_counter_event_group(event))
1431 continue;
1432 __intel_pmu_pebs_update_cfg(event, cpuc->assign[i], &pebs_data_cfg);
1433 }
1434
1435 if (pebs_data_cfg & ~cpuc->pebs_data_cfg)
1436 cpuc->pebs_data_cfg |= pebs_data_cfg | PEBS_UPDATE_DS_SW;
1437 }
1438
1439 #define PERF_PEBS_MEMINFO_TYPE (PERF_SAMPLE_ADDR | PERF_SAMPLE_DATA_SRC | \
1440 PERF_SAMPLE_PHYS_ADDR | \
1441 PERF_SAMPLE_WEIGHT_TYPE | \
1442 PERF_SAMPLE_TRANSACTION | \
1443 PERF_SAMPLE_DATA_PAGE_SIZE)
1444
pebs_update_adaptive_cfg(struct perf_event * event)1445 static u64 pebs_update_adaptive_cfg(struct perf_event *event)
1446 {
1447 struct perf_event_attr *attr = &event->attr;
1448 u64 sample_type = attr->sample_type;
1449 u64 pebs_data_cfg = 0;
1450 bool gprs, tsx_weight;
1451
1452 if (!(sample_type & ~(PERF_SAMPLE_IP|PERF_SAMPLE_TIME)) &&
1453 attr->precise_ip > 1)
1454 return pebs_data_cfg;
1455
1456 if (sample_type & PERF_PEBS_MEMINFO_TYPE)
1457 pebs_data_cfg |= PEBS_DATACFG_MEMINFO;
1458
1459 /*
1460 * We need GPRs when:
1461 * + user requested them
1462 * + precise_ip < 2 for the non event IP
1463 * + For RTM TSX weight we need GPRs for the abort code.
1464 */
1465 gprs = ((sample_type & PERF_SAMPLE_REGS_INTR) &&
1466 (attr->sample_regs_intr & PEBS_GP_REGS)) ||
1467 ((sample_type & PERF_SAMPLE_REGS_USER) &&
1468 (attr->sample_regs_user & PEBS_GP_REGS));
1469
1470 tsx_weight = (sample_type & PERF_SAMPLE_WEIGHT_TYPE) &&
1471 ((attr->config & INTEL_ARCH_EVENT_MASK) ==
1472 x86_pmu.rtm_abort_event);
1473
1474 if (gprs || (attr->precise_ip < 2) || tsx_weight)
1475 pebs_data_cfg |= PEBS_DATACFG_GP;
1476
1477 if ((sample_type & PERF_SAMPLE_REGS_INTR) &&
1478 (attr->sample_regs_intr & PERF_REG_EXTENDED_MASK))
1479 pebs_data_cfg |= PEBS_DATACFG_XMMS;
1480
1481 if (sample_type & PERF_SAMPLE_BRANCH_STACK) {
1482 /*
1483 * For now always log all LBRs. Could configure this
1484 * later.
1485 */
1486 pebs_data_cfg |= PEBS_DATACFG_LBRS |
1487 ((x86_pmu.lbr_nr-1) << PEBS_DATACFG_LBR_SHIFT);
1488 }
1489
1490 return pebs_data_cfg;
1491 }
1492
1493 static void
pebs_update_state(bool needed_cb,struct cpu_hw_events * cpuc,struct perf_event * event,bool add)1494 pebs_update_state(bool needed_cb, struct cpu_hw_events *cpuc,
1495 struct perf_event *event, bool add)
1496 {
1497 struct pmu *pmu = event->pmu;
1498
1499 /*
1500 * Make sure we get updated with the first PEBS event.
1501 * During removal, ->pebs_data_cfg is still valid for
1502 * the last PEBS event. Don't clear it.
1503 */
1504 if ((cpuc->n_pebs == 1) && add)
1505 cpuc->pebs_data_cfg = PEBS_UPDATE_DS_SW;
1506
1507 if (needed_cb != pebs_needs_sched_cb(cpuc)) {
1508 if (!needed_cb)
1509 perf_sched_cb_inc(pmu);
1510 else
1511 perf_sched_cb_dec(pmu);
1512
1513 cpuc->pebs_data_cfg |= PEBS_UPDATE_DS_SW;
1514 }
1515
1516 /*
1517 * The PEBS record doesn't shrink on pmu::del(). Doing so would require
1518 * iterating all remaining PEBS events to reconstruct the config.
1519 */
1520 if (x86_pmu.intel_cap.pebs_baseline && add) {
1521 u64 pebs_data_cfg;
1522
1523 pebs_data_cfg = pebs_update_adaptive_cfg(event);
1524 /*
1525 * Be sure to update the thresholds when we change the record.
1526 */
1527 if (pebs_data_cfg & ~cpuc->pebs_data_cfg)
1528 cpuc->pebs_data_cfg |= pebs_data_cfg | PEBS_UPDATE_DS_SW;
1529 }
1530 }
1531
intel_get_arch_pebs_data_config(struct perf_event * event)1532 u64 intel_get_arch_pebs_data_config(struct perf_event *event)
1533 {
1534 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1535 u64 pebs_data_cfg = 0;
1536 u64 cntr_mask;
1537
1538 if (WARN_ON(event->hw.idx < 0 || event->hw.idx >= X86_PMC_IDX_MAX))
1539 return 0;
1540
1541 pebs_data_cfg |= pebs_update_adaptive_cfg(event);
1542
1543 cntr_mask = (PEBS_DATACFG_CNTR_MASK << PEBS_DATACFG_CNTR_SHIFT) |
1544 (PEBS_DATACFG_FIX_MASK << PEBS_DATACFG_FIX_SHIFT) |
1545 PEBS_DATACFG_CNTR | PEBS_DATACFG_METRICS;
1546 pebs_data_cfg |= cpuc->pebs_data_cfg & cntr_mask;
1547
1548 return pebs_data_cfg;
1549 }
1550
intel_pmu_pebs_add(struct perf_event * event)1551 void intel_pmu_pebs_add(struct perf_event *event)
1552 {
1553 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1554 struct hw_perf_event *hwc = &event->hw;
1555 bool needed_cb = pebs_needs_sched_cb(cpuc);
1556
1557 cpuc->n_pebs++;
1558 if (hwc->flags & PERF_X86_EVENT_LARGE_PEBS)
1559 cpuc->n_large_pebs++;
1560 if (hwc->flags & PERF_X86_EVENT_PEBS_VIA_PT)
1561 cpuc->n_pebs_via_pt++;
1562
1563 pebs_update_state(needed_cb, cpuc, event, true);
1564 }
1565
intel_pmu_pebs_via_pt_disable(struct perf_event * event)1566 static void intel_pmu_pebs_via_pt_disable(struct perf_event *event)
1567 {
1568 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1569
1570 if (!is_pebs_pt(event))
1571 return;
1572
1573 if (!(cpuc->pebs_enabled & ~PEBS_VIA_PT_MASK))
1574 cpuc->pebs_enabled &= ~PEBS_VIA_PT_MASK;
1575 }
1576
intel_pmu_pebs_via_pt_enable(struct perf_event * event)1577 static void intel_pmu_pebs_via_pt_enable(struct perf_event *event)
1578 {
1579 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1580 struct hw_perf_event *hwc = &event->hw;
1581 struct debug_store *ds = cpuc->ds;
1582 u64 value = ds->pebs_event_reset[hwc->idx];
1583 u32 base = MSR_RELOAD_PMC0;
1584 unsigned int idx = hwc->idx;
1585
1586 if (!is_pebs_pt(event))
1587 return;
1588
1589 if (!(event->hw.flags & PERF_X86_EVENT_LARGE_PEBS))
1590 cpuc->pebs_enabled |= PEBS_PMI_AFTER_EACH_RECORD;
1591
1592 cpuc->pebs_enabled |= PEBS_OUTPUT_PT;
1593
1594 if (hwc->idx >= INTEL_PMC_IDX_FIXED) {
1595 base = MSR_RELOAD_FIXED_CTR0;
1596 idx = hwc->idx - INTEL_PMC_IDX_FIXED;
1597 if (x86_pmu.intel_cap.pebs_format < 5)
1598 value = ds->pebs_event_reset[MAX_PEBS_EVENTS_FMT4 + idx];
1599 else
1600 value = ds->pebs_event_reset[MAX_PEBS_EVENTS + idx];
1601 }
1602 wrmsrq(base + idx, value);
1603 }
1604
intel_pmu_drain_large_pebs(struct cpu_hw_events * cpuc)1605 static inline void intel_pmu_drain_large_pebs(struct cpu_hw_events *cpuc)
1606 {
1607 if (cpuc->n_pebs == cpuc->n_large_pebs &&
1608 cpuc->n_pebs != cpuc->n_pebs_via_pt)
1609 intel_pmu_drain_pebs_buffer();
1610 }
1611
__intel_pmu_pebs_enable(struct perf_event * event)1612 static void __intel_pmu_pebs_enable(struct perf_event *event)
1613 {
1614 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1615 struct hw_perf_event *hwc = &event->hw;
1616
1617 hwc->config &= ~ARCH_PERFMON_EVENTSEL_INT;
1618 cpuc->pebs_enabled |= 1ULL << hwc->idx;
1619 }
1620
intel_pmu_pebs_enable(struct perf_event * event)1621 void intel_pmu_pebs_enable(struct perf_event *event)
1622 {
1623 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1624 u64 pebs_data_cfg = cpuc->pebs_data_cfg & ~PEBS_UPDATE_DS_SW;
1625 struct hw_perf_event *hwc = &event->hw;
1626 struct debug_store *ds = cpuc->ds;
1627 unsigned int idx = hwc->idx;
1628
1629 __intel_pmu_pebs_enable(event);
1630
1631 if ((event->hw.flags & PERF_X86_EVENT_PEBS_LDLAT) && (x86_pmu.version < 5))
1632 cpuc->pebs_enabled |= 1ULL << (hwc->idx + 32);
1633 else if (event->hw.flags & PERF_X86_EVENT_PEBS_ST)
1634 cpuc->pebs_enabled |= 1ULL << 63;
1635
1636 if (x86_pmu.intel_cap.pebs_baseline) {
1637 hwc->config |= ICL_EVENTSEL_ADAPTIVE;
1638 if (pebs_data_cfg != cpuc->active_pebs_data_cfg) {
1639 /*
1640 * drain_pebs() assumes uniform record size;
1641 * hence we need to drain when changing said
1642 * size.
1643 */
1644 intel_pmu_drain_pebs_buffer();
1645 adaptive_pebs_record_size_update();
1646 wrmsrq(MSR_PEBS_DATA_CFG, pebs_data_cfg);
1647 cpuc->active_pebs_data_cfg = pebs_data_cfg;
1648 }
1649 }
1650 if (cpuc->pebs_data_cfg & PEBS_UPDATE_DS_SW) {
1651 cpuc->pebs_data_cfg = pebs_data_cfg;
1652 pebs_update_threshold(cpuc);
1653 }
1654
1655 if (idx >= INTEL_PMC_IDX_FIXED) {
1656 if (x86_pmu.intel_cap.pebs_format < 5)
1657 idx = MAX_PEBS_EVENTS_FMT4 + (idx - INTEL_PMC_IDX_FIXED);
1658 else
1659 idx = MAX_PEBS_EVENTS + (idx - INTEL_PMC_IDX_FIXED);
1660 }
1661
1662 /*
1663 * Use auto-reload if possible to save a MSR write in the PMI.
1664 * This must be done in pmu::start(), because PERF_EVENT_IOC_PERIOD.
1665 */
1666 if (hwc->flags & PERF_X86_EVENT_AUTO_RELOAD) {
1667 ds->pebs_event_reset[idx] =
1668 (u64)(-hwc->sample_period) & x86_pmu.cntval_mask;
1669 } else {
1670 ds->pebs_event_reset[idx] = 0;
1671 }
1672
1673 intel_pmu_pebs_via_pt_enable(event);
1674 }
1675
intel_pmu_pebs_del(struct perf_event * event)1676 void intel_pmu_pebs_del(struct perf_event *event)
1677 {
1678 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1679 struct hw_perf_event *hwc = &event->hw;
1680 bool needed_cb = pebs_needs_sched_cb(cpuc);
1681
1682 cpuc->n_pebs--;
1683 if (hwc->flags & PERF_X86_EVENT_LARGE_PEBS)
1684 cpuc->n_large_pebs--;
1685 if (hwc->flags & PERF_X86_EVENT_PEBS_VIA_PT)
1686 cpuc->n_pebs_via_pt--;
1687
1688 pebs_update_state(needed_cb, cpuc, event, false);
1689 }
1690
__intel_pmu_pebs_disable(struct perf_event * event)1691 static void __intel_pmu_pebs_disable(struct perf_event *event)
1692 {
1693 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1694 struct hw_perf_event *hwc = &event->hw;
1695
1696 intel_pmu_drain_large_pebs(cpuc);
1697 cpuc->pebs_enabled &= ~(1ULL << hwc->idx);
1698 hwc->config |= ARCH_PERFMON_EVENTSEL_INT;
1699 }
1700
intel_pmu_pebs_disable(struct perf_event * event)1701 void intel_pmu_pebs_disable(struct perf_event *event)
1702 {
1703 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1704 struct hw_perf_event *hwc = &event->hw;
1705
1706 __intel_pmu_pebs_disable(event);
1707
1708 if ((event->hw.flags & PERF_X86_EVENT_PEBS_LDLAT) &&
1709 (x86_pmu.version < 5))
1710 cpuc->pebs_enabled &= ~(1ULL << (hwc->idx + 32));
1711 else if (event->hw.flags & PERF_X86_EVENT_PEBS_ST)
1712 cpuc->pebs_enabled &= ~(1ULL << 63);
1713
1714 intel_pmu_pebs_via_pt_disable(event);
1715
1716 if (cpuc->enabled)
1717 wrmsrq(MSR_IA32_PEBS_ENABLE, cpuc->pebs_enabled);
1718 }
1719
intel_pmu_pebs_enable_all(void)1720 void intel_pmu_pebs_enable_all(void)
1721 {
1722 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1723
1724 if (cpuc->pebs_enabled)
1725 wrmsrq(MSR_IA32_PEBS_ENABLE, cpuc->pebs_enabled);
1726 }
1727
intel_pmu_pebs_disable_all(void)1728 void intel_pmu_pebs_disable_all(void)
1729 {
1730 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1731
1732 if (cpuc->pebs_enabled)
1733 __intel_pmu_pebs_disable_all();
1734 }
1735
intel_pmu_pebs_fixup_ip(struct pt_regs * regs)1736 static int intel_pmu_pebs_fixup_ip(struct pt_regs *regs)
1737 {
1738 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1739 unsigned long from = cpuc->lbr_entries[0].from;
1740 unsigned long old_to, to = cpuc->lbr_entries[0].to;
1741 unsigned long ip = regs->ip;
1742 int is_64bit = 0;
1743 void *kaddr;
1744 int size;
1745
1746 /*
1747 * We don't need to fixup if the PEBS assist is fault like
1748 */
1749 if (!x86_pmu.intel_cap.pebs_trap)
1750 return 1;
1751
1752 /*
1753 * No LBR entry, no basic block, no rewinding
1754 */
1755 if (!cpuc->lbr_stack.nr || !from || !to)
1756 return 0;
1757
1758 /*
1759 * Basic blocks should never cross user/kernel boundaries
1760 */
1761 if (kernel_ip(ip) != kernel_ip(to))
1762 return 0;
1763
1764 /*
1765 * unsigned math, either ip is before the start (impossible) or
1766 * the basic block is larger than 1 page (sanity)
1767 */
1768 if ((ip - to) > PEBS_FIXUP_SIZE)
1769 return 0;
1770
1771 /*
1772 * We sampled a branch insn, rewind using the LBR stack
1773 */
1774 if (ip == to) {
1775 set_linear_ip(regs, from);
1776 return 1;
1777 }
1778
1779 size = ip - to;
1780 if (!kernel_ip(ip)) {
1781 int bytes;
1782 u8 *buf = this_cpu_read(insn_buffer);
1783
1784 /* 'size' must fit our buffer, see above */
1785 bytes = copy_from_user_nmi(buf, (void __user *)to, size);
1786 if (bytes != 0)
1787 return 0;
1788
1789 kaddr = buf;
1790 } else {
1791 kaddr = (void *)to;
1792 }
1793
1794 do {
1795 struct insn insn;
1796
1797 old_to = to;
1798
1799 #ifdef CONFIG_X86_64
1800 is_64bit = kernel_ip(to) || any_64bit_mode(regs);
1801 #endif
1802 insn_init(&insn, kaddr, size, is_64bit);
1803
1804 /*
1805 * Make sure there was not a problem decoding the instruction.
1806 * This is doubly important because we have an infinite loop if
1807 * insn.length=0.
1808 */
1809 if (insn_get_length(&insn))
1810 break;
1811
1812 to += insn.length;
1813 kaddr += insn.length;
1814 size -= insn.length;
1815 } while (to < ip);
1816
1817 if (to == ip) {
1818 set_linear_ip(regs, old_to);
1819 return 1;
1820 }
1821
1822 /*
1823 * Even though we decoded the basic block, the instruction stream
1824 * never matched the given IP, either the TO or the IP got corrupted.
1825 */
1826 return 0;
1827 }
1828
intel_get_tsx_weight(u64 tsx_tuning)1829 static inline u64 intel_get_tsx_weight(u64 tsx_tuning)
1830 {
1831 if (tsx_tuning) {
1832 union hsw_tsx_tuning tsx = { .value = tsx_tuning };
1833 return tsx.cycles_last_block;
1834 }
1835 return 0;
1836 }
1837
intel_get_tsx_transaction(u64 tsx_tuning,u64 ax)1838 static inline u64 intel_get_tsx_transaction(u64 tsx_tuning, u64 ax)
1839 {
1840 u64 txn = (tsx_tuning & PEBS_HSW_TSX_FLAGS) >> 32;
1841
1842 /* For RTM XABORTs also log the abort code from AX */
1843 if ((txn & PERF_TXN_TRANSACTION) && (ax & 1))
1844 txn |= ((ax >> 24) & 0xff) << PERF_TXN_ABORT_SHIFT;
1845 return txn;
1846 }
1847
get_pebs_status(void * n)1848 static inline u64 get_pebs_status(void *n)
1849 {
1850 if (x86_pmu.intel_cap.pebs_format < 4)
1851 return ((struct pebs_record_nhm *)n)->status;
1852 return ((struct pebs_basic *)n)->applicable_counters;
1853 }
1854
1855 #define PERF_X86_EVENT_PEBS_HSW_PREC \
1856 (PERF_X86_EVENT_PEBS_ST_HSW | \
1857 PERF_X86_EVENT_PEBS_LD_HSW | \
1858 PERF_X86_EVENT_PEBS_NA_HSW)
1859
get_data_src(struct perf_event * event,u64 aux)1860 static u64 get_data_src(struct perf_event *event, u64 aux)
1861 {
1862 u64 val = PERF_MEM_NA;
1863 int fl = event->hw.flags;
1864 bool fst = fl & (PERF_X86_EVENT_PEBS_ST | PERF_X86_EVENT_PEBS_HSW_PREC);
1865
1866 if (fl & PERF_X86_EVENT_PEBS_LDLAT)
1867 val = load_latency_data(event, aux);
1868 else if (fl & PERF_X86_EVENT_PEBS_STLAT)
1869 val = store_latency_data(event, aux);
1870 else if (fl & PERF_X86_EVENT_PEBS_LAT_HYBRID)
1871 val = x86_pmu.pebs_latency_data(event, aux);
1872 else if (fst && (fl & PERF_X86_EVENT_PEBS_HSW_PREC))
1873 val = precise_datala_hsw(event, aux);
1874 else if (fst)
1875 val = precise_store_data(aux);
1876 return val;
1877 }
1878
setup_pebs_time(struct perf_event * event,struct perf_sample_data * data,u64 tsc)1879 static void setup_pebs_time(struct perf_event *event,
1880 struct perf_sample_data *data,
1881 u64 tsc)
1882 {
1883 /* Converting to a user-defined clock is not supported yet. */
1884 if (event->attr.use_clockid != 0)
1885 return;
1886
1887 /*
1888 * Doesn't support the conversion when the TSC is unstable.
1889 * The TSC unstable case is a corner case and very unlikely to
1890 * happen. If it happens, the TSC in a PEBS record will be
1891 * dropped and fall back to perf_event_clock().
1892 */
1893 if (!using_native_sched_clock() || !sched_clock_stable())
1894 return;
1895
1896 data->time = native_sched_clock_from_tsc(tsc) + __sched_clock_offset;
1897 data->sample_flags |= PERF_SAMPLE_TIME;
1898 }
1899
1900 #define PERF_SAMPLE_ADDR_TYPE (PERF_SAMPLE_ADDR | \
1901 PERF_SAMPLE_PHYS_ADDR | \
1902 PERF_SAMPLE_DATA_PAGE_SIZE)
1903
setup_pebs_fixed_sample_data(struct perf_event * event,struct pt_regs * iregs,void * __pebs,struct perf_sample_data * data,struct pt_regs * regs)1904 static void setup_pebs_fixed_sample_data(struct perf_event *event,
1905 struct pt_regs *iregs, void *__pebs,
1906 struct perf_sample_data *data,
1907 struct pt_regs *regs)
1908 {
1909 /*
1910 * We cast to the biggest pebs_record but are careful not to
1911 * unconditionally access the 'extra' entries.
1912 */
1913 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1914 struct pebs_record_skl *pebs = __pebs;
1915 u64 sample_type;
1916 int fll;
1917
1918 if (pebs == NULL)
1919 return;
1920
1921 sample_type = event->attr.sample_type;
1922 fll = event->hw.flags & PERF_X86_EVENT_PEBS_LDLAT;
1923
1924 perf_sample_data_init(data, 0, event->hw.last_period);
1925
1926 /*
1927 * Use latency for weight (only avail with PEBS-LL)
1928 */
1929 if (fll && (sample_type & PERF_SAMPLE_WEIGHT_TYPE)) {
1930 data->weight.full = pebs->lat;
1931 data->sample_flags |= PERF_SAMPLE_WEIGHT_TYPE;
1932 }
1933
1934 /*
1935 * data.data_src encodes the data source
1936 */
1937 if (sample_type & PERF_SAMPLE_DATA_SRC) {
1938 data->data_src.val = get_data_src(event, pebs->dse);
1939 data->sample_flags |= PERF_SAMPLE_DATA_SRC;
1940 }
1941
1942 /*
1943 * We must however always use iregs for the unwinder to stay sane; the
1944 * record BP,SP,IP can point into thin air when the record is from a
1945 * previous PMI context or an (I)RET happened between the record and
1946 * PMI.
1947 */
1948 perf_sample_save_callchain(data, event, iregs);
1949
1950 /*
1951 * We use the interrupt regs as a base because the PEBS record does not
1952 * contain a full regs set, specifically it seems to lack segment
1953 * descriptors, which get used by things like user_mode().
1954 *
1955 * In the simple case fix up only the IP for PERF_SAMPLE_IP.
1956 */
1957 *regs = *iregs;
1958
1959 /*
1960 * Initialize regs_>flags from PEBS,
1961 * Clear exact bit (which uses x86 EFLAGS Reserved bit 3),
1962 * i.e., do not rely on it being zero:
1963 */
1964 regs->flags = pebs->flags & ~PERF_EFLAGS_EXACT;
1965
1966 if (sample_type & PERF_SAMPLE_REGS_INTR) {
1967 regs->ax = pebs->ax;
1968 regs->bx = pebs->bx;
1969 regs->cx = pebs->cx;
1970 regs->dx = pebs->dx;
1971 regs->si = pebs->si;
1972 regs->di = pebs->di;
1973
1974 regs->bp = pebs->bp;
1975 regs->sp = pebs->sp;
1976
1977 #ifndef CONFIG_X86_32
1978 regs->r8 = pebs->r8;
1979 regs->r9 = pebs->r9;
1980 regs->r10 = pebs->r10;
1981 regs->r11 = pebs->r11;
1982 regs->r12 = pebs->r12;
1983 regs->r13 = pebs->r13;
1984 regs->r14 = pebs->r14;
1985 regs->r15 = pebs->r15;
1986 #endif
1987 }
1988
1989 if (event->attr.precise_ip > 1) {
1990 /*
1991 * Haswell and later processors have an 'eventing IP'
1992 * (real IP) which fixes the off-by-1 skid in hardware.
1993 * Use it when precise_ip >= 2 :
1994 */
1995 if (x86_pmu.intel_cap.pebs_format >= 2) {
1996 set_linear_ip(regs, pebs->real_ip);
1997 regs->flags |= PERF_EFLAGS_EXACT;
1998 } else {
1999 /* Otherwise, use PEBS off-by-1 IP: */
2000 set_linear_ip(regs, pebs->ip);
2001
2002 /*
2003 * With precise_ip >= 2, try to fix up the off-by-1 IP
2004 * using the LBR. If successful, the fixup function
2005 * corrects regs->ip and calls set_linear_ip() on regs:
2006 */
2007 if (intel_pmu_pebs_fixup_ip(regs))
2008 regs->flags |= PERF_EFLAGS_EXACT;
2009 }
2010 } else {
2011 /*
2012 * When precise_ip == 1, return the PEBS off-by-1 IP,
2013 * no fixup attempted:
2014 */
2015 set_linear_ip(regs, pebs->ip);
2016 }
2017
2018
2019 if ((sample_type & PERF_SAMPLE_ADDR_TYPE) &&
2020 x86_pmu.intel_cap.pebs_format >= 1) {
2021 data->addr = pebs->dla;
2022 data->sample_flags |= PERF_SAMPLE_ADDR;
2023 }
2024
2025 if (x86_pmu.intel_cap.pebs_format >= 2) {
2026 /* Only set the TSX weight when no memory weight. */
2027 if ((sample_type & PERF_SAMPLE_WEIGHT_TYPE) && !fll) {
2028 data->weight.full = intel_get_tsx_weight(pebs->tsx_tuning);
2029 data->sample_flags |= PERF_SAMPLE_WEIGHT_TYPE;
2030 }
2031 if (sample_type & PERF_SAMPLE_TRANSACTION) {
2032 data->txn = intel_get_tsx_transaction(pebs->tsx_tuning,
2033 pebs->ax);
2034 data->sample_flags |= PERF_SAMPLE_TRANSACTION;
2035 }
2036 }
2037
2038 /*
2039 * v3 supplies an accurate time stamp, so we use that
2040 * for the time stamp.
2041 *
2042 * We can only do this for the default trace clock.
2043 */
2044 if (x86_pmu.intel_cap.pebs_format >= 3)
2045 setup_pebs_time(event, data, pebs->tsc);
2046
2047 perf_sample_save_brstack(data, event, &cpuc->lbr_stack, NULL);
2048 }
2049
adaptive_pebs_save_regs(struct pt_regs * regs,struct pebs_gprs * gprs)2050 static void adaptive_pebs_save_regs(struct pt_regs *regs,
2051 struct pebs_gprs *gprs)
2052 {
2053 regs->ax = gprs->ax;
2054 regs->bx = gprs->bx;
2055 regs->cx = gprs->cx;
2056 regs->dx = gprs->dx;
2057 regs->si = gprs->si;
2058 regs->di = gprs->di;
2059 regs->bp = gprs->bp;
2060 regs->sp = gprs->sp;
2061 #ifndef CONFIG_X86_32
2062 regs->r8 = gprs->r8;
2063 regs->r9 = gprs->r9;
2064 regs->r10 = gprs->r10;
2065 regs->r11 = gprs->r11;
2066 regs->r12 = gprs->r12;
2067 regs->r13 = gprs->r13;
2068 regs->r14 = gprs->r14;
2069 regs->r15 = gprs->r15;
2070 #endif
2071 }
2072
intel_perf_event_update_pmc(struct perf_event * event,u64 pmc)2073 static void intel_perf_event_update_pmc(struct perf_event *event, u64 pmc)
2074 {
2075 int shift = 64 - x86_pmu.cntval_bits;
2076 struct hw_perf_event *hwc;
2077 u64 delta, prev_pmc;
2078
2079 /*
2080 * A recorded counter may not have an assigned event in the
2081 * following cases. The value should be dropped.
2082 * - An event is deleted. There is still an active PEBS event.
2083 * The PEBS record doesn't shrink on pmu::del().
2084 * If the counter of the deleted event once occurred in a PEBS
2085 * record, PEBS still records the counter until the counter is
2086 * reassigned.
2087 * - An event is stopped for some reason, e.g., throttled.
2088 * During this period, another event is added and takes the
2089 * counter of the stopped event. The stopped event is assigned
2090 * to another new and uninitialized counter, since the
2091 * x86_pmu_start(RELOAD) is not invoked for a stopped event.
2092 * The PEBS__DATA_CFG is updated regardless of the event state.
2093 * The uninitialized counter can be recorded in a PEBS record.
2094 * But the cpuc->events[uninitialized_counter] is always NULL,
2095 * because the event is stopped. The uninitialized value is
2096 * safely dropped.
2097 */
2098 if (!event)
2099 return;
2100
2101 hwc = &event->hw;
2102 prev_pmc = local64_read(&hwc->prev_count);
2103
2104 /* Only update the count when the PMU is disabled */
2105 WARN_ON(this_cpu_read(cpu_hw_events.enabled));
2106 local64_set(&hwc->prev_count, pmc);
2107
2108 delta = (pmc << shift) - (prev_pmc << shift);
2109 delta >>= shift;
2110
2111 local64_add(delta, &event->count);
2112 local64_sub(delta, &hwc->period_left);
2113 }
2114
__setup_pebs_counter_group(struct cpu_hw_events * cpuc,struct perf_event * event,struct pebs_cntr_header * cntr,void * next_record)2115 static inline void __setup_pebs_counter_group(struct cpu_hw_events *cpuc,
2116 struct perf_event *event,
2117 struct pebs_cntr_header *cntr,
2118 void *next_record)
2119 {
2120 int bit;
2121
2122 for_each_set_bit(bit, (unsigned long *)&cntr->cntr, INTEL_PMC_MAX_GENERIC) {
2123 intel_perf_event_update_pmc(cpuc->events[bit], *(u64 *)next_record);
2124 next_record += sizeof(u64);
2125 }
2126
2127 for_each_set_bit(bit, (unsigned long *)&cntr->fixed, INTEL_PMC_MAX_FIXED) {
2128 /* The slots event will be handled with perf_metric later */
2129 if ((cntr->metrics == INTEL_CNTR_METRICS) &&
2130 (bit + INTEL_PMC_IDX_FIXED == INTEL_PMC_IDX_FIXED_SLOTS)) {
2131 next_record += sizeof(u64);
2132 continue;
2133 }
2134 intel_perf_event_update_pmc(cpuc->events[bit + INTEL_PMC_IDX_FIXED],
2135 *(u64 *)next_record);
2136 next_record += sizeof(u64);
2137 }
2138
2139 /* HW will reload the value right after the overflow. */
2140 if (event->hw.flags & PERF_X86_EVENT_AUTO_RELOAD)
2141 local64_set(&event->hw.prev_count, (u64)-event->hw.sample_period);
2142
2143 if (cntr->metrics == INTEL_CNTR_METRICS) {
2144 static_call(intel_pmu_update_topdown_event)
2145 (cpuc->events[INTEL_PMC_IDX_FIXED_SLOTS],
2146 (u64 *)next_record);
2147 next_record += 2 * sizeof(u64);
2148 }
2149 }
2150
2151 #define PEBS_LATENCY_MASK 0xffff
2152
__setup_perf_sample_data(struct perf_event * event,struct pt_regs * iregs,struct perf_sample_data * data)2153 static inline void __setup_perf_sample_data(struct perf_event *event,
2154 struct pt_regs *iregs,
2155 struct perf_sample_data *data)
2156 {
2157 perf_sample_data_init(data, 0, event->hw.last_period);
2158
2159 /*
2160 * We must however always use iregs for the unwinder to stay sane; the
2161 * record BP,SP,IP can point into thin air when the record is from a
2162 * previous PMI context or an (I)RET happened between the record and
2163 * PMI.
2164 */
2165 perf_sample_save_callchain(data, event, iregs);
2166 }
2167
__setup_pebs_basic_group(struct perf_event * event,struct pt_regs * regs,struct perf_sample_data * data,u64 sample_type,u64 ip,u64 tsc,u16 retire)2168 static inline void __setup_pebs_basic_group(struct perf_event *event,
2169 struct pt_regs *regs,
2170 struct perf_sample_data *data,
2171 u64 sample_type, u64 ip,
2172 u64 tsc, u16 retire)
2173 {
2174 /* The ip in basic is EventingIP */
2175 set_linear_ip(regs, ip);
2176 regs->flags = PERF_EFLAGS_EXACT;
2177 setup_pebs_time(event, data, tsc);
2178
2179 if (sample_type & PERF_SAMPLE_WEIGHT_STRUCT)
2180 data->weight.var3_w = retire;
2181 }
2182
__setup_pebs_gpr_group(struct perf_event * event,struct pt_regs * regs,struct pebs_gprs * gprs,u64 sample_type)2183 static inline void __setup_pebs_gpr_group(struct perf_event *event,
2184 struct pt_regs *regs,
2185 struct pebs_gprs *gprs,
2186 u64 sample_type)
2187 {
2188 if (event->attr.precise_ip < 2) {
2189 set_linear_ip(regs, gprs->ip);
2190 regs->flags &= ~PERF_EFLAGS_EXACT;
2191 }
2192
2193 if (sample_type & (PERF_SAMPLE_REGS_INTR | PERF_SAMPLE_REGS_USER))
2194 adaptive_pebs_save_regs(regs, gprs);
2195 }
2196
__setup_pebs_meminfo_group(struct perf_event * event,struct perf_sample_data * data,u64 sample_type,u64 latency,u16 instr_latency,u64 address,u64 aux,u64 tsx_tuning,u64 ax)2197 static inline void __setup_pebs_meminfo_group(struct perf_event *event,
2198 struct perf_sample_data *data,
2199 u64 sample_type, u64 latency,
2200 u16 instr_latency, u64 address,
2201 u64 aux, u64 tsx_tuning, u64 ax)
2202 {
2203 if (sample_type & PERF_SAMPLE_WEIGHT_TYPE) {
2204 u64 tsx_latency = intel_get_tsx_weight(tsx_tuning);
2205
2206 data->weight.var2_w = instr_latency;
2207
2208 /*
2209 * Although meminfo::latency is defined as a u64,
2210 * only the lower 32 bits include the valid data
2211 * in practice on Ice Lake and earlier platforms.
2212 */
2213 if (sample_type & PERF_SAMPLE_WEIGHT)
2214 data->weight.full = latency ?: tsx_latency;
2215 else
2216 data->weight.var1_dw = (u32)latency ?: tsx_latency;
2217
2218 data->sample_flags |= PERF_SAMPLE_WEIGHT_TYPE;
2219 }
2220
2221 if (sample_type & PERF_SAMPLE_DATA_SRC) {
2222 data->data_src.val = get_data_src(event, aux);
2223 data->sample_flags |= PERF_SAMPLE_DATA_SRC;
2224 }
2225
2226 if (sample_type & PERF_SAMPLE_ADDR_TYPE) {
2227 data->addr = address;
2228 data->sample_flags |= PERF_SAMPLE_ADDR;
2229 }
2230
2231 if (sample_type & PERF_SAMPLE_TRANSACTION) {
2232 data->txn = intel_get_tsx_transaction(tsx_tuning, ax);
2233 data->sample_flags |= PERF_SAMPLE_TRANSACTION;
2234 }
2235 }
2236
2237 /*
2238 * With adaptive PEBS the layout depends on what fields are configured.
2239 */
setup_pebs_adaptive_sample_data(struct perf_event * event,struct pt_regs * iregs,void * __pebs,struct perf_sample_data * data,struct pt_regs * regs)2240 static void setup_pebs_adaptive_sample_data(struct perf_event *event,
2241 struct pt_regs *iregs, void *__pebs,
2242 struct perf_sample_data *data,
2243 struct pt_regs *regs)
2244 {
2245 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
2246 u64 sample_type = event->attr.sample_type;
2247 struct pebs_basic *basic = __pebs;
2248 void *next_record = basic + 1;
2249 struct pebs_meminfo *meminfo = NULL;
2250 struct pebs_gprs *gprs = NULL;
2251 struct x86_perf_regs *perf_regs;
2252 u64 format_group;
2253 u16 retire;
2254
2255 if (basic == NULL)
2256 return;
2257
2258 perf_regs = container_of(regs, struct x86_perf_regs, regs);
2259 perf_regs->xmm_regs = NULL;
2260
2261 format_group = basic->format_group;
2262
2263 __setup_perf_sample_data(event, iregs, data);
2264
2265 *regs = *iregs;
2266
2267 /* basic group */
2268 retire = x86_pmu.flags & PMU_FL_RETIRE_LATENCY ?
2269 basic->retire_latency : 0;
2270 __setup_pebs_basic_group(event, regs, data, sample_type,
2271 basic->ip, basic->tsc, retire);
2272
2273 /*
2274 * The record for MEMINFO is in front of GP
2275 * But PERF_SAMPLE_TRANSACTION needs gprs->ax.
2276 * Save the pointer here but process later.
2277 */
2278 if (format_group & PEBS_DATACFG_MEMINFO) {
2279 meminfo = next_record;
2280 next_record = meminfo + 1;
2281 }
2282
2283 if (format_group & PEBS_DATACFG_GP) {
2284 gprs = next_record;
2285 next_record = gprs + 1;
2286
2287 __setup_pebs_gpr_group(event, regs, gprs, sample_type);
2288 }
2289
2290 if (format_group & PEBS_DATACFG_MEMINFO) {
2291 u64 latency = x86_pmu.flags & PMU_FL_INSTR_LATENCY ?
2292 meminfo->cache_latency : meminfo->mem_latency;
2293 u64 instr_latency = x86_pmu.flags & PMU_FL_INSTR_LATENCY ?
2294 meminfo->instr_latency : 0;
2295 u64 ax = gprs ? gprs->ax : 0;
2296
2297 __setup_pebs_meminfo_group(event, data, sample_type, latency,
2298 instr_latency, meminfo->address,
2299 meminfo->aux, meminfo->tsx_tuning,
2300 ax);
2301 }
2302
2303 if (format_group & PEBS_DATACFG_XMMS) {
2304 struct pebs_xmm *xmm = next_record;
2305
2306 next_record = xmm + 1;
2307 perf_regs->xmm_regs = xmm->xmm;
2308 }
2309
2310 if (format_group & PEBS_DATACFG_LBRS) {
2311 struct lbr_entry *lbr = next_record;
2312 int num_lbr = ((format_group >> PEBS_DATACFG_LBR_SHIFT)
2313 & 0xff) + 1;
2314 next_record = next_record + num_lbr * sizeof(struct lbr_entry);
2315
2316 if (has_branch_stack(event)) {
2317 intel_pmu_store_pebs_lbrs(lbr);
2318 intel_pmu_lbr_save_brstack(data, cpuc, event);
2319 }
2320 }
2321
2322 if (format_group & (PEBS_DATACFG_CNTR | PEBS_DATACFG_METRICS)) {
2323 struct pebs_cntr_header *cntr = next_record;
2324 unsigned int nr;
2325
2326 next_record += sizeof(struct pebs_cntr_header);
2327 /*
2328 * The PEBS_DATA_CFG is a global register, which is the
2329 * superset configuration for all PEBS events.
2330 * For the PEBS record of non-sample-read group, ignore
2331 * the counter snapshot fields.
2332 */
2333 if (is_pebs_counter_event_group(event)) {
2334 __setup_pebs_counter_group(cpuc, event, cntr, next_record);
2335 data->sample_flags |= PERF_SAMPLE_READ;
2336 }
2337
2338 nr = hweight32(cntr->cntr) + hweight32(cntr->fixed);
2339 if (cntr->metrics == INTEL_CNTR_METRICS)
2340 nr += 2;
2341 next_record += nr * sizeof(u64);
2342 }
2343
2344 WARN_ONCE(next_record != __pebs + basic->format_size,
2345 "PEBS record size %u, expected %llu, config %llx\n",
2346 basic->format_size,
2347 (u64)(next_record - __pebs),
2348 format_group);
2349 }
2350
arch_pebs_record_continued(struct arch_pebs_header * header)2351 static inline bool arch_pebs_record_continued(struct arch_pebs_header *header)
2352 {
2353 /* Continue bit or null PEBS record indicates fragment follows. */
2354 return header->cont || !(header->format & GENMASK_ULL(63, 16));
2355 }
2356
setup_arch_pebs_sample_data(struct perf_event * event,struct pt_regs * iregs,void * __pebs,struct perf_sample_data * data,struct pt_regs * regs)2357 static void setup_arch_pebs_sample_data(struct perf_event *event,
2358 struct pt_regs *iregs,
2359 void *__pebs,
2360 struct perf_sample_data *data,
2361 struct pt_regs *regs)
2362 {
2363 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
2364 u64 sample_type = event->attr.sample_type;
2365 struct arch_pebs_header *header = NULL;
2366 struct arch_pebs_aux *meminfo = NULL;
2367 struct arch_pebs_gprs *gprs = NULL;
2368 struct x86_perf_regs *perf_regs;
2369 void *next_record;
2370 void *at = __pebs;
2371
2372 if (at == NULL)
2373 return;
2374
2375 perf_regs = container_of(regs, struct x86_perf_regs, regs);
2376 perf_regs->xmm_regs = NULL;
2377
2378 __setup_perf_sample_data(event, iregs, data);
2379
2380 *regs = *iregs;
2381
2382 again:
2383 header = at;
2384 next_record = at + sizeof(struct arch_pebs_header);
2385 if (header->basic) {
2386 struct arch_pebs_basic *basic = next_record;
2387 u16 retire = 0;
2388
2389 next_record = basic + 1;
2390
2391 if (sample_type & PERF_SAMPLE_WEIGHT_STRUCT)
2392 retire = basic->valid ? basic->retire : 0;
2393 __setup_pebs_basic_group(event, regs, data, sample_type,
2394 basic->ip, basic->tsc, retire);
2395 }
2396
2397 /*
2398 * The record for MEMINFO is in front of GP
2399 * But PERF_SAMPLE_TRANSACTION needs gprs->ax.
2400 * Save the pointer here but process later.
2401 */
2402 if (header->aux) {
2403 meminfo = next_record;
2404 next_record = meminfo + 1;
2405 }
2406
2407 if (header->gpr) {
2408 gprs = next_record;
2409 next_record = gprs + 1;
2410
2411 __setup_pebs_gpr_group(event, regs,
2412 (struct pebs_gprs *)gprs,
2413 sample_type);
2414 }
2415
2416 if (header->aux) {
2417 u64 ax = gprs ? gprs->ax : 0;
2418
2419 __setup_pebs_meminfo_group(event, data, sample_type,
2420 meminfo->cache_latency,
2421 meminfo->instr_latency,
2422 meminfo->address, meminfo->aux,
2423 meminfo->tsx_tuning, ax);
2424 }
2425
2426 if (header->xmm) {
2427 struct pebs_xmm *xmm;
2428
2429 next_record += sizeof(struct arch_pebs_xer_header);
2430
2431 xmm = next_record;
2432 perf_regs->xmm_regs = xmm->xmm;
2433 next_record = xmm + 1;
2434 }
2435
2436 if (header->lbr) {
2437 struct arch_pebs_lbr_header *lbr_header = next_record;
2438 struct lbr_entry *lbr;
2439 int num_lbr;
2440
2441 next_record = lbr_header + 1;
2442 lbr = next_record;
2443
2444 num_lbr = header->lbr == ARCH_PEBS_LBR_NUM_VAR ?
2445 lbr_header->depth :
2446 header->lbr * ARCH_PEBS_BASE_LBR_ENTRIES;
2447 next_record += num_lbr * sizeof(struct lbr_entry);
2448
2449 if (has_branch_stack(event)) {
2450 intel_pmu_store_pebs_lbrs(lbr);
2451 intel_pmu_lbr_save_brstack(data, cpuc, event);
2452 }
2453 }
2454
2455 if (header->cntr) {
2456 struct arch_pebs_cntr_header *cntr = next_record;
2457 unsigned int nr;
2458
2459 next_record += sizeof(struct arch_pebs_cntr_header);
2460
2461 if (is_pebs_counter_event_group(event)) {
2462 __setup_pebs_counter_group(cpuc, event,
2463 (struct pebs_cntr_header *)cntr, next_record);
2464 data->sample_flags |= PERF_SAMPLE_READ;
2465 }
2466
2467 nr = hweight32(cntr->cntr) + hweight32(cntr->fixed);
2468 if (cntr->metrics == INTEL_CNTR_METRICS)
2469 nr += 2;
2470 next_record += nr * sizeof(u64);
2471 }
2472
2473 /* Parse followed fragments if there are. */
2474 if (arch_pebs_record_continued(header)) {
2475 at = at + header->size;
2476 goto again;
2477 }
2478 }
2479
2480 static inline void *
get_next_pebs_record_by_bit(void * base,void * top,int bit)2481 get_next_pebs_record_by_bit(void *base, void *top, int bit)
2482 {
2483 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
2484 void *at;
2485 u64 pebs_status;
2486
2487 /*
2488 * fmt0 does not have a status bitfield (does not use
2489 * perf_record_nhm format)
2490 */
2491 if (x86_pmu.intel_cap.pebs_format < 1)
2492 return base;
2493
2494 if (base == NULL)
2495 return NULL;
2496
2497 for (at = base; at < top; at += cpuc->pebs_record_size) {
2498 unsigned long status = get_pebs_status(at);
2499
2500 if (test_bit(bit, (unsigned long *)&status)) {
2501 /* PEBS v3 has accurate status bits */
2502 if (x86_pmu.intel_cap.pebs_format >= 3)
2503 return at;
2504
2505 if (status == (1 << bit))
2506 return at;
2507
2508 /* clear non-PEBS bit and re-check */
2509 pebs_status = status & cpuc->pebs_enabled;
2510 pebs_status &= PEBS_COUNTER_MASK;
2511 if (pebs_status == (1 << bit))
2512 return at;
2513 }
2514 }
2515 return NULL;
2516 }
2517
2518 /*
2519 * Special variant of intel_pmu_save_and_restart() for auto-reload.
2520 */
2521 static int
intel_pmu_save_and_restart_reload(struct perf_event * event,int count)2522 intel_pmu_save_and_restart_reload(struct perf_event *event, int count)
2523 {
2524 struct hw_perf_event *hwc = &event->hw;
2525 int shift = 64 - x86_pmu.cntval_bits;
2526 u64 period = hwc->sample_period;
2527 u64 prev_raw_count, new_raw_count;
2528 s64 new, old;
2529
2530 WARN_ON(!period);
2531
2532 /*
2533 * drain_pebs() only happens when the PMU is disabled.
2534 */
2535 WARN_ON(this_cpu_read(cpu_hw_events.enabled));
2536
2537 prev_raw_count = local64_read(&hwc->prev_count);
2538 new_raw_count = rdpmc(hwc->event_base_rdpmc);
2539 local64_set(&hwc->prev_count, new_raw_count);
2540
2541 /*
2542 * Since the counter increments a negative counter value and
2543 * overflows on the sign switch, giving the interval:
2544 *
2545 * [-period, 0]
2546 *
2547 * the difference between two consecutive reads is:
2548 *
2549 * A) value2 - value1;
2550 * when no overflows have happened in between,
2551 *
2552 * B) (0 - value1) + (value2 - (-period));
2553 * when one overflow happened in between,
2554 *
2555 * C) (0 - value1) + (n - 1) * (period) + (value2 - (-period));
2556 * when @n overflows happened in between.
2557 *
2558 * Here A) is the obvious difference, B) is the extension to the
2559 * discrete interval, where the first term is to the top of the
2560 * interval and the second term is from the bottom of the next
2561 * interval and C) the extension to multiple intervals, where the
2562 * middle term is the whole intervals covered.
2563 *
2564 * An equivalent of C, by reduction, is:
2565 *
2566 * value2 - value1 + n * period
2567 */
2568 new = ((s64)(new_raw_count << shift) >> shift);
2569 old = ((s64)(prev_raw_count << shift) >> shift);
2570 local64_add(new - old + count * period, &event->count);
2571
2572 local64_set(&hwc->period_left, -new);
2573
2574 perf_event_update_userpage(event);
2575
2576 return 0;
2577 }
2578
2579 typedef void (*setup_fn)(struct perf_event *, struct pt_regs *, void *,
2580 struct perf_sample_data *, struct pt_regs *);
2581
2582 static struct pt_regs dummy_iregs;
2583
2584 static __always_inline void
__intel_pmu_pebs_event(struct perf_event * event,struct pt_regs * iregs,struct pt_regs * regs,struct perf_sample_data * data,void * at,setup_fn setup_sample)2585 __intel_pmu_pebs_event(struct perf_event *event,
2586 struct pt_regs *iregs,
2587 struct pt_regs *regs,
2588 struct perf_sample_data *data,
2589 void *at,
2590 setup_fn setup_sample)
2591 {
2592 setup_sample(event, iregs, at, data, regs);
2593 perf_event_output(event, data, regs);
2594 }
2595
2596 static __always_inline void
__intel_pmu_pebs_last_event(struct perf_event * event,struct pt_regs * iregs,struct pt_regs * regs,struct perf_sample_data * data,void * at,int count,setup_fn setup_sample)2597 __intel_pmu_pebs_last_event(struct perf_event *event,
2598 struct pt_regs *iregs,
2599 struct pt_regs *regs,
2600 struct perf_sample_data *data,
2601 void *at,
2602 int count,
2603 setup_fn setup_sample)
2604 {
2605 struct hw_perf_event *hwc = &event->hw;
2606
2607 setup_sample(event, iregs, at, data, regs);
2608 if (iregs == &dummy_iregs) {
2609 /*
2610 * The PEBS records may be drained in the non-overflow context,
2611 * e.g., large PEBS + context switch. Perf should treat the
2612 * last record the same as other PEBS records, and doesn't
2613 * invoke the generic overflow handler.
2614 */
2615 perf_event_output(event, data, regs);
2616 } else {
2617 /*
2618 * All but the last records are processed.
2619 * The last one is left to be able to call the overflow handler.
2620 */
2621 perf_event_overflow(event, data, regs);
2622 }
2623
2624 if (hwc->flags & PERF_X86_EVENT_AUTO_RELOAD) {
2625 if ((is_pebs_counter_event_group(event))) {
2626 /*
2627 * The value of each sample has been updated when setup
2628 * the corresponding sample data.
2629 */
2630 perf_event_update_userpage(event);
2631 } else {
2632 /*
2633 * Now, auto-reload is only enabled in fixed period mode.
2634 * The reload value is always hwc->sample_period.
2635 * May need to change it, if auto-reload is enabled in
2636 * freq mode later.
2637 */
2638 intel_pmu_save_and_restart_reload(event, count);
2639 }
2640 } else {
2641 /*
2642 * For a non-precise event, it's possible the
2643 * counters-snapshotting records a positive value for the
2644 * overflowed event. Then the HW auto-reload mechanism
2645 * reset the counter to 0 immediately, because the
2646 * pebs_event_reset is cleared if the PERF_X86_EVENT_AUTO_RELOAD
2647 * is not set. The counter backwards may be observed in a
2648 * PMI handler.
2649 *
2650 * Since the event value has been updated when processing the
2651 * counters-snapshotting record, only needs to set the new
2652 * period for the counter.
2653 */
2654 if (is_pebs_counter_event_group(event))
2655 static_call(x86_pmu_set_period)(event);
2656 else
2657 intel_pmu_save_and_restart(event);
2658 }
2659 }
2660
2661 static __always_inline void
__intel_pmu_pebs_events(struct perf_event * event,struct pt_regs * iregs,struct perf_sample_data * data,void * base,void * top,int bit,int count,setup_fn setup_sample)2662 __intel_pmu_pebs_events(struct perf_event *event,
2663 struct pt_regs *iregs,
2664 struct perf_sample_data *data,
2665 void *base, void *top,
2666 int bit, int count,
2667 setup_fn setup_sample)
2668 {
2669 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
2670 struct x86_perf_regs perf_regs;
2671 struct pt_regs *regs = &perf_regs.regs;
2672 void *at = get_next_pebs_record_by_bit(base, top, bit);
2673 int cnt = count;
2674
2675 if (!iregs)
2676 iregs = &dummy_iregs;
2677
2678 while (cnt > 1) {
2679 __intel_pmu_pebs_event(event, iregs, regs, data, at, setup_sample);
2680 at += cpuc->pebs_record_size;
2681 at = get_next_pebs_record_by_bit(at, top, bit);
2682 cnt--;
2683 }
2684
2685 __intel_pmu_pebs_last_event(event, iregs, regs, data, at, count, setup_sample);
2686 }
2687
intel_pmu_drain_pebs_core(struct pt_regs * iregs,struct perf_sample_data * data)2688 static void intel_pmu_drain_pebs_core(struct pt_regs *iregs, struct perf_sample_data *data)
2689 {
2690 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
2691 struct debug_store *ds = cpuc->ds;
2692 struct perf_event *event = cpuc->events[0]; /* PMC0 only */
2693 struct pebs_record_core *at, *top;
2694 int n;
2695
2696 if (!x86_pmu.pebs_active)
2697 return;
2698
2699 at = (struct pebs_record_core *)(unsigned long)ds->pebs_buffer_base;
2700 top = (struct pebs_record_core *)(unsigned long)ds->pebs_index;
2701
2702 /*
2703 * Whatever else happens, drain the thing
2704 */
2705 ds->pebs_index = ds->pebs_buffer_base;
2706
2707 if (!test_bit(0, cpuc->active_mask))
2708 return;
2709
2710 WARN_ON_ONCE(!event);
2711
2712 if (!event->attr.precise_ip)
2713 return;
2714
2715 n = top - at;
2716 if (n <= 0) {
2717 if (event->hw.flags & PERF_X86_EVENT_AUTO_RELOAD)
2718 intel_pmu_save_and_restart_reload(event, 0);
2719 return;
2720 }
2721
2722 __intel_pmu_pebs_events(event, iregs, data, at, top, 0, n,
2723 setup_pebs_fixed_sample_data);
2724 }
2725
intel_pmu_pebs_event_update_no_drain(struct cpu_hw_events * cpuc,u64 mask)2726 static void intel_pmu_pebs_event_update_no_drain(struct cpu_hw_events *cpuc, u64 mask)
2727 {
2728 u64 pebs_enabled = cpuc->pebs_enabled & mask;
2729 struct perf_event *event;
2730 int bit;
2731
2732 /*
2733 * The drain_pebs() could be called twice in a short period
2734 * for auto-reload event in pmu::read(). There are no
2735 * overflows have happened in between.
2736 * It needs to call intel_pmu_save_and_restart_reload() to
2737 * update the event->count for this case.
2738 */
2739 for_each_set_bit(bit, (unsigned long *)&pebs_enabled, X86_PMC_IDX_MAX) {
2740 event = cpuc->events[bit];
2741 if (event->hw.flags & PERF_X86_EVENT_AUTO_RELOAD)
2742 intel_pmu_save_and_restart_reload(event, 0);
2743 }
2744 }
2745
intel_pmu_drain_pebs_nhm(struct pt_regs * iregs,struct perf_sample_data * data)2746 static void intel_pmu_drain_pebs_nhm(struct pt_regs *iregs, struct perf_sample_data *data)
2747 {
2748 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
2749 struct debug_store *ds = cpuc->ds;
2750 struct perf_event *event;
2751 void *base, *at, *top;
2752 short counts[INTEL_PMC_IDX_FIXED + MAX_FIXED_PEBS_EVENTS] = {};
2753 short error[INTEL_PMC_IDX_FIXED + MAX_FIXED_PEBS_EVENTS] = {};
2754 int max_pebs_events = intel_pmu_max_num_pebs(NULL);
2755 int bit, i, size;
2756 u64 mask;
2757
2758 if (!x86_pmu.pebs_active)
2759 return;
2760
2761 base = (struct pebs_record_nhm *)(unsigned long)ds->pebs_buffer_base;
2762 top = (struct pebs_record_nhm *)(unsigned long)ds->pebs_index;
2763
2764 ds->pebs_index = ds->pebs_buffer_base;
2765
2766 mask = x86_pmu.pebs_events_mask;
2767 size = max_pebs_events;
2768 if (x86_pmu.flags & PMU_FL_PEBS_ALL) {
2769 mask |= x86_pmu.fixed_cntr_mask64 << INTEL_PMC_IDX_FIXED;
2770 size = INTEL_PMC_IDX_FIXED + x86_pmu_max_num_counters_fixed(NULL);
2771 }
2772
2773 if (unlikely(base >= top)) {
2774 intel_pmu_pebs_event_update_no_drain(cpuc, mask);
2775 return;
2776 }
2777
2778 for (at = base; at < top; at += x86_pmu.pebs_record_size) {
2779 struct pebs_record_nhm *p = at;
2780 u64 pebs_status;
2781
2782 pebs_status = p->status & cpuc->pebs_enabled;
2783 pebs_status &= mask;
2784
2785 /* PEBS v3 has more accurate status bits */
2786 if (x86_pmu.intel_cap.pebs_format >= 3) {
2787 for_each_set_bit(bit, (unsigned long *)&pebs_status, size)
2788 counts[bit]++;
2789
2790 continue;
2791 }
2792
2793 /*
2794 * On some CPUs the PEBS status can be zero when PEBS is
2795 * racing with clearing of GLOBAL_STATUS.
2796 *
2797 * Normally we would drop that record, but in the
2798 * case when there is only a single active PEBS event
2799 * we can assume it's for that event.
2800 */
2801 if (!pebs_status && cpuc->pebs_enabled &&
2802 !(cpuc->pebs_enabled & (cpuc->pebs_enabled-1)))
2803 pebs_status = p->status = cpuc->pebs_enabled;
2804
2805 bit = find_first_bit((unsigned long *)&pebs_status,
2806 max_pebs_events);
2807
2808 if (!(x86_pmu.pebs_events_mask & (1 << bit)))
2809 continue;
2810
2811 /*
2812 * The PEBS hardware does not deal well with the situation
2813 * when events happen near to each other and multiple bits
2814 * are set. But it should happen rarely.
2815 *
2816 * If these events include one PEBS and multiple non-PEBS
2817 * events, it doesn't impact PEBS record. The record will
2818 * be handled normally. (slow path)
2819 *
2820 * If these events include two or more PEBS events, the
2821 * records for the events can be collapsed into a single
2822 * one, and it's not possible to reconstruct all events
2823 * that caused the PEBS record. It's called collision.
2824 * If collision happened, the record will be dropped.
2825 */
2826 if (pebs_status != (1ULL << bit)) {
2827 for_each_set_bit(i, (unsigned long *)&pebs_status, size)
2828 error[i]++;
2829 continue;
2830 }
2831
2832 counts[bit]++;
2833 }
2834
2835 for_each_set_bit(bit, (unsigned long *)&mask, size) {
2836 if ((counts[bit] == 0) && (error[bit] == 0))
2837 continue;
2838
2839 event = cpuc->events[bit];
2840 if (WARN_ON_ONCE(!event))
2841 continue;
2842
2843 if (WARN_ON_ONCE(!event->attr.precise_ip))
2844 continue;
2845
2846 /* log dropped samples number */
2847 if (error[bit]) {
2848 perf_log_lost_samples(event, error[bit]);
2849
2850 if (iregs)
2851 perf_event_account_interrupt(event);
2852 }
2853
2854 if (counts[bit]) {
2855 __intel_pmu_pebs_events(event, iregs, data, base,
2856 top, bit, counts[bit],
2857 setup_pebs_fixed_sample_data);
2858 }
2859 }
2860 }
2861
2862 static __always_inline void
__intel_pmu_handle_pebs_record(struct pt_regs * iregs,struct pt_regs * regs,struct perf_sample_data * data,void * at,u64 pebs_status,short * counts,void ** last,setup_fn setup_sample)2863 __intel_pmu_handle_pebs_record(struct pt_regs *iregs,
2864 struct pt_regs *regs,
2865 struct perf_sample_data *data,
2866 void *at, u64 pebs_status,
2867 short *counts, void **last,
2868 setup_fn setup_sample)
2869 {
2870 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
2871 struct perf_event *event;
2872 int bit;
2873
2874 for_each_set_bit(bit, (unsigned long *)&pebs_status, X86_PMC_IDX_MAX) {
2875 event = cpuc->events[bit];
2876
2877 if (WARN_ON_ONCE(!event) ||
2878 WARN_ON_ONCE(!event->attr.precise_ip))
2879 continue;
2880
2881 if (counts[bit]++) {
2882 __intel_pmu_pebs_event(event, iregs, regs, data,
2883 last[bit], setup_sample);
2884 }
2885
2886 last[bit] = at;
2887 }
2888 }
2889
2890 static __always_inline void
__intel_pmu_handle_last_pebs_record(struct pt_regs * iregs,struct pt_regs * regs,struct perf_sample_data * data,u64 mask,short * counts,void ** last,setup_fn setup_sample)2891 __intel_pmu_handle_last_pebs_record(struct pt_regs *iregs,
2892 struct pt_regs *regs,
2893 struct perf_sample_data *data,
2894 u64 mask, short *counts, void **last,
2895 setup_fn setup_sample)
2896 {
2897 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
2898 struct perf_event *event;
2899 int bit;
2900
2901 for_each_set_bit(bit, (unsigned long *)&mask, X86_PMC_IDX_MAX) {
2902 if (!counts[bit])
2903 continue;
2904
2905 event = cpuc->events[bit];
2906
2907 __intel_pmu_pebs_last_event(event, iregs, regs, data, last[bit],
2908 counts[bit], setup_sample);
2909 }
2910
2911 }
2912
intel_pmu_drain_pebs_icl(struct pt_regs * iregs,struct perf_sample_data * data)2913 static void intel_pmu_drain_pebs_icl(struct pt_regs *iregs, struct perf_sample_data *data)
2914 {
2915 short counts[INTEL_PMC_IDX_FIXED + MAX_FIXED_PEBS_EVENTS] = {};
2916 void *last[INTEL_PMC_IDX_FIXED + MAX_FIXED_PEBS_EVENTS];
2917 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
2918 struct debug_store *ds = cpuc->ds;
2919 struct x86_perf_regs perf_regs;
2920 struct pt_regs *regs = &perf_regs.regs;
2921 struct pebs_basic *basic;
2922 void *base, *at, *top;
2923 u64 mask;
2924
2925 if (!x86_pmu.pebs_active)
2926 return;
2927
2928 base = (struct pebs_basic *)(unsigned long)ds->pebs_buffer_base;
2929 top = (struct pebs_basic *)(unsigned long)ds->pebs_index;
2930
2931 ds->pebs_index = ds->pebs_buffer_base;
2932
2933 mask = hybrid(cpuc->pmu, pebs_events_mask) |
2934 (hybrid(cpuc->pmu, fixed_cntr_mask64) << INTEL_PMC_IDX_FIXED);
2935 mask &= cpuc->pebs_enabled;
2936
2937 if (unlikely(base >= top)) {
2938 intel_pmu_pebs_event_update_no_drain(cpuc, mask);
2939 return;
2940 }
2941
2942 if (!iregs)
2943 iregs = &dummy_iregs;
2944
2945 /* Process all but the last event for each counter. */
2946 for (at = base; at < top; at += basic->format_size) {
2947 u64 pebs_status;
2948
2949 basic = at;
2950 if (basic->format_size != cpuc->pebs_record_size)
2951 continue;
2952
2953 pebs_status = mask & basic->applicable_counters;
2954 __intel_pmu_handle_pebs_record(iregs, regs, data, at,
2955 pebs_status, counts, last,
2956 setup_pebs_adaptive_sample_data);
2957 }
2958
2959 __intel_pmu_handle_last_pebs_record(iregs, regs, data, mask, counts, last,
2960 setup_pebs_adaptive_sample_data);
2961 }
2962
intel_pmu_drain_arch_pebs(struct pt_regs * iregs,struct perf_sample_data * data)2963 static void intel_pmu_drain_arch_pebs(struct pt_regs *iregs,
2964 struct perf_sample_data *data)
2965 {
2966 short counts[INTEL_PMC_IDX_FIXED + MAX_FIXED_PEBS_EVENTS] = {};
2967 void *last[INTEL_PMC_IDX_FIXED + MAX_FIXED_PEBS_EVENTS];
2968 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
2969 union arch_pebs_index index;
2970 struct x86_perf_regs perf_regs;
2971 struct pt_regs *regs = &perf_regs.regs;
2972 void *base, *at, *top;
2973 u64 mask;
2974
2975 rdmsrq(MSR_IA32_PEBS_INDEX, index.whole);
2976
2977 if (unlikely(!index.wr)) {
2978 intel_pmu_pebs_event_update_no_drain(cpuc, X86_PMC_IDX_MAX);
2979 return;
2980 }
2981
2982 base = cpuc->pebs_vaddr;
2983 top = cpuc->pebs_vaddr + (index.wr << ARCH_PEBS_INDEX_WR_SHIFT);
2984
2985 index.wr = 0;
2986 index.full = 0;
2987 index.en = 1;
2988 if (cpuc->n_pebs == cpuc->n_large_pebs)
2989 index.thresh = ARCH_PEBS_THRESH_MULTI;
2990 else
2991 index.thresh = ARCH_PEBS_THRESH_SINGLE;
2992 wrmsrq(MSR_IA32_PEBS_INDEX, index.whole);
2993
2994 mask = hybrid(cpuc->pmu, arch_pebs_cap).counters & cpuc->pebs_enabled;
2995
2996 if (!iregs)
2997 iregs = &dummy_iregs;
2998
2999 /* Process all but the last event for each counter. */
3000 for (at = base; at < top;) {
3001 struct arch_pebs_header *header;
3002 struct arch_pebs_basic *basic;
3003 u64 pebs_status;
3004
3005 header = at;
3006
3007 if (WARN_ON_ONCE(!header->size))
3008 break;
3009
3010 /* 1st fragment or single record must have basic group */
3011 if (!header->basic) {
3012 at += header->size;
3013 continue;
3014 }
3015
3016 basic = at + sizeof(struct arch_pebs_header);
3017 pebs_status = mask & basic->applicable_counters;
3018 __intel_pmu_handle_pebs_record(iregs, regs, data, at,
3019 pebs_status, counts, last,
3020 setup_arch_pebs_sample_data);
3021
3022 /* Skip non-last fragments */
3023 while (arch_pebs_record_continued(header)) {
3024 if (!header->size)
3025 break;
3026 at += header->size;
3027 header = at;
3028 }
3029
3030 /* Skip last fragment or the single record */
3031 at += header->size;
3032 }
3033
3034 __intel_pmu_handle_last_pebs_record(iregs, regs, data, mask,
3035 counts, last,
3036 setup_arch_pebs_sample_data);
3037 }
3038
intel_arch_pebs_init(void)3039 static void __init intel_arch_pebs_init(void)
3040 {
3041 /*
3042 * Current hybrid platforms always both support arch-PEBS or not
3043 * on all kinds of cores. So directly set x86_pmu.arch_pebs flag
3044 * if boot cpu supports arch-PEBS.
3045 */
3046 x86_pmu.arch_pebs = 1;
3047 x86_pmu.pebs_buffer_size = PEBS_BUFFER_SIZE;
3048 x86_pmu.drain_pebs = intel_pmu_drain_arch_pebs;
3049 x86_pmu.pebs_capable = ~0ULL;
3050 x86_pmu.flags |= PMU_FL_PEBS_ALL;
3051
3052 x86_pmu.pebs_enable = __intel_pmu_pebs_enable;
3053 x86_pmu.pebs_disable = __intel_pmu_pebs_disable;
3054 }
3055
3056 /*
3057 * PEBS probe and setup
3058 */
3059
intel_ds_pebs_init(void)3060 static void __init intel_ds_pebs_init(void)
3061 {
3062 /*
3063 * No support for 32bit formats
3064 */
3065 if (!boot_cpu_has(X86_FEATURE_DTES64))
3066 return;
3067
3068 x86_pmu.ds_pebs = boot_cpu_has(X86_FEATURE_PEBS);
3069 x86_pmu.pebs_buffer_size = PEBS_BUFFER_SIZE;
3070 if (x86_pmu.version <= 4)
3071 x86_pmu.pebs_no_isolation = 1;
3072
3073 if (x86_pmu.ds_pebs) {
3074 char pebs_type = x86_pmu.intel_cap.pebs_trap ? '+' : '-';
3075 char *pebs_qual = "";
3076 int format = x86_pmu.intel_cap.pebs_format;
3077
3078 if (format < 4)
3079 x86_pmu.intel_cap.pebs_baseline = 0;
3080
3081 x86_pmu.pebs_enable = intel_pmu_pebs_enable;
3082 x86_pmu.pebs_disable = intel_pmu_pebs_disable;
3083 x86_pmu.pebs_enable_all = intel_pmu_pebs_enable_all;
3084 x86_pmu.pebs_disable_all = intel_pmu_pebs_disable_all;
3085
3086 switch (format) {
3087 case 0:
3088 pr_cont("PEBS fmt0%c, ", pebs_type);
3089 x86_pmu.pebs_record_size = sizeof(struct pebs_record_core);
3090 /*
3091 * Using >PAGE_SIZE buffers makes the WRMSR to
3092 * PERF_GLOBAL_CTRL in intel_pmu_enable_all()
3093 * mysteriously hang on Core2.
3094 *
3095 * As a workaround, we don't do this.
3096 */
3097 x86_pmu.pebs_buffer_size = PAGE_SIZE;
3098 x86_pmu.drain_pebs = intel_pmu_drain_pebs_core;
3099 break;
3100
3101 case 1:
3102 pr_cont("PEBS fmt1%c, ", pebs_type);
3103 x86_pmu.pebs_record_size = sizeof(struct pebs_record_nhm);
3104 x86_pmu.drain_pebs = intel_pmu_drain_pebs_nhm;
3105 break;
3106
3107 case 2:
3108 pr_cont("PEBS fmt2%c, ", pebs_type);
3109 x86_pmu.pebs_record_size = sizeof(struct pebs_record_hsw);
3110 x86_pmu.drain_pebs = intel_pmu_drain_pebs_nhm;
3111 break;
3112
3113 case 3:
3114 pr_cont("PEBS fmt3%c, ", pebs_type);
3115 x86_pmu.pebs_record_size =
3116 sizeof(struct pebs_record_skl);
3117 x86_pmu.drain_pebs = intel_pmu_drain_pebs_nhm;
3118 x86_pmu.large_pebs_flags |= PERF_SAMPLE_TIME;
3119 break;
3120
3121 case 6:
3122 if (x86_pmu.intel_cap.pebs_baseline)
3123 x86_pmu.large_pebs_flags |= PERF_SAMPLE_READ;
3124 fallthrough;
3125 case 5:
3126 x86_pmu.pebs_ept = 1;
3127 fallthrough;
3128 case 4:
3129 x86_pmu.drain_pebs = intel_pmu_drain_pebs_icl;
3130 x86_pmu.pebs_record_size = sizeof(struct pebs_basic);
3131 if (x86_pmu.intel_cap.pebs_baseline) {
3132 x86_pmu.large_pebs_flags |=
3133 PERF_SAMPLE_BRANCH_STACK |
3134 PERF_SAMPLE_TIME;
3135 x86_pmu.flags |= PMU_FL_PEBS_ALL;
3136 x86_pmu.pebs_capable = ~0ULL;
3137 pebs_qual = "-baseline";
3138 x86_get_pmu(smp_processor_id())->capabilities |= PERF_PMU_CAP_EXTENDED_REGS;
3139 } else {
3140 /* Only basic record supported */
3141 x86_pmu.large_pebs_flags &=
3142 ~(PERF_SAMPLE_ADDR |
3143 PERF_SAMPLE_TIME |
3144 PERF_SAMPLE_DATA_SRC |
3145 PERF_SAMPLE_TRANSACTION |
3146 PERF_SAMPLE_REGS_USER |
3147 PERF_SAMPLE_REGS_INTR);
3148 }
3149 pr_cont("PEBS fmt%d%c%s, ", format, pebs_type, pebs_qual);
3150
3151 /*
3152 * The PEBS-via-PT is not supported on hybrid platforms,
3153 * because not all CPUs of a hybrid machine support it.
3154 * The global x86_pmu.intel_cap, which only contains the
3155 * common capabilities, is used to check the availability
3156 * of the feature. The per-PMU pebs_output_pt_available
3157 * in a hybrid machine should be ignored.
3158 */
3159 if (x86_pmu.intel_cap.pebs_output_pt_available) {
3160 pr_cont("PEBS-via-PT, ");
3161 x86_get_pmu(smp_processor_id())->capabilities |= PERF_PMU_CAP_AUX_OUTPUT;
3162 }
3163
3164 break;
3165
3166 default:
3167 pr_cont("no PEBS fmt%d%c, ", format, pebs_type);
3168 x86_pmu.ds_pebs = 0;
3169 }
3170 }
3171 }
3172
intel_pebs_init(void)3173 void __init intel_pebs_init(void)
3174 {
3175 if (x86_pmu.intel_cap.pebs_format == 0xf)
3176 intel_arch_pebs_init();
3177 else
3178 intel_ds_pebs_init();
3179 }
3180
perf_restore_debug_store(void)3181 void perf_restore_debug_store(void)
3182 {
3183 struct debug_store *ds = __this_cpu_read(cpu_hw_events.ds);
3184
3185 if (!x86_pmu.bts && !x86_pmu.ds_pebs)
3186 return;
3187
3188 wrmsrq(MSR_IA32_DS_AREA, (unsigned long)ds);
3189 }
3190