xref: /linux/arch/x86/events/intel/core.c (revision 507e190946297c34a27d9366b0661d5e506fdd03)
1 /*
2  * Per core/cpu state
3  *
4  * Used to coordinate shared registers between HT threads or
5  * among events on a single PMU.
6  */
7 
8 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
9 
10 #include <linux/stddef.h>
11 #include <linux/types.h>
12 #include <linux/init.h>
13 #include <linux/slab.h>
14 #include <linux/export.h>
15 #include <linux/nmi.h>
16 
17 #include <asm/cpufeature.h>
18 #include <asm/hardirq.h>
19 #include <asm/intel-family.h>
20 #include <asm/apic.h>
21 
22 #include "../perf_event.h"
23 
24 /*
25  * Intel PerfMon, used on Core and later.
26  */
27 static u64 intel_perfmon_event_map[PERF_COUNT_HW_MAX] __read_mostly =
28 {
29 	[PERF_COUNT_HW_CPU_CYCLES]		= 0x003c,
30 	[PERF_COUNT_HW_INSTRUCTIONS]		= 0x00c0,
31 	[PERF_COUNT_HW_CACHE_REFERENCES]	= 0x4f2e,
32 	[PERF_COUNT_HW_CACHE_MISSES]		= 0x412e,
33 	[PERF_COUNT_HW_BRANCH_INSTRUCTIONS]	= 0x00c4,
34 	[PERF_COUNT_HW_BRANCH_MISSES]		= 0x00c5,
35 	[PERF_COUNT_HW_BUS_CYCLES]		= 0x013c,
36 	[PERF_COUNT_HW_REF_CPU_CYCLES]		= 0x0300, /* pseudo-encoding */
37 };
38 
39 static struct event_constraint intel_core_event_constraints[] __read_mostly =
40 {
41 	INTEL_EVENT_CONSTRAINT(0x11, 0x2), /* FP_ASSIST */
42 	INTEL_EVENT_CONSTRAINT(0x12, 0x2), /* MUL */
43 	INTEL_EVENT_CONSTRAINT(0x13, 0x2), /* DIV */
44 	INTEL_EVENT_CONSTRAINT(0x14, 0x1), /* CYCLES_DIV_BUSY */
45 	INTEL_EVENT_CONSTRAINT(0x19, 0x2), /* DELAYED_BYPASS */
46 	INTEL_EVENT_CONSTRAINT(0xc1, 0x1), /* FP_COMP_INSTR_RET */
47 	EVENT_CONSTRAINT_END
48 };
49 
50 static struct event_constraint intel_core2_event_constraints[] __read_mostly =
51 {
52 	FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */
53 	FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */
54 	FIXED_EVENT_CONSTRAINT(0x0300, 2), /* CPU_CLK_UNHALTED.REF */
55 	INTEL_EVENT_CONSTRAINT(0x10, 0x1), /* FP_COMP_OPS_EXE */
56 	INTEL_EVENT_CONSTRAINT(0x11, 0x2), /* FP_ASSIST */
57 	INTEL_EVENT_CONSTRAINT(0x12, 0x2), /* MUL */
58 	INTEL_EVENT_CONSTRAINT(0x13, 0x2), /* DIV */
59 	INTEL_EVENT_CONSTRAINT(0x14, 0x1), /* CYCLES_DIV_BUSY */
60 	INTEL_EVENT_CONSTRAINT(0x18, 0x1), /* IDLE_DURING_DIV */
61 	INTEL_EVENT_CONSTRAINT(0x19, 0x2), /* DELAYED_BYPASS */
62 	INTEL_EVENT_CONSTRAINT(0xa1, 0x1), /* RS_UOPS_DISPATCH_CYCLES */
63 	INTEL_EVENT_CONSTRAINT(0xc9, 0x1), /* ITLB_MISS_RETIRED (T30-9) */
64 	INTEL_EVENT_CONSTRAINT(0xcb, 0x1), /* MEM_LOAD_RETIRED */
65 	EVENT_CONSTRAINT_END
66 };
67 
68 static struct event_constraint intel_nehalem_event_constraints[] __read_mostly =
69 {
70 	FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */
71 	FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */
72 	FIXED_EVENT_CONSTRAINT(0x0300, 2), /* CPU_CLK_UNHALTED.REF */
73 	INTEL_EVENT_CONSTRAINT(0x40, 0x3), /* L1D_CACHE_LD */
74 	INTEL_EVENT_CONSTRAINT(0x41, 0x3), /* L1D_CACHE_ST */
75 	INTEL_EVENT_CONSTRAINT(0x42, 0x3), /* L1D_CACHE_LOCK */
76 	INTEL_EVENT_CONSTRAINT(0x43, 0x3), /* L1D_ALL_REF */
77 	INTEL_EVENT_CONSTRAINT(0x48, 0x3), /* L1D_PEND_MISS */
78 	INTEL_EVENT_CONSTRAINT(0x4e, 0x3), /* L1D_PREFETCH */
79 	INTEL_EVENT_CONSTRAINT(0x51, 0x3), /* L1D */
80 	INTEL_EVENT_CONSTRAINT(0x63, 0x3), /* CACHE_LOCK_CYCLES */
81 	EVENT_CONSTRAINT_END
82 };
83 
84 static struct extra_reg intel_nehalem_extra_regs[] __read_mostly =
85 {
86 	/* must define OFFCORE_RSP_X first, see intel_fixup_er() */
87 	INTEL_UEVENT_EXTRA_REG(0x01b7, MSR_OFFCORE_RSP_0, 0xffff, RSP_0),
88 	INTEL_UEVENT_PEBS_LDLAT_EXTRA_REG(0x100b),
89 	EVENT_EXTRA_END
90 };
91 
92 static struct event_constraint intel_westmere_event_constraints[] __read_mostly =
93 {
94 	FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */
95 	FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */
96 	FIXED_EVENT_CONSTRAINT(0x0300, 2), /* CPU_CLK_UNHALTED.REF */
97 	INTEL_EVENT_CONSTRAINT(0x51, 0x3), /* L1D */
98 	INTEL_EVENT_CONSTRAINT(0x60, 0x1), /* OFFCORE_REQUESTS_OUTSTANDING */
99 	INTEL_EVENT_CONSTRAINT(0x63, 0x3), /* CACHE_LOCK_CYCLES */
100 	INTEL_EVENT_CONSTRAINT(0xb3, 0x1), /* SNOOPQ_REQUEST_OUTSTANDING */
101 	EVENT_CONSTRAINT_END
102 };
103 
104 static struct event_constraint intel_snb_event_constraints[] __read_mostly =
105 {
106 	FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */
107 	FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */
108 	FIXED_EVENT_CONSTRAINT(0x0300, 2), /* CPU_CLK_UNHALTED.REF */
109 	INTEL_UEVENT_CONSTRAINT(0x04a3, 0xf), /* CYCLE_ACTIVITY.CYCLES_NO_DISPATCH */
110 	INTEL_UEVENT_CONSTRAINT(0x05a3, 0xf), /* CYCLE_ACTIVITY.STALLS_L2_PENDING */
111 	INTEL_UEVENT_CONSTRAINT(0x02a3, 0x4), /* CYCLE_ACTIVITY.CYCLES_L1D_PENDING */
112 	INTEL_UEVENT_CONSTRAINT(0x06a3, 0x4), /* CYCLE_ACTIVITY.STALLS_L1D_PENDING */
113 	INTEL_EVENT_CONSTRAINT(0x48, 0x4), /* L1D_PEND_MISS.PENDING */
114 	INTEL_UEVENT_CONSTRAINT(0x01c0, 0x2), /* INST_RETIRED.PREC_DIST */
115 	INTEL_EVENT_CONSTRAINT(0xcd, 0x8), /* MEM_TRANS_RETIRED.LOAD_LATENCY */
116 	INTEL_UEVENT_CONSTRAINT(0x04a3, 0xf), /* CYCLE_ACTIVITY.CYCLES_NO_DISPATCH */
117 	INTEL_UEVENT_CONSTRAINT(0x02a3, 0x4), /* CYCLE_ACTIVITY.CYCLES_L1D_PENDING */
118 
119 	/*
120 	 * When HT is off these events can only run on the bottom 4 counters
121 	 * When HT is on, they are impacted by the HT bug and require EXCL access
122 	 */
123 	INTEL_EXCLEVT_CONSTRAINT(0xd0, 0xf), /* MEM_UOPS_RETIRED.* */
124 	INTEL_EXCLEVT_CONSTRAINT(0xd1, 0xf), /* MEM_LOAD_UOPS_RETIRED.* */
125 	INTEL_EXCLEVT_CONSTRAINT(0xd2, 0xf), /* MEM_LOAD_UOPS_LLC_HIT_RETIRED.* */
126 	INTEL_EXCLEVT_CONSTRAINT(0xd3, 0xf), /* MEM_LOAD_UOPS_LLC_MISS_RETIRED.* */
127 
128 	EVENT_CONSTRAINT_END
129 };
130 
131 static struct event_constraint intel_ivb_event_constraints[] __read_mostly =
132 {
133 	FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */
134 	FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */
135 	FIXED_EVENT_CONSTRAINT(0x0300, 2), /* CPU_CLK_UNHALTED.REF */
136 	INTEL_UEVENT_CONSTRAINT(0x0148, 0x4), /* L1D_PEND_MISS.PENDING */
137 	INTEL_UEVENT_CONSTRAINT(0x0279, 0xf), /* IDQ.EMTPY */
138 	INTEL_UEVENT_CONSTRAINT(0x019c, 0xf), /* IDQ_UOPS_NOT_DELIVERED.CORE */
139 	INTEL_UEVENT_CONSTRAINT(0x02a3, 0xf), /* CYCLE_ACTIVITY.CYCLES_LDM_PENDING */
140 	INTEL_UEVENT_CONSTRAINT(0x04a3, 0xf), /* CYCLE_ACTIVITY.CYCLES_NO_EXECUTE */
141 	INTEL_UEVENT_CONSTRAINT(0x05a3, 0xf), /* CYCLE_ACTIVITY.STALLS_L2_PENDING */
142 	INTEL_UEVENT_CONSTRAINT(0x06a3, 0xf), /* CYCLE_ACTIVITY.STALLS_LDM_PENDING */
143 	INTEL_UEVENT_CONSTRAINT(0x08a3, 0x4), /* CYCLE_ACTIVITY.CYCLES_L1D_PENDING */
144 	INTEL_UEVENT_CONSTRAINT(0x0ca3, 0x4), /* CYCLE_ACTIVITY.STALLS_L1D_PENDING */
145 	INTEL_UEVENT_CONSTRAINT(0x01c0, 0x2), /* INST_RETIRED.PREC_DIST */
146 
147 	/*
148 	 * When HT is off these events can only run on the bottom 4 counters
149 	 * When HT is on, they are impacted by the HT bug and require EXCL access
150 	 */
151 	INTEL_EXCLEVT_CONSTRAINT(0xd0, 0xf), /* MEM_UOPS_RETIRED.* */
152 	INTEL_EXCLEVT_CONSTRAINT(0xd1, 0xf), /* MEM_LOAD_UOPS_RETIRED.* */
153 	INTEL_EXCLEVT_CONSTRAINT(0xd2, 0xf), /* MEM_LOAD_UOPS_LLC_HIT_RETIRED.* */
154 	INTEL_EXCLEVT_CONSTRAINT(0xd3, 0xf), /* MEM_LOAD_UOPS_LLC_MISS_RETIRED.* */
155 
156 	EVENT_CONSTRAINT_END
157 };
158 
159 static struct extra_reg intel_westmere_extra_regs[] __read_mostly =
160 {
161 	/* must define OFFCORE_RSP_X first, see intel_fixup_er() */
162 	INTEL_UEVENT_EXTRA_REG(0x01b7, MSR_OFFCORE_RSP_0, 0xffff, RSP_0),
163 	INTEL_UEVENT_EXTRA_REG(0x01bb, MSR_OFFCORE_RSP_1, 0xffff, RSP_1),
164 	INTEL_UEVENT_PEBS_LDLAT_EXTRA_REG(0x100b),
165 	EVENT_EXTRA_END
166 };
167 
168 static struct event_constraint intel_v1_event_constraints[] __read_mostly =
169 {
170 	EVENT_CONSTRAINT_END
171 };
172 
173 static struct event_constraint intel_gen_event_constraints[] __read_mostly =
174 {
175 	FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */
176 	FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */
177 	FIXED_EVENT_CONSTRAINT(0x0300, 2), /* CPU_CLK_UNHALTED.REF */
178 	EVENT_CONSTRAINT_END
179 };
180 
181 static struct event_constraint intel_slm_event_constraints[] __read_mostly =
182 {
183 	FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */
184 	FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */
185 	FIXED_EVENT_CONSTRAINT(0x0300, 2), /* pseudo CPU_CLK_UNHALTED.REF */
186 	EVENT_CONSTRAINT_END
187 };
188 
189 static struct event_constraint intel_skl_event_constraints[] = {
190 	FIXED_EVENT_CONSTRAINT(0x00c0, 0),	/* INST_RETIRED.ANY */
191 	FIXED_EVENT_CONSTRAINT(0x003c, 1),	/* CPU_CLK_UNHALTED.CORE */
192 	FIXED_EVENT_CONSTRAINT(0x0300, 2),	/* CPU_CLK_UNHALTED.REF */
193 	INTEL_UEVENT_CONSTRAINT(0x1c0, 0x2),	/* INST_RETIRED.PREC_DIST */
194 
195 	/*
196 	 * when HT is off, these can only run on the bottom 4 counters
197 	 */
198 	INTEL_EVENT_CONSTRAINT(0xd0, 0xf),	/* MEM_INST_RETIRED.* */
199 	INTEL_EVENT_CONSTRAINT(0xd1, 0xf),	/* MEM_LOAD_RETIRED.* */
200 	INTEL_EVENT_CONSTRAINT(0xd2, 0xf),	/* MEM_LOAD_L3_HIT_RETIRED.* */
201 	INTEL_EVENT_CONSTRAINT(0xcd, 0xf),	/* MEM_TRANS_RETIRED.* */
202 	INTEL_EVENT_CONSTRAINT(0xc6, 0xf),	/* FRONTEND_RETIRED.* */
203 
204 	EVENT_CONSTRAINT_END
205 };
206 
207 static struct extra_reg intel_knl_extra_regs[] __read_mostly = {
208 	INTEL_UEVENT_EXTRA_REG(0x01b7, MSR_OFFCORE_RSP_0, 0x799ffbb6e7ull, RSP_0),
209 	INTEL_UEVENT_EXTRA_REG(0x02b7, MSR_OFFCORE_RSP_1, 0x399ffbffe7ull, RSP_1),
210 	EVENT_EXTRA_END
211 };
212 
213 static struct extra_reg intel_snb_extra_regs[] __read_mostly = {
214 	/* must define OFFCORE_RSP_X first, see intel_fixup_er() */
215 	INTEL_UEVENT_EXTRA_REG(0x01b7, MSR_OFFCORE_RSP_0, 0x3f807f8fffull, RSP_0),
216 	INTEL_UEVENT_EXTRA_REG(0x01bb, MSR_OFFCORE_RSP_1, 0x3f807f8fffull, RSP_1),
217 	INTEL_UEVENT_PEBS_LDLAT_EXTRA_REG(0x01cd),
218 	EVENT_EXTRA_END
219 };
220 
221 static struct extra_reg intel_snbep_extra_regs[] __read_mostly = {
222 	/* must define OFFCORE_RSP_X first, see intel_fixup_er() */
223 	INTEL_UEVENT_EXTRA_REG(0x01b7, MSR_OFFCORE_RSP_0, 0x3fffff8fffull, RSP_0),
224 	INTEL_UEVENT_EXTRA_REG(0x01bb, MSR_OFFCORE_RSP_1, 0x3fffff8fffull, RSP_1),
225 	INTEL_UEVENT_PEBS_LDLAT_EXTRA_REG(0x01cd),
226 	EVENT_EXTRA_END
227 };
228 
229 static struct extra_reg intel_skl_extra_regs[] __read_mostly = {
230 	INTEL_UEVENT_EXTRA_REG(0x01b7, MSR_OFFCORE_RSP_0, 0x3fffff8fffull, RSP_0),
231 	INTEL_UEVENT_EXTRA_REG(0x01bb, MSR_OFFCORE_RSP_1, 0x3fffff8fffull, RSP_1),
232 	INTEL_UEVENT_PEBS_LDLAT_EXTRA_REG(0x01cd),
233 	/*
234 	 * Note the low 8 bits eventsel code is not a continuous field, containing
235 	 * some #GPing bits. These are masked out.
236 	 */
237 	INTEL_UEVENT_EXTRA_REG(0x01c6, MSR_PEBS_FRONTEND, 0x7fff17, FE),
238 	EVENT_EXTRA_END
239 };
240 
241 EVENT_ATTR_STR(mem-loads,	mem_ld_nhm,	"event=0x0b,umask=0x10,ldlat=3");
242 EVENT_ATTR_STR(mem-loads,	mem_ld_snb,	"event=0xcd,umask=0x1,ldlat=3");
243 EVENT_ATTR_STR(mem-stores,	mem_st_snb,	"event=0xcd,umask=0x2");
244 
245 static struct attribute *nhm_events_attrs[] = {
246 	EVENT_PTR(mem_ld_nhm),
247 	NULL,
248 };
249 
250 /*
251  * topdown events for Intel Core CPUs.
252  *
253  * The events are all in slots, which is a free slot in a 4 wide
254  * pipeline. Some events are already reported in slots, for cycle
255  * events we multiply by the pipeline width (4).
256  *
257  * With Hyper Threading on, topdown metrics are either summed or averaged
258  * between the threads of a core: (count_t0 + count_t1).
259  *
260  * For the average case the metric is always scaled to pipeline width,
261  * so we use factor 2 ((count_t0 + count_t1) / 2 * 4)
262  */
263 
264 EVENT_ATTR_STR_HT(topdown-total-slots, td_total_slots,
265 	"event=0x3c,umask=0x0",			/* cpu_clk_unhalted.thread */
266 	"event=0x3c,umask=0x0,any=1");		/* cpu_clk_unhalted.thread_any */
267 EVENT_ATTR_STR_HT(topdown-total-slots.scale, td_total_slots_scale, "4", "2");
268 EVENT_ATTR_STR(topdown-slots-issued, td_slots_issued,
269 	"event=0xe,umask=0x1");			/* uops_issued.any */
270 EVENT_ATTR_STR(topdown-slots-retired, td_slots_retired,
271 	"event=0xc2,umask=0x2");		/* uops_retired.retire_slots */
272 EVENT_ATTR_STR(topdown-fetch-bubbles, td_fetch_bubbles,
273 	"event=0x9c,umask=0x1");		/* idq_uops_not_delivered_core */
274 EVENT_ATTR_STR_HT(topdown-recovery-bubbles, td_recovery_bubbles,
275 	"event=0xd,umask=0x3,cmask=1",		/* int_misc.recovery_cycles */
276 	"event=0xd,umask=0x3,cmask=1,any=1");	/* int_misc.recovery_cycles_any */
277 EVENT_ATTR_STR_HT(topdown-recovery-bubbles.scale, td_recovery_bubbles_scale,
278 	"4", "2");
279 
280 static struct attribute *snb_events_attrs[] = {
281 	EVENT_PTR(mem_ld_snb),
282 	EVENT_PTR(mem_st_snb),
283 	EVENT_PTR(td_slots_issued),
284 	EVENT_PTR(td_slots_retired),
285 	EVENT_PTR(td_fetch_bubbles),
286 	EVENT_PTR(td_total_slots),
287 	EVENT_PTR(td_total_slots_scale),
288 	EVENT_PTR(td_recovery_bubbles),
289 	EVENT_PTR(td_recovery_bubbles_scale),
290 	NULL,
291 };
292 
293 static struct event_constraint intel_hsw_event_constraints[] = {
294 	FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */
295 	FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */
296 	FIXED_EVENT_CONSTRAINT(0x0300, 2), /* CPU_CLK_UNHALTED.REF */
297 	INTEL_UEVENT_CONSTRAINT(0x148, 0x4),	/* L1D_PEND_MISS.PENDING */
298 	INTEL_UEVENT_CONSTRAINT(0x01c0, 0x2), /* INST_RETIRED.PREC_DIST */
299 	INTEL_EVENT_CONSTRAINT(0xcd, 0x8), /* MEM_TRANS_RETIRED.LOAD_LATENCY */
300 	/* CYCLE_ACTIVITY.CYCLES_L1D_PENDING */
301 	INTEL_UEVENT_CONSTRAINT(0x08a3, 0x4),
302 	/* CYCLE_ACTIVITY.STALLS_L1D_PENDING */
303 	INTEL_UEVENT_CONSTRAINT(0x0ca3, 0x4),
304 	/* CYCLE_ACTIVITY.CYCLES_NO_EXECUTE */
305 	INTEL_UEVENT_CONSTRAINT(0x04a3, 0xf),
306 
307 	/*
308 	 * When HT is off these events can only run on the bottom 4 counters
309 	 * When HT is on, they are impacted by the HT bug and require EXCL access
310 	 */
311 	INTEL_EXCLEVT_CONSTRAINT(0xd0, 0xf), /* MEM_UOPS_RETIRED.* */
312 	INTEL_EXCLEVT_CONSTRAINT(0xd1, 0xf), /* MEM_LOAD_UOPS_RETIRED.* */
313 	INTEL_EXCLEVT_CONSTRAINT(0xd2, 0xf), /* MEM_LOAD_UOPS_LLC_HIT_RETIRED.* */
314 	INTEL_EXCLEVT_CONSTRAINT(0xd3, 0xf), /* MEM_LOAD_UOPS_LLC_MISS_RETIRED.* */
315 
316 	EVENT_CONSTRAINT_END
317 };
318 
319 static struct event_constraint intel_bdw_event_constraints[] = {
320 	FIXED_EVENT_CONSTRAINT(0x00c0, 0),	/* INST_RETIRED.ANY */
321 	FIXED_EVENT_CONSTRAINT(0x003c, 1),	/* CPU_CLK_UNHALTED.CORE */
322 	FIXED_EVENT_CONSTRAINT(0x0300, 2),	/* CPU_CLK_UNHALTED.REF */
323 	INTEL_UEVENT_CONSTRAINT(0x148, 0x4),	/* L1D_PEND_MISS.PENDING */
324 	INTEL_UBIT_EVENT_CONSTRAINT(0x8a3, 0x4),	/* CYCLE_ACTIVITY.CYCLES_L1D_MISS */
325 	/*
326 	 * when HT is off, these can only run on the bottom 4 counters
327 	 */
328 	INTEL_EVENT_CONSTRAINT(0xd0, 0xf),	/* MEM_INST_RETIRED.* */
329 	INTEL_EVENT_CONSTRAINT(0xd1, 0xf),	/* MEM_LOAD_RETIRED.* */
330 	INTEL_EVENT_CONSTRAINT(0xd2, 0xf),	/* MEM_LOAD_L3_HIT_RETIRED.* */
331 	INTEL_EVENT_CONSTRAINT(0xcd, 0xf),	/* MEM_TRANS_RETIRED.* */
332 	EVENT_CONSTRAINT_END
333 };
334 
335 static u64 intel_pmu_event_map(int hw_event)
336 {
337 	return intel_perfmon_event_map[hw_event];
338 }
339 
340 /*
341  * Notes on the events:
342  * - data reads do not include code reads (comparable to earlier tables)
343  * - data counts include speculative execution (except L1 write, dtlb, bpu)
344  * - remote node access includes remote memory, remote cache, remote mmio.
345  * - prefetches are not included in the counts.
346  * - icache miss does not include decoded icache
347  */
348 
349 #define SKL_DEMAND_DATA_RD		BIT_ULL(0)
350 #define SKL_DEMAND_RFO			BIT_ULL(1)
351 #define SKL_ANY_RESPONSE		BIT_ULL(16)
352 #define SKL_SUPPLIER_NONE		BIT_ULL(17)
353 #define SKL_L3_MISS_LOCAL_DRAM		BIT_ULL(26)
354 #define SKL_L3_MISS_REMOTE_HOP0_DRAM	BIT_ULL(27)
355 #define SKL_L3_MISS_REMOTE_HOP1_DRAM	BIT_ULL(28)
356 #define SKL_L3_MISS_REMOTE_HOP2P_DRAM	BIT_ULL(29)
357 #define SKL_L3_MISS			(SKL_L3_MISS_LOCAL_DRAM| \
358 					 SKL_L3_MISS_REMOTE_HOP0_DRAM| \
359 					 SKL_L3_MISS_REMOTE_HOP1_DRAM| \
360 					 SKL_L3_MISS_REMOTE_HOP2P_DRAM)
361 #define SKL_SPL_HIT			BIT_ULL(30)
362 #define SKL_SNOOP_NONE			BIT_ULL(31)
363 #define SKL_SNOOP_NOT_NEEDED		BIT_ULL(32)
364 #define SKL_SNOOP_MISS			BIT_ULL(33)
365 #define SKL_SNOOP_HIT_NO_FWD		BIT_ULL(34)
366 #define SKL_SNOOP_HIT_WITH_FWD		BIT_ULL(35)
367 #define SKL_SNOOP_HITM			BIT_ULL(36)
368 #define SKL_SNOOP_NON_DRAM		BIT_ULL(37)
369 #define SKL_ANY_SNOOP			(SKL_SPL_HIT|SKL_SNOOP_NONE| \
370 					 SKL_SNOOP_NOT_NEEDED|SKL_SNOOP_MISS| \
371 					 SKL_SNOOP_HIT_NO_FWD|SKL_SNOOP_HIT_WITH_FWD| \
372 					 SKL_SNOOP_HITM|SKL_SNOOP_NON_DRAM)
373 #define SKL_DEMAND_READ			SKL_DEMAND_DATA_RD
374 #define SKL_SNOOP_DRAM			(SKL_SNOOP_NONE| \
375 					 SKL_SNOOP_NOT_NEEDED|SKL_SNOOP_MISS| \
376 					 SKL_SNOOP_HIT_NO_FWD|SKL_SNOOP_HIT_WITH_FWD| \
377 					 SKL_SNOOP_HITM|SKL_SPL_HIT)
378 #define SKL_DEMAND_WRITE		SKL_DEMAND_RFO
379 #define SKL_LLC_ACCESS			SKL_ANY_RESPONSE
380 #define SKL_L3_MISS_REMOTE		(SKL_L3_MISS_REMOTE_HOP0_DRAM| \
381 					 SKL_L3_MISS_REMOTE_HOP1_DRAM| \
382 					 SKL_L3_MISS_REMOTE_HOP2P_DRAM)
383 
384 static __initconst const u64 skl_hw_cache_event_ids
385 				[PERF_COUNT_HW_CACHE_MAX]
386 				[PERF_COUNT_HW_CACHE_OP_MAX]
387 				[PERF_COUNT_HW_CACHE_RESULT_MAX] =
388 {
389  [ C(L1D ) ] = {
390 	[ C(OP_READ) ] = {
391 		[ C(RESULT_ACCESS) ] = 0x81d0,	/* MEM_INST_RETIRED.ALL_LOADS */
392 		[ C(RESULT_MISS)   ] = 0x151,	/* L1D.REPLACEMENT */
393 	},
394 	[ C(OP_WRITE) ] = {
395 		[ C(RESULT_ACCESS) ] = 0x82d0,	/* MEM_INST_RETIRED.ALL_STORES */
396 		[ C(RESULT_MISS)   ] = 0x0,
397 	},
398 	[ C(OP_PREFETCH) ] = {
399 		[ C(RESULT_ACCESS) ] = 0x0,
400 		[ C(RESULT_MISS)   ] = 0x0,
401 	},
402  },
403  [ C(L1I ) ] = {
404 	[ C(OP_READ) ] = {
405 		[ C(RESULT_ACCESS) ] = 0x0,
406 		[ C(RESULT_MISS)   ] = 0x283,	/* ICACHE_64B.MISS */
407 	},
408 	[ C(OP_WRITE) ] = {
409 		[ C(RESULT_ACCESS) ] = -1,
410 		[ C(RESULT_MISS)   ] = -1,
411 	},
412 	[ C(OP_PREFETCH) ] = {
413 		[ C(RESULT_ACCESS) ] = 0x0,
414 		[ C(RESULT_MISS)   ] = 0x0,
415 	},
416  },
417  [ C(LL  ) ] = {
418 	[ C(OP_READ) ] = {
419 		[ C(RESULT_ACCESS) ] = 0x1b7,	/* OFFCORE_RESPONSE */
420 		[ C(RESULT_MISS)   ] = 0x1b7,	/* OFFCORE_RESPONSE */
421 	},
422 	[ C(OP_WRITE) ] = {
423 		[ C(RESULT_ACCESS) ] = 0x1b7,	/* OFFCORE_RESPONSE */
424 		[ C(RESULT_MISS)   ] = 0x1b7,	/* OFFCORE_RESPONSE */
425 	},
426 	[ C(OP_PREFETCH) ] = {
427 		[ C(RESULT_ACCESS) ] = 0x0,
428 		[ C(RESULT_MISS)   ] = 0x0,
429 	},
430  },
431  [ C(DTLB) ] = {
432 	[ C(OP_READ) ] = {
433 		[ C(RESULT_ACCESS) ] = 0x81d0,	/* MEM_INST_RETIRED.ALL_LOADS */
434 		[ C(RESULT_MISS)   ] = 0xe08,	/* DTLB_LOAD_MISSES.WALK_COMPLETED */
435 	},
436 	[ C(OP_WRITE) ] = {
437 		[ C(RESULT_ACCESS) ] = 0x82d0,	/* MEM_INST_RETIRED.ALL_STORES */
438 		[ C(RESULT_MISS)   ] = 0xe49,	/* DTLB_STORE_MISSES.WALK_COMPLETED */
439 	},
440 	[ C(OP_PREFETCH) ] = {
441 		[ C(RESULT_ACCESS) ] = 0x0,
442 		[ C(RESULT_MISS)   ] = 0x0,
443 	},
444  },
445  [ C(ITLB) ] = {
446 	[ C(OP_READ) ] = {
447 		[ C(RESULT_ACCESS) ] = 0x2085,	/* ITLB_MISSES.STLB_HIT */
448 		[ C(RESULT_MISS)   ] = 0xe85,	/* ITLB_MISSES.WALK_COMPLETED */
449 	},
450 	[ C(OP_WRITE) ] = {
451 		[ C(RESULT_ACCESS) ] = -1,
452 		[ C(RESULT_MISS)   ] = -1,
453 	},
454 	[ C(OP_PREFETCH) ] = {
455 		[ C(RESULT_ACCESS) ] = -1,
456 		[ C(RESULT_MISS)   ] = -1,
457 	},
458  },
459  [ C(BPU ) ] = {
460 	[ C(OP_READ) ] = {
461 		[ C(RESULT_ACCESS) ] = 0xc4,	/* BR_INST_RETIRED.ALL_BRANCHES */
462 		[ C(RESULT_MISS)   ] = 0xc5,	/* BR_MISP_RETIRED.ALL_BRANCHES */
463 	},
464 	[ C(OP_WRITE) ] = {
465 		[ C(RESULT_ACCESS) ] = -1,
466 		[ C(RESULT_MISS)   ] = -1,
467 	},
468 	[ C(OP_PREFETCH) ] = {
469 		[ C(RESULT_ACCESS) ] = -1,
470 		[ C(RESULT_MISS)   ] = -1,
471 	},
472  },
473  [ C(NODE) ] = {
474 	[ C(OP_READ) ] = {
475 		[ C(RESULT_ACCESS) ] = 0x1b7,	/* OFFCORE_RESPONSE */
476 		[ C(RESULT_MISS)   ] = 0x1b7,	/* OFFCORE_RESPONSE */
477 	},
478 	[ C(OP_WRITE) ] = {
479 		[ C(RESULT_ACCESS) ] = 0x1b7,	/* OFFCORE_RESPONSE */
480 		[ C(RESULT_MISS)   ] = 0x1b7,	/* OFFCORE_RESPONSE */
481 	},
482 	[ C(OP_PREFETCH) ] = {
483 		[ C(RESULT_ACCESS) ] = 0x0,
484 		[ C(RESULT_MISS)   ] = 0x0,
485 	},
486  },
487 };
488 
489 static __initconst const u64 skl_hw_cache_extra_regs
490 				[PERF_COUNT_HW_CACHE_MAX]
491 				[PERF_COUNT_HW_CACHE_OP_MAX]
492 				[PERF_COUNT_HW_CACHE_RESULT_MAX] =
493 {
494  [ C(LL  ) ] = {
495 	[ C(OP_READ) ] = {
496 		[ C(RESULT_ACCESS) ] = SKL_DEMAND_READ|
497 				       SKL_LLC_ACCESS|SKL_ANY_SNOOP,
498 		[ C(RESULT_MISS)   ] = SKL_DEMAND_READ|
499 				       SKL_L3_MISS|SKL_ANY_SNOOP|
500 				       SKL_SUPPLIER_NONE,
501 	},
502 	[ C(OP_WRITE) ] = {
503 		[ C(RESULT_ACCESS) ] = SKL_DEMAND_WRITE|
504 				       SKL_LLC_ACCESS|SKL_ANY_SNOOP,
505 		[ C(RESULT_MISS)   ] = SKL_DEMAND_WRITE|
506 				       SKL_L3_MISS|SKL_ANY_SNOOP|
507 				       SKL_SUPPLIER_NONE,
508 	},
509 	[ C(OP_PREFETCH) ] = {
510 		[ C(RESULT_ACCESS) ] = 0x0,
511 		[ C(RESULT_MISS)   ] = 0x0,
512 	},
513  },
514  [ C(NODE) ] = {
515 	[ C(OP_READ) ] = {
516 		[ C(RESULT_ACCESS) ] = SKL_DEMAND_READ|
517 				       SKL_L3_MISS_LOCAL_DRAM|SKL_SNOOP_DRAM,
518 		[ C(RESULT_MISS)   ] = SKL_DEMAND_READ|
519 				       SKL_L3_MISS_REMOTE|SKL_SNOOP_DRAM,
520 	},
521 	[ C(OP_WRITE) ] = {
522 		[ C(RESULT_ACCESS) ] = SKL_DEMAND_WRITE|
523 				       SKL_L3_MISS_LOCAL_DRAM|SKL_SNOOP_DRAM,
524 		[ C(RESULT_MISS)   ] = SKL_DEMAND_WRITE|
525 				       SKL_L3_MISS_REMOTE|SKL_SNOOP_DRAM,
526 	},
527 	[ C(OP_PREFETCH) ] = {
528 		[ C(RESULT_ACCESS) ] = 0x0,
529 		[ C(RESULT_MISS)   ] = 0x0,
530 	},
531  },
532 };
533 
534 #define SNB_DMND_DATA_RD	(1ULL << 0)
535 #define SNB_DMND_RFO		(1ULL << 1)
536 #define SNB_DMND_IFETCH		(1ULL << 2)
537 #define SNB_DMND_WB		(1ULL << 3)
538 #define SNB_PF_DATA_RD		(1ULL << 4)
539 #define SNB_PF_RFO		(1ULL << 5)
540 #define SNB_PF_IFETCH		(1ULL << 6)
541 #define SNB_LLC_DATA_RD		(1ULL << 7)
542 #define SNB_LLC_RFO		(1ULL << 8)
543 #define SNB_LLC_IFETCH		(1ULL << 9)
544 #define SNB_BUS_LOCKS		(1ULL << 10)
545 #define SNB_STRM_ST		(1ULL << 11)
546 #define SNB_OTHER		(1ULL << 15)
547 #define SNB_RESP_ANY		(1ULL << 16)
548 #define SNB_NO_SUPP		(1ULL << 17)
549 #define SNB_LLC_HITM		(1ULL << 18)
550 #define SNB_LLC_HITE		(1ULL << 19)
551 #define SNB_LLC_HITS		(1ULL << 20)
552 #define SNB_LLC_HITF		(1ULL << 21)
553 #define SNB_LOCAL		(1ULL << 22)
554 #define SNB_REMOTE		(0xffULL << 23)
555 #define SNB_SNP_NONE		(1ULL << 31)
556 #define SNB_SNP_NOT_NEEDED	(1ULL << 32)
557 #define SNB_SNP_MISS		(1ULL << 33)
558 #define SNB_NO_FWD		(1ULL << 34)
559 #define SNB_SNP_FWD		(1ULL << 35)
560 #define SNB_HITM		(1ULL << 36)
561 #define SNB_NON_DRAM		(1ULL << 37)
562 
563 #define SNB_DMND_READ		(SNB_DMND_DATA_RD|SNB_LLC_DATA_RD)
564 #define SNB_DMND_WRITE		(SNB_DMND_RFO|SNB_LLC_RFO)
565 #define SNB_DMND_PREFETCH	(SNB_PF_DATA_RD|SNB_PF_RFO)
566 
567 #define SNB_SNP_ANY		(SNB_SNP_NONE|SNB_SNP_NOT_NEEDED| \
568 				 SNB_SNP_MISS|SNB_NO_FWD|SNB_SNP_FWD| \
569 				 SNB_HITM)
570 
571 #define SNB_DRAM_ANY		(SNB_LOCAL|SNB_REMOTE|SNB_SNP_ANY)
572 #define SNB_DRAM_REMOTE		(SNB_REMOTE|SNB_SNP_ANY)
573 
574 #define SNB_L3_ACCESS		SNB_RESP_ANY
575 #define SNB_L3_MISS		(SNB_DRAM_ANY|SNB_NON_DRAM)
576 
577 static __initconst const u64 snb_hw_cache_extra_regs
578 				[PERF_COUNT_HW_CACHE_MAX]
579 				[PERF_COUNT_HW_CACHE_OP_MAX]
580 				[PERF_COUNT_HW_CACHE_RESULT_MAX] =
581 {
582  [ C(LL  ) ] = {
583 	[ C(OP_READ) ] = {
584 		[ C(RESULT_ACCESS) ] = SNB_DMND_READ|SNB_L3_ACCESS,
585 		[ C(RESULT_MISS)   ] = SNB_DMND_READ|SNB_L3_MISS,
586 	},
587 	[ C(OP_WRITE) ] = {
588 		[ C(RESULT_ACCESS) ] = SNB_DMND_WRITE|SNB_L3_ACCESS,
589 		[ C(RESULT_MISS)   ] = SNB_DMND_WRITE|SNB_L3_MISS,
590 	},
591 	[ C(OP_PREFETCH) ] = {
592 		[ C(RESULT_ACCESS) ] = SNB_DMND_PREFETCH|SNB_L3_ACCESS,
593 		[ C(RESULT_MISS)   ] = SNB_DMND_PREFETCH|SNB_L3_MISS,
594 	},
595  },
596  [ C(NODE) ] = {
597 	[ C(OP_READ) ] = {
598 		[ C(RESULT_ACCESS) ] = SNB_DMND_READ|SNB_DRAM_ANY,
599 		[ C(RESULT_MISS)   ] = SNB_DMND_READ|SNB_DRAM_REMOTE,
600 	},
601 	[ C(OP_WRITE) ] = {
602 		[ C(RESULT_ACCESS) ] = SNB_DMND_WRITE|SNB_DRAM_ANY,
603 		[ C(RESULT_MISS)   ] = SNB_DMND_WRITE|SNB_DRAM_REMOTE,
604 	},
605 	[ C(OP_PREFETCH) ] = {
606 		[ C(RESULT_ACCESS) ] = SNB_DMND_PREFETCH|SNB_DRAM_ANY,
607 		[ C(RESULT_MISS)   ] = SNB_DMND_PREFETCH|SNB_DRAM_REMOTE,
608 	},
609  },
610 };
611 
612 static __initconst const u64 snb_hw_cache_event_ids
613 				[PERF_COUNT_HW_CACHE_MAX]
614 				[PERF_COUNT_HW_CACHE_OP_MAX]
615 				[PERF_COUNT_HW_CACHE_RESULT_MAX] =
616 {
617  [ C(L1D) ] = {
618 	[ C(OP_READ) ] = {
619 		[ C(RESULT_ACCESS) ] = 0xf1d0, /* MEM_UOP_RETIRED.LOADS        */
620 		[ C(RESULT_MISS)   ] = 0x0151, /* L1D.REPLACEMENT              */
621 	},
622 	[ C(OP_WRITE) ] = {
623 		[ C(RESULT_ACCESS) ] = 0xf2d0, /* MEM_UOP_RETIRED.STORES       */
624 		[ C(RESULT_MISS)   ] = 0x0851, /* L1D.ALL_M_REPLACEMENT        */
625 	},
626 	[ C(OP_PREFETCH) ] = {
627 		[ C(RESULT_ACCESS) ] = 0x0,
628 		[ C(RESULT_MISS)   ] = 0x024e, /* HW_PRE_REQ.DL1_MISS          */
629 	},
630  },
631  [ C(L1I ) ] = {
632 	[ C(OP_READ) ] = {
633 		[ C(RESULT_ACCESS) ] = 0x0,
634 		[ C(RESULT_MISS)   ] = 0x0280, /* ICACHE.MISSES */
635 	},
636 	[ C(OP_WRITE) ] = {
637 		[ C(RESULT_ACCESS) ] = -1,
638 		[ C(RESULT_MISS)   ] = -1,
639 	},
640 	[ C(OP_PREFETCH) ] = {
641 		[ C(RESULT_ACCESS) ] = 0x0,
642 		[ C(RESULT_MISS)   ] = 0x0,
643 	},
644  },
645  [ C(LL  ) ] = {
646 	[ C(OP_READ) ] = {
647 		/* OFFCORE_RESPONSE.ANY_DATA.LOCAL_CACHE */
648 		[ C(RESULT_ACCESS) ] = 0x01b7,
649 		/* OFFCORE_RESPONSE.ANY_DATA.ANY_LLC_MISS */
650 		[ C(RESULT_MISS)   ] = 0x01b7,
651 	},
652 	[ C(OP_WRITE) ] = {
653 		/* OFFCORE_RESPONSE.ANY_RFO.LOCAL_CACHE */
654 		[ C(RESULT_ACCESS) ] = 0x01b7,
655 		/* OFFCORE_RESPONSE.ANY_RFO.ANY_LLC_MISS */
656 		[ C(RESULT_MISS)   ] = 0x01b7,
657 	},
658 	[ C(OP_PREFETCH) ] = {
659 		/* OFFCORE_RESPONSE.PREFETCH.LOCAL_CACHE */
660 		[ C(RESULT_ACCESS) ] = 0x01b7,
661 		/* OFFCORE_RESPONSE.PREFETCH.ANY_LLC_MISS */
662 		[ C(RESULT_MISS)   ] = 0x01b7,
663 	},
664  },
665  [ C(DTLB) ] = {
666 	[ C(OP_READ) ] = {
667 		[ C(RESULT_ACCESS) ] = 0x81d0, /* MEM_UOP_RETIRED.ALL_LOADS */
668 		[ C(RESULT_MISS)   ] = 0x0108, /* DTLB_LOAD_MISSES.CAUSES_A_WALK */
669 	},
670 	[ C(OP_WRITE) ] = {
671 		[ C(RESULT_ACCESS) ] = 0x82d0, /* MEM_UOP_RETIRED.ALL_STORES */
672 		[ C(RESULT_MISS)   ] = 0x0149, /* DTLB_STORE_MISSES.MISS_CAUSES_A_WALK */
673 	},
674 	[ C(OP_PREFETCH) ] = {
675 		[ C(RESULT_ACCESS) ] = 0x0,
676 		[ C(RESULT_MISS)   ] = 0x0,
677 	},
678  },
679  [ C(ITLB) ] = {
680 	[ C(OP_READ) ] = {
681 		[ C(RESULT_ACCESS) ] = 0x1085, /* ITLB_MISSES.STLB_HIT         */
682 		[ C(RESULT_MISS)   ] = 0x0185, /* ITLB_MISSES.CAUSES_A_WALK    */
683 	},
684 	[ C(OP_WRITE) ] = {
685 		[ C(RESULT_ACCESS) ] = -1,
686 		[ C(RESULT_MISS)   ] = -1,
687 	},
688 	[ C(OP_PREFETCH) ] = {
689 		[ C(RESULT_ACCESS) ] = -1,
690 		[ C(RESULT_MISS)   ] = -1,
691 	},
692  },
693  [ C(BPU ) ] = {
694 	[ C(OP_READ) ] = {
695 		[ C(RESULT_ACCESS) ] = 0x00c4, /* BR_INST_RETIRED.ALL_BRANCHES */
696 		[ C(RESULT_MISS)   ] = 0x00c5, /* BR_MISP_RETIRED.ALL_BRANCHES */
697 	},
698 	[ C(OP_WRITE) ] = {
699 		[ C(RESULT_ACCESS) ] = -1,
700 		[ C(RESULT_MISS)   ] = -1,
701 	},
702 	[ C(OP_PREFETCH) ] = {
703 		[ C(RESULT_ACCESS) ] = -1,
704 		[ C(RESULT_MISS)   ] = -1,
705 	},
706  },
707  [ C(NODE) ] = {
708 	[ C(OP_READ) ] = {
709 		[ C(RESULT_ACCESS) ] = 0x01b7,
710 		[ C(RESULT_MISS)   ] = 0x01b7,
711 	},
712 	[ C(OP_WRITE) ] = {
713 		[ C(RESULT_ACCESS) ] = 0x01b7,
714 		[ C(RESULT_MISS)   ] = 0x01b7,
715 	},
716 	[ C(OP_PREFETCH) ] = {
717 		[ C(RESULT_ACCESS) ] = 0x01b7,
718 		[ C(RESULT_MISS)   ] = 0x01b7,
719 	},
720  },
721 
722 };
723 
724 /*
725  * Notes on the events:
726  * - data reads do not include code reads (comparable to earlier tables)
727  * - data counts include speculative execution (except L1 write, dtlb, bpu)
728  * - remote node access includes remote memory, remote cache, remote mmio.
729  * - prefetches are not included in the counts because they are not
730  *   reliably counted.
731  */
732 
733 #define HSW_DEMAND_DATA_RD		BIT_ULL(0)
734 #define HSW_DEMAND_RFO			BIT_ULL(1)
735 #define HSW_ANY_RESPONSE		BIT_ULL(16)
736 #define HSW_SUPPLIER_NONE		BIT_ULL(17)
737 #define HSW_L3_MISS_LOCAL_DRAM		BIT_ULL(22)
738 #define HSW_L3_MISS_REMOTE_HOP0		BIT_ULL(27)
739 #define HSW_L3_MISS_REMOTE_HOP1		BIT_ULL(28)
740 #define HSW_L3_MISS_REMOTE_HOP2P	BIT_ULL(29)
741 #define HSW_L3_MISS			(HSW_L3_MISS_LOCAL_DRAM| \
742 					 HSW_L3_MISS_REMOTE_HOP0|HSW_L3_MISS_REMOTE_HOP1| \
743 					 HSW_L3_MISS_REMOTE_HOP2P)
744 #define HSW_SNOOP_NONE			BIT_ULL(31)
745 #define HSW_SNOOP_NOT_NEEDED		BIT_ULL(32)
746 #define HSW_SNOOP_MISS			BIT_ULL(33)
747 #define HSW_SNOOP_HIT_NO_FWD		BIT_ULL(34)
748 #define HSW_SNOOP_HIT_WITH_FWD		BIT_ULL(35)
749 #define HSW_SNOOP_HITM			BIT_ULL(36)
750 #define HSW_SNOOP_NON_DRAM		BIT_ULL(37)
751 #define HSW_ANY_SNOOP			(HSW_SNOOP_NONE| \
752 					 HSW_SNOOP_NOT_NEEDED|HSW_SNOOP_MISS| \
753 					 HSW_SNOOP_HIT_NO_FWD|HSW_SNOOP_HIT_WITH_FWD| \
754 					 HSW_SNOOP_HITM|HSW_SNOOP_NON_DRAM)
755 #define HSW_SNOOP_DRAM			(HSW_ANY_SNOOP & ~HSW_SNOOP_NON_DRAM)
756 #define HSW_DEMAND_READ			HSW_DEMAND_DATA_RD
757 #define HSW_DEMAND_WRITE		HSW_DEMAND_RFO
758 #define HSW_L3_MISS_REMOTE		(HSW_L3_MISS_REMOTE_HOP0|\
759 					 HSW_L3_MISS_REMOTE_HOP1|HSW_L3_MISS_REMOTE_HOP2P)
760 #define HSW_LLC_ACCESS			HSW_ANY_RESPONSE
761 
762 #define BDW_L3_MISS_LOCAL		BIT(26)
763 #define BDW_L3_MISS			(BDW_L3_MISS_LOCAL| \
764 					 HSW_L3_MISS_REMOTE_HOP0|HSW_L3_MISS_REMOTE_HOP1| \
765 					 HSW_L3_MISS_REMOTE_HOP2P)
766 
767 
768 static __initconst const u64 hsw_hw_cache_event_ids
769 				[PERF_COUNT_HW_CACHE_MAX]
770 				[PERF_COUNT_HW_CACHE_OP_MAX]
771 				[PERF_COUNT_HW_CACHE_RESULT_MAX] =
772 {
773  [ C(L1D ) ] = {
774 	[ C(OP_READ) ] = {
775 		[ C(RESULT_ACCESS) ] = 0x81d0,	/* MEM_UOPS_RETIRED.ALL_LOADS */
776 		[ C(RESULT_MISS)   ] = 0x151,	/* L1D.REPLACEMENT */
777 	},
778 	[ C(OP_WRITE) ] = {
779 		[ C(RESULT_ACCESS) ] = 0x82d0,	/* MEM_UOPS_RETIRED.ALL_STORES */
780 		[ C(RESULT_MISS)   ] = 0x0,
781 	},
782 	[ C(OP_PREFETCH) ] = {
783 		[ C(RESULT_ACCESS) ] = 0x0,
784 		[ C(RESULT_MISS)   ] = 0x0,
785 	},
786  },
787  [ C(L1I ) ] = {
788 	[ C(OP_READ) ] = {
789 		[ C(RESULT_ACCESS) ] = 0x0,
790 		[ C(RESULT_MISS)   ] = 0x280,	/* ICACHE.MISSES */
791 	},
792 	[ C(OP_WRITE) ] = {
793 		[ C(RESULT_ACCESS) ] = -1,
794 		[ C(RESULT_MISS)   ] = -1,
795 	},
796 	[ C(OP_PREFETCH) ] = {
797 		[ C(RESULT_ACCESS) ] = 0x0,
798 		[ C(RESULT_MISS)   ] = 0x0,
799 	},
800  },
801  [ C(LL  ) ] = {
802 	[ C(OP_READ) ] = {
803 		[ C(RESULT_ACCESS) ] = 0x1b7,	/* OFFCORE_RESPONSE */
804 		[ C(RESULT_MISS)   ] = 0x1b7,	/* OFFCORE_RESPONSE */
805 	},
806 	[ C(OP_WRITE) ] = {
807 		[ C(RESULT_ACCESS) ] = 0x1b7,	/* OFFCORE_RESPONSE */
808 		[ C(RESULT_MISS)   ] = 0x1b7,	/* OFFCORE_RESPONSE */
809 	},
810 	[ C(OP_PREFETCH) ] = {
811 		[ C(RESULT_ACCESS) ] = 0x0,
812 		[ C(RESULT_MISS)   ] = 0x0,
813 	},
814  },
815  [ C(DTLB) ] = {
816 	[ C(OP_READ) ] = {
817 		[ C(RESULT_ACCESS) ] = 0x81d0,	/* MEM_UOPS_RETIRED.ALL_LOADS */
818 		[ C(RESULT_MISS)   ] = 0x108,	/* DTLB_LOAD_MISSES.MISS_CAUSES_A_WALK */
819 	},
820 	[ C(OP_WRITE) ] = {
821 		[ C(RESULT_ACCESS) ] = 0x82d0,	/* MEM_UOPS_RETIRED.ALL_STORES */
822 		[ C(RESULT_MISS)   ] = 0x149,	/* DTLB_STORE_MISSES.MISS_CAUSES_A_WALK */
823 	},
824 	[ C(OP_PREFETCH) ] = {
825 		[ C(RESULT_ACCESS) ] = 0x0,
826 		[ C(RESULT_MISS)   ] = 0x0,
827 	},
828  },
829  [ C(ITLB) ] = {
830 	[ C(OP_READ) ] = {
831 		[ C(RESULT_ACCESS) ] = 0x6085,	/* ITLB_MISSES.STLB_HIT */
832 		[ C(RESULT_MISS)   ] = 0x185,	/* ITLB_MISSES.MISS_CAUSES_A_WALK */
833 	},
834 	[ C(OP_WRITE) ] = {
835 		[ C(RESULT_ACCESS) ] = -1,
836 		[ C(RESULT_MISS)   ] = -1,
837 	},
838 	[ C(OP_PREFETCH) ] = {
839 		[ C(RESULT_ACCESS) ] = -1,
840 		[ C(RESULT_MISS)   ] = -1,
841 	},
842  },
843  [ C(BPU ) ] = {
844 	[ C(OP_READ) ] = {
845 		[ C(RESULT_ACCESS) ] = 0xc4,	/* BR_INST_RETIRED.ALL_BRANCHES */
846 		[ C(RESULT_MISS)   ] = 0xc5,	/* BR_MISP_RETIRED.ALL_BRANCHES */
847 	},
848 	[ C(OP_WRITE) ] = {
849 		[ C(RESULT_ACCESS) ] = -1,
850 		[ C(RESULT_MISS)   ] = -1,
851 	},
852 	[ C(OP_PREFETCH) ] = {
853 		[ C(RESULT_ACCESS) ] = -1,
854 		[ C(RESULT_MISS)   ] = -1,
855 	},
856  },
857  [ C(NODE) ] = {
858 	[ C(OP_READ) ] = {
859 		[ C(RESULT_ACCESS) ] = 0x1b7,	/* OFFCORE_RESPONSE */
860 		[ C(RESULT_MISS)   ] = 0x1b7,	/* OFFCORE_RESPONSE */
861 	},
862 	[ C(OP_WRITE) ] = {
863 		[ C(RESULT_ACCESS) ] = 0x1b7,	/* OFFCORE_RESPONSE */
864 		[ C(RESULT_MISS)   ] = 0x1b7,	/* OFFCORE_RESPONSE */
865 	},
866 	[ C(OP_PREFETCH) ] = {
867 		[ C(RESULT_ACCESS) ] = 0x0,
868 		[ C(RESULT_MISS)   ] = 0x0,
869 	},
870  },
871 };
872 
873 static __initconst const u64 hsw_hw_cache_extra_regs
874 				[PERF_COUNT_HW_CACHE_MAX]
875 				[PERF_COUNT_HW_CACHE_OP_MAX]
876 				[PERF_COUNT_HW_CACHE_RESULT_MAX] =
877 {
878  [ C(LL  ) ] = {
879 	[ C(OP_READ) ] = {
880 		[ C(RESULT_ACCESS) ] = HSW_DEMAND_READ|
881 				       HSW_LLC_ACCESS,
882 		[ C(RESULT_MISS)   ] = HSW_DEMAND_READ|
883 				       HSW_L3_MISS|HSW_ANY_SNOOP,
884 	},
885 	[ C(OP_WRITE) ] = {
886 		[ C(RESULT_ACCESS) ] = HSW_DEMAND_WRITE|
887 				       HSW_LLC_ACCESS,
888 		[ C(RESULT_MISS)   ] = HSW_DEMAND_WRITE|
889 				       HSW_L3_MISS|HSW_ANY_SNOOP,
890 	},
891 	[ C(OP_PREFETCH) ] = {
892 		[ C(RESULT_ACCESS) ] = 0x0,
893 		[ C(RESULT_MISS)   ] = 0x0,
894 	},
895  },
896  [ C(NODE) ] = {
897 	[ C(OP_READ) ] = {
898 		[ C(RESULT_ACCESS) ] = HSW_DEMAND_READ|
899 				       HSW_L3_MISS_LOCAL_DRAM|
900 				       HSW_SNOOP_DRAM,
901 		[ C(RESULT_MISS)   ] = HSW_DEMAND_READ|
902 				       HSW_L3_MISS_REMOTE|
903 				       HSW_SNOOP_DRAM,
904 	},
905 	[ C(OP_WRITE) ] = {
906 		[ C(RESULT_ACCESS) ] = HSW_DEMAND_WRITE|
907 				       HSW_L3_MISS_LOCAL_DRAM|
908 				       HSW_SNOOP_DRAM,
909 		[ C(RESULT_MISS)   ] = HSW_DEMAND_WRITE|
910 				       HSW_L3_MISS_REMOTE|
911 				       HSW_SNOOP_DRAM,
912 	},
913 	[ C(OP_PREFETCH) ] = {
914 		[ C(RESULT_ACCESS) ] = 0x0,
915 		[ C(RESULT_MISS)   ] = 0x0,
916 	},
917  },
918 };
919 
920 static __initconst const u64 westmere_hw_cache_event_ids
921 				[PERF_COUNT_HW_CACHE_MAX]
922 				[PERF_COUNT_HW_CACHE_OP_MAX]
923 				[PERF_COUNT_HW_CACHE_RESULT_MAX] =
924 {
925  [ C(L1D) ] = {
926 	[ C(OP_READ) ] = {
927 		[ C(RESULT_ACCESS) ] = 0x010b, /* MEM_INST_RETIRED.LOADS       */
928 		[ C(RESULT_MISS)   ] = 0x0151, /* L1D.REPL                     */
929 	},
930 	[ C(OP_WRITE) ] = {
931 		[ C(RESULT_ACCESS) ] = 0x020b, /* MEM_INST_RETURED.STORES      */
932 		[ C(RESULT_MISS)   ] = 0x0251, /* L1D.M_REPL                   */
933 	},
934 	[ C(OP_PREFETCH) ] = {
935 		[ C(RESULT_ACCESS) ] = 0x014e, /* L1D_PREFETCH.REQUESTS        */
936 		[ C(RESULT_MISS)   ] = 0x024e, /* L1D_PREFETCH.MISS            */
937 	},
938  },
939  [ C(L1I ) ] = {
940 	[ C(OP_READ) ] = {
941 		[ C(RESULT_ACCESS) ] = 0x0380, /* L1I.READS                    */
942 		[ C(RESULT_MISS)   ] = 0x0280, /* L1I.MISSES                   */
943 	},
944 	[ C(OP_WRITE) ] = {
945 		[ C(RESULT_ACCESS) ] = -1,
946 		[ C(RESULT_MISS)   ] = -1,
947 	},
948 	[ C(OP_PREFETCH) ] = {
949 		[ C(RESULT_ACCESS) ] = 0x0,
950 		[ C(RESULT_MISS)   ] = 0x0,
951 	},
952  },
953  [ C(LL  ) ] = {
954 	[ C(OP_READ) ] = {
955 		/* OFFCORE_RESPONSE.ANY_DATA.LOCAL_CACHE */
956 		[ C(RESULT_ACCESS) ] = 0x01b7,
957 		/* OFFCORE_RESPONSE.ANY_DATA.ANY_LLC_MISS */
958 		[ C(RESULT_MISS)   ] = 0x01b7,
959 	},
960 	/*
961 	 * Use RFO, not WRITEBACK, because a write miss would typically occur
962 	 * on RFO.
963 	 */
964 	[ C(OP_WRITE) ] = {
965 		/* OFFCORE_RESPONSE.ANY_RFO.LOCAL_CACHE */
966 		[ C(RESULT_ACCESS) ] = 0x01b7,
967 		/* OFFCORE_RESPONSE.ANY_RFO.ANY_LLC_MISS */
968 		[ C(RESULT_MISS)   ] = 0x01b7,
969 	},
970 	[ C(OP_PREFETCH) ] = {
971 		/* OFFCORE_RESPONSE.PREFETCH.LOCAL_CACHE */
972 		[ C(RESULT_ACCESS) ] = 0x01b7,
973 		/* OFFCORE_RESPONSE.PREFETCH.ANY_LLC_MISS */
974 		[ C(RESULT_MISS)   ] = 0x01b7,
975 	},
976  },
977  [ C(DTLB) ] = {
978 	[ C(OP_READ) ] = {
979 		[ C(RESULT_ACCESS) ] = 0x010b, /* MEM_INST_RETIRED.LOADS       */
980 		[ C(RESULT_MISS)   ] = 0x0108, /* DTLB_LOAD_MISSES.ANY         */
981 	},
982 	[ C(OP_WRITE) ] = {
983 		[ C(RESULT_ACCESS) ] = 0x020b, /* MEM_INST_RETURED.STORES      */
984 		[ C(RESULT_MISS)   ] = 0x010c, /* MEM_STORE_RETIRED.DTLB_MISS  */
985 	},
986 	[ C(OP_PREFETCH) ] = {
987 		[ C(RESULT_ACCESS) ] = 0x0,
988 		[ C(RESULT_MISS)   ] = 0x0,
989 	},
990  },
991  [ C(ITLB) ] = {
992 	[ C(OP_READ) ] = {
993 		[ C(RESULT_ACCESS) ] = 0x01c0, /* INST_RETIRED.ANY_P           */
994 		[ C(RESULT_MISS)   ] = 0x0185, /* ITLB_MISSES.ANY              */
995 	},
996 	[ C(OP_WRITE) ] = {
997 		[ C(RESULT_ACCESS) ] = -1,
998 		[ C(RESULT_MISS)   ] = -1,
999 	},
1000 	[ C(OP_PREFETCH) ] = {
1001 		[ C(RESULT_ACCESS) ] = -1,
1002 		[ C(RESULT_MISS)   ] = -1,
1003 	},
1004  },
1005  [ C(BPU ) ] = {
1006 	[ C(OP_READ) ] = {
1007 		[ C(RESULT_ACCESS) ] = 0x00c4, /* BR_INST_RETIRED.ALL_BRANCHES */
1008 		[ C(RESULT_MISS)   ] = 0x03e8, /* BPU_CLEARS.ANY               */
1009 	},
1010 	[ C(OP_WRITE) ] = {
1011 		[ C(RESULT_ACCESS) ] = -1,
1012 		[ C(RESULT_MISS)   ] = -1,
1013 	},
1014 	[ C(OP_PREFETCH) ] = {
1015 		[ C(RESULT_ACCESS) ] = -1,
1016 		[ C(RESULT_MISS)   ] = -1,
1017 	},
1018  },
1019  [ C(NODE) ] = {
1020 	[ C(OP_READ) ] = {
1021 		[ C(RESULT_ACCESS) ] = 0x01b7,
1022 		[ C(RESULT_MISS)   ] = 0x01b7,
1023 	},
1024 	[ C(OP_WRITE) ] = {
1025 		[ C(RESULT_ACCESS) ] = 0x01b7,
1026 		[ C(RESULT_MISS)   ] = 0x01b7,
1027 	},
1028 	[ C(OP_PREFETCH) ] = {
1029 		[ C(RESULT_ACCESS) ] = 0x01b7,
1030 		[ C(RESULT_MISS)   ] = 0x01b7,
1031 	},
1032  },
1033 };
1034 
1035 /*
1036  * Nehalem/Westmere MSR_OFFCORE_RESPONSE bits;
1037  * See IA32 SDM Vol 3B 30.6.1.3
1038  */
1039 
1040 #define NHM_DMND_DATA_RD	(1 << 0)
1041 #define NHM_DMND_RFO		(1 << 1)
1042 #define NHM_DMND_IFETCH		(1 << 2)
1043 #define NHM_DMND_WB		(1 << 3)
1044 #define NHM_PF_DATA_RD		(1 << 4)
1045 #define NHM_PF_DATA_RFO		(1 << 5)
1046 #define NHM_PF_IFETCH		(1 << 6)
1047 #define NHM_OFFCORE_OTHER	(1 << 7)
1048 #define NHM_UNCORE_HIT		(1 << 8)
1049 #define NHM_OTHER_CORE_HIT_SNP	(1 << 9)
1050 #define NHM_OTHER_CORE_HITM	(1 << 10)
1051         			/* reserved */
1052 #define NHM_REMOTE_CACHE_FWD	(1 << 12)
1053 #define NHM_REMOTE_DRAM		(1 << 13)
1054 #define NHM_LOCAL_DRAM		(1 << 14)
1055 #define NHM_NON_DRAM		(1 << 15)
1056 
1057 #define NHM_LOCAL		(NHM_LOCAL_DRAM|NHM_REMOTE_CACHE_FWD)
1058 #define NHM_REMOTE		(NHM_REMOTE_DRAM)
1059 
1060 #define NHM_DMND_READ		(NHM_DMND_DATA_RD)
1061 #define NHM_DMND_WRITE		(NHM_DMND_RFO|NHM_DMND_WB)
1062 #define NHM_DMND_PREFETCH	(NHM_PF_DATA_RD|NHM_PF_DATA_RFO)
1063 
1064 #define NHM_L3_HIT	(NHM_UNCORE_HIT|NHM_OTHER_CORE_HIT_SNP|NHM_OTHER_CORE_HITM)
1065 #define NHM_L3_MISS	(NHM_NON_DRAM|NHM_LOCAL_DRAM|NHM_REMOTE_DRAM|NHM_REMOTE_CACHE_FWD)
1066 #define NHM_L3_ACCESS	(NHM_L3_HIT|NHM_L3_MISS)
1067 
1068 static __initconst const u64 nehalem_hw_cache_extra_regs
1069 				[PERF_COUNT_HW_CACHE_MAX]
1070 				[PERF_COUNT_HW_CACHE_OP_MAX]
1071 				[PERF_COUNT_HW_CACHE_RESULT_MAX] =
1072 {
1073  [ C(LL  ) ] = {
1074 	[ C(OP_READ) ] = {
1075 		[ C(RESULT_ACCESS) ] = NHM_DMND_READ|NHM_L3_ACCESS,
1076 		[ C(RESULT_MISS)   ] = NHM_DMND_READ|NHM_L3_MISS,
1077 	},
1078 	[ C(OP_WRITE) ] = {
1079 		[ C(RESULT_ACCESS) ] = NHM_DMND_WRITE|NHM_L3_ACCESS,
1080 		[ C(RESULT_MISS)   ] = NHM_DMND_WRITE|NHM_L3_MISS,
1081 	},
1082 	[ C(OP_PREFETCH) ] = {
1083 		[ C(RESULT_ACCESS) ] = NHM_DMND_PREFETCH|NHM_L3_ACCESS,
1084 		[ C(RESULT_MISS)   ] = NHM_DMND_PREFETCH|NHM_L3_MISS,
1085 	},
1086  },
1087  [ C(NODE) ] = {
1088 	[ C(OP_READ) ] = {
1089 		[ C(RESULT_ACCESS) ] = NHM_DMND_READ|NHM_LOCAL|NHM_REMOTE,
1090 		[ C(RESULT_MISS)   ] = NHM_DMND_READ|NHM_REMOTE,
1091 	},
1092 	[ C(OP_WRITE) ] = {
1093 		[ C(RESULT_ACCESS) ] = NHM_DMND_WRITE|NHM_LOCAL|NHM_REMOTE,
1094 		[ C(RESULT_MISS)   ] = NHM_DMND_WRITE|NHM_REMOTE,
1095 	},
1096 	[ C(OP_PREFETCH) ] = {
1097 		[ C(RESULT_ACCESS) ] = NHM_DMND_PREFETCH|NHM_LOCAL|NHM_REMOTE,
1098 		[ C(RESULT_MISS)   ] = NHM_DMND_PREFETCH|NHM_REMOTE,
1099 	},
1100  },
1101 };
1102 
1103 static __initconst const u64 nehalem_hw_cache_event_ids
1104 				[PERF_COUNT_HW_CACHE_MAX]
1105 				[PERF_COUNT_HW_CACHE_OP_MAX]
1106 				[PERF_COUNT_HW_CACHE_RESULT_MAX] =
1107 {
1108  [ C(L1D) ] = {
1109 	[ C(OP_READ) ] = {
1110 		[ C(RESULT_ACCESS) ] = 0x010b, /* MEM_INST_RETIRED.LOADS       */
1111 		[ C(RESULT_MISS)   ] = 0x0151, /* L1D.REPL                     */
1112 	},
1113 	[ C(OP_WRITE) ] = {
1114 		[ C(RESULT_ACCESS) ] = 0x020b, /* MEM_INST_RETURED.STORES      */
1115 		[ C(RESULT_MISS)   ] = 0x0251, /* L1D.M_REPL                   */
1116 	},
1117 	[ C(OP_PREFETCH) ] = {
1118 		[ C(RESULT_ACCESS) ] = 0x014e, /* L1D_PREFETCH.REQUESTS        */
1119 		[ C(RESULT_MISS)   ] = 0x024e, /* L1D_PREFETCH.MISS            */
1120 	},
1121  },
1122  [ C(L1I ) ] = {
1123 	[ C(OP_READ) ] = {
1124 		[ C(RESULT_ACCESS) ] = 0x0380, /* L1I.READS                    */
1125 		[ C(RESULT_MISS)   ] = 0x0280, /* L1I.MISSES                   */
1126 	},
1127 	[ C(OP_WRITE) ] = {
1128 		[ C(RESULT_ACCESS) ] = -1,
1129 		[ C(RESULT_MISS)   ] = -1,
1130 	},
1131 	[ C(OP_PREFETCH) ] = {
1132 		[ C(RESULT_ACCESS) ] = 0x0,
1133 		[ C(RESULT_MISS)   ] = 0x0,
1134 	},
1135  },
1136  [ C(LL  ) ] = {
1137 	[ C(OP_READ) ] = {
1138 		/* OFFCORE_RESPONSE.ANY_DATA.LOCAL_CACHE */
1139 		[ C(RESULT_ACCESS) ] = 0x01b7,
1140 		/* OFFCORE_RESPONSE.ANY_DATA.ANY_LLC_MISS */
1141 		[ C(RESULT_MISS)   ] = 0x01b7,
1142 	},
1143 	/*
1144 	 * Use RFO, not WRITEBACK, because a write miss would typically occur
1145 	 * on RFO.
1146 	 */
1147 	[ C(OP_WRITE) ] = {
1148 		/* OFFCORE_RESPONSE.ANY_RFO.LOCAL_CACHE */
1149 		[ C(RESULT_ACCESS) ] = 0x01b7,
1150 		/* OFFCORE_RESPONSE.ANY_RFO.ANY_LLC_MISS */
1151 		[ C(RESULT_MISS)   ] = 0x01b7,
1152 	},
1153 	[ C(OP_PREFETCH) ] = {
1154 		/* OFFCORE_RESPONSE.PREFETCH.LOCAL_CACHE */
1155 		[ C(RESULT_ACCESS) ] = 0x01b7,
1156 		/* OFFCORE_RESPONSE.PREFETCH.ANY_LLC_MISS */
1157 		[ C(RESULT_MISS)   ] = 0x01b7,
1158 	},
1159  },
1160  [ C(DTLB) ] = {
1161 	[ C(OP_READ) ] = {
1162 		[ C(RESULT_ACCESS) ] = 0x0f40, /* L1D_CACHE_LD.MESI   (alias)  */
1163 		[ C(RESULT_MISS)   ] = 0x0108, /* DTLB_LOAD_MISSES.ANY         */
1164 	},
1165 	[ C(OP_WRITE) ] = {
1166 		[ C(RESULT_ACCESS) ] = 0x0f41, /* L1D_CACHE_ST.MESI   (alias)  */
1167 		[ C(RESULT_MISS)   ] = 0x010c, /* MEM_STORE_RETIRED.DTLB_MISS  */
1168 	},
1169 	[ C(OP_PREFETCH) ] = {
1170 		[ C(RESULT_ACCESS) ] = 0x0,
1171 		[ C(RESULT_MISS)   ] = 0x0,
1172 	},
1173  },
1174  [ C(ITLB) ] = {
1175 	[ C(OP_READ) ] = {
1176 		[ C(RESULT_ACCESS) ] = 0x01c0, /* INST_RETIRED.ANY_P           */
1177 		[ C(RESULT_MISS)   ] = 0x20c8, /* ITLB_MISS_RETIRED            */
1178 	},
1179 	[ C(OP_WRITE) ] = {
1180 		[ C(RESULT_ACCESS) ] = -1,
1181 		[ C(RESULT_MISS)   ] = -1,
1182 	},
1183 	[ C(OP_PREFETCH) ] = {
1184 		[ C(RESULT_ACCESS) ] = -1,
1185 		[ C(RESULT_MISS)   ] = -1,
1186 	},
1187  },
1188  [ C(BPU ) ] = {
1189 	[ C(OP_READ) ] = {
1190 		[ C(RESULT_ACCESS) ] = 0x00c4, /* BR_INST_RETIRED.ALL_BRANCHES */
1191 		[ C(RESULT_MISS)   ] = 0x03e8, /* BPU_CLEARS.ANY               */
1192 	},
1193 	[ C(OP_WRITE) ] = {
1194 		[ C(RESULT_ACCESS) ] = -1,
1195 		[ C(RESULT_MISS)   ] = -1,
1196 	},
1197 	[ C(OP_PREFETCH) ] = {
1198 		[ C(RESULT_ACCESS) ] = -1,
1199 		[ C(RESULT_MISS)   ] = -1,
1200 	},
1201  },
1202  [ C(NODE) ] = {
1203 	[ C(OP_READ) ] = {
1204 		[ C(RESULT_ACCESS) ] = 0x01b7,
1205 		[ C(RESULT_MISS)   ] = 0x01b7,
1206 	},
1207 	[ C(OP_WRITE) ] = {
1208 		[ C(RESULT_ACCESS) ] = 0x01b7,
1209 		[ C(RESULT_MISS)   ] = 0x01b7,
1210 	},
1211 	[ C(OP_PREFETCH) ] = {
1212 		[ C(RESULT_ACCESS) ] = 0x01b7,
1213 		[ C(RESULT_MISS)   ] = 0x01b7,
1214 	},
1215  },
1216 };
1217 
1218 static __initconst const u64 core2_hw_cache_event_ids
1219 				[PERF_COUNT_HW_CACHE_MAX]
1220 				[PERF_COUNT_HW_CACHE_OP_MAX]
1221 				[PERF_COUNT_HW_CACHE_RESULT_MAX] =
1222 {
1223  [ C(L1D) ] = {
1224 	[ C(OP_READ) ] = {
1225 		[ C(RESULT_ACCESS) ] = 0x0f40, /* L1D_CACHE_LD.MESI          */
1226 		[ C(RESULT_MISS)   ] = 0x0140, /* L1D_CACHE_LD.I_STATE       */
1227 	},
1228 	[ C(OP_WRITE) ] = {
1229 		[ C(RESULT_ACCESS) ] = 0x0f41, /* L1D_CACHE_ST.MESI          */
1230 		[ C(RESULT_MISS)   ] = 0x0141, /* L1D_CACHE_ST.I_STATE       */
1231 	},
1232 	[ C(OP_PREFETCH) ] = {
1233 		[ C(RESULT_ACCESS) ] = 0x104e, /* L1D_PREFETCH.REQUESTS      */
1234 		[ C(RESULT_MISS)   ] = 0,
1235 	},
1236  },
1237  [ C(L1I ) ] = {
1238 	[ C(OP_READ) ] = {
1239 		[ C(RESULT_ACCESS) ] = 0x0080, /* L1I.READS                  */
1240 		[ C(RESULT_MISS)   ] = 0x0081, /* L1I.MISSES                 */
1241 	},
1242 	[ C(OP_WRITE) ] = {
1243 		[ C(RESULT_ACCESS) ] = -1,
1244 		[ C(RESULT_MISS)   ] = -1,
1245 	},
1246 	[ C(OP_PREFETCH) ] = {
1247 		[ C(RESULT_ACCESS) ] = 0,
1248 		[ C(RESULT_MISS)   ] = 0,
1249 	},
1250  },
1251  [ C(LL  ) ] = {
1252 	[ C(OP_READ) ] = {
1253 		[ C(RESULT_ACCESS) ] = 0x4f29, /* L2_LD.MESI                 */
1254 		[ C(RESULT_MISS)   ] = 0x4129, /* L2_LD.ISTATE               */
1255 	},
1256 	[ C(OP_WRITE) ] = {
1257 		[ C(RESULT_ACCESS) ] = 0x4f2A, /* L2_ST.MESI                 */
1258 		[ C(RESULT_MISS)   ] = 0x412A, /* L2_ST.ISTATE               */
1259 	},
1260 	[ C(OP_PREFETCH) ] = {
1261 		[ C(RESULT_ACCESS) ] = 0,
1262 		[ C(RESULT_MISS)   ] = 0,
1263 	},
1264  },
1265  [ C(DTLB) ] = {
1266 	[ C(OP_READ) ] = {
1267 		[ C(RESULT_ACCESS) ] = 0x0f40, /* L1D_CACHE_LD.MESI  (alias) */
1268 		[ C(RESULT_MISS)   ] = 0x0208, /* DTLB_MISSES.MISS_LD        */
1269 	},
1270 	[ C(OP_WRITE) ] = {
1271 		[ C(RESULT_ACCESS) ] = 0x0f41, /* L1D_CACHE_ST.MESI  (alias) */
1272 		[ C(RESULT_MISS)   ] = 0x0808, /* DTLB_MISSES.MISS_ST        */
1273 	},
1274 	[ C(OP_PREFETCH) ] = {
1275 		[ C(RESULT_ACCESS) ] = 0,
1276 		[ C(RESULT_MISS)   ] = 0,
1277 	},
1278  },
1279  [ C(ITLB) ] = {
1280 	[ C(OP_READ) ] = {
1281 		[ C(RESULT_ACCESS) ] = 0x00c0, /* INST_RETIRED.ANY_P         */
1282 		[ C(RESULT_MISS)   ] = 0x1282, /* ITLBMISSES                 */
1283 	},
1284 	[ C(OP_WRITE) ] = {
1285 		[ C(RESULT_ACCESS) ] = -1,
1286 		[ C(RESULT_MISS)   ] = -1,
1287 	},
1288 	[ C(OP_PREFETCH) ] = {
1289 		[ C(RESULT_ACCESS) ] = -1,
1290 		[ C(RESULT_MISS)   ] = -1,
1291 	},
1292  },
1293  [ C(BPU ) ] = {
1294 	[ C(OP_READ) ] = {
1295 		[ C(RESULT_ACCESS) ] = 0x00c4, /* BR_INST_RETIRED.ANY        */
1296 		[ C(RESULT_MISS)   ] = 0x00c5, /* BP_INST_RETIRED.MISPRED    */
1297 	},
1298 	[ C(OP_WRITE) ] = {
1299 		[ C(RESULT_ACCESS) ] = -1,
1300 		[ C(RESULT_MISS)   ] = -1,
1301 	},
1302 	[ C(OP_PREFETCH) ] = {
1303 		[ C(RESULT_ACCESS) ] = -1,
1304 		[ C(RESULT_MISS)   ] = -1,
1305 	},
1306  },
1307 };
1308 
1309 static __initconst const u64 atom_hw_cache_event_ids
1310 				[PERF_COUNT_HW_CACHE_MAX]
1311 				[PERF_COUNT_HW_CACHE_OP_MAX]
1312 				[PERF_COUNT_HW_CACHE_RESULT_MAX] =
1313 {
1314  [ C(L1D) ] = {
1315 	[ C(OP_READ) ] = {
1316 		[ C(RESULT_ACCESS) ] = 0x2140, /* L1D_CACHE.LD               */
1317 		[ C(RESULT_MISS)   ] = 0,
1318 	},
1319 	[ C(OP_WRITE) ] = {
1320 		[ C(RESULT_ACCESS) ] = 0x2240, /* L1D_CACHE.ST               */
1321 		[ C(RESULT_MISS)   ] = 0,
1322 	},
1323 	[ C(OP_PREFETCH) ] = {
1324 		[ C(RESULT_ACCESS) ] = 0x0,
1325 		[ C(RESULT_MISS)   ] = 0,
1326 	},
1327  },
1328  [ C(L1I ) ] = {
1329 	[ C(OP_READ) ] = {
1330 		[ C(RESULT_ACCESS) ] = 0x0380, /* L1I.READS                  */
1331 		[ C(RESULT_MISS)   ] = 0x0280, /* L1I.MISSES                 */
1332 	},
1333 	[ C(OP_WRITE) ] = {
1334 		[ C(RESULT_ACCESS) ] = -1,
1335 		[ C(RESULT_MISS)   ] = -1,
1336 	},
1337 	[ C(OP_PREFETCH) ] = {
1338 		[ C(RESULT_ACCESS) ] = 0,
1339 		[ C(RESULT_MISS)   ] = 0,
1340 	},
1341  },
1342  [ C(LL  ) ] = {
1343 	[ C(OP_READ) ] = {
1344 		[ C(RESULT_ACCESS) ] = 0x4f29, /* L2_LD.MESI                 */
1345 		[ C(RESULT_MISS)   ] = 0x4129, /* L2_LD.ISTATE               */
1346 	},
1347 	[ C(OP_WRITE) ] = {
1348 		[ C(RESULT_ACCESS) ] = 0x4f2A, /* L2_ST.MESI                 */
1349 		[ C(RESULT_MISS)   ] = 0x412A, /* L2_ST.ISTATE               */
1350 	},
1351 	[ C(OP_PREFETCH) ] = {
1352 		[ C(RESULT_ACCESS) ] = 0,
1353 		[ C(RESULT_MISS)   ] = 0,
1354 	},
1355  },
1356  [ C(DTLB) ] = {
1357 	[ C(OP_READ) ] = {
1358 		[ C(RESULT_ACCESS) ] = 0x2140, /* L1D_CACHE_LD.MESI  (alias) */
1359 		[ C(RESULT_MISS)   ] = 0x0508, /* DTLB_MISSES.MISS_LD        */
1360 	},
1361 	[ C(OP_WRITE) ] = {
1362 		[ C(RESULT_ACCESS) ] = 0x2240, /* L1D_CACHE_ST.MESI  (alias) */
1363 		[ C(RESULT_MISS)   ] = 0x0608, /* DTLB_MISSES.MISS_ST        */
1364 	},
1365 	[ C(OP_PREFETCH) ] = {
1366 		[ C(RESULT_ACCESS) ] = 0,
1367 		[ C(RESULT_MISS)   ] = 0,
1368 	},
1369  },
1370  [ C(ITLB) ] = {
1371 	[ C(OP_READ) ] = {
1372 		[ C(RESULT_ACCESS) ] = 0x00c0, /* INST_RETIRED.ANY_P         */
1373 		[ C(RESULT_MISS)   ] = 0x0282, /* ITLB.MISSES                */
1374 	},
1375 	[ C(OP_WRITE) ] = {
1376 		[ C(RESULT_ACCESS) ] = -1,
1377 		[ C(RESULT_MISS)   ] = -1,
1378 	},
1379 	[ C(OP_PREFETCH) ] = {
1380 		[ C(RESULT_ACCESS) ] = -1,
1381 		[ C(RESULT_MISS)   ] = -1,
1382 	},
1383  },
1384  [ C(BPU ) ] = {
1385 	[ C(OP_READ) ] = {
1386 		[ C(RESULT_ACCESS) ] = 0x00c4, /* BR_INST_RETIRED.ANY        */
1387 		[ C(RESULT_MISS)   ] = 0x00c5, /* BP_INST_RETIRED.MISPRED    */
1388 	},
1389 	[ C(OP_WRITE) ] = {
1390 		[ C(RESULT_ACCESS) ] = -1,
1391 		[ C(RESULT_MISS)   ] = -1,
1392 	},
1393 	[ C(OP_PREFETCH) ] = {
1394 		[ C(RESULT_ACCESS) ] = -1,
1395 		[ C(RESULT_MISS)   ] = -1,
1396 	},
1397  },
1398 };
1399 
1400 EVENT_ATTR_STR(topdown-total-slots, td_total_slots_slm, "event=0x3c");
1401 EVENT_ATTR_STR(topdown-total-slots.scale, td_total_slots_scale_slm, "2");
1402 /* no_alloc_cycles.not_delivered */
1403 EVENT_ATTR_STR(topdown-fetch-bubbles, td_fetch_bubbles_slm,
1404 	       "event=0xca,umask=0x50");
1405 EVENT_ATTR_STR(topdown-fetch-bubbles.scale, td_fetch_bubbles_scale_slm, "2");
1406 /* uops_retired.all */
1407 EVENT_ATTR_STR(topdown-slots-issued, td_slots_issued_slm,
1408 	       "event=0xc2,umask=0x10");
1409 /* uops_retired.all */
1410 EVENT_ATTR_STR(topdown-slots-retired, td_slots_retired_slm,
1411 	       "event=0xc2,umask=0x10");
1412 
1413 static struct attribute *slm_events_attrs[] = {
1414 	EVENT_PTR(td_total_slots_slm),
1415 	EVENT_PTR(td_total_slots_scale_slm),
1416 	EVENT_PTR(td_fetch_bubbles_slm),
1417 	EVENT_PTR(td_fetch_bubbles_scale_slm),
1418 	EVENT_PTR(td_slots_issued_slm),
1419 	EVENT_PTR(td_slots_retired_slm),
1420 	NULL
1421 };
1422 
1423 static struct extra_reg intel_slm_extra_regs[] __read_mostly =
1424 {
1425 	/* must define OFFCORE_RSP_X first, see intel_fixup_er() */
1426 	INTEL_UEVENT_EXTRA_REG(0x01b7, MSR_OFFCORE_RSP_0, 0x768005ffffull, RSP_0),
1427 	INTEL_UEVENT_EXTRA_REG(0x02b7, MSR_OFFCORE_RSP_1, 0x368005ffffull, RSP_1),
1428 	EVENT_EXTRA_END
1429 };
1430 
1431 #define SLM_DMND_READ		SNB_DMND_DATA_RD
1432 #define SLM_DMND_WRITE		SNB_DMND_RFO
1433 #define SLM_DMND_PREFETCH	(SNB_PF_DATA_RD|SNB_PF_RFO)
1434 
1435 #define SLM_SNP_ANY		(SNB_SNP_NONE|SNB_SNP_MISS|SNB_NO_FWD|SNB_HITM)
1436 #define SLM_LLC_ACCESS		SNB_RESP_ANY
1437 #define SLM_LLC_MISS		(SLM_SNP_ANY|SNB_NON_DRAM)
1438 
1439 static __initconst const u64 slm_hw_cache_extra_regs
1440 				[PERF_COUNT_HW_CACHE_MAX]
1441 				[PERF_COUNT_HW_CACHE_OP_MAX]
1442 				[PERF_COUNT_HW_CACHE_RESULT_MAX] =
1443 {
1444  [ C(LL  ) ] = {
1445 	[ C(OP_READ) ] = {
1446 		[ C(RESULT_ACCESS) ] = SLM_DMND_READ|SLM_LLC_ACCESS,
1447 		[ C(RESULT_MISS)   ] = 0,
1448 	},
1449 	[ C(OP_WRITE) ] = {
1450 		[ C(RESULT_ACCESS) ] = SLM_DMND_WRITE|SLM_LLC_ACCESS,
1451 		[ C(RESULT_MISS)   ] = SLM_DMND_WRITE|SLM_LLC_MISS,
1452 	},
1453 	[ C(OP_PREFETCH) ] = {
1454 		[ C(RESULT_ACCESS) ] = SLM_DMND_PREFETCH|SLM_LLC_ACCESS,
1455 		[ C(RESULT_MISS)   ] = SLM_DMND_PREFETCH|SLM_LLC_MISS,
1456 	},
1457  },
1458 };
1459 
1460 static __initconst const u64 slm_hw_cache_event_ids
1461 				[PERF_COUNT_HW_CACHE_MAX]
1462 				[PERF_COUNT_HW_CACHE_OP_MAX]
1463 				[PERF_COUNT_HW_CACHE_RESULT_MAX] =
1464 {
1465  [ C(L1D) ] = {
1466 	[ C(OP_READ) ] = {
1467 		[ C(RESULT_ACCESS) ] = 0,
1468 		[ C(RESULT_MISS)   ] = 0x0104, /* LD_DCU_MISS */
1469 	},
1470 	[ C(OP_WRITE) ] = {
1471 		[ C(RESULT_ACCESS) ] = 0,
1472 		[ C(RESULT_MISS)   ] = 0,
1473 	},
1474 	[ C(OP_PREFETCH) ] = {
1475 		[ C(RESULT_ACCESS) ] = 0,
1476 		[ C(RESULT_MISS)   ] = 0,
1477 	},
1478  },
1479  [ C(L1I ) ] = {
1480 	[ C(OP_READ) ] = {
1481 		[ C(RESULT_ACCESS) ] = 0x0380, /* ICACHE.ACCESSES */
1482 		[ C(RESULT_MISS)   ] = 0x0280, /* ICACGE.MISSES */
1483 	},
1484 	[ C(OP_WRITE) ] = {
1485 		[ C(RESULT_ACCESS) ] = -1,
1486 		[ C(RESULT_MISS)   ] = -1,
1487 	},
1488 	[ C(OP_PREFETCH) ] = {
1489 		[ C(RESULT_ACCESS) ] = 0,
1490 		[ C(RESULT_MISS)   ] = 0,
1491 	},
1492  },
1493  [ C(LL  ) ] = {
1494 	[ C(OP_READ) ] = {
1495 		/* OFFCORE_RESPONSE.ANY_DATA.LOCAL_CACHE */
1496 		[ C(RESULT_ACCESS) ] = 0x01b7,
1497 		[ C(RESULT_MISS)   ] = 0,
1498 	},
1499 	[ C(OP_WRITE) ] = {
1500 		/* OFFCORE_RESPONSE.ANY_RFO.LOCAL_CACHE */
1501 		[ C(RESULT_ACCESS) ] = 0x01b7,
1502 		/* OFFCORE_RESPONSE.ANY_RFO.ANY_LLC_MISS */
1503 		[ C(RESULT_MISS)   ] = 0x01b7,
1504 	},
1505 	[ C(OP_PREFETCH) ] = {
1506 		/* OFFCORE_RESPONSE.PREFETCH.LOCAL_CACHE */
1507 		[ C(RESULT_ACCESS) ] = 0x01b7,
1508 		/* OFFCORE_RESPONSE.PREFETCH.ANY_LLC_MISS */
1509 		[ C(RESULT_MISS)   ] = 0x01b7,
1510 	},
1511  },
1512  [ C(DTLB) ] = {
1513 	[ C(OP_READ) ] = {
1514 		[ C(RESULT_ACCESS) ] = 0,
1515 		[ C(RESULT_MISS)   ] = 0x0804, /* LD_DTLB_MISS */
1516 	},
1517 	[ C(OP_WRITE) ] = {
1518 		[ C(RESULT_ACCESS) ] = 0,
1519 		[ C(RESULT_MISS)   ] = 0,
1520 	},
1521 	[ C(OP_PREFETCH) ] = {
1522 		[ C(RESULT_ACCESS) ] = 0,
1523 		[ C(RESULT_MISS)   ] = 0,
1524 	},
1525  },
1526  [ C(ITLB) ] = {
1527 	[ C(OP_READ) ] = {
1528 		[ C(RESULT_ACCESS) ] = 0x00c0, /* INST_RETIRED.ANY_P */
1529 		[ C(RESULT_MISS)   ] = 0x40205, /* PAGE_WALKS.I_SIDE_WALKS */
1530 	},
1531 	[ C(OP_WRITE) ] = {
1532 		[ C(RESULT_ACCESS) ] = -1,
1533 		[ C(RESULT_MISS)   ] = -1,
1534 	},
1535 	[ C(OP_PREFETCH) ] = {
1536 		[ C(RESULT_ACCESS) ] = -1,
1537 		[ C(RESULT_MISS)   ] = -1,
1538 	},
1539  },
1540  [ C(BPU ) ] = {
1541 	[ C(OP_READ) ] = {
1542 		[ C(RESULT_ACCESS) ] = 0x00c4, /* BR_INST_RETIRED.ANY */
1543 		[ C(RESULT_MISS)   ] = 0x00c5, /* BP_INST_RETIRED.MISPRED */
1544 	},
1545 	[ C(OP_WRITE) ] = {
1546 		[ C(RESULT_ACCESS) ] = -1,
1547 		[ C(RESULT_MISS)   ] = -1,
1548 	},
1549 	[ C(OP_PREFETCH) ] = {
1550 		[ C(RESULT_ACCESS) ] = -1,
1551 		[ C(RESULT_MISS)   ] = -1,
1552 	},
1553  },
1554 };
1555 
1556 EVENT_ATTR_STR(topdown-total-slots, td_total_slots_glm, "event=0x3c");
1557 EVENT_ATTR_STR(topdown-total-slots.scale, td_total_slots_scale_glm, "3");
1558 /* UOPS_NOT_DELIVERED.ANY */
1559 EVENT_ATTR_STR(topdown-fetch-bubbles, td_fetch_bubbles_glm, "event=0x9c");
1560 /* ISSUE_SLOTS_NOT_CONSUMED.RECOVERY */
1561 EVENT_ATTR_STR(topdown-recovery-bubbles, td_recovery_bubbles_glm, "event=0xca,umask=0x02");
1562 /* UOPS_RETIRED.ANY */
1563 EVENT_ATTR_STR(topdown-slots-retired, td_slots_retired_glm, "event=0xc2");
1564 /* UOPS_ISSUED.ANY */
1565 EVENT_ATTR_STR(topdown-slots-issued, td_slots_issued_glm, "event=0x0e");
1566 
1567 static struct attribute *glm_events_attrs[] = {
1568 	EVENT_PTR(td_total_slots_glm),
1569 	EVENT_PTR(td_total_slots_scale_glm),
1570 	EVENT_PTR(td_fetch_bubbles_glm),
1571 	EVENT_PTR(td_recovery_bubbles_glm),
1572 	EVENT_PTR(td_slots_issued_glm),
1573 	EVENT_PTR(td_slots_retired_glm),
1574 	NULL
1575 };
1576 
1577 static struct extra_reg intel_glm_extra_regs[] __read_mostly = {
1578 	/* must define OFFCORE_RSP_X first, see intel_fixup_er() */
1579 	INTEL_UEVENT_EXTRA_REG(0x01b7, MSR_OFFCORE_RSP_0, 0x760005ffbfull, RSP_0),
1580 	INTEL_UEVENT_EXTRA_REG(0x02b7, MSR_OFFCORE_RSP_1, 0x360005ffbfull, RSP_1),
1581 	EVENT_EXTRA_END
1582 };
1583 
1584 #define GLM_DEMAND_DATA_RD		BIT_ULL(0)
1585 #define GLM_DEMAND_RFO			BIT_ULL(1)
1586 #define GLM_ANY_RESPONSE		BIT_ULL(16)
1587 #define GLM_SNP_NONE_OR_MISS		BIT_ULL(33)
1588 #define GLM_DEMAND_READ			GLM_DEMAND_DATA_RD
1589 #define GLM_DEMAND_WRITE		GLM_DEMAND_RFO
1590 #define GLM_DEMAND_PREFETCH		(SNB_PF_DATA_RD|SNB_PF_RFO)
1591 #define GLM_LLC_ACCESS			GLM_ANY_RESPONSE
1592 #define GLM_SNP_ANY			(GLM_SNP_NONE_OR_MISS|SNB_NO_FWD|SNB_HITM)
1593 #define GLM_LLC_MISS			(GLM_SNP_ANY|SNB_NON_DRAM)
1594 
1595 static __initconst const u64 glm_hw_cache_event_ids
1596 				[PERF_COUNT_HW_CACHE_MAX]
1597 				[PERF_COUNT_HW_CACHE_OP_MAX]
1598 				[PERF_COUNT_HW_CACHE_RESULT_MAX] = {
1599 	[C(L1D)] = {
1600 		[C(OP_READ)] = {
1601 			[C(RESULT_ACCESS)]	= 0x81d0,	/* MEM_UOPS_RETIRED.ALL_LOADS */
1602 			[C(RESULT_MISS)]	= 0x0,
1603 		},
1604 		[C(OP_WRITE)] = {
1605 			[C(RESULT_ACCESS)]	= 0x82d0,	/* MEM_UOPS_RETIRED.ALL_STORES */
1606 			[C(RESULT_MISS)]	= 0x0,
1607 		},
1608 		[C(OP_PREFETCH)] = {
1609 			[C(RESULT_ACCESS)]	= 0x0,
1610 			[C(RESULT_MISS)]	= 0x0,
1611 		},
1612 	},
1613 	[C(L1I)] = {
1614 		[C(OP_READ)] = {
1615 			[C(RESULT_ACCESS)]	= 0x0380,	/* ICACHE.ACCESSES */
1616 			[C(RESULT_MISS)]	= 0x0280,	/* ICACHE.MISSES */
1617 		},
1618 		[C(OP_WRITE)] = {
1619 			[C(RESULT_ACCESS)]	= -1,
1620 			[C(RESULT_MISS)]	= -1,
1621 		},
1622 		[C(OP_PREFETCH)] = {
1623 			[C(RESULT_ACCESS)]	= 0x0,
1624 			[C(RESULT_MISS)]	= 0x0,
1625 		},
1626 	},
1627 	[C(LL)] = {
1628 		[C(OP_READ)] = {
1629 			[C(RESULT_ACCESS)]	= 0x1b7,	/* OFFCORE_RESPONSE */
1630 			[C(RESULT_MISS)]	= 0x1b7,	/* OFFCORE_RESPONSE */
1631 		},
1632 		[C(OP_WRITE)] = {
1633 			[C(RESULT_ACCESS)]	= 0x1b7,	/* OFFCORE_RESPONSE */
1634 			[C(RESULT_MISS)]	= 0x1b7,	/* OFFCORE_RESPONSE */
1635 		},
1636 		[C(OP_PREFETCH)] = {
1637 			[C(RESULT_ACCESS)]	= 0x1b7,	/* OFFCORE_RESPONSE */
1638 			[C(RESULT_MISS)]	= 0x1b7,	/* OFFCORE_RESPONSE */
1639 		},
1640 	},
1641 	[C(DTLB)] = {
1642 		[C(OP_READ)] = {
1643 			[C(RESULT_ACCESS)]	= 0x81d0,	/* MEM_UOPS_RETIRED.ALL_LOADS */
1644 			[C(RESULT_MISS)]	= 0x0,
1645 		},
1646 		[C(OP_WRITE)] = {
1647 			[C(RESULT_ACCESS)]	= 0x82d0,	/* MEM_UOPS_RETIRED.ALL_STORES */
1648 			[C(RESULT_MISS)]	= 0x0,
1649 		},
1650 		[C(OP_PREFETCH)] = {
1651 			[C(RESULT_ACCESS)]	= 0x0,
1652 			[C(RESULT_MISS)]	= 0x0,
1653 		},
1654 	},
1655 	[C(ITLB)] = {
1656 		[C(OP_READ)] = {
1657 			[C(RESULT_ACCESS)]	= 0x00c0,	/* INST_RETIRED.ANY_P */
1658 			[C(RESULT_MISS)]	= 0x0481,	/* ITLB.MISS */
1659 		},
1660 		[C(OP_WRITE)] = {
1661 			[C(RESULT_ACCESS)]	= -1,
1662 			[C(RESULT_MISS)]	= -1,
1663 		},
1664 		[C(OP_PREFETCH)] = {
1665 			[C(RESULT_ACCESS)]	= -1,
1666 			[C(RESULT_MISS)]	= -1,
1667 		},
1668 	},
1669 	[C(BPU)] = {
1670 		[C(OP_READ)] = {
1671 			[C(RESULT_ACCESS)]	= 0x00c4,	/* BR_INST_RETIRED.ALL_BRANCHES */
1672 			[C(RESULT_MISS)]	= 0x00c5,	/* BR_MISP_RETIRED.ALL_BRANCHES */
1673 		},
1674 		[C(OP_WRITE)] = {
1675 			[C(RESULT_ACCESS)]	= -1,
1676 			[C(RESULT_MISS)]	= -1,
1677 		},
1678 		[C(OP_PREFETCH)] = {
1679 			[C(RESULT_ACCESS)]	= -1,
1680 			[C(RESULT_MISS)]	= -1,
1681 		},
1682 	},
1683 };
1684 
1685 static __initconst const u64 glm_hw_cache_extra_regs
1686 				[PERF_COUNT_HW_CACHE_MAX]
1687 				[PERF_COUNT_HW_CACHE_OP_MAX]
1688 				[PERF_COUNT_HW_CACHE_RESULT_MAX] = {
1689 	[C(LL)] = {
1690 		[C(OP_READ)] = {
1691 			[C(RESULT_ACCESS)]	= GLM_DEMAND_READ|
1692 						  GLM_LLC_ACCESS,
1693 			[C(RESULT_MISS)]	= GLM_DEMAND_READ|
1694 						  GLM_LLC_MISS,
1695 		},
1696 		[C(OP_WRITE)] = {
1697 			[C(RESULT_ACCESS)]	= GLM_DEMAND_WRITE|
1698 						  GLM_LLC_ACCESS,
1699 			[C(RESULT_MISS)]	= GLM_DEMAND_WRITE|
1700 						  GLM_LLC_MISS,
1701 		},
1702 		[C(OP_PREFETCH)] = {
1703 			[C(RESULT_ACCESS)]	= GLM_DEMAND_PREFETCH|
1704 						  GLM_LLC_ACCESS,
1705 			[C(RESULT_MISS)]	= GLM_DEMAND_PREFETCH|
1706 						  GLM_LLC_MISS,
1707 		},
1708 	},
1709 };
1710 
1711 static __initconst const u64 glp_hw_cache_event_ids
1712 				[PERF_COUNT_HW_CACHE_MAX]
1713 				[PERF_COUNT_HW_CACHE_OP_MAX]
1714 				[PERF_COUNT_HW_CACHE_RESULT_MAX] = {
1715 	[C(L1D)] = {
1716 		[C(OP_READ)] = {
1717 			[C(RESULT_ACCESS)]	= 0x81d0,	/* MEM_UOPS_RETIRED.ALL_LOADS */
1718 			[C(RESULT_MISS)]	= 0x0,
1719 		},
1720 		[C(OP_WRITE)] = {
1721 			[C(RESULT_ACCESS)]	= 0x82d0,	/* MEM_UOPS_RETIRED.ALL_STORES */
1722 			[C(RESULT_MISS)]	= 0x0,
1723 		},
1724 		[C(OP_PREFETCH)] = {
1725 			[C(RESULT_ACCESS)]	= 0x0,
1726 			[C(RESULT_MISS)]	= 0x0,
1727 		},
1728 	},
1729 	[C(L1I)] = {
1730 		[C(OP_READ)] = {
1731 			[C(RESULT_ACCESS)]	= 0x0380,	/* ICACHE.ACCESSES */
1732 			[C(RESULT_MISS)]	= 0x0280,	/* ICACHE.MISSES */
1733 		},
1734 		[C(OP_WRITE)] = {
1735 			[C(RESULT_ACCESS)]	= -1,
1736 			[C(RESULT_MISS)]	= -1,
1737 		},
1738 		[C(OP_PREFETCH)] = {
1739 			[C(RESULT_ACCESS)]	= 0x0,
1740 			[C(RESULT_MISS)]	= 0x0,
1741 		},
1742 	},
1743 	[C(LL)] = {
1744 		[C(OP_READ)] = {
1745 			[C(RESULT_ACCESS)]	= 0x1b7,	/* OFFCORE_RESPONSE */
1746 			[C(RESULT_MISS)]	= 0x1b7,	/* OFFCORE_RESPONSE */
1747 		},
1748 		[C(OP_WRITE)] = {
1749 			[C(RESULT_ACCESS)]	= 0x1b7,	/* OFFCORE_RESPONSE */
1750 			[C(RESULT_MISS)]	= 0x1b7,	/* OFFCORE_RESPONSE */
1751 		},
1752 		[C(OP_PREFETCH)] = {
1753 			[C(RESULT_ACCESS)]	= 0x0,
1754 			[C(RESULT_MISS)]	= 0x0,
1755 		},
1756 	},
1757 	[C(DTLB)] = {
1758 		[C(OP_READ)] = {
1759 			[C(RESULT_ACCESS)]	= 0x81d0,	/* MEM_UOPS_RETIRED.ALL_LOADS */
1760 			[C(RESULT_MISS)]	= 0xe08,	/* DTLB_LOAD_MISSES.WALK_COMPLETED */
1761 		},
1762 		[C(OP_WRITE)] = {
1763 			[C(RESULT_ACCESS)]	= 0x82d0,	/* MEM_UOPS_RETIRED.ALL_STORES */
1764 			[C(RESULT_MISS)]	= 0xe49,	/* DTLB_STORE_MISSES.WALK_COMPLETED */
1765 		},
1766 		[C(OP_PREFETCH)] = {
1767 			[C(RESULT_ACCESS)]	= 0x0,
1768 			[C(RESULT_MISS)]	= 0x0,
1769 		},
1770 	},
1771 	[C(ITLB)] = {
1772 		[C(OP_READ)] = {
1773 			[C(RESULT_ACCESS)]	= 0x00c0,	/* INST_RETIRED.ANY_P */
1774 			[C(RESULT_MISS)]	= 0x0481,	/* ITLB.MISS */
1775 		},
1776 		[C(OP_WRITE)] = {
1777 			[C(RESULT_ACCESS)]	= -1,
1778 			[C(RESULT_MISS)]	= -1,
1779 		},
1780 		[C(OP_PREFETCH)] = {
1781 			[C(RESULT_ACCESS)]	= -1,
1782 			[C(RESULT_MISS)]	= -1,
1783 		},
1784 	},
1785 	[C(BPU)] = {
1786 		[C(OP_READ)] = {
1787 			[C(RESULT_ACCESS)]	= 0x00c4,	/* BR_INST_RETIRED.ALL_BRANCHES */
1788 			[C(RESULT_MISS)]	= 0x00c5,	/* BR_MISP_RETIRED.ALL_BRANCHES */
1789 		},
1790 		[C(OP_WRITE)] = {
1791 			[C(RESULT_ACCESS)]	= -1,
1792 			[C(RESULT_MISS)]	= -1,
1793 		},
1794 		[C(OP_PREFETCH)] = {
1795 			[C(RESULT_ACCESS)]	= -1,
1796 			[C(RESULT_MISS)]	= -1,
1797 		},
1798 	},
1799 };
1800 
1801 static __initconst const u64 glp_hw_cache_extra_regs
1802 				[PERF_COUNT_HW_CACHE_MAX]
1803 				[PERF_COUNT_HW_CACHE_OP_MAX]
1804 				[PERF_COUNT_HW_CACHE_RESULT_MAX] = {
1805 	[C(LL)] = {
1806 		[C(OP_READ)] = {
1807 			[C(RESULT_ACCESS)]	= GLM_DEMAND_READ|
1808 						  GLM_LLC_ACCESS,
1809 			[C(RESULT_MISS)]	= GLM_DEMAND_READ|
1810 						  GLM_LLC_MISS,
1811 		},
1812 		[C(OP_WRITE)] = {
1813 			[C(RESULT_ACCESS)]	= GLM_DEMAND_WRITE|
1814 						  GLM_LLC_ACCESS,
1815 			[C(RESULT_MISS)]	= GLM_DEMAND_WRITE|
1816 						  GLM_LLC_MISS,
1817 		},
1818 		[C(OP_PREFETCH)] = {
1819 			[C(RESULT_ACCESS)]	= 0x0,
1820 			[C(RESULT_MISS)]	= 0x0,
1821 		},
1822 	},
1823 };
1824 
1825 #define KNL_OT_L2_HITE		BIT_ULL(19) /* Other Tile L2 Hit */
1826 #define KNL_OT_L2_HITF		BIT_ULL(20) /* Other Tile L2 Hit */
1827 #define KNL_MCDRAM_LOCAL	BIT_ULL(21)
1828 #define KNL_MCDRAM_FAR		BIT_ULL(22)
1829 #define KNL_DDR_LOCAL		BIT_ULL(23)
1830 #define KNL_DDR_FAR		BIT_ULL(24)
1831 #define KNL_DRAM_ANY		(KNL_MCDRAM_LOCAL | KNL_MCDRAM_FAR | \
1832 				    KNL_DDR_LOCAL | KNL_DDR_FAR)
1833 #define KNL_L2_READ		SLM_DMND_READ
1834 #define KNL_L2_WRITE		SLM_DMND_WRITE
1835 #define KNL_L2_PREFETCH		SLM_DMND_PREFETCH
1836 #define KNL_L2_ACCESS		SLM_LLC_ACCESS
1837 #define KNL_L2_MISS		(KNL_OT_L2_HITE | KNL_OT_L2_HITF | \
1838 				   KNL_DRAM_ANY | SNB_SNP_ANY | \
1839 						  SNB_NON_DRAM)
1840 
1841 static __initconst const u64 knl_hw_cache_extra_regs
1842 				[PERF_COUNT_HW_CACHE_MAX]
1843 				[PERF_COUNT_HW_CACHE_OP_MAX]
1844 				[PERF_COUNT_HW_CACHE_RESULT_MAX] = {
1845 	[C(LL)] = {
1846 		[C(OP_READ)] = {
1847 			[C(RESULT_ACCESS)] = KNL_L2_READ | KNL_L2_ACCESS,
1848 			[C(RESULT_MISS)]   = 0,
1849 		},
1850 		[C(OP_WRITE)] = {
1851 			[C(RESULT_ACCESS)] = KNL_L2_WRITE | KNL_L2_ACCESS,
1852 			[C(RESULT_MISS)]   = KNL_L2_WRITE | KNL_L2_MISS,
1853 		},
1854 		[C(OP_PREFETCH)] = {
1855 			[C(RESULT_ACCESS)] = KNL_L2_PREFETCH | KNL_L2_ACCESS,
1856 			[C(RESULT_MISS)]   = KNL_L2_PREFETCH | KNL_L2_MISS,
1857 		},
1858 	},
1859 };
1860 
1861 /*
1862  * Used from PMIs where the LBRs are already disabled.
1863  *
1864  * This function could be called consecutively. It is required to remain in
1865  * disabled state if called consecutively.
1866  *
1867  * During consecutive calls, the same disable value will be written to related
1868  * registers, so the PMU state remains unchanged.
1869  *
1870  * intel_bts events don't coexist with intel PMU's BTS events because of
1871  * x86_add_exclusive(x86_lbr_exclusive_lbr); there's no need to keep them
1872  * disabled around intel PMU's event batching etc, only inside the PMI handler.
1873  */
1874 static void __intel_pmu_disable_all(void)
1875 {
1876 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1877 
1878 	wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, 0);
1879 
1880 	if (test_bit(INTEL_PMC_IDX_FIXED_BTS, cpuc->active_mask))
1881 		intel_pmu_disable_bts();
1882 
1883 	intel_pmu_pebs_disable_all();
1884 }
1885 
1886 static void intel_pmu_disable_all(void)
1887 {
1888 	__intel_pmu_disable_all();
1889 	intel_pmu_lbr_disable_all();
1890 }
1891 
1892 static void __intel_pmu_enable_all(int added, bool pmi)
1893 {
1894 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1895 
1896 	intel_pmu_pebs_enable_all();
1897 	intel_pmu_lbr_enable_all(pmi);
1898 	wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL,
1899 			x86_pmu.intel_ctrl & ~cpuc->intel_ctrl_guest_mask);
1900 
1901 	if (test_bit(INTEL_PMC_IDX_FIXED_BTS, cpuc->active_mask)) {
1902 		struct perf_event *event =
1903 			cpuc->events[INTEL_PMC_IDX_FIXED_BTS];
1904 
1905 		if (WARN_ON_ONCE(!event))
1906 			return;
1907 
1908 		intel_pmu_enable_bts(event->hw.config);
1909 	}
1910 }
1911 
1912 static void intel_pmu_enable_all(int added)
1913 {
1914 	__intel_pmu_enable_all(added, false);
1915 }
1916 
1917 /*
1918  * Workaround for:
1919  *   Intel Errata AAK100 (model 26)
1920  *   Intel Errata AAP53  (model 30)
1921  *   Intel Errata BD53   (model 44)
1922  *
1923  * The official story:
1924  *   These chips need to be 'reset' when adding counters by programming the
1925  *   magic three (non-counting) events 0x4300B5, 0x4300D2, and 0x4300B1 either
1926  *   in sequence on the same PMC or on different PMCs.
1927  *
1928  * In practise it appears some of these events do in fact count, and
1929  * we need to programm all 4 events.
1930  */
1931 static void intel_pmu_nhm_workaround(void)
1932 {
1933 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1934 	static const unsigned long nhm_magic[4] = {
1935 		0x4300B5,
1936 		0x4300D2,
1937 		0x4300B1,
1938 		0x4300B1
1939 	};
1940 	struct perf_event *event;
1941 	int i;
1942 
1943 	/*
1944 	 * The Errata requires below steps:
1945 	 * 1) Clear MSR_IA32_PEBS_ENABLE and MSR_CORE_PERF_GLOBAL_CTRL;
1946 	 * 2) Configure 4 PERFEVTSELx with the magic events and clear
1947 	 *    the corresponding PMCx;
1948 	 * 3) set bit0~bit3 of MSR_CORE_PERF_GLOBAL_CTRL;
1949 	 * 4) Clear MSR_CORE_PERF_GLOBAL_CTRL;
1950 	 * 5) Clear 4 pairs of ERFEVTSELx and PMCx;
1951 	 */
1952 
1953 	/*
1954 	 * The real steps we choose are a little different from above.
1955 	 * A) To reduce MSR operations, we don't run step 1) as they
1956 	 *    are already cleared before this function is called;
1957 	 * B) Call x86_perf_event_update to save PMCx before configuring
1958 	 *    PERFEVTSELx with magic number;
1959 	 * C) With step 5), we do clear only when the PERFEVTSELx is
1960 	 *    not used currently.
1961 	 * D) Call x86_perf_event_set_period to restore PMCx;
1962 	 */
1963 
1964 	/* We always operate 4 pairs of PERF Counters */
1965 	for (i = 0; i < 4; i++) {
1966 		event = cpuc->events[i];
1967 		if (event)
1968 			x86_perf_event_update(event);
1969 	}
1970 
1971 	for (i = 0; i < 4; i++) {
1972 		wrmsrl(MSR_ARCH_PERFMON_EVENTSEL0 + i, nhm_magic[i]);
1973 		wrmsrl(MSR_ARCH_PERFMON_PERFCTR0 + i, 0x0);
1974 	}
1975 
1976 	wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, 0xf);
1977 	wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, 0x0);
1978 
1979 	for (i = 0; i < 4; i++) {
1980 		event = cpuc->events[i];
1981 
1982 		if (event) {
1983 			x86_perf_event_set_period(event);
1984 			__x86_pmu_enable_event(&event->hw,
1985 					ARCH_PERFMON_EVENTSEL_ENABLE);
1986 		} else
1987 			wrmsrl(MSR_ARCH_PERFMON_EVENTSEL0 + i, 0x0);
1988 	}
1989 }
1990 
1991 static void intel_pmu_nhm_enable_all(int added)
1992 {
1993 	if (added)
1994 		intel_pmu_nhm_workaround();
1995 	intel_pmu_enable_all(added);
1996 }
1997 
1998 static inline u64 intel_pmu_get_status(void)
1999 {
2000 	u64 status;
2001 
2002 	rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status);
2003 
2004 	return status;
2005 }
2006 
2007 static inline void intel_pmu_ack_status(u64 ack)
2008 {
2009 	wrmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, ack);
2010 }
2011 
2012 static void intel_pmu_disable_fixed(struct hw_perf_event *hwc)
2013 {
2014 	int idx = hwc->idx - INTEL_PMC_IDX_FIXED;
2015 	u64 ctrl_val, mask;
2016 
2017 	mask = 0xfULL << (idx * 4);
2018 
2019 	rdmsrl(hwc->config_base, ctrl_val);
2020 	ctrl_val &= ~mask;
2021 	wrmsrl(hwc->config_base, ctrl_val);
2022 }
2023 
2024 static inline bool event_is_checkpointed(struct perf_event *event)
2025 {
2026 	return (event->hw.config & HSW_IN_TX_CHECKPOINTED) != 0;
2027 }
2028 
2029 static void intel_pmu_disable_event(struct perf_event *event)
2030 {
2031 	struct hw_perf_event *hwc = &event->hw;
2032 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
2033 
2034 	if (unlikely(hwc->idx == INTEL_PMC_IDX_FIXED_BTS)) {
2035 		intel_pmu_disable_bts();
2036 		intel_pmu_drain_bts_buffer();
2037 		return;
2038 	}
2039 
2040 	cpuc->intel_ctrl_guest_mask &= ~(1ull << hwc->idx);
2041 	cpuc->intel_ctrl_host_mask &= ~(1ull << hwc->idx);
2042 	cpuc->intel_cp_status &= ~(1ull << hwc->idx);
2043 
2044 	if (unlikely(hwc->config_base == MSR_ARCH_PERFMON_FIXED_CTR_CTRL)) {
2045 		intel_pmu_disable_fixed(hwc);
2046 		return;
2047 	}
2048 
2049 	x86_pmu_disable_event(event);
2050 
2051 	if (unlikely(event->attr.precise_ip))
2052 		intel_pmu_pebs_disable(event);
2053 }
2054 
2055 static void intel_pmu_del_event(struct perf_event *event)
2056 {
2057 	if (needs_branch_stack(event))
2058 		intel_pmu_lbr_del(event);
2059 	if (event->attr.precise_ip)
2060 		intel_pmu_pebs_del(event);
2061 }
2062 
2063 static void intel_pmu_enable_fixed(struct hw_perf_event *hwc)
2064 {
2065 	int idx = hwc->idx - INTEL_PMC_IDX_FIXED;
2066 	u64 ctrl_val, bits, mask;
2067 
2068 	/*
2069 	 * Enable IRQ generation (0x8),
2070 	 * and enable ring-3 counting (0x2) and ring-0 counting (0x1)
2071 	 * if requested:
2072 	 */
2073 	bits = 0x8ULL;
2074 	if (hwc->config & ARCH_PERFMON_EVENTSEL_USR)
2075 		bits |= 0x2;
2076 	if (hwc->config & ARCH_PERFMON_EVENTSEL_OS)
2077 		bits |= 0x1;
2078 
2079 	/*
2080 	 * ANY bit is supported in v3 and up
2081 	 */
2082 	if (x86_pmu.version > 2 && hwc->config & ARCH_PERFMON_EVENTSEL_ANY)
2083 		bits |= 0x4;
2084 
2085 	bits <<= (idx * 4);
2086 	mask = 0xfULL << (idx * 4);
2087 
2088 	rdmsrl(hwc->config_base, ctrl_val);
2089 	ctrl_val &= ~mask;
2090 	ctrl_val |= bits;
2091 	wrmsrl(hwc->config_base, ctrl_val);
2092 }
2093 
2094 static void intel_pmu_enable_event(struct perf_event *event)
2095 {
2096 	struct hw_perf_event *hwc = &event->hw;
2097 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
2098 
2099 	if (unlikely(hwc->idx == INTEL_PMC_IDX_FIXED_BTS)) {
2100 		if (!__this_cpu_read(cpu_hw_events.enabled))
2101 			return;
2102 
2103 		intel_pmu_enable_bts(hwc->config);
2104 		return;
2105 	}
2106 
2107 	if (event->attr.exclude_host)
2108 		cpuc->intel_ctrl_guest_mask |= (1ull << hwc->idx);
2109 	if (event->attr.exclude_guest)
2110 		cpuc->intel_ctrl_host_mask |= (1ull << hwc->idx);
2111 
2112 	if (unlikely(event_is_checkpointed(event)))
2113 		cpuc->intel_cp_status |= (1ull << hwc->idx);
2114 
2115 	if (unlikely(hwc->config_base == MSR_ARCH_PERFMON_FIXED_CTR_CTRL)) {
2116 		intel_pmu_enable_fixed(hwc);
2117 		return;
2118 	}
2119 
2120 	if (unlikely(event->attr.precise_ip))
2121 		intel_pmu_pebs_enable(event);
2122 
2123 	__x86_pmu_enable_event(hwc, ARCH_PERFMON_EVENTSEL_ENABLE);
2124 }
2125 
2126 static void intel_pmu_add_event(struct perf_event *event)
2127 {
2128 	if (event->attr.precise_ip)
2129 		intel_pmu_pebs_add(event);
2130 	if (needs_branch_stack(event))
2131 		intel_pmu_lbr_add(event);
2132 }
2133 
2134 /*
2135  * Save and restart an expired event. Called by NMI contexts,
2136  * so it has to be careful about preempting normal event ops:
2137  */
2138 int intel_pmu_save_and_restart(struct perf_event *event)
2139 {
2140 	x86_perf_event_update(event);
2141 	/*
2142 	 * For a checkpointed counter always reset back to 0.  This
2143 	 * avoids a situation where the counter overflows, aborts the
2144 	 * transaction and is then set back to shortly before the
2145 	 * overflow, and overflows and aborts again.
2146 	 */
2147 	if (unlikely(event_is_checkpointed(event))) {
2148 		/* No race with NMIs because the counter should not be armed */
2149 		wrmsrl(event->hw.event_base, 0);
2150 		local64_set(&event->hw.prev_count, 0);
2151 	}
2152 	return x86_perf_event_set_period(event);
2153 }
2154 
2155 static void intel_pmu_reset(void)
2156 {
2157 	struct debug_store *ds = __this_cpu_read(cpu_hw_events.ds);
2158 	unsigned long flags;
2159 	int idx;
2160 
2161 	if (!x86_pmu.num_counters)
2162 		return;
2163 
2164 	local_irq_save(flags);
2165 
2166 	pr_info("clearing PMU state on CPU#%d\n", smp_processor_id());
2167 
2168 	for (idx = 0; idx < x86_pmu.num_counters; idx++) {
2169 		wrmsrl_safe(x86_pmu_config_addr(idx), 0ull);
2170 		wrmsrl_safe(x86_pmu_event_addr(idx),  0ull);
2171 	}
2172 	for (idx = 0; idx < x86_pmu.num_counters_fixed; idx++)
2173 		wrmsrl_safe(MSR_ARCH_PERFMON_FIXED_CTR0 + idx, 0ull);
2174 
2175 	if (ds)
2176 		ds->bts_index = ds->bts_buffer_base;
2177 
2178 	/* Ack all overflows and disable fixed counters */
2179 	if (x86_pmu.version >= 2) {
2180 		intel_pmu_ack_status(intel_pmu_get_status());
2181 		wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, 0);
2182 	}
2183 
2184 	/* Reset LBRs and LBR freezing */
2185 	if (x86_pmu.lbr_nr) {
2186 		update_debugctlmsr(get_debugctlmsr() &
2187 			~(DEBUGCTLMSR_FREEZE_LBRS_ON_PMI|DEBUGCTLMSR_LBR));
2188 	}
2189 
2190 	local_irq_restore(flags);
2191 }
2192 
2193 /*
2194  * This handler is triggered by the local APIC, so the APIC IRQ handling
2195  * rules apply:
2196  */
2197 static int intel_pmu_handle_irq(struct pt_regs *regs)
2198 {
2199 	struct perf_sample_data data;
2200 	struct cpu_hw_events *cpuc;
2201 	int bit, loops;
2202 	u64 status;
2203 	int handled;
2204 
2205 	cpuc = this_cpu_ptr(&cpu_hw_events);
2206 
2207 	/*
2208 	 * No known reason to not always do late ACK,
2209 	 * but just in case do it opt-in.
2210 	 */
2211 	if (!x86_pmu.late_ack)
2212 		apic_write(APIC_LVTPC, APIC_DM_NMI);
2213 	intel_bts_disable_local();
2214 	__intel_pmu_disable_all();
2215 	handled = intel_pmu_drain_bts_buffer();
2216 	handled += intel_bts_interrupt();
2217 	status = intel_pmu_get_status();
2218 	if (!status)
2219 		goto done;
2220 
2221 	loops = 0;
2222 again:
2223 	intel_pmu_lbr_read();
2224 	intel_pmu_ack_status(status);
2225 	if (++loops > 100) {
2226 		static bool warned = false;
2227 		if (!warned) {
2228 			WARN(1, "perfevents: irq loop stuck!\n");
2229 			perf_event_print_debug();
2230 			warned = true;
2231 		}
2232 		intel_pmu_reset();
2233 		goto done;
2234 	}
2235 
2236 	inc_irq_stat(apic_perf_irqs);
2237 
2238 
2239 	/*
2240 	 * Ignore a range of extra bits in status that do not indicate
2241 	 * overflow by themselves.
2242 	 */
2243 	status &= ~(GLOBAL_STATUS_COND_CHG |
2244 		    GLOBAL_STATUS_ASIF |
2245 		    GLOBAL_STATUS_LBRS_FROZEN);
2246 	if (!status)
2247 		goto done;
2248 	/*
2249 	 * In case multiple PEBS events are sampled at the same time,
2250 	 * it is possible to have GLOBAL_STATUS bit 62 set indicating
2251 	 * PEBS buffer overflow and also seeing at most 3 PEBS counters
2252 	 * having their bits set in the status register. This is a sign
2253 	 * that there was at least one PEBS record pending at the time
2254 	 * of the PMU interrupt. PEBS counters must only be processed
2255 	 * via the drain_pebs() calls and not via the regular sample
2256 	 * processing loop coming after that the function, otherwise
2257 	 * phony regular samples may be generated in the sampling buffer
2258 	 * not marked with the EXACT tag. Another possibility is to have
2259 	 * one PEBS event and at least one non-PEBS event whic hoverflows
2260 	 * while PEBS has armed. In this case, bit 62 of GLOBAL_STATUS will
2261 	 * not be set, yet the overflow status bit for the PEBS counter will
2262 	 * be on Skylake.
2263 	 *
2264 	 * To avoid this problem, we systematically ignore the PEBS-enabled
2265 	 * counters from the GLOBAL_STATUS mask and we always process PEBS
2266 	 * events via drain_pebs().
2267 	 */
2268 	status &= ~(cpuc->pebs_enabled & PEBS_COUNTER_MASK);
2269 
2270 	/*
2271 	 * PEBS overflow sets bit 62 in the global status register
2272 	 */
2273 	if (__test_and_clear_bit(62, (unsigned long *)&status)) {
2274 		handled++;
2275 		x86_pmu.drain_pebs(regs);
2276 		status &= x86_pmu.intel_ctrl | GLOBAL_STATUS_TRACE_TOPAPMI;
2277 	}
2278 
2279 	/*
2280 	 * Intel PT
2281 	 */
2282 	if (__test_and_clear_bit(55, (unsigned long *)&status)) {
2283 		handled++;
2284 		intel_pt_interrupt();
2285 	}
2286 
2287 	/*
2288 	 * Checkpointed counters can lead to 'spurious' PMIs because the
2289 	 * rollback caused by the PMI will have cleared the overflow status
2290 	 * bit. Therefore always force probe these counters.
2291 	 */
2292 	status |= cpuc->intel_cp_status;
2293 
2294 	for_each_set_bit(bit, (unsigned long *)&status, X86_PMC_IDX_MAX) {
2295 		struct perf_event *event = cpuc->events[bit];
2296 
2297 		handled++;
2298 
2299 		if (!test_bit(bit, cpuc->active_mask))
2300 			continue;
2301 
2302 		if (!intel_pmu_save_and_restart(event))
2303 			continue;
2304 
2305 		perf_sample_data_init(&data, 0, event->hw.last_period);
2306 
2307 		if (has_branch_stack(event))
2308 			data.br_stack = &cpuc->lbr_stack;
2309 
2310 		if (perf_event_overflow(event, &data, regs))
2311 			x86_pmu_stop(event, 0);
2312 	}
2313 
2314 	/*
2315 	 * Repeat if there is more work to be done:
2316 	 */
2317 	status = intel_pmu_get_status();
2318 	if (status)
2319 		goto again;
2320 
2321 done:
2322 	/* Only restore PMU state when it's active. See x86_pmu_disable(). */
2323 	if (cpuc->enabled)
2324 		__intel_pmu_enable_all(0, true);
2325 	intel_bts_enable_local();
2326 
2327 	/*
2328 	 * Only unmask the NMI after the overflow counters
2329 	 * have been reset. This avoids spurious NMIs on
2330 	 * Haswell CPUs.
2331 	 */
2332 	if (x86_pmu.late_ack)
2333 		apic_write(APIC_LVTPC, APIC_DM_NMI);
2334 	return handled;
2335 }
2336 
2337 static struct event_constraint *
2338 intel_bts_constraints(struct perf_event *event)
2339 {
2340 	struct hw_perf_event *hwc = &event->hw;
2341 	unsigned int hw_event, bts_event;
2342 
2343 	if (event->attr.freq)
2344 		return NULL;
2345 
2346 	hw_event = hwc->config & INTEL_ARCH_EVENT_MASK;
2347 	bts_event = x86_pmu.event_map(PERF_COUNT_HW_BRANCH_INSTRUCTIONS);
2348 
2349 	if (unlikely(hw_event == bts_event && hwc->sample_period == 1))
2350 		return &bts_constraint;
2351 
2352 	return NULL;
2353 }
2354 
2355 static int intel_alt_er(int idx, u64 config)
2356 {
2357 	int alt_idx = idx;
2358 
2359 	if (!(x86_pmu.flags & PMU_FL_HAS_RSP_1))
2360 		return idx;
2361 
2362 	if (idx == EXTRA_REG_RSP_0)
2363 		alt_idx = EXTRA_REG_RSP_1;
2364 
2365 	if (idx == EXTRA_REG_RSP_1)
2366 		alt_idx = EXTRA_REG_RSP_0;
2367 
2368 	if (config & ~x86_pmu.extra_regs[alt_idx].valid_mask)
2369 		return idx;
2370 
2371 	return alt_idx;
2372 }
2373 
2374 static void intel_fixup_er(struct perf_event *event, int idx)
2375 {
2376 	event->hw.extra_reg.idx = idx;
2377 
2378 	if (idx == EXTRA_REG_RSP_0) {
2379 		event->hw.config &= ~INTEL_ARCH_EVENT_MASK;
2380 		event->hw.config |= x86_pmu.extra_regs[EXTRA_REG_RSP_0].event;
2381 		event->hw.extra_reg.reg = MSR_OFFCORE_RSP_0;
2382 	} else if (idx == EXTRA_REG_RSP_1) {
2383 		event->hw.config &= ~INTEL_ARCH_EVENT_MASK;
2384 		event->hw.config |= x86_pmu.extra_regs[EXTRA_REG_RSP_1].event;
2385 		event->hw.extra_reg.reg = MSR_OFFCORE_RSP_1;
2386 	}
2387 }
2388 
2389 /*
2390  * manage allocation of shared extra msr for certain events
2391  *
2392  * sharing can be:
2393  * per-cpu: to be shared between the various events on a single PMU
2394  * per-core: per-cpu + shared by HT threads
2395  */
2396 static struct event_constraint *
2397 __intel_shared_reg_get_constraints(struct cpu_hw_events *cpuc,
2398 				   struct perf_event *event,
2399 				   struct hw_perf_event_extra *reg)
2400 {
2401 	struct event_constraint *c = &emptyconstraint;
2402 	struct er_account *era;
2403 	unsigned long flags;
2404 	int idx = reg->idx;
2405 
2406 	/*
2407 	 * reg->alloc can be set due to existing state, so for fake cpuc we
2408 	 * need to ignore this, otherwise we might fail to allocate proper fake
2409 	 * state for this extra reg constraint. Also see the comment below.
2410 	 */
2411 	if (reg->alloc && !cpuc->is_fake)
2412 		return NULL; /* call x86_get_event_constraint() */
2413 
2414 again:
2415 	era = &cpuc->shared_regs->regs[idx];
2416 	/*
2417 	 * we use spin_lock_irqsave() to avoid lockdep issues when
2418 	 * passing a fake cpuc
2419 	 */
2420 	raw_spin_lock_irqsave(&era->lock, flags);
2421 
2422 	if (!atomic_read(&era->ref) || era->config == reg->config) {
2423 
2424 		/*
2425 		 * If its a fake cpuc -- as per validate_{group,event}() we
2426 		 * shouldn't touch event state and we can avoid doing so
2427 		 * since both will only call get_event_constraints() once
2428 		 * on each event, this avoids the need for reg->alloc.
2429 		 *
2430 		 * Not doing the ER fixup will only result in era->reg being
2431 		 * wrong, but since we won't actually try and program hardware
2432 		 * this isn't a problem either.
2433 		 */
2434 		if (!cpuc->is_fake) {
2435 			if (idx != reg->idx)
2436 				intel_fixup_er(event, idx);
2437 
2438 			/*
2439 			 * x86_schedule_events() can call get_event_constraints()
2440 			 * multiple times on events in the case of incremental
2441 			 * scheduling(). reg->alloc ensures we only do the ER
2442 			 * allocation once.
2443 			 */
2444 			reg->alloc = 1;
2445 		}
2446 
2447 		/* lock in msr value */
2448 		era->config = reg->config;
2449 		era->reg = reg->reg;
2450 
2451 		/* one more user */
2452 		atomic_inc(&era->ref);
2453 
2454 		/*
2455 		 * need to call x86_get_event_constraint()
2456 		 * to check if associated event has constraints
2457 		 */
2458 		c = NULL;
2459 	} else {
2460 		idx = intel_alt_er(idx, reg->config);
2461 		if (idx != reg->idx) {
2462 			raw_spin_unlock_irqrestore(&era->lock, flags);
2463 			goto again;
2464 		}
2465 	}
2466 	raw_spin_unlock_irqrestore(&era->lock, flags);
2467 
2468 	return c;
2469 }
2470 
2471 static void
2472 __intel_shared_reg_put_constraints(struct cpu_hw_events *cpuc,
2473 				   struct hw_perf_event_extra *reg)
2474 {
2475 	struct er_account *era;
2476 
2477 	/*
2478 	 * Only put constraint if extra reg was actually allocated. Also takes
2479 	 * care of event which do not use an extra shared reg.
2480 	 *
2481 	 * Also, if this is a fake cpuc we shouldn't touch any event state
2482 	 * (reg->alloc) and we don't care about leaving inconsistent cpuc state
2483 	 * either since it'll be thrown out.
2484 	 */
2485 	if (!reg->alloc || cpuc->is_fake)
2486 		return;
2487 
2488 	era = &cpuc->shared_regs->regs[reg->idx];
2489 
2490 	/* one fewer user */
2491 	atomic_dec(&era->ref);
2492 
2493 	/* allocate again next time */
2494 	reg->alloc = 0;
2495 }
2496 
2497 static struct event_constraint *
2498 intel_shared_regs_constraints(struct cpu_hw_events *cpuc,
2499 			      struct perf_event *event)
2500 {
2501 	struct event_constraint *c = NULL, *d;
2502 	struct hw_perf_event_extra *xreg, *breg;
2503 
2504 	xreg = &event->hw.extra_reg;
2505 	if (xreg->idx != EXTRA_REG_NONE) {
2506 		c = __intel_shared_reg_get_constraints(cpuc, event, xreg);
2507 		if (c == &emptyconstraint)
2508 			return c;
2509 	}
2510 	breg = &event->hw.branch_reg;
2511 	if (breg->idx != EXTRA_REG_NONE) {
2512 		d = __intel_shared_reg_get_constraints(cpuc, event, breg);
2513 		if (d == &emptyconstraint) {
2514 			__intel_shared_reg_put_constraints(cpuc, xreg);
2515 			c = d;
2516 		}
2517 	}
2518 	return c;
2519 }
2520 
2521 struct event_constraint *
2522 x86_get_event_constraints(struct cpu_hw_events *cpuc, int idx,
2523 			  struct perf_event *event)
2524 {
2525 	struct event_constraint *c;
2526 
2527 	if (x86_pmu.event_constraints) {
2528 		for_each_event_constraint(c, x86_pmu.event_constraints) {
2529 			if ((event->hw.config & c->cmask) == c->code) {
2530 				event->hw.flags |= c->flags;
2531 				return c;
2532 			}
2533 		}
2534 	}
2535 
2536 	return &unconstrained;
2537 }
2538 
2539 static struct event_constraint *
2540 __intel_get_event_constraints(struct cpu_hw_events *cpuc, int idx,
2541 			    struct perf_event *event)
2542 {
2543 	struct event_constraint *c;
2544 
2545 	c = intel_bts_constraints(event);
2546 	if (c)
2547 		return c;
2548 
2549 	c = intel_shared_regs_constraints(cpuc, event);
2550 	if (c)
2551 		return c;
2552 
2553 	c = intel_pebs_constraints(event);
2554 	if (c)
2555 		return c;
2556 
2557 	return x86_get_event_constraints(cpuc, idx, event);
2558 }
2559 
2560 static void
2561 intel_start_scheduling(struct cpu_hw_events *cpuc)
2562 {
2563 	struct intel_excl_cntrs *excl_cntrs = cpuc->excl_cntrs;
2564 	struct intel_excl_states *xl;
2565 	int tid = cpuc->excl_thread_id;
2566 
2567 	/*
2568 	 * nothing needed if in group validation mode
2569 	 */
2570 	if (cpuc->is_fake || !is_ht_workaround_enabled())
2571 		return;
2572 
2573 	/*
2574 	 * no exclusion needed
2575 	 */
2576 	if (WARN_ON_ONCE(!excl_cntrs))
2577 		return;
2578 
2579 	xl = &excl_cntrs->states[tid];
2580 
2581 	xl->sched_started = true;
2582 	/*
2583 	 * lock shared state until we are done scheduling
2584 	 * in stop_event_scheduling()
2585 	 * makes scheduling appear as a transaction
2586 	 */
2587 	raw_spin_lock(&excl_cntrs->lock);
2588 }
2589 
2590 static void intel_commit_scheduling(struct cpu_hw_events *cpuc, int idx, int cntr)
2591 {
2592 	struct intel_excl_cntrs *excl_cntrs = cpuc->excl_cntrs;
2593 	struct event_constraint *c = cpuc->event_constraint[idx];
2594 	struct intel_excl_states *xl;
2595 	int tid = cpuc->excl_thread_id;
2596 
2597 	if (cpuc->is_fake || !is_ht_workaround_enabled())
2598 		return;
2599 
2600 	if (WARN_ON_ONCE(!excl_cntrs))
2601 		return;
2602 
2603 	if (!(c->flags & PERF_X86_EVENT_DYNAMIC))
2604 		return;
2605 
2606 	xl = &excl_cntrs->states[tid];
2607 
2608 	lockdep_assert_held(&excl_cntrs->lock);
2609 
2610 	if (c->flags & PERF_X86_EVENT_EXCL)
2611 		xl->state[cntr] = INTEL_EXCL_EXCLUSIVE;
2612 	else
2613 		xl->state[cntr] = INTEL_EXCL_SHARED;
2614 }
2615 
2616 static void
2617 intel_stop_scheduling(struct cpu_hw_events *cpuc)
2618 {
2619 	struct intel_excl_cntrs *excl_cntrs = cpuc->excl_cntrs;
2620 	struct intel_excl_states *xl;
2621 	int tid = cpuc->excl_thread_id;
2622 
2623 	/*
2624 	 * nothing needed if in group validation mode
2625 	 */
2626 	if (cpuc->is_fake || !is_ht_workaround_enabled())
2627 		return;
2628 	/*
2629 	 * no exclusion needed
2630 	 */
2631 	if (WARN_ON_ONCE(!excl_cntrs))
2632 		return;
2633 
2634 	xl = &excl_cntrs->states[tid];
2635 
2636 	xl->sched_started = false;
2637 	/*
2638 	 * release shared state lock (acquired in intel_start_scheduling())
2639 	 */
2640 	raw_spin_unlock(&excl_cntrs->lock);
2641 }
2642 
2643 static struct event_constraint *
2644 intel_get_excl_constraints(struct cpu_hw_events *cpuc, struct perf_event *event,
2645 			   int idx, struct event_constraint *c)
2646 {
2647 	struct intel_excl_cntrs *excl_cntrs = cpuc->excl_cntrs;
2648 	struct intel_excl_states *xlo;
2649 	int tid = cpuc->excl_thread_id;
2650 	int is_excl, i;
2651 
2652 	/*
2653 	 * validating a group does not require
2654 	 * enforcing cross-thread  exclusion
2655 	 */
2656 	if (cpuc->is_fake || !is_ht_workaround_enabled())
2657 		return c;
2658 
2659 	/*
2660 	 * no exclusion needed
2661 	 */
2662 	if (WARN_ON_ONCE(!excl_cntrs))
2663 		return c;
2664 
2665 	/*
2666 	 * because we modify the constraint, we need
2667 	 * to make a copy. Static constraints come
2668 	 * from static const tables.
2669 	 *
2670 	 * only needed when constraint has not yet
2671 	 * been cloned (marked dynamic)
2672 	 */
2673 	if (!(c->flags & PERF_X86_EVENT_DYNAMIC)) {
2674 		struct event_constraint *cx;
2675 
2676 		/*
2677 		 * grab pre-allocated constraint entry
2678 		 */
2679 		cx = &cpuc->constraint_list[idx];
2680 
2681 		/*
2682 		 * initialize dynamic constraint
2683 		 * with static constraint
2684 		 */
2685 		*cx = *c;
2686 
2687 		/*
2688 		 * mark constraint as dynamic, so we
2689 		 * can free it later on
2690 		 */
2691 		cx->flags |= PERF_X86_EVENT_DYNAMIC;
2692 		c = cx;
2693 	}
2694 
2695 	/*
2696 	 * From here on, the constraint is dynamic.
2697 	 * Either it was just allocated above, or it
2698 	 * was allocated during a earlier invocation
2699 	 * of this function
2700 	 */
2701 
2702 	/*
2703 	 * state of sibling HT
2704 	 */
2705 	xlo = &excl_cntrs->states[tid ^ 1];
2706 
2707 	/*
2708 	 * event requires exclusive counter access
2709 	 * across HT threads
2710 	 */
2711 	is_excl = c->flags & PERF_X86_EVENT_EXCL;
2712 	if (is_excl && !(event->hw.flags & PERF_X86_EVENT_EXCL_ACCT)) {
2713 		event->hw.flags |= PERF_X86_EVENT_EXCL_ACCT;
2714 		if (!cpuc->n_excl++)
2715 			WRITE_ONCE(excl_cntrs->has_exclusive[tid], 1);
2716 	}
2717 
2718 	/*
2719 	 * Modify static constraint with current dynamic
2720 	 * state of thread
2721 	 *
2722 	 * EXCLUSIVE: sibling counter measuring exclusive event
2723 	 * SHARED   : sibling counter measuring non-exclusive event
2724 	 * UNUSED   : sibling counter unused
2725 	 */
2726 	for_each_set_bit(i, c->idxmsk, X86_PMC_IDX_MAX) {
2727 		/*
2728 		 * exclusive event in sibling counter
2729 		 * our corresponding counter cannot be used
2730 		 * regardless of our event
2731 		 */
2732 		if (xlo->state[i] == INTEL_EXCL_EXCLUSIVE)
2733 			__clear_bit(i, c->idxmsk);
2734 		/*
2735 		 * if measuring an exclusive event, sibling
2736 		 * measuring non-exclusive, then counter cannot
2737 		 * be used
2738 		 */
2739 		if (is_excl && xlo->state[i] == INTEL_EXCL_SHARED)
2740 			__clear_bit(i, c->idxmsk);
2741 	}
2742 
2743 	/*
2744 	 * recompute actual bit weight for scheduling algorithm
2745 	 */
2746 	c->weight = hweight64(c->idxmsk64);
2747 
2748 	/*
2749 	 * if we return an empty mask, then switch
2750 	 * back to static empty constraint to avoid
2751 	 * the cost of freeing later on
2752 	 */
2753 	if (c->weight == 0)
2754 		c = &emptyconstraint;
2755 
2756 	return c;
2757 }
2758 
2759 static struct event_constraint *
2760 intel_get_event_constraints(struct cpu_hw_events *cpuc, int idx,
2761 			    struct perf_event *event)
2762 {
2763 	struct event_constraint *c1 = NULL;
2764 	struct event_constraint *c2;
2765 
2766 	if (idx >= 0) /* fake does < 0 */
2767 		c1 = cpuc->event_constraint[idx];
2768 
2769 	/*
2770 	 * first time only
2771 	 * - static constraint: no change across incremental scheduling calls
2772 	 * - dynamic constraint: handled by intel_get_excl_constraints()
2773 	 */
2774 	c2 = __intel_get_event_constraints(cpuc, idx, event);
2775 	if (c1 && (c1->flags & PERF_X86_EVENT_DYNAMIC)) {
2776 		bitmap_copy(c1->idxmsk, c2->idxmsk, X86_PMC_IDX_MAX);
2777 		c1->weight = c2->weight;
2778 		c2 = c1;
2779 	}
2780 
2781 	if (cpuc->excl_cntrs)
2782 		return intel_get_excl_constraints(cpuc, event, idx, c2);
2783 
2784 	return c2;
2785 }
2786 
2787 static void intel_put_excl_constraints(struct cpu_hw_events *cpuc,
2788 		struct perf_event *event)
2789 {
2790 	struct hw_perf_event *hwc = &event->hw;
2791 	struct intel_excl_cntrs *excl_cntrs = cpuc->excl_cntrs;
2792 	int tid = cpuc->excl_thread_id;
2793 	struct intel_excl_states *xl;
2794 
2795 	/*
2796 	 * nothing needed if in group validation mode
2797 	 */
2798 	if (cpuc->is_fake)
2799 		return;
2800 
2801 	if (WARN_ON_ONCE(!excl_cntrs))
2802 		return;
2803 
2804 	if (hwc->flags & PERF_X86_EVENT_EXCL_ACCT) {
2805 		hwc->flags &= ~PERF_X86_EVENT_EXCL_ACCT;
2806 		if (!--cpuc->n_excl)
2807 			WRITE_ONCE(excl_cntrs->has_exclusive[tid], 0);
2808 	}
2809 
2810 	/*
2811 	 * If event was actually assigned, then mark the counter state as
2812 	 * unused now.
2813 	 */
2814 	if (hwc->idx >= 0) {
2815 		xl = &excl_cntrs->states[tid];
2816 
2817 		/*
2818 		 * put_constraint may be called from x86_schedule_events()
2819 		 * which already has the lock held so here make locking
2820 		 * conditional.
2821 		 */
2822 		if (!xl->sched_started)
2823 			raw_spin_lock(&excl_cntrs->lock);
2824 
2825 		xl->state[hwc->idx] = INTEL_EXCL_UNUSED;
2826 
2827 		if (!xl->sched_started)
2828 			raw_spin_unlock(&excl_cntrs->lock);
2829 	}
2830 }
2831 
2832 static void
2833 intel_put_shared_regs_event_constraints(struct cpu_hw_events *cpuc,
2834 					struct perf_event *event)
2835 {
2836 	struct hw_perf_event_extra *reg;
2837 
2838 	reg = &event->hw.extra_reg;
2839 	if (reg->idx != EXTRA_REG_NONE)
2840 		__intel_shared_reg_put_constraints(cpuc, reg);
2841 
2842 	reg = &event->hw.branch_reg;
2843 	if (reg->idx != EXTRA_REG_NONE)
2844 		__intel_shared_reg_put_constraints(cpuc, reg);
2845 }
2846 
2847 static void intel_put_event_constraints(struct cpu_hw_events *cpuc,
2848 					struct perf_event *event)
2849 {
2850 	intel_put_shared_regs_event_constraints(cpuc, event);
2851 
2852 	/*
2853 	 * is PMU has exclusive counter restrictions, then
2854 	 * all events are subject to and must call the
2855 	 * put_excl_constraints() routine
2856 	 */
2857 	if (cpuc->excl_cntrs)
2858 		intel_put_excl_constraints(cpuc, event);
2859 }
2860 
2861 static void intel_pebs_aliases_core2(struct perf_event *event)
2862 {
2863 	if ((event->hw.config & X86_RAW_EVENT_MASK) == 0x003c) {
2864 		/*
2865 		 * Use an alternative encoding for CPU_CLK_UNHALTED.THREAD_P
2866 		 * (0x003c) so that we can use it with PEBS.
2867 		 *
2868 		 * The regular CPU_CLK_UNHALTED.THREAD_P event (0x003c) isn't
2869 		 * PEBS capable. However we can use INST_RETIRED.ANY_P
2870 		 * (0x00c0), which is a PEBS capable event, to get the same
2871 		 * count.
2872 		 *
2873 		 * INST_RETIRED.ANY_P counts the number of cycles that retires
2874 		 * CNTMASK instructions. By setting CNTMASK to a value (16)
2875 		 * larger than the maximum number of instructions that can be
2876 		 * retired per cycle (4) and then inverting the condition, we
2877 		 * count all cycles that retire 16 or less instructions, which
2878 		 * is every cycle.
2879 		 *
2880 		 * Thereby we gain a PEBS capable cycle counter.
2881 		 */
2882 		u64 alt_config = X86_CONFIG(.event=0xc0, .inv=1, .cmask=16);
2883 
2884 		alt_config |= (event->hw.config & ~X86_RAW_EVENT_MASK);
2885 		event->hw.config = alt_config;
2886 	}
2887 }
2888 
2889 static void intel_pebs_aliases_snb(struct perf_event *event)
2890 {
2891 	if ((event->hw.config & X86_RAW_EVENT_MASK) == 0x003c) {
2892 		/*
2893 		 * Use an alternative encoding for CPU_CLK_UNHALTED.THREAD_P
2894 		 * (0x003c) so that we can use it with PEBS.
2895 		 *
2896 		 * The regular CPU_CLK_UNHALTED.THREAD_P event (0x003c) isn't
2897 		 * PEBS capable. However we can use UOPS_RETIRED.ALL
2898 		 * (0x01c2), which is a PEBS capable event, to get the same
2899 		 * count.
2900 		 *
2901 		 * UOPS_RETIRED.ALL counts the number of cycles that retires
2902 		 * CNTMASK micro-ops. By setting CNTMASK to a value (16)
2903 		 * larger than the maximum number of micro-ops that can be
2904 		 * retired per cycle (4) and then inverting the condition, we
2905 		 * count all cycles that retire 16 or less micro-ops, which
2906 		 * is every cycle.
2907 		 *
2908 		 * Thereby we gain a PEBS capable cycle counter.
2909 		 */
2910 		u64 alt_config = X86_CONFIG(.event=0xc2, .umask=0x01, .inv=1, .cmask=16);
2911 
2912 		alt_config |= (event->hw.config & ~X86_RAW_EVENT_MASK);
2913 		event->hw.config = alt_config;
2914 	}
2915 }
2916 
2917 static void intel_pebs_aliases_precdist(struct perf_event *event)
2918 {
2919 	if ((event->hw.config & X86_RAW_EVENT_MASK) == 0x003c) {
2920 		/*
2921 		 * Use an alternative encoding for CPU_CLK_UNHALTED.THREAD_P
2922 		 * (0x003c) so that we can use it with PEBS.
2923 		 *
2924 		 * The regular CPU_CLK_UNHALTED.THREAD_P event (0x003c) isn't
2925 		 * PEBS capable. However we can use INST_RETIRED.PREC_DIST
2926 		 * (0x01c0), which is a PEBS capable event, to get the same
2927 		 * count.
2928 		 *
2929 		 * The PREC_DIST event has special support to minimize sample
2930 		 * shadowing effects. One drawback is that it can be
2931 		 * only programmed on counter 1, but that seems like an
2932 		 * acceptable trade off.
2933 		 */
2934 		u64 alt_config = X86_CONFIG(.event=0xc0, .umask=0x01, .inv=1, .cmask=16);
2935 
2936 		alt_config |= (event->hw.config & ~X86_RAW_EVENT_MASK);
2937 		event->hw.config = alt_config;
2938 	}
2939 }
2940 
2941 static void intel_pebs_aliases_ivb(struct perf_event *event)
2942 {
2943 	if (event->attr.precise_ip < 3)
2944 		return intel_pebs_aliases_snb(event);
2945 	return intel_pebs_aliases_precdist(event);
2946 }
2947 
2948 static void intel_pebs_aliases_skl(struct perf_event *event)
2949 {
2950 	if (event->attr.precise_ip < 3)
2951 		return intel_pebs_aliases_core2(event);
2952 	return intel_pebs_aliases_precdist(event);
2953 }
2954 
2955 static unsigned long intel_pmu_free_running_flags(struct perf_event *event)
2956 {
2957 	unsigned long flags = x86_pmu.free_running_flags;
2958 
2959 	if (event->attr.use_clockid)
2960 		flags &= ~PERF_SAMPLE_TIME;
2961 	return flags;
2962 }
2963 
2964 static int intel_pmu_hw_config(struct perf_event *event)
2965 {
2966 	int ret = x86_pmu_hw_config(event);
2967 
2968 	if (ret)
2969 		return ret;
2970 
2971 	if (event->attr.precise_ip) {
2972 		if (!event->attr.freq) {
2973 			event->hw.flags |= PERF_X86_EVENT_AUTO_RELOAD;
2974 			if (!(event->attr.sample_type &
2975 			      ~intel_pmu_free_running_flags(event)))
2976 				event->hw.flags |= PERF_X86_EVENT_FREERUNNING;
2977 		}
2978 		if (x86_pmu.pebs_aliases)
2979 			x86_pmu.pebs_aliases(event);
2980 	}
2981 
2982 	if (needs_branch_stack(event)) {
2983 		ret = intel_pmu_setup_lbr_filter(event);
2984 		if (ret)
2985 			return ret;
2986 
2987 		/*
2988 		 * BTS is set up earlier in this path, so don't account twice
2989 		 */
2990 		if (!intel_pmu_has_bts(event)) {
2991 			/* disallow lbr if conflicting events are present */
2992 			if (x86_add_exclusive(x86_lbr_exclusive_lbr))
2993 				return -EBUSY;
2994 
2995 			event->destroy = hw_perf_lbr_event_destroy;
2996 		}
2997 	}
2998 
2999 	if (event->attr.type != PERF_TYPE_RAW)
3000 		return 0;
3001 
3002 	if (!(event->attr.config & ARCH_PERFMON_EVENTSEL_ANY))
3003 		return 0;
3004 
3005 	if (x86_pmu.version < 3)
3006 		return -EINVAL;
3007 
3008 	if (perf_paranoid_cpu() && !capable(CAP_SYS_ADMIN))
3009 		return -EACCES;
3010 
3011 	event->hw.config |= ARCH_PERFMON_EVENTSEL_ANY;
3012 
3013 	return 0;
3014 }
3015 
3016 struct perf_guest_switch_msr *perf_guest_get_msrs(int *nr)
3017 {
3018 	if (x86_pmu.guest_get_msrs)
3019 		return x86_pmu.guest_get_msrs(nr);
3020 	*nr = 0;
3021 	return NULL;
3022 }
3023 EXPORT_SYMBOL_GPL(perf_guest_get_msrs);
3024 
3025 static struct perf_guest_switch_msr *intel_guest_get_msrs(int *nr)
3026 {
3027 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
3028 	struct perf_guest_switch_msr *arr = cpuc->guest_switch_msrs;
3029 
3030 	arr[0].msr = MSR_CORE_PERF_GLOBAL_CTRL;
3031 	arr[0].host = x86_pmu.intel_ctrl & ~cpuc->intel_ctrl_guest_mask;
3032 	arr[0].guest = x86_pmu.intel_ctrl & ~cpuc->intel_ctrl_host_mask;
3033 	/*
3034 	 * If PMU counter has PEBS enabled it is not enough to disable counter
3035 	 * on a guest entry since PEBS memory write can overshoot guest entry
3036 	 * and corrupt guest memory. Disabling PEBS solves the problem.
3037 	 */
3038 	arr[1].msr = MSR_IA32_PEBS_ENABLE;
3039 	arr[1].host = cpuc->pebs_enabled;
3040 	arr[1].guest = 0;
3041 
3042 	*nr = 2;
3043 	return arr;
3044 }
3045 
3046 static struct perf_guest_switch_msr *core_guest_get_msrs(int *nr)
3047 {
3048 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
3049 	struct perf_guest_switch_msr *arr = cpuc->guest_switch_msrs;
3050 	int idx;
3051 
3052 	for (idx = 0; idx < x86_pmu.num_counters; idx++)  {
3053 		struct perf_event *event = cpuc->events[idx];
3054 
3055 		arr[idx].msr = x86_pmu_config_addr(idx);
3056 		arr[idx].host = arr[idx].guest = 0;
3057 
3058 		if (!test_bit(idx, cpuc->active_mask))
3059 			continue;
3060 
3061 		arr[idx].host = arr[idx].guest =
3062 			event->hw.config | ARCH_PERFMON_EVENTSEL_ENABLE;
3063 
3064 		if (event->attr.exclude_host)
3065 			arr[idx].host &= ~ARCH_PERFMON_EVENTSEL_ENABLE;
3066 		else if (event->attr.exclude_guest)
3067 			arr[idx].guest &= ~ARCH_PERFMON_EVENTSEL_ENABLE;
3068 	}
3069 
3070 	*nr = x86_pmu.num_counters;
3071 	return arr;
3072 }
3073 
3074 static void core_pmu_enable_event(struct perf_event *event)
3075 {
3076 	if (!event->attr.exclude_host)
3077 		x86_pmu_enable_event(event);
3078 }
3079 
3080 static void core_pmu_enable_all(int added)
3081 {
3082 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
3083 	int idx;
3084 
3085 	for (idx = 0; idx < x86_pmu.num_counters; idx++) {
3086 		struct hw_perf_event *hwc = &cpuc->events[idx]->hw;
3087 
3088 		if (!test_bit(idx, cpuc->active_mask) ||
3089 				cpuc->events[idx]->attr.exclude_host)
3090 			continue;
3091 
3092 		__x86_pmu_enable_event(hwc, ARCH_PERFMON_EVENTSEL_ENABLE);
3093 	}
3094 }
3095 
3096 static int hsw_hw_config(struct perf_event *event)
3097 {
3098 	int ret = intel_pmu_hw_config(event);
3099 
3100 	if (ret)
3101 		return ret;
3102 	if (!boot_cpu_has(X86_FEATURE_RTM) && !boot_cpu_has(X86_FEATURE_HLE))
3103 		return 0;
3104 	event->hw.config |= event->attr.config & (HSW_IN_TX|HSW_IN_TX_CHECKPOINTED);
3105 
3106 	/*
3107 	 * IN_TX/IN_TX-CP filters are not supported by the Haswell PMU with
3108 	 * PEBS or in ANY thread mode. Since the results are non-sensical forbid
3109 	 * this combination.
3110 	 */
3111 	if ((event->hw.config & (HSW_IN_TX|HSW_IN_TX_CHECKPOINTED)) &&
3112 	     ((event->hw.config & ARCH_PERFMON_EVENTSEL_ANY) ||
3113 	      event->attr.precise_ip > 0))
3114 		return -EOPNOTSUPP;
3115 
3116 	if (event_is_checkpointed(event)) {
3117 		/*
3118 		 * Sampling of checkpointed events can cause situations where
3119 		 * the CPU constantly aborts because of a overflow, which is
3120 		 * then checkpointed back and ignored. Forbid checkpointing
3121 		 * for sampling.
3122 		 *
3123 		 * But still allow a long sampling period, so that perf stat
3124 		 * from KVM works.
3125 		 */
3126 		if (event->attr.sample_period > 0 &&
3127 		    event->attr.sample_period < 0x7fffffff)
3128 			return -EOPNOTSUPP;
3129 	}
3130 	return 0;
3131 }
3132 
3133 static struct event_constraint counter0_constraint =
3134 			INTEL_ALL_EVENT_CONSTRAINT(0, 0x1);
3135 
3136 static struct event_constraint counter2_constraint =
3137 			EVENT_CONSTRAINT(0, 0x4, 0);
3138 
3139 static struct event_constraint *
3140 hsw_get_event_constraints(struct cpu_hw_events *cpuc, int idx,
3141 			  struct perf_event *event)
3142 {
3143 	struct event_constraint *c;
3144 
3145 	c = intel_get_event_constraints(cpuc, idx, event);
3146 
3147 	/* Handle special quirk on in_tx_checkpointed only in counter 2 */
3148 	if (event->hw.config & HSW_IN_TX_CHECKPOINTED) {
3149 		if (c->idxmsk64 & (1U << 2))
3150 			return &counter2_constraint;
3151 		return &emptyconstraint;
3152 	}
3153 
3154 	return c;
3155 }
3156 
3157 static struct event_constraint *
3158 glp_get_event_constraints(struct cpu_hw_events *cpuc, int idx,
3159 			  struct perf_event *event)
3160 {
3161 	struct event_constraint *c;
3162 
3163 	/* :ppp means to do reduced skid PEBS which is PMC0 only. */
3164 	if (event->attr.precise_ip == 3)
3165 		return &counter0_constraint;
3166 
3167 	c = intel_get_event_constraints(cpuc, idx, event);
3168 
3169 	return c;
3170 }
3171 
3172 /*
3173  * Broadwell:
3174  *
3175  * The INST_RETIRED.ALL period always needs to have lowest 6 bits cleared
3176  * (BDM55) and it must not use a period smaller than 100 (BDM11). We combine
3177  * the two to enforce a minimum period of 128 (the smallest value that has bits
3178  * 0-5 cleared and >= 100).
3179  *
3180  * Because of how the code in x86_perf_event_set_period() works, the truncation
3181  * of the lower 6 bits is 'harmless' as we'll occasionally add a longer period
3182  * to make up for the 'lost' events due to carrying the 'error' in period_left.
3183  *
3184  * Therefore the effective (average) period matches the requested period,
3185  * despite coarser hardware granularity.
3186  */
3187 static unsigned bdw_limit_period(struct perf_event *event, unsigned left)
3188 {
3189 	if ((event->hw.config & INTEL_ARCH_EVENT_MASK) ==
3190 			X86_CONFIG(.event=0xc0, .umask=0x01)) {
3191 		if (left < 128)
3192 			left = 128;
3193 		left &= ~0x3fu;
3194 	}
3195 	return left;
3196 }
3197 
3198 PMU_FORMAT_ATTR(event,	"config:0-7"	);
3199 PMU_FORMAT_ATTR(umask,	"config:8-15"	);
3200 PMU_FORMAT_ATTR(edge,	"config:18"	);
3201 PMU_FORMAT_ATTR(pc,	"config:19"	);
3202 PMU_FORMAT_ATTR(any,	"config:21"	); /* v3 + */
3203 PMU_FORMAT_ATTR(inv,	"config:23"	);
3204 PMU_FORMAT_ATTR(cmask,	"config:24-31"	);
3205 PMU_FORMAT_ATTR(in_tx,  "config:32");
3206 PMU_FORMAT_ATTR(in_tx_cp, "config:33");
3207 
3208 static struct attribute *intel_arch_formats_attr[] = {
3209 	&format_attr_event.attr,
3210 	&format_attr_umask.attr,
3211 	&format_attr_edge.attr,
3212 	&format_attr_pc.attr,
3213 	&format_attr_inv.attr,
3214 	&format_attr_cmask.attr,
3215 	NULL,
3216 };
3217 
3218 ssize_t intel_event_sysfs_show(char *page, u64 config)
3219 {
3220 	u64 event = (config & ARCH_PERFMON_EVENTSEL_EVENT);
3221 
3222 	return x86_event_sysfs_show(page, config, event);
3223 }
3224 
3225 struct intel_shared_regs *allocate_shared_regs(int cpu)
3226 {
3227 	struct intel_shared_regs *regs;
3228 	int i;
3229 
3230 	regs = kzalloc_node(sizeof(struct intel_shared_regs),
3231 			    GFP_KERNEL, cpu_to_node(cpu));
3232 	if (regs) {
3233 		/*
3234 		 * initialize the locks to keep lockdep happy
3235 		 */
3236 		for (i = 0; i < EXTRA_REG_MAX; i++)
3237 			raw_spin_lock_init(&regs->regs[i].lock);
3238 
3239 		regs->core_id = -1;
3240 	}
3241 	return regs;
3242 }
3243 
3244 static struct intel_excl_cntrs *allocate_excl_cntrs(int cpu)
3245 {
3246 	struct intel_excl_cntrs *c;
3247 
3248 	c = kzalloc_node(sizeof(struct intel_excl_cntrs),
3249 			 GFP_KERNEL, cpu_to_node(cpu));
3250 	if (c) {
3251 		raw_spin_lock_init(&c->lock);
3252 		c->core_id = -1;
3253 	}
3254 	return c;
3255 }
3256 
3257 static int intel_pmu_cpu_prepare(int cpu)
3258 {
3259 	struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu);
3260 
3261 	if (x86_pmu.extra_regs || x86_pmu.lbr_sel_map) {
3262 		cpuc->shared_regs = allocate_shared_regs(cpu);
3263 		if (!cpuc->shared_regs)
3264 			goto err;
3265 	}
3266 
3267 	if (x86_pmu.flags & PMU_FL_EXCL_CNTRS) {
3268 		size_t sz = X86_PMC_IDX_MAX * sizeof(struct event_constraint);
3269 
3270 		cpuc->constraint_list = kzalloc(sz, GFP_KERNEL);
3271 		if (!cpuc->constraint_list)
3272 			goto err_shared_regs;
3273 
3274 		cpuc->excl_cntrs = allocate_excl_cntrs(cpu);
3275 		if (!cpuc->excl_cntrs)
3276 			goto err_constraint_list;
3277 
3278 		cpuc->excl_thread_id = 0;
3279 	}
3280 
3281 	return 0;
3282 
3283 err_constraint_list:
3284 	kfree(cpuc->constraint_list);
3285 	cpuc->constraint_list = NULL;
3286 
3287 err_shared_regs:
3288 	kfree(cpuc->shared_regs);
3289 	cpuc->shared_regs = NULL;
3290 
3291 err:
3292 	return -ENOMEM;
3293 }
3294 
3295 static void flip_smm_bit(void *data)
3296 {
3297 	unsigned long set = *(unsigned long *)data;
3298 
3299 	if (set > 0) {
3300 		msr_set_bit(MSR_IA32_DEBUGCTLMSR,
3301 			    DEBUGCTLMSR_FREEZE_IN_SMM_BIT);
3302 	} else {
3303 		msr_clear_bit(MSR_IA32_DEBUGCTLMSR,
3304 			      DEBUGCTLMSR_FREEZE_IN_SMM_BIT);
3305 	}
3306 }
3307 
3308 static void intel_pmu_cpu_starting(int cpu)
3309 {
3310 	struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu);
3311 	int core_id = topology_core_id(cpu);
3312 	int i;
3313 
3314 	init_debug_store_on_cpu(cpu);
3315 	/*
3316 	 * Deal with CPUs that don't clear their LBRs on power-up.
3317 	 */
3318 	intel_pmu_lbr_reset();
3319 
3320 	cpuc->lbr_sel = NULL;
3321 
3322 	flip_smm_bit(&x86_pmu.attr_freeze_on_smi);
3323 
3324 	if (!cpuc->shared_regs)
3325 		return;
3326 
3327 	if (!(x86_pmu.flags & PMU_FL_NO_HT_SHARING)) {
3328 		for_each_cpu(i, topology_sibling_cpumask(cpu)) {
3329 			struct intel_shared_regs *pc;
3330 
3331 			pc = per_cpu(cpu_hw_events, i).shared_regs;
3332 			if (pc && pc->core_id == core_id) {
3333 				cpuc->kfree_on_online[0] = cpuc->shared_regs;
3334 				cpuc->shared_regs = pc;
3335 				break;
3336 			}
3337 		}
3338 		cpuc->shared_regs->core_id = core_id;
3339 		cpuc->shared_regs->refcnt++;
3340 	}
3341 
3342 	if (x86_pmu.lbr_sel_map)
3343 		cpuc->lbr_sel = &cpuc->shared_regs->regs[EXTRA_REG_LBR];
3344 
3345 	if (x86_pmu.flags & PMU_FL_EXCL_CNTRS) {
3346 		for_each_cpu(i, topology_sibling_cpumask(cpu)) {
3347 			struct cpu_hw_events *sibling;
3348 			struct intel_excl_cntrs *c;
3349 
3350 			sibling = &per_cpu(cpu_hw_events, i);
3351 			c = sibling->excl_cntrs;
3352 			if (c && c->core_id == core_id) {
3353 				cpuc->kfree_on_online[1] = cpuc->excl_cntrs;
3354 				cpuc->excl_cntrs = c;
3355 				if (!sibling->excl_thread_id)
3356 					cpuc->excl_thread_id = 1;
3357 				break;
3358 			}
3359 		}
3360 		cpuc->excl_cntrs->core_id = core_id;
3361 		cpuc->excl_cntrs->refcnt++;
3362 	}
3363 }
3364 
3365 static void free_excl_cntrs(int cpu)
3366 {
3367 	struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu);
3368 	struct intel_excl_cntrs *c;
3369 
3370 	c = cpuc->excl_cntrs;
3371 	if (c) {
3372 		if (c->core_id == -1 || --c->refcnt == 0)
3373 			kfree(c);
3374 		cpuc->excl_cntrs = NULL;
3375 		kfree(cpuc->constraint_list);
3376 		cpuc->constraint_list = NULL;
3377 	}
3378 }
3379 
3380 static void intel_pmu_cpu_dying(int cpu)
3381 {
3382 	struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu);
3383 	struct intel_shared_regs *pc;
3384 
3385 	pc = cpuc->shared_regs;
3386 	if (pc) {
3387 		if (pc->core_id == -1 || --pc->refcnt == 0)
3388 			kfree(pc);
3389 		cpuc->shared_regs = NULL;
3390 	}
3391 
3392 	free_excl_cntrs(cpu);
3393 
3394 	fini_debug_store_on_cpu(cpu);
3395 }
3396 
3397 static void intel_pmu_sched_task(struct perf_event_context *ctx,
3398 				 bool sched_in)
3399 {
3400 	intel_pmu_pebs_sched_task(ctx, sched_in);
3401 	intel_pmu_lbr_sched_task(ctx, sched_in);
3402 }
3403 
3404 PMU_FORMAT_ATTR(offcore_rsp, "config1:0-63");
3405 
3406 PMU_FORMAT_ATTR(ldlat, "config1:0-15");
3407 
3408 PMU_FORMAT_ATTR(frontend, "config1:0-23");
3409 
3410 static struct attribute *intel_arch3_formats_attr[] = {
3411 	&format_attr_event.attr,
3412 	&format_attr_umask.attr,
3413 	&format_attr_edge.attr,
3414 	&format_attr_pc.attr,
3415 	&format_attr_any.attr,
3416 	&format_attr_inv.attr,
3417 	&format_attr_cmask.attr,
3418 	&format_attr_in_tx.attr,
3419 	&format_attr_in_tx_cp.attr,
3420 
3421 	&format_attr_offcore_rsp.attr, /* XXX do NHM/WSM + SNB breakout */
3422 	&format_attr_ldlat.attr, /* PEBS load latency */
3423 	NULL,
3424 };
3425 
3426 static struct attribute *skl_format_attr[] = {
3427 	&format_attr_frontend.attr,
3428 	NULL,
3429 };
3430 
3431 static __initconst const struct x86_pmu core_pmu = {
3432 	.name			= "core",
3433 	.handle_irq		= x86_pmu_handle_irq,
3434 	.disable_all		= x86_pmu_disable_all,
3435 	.enable_all		= core_pmu_enable_all,
3436 	.enable			= core_pmu_enable_event,
3437 	.disable		= x86_pmu_disable_event,
3438 	.hw_config		= x86_pmu_hw_config,
3439 	.schedule_events	= x86_schedule_events,
3440 	.eventsel		= MSR_ARCH_PERFMON_EVENTSEL0,
3441 	.perfctr		= MSR_ARCH_PERFMON_PERFCTR0,
3442 	.event_map		= intel_pmu_event_map,
3443 	.max_events		= ARRAY_SIZE(intel_perfmon_event_map),
3444 	.apic			= 1,
3445 	.free_running_flags	= PEBS_FREERUNNING_FLAGS,
3446 
3447 	/*
3448 	 * Intel PMCs cannot be accessed sanely above 32-bit width,
3449 	 * so we install an artificial 1<<31 period regardless of
3450 	 * the generic event period:
3451 	 */
3452 	.max_period		= (1ULL<<31) - 1,
3453 	.get_event_constraints	= intel_get_event_constraints,
3454 	.put_event_constraints	= intel_put_event_constraints,
3455 	.event_constraints	= intel_core_event_constraints,
3456 	.guest_get_msrs		= core_guest_get_msrs,
3457 	.format_attrs		= intel_arch_formats_attr,
3458 	.events_sysfs_show	= intel_event_sysfs_show,
3459 
3460 	/*
3461 	 * Virtual (or funny metal) CPU can define x86_pmu.extra_regs
3462 	 * together with PMU version 1 and thus be using core_pmu with
3463 	 * shared_regs. We need following callbacks here to allocate
3464 	 * it properly.
3465 	 */
3466 	.cpu_prepare		= intel_pmu_cpu_prepare,
3467 	.cpu_starting		= intel_pmu_cpu_starting,
3468 	.cpu_dying		= intel_pmu_cpu_dying,
3469 };
3470 
3471 static __initconst const struct x86_pmu intel_pmu = {
3472 	.name			= "Intel",
3473 	.handle_irq		= intel_pmu_handle_irq,
3474 	.disable_all		= intel_pmu_disable_all,
3475 	.enable_all		= intel_pmu_enable_all,
3476 	.enable			= intel_pmu_enable_event,
3477 	.disable		= intel_pmu_disable_event,
3478 	.add			= intel_pmu_add_event,
3479 	.del			= intel_pmu_del_event,
3480 	.hw_config		= intel_pmu_hw_config,
3481 	.schedule_events	= x86_schedule_events,
3482 	.eventsel		= MSR_ARCH_PERFMON_EVENTSEL0,
3483 	.perfctr		= MSR_ARCH_PERFMON_PERFCTR0,
3484 	.event_map		= intel_pmu_event_map,
3485 	.max_events		= ARRAY_SIZE(intel_perfmon_event_map),
3486 	.apic			= 1,
3487 	.free_running_flags	= PEBS_FREERUNNING_FLAGS,
3488 	/*
3489 	 * Intel PMCs cannot be accessed sanely above 32 bit width,
3490 	 * so we install an artificial 1<<31 period regardless of
3491 	 * the generic event period:
3492 	 */
3493 	.max_period		= (1ULL << 31) - 1,
3494 	.get_event_constraints	= intel_get_event_constraints,
3495 	.put_event_constraints	= intel_put_event_constraints,
3496 	.pebs_aliases		= intel_pebs_aliases_core2,
3497 
3498 	.format_attrs		= intel_arch3_formats_attr,
3499 	.events_sysfs_show	= intel_event_sysfs_show,
3500 
3501 	.cpu_prepare		= intel_pmu_cpu_prepare,
3502 	.cpu_starting		= intel_pmu_cpu_starting,
3503 	.cpu_dying		= intel_pmu_cpu_dying,
3504 	.guest_get_msrs		= intel_guest_get_msrs,
3505 	.sched_task		= intel_pmu_sched_task,
3506 };
3507 
3508 static __init void intel_clovertown_quirk(void)
3509 {
3510 	/*
3511 	 * PEBS is unreliable due to:
3512 	 *
3513 	 *   AJ67  - PEBS may experience CPL leaks
3514 	 *   AJ68  - PEBS PMI may be delayed by one event
3515 	 *   AJ69  - GLOBAL_STATUS[62] will only be set when DEBUGCTL[12]
3516 	 *   AJ106 - FREEZE_LBRS_ON_PMI doesn't work in combination with PEBS
3517 	 *
3518 	 * AJ67 could be worked around by restricting the OS/USR flags.
3519 	 * AJ69 could be worked around by setting PMU_FREEZE_ON_PMI.
3520 	 *
3521 	 * AJ106 could possibly be worked around by not allowing LBR
3522 	 *       usage from PEBS, including the fixup.
3523 	 * AJ68  could possibly be worked around by always programming
3524 	 *	 a pebs_event_reset[0] value and coping with the lost events.
3525 	 *
3526 	 * But taken together it might just make sense to not enable PEBS on
3527 	 * these chips.
3528 	 */
3529 	pr_warn("PEBS disabled due to CPU errata\n");
3530 	x86_pmu.pebs = 0;
3531 	x86_pmu.pebs_constraints = NULL;
3532 }
3533 
3534 static int intel_snb_pebs_broken(int cpu)
3535 {
3536 	u32 rev = UINT_MAX; /* default to broken for unknown models */
3537 
3538 	switch (cpu_data(cpu).x86_model) {
3539 	case INTEL_FAM6_SANDYBRIDGE:
3540 		rev = 0x28;
3541 		break;
3542 
3543 	case INTEL_FAM6_SANDYBRIDGE_X:
3544 		switch (cpu_data(cpu).x86_mask) {
3545 		case 6: rev = 0x618; break;
3546 		case 7: rev = 0x70c; break;
3547 		}
3548 	}
3549 
3550 	return (cpu_data(cpu).microcode < rev);
3551 }
3552 
3553 static void intel_snb_check_microcode(void)
3554 {
3555 	int pebs_broken = 0;
3556 	int cpu;
3557 
3558 	for_each_online_cpu(cpu) {
3559 		if ((pebs_broken = intel_snb_pebs_broken(cpu)))
3560 			break;
3561 	}
3562 
3563 	if (pebs_broken == x86_pmu.pebs_broken)
3564 		return;
3565 
3566 	/*
3567 	 * Serialized by the microcode lock..
3568 	 */
3569 	if (x86_pmu.pebs_broken) {
3570 		pr_info("PEBS enabled due to microcode update\n");
3571 		x86_pmu.pebs_broken = 0;
3572 	} else {
3573 		pr_info("PEBS disabled due to CPU errata, please upgrade microcode\n");
3574 		x86_pmu.pebs_broken = 1;
3575 	}
3576 }
3577 
3578 static bool is_lbr_from(unsigned long msr)
3579 {
3580 	unsigned long lbr_from_nr = x86_pmu.lbr_from + x86_pmu.lbr_nr;
3581 
3582 	return x86_pmu.lbr_from <= msr && msr < lbr_from_nr;
3583 }
3584 
3585 /*
3586  * Under certain circumstances, access certain MSR may cause #GP.
3587  * The function tests if the input MSR can be safely accessed.
3588  */
3589 static bool check_msr(unsigned long msr, u64 mask)
3590 {
3591 	u64 val_old, val_new, val_tmp;
3592 
3593 	/*
3594 	 * Read the current value, change it and read it back to see if it
3595 	 * matches, this is needed to detect certain hardware emulators
3596 	 * (qemu/kvm) that don't trap on the MSR access and always return 0s.
3597 	 */
3598 	if (rdmsrl_safe(msr, &val_old))
3599 		return false;
3600 
3601 	/*
3602 	 * Only change the bits which can be updated by wrmsrl.
3603 	 */
3604 	val_tmp = val_old ^ mask;
3605 
3606 	if (is_lbr_from(msr))
3607 		val_tmp = lbr_from_signext_quirk_wr(val_tmp);
3608 
3609 	if (wrmsrl_safe(msr, val_tmp) ||
3610 	    rdmsrl_safe(msr, &val_new))
3611 		return false;
3612 
3613 	/*
3614 	 * Quirk only affects validation in wrmsr(), so wrmsrl()'s value
3615 	 * should equal rdmsrl()'s even with the quirk.
3616 	 */
3617 	if (val_new != val_tmp)
3618 		return false;
3619 
3620 	if (is_lbr_from(msr))
3621 		val_old = lbr_from_signext_quirk_wr(val_old);
3622 
3623 	/* Here it's sure that the MSR can be safely accessed.
3624 	 * Restore the old value and return.
3625 	 */
3626 	wrmsrl(msr, val_old);
3627 
3628 	return true;
3629 }
3630 
3631 static __init void intel_sandybridge_quirk(void)
3632 {
3633 	x86_pmu.check_microcode = intel_snb_check_microcode;
3634 	cpus_read_lock();
3635 	intel_snb_check_microcode();
3636 	cpus_read_unlock();
3637 }
3638 
3639 static const struct { int id; char *name; } intel_arch_events_map[] __initconst = {
3640 	{ PERF_COUNT_HW_CPU_CYCLES, "cpu cycles" },
3641 	{ PERF_COUNT_HW_INSTRUCTIONS, "instructions" },
3642 	{ PERF_COUNT_HW_BUS_CYCLES, "bus cycles" },
3643 	{ PERF_COUNT_HW_CACHE_REFERENCES, "cache references" },
3644 	{ PERF_COUNT_HW_CACHE_MISSES, "cache misses" },
3645 	{ PERF_COUNT_HW_BRANCH_INSTRUCTIONS, "branch instructions" },
3646 	{ PERF_COUNT_HW_BRANCH_MISSES, "branch misses" },
3647 };
3648 
3649 static __init void intel_arch_events_quirk(void)
3650 {
3651 	int bit;
3652 
3653 	/* disable event that reported as not presend by cpuid */
3654 	for_each_set_bit(bit, x86_pmu.events_mask, ARRAY_SIZE(intel_arch_events_map)) {
3655 		intel_perfmon_event_map[intel_arch_events_map[bit].id] = 0;
3656 		pr_warn("CPUID marked event: \'%s\' unavailable\n",
3657 			intel_arch_events_map[bit].name);
3658 	}
3659 }
3660 
3661 static __init void intel_nehalem_quirk(void)
3662 {
3663 	union cpuid10_ebx ebx;
3664 
3665 	ebx.full = x86_pmu.events_maskl;
3666 	if (ebx.split.no_branch_misses_retired) {
3667 		/*
3668 		 * Erratum AAJ80 detected, we work it around by using
3669 		 * the BR_MISP_EXEC.ANY event. This will over-count
3670 		 * branch-misses, but it's still much better than the
3671 		 * architectural event which is often completely bogus:
3672 		 */
3673 		intel_perfmon_event_map[PERF_COUNT_HW_BRANCH_MISSES] = 0x7f89;
3674 		ebx.split.no_branch_misses_retired = 0;
3675 		x86_pmu.events_maskl = ebx.full;
3676 		pr_info("CPU erratum AAJ80 worked around\n");
3677 	}
3678 }
3679 
3680 /*
3681  * enable software workaround for errata:
3682  * SNB: BJ122
3683  * IVB: BV98
3684  * HSW: HSD29
3685  *
3686  * Only needed when HT is enabled. However detecting
3687  * if HT is enabled is difficult (model specific). So instead,
3688  * we enable the workaround in the early boot, and verify if
3689  * it is needed in a later initcall phase once we have valid
3690  * topology information to check if HT is actually enabled
3691  */
3692 static __init void intel_ht_bug(void)
3693 {
3694 	x86_pmu.flags |= PMU_FL_EXCL_CNTRS | PMU_FL_EXCL_ENABLED;
3695 
3696 	x86_pmu.start_scheduling = intel_start_scheduling;
3697 	x86_pmu.commit_scheduling = intel_commit_scheduling;
3698 	x86_pmu.stop_scheduling = intel_stop_scheduling;
3699 }
3700 
3701 EVENT_ATTR_STR(mem-loads,	mem_ld_hsw,	"event=0xcd,umask=0x1,ldlat=3");
3702 EVENT_ATTR_STR(mem-stores,	mem_st_hsw,	"event=0xd0,umask=0x82")
3703 
3704 /* Haswell special events */
3705 EVENT_ATTR_STR(tx-start,	tx_start,	"event=0xc9,umask=0x1");
3706 EVENT_ATTR_STR(tx-commit,	tx_commit,	"event=0xc9,umask=0x2");
3707 EVENT_ATTR_STR(tx-abort,	tx_abort,	"event=0xc9,umask=0x4");
3708 EVENT_ATTR_STR(tx-capacity,	tx_capacity,	"event=0x54,umask=0x2");
3709 EVENT_ATTR_STR(tx-conflict,	tx_conflict,	"event=0x54,umask=0x1");
3710 EVENT_ATTR_STR(el-start,	el_start,	"event=0xc8,umask=0x1");
3711 EVENT_ATTR_STR(el-commit,	el_commit,	"event=0xc8,umask=0x2");
3712 EVENT_ATTR_STR(el-abort,	el_abort,	"event=0xc8,umask=0x4");
3713 EVENT_ATTR_STR(el-capacity,	el_capacity,	"event=0x54,umask=0x2");
3714 EVENT_ATTR_STR(el-conflict,	el_conflict,	"event=0x54,umask=0x1");
3715 EVENT_ATTR_STR(cycles-t,	cycles_t,	"event=0x3c,in_tx=1");
3716 EVENT_ATTR_STR(cycles-ct,	cycles_ct,	"event=0x3c,in_tx=1,in_tx_cp=1");
3717 
3718 static struct attribute *hsw_events_attrs[] = {
3719 	EVENT_PTR(tx_start),
3720 	EVENT_PTR(tx_commit),
3721 	EVENT_PTR(tx_abort),
3722 	EVENT_PTR(tx_capacity),
3723 	EVENT_PTR(tx_conflict),
3724 	EVENT_PTR(el_start),
3725 	EVENT_PTR(el_commit),
3726 	EVENT_PTR(el_abort),
3727 	EVENT_PTR(el_capacity),
3728 	EVENT_PTR(el_conflict),
3729 	EVENT_PTR(cycles_t),
3730 	EVENT_PTR(cycles_ct),
3731 	EVENT_PTR(mem_ld_hsw),
3732 	EVENT_PTR(mem_st_hsw),
3733 	EVENT_PTR(td_slots_issued),
3734 	EVENT_PTR(td_slots_retired),
3735 	EVENT_PTR(td_fetch_bubbles),
3736 	EVENT_PTR(td_total_slots),
3737 	EVENT_PTR(td_total_slots_scale),
3738 	EVENT_PTR(td_recovery_bubbles),
3739 	EVENT_PTR(td_recovery_bubbles_scale),
3740 	NULL
3741 };
3742 
3743 static ssize_t freeze_on_smi_show(struct device *cdev,
3744 				  struct device_attribute *attr,
3745 				  char *buf)
3746 {
3747 	return sprintf(buf, "%lu\n", x86_pmu.attr_freeze_on_smi);
3748 }
3749 
3750 static DEFINE_MUTEX(freeze_on_smi_mutex);
3751 
3752 static ssize_t freeze_on_smi_store(struct device *cdev,
3753 				   struct device_attribute *attr,
3754 				   const char *buf, size_t count)
3755 {
3756 	unsigned long val;
3757 	ssize_t ret;
3758 
3759 	ret = kstrtoul(buf, 0, &val);
3760 	if (ret)
3761 		return ret;
3762 
3763 	if (val > 1)
3764 		return -EINVAL;
3765 
3766 	mutex_lock(&freeze_on_smi_mutex);
3767 
3768 	if (x86_pmu.attr_freeze_on_smi == val)
3769 		goto done;
3770 
3771 	x86_pmu.attr_freeze_on_smi = val;
3772 
3773 	get_online_cpus();
3774 	on_each_cpu(flip_smm_bit, &val, 1);
3775 	put_online_cpus();
3776 done:
3777 	mutex_unlock(&freeze_on_smi_mutex);
3778 
3779 	return count;
3780 }
3781 
3782 static DEVICE_ATTR_RW(freeze_on_smi);
3783 
3784 static struct attribute *intel_pmu_attrs[] = {
3785 	&dev_attr_freeze_on_smi.attr,
3786 	NULL,
3787 };
3788 
3789 __init int intel_pmu_init(void)
3790 {
3791 	union cpuid10_edx edx;
3792 	union cpuid10_eax eax;
3793 	union cpuid10_ebx ebx;
3794 	struct event_constraint *c;
3795 	unsigned int unused;
3796 	struct extra_reg *er;
3797 	int version, i;
3798 
3799 	if (!cpu_has(&boot_cpu_data, X86_FEATURE_ARCH_PERFMON)) {
3800 		switch (boot_cpu_data.x86) {
3801 		case 0x6:
3802 			return p6_pmu_init();
3803 		case 0xb:
3804 			return knc_pmu_init();
3805 		case 0xf:
3806 			return p4_pmu_init();
3807 		}
3808 		return -ENODEV;
3809 	}
3810 
3811 	/*
3812 	 * Check whether the Architectural PerfMon supports
3813 	 * Branch Misses Retired hw_event or not.
3814 	 */
3815 	cpuid(10, &eax.full, &ebx.full, &unused, &edx.full);
3816 	if (eax.split.mask_length < ARCH_PERFMON_EVENTS_COUNT)
3817 		return -ENODEV;
3818 
3819 	version = eax.split.version_id;
3820 	if (version < 2)
3821 		x86_pmu = core_pmu;
3822 	else
3823 		x86_pmu = intel_pmu;
3824 
3825 	x86_pmu.version			= version;
3826 	x86_pmu.num_counters		= eax.split.num_counters;
3827 	x86_pmu.cntval_bits		= eax.split.bit_width;
3828 	x86_pmu.cntval_mask		= (1ULL << eax.split.bit_width) - 1;
3829 
3830 	x86_pmu.events_maskl		= ebx.full;
3831 	x86_pmu.events_mask_len		= eax.split.mask_length;
3832 
3833 	x86_pmu.max_pebs_events		= min_t(unsigned, MAX_PEBS_EVENTS, x86_pmu.num_counters);
3834 
3835 
3836 	x86_pmu.attrs			= intel_pmu_attrs;
3837 	/*
3838 	 * Quirk: v2 perfmon does not report fixed-purpose events, so
3839 	 * assume at least 3 events, when not running in a hypervisor:
3840 	 */
3841 	if (version > 1) {
3842 		int assume = 3 * !boot_cpu_has(X86_FEATURE_HYPERVISOR);
3843 
3844 		x86_pmu.num_counters_fixed =
3845 			max((int)edx.split.num_counters_fixed, assume);
3846 	}
3847 
3848 	if (boot_cpu_has(X86_FEATURE_PDCM)) {
3849 		u64 capabilities;
3850 
3851 		rdmsrl(MSR_IA32_PERF_CAPABILITIES, capabilities);
3852 		x86_pmu.intel_cap.capabilities = capabilities;
3853 	}
3854 
3855 	intel_ds_init();
3856 
3857 	x86_add_quirk(intel_arch_events_quirk); /* Install first, so it runs last */
3858 
3859 	/*
3860 	 * Install the hw-cache-events table:
3861 	 */
3862 	switch (boot_cpu_data.x86_model) {
3863 	case INTEL_FAM6_CORE_YONAH:
3864 		pr_cont("Core events, ");
3865 		break;
3866 
3867 	case INTEL_FAM6_CORE2_MEROM:
3868 		x86_add_quirk(intel_clovertown_quirk);
3869 	case INTEL_FAM6_CORE2_MEROM_L:
3870 	case INTEL_FAM6_CORE2_PENRYN:
3871 	case INTEL_FAM6_CORE2_DUNNINGTON:
3872 		memcpy(hw_cache_event_ids, core2_hw_cache_event_ids,
3873 		       sizeof(hw_cache_event_ids));
3874 
3875 		intel_pmu_lbr_init_core();
3876 
3877 		x86_pmu.event_constraints = intel_core2_event_constraints;
3878 		x86_pmu.pebs_constraints = intel_core2_pebs_event_constraints;
3879 		pr_cont("Core2 events, ");
3880 		break;
3881 
3882 	case INTEL_FAM6_NEHALEM:
3883 	case INTEL_FAM6_NEHALEM_EP:
3884 	case INTEL_FAM6_NEHALEM_EX:
3885 		memcpy(hw_cache_event_ids, nehalem_hw_cache_event_ids,
3886 		       sizeof(hw_cache_event_ids));
3887 		memcpy(hw_cache_extra_regs, nehalem_hw_cache_extra_regs,
3888 		       sizeof(hw_cache_extra_regs));
3889 
3890 		intel_pmu_lbr_init_nhm();
3891 
3892 		x86_pmu.event_constraints = intel_nehalem_event_constraints;
3893 		x86_pmu.pebs_constraints = intel_nehalem_pebs_event_constraints;
3894 		x86_pmu.enable_all = intel_pmu_nhm_enable_all;
3895 		x86_pmu.extra_regs = intel_nehalem_extra_regs;
3896 
3897 		x86_pmu.cpu_events = nhm_events_attrs;
3898 
3899 		/* UOPS_ISSUED.STALLED_CYCLES */
3900 		intel_perfmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] =
3901 			X86_CONFIG(.event=0x0e, .umask=0x01, .inv=1, .cmask=1);
3902 		/* UOPS_EXECUTED.CORE_ACTIVE_CYCLES,c=1,i=1 */
3903 		intel_perfmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_BACKEND] =
3904 			X86_CONFIG(.event=0xb1, .umask=0x3f, .inv=1, .cmask=1);
3905 
3906 		intel_pmu_pebs_data_source_nhm();
3907 		x86_add_quirk(intel_nehalem_quirk);
3908 
3909 		pr_cont("Nehalem events, ");
3910 		break;
3911 
3912 	case INTEL_FAM6_ATOM_PINEVIEW:
3913 	case INTEL_FAM6_ATOM_LINCROFT:
3914 	case INTEL_FAM6_ATOM_PENWELL:
3915 	case INTEL_FAM6_ATOM_CLOVERVIEW:
3916 	case INTEL_FAM6_ATOM_CEDARVIEW:
3917 		memcpy(hw_cache_event_ids, atom_hw_cache_event_ids,
3918 		       sizeof(hw_cache_event_ids));
3919 
3920 		intel_pmu_lbr_init_atom();
3921 
3922 		x86_pmu.event_constraints = intel_gen_event_constraints;
3923 		x86_pmu.pebs_constraints = intel_atom_pebs_event_constraints;
3924 		x86_pmu.pebs_aliases = intel_pebs_aliases_core2;
3925 		pr_cont("Atom events, ");
3926 		break;
3927 
3928 	case INTEL_FAM6_ATOM_SILVERMONT1:
3929 	case INTEL_FAM6_ATOM_SILVERMONT2:
3930 	case INTEL_FAM6_ATOM_AIRMONT:
3931 		memcpy(hw_cache_event_ids, slm_hw_cache_event_ids,
3932 			sizeof(hw_cache_event_ids));
3933 		memcpy(hw_cache_extra_regs, slm_hw_cache_extra_regs,
3934 		       sizeof(hw_cache_extra_regs));
3935 
3936 		intel_pmu_lbr_init_slm();
3937 
3938 		x86_pmu.event_constraints = intel_slm_event_constraints;
3939 		x86_pmu.pebs_constraints = intel_slm_pebs_event_constraints;
3940 		x86_pmu.extra_regs = intel_slm_extra_regs;
3941 		x86_pmu.flags |= PMU_FL_HAS_RSP_1;
3942 		x86_pmu.cpu_events = slm_events_attrs;
3943 		pr_cont("Silvermont events, ");
3944 		break;
3945 
3946 	case INTEL_FAM6_ATOM_GOLDMONT:
3947 	case INTEL_FAM6_ATOM_DENVERTON:
3948 		memcpy(hw_cache_event_ids, glm_hw_cache_event_ids,
3949 		       sizeof(hw_cache_event_ids));
3950 		memcpy(hw_cache_extra_regs, glm_hw_cache_extra_regs,
3951 		       sizeof(hw_cache_extra_regs));
3952 
3953 		intel_pmu_lbr_init_skl();
3954 
3955 		x86_pmu.event_constraints = intel_slm_event_constraints;
3956 		x86_pmu.pebs_constraints = intel_glm_pebs_event_constraints;
3957 		x86_pmu.extra_regs = intel_glm_extra_regs;
3958 		/*
3959 		 * It's recommended to use CPU_CLK_UNHALTED.CORE_P + NPEBS
3960 		 * for precise cycles.
3961 		 * :pp is identical to :ppp
3962 		 */
3963 		x86_pmu.pebs_aliases = NULL;
3964 		x86_pmu.pebs_prec_dist = true;
3965 		x86_pmu.lbr_pt_coexist = true;
3966 		x86_pmu.flags |= PMU_FL_HAS_RSP_1;
3967 		x86_pmu.cpu_events = glm_events_attrs;
3968 		pr_cont("Goldmont events, ");
3969 		break;
3970 
3971 	case INTEL_FAM6_ATOM_GEMINI_LAKE:
3972 		memcpy(hw_cache_event_ids, glp_hw_cache_event_ids,
3973 		       sizeof(hw_cache_event_ids));
3974 		memcpy(hw_cache_extra_regs, glp_hw_cache_extra_regs,
3975 		       sizeof(hw_cache_extra_regs));
3976 
3977 		intel_pmu_lbr_init_skl();
3978 
3979 		x86_pmu.event_constraints = intel_slm_event_constraints;
3980 		x86_pmu.pebs_constraints = intel_glp_pebs_event_constraints;
3981 		x86_pmu.extra_regs = intel_glm_extra_regs;
3982 		/*
3983 		 * It's recommended to use CPU_CLK_UNHALTED.CORE_P + NPEBS
3984 		 * for precise cycles.
3985 		 */
3986 		x86_pmu.pebs_aliases = NULL;
3987 		x86_pmu.pebs_prec_dist = true;
3988 		x86_pmu.lbr_pt_coexist = true;
3989 		x86_pmu.flags |= PMU_FL_HAS_RSP_1;
3990 		x86_pmu.get_event_constraints = glp_get_event_constraints;
3991 		x86_pmu.cpu_events = glm_events_attrs;
3992 		/* Goldmont Plus has 4-wide pipeline */
3993 		event_attr_td_total_slots_scale_glm.event_str = "4";
3994 		pr_cont("Goldmont plus events, ");
3995 		break;
3996 
3997 	case INTEL_FAM6_WESTMERE:
3998 	case INTEL_FAM6_WESTMERE_EP:
3999 	case INTEL_FAM6_WESTMERE_EX:
4000 		memcpy(hw_cache_event_ids, westmere_hw_cache_event_ids,
4001 		       sizeof(hw_cache_event_ids));
4002 		memcpy(hw_cache_extra_regs, nehalem_hw_cache_extra_regs,
4003 		       sizeof(hw_cache_extra_regs));
4004 
4005 		intel_pmu_lbr_init_nhm();
4006 
4007 		x86_pmu.event_constraints = intel_westmere_event_constraints;
4008 		x86_pmu.enable_all = intel_pmu_nhm_enable_all;
4009 		x86_pmu.pebs_constraints = intel_westmere_pebs_event_constraints;
4010 		x86_pmu.extra_regs = intel_westmere_extra_regs;
4011 		x86_pmu.flags |= PMU_FL_HAS_RSP_1;
4012 
4013 		x86_pmu.cpu_events = nhm_events_attrs;
4014 
4015 		/* UOPS_ISSUED.STALLED_CYCLES */
4016 		intel_perfmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] =
4017 			X86_CONFIG(.event=0x0e, .umask=0x01, .inv=1, .cmask=1);
4018 		/* UOPS_EXECUTED.CORE_ACTIVE_CYCLES,c=1,i=1 */
4019 		intel_perfmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_BACKEND] =
4020 			X86_CONFIG(.event=0xb1, .umask=0x3f, .inv=1, .cmask=1);
4021 
4022 		intel_pmu_pebs_data_source_nhm();
4023 		pr_cont("Westmere events, ");
4024 		break;
4025 
4026 	case INTEL_FAM6_SANDYBRIDGE:
4027 	case INTEL_FAM6_SANDYBRIDGE_X:
4028 		x86_add_quirk(intel_sandybridge_quirk);
4029 		x86_add_quirk(intel_ht_bug);
4030 		memcpy(hw_cache_event_ids, snb_hw_cache_event_ids,
4031 		       sizeof(hw_cache_event_ids));
4032 		memcpy(hw_cache_extra_regs, snb_hw_cache_extra_regs,
4033 		       sizeof(hw_cache_extra_regs));
4034 
4035 		intel_pmu_lbr_init_snb();
4036 
4037 		x86_pmu.event_constraints = intel_snb_event_constraints;
4038 		x86_pmu.pebs_constraints = intel_snb_pebs_event_constraints;
4039 		x86_pmu.pebs_aliases = intel_pebs_aliases_snb;
4040 		if (boot_cpu_data.x86_model == INTEL_FAM6_SANDYBRIDGE_X)
4041 			x86_pmu.extra_regs = intel_snbep_extra_regs;
4042 		else
4043 			x86_pmu.extra_regs = intel_snb_extra_regs;
4044 
4045 
4046 		/* all extra regs are per-cpu when HT is on */
4047 		x86_pmu.flags |= PMU_FL_HAS_RSP_1;
4048 		x86_pmu.flags |= PMU_FL_NO_HT_SHARING;
4049 
4050 		x86_pmu.cpu_events = snb_events_attrs;
4051 
4052 		/* UOPS_ISSUED.ANY,c=1,i=1 to count stall cycles */
4053 		intel_perfmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] =
4054 			X86_CONFIG(.event=0x0e, .umask=0x01, .inv=1, .cmask=1);
4055 		/* UOPS_DISPATCHED.THREAD,c=1,i=1 to count stall cycles*/
4056 		intel_perfmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_BACKEND] =
4057 			X86_CONFIG(.event=0xb1, .umask=0x01, .inv=1, .cmask=1);
4058 
4059 		pr_cont("SandyBridge events, ");
4060 		break;
4061 
4062 	case INTEL_FAM6_IVYBRIDGE:
4063 	case INTEL_FAM6_IVYBRIDGE_X:
4064 		x86_add_quirk(intel_ht_bug);
4065 		memcpy(hw_cache_event_ids, snb_hw_cache_event_ids,
4066 		       sizeof(hw_cache_event_ids));
4067 		/* dTLB-load-misses on IVB is different than SNB */
4068 		hw_cache_event_ids[C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = 0x8108; /* DTLB_LOAD_MISSES.DEMAND_LD_MISS_CAUSES_A_WALK */
4069 
4070 		memcpy(hw_cache_extra_regs, snb_hw_cache_extra_regs,
4071 		       sizeof(hw_cache_extra_regs));
4072 
4073 		intel_pmu_lbr_init_snb();
4074 
4075 		x86_pmu.event_constraints = intel_ivb_event_constraints;
4076 		x86_pmu.pebs_constraints = intel_ivb_pebs_event_constraints;
4077 		x86_pmu.pebs_aliases = intel_pebs_aliases_ivb;
4078 		x86_pmu.pebs_prec_dist = true;
4079 		if (boot_cpu_data.x86_model == INTEL_FAM6_IVYBRIDGE_X)
4080 			x86_pmu.extra_regs = intel_snbep_extra_regs;
4081 		else
4082 			x86_pmu.extra_regs = intel_snb_extra_regs;
4083 		/* all extra regs are per-cpu when HT is on */
4084 		x86_pmu.flags |= PMU_FL_HAS_RSP_1;
4085 		x86_pmu.flags |= PMU_FL_NO_HT_SHARING;
4086 
4087 		x86_pmu.cpu_events = snb_events_attrs;
4088 
4089 		/* UOPS_ISSUED.ANY,c=1,i=1 to count stall cycles */
4090 		intel_perfmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] =
4091 			X86_CONFIG(.event=0x0e, .umask=0x01, .inv=1, .cmask=1);
4092 
4093 		pr_cont("IvyBridge events, ");
4094 		break;
4095 
4096 
4097 	case INTEL_FAM6_HASWELL_CORE:
4098 	case INTEL_FAM6_HASWELL_X:
4099 	case INTEL_FAM6_HASWELL_ULT:
4100 	case INTEL_FAM6_HASWELL_GT3E:
4101 		x86_add_quirk(intel_ht_bug);
4102 		x86_pmu.late_ack = true;
4103 		memcpy(hw_cache_event_ids, hsw_hw_cache_event_ids, sizeof(hw_cache_event_ids));
4104 		memcpy(hw_cache_extra_regs, hsw_hw_cache_extra_regs, sizeof(hw_cache_extra_regs));
4105 
4106 		intel_pmu_lbr_init_hsw();
4107 
4108 		x86_pmu.event_constraints = intel_hsw_event_constraints;
4109 		x86_pmu.pebs_constraints = intel_hsw_pebs_event_constraints;
4110 		x86_pmu.extra_regs = intel_snbep_extra_regs;
4111 		x86_pmu.pebs_aliases = intel_pebs_aliases_ivb;
4112 		x86_pmu.pebs_prec_dist = true;
4113 		/* all extra regs are per-cpu when HT is on */
4114 		x86_pmu.flags |= PMU_FL_HAS_RSP_1;
4115 		x86_pmu.flags |= PMU_FL_NO_HT_SHARING;
4116 
4117 		x86_pmu.hw_config = hsw_hw_config;
4118 		x86_pmu.get_event_constraints = hsw_get_event_constraints;
4119 		x86_pmu.cpu_events = hsw_events_attrs;
4120 		x86_pmu.lbr_double_abort = true;
4121 		pr_cont("Haswell events, ");
4122 		break;
4123 
4124 	case INTEL_FAM6_BROADWELL_CORE:
4125 	case INTEL_FAM6_BROADWELL_XEON_D:
4126 	case INTEL_FAM6_BROADWELL_GT3E:
4127 	case INTEL_FAM6_BROADWELL_X:
4128 		x86_pmu.late_ack = true;
4129 		memcpy(hw_cache_event_ids, hsw_hw_cache_event_ids, sizeof(hw_cache_event_ids));
4130 		memcpy(hw_cache_extra_regs, hsw_hw_cache_extra_regs, sizeof(hw_cache_extra_regs));
4131 
4132 		/* L3_MISS_LOCAL_DRAM is BIT(26) in Broadwell */
4133 		hw_cache_extra_regs[C(LL)][C(OP_READ)][C(RESULT_MISS)] = HSW_DEMAND_READ |
4134 									 BDW_L3_MISS|HSW_SNOOP_DRAM;
4135 		hw_cache_extra_regs[C(LL)][C(OP_WRITE)][C(RESULT_MISS)] = HSW_DEMAND_WRITE|BDW_L3_MISS|
4136 									  HSW_SNOOP_DRAM;
4137 		hw_cache_extra_regs[C(NODE)][C(OP_READ)][C(RESULT_ACCESS)] = HSW_DEMAND_READ|
4138 									     BDW_L3_MISS_LOCAL|HSW_SNOOP_DRAM;
4139 		hw_cache_extra_regs[C(NODE)][C(OP_WRITE)][C(RESULT_ACCESS)] = HSW_DEMAND_WRITE|
4140 									      BDW_L3_MISS_LOCAL|HSW_SNOOP_DRAM;
4141 
4142 		intel_pmu_lbr_init_hsw();
4143 
4144 		x86_pmu.event_constraints = intel_bdw_event_constraints;
4145 		x86_pmu.pebs_constraints = intel_bdw_pebs_event_constraints;
4146 		x86_pmu.extra_regs = intel_snbep_extra_regs;
4147 		x86_pmu.pebs_aliases = intel_pebs_aliases_ivb;
4148 		x86_pmu.pebs_prec_dist = true;
4149 		/* all extra regs are per-cpu when HT is on */
4150 		x86_pmu.flags |= PMU_FL_HAS_RSP_1;
4151 		x86_pmu.flags |= PMU_FL_NO_HT_SHARING;
4152 
4153 		x86_pmu.hw_config = hsw_hw_config;
4154 		x86_pmu.get_event_constraints = hsw_get_event_constraints;
4155 		x86_pmu.cpu_events = hsw_events_attrs;
4156 		x86_pmu.limit_period = bdw_limit_period;
4157 		pr_cont("Broadwell events, ");
4158 		break;
4159 
4160 	case INTEL_FAM6_XEON_PHI_KNL:
4161 	case INTEL_FAM6_XEON_PHI_KNM:
4162 		memcpy(hw_cache_event_ids,
4163 		       slm_hw_cache_event_ids, sizeof(hw_cache_event_ids));
4164 		memcpy(hw_cache_extra_regs,
4165 		       knl_hw_cache_extra_regs, sizeof(hw_cache_extra_regs));
4166 		intel_pmu_lbr_init_knl();
4167 
4168 		x86_pmu.event_constraints = intel_slm_event_constraints;
4169 		x86_pmu.pebs_constraints = intel_slm_pebs_event_constraints;
4170 		x86_pmu.extra_regs = intel_knl_extra_regs;
4171 
4172 		/* all extra regs are per-cpu when HT is on */
4173 		x86_pmu.flags |= PMU_FL_HAS_RSP_1;
4174 		x86_pmu.flags |= PMU_FL_NO_HT_SHARING;
4175 
4176 		pr_cont("Knights Landing/Mill events, ");
4177 		break;
4178 
4179 	case INTEL_FAM6_SKYLAKE_MOBILE:
4180 	case INTEL_FAM6_SKYLAKE_DESKTOP:
4181 	case INTEL_FAM6_SKYLAKE_X:
4182 	case INTEL_FAM6_KABYLAKE_MOBILE:
4183 	case INTEL_FAM6_KABYLAKE_DESKTOP:
4184 		x86_pmu.late_ack = true;
4185 		memcpy(hw_cache_event_ids, skl_hw_cache_event_ids, sizeof(hw_cache_event_ids));
4186 		memcpy(hw_cache_extra_regs, skl_hw_cache_extra_regs, sizeof(hw_cache_extra_regs));
4187 		intel_pmu_lbr_init_skl();
4188 
4189 		/* INT_MISC.RECOVERY_CYCLES has umask 1 in Skylake */
4190 		event_attr_td_recovery_bubbles.event_str_noht =
4191 			"event=0xd,umask=0x1,cmask=1";
4192 		event_attr_td_recovery_bubbles.event_str_ht =
4193 			"event=0xd,umask=0x1,cmask=1,any=1";
4194 
4195 		x86_pmu.event_constraints = intel_skl_event_constraints;
4196 		x86_pmu.pebs_constraints = intel_skl_pebs_event_constraints;
4197 		x86_pmu.extra_regs = intel_skl_extra_regs;
4198 		x86_pmu.pebs_aliases = intel_pebs_aliases_skl;
4199 		x86_pmu.pebs_prec_dist = true;
4200 		/* all extra regs are per-cpu when HT is on */
4201 		x86_pmu.flags |= PMU_FL_HAS_RSP_1;
4202 		x86_pmu.flags |= PMU_FL_NO_HT_SHARING;
4203 
4204 		x86_pmu.hw_config = hsw_hw_config;
4205 		x86_pmu.get_event_constraints = hsw_get_event_constraints;
4206 		x86_pmu.format_attrs = merge_attr(intel_arch3_formats_attr,
4207 						  skl_format_attr);
4208 		WARN_ON(!x86_pmu.format_attrs);
4209 		x86_pmu.cpu_events = hsw_events_attrs;
4210 		pr_cont("Skylake events, ");
4211 		break;
4212 
4213 	default:
4214 		switch (x86_pmu.version) {
4215 		case 1:
4216 			x86_pmu.event_constraints = intel_v1_event_constraints;
4217 			pr_cont("generic architected perfmon v1, ");
4218 			break;
4219 		default:
4220 			/*
4221 			 * default constraints for v2 and up
4222 			 */
4223 			x86_pmu.event_constraints = intel_gen_event_constraints;
4224 			pr_cont("generic architected perfmon, ");
4225 			break;
4226 		}
4227 	}
4228 
4229 	if (x86_pmu.num_counters > INTEL_PMC_MAX_GENERIC) {
4230 		WARN(1, KERN_ERR "hw perf events %d > max(%d), clipping!",
4231 		     x86_pmu.num_counters, INTEL_PMC_MAX_GENERIC);
4232 		x86_pmu.num_counters = INTEL_PMC_MAX_GENERIC;
4233 	}
4234 	x86_pmu.intel_ctrl = (1ULL << x86_pmu.num_counters) - 1;
4235 
4236 	if (x86_pmu.num_counters_fixed > INTEL_PMC_MAX_FIXED) {
4237 		WARN(1, KERN_ERR "hw perf events fixed %d > max(%d), clipping!",
4238 		     x86_pmu.num_counters_fixed, INTEL_PMC_MAX_FIXED);
4239 		x86_pmu.num_counters_fixed = INTEL_PMC_MAX_FIXED;
4240 	}
4241 
4242 	x86_pmu.intel_ctrl |=
4243 		((1LL << x86_pmu.num_counters_fixed)-1) << INTEL_PMC_IDX_FIXED;
4244 
4245 	if (x86_pmu.event_constraints) {
4246 		/*
4247 		 * event on fixed counter2 (REF_CYCLES) only works on this
4248 		 * counter, so do not extend mask to generic counters
4249 		 */
4250 		for_each_event_constraint(c, x86_pmu.event_constraints) {
4251 			if (c->cmask == FIXED_EVENT_FLAGS
4252 			    && c->idxmsk64 != INTEL_PMC_MSK_FIXED_REF_CYCLES) {
4253 				c->idxmsk64 |= (1ULL << x86_pmu.num_counters) - 1;
4254 			}
4255 			c->idxmsk64 &=
4256 				~(~0ULL << (INTEL_PMC_IDX_FIXED + x86_pmu.num_counters_fixed));
4257 			c->weight = hweight64(c->idxmsk64);
4258 		}
4259 	}
4260 
4261 	/*
4262 	 * Access LBR MSR may cause #GP under certain circumstances.
4263 	 * E.g. KVM doesn't support LBR MSR
4264 	 * Check all LBT MSR here.
4265 	 * Disable LBR access if any LBR MSRs can not be accessed.
4266 	 */
4267 	if (x86_pmu.lbr_nr && !check_msr(x86_pmu.lbr_tos, 0x3UL))
4268 		x86_pmu.lbr_nr = 0;
4269 	for (i = 0; i < x86_pmu.lbr_nr; i++) {
4270 		if (!(check_msr(x86_pmu.lbr_from + i, 0xffffUL) &&
4271 		      check_msr(x86_pmu.lbr_to + i, 0xffffUL)))
4272 			x86_pmu.lbr_nr = 0;
4273 	}
4274 
4275 	if (x86_pmu.lbr_nr)
4276 		pr_cont("%d-deep LBR, ", x86_pmu.lbr_nr);
4277 	/*
4278 	 * Access extra MSR may cause #GP under certain circumstances.
4279 	 * E.g. KVM doesn't support offcore event
4280 	 * Check all extra_regs here.
4281 	 */
4282 	if (x86_pmu.extra_regs) {
4283 		for (er = x86_pmu.extra_regs; er->msr; er++) {
4284 			er->extra_msr_access = check_msr(er->msr, 0x11UL);
4285 			/* Disable LBR select mapping */
4286 			if ((er->idx == EXTRA_REG_LBR) && !er->extra_msr_access)
4287 				x86_pmu.lbr_sel_map = NULL;
4288 		}
4289 	}
4290 
4291 	/* Support full width counters using alternative MSR range */
4292 	if (x86_pmu.intel_cap.full_width_write) {
4293 		x86_pmu.max_period = x86_pmu.cntval_mask >> 1;
4294 		x86_pmu.perfctr = MSR_IA32_PMC0;
4295 		pr_cont("full-width counters, ");
4296 	}
4297 
4298 	return 0;
4299 }
4300 
4301 /*
4302  * HT bug: phase 2 init
4303  * Called once we have valid topology information to check
4304  * whether or not HT is enabled
4305  * If HT is off, then we disable the workaround
4306  */
4307 static __init int fixup_ht_bug(void)
4308 {
4309 	int c;
4310 	/*
4311 	 * problem not present on this CPU model, nothing to do
4312 	 */
4313 	if (!(x86_pmu.flags & PMU_FL_EXCL_ENABLED))
4314 		return 0;
4315 
4316 	if (topology_max_smt_threads() > 1) {
4317 		pr_info("PMU erratum BJ122, BV98, HSD29 worked around, HT is on\n");
4318 		return 0;
4319 	}
4320 
4321 	if (lockup_detector_suspend() != 0) {
4322 		pr_debug("failed to disable PMU erratum BJ122, BV98, HSD29 workaround\n");
4323 		return 0;
4324 	}
4325 
4326 	x86_pmu.flags &= ~(PMU_FL_EXCL_CNTRS | PMU_FL_EXCL_ENABLED);
4327 
4328 	x86_pmu.start_scheduling = NULL;
4329 	x86_pmu.commit_scheduling = NULL;
4330 	x86_pmu.stop_scheduling = NULL;
4331 
4332 	lockup_detector_resume();
4333 
4334 	cpus_read_lock();
4335 
4336 	for_each_online_cpu(c)
4337 		free_excl_cntrs(c);
4338 
4339 	cpus_read_unlock();
4340 	pr_info("PMU erratum BJ122, BV98, HSD29 workaround disabled, HT off\n");
4341 	return 0;
4342 }
4343 subsys_initcall(fixup_ht_bug)
4344