1 /*
2 * Support cstate residency counters
3 *
4 * Copyright (C) 2015, Intel Corp.
5 * Author: Kan Liang (kan.liang@intel.com)
6 *
7 * This library is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU Library General Public
9 * License as published by the Free Software Foundation; either
10 * version 2 of the License, or (at your option) any later version.
11 *
12 * This library is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * Library General Public License for more details.
16 *
17 */
18
19 /*
20 * This file export cstate related free running (read-only) counters
21 * for perf. These counters may be use simultaneously by other tools,
22 * such as turbostat. However, it still make sense to implement them
23 * in perf. Because we can conveniently collect them together with
24 * other events, and allow to use them from tools without special MSR
25 * access code.
26 *
27 * The events only support system-wide mode counting. There is no
28 * sampling support because it is not supported by the hardware.
29 *
30 * According to counters' scope and category, two PMUs are registered
31 * with the perf_event core subsystem.
32 * - 'cstate_core': The counter is available for each physical core.
33 * The counters include CORE_C*_RESIDENCY.
34 * - 'cstate_pkg': The counter is available for each physical package.
35 * The counters include PKG_C*_RESIDENCY.
36 *
37 * All of these counters are specified in the Intel® 64 and IA-32
38 * Architectures Software Developer.s Manual Vol3b.
39 *
40 * Model specific counters:
41 * MSR_CORE_C1_RES: CORE C1 Residency Counter
42 * perf code: 0x00
43 * Available model: SLM,AMT,GLM,CNL,ICX,TNT,ADL,RPL
44 * MTL,SRF,GRR,ARL,LNL
45 * Scope: Core (each processor core has a MSR)
46 * MSR_CORE_C3_RESIDENCY: CORE C3 Residency Counter
47 * perf code: 0x01
48 * Available model: NHM,WSM,SNB,IVB,HSW,BDW,SKL,GLM,
49 * CNL,KBL,CML,TNT
50 * Scope: Core
51 * MSR_CORE_C6_RESIDENCY: CORE C6 Residency Counter
52 * perf code: 0x02
53 * Available model: SLM,AMT,NHM,WSM,SNB,IVB,HSW,BDW,
54 * SKL,KNL,GLM,CNL,KBL,CML,ICL,ICX,
55 * TGL,TNT,RKL,ADL,RPL,SPR,MTL,SRF,
56 * GRR,ARL,LNL
57 * Scope: Core
58 * MSR_CORE_C7_RESIDENCY: CORE C7 Residency Counter
59 * perf code: 0x03
60 * Available model: SNB,IVB,HSW,BDW,SKL,CNL,KBL,CML,
61 * ICL,TGL,RKL,ADL,RPL,MTL,ARL,LNL
62 * Scope: Core
63 * MSR_PKG_C2_RESIDENCY: Package C2 Residency Counter.
64 * perf code: 0x00
65 * Available model: SNB,IVB,HSW,BDW,SKL,KNL,GLM,CNL,
66 * KBL,CML,ICL,ICX,TGL,TNT,RKL,ADL,
67 * RPL,SPR,MTL,ARL,LNL,SRF
68 * Scope: Package (physical package)
69 * MSR_PKG_C3_RESIDENCY: Package C3 Residency Counter.
70 * perf code: 0x01
71 * Available model: NHM,WSM,SNB,IVB,HSW,BDW,SKL,KNL,
72 * GLM,CNL,KBL,CML,ICL,TGL,TNT,RKL,
73 * ADL,RPL,MTL,ARL,LNL
74 * Scope: Package (physical package)
75 * MSR_PKG_C6_RESIDENCY: Package C6 Residency Counter.
76 * perf code: 0x02
77 * Available model: SLM,AMT,NHM,WSM,SNB,IVB,HSW,BDW,
78 * SKL,KNL,GLM,CNL,KBL,CML,ICL,ICX,
79 * TGL,TNT,RKL,ADL,RPL,SPR,MTL,SRF,
80 * ARL,LNL
81 * Scope: Package (physical package)
82 * MSR_PKG_C7_RESIDENCY: Package C7 Residency Counter.
83 * perf code: 0x03
84 * Available model: NHM,WSM,SNB,IVB,HSW,BDW,SKL,CNL,
85 * KBL,CML,ICL,TGL,RKL
86 * Scope: Package (physical package)
87 * MSR_PKG_C8_RESIDENCY: Package C8 Residency Counter.
88 * perf code: 0x04
89 * Available model: HSW ULT,KBL,CNL,CML,ICL,TGL,RKL,
90 * ADL,RPL,MTL,ARL
91 * Scope: Package (physical package)
92 * MSR_PKG_C9_RESIDENCY: Package C9 Residency Counter.
93 * perf code: 0x05
94 * Available model: HSW ULT,KBL,CNL,CML,ICL,TGL,RKL
95 * Scope: Package (physical package)
96 * MSR_PKG_C10_RESIDENCY: Package C10 Residency Counter.
97 * perf code: 0x06
98 * Available model: HSW ULT,KBL,GLM,CNL,CML,ICL,TGL,
99 * TNT,RKL,ADL,RPL,MTL,ARL,LNL
100 * Scope: Package (physical package)
101 * MSR_MODULE_C6_RES_MS: Module C6 Residency Counter.
102 * perf code: 0x00
103 * Available model: SRF,GRR
104 * Scope: A cluster of cores shared L2 cache
105 *
106 */
107
108 #include <linux/module.h>
109 #include <linux/slab.h>
110 #include <linux/perf_event.h>
111 #include <linux/nospec.h>
112 #include <asm/cpu_device_id.h>
113 #include <asm/intel-family.h>
114 #include "../perf_event.h"
115 #include "../probe.h"
116
117 MODULE_DESCRIPTION("Support for Intel cstate performance events");
118 MODULE_LICENSE("GPL");
119
120 #define DEFINE_CSTATE_FORMAT_ATTR(_var, _name, _format) \
121 static ssize_t __cstate_##_var##_show(struct device *dev, \
122 struct device_attribute *attr, \
123 char *page) \
124 { \
125 BUILD_BUG_ON(sizeof(_format) >= PAGE_SIZE); \
126 return sprintf(page, _format "\n"); \
127 } \
128 static struct device_attribute format_attr_##_var = \
129 __ATTR(_name, 0444, __cstate_##_var##_show, NULL)
130
131 /* Model -> events mapping */
132 struct cstate_model {
133 unsigned long core_events;
134 unsigned long pkg_events;
135 unsigned long module_events;
136 unsigned long quirks;
137 };
138
139 /* Quirk flags */
140 #define SLM_PKG_C6_USE_C7_MSR (1UL << 0)
141 #define KNL_CORE_C6_MSR (1UL << 1)
142
143 /* cstate_core PMU */
144 static struct pmu cstate_core_pmu;
145 static bool has_cstate_core;
146
147 enum perf_cstate_core_events {
148 PERF_CSTATE_CORE_C1_RES = 0,
149 PERF_CSTATE_CORE_C3_RES,
150 PERF_CSTATE_CORE_C6_RES,
151 PERF_CSTATE_CORE_C7_RES,
152
153 PERF_CSTATE_CORE_EVENT_MAX,
154 };
155
156 PMU_EVENT_ATTR_STRING(c1-residency, attr_cstate_core_c1, "event=0x00");
157 PMU_EVENT_ATTR_STRING(c3-residency, attr_cstate_core_c3, "event=0x01");
158 PMU_EVENT_ATTR_STRING(c6-residency, attr_cstate_core_c6, "event=0x02");
159 PMU_EVENT_ATTR_STRING(c7-residency, attr_cstate_core_c7, "event=0x03");
160
161 static unsigned long core_msr_mask;
162
163 PMU_EVENT_GROUP(events, cstate_core_c1);
164 PMU_EVENT_GROUP(events, cstate_core_c3);
165 PMU_EVENT_GROUP(events, cstate_core_c6);
166 PMU_EVENT_GROUP(events, cstate_core_c7);
167
test_msr(int idx,void * data)168 static bool test_msr(int idx, void *data)
169 {
170 return test_bit(idx, (unsigned long *) data);
171 }
172
173 static struct perf_msr core_msr[] = {
174 [PERF_CSTATE_CORE_C1_RES] = { MSR_CORE_C1_RES, &group_cstate_core_c1, test_msr },
175 [PERF_CSTATE_CORE_C3_RES] = { MSR_CORE_C3_RESIDENCY, &group_cstate_core_c3, test_msr },
176 [PERF_CSTATE_CORE_C6_RES] = { MSR_CORE_C6_RESIDENCY, &group_cstate_core_c6, test_msr },
177 [PERF_CSTATE_CORE_C7_RES] = { MSR_CORE_C7_RESIDENCY, &group_cstate_core_c7, test_msr },
178 };
179
180 static struct attribute *attrs_empty[] = {
181 NULL,
182 };
183
184 /*
185 * There are no default events, but we need to create
186 * "events" group (with empty attrs) before updating
187 * it with detected events.
188 */
189 static struct attribute_group cstate_events_attr_group = {
190 .name = "events",
191 .attrs = attrs_empty,
192 };
193
194 DEFINE_CSTATE_FORMAT_ATTR(cstate_event, event, "config:0-63");
195 static struct attribute *cstate_format_attrs[] = {
196 &format_attr_cstate_event.attr,
197 NULL,
198 };
199
200 static struct attribute_group cstate_format_attr_group = {
201 .name = "format",
202 .attrs = cstate_format_attrs,
203 };
204
205 static const struct attribute_group *cstate_attr_groups[] = {
206 &cstate_events_attr_group,
207 &cstate_format_attr_group,
208 NULL,
209 };
210
211 /* cstate_pkg PMU */
212 static struct pmu cstate_pkg_pmu;
213 static bool has_cstate_pkg;
214
215 enum perf_cstate_pkg_events {
216 PERF_CSTATE_PKG_C2_RES = 0,
217 PERF_CSTATE_PKG_C3_RES,
218 PERF_CSTATE_PKG_C6_RES,
219 PERF_CSTATE_PKG_C7_RES,
220 PERF_CSTATE_PKG_C8_RES,
221 PERF_CSTATE_PKG_C9_RES,
222 PERF_CSTATE_PKG_C10_RES,
223
224 PERF_CSTATE_PKG_EVENT_MAX,
225 };
226
227 PMU_EVENT_ATTR_STRING(c2-residency, attr_cstate_pkg_c2, "event=0x00");
228 PMU_EVENT_ATTR_STRING(c3-residency, attr_cstate_pkg_c3, "event=0x01");
229 PMU_EVENT_ATTR_STRING(c6-residency, attr_cstate_pkg_c6, "event=0x02");
230 PMU_EVENT_ATTR_STRING(c7-residency, attr_cstate_pkg_c7, "event=0x03");
231 PMU_EVENT_ATTR_STRING(c8-residency, attr_cstate_pkg_c8, "event=0x04");
232 PMU_EVENT_ATTR_STRING(c9-residency, attr_cstate_pkg_c9, "event=0x05");
233 PMU_EVENT_ATTR_STRING(c10-residency, attr_cstate_pkg_c10, "event=0x06");
234
235 static unsigned long pkg_msr_mask;
236
237 PMU_EVENT_GROUP(events, cstate_pkg_c2);
238 PMU_EVENT_GROUP(events, cstate_pkg_c3);
239 PMU_EVENT_GROUP(events, cstate_pkg_c6);
240 PMU_EVENT_GROUP(events, cstate_pkg_c7);
241 PMU_EVENT_GROUP(events, cstate_pkg_c8);
242 PMU_EVENT_GROUP(events, cstate_pkg_c9);
243 PMU_EVENT_GROUP(events, cstate_pkg_c10);
244
245 static struct perf_msr pkg_msr[] = {
246 [PERF_CSTATE_PKG_C2_RES] = { MSR_PKG_C2_RESIDENCY, &group_cstate_pkg_c2, test_msr },
247 [PERF_CSTATE_PKG_C3_RES] = { MSR_PKG_C3_RESIDENCY, &group_cstate_pkg_c3, test_msr },
248 [PERF_CSTATE_PKG_C6_RES] = { MSR_PKG_C6_RESIDENCY, &group_cstate_pkg_c6, test_msr },
249 [PERF_CSTATE_PKG_C7_RES] = { MSR_PKG_C7_RESIDENCY, &group_cstate_pkg_c7, test_msr },
250 [PERF_CSTATE_PKG_C8_RES] = { MSR_PKG_C8_RESIDENCY, &group_cstate_pkg_c8, test_msr },
251 [PERF_CSTATE_PKG_C9_RES] = { MSR_PKG_C9_RESIDENCY, &group_cstate_pkg_c9, test_msr },
252 [PERF_CSTATE_PKG_C10_RES] = { MSR_PKG_C10_RESIDENCY, &group_cstate_pkg_c10, test_msr },
253 };
254
255 /* cstate_module PMU */
256 static struct pmu cstate_module_pmu;
257 static bool has_cstate_module;
258
259 enum perf_cstate_module_events {
260 PERF_CSTATE_MODULE_C6_RES = 0,
261
262 PERF_CSTATE_MODULE_EVENT_MAX,
263 };
264
265 PMU_EVENT_ATTR_STRING(c6-residency, attr_cstate_module_c6, "event=0x00");
266
267 static unsigned long module_msr_mask;
268
269 PMU_EVENT_GROUP(events, cstate_module_c6);
270
271 static struct perf_msr module_msr[] = {
272 [PERF_CSTATE_MODULE_C6_RES] = { MSR_MODULE_C6_RES_MS, &group_cstate_module_c6, test_msr },
273 };
274
cstate_pmu_event_init(struct perf_event * event)275 static int cstate_pmu_event_init(struct perf_event *event)
276 {
277 u64 cfg = event->attr.config;
278
279 if (event->attr.type != event->pmu->type)
280 return -ENOENT;
281
282 /* unsupported modes and filters */
283 if (event->attr.sample_period) /* no sampling */
284 return -EINVAL;
285
286 if (event->cpu < 0)
287 return -EINVAL;
288
289 if (event->pmu == &cstate_core_pmu) {
290 if (cfg >= PERF_CSTATE_CORE_EVENT_MAX)
291 return -EINVAL;
292 cfg = array_index_nospec((unsigned long)cfg, PERF_CSTATE_CORE_EVENT_MAX);
293 if (!(core_msr_mask & (1 << cfg)))
294 return -EINVAL;
295 event->hw.event_base = core_msr[cfg].msr;
296 } else if (event->pmu == &cstate_pkg_pmu) {
297 if (cfg >= PERF_CSTATE_PKG_EVENT_MAX)
298 return -EINVAL;
299 cfg = array_index_nospec((unsigned long)cfg, PERF_CSTATE_PKG_EVENT_MAX);
300 if (!(pkg_msr_mask & (1 << cfg)))
301 return -EINVAL;
302 event->hw.event_base = pkg_msr[cfg].msr;
303 } else if (event->pmu == &cstate_module_pmu) {
304 if (cfg >= PERF_CSTATE_MODULE_EVENT_MAX)
305 return -EINVAL;
306 cfg = array_index_nospec((unsigned long)cfg, PERF_CSTATE_MODULE_EVENT_MAX);
307 if (!(module_msr_mask & (1 << cfg)))
308 return -EINVAL;
309 event->hw.event_base = module_msr[cfg].msr;
310 } else {
311 return -ENOENT;
312 }
313
314 event->hw.config = cfg;
315 event->hw.idx = -1;
316 return 0;
317 }
318
cstate_pmu_read_counter(struct perf_event * event)319 static inline u64 cstate_pmu_read_counter(struct perf_event *event)
320 {
321 u64 val;
322
323 rdmsrl(event->hw.event_base, val);
324 return val;
325 }
326
cstate_pmu_event_update(struct perf_event * event)327 static void cstate_pmu_event_update(struct perf_event *event)
328 {
329 struct hw_perf_event *hwc = &event->hw;
330 u64 prev_raw_count, new_raw_count;
331
332 prev_raw_count = local64_read(&hwc->prev_count);
333 do {
334 new_raw_count = cstate_pmu_read_counter(event);
335 } while (!local64_try_cmpxchg(&hwc->prev_count,
336 &prev_raw_count, new_raw_count));
337
338 local64_add(new_raw_count - prev_raw_count, &event->count);
339 }
340
cstate_pmu_event_start(struct perf_event * event,int mode)341 static void cstate_pmu_event_start(struct perf_event *event, int mode)
342 {
343 local64_set(&event->hw.prev_count, cstate_pmu_read_counter(event));
344 }
345
cstate_pmu_event_stop(struct perf_event * event,int mode)346 static void cstate_pmu_event_stop(struct perf_event *event, int mode)
347 {
348 cstate_pmu_event_update(event);
349 }
350
cstate_pmu_event_del(struct perf_event * event,int mode)351 static void cstate_pmu_event_del(struct perf_event *event, int mode)
352 {
353 cstate_pmu_event_stop(event, PERF_EF_UPDATE);
354 }
355
cstate_pmu_event_add(struct perf_event * event,int mode)356 static int cstate_pmu_event_add(struct perf_event *event, int mode)
357 {
358 if (mode & PERF_EF_START)
359 cstate_pmu_event_start(event, mode);
360
361 return 0;
362 }
363
364 static const struct attribute_group *core_attr_update[] = {
365 &group_cstate_core_c1,
366 &group_cstate_core_c3,
367 &group_cstate_core_c6,
368 &group_cstate_core_c7,
369 NULL,
370 };
371
372 static const struct attribute_group *pkg_attr_update[] = {
373 &group_cstate_pkg_c2,
374 &group_cstate_pkg_c3,
375 &group_cstate_pkg_c6,
376 &group_cstate_pkg_c7,
377 &group_cstate_pkg_c8,
378 &group_cstate_pkg_c9,
379 &group_cstate_pkg_c10,
380 NULL,
381 };
382
383 static const struct attribute_group *module_attr_update[] = {
384 &group_cstate_module_c6,
385 NULL
386 };
387
388 static struct pmu cstate_core_pmu = {
389 .attr_groups = cstate_attr_groups,
390 .attr_update = core_attr_update,
391 .name = "cstate_core",
392 .task_ctx_nr = perf_invalid_context,
393 .event_init = cstate_pmu_event_init,
394 .add = cstate_pmu_event_add,
395 .del = cstate_pmu_event_del,
396 .start = cstate_pmu_event_start,
397 .stop = cstate_pmu_event_stop,
398 .read = cstate_pmu_event_update,
399 .capabilities = PERF_PMU_CAP_NO_INTERRUPT | PERF_PMU_CAP_NO_EXCLUDE,
400 .scope = PERF_PMU_SCOPE_CORE,
401 .module = THIS_MODULE,
402 };
403
404 static struct pmu cstate_pkg_pmu = {
405 .attr_groups = cstate_attr_groups,
406 .attr_update = pkg_attr_update,
407 .name = "cstate_pkg",
408 .task_ctx_nr = perf_invalid_context,
409 .event_init = cstate_pmu_event_init,
410 .add = cstate_pmu_event_add,
411 .del = cstate_pmu_event_del,
412 .start = cstate_pmu_event_start,
413 .stop = cstate_pmu_event_stop,
414 .read = cstate_pmu_event_update,
415 .capabilities = PERF_PMU_CAP_NO_INTERRUPT | PERF_PMU_CAP_NO_EXCLUDE,
416 .scope = PERF_PMU_SCOPE_PKG,
417 .module = THIS_MODULE,
418 };
419
420 static struct pmu cstate_module_pmu = {
421 .attr_groups = cstate_attr_groups,
422 .attr_update = module_attr_update,
423 .name = "cstate_module",
424 .task_ctx_nr = perf_invalid_context,
425 .event_init = cstate_pmu_event_init,
426 .add = cstate_pmu_event_add,
427 .del = cstate_pmu_event_del,
428 .start = cstate_pmu_event_start,
429 .stop = cstate_pmu_event_stop,
430 .read = cstate_pmu_event_update,
431 .capabilities = PERF_PMU_CAP_NO_INTERRUPT | PERF_PMU_CAP_NO_EXCLUDE,
432 .scope = PERF_PMU_SCOPE_CLUSTER,
433 .module = THIS_MODULE,
434 };
435
436 static const struct cstate_model nhm_cstates __initconst = {
437 .core_events = BIT(PERF_CSTATE_CORE_C3_RES) |
438 BIT(PERF_CSTATE_CORE_C6_RES),
439
440 .pkg_events = BIT(PERF_CSTATE_PKG_C3_RES) |
441 BIT(PERF_CSTATE_PKG_C6_RES) |
442 BIT(PERF_CSTATE_PKG_C7_RES),
443 };
444
445 static const struct cstate_model snb_cstates __initconst = {
446 .core_events = BIT(PERF_CSTATE_CORE_C3_RES) |
447 BIT(PERF_CSTATE_CORE_C6_RES) |
448 BIT(PERF_CSTATE_CORE_C7_RES),
449
450 .pkg_events = BIT(PERF_CSTATE_PKG_C2_RES) |
451 BIT(PERF_CSTATE_PKG_C3_RES) |
452 BIT(PERF_CSTATE_PKG_C6_RES) |
453 BIT(PERF_CSTATE_PKG_C7_RES),
454 };
455
456 static const struct cstate_model hswult_cstates __initconst = {
457 .core_events = BIT(PERF_CSTATE_CORE_C3_RES) |
458 BIT(PERF_CSTATE_CORE_C6_RES) |
459 BIT(PERF_CSTATE_CORE_C7_RES),
460
461 .pkg_events = BIT(PERF_CSTATE_PKG_C2_RES) |
462 BIT(PERF_CSTATE_PKG_C3_RES) |
463 BIT(PERF_CSTATE_PKG_C6_RES) |
464 BIT(PERF_CSTATE_PKG_C7_RES) |
465 BIT(PERF_CSTATE_PKG_C8_RES) |
466 BIT(PERF_CSTATE_PKG_C9_RES) |
467 BIT(PERF_CSTATE_PKG_C10_RES),
468 };
469
470 static const struct cstate_model cnl_cstates __initconst = {
471 .core_events = BIT(PERF_CSTATE_CORE_C1_RES) |
472 BIT(PERF_CSTATE_CORE_C3_RES) |
473 BIT(PERF_CSTATE_CORE_C6_RES) |
474 BIT(PERF_CSTATE_CORE_C7_RES),
475
476 .pkg_events = BIT(PERF_CSTATE_PKG_C2_RES) |
477 BIT(PERF_CSTATE_PKG_C3_RES) |
478 BIT(PERF_CSTATE_PKG_C6_RES) |
479 BIT(PERF_CSTATE_PKG_C7_RES) |
480 BIT(PERF_CSTATE_PKG_C8_RES) |
481 BIT(PERF_CSTATE_PKG_C9_RES) |
482 BIT(PERF_CSTATE_PKG_C10_RES),
483 };
484
485 static const struct cstate_model icl_cstates __initconst = {
486 .core_events = BIT(PERF_CSTATE_CORE_C6_RES) |
487 BIT(PERF_CSTATE_CORE_C7_RES),
488
489 .pkg_events = BIT(PERF_CSTATE_PKG_C2_RES) |
490 BIT(PERF_CSTATE_PKG_C3_RES) |
491 BIT(PERF_CSTATE_PKG_C6_RES) |
492 BIT(PERF_CSTATE_PKG_C7_RES) |
493 BIT(PERF_CSTATE_PKG_C8_RES) |
494 BIT(PERF_CSTATE_PKG_C9_RES) |
495 BIT(PERF_CSTATE_PKG_C10_RES),
496 };
497
498 static const struct cstate_model icx_cstates __initconst = {
499 .core_events = BIT(PERF_CSTATE_CORE_C1_RES) |
500 BIT(PERF_CSTATE_CORE_C6_RES),
501
502 .pkg_events = BIT(PERF_CSTATE_PKG_C2_RES) |
503 BIT(PERF_CSTATE_PKG_C6_RES),
504 };
505
506 static const struct cstate_model adl_cstates __initconst = {
507 .core_events = BIT(PERF_CSTATE_CORE_C1_RES) |
508 BIT(PERF_CSTATE_CORE_C6_RES) |
509 BIT(PERF_CSTATE_CORE_C7_RES),
510
511 .pkg_events = BIT(PERF_CSTATE_PKG_C2_RES) |
512 BIT(PERF_CSTATE_PKG_C3_RES) |
513 BIT(PERF_CSTATE_PKG_C6_RES) |
514 BIT(PERF_CSTATE_PKG_C8_RES) |
515 BIT(PERF_CSTATE_PKG_C10_RES),
516 };
517
518 static const struct cstate_model lnl_cstates __initconst = {
519 .core_events = BIT(PERF_CSTATE_CORE_C1_RES) |
520 BIT(PERF_CSTATE_CORE_C6_RES) |
521 BIT(PERF_CSTATE_CORE_C7_RES),
522
523 .pkg_events = BIT(PERF_CSTATE_PKG_C2_RES) |
524 BIT(PERF_CSTATE_PKG_C3_RES) |
525 BIT(PERF_CSTATE_PKG_C6_RES) |
526 BIT(PERF_CSTATE_PKG_C10_RES),
527 };
528
529 static const struct cstate_model slm_cstates __initconst = {
530 .core_events = BIT(PERF_CSTATE_CORE_C1_RES) |
531 BIT(PERF_CSTATE_CORE_C6_RES),
532
533 .pkg_events = BIT(PERF_CSTATE_PKG_C6_RES),
534 .quirks = SLM_PKG_C6_USE_C7_MSR,
535 };
536
537
538 static const struct cstate_model knl_cstates __initconst = {
539 .core_events = BIT(PERF_CSTATE_CORE_C6_RES),
540
541 .pkg_events = BIT(PERF_CSTATE_PKG_C2_RES) |
542 BIT(PERF_CSTATE_PKG_C3_RES) |
543 BIT(PERF_CSTATE_PKG_C6_RES),
544 .quirks = KNL_CORE_C6_MSR,
545 };
546
547
548 static const struct cstate_model glm_cstates __initconst = {
549 .core_events = BIT(PERF_CSTATE_CORE_C1_RES) |
550 BIT(PERF_CSTATE_CORE_C3_RES) |
551 BIT(PERF_CSTATE_CORE_C6_RES),
552
553 .pkg_events = BIT(PERF_CSTATE_PKG_C2_RES) |
554 BIT(PERF_CSTATE_PKG_C3_RES) |
555 BIT(PERF_CSTATE_PKG_C6_RES) |
556 BIT(PERF_CSTATE_PKG_C10_RES),
557 };
558
559 static const struct cstate_model grr_cstates __initconst = {
560 .core_events = BIT(PERF_CSTATE_CORE_C1_RES) |
561 BIT(PERF_CSTATE_CORE_C6_RES),
562
563 .module_events = BIT(PERF_CSTATE_MODULE_C6_RES),
564 };
565
566 static const struct cstate_model srf_cstates __initconst = {
567 .core_events = BIT(PERF_CSTATE_CORE_C1_RES) |
568 BIT(PERF_CSTATE_CORE_C6_RES),
569
570 .pkg_events = BIT(PERF_CSTATE_PKG_C2_RES) |
571 BIT(PERF_CSTATE_PKG_C6_RES),
572
573 .module_events = BIT(PERF_CSTATE_MODULE_C6_RES),
574 };
575
576
577 static const struct x86_cpu_id intel_cstates_match[] __initconst = {
578 X86_MATCH_VFM(INTEL_NEHALEM, &nhm_cstates),
579 X86_MATCH_VFM(INTEL_NEHALEM_EP, &nhm_cstates),
580 X86_MATCH_VFM(INTEL_NEHALEM_EX, &nhm_cstates),
581
582 X86_MATCH_VFM(INTEL_WESTMERE, &nhm_cstates),
583 X86_MATCH_VFM(INTEL_WESTMERE_EP, &nhm_cstates),
584 X86_MATCH_VFM(INTEL_WESTMERE_EX, &nhm_cstates),
585
586 X86_MATCH_VFM(INTEL_SANDYBRIDGE, &snb_cstates),
587 X86_MATCH_VFM(INTEL_SANDYBRIDGE_X, &snb_cstates),
588
589 X86_MATCH_VFM(INTEL_IVYBRIDGE, &snb_cstates),
590 X86_MATCH_VFM(INTEL_IVYBRIDGE_X, &snb_cstates),
591
592 X86_MATCH_VFM(INTEL_HASWELL, &snb_cstates),
593 X86_MATCH_VFM(INTEL_HASWELL_X, &snb_cstates),
594 X86_MATCH_VFM(INTEL_HASWELL_G, &snb_cstates),
595
596 X86_MATCH_VFM(INTEL_HASWELL_L, &hswult_cstates),
597
598 X86_MATCH_VFM(INTEL_ATOM_SILVERMONT, &slm_cstates),
599 X86_MATCH_VFM(INTEL_ATOM_SILVERMONT_D, &slm_cstates),
600 X86_MATCH_VFM(INTEL_ATOM_AIRMONT, &slm_cstates),
601
602 X86_MATCH_VFM(INTEL_BROADWELL, &snb_cstates),
603 X86_MATCH_VFM(INTEL_BROADWELL_D, &snb_cstates),
604 X86_MATCH_VFM(INTEL_BROADWELL_G, &snb_cstates),
605 X86_MATCH_VFM(INTEL_BROADWELL_X, &snb_cstates),
606
607 X86_MATCH_VFM(INTEL_SKYLAKE_L, &snb_cstates),
608 X86_MATCH_VFM(INTEL_SKYLAKE, &snb_cstates),
609 X86_MATCH_VFM(INTEL_SKYLAKE_X, &snb_cstates),
610
611 X86_MATCH_VFM(INTEL_KABYLAKE_L, &hswult_cstates),
612 X86_MATCH_VFM(INTEL_KABYLAKE, &hswult_cstates),
613 X86_MATCH_VFM(INTEL_COMETLAKE_L, &hswult_cstates),
614 X86_MATCH_VFM(INTEL_COMETLAKE, &hswult_cstates),
615
616 X86_MATCH_VFM(INTEL_CANNONLAKE_L, &cnl_cstates),
617
618 X86_MATCH_VFM(INTEL_XEON_PHI_KNL, &knl_cstates),
619 X86_MATCH_VFM(INTEL_XEON_PHI_KNM, &knl_cstates),
620
621 X86_MATCH_VFM(INTEL_ATOM_GOLDMONT, &glm_cstates),
622 X86_MATCH_VFM(INTEL_ATOM_GOLDMONT_D, &glm_cstates),
623 X86_MATCH_VFM(INTEL_ATOM_GOLDMONT_PLUS, &glm_cstates),
624 X86_MATCH_VFM(INTEL_ATOM_TREMONT_D, &glm_cstates),
625 X86_MATCH_VFM(INTEL_ATOM_TREMONT, &glm_cstates),
626 X86_MATCH_VFM(INTEL_ATOM_TREMONT_L, &glm_cstates),
627 X86_MATCH_VFM(INTEL_ATOM_GRACEMONT, &adl_cstates),
628 X86_MATCH_VFM(INTEL_ATOM_CRESTMONT_X, &srf_cstates),
629 X86_MATCH_VFM(INTEL_ATOM_CRESTMONT, &grr_cstates),
630
631 X86_MATCH_VFM(INTEL_ICELAKE_L, &icl_cstates),
632 X86_MATCH_VFM(INTEL_ICELAKE, &icl_cstates),
633 X86_MATCH_VFM(INTEL_ICELAKE_X, &icx_cstates),
634 X86_MATCH_VFM(INTEL_ICELAKE_D, &icx_cstates),
635 X86_MATCH_VFM(INTEL_SAPPHIRERAPIDS_X, &icx_cstates),
636 X86_MATCH_VFM(INTEL_EMERALDRAPIDS_X, &icx_cstates),
637 X86_MATCH_VFM(INTEL_GRANITERAPIDS_X, &icx_cstates),
638 X86_MATCH_VFM(INTEL_GRANITERAPIDS_D, &icx_cstates),
639
640 X86_MATCH_VFM(INTEL_TIGERLAKE_L, &icl_cstates),
641 X86_MATCH_VFM(INTEL_TIGERLAKE, &icl_cstates),
642 X86_MATCH_VFM(INTEL_ROCKETLAKE, &icl_cstates),
643 X86_MATCH_VFM(INTEL_ALDERLAKE, &adl_cstates),
644 X86_MATCH_VFM(INTEL_ALDERLAKE_L, &adl_cstates),
645 X86_MATCH_VFM(INTEL_RAPTORLAKE, &adl_cstates),
646 X86_MATCH_VFM(INTEL_RAPTORLAKE_P, &adl_cstates),
647 X86_MATCH_VFM(INTEL_RAPTORLAKE_S, &adl_cstates),
648 X86_MATCH_VFM(INTEL_METEORLAKE, &adl_cstates),
649 X86_MATCH_VFM(INTEL_METEORLAKE_L, &adl_cstates),
650 X86_MATCH_VFM(INTEL_ARROWLAKE, &adl_cstates),
651 X86_MATCH_VFM(INTEL_ARROWLAKE_H, &adl_cstates),
652 X86_MATCH_VFM(INTEL_ARROWLAKE_U, &adl_cstates),
653 X86_MATCH_VFM(INTEL_LUNARLAKE_M, &lnl_cstates),
654 { },
655 };
656 MODULE_DEVICE_TABLE(x86cpu, intel_cstates_match);
657
cstate_probe(const struct cstate_model * cm)658 static int __init cstate_probe(const struct cstate_model *cm)
659 {
660 /* SLM has different MSR for PKG C6 */
661 if (cm->quirks & SLM_PKG_C6_USE_C7_MSR)
662 pkg_msr[PERF_CSTATE_PKG_C6_RES].msr = MSR_PKG_C7_RESIDENCY;
663
664 /* KNL has different MSR for CORE C6 */
665 if (cm->quirks & KNL_CORE_C6_MSR)
666 pkg_msr[PERF_CSTATE_CORE_C6_RES].msr = MSR_KNL_CORE_C6_RESIDENCY;
667
668
669 core_msr_mask = perf_msr_probe(core_msr, PERF_CSTATE_CORE_EVENT_MAX,
670 true, (void *) &cm->core_events);
671
672 pkg_msr_mask = perf_msr_probe(pkg_msr, PERF_CSTATE_PKG_EVENT_MAX,
673 true, (void *) &cm->pkg_events);
674
675 module_msr_mask = perf_msr_probe(module_msr, PERF_CSTATE_MODULE_EVENT_MAX,
676 true, (void *) &cm->module_events);
677
678 has_cstate_core = !!core_msr_mask;
679 has_cstate_pkg = !!pkg_msr_mask;
680 has_cstate_module = !!module_msr_mask;
681
682 return (has_cstate_core || has_cstate_pkg || has_cstate_module) ? 0 : -ENODEV;
683 }
684
cstate_cleanup(void)685 static inline void cstate_cleanup(void)
686 {
687 if (has_cstate_core)
688 perf_pmu_unregister(&cstate_core_pmu);
689
690 if (has_cstate_pkg)
691 perf_pmu_unregister(&cstate_pkg_pmu);
692
693 if (has_cstate_module)
694 perf_pmu_unregister(&cstate_module_pmu);
695 }
696
cstate_init(void)697 static int __init cstate_init(void)
698 {
699 int err;
700
701 if (has_cstate_core) {
702 err = perf_pmu_register(&cstate_core_pmu, cstate_core_pmu.name, -1);
703 if (err) {
704 has_cstate_core = false;
705 pr_info("Failed to register cstate core pmu\n");
706 cstate_cleanup();
707 return err;
708 }
709 }
710
711 if (has_cstate_pkg) {
712 if (topology_max_dies_per_package() > 1) {
713 /* CLX-AP is multi-die and the cstate is die-scope */
714 cstate_pkg_pmu.scope = PERF_PMU_SCOPE_DIE;
715 err = perf_pmu_register(&cstate_pkg_pmu,
716 "cstate_die", -1);
717 } else {
718 err = perf_pmu_register(&cstate_pkg_pmu,
719 cstate_pkg_pmu.name, -1);
720 }
721 if (err) {
722 has_cstate_pkg = false;
723 pr_info("Failed to register cstate pkg pmu\n");
724 cstate_cleanup();
725 return err;
726 }
727 }
728
729 if (has_cstate_module) {
730 err = perf_pmu_register(&cstate_module_pmu, cstate_module_pmu.name, -1);
731 if (err) {
732 has_cstate_module = false;
733 pr_info("Failed to register cstate cluster pmu\n");
734 cstate_cleanup();
735 return err;
736 }
737 }
738 return 0;
739 }
740
cstate_pmu_init(void)741 static int __init cstate_pmu_init(void)
742 {
743 const struct x86_cpu_id *id;
744 int err;
745
746 if (boot_cpu_has(X86_FEATURE_HYPERVISOR))
747 return -ENODEV;
748
749 id = x86_match_cpu(intel_cstates_match);
750 if (!id)
751 return -ENODEV;
752
753 err = cstate_probe((const struct cstate_model *) id->driver_data);
754 if (err)
755 return err;
756
757 return cstate_init();
758 }
759 module_init(cstate_pmu_init);
760
cstate_pmu_exit(void)761 static void __exit cstate_pmu_exit(void)
762 {
763 cstate_cleanup();
764 }
765 module_exit(cstate_pmu_exit);
766