1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Resource Director Technology(RDT)
4 * - Intel Application Energy Telemetry
5 *
6 * Copyright (C) 2025 Intel Corporation
7 *
8 * Author:
9 * Tony Luck <tony.luck@intel.com>
10 */
11
12 #define pr_fmt(fmt) "resctrl: " fmt
13
14 #include <linux/bits.h>
15 #include <linux/compiler_types.h>
16 #include <linux/container_of.h>
17 #include <linux/cpumask.h>
18 #include <linux/err.h>
19 #include <linux/errno.h>
20 #include <linux/gfp_types.h>
21 #include <linux/init.h>
22 #include <linux/intel_pmt_features.h>
23 #include <linux/intel_vsec.h>
24 #include <linux/io.h>
25 #include <linux/minmax.h>
26 #include <linux/printk.h>
27 #include <linux/rculist.h>
28 #include <linux/rcupdate.h>
29 #include <linux/resctrl.h>
30 #include <linux/resctrl_types.h>
31 #include <linux/slab.h>
32 #include <linux/stddef.h>
33 #include <linux/topology.h>
34 #include <linux/types.h>
35
36 #include "internal.h"
37
38 /**
39 * struct pmt_event - Telemetry event.
40 * @id: Resctrl event id.
41 * @idx: Counter index within each per-RMID block of counters.
42 * @bin_bits: Zero for integer valued events, else number bits in fraction
43 * part of fixed-point.
44 */
45 struct pmt_event {
46 enum resctrl_event_id id;
47 unsigned int idx;
48 unsigned int bin_bits;
49 };
50
51 #define EVT(_id, _idx, _bits) { .id = _id, .idx = _idx, .bin_bits = _bits }
52
53 /**
54 * struct event_group - Events with the same feature type ("energy" or "perf") and GUID.
55 * @pfname: PMT feature name ("energy" or "perf") of this event group.
56 * Used by boot rdt= option.
57 * @pfg: Points to the aggregated telemetry space information
58 * returned by the intel_pmt_get_regions_by_feature()
59 * call to the INTEL_PMT_TELEMETRY driver that contains
60 * data for all telemetry regions of type @pfname.
61 * Valid if the system supports the event group,
62 * NULL otherwise.
63 * @force_off: True when "rdt" command line or architecture code disables
64 * this event group due to insufficient RMIDs.
65 * @force_on: True when "rdt" command line overrides disable of this
66 * event group.
67 * @guid: Unique number per XML description file.
68 * @num_rmid: Number of RMIDs supported by this group. May be
69 * adjusted downwards if enumeration from
70 * intel_pmt_get_regions_by_feature() indicates fewer
71 * RMIDs can be tracked simultaneously.
72 * @mmio_size: Number of bytes of MMIO registers for this group.
73 * @num_events: Number of events in this group.
74 * @evts: Array of event descriptors.
75 */
76 struct event_group {
77 /* Data fields for additional structures to manage this group. */
78 const char *pfname;
79 struct pmt_feature_group *pfg;
80 bool force_off, force_on;
81
82 /* Remaining fields initialized from XML file. */
83 u32 guid;
84 u32 num_rmid;
85 size_t mmio_size;
86 unsigned int num_events;
87 struct pmt_event evts[] __counted_by(num_events);
88 };
89
90 #define XML_MMIO_SIZE(num_rmids, num_events, num_extra_status) \
91 (((num_rmids) * (num_events) + (num_extra_status)) * sizeof(u64))
92
93 /*
94 * Link: https://github.com/intel/Intel-PMT/blob/main/xml/CWF/OOBMSM/RMID-ENERGY/cwf_aggregator.xml
95 */
96 static struct event_group energy_0x26696143 = {
97 .pfname = "energy",
98 .guid = 0x26696143,
99 .num_rmid = 576,
100 .mmio_size = XML_MMIO_SIZE(576, 2, 3),
101 .num_events = 2,
102 .evts = {
103 EVT(PMT_EVENT_ENERGY, 0, 18),
104 EVT(PMT_EVENT_ACTIVITY, 1, 18),
105 }
106 };
107
108 /*
109 * Link: https://github.com/intel/Intel-PMT/blob/main/xml/CWF/OOBMSM/RMID-PERF/cwf_aggregator.xml
110 */
111 static struct event_group perf_0x26557651 = {
112 .pfname = "perf",
113 .guid = 0x26557651,
114 .num_rmid = 576,
115 .mmio_size = XML_MMIO_SIZE(576, 7, 3),
116 .num_events = 7,
117 .evts = {
118 EVT(PMT_EVENT_STALLS_LLC_HIT, 0, 0),
119 EVT(PMT_EVENT_C1_RES, 1, 0),
120 EVT(PMT_EVENT_UNHALTED_CORE_CYCLES, 2, 0),
121 EVT(PMT_EVENT_STALLS_LLC_MISS, 3, 0),
122 EVT(PMT_EVENT_AUTO_C6_RES, 4, 0),
123 EVT(PMT_EVENT_UNHALTED_REF_CYCLES, 5, 0),
124 EVT(PMT_EVENT_UOPS_RETIRED, 6, 0),
125 }
126 };
127
128 /*
129 * Link: https://github.com/intel/Intel-PMT/blob/main/xml/DMR/OOBMSM/RMID-ENERGY/dmr_aggregator.xml
130 */
131 static struct event_group energy_0x47903099 = {
132 .pfname = "energy",
133 .guid = 0x47903099,
134 .num_rmid = 256,
135 .mmio_size = XML_MMIO_SIZE(256, 2, 3),
136 .num_events = 2,
137 .evts = {
138 EVT(PMT_EVENT_ENERGY, 0, 18),
139 EVT(PMT_EVENT_ACTIVITY, 1, 18),
140 }
141 };
142
143 /*
144 * Link: https://github.com/intel/Intel-PMT/blob/main/xml/DMR/OOBMSM/RMID-PERF/dmr_aggregator.xml
145 */
146 static struct event_group perf_0x1415910a = {
147 .pfname = "perf",
148 .guid = 0x1415910a,
149 .num_rmid = 256,
150 .mmio_size = XML_MMIO_SIZE(256, 4, 3),
151 .num_events = 4,
152 .evts = {
153 EVT(PMT_EVENT_UNHALTED_REF_CYCLES, 0, 0),
154 EVT(PMT_EVENT_UNHALTED_CORE_CYCLES, 1, 0),
155 EVT(PMT_EVENT_INST_RETIRED, 2, 0),
156 EVT(PMT_EVENT_PCNT, 3, 0),
157 }
158 };
159
160 static struct event_group *known_event_groups[] = {
161 &energy_0x26696143,
162 &energy_0x47903099,
163 &perf_0x26557651,
164 &perf_0x1415910a,
165 };
166
167 #define for_each_event_group(_peg) \
168 for (_peg = known_event_groups; \
169 _peg < &known_event_groups[ARRAY_SIZE(known_event_groups)]; \
170 _peg++)
171
intel_handle_aet_option(bool force_off,char * tok)172 bool intel_handle_aet_option(bool force_off, char *tok)
173 {
174 struct event_group **peg;
175 bool ret = false;
176 u32 guid = 0;
177 char *name;
178
179 if (!tok)
180 return false;
181
182 name = strsep(&tok, ":");
183 if (tok && kstrtou32(tok, 16, &guid))
184 return false;
185
186 for_each_event_group(peg) {
187 if (strcmp(name, (*peg)->pfname))
188 continue;
189 if (guid && (*peg)->guid != guid)
190 continue;
191 if (force_off)
192 (*peg)->force_off = true;
193 else
194 (*peg)->force_on = true;
195 ret = true;
196 }
197
198 return ret;
199 }
200
skip_telem_region(struct telemetry_region * tr,struct event_group * e)201 static bool skip_telem_region(struct telemetry_region *tr, struct event_group *e)
202 {
203 if (tr->guid != e->guid)
204 return true;
205 if (tr->plat_info.package_id >= topology_max_packages()) {
206 pr_warn("Bad package %u in guid 0x%x\n", tr->plat_info.package_id,
207 tr->guid);
208 return true;
209 }
210 if (tr->size != e->mmio_size) {
211 pr_warn("MMIO space wrong size (%zu bytes) for guid 0x%x. Expected %zu bytes.\n",
212 tr->size, e->guid, e->mmio_size);
213 return true;
214 }
215
216 return false;
217 }
218
group_has_usable_regions(struct event_group * e,struct pmt_feature_group * p)219 static bool group_has_usable_regions(struct event_group *e, struct pmt_feature_group *p)
220 {
221 bool usable_regions = false;
222
223 for (int i = 0; i < p->count; i++) {
224 if (skip_telem_region(&p->regions[i], e)) {
225 /*
226 * Clear the address field of regions that did not pass the checks in
227 * skip_telem_region() so they will not be used by intel_aet_read_event().
228 * This is safe to do because intel_pmt_get_regions_by_feature() allocates
229 * a new pmt_feature_group structure to return to each caller and only makes
230 * use of the pmt_feature_group::kref field when intel_pmt_put_feature_group()
231 * returns the structure.
232 */
233 p->regions[i].addr = NULL;
234
235 continue;
236 }
237 usable_regions = true;
238 }
239
240 return usable_regions;
241 }
242
all_regions_have_sufficient_rmid(struct event_group * e,struct pmt_feature_group * p)243 static bool all_regions_have_sufficient_rmid(struct event_group *e, struct pmt_feature_group *p)
244 {
245 struct telemetry_region *tr;
246
247 for (int i = 0; i < p->count; i++) {
248 if (!p->regions[i].addr)
249 continue;
250 tr = &p->regions[i];
251 if (tr->num_rmids < e->num_rmid) {
252 e->force_off = true;
253 return false;
254 }
255 }
256
257 return true;
258 }
259
enable_events(struct event_group * e,struct pmt_feature_group * p)260 static bool enable_events(struct event_group *e, struct pmt_feature_group *p)
261 {
262 struct rdt_resource *r = &rdt_resources_all[RDT_RESOURCE_PERF_PKG].r_resctrl;
263 int skipped_events = 0;
264
265 if (e->force_off)
266 return false;
267
268 if (!group_has_usable_regions(e, p))
269 return false;
270
271 /*
272 * Only enable event group with insufficient RMIDs if the user requested
273 * it from the kernel command line.
274 */
275 if (!all_regions_have_sufficient_rmid(e, p) && !e->force_on) {
276 pr_info("%s %s:0x%x monitoring not enabled due to insufficient RMIDs\n",
277 r->name, e->pfname, e->guid);
278 return false;
279 }
280
281 for (int i = 0; i < p->count; i++) {
282 if (!p->regions[i].addr)
283 continue;
284 /*
285 * e->num_rmid only adjusted lower if user (via rdt= kernel
286 * parameter) forces an event group with insufficient RMID
287 * to be enabled.
288 */
289 e->num_rmid = min(e->num_rmid, p->regions[i].num_rmids);
290 }
291
292 for (int j = 0; j < e->num_events; j++) {
293 if (!resctrl_enable_mon_event(e->evts[j].id, true,
294 e->evts[j].bin_bits, &e->evts[j]))
295 skipped_events++;
296 }
297 if (e->num_events == skipped_events) {
298 pr_info("No events enabled in %s %s:0x%x\n", r->name, e->pfname, e->guid);
299 return false;
300 }
301
302 if (r->mon.num_rmid)
303 r->mon.num_rmid = min(r->mon.num_rmid, e->num_rmid);
304 else
305 r->mon.num_rmid = e->num_rmid;
306
307 if (skipped_events)
308 pr_info("%s %s:0x%x monitoring detected (skipped %d events)\n", r->name,
309 e->pfname, e->guid, skipped_events);
310 else
311 pr_info("%s %s:0x%x monitoring detected\n", r->name, e->pfname, e->guid);
312
313 return true;
314 }
315
lookup_pfid(const char * pfname)316 static enum pmt_feature_id lookup_pfid(const char *pfname)
317 {
318 if (!strcmp(pfname, "energy"))
319 return FEATURE_PER_RMID_ENERGY_TELEM;
320 else if (!strcmp(pfname, "perf"))
321 return FEATURE_PER_RMID_PERF_TELEM;
322
323 pr_warn("Unknown PMT feature name '%s'\n", pfname);
324
325 return FEATURE_INVALID;
326 }
327
328 /*
329 * Request a copy of struct pmt_feature_group for each event group. If there is
330 * one, the returned structure has an array of telemetry_region structures,
331 * each element of the array describes one telemetry aggregator. The
332 * telemetry aggregators may have different GUIDs so obtain duplicate struct
333 * pmt_feature_group for event groups with same feature type but different
334 * GUID. Post-processing ensures an event group can only use the telemetry
335 * aggregators that match its GUID. An event group keeps a pointer to its
336 * struct pmt_feature_group to indicate that its events are successfully
337 * enabled.
338 */
intel_aet_get_events(void)339 bool intel_aet_get_events(void)
340 {
341 struct pmt_feature_group *p;
342 enum pmt_feature_id pfid;
343 struct event_group **peg;
344 bool ret = false;
345
346 for_each_event_group(peg) {
347 pfid = lookup_pfid((*peg)->pfname);
348 p = intel_pmt_get_regions_by_feature(pfid);
349 if (IS_ERR_OR_NULL(p))
350 continue;
351 if (enable_events(*peg, p)) {
352 (*peg)->pfg = p;
353 ret = true;
354 } else {
355 intel_pmt_put_feature_group(p);
356 }
357 }
358
359 return ret;
360 }
361
intel_aet_exit(void)362 void __exit intel_aet_exit(void)
363 {
364 struct event_group **peg;
365
366 for_each_event_group(peg) {
367 if ((*peg)->pfg) {
368 intel_pmt_put_feature_group((*peg)->pfg);
369 (*peg)->pfg = NULL;
370 }
371 }
372 }
373
374 #define DATA_VALID BIT_ULL(63)
375 #define DATA_BITS GENMASK_ULL(62, 0)
376
377 /*
378 * Read counter for an event on a domain (summing all aggregators on the
379 * domain). If an aggregator hasn't received any data for a specific RMID,
380 * the MMIO read indicates that data is not valid. Return success if at
381 * least one aggregator has valid data.
382 */
intel_aet_read_event(int domid,u32 rmid,void * arch_priv,u64 * val)383 int intel_aet_read_event(int domid, u32 rmid, void *arch_priv, u64 *val)
384 {
385 struct pmt_event *pevt = arch_priv;
386 struct event_group *e;
387 bool valid = false;
388 u64 total = 0;
389 u64 evtcount;
390 void *pevt0;
391 u32 idx;
392
393 pevt0 = pevt - pevt->idx;
394 e = container_of(pevt0, struct event_group, evts);
395 idx = rmid * e->num_events;
396 idx += pevt->idx;
397
398 if (idx * sizeof(u64) + sizeof(u64) > e->mmio_size) {
399 pr_warn_once("MMIO index %u out of range\n", idx);
400 return -EIO;
401 }
402
403 for (int i = 0; i < e->pfg->count; i++) {
404 if (!e->pfg->regions[i].addr)
405 continue;
406 if (e->pfg->regions[i].plat_info.package_id != domid)
407 continue;
408 evtcount = readq(e->pfg->regions[i].addr + idx * sizeof(u64));
409 if (!(evtcount & DATA_VALID))
410 continue;
411 total += evtcount & DATA_BITS;
412 valid = true;
413 }
414
415 if (valid)
416 *val = total;
417
418 return valid ? 0 : -EINVAL;
419 }
420
intel_aet_mon_domain_setup(int cpu,int id,struct rdt_resource * r,struct list_head * add_pos)421 void intel_aet_mon_domain_setup(int cpu, int id, struct rdt_resource *r,
422 struct list_head *add_pos)
423 {
424 struct rdt_perf_pkg_mon_domain *d;
425 int err;
426
427 d = kzalloc_node(sizeof(*d), GFP_KERNEL, cpu_to_node(cpu));
428 if (!d)
429 return;
430
431 d->hdr.id = id;
432 d->hdr.type = RESCTRL_MON_DOMAIN;
433 d->hdr.rid = RDT_RESOURCE_PERF_PKG;
434 cpumask_set_cpu(cpu, &d->hdr.cpu_mask);
435
436 err = resctrl_online_mon_domain(r, &d->hdr);
437 if (err) {
438 kfree(d);
439 return;
440 }
441 list_add_tail_rcu(&d->hdr.list, add_pos);
442 }
443