1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Resource Director Technology(RDT) 4 * - Intel Application Energy Telemetry 5 * 6 * Copyright (C) 2025 Intel Corporation 7 * 8 * Author: 9 * Tony Luck <tony.luck@intel.com> 10 */ 11 12 #define pr_fmt(fmt) "resctrl: " fmt 13 14 #include <linux/bits.h> 15 #include <linux/compiler_types.h> 16 #include <linux/container_of.h> 17 #include <linux/cpumask.h> 18 #include <linux/err.h> 19 #include <linux/errno.h> 20 #include <linux/gfp_types.h> 21 #include <linux/init.h> 22 #include <linux/intel_pmt_features.h> 23 #include <linux/intel_vsec.h> 24 #include <linux/io.h> 25 #include <linux/minmax.h> 26 #include <linux/printk.h> 27 #include <linux/rculist.h> 28 #include <linux/rcupdate.h> 29 #include <linux/resctrl.h> 30 #include <linux/resctrl_types.h> 31 #include <linux/slab.h> 32 #include <linux/stddef.h> 33 #include <linux/topology.h> 34 #include <linux/types.h> 35 36 #include "internal.h" 37 38 /** 39 * struct pmt_event - Telemetry event. 40 * @id: Resctrl event id. 41 * @idx: Counter index within each per-RMID block of counters. 42 * @bin_bits: Zero for integer valued events, else number bits in fraction 43 * part of fixed-point. 44 */ 45 struct pmt_event { 46 enum resctrl_event_id id; 47 unsigned int idx; 48 unsigned int bin_bits; 49 }; 50 51 #define EVT(_id, _idx, _bits) { .id = _id, .idx = _idx, .bin_bits = _bits } 52 53 /** 54 * struct event_group - Events with the same feature type ("energy" or "perf") and GUID. 55 * @pfname: PMT feature name ("energy" or "perf") of this event group. 56 * Used by boot rdt= option. 57 * @pfg: Points to the aggregated telemetry space information 58 * returned by the intel_pmt_get_regions_by_feature() 59 * call to the INTEL_PMT_TELEMETRY driver that contains 60 * data for all telemetry regions of type @pfname. 61 * Valid if the system supports the event group, 62 * NULL otherwise. 63 * @force_off: True when "rdt" command line or architecture code disables 64 * this event group due to insufficient RMIDs. 65 * @force_on: True when "rdt" command line overrides disable of this 66 * event group. 67 * @guid: Unique number per XML description file. 68 * @num_rmid: Number of RMIDs supported by this group. May be 69 * adjusted downwards if enumeration from 70 * intel_pmt_get_regions_by_feature() indicates fewer 71 * RMIDs can be tracked simultaneously. 72 * @mmio_size: Number of bytes of MMIO registers for this group. 73 * @num_events: Number of events in this group. 74 * @evts: Array of event descriptors. 75 */ 76 struct event_group { 77 /* Data fields for additional structures to manage this group. */ 78 const char *pfname; 79 struct pmt_feature_group *pfg; 80 bool force_off, force_on; 81 82 /* Remaining fields initialized from XML file. */ 83 u32 guid; 84 u32 num_rmid; 85 size_t mmio_size; 86 unsigned int num_events; 87 struct pmt_event evts[] __counted_by(num_events); 88 }; 89 90 #define XML_MMIO_SIZE(num_rmids, num_events, num_extra_status) \ 91 (((num_rmids) * (num_events) + (num_extra_status)) * sizeof(u64)) 92 93 /* 94 * Link: https://github.com/intel/Intel-PMT/blob/main/xml/CWF/OOBMSM/RMID-ENERGY/cwf_aggregator.xml 95 */ 96 static struct event_group energy_0x26696143 = { 97 .pfname = "energy", 98 .guid = 0x26696143, 99 .num_rmid = 576, 100 .mmio_size = XML_MMIO_SIZE(576, 2, 3), 101 .num_events = 2, 102 .evts = { 103 EVT(PMT_EVENT_ENERGY, 0, 18), 104 EVT(PMT_EVENT_ACTIVITY, 1, 18), 105 } 106 }; 107 108 /* 109 * Link: https://github.com/intel/Intel-PMT/blob/main/xml/CWF/OOBMSM/RMID-PERF/cwf_aggregator.xml 110 */ 111 static struct event_group perf_0x26557651 = { 112 .pfname = "perf", 113 .guid = 0x26557651, 114 .num_rmid = 576, 115 .mmio_size = XML_MMIO_SIZE(576, 7, 3), 116 .num_events = 7, 117 .evts = { 118 EVT(PMT_EVENT_STALLS_LLC_HIT, 0, 0), 119 EVT(PMT_EVENT_C1_RES, 1, 0), 120 EVT(PMT_EVENT_UNHALTED_CORE_CYCLES, 2, 0), 121 EVT(PMT_EVENT_STALLS_LLC_MISS, 3, 0), 122 EVT(PMT_EVENT_AUTO_C6_RES, 4, 0), 123 EVT(PMT_EVENT_UNHALTED_REF_CYCLES, 5, 0), 124 EVT(PMT_EVENT_UOPS_RETIRED, 6, 0), 125 } 126 }; 127 128 /* 129 * Link: https://github.com/intel/Intel-PMT/blob/main/xml/DMR/OOBMSM/RMID-ENERGY/dmr_aggregator.xml 130 */ 131 static struct event_group energy_0x47903099 = { 132 .pfname = "energy", 133 .guid = 0x47903099, 134 .num_rmid = 256, 135 .mmio_size = XML_MMIO_SIZE(256, 2, 3), 136 .num_events = 2, 137 .evts = { 138 EVT(PMT_EVENT_ENERGY, 0, 18), 139 EVT(PMT_EVENT_ACTIVITY, 1, 18), 140 } 141 }; 142 143 /* 144 * Link: https://github.com/intel/Intel-PMT/blob/main/xml/DMR/OOBMSM/RMID-PERF/dmr_aggregator.xml 145 */ 146 static struct event_group perf_0x1415910a = { 147 .pfname = "perf", 148 .guid = 0x1415910a, 149 .num_rmid = 256, 150 .mmio_size = XML_MMIO_SIZE(256, 4, 3), 151 .num_events = 4, 152 .evts = { 153 EVT(PMT_EVENT_UNHALTED_REF_CYCLES, 0, 0), 154 EVT(PMT_EVENT_UNHALTED_CORE_CYCLES, 1, 0), 155 EVT(PMT_EVENT_INST_RETIRED, 2, 0), 156 EVT(PMT_EVENT_PCNT, 3, 0), 157 } 158 }; 159 160 static struct event_group *known_event_groups[] = { 161 &energy_0x26696143, 162 &energy_0x47903099, 163 &perf_0x26557651, 164 &perf_0x1415910a, 165 }; 166 167 #define for_each_event_group(_peg) \ 168 for (_peg = known_event_groups; \ 169 _peg < &known_event_groups[ARRAY_SIZE(known_event_groups)]; \ 170 _peg++) 171 172 bool intel_handle_aet_option(bool force_off, char *tok) 173 { 174 struct event_group **peg; 175 bool ret = false; 176 u32 guid = 0; 177 char *name; 178 179 if (!tok) 180 return false; 181 182 name = strsep(&tok, ":"); 183 if (tok && kstrtou32(tok, 16, &guid)) 184 return false; 185 186 for_each_event_group(peg) { 187 if (strcmp(name, (*peg)->pfname)) 188 continue; 189 if (guid && (*peg)->guid != guid) 190 continue; 191 if (force_off) 192 (*peg)->force_off = true; 193 else 194 (*peg)->force_on = true; 195 ret = true; 196 } 197 198 return ret; 199 } 200 201 static bool skip_telem_region(struct telemetry_region *tr, struct event_group *e) 202 { 203 if (tr->guid != e->guid) 204 return true; 205 if (tr->plat_info.package_id >= topology_max_packages()) { 206 pr_warn("Bad package %u in guid 0x%x\n", tr->plat_info.package_id, 207 tr->guid); 208 return true; 209 } 210 if (tr->size != e->mmio_size) { 211 pr_warn("MMIO space wrong size (%zu bytes) for guid 0x%x. Expected %zu bytes.\n", 212 tr->size, e->guid, e->mmio_size); 213 return true; 214 } 215 216 return false; 217 } 218 219 static bool group_has_usable_regions(struct event_group *e, struct pmt_feature_group *p) 220 { 221 bool usable_regions = false; 222 223 for (int i = 0; i < p->count; i++) { 224 if (skip_telem_region(&p->regions[i], e)) { 225 /* 226 * Clear the address field of regions that did not pass the checks in 227 * skip_telem_region() so they will not be used by intel_aet_read_event(). 228 * This is safe to do because intel_pmt_get_regions_by_feature() allocates 229 * a new pmt_feature_group structure to return to each caller and only makes 230 * use of the pmt_feature_group::kref field when intel_pmt_put_feature_group() 231 * returns the structure. 232 */ 233 p->regions[i].addr = NULL; 234 235 continue; 236 } 237 usable_regions = true; 238 } 239 240 return usable_regions; 241 } 242 243 static bool all_regions_have_sufficient_rmid(struct event_group *e, struct pmt_feature_group *p) 244 { 245 struct telemetry_region *tr; 246 247 for (int i = 0; i < p->count; i++) { 248 if (!p->regions[i].addr) 249 continue; 250 tr = &p->regions[i]; 251 if (tr->num_rmids < e->num_rmid) { 252 e->force_off = true; 253 return false; 254 } 255 } 256 257 return true; 258 } 259 260 static bool enable_events(struct event_group *e, struct pmt_feature_group *p) 261 { 262 struct rdt_resource *r = &rdt_resources_all[RDT_RESOURCE_PERF_PKG].r_resctrl; 263 int skipped_events = 0; 264 265 if (e->force_off) 266 return false; 267 268 if (!group_has_usable_regions(e, p)) 269 return false; 270 271 /* 272 * Only enable event group with insufficient RMIDs if the user requested 273 * it from the kernel command line. 274 */ 275 if (!all_regions_have_sufficient_rmid(e, p) && !e->force_on) { 276 pr_info("%s %s:0x%x monitoring not enabled due to insufficient RMIDs\n", 277 r->name, e->pfname, e->guid); 278 return false; 279 } 280 281 for (int i = 0; i < p->count; i++) { 282 if (!p->regions[i].addr) 283 continue; 284 /* 285 * e->num_rmid only adjusted lower if user (via rdt= kernel 286 * parameter) forces an event group with insufficient RMID 287 * to be enabled. 288 */ 289 e->num_rmid = min(e->num_rmid, p->regions[i].num_rmids); 290 } 291 292 for (int j = 0; j < e->num_events; j++) { 293 if (!resctrl_enable_mon_event(e->evts[j].id, true, 294 e->evts[j].bin_bits, &e->evts[j])) 295 skipped_events++; 296 } 297 if (e->num_events == skipped_events) { 298 pr_info("No events enabled in %s %s:0x%x\n", r->name, e->pfname, e->guid); 299 return false; 300 } 301 302 if (r->mon.num_rmid) 303 r->mon.num_rmid = min(r->mon.num_rmid, e->num_rmid); 304 else 305 r->mon.num_rmid = e->num_rmid; 306 307 if (skipped_events) 308 pr_info("%s %s:0x%x monitoring detected (skipped %d events)\n", r->name, 309 e->pfname, e->guid, skipped_events); 310 else 311 pr_info("%s %s:0x%x monitoring detected\n", r->name, e->pfname, e->guid); 312 313 return true; 314 } 315 316 static enum pmt_feature_id lookup_pfid(const char *pfname) 317 { 318 if (!strcmp(pfname, "energy")) 319 return FEATURE_PER_RMID_ENERGY_TELEM; 320 else if (!strcmp(pfname, "perf")) 321 return FEATURE_PER_RMID_PERF_TELEM; 322 323 pr_warn("Unknown PMT feature name '%s'\n", pfname); 324 325 return FEATURE_INVALID; 326 } 327 328 /* 329 * Request a copy of struct pmt_feature_group for each event group. If there is 330 * one, the returned structure has an array of telemetry_region structures, 331 * each element of the array describes one telemetry aggregator. The 332 * telemetry aggregators may have different GUIDs so obtain duplicate struct 333 * pmt_feature_group for event groups with same feature type but different 334 * GUID. Post-processing ensures an event group can only use the telemetry 335 * aggregators that match its GUID. An event group keeps a pointer to its 336 * struct pmt_feature_group to indicate that its events are successfully 337 * enabled. 338 */ 339 bool intel_aet_get_events(void) 340 { 341 struct pmt_feature_group *p; 342 enum pmt_feature_id pfid; 343 struct event_group **peg; 344 bool ret = false; 345 346 for_each_event_group(peg) { 347 pfid = lookup_pfid((*peg)->pfname); 348 p = intel_pmt_get_regions_by_feature(pfid); 349 if (IS_ERR_OR_NULL(p)) 350 continue; 351 if (enable_events(*peg, p)) { 352 (*peg)->pfg = p; 353 ret = true; 354 } else { 355 intel_pmt_put_feature_group(p); 356 } 357 } 358 359 return ret; 360 } 361 362 void __exit intel_aet_exit(void) 363 { 364 struct event_group **peg; 365 366 for_each_event_group(peg) { 367 if ((*peg)->pfg) { 368 intel_pmt_put_feature_group((*peg)->pfg); 369 (*peg)->pfg = NULL; 370 } 371 } 372 } 373 374 #define DATA_VALID BIT_ULL(63) 375 #define DATA_BITS GENMASK_ULL(62, 0) 376 377 /* 378 * Read counter for an event on a domain (summing all aggregators on the 379 * domain). If an aggregator hasn't received any data for a specific RMID, 380 * the MMIO read indicates that data is not valid. Return success if at 381 * least one aggregator has valid data. 382 */ 383 int intel_aet_read_event(int domid, u32 rmid, void *arch_priv, u64 *val) 384 { 385 struct pmt_event *pevt = arch_priv; 386 struct event_group *e; 387 bool valid = false; 388 u64 total = 0; 389 u64 evtcount; 390 void *pevt0; 391 u32 idx; 392 393 pevt0 = pevt - pevt->idx; 394 e = container_of(pevt0, struct event_group, evts); 395 idx = rmid * e->num_events; 396 idx += pevt->idx; 397 398 if (idx * sizeof(u64) + sizeof(u64) > e->mmio_size) { 399 pr_warn_once("MMIO index %u out of range\n", idx); 400 return -EIO; 401 } 402 403 for (int i = 0; i < e->pfg->count; i++) { 404 if (!e->pfg->regions[i].addr) 405 continue; 406 if (e->pfg->regions[i].plat_info.package_id != domid) 407 continue; 408 evtcount = readq(e->pfg->regions[i].addr + idx * sizeof(u64)); 409 if (!(evtcount & DATA_VALID)) 410 continue; 411 total += evtcount & DATA_BITS; 412 valid = true; 413 } 414 415 if (valid) 416 *val = total; 417 418 return valid ? 0 : -EINVAL; 419 } 420 421 void intel_aet_mon_domain_setup(int cpu, int id, struct rdt_resource *r, 422 struct list_head *add_pos) 423 { 424 struct rdt_perf_pkg_mon_domain *d; 425 int err; 426 427 d = kzalloc_node(sizeof(*d), GFP_KERNEL, cpu_to_node(cpu)); 428 if (!d) 429 return; 430 431 d->hdr.id = id; 432 d->hdr.type = RESCTRL_MON_DOMAIN; 433 d->hdr.rid = RDT_RESOURCE_PERF_PKG; 434 cpumask_set_cpu(cpu, &d->hdr.cpu_mask); 435 436 err = resctrl_online_mon_domain(r, &d->hdr); 437 if (err) { 438 kfree(d); 439 return; 440 } 441 list_add_tail_rcu(&d->hdr.list, add_pos); 442 } 443