1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2025 Intel Corporation 4 */ 5 6 #include <drm/drm_drv.h> 7 #include <linux/device.h> 8 9 #include "xe_device.h" 10 #include "xe_force_wake.h" 11 #include "xe_gt_idle.h" 12 #include "xe_guc_engine_activity.h" 13 #include "xe_hw_engine.h" 14 #include "xe_pm.h" 15 #include "xe_pmu.h" 16 17 /** 18 * DOC: Xe PMU (Performance Monitoring Unit) 19 * 20 * Expose events/counters like GT-C6 residency, GT frequency and per-class-engine 21 * activity to user land via the perf interface. Events are per device. 22 * 23 * All events are listed in sysfs: 24 * 25 * $ ls -ld /sys/bus/event_source/devices/xe_* 26 * $ ls /sys/bus/event_source/devices/xe_0000_00_02.0/events/ 27 * $ ls /sys/bus/event_source/devices/xe_0000_00_02.0/format/ 28 * 29 * The following format parameters are available to read events, 30 * but only few are valid with each event: 31 * 32 * gt[60:63] Selects gt for the event 33 * engine_class[20:27] Selects engine-class for event 34 * engine_instance[12:19] Selects the engine-instance for the event 35 * 36 * For engine specific events (engine-*), gt, engine_class and engine_instance parameters must be 37 * set as populated by DRM_XE_DEVICE_QUERY_ENGINES. 38 * 39 * For gt specific events (gt-*) gt parameter must be passed. All other parameters will be 0. 40 * 41 * The standard perf tool can be used to grep for a certain event as well. 42 * Example: 43 * 44 * $ perf list | grep gt-c6 45 * 46 * To sample a specific event for a GT at regular intervals: 47 * 48 * $ perf stat -e <event_name,gt=> -I <interval> 49 */ 50 51 #define XE_PMU_EVENT_GT_MASK GENMASK_ULL(63, 60) 52 #define XE_PMU_EVENT_ENGINE_CLASS_MASK GENMASK_ULL(27, 20) 53 #define XE_PMU_EVENT_ENGINE_INSTANCE_MASK GENMASK_ULL(19, 12) 54 #define XE_PMU_EVENT_ID_MASK GENMASK_ULL(11, 0) 55 56 static unsigned int config_to_event_id(u64 config) 57 { 58 return FIELD_GET(XE_PMU_EVENT_ID_MASK, config); 59 } 60 61 static unsigned int config_to_engine_class(u64 config) 62 { 63 return FIELD_GET(XE_PMU_EVENT_ENGINE_CLASS_MASK, config); 64 } 65 66 static unsigned int config_to_engine_instance(u64 config) 67 { 68 return FIELD_GET(XE_PMU_EVENT_ENGINE_INSTANCE_MASK, config); 69 } 70 71 static unsigned int config_to_gt_id(u64 config) 72 { 73 return FIELD_GET(XE_PMU_EVENT_GT_MASK, config); 74 } 75 76 #define XE_PMU_EVENT_GT_C6_RESIDENCY 0x01 77 #define XE_PMU_EVENT_ENGINE_ACTIVE_TICKS 0x02 78 #define XE_PMU_EVENT_ENGINE_TOTAL_TICKS 0x03 79 80 static struct xe_gt *event_to_gt(struct perf_event *event) 81 { 82 struct xe_device *xe = container_of(event->pmu, typeof(*xe), pmu.base); 83 u64 gt = config_to_gt_id(event->attr.config); 84 85 return xe_device_get_gt(xe, gt); 86 } 87 88 static struct xe_hw_engine *event_to_hwe(struct perf_event *event) 89 { 90 struct xe_device *xe = container_of(event->pmu, typeof(*xe), pmu.base); 91 struct drm_xe_engine_class_instance eci; 92 u64 config = event->attr.config; 93 struct xe_hw_engine *hwe; 94 95 eci.engine_class = config_to_engine_class(config); 96 eci.engine_instance = config_to_engine_instance(config); 97 eci.gt_id = config_to_gt_id(config); 98 99 hwe = xe_hw_engine_lookup(xe, eci); 100 if (!hwe || xe_hw_engine_is_reserved(hwe)) 101 return NULL; 102 103 return hwe; 104 } 105 106 static bool is_engine_event(u64 config) 107 { 108 unsigned int event_id = config_to_event_id(config); 109 110 return (event_id == XE_PMU_EVENT_ENGINE_TOTAL_TICKS || 111 event_id == XE_PMU_EVENT_ENGINE_ACTIVE_TICKS); 112 } 113 114 static bool event_gt_forcewake(struct perf_event *event) 115 { 116 struct xe_device *xe = container_of(event->pmu, typeof(*xe), pmu.base); 117 u64 config = event->attr.config; 118 struct xe_gt *gt; 119 unsigned int *fw_ref; 120 121 if (!is_engine_event(config)) 122 return true; 123 124 gt = xe_device_get_gt(xe, config_to_gt_id(config)); 125 126 fw_ref = kzalloc(sizeof(*fw_ref), GFP_KERNEL); 127 if (!fw_ref) 128 return false; 129 130 *fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT); 131 if (!*fw_ref) { 132 kfree(fw_ref); 133 return false; 134 } 135 136 event->pmu_private = fw_ref; 137 138 return true; 139 } 140 141 static bool event_supported(struct xe_pmu *pmu, unsigned int gt, 142 unsigned int id) 143 { 144 if (gt >= XE_MAX_GT_PER_TILE) 145 return false; 146 147 return id < sizeof(pmu->supported_events) * BITS_PER_BYTE && 148 pmu->supported_events & BIT_ULL(id); 149 } 150 151 static bool event_param_valid(struct perf_event *event) 152 { 153 struct xe_device *xe = container_of(event->pmu, typeof(*xe), pmu.base); 154 unsigned int engine_class, engine_instance; 155 u64 config = event->attr.config; 156 struct xe_gt *gt; 157 158 gt = xe_device_get_gt(xe, config_to_gt_id(config)); 159 if (!gt) 160 return false; 161 162 engine_class = config_to_engine_class(config); 163 engine_instance = config_to_engine_instance(config); 164 165 switch (config_to_event_id(config)) { 166 case XE_PMU_EVENT_GT_C6_RESIDENCY: 167 if (engine_class || engine_instance) 168 return false; 169 break; 170 case XE_PMU_EVENT_ENGINE_ACTIVE_TICKS: 171 case XE_PMU_EVENT_ENGINE_TOTAL_TICKS: 172 if (!event_to_hwe(event)) 173 return false; 174 break; 175 } 176 177 return true; 178 } 179 180 static void xe_pmu_event_destroy(struct perf_event *event) 181 { 182 struct xe_device *xe = container_of(event->pmu, typeof(*xe), pmu.base); 183 struct xe_gt *gt; 184 unsigned int *fw_ref = event->pmu_private; 185 186 if (fw_ref) { 187 gt = xe_device_get_gt(xe, config_to_gt_id(event->attr.config)); 188 xe_force_wake_put(gt_to_fw(gt), *fw_ref); 189 kfree(fw_ref); 190 event->pmu_private = NULL; 191 } 192 193 drm_WARN_ON(&xe->drm, event->parent); 194 xe_pm_runtime_put(xe); 195 drm_dev_put(&xe->drm); 196 } 197 198 static int xe_pmu_event_init(struct perf_event *event) 199 { 200 struct xe_device *xe = container_of(event->pmu, typeof(*xe), pmu.base); 201 struct xe_pmu *pmu = &xe->pmu; 202 unsigned int id, gt; 203 204 if (!pmu->registered) 205 return -ENODEV; 206 207 if (event->attr.type != event->pmu->type) 208 return -ENOENT; 209 210 /* unsupported modes and filters */ 211 if (event->attr.sample_period) /* no sampling */ 212 return -EINVAL; 213 214 if (event->cpu < 0) 215 return -EINVAL; 216 217 gt = config_to_gt_id(event->attr.config); 218 id = config_to_event_id(event->attr.config); 219 if (!event_supported(pmu, gt, id)) 220 return -ENOENT; 221 222 if (has_branch_stack(event)) 223 return -EOPNOTSUPP; 224 225 if (!event_param_valid(event)) 226 return -ENOENT; 227 228 if (!event->parent) { 229 drm_dev_get(&xe->drm); 230 xe_pm_runtime_get(xe); 231 if (!event_gt_forcewake(event)) { 232 xe_pm_runtime_put(xe); 233 drm_dev_put(&xe->drm); 234 return -EINVAL; 235 } 236 event->destroy = xe_pmu_event_destroy; 237 } 238 239 return 0; 240 } 241 242 static u64 read_engine_events(struct xe_gt *gt, struct perf_event *event) 243 { 244 struct xe_hw_engine *hwe; 245 u64 val = 0; 246 247 hwe = event_to_hwe(event); 248 if (config_to_event_id(event->attr.config) == XE_PMU_EVENT_ENGINE_ACTIVE_TICKS) 249 val = xe_guc_engine_activity_active_ticks(>->uc.guc, hwe); 250 else 251 val = xe_guc_engine_activity_total_ticks(>->uc.guc, hwe); 252 253 return val; 254 } 255 256 static u64 __xe_pmu_event_read(struct perf_event *event) 257 { 258 struct xe_gt *gt = event_to_gt(event); 259 260 if (!gt) 261 return 0; 262 263 switch (config_to_event_id(event->attr.config)) { 264 case XE_PMU_EVENT_GT_C6_RESIDENCY: 265 return xe_gt_idle_residency_msec(>->gtidle); 266 case XE_PMU_EVENT_ENGINE_ACTIVE_TICKS: 267 case XE_PMU_EVENT_ENGINE_TOTAL_TICKS: 268 return read_engine_events(gt, event); 269 } 270 271 return 0; 272 } 273 274 static void xe_pmu_event_update(struct perf_event *event) 275 { 276 struct hw_perf_event *hwc = &event->hw; 277 u64 prev, new; 278 279 prev = local64_read(&hwc->prev_count); 280 do { 281 new = __xe_pmu_event_read(event); 282 } while (!local64_try_cmpxchg(&hwc->prev_count, &prev, new)); 283 284 local64_add(new - prev, &event->count); 285 } 286 287 static void xe_pmu_event_read(struct perf_event *event) 288 { 289 struct xe_device *xe = container_of(event->pmu, typeof(*xe), pmu.base); 290 struct xe_pmu *pmu = &xe->pmu; 291 292 if (!pmu->registered) { 293 event->hw.state = PERF_HES_STOPPED; 294 return; 295 } 296 297 xe_pmu_event_update(event); 298 } 299 300 static void xe_pmu_enable(struct perf_event *event) 301 { 302 /* 303 * Store the current counter value so we can report the correct delta 304 * for all listeners. Even when the event was already enabled and has 305 * an existing non-zero value. 306 */ 307 local64_set(&event->hw.prev_count, __xe_pmu_event_read(event)); 308 } 309 310 static void xe_pmu_event_start(struct perf_event *event, int flags) 311 { 312 struct xe_device *xe = container_of(event->pmu, typeof(*xe), pmu.base); 313 struct xe_pmu *pmu = &xe->pmu; 314 315 if (!pmu->registered) 316 return; 317 318 xe_pmu_enable(event); 319 event->hw.state = 0; 320 } 321 322 static void xe_pmu_event_stop(struct perf_event *event, int flags) 323 { 324 struct xe_device *xe = container_of(event->pmu, typeof(*xe), pmu.base); 325 struct xe_pmu *pmu = &xe->pmu; 326 327 if (pmu->registered) 328 if (flags & PERF_EF_UPDATE) 329 xe_pmu_event_update(event); 330 331 event->hw.state = PERF_HES_STOPPED; 332 } 333 334 static int xe_pmu_event_add(struct perf_event *event, int flags) 335 { 336 struct xe_device *xe = container_of(event->pmu, typeof(*xe), pmu.base); 337 struct xe_pmu *pmu = &xe->pmu; 338 339 if (!pmu->registered) 340 return -ENODEV; 341 342 if (flags & PERF_EF_START) 343 xe_pmu_event_start(event, flags); 344 345 return 0; 346 } 347 348 static void xe_pmu_event_del(struct perf_event *event, int flags) 349 { 350 xe_pmu_event_stop(event, PERF_EF_UPDATE); 351 } 352 353 PMU_FORMAT_ATTR(gt, "config:60-63"); 354 PMU_FORMAT_ATTR(engine_class, "config:20-27"); 355 PMU_FORMAT_ATTR(engine_instance, "config:12-19"); 356 PMU_FORMAT_ATTR(event, "config:0-11"); 357 358 static struct attribute *pmu_format_attrs[] = { 359 &format_attr_event.attr, 360 &format_attr_engine_class.attr, 361 &format_attr_engine_instance.attr, 362 &format_attr_gt.attr, 363 NULL, 364 }; 365 366 static const struct attribute_group pmu_format_attr_group = { 367 .name = "format", 368 .attrs = pmu_format_attrs, 369 }; 370 371 static ssize_t event_attr_show(struct device *dev, 372 struct device_attribute *attr, char *buf) 373 { 374 struct perf_pmu_events_attr *pmu_attr = 375 container_of(attr, struct perf_pmu_events_attr, attr); 376 377 return sprintf(buf, "event=%#04llx\n", pmu_attr->id); 378 } 379 380 #define XE_EVENT_ATTR(name_, v_, id_) \ 381 PMU_EVENT_ATTR(name_, pmu_event_ ## v_, id_, event_attr_show) 382 383 #define XE_EVENT_ATTR_UNIT(name_, v_, unit_) \ 384 PMU_EVENT_ATTR_STRING(name_.unit, pmu_event_unit_ ## v_, unit_) 385 386 #define XE_EVENT_ATTR_GROUP(v_, id_, ...) \ 387 static struct attribute *pmu_attr_ ##v_[] = { \ 388 __VA_ARGS__, \ 389 NULL \ 390 }; \ 391 static umode_t is_visible_##v_(struct kobject *kobj, \ 392 struct attribute *attr, int idx) \ 393 { \ 394 struct perf_pmu_events_attr *pmu_attr; \ 395 struct xe_pmu *pmu; \ 396 \ 397 pmu_attr = container_of(attr, typeof(*pmu_attr), attr.attr); \ 398 pmu = container_of(dev_get_drvdata(kobj_to_dev(kobj)), \ 399 typeof(*pmu), base); \ 400 \ 401 return event_supported(pmu, 0, id_) ? attr->mode : 0; \ 402 } \ 403 static const struct attribute_group pmu_group_ ##v_ = { \ 404 .name = "events", \ 405 .attrs = pmu_attr_ ## v_, \ 406 .is_visible = is_visible_ ## v_, \ 407 } 408 409 #define XE_EVENT_ATTR_SIMPLE(name_, v_, id_, unit_) \ 410 XE_EVENT_ATTR(name_, v_, id_) \ 411 XE_EVENT_ATTR_UNIT(name_, v_, unit_) \ 412 XE_EVENT_ATTR_GROUP(v_, id_, &pmu_event_ ##v_.attr.attr, \ 413 &pmu_event_unit_ ##v_.attr.attr) 414 415 #define XE_EVENT_ATTR_NOUNIT(name_, v_, id_) \ 416 XE_EVENT_ATTR(name_, v_, id_) \ 417 XE_EVENT_ATTR_GROUP(v_, id_, &pmu_event_ ##v_.attr.attr) 418 419 XE_EVENT_ATTR_SIMPLE(gt-c6-residency, gt_c6_residency, XE_PMU_EVENT_GT_C6_RESIDENCY, "ms"); 420 XE_EVENT_ATTR_NOUNIT(engine-active-ticks, engine_active_ticks, XE_PMU_EVENT_ENGINE_ACTIVE_TICKS); 421 XE_EVENT_ATTR_NOUNIT(engine-total-ticks, engine_total_ticks, XE_PMU_EVENT_ENGINE_TOTAL_TICKS); 422 423 static struct attribute *pmu_empty_event_attrs[] = { 424 /* Empty - all events are added as groups with .attr_update() */ 425 NULL, 426 }; 427 428 static const struct attribute_group pmu_events_attr_group = { 429 .name = "events", 430 .attrs = pmu_empty_event_attrs, 431 }; 432 433 static const struct attribute_group *pmu_events_attr_update[] = { 434 &pmu_group_gt_c6_residency, 435 &pmu_group_engine_active_ticks, 436 &pmu_group_engine_total_ticks, 437 NULL, 438 }; 439 440 static void set_supported_events(struct xe_pmu *pmu) 441 { 442 struct xe_device *xe = container_of(pmu, typeof(*xe), pmu); 443 struct xe_gt *gt = xe_device_get_gt(xe, 0); 444 445 if (!xe->info.skip_guc_pc) 446 pmu->supported_events |= BIT_ULL(XE_PMU_EVENT_GT_C6_RESIDENCY); 447 448 if (xe_guc_engine_activity_supported(>->uc.guc)) { 449 pmu->supported_events |= BIT_ULL(XE_PMU_EVENT_ENGINE_ACTIVE_TICKS); 450 pmu->supported_events |= BIT_ULL(XE_PMU_EVENT_ENGINE_TOTAL_TICKS); 451 } 452 } 453 454 /** 455 * xe_pmu_unregister() - Remove/cleanup PMU registration 456 * @arg: Ptr to pmu 457 */ 458 static void xe_pmu_unregister(void *arg) 459 { 460 struct xe_pmu *pmu = arg; 461 struct xe_device *xe = container_of(pmu, typeof(*xe), pmu); 462 463 if (!pmu->registered) 464 return; 465 466 pmu->registered = false; 467 468 perf_pmu_unregister(&pmu->base); 469 kfree(pmu->name); 470 } 471 472 /** 473 * xe_pmu_register() - Define basic PMU properties for Xe and add event callbacks. 474 * @pmu: the PMU object 475 * 476 * Returns 0 on success and an appropriate error code otherwise 477 */ 478 int xe_pmu_register(struct xe_pmu *pmu) 479 { 480 struct xe_device *xe = container_of(pmu, typeof(*xe), pmu); 481 static const struct attribute_group *attr_groups[] = { 482 &pmu_format_attr_group, 483 &pmu_events_attr_group, 484 NULL 485 }; 486 int ret = -ENOMEM; 487 char *name; 488 489 BUILD_BUG_ON(XE_MAX_GT_PER_TILE != XE_PMU_MAX_GT); 490 491 if (IS_SRIOV_VF(xe)) 492 return 0; 493 494 name = kasprintf(GFP_KERNEL, "xe_%s", 495 dev_name(xe->drm.dev)); 496 if (!name) 497 goto err; 498 499 /* tools/perf reserves colons as special. */ 500 strreplace(name, ':', '_'); 501 502 pmu->name = name; 503 pmu->base.attr_groups = attr_groups; 504 pmu->base.attr_update = pmu_events_attr_update; 505 pmu->base.scope = PERF_PMU_SCOPE_SYS_WIDE; 506 pmu->base.module = THIS_MODULE; 507 pmu->base.task_ctx_nr = perf_invalid_context; 508 pmu->base.event_init = xe_pmu_event_init; 509 pmu->base.add = xe_pmu_event_add; 510 pmu->base.del = xe_pmu_event_del; 511 pmu->base.start = xe_pmu_event_start; 512 pmu->base.stop = xe_pmu_event_stop; 513 pmu->base.read = xe_pmu_event_read; 514 515 set_supported_events(pmu); 516 517 ret = perf_pmu_register(&pmu->base, pmu->name, -1); 518 if (ret) 519 goto err_name; 520 521 pmu->registered = true; 522 523 return devm_add_action_or_reset(xe->drm.dev, xe_pmu_unregister, pmu); 524 525 err_name: 526 kfree(name); 527 err: 528 drm_err(&xe->drm, "Failed to register PMU (ret=%d)!\n", ret); 529 530 return ret; 531 } 532