1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2025 Intel Corporation 4 */ 5 6 #include <drm/drm_managed.h> 7 8 #include "abi/guc_actions_abi.h" 9 #include "regs/xe_gt_regs.h" 10 11 #include "xe_bo.h" 12 #include "xe_force_wake.h" 13 #include "xe_gt_printk.h" 14 #include "xe_guc.h" 15 #include "xe_guc_engine_activity.h" 16 #include "xe_guc_ct.h" 17 #include "xe_hw_engine.h" 18 #include "xe_map.h" 19 #include "xe_mmio.h" 20 #include "xe_sriov_pf_helpers.h" 21 #include "xe_trace_guc.h" 22 23 #define TOTAL_QUANTA 0x8000 24 25 static struct iosys_map engine_activity_map(struct xe_guc *guc, struct xe_hw_engine *hwe, 26 unsigned int index) 27 { 28 struct xe_guc_engine_activity *engine_activity = &guc->engine_activity; 29 struct engine_activity_buffer *buffer; 30 u16 guc_class = xe_hwe_to_guc_class(hwe); 31 u16 guc_logical_instance = xe_hwe_guc_logical_instance(hwe); 32 size_t offset; 33 34 if (engine_activity->num_functions) { 35 buffer = &engine_activity->function_buffer; 36 offset = sizeof(struct guc_engine_activity_data) * index; 37 } else { 38 buffer = &engine_activity->device_buffer; 39 offset = 0; 40 } 41 42 offset += offsetof(struct guc_engine_activity_data, 43 engine_activity[guc_class][guc_logical_instance]); 44 45 return IOSYS_MAP_INIT_OFFSET(&buffer->activity_bo->vmap, offset); 46 } 47 48 static struct iosys_map engine_metadata_map(struct xe_guc *guc, 49 unsigned int index) 50 { 51 struct xe_guc_engine_activity *engine_activity = &guc->engine_activity; 52 struct engine_activity_buffer *buffer; 53 size_t offset; 54 55 if (engine_activity->num_functions) { 56 buffer = &engine_activity->function_buffer; 57 offset = sizeof(struct guc_engine_activity_metadata) * index; 58 } else { 59 buffer = &engine_activity->device_buffer; 60 offset = 0; 61 } 62 63 return IOSYS_MAP_INIT_OFFSET(&buffer->metadata_bo->vmap, offset); 64 } 65 66 static int allocate_engine_activity_group(struct xe_guc *guc) 67 { 68 struct xe_guc_engine_activity *engine_activity = &guc->engine_activity; 69 struct xe_device *xe = guc_to_xe(guc); 70 u32 num_activity_group; 71 72 /* 73 * An additional activity group is allocated for PF 74 */ 75 num_activity_group = IS_SRIOV_PF(xe) ? xe_sriov_pf_get_totalvfs(xe) + 1 : 1; 76 77 engine_activity->eag = drmm_kcalloc(&xe->drm, num_activity_group, 78 sizeof(struct engine_activity_group), GFP_KERNEL); 79 80 if (!engine_activity->eag) 81 return -ENOMEM; 82 83 engine_activity->num_activity_group = num_activity_group; 84 85 return 0; 86 } 87 88 static int allocate_engine_activity_buffers(struct xe_guc *guc, 89 struct engine_activity_buffer *buffer, 90 int count) 91 { 92 u32 metadata_size = sizeof(struct guc_engine_activity_metadata) * count; 93 u32 size = sizeof(struct guc_engine_activity_data) * count; 94 struct xe_gt *gt = guc_to_gt(guc); 95 struct xe_tile *tile = gt_to_tile(gt); 96 struct xe_bo *bo, *metadata_bo; 97 98 metadata_bo = xe_bo_create_pin_map_novm(gt_to_xe(gt), tile, PAGE_ALIGN(metadata_size), 99 ttm_bo_type_kernel, XE_BO_FLAG_SYSTEM | 100 XE_BO_FLAG_GGTT | XE_BO_FLAG_GGTT_INVALIDATE | 101 xe_guc_bo_wa_flags(gt), 102 false); 103 104 if (IS_ERR(metadata_bo)) 105 return PTR_ERR(metadata_bo); 106 107 bo = xe_bo_create_pin_map_novm(gt_to_xe(gt), tile, PAGE_ALIGN(size), 108 ttm_bo_type_kernel, XE_BO_FLAG_VRAM_IF_DGFX(tile) | 109 XE_BO_FLAG_GGTT | XE_BO_FLAG_GGTT_INVALIDATE | 110 xe_guc_bo_wa_flags(gt), false); 111 112 if (IS_ERR(bo)) { 113 xe_bo_unpin_map_no_vm(metadata_bo); 114 return PTR_ERR(bo); 115 } 116 117 buffer->metadata_bo = metadata_bo; 118 buffer->activity_bo = bo; 119 return 0; 120 } 121 122 static void free_engine_activity_buffers(struct engine_activity_buffer *buffer) 123 { 124 xe_bo_unpin_map_no_vm(buffer->metadata_bo); 125 xe_bo_unpin_map_no_vm(buffer->activity_bo); 126 } 127 128 static bool is_engine_activity_supported(struct xe_guc *guc) 129 { 130 struct xe_uc_fw_version *version = &guc->fw.versions.found[XE_UC_FW_VER_COMPATIBILITY]; 131 struct xe_uc_fw_version required = { .major = 1, .minor = 14, .patch = 1 }; 132 struct xe_gt *gt = guc_to_gt(guc); 133 134 if (IS_SRIOV_VF(gt_to_xe(gt))) { 135 xe_gt_info(gt, "engine activity stats not supported on VFs\n"); 136 return false; 137 } 138 139 /* engine activity stats is supported from GuC interface version (1.14.1) */ 140 if (GUC_SUBMIT_VER(guc) < MAKE_GUC_VER_STRUCT(required)) { 141 xe_gt_info(gt, 142 "engine activity stats unsupported in GuC interface v%u.%u.%u, need v%u.%u.%u or higher\n", 143 version->major, version->minor, version->patch, required.major, 144 required.minor, required.patch); 145 return false; 146 } 147 148 return true; 149 } 150 151 static struct engine_activity *hw_engine_to_engine_activity(struct xe_hw_engine *hwe, 152 unsigned int index) 153 { 154 struct xe_guc *guc = &hwe->gt->uc.guc; 155 struct engine_activity_group *eag = &guc->engine_activity.eag[index]; 156 u16 guc_class = xe_hwe_to_guc_class(hwe); 157 u16 guc_logical_instance = xe_hwe_guc_logical_instance(hwe); 158 159 return &eag->engine[guc_class][guc_logical_instance]; 160 } 161 162 static u64 cpu_ns_to_guc_tsc_tick(ktime_t ns, u32 freq) 163 { 164 return mul_u64_u32_div(ns, freq, NSEC_PER_SEC); 165 } 166 167 #define read_engine_activity_record(xe_, map_, field_) \ 168 xe_map_rd_field(xe_, map_, 0, struct guc_engine_activity, field_) 169 170 #define read_metadata_record(xe_, map_, field_) \ 171 xe_map_rd_field(xe_, map_, 0, struct guc_engine_activity_metadata, field_) 172 173 static u64 get_engine_active_ticks(struct xe_guc *guc, struct xe_hw_engine *hwe, 174 unsigned int index) 175 { 176 struct engine_activity *ea = hw_engine_to_engine_activity(hwe, index); 177 struct guc_engine_activity *cached_activity = &ea->activity; 178 struct guc_engine_activity_metadata *cached_metadata = &ea->metadata; 179 struct xe_guc_engine_activity *engine_activity = &guc->engine_activity; 180 struct iosys_map activity_map, metadata_map; 181 struct xe_device *xe = guc_to_xe(guc); 182 struct xe_gt *gt = guc_to_gt(guc); 183 u32 last_update_tick, global_change_num; 184 u64 active_ticks, gpm_ts; 185 u16 change_num; 186 187 activity_map = engine_activity_map(guc, hwe, index); 188 metadata_map = engine_metadata_map(guc, index); 189 global_change_num = read_metadata_record(xe, &metadata_map, global_change_num); 190 191 /* GuC has not initialized activity data yet, return 0 */ 192 if (!global_change_num) 193 goto update; 194 195 if (global_change_num == cached_metadata->global_change_num) 196 goto update; 197 198 cached_metadata->global_change_num = global_change_num; 199 change_num = read_engine_activity_record(xe, &activity_map, change_num); 200 201 if (!change_num || change_num == cached_activity->change_num) 202 goto update; 203 204 /* read engine activity values */ 205 last_update_tick = read_engine_activity_record(xe, &activity_map, last_update_tick); 206 active_ticks = read_engine_activity_record(xe, &activity_map, active_ticks); 207 208 /* activity calculations */ 209 ea->running = !!last_update_tick; 210 ea->total += active_ticks - cached_activity->active_ticks; 211 ea->active = 0; 212 213 /* cache the counter */ 214 cached_activity->change_num = change_num; 215 cached_activity->last_update_tick = last_update_tick; 216 cached_activity->active_ticks = active_ticks; 217 218 update: 219 if (ea->running) { 220 gpm_ts = xe_mmio_read64_2x32(>->mmio, MISC_STATUS_0) >> 221 engine_activity->gpm_timestamp_shift; 222 ea->active = lower_32_bits(gpm_ts) - cached_activity->last_update_tick; 223 } 224 225 trace_xe_guc_engine_activity(xe, ea, hwe->name, hwe->instance); 226 227 return ea->total + ea->active; 228 } 229 230 static u64 get_engine_total_ticks(struct xe_guc *guc, struct xe_hw_engine *hwe, unsigned int index) 231 { 232 struct engine_activity *ea = hw_engine_to_engine_activity(hwe, index); 233 struct guc_engine_activity_metadata *cached_metadata = &ea->metadata; 234 struct guc_engine_activity *cached_activity = &ea->activity; 235 struct iosys_map activity_map, metadata_map; 236 struct xe_device *xe = guc_to_xe(guc); 237 ktime_t now, cpu_delta; 238 u64 numerator; 239 u16 quanta_ratio; 240 241 activity_map = engine_activity_map(guc, hwe, index); 242 metadata_map = engine_metadata_map(guc, index); 243 244 if (!cached_metadata->guc_tsc_frequency_hz) 245 cached_metadata->guc_tsc_frequency_hz = read_metadata_record(xe, &metadata_map, 246 guc_tsc_frequency_hz); 247 248 quanta_ratio = read_engine_activity_record(xe, &activity_map, quanta_ratio); 249 cached_activity->quanta_ratio = quanta_ratio; 250 251 /* Total ticks calculations */ 252 now = ktime_get(); 253 cpu_delta = now - ea->last_cpu_ts; 254 ea->last_cpu_ts = now; 255 numerator = (ea->quanta_remainder_ns + cpu_delta) * cached_activity->quanta_ratio; 256 ea->quanta_ns += numerator / TOTAL_QUANTA; 257 ea->quanta_remainder_ns = numerator % TOTAL_QUANTA; 258 ea->quanta = cpu_ns_to_guc_tsc_tick(ea->quanta_ns, cached_metadata->guc_tsc_frequency_hz); 259 260 trace_xe_guc_engine_activity(xe, ea, hwe->name, hwe->instance); 261 262 return ea->quanta; 263 } 264 265 static int enable_engine_activity_stats(struct xe_guc *guc) 266 { 267 struct xe_guc_engine_activity *engine_activity = &guc->engine_activity; 268 struct engine_activity_buffer *buffer = &engine_activity->device_buffer; 269 u32 action[] = { 270 XE_GUC_ACTION_SET_DEVICE_ENGINE_ACTIVITY_BUFFER, 271 xe_bo_ggtt_addr(buffer->metadata_bo), 272 0, 273 xe_bo_ggtt_addr(buffer->activity_bo), 274 0, 275 }; 276 277 /* Blocking here to ensure the buffers are ready before reading them */ 278 return xe_guc_ct_send_block(&guc->ct, action, ARRAY_SIZE(action)); 279 } 280 281 static int enable_function_engine_activity_stats(struct xe_guc *guc, bool enable) 282 { 283 struct xe_guc_engine_activity *engine_activity = &guc->engine_activity; 284 u32 metadata_ggtt_addr = 0, ggtt_addr = 0, num_functions = 0; 285 struct engine_activity_buffer *buffer = &engine_activity->function_buffer; 286 u32 action[6]; 287 int len = 0; 288 289 if (enable) { 290 metadata_ggtt_addr = xe_bo_ggtt_addr(buffer->metadata_bo); 291 ggtt_addr = xe_bo_ggtt_addr(buffer->activity_bo); 292 num_functions = engine_activity->num_functions; 293 } 294 295 action[len++] = XE_GUC_ACTION_SET_FUNCTION_ENGINE_ACTIVITY_BUFFER; 296 action[len++] = num_functions; 297 action[len++] = metadata_ggtt_addr; 298 action[len++] = 0; 299 action[len++] = ggtt_addr; 300 action[len++] = 0; 301 302 /* Blocking here to ensure the buffers are ready before reading them */ 303 return xe_guc_ct_send_block(&guc->ct, action, ARRAY_SIZE(action)); 304 } 305 306 static void engine_activity_set_cpu_ts(struct xe_guc *guc, unsigned int index) 307 { 308 struct xe_guc_engine_activity *engine_activity = &guc->engine_activity; 309 struct engine_activity_group *eag = &engine_activity->eag[index]; 310 int i, j; 311 312 xe_gt_assert(guc_to_gt(guc), index < engine_activity->num_activity_group); 313 314 for (i = 0; i < GUC_MAX_ENGINE_CLASSES; i++) 315 for (j = 0; j < GUC_MAX_INSTANCES_PER_CLASS; j++) 316 eag->engine[i][j].last_cpu_ts = ktime_get(); 317 } 318 319 static u32 gpm_timestamp_shift(struct xe_gt *gt) 320 { 321 u32 reg; 322 323 reg = xe_mmio_read32(>->mmio, RPM_CONFIG0); 324 325 return 3 - REG_FIELD_GET(RPM_CONFIG0_CTC_SHIFT_PARAMETER_MASK, reg); 326 } 327 328 static bool is_function_valid(struct xe_guc *guc, unsigned int fn_id) 329 { 330 struct xe_device *xe = guc_to_xe(guc); 331 struct xe_guc_engine_activity *engine_activity = &guc->engine_activity; 332 333 if (!IS_SRIOV_PF(xe) && fn_id) 334 return false; 335 336 if (engine_activity->num_functions && fn_id >= engine_activity->num_functions) 337 return false; 338 339 return true; 340 } 341 342 static int engine_activity_disable_function_stats(struct xe_guc *guc) 343 { 344 struct xe_guc_engine_activity *engine_activity = &guc->engine_activity; 345 struct engine_activity_buffer *buffer = &engine_activity->function_buffer; 346 int ret; 347 348 if (!engine_activity->num_functions) 349 return 0; 350 351 ret = enable_function_engine_activity_stats(guc, false); 352 if (ret) 353 return ret; 354 355 free_engine_activity_buffers(buffer); 356 engine_activity->num_functions = 0; 357 358 return 0; 359 } 360 361 static int engine_activity_enable_function_stats(struct xe_guc *guc, int num_vfs) 362 { 363 struct xe_guc_engine_activity *engine_activity = &guc->engine_activity; 364 struct engine_activity_buffer *buffer = &engine_activity->function_buffer; 365 int ret, i; 366 367 if (!num_vfs) 368 return 0; 369 370 /* This includes 1 PF and num_vfs */ 371 engine_activity->num_functions = num_vfs + 1; 372 373 ret = allocate_engine_activity_buffers(guc, buffer, engine_activity->num_functions); 374 if (ret) 375 return ret; 376 377 ret = enable_function_engine_activity_stats(guc, true); 378 if (ret) { 379 free_engine_activity_buffers(buffer); 380 engine_activity->num_functions = 0; 381 return ret; 382 } 383 384 /* skip PF as it was already setup */ 385 for (i = 1; i < engine_activity->num_functions; i++) 386 engine_activity_set_cpu_ts(guc, i); 387 388 return 0; 389 } 390 391 /** 392 * xe_guc_engine_activity_active_ticks - Get engine active ticks 393 * @guc: The GuC object 394 * @hwe: The hw_engine object 395 * @fn_id: function id to report on 396 * 397 * Return: accumulated ticks @hwe was active since engine activity stats were enabled. 398 */ 399 u64 xe_guc_engine_activity_active_ticks(struct xe_guc *guc, struct xe_hw_engine *hwe, 400 unsigned int fn_id) 401 { 402 if (!xe_guc_engine_activity_supported(guc)) 403 return 0; 404 405 if (!is_function_valid(guc, fn_id)) 406 return 0; 407 408 return get_engine_active_ticks(guc, hwe, fn_id); 409 } 410 411 /** 412 * xe_guc_engine_activity_total_ticks - Get engine total ticks 413 * @guc: The GuC object 414 * @hwe: The hw_engine object 415 * @fn_id: function id to report on 416 * 417 * Return: accumulated quanta of ticks allocated for the engine 418 */ 419 u64 xe_guc_engine_activity_total_ticks(struct xe_guc *guc, struct xe_hw_engine *hwe, 420 unsigned int fn_id) 421 { 422 if (!xe_guc_engine_activity_supported(guc)) 423 return 0; 424 425 if (!is_function_valid(guc, fn_id)) 426 return 0; 427 428 return get_engine_total_ticks(guc, hwe, fn_id); 429 } 430 431 /** 432 * xe_guc_engine_activity_supported - Check support for engine activity stats 433 * @guc: The GuC object 434 * 435 * Engine activity stats is supported from GuC interface version (1.14.1) 436 * 437 * Return: true if engine activity stats supported, false otherwise 438 */ 439 bool xe_guc_engine_activity_supported(struct xe_guc *guc) 440 { 441 struct xe_guc_engine_activity *engine_activity = &guc->engine_activity; 442 443 return engine_activity->supported; 444 } 445 446 /** 447 * xe_guc_engine_activity_function_stats - Enable/Disable per-function engine activity stats 448 * @guc: The GuC object 449 * @num_vfs: number of vfs 450 * @enable: true to enable, false otherwise 451 * 452 * Return: 0 on success, negative error code otherwise 453 */ 454 int xe_guc_engine_activity_function_stats(struct xe_guc *guc, int num_vfs, bool enable) 455 { 456 if (!xe_guc_engine_activity_supported(guc)) 457 return 0; 458 459 if (enable) 460 return engine_activity_enable_function_stats(guc, num_vfs); 461 462 return engine_activity_disable_function_stats(guc); 463 } 464 465 /** 466 * xe_guc_engine_activity_enable_stats - Enable engine activity stats 467 * @guc: The GuC object 468 * 469 * Enable engine activity stats and set initial timestamps 470 */ 471 void xe_guc_engine_activity_enable_stats(struct xe_guc *guc) 472 { 473 int ret; 474 475 if (!xe_guc_engine_activity_supported(guc)) 476 return; 477 478 ret = enable_engine_activity_stats(guc); 479 if (ret) 480 xe_gt_err(guc_to_gt(guc), "failed to enable activity stats: %pe\n", ERR_PTR(ret)); 481 else 482 engine_activity_set_cpu_ts(guc, 0); 483 } 484 485 static void engine_activity_fini(void *arg) 486 { 487 struct xe_guc_engine_activity *engine_activity = arg; 488 struct engine_activity_buffer *buffer = &engine_activity->device_buffer; 489 490 free_engine_activity_buffers(buffer); 491 } 492 493 /** 494 * xe_guc_engine_activity_init - Initialize the engine activity data 495 * @guc: The GuC object 496 * 497 * Return: 0 on success, negative error code otherwise. 498 */ 499 int xe_guc_engine_activity_init(struct xe_guc *guc) 500 { 501 struct xe_guc_engine_activity *engine_activity = &guc->engine_activity; 502 struct xe_gt *gt = guc_to_gt(guc); 503 int ret; 504 505 engine_activity->supported = is_engine_activity_supported(guc); 506 if (!engine_activity->supported) 507 return 0; 508 509 ret = allocate_engine_activity_group(guc); 510 if (ret) { 511 xe_gt_err(gt, "failed to allocate engine activity group (%pe)\n", ERR_PTR(ret)); 512 return ret; 513 } 514 515 ret = allocate_engine_activity_buffers(guc, &engine_activity->device_buffer, 1); 516 if (ret) { 517 xe_gt_err(gt, "failed to allocate engine activity buffers (%pe)\n", ERR_PTR(ret)); 518 return ret; 519 } 520 521 engine_activity->gpm_timestamp_shift = gpm_timestamp_shift(gt); 522 523 return devm_add_action_or_reset(gt_to_xe(gt)->drm.dev, engine_activity_fini, 524 engine_activity); 525 } 526